Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve performance, upgrade deps and logging, remove numpy, numba, pandas #816

Merged
merged 16 commits into from
Jan 12, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 15 additions & 0 deletions .vscode/launch.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
{
// Use IntelliSense to learn about possible attributes.
// Hover to view descriptions of existing attributes.
// For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
"version": "0.2.0",
"configurations": [
{
"name": "Python: Module",
"type": "python",
"request": "launch",
"module": "tdp_core",
"justMyCode": false
}
]
}
7 changes: 7 additions & 0 deletions .vscode/settings.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
{
"python.testing.pytestArgs": [
"tdp_core"
],
"python.testing.unittestEnabled": false,
"python.testing.pytestEnabled": true
}
30 changes: 22 additions & 8 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
.DEFAULT_GOAL := help
pkg_src = tdp_core

flake8 = flake8 $(pkg_src) setup.py
isort = isort $(pkg_src) setup.py
black = black --line-length 140 $(pkg_src) setup.py
pyright = pyright $(pkg_src) setup.py
ruff = ruff $(pkg_src) setup.py --line-length 140 --select E,W,F,N,I,C,B,UP,PT,SIM,RUF --ignore E501,C901,B008

.PHONY: start ## Start the development server
start:
Expand All @@ -17,25 +17,25 @@ ci: check-format lint test

.PHONY: format ## Auto-format the source code
format:
$(isort)
$(ruff) --fix
$(black)

.PHONY: check-format ## Check the source code format without changes
check-format:
$(isort) --check-only
$(black) --check

.PHONY: lint ## Run flake8
.PHONY: lint ## Run flake8 and pyright
lint:
$(flake8)
$(ruff) --format=github
$(pyright)

.PHONY: test ## Run tests
test:
pytest $(pkg_src)

.PHONEY: documentation ## Generate docs
documentation:
mkdocs build
documentation:
echo "TODO"

.PHONY: install ## Install the requirements
install:
Expand All @@ -45,6 +45,20 @@ install:
develop:
pip install -e .[develop]

.PHONY: env_encrypt ## Encrypts the current ./<app>/.env
env_encrypt:
openssl aes-256-cbc -pbkdf2 -in ./$(pkg_src)/.env -out ./$(pkg_src)/.env.enc

.PHONY: env_decrypt ## Decrypts the ./<app>/.env.enc
env_decrypt:
@if [ -z "${ENV_PASSWORD}" ]; then \
echo "No ENV_PASSWORD set, prompting for password..."; \
openssl aes-256-cbc -pbkdf2 -d -in ./$(pkg_src)/.env.enc -out ./$(pkg_src)/.env; \
else \
echo "ENV_PASSWORD set, using it..."; \
openssl aes-256-cbc -pbkdf2 -d -in ./$(pkg_src)/.env.enc -out ./$(pkg_src)/.env -pass env:ENV_PASSWORD; \
fi

.PHONY: build ## Build a wheel
build:
python setup.py sdist bdist_wheel --dist-dir dist_python
Expand Down

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 1 addition & 2 deletions dist/lineup/structureImage/StructureImageRenderer.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion dist/lineup/structureImage/StructureImageRenderer.js.map

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 4 additions & 0 deletions pyrightconfig.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@
{
"typeCheckingMode": "basic",
"useLibraryCodeForTypes": true,
}
39 changes: 18 additions & 21 deletions requirements.txt
Original file line number Diff line number Diff line change
@@ -1,30 +1,27 @@
alembic==1.7.7
fastapi==0.75.0
cachetools==5.0.0
flask-smorest==0.37.0
# a2wsgi==1.6.0 # This WSIGMiddleware is not compatible with starlette_context
alembic==1.9.0
cachetools==5.2.0
fastapi[all]==0.89.0
flask-swagger-ui==3.36.0
Flask==2.0.3
Flask[async]>=2.1.0,<=2.2.2
json-cfg==0.4.2
marshmallow-sqlalchemy==0.28.0
marshmallow==3.15.0
numba==0.55.1
numpy==1.21.6
marshmallow-sqlalchemy>=0.26.0,<=0.28.1
marshmallow==3.19.0
openpyxl==3.0.9
pandas==1.4.1
Pillow==9.0.1
psycopg2==2.9.3
pydantic==1.9.0
pyjwt==2.4.0
Pillow==9.3.0
psycopg==3.1.7
psycopg2==2.9.5
pydantic==1.10.2
pyjwt==2.6.0
pymongo~=3.11.1
pytest-postgresql==4.1.1
python-dateutil==2.8.2
python-memcached==1.59
python-multipart==0.0.5
requests==2.27.1
SQLAlchemy==1.4.32
sqlmodel==0.0.6
rdkit==2022.3.5
requests==2.28.1
SQLAlchemy>=1.4.40,<=1.4.46
starlette-context==0.3.5
urllib3==1.26.9
uvicorn[standard]==0.17.6
uvicorn[standard]==0.20.0
yamlreader==3.0.4
pytest-postgresql==4.1.1
psycopg==3.0.16
rdkit==2022.3.5
12 changes: 4 additions & 8 deletions requirements_dev.txt
Original file line number Diff line number Diff line change
@@ -1,9 +1,5 @@
black~=22.3.0
debugpy~=1.5.1
flake8~=4.0.1
isort~=5.10.1
mkdocs-material~=8.2.8
pep8-naming~=0.12.1
black~=22.12.0
pyright~=1.1.285
pytest-runner~=6.0.0
pytest~=7.1.1
recommonmark~=0.7.1
pytest~=7.2.0
ruff==0.0.218
2 changes: 1 addition & 1 deletion setup.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ def requirements(file):
package_data={},
# Although 'package_data' is the preferred approach, in some case you may
# need to place data files outside of your packages. See:
# http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files # noqa
# http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
# In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
data_files=[], # [('my_data', ['data/data_file'])],
)
3 changes: 1 addition & 2 deletions src/lineup/structureImage/StructureImageRenderer.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@ import { abortAble } from 'lineupengine';
import { StructureImageColumn } from './StructureImageColumn';
import { I18nextManager } from '../../i18n';

const template = '<a target="_blank" rel="noopener" style="background-size: contain; background-position: center; background-repeat: no-repeat;"></a>';
const template = '<div style="background-size: contain; background-position: center; background-repeat: no-repeat;"></div>';

function getImageURL(structure: string, substructure: string | null = null, align: string | null = null): string {
return `/api/rdkit/?structure=${encodeURIComponent(structure)}${substructure ? `&substructure=${encodeURIComponent(substructure)}` : ''}${
Expand Down Expand Up @@ -94,7 +94,6 @@ export class StructureImageRenderer implements ICellRendererFactory {
}
n.style.backgroundImage = `url('${getImageURL(value, col.getFilter()?.filter, col.getAlign())}')`;
n.title = value;
n.href = `https://pubchem.ncbi.nlm.nih.gov/#query=${value}`;
});
}
return null;
Expand Down
15 changes: 12 additions & 3 deletions tdp_core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@ def init_app(self, app: FastAPI):
app.include_router(img_api.app)

def register(self, registry: RegHelper):
import logging

_log = logging.getLogger(__name__)

# phovea_server
registry.append(
"namespace",
Expand All @@ -23,7 +27,14 @@ def register(self, registry: RegHelper):
"tdp_core.id_mapping.idtype_api",
{"namespace": "/api/idtype", "factory": "create_idtype"},
)
registry.append("json-encoder", "numpy", "tdp_core.encoder.json_encoder")

try:
import numpy # noqa, type: ignore

registry.append("json-encoder", "numpy", "tdp_core.encoder.json_encoder")
except ImportError:
_log.info('numpy not available, skipping "numpy" json encoder')

registry.append("json-encoder", "set-encoder", "tdp_core.encoder.set_encoder", {})

registry.append(
Expand Down Expand Up @@ -63,8 +74,6 @@ def register(self, registry: RegHelper):
registry.append_router("tdp_plugin_router", "tdp_core.plugin.router", {})
registry.append("namespace", "tdp_xlsx2json", "tdp_core.xlsx", {"namespace": "/api/tdp/xlsx"})
registry.append("mapping_provider", "tdp_core", "tdp_core.mapping_table")
# TODO: Check if this is still required?
registry.append("greenifier", "psycopg2", "tdp_core.sql_use_gevent", {})

# DB migration plugins
registry.append(
Expand Down
14 changes: 7 additions & 7 deletions tdp_core/dataset/dataset.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
import itertools
from builtins import str

from .. import manager

Expand Down Expand Up @@ -41,29 +40,33 @@ def get(dataset_id):
return None


def add(desc, files=[], id=None):
def add(desc, files=None, id=None):
"""
adds a new dataset to this storage
:param desc: the dict description information
:param files: a list of FileStorage
:param id: optional the unique id to use
:return: the newly created dataset or None if an error occurred
"""
if files is None:
files = []
for p in _providers():
r = p.upload(desc, files, id)
if r:
return r
return None


def update(dataset, desc, files=[]):
def update(dataset, desc, files=None):
"""
updates the given dataset
:param dataset: a dataset or a dataset id
:param desc: the dict description information
:param files: a list of FileStorage
:return:
"""
if files is None:
files = []
old = get(dataset) if isinstance(dataset, str) else dataset
if old is None:
return add(desc, files)
Expand All @@ -80,7 +83,4 @@ def remove(dataset):
old = get(dataset) if isinstance(dataset, str) else dataset
if old is None:
return False
for p in _providers():
if p.remove(old):
return True
return False
return any(p.remove(old) for p in _providers())
Loading