From fbae4e00e68086e9c0ead262191aac57de62c22d Mon Sep 17 00:00:00 2001 From: aras2137 Date: Tue, 12 Dec 2023 14:24:21 +0100 Subject: [PATCH 1/4] sqlalchemy 2.0 setup, use tox to check + test both sqlalchemy 1.4 and 2.0 --- CHANGELOG.md | 10 ++++++ nameko_sqlalchemy/database.py | 2 +- nameko_sqlalchemy/database_session.py | 2 +- nameko_sqlalchemy/pytest_fixtures.py | 8 ++--- pyproject.toml | 4 +-- requirements.txt | 2 -- test/conftest.py | 10 +++--- test/test_database.py | 46 +++++++++++++-------------- test/test_database_session.py | 14 ++++---- test/test_pytest_fixtures.py | 12 +++---- test/test_transaction_retry.py | 5 ++- test_requirements.txt | 2 -- tox.ini | 31 ++++++++++++++++-- 13 files changed, 90 insertions(+), 58 deletions(-) delete mode 100644 requirements.txt diff --git a/CHANGELOG.md b/CHANGELOG.md index fd4d55c..9ed370e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,16 @@ Release Notes ============= +Version 2.1.0 +------------- + +(Unreleased yet) + +* Added support for SQLAlchemy 2.0 while still allowing 1.4 usage +* Changed tests to use new instead of deprecated functions, excluding declarative_base() +* Make tox run check + tests on both SQLAlchemy versions + + Version 2.0.0 ------------- diff --git a/nameko_sqlalchemy/database.py b/nameko_sqlalchemy/database.py index 67b7b69..9de690d 100644 --- a/nameko_sqlalchemy/database.py +++ b/nameko_sqlalchemy/database.py @@ -72,7 +72,7 @@ def __init__( def setup(self): service_name = self.container.service_name declarative_base_name = self.declarative_base.__name__ - uri_key = '{}:{}'.format(service_name, declarative_base_name) + uri_key = f'{service_name}:{declarative_base_name}' db_uris = self.container.config[DB_URIS_KEY] self.db_uri = db_uris[uri_key].format({ diff --git a/nameko_sqlalchemy/database_session.py b/nameko_sqlalchemy/database_session.py index 66b0860..5811178 100644 --- a/nameko_sqlalchemy/database_session.py +++ b/nameko_sqlalchemy/database_session.py @@ -19,7 +19,7 @@ def __init__( def setup(self): service_name = self.container.service_name decl_base_name = self.declarative_base.__name__ - uri_key = '{}:{}'.format(service_name, decl_base_name) + uri_key = f'{service_name}:{decl_base_name}' db_uris = self.container.config[DB_URIS_KEY] self.db_uri = db_uris[uri_key].format({ diff --git a/nameko_sqlalchemy/pytest_fixtures.py b/nameko_sqlalchemy/pytest_fixtures.py index 00a2481..3245f70 100644 --- a/nameko_sqlalchemy/pytest_fixtures.py +++ b/nameko_sqlalchemy/pytest_fixtures.py @@ -114,7 +114,7 @@ def model_base(): raise NotImplementedError("Fixture `model_base` has to be overwritten") -@pytest.yield_fixture(scope='session') +@pytest.fixture(scope='session') def db_connection(db_url, model_base, db_engine_options): engine = create_engine(db_url, **db_engine_options) model_base.metadata.create_all(engine) @@ -123,11 +123,11 @@ def db_connection(db_url, model_base, db_engine_options): yield connection - model_base.metadata.drop_all() + model_base.metadata.drop_all(bind=engine) engine.dispose() -@pytest.yield_fixture +@pytest.fixture def db_session(db_connection, model_base): session = sessionmaker(bind=db_connection, class_=Session) db_session = session() @@ -143,7 +143,7 @@ def db_session(db_connection, model_base): db_session.close() -@pytest.yield_fixture +@pytest.fixture def database(db_connection, model_base): database = DatabaseWrapper( diff --git a/pyproject.toml b/pyproject.toml index c66e6e5..d3ac09b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "nameko-sqlalchemy" -version = "2.0.0" +version = "2.1.0" description = "SQLAlchemy dependency for nameko services" license = {file = "LICENSE.txt"} readme = "README.rst" @@ -25,7 +25,7 @@ classifiers = [ ] dependencies = [ "nameko>=2.0.0", - "sqlalchemy>=1.4,<2" + "sqlalchemy>=1.4,<3" ] [project.urls] diff --git a/requirements.txt b/requirements.txt deleted file mode 100644 index fd2cd20..0000000 --- a/requirements.txt +++ /dev/null @@ -1,2 +0,0 @@ -nameko>=2.0.0 -sqlalchemy>=1.4,<2 \ No newline at end of file diff --git a/test/conftest.py b/test/conftest.py index cb147f7..ab651f8 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -11,7 +11,7 @@ import pytest import requests from sqlalchemy import Column, Integer, String -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base TOXIPROXY_PROXY_NAME = 'nameko_sqlalchemy_test_mysql' @@ -25,7 +25,7 @@ class ExampleModel(DeclarativeBase): data = Column(String(100)) -@pytest.yield_fixture +@pytest.fixture def toxiproxy(toxiproxy_api_url, toxiproxy_db_url): class Controller(object): @@ -33,13 +33,11 @@ def __init__(self, api_url): self.api_url = api_url def enable(self): - resource = 'http://{}/reset'.format(self.api_url) + resource = f'http://{self.api_url}/reset' requests.post(resource) def disable(self): - resource = 'http://{}/proxies/{}'.format( - self.api_url, TOXIPROXY_PROXY_NAME - ) + resource = f'http://{self.api_url}/proxies/{TOXIPROXY_PROXY_NAME}' data = { 'enabled': False } diff --git a/test/test_database.py b/test/test_database.py index 0fbf06f..fc24339 100644 --- a/test/test_database.py +++ b/test/test_database.py @@ -4,9 +4,9 @@ from mock import Mock, patch from nameko.containers import ServiceContainer, WorkerContext from nameko.testing.services import dummy, entrypoint_hook -from sqlalchemy import Column, String, create_engine +from sqlalchemy import Column, String, create_engine, text from sqlalchemy.engine import Engine -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from nameko_sqlalchemy.database import DB_URIS_KEY, Database, Session @@ -300,7 +300,7 @@ class BaseTestEndToEnd: @pytest.fixture def db_uri(self, tmpdir): - return 'sqlite:///{}'.format(tmpdir.join("db").strpath) + return f'sqlite:///{tmpdir.join("db").strpath}' @pytest.fixture def container(self, container_factory, db_uri): @@ -338,7 +338,7 @@ def write(self, key, value): @dummy def read(self, key): session = self.db.get_session() - value = session.query(ExampleModel).get(key).value + value = session.get(ident=key, entity=ExampleModel).value session.close() return value @@ -349,10 +349,10 @@ def test_successful_write_and_read(slf, container, db_uri): write(key='spam', value='ham') # verify changes written to disk - entries = list( - create_engine(db_uri).execute( - 'SELECT key, value FROM example LIMIT 1')) - assert entries == [('spam', 'ham',)] + engine = create_engine(db_uri) + with engine.connect() as conn: + entries = list(conn.execute(text('SELECT key, value FROM example LIMIT 1'))) + assert entries == [('spam', 'ham',)] # read through the service with entrypoint_hook(container, 'read') as read: @@ -375,7 +375,7 @@ def write(self, key, value): @dummy def read(self, key): with self.db.get_session() as session: - return session.query(ExampleModel).get(key).value + return session.get(ident=key, entity=ExampleModel).value def test_successful_write_and_read(slf, container, db_uri): @@ -384,10 +384,10 @@ def test_successful_write_and_read(slf, container, db_uri): write(key='spam', value='ham') # verify changes written to disk - entries = list( - create_engine(db_uri).execute( - 'SELECT key, value FROM example LIMIT 1')) - assert entries == [('spam', 'ham',)] + engine = create_engine(db_uri) + with engine.connect() as conn: + entries = list(conn.execute(text('SELECT key, value FROM example LIMIT 1'))) + assert entries == [('spam', 'ham',)] # read through the service with entrypoint_hook(container, 'read') as read: @@ -409,7 +409,7 @@ def write(self, key, value): @dummy def read(self, key): - return self.db.session.query(ExampleModel).get(key).value + return self.db.session.get(ident=key, entity=ExampleModel).value def test_successful_write_and_read(slf, container, db_uri): @@ -418,10 +418,10 @@ def test_successful_write_and_read(slf, container, db_uri): write(key='spam', value='ham') # verify changes written to disk - entries = list( - create_engine(db_uri).execute( - 'SELECT key, value FROM example LIMIT 1')) - assert entries == [('spam', 'ham',)] + engine = create_engine(db_uri) + with engine.connect() as conn: + entries = list(conn.execute(text('SELECT key, value FROM example LIMIT 1'))) + assert entries == [('spam', 'ham',)] # read through the service with entrypoint_hook(container, 'read') as read: @@ -444,7 +444,7 @@ def write(self, key, value): @dummy def read(self, key): with self.db.session as session: - return session.query(ExampleModel).get(key).value + return session.get(ident=key, entity=ExampleModel).value def test_successful_write_and_read(slf, container, db_uri): @@ -453,10 +453,10 @@ def test_successful_write_and_read(slf, container, db_uri): write(key='spam', value='ham') # verify changes written to disk - entries = list( - create_engine(db_uri).execute( - 'SELECT key, value FROM example LIMIT 1')) - assert entries == [('spam', 'ham',)] + engine = create_engine(db_uri) + with engine.connect() as conn: + entries = list(conn.execute(text('SELECT key, value FROM example LIMIT 1'))) + assert entries == [('spam', 'ham',)] # read through the service with entrypoint_hook(container, 'read') as read: diff --git a/test/test_database_session.py b/test/test_database_session.py index f8d0671..30a29a4 100644 --- a/test/test_database_session.py +++ b/test/test_database_session.py @@ -4,9 +4,9 @@ from mock import Mock from nameko.containers import ServiceContainer, WorkerContext from nameko.testing.services import dummy, entrypoint_hook -from sqlalchemy import Column, Integer, String, create_engine +from sqlalchemy import Column, Integer, String, create_engine, text from sqlalchemy.engine import Engine -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from sqlalchemy.orm.session import Session from nameko_sqlalchemy.database import DB_URIS_KEY @@ -35,7 +35,7 @@ def write(self, value): @dummy def read(self, id): - return self.session.query(ExampleModel).get(id).data + return self.session.get(ident=id, entity=ExampleModel).data @pytest.fixture @@ -174,7 +174,7 @@ def test_worker_teardown(db_session): def test_end_to_end(container_factory, tmpdir): # create a temporary database - db_uri = 'sqlite:///{}'.format(tmpdir.join("db").strpath) + db_uri = f'sqlite:///{tmpdir.join("db").strpath}' engine = create_engine(db_uri) ExampleModel.metadata.create_all(engine) @@ -192,8 +192,10 @@ def test_end_to_end(container_factory, tmpdir): pk = write("foobar") # verify changes written to disk - entries = list(engine.execute('SELECT data FROM example LIMIT 1')) - assert entries == [('foobar',)] + engine = create_engine(db_uri) + with engine.connect() as conn: + entries = list(conn.execute(text('SELECT data FROM example LIMIT 1'))) + assert entries == [('foobar',)] # read through the service with entrypoint_hook(container, "read") as read: diff --git a/test/test_pytest_fixtures.py b/test/test_pytest_fixtures.py index 0fef5db..22bc97b 100644 --- a/test/test_pytest_fixtures.py +++ b/test/test_pytest_fixtures.py @@ -1,7 +1,7 @@ import pytest from nameko.testing.services import dummy, worker_factory from sqlalchemy import Column, Integer, String -from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import declarative_base from nameko_sqlalchemy.database import Database @@ -31,7 +31,7 @@ def test_can_save_model(db_session): user = User(id=1, name='Joe') db_session.add(user) db_session.commit() - saved_user = db_session.query(User).get(user.id) + saved_user = db_session.get(ident=user.id, entity=User) assert saved_user.id > 0 assert saved_user.name == 'Joe' @@ -136,7 +136,7 @@ def write(self, id_, name): @dummy def read(self, id_): session = self.db.get_session() - name = session.query(User).get(id_).name + name = session.get(ident=id_, entity=User).name session.close() return name @@ -164,7 +164,7 @@ def write(self, id_, name): @dummy def read(self, id_): with self.db.get_session() as session: - return session.query(User).get(id_).name + return session.get(ident=id_, entity=User).name def test_database_fixture(self, database): @@ -189,7 +189,7 @@ def write(self, id_, name): @dummy def read(self, id_): - return self.db.session.query(User).get(id_).name + return self.db.session.get(ident=id_, entity=User).name def test_database_fixture(self, database): @@ -215,7 +215,7 @@ def write(self, id_, name): @dummy def read(self, id_): with self.db.session as session: - return session.query(User).get(id_).name + return session.get(ident=id_, entity=User).name def test_database_fixture(self, database): diff --git a/test/test_transaction_retry.py b/test/test_transaction_retry.py index 7900140..2880db2 100644 --- a/test/test_transaction_retry.py +++ b/test/test_transaction_retry.py @@ -8,8 +8,7 @@ from nameko.testing.services import dummy, entrypoint_hook from sqlalchemy import create_engine from sqlalchemy.exc import OperationalError, StatementError -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import sessionmaker +from sqlalchemy.orm import declarative_base, sessionmaker from nameko_sqlalchemy.database import DB_URIS_KEY, Database from nameko_sqlalchemy.database_session import DatabaseSession @@ -18,7 +17,7 @@ DeclBase = declarative_base(name='examplebase') -@pytest.yield_fixture +@pytest.fixture def toxiproxy_db_session(toxiproxy_db_url): engine = create_engine(toxiproxy_db_url) Session = sessionmaker(bind=engine) diff --git a/test_requirements.txt b/test_requirements.txt index 09ae17f..8f8f323 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,4 +1,3 @@ --r requirements.txt coverage==7.3.2 isort==5.12.0 mypy==1.7.1 @@ -6,6 +5,5 @@ pytest==7.4.3 requests==2.31.0 PyMySQL==1.1.0 ruff==0.1.6 -sqlalchemy2-stubs==0.0.2a37 types-mock==5.1.0.3 types-requests==2.31.0.10 \ No newline at end of file diff --git a/tox.ini b/tox.ini index e186ccc..f2625f4 100644 --- a/tox.ini +++ b/tox.ini @@ -4,12 +4,26 @@ envlist = check, py{38,39,310}, report + check14, + test14, skipsdist = True [testenv] deps = -e{toxinidir}/ -r{toxinidir}/test_requirements.txt + sqlalchemy>=2.0,<3 +allowlist_externals = + make +commands = + make test +usedevelop = true + +[testenv:test14] +deps = + -e{toxinidir}/ + -r{toxinidir}/test_requirements.txt + sqlalchemy>=1.4,<2 allowlist_externals = make commands = @@ -17,6 +31,19 @@ commands = usedevelop = true [testenv:check] +deps = + -r{toxinidir}/test_requirements.txt + sqlalchemy>=2.0,<3 +commands = + isort --verbose --check-only --diff nameko_sqlalchemy test + ruff nameko_sqlalchemy test + mypy nameko_sqlalchemy test + +[testenv:check14] +deps = + -r{toxinidir}/test_requirements.txt + sqlalchemy>=1.4,<2 + sqlalchemy2-stubs commands = isort --verbose --check-only --diff nameko_sqlalchemy test ruff nameko_sqlalchemy test @@ -30,12 +57,12 @@ commands = coverage html [testenv:clean] +deps = coverage[toml] commands = coverage erase skip_install = true -deps = coverage[toml] [gh-actions] python = 3.8: py38 3.9: py39 - 3.10: py310, clean, check, report + 3.10: py310, clean, check, report, check14, test14 From 1ee8510b0c838cda36176af8527c0c9c5fdba3f5 Mon Sep 17 00:00:00 2001 From: Zbigniew Siciarz Date: Fri, 25 Oct 2024 12:27:43 +0200 Subject: [PATCH 2/4] Drop Python 3.8, add 3.11 and 3.12 to test matrix. --- .github/workflows/build.yml | 4 ++-- pyproject.toml | 7 ++++--- tox.ini | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 60fa64f..210a36c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -6,10 +6,10 @@ jobs: strategy: max-parallel: 2 matrix: - python-version: ["3.8", "3.9", "3.10"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: "${{ matrix.python-version }}" - name: Upgrade pip diff --git a/pyproject.toml b/pyproject.toml index d3ac09b..dd6f13a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,7 +8,7 @@ version = "2.1.0" description = "SQLAlchemy dependency for nameko services" license = {file = "LICENSE.txt"} readme = "README.rst" -requires-python = ">=3.8" +requires-python = ">=3.9" authors = [{name="onefinestay", email="engineering@onefinestay.com"}] classifiers = [ "Intended Audience :: Developers", @@ -17,9 +17,10 @@ classifiers = [ "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", "Topic :: Internet", "Topic :: Software Development :: Libraries :: Python Modules", ] @@ -76,7 +77,7 @@ select = [ ] [tool.mypy] -python_version = "3.10" +python_version = "3.12" plugins = "sqlalchemy.ext.mypy.plugin" mypy_path = "nameko_sqlalchemy/" namespace_packages = true diff --git a/tox.ini b/tox.ini index f2625f4..cd736aa 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ envlist = clean, check, - py{38,39,310}, + py{39,310,311,312}, report check14, test14, @@ -63,6 +63,7 @@ skip_install = true [gh-actions] python = - 3.8: py38 3.9: py39 - 3.10: py310, clean, check, report, check14, test14 + 3.10: py310 + 3.11: py311 + 3.12: py312, clean, check, report, check14, test14 From 6d7a8d246335f8d9727e810c6ec7e2b29e47ee17 Mon Sep 17 00:00:00 2001 From: Zbigniew Siciarz Date: Fri, 25 Oct 2024 12:28:04 +0200 Subject: [PATCH 3/4] Update development dependencies. --- pyproject.toml | 21 +++++++++++---------- test_requirements.txt | 18 +++++++++--------- tox.ini | 4 ++-- 3 files changed, 22 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index dd6f13a..6ad5896 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,15 +34,15 @@ Homepage = "https://github.com/onefinestay/nameko-sqlalchemy" [project.optional-dependencies] dev = [ - "coverage==7.3.2", - "isort==5.12.0", - "mypy==1.7.1", - "pytest==7.4.3", - "requests==2.31.0", - "ruff==0.1.6", - "PyMySQL==1.1.0", - "types-mock==5.1.0.3", - "types-requests==2.31.0.10", + "coverage==7.6.4", + "isort==5.13.2", + "mypy==1.13.0", + "pytest==8.3.3", + "requests==2.32.3", + "PyMySQL==1.1.1", + "ruff==0.7.1", + "types-mock==5.1.0.20240425", + "types-requests==2.32.0.20241016", ] [project.entry-points."pytest11"] @@ -66,6 +66,7 @@ extend-exclude = [ ".venv", "migrations", ] +[tool.ruff.lint] ignore = [ "E402", "E501", @@ -90,4 +91,4 @@ ignore_missing_imports = true [tool.pytest.ini_options] norecursedirs = [".git", ".tox", "dist", "build"] -testpaths = ["test"] \ No newline at end of file +testpaths = ["test"] diff --git a/test_requirements.txt b/test_requirements.txt index 8f8f323..f809a14 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,9 +1,9 @@ -coverage==7.3.2 -isort==5.12.0 -mypy==1.7.1 -pytest==7.4.3 -requests==2.31.0 -PyMySQL==1.1.0 -ruff==0.1.6 -types-mock==5.1.0.3 -types-requests==2.31.0.10 \ No newline at end of file +coverage==7.6.4 +isort==5.13.2 +mypy==1.13.0 +pytest==8.3.3 +requests==2.32.3 +PyMySQL==1.1.1 +ruff==0.7.1 +types-mock==5.1.0.20240425 +types-requests==2.32.0.20241016 diff --git a/tox.ini b/tox.ini index cd736aa..927308f 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,7 @@ deps = sqlalchemy>=2.0,<3 commands = isort --verbose --check-only --diff nameko_sqlalchemy test - ruff nameko_sqlalchemy test + ruff check nameko_sqlalchemy test mypy nameko_sqlalchemy test [testenv:check14] @@ -46,7 +46,7 @@ deps = sqlalchemy2-stubs commands = isort --verbose --check-only --diff nameko_sqlalchemy test - ruff nameko_sqlalchemy test + ruff check nameko_sqlalchemy test mypy nameko_sqlalchemy test [testenv:report] From eb3c3956638650f2db7ee593b5471fb41db0872a Mon Sep 17 00:00:00 2001 From: Zbigniew Siciarz Date: Fri, 25 Oct 2024 12:28:18 +0200 Subject: [PATCH 4/4] Fix copypaste error in known_first_party --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 6ad5896..33f3247 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -59,7 +59,7 @@ src_paths = [ "nameko_sqlalchemy/", "test/", ] -known_first_party = "nameko_chassis" +known_first_party = "nameko_sqlalchemy" [tool.ruff] extend-exclude = [