From b9767b192f7c6978ff872878a781dda8100219ff Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 10:01:52 +0100 Subject: [PATCH 1/7] Remove old compatibility code --- ckanext/dcat/harvesters/_json.py | 1 - ckanext/dcat/harvesters/rdf.py | 1 - ckanext/dcat/logic.py | 1 - ckanext/dcat/plugins/__init__.py | 1 - ckanext/dcat/processors.py | 4 ---- ckanext/dcat/tests/harvester/test_harvester.py | 3 --- ckanext/dcat/tests/harvester/test_json_harvester.py | 3 --- ckanext/dcat/tests/logic/test_logic.py | 2 -- ckanext/dcat/tests/profiles/base/test_base_parser.py | 2 -- ckanext/dcat/tests/profiles/base/test_base_profile.py | 2 -- ckanext/dcat/tests/profiles/base/test_base_serializer.py | 1 - ckanext/dcat/tests/profiles/croissant/test_serialize.py | 1 - .../tests/profiles/dcat_ap/test_euro_dcatap_profile_parse.py | 2 -- .../profiles/dcat_ap/test_euro_dcatap_profile_serialize.py | 2 -- .../profiles/dcat_ap_2/test_euro_dcatap_2_profile_parse.py | 2 -- .../dcat_ap_2/test_euro_dcatap_2_profile_serialize.py | 2 -- .../profiles/schemaorg/test_schemaorg_profile_serialize.py | 1 - ckanext/dcat/tests/test_blueprints.py | 2 -- ckanext/dcat/tests/test_converters.py | 1 - ckanext/dcat/utils.py | 1 - requirements.txt | 1 - 21 files changed, 36 deletions(-) diff --git a/ckanext/dcat/harvesters/_json.py b/ckanext/dcat/harvesters/_json.py index c058bc54..6600ec86 100644 --- a/ckanext/dcat/harvesters/_json.py +++ b/ckanext/dcat/harvesters/_json.py @@ -1,4 +1,3 @@ -from builtins import str import json import logging from hashlib import sha1 diff --git a/ckanext/dcat/harvesters/rdf.py b/ckanext/dcat/harvesters/rdf.py index 7bc402e3..dddd097a 100644 --- a/ckanext/dcat/harvesters/rdf.py +++ b/ckanext/dcat/harvesters/rdf.py @@ -1,4 +1,3 @@ -from builtins import str from past.builtins import basestring import json import uuid diff --git a/ckanext/dcat/logic.py b/ckanext/dcat/logic.py index e476e884..f5af6418 100644 --- a/ckanext/dcat/logic.py +++ b/ckanext/dcat/logic.py @@ -1,4 +1,3 @@ -from __future__ import division import math from ckantoolkit import config diff --git a/ckanext/dcat/plugins/__init__.py b/ckanext/dcat/plugins/__init__.py index bd5cf9d3..3e3625d9 100644 --- a/ckanext/dcat/plugins/__init__.py +++ b/ckanext/dcat/plugins/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from builtins import object from functools import wraps import os import json diff --git a/ckanext/dcat/processors.py b/ckanext/dcat/processors.py index c54c0165..79f35821 100644 --- a/ckanext/dcat/processors.py +++ b/ckanext/dcat/processors.py @@ -1,7 +1,3 @@ -from __future__ import print_function - -from builtins import str -from builtins import object import sys import argparse import xml diff --git a/ckanext/dcat/tests/harvester/test_harvester.py b/ckanext/dcat/tests/harvester/test_harvester.py index fda47e5f..8795f58c 100644 --- a/ckanext/dcat/tests/harvester/test_harvester.py +++ b/ckanext/dcat/tests/harvester/test_harvester.py @@ -1,8 +1,5 @@ # -*- coding: utf-8 -*- -from builtins import str -from builtins import range -from builtins import object from collections import defaultdict import re diff --git a/ckanext/dcat/tests/harvester/test_json_harvester.py b/ckanext/dcat/tests/harvester/test_json_harvester.py index 12e9bdf1..2414a7c8 100644 --- a/ckanext/dcat/tests/harvester/test_json_harvester.py +++ b/ckanext/dcat/tests/harvester/test_json_harvester.py @@ -1,6 +1,3 @@ -from __future__ import absolute_import -from builtins import object - import responses import pytest try: diff --git a/ckanext/dcat/tests/logic/test_logic.py b/ckanext/dcat/tests/logic/test_logic.py index 0b57e8a1..bbedfa49 100644 --- a/ckanext/dcat/tests/logic/test_logic.py +++ b/ckanext/dcat/tests/logic/test_logic.py @@ -1,5 +1,3 @@ -from builtins import range -from builtins import object try: from unittest import mock diff --git a/ckanext/dcat/tests/profiles/base/test_base_parser.py b/ckanext/dcat/tests/profiles/base/test_base_parser.py index 20a409ac..51575c91 100644 --- a/ckanext/dcat/tests/profiles/base/test_base_parser.py +++ b/ckanext/dcat/tests/profiles/base/test_base_parser.py @@ -1,5 +1,3 @@ -from builtins import str -from builtins import object import pytest diff --git a/ckanext/dcat/tests/profiles/base/test_base_profile.py b/ckanext/dcat/tests/profiles/base/test_base_profile.py index f7b6da9f..42d36f78 100644 --- a/ckanext/dcat/tests/profiles/base/test_base_profile.py +++ b/ckanext/dcat/tests/profiles/base/test_base_profile.py @@ -1,5 +1,3 @@ -from builtins import str -from builtins import object import pytest diff --git a/ckanext/dcat/tests/profiles/base/test_base_serializer.py b/ckanext/dcat/tests/profiles/base/test_base_serializer.py index 1ff7a356..d8c7af00 100644 --- a/ckanext/dcat/tests/profiles/base/test_base_serializer.py +++ b/ckanext/dcat/tests/profiles/base/test_base_serializer.py @@ -1,4 +1,3 @@ -from builtins import str from ckantoolkit import config diff --git a/ckanext/dcat/tests/profiles/croissant/test_serialize.py b/ckanext/dcat/tests/profiles/croissant/test_serialize.py index e7c5265c..e2b7ed16 100644 --- a/ckanext/dcat/tests/profiles/croissant/test_serialize.py +++ b/ckanext/dcat/tests/profiles/croissant/test_serialize.py @@ -1,4 +1,3 @@ -from builtins import str import json from unittest import mock import uuid diff --git a/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_parse.py b/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_parse.py index 3300f8c3..8da1c634 100644 --- a/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_parse.py +++ b/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_parse.py @@ -1,5 +1,3 @@ -from builtins import str -from builtins import object import json import pytest diff --git a/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_serialize.py b/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_serialize.py index 9ba200be..826aef47 100644 --- a/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_serialize.py +++ b/ckanext/dcat/tests/profiles/dcat_ap/test_euro_dcatap_profile_serialize.py @@ -1,5 +1,3 @@ -from builtins import str -from builtins import object import json import uuid from decimal import Decimal diff --git a/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_parse.py b/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_parse.py index 1bce901c..04363e2f 100644 --- a/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_parse.py +++ b/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_parse.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from builtins import str -from builtins import object import os import json diff --git a/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_serialize.py b/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_serialize.py index f6e8ae6d..8f9ebd87 100644 --- a/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_serialize.py +++ b/ckanext/dcat/tests/profiles/dcat_ap_2/test_euro_dcatap_2_profile_serialize.py @@ -1,7 +1,5 @@ # -*- coding: utf-8 -*- -from builtins import str -from builtins import object import json from decimal import Decimal import six diff --git a/ckanext/dcat/tests/profiles/schemaorg/test_schemaorg_profile_serialize.py b/ckanext/dcat/tests/profiles/schemaorg/test_schemaorg_profile_serialize.py index dec37644..8ec92e1c 100644 --- a/ckanext/dcat/tests/profiles/schemaorg/test_schemaorg_profile_serialize.py +++ b/ckanext/dcat/tests/profiles/schemaorg/test_schemaorg_profile_serialize.py @@ -1,4 +1,3 @@ -from builtins import str import json import pytest diff --git a/ckanext/dcat/tests/test_blueprints.py b/ckanext/dcat/tests/test_blueprints.py index e6bb3964..6d42f26d 100644 --- a/ckanext/dcat/tests/test_blueprints.py +++ b/ckanext/dcat/tests/test_blueprints.py @@ -1,6 +1,4 @@ # -*- coding: utf-8 -*- -from builtins import str -from builtins import range import json import time diff --git a/ckanext/dcat/tests/test_converters.py b/ckanext/dcat/tests/test_converters.py index 249bec68..5b1578b0 100644 --- a/ckanext/dcat/tests/test_converters.py +++ b/ckanext/dcat/tests/test_converters.py @@ -1,4 +1,3 @@ -from builtins import object import os import json import difflib diff --git a/ckanext/dcat/utils.py b/ckanext/dcat/utils.py index 99254d7a..0931aecc 100644 --- a/ckanext/dcat/utils.py +++ b/ckanext/dcat/utils.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- -from builtins import str import logging import uuid import simplejson as json diff --git a/requirements.txt b/requirements.txt index f55a831e..0ef211b4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,4 +2,3 @@ rdflib>=6.1.1,<7.2.0 pyld>=2.0.4,<3.0.0 geomet>=0.2.0 ckantoolkit>=0.0.7 -future>=0.18.2 From 02a4e7d3712d980a3b1a897483f28a3b3c3ed869 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 10:38:31 +0100 Subject: [PATCH 2/7] Include dependencies dynamically in pyproject.toml Keep the requirements.txt and dev-requirements.txt around and acting like the source of truth, o all existing installation processes keep working as intended but include them dynamically in pyproject.toml to integrate with modern workflows, and be able to just run pip install ckanext-dcat --- MANIFEST.in | 4 ++++ pyproject.toml | 8 ++++++-- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 02c1cab4..82741144 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,2 +1,6 @@ recursive-include ckanext/dcat/i18n * recursive-include ckanext/dcat/templates * +recursive-include ckanext *.yaml +recursive-include ckanext *.yml +include requirements.txt +include dev-requirements.txt diff --git a/pyproject.toml b/pyproject.toml index 6adb8439..9c595e83 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,7 @@ [project] name = "ckanext-dcat" version = "2.2.0" +dynamic = ["dependencies", "optional-dependencies"] description = "Plugins for exposing and consuming DCAT metadata on CKAN" authors = [ {name = "Adrià Mercader", email = "amercadero@gmail.com"} @@ -31,7 +32,6 @@ keywords = [ "RDF", "Semantic data" ] -dependencies = [] [project.urls] Documentation = "https://docs.ckan.org/projects/ckanext-dcat" @@ -40,9 +40,13 @@ Issues = "https://github.com/ckan/ckanext-dcat/issues" Changelog = "https://docs.ckan.org/projects/ckanext-dcat/en/latest/changelog/" [build-system] -requires = ["setuptools"] +requires = ["setuptools>=62.6.0"] build-backend = "setuptools.build_meta" +[tool.setuptools.dynamic] +dependencies = {file = ["requirements.txt"]} +optional-dependencies = {dev = { file = ["dev-requirements.txt"] }} + [project.entry-points."ckan.plugins"] dcat_xml_harvester = "ckanext.dcat.harvesters:DCATXMLHarvester" dcat_json_harvester = "ckanext.dcat.harvesters:DCATJSONHarvester" From d11755cb1b87dc3b50dcbceeb20e700cad44c186 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 10:54:32 +0100 Subject: [PATCH 3/7] Update installation docs --- docs/getting-started.md | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/docs/getting-started.md b/docs/getting-started.md index f259c1b4..1e3e26f5 100644 --- a/docs/getting-started.md +++ b/docs/getting-started.md @@ -5,17 +5,23 @@ 1. Install the extension in your virtualenv: - (pyenv) $ pip install -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat + pip install ckanext-dcat -2. Install the extension requirements: - (pyenv) $ pip install -r ckanext-dcat/requirements.txt + !!! Note -3. Enable the required plugins in your ini file: + For versions older than **ckanext-dcat 2.3.0** use the old legacy version of the install: + + pip install -e git+https://github.com/ckan/ckanext-dcat.git#egg=ckanext-dcat + + pip install -r ckanext-dcat/requirements.txt + + +2. Enable the required plugins in your ini file: ckan.plugins = dcat dcat_rdf_harvester structured_data -4. To use the pre-built schemas, install [ckanext-scheming](https://github.com/ckan/ckanext-scheming): +3. To use the pre-built schemas, install [ckanext-scheming](https://github.com/ckan/ckanext-scheming): pip install -e "git+https://github.com/ckan/ckanext-scheming.git#egg=ckanext-scheming" From 7f5d7a6272e8b2df1390b15ce1d3b7acc7a00f8e Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 11:00:49 +0100 Subject: [PATCH 4/7] Remove old compatibility code --- ckanext/dcat/converters.py | 1 - ckanext/dcat/harvesters/rdf.py | 1 - 2 files changed, 2 deletions(-) diff --git a/ckanext/dcat/converters.py b/ckanext/dcat/converters.py index 8129f7ad..0b9fd09d 100644 --- a/ckanext/dcat/converters.py +++ b/ckanext/dcat/converters.py @@ -1,4 +1,3 @@ -from past.builtins import basestring import logging log = logging.getLogger(__name__) diff --git a/ckanext/dcat/harvesters/rdf.py b/ckanext/dcat/harvesters/rdf.py index dddd097a..0381da5d 100644 --- a/ckanext/dcat/harvesters/rdf.py +++ b/ckanext/dcat/harvesters/rdf.py @@ -1,4 +1,3 @@ -from past.builtins import basestring import json import uuid import logging From fde6330e1c57943d9e973e8c078086fa95304c66 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 11:06:19 +0100 Subject: [PATCH 5/7] Replace usage of basestring --- ckanext/dcat/converters.py | 4 ++-- ckanext/dcat/harvesters/rdf.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ckanext/dcat/converters.py b/ckanext/dcat/converters.py index 0b9fd09d..cb0d89e2 100644 --- a/ckanext/dcat/converters.py +++ b/ckanext/dcat/converters.py @@ -23,7 +23,7 @@ def dcat_to_ckan(dcat_dict): package_dict['extras'].append({'key': 'guid', 'value': dcat_dict.get('identifier')}) dcat_publisher = dcat_dict.get('publisher') - if isinstance(dcat_publisher, basestring): + if isinstance(dcat_publisher, str): package_dict['extras'].append({'key': 'dcat_publisher_name', 'value': dcat_publisher}) elif isinstance(dcat_publisher, dict) and dcat_publisher.get('name'): package_dict['extras'].append({'key': 'dcat_publisher_name', 'value': dcat_publisher.get('name')}) @@ -38,7 +38,7 @@ def dcat_to_ckan(dcat_dict): }) dcat_creator = dcat_dict.get('creator') - if isinstance(dcat_creator, basestring): + if isinstance(dcat_creator, str): package_dict['extras'].append({'key': 'dcat_creator_name', 'value': dcat_creator}) elif isinstance(dcat_creator, dict) and dcat_creator.get('name'): if dcat_creator.get('name'): diff --git a/ckanext/dcat/harvesters/rdf.py b/ckanext/dcat/harvesters/rdf.py index 0381da5d..a22e0b97 100644 --- a/ckanext/dcat/harvesters/rdf.py +++ b/ckanext/dcat/harvesters/rdf.py @@ -134,7 +134,7 @@ def validate_config(self, source_config): source_config_obj = json.loads(source_config) if 'rdf_format' in source_config_obj: rdf_format = source_config_obj['rdf_format'] - if not isinstance(rdf_format, basestring): + if not isinstance(rdf_format, str): raise ValueError('rdf_format must be a string') supported_formats = RDFParser().supported_formats() if rdf_format not in supported_formats: From e26afbe866d3babce86a03a127efe2abd9db32a2 Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 11:07:30 +0100 Subject: [PATCH 6/7] Remove upper bounds in requirements As per these recommendations: https://github.com/ckan/ckan/issues/8382#issuecomment-2353188970 --- requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 0ef211b4..73105e48 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -rdflib>=6.1.1,<7.2.0 -pyld>=2.0.4,<3.0.0 +rdflib>=6.1.1 +pyld>=2.0.4 geomet>=0.2.0 ckantoolkit>=0.0.7 From 275d1bfc3804fa3e4133c959ddde2132c2e861ba Mon Sep 17 00:00:00 2001 From: amercader Date: Tue, 25 Feb 2025 11:19:09 +0100 Subject: [PATCH 7/7] Add Github workflow for publishing to PyPI --- .github/workflows/publish-pypi.yml | 54 ++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 .github/workflows/publish-pypi.yml diff --git a/.github/workflows/publish-pypi.yml b/.github/workflows/publish-pypi.yml new file mode 100644 index 00000000..a48df56d --- /dev/null +++ b/.github/workflows/publish-pypi.yml @@ -0,0 +1,54 @@ +name: Publish to PyPI + +# Publish to PyPI when a tag is pushed +on: + push: + tags: + - 'v**' + +jobs: + build: + if: github.repository == 'ckan/ckanext-dcat' + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.9" + - name: Install pypa/build + run: python3 -m pip install build --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artifact@v4 + with: + name: python-package-distributions + path: dist/ + + publish-to-pypi: + name: Publish Python distribution on PyPI + needs: + - build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/ckanext-dcat + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/download-artifact@v4 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 + + publishSkipped: + if: github.repository != 'ckan/ckanext-dcat' + runs-on: ubuntu-latest + steps: + - run: | + echo "## Skipping PyPI publish on downstream repository" >> $GITHUB_STEP_SUMMARY