Remove support for old CKAN versions and Python 2

.github/workflows/test.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v3
-      - uses: actions/setup-python@v2
+      - uses: actions/setup-python@v4
         with:
           python-version: '3.7'
       - name: Install requirements
@@ -17,7 +17,7 @@ jobs:
     needs: lint
     strategy:
       matrix:
-        ckan-version: ["2.10", 2.9, 2.9-py2, 2.8, 2.7]
+        ckan-version: ["2.10", 2.9]
       fail-fast: false
 
     name: CKAN ${{ matrix.ckan-version }}
@@ -45,43 +45,24 @@ jobs:
 
     steps:
     - uses: actions/checkout@v3
-    - name: Install requirements (Python 3)
-      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.9-py2'}}
+    - name: Install requirements
       run: |
         pip install -r requirements.txt
         pip install -r dev-requirements.txt
-    - name: Install requirements (Python 2)
-      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.9-py2'}}
-      run: |
-        pip install -r requirements-py2.txt
-        pip install -r dev-requirements-py2.txt
     - name: Install requirements (common)
       run: |
         pip install -e .
         # Replace default path to CKAN core config file with the one on the container
         sed -i -e 's/use = config:.*/use = config:\/srv\/app\/src\/ckan\/test-core.ini/' test.ini
-    - name: Setup harvest extension (py3)
-      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' && matrix.ckan-version != '2.9-py2'}}
+    - name: Setup harvest extension
       run: |
         git clone https://github.com/ckan/ckanext-harvest
         pip install -e ckanext-harvest
         pip install -r ckanext-harvest/pip-requirements.txt
-    - name: Setup harvest extension (py2)
-      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' || matrix.ckan-version == '2.9-py2'}}
-      run: |
-        git clone https://github.com/ckan/ckanext-harvest --branch v1.4.2 --single-branch
-        pip install -e ckanext-harvest
-        pip install -r ckanext-harvest/requirements.txt
-    - name: Setup extension (CKAN >= 2.9)
-      if: ${{ matrix.ckan-version != '2.7' && matrix.ckan-version != '2.8' }}
+    - name: Setup extension
       run: |
         ckan -c test.ini db init
         ckan -c test.ini harvester initdb
-    - name: Setup extension (CKAN < 2.9)
-      if: ${{ matrix.ckan-version == '2.7' || matrix.ckan-version == '2.8' }}
-      run: |
-        paster --plugin=ckan db init -c test.ini
-        paster --plugin=ckanext-harvest harvester initdb -c test.ini
     - name: Run tests
       run: pytest --ckan-ini=test.ini --cov=ckanext.dcat --cov-report=xml --cov-append --disable-warnings ckanext/dcat/tests
     - name: Upload coverage report to codecov

CHANGELOG.md

@@ -3,6 +3,7 @@
 
 ## [Unreleased](https://github.com/ckan/ckanext-dcat/compare/v1.4.0...HEAD)
 
+* Remove support for old CKAN versions prior 2.9 and Python 2 (#244)
 * Update hooks to support CKAN 2.10 (#241)
 
 ## [v1.4.0](https://github.com/ckan/ckanext-dcat/compare/v1.3.0...v1.4.0) - 2022-12-05

README.md

@@ -78,10 +78,6 @@ These are implemented internally using:
 
         (pyenv) $ pip install -r ckanext-dcat/requirements.txt
 
-    > **Note**
-    >
-    > If you are running on Python 2.7 or 3.6 please use `requirements-py2-py36.txt` instead
-
 4.  Enable the required plugins in your ini file:
 
         ckan.plugins = dcat dcat_rdf_harvester dcat_json_harvester dcat_json_interface structured_data

ckanext/dcat/controllers.py

@@ -4,13 +4,7 @@
 import ckanext.dcat.utils as utils
 
 
-if toolkit.check_ckan_version(min_version='2.1'):
-    BaseController = toolkit.BaseController
-else:
-    from ckan.lib.base import BaseController
-
-
-class DCATController(BaseController):
+class DCATController(toolkit.BaseController):
 
     def read_catalog(self, _format=None):
         return utils.read_catalog_page(_format)

ckanext/dcat/harvesters/base.py

@@ -1,7 +1,6 @@
 import os
 import logging
 
-import six
 import requests
 import rdflib
 
@@ -86,7 +85,7 @@ def _get_content_and_type(self, url, harvest_job, page=1,
                 r = session.get(url, stream=True)
 
             length = 0
-            content = '' if six.PY2 else b''
+            content = b''
             for chunk in r.iter_content(chunk_size=self.CHUNK_SIZE):
                 content = content + chunk
 
@@ -97,8 +96,7 @@ def _get_content_and_type(self, url, harvest_job, page=1,
                                             harvest_job)
                     return None, None
 
-            if not six.PY2:
-                content = content.decode('utf-8')
+            content = content.decode('utf-8')
 
             if content_type is None and r.headers.get('content-type'):
                 content_type = r.headers.get('content-type').split(";", 1)[0]

ckanext/dcat/harvesters/rdf.py

@@ -6,8 +6,6 @@
 import hashlib
 import traceback
 
-import six
-
 import ckan.plugins as p
 import ckan.model as model
 
@@ -174,10 +172,7 @@ def gather_stage(self, harvest_job):
 
             content_hash = hashlib.md5()
             if content:
-                if six.PY2:
-                    content_hash.update(content)
-                else:
-                    content_hash.update(content.encode('utf8'))
+                content_hash.update(content.encode('utf8'))
 
             if last_content_hash:
                 if content_hash.digest() == last_content_hash.digest():

ckanext/dcat/logic.py

@@ -1,7 +1,6 @@
 from __future__ import division
 import math
 
-import six
 from ckantoolkit import config
 from dateutil.parser import parse as dateutil_parse
 
@@ -155,7 +154,7 @@ def _page_url(page):
             qs = '&'.join(
                 ['{0}={1}'.format(
                     p[0],
-                    p[1].encode('utf8') if six.PY2 else p[1]
+                    p[1]
                     ) for p in params
                 ]
             )

ckanext/dcat/plugins/__init__.py

@@ -3,8 +3,6 @@
 from builtins import object
 import os
 
-import six
-
 from ckantoolkit import config
 
 from ckan import plugins as p
@@ -23,14 +21,7 @@ class DefaultTranslation(object):
                                 )
 from ckanext.dcat import utils
 
-if p.toolkit.check_ckan_version('2.9'):
-    from ckanext.dcat.plugins.flask_plugin import (
-        MixinDCATPlugin, MixinDCATJSONInterface
-    )
-else:
-    from ckanext.dcat.plugins.pylons_plugin import (
-        MixinDCATPlugin, MixinDCATJSONInterface
-    )
+from ckanext.dcat.plugins.flask_plugin import MixinDCATPlugin, MixinDCATJSONInterface
 
 
 CUSTOM_ENDPOINT_CONFIG = 'ckanext.dcat.catalog_endpoint'
@@ -47,8 +38,7 @@ class DCATPlugin(MixinDCATPlugin, p.SingletonPlugin, DefaultTranslation):
     p.implements(p.IActions, inherit=True)
     p.implements(p.IAuthFunctions, inherit=True)
     p.implements(p.IPackageController, inherit=True)
-    if p.toolkit.check_ckan_version(min_version='2.5.0'):
-        p.implements(p.ITranslation, inherit=True)
+    p.implements(p.ITranslation, inherit=True)
 
     # ITranslation
 
@@ -118,7 +108,7 @@ def after_dataset_show(self, context, data_dict):
             field_labels = utils.field_labels()
 
             def set_titles(object_dict):
-                for key, value in six.iteritems(object_dict.copy()):
+                for key, value in object_dict.copy().items():
                     if key in field_labels:
                         object_dict[field_labels[key]] = object_dict[key]
                         del object_dict[key]

ckanext/dcat/profiles.py

@@ -4,12 +4,12 @@
 import datetime
 import json
 
-import six
-from six.moves.urllib.parse import quote
+from urllib.parse import quote
 
 from dateutil.parser import parse as parse_date
 
 from ckantoolkit import config
+from ckantoolkit import url_for
 
 import rdflib
 from rdflib import URIRef, BNode, Literal
@@ -20,7 +20,6 @@
 from ckan.model.license import LicenseRegister
 from ckan.plugins import toolkit
 from ckan.lib.munge import munge_tag
-from ckanext.dcat.urls import url_for
 from ckanext.dcat.utils import resource_uri, publisher_uri_organization_fallback, DCAT_EXPOSE_SUBCATALOGS, DCAT_CLEAN_TAGS
 
 DCT = Namespace("http://purl.org/dc/terms/")
@@ -560,7 +559,7 @@ def _access_rights(self, subject, predicate):
                 result = self._object_value(obj, RDFS.label)
             elif isinstance(obj, Literal) or isinstance(obj, URIRef):
                 # unicode_safe not include Literal or URIRef
-                result = six.text_type(obj)
+                result = str(obj)
         return result
 
     def _distribution_format(self, distribution, normalize_ckan_format=True):
@@ -588,7 +587,7 @@ def _distribution_format(self, distribution, normalize_ckan_format=True):
         2. label of dct:format if it is an instance of dct:IMT (see above)
         3. value of dct:format if it is an URIRef and doesn't look like an IANA type
 
-        If `normalize_ckan_format` is True and using CKAN>=2.3, the label will
+        If `normalize_ckan_format` is True the label will
         be tried to match against the standard list of formats that is included
         with CKAN core
         (https://github.com/ckan/ckan/blob/master/ckan/config/resource_formats.json)
@@ -625,8 +624,7 @@ def _distribution_format(self, distribution, normalize_ckan_format=True):
                 else:
                     label = format_uri
 
-        if ((imt or label) and normalize_ckan_format and
-                toolkit.check_ckan_version(min_version='2.3')):
+        if ((imt or label) and normalize_ckan_format):
             import ckan.config
             from ckan.lib import helpers
 

ckanext/dcat/tests/test_base_profile.py

@@ -2,7 +2,6 @@
 from builtins import object
 
 import pytest
-from six import string_types
 
 from rdflib import Graph, URIRef, Literal
 from rdflib.namespace import Namespace
@@ -103,7 +102,7 @@ def test_object_value(self):
         value = p._object_value(URIRef('http://example.org/datasets/1'),
                                 DCT.title)
 
-        assert isinstance(value, string_types)
+        assert isinstance(value, str)
         assert value == 'Test Dataset 1'
 
     def test_object_value_not_found(self):
@@ -127,7 +126,7 @@ def test_object_value_default_lang(self):
         value = p._object_value(URIRef('http://example.org/datasets/1'),
                                 DCT.title)
 
-        assert isinstance(value, string_types)
+        assert isinstance(value, str)
         assert value == 'Test Datensatz 1'
 
     @pytest.mark.ckan_config('ckan.locale_default', 'fr')
@@ -140,7 +139,7 @@ def test_object_value_default_lang_not_in_graph(self):
         value = p._object_value(URIRef('http://example.org/datasets/1'),
                                 DCT.title)
 
-        assert isinstance(value, string_types)
+        assert isinstance(value, str)
         # FR is not in graph, so either node may be used
         assert value.startswith('Test D')
         assert value.endswith(' 1')
@@ -156,7 +155,7 @@ def test_object_value_default_lang_fallback(self):
         value = p._object_value(URIRef('http://example.org/datasets/1'),
                                 DCT.title)
 
-        assert isinstance(value, string_types)
+        assert isinstance(value, str)
         # without config parameter, EN is used as default
         assert value == 'Test Dataset 1 (EN)'
 
@@ -166,7 +165,7 @@ def test_object_value_default_lang_missing_lang_param(self):
         value = p._object_value(URIRef('http://example.org/datasets/1'),
                                 DCT.title)
 
-        assert isinstance(value, string_types)
+        assert isinstance(value, str)
         assert value == 'Test Dataset 1'
 
     def test_object_int(self):
@@ -234,7 +233,7 @@ def test_object_list(self):
                                      DCAT.keyword)
 
         assert isinstance(value, list)
-        assert isinstance(value[0], string_types)
+        assert isinstance(value[0], str)
         assert len(value) == 2
         assert sorted(value) == ['moon', 'space']
 

ckanext/dcat/tests/test_blueprints.py

@@ -4,19 +4,19 @@
 import time
 
 from collections import OrderedDict
-from six.moves.urllib.parse import urlparse, parse_qs, urlencode, urlunparse
+from urllib.parse import urlparse, parse_qs, urlencode, urlunparse
 
 import pytest
 
 from ckan import plugins as p
 
 from rdflib import Graph
+from ckantoolkit import url_for
 from ckantoolkit.tests import factories
 
 from ckanext.dcat.processors import RDFParser
 from ckanext.dcat.profiles import RDF, DCAT
 from ckanext.dcat.processors import HYDRA
-from ckanext.dcat.urls import url_for
 
 
 def _sort_query_params(url):

ckanext/dcat/tests/test_euro_dcatap_profile_parse.py

@@ -499,11 +499,8 @@ def test_distribution_format_imt_only(self):
         datasets = [d for d in p.datasets()]
 
         resource = datasets[0]['resources'][0]
-        if toolkit.check_ckan_version(min_version='2.3'):
-            assert resource['format'] == u'CSV'
-            assert resource['mimetype'] == u'text/csv'
-        else:
-            assert resource['format'] == u'text/csv'
+        assert resource['format'] == u'CSV'
+        assert resource['mimetype'] == u'text/csv'
 
     @pytest.mark.ckan_config('ckanext.dcat.normalize_ckan_format', False)
     def test_distribution_format_imt_only_normalize_false(self):
@@ -638,11 +635,8 @@ def test_distribution_format_format_normalized(self):
 
         resource = datasets[0]['resources'][0]
 
-        if toolkit.check_ckan_version(min_version='2.3'):
-            assert resource['format'] == u'CSV'
-            assert resource['mimetype'] == u'text/csv'
-        else:
-            assert resource['format'] == u'Comma Separated Values'
+        assert resource['format'] == u'CSV'
+        assert resource['mimetype'] == u'text/csv'
 
     def test_distribution_format_IMT_field(self):
         g = Graph()

ckanext/dcat/tests/test_harvester.py

@@ -6,7 +6,6 @@
 from collections import defaultdict
 import re
 
-import six
 import pytest
 import responses
 try:
@@ -700,7 +699,7 @@ def test_harvest_default_file_size(self, mock_save_gather_error):
         # this as well
         responses.add(responses.HEAD, self.ttl_mock_url,
                                status=200, content_type=self.ttl_content_type,
-                               adding_headers = {'content-length': six.text_type(actual_file_size)})
+                               adding_headers = {'content-length': str(actual_file_size)})
 
         harvest_source = self._create_harvest_source(self.ttl_mock_url)
         # Create new job for the source
@@ -729,7 +728,7 @@ def test_harvest_config_file_size(self, mock_save_gather_error):
         # this as well  , content_length=file_size
         responses.add(responses.HEAD, self.ttl_mock_url,
                                status=200, content_type=self.ttl_content_type,
-                               adding_headers = {'content-length': six.text_type(actual_file_size)})
+                               adding_headers = {'content-length': str(actual_file_size)})
 
         harvest_source = self._create_harvest_source(self.ttl_mock_url)
         # Create new job for the source

ckanext/dcat/utils.py

@@ -10,27 +10,15 @@
 
 from ckantoolkit import config, h
 
-try:
-    # CKAN >= 2.6
-    from ckan.exceptions import HelperError
-except ImportError:
-    # CKAN < 2.6
-    class HelperError(Exception):
-        pass
+from ckan.exceptions import HelperError
 
 from ckan import model
 import ckan.plugins.toolkit as toolkit
 
 from ckanext.dcat.exceptions import RDFProfileException
 
-if toolkit.check_ckan_version(max_version='2.8.99'):
-    from ckan.controllers.package import PackageController
-    from ckan.controllers.home import HomeController
-    read_endpoint = PackageController().read
-    index_endpoint = HomeController().index
-else:
-    from ckan.views.home import index as index_endpoint
-    from ckan.views.dataset import read as read_endpoint
+from ckan.views.home import index as index_endpoint
+from ckan.views.dataset import read as read_endpoint
 
 _ = toolkit._
 
@@ -412,10 +400,7 @@ def read_dataset_page(_id, _format):
         _format = check_access_header()
 
     if not _format:
-        if toolkit.check_ckan_version(max_version='2.8.99'):
-            return read_endpoint(_id)
-        else:
-            return read_endpoint(_get_package_type(_id), _id)
+        return read_endpoint(_get_package_type(_id), _id)
 
     _profiles = toolkit.request.params.get('profiles')
     if _profiles:
@@ -429,12 +414,9 @@ def read_dataset_page(_id, _format):
     except (toolkit.ValidationError, RDFProfileException) as e:
         toolkit.abort(409, str(e))
 
-    if toolkit.check_ckan_version(max_version='2.8.99'):
-        toolkit.response.headers.update({'Content-type': CONTENT_TYPES[_format]})
-    else:
-        from flask import make_response
-        response = make_response(response)
-        response.headers['Content-type'] = CONTENT_TYPES[_format]
+    from flask import make_response
+    response = make_response(response)
+    response.headers['Content-type'] = CONTENT_TYPES[_format]
 
     return response
 
@@ -463,19 +445,12 @@ def read_catalog_page(_format):
     except (toolkit.ValidationError, RDFProfileException) as e:
         toolkit.abort(409, str(e))
 
-    if toolkit.check_ckan_version(max_version='2.8.99'):
-        toolkit.response.headers.update(
-            {'Content-type': CONTENT_TYPES[_format]})
-    else:
-        from flask import make_response
-        response = make_response(response)
-        response.headers['Content-type'] = CONTENT_TYPES[_format]
+    from flask import make_response
+    response = make_response(response)
+    response.headers['Content-type'] = CONTENT_TYPES[_format]
 
     return response
 
 
 def get_endpoint(_type='dataset'):
-    if toolkit.check_ckan_version(min_version='2.9'):
-        return 'dcat.read_dataset' if _type == 'dataset' else 'dcat.read_catalog'
-    else:
-        return 'dcat_dataset' if _type == 'dataset' else 'dcat_catalog'
+    return 'dcat.read_dataset' if _type == 'dataset' else 'dcat.read_catalog'

requirements.txt

@@ -1,5 +1,4 @@
 rdflib==6.1.1
 geomet>=0.2.0
 ckantoolkit>=0.0.7
-future>=0.18.2
-six
+future>=0.18.2

setup.py

@@ -45,9 +45,6 @@
     euro_dcat_ap_2=ckanext.dcat.profiles:EuropeanDCATAP2Profile
     schemaorg=ckanext.dcat.profiles:SchemaOrgProfile
 
-    [paste.paster_command]
-    generate_static = ckanext.dcat.commands:GenerateStaticDCATCommand
-
     [babel.extractors]
     ckan = ckan.lib.extract:extract_ckan
     ''',