Migration to Python 3

.github/workflows/python_actions.yml

@@ -11,11 +11,11 @@ jobs:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2
       with:
-        python-version: 2.7
+        python-version: 3.8
 
     - name: Install dependencies
       run: |
-        python -m pip install --upgrade wheel setuptools pip
+        python -m pip install --upgrade setuptools pip
         pip install -U -r requirements.txt
         pip install -U -r dev-requirements.txt
 

.gitignore

@@ -1,3 +1,4 @@
 *pyc
 .idea
 python
+venv

alembic/env.py

@@ -1,4 +1,5 @@
 from __future__ import with_statement
+from __future__ import print_function
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
@@ -51,7 +52,7 @@ def get_app_config(key):
     app = application.create_app()
 
     with app.app_context() as c:
-        print 'Getting actual config for', key, app.config.get(key)
+        print('Getting actual config for', key, app.config.get(key))
         return app.config.get(key)
 
 def run_migrations_online():

dev-requirements.txt

@@ -1,10 +1,10 @@
 flask-cors==2.0.0
-flask-testing==0.6.2
-httpretty==0.8.10
+Flask-Testing==0.8.1
+httpretty==0.9.5
 pep257==0.5.0
-pytest==2.8.2
+pytest==6.2.4
 pytest-cache==1.0
-pytest-cov==2.2.0
+pytest-cov==2.12.0
 pytest-pep8==1.0.6
 mock==2.0.0
 coverage==5.2.1

requirements.txt

@@ -1,2 +1,3 @@
-git+https://github.com/adsabs/ADSMicroserviceUtils.git@v1.1.9
+git+https://github.com/adsabs/ADSMicroserviceUtils.git@v1.2.0
 alembic==0.8.1
+future==0.18.2

solr/__init__.py

@@ -1,2 +1,3 @@
-import app
-from app import create_app
+from __future__ import absolute_import
+from . import app
+from .app import create_app

solr/app.py

@@ -1,8 +1,10 @@
+from __future__ import absolute_import
+from past.builtins import basestring
 from flask import Flask, make_response, jsonify
-from flask.ext.restful import Api
-from flask.ext.discoverer import Discoverer
-from flask.ext.sqlalchemy import SQLAlchemy
-from views import StatusView, Tvrh, Search, Qtree, BigQuery
+from flask_restful import Api
+from flask_discoverer import Discoverer
+from flask_sqlalchemy import SQLAlchemy
+from .views import StatusView, Tvrh, Search, Qtree, BigQuery
 from adsmutils import ADSFlask
 
 def create_app(**config):

solr/tests/unittests/test_solr.py

@@ -1,21 +1,20 @@
 import sys, os
 PROJECT_HOME = os.path.abspath(os.path.join(os.path.dirname(__file__), '../../'))
 sys.path.append(PROJECT_HOME)
-from flask.ext.testing import TestCase
+from flask_testing import TestCase
 from flask import url_for
 import unittest
 import httpretty
 import json
-import app
+from solr import app
 from werkzeug.security import gen_salt
 from werkzeug.datastructures import MultiDict
-from StringIO import StringIO
+from io import BytesIO
 from solr.tests.mocks import MockSolrResponse
-from views import SolrInterface
 from solr import views
+from solr.views import SolrInterface
 from models import Limits, Base
 import mock
-from StringIO import StringIO
 
 class TestSolrInterface(TestCase):
 
@@ -248,11 +247,11 @@ def request_callback(request, uri, headers):
             r = c.get(url_for('search'), query_string={'q': 'star'})
 
             # Two cookies (session and sroute)
-            self.assertEqual(len(r.data.split(';')), len(self.app.config.get("SOLR_SERVICE_FORWARDED_COOKIES")))
+            self.assertEqual(len(r.data.decode('utf-8').split(';')), len(self.app.config.get("SOLR_SERVICE_FORWARDED_COOKIES")))
 
             # This forwarded cookie should match the one we gave originally
             n_found_cookies_with_good_value = 0
-            for cookie in r.data.split(';'):
+            for cookie in r.data.decode('utf-8').split(';'):
                 key, value = cookie.split('=')
                 if key in self.app.config.get("SOLR_SERVICE_FORWARDED_COOKIES"):
                     self.assertEqual(value.strip(), cookie_value)
@@ -417,7 +416,7 @@ def test_docs_subquery(self):
                                  data={
                                      'q': '*:*',
                                      'fq': 'docs(hHGU1Ef-TpacAhicI3J8kQ)',
-                                     'big': (StringIO('foo\nbar'), 'bigname', 'big-query/csv')
+                                     'big': (BytesIO('foo\nbar'.encode('utf-8')), 'bigname', 'big-query/csv')
                                      },
                                  headers={'Authorization': 'Bearer foo'})
             # it made a request to retrieve library
@@ -568,7 +567,7 @@ def request_callback(request, uri, headers):
                     'q': '*:*',
                     'fl': 'bibcode',
                     'fq': '{!bitset}',
-                    'file_field': (StringIO(bibcodes), 'file', 'big-query/csv')
+                    'file_field': (BytesIO(bibcodes.encode('utf-8')), 'file', 'big-query/csv')
                 }
         )
 
@@ -581,13 +580,13 @@ def request_callback(request, uri, headers):
                 data={
                     'q': '*:*',
                     'fl': 'bibcode',
-                    'file_field': (StringIO(bibcodes), 'filename', 'big-query/csv'),
+                    'file_field': (BytesIO(bibcodes.encode('utf-8')), 'filename', 'big-query/csv'),
                     }
         )
 
         self.assertEqual(resp.status_code, 200)
-        self.assertTrue('bitset' not in resp.data)
-        self.assertTrue("Content-Disposition: form-data; name=\"file_field\"; filename=\"file_field\"\r\nContent-Type: big-query/csv" in resp.data)
+        self.assertTrue('bitset' not in resp.data.decode('utf-8'))
+        self.assertTrue("Content-Disposition: form-data; name=\"file_field\"; filename=\"file_field\"\\r\\nContent-Type: big-query/csv" in resp.data.decode('utf-8'))
 
 
         # Missing 'fq' parameter is filled in - but only when data (request.post(data=...)
@@ -599,8 +598,8 @@ def request_callback(request, uri, headers):
         )
 
         self.assertEqual(resp.status_code, 200)
-        self.assertTrue('fq=%7B%21bitset%7D' in resp.data)
-        self.assertTrue("Content-Disposition: form-data; name=\"old-bad-behaviour\"; filename=\"old-bad-behaviour\"\r\nContent-Type: big-query/csv" in resp.data)
+        self.assertTrue('fq=%7B%21bitset%7D' in resp.data.decode('utf-8'))
+        self.assertTrue("Content-Disposition: form-data; name=\"old-bad-behaviour\"; filename=\"old-bad-behaviour\"\\r\\nContent-Type: big-query/csv" in resp.data.decode('utf-8'))
 
 
 
@@ -612,20 +611,20 @@ def request_callback(request, uri, headers):
                     'q': '*:*',
                     'fl': 'bibcode',
                     'fq': '{!bitset compression = true}',
-                    'file_field': (StringIO(bibcodes), 'filename', 'big-query/csv'),
+                    'file_field': (BytesIO(bibcodes.encode('utf-8')), 'filename', 'big-query/csv'),
                     }
         )
 
         self.assertEqual(resp.status_code, 200)
-        self.assertTrue('fq=%7B%21bitset+compression+%3D+true%7D' in resp.data)
+        self.assertTrue('fq=%7B%21bitset+compression+%3D+true%7D' in resp.data.decode('utf-8'))
 
         # We now allow more content streams to be sent
         data = MultiDict([
             ('q', '*:*'),
             ('fl', 'bibcode'),
             ('fq', '{!bitset}'),
-            ('file_field', (StringIO(bibcodes), 'filename', 'big-query/csv')),
-            ('file_field', (StringIO(bibcodes), 'filename', 'big-query/csv')),
+            ('file_field', (BytesIO(bibcodes.encode('utf-8')), 'filename', 'big-query/csv')),
+            ('file_field', (BytesIO(bibcodes.encode('utf-8')), 'filename', 'big-query/csv')),
         ])
 
         resp = self.client.post(

solr/tests/unittests/test_webservices.py

@@ -1,6 +1,7 @@
-from flask.ext.testing import TestCase
+from flask_testing import TestCase
+from past.builtins import basestring
 import unittest
-import app
+from solr import app
 
 
 class TestWebservices(TestCase):
@@ -23,7 +24,7 @@ def test_ResourcesRoute(self):
             'methods': list,
             'description': basestring,
             'rate_limit': list,
-        }.iteritems():
+        }.items():
             # Assert each resource is described has the expected_field
             [self.assertIn(expected_field, v) for v in r.json.values()]
 

solr/views.py

@@ -1,21 +1,23 @@
+from __future__ import absolute_import
+from future import standard_library
+standard_library.install_aliases()
+from builtins import str
+from past.builtins import basestring
 from flask import current_app, request
-from flask.ext.restful import Resource
-from flask.ext.discoverer import advertise
+from flask_restful import Resource
+from flask_discoverer import advertise
 try:
     from flask_login import current_user
 except:
     # If solr service is not shipped with adsws, this will fail and it is ok
     pass
 import json
-from models import Limits
+from .models import Limits
 from sqlalchemy import or_
 from werkzeug.datastructures import MultiDict
-try:
-    from cStringIO import StringIO
-except:
-    from StringIO import StringIO
+from io import StringIO
 from io import BytesIO
-from urlparse import parse_qs
+from urllib.parse import parse_qs
 
 import requests # Do not use current_app.client but requests, to avoid re-using
                 # connections from a pool which would make solr ingress nginx
@@ -145,7 +147,7 @@ def apply_protective_filters(self, payload, user_id, protected_fields, key):
             for f in session.query(Limits).filter(Limits.uid==user_id, or_(Limits.field==x for x in protected_fields)).all():
                 if f.filter:
                     fl = u'{0},{1}'.format(fl, f.field)
-                    fq.append(unicode(f.filter))
+                    fq.append(str(f.filter))
                     payload['fl'] = fl
             session.commit()
 
@@ -173,7 +175,7 @@ def cleanup_solr_request(self, payload, user_id=None, handler_class="default"):
         handler = self.handler.get(handler_class, self.handler.get("default", "-"))
         headers['Host'] = self.get_host(current_app.config.get(handler))
         internal_logging = []
-        for internal_param, default in self.internal_logging_params.iteritems():
+        for internal_param, default in self.internal_logging_params.items():
             if internal_param in request.headers:
                 internal_logging.append("{}={}".format(internal_param, request.headers[internal_param]))
                 headers[internal_param] = request.headers[internal_param]
@@ -208,7 +210,7 @@ def cleanup_solr_request(self, payload, user_id=None, handler_class="default"):
 
         max_hl = current_app.config.get('SOLR_SERVICE_MAX_SNIPPETS', 4)
         max_frag = current_app.config.get('SOLR_SERVICE_MAX_FRAGSIZE', 100)
-        for k,v in payload.items():
+        for k,v in list(payload.items()):
             if 'hl.' in k:
                 if '.snippets' in k:
                     payload[k] = max(0, min(_safe_int(v, default=max_hl), max_hl))
@@ -242,7 +244,7 @@ def _cleanup_fields(self, payload, key, allowed_fields):
             fields.extend([i.strip().lower() for i in y.split(',')])
 
         if allowed_fields:
-            fields = filter(lambda x: x in allowed_fields, fields)
+            fields = [x for x in fields if x in allowed_fields]
 
         payload[key] = ','.join(fields)
 
@@ -271,8 +273,8 @@ def _cleanup_fl(self, payload, user_id, key):
 
         protected_fields = []
         if disallowed:
-            protected_fields = filter(lambda x: x in disallowed, fields)
-            fields = filter(lambda x: x not in disallowed, fields)
+            protected_fields = [x for x in fields if x in disallowed]
+            fields = [x for x in fields if x not in disallowed]
 
         if len(fields) == 0:
             fields.append('id')
@@ -340,7 +342,9 @@ def check_for_embedded_bigquery(self, params, request, headers, handler_class="d
         if request.data and isinstance(request.data, basestring) and len(request.data) > 0:
             if 'fq' not in params:
                 params['fq'] = [u'{!bitset}']
-            elif len(filter(lambda x: '!bitset' in x, params['fq'])) == 0:
+            elif isinstance(params['fq'], str) and '{!bitset}' not in params['fq']:
+                params['fq'] += u' {!bitset}'
+            elif isinstance(params['fq'], list) and len([x for x in params['fq'] if '!bitset' in x]) == 0:
                 params['fq'].append(u'{!bitset}')
 
             # we'll package request.data into files

wsgi.py

@@ -1,5 +1,5 @@
 from werkzeug.serving import run_simple
-from werkzeug.wsgi import DispatcherMiddleware
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
 from solr import app
 
 application = app.create_app()