Updating to Django 5 and Ubuntu 24.04 (#65)

* Updating to Django 5 and Ubuntu 24.04

* Temporarily remove db version check to support mariadb server on EL8 without appstream enabled

* Configurable workers and threads

* Configuring Content Security Policy

* Fix IDP-less debugging on django5

* Switching charset to utf8mb4

Author: guilbaults

Containerfile

@@ -1,21 +1,26 @@
-FROM ubuntu:22.04
+FROM ubuntu:24.04
 
 ENV PYTHONDONTWRITEBYTECODE 1
 ENV PYTHONUNBUFFERED 1
 
-RUN apt-get update && apt-get install -y tzdata && apt install -y python3.10 python3-pip python3-dev python3-numpy libpq-dev nginx libmysqlclient-dev build-essential libsasl2-dev libldap2-dev libssl-dev xmlsec1 gettext
+RUN apt-get update && apt-get install -y tzdata && apt install -y python3.12 python3-pip python3-dev python3.12-venv libpq-dev nginx libmysqlclient-dev pkg-config build-essential libsasl2-dev libldap2-dev libssl-dev xmlsec1 gettext git
 
 WORKDIR /opt/userportal
 
 COPY requirements.txt ./
 
-RUN pip3 install --upgrade pip && \
-    pip3 install --no-cache-dir -r requirements.txt
+RUN python3 -m venv /opt/userportal-env && \
+    /opt/userportal-env/bin/pip install --upgrade pip && \
+    /opt/userportal-env/bin/pip install --no-cache-dir -r requirements.txt
 
 COPY . .
 
-RUN patch /usr/local/lib/python3.10/dist-packages/ldapdb/backends/ldap/base.py < /opt/userportal/ldapdb.patch
-RUN python3 manage.py collectstatic --noinput && python3 manage.py compilemessages
+RUN patch /opt/userportal-env/lib/python3.12/site-packages/ldapdb/backends/ldap/base.py < /opt/userportal/ldapdb.patch
+
+# Temporarily remove db version check to support mariadb server on EL8 without appstream enabled
+RUN patch /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py < /opt/userportal/dbcheck.patch
+
+RUN /opt/userportal-env/bin/python manage.py collectstatic --noinput && /opt/userportal-env/bin/python manage.py compilemessages
 
 EXPOSE 8000
 CMD ["./run-django-production.sh"]

dbcheck.patch

@@ -0,0 +1,11 @@
+--- /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py    2024-07-17 17:45:44.000000000 +0000
++++ /opt/userportal-env/lib/python3.12/site-packages/django/db/backends/base/base.py.modified   2024-07-17 19:21:39.250937980 +0000
+@@ -222,7 +222,7 @@
+         """Initialize the database connection settings."""
+         global RAN_DB_VERSION_CHECK
+         if self.alias not in RAN_DB_VERSION_CHECK:
+-            self.check_database_version_supported()
++            #self.check_database_version_supported()
+             RAN_DB_VERSION_CHECK.add(self.alias)
+
+     def create_cursor(self, name=None):

docs/install.md

@@ -5,7 +5,18 @@ Before installing in production, [a test environment should be set up to test th
 The portal can be installed directly on a Rocky8 Apache web server or with Nginx and Gunicorn. The portal can also be deployed as a container with Podman or Kubernetes. Some scripts used to deploy both Nginx and Django containers inside the same pod are provided in the `podman` directory.
 The various recommendations for any normal Django production deployment can be followed.
 
-[Deploying Django](https://docs.djangoproject.com/en/3.2/howto/deployment/)
+[Deploying Django](https://docs.djangoproject.com/en/5.0/howto/deployment/)
+
+The database should support UTF8. With MariaDB, the default collation can be changed with the following command:
+
+```
+ALTER DATABASE userportal CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;
+```
+
+Migration scripts will also ensure that the tables and columns are converted to the correct collation.
+
+# Production with containers
+Using containers is the recommended way to deploy the portal. The container is automatically built in the CI pipeline and pushed to the Github registry. This container is handling the django application. The static files are served by a standard Nginx container. Both containers are deployed in the same pod with a shared volume containing the static files.
 
 # Production without containers on Rocky Linux 8
 

jobstats/migrations/0002_utf8.py

@@ -0,0 +1,15 @@
+# Generated by Django 5.0.7 on 2024-10-16 18:38
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobstats', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunSQL('ALTER TABLE jobstats_jobscript CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
+        migrations.RunSQL('ALTER TABLE jobstats_jobscript MODIFY submit_script LONGTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
+    ]

notes/migrations/0002_utf8.py

@@ -0,0 +1,16 @@
+# Generated by Django 5.0.7 on 2024-10-16 18:03
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('notes', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.RunSQL('ALTER TABLE notes_note CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
+        migrations.RunSQL('ALTER TABLE notes_note MODIFY title VARCHAR(255) CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
+        migrations.RunSQL('ALTER TABLE notes_note MODIFY notes LONGTEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;'),
+    ]

podman/deploy.sh

@@ -20,6 +20,16 @@ if [ -z "$VERSION" ]; then
     VERSION=latest
 fi
 
+if [ -z "$WORKERS" ]; then
+    echo "WORKERS is not set in the config file, using 4"
+    WORKERS=4
+fi
+
+if [ -z "$THREADS" ]; then
+    echo "THREADS is not set in the config file, using 4"
+    THREADS=4
+fi
+
 podman pod stop TT-$TT_ENV
 podman pod rm -f TT-$TT_ENV
 podman volume rm -f TT-$TT_ENV-static
@@ -28,6 +38,8 @@ podman pod create -p $PORT:80 --name TT-$TT_ENV
 podman volume create TT-$TT_ENV-static
 
 podman run --detach --pod TT-$TT_ENV --name=TT-$TT_ENV-django \
+    --env WORKERS=${WORKERS} \
+    --env THREADS=${THREADS} \
     -v $TT_ENV_PATH/99-local.py:/secrets/settings/99-local.py:Z \
     -v $TT_ENV_PATH/private.key:/opt/userportal/private.key:Z \
     -v $TT_ENV_PATH/public.cert:/opt/userportal/public.cert:Z \

requirements.txt

@@ -1,56 +1,55 @@
-asgiref==3.4.1
-certifi==2024.07.04
-cffi==1.15.1
-chardet==4.0.0
-charset-normalizer==3.3.1
-cryptography==42.0.4
-dateparser==1.0.0
+asgiref==3.8.1
+certifi==2024.7.4
+cffi==1.16.0
+charset-normalizer==3.3.2
+contourpy==1.2.1
+cryptography==42.0.8
+cycler==0.12.1
+dateparser==1.2.0
 defusedxml==0.7.1
-Django==3.2.25
-django-auth-ldap==2.3.0
-django-bootstrap-pagination==1.7.1
-django-debug-toolbar==3.2.4
-django-ldapdb==1.5.1
+Django==5.0.9
+django-bootstrap-pagination @ git+https://github.com/teejaydub/django-bootstrap-pagination@82ea7a213860a741890f324e025f6ec215f24a32
+django-crispy-forms==2.2
+django-debug-toolbar==4.4.6
+django-ldapdb @ git+https://github.com/nikolaik/django-ldapdb@381e4f40032c96ea6ae6623a5a0339b3ea526ae5
 django-settings-export==1.2.1
 django-watchman==1.3.0
-djangorestframework==3.13.1
-djangorestframework-api-key==2.0.0
-djangorestframework-datatables==0.7.0
-djangosaml2==1.5.1
-elementpath==2.4.0
-flake8==3.9.0
+django_csp==3.8
+djangorestframework==3.15.2
+djangorestframework-datatables==0.7.2
+djangosaml2==1.9.3
+elementpath==4.4.0
+flake8==7.1.0
+fonttools==4.53.1
 gunicorn==22.0.0
+httmock==1.4.0
 idna==3.7
-importlib-metadata==3.7.3
-importlib-resources==5.4.0
-install==1.3.5
-IPy==1.1
-mccabe==0.6.1
-mysqlclient==2.0.3
-numpy==1.23.4
-pandas==1.1.5
-plotly==4.14.3
-prometheus-api-client==0.4.2
-pyasn1==0.4.8
-pyasn1-modules==0.2.8
-pycodestyle==2.7.0
-pycparser==2.21
-pyflakes==2.3.0
-PyJWT==2.7.0
-Pyment==0.3.3
+kiwisolver==1.4.5
+matplotlib==3.9.2
+mccabe==0.7.0
+mysqlclient==2.2.4
+numpy==2.0.0
+packaging==24.1
+pandas==2.2.2
+pillow==10.4.0
+prometheus-api-client==0.5.5
+pyasn1==0.6.0
+pyasn1_modules==0.4.0
+pycodestyle==2.12.0
+pycparser==2.22
+pyflakes==3.2.0
 pyOpenSSL==24.1.0
-pysaml2==7.1.2
-python-dateutil==2.8.1
-python-ldap==3.4.3
-pytz==2021.1
-PyYAML==6.0
-regex==2020.11.13
+pyparsing==3.1.2
+pysaml2==7.5.0
+python-dateutil==2.9.0.post0
+python-ldap==3.4.4
+pytz==2024.1
+PyYAML==6.0.1
+regex==2024.5.15
 requests==2.32.3
-retrying==1.3.3
-six==1.15.0
+six==1.16.0
 sqlparse==0.5.0
-typing-extensions==3.7.4.3
-tzlocal==2.1
-urllib3==1.26.19
-xmlschema==1.9.2
-zipp==3.19.1
+tzdata==2024.1
+tzlocal==5.2
+urllib3==2.2.2
+xmlschema==2.5.1

run-django-production.sh

@@ -7,4 +7,4 @@ set -o nounset
 cp /secrets/settings/99-local.py /opt/userportal/userportal/settings/99-local.py
 mkdir -p /var/www/api/static
 cp -r /opt/userportal/collected-static/* /var/www/api/static/
-gunicorn --bind :8000 --workers 1 --timeout 90 userportal.wsgi
+/opt/userportal-env/bin/gunicorn --bind :8000 --workers $WORKERS --threads $THREADS --timeout 90 userportal.wsgi

userportal/authentication.py

@@ -23,7 +23,7 @@ def clean_username(self, username):
             username = username.split('@')[0]
         return username
 
-    def configure_user(self, request, user):
+    def configure_user(self, request, user, created=True):
         if '[email protected]' in request.META['affiliation'] \
                 or '[email protected]' in request.META['affiliation']:
             user.is_staff = True

userportal/settings/10-base.py

@@ -1,13 +1,5 @@
 """
 Django settings for userportal project.
-
-Generated by 'django-admin startproject' using Django 3.1.7.
-
-For more information on this file, see
-https://docs.djangoproject.com/en/3.1/topics/settings/
-
-For the full list of settings and their values, see
-https://docs.djangoproject.com/en/3.1/ref/settings/
 """
 
 from pathlib import Path
@@ -43,6 +35,7 @@
     'django.contrib.staticfiles',
     'django.contrib.humanize',
     'djangosaml2',
+    'csp',
     'watchman',
 
     'pages',
@@ -78,6 +71,7 @@
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware',
     'djangosaml2.middleware.SamlSessionMiddleware',
+    'csp.middleware.CSPMiddleware',
 ]
 
 ROOT_URLCONF = 'userportal.urls'
@@ -204,3 +198,9 @@
     'DEFAULT_PAGINATION_CLASS': 'rest_framework_datatables.pagination.DatatablesPageNumberPagination',
     'PAGE_SIZE': 100,
 }
+
+# Content Security Policy
+CSP_DEFAULT_SRC = ("'self'")
+CSP_IMG_SRC = ("'self'", "data:", 'object-arbutus.cloud.computecanada.ca')
+CSP_STYLE_SRC = ("'self'", "'unsafe-inline'", 'cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'cdn.datatables.net')
+CSP_SCRIPT_SRC = ("'self'", "'unsafe-inline'", 'cdn.jsdelivr.net', 'cdnjs.cloudflare.com', 'cdn.datatables.net', 'code.jquery.com', 'cdn.plot.ly')

userportal/settings/20-databases.py

@@ -10,7 +10,9 @@
         'HOST': 'dbserver',
         'PORT': '3128',
         'OPTIONS': {
-        }
+            'charset': 'utf8mb4',
+            'use_unicode': True,
+        },
     },
     'slurm': {
         'ENGINE': 'django.db.backends.mysql',