From 5518e6c65367acfc73f439a6ad321460687bdf64 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Tue, 10 Dec 2024 16:49:27 -0800 Subject: [PATCH 1/3] test: add test for admin export --- backend/mlarchive/tests/archive/conftest.py | 2 ++ backend/mlarchive/tests/archive/views.py | 25 +++++++++++++++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/backend/mlarchive/tests/archive/conftest.py b/backend/mlarchive/tests/archive/conftest.py index 736adaef..e539fa56 100644 --- a/backend/mlarchive/tests/archive/conftest.py +++ b/backend/mlarchive/tests/archive/conftest.py @@ -58,6 +58,8 @@ def data_dir(tmp_dir, settings): """Use temporary directory""" DATA_ROOT = tmp_dir settings.ARCHIVE_DIR = os.path.join(DATA_ROOT, 'archive') + if not os.path.exists(settings.ARCHIVE_DIR): + os.mkdir(settings.ARCHIVE_DIR) settings.EXPORT_DIR = os.path.join(DATA_ROOT, 'export') yield diff --git a/backend/mlarchive/tests/archive/views.py b/backend/mlarchive/tests/archive/views.py index 9f71a0a2..47b3021d 100644 --- a/backend/mlarchive/tests/archive/views.py +++ b/backend/mlarchive/tests/archive/views.py @@ -166,7 +166,7 @@ def test_admin_search_from(client, messages): assert response.status_code == 200 results = response.context['results'] assert str(msg.pk) in [r.django_id for r in results] - + @pytest.mark.django_db(transaction=True) def test_admin_menu(client, admin_client): @@ -700,10 +700,11 @@ def test_export_not_logged_in(client, messages): @pytest.mark.django_db(transaction=True) -def test_export_limit(admin_client, messages, settings): +def test_export_limit(client, admin_client, messages, settings, users): settings.EXPORT_LIMIT = 0 url = reverse('archive_browse_list', kwargs={'list_name': 'pubone'}) - response = admin_client.get(url) + assert client.login(username='unprivileged@example.com', password='password') + response = client.get(url) # print(response.content) print(type(response.content)) assert response.status_code == 200 @@ -711,10 +712,26 @@ def test_export_limit(admin_client, messages, settings): q = PyQuery(response.content) assert len(q('.export-link.disabled')) == 3 url = reverse('archive_export', kwargs={'type': 'mbox'}) + '?email_list=pubone' - response = admin_client.get(url) + response = client.get(url) assert response.status_code == 302 +@pytest.mark.django_db(transaction=True) +def test_export_limit_admin(admin_client, export_messages, settings, users): + '''Limits do not apply to superuser''' + settings.EXPORT_LIMIT = 0 + url = reverse('archive_browse_list', kwargs={'list_name': 'acme'}) + response = admin_client.get(url) + print(type(response.content)) + assert response.status_code == 200 + assert 'Export is limited to 0 messages.' not in smart_str(response.content) + q = PyQuery(response.content) + assert len(q('.export-link.disabled')) == 0 + url = reverse('archive_export', kwargs={'type': 'mbox'}) + '?email_list=acme' + response = admin_client.get(url) + assert response.status_code == 200 + + @pytest.mark.django_db(transaction=True) def test_export_bad_query(admin_client, messages, settings): settings.EXPORT_LIMIT = 0 From 9fa0c0333ffa9ad7e2379ea51b8c605aac7d8766 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Tue, 10 Dec 2024 16:52:34 -0800 Subject: [PATCH 2/3] fix: fix deprecation warnings --- .../archive/backends/elasticsearch.py | 80 +++++++++---------- backend/mlarchive/tests/archive/factories.py | 9 +++ .../tests/archive/test_elasticsearch.py | 4 +- 3 files changed, 50 insertions(+), 43 deletions(-) diff --git a/backend/mlarchive/archive/backends/elasticsearch.py b/backend/mlarchive/archive/backends/elasticsearch.py index 059b7593..2ec3f8da 100644 --- a/backend/mlarchive/archive/backends/elasticsearch.py +++ b/backend/mlarchive/archive/backends/elasticsearch.py @@ -56,7 +56,7 @@ def full_prepare(message): class ESBackend(): """Elasticsearch Backend""" - + # Characters reserved by Elasticsearch for special use. # The '\\' must come first, so as not to overwrite the other slash replacements. RESERVED_CHARACTERS = ( @@ -67,44 +67,42 @@ class ESBackend(): # Settings to add an n-gram & edge n-gram analyzers # for use in autocomplete feature DEFAULT_SETTINGS = { - 'settings': { - "analysis": { - "analyzer": { - "ngram_analyzer": { - "type": "custom", - "tokenizer": "standard", - "filter": ["ngram_filter", "lowercase"] - }, - "edgengram_analyzer": { - "type": "custom", - "tokenizer": "standard", - "filter": ["edgengram_filter", "lowercase"] - } + "analysis": { + "analyzer": { + "ngram_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["ngram_filter", "lowercase"] }, - "tokenizer": { - "custom_ngram_tokenizer": { - "type": "ngram", - "min_gram": 4, - "max_gram": 4, - }, - "custom_edgengram_tokenizer": { - "type": "edge_ngram", - "min_gram": 4, - "max_gram": 4, - "side": "front" - } + "edgengram_analyzer": { + "type": "custom", + "tokenizer": "standard", + "filter": ["edgengram_filter", "lowercase"] + } + }, + "tokenizer": { + "custom_ngram_tokenizer": { + "type": "ngram", + "min_gram": 4, + "max_gram": 4, }, - "filter": { - "ngram_filter": { - "type": "ngram", - "min_gram": 4, - "max_gram": 4 - }, - "edgengram_filter": { - "type": "edge_ngram", - "min_gram": 4, - "max_gram": 4 - } + "custom_edgengram_tokenizer": { + "type": "edge_ngram", + "min_gram": 4, + "max_gram": 4, + "side": "front" + } + }, + "filter": { + "ngram_filter": { + "type": "ngram", + "min_gram": 4, + "max_gram": 4 + }, + "edgengram_filter": { + "type": "edge_ngram", + "min_gram": 4, + "max_gram": 4 } } } @@ -134,16 +132,16 @@ def setup(self): If the index doesn't exist, create it and set mappings. You can't change mappings of existing indexes. """ - if not self.client.indices.exists(self.index_name): + if not self.client.indices.exists(index=self.index_name): self.client.indices.create(index=self.index_name, - body=self.DEFAULT_SETTINGS) + settings=self.DEFAULT_SETTINGS) self.client.indices.put_mapping(index=self.index_name, body=self.mapping) self.setup_complete = True def clear(self, commit=True): - '''Clears index of all data, and runs setup, leaving + '''Clears index of all data, and runs setup, leaving an empty index.''' logger.debug('ESBackend.clear() called.') self.client.indices.delete(index=self.index_name, ignore=404) @@ -153,7 +151,7 @@ def update(self, iterable, commit=True): '''Update index records using iterable of instances''' logger.debug('ESBackend.update() called. iterable={}, iterable_length={}, last_message={}, commit={}, setup_complete={}'.format( type(iterable), len(iterable), iterable[-1].django_id, commit, self.setup_complete)) - + if not self.setup_complete: try: self.setup() diff --git a/backend/mlarchive/tests/archive/factories.py b/backend/mlarchive/tests/archive/factories.py index a8eec59a..7d86d42e 100644 --- a/backend/mlarchive/tests/archive/factories.py +++ b/backend/mlarchive/tests/archive/factories.py @@ -57,11 +57,20 @@ class Meta: class UserFactory(factory.django.DjangoModelFactory): class Meta: model = User + skip_postgeneration_save = True email = 'admin@admin.com' username = 'admin' password = factory.PostGenerationMethodCall('set_password', 'admin') + @factory.post_generation + def finalize_instance(self, create, extracted, **kwargs): + """ + Custom post-generation hook to save the instance after any modifications. + """ + if create: # Only save if the instance was created + self.save() + class SubscriberFactory(factory.django.DjangoModelFactory): class Meta: diff --git a/backend/mlarchive/tests/archive/test_elasticsearch.py b/backend/mlarchive/tests/archive/test_elasticsearch.py index 5c6a6caa..c2e5eb70 100644 --- a/backend/mlarchive/tests/archive/test_elasticsearch.py +++ b/backend/mlarchive/tests/archive/test_elasticsearch.py @@ -30,7 +30,7 @@ def test_rebuild_index(db_only): client = ESBackend().client s = Search(using=client, index=settings.ELASTICSEARCH_INDEX_NAME) assert s.count() == 3 - info = client.cat.indices(settings.ELASTICSEARCH_INDEX_NAME) + info = client.cat.indices(index=settings.ELASTICSEARCH_INDEX_NAME) uuid = info.split()[3] # delete a record Message.objects.get(msgid='x001').delete() @@ -47,7 +47,7 @@ def test_rebuild_index(db_only): assert 'Indexing 2 Messages' in out.getvalue() s = Search(using=client, index=settings.ELASTICSEARCH_INDEX_NAME) assert s.count() == 2 - info = client.cat.indices(settings.ELASTICSEARCH_INDEX_NAME) + info = client.cat.indices(index=settings.ELASTICSEARCH_INDEX_NAME) new_uuid = info.split()[3] assert new_uuid != uuid s = Search(using=client, index=settings.ELASTICSEARCH_INDEX_NAME) From 1caed6c61b485628aef3fd529b0919ee3f196095 Mon Sep 17 00:00:00 2001 From: Ryan Cross Date: Tue, 10 Dec 2024 16:55:44 -0800 Subject: [PATCH 3/3] chore: fix vscode task and move STATUS settings to vault --- .vscode/tasks.json | 7 +++++-- docker/configs/docker_env | 4 ---- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 380f61f1..8d875593 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -50,8 +50,11 @@ "-v" ], "options": { - "cwd": "${workspaceFolder}/backend/mlarchive" - } + "cwd": "${workspaceFolder}/backend/mlarchive", + "env": { + "DJANGO_SETTINGS_MODULE": "mlarchive.settings.test" + } + }, "group": "test", "presentation": { "echo": true, diff --git a/docker/configs/docker_env b/docker/configs/docker_env index 7ee89448..246b843b 100644 --- a/docker/configs/docker_env +++ b/docker/configs/docker_env @@ -8,7 +8,3 @@ ELASTICSEARCH_HOST='es' # CELERY SETTINGS CELERY_BROKER_URL="amqp://guest:guest@rabbit:5672//" - -# STATIC FILES -STATIC_URL="https://static.ietf.org/mailarchive/{}/" -STATIC_FRAG="/../../static/"