From bfb7aaf181a921d85d6faf6955d7bcc9b8e23c1c Mon Sep 17 00:00:00 2001 From: Nils Friedrich Date: Tue, 7 Jun 2022 18:19:37 +0200 Subject: [PATCH 1/6] [TASK] ADD local testing --- .github/workflows/ci.yml | 1 + docker-compose.yml | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e483f27..02e62c13 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,6 +26,7 @@ jobs: - '^10.4' elasticsearch: - '5' + - '7' steps: - uses: actions/checkout@v3 diff --git a/docker-compose.yml b/docker-compose.yml index 628b3de3..57d1da02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,4 +27,8 @@ services: - /var/lib/mysql elasticsearch: - image: elasticsearch:${ELASTICSEARCH_VERSION:-5} + image: elasticsearch:${ELASTICSEARCH_VERSION:-7.17.4} + environment: + - ingest.geoip.downloader.enabled=false + - discovery.type=single-node + command: ["elasticsearch", "-Elogger.level=DEBUG"] From cecb81c302ac9a43d4ae27ea257fc482c9bf3621 Mon Sep 17 00:00:00 2001 From: Nils Friedrich Date: Tue, 7 Jun 2022 18:25:57 +0200 Subject: [PATCH 2/6] Revert "[TASK] ADD local testing" This reverts commit bfb7aaf181a921d85d6faf6955d7bcc9b8e23c1c. --- .github/workflows/ci.yml | 1 - docker-compose.yml | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 02e62c13..8e483f27 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,7 +26,6 @@ jobs: - '^10.4' elasticsearch: - '5' - - '7' steps: - uses: actions/checkout@v3 diff --git a/docker-compose.yml b/docker-compose.yml index 57d1da02..628b3de3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,4 @@ services: - /var/lib/mysql elasticsearch: - image: elasticsearch:${ELASTICSEARCH_VERSION:-7.17.4} - environment: - - ingest.geoip.downloader.enabled=false - - discovery.type=single-node - command: ["elasticsearch", "-Elogger.level=DEBUG"] + image: elasticsearch:${ELASTICSEARCH_VERSION:-5} From cf1c9db256fe10c93ff0937377572d4acc852017 Mon Sep 17 00:00:00 2001 From: Nils Friedrich Date: Tue, 7 Jun 2022 18:27:08 +0200 Subject: [PATCH 3/6] [TASK] Add elastic7 image for local testing --- docker-compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 628b3de3..57d1da02 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,4 +27,8 @@ services: - /var/lib/mysql elasticsearch: - image: elasticsearch:${ELASTICSEARCH_VERSION:-5} + image: elasticsearch:${ELASTICSEARCH_VERSION:-7.17.4} + environment: + - ingest.geoip.downloader.enabled=false + - discovery.type=single-node + command: ["elasticsearch", "-Elogger.level=DEBUG"] From fcb4416678d22dc19966bdec5fe5c22d3b34ead3 Mon Sep 17 00:00:00 2001 From: Nils Friedrich Date: Wed, 8 Jun 2022 16:10:20 +0200 Subject: [PATCH 4/6] [TASK] Refresh indices and add sleep time for tets --- Tests/Functional/AbstractElasticsearchTest.php | 9 ++++++++- Tests/Functional/Service/IndexingServiceTest.php | 5 ++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/Tests/Functional/AbstractElasticsearchTest.php b/Tests/Functional/AbstractElasticsearchTest.php index 6005d0b6..33073ada 100644 --- a/Tests/Functional/AbstractElasticsearchTest.php +++ b/Tests/Functional/AbstractElasticsearchTest.php @@ -243,6 +243,9 @@ protected function assertIndexeEmpty(int $languageId = 0): void $this->syncIndices(); $indexe = ExtconfService::getLanguageIndicies($languageId); $indexString = implode(',', $indexe); + $client->indices()->refresh([ + 'index' => $indexString, + ]); $response = $client->search([ 'index' => $indexString, @@ -281,6 +284,10 @@ protected function searchDocumentByUid(int $uid, int $languageId): array $indexe = ExtconfService::getLanguageIndicies($languageId); $indexString = implode(',', $indexe); + $client->indices()->refresh([ + 'index' => $indexString, + ]); + $response = $client->search([ 'index' => $indexString, 'body' => [ @@ -304,7 +311,7 @@ protected function searchDocumentByUid(int $uid, int $languageId): array */ protected function syncIndices(): void { - $this->getElasticsearchClient()->indices()->flushSynced([ + $this->getElasticsearchClient()->indices()->flush([ 'index' => implode(',', array_merge( $this->indexNames, ['searchable_updates'] diff --git a/Tests/Functional/Service/IndexingServiceTest.php b/Tests/Functional/Service/IndexingServiceTest.php index 5e524731..00ae2bcd 100644 --- a/Tests/Functional/Service/IndexingServiceTest.php +++ b/Tests/Functional/Service/IndexingServiceTest.php @@ -193,9 +193,12 @@ public function indexesRecordsPartially(): void ); $this->syncIndices(); - + sleep(7); $this->indexingService->indexPartial(); + $client = $this->getElasticsearchClient(); + $client->indices()->refresh(); + sleep(3); $this->assertDocumentInIndex( 2, [ From 9aacb450689e9323ff7f650a4ae8e0456495b968 Mon Sep 17 00:00:00 2001 From: Nils Friedrich Date: Wed, 15 Jun 2022 13:53:02 +0200 Subject: [PATCH 5/6] [TASK] Fix SearchQueryTest.php The problem is the TermSuggestFeature, because of the missing _all field Added an _all field with copy_to -> https://stackoverflow.com/a/60229131 --- .github/workflows/ci.yml | 3 +- .../Functional/AbstractElasticsearchTest.php | 76 ++++++++++++++++++- Tests/Functional/Query/SearchQueryTest.php | 5 +- .../Service/IndexingServiceTest.php | 10 ++- 4 files changed, 87 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e483f27..047ae27e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -10,7 +10,7 @@ on: pull_request: branches: - master - - elasticsearch7 + - elasticsearch7-testing schedule: - cron: '0 7 * * *' @@ -26,6 +26,7 @@ jobs: - '^10.4' elasticsearch: - '5' + - '7' steps: - uses: actions/checkout@v3 diff --git a/Tests/Functional/AbstractElasticsearchTest.php b/Tests/Functional/AbstractElasticsearchTest.php index 33073ada..6b40233f 100644 --- a/Tests/Functional/AbstractElasticsearchTest.php +++ b/Tests/Functional/AbstractElasticsearchTest.php @@ -126,8 +126,26 @@ protected function setUp(): void 'pid' => 1, ], ], + 'mapping' => [ + "dynamic_templates" => [ + [ + "all_text" => [ + "match_mapping_type" => "string", + "mapping" => [ + "copy_to" => "_all", + "type" => "text", + ], + ], + ], + ], + "properties" => [ + "_all" => [ + "type" => "text", + ], + ], + ], + ], ], - ], 'bar_pages' => [ 'className' => PagesIndexer::class, 'config' => [ @@ -137,6 +155,24 @@ protected function setUp(): void 'pid' => 100, ], ], + 'mapping' => [ + "dynamic_templates" => [ + [ + "all_text" => [ + "match_mapping_type" => "string", + "mapping" => [ + "copy_to" => "_all", + "type" => "text", + ], + ], + ], + ], + "properties" => [ + "_all" => [ + "type" => "text", + ], + ], + ], ], ], 'qux_pages' => [ @@ -148,6 +184,24 @@ protected function setUp(): void 'pid' => 200, ], ], + 'mapping' => [ + "dynamic_templates" => [ + [ + "all_text" => [ + "match_mapping_type" => "string", + "mapping" => [ + "copy_to" => "_all", + "type" => "text", + ], + ], + ], + ], + "properties" => [ + "_all" => [ + "type" => "text", + ], + ], + ], ], ], 'content' => [ @@ -166,10 +220,28 @@ protected function setUp(): void 'preview' => [ 'className' => ContentPreviewRenderer::class, ], + 'mapping' => [ + "dynamic_templates" => [ + [ + "all_text" => [ + "match_mapping_type" => "string", + "mapping" => [ + "copy_to" => "_all", + "type" => "text", + ], + ], + ], + ], + "properties" => [ + "_all" => [ + "type" => "text", + ], + ], + ], ], ], ], - ] + ], ); $this->getDatabaseConnection()->insertArray('pages', [ diff --git a/Tests/Functional/Query/SearchQueryTest.php b/Tests/Functional/Query/SearchQueryTest.php index 18300a76..6a6f79e3 100644 --- a/Tests/Functional/Query/SearchQueryTest.php +++ b/Tests/Functional/Query/SearchQueryTest.php @@ -36,12 +36,13 @@ public function searchesByTerm(): void 'doktype' => PageRepository::DOKTYPE_DEFAULT, 'title' => 'Unrelated page', ]); - + $this->indexingService->indexFull(); $this->syncIndices(); - $query = GeneralUtility::makeInstance(SearchQuery::class); $query->setTerm('test'); + $client = $this->getElasticsearchClient(); + $client->indices()->refresh(); $result = $query->execute(); diff --git a/Tests/Functional/Service/IndexingServiceTest.php b/Tests/Functional/Service/IndexingServiceTest.php index 00ae2bcd..93286b7f 100644 --- a/Tests/Functional/Service/IndexingServiceTest.php +++ b/Tests/Functional/Service/IndexingServiceTest.php @@ -136,6 +136,9 @@ public function appliesLanguageForRecordTranslationIndexing(): void $this->indexingService->setup(); $this->indexingService->indexFull('content'); + $client = $this->getElasticsearchClient(); + $client->indices()->refresh(); + $this->assertDocumentInIndex( 1, [ @@ -193,12 +196,12 @@ public function indexesRecordsPartially(): void ); $this->syncIndices(); - sleep(7); + $client = $this->getElasticsearchClient(); + $client->indices()->refresh(); $this->indexingService->indexPartial(); $client = $this->getElasticsearchClient(); $client->indices()->refresh(); - sleep(3); $this->assertDocumentInIndex( 2, [ @@ -285,6 +288,9 @@ public function respectsSiteBase(): void $this->indexingService->indexFull(); + $client = $this->getElasticsearchClient(); + $client->indices()->refresh(); + $this->assertDocumentInIndex( 101, [ From d2cdd69dfbb0b7abcac543ae45db80e9e7bb7c00 Mon Sep 17 00:00:00 2001 From: Nils Friedrich Date: Mon, 20 Jun 2022 15:27:50 +0200 Subject: [PATCH 6/6] [TASK] Comment out preferred elasticsearch 7 --- docker-compose.yml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 57d1da02..5d30cf10 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,8 +27,9 @@ services: - /var/lib/mysql elasticsearch: - image: elasticsearch:${ELASTICSEARCH_VERSION:-7.17.4} - environment: - - ingest.geoip.downloader.enabled=false - - discovery.type=single-node - command: ["elasticsearch", "-Elogger.level=DEBUG"] + image: elasticsearch:${ELASTICSEARCH_VERSION:-5} + # image: elasticsearch:${ELASTICSEARCH_VERSION:-7.17.4} + # environment: + # - ingest.geoip.downloader.enabled=false + # - discovery.type=single-node + # command: ["elasticsearch", "-Elogger.level=DEBUG"]