[Backport 2.x] Add CI bundle pattern to distribution download (#5573)

* Add CI bundle pattern to distribution download (#5348)

* Add CI bundle pattern for ivy repo

Signed-off-by: Zelin Hao <[email protected]>

* Gradle update

Signed-off-by: Zelin Hao <[email protected]>

* Extract path

Signed-off-by: Zelin Hao <[email protected]>

* Change with customDistributionDownloadType

Signed-off-by: Zelin Hao <[email protected]>

* Add default for exception handle

Signed-off-by: Zelin Hao <[email protected]>

* Add documentations

Signed-off-by: Zelin Hao <[email protected]>

Signed-off-by: Zelin Hao <[email protected]>
(cherry picked from commit fe8fd67)
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>

* Reorder logics to fix tests

Signed-off-by: Zelin Hao <[email protected]>

Signed-off-by: Zelin Hao <[email protected]>
Signed-off-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Zelin Hao <[email protected]>

Author: 3 people

CHANGELOG.md

@@ -16,6 +16,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Adding support to register settings dynamically ([#5495](https://github.com/opensearch-project/OpenSearch/pull/5495))
 - Adding auto release workflow ([#5582](https://github.com/opensearch-project/OpenSearch/pull/5582))
 - Added experimental support for extensions ([#5347](https://github.com/opensearch-project/OpenSearch/pull/5347)), ([#5518](https://github.com/opensearch-project/OpenSearch/pull/5518))
+- Add CI bundle pattern to distribution download ([#5348](https://github.com/opensearch-project/OpenSearch/pull/5348))
 
 
 ### Dependencies

TESTING.md

@@ -383,6 +383,10 @@ Use -Dtest.class and -Dtests.method to run a specific bwcTest test. For example
      -Dtests.class=org.opensearch.upgrades.RecoveryIT \
      -Dtests.method=testHistoryUUIDIsGenerated
 
+Use `-PcustomDistributionDownloadType=bundle` to run the bwcTest against the test cluster with latest CI distribution bundle set up for the specified version; this property is default to min and exclusive choices between `bundle` and `min`:
+
+    ./gradlew bwcTest -PcustomDistributionDownloadType=bundle
+
 When running `./gradlew check`, minimal bwc checks are also run against compatible versions that are not yet released.
 
 ## BWC Testing against a specific remote/branch

buildSrc/src/main/java/org/opensearch/gradle/DistributionDownloadPlugin.java

@@ -80,6 +80,8 @@ public class DistributionDownloadPlugin implements Plugin<Project> {
     private static final String RELEASE_PATTERN_LAYOUT = "/core/opensearch/[revision]/[module]-min-[revision](-[classifier]).[ext]";
     private static final String SNAPSHOT_PATTERN_LAYOUT =
         "/snapshots/core/opensearch/[revision]/[module]-min-[revision](-[classifier])-latest.[ext]";
+    private static final String BUNDLE_PATTERN_LAYOUT =
+        "/ci/dbc/distribution-build-opensearch/[revision]/latest/linux/x64/tar/dist/opensearch/[module]-[revision](-[classifier]).[ext]";
 
     private NamedDomainObjectContainer<OpenSearchDistribution> distributionsContainer;
     private NamedDomainObjectContainer<DistributionResolution> distributionsResolutionStrategiesContainer;
@@ -196,24 +198,45 @@ private static void setupDownloadServiceRepo(Project project) {
             return;
         }
         Object customDistributionUrl = project.findProperty("customDistributionUrl");
-        // checks if custom Distribution Url has been passed by user from plugins
+        Object customDistributionDownloadType = project.findProperty("customDistributionDownloadType");
+        // distributionDownloadType is default min if is not specified; download the distribution from CI if is bundle
+        String distributionDownloadType = customDistributionDownloadType != null
+            && customDistributionDownloadType.toString().equals("bundle") ? "bundle" : "min";
+
+        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
+        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
+
         if (customDistributionUrl != null) {
             addIvyRepo(project, DOWNLOAD_REPO_NAME, customDistributionUrl.toString(), FAKE_IVY_GROUP, "");
             addIvyRepo(project, SNAPSHOT_REPO_NAME, customDistributionUrl.toString(), FAKE_SNAPSHOT_IVY_GROUP, "");
-        } else {
-            addIvyRepo(
-                project,
-                DOWNLOAD_REPO_NAME,
-                "https://artifacts.opensearch.org",
-                FAKE_IVY_GROUP,
-                "/releases" + RELEASE_PATTERN_LAYOUT,
-                "/release-candidates" + RELEASE_PATTERN_LAYOUT
-            );
-            addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://artifacts.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, SNAPSHOT_PATTERN_LAYOUT);
+            return;
+        }
+        switch (distributionDownloadType) {
+            case "bundle":
+                addIvyRepo(project, DOWNLOAD_REPO_NAME, "https://ci.opensearch.org", FAKE_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                addIvyRepo(project, SNAPSHOT_REPO_NAME, "https://ci.opensearch.org", FAKE_SNAPSHOT_IVY_GROUP, BUNDLE_PATTERN_LAYOUT);
+                break;
+            case "min":
+                addIvyRepo(
+                    project,
+                    DOWNLOAD_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_IVY_GROUP,
+                    "/releases" + RELEASE_PATTERN_LAYOUT,
+                    "/release-candidates" + RELEASE_PATTERN_LAYOUT
+                );
+                addIvyRepo(
+                    project,
+                    SNAPSHOT_REPO_NAME,
+                    "https://artifacts.opensearch.org",
+                    FAKE_SNAPSHOT_IVY_GROUP,
+                    SNAPSHOT_PATTERN_LAYOUT
+                );
+                break;
+            default:
+                throw new IllegalArgumentException("Unsupported property argument: " + distributionDownloadType);
         }
 
-        addIvyRepo2(project, DOWNLOAD_REPO_NAME_ES, "https://artifacts-no-kpi.elastic.co", FAKE_IVY_GROUP_ES);
-        addIvyRepo2(project, SNAPSHOT_REPO_NAME_ES, "https://snapshots-no-kpi.elastic.co", FAKE_SNAPSHOT_IVY_GROUP_ES);
     }
 
     /**