Merge branch 'main' into flaky

Author: bowenlan-amzn

CHANGELOG.md

@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - GHA to verify checklist items completion in PR descriptions ([#10800](https://github.com/opensearch-project/OpenSearch/pull/10800))
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
+- Add support for dependencies in plugin descriptor properties with semver range ([#11441](https://github.com/opensearch-project/OpenSearch/pull/11441))
 - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
 
 ### Dependencies

buildSrc/src/main/java/org/opensearch/gradle/RepositoriesSetupPlugin.java

@@ -94,7 +94,7 @@ public static void configureRepositories(Project project) {
             String revision = matcher.group(1);
             MavenArtifactRepository luceneRepo = repos.maven(repo -> {
                 repo.setName("lucene-snapshots");
-                repo.setUrl("https://artifacts.opensearch.org/snapshots/lucene/");
+                repo.setUrl("https://ci.opensearch.org/ci/dbc/snapshots/lucene/");
             });
             repos.exclusiveContent(exclusiveRepo -> {
                 exclusiveRepo.filter(

distribution/tools/plugin-cli/src/main/java/org/opensearch/plugins/ListPluginsCommand.java

@@ -78,15 +78,14 @@ private void printPlugin(Environment env, Terminal terminal, Path plugin, String
         PluginInfo info = PluginInfo.readFromProperties(env.pluginsDir().resolve(plugin));
         terminal.println(Terminal.Verbosity.SILENT, prefix + info.getName());
         terminal.println(Terminal.Verbosity.VERBOSE, info.toString(prefix));
-        if (info.getOpenSearchVersion().equals(Version.CURRENT) == false) {
+        if (!PluginsService.isPluginVersionCompatible(info, Version.CURRENT)) {
             terminal.errorPrintln(
                 "WARNING: plugin ["
                     + info.getName()
                     + "] was built for OpenSearch version "
-                    + info.getVersion()
-                    + " but version "
+                    + info.getOpenSearchVersionRangesString()
+                    + " and is not compatible with "
                     + Version.CURRENT
-                    + " is required"
             );
         }
     }

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/InstallPluginCommandTests.java

@@ -70,8 +70,10 @@
 import org.opensearch.core.util.FileSystemUtils;
 import org.opensearch.env.Environment;
 import org.opensearch.env.TestEnvironment;
+import org.opensearch.semver.SemverRange;
 import org.opensearch.test.OpenSearchTestCase;
 import org.opensearch.test.PosixPermissionsResetter;
+import org.opensearch.test.VersionUtils;
 import org.junit.After;
 import org.junit.Before;
 
@@ -284,6 +286,35 @@ static void writePlugin(String name, Path structure, String... additionalProps)
         writeJar(structure.resolve("plugin.jar"), className);
     }
 
+    static void writePlugin(String name, Path structure, SemverRange opensearchVersionRange, String... additionalProps) throws IOException {
+        String[] properties = Stream.concat(
+            Stream.of(
+                "description",
+                "fake desc",
+                "name",
+                name,
+                "version",
+                "1.0",
+                "dependencies",
+                "{opensearch:\"" + opensearchVersionRange + "\"}",
+                "java.version",
+                System.getProperty("java.specification.version"),
+                "classname",
+                "FakePlugin"
+            ),
+            Arrays.stream(additionalProps)
+        ).toArray(String[]::new);
+        PluginTestUtil.writePluginProperties(structure, properties);
+        String className = name.substring(0, 1).toUpperCase(Locale.ENGLISH) + name.substring(1) + "Plugin";
+        writeJar(structure.resolve("plugin.jar"), className);
+    }
+
+    static Path createPlugin(String name, Path structure, SemverRange opensearchVersionRange, String... additionalProps)
+        throws IOException {
+        writePlugin(name, structure, opensearchVersionRange, additionalProps);
+        return writeZip(structure, null);
+    }
+
     static void writePluginSecurityPolicy(Path pluginDir, String... permissions) throws IOException {
         StringBuilder securityPolicyContent = new StringBuilder("grant {\n  ");
         for (String permission : permissions) {
@@ -867,6 +898,32 @@ public void testInstallMisspelledOfficialPlugins() throws Exception {
         assertThat(e.getMessage(), containsString("Unknown plugin unknown_plugin"));
     }
 
+    public void testInstallPluginWithCompatibleDependencies() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        String pluginZip = createPlugin("fake", pluginDir, SemverRange.fromString("~" + Version.CURRENT.toString())).toUri()
+            .toURL()
+            .toString();
+        skipJarHellCommand.execute(terminal, Collections.singletonList(pluginZip), false, env.v2());
+        assertThat(terminal.getOutput(), containsString("100%"));
+    }
+
+    public void testInstallPluginWithIncompatibleDependencies() throws Exception {
+        Tuple<Path, Environment> env = createEnv(fs, temp);
+        Path pluginDir = createPluginDir(temp);
+        // Core version is behind plugin version by one w.r.t patch, hence incompatible
+        Version coreVersion = Version.CURRENT;
+        Version pluginVersion = VersionUtils.getVersion(coreVersion.major, coreVersion.minor, (byte) (coreVersion.revision + 1));
+        String pluginZip = createPlugin("fake", pluginDir, SemverRange.fromString("~" + pluginVersion.toString())).toUri()
+            .toURL()
+            .toString();
+        IllegalArgumentException e = expectThrows(
+            IllegalArgumentException.class,
+            () -> skipJarHellCommand.execute(terminal, Collections.singletonList(pluginZip), false, env.v2())
+        );
+        assertThat(e.getMessage(), containsString("Plugin [fake] was built for OpenSearch version ~" + pluginVersion));
+    }
+
     public void testBatchFlag() throws Exception {
         MockTerminal terminal = new MockTerminal();
         installPlugin(terminal, true);

distribution/tools/plugin-cli/src/test/java/org/opensearch/plugins/ListPluginsCommandTests.java

@@ -278,12 +278,49 @@ public void testExistingIncompatiblePlugin() throws Exception {
         buildFakePlugin(env, "fake desc 2", "fake_plugin2", "org.fake2");
 
         MockTerminal terminal = listPlugins(home);
-        String message = "plugin [fake_plugin1] was built for OpenSearch version 1.0 but version " + Version.CURRENT + " is required";
+        String message = "plugin [fake_plugin1] was built for OpenSearch version 5.0.0 and is not compatible with " + Version.CURRENT;
         assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput());
         assertEquals("WARNING: " + message + "\n", terminal.getErrorOutput());
 
         String[] params = { "-s" };
         terminal = listPlugins(home, params);
         assertEquals("fake_plugin1\nfake_plugin2\n", terminal.getOutput());
     }
+
+    public void testPluginWithDependencies() throws Exception {
+        PluginTestUtil.writePluginProperties(
+            env.pluginsDir().resolve("fake_plugin1"),
+            "description",
+            "fake desc 1",
+            "name",
+            "fake_plugin1",
+            "version",
+            "1.0",
+            "dependencies",
+            "{opensearch:\"" + Version.CURRENT + "\"}",
+            "java.version",
+            System.getProperty("java.specification.version"),
+            "classname",
+            "org.fake1"
+        );
+        String[] params = { "-v" };
+        MockTerminal terminal = listPlugins(home, params);
+        assertEquals(
+            buildMultiline(
+                "Plugins directory: " + env.pluginsDir(),
+                "fake_plugin1",
+                "- Plugin information:",
+                "Name: fake_plugin1",
+                "Description: fake desc 1",
+                "Version: 1.0",
+                "OpenSearch Version: " + Version.CURRENT.toString(),
+                "Java Version: " + System.getProperty("java.specification.version"),
+                "Native Controller: false",
+                "Extended Plugins: []",
+                " * Classname: org.fake1",
+                "Folder name: null"
+            ),
+            terminal.getOutput()
+        );
+    }
 }

gradle/code-coverage.gradle

@@ -13,7 +13,7 @@ repositories {
   gradlePluginPortal()
   // TODO: Find the way to use the repositories from RepositoriesSetupPlugin
   maven {
-    url = "https://artifacts.opensearch.org/snapshots/lucene/"
+    url = "https://ci.opensearch.org/ci/dbc/snapshots/lucene/"
   }
 }
 
@@ -37,14 +37,14 @@ tasks.withType(JacocoReport).configureEach {
 if (System.getProperty("tests.coverage")) {
   reporting {
     reports {
-      testCodeCoverageReport(JacocoCoverageReport) { 
+      testCodeCoverageReport(JacocoCoverageReport) {
         testType = TestSuiteType.UNIT_TEST
       }
     }
   }
 
   // Attach code coverage report task to Gradle check task
   project.getTasks().named(JavaBasePlugin.CHECK_TASK_NAME).configure {
-    dependsOn tasks.named('testCodeCoverageReport', JacocoReport) 
+    dependsOn tasks.named('testCodeCoverageReport', JacocoReport)
   }
 }

libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamInput.java

@@ -56,6 +56,7 @@
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
 import org.opensearch.core.xcontent.MediaType;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
+import org.opensearch.semver.SemverRange;
 
 import java.io.ByteArrayInputStream;
 import java.io.EOFException;
@@ -750,6 +751,8 @@ public Object readGenericValue() throws IOException {
                 return readCollection(StreamInput::readGenericValue, HashSet::new, Collections.emptySet());
             case 26:
                 return readBigInteger();
+            case 27:
+                return readSemverRange();
             default:
                 throw new IOException("Can't read unknown type [" + type + "]");
         }
@@ -1090,6 +1093,10 @@ public Version readVersion() throws IOException {
         return Version.fromId(readVInt());
     }
 
+    public SemverRange readSemverRange() throws IOException {
+        return SemverRange.fromString(readString());
+    }
+
     /** Reads the {@link Version} from the input stream */
     public Build readBuild() throws IOException {
         // the following is new for opensearch: we write the distribution to support any "forks"

libs/core/src/main/java/org/opensearch/core/common/io/stream/StreamOutput.java

@@ -54,6 +54,7 @@
 import org.opensearch.core.common.settings.SecureString;
 import org.opensearch.core.common.text.Text;
 import org.opensearch.core.concurrency.OpenSearchRejectedExecutionException;
+import org.opensearch.semver.SemverRange;
 
 import java.io.EOFException;
 import java.io.FileNotFoundException;
@@ -784,6 +785,10 @@ public final void writeOptionalInstant(@Nullable Instant instant) throws IOExcep
             o.writeByte((byte) 26);
             o.writeString(v.toString());
         });
+        writers.put(SemverRange.class, (o, v) -> {
+            o.writeByte((byte) 27);
+            o.writeSemverRange((SemverRange) v);
+        });
         WRITERS = Collections.unmodifiableMap(writers);
     }
 
@@ -1101,6 +1106,10 @@ public void writeVersion(final Version version) throws IOException {
         writeVInt(version.id);
     }
 
+    public void writeSemverRange(final SemverRange range) throws IOException {
+        writeString(range.toString());
+    }
+
     /** Writes the OpenSearch {@link Build} informn to the output stream */
     public void writeBuild(final Build build) throws IOException {
         // the following is new for opensearch: we write the distribution name to support any "forks" of the code