Merge remote-tracking branch 'elastic/master' into opt-out-query-cach…

…e-default-distribution * elastic/master: (213 commits) ML: Fix build after HLRC change Fix inner hits retrieval when stored fields are disabled (_none_) (elastic#33018) SQL: Show/desc commands now support table ids (elastic#33363) Mute testValidateFollowingIndexSettings HLRC: Add delete by query API (elastic#32782) [ML] The sort field on get records should default to the record_score (elastic#33358) [ML] Minor improvements to categorization Grok pattern creation (elastic#33353) [DOCS] fix a couple of typos (elastic#33356) Disable assemble task instead of removing it (elastic#33348) Simplify the return type of FieldMapper#parse. (elastic#32654) [ML] Delete forecast API (elastic#31134) (elastic#33218) Introduce private settings (elastic#33327) [Docs] Add search timeout caveats (elastic#33354) TESTS: Fix Race Condition in Temp Path Creation (elastic#33352) Fix from_range in search_after in changes snapshot (elastic#33335) TESTS+DISTR.: Fix testIndexCheckOnStartup Flake (elastic#33349) Null completion field should not throw IAE (elastic#33268) Adds code to help with IndicesRequestCacheIT failures (elastic#33313) Prevent NPE parsing the stop datafeed request. (elastic#33347) HLRC: Add ML get overall buckets API (elastic#33297) ...
jasontedor · Sep 4, 2018 · 2133855 · 2133855
2 parents 3e378b2 + ebd5eb6
commit 2133855
Show file tree

Hide file tree

Showing 332 changed files with 17,858 additions and 3,164 deletions.
diff --git a/benchmarks/build.gradle b/benchmarks/build.gradle
@@ -21,10 +21,7 @@ apply plugin: 'elasticsearch.build'
 apply plugin: 'application'
 mainClassName = 'org.openjdk.jmh.Main'
 
-// Not published so no need to assemble
-tasks.remove(assemble)
-build.dependsOn.remove('assemble')
-
+assemble.enabled = false
 archivesBaseName = 'elasticsearch-benchmarks'
 
 test.enabled = false

diff --git a/build.gradle b/build.gradle
@@ -20,16 +20,12 @@
 import com.github.jengelman.gradle.plugins.shadow.ShadowPlugin
 import org.apache.tools.ant.taskdefs.condition.Os
 import org.elasticsearch.gradle.BuildPlugin
-import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.plugin.PluginBuildPlugin
 import org.gradle.plugins.ide.eclipse.model.SourceFolder
 
-import java.nio.file.Files
-import java.nio.file.Path
-
 plugins {
     id 'com.gradle.build-scan' version '1.13.2'
 }
@@ -576,76 +572,18 @@ wrapper {
     }
 }
 
-static void assertLinesInFile(final Path path, final List<String> expectedLines) {
-  final List<String> actualLines = Files.readAllLines(path)
-  int line = 0
-  for (final String expectedLine : expectedLines) {
-    final String actualLine = actualLines.get(line)
-    if (expectedLine != actualLine) {
-      throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]")
-    }
-    line++
-  }
-}
-
-/*
- * Check that all generated JARs have our NOTICE.txt and an appropriate
- * LICENSE.txt in them. We configurate this in gradle but we'd like to
- * be extra paranoid.
- */
-subprojects { project ->
-  project.tasks.withType(Jar).whenTaskAdded { jarTask ->
-    final Task extract = project.task("extract${jarTask.name.capitalize()}", type: LoggedExec) {
-      dependsOn jarTask
-      ext.destination = project.buildDir.toPath().resolve("jar-extracted/${jarTask.name}")
-      commandLine "${->new File(rootProject.compilerJavaHome, 'bin/jar')}",
-          'xf', "${-> jarTask.outputs.files.singleFile}", 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'
-      workingDir destination
-      onlyIf {jarTask.enabled}
-      doFirst {
-        project.delete(destination)
-        Files.createDirectories(destination)
-      }
-    }
-
-    final Task checkNotice = project.task("verify${jarTask.name.capitalize()}Notice") {
-      dependsOn extract
-      onlyIf {jarTask.enabled}
-      doLast {
-        final List<String> noticeLines = Files.readAllLines(project.noticeFile.toPath())
-        final Path noticePath = extract.destination.resolve('META-INF/NOTICE.txt')
-        assertLinesInFile(noticePath, noticeLines)
-      }
-    }
-    project.check.dependsOn checkNotice
-
-    final Task checkLicense = project.task("verify${jarTask.name.capitalize()}License") {
-      dependsOn extract
-      onlyIf {jarTask.enabled}
-      doLast {
-        final List<String> licenseLines = Files.readAllLines(project.licenseFile.toPath())
-        final Path licensePath = extract.destination.resolve('META-INF/LICENSE.txt')
-        assertLinesInFile(licensePath, licenseLines)
-      }
-    }
-    project.check.dependsOn checkLicense
-  }
-}
-
 /* Remove assemble/dependenciesInfo on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {
   subprojects {
     if (project.path.startsWith(':qa')) {
       Task assemble = project.tasks.findByName('assemble')
       if (assemble) {
-        project.tasks.remove(assemble)
-        project.build.dependsOn.remove('assemble')
+        assemble.enabled = false
       }
       Task dependenciesInfo = project.tasks.findByName('dependenciesInfo')
       if (dependenciesInfo) {
-        project.tasks.remove(dependenciesInfo)
-        project.precommit.dependsOn.remove('dependenciesInfo')
+        dependenciesInfo.enabled = false
       }
     }
   }

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy
@@ -56,6 +56,7 @@ import org.gradle.util.GradleVersion
 import java.nio.charset.StandardCharsets
 import java.time.ZoneOffset
 import java.time.ZonedDateTime
+
 /**
  * Encapsulates build configuration for elasticsearch projects.
  */
@@ -546,7 +547,7 @@ class BuildPlugin implements Plugin<Project> {
             }
             // build poms with assemble (if the assemble task exists)
             Task assemble = project.tasks.findByName('assemble')
-            if (assemble) {
+            if (assemble && assemble.enabled) {
                 assemble.dependsOn(generatePOMTask)
             }
         }
@@ -739,6 +740,7 @@ class BuildPlugin implements Plugin<Project> {
                     }
                     from(project.noticeFile.parent) {
                         include project.noticeFile.name
+                        rename { 'NOTICE.txt' }
                     }
                 }
             }

diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/doc/DocsTestPlugin.groovy
@@ -35,8 +35,7 @@ public class DocsTestPlugin extends RestTestPlugin {
         // The distribution can be configured with -Dtests.distribution on the command line
         project.integTestCluster.distribution = System.getProperty('tests.distribution', 'zip')
         // Docs are published separately so no need to assemble
-        project.tasks.remove(project.assemble)
-        project.build.dependsOn.remove('assemble')
+        project.tasks.assemble.enabled = false
         Map<String, String> defaultSubstitutions = [
             /* These match up with the asciidoc syntax for substitutions but
              * the values may differ. In particular {version} needs to resolve

diff --git a/buildSrc/src/main/resources/checkstyle_suppressions.xml b/buildSrc/src/main/resources/checkstyle_suppressions.xml
@@ -539,7 +539,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentMapperMergeTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DocumentParserTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]DynamicMappingTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalFieldMapperTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMapper.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]ExternalMetadataMapper.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]mapper[/\\]FieldNamesFieldMapperTests.java" checks="LineLength" />

diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/BuildPluginIT.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.gradle;
+
+import org.apache.commons.io.IOUtils;
+import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
+import org.gradle.testkit.runner.BuildResult;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipFile;
+
+public class BuildPluginIT extends GradleIntegrationTestCase {
+
+    public void testPluginCanBeApplied() {
+        BuildResult result = getGradleRunner("elasticsearch.build")
+            .withArguments("hello", "-s")
+            .build();
+        assertTaskSuccessful(result, ":hello");
+        assertOutputContains("build plugin can be applied");
+    }
+
+    public void testCheckTask() {
+        BuildResult result = getGradleRunner("elasticsearch.build")
+            .withArguments("check", "assemble", "-s", "-Dlocal.repo.path=" + getLocalTestRepoPath())
+            .build();
+        assertTaskSuccessful(result, ":check");
+    }
+
+    public void testLicenseAndNotice() throws IOException {
+        BuildResult result = getGradleRunner("elasticsearch.build")
+            .withArguments("clean", "assemble", "-s", "-Dlocal.repo.path=" + getLocalTestRepoPath())
+            .build();
+
+        assertTaskSuccessful(result, ":assemble");
+
+        assertBuildFileExists(result, "elasticsearch.build", "distributions/elasticsearch.build.jar");
+
+        try (ZipFile zipFile = new ZipFile(new File(
+            getBuildDir("elasticsearch.build"), "distributions/elasticsearch.build.jar"
+        ))) {
+            ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt");
+            ZipEntry noticeEntry = zipFile.getEntry("META-INF/NOTICE.txt");
+            assertNotNull("Jar does not have META-INF/LICENSE.txt", licenseEntry);
+            assertNotNull("Jar does not have META-INF/NOTICE.txt", noticeEntry);
+            try (
+                InputStream license = zipFile.getInputStream(licenseEntry);
+                InputStream notice = zipFile.getInputStream(noticeEntry)
+            ) {
+                assertEquals("this is a test license file", IOUtils.toString(license, StandardCharsets.UTF_8.name()));
+                assertEquals("this is a test notice file", IOUtils.toString(notice, StandardCharsets.UTF_8.name()));
+            }
+        }
+    }
+
+
+}
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java b/buildSrc/src/test/java/org/elasticsearch/gradle/precommit/NamingConventionsTaskIT.java
@@ -2,74 +2,57 @@
 
 import org.elasticsearch.gradle.test.GradleIntegrationTestCase;
 import org.gradle.testkit.runner.BuildResult;
-import org.gradle.testkit.runner.GradleRunner;
-import org.gradle.testkit.runner.TaskOutcome;
 
 import java.util.Arrays;
+import java.util.HashSet;
 
 public class NamingConventionsTaskIT extends GradleIntegrationTestCase {
 
-    public void testPluginCanBeApplied() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("namingConventionsSelfTest"))
-            .withArguments("hello", "-s", "-PcheckForTestsInMain=false")
-            .withPluginClasspath()
-            .build();
-
-        assertEquals(TaskOutcome.SUCCESS, result.task(":hello").getOutcome());
-        String output = result.getOutput();
-        assertTrue(output, output.contains("build plugin can be applied"));
-    }
-
     public void testNameCheckFailsAsItShould() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("namingConventionsSelfTest"))
+        BuildResult result = getGradleRunner("namingConventionsSelfTest")
             .withArguments("namingConventions", "-s", "-PcheckForTestsInMain=false")
-            .withPluginClasspath()
             .buildAndFail();
 
-        assertNotNull("task did not run", result.task(":namingConventions"));
-        assertEquals(TaskOutcome.FAILED, result.task(":namingConventions").getOutcome());
-        String output = result.getOutput();
-        for (String line : Arrays.asList(
-            "Found inner classes that are tests, which are excluded from the test runner:",
-            "* org.elasticsearch.test.NamingConventionsCheckInMainIT$InternalInvalidTests",
-            "Classes ending with [Tests] must subclass [UnitTestCase]:",
-            "* org.elasticsearch.test.NamingConventionsCheckInMainTests",
-            "* org.elasticsearch.test.NamingConventionsCheckInMainIT",
-            "Not all subclasses of UnitTestCase match the naming convention. Concrete classes must end with [Tests]:",
-            "* org.elasticsearch.test.WrongName")) {
-            assertTrue(
-                "expected:  '" + line + "' but it was not found in the output:\n" + output,
-                output.contains(line)
-            );
-        }
+        assertTaskFailed(result, ":namingConventions");
+        assertOutputContains(
+            result.getOutput(),
+            // TODO: java9 Set.of
+            new HashSet<>(
+                Arrays.asList(
+                    "Not all subclasses of UnitTestCase match the naming convention. Concrete classes must end with [Tests]:",
+                    "* org.elasticsearch.test.WrongName",
+                    "Found inner classes that are tests, which are excluded from the test runner:",
+                    "* org.elasticsearch.test.NamingConventionsCheckInMainIT$InternalInvalidTests",
+                    "Classes ending with [Tests] must subclass [UnitTestCase]:",
+                    "* org.elasticsearch.test.NamingConventionsCheckInMainTests",
+                    "* org.elasticsearch.test.NamingConventionsCheckInMainIT"
+                )
+            )
+        );
     }
 
     public void testNameCheckFailsAsItShouldWithMain() {
-        BuildResult result = GradleRunner.create()
-            .withProjectDir(getProjectDir("namingConventionsSelfTest"))
+        BuildResult result = getGradleRunner("namingConventionsSelfTest")
             .withArguments("namingConventions", "-s", "-PcheckForTestsInMain=true")
-            .withPluginClasspath()
             .buildAndFail();
 
-        assertNotNull("task did not run", result.task(":namingConventions"));
-        assertEquals(TaskOutcome.FAILED, result.task(":namingConventions").getOutcome());
-
-        String output = result.getOutput();
-        for (String line : Arrays.asList(
-            "Classes ending with [Tests] or [IT] or extending [UnitTestCase] must be in src/test/java:",
-            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyInterfaceTests",
-            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyAbstractTests",
-            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$InnerTests",
-            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$NotImplementingTests",
-            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$WrongNameTheSecond",
-            "* org.elasticsearch.test.NamingConventionsCheckBadClasses$WrongName")) {
-            assertTrue(
-                "expected:  '" + line + "' but it was not found in the output:\n"+output,
-                output.contains(line)
-            );
-        }
+        assertTaskFailed(result, ":namingConventions");
+        assertOutputContains(
+            result.getOutput(),
+            // TODO: java9 Set.of
+            new HashSet<>(
+                Arrays.asList(
+                    "Classes ending with [Tests] or [IT] or extending [UnitTestCase] must be in src/test/java:",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyInterfaceTests",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyInterfaceTests",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$DummyAbstractTests",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$InnerTests",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$NotImplementingTests",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$WrongNameTheSecond",
+                    "* org.elasticsearch.test.NamingConventionsCheckBadClasses$WrongName"
+                )
+            )
+        );
     }
 
 }
diff --git a/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java b/buildSrc/src/test/java/org/elasticsearch/gradle/test/GradleIntegrationTestCase.java
@@ -10,6 +10,7 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Objects;
+import java.util.Set;
 import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
@@ -47,6 +48,12 @@ protected void assertOutputContains(String output, String... lines) {
         }
     }
 
+    protected void assertOutputContains(String output, Set<String> lines) {
+        for (String line : lines) {
+            assertOutputContains(output, line);
+        }
+    }
+
     protected void assertOutputContains(String output, String line) {
         assertTrue(
             "Expected the following line in output:\n\n" + line + "\n\nOutput is:\n" + output,
@@ -82,7 +89,7 @@ private void assertTaskOutcome(BuildResult result, String taskName, TaskOutcome
                 "\n\nOutput is:\n" + result.getOutput());
         }
         assertEquals(
-            "Expected task to be successful but it was: " + task.getOutcome() +
+            "Expected task `" + taskName +"` to be successful but it was: " + task.getOutcome() +
                 taskOutcome + "\n\nOutput is:\n" + result.getOutput() ,
             taskOutcome,
             task.getOutcome()

diff --git a/buildSrc/src/testKit/elasticsearch.build/LICENSE b/buildSrc/src/testKit/elasticsearch.build/LICENSE
@@ -0,0 +1 @@
+this is a test license file
diff --git a/buildSrc/src/testKit/elasticsearch.build/NOTICE b/buildSrc/src/testKit/elasticsearch.build/NOTICE
@@ -0,0 +1 @@
+this is a test notice file