Merge branch 'master' into allowed-index-store-types

* master:
  Fix ordering of bootstrap checks in docs (elastic#32417)
  [TEST] Mute failing InternalEngineTests#testSeqNoAndCheckpoints
  [TEST] Mute failing testConvertLongHexError
  bump lucene version after backport
  Upgrade to Lucene-7.5.0-snapshot-608f0277b0 (elastic#32390)
  [Kerberos] Avoid vagrant update on precommit (elastic#32416)

Author: jasontedor

buildSrc/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 7.0.0-alpha1
-lucene            = 7.4.0
+lucene            = 7.5.0-snapshot-608f0277b0
 
 # optional dependencies
 spatial4j         = 0.7

docs/Versions.asciidoc

@@ -19,24 +19,24 @@
 
 package org.elasticsearch.ingest.common;
 
-import org.elasticsearch.ingest.IngestDocument;
-import org.elasticsearch.ingest.Processor;
-import org.elasticsearch.ingest.RandomDocumentPicks;
-import org.elasticsearch.test.ESTestCase;
-
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Locale;
 import java.util.Map;
 
+import org.elasticsearch.ingest.IngestDocument;
+import org.elasticsearch.ingest.Processor;
+import org.elasticsearch.ingest.RandomDocumentPicks;
+import org.elasticsearch.test.ESTestCase;
+
 import static org.elasticsearch.ingest.IngestDocumentMatcher.assertIngestDocument;
 import static org.elasticsearch.ingest.common.ConvertProcessor.Type;
 import static org.hamcrest.Matchers.containsString;
 import static org.hamcrest.Matchers.equalTo;
-import static org.hamcrest.Matchers.sameInstance;
 import static org.hamcrest.Matchers.not;
+import static org.hamcrest.Matchers.sameInstance;
 
 public class ConvertProcessorTests extends ESTestCase {
 
@@ -138,6 +138,7 @@ public void testConvertLongLeadingZero() throws Exception {
         assertThat(ingestDocument.getFieldValue(fieldName, Long.class), equalTo(10L));
     }
 
+    @AwaitsFix( bugUrl = "https://github.com/elastic/elasticsearch/issues/32370")
     public void testConvertLongHexError() {
         IngestDocument ingestDocument = RandomDocumentPicks.randomIngestDocument(random());
         String value = "0x" + randomAlphaOfLengthBetween(1, 10);

docs/reference/setup/bootstrap-checks.asciidoc

@@ -0,0 +1 @@
+bd7d8078a2d0ad11a24f54156cc015630c96858a

modules/ingest-common/src/test/java/org/elasticsearch/ingest/common/ConvertProcessorTests.java

@@ -30,7 +30,7 @@ forbiddenApis {
 
 dependencies {
   compile "org.apache.lucene:lucene-analyzers-icu:${versions.lucene}"
-  compile 'com.ibm.icu:icu4j:61.1'
+  compile 'com.ibm.icu:icu4j:62.1'
 }
 
 dependencyLicenses {

modules/lang-expression/licenses/lucene-expressions-7.4.0.jar.sha1

@@ -0,0 +1 @@
+7a4d00d5ec5febd252a6182e8b6e87a0a9821f81

modules/lang-expression/licenses/lucene-expressions-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+7a37816def72a748416c4ae8b0f6817e30efb99f

plugins/analysis-icu/build.gradle

@@ -0,0 +1 @@
+ca7437178cdbf7b8bfe0d75c75e3c8eb93925724

plugins/analysis-icu/licenses/icu4j-61.1.jar.sha1

@@ -0,0 +1 @@
+3f5dec44f380d6d58bc1c8aec51964fcb5390b60

plugins/analysis-icu/licenses/icu4j-62.1.jar.sha1

@@ -0,0 +1 @@
+453bf1d60df0415439095624e0b3e42492ad4716

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.4.0.jar.sha1

@@ -0,0 +1 @@
+70095a45257bca9f46629b5fb6cedf9eff5e2b07

plugins/analysis-icu/licenses/lucene-analyzers-icu-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+7199d6962d268b7877f7b5160e98e4ff21cce5c7

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.4.0.jar.sha1

@@ -0,0 +1 @@
+12aff508d39d206a1aead5013ecd11882062eb06

plugins/analysis-kuromoji/licenses/lucene-analyzers-kuromoji-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -30,4 +30,12 @@ public class SmbMMapDirectoryTests extends EsBaseDirectoryTestCase {
     protected Directory getDirectory(Path file) throws IOException {
         return new SmbDirectoryWrapper(new MMapDirectory(file));
     }
+
+    @Override
+    public void testCreateOutputForExistingFile() throws IOException {
+        /**
+         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+         */
+    }
 }

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.4.0.jar.sha1

@@ -30,4 +30,12 @@ public class SmbSimpleFSDirectoryTests extends EsBaseDirectoryTestCase {
     protected Directory getDirectory(Path file) throws IOException {
         return new SmbDirectoryWrapper(new SimpleFSDirectory(file));
     }
+
+    @Override
+    public void testCreateOutputForExistingFile() throws IOException {
+        /**
+         * This test is disabled because {@link SmbDirectoryWrapper} opens existing file
+         * with an explicit StandardOpenOption.TRUNCATE_EXISTING option.
+         */
+    }
 }

plugins/analysis-nori/licenses/lucene-analyzers-nori-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+d27958843ca118db2ffd2c242ae3761bd5a47328

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.4.0.jar.sha1

@@ -0,0 +1 @@
+7ea220ba8e4accb8b04e280463042ad470e23bc0

plugins/analysis-phonetic/licenses/lucene-analyzers-phonetic-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+471096d6e92338b208aa91f3a85feb2f9cfc4afd

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.4.0.jar.sha1

@@ -0,0 +1 @@
+f0af947c60d24f779c22f774e81ebd7dd91cc932

plugins/analysis-smartcn/licenses/lucene-analyzers-smartcn-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+fbc83ac5a0139ed7e7faf6c95a2718f46f28c641

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.4.0.jar.sha1

@@ -0,0 +1 @@
+30adfe493982b0db059dc243e269eea38d850d46

plugins/analysis-stempel/licenses/lucene-analyzers-stempel-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+656f304261d9aad05070fb68593beffafe9147e3

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.4.0.jar.sha1

@@ -0,0 +1 @@
+8bf22ad81a7480c255b55bada401eb131bfdb4df

plugins/analysis-ukrainian/licenses/lucene-analyzers-morfologik-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+edb3de4d68a34c1e1ca08f79fe4d103b10e98ad1

plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbMMapDirectoryTests.java

@@ -0,0 +1 @@
+7ece30d5f1e18d96f61644451c858c3d9960558f

plugins/store-smb/src/test/java/org/elasticsearch/index/store/SmbSimpleFSDirectoryTests.java

@@ -0,0 +1 @@
+ad3bd0c2ed96556193c7215bef328e689d0b157f

server/licenses/lucene-analyzers-common-7.4.0.jar.sha1

@@ -0,0 +1 @@
+8a6bd97e39ee5af60126adbe8c8375dc41b1ea8e

server/licenses/lucene-analyzers-common-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+07e748d2d80000a7a213f3405b82b6e26b452948

server/licenses/lucene-backward-codecs-7.4.0.jar.sha1

@@ -0,0 +1 @@
+fd737bd5562f3943618ee7e73a0aaffb6319fdb2

server/licenses/lucene-backward-codecs-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -0,0 +1 @@
+ff3f260d1dc8c18bc67f3c33aa84a0ad290daac5

server/licenses/lucene-core-7.4.0.jar.sha1

@@ -179,10 +179,10 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final int V_6_4_0_ID = 6040099;
     public static final Version V_6_4_0 = new Version(V_6_4_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
     public static final int V_6_5_0_ID = 6050099;
-    public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+    public static final Version V_6_5_0 = new Version(V_6_5_0_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
     public static final int V_7_0_0_alpha1_ID = 7000001;
     public static final Version V_7_0_0_alpha1 =
-        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_4_0);
+        new Version(V_7_0_0_alpha1_ID, org.apache.lucene.util.Version.LUCENE_7_5_0);
     public static final Version CURRENT = V_7_0_0_alpha1;
 
     static {

server/licenses/lucene-core-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -52,7 +52,7 @@ public MergeSpecification findForcedMerges(SegmentInfos infos, int maxSegmentCou
 
     @Override
     public MergeSpecification findForcedDeletesMerges(SegmentInfos infos, MergeContext mergeContext) throws IOException {
-        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);    
+        return forcedMergePolicy.findForcedDeletesMerges(infos, mergeContext);
     }
 
     public void setForceMergeDeletesPctAllowed(double forceMergeDeletesPctAllowed) {
@@ -108,13 +108,4 @@ public void setSegmentsPerTier(double segmentsPerTier) {
     public double getSegmentsPerTier() {
         return regularMergePolicy.getSegmentsPerTier();
     }
-
-    public void setReclaimDeletesWeight(double reclaimDeletesWeight) {
-        regularMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-        forcedMergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-    }
-
-    public double getReclaimDeletesWeight() {
-        return regularMergePolicy.getReclaimDeletesWeight();
-    }
 }

server/licenses/lucene-grouping-7.4.0.jar.sha1

@@ -416,7 +416,6 @@ public IndexSettings(final IndexMetaData indexMetaData, final Settings nodeSetti
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGE_AT_ONCE_EXPLICIT_SETTING, mergePolicyConfig::setMaxMergesAtOnceExplicit);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING, mergePolicyConfig::setMaxMergedSegment);
         scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING, mergePolicyConfig::setSegmentsPerTier);
-        scopedSettings.addSettingsUpdateConsumer(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING, mergePolicyConfig::setReclaimDeletesWeight);
 
         scopedSettings.addSettingsUpdateConsumer(MergeSchedulerConfig.MAX_THREAD_COUNT_SETTING, MergeSchedulerConfig.MAX_MERGE_COUNT_SETTING,
             mergeSchedulerConfig::setMaxThreadAndMergeCount);

server/licenses/lucene-grouping-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -150,7 +150,7 @@ public final class MergePolicyConfig {
             Property.Dynamic, Property.IndexScope);
     public static final Setting<Double> INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING =
         Setting.doubleSetting("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT, 0.0d,
-            Property.Dynamic, Property.IndexScope);
+            Property.Dynamic, Property.IndexScope, Property.Deprecated);
     public static final String INDEX_MERGE_ENABLED = "index.merge.enabled"; // don't convert to Setting<> and register... we only set this in tests and register via a plugin
 
 
@@ -176,17 +176,12 @@ public final class MergePolicyConfig {
         mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit);
         mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.getMbFrac());
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
-        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
         if (logger.isTraceEnabled()) {
             logger.trace("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]",
                 forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight);
         }
     }
 
-    void setReclaimDeletesWeight(Double reclaimDeletesWeight) {
-        mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight);
-    }
-
     void setSegmentsPerTier(Double segmentsPerTier) {
         mergePolicy.setSegmentsPerTier(segmentsPerTier);
     }

server/licenses/lucene-highlighter-7.4.0.jar.sha1

@@ -29,16 +29,14 @@ public void testDefaults() {
         assertEquals(
                 new TieredMergePolicy().getMaxMergedSegmentMB(),
                 policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
-        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
-        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+        assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
     }
 
     public void testSetMaxMergedSegmentMB() {
         EsTieredMergePolicy policy = new EsTieredMergePolicy();
         policy.setMaxMergedSegmentMB(10 * 1024);
         assertEquals(10 * 1024, policy.regularMergePolicy.getMaxMergedSegmentMB(), 0d);
-        // TODO: fix when incorporating https://issues.apache.org/jira/browse/LUCENE-8398, the first divisor must be a double
-        assertEquals(Long.MAX_VALUE / 1024 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
+        assertEquals(Long.MAX_VALUE / 1024.0 / 1024.0, policy.forcedMergePolicy.getMaxMergedSegmentMB(), 0d);
     }
 
     public void testSetForceMergeDeletesPctAllowed() {
@@ -71,10 +69,4 @@ public void testSetSegmentsPerTier() {
         policy.setSegmentsPerTier(42);
         assertEquals(42, policy.regularMergePolicy.getSegmentsPerTier(), 0);
     }
-
-    public void testSetReclaimDeletesWeight() {
-        EsTieredMergePolicy policy = new EsTieredMergePolicy();
-        policy.setReclaimDeletesWeight(42);
-        assertEquals(42, policy.regularMergePolicy.getReclaimDeletesWeight(), 0);
-    }
 }

server/licenses/lucene-highlighter-7.5.0-snapshot-608f0277b0.jar.sha1

@@ -96,10 +96,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_MAX_MERGED_SEGMENT_SETTING.getKey(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1)).build()));
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
 
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
-        indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_RECLAIM_DELETES_WEIGHT_SETTING.getKey(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1).build()));
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT + 1, 0);
-
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
         indexSettings.updateIndexMetaData(newIndexMeta("index", Settings.builder().put(MergePolicyConfig.INDEX_MERGE_POLICY_SEGMENTS_PER_TIER_SETTING.getKey(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1).build()));
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER + 1, 0);
@@ -110,7 +106,6 @@ public void testTieredMergePolicySettingsUpdate() throws IOException {
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnce(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergeAtOnceExplicit(), MergePolicyConfig.DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getMaxMergedSegmentMB(), new ByteSizeValue(MergePolicyConfig.DEFAULT_MAX_MERGED_SEGMENT.getBytes() + 1).getMbFrac(), 0.0001);
-        assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getReclaimDeletesWeight(), MergePolicyConfig.DEFAULT_RECLAIM_DELETES_WEIGHT, 0);
         assertEquals(((EsTieredMergePolicy) indexSettings.getMergePolicy()).getSegmentsPerTier(), MergePolicyConfig.DEFAULT_SEGMENTS_PER_TIER, 0);
     }