Merge branch 'master' into zen2_tests_bootstrap

# Conflicts:
#	server/src/test/java/org/elasticsearch/cluster/MinimumMasterNodesIT.java
#	server/src/test/java/org/elasticsearch/cluster/SpecificMasterNodesIT.java
#	server/src/test/java/org/elasticsearch/cluster/coordination/UnsafeBootstrapMasterIT.java

Author: Andrey Ershov

benchmarks/src/main/java/org/elasticsearch/benchmark/time/DateFormatterFromBenchmark.java

@@ -0,0 +1,53 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.elasticsearch.benchmark.time;
+
+import org.elasticsearch.common.time.DateFormatter;
+import org.elasticsearch.common.time.DateFormatters;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Warmup;
+
+import java.time.temporal.TemporalAccessor;
+import java.util.concurrent.TimeUnit;
+
+@Fork(3)
+@Warmup(iterations = 10)
+@Measurement(iterations = 10)
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.NANOSECONDS)
+@State(Scope.Benchmark)
+@SuppressWarnings("unused") //invoked by benchmarking framework
+public class DateFormatterFromBenchmark {
+
+    private final TemporalAccessor accessor = DateFormatter.forPattern("epoch_millis").parse("1234567890");
+
+    @Benchmark
+    public TemporalAccessor benchmarkFrom() {
+        // benchmark an accessor that does not contain a timezone
+        // this used to throw an exception earlier and thus was very very slow
+        return DateFormatters.from(accessor);
+    }
+}

build.gradle

@@ -159,8 +159,8 @@ task verifyVersions {
  * the enabled state of every bwc task. It should be set back to true
  * after the backport of the backcompat code is complete.
  */
-final boolean bwc_tests_enabled = true
-final String bwc_tests_disabled_issue = "" /* place a PR link here when committing bwc changes */
+final boolean bwc_tests_enabled = false
+final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/37899" /* place a PR link here when committing bwc changes */
 if (bwc_tests_enabled == false) {
   if (bwc_tests_disabled_issue.isEmpty()) {
     throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false")
@@ -201,7 +201,7 @@ allprojects {
   }
 
   /* Sets up the dependencies that we build as part of this project but
-    register as thought they were external to resolve internally. We register
+    register as though they were external to resolve internally. We register
     them as external dependencies so the build plugin that we use can be used
     to build elasticsearch plugins outside of the elasticsearch source tree. */
   ext.projectSubstitutions = [
@@ -214,6 +214,7 @@ allprojects {
     "org.elasticsearch:elasticsearch-x-content:${version}": ':libs:x-content',
     "org.elasticsearch:elasticsearch-geo:${version}": ':libs:elasticsearch-geo',
     "org.elasticsearch:elasticsearch-secure-sm:${version}": ':libs:secure-sm',
+    "org.elasticsearch:elasticsearch-ssl-config:${version}": ':libs:elasticsearch-ssl-config',
     "org.elasticsearch.client:elasticsearch-rest-client:${version}": ':client:rest',
     "org.elasticsearch.client:elasticsearch-rest-client-sniffer:${version}": ':client:sniffer',
     "org.elasticsearch.client:elasticsearch-rest-high-level-client:${version}": ':client:rest-high-level',

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -373,18 +373,15 @@ class ClusterFormationTasks {
                 'path.repo'                    : "${node.sharedDir}/repo",
                 'path.shared_data'             : "${node.sharedDir}/",
                 // Define a node attribute so we can test that it exists
-                'node.attr.testattr'           : 'test'
+                'node.attr.testattr'           : 'test',
+                // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
+                'discovery.initial_state_timeout' : '0s'
         ]
         int minimumMasterNodes = node.config.minimumMasterNodes.call()
-        if (minimumMasterNodes > 0) {
+        if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) {
             esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
         }
-        if (minimumMasterNodes > 1) {
-            // don't wait for state.. just start up quickly
-            // this will also allow new and old nodes in the BWC case to become the master
-            esConfig['discovery.initial_state_timeout'] = '0s'
-        }
-        if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
+        if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
             // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct
             // its mistake. Instead, only wait 5s to do another round of pinging.
             // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation

buildSrc/version.properties

@@ -21,16 +21,13 @@ joda              = 2.10.1
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.12
-httpclient        = 4.5.2
-# When updating httpcore, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpcore          = 4.4.5
-# When updating httpasyncclient, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
-httpasyncclient   = 4.1.2
+httpclient        = 4.5.7
+httpcore          = 4.4.11
+httpasyncclient   = 4.1.4
 commonslogging    = 1.1.3
-commonscodec      = 1.10
+commonscodec      = 1.11
 hamcrest          = 1.3
 securemock        = 1.2
-# When updating mocksocket, please also update server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy
 mocksocket        = 1.2
 
 # benchmark dependencies

client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java

@@ -76,6 +76,7 @@
 import org.elasticsearch.index.reindex.DeleteByQueryRequest;
 import org.elasticsearch.index.reindex.ReindexRequest;
 import org.elasticsearch.index.reindex.UpdateByQueryRequest;
+import org.elasticsearch.index.seqno.SequenceNumbers;
 import org.elasticsearch.rest.action.search.RestSearchAction;
 import org.elasticsearch.script.mustache.MultiSearchTemplateRequest;
 import org.elasticsearch.script.mustache.SearchTemplateRequest;
@@ -885,6 +886,20 @@ Params withVersionType(VersionType versionType) {
             return this;
         }
 
+        Params withIfSeqNo(long ifSeqNo) {
+            if (ifSeqNo != SequenceNumbers.UNASSIGNED_SEQ_NO) {
+                return putParam("if_seq_no", Long.toString(ifSeqNo));
+            }
+            return this;
+        }
+
+        Params withIfPrimaryTerm(long ifPrimaryTerm) {
+            if (ifPrimaryTerm != SequenceNumbers.UNASSIGNED_PRIMARY_TERM) {
+                return putParam("if_primary_term", Long.toString(ifPrimaryTerm));
+            }
+            return this;
+        }
+
         Params withWaitForActiveShards(ActiveShardCount activeShardCount) {
             return withWaitForActiveShards(activeShardCount, ActiveShardCount.DEFAULT);
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java

@@ -1671,15 +1671,16 @@ protected final ElasticsearchStatusException parseResponseException(ResponseExce
         Response response = responseException.getResponse();
         HttpEntity entity = response.getEntity();
         ElasticsearchStatusException elasticsearchException;
+        RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode());
+
         if (entity == null) {
             elasticsearchException = new ElasticsearchStatusException(
-                    responseException.getMessage(), RestStatus.fromCode(response.getStatusLine().getStatusCode()), responseException);
+                    responseException.getMessage(), restStatus, responseException);
         } else {
             try {
                 elasticsearchException = parseEntity(entity, BytesRestResponse::errorFromXContent);
                 elasticsearchException.addSuppressed(responseException);
             } catch (Exception e) {
-                RestStatus restStatus = RestStatus.fromCode(response.getStatusLine().getStatusCode());
                 elasticsearchException = new ElasticsearchStatusException("Unable to parse response body", restStatus, responseException);
                 elasticsearchException.addSuppressed(e);
             }

client/rest-high-level/src/main/java/org/elasticsearch/client/WatcherRequestConverters.java

@@ -69,7 +69,10 @@ static Request putWatch(PutWatchRequest putWatchRequest) {
             .build();
 
         Request request = new Request(HttpPut.METHOD_NAME, endpoint);
-        RequestConverters.Params params = new RequestConverters.Params(request).withVersion(putWatchRequest.getVersion());
+        RequestConverters.Params params = new RequestConverters.Params(request)
+            .withVersion(putWatchRequest.getVersion())
+            .withIfSeqNo(putWatchRequest.ifSeqNo())
+            .withIfPrimaryTerm(putWatchRequest.ifPrimaryTerm());
         if (putWatchRequest.isActive() == false) {
             params.putParam("active", "false");
         }

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/AllocateAction.java

@@ -40,7 +40,7 @@ public class AllocateAction implements LifecycleAction, ToXContentObject {
     static final ParseField REQUIRE_FIELD = new ParseField("require");
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+    private static final ConstructingObjectParser<AllocateAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
             a -> new AllocateAction((Integer) a[0], (Map<String, String>) a[1], (Map<String, String>) a[2], (Map<String, String>) a[3]));
 
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/DeleteAction.java

@@ -30,7 +30,7 @@
 public class DeleteAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "delete";
 
-    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, DeleteAction::new);
+    private static final ObjectParser<DeleteAction, Void> PARSER = new ObjectParser<>(NAME, true, DeleteAction::new);
 
     public static DeleteAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ForceMergeAction.java

@@ -33,7 +33,7 @@ public class ForceMergeAction implements LifecycleAction, ToXContentObject {
     private static final ParseField MAX_NUM_SEGMENTS_FIELD = new ParseField("max_num_segments");
 
     private static final ConstructingObjectParser<ForceMergeAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
-        false, a -> {
+        true, a -> {
         int maxNumSegments = (int) a[0];
         return new ForceMergeAction(maxNumSegments);
     });

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/FreezeAction.java

@@ -29,7 +29,7 @@
 public class FreezeAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "freeze";
 
-    private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, FreezeAction::new);
+    private static final ObjectParser<FreezeAction, Void> PARSER = new ObjectParser<>(NAME, true, FreezeAction::new);
 
     public static FreezeAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecycleManagementStatusResponse.java

@@ -34,7 +34,7 @@ public class LifecycleManagementStatusResponse {
     private static final String OPERATION_MODE = "operation_mode";
     @SuppressWarnings("unchecked")
     private static final ConstructingObjectParser<LifecycleManagementStatusResponse, Void> PARSER = new ConstructingObjectParser<>(
-        OPERATION_MODE, a -> new LifecycleManagementStatusResponse((String) a[0]));
+        OPERATION_MODE, true, a -> new LifecycleManagementStatusResponse((String) a[0]));
 
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), new ParseField(OPERATION_MODE));

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicy.java

@@ -44,7 +44,7 @@ public class LifecyclePolicy implements ToXContentObject {
     static final ParseField PHASES_FIELD = new ParseField("phases");
 
     @SuppressWarnings("unchecked")
-    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", false,
+    public static ConstructingObjectParser<LifecyclePolicy, String> PARSER = new ConstructingObjectParser<>("lifecycle_policy", true,
         (a, name) -> {
             List<Phase> phases = (List<Phase>) a[0];
             Map<String, Phase> phaseMap = phases.stream().collect(Collectors.toMap(Phase::getName, Function.identity()));

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/LifecyclePolicyMetadata.java

@@ -38,7 +38,8 @@ public class LifecyclePolicyMetadata implements ToXContentObject {
     static final ParseField MODIFIED_DATE = new ParseField("modified_date");
 
     @SuppressWarnings("unchecked")
-    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>("policy_metadata",
+    public static final ConstructingObjectParser<LifecyclePolicyMetadata, String> PARSER = new ConstructingObjectParser<>(
+            "policy_metadata", true,
             a -> {
                 LifecyclePolicy policy = (LifecyclePolicy) a[0];
                 return new LifecyclePolicyMetadata(policy, (long) a[1], ZonedDateTime.parse((String) a[2]).toInstant().toEpochMilli());

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/Phase.java

@@ -44,7 +44,7 @@ public class Phase implements ToXContentObject {
     static final ParseField ACTIONS_FIELD = new ParseField("actions");
 
     @SuppressWarnings("unchecked")
-    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", false,
+    private static final ConstructingObjectParser<Phase, String> PARSER = new ConstructingObjectParser<>("phase", true,
         (a, name) -> new Phase(name, (TimeValue) a[0], ((List<LifecycleAction>) a[1]).stream()
             .collect(Collectors.toMap(LifecycleAction::getName, Function.identity()))));
     static {

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/PhaseExecutionInfo.java

@@ -40,7 +40,7 @@ public class PhaseExecutionInfo implements ToXContentObject {
     private static final ParseField MODIFIED_DATE_IN_MILLIS_FIELD = new ParseField("modified_date_in_millis");
 
     private static final ConstructingObjectParser<PhaseExecutionInfo, String> PARSER = new ConstructingObjectParser<>(
-        "phase_execution_info", false,
+        "phase_execution_info", true,
         (a, name) -> new PhaseExecutionInfo((String) a[0], (Phase) a[1], (long) a[2], (long) a[3]));
     static {
         PARSER.declareString(ConstructingObjectParser.constructorArg(), POLICY_NAME_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ReadOnlyAction.java

@@ -29,7 +29,7 @@
 public class ReadOnlyAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "readonly";
 
-    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, false, ReadOnlyAction::new);
+    private static final ObjectParser<ReadOnlyAction, Void> PARSER = new ObjectParser<>(NAME, true, ReadOnlyAction::new);
 
     public static ReadOnlyAction parse(XContentParser parser) {
         return PARSER.apply(parser, null);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/RolloverAction.java

@@ -38,7 +38,7 @@ public class RolloverAction implements LifecycleAction, ToXContentObject {
     private static final ParseField MAX_DOCS_FIELD = new ParseField("max_docs");
     private static final ParseField MAX_AGE_FIELD = new ParseField("max_age");
 
-    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME,
+    private static final ConstructingObjectParser<RolloverAction, Void> PARSER = new ConstructingObjectParser<>(NAME, true,
         a -> new RolloverAction((ByteSizeValue) a[0], (TimeValue) a[1], (Long) a[2]));
     static {
         PARSER.declareField(ConstructingObjectParser.optionalConstructorArg(),

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/SetPriorityAction.java

@@ -55,7 +55,7 @@ public static SetPriorityAction parse(XContentParser parser) {
     }
 
     public SetPriorityAction(@Nullable Integer recoveryPriority) {
-        if (recoveryPriority != null && recoveryPriority <= 0) {
+        if (recoveryPriority != null && recoveryPriority < 0) {
             throw new IllegalArgumentException("[" + RECOVERY_PRIORITY_FIELD.getPreferredName() + "] must be 0 or greater");
         }
         this.recoveryPriority = recoveryPriority;

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/ShrinkAction.java

@@ -33,7 +33,7 @@ public class ShrinkAction implements LifecycleAction, ToXContentObject {
     private static final ParseField NUMBER_OF_SHARDS_FIELD = new ParseField("number_of_shards");
 
     private static final ConstructingObjectParser<ShrinkAction, Void> PARSER =
-        new ConstructingObjectParser<>(NAME, a -> new ShrinkAction((Integer) a[0]));
+        new ConstructingObjectParser<>(NAME, true, a -> new ShrinkAction((Integer) a[0]));
 
     static {
         PARSER.declareInt(ConstructingObjectParser.constructorArg(), NUMBER_OF_SHARDS_FIELD);

client/rest-high-level/src/main/java/org/elasticsearch/client/indexlifecycle/UnfollowAction.java

@@ -31,7 +31,7 @@
 public class UnfollowAction implements LifecycleAction, ToXContentObject {
     public static final String NAME = "unfollow";
 
-    private static final ObjectParser<UnfollowAction, Void> PARSER = new ObjectParser<>(NAME, UnfollowAction::new);
+    private static final ObjectParser<UnfollowAction, Void> PARSER = new ObjectParser<>(NAME, true, UnfollowAction::new);
 
     public UnfollowAction() {}
 

client/rest-high-level/src/main/java/org/elasticsearch/client/ml/job/util/TimeUtil.java

@@ -39,7 +39,7 @@ public static Date parseTimeField(XContentParser parser, String fieldName) throw
         if (parser.currentToken() == XContentParser.Token.VALUE_NUMBER) {
             return new Date(parser.longValue());
         } else if (parser.currentToken() == XContentParser.Token.VALUE_STRING) {
-            return new Date(DateFormatters.toZonedDateTime(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli());
+            return new Date(DateFormatters.from(DateTimeFormatter.ISO_INSTANT.parse(parser.text())).toInstant().toEpochMilli());
         }
         throw new IllegalArgumentException(
             "unexpected token [" + parser.currentToken() + "] for [" + fieldName + "]");