DISCOVERY: Use Realistic Num. of Min Master Nodes (#34854)

* DISCOVERY: Use Realistic Num. of Min Master Nodes

* With all 3 nodes starting in parallel 2 nodes can win a master election and
start waiting for nodes to join when started in parallel. We wait for 30s on the
rest tests for the cluster health endpoint to show 3 nodes which breaks if
one of the master nodes itself waits for 30s for joining nodes, waiting for 5 seconds
fixes the issue and allows us to run with `minimum master nodes < node count`
* relates #33675

Author: original-brownbear

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -69,7 +69,11 @@ class ClusterConfiguration {
      */
     @Input
     Closure<Integer> minimumMasterNodes = {
-        return getNumNodes() > 1 ? getNumNodes() : -1
+        if (bwcVersion != null && bwcVersion.before("6.5.0-SNAPSHOT")) {
+            return numNodes > 1 ? numNodes : -1
+        } else {
+            return numNodes > 1 ? numNodes.intdiv(2) + 1 : -1
+        }
     }
 
     @Input

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -343,6 +343,13 @@ class ClusterFormationTasks {
             // this will also allow new and old nodes in the BWC case to become the master
             esConfig['discovery.initial_state_timeout'] = '0s'
         }
+        if (esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
+            // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct
+            // its mistake. Instead, only wait 5s to do another round of pinging.
+            // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation
+            // so we need to bail quicker than the default 30s for the cluster to form in time.
+            esConfig['discovery.zen.master_election.wait_for_joins_timeout'] = '5s'
+        }
         esConfig['node.max_local_storage_nodes'] = node.config.numNodes
         esConfig['http.port'] = node.config.httpPort
         esConfig['transport.tcp.port'] =  node.config.transportPort