Skip to content

Commit 3fa67c5

Browse files
original-brownbearoriginal-brownbear
authored
DISCOVERY: Cleanup AbstractDisruptionTestCase (#34808)
* DISCOVERY: Cleanup AbstractDisruptionTestCase * Make the internal test cluster manage minimum master nodes where we used the default of (nodes / 2 + 1) before * Remove use of the `NodeConfigurationSource` indirection * Relates #33675
1 parent 455906d commit 3fa67c5

6 files changed

+113
-119
lines changed

‎server/src/test/java/org/elasticsearch/discovery/AbstractDisruptionTestCase.java

+1-49
Original file line numberDiff line numberDiff line change
@@ -19,23 +19,19 @@
1919

2020
package org.elasticsearch.discovery;
2121

22-
import java.nio.file.Path;
2322
import org.elasticsearch.cluster.ClusterState;
2423
import org.elasticsearch.cluster.block.ClusterBlock;
2524
import org.elasticsearch.cluster.block.ClusterBlockLevel;
2625
import org.elasticsearch.cluster.node.DiscoveryNodes;
2726
import org.elasticsearch.common.Nullable;
2827
import org.elasticsearch.common.settings.Settings;
2928
import org.elasticsearch.common.unit.TimeValue;
30-
import org.elasticsearch.discovery.zen.ElectMasterService;
3129
import org.elasticsearch.discovery.zen.FaultDetection;
3230
import org.elasticsearch.discovery.zen.UnicastZenPing;
3331
import org.elasticsearch.discovery.zen.ZenPing;
34-
import org.elasticsearch.env.NodeEnvironment;
3532
import org.elasticsearch.plugins.Plugin;
3633
import org.elasticsearch.test.ESIntegTestCase;
3734
import org.elasticsearch.test.InternalTestCluster;
38-
import org.elasticsearch.test.NodeConfigurationSource;
3935
import org.elasticsearch.test.discovery.TestZenDiscovery;
4036
import org.elasticsearch.test.disruption.NetworkDisruption;
4137
import org.elasticsearch.test.disruption.NetworkDisruption.Bridge;
@@ -56,27 +52,20 @@
5652
import java.util.Set;
5753
import java.util.concurrent.TimeUnit;
5854

59-
import static org.elasticsearch.discovery.DiscoveryModule.DISCOVERY_HOSTS_PROVIDER_SETTING;
6055
import static org.hamcrest.Matchers.equalTo;
6156
import static org.hamcrest.Matchers.not;
6257

6358
public abstract class AbstractDisruptionTestCase extends ESIntegTestCase {
6459

6560
static final TimeValue DISRUPTION_HEALING_OVERHEAD = TimeValue.timeValueSeconds(40); // we use 30s as timeout in many places.
6661

67-
private NodeConfigurationSource discoveryConfig;
6862

6963
@Override
7064
protected Settings nodeSettings(int nodeOrdinal) {
71-
return Settings.builder().put(discoveryConfig.nodeSettings(nodeOrdinal))
65+
return Settings.builder().put(super.nodeSettings(nodeOrdinal)).put(DEFAULT_SETTINGS)
7266
.put(TestZenDiscovery.USE_MOCK_PINGS.getKey(), false).build();
7367
}
7468

75-
@Before
76-
public void clearConfig() {
77-
discoveryConfig = null;
78-
}
79-
8069
@Override
8170
protected int numberOfShards() {
8271
return 3;
@@ -119,11 +108,6 @@ protected void beforeIndexDeletion() throws Exception {
119108
}
120109

121110
List<String> startCluster(int numberOfNodes) {
122-
return startCluster(numberOfNodes, -1);
123-
}
124-
125-
List<String> startCluster(int numberOfNodes, int minimumMasterNode) {
126-
configureCluster(numberOfNodes, minimumMasterNode);
127111
InternalTestCluster internalCluster = internalCluster();
128112
List<String> nodes = internalCluster.startNodes(numberOfNodes);
129113
ensureStableCluster(numberOfNodes);
@@ -152,38 +136,6 @@ protected Collection<Class<? extends Plugin>> nodePlugins() {
152136
return Arrays.asList(MockTransportService.TestPlugin.class);
153137
}
154138

155-
void configureCluster(int numberOfNodes, int minimumMasterNode) {
156-
configureCluster(DEFAULT_SETTINGS, numberOfNodes, minimumMasterNode);
157-
}
158-
159-
void configureCluster(Settings settings, int numberOfNodes, int minimumMasterNode) {
160-
if (minimumMasterNode < 0) {
161-
minimumMasterNode = numberOfNodes / 2 + 1;
162-
}
163-
logger.info("---> configured unicast");
164-
// TODO: Rarely use default settings form some of these
165-
Settings nodeSettings = Settings.builder()
166-
.put(settings)
167-
.put(NodeEnvironment.MAX_LOCAL_STORAGE_NODES_SETTING.getKey(), numberOfNodes)
168-
.put(ElectMasterService.DISCOVERY_ZEN_MINIMUM_MASTER_NODES_SETTING.getKey(), minimumMasterNode)
169-
.putList(DISCOVERY_HOSTS_PROVIDER_SETTING.getKey(), "file")
170-
.build();
171-
172-
if (discoveryConfig == null) {
173-
discoveryConfig = new NodeConfigurationSource() {
174-
@Override
175-
public Settings nodeSettings(final int nodeOrdinal) {
176-
return nodeSettings;
177-
}
178-
179-
@Override
180-
public Path nodeConfigPath(final int nodeOrdinal) {
181-
return null;
182-
}
183-
};
184-
}
185-
}
186-
187139
ClusterState getNodeClusterState(String node) {
188140
return client(node).admin().cluster().prepareState().setLocal(true).get().getState();
189141
}

‎server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionCleanSettingsIT.java

+76
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Licensed to Elasticsearch under one or more contributor
3+
* license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright
5+
* ownership. Elasticsearch licenses this file to you under
6+
* the Apache License, Version 2.0 (the "License"); you may
7+
* not use this file except in compliance with the License.
8+
* You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.elasticsearch.discovery;
21+
22+
import java.util.ArrayList;
23+
import java.util.Arrays;
24+
import java.util.Collection;
25+
import java.util.List;
26+
import org.elasticsearch.action.index.IndexRequestBuilder;
27+
import org.elasticsearch.cluster.metadata.IndexMetaData;
28+
import org.elasticsearch.common.settings.Settings;
29+
import org.elasticsearch.common.xcontent.XContentType;
30+
import org.elasticsearch.indices.store.IndicesStoreIntegrationIT;
31+
import org.elasticsearch.plugins.Plugin;
32+
import org.elasticsearch.test.ESIntegTestCase;
33+
import org.elasticsearch.test.transport.MockTransportService;
34+
35+
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
36+
import static org.hamcrest.Matchers.equalTo;
37+
38+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
39+
public class ClusterDisruptionCleanSettingsIT extends ESIntegTestCase {
40+
41+
@Override
42+
protected Collection<Class<? extends Plugin>> nodePlugins() {
43+
return Arrays.asList(MockTransportService.TestPlugin.class);
44+
}
45+
46+
/**
47+
* This test creates a scenario where a primary shard (0 replicas) relocates and is in POST_RECOVERY on the target
48+
* node but already deleted on the source node. Search request should still work.
49+
*/
50+
public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Exception {
51+
// Don't use AbstractDisruptionTestCase.DEFAULT_SETTINGS as settings
52+
// (which can cause node disconnects on a slow CI machine)
53+
internalCluster().startMasterOnlyNode();
54+
final String node_1 = internalCluster().startDataOnlyNode();
55+
56+
logger.info("--> creating index [test] with one shard and on replica");
57+
assertAcked(prepareCreate("test").setSettings(
58+
Settings.builder().put(indexSettings())
59+
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
60+
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
61+
);
62+
ensureGreen("test");
63+
64+
final String node_2 = internalCluster().startDataOnlyNode();
65+
List<IndexRequestBuilder> indexRequestBuilderList = new ArrayList<>();
66+
for (int i = 0; i < 100; i++) {
67+
indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("_doc")
68+
.setSource("{\"int_field\":1}", XContentType.JSON));
69+
}
70+
indexRandom(true, indexRequestBuilderList);
71+
72+
IndicesStoreIntegrationIT.relocateAndBlockCompletion(logger, "test", 0, node_1, node_2);
73+
// now search for the documents and see if we get a reply
74+
assertThat(client().prepareSearch().setSize(0).get().getHits().getTotalHits(), equalTo(100L));
75+
}
76+
}

‎server/src/test/java/org/elasticsearch/discovery/ClusterDisruptionIT.java

+5-43
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,6 @@
2424
import org.elasticsearch.ElasticsearchException;
2525
import org.elasticsearch.action.NoShardAvailableActionException;
2626
import org.elasticsearch.action.get.GetResponse;
27-
import org.elasticsearch.action.index.IndexRequestBuilder;
2827
import org.elasticsearch.action.index.IndexResponse;
2928
import org.elasticsearch.client.Client;
3029
import org.elasticsearch.cluster.ClusterState;
@@ -36,9 +35,7 @@
3635
import org.elasticsearch.common.settings.Settings;
3736
import org.elasticsearch.common.unit.TimeValue;
3837
import org.elasticsearch.common.xcontent.XContentType;
39-
import org.elasticsearch.indices.store.IndicesStoreIntegrationIT;
40-
import org.elasticsearch.test.ESIntegTestCase.ClusterScope;
41-
import org.elasticsearch.test.ESIntegTestCase.Scope;
38+
import org.elasticsearch.test.ESIntegTestCase;
4239
import org.elasticsearch.test.InternalTestCluster;
4340
import org.elasticsearch.test.disruption.NetworkDisruption;
4441
import org.elasticsearch.test.disruption.NetworkDisruption.Bridge;
@@ -72,8 +69,8 @@
7269
/**
7370
* Tests various cluster operations (e.g., indexing) during disruptions.
7471
*/
75-
@ClusterScope(scope = Scope.TEST, numDataNodes = 0, transportClientRatio = 0, autoMinMasterNodes = false)
7672
@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
73+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
7774
public class ClusterDisruptionIT extends AbstractDisruptionTestCase {
7875

7976
/**
@@ -289,7 +286,7 @@ public void testRejoinDocumentExistsInAllShardCopies() throws Exception {
289286

290287
// simulate handling of sending shard failure during an isolation
291288
public void testSendingShardFailure() throws Exception {
292-
List<String> nodes = startCluster(3, 2);
289+
List<String> nodes = startCluster(3);
293290
String masterNode = internalCluster().getMasterName();
294291
List<String> nonMasterNodes = nodes.stream().filter(node -> !node.equals(masterNode)).collect(Collectors.toList());
295292
String nonMasterNode = randomFrom(nonMasterNodes);
@@ -357,43 +354,10 @@ public void onFailure(Exception e) {
357354
}
358355
}
359356

360-
/**
361-
* This test creates a scenario where a primary shard (0 replicas) relocates and is in POST_RECOVERY on the target
362-
* node but already deleted on the source node. Search request should still work.
363-
*/
364-
public void testSearchWithRelocationAndSlowClusterStateProcessing() throws Exception {
365-
// don't use DEFAULT settings (which can cause node disconnects on a slow CI machine)
366-
configureCluster(Settings.EMPTY, 3, 1);
367-
internalCluster().startMasterOnlyNode();
368-
final String node_1 = internalCluster().startDataOnlyNode();
369-
370-
logger.info("--> creating index [test] with one shard and on replica");
371-
assertAcked(prepareCreate("test").setSettings(
372-
Settings.builder().put(indexSettings())
373-
.put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1)
374-
.put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0))
375-
);
376-
ensureGreen("test");
377-
378-
final String node_2 = internalCluster().startDataOnlyNode();
379-
List<IndexRequestBuilder> indexRequestBuilderList = new ArrayList<>();
380-
for (int i = 0; i < 100; i++) {
381-
indexRequestBuilderList.add(client().prepareIndex().setIndex("test").setType("_doc")
382-
.setSource("{\"int_field\":1}", XContentType.JSON));
383-
}
384-
indexRandom(true, indexRequestBuilderList);
385-
386-
IndicesStoreIntegrationIT.relocateAndBlockCompletion(logger, "test", 0, node_1, node_2);
387-
// now search for the documents and see if we get a reply
388-
assertThat(client().prepareSearch().setSize(0).get().getHits().getTotalHits(), equalTo(100L));
389-
}
390-
391357
public void testIndexImportedFromDataOnlyNodesIfMasterLostDataFolder() throws Exception {
392358
// test for https://github.com/elastic/elasticsearch/issues/8823
393-
configureCluster(2, 1);
394359
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
395360
internalCluster().startDataOnlyNode(Settings.EMPTY);
396-
397361
ensureStableCluster(2);
398362
assertAcked(prepareCreate("index").setSettings(Settings.builder().put("index.number_of_replicas", 0)));
399363
index("index", "_doc", "1", jsonBuilder().startObject().field("text", "some text").endObject());
@@ -416,14 +380,12 @@ public boolean clearData(String nodeName) {
416380
*/
417381
public void testIndicesDeleted() throws Exception {
418382
final Settings settings = Settings.builder()
419-
.put(DEFAULT_SETTINGS)
420383
.put(DiscoverySettings.PUBLISH_TIMEOUT_SETTING.getKey(), "0s") // don't wait on isolated data node
421384
.put(DiscoverySettings.COMMIT_TIMEOUT_SETTING.getKey(), "30s") // wait till cluster state is committed
422385
.build();
423386
final String idxName = "test";
424-
configureCluster(settings, 3, 2);
425-
final List<String> allMasterEligibleNodes = internalCluster().startMasterOnlyNodes(2);
426-
final String dataNode = internalCluster().startDataOnlyNode();
387+
final List<String> allMasterEligibleNodes = internalCluster().startMasterOnlyNodes(2, settings);
388+
final String dataNode = internalCluster().startDataOnlyNode(settings);
427389
ensureStableCluster(3);
428390
assertAcked(prepareCreate("test"));
429391

‎server/src/test/java/org/elasticsearch/discovery/DiscoveryDisruptionIT.java

+5-14
Original file line numberDiff line numberDiff line change
@@ -54,13 +54,13 @@
5454
/**
5555
* Tests for discovery during disruptions.
5656
*/
57-
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0, autoMinMasterNodes = false)
5857
@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
58+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
5959
public class DiscoveryDisruptionIT extends AbstractDisruptionTestCase {
6060

6161
public void testIsolatedUnicastNodes() throws Exception {
6262
internalCluster().setHostsListContainsOnlyFirstNode(true);
63-
List<String> nodes = startCluster(4, -1);
63+
List<String> nodes = startCluster(4);
6464
// Figure out what is the elected master node
6565
final String unicastTarget = nodes.get(0);
6666

@@ -100,7 +100,7 @@ public void testIsolatedUnicastNodes() throws Exception {
100100
*/
101101
public void testUnicastSinglePingResponseContainsMaster() throws Exception {
102102
internalCluster().setHostsListContainsOnlyFirstNode(true);
103-
List<String> nodes = startCluster(4, -1);
103+
List<String> nodes = startCluster(4);
104104
// Figure out what is the elected master node
105105
final String masterNode = internalCluster().getMasterName();
106106
logger.info("---> legit elected master node={}", masterNode);
@@ -138,15 +138,8 @@ public void testUnicastSinglePingResponseContainsMaster() throws Exception {
138138
* Test cluster join with issues in cluster state publishing *
139139
*/
140140
public void testClusterJoinDespiteOfPublishingIssues() throws Exception {
141-
List<String> nodes = startCluster(2, 1);
142-
143-
String masterNode = internalCluster().getMasterName();
144-
String nonMasterNode;
145-
if (masterNode.equals(nodes.get(0))) {
146-
nonMasterNode = nodes.get(1);
147-
} else {
148-
nonMasterNode = nodes.get(0);
149-
}
141+
String masterNode = internalCluster().startMasterOnlyNode(Settings.EMPTY);
142+
String nonMasterNode = internalCluster().startDataOnlyNode(Settings.EMPTY);
150143

151144
DiscoveryNodes discoveryNodes = internalCluster().getInstance(ClusterService.class, nonMasterNode).state().nodes();
152145

@@ -196,7 +189,6 @@ public void testClusterJoinDespiteOfPublishingIssues() throws Exception {
196189
}
197190

198191
public void testClusterFormingWithASlowNode() throws Exception {
199-
configureCluster(3, 2);
200192

201193
SlowClusterStateProcessing disruption = new SlowClusterStateProcessing(random(), 0, 0, 1000, 2000);
202194

@@ -212,7 +204,6 @@ public void testClusterFormingWithASlowNode() throws Exception {
212204
}
213205

214206
public void testElectMasterWithLatestVersion() throws Exception {
215-
configureCluster(3, 2);
216207
final Set<String> nodes = new HashSet<>(internalCluster().startNodes(3));
217208
ensureStableCluster(3);
218209
ServiceDisruptionScheme isolateAllNodes =

‎server/src/test/java/org/elasticsearch/discovery/MasterDisruptionIT.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,8 @@
6767
/**
6868
* Tests relating to the loss of the master.
6969
*/
70-
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0, autoMinMasterNodes = false)
7170
@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE")
71+
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.TEST, numDataNodes = 0, transportClientRatio = 0)
7272
public class MasterDisruptionIT extends AbstractDisruptionTestCase {
7373

7474
/**
@@ -153,8 +153,8 @@ public void testNodesFDAfterMasterReelection() throws Exception {
153153
*/
154154
@TestLogging("_root:DEBUG,org.elasticsearch.cluster.service:TRACE,org.elasticsearch.test.disruption:TRACE")
155155
public void testStaleMasterNotHijackingMajority() throws Exception {
156-
// 3 node cluster with unicast discovery and minimum_master_nodes set to 2:
157-
final List<String> nodes = startCluster(3, 2);
156+
// 3 node cluster with unicast discovery and minimum_master_nodes set to the default of 2:
157+
final List<String> nodes = startCluster(3);
158158

159159
// Save the current master node as old master node, because that node will get frozen
160160
final String oldMasterNode = internalCluster().getMasterName();
@@ -267,7 +267,7 @@ public void onFailure(String source, Exception e) {
267267
* Test that cluster recovers from a long GC on master that causes other nodes to elect a new one
268268
*/
269269
public void testMasterNodeGCs() throws Exception {
270-
List<String> nodes = startCluster(3, -1);
270+
List<String> nodes = startCluster(3);
271271

272272
String oldMasterNode = internalCluster().getMasterName();
273273
// a very long GC, but it's OK as we remove the disruption when it has had an effect

0 commit comments

Comments
 (0)