Skip to content

Commit 758c2aa

Browse files
ashking94ashking94
authored
Fix flaky tests with hashed prefix path type snapshots (#15560)
Signed-off-by: Ashish Singh <[email protected]>
1 parent ef47252 commit 758c2aa

File tree

4 files changed

+30
-22
lines changed

4 files changed

+30
-22
lines changed

‎server/src/internalClusterTest/java/org/opensearch/snapshots/CorruptedBlobStoreRepositoryIT.java

+1-17
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,9 @@
4040
import org.opensearch.cluster.ClusterState;
4141
import org.opensearch.cluster.metadata.Metadata;
4242
import org.opensearch.cluster.metadata.RepositoriesMetadata;
43-
import org.opensearch.common.blobstore.BlobPath;
4443
import org.opensearch.common.settings.Settings;
4544
import org.opensearch.common.util.io.IOUtils;
4645
import org.opensearch.core.common.unit.ByteSizeUnit;
47-
import org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm;
48-
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
49-
import org.opensearch.index.remote.RemoteStorePathStrategy;
5046
import org.opensearch.repositories.IndexId;
5147
import org.opensearch.repositories.RepositoriesService;
5248
import org.opensearch.repositories.Repository;
@@ -62,7 +58,7 @@
6258
import java.util.Map;
6359
import java.util.stream.Stream;
6460

65-
import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1;
61+
import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath;
6662
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
6763
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows;
6864
import static org.hamcrest.Matchers.containsString;
@@ -683,16 +679,4 @@ private void assertRepositoryBlocked(Client client, String repo, String existing
683679
containsString("Could not read repository data because the contents of the repository do not match its expected state.")
684680
);
685681
}
686-
687-
private static String resolvePath(IndexId indexId, String shardId) {
688-
PathType pathType = PathType.fromCode(indexId.getShardPathType());
689-
RemoteStorePathStrategy.SnapshotShardPathInput shardPathInput = new RemoteStorePathStrategy.SnapshotShardPathInput.Builder()
690-
.basePath(BlobPath.cleanPath())
691-
.indexUUID(indexId.getId())
692-
.shardId(shardId)
693-
.build();
694-
PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null;
695-
BlobPath blobPath = pathType.path(shardPathInput, pathHashAlgorithm);
696-
return blobPath.buildAsString();
697-
}
698682
}

‎server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java

+8-1
Original file line numberDiff line numberDiff line change
@@ -110,10 +110,17 @@ public void testRepositoryCreation() throws Exception {
110110
assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-1"), notNullValue());
111111
assertThat(findRepository(repositoriesResponse.repositories(), "test-repo-2"), notNullValue());
112112

113+
RepositoryMetadata testRepo1Md = findRepository(repositoriesResponse.repositories(), "test-repo-1");
114+
113115
logger.info("--> check that trying to create a repository with the same settings repeatedly does not update cluster state");
114116
String beforeStateUuid = clusterStateResponse.getState().stateUUID();
115117
createRepository("test-repo-1", "fs", Settings.builder().put("location", location));
116-
assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID());
118+
repositoriesResponse = client.admin().cluster().prepareGetRepositories(randomFrom("_all", "*", "test-repo-*")).get();
119+
RepositoryMetadata testRepo1MdAfterUpdate = findRepository(repositoriesResponse.repositories(), "test-repo-1");
120+
121+
if (testRepo1Md.settings().equals(testRepo1MdAfterUpdate.settings())) {
122+
assertEquals(beforeStateUuid, client.admin().cluster().prepareState().clear().get().getState().stateUUID());
123+
}
117124

118125
logger.info("--> delete repository test-repo-1");
119126
client.admin().cluster().prepareDeleteRepository("test-repo-1").get();

‎server/src/internalClusterTest/java/org/opensearch/snapshots/SnapshotStatusApisIT.java

+4-4
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,7 @@
4949
import org.opensearch.common.util.io.IOUtils;
5050
import org.opensearch.core.common.Strings;
5151
import org.opensearch.core.common.unit.ByteSizeUnit;
52+
import org.opensearch.repositories.IndexId;
5253
import org.opensearch.repositories.blobstore.BlobStoreRepository;
5354
import org.opensearch.threadpool.ThreadPool;
5455

@@ -62,6 +63,7 @@
6263
import java.util.concurrent.TimeUnit;
6364
import java.util.stream.Collectors;
6465

66+
import static org.opensearch.test.OpenSearchIntegTestCase.resolvePath;
6567
import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked;
6668
import static org.hamcrest.Matchers.equalTo;
6769
import static org.hamcrest.Matchers.greaterThan;
@@ -200,11 +202,9 @@ public void testExceptionOnMissingShardLevelSnapBlob() throws IOException {
200202
final SnapshotInfo snapshotInfo = createFullSnapshot("test-repo", "test-snap");
201203

202204
logger.info("--> delete shard-level snap-${uuid}.dat file for one shard in this snapshot to simulate concurrent delete");
203-
final String indexRepoId = getRepositoryData("test-repo").resolveIndexId(snapshotInfo.indices().get(0)).getId();
205+
IndexId indexId = getRepositoryData("test-repo").resolveIndexId(snapshotInfo.indices().get(0));
204206
IOUtils.rm(
205-
repoPath.resolve("indices")
206-
.resolve(indexRepoId)
207-
.resolve("0")
207+
repoPath.resolve(resolvePath(indexId, "0"))
208208
.resolve(BlobStoreRepository.SNAPSHOT_PREFIX + snapshotInfo.snapshotId().getUUID() + ".dat")
209209
);
210210

‎test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java

+17
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
import org.opensearch.cluster.service.applicationtemplates.TestSystemTemplatesRepositoryPlugin;
9797
import org.opensearch.common.Nullable;
9898
import org.opensearch.common.Priority;
99+
import org.opensearch.common.blobstore.BlobPath;
99100
import org.opensearch.common.collect.Tuple;
100101
import org.opensearch.common.concurrent.GatedCloseable;
101102
import org.opensearch.common.network.NetworkModule;
@@ -140,7 +141,9 @@
140141
import org.opensearch.index.engine.Segment;
141142
import org.opensearch.index.mapper.CompletionFieldMapper;
142143
import org.opensearch.index.mapper.MockFieldFilterPlugin;
144+
import org.opensearch.index.remote.RemoteStoreEnums;
143145
import org.opensearch.index.remote.RemoteStoreEnums.PathType;
146+
import org.opensearch.index.remote.RemoteStorePathStrategy;
144147
import org.opensearch.index.shard.IndexShard;
145148
import org.opensearch.index.store.Store;
146149
import org.opensearch.index.translog.Translog;
@@ -155,6 +158,7 @@
155158
import org.opensearch.node.remotestore.RemoteStoreNodeService;
156159
import org.opensearch.plugins.NetworkPlugin;
157160
import org.opensearch.plugins.Plugin;
161+
import org.opensearch.repositories.IndexId;
158162
import org.opensearch.repositories.blobstore.BlobStoreRepository;
159163
import org.opensearch.repositories.fs.FsRepository;
160164
import org.opensearch.repositories.fs.ReloadableFsRepository;
@@ -220,6 +224,7 @@
220224
import static org.opensearch.index.IndexSettings.INDEX_DOC_ID_FUZZY_SET_FALSE_POSITIVE_PROBABILITY_SETTING;
221225
import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_RETENTION_LEASE_PERIOD_SETTING;
222226
import static org.opensearch.index.query.QueryBuilders.matchAllQuery;
227+
import static org.opensearch.index.remote.RemoteStoreEnums.PathHashAlgorithm.FNV_1A_COMPOSITE_1;
223228
import static org.opensearch.indices.IndicesService.CLUSTER_REPLICATION_TYPE_SETTING;
224229
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_CLUSTER_STATE_REPOSITORY_NAME_ATTRIBUTE_KEY;
225230
import static org.opensearch.node.remotestore.RemoteStoreNodeAttribute.REMOTE_STORE_REPOSITORY_SETTINGS_ATTRIBUTE_KEY_PREFIX;
@@ -2884,4 +2889,16 @@ private static Settings buildRemoteStoreNodeAttributes(
28842889
settings.put(RemoteStoreSettings.CLUSTER_REMOTE_STORE_PINNED_TIMESTAMP_ENABLED.getKey(), randomBoolean());
28852890
return settings.build();
28862891
}
2892+
2893+
public static String resolvePath(IndexId indexId, String shardId) {
2894+
PathType pathType = PathType.fromCode(indexId.getShardPathType());
2895+
RemoteStorePathStrategy.SnapshotShardPathInput shardPathInput = new RemoteStorePathStrategy.SnapshotShardPathInput.Builder()
2896+
.basePath(BlobPath.cleanPath())
2897+
.indexUUID(indexId.getId())
2898+
.shardId(shardId)
2899+
.build();
2900+
RemoteStoreEnums.PathHashAlgorithm pathHashAlgorithm = pathType != PathType.FIXED ? FNV_1A_COMPOSITE_1 : null;
2901+
BlobPath blobPath = pathType.path(shardPathInput, pathHashAlgorithm);
2902+
return blobPath.buildAsString();
2903+
}
28872904
}

0 commit comments

Comments
 (0)