forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 2
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add IT for multiple writer validation
Signed-off-by: Ashish Singh <[email protected]>
- Loading branch information
Showing
6 changed files
with
280 additions
and
21 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
147 changes: 147 additions & 0 deletions
147
...lClusterTest/java/org/opensearch/remotestore/RemoteStoreMultipleWriterVerificationIT.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,147 @@ | ||
| /* | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| * | ||
| * The OpenSearch Contributors require contributions made to | ||
| * this file be licensed under the Apache-2.0 license or a | ||
| * compatible open source license. | ||
| */ | ||
|
|
||
| package org.opensearch.remotestore; | ||
|
|
||
| import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; | ||
| import org.opensearch.action.admin.indices.flush.FlushRequest; | ||
| import org.opensearch.action.admin.indices.settings.put.UpdateSettingsRequest; | ||
| import org.opensearch.cluster.health.ClusterHealthStatus; | ||
| import org.opensearch.cluster.metadata.IndexMetadata; | ||
| import org.opensearch.cluster.routing.allocation.command.MoveAllocationCommand; | ||
| import org.opensearch.common.Priority; | ||
| import org.opensearch.common.settings.Settings; | ||
| import org.opensearch.common.unit.TimeValue; | ||
| import org.opensearch.index.shard.IndexShard; | ||
| import org.opensearch.indices.recovery.PeerRecoveryTargetService; | ||
| import org.opensearch.plugins.Plugin; | ||
| import org.opensearch.remotestore.multipart.mocks.MockFsRepository; | ||
| import org.opensearch.remotestore.multipart.mocks.MockFsRepositoryPlugin; | ||
| import org.opensearch.repositories.RepositoriesService; | ||
| import org.opensearch.test.OpenSearchIntegTestCase; | ||
| import org.opensearch.test.transport.MockTransportService; | ||
| import org.opensearch.transport.TransportService; | ||
|
|
||
| import java.util.Arrays; | ||
| import java.util.Collection; | ||
| import java.util.concurrent.CountDownLatch; | ||
| import java.util.concurrent.ExecutionException; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| @OpenSearchIntegTestCase.ClusterScope(scope = OpenSearchIntegTestCase.Scope.TEST, numDataNodes = 0) | ||
| public class RemoteStoreMultipleWriterVerificationIT extends RemoteStoreBaseIntegTestCase { | ||
|
|
||
| protected final String INDEX_NAME = "remote-store-test-idx-1"; | ||
|
|
||
| @Override | ||
| protected Settings nodeSettings(int nodeOrdinal) { | ||
| return Settings.builder() | ||
| .put(super.nodeSettings(nodeOrdinal)) | ||
| .put( | ||
| remoteStoreClusterSettings( | ||
| REPOSITORY_NAME, | ||
| segmentRepoPath, | ||
| MockFsRepositoryPlugin.TYPE, | ||
| REPOSITORY_2_NAME, | ||
| translogRepoPath, | ||
| MockFsRepositoryPlugin.TYPE | ||
| ) | ||
| ) | ||
| .build(); | ||
| } | ||
|
|
||
| @Override | ||
| protected Collection<Class<? extends Plugin>> nodePlugins() { | ||
| return Arrays.asList(MockTransportService.TestPlugin.class, MockFsRepositoryPlugin.class); | ||
| } | ||
|
|
||
| public void testNoMultipleWriterDuringPrimaryRelocation() throws ExecutionException, InterruptedException { | ||
| // In this test, we trigger a force flush on existing primary while the primary mode on new primary has been | ||
| // activated. There was a bug in primary relocation of remote store enabled indexes where the new primary | ||
| // starts uploading translog and segments even before the cluster manager has started this shard. With this test, | ||
| // we check that we do not overwrite any file on remote store. Here we will also increase the replica count to | ||
| // check that there are no duplicate metadata files for translog or upload. | ||
|
|
||
| internalCluster().startClusterManagerOnlyNode(); | ||
| String oldPrimary = internalCluster().startDataOnlyNodes(1).get(0); | ||
| createIndex(INDEX_NAME, remoteStoreIndexSettings(0)); | ||
| ensureGreen(INDEX_NAME); | ||
| indexBulk(INDEX_NAME, randomIntBetween(5, 10)); | ||
| String newPrimary = internalCluster().startDataOnlyNodes(1).get(0); | ||
| ensureStableCluster(3); | ||
|
|
||
| IndexShard oldPrimaryIndexShard = getIndexShard(oldPrimary, INDEX_NAME); | ||
| RepositoriesService newPrimaryRepositories = internalCluster().getInstance(RepositoriesService.class, newPrimary); | ||
| MockFsRepository newPrimaryTranslogRepo = (MockFsRepository) newPrimaryRepositories.repository(REPOSITORY_2_NAME); | ||
| CountDownLatch flushLatch = new CountDownLatch(1); | ||
|
|
||
| MockTransportService mockTargetTransportService = ((MockTransportService) internalCluster().getInstance( | ||
| TransportService.class, | ||
| oldPrimary | ||
| )); | ||
| mockTargetTransportService.addSendBehavior((connection, requestId, action, request, options) -> { | ||
| if (PeerRecoveryTargetService.Actions.HANDOFF_PRIMARY_CONTEXT.equals(action)) { | ||
| newPrimaryTranslogRepo.setSleepSeconds(5); | ||
| flushLatch.countDown(); | ||
| } | ||
| connection.sendRequest(requestId, action, request, options); | ||
| }); | ||
|
|
||
| logger.info("--> relocate the shard"); | ||
| client().admin() | ||
| .cluster() | ||
| .prepareReroute() | ||
| .add(new MoveAllocationCommand(INDEX_NAME, 0, oldPrimary, newPrimary)) | ||
| .execute() | ||
| .actionGet(); | ||
|
|
||
| CountDownLatch flushDone = new CountDownLatch(1); | ||
| Thread flushThread = new Thread(() -> { | ||
| try { | ||
| flushLatch.await(2, TimeUnit.SECONDS); | ||
| oldPrimaryIndexShard.flush(new FlushRequest().waitIfOngoing(true).force(true)); | ||
| newPrimaryTranslogRepo.setSleepSeconds(0); | ||
| flushDone.countDown(); | ||
| } catch (InterruptedException e) { | ||
| throw new AssertionError(e); | ||
| } | ||
| }); | ||
| flushThread.start(); | ||
| flushDone.await(5, TimeUnit.SECONDS); | ||
| flushThread.join(); | ||
|
|
||
| ClusterHealthResponse clusterHealthResponse = client().admin() | ||
| .cluster() | ||
| .prepareHealth() | ||
| .setWaitForStatus(ClusterHealthStatus.GREEN) | ||
| .setWaitForEvents(Priority.LANGUID) | ||
| .setWaitForNoRelocatingShards(true) | ||
| .setTimeout(TimeValue.timeValueSeconds(5)) | ||
| .execute() | ||
| .actionGet(); | ||
| assertFalse(clusterHealthResponse.isTimedOut()); | ||
|
|
||
| client().admin() | ||
| .indices() | ||
| .updateSettings( | ||
| new UpdateSettingsRequest(INDEX_NAME).settings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) | ||
| ) | ||
| .get(); | ||
|
|
||
| clusterHealthResponse = client().admin() | ||
| .cluster() | ||
| .prepareHealth() | ||
| .setWaitForStatus(ClusterHealthStatus.GREEN) | ||
| .setWaitForEvents(Priority.LANGUID) | ||
| .setWaitForNoRelocatingShards(true) | ||
| .setTimeout(TimeValue.timeValueSeconds(5)) | ||
| .execute() | ||
| .actionGet(); | ||
| assertFalse(clusterHealthResponse.isTimedOut()); | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.