From 27a1d5520f7a9262b992bb9922e0710175841dd6 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 12:57:33 +0000 Subject: [PATCH 01/16] Initial plan From 3fc7036304dfab7ea56a7f8a9b3d65d50abf40d0 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:06:58 +0000 Subject: [PATCH 02/16] Merge AbstractRestartWhileUpdatingTestBase into RestartWhileUpdatingTest Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../solr/cloud/RestartWhileUpdatingTest.java | 177 +++++++++++++++- .../AbstractRestartWhileUpdatingTestBase.java | 199 ------------------ 2 files changed, 170 insertions(+), 206 deletions(-) delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java diff --git a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java index 1bc74fff254..22b0084e989 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java @@ -16,23 +16,186 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase.Nightly; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; -/** - * Implementation moved to AbstractRestartWhileUpdatingTestBase because it is used by HDFS contrib - * module tests - */ @Nightly -public class RestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTestBase { +public class RestartWhileUpdatingTest extends AbstractFullDistribZkTestBase { + + // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain"; + private List threads; + + private volatile boolean stopExpire = false; public RestartWhileUpdatingTest() throws Exception { super(); + sliceCount = 1; + fixShardCount(3); + schemaString = "schema15.xml"; // we need a string id + useFactory("solr.StandardDirectoryFactory"); } - @Test + public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; + public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; + @Override + protected String[] getFieldNames() { + return fieldNames; + } + + @Override + protected RandVal[] getRandValues() { + return randVals; + } + + @BeforeClass + public static void beforeRestartWhileUpdatingTest() { + System.setProperty("leaderVoteWait", "300000"); + System.setProperty("solr.autoCommit.maxTime", "30000"); + System.setProperty("solr.autoSoftCommit.maxTime", "3000"); + // SOLR-13212 // TestInjection.nonGracefullClose = "true:60"; + // SOLR-13189 // TestInjection.failReplicaRequests = "true:03"; + } + + @AfterClass + public static void afterRestartWhileUpdatingTest() { + System.clearProperty("leaderVoteWait"); + System.clearProperty("solr.autoCommit.maxTime"); + System.clearProperty("solr.autoSoftCommit.maxTime"); + } + + @Test public void test() throws Exception { - super.test(); + handle.clear(); + handle.put("timestamp", SKIPVAL); + + // start a couple indexing threads + + int[] maxDocList = new int[] {5000, 10000}; + + int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; + + int numThreads = random().nextInt(4) + 1; + + threads = new ArrayList<>(numThreads); + + Thread expireThread = + new Thread("expireThread") { + @Override + public void run() { + while (!stopExpire) { + try { + Thread.sleep(random().nextInt(15000)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // try { + // chaosMonkey.expireRandomSession(); + // } catch (KeeperException e) { + // throw new RuntimeException(e); + // } catch (InterruptedException e) { + // throw new RuntimeException(e); + // } + } + } + }; + + // Currently unused + // expireThread.start(); + + StoppableIndexingThread indexThread; + for (int i = 0; i < numThreads; i++) { + indexThread = + new StoppableIndexingThread( + controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true); + threads.add(indexThread); + indexThread.start(); + } + + Thread.sleep(2000); + + int restartTimes = 1; // random().nextInt(4) + 1;; + for (int i = 0; i < restartTimes; i++) { + Thread.sleep(random().nextInt(30000)); + stopAndStartAllReplicas(); + Thread.sleep(random().nextInt(30000)); + } + + Thread.sleep(2000); + + // stop indexing threads + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + stopExpire = true; + expireThread.join(); + + Thread.sleep(1000); + + waitForThingsToLevelOut(320, TimeUnit.SECONDS); + + Thread.sleep(2000); + + waitForThingsToLevelOut(30, TimeUnit.SECONDS); + + Thread.sleep(5000); + + waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); + + for (StoppableIndexingThread thread : threads) { + thread.join(); + } + + checkShardConsistency(false, false); + } + + public void stopAndStartAllReplicas() throws Exception, InterruptedException { + chaosMonkey.stopAll(random().nextInt(1)); + + if (random().nextBoolean()) { + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + } + Thread.sleep(1000); + + chaosMonkey.startAll(); + } + + @Override + protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { + cloudClient.add(doc); + } + + @Override + public void distribTearDown() throws Exception { + // make sure threads have been stopped... + if (threads != null) { + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + thread.safeStop(); + } + } + + super.distribTearDown(); + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java deleted file mode 100644 index 38733b30fbe..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public abstract class AbstractRestartWhileUpdatingTestBase extends AbstractFullDistribZkTestBase { - - // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain"; - private List threads; - - private volatile boolean stopExpire = false; - - public AbstractRestartWhileUpdatingTestBase() throws Exception { - super(); - sliceCount = 1; - fixShardCount(3); - schemaString = "schema15.xml"; // we need a string id - useFactory("solr.StandardDirectoryFactory"); - } - - public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; - public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; - - @Override - protected String[] getFieldNames() { - return fieldNames; - } - - @Override - protected RandVal[] getRandValues() { - return randVals; - } - - @BeforeClass - public static void beforeRestartWhileUpdatingTest() { - System.setProperty("leaderVoteWait", "300000"); - System.setProperty("solr.autoCommit.maxTime", "30000"); - System.setProperty("solr.autoSoftCommit.maxTime", "3000"); - // SOLR-13212 // TestInjection.nonGracefullClose = "true:60"; - // SOLR-13189 // TestInjection.failReplicaRequests = "true:03"; - } - - @AfterClass - public static void afterRestartWhileUpdatingTest() { - System.clearProperty("leaderVoteWait"); - System.clearProperty("solr.autoCommit.maxTime"); - System.clearProperty("solr.autoSoftCommit.maxTime"); - } - - @Test - public void test() throws Exception { - handle.clear(); - handle.put("timestamp", SKIPVAL); - - // start a couple indexing threads - - int[] maxDocList = new int[] {5000, 10000}; - - int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; - - int numThreads = random().nextInt(4) + 1; - - threads = new ArrayList<>(numThreads); - - Thread expireThread = - new Thread("expireThread") { - @Override - public void run() { - while (!stopExpire) { - try { - Thread.sleep(random().nextInt(15000)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - // try { - // chaosMonkey.expireRandomSession(); - // } catch (KeeperException e) { - // throw new RuntimeException(e); - // } catch (InterruptedException e) { - // throw new RuntimeException(e); - // } - } - } - }; - - // Currently unused - // expireThread.start(); - - StoppableIndexingThread indexThread; - for (int i = 0; i < numThreads; i++) { - indexThread = - new StoppableIndexingThread( - controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true); - threads.add(indexThread); - indexThread.start(); - } - - Thread.sleep(2000); - - int restartTimes = 1; // random().nextInt(4) + 1;; - for (int i = 0; i < restartTimes; i++) { - Thread.sleep(random().nextInt(30000)); - stopAndStartAllReplicas(); - Thread.sleep(random().nextInt(30000)); - } - - Thread.sleep(2000); - - // stop indexing threads - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - stopExpire = true; - expireThread.join(); - - Thread.sleep(1000); - - waitForThingsToLevelOut(320, TimeUnit.SECONDS); - - Thread.sleep(2000); - - waitForThingsToLevelOut(30, TimeUnit.SECONDS); - - Thread.sleep(5000); - - waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); - - for (StoppableIndexingThread thread : threads) { - thread.join(); - } - - checkShardConsistency(false, false); - } - - public void stopAndStartAllReplicas() throws Exception, InterruptedException { - chaosMonkey.stopAll(random().nextInt(1)); - - if (random().nextBoolean()) { - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - } - Thread.sleep(1000); - - chaosMonkey.startAll(); - } - - @Override - protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { - cloudClient.add(doc); - } - - @Override - public void distribTearDown() throws Exception { - // make sure threads have been stopped... - if (threads != null) { - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - thread.safeStop(); - } - } - - super.distribTearDown(); - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} From 341f645a02dda95724e58550d00d528693745408 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:21:45 +0000 Subject: [PATCH 03/16] Merge abstract test classes with single subclasses into concrete test classes Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../solr/cloud/BasicDistributedZk2Test.java | 463 ++++- .../solr/cloud/BasicDistributedZkTest.java | 1717 +++++++++++++++- .../cloud/ChaosMonkeyNothingIsSafeTest.java | 324 ++- .../solr/cloud/ChaosMonkeySafeLeaderTest.java | 216 +- .../apache/solr/cloud/MoveReplicaTest.java | 389 +++- ...OfBoxZkACLAndCredentialsProvidersTest.java | 2 +- ...iddenZkACLAndCredentialsProvidersTest.java | 16 +- .../org/apache/solr/cloud/RecoveryZkTest.java | 139 +- .../org/apache/solr/cloud/SyncSliceTest.java | 287 ++- .../TlogReplayBufferedWhileIndexingTest.java | 125 +- .../solr/cloud/UnloadDistributedZkTest.java | 444 ++++- ...aramsZkACLAndCredentialsProvidersTest.java | 551 +++++- .../CollectionsAPIDistributedZkTest.java | 682 ++++++- .../AbstractBasicDistributedZk2TestBase.java | 487 ----- .../AbstractBasicDistributedZkTestBase.java | 1737 ----------------- ...tractChaosMonkeyNothingIsSafeTestBase.java | 338 ---- ...AbstractChaosMonkeySafeLeaderTestBase.java | 231 --- ...tZkACLAndCredentialsProvidersTestBase.java | 567 ------ .../cloud/AbstractMoveReplicaTestBase.java | 410 ---- .../cloud/AbstractRecoveryZkTestBase.java | 158 -- .../solr/cloud/AbstractSyncSliceTestBase.java | 300 --- ...ogReplayBufferedWhileIndexingTestBase.java | 146 -- .../AbstractUnloadDistributedZkTestBase.java | 463 ----- ...ctCollectionsAPIDistributedZkTestBase.java | 700 ------- 24 files changed, 5288 insertions(+), 5604 deletions(-) delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index b99330645f1..f975deca67c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -16,21 +16,474 @@ */ package org.apache.solr.cloud; +import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.mockfile.FilterPath; +import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.handler.BackupStatusChecker; +import org.apache.solr.handler.ReplicationHandler; +import org.junit.BeforeClass; import org.junit.Test; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. Implementation moved to AbstractBasicDistributedZk2TestBase as it is used by HDFS - * contrib module tests. + * expected. */ +// Backups do checksum validation against a footer value not present in 'SimpleText' +@LuceneTestCase.SuppressCodecs({"SimpleText"}) @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class BasicDistributedZk2Test extends AbstractBasicDistributedZk2TestBase { +public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase { + private static final String SHARD2 = "shard2"; + private static final String SHARD1 = "shard1"; + private static final String ONE_NODE_COLLECTION = "onenodecollection"; + private final boolean onlyLeaderIndexes = random().nextBoolean(); + + public BasicDistributedZk2Test() { + super(); + // we need DVs on point fields to compute stats & facets + if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) + System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); + + sliceCount = 2; + } - @Test @Override + protected boolean useTlogReplicas() { + return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug + // and it's TestInjection use + } + + @BeforeClass + public static void _setPrsDefault() { + configurePrsDefault(); + } + + @Test @ShardsFixed(num = 4) public void test() throws Exception { - super.test(); + boolean testFinished = false; + try { + handle.clear(); + handle.put("timestamp", SKIPVAL); + + testNodeWithoutCollectionForwarding(); + + indexr( + id, + 1, + i1, + 100, + tlong, + 100, + t1, + "now is the time for all good men", + "foo_f", + 1.414f, + "foo_b", + "true", + "foo_d", + 1.414d); + + commit(); + + // make sure we are in a steady state... + waitForRecoveriesToFinish(false); + + assertDocCounts(false); + + indexAbunchOfDocs(); + + // check again + waitForRecoveriesToFinish(false); + + commit(); + + assertDocCounts(VERBOSE); + checkQueries(); + + assertDocCounts(VERBOSE); + + query("q", "*:*", "sort", "n_tl1 desc"); + + bringDownShardIndexSomeDocsAndRecover(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + // test adding another replica to a shard - it should do a + // recovery/replication to pick up the index from the leader + addNewReplica(); + + long docId = testUpdateAndDelete(); + + // index a bad doc... + expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id")); + + // TODO: bring this to its own method? + // try indexing to a leader that has no replicas up + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2); + + String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP); + chaosMonkey.stopShardExcept(SHARD2, nodeName); + + SolrClient client = getClient(nodeName); + + index_specific(client, "id", docId + 1, t1, "what happens here?"); + + // expire a session... + CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0); + chaosMonkey.expireSession(cloudJetty.jetty); + // Wait until the jetty is reconnected, otherwise the following index command could fail + cloudJetty + .jetty + .getCoreContainer() + .getZkController() + .getZkClient() + .getCuratorFramework() + .blockUntilConnected(50, TimeUnit.MILLISECONDS); + + indexr("id", docId + 1, t1, "slip this doc in"); + + waitForRecoveriesToFinish(false); + + checkShardConsistency(SHARD1); + checkShardConsistency(SHARD2); + + testFinished = true; + } finally { + if (!testFinished) { + printLayoutOnTearDown = true; + } + } + } + + private void testNodeWithoutCollectionForwarding() throws Exception { + assertEquals( + 0, + CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1") + .setCoreName(ONE_NODE_COLLECTION + "core") + .process(cloudClient) + .isSuccess()); + + waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1); + waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false); + + ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000); + + int docs = 2; + for (JettySolrRunner jetty : jettys) { + final String clientUrl = getBaseUrl(jetty); + addAndQueryDocs(clientUrl, docs); + docs += 2; + } + } + + // 2 docs added every call + private void addAndQueryDocs(final String baseUrl, int docs) throws Exception { + + SolrQuery query = new SolrQuery("*:*"); + + try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) { + // add a doc + client.add(sdoc("id", docs)); + client.commit(); + + QueryResponse results = client.query(query); + assertEquals(docs - 1, results.getResults().getNumFound()); + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", docs + 1); + client.add(doc); + client.commit(); + + query = new SolrQuery("*:*"); + query.set("rows", 0); + results = client.query(query); + assertEquals(docs, results.getResults().getNumFound()); + } + } + + private long testUpdateAndDelete() throws Exception { + long docId = 99999999L; + indexr("id", docId, t1, "originalcontent"); + + commit(); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("q", t1 + ":originalcontent"); + QueryResponse results = clients.get(0).query(params); + assertEquals(1, results.getResults().getNumFound()); + + // update doc + indexr("id", docId, t1, "updatedcontent"); + + commit(); + + results = clients.get(0).query(params); + assertEquals(0, results.getResults().getNumFound()); + + params.set("q", t1 + ":updatedcontent"); + + results = clients.get(0).query(params); + assertEquals(1, results.getResults().getNumFound()); + + UpdateRequest uReq = new UpdateRequest(); + // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN); + uReq.deleteById(Long.toString(docId)).process(clients.get(0)); + + commit(); + + results = clients.get(0).query(params); + assertEquals(0, results.getResults().getNumFound()); + return docId; + } + + private void bringDownShardIndexSomeDocsAndRecover() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.set("distrib", false); + + commit(); + + long deadShardCount = + shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + int oldLiveNodes = + ZkStateReader.from(cloudClient) + .getZkClient() + .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true) + .size(); + + assertEquals(5, oldLiveNodes); + + // kill a shard + CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0); + + // ensure shard is dead + expectThrows( + SolrServerException.class, + "This server should be down and this update should have failed", + () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!")); + + commit(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + // long cloudClientDocs = cloudClient.query(new + // SolrQuery("*:*")).getResults().getNumFound(); + // System.out.println("clouddocs:" + cloudClientDocs); + + // try to index to a living shard at shard2 + + long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000); + + try { + index_specific( + shardToJetty.get(SHARD1).get(1).client.solrClient, + id, + 1000, + i1, + 108, + t1, + "specific doc!"); + } catch (Exception e) { + // wait and try again + Thread.sleep(4000); + index_specific( + shardToJetty.get(SHARD1).get(1).client.solrClient, + id, + 1000, + i1, + 108, + t1, + "specific doc!"); + } + + commit(); + + checkShardConsistency(true, false); + + query("q", "*:*", "sort", "n_tl1 desc"); + + long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + assertEquals(numFound1 + 1, numFound2); + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", 1001); + + controlClient.add(doc); + + // try adding a doc with CloudSolrServer + UpdateRequest ureq = new UpdateRequest(); + ureq.add(doc); + // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); + + try { + ureq.process(cloudClient); + } catch (SolrServerException e) { + // try again + Thread.sleep(3500); + ureq.process(cloudClient); + } + + commit(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + // lets just check that the one doc since last commit made it in... + assertEquals(numFound2 + 1, numFound3); + + // test debugging + testDebugQueries(); + + if (VERBOSE) { + System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()); + + for (SolrClient client : clients) { + try { + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + System.err.println(client.query(q).getResults().getNumFound()); + } catch (Exception e) { + + } + } + } + // TODO: This test currently fails because debug info is obtained only + // on shards with matches. + // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); + + // this should trigger a recovery phase on deadShard + deadShard.jetty.start(); + + // make sure we have published we are recovering + Thread.sleep(1500); + + waitForRecoveriesToFinish(false); + + deadShardCount = + shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound(); + // if we properly recovered, we should now have the couple missing docs that + // came in while shard was down + checkShardConsistency(true, false); + + // recover over 100 docs so we do more than just peer sync (replicate recovery) + chaosMonkey.stopJetty(deadShard); + + for (int i = 0; i < 226; i++) { + doc = new SolrInputDocument(); + doc.addField("id", 2000 + i); + controlClient.add(doc); + ureq = new UpdateRequest(); + ureq.add(doc); + // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); + ureq.process(cloudClient); + } + commit(); + + Thread.sleep(1500); + + deadShard.jetty.start(); + + // make sure we have published we are recovering + Thread.sleep(1500); + + waitForThingsToLevelOut(1, TimeUnit.MINUTES); + + Thread.sleep(500); + + waitForRecoveriesToFinish(false); + + checkShardConsistency(true, false); + + // try a backup command + try (final SolrClient client = + getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) { + final String backupName = "the_backup"; + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("qt", ReplicationHandler.PATH); + params.set("command", "backup"); + params.set("name", backupName); + final Path location = FilterPath.unwrap(createTempDir()).toRealPath(); + // Allow non-standard location outside SOLR_HOME + jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location)); + params.set("location", location.toString()); + + QueryRequest request = new QueryRequest(params); + client.request(request, DEFAULT_TEST_COLLECTION_NAME); + + final BackupStatusChecker backupStatus = + new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication"); + final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30); + assertTrue( + "Backup dir does not exist: " + backupDirName, + Files.exists(location.resolve(backupDirName))); + } + } + + private void addNewReplica() throws Exception { + + waitForRecoveriesToFinish(false); + + // new server should be part of first shard + // how many docs are on the new shard? + for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) { + if (VERBOSE) + System.err.println( + "shard1 total:" + + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) { + if (VERBOSE) + System.err.println( + "shard2 total:" + + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + + checkShardConsistency(SHARD1); + checkShardConsistency(SHARD2); + + assertDocCounts(VERBOSE); + } + + private void testDebugQueries() throws Exception { + handle.put("explain", SKIPVAL); + handle.put("debug", UNORDERED); + handle.put("time", SKIPVAL); + handle.put("track", SKIP); + query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 7975cfb68d3..57b71796da8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -16,21 +16,1724 @@ */ package org.apache.solr.cloud; -import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.JSONTestUtil; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.impl.InputStreamResponseParser; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; +import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.StreamingUpdateRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.Group; +import org.apache.solr.client.solrj.response.GroupCommand; +import org.apache.solr.client.solrj.response.GroupResponse; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.apache.solr.util.TestInjection.Hook; +import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. Implementation moved to AbstractBasicDistributedZkTestBase as it is used by many HDFS - * contrib tests. + * expected. */ @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class BasicDistributedZkTest extends AbstractBasicDistributedZkTestBase { +public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String DEFAULT_COLLECTION = "collection1"; + + private final boolean onlyLeaderIndexes = random().nextBoolean(); + + String tsort = "t_sortable"; + + private Map> otherCollectionClients = new HashMap<>(); + + private String oneInstanceCollection = "oneInstanceCollection"; + private String oneInstanceCollection2 = "oneInstanceCollection2"; + + private AtomicInteger nodeCounter = new AtomicInteger(); + + CompletionService completionService; + Set> pending; + + private static Hook newSearcherHook = + new Hook() { + volatile CountDownLatch latch; + AtomicReference collection = new AtomicReference<>(); + + @Override + public void newSearcher(String collectionName) { + String c = collection.get(); + if (c != null && c.equals(collectionName)) { + log.info("Hook detected newSearcher"); + try { + latch.countDown(); + } catch (NullPointerException e) { + + } + } + } + + @Override + public void waitForSearcher( + String collection, int cnt, int timeoutms, boolean failOnTimeout) + throws InterruptedException { + latch = new CountDownLatch(cnt); + this.collection.set(collection); + boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS); + if (timeout && failOnTimeout) { + fail("timed out waiting for new searcher event " + latch.getCount()); + } + } + }; + + public BasicDistributedZkTest() { + // we need DVs on point fields to compute stats & facets + if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) + System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); + + sliceCount = 2; + completionService = new ExecutorCompletionService<>(executor); + pending = new HashSet<>(); + } + + @BeforeClass + public static void beforeBDZKTClass() { + TestInjection.newSearcherHook(newSearcherHook); + } + + @Override + protected boolean useTlogReplicas() { + // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's + // TestInjection use + return false; + } - @Test @Override + protected void setDistributedParams(ModifiableSolrParams params) { + + if (r.nextBoolean()) { + // don't set shards, let that be figured out from the cloud state + } else { + // use shard ids rather than physical locations + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < getShardCount(); i++) { + if (i > 0) sb.append(','); + sb.append("shard" + (i + 3)); + } + params.set("shards", sb.toString()); + } + } + + @Test @ShardsFixed(num = 4) - public void test() throws Exception { - super.test(); + protected void test() throws Exception { + // setLoggingLevel(null); + + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + // make sure we have leaders for each shard + for (int j = 1; j < sliceCount; j++) { + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); + } // make sure we again have leaders for each shard + + waitForRecoveriesToFinish(false); + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + del("*:*"); + queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty")); + + // ask every individual replica of every shard to update+commit the same doc id + // with an incrementing counter on each update+commit + int foo_i_counter = 0; + for (SolrClient client : clients) { + foo_i_counter++; + indexDoc( + client, + params("commit", "true"), // SOLR-4923 + sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter)); + // after every update+commit, check all the shards consistency + queryAndCompareShards( + params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup")); + queryAndCompareShards( + params( + "q", "id:1", + "sanity_check", "distrib_id_1_lookup")); + } + + indexr( + id, + 1, + i1, + 100, + tlong, + 100, + t1, + "now is the time for all good men", + "foo_f", + 1.414f, + "foo_b", + "true", + "foo_d", + 1.414d, + tsort, + "now is the time for all good men"); + indexr( + id, + 2, + i1, + 50, + tlong, + 50, + t1, + "to come to the aid of their country.", + tsort, + "to come to the aid of their country."); + indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow"); + indexr( + id, + 4, + i1, + -100, + tlong, + 101, + t1, + "the quick fox jumped over the lazy dog", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 5, + i1, + 500, + tlong, + 500, + t1, + "the quick fox jumped way over the lazy dog", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 6, + i1, + -600, + tlong, + 600, + t1, + "humpty dumpy sat on a wall", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 7, + i1, + 123, + tlong, + 123, + t1, + "humpty dumpy had a great fall", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 8, + i1, + 876, + tlong, + 876, + t1, + "all the kings horses and all the kings men", + tsort, + "all the kings horses and all the kings men"); + indexr( + id, + 9, + i1, + 7, + tlong, + 7, + t1, + "couldn't put humpty together again", + tsort, + "the quick fox jumped over the lazy dog"); + indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass"); + indexr( + id, + 11, + i1, + -987, + tlong, + 987, + t1, + "An eye for eye only ends up making the whole world blind.", + tsort, + "An eye for eye only ends up making the whole world blind."); + indexr( + id, + 12, + i1, + 379, + tlong, + 379, + t1, + "Great works are performed, not by strength, but by perseverance.", + tsort, + "Great works are performed, not by strength, but by perseverance."); + indexr( + id, + 13, + i1, + 232, + tlong, + 232, + t1, + "no eggs on wall, lesson learned", + oddField, + "odd man out", + tsort, + "no eggs on wall, lesson learned"); + + indexr( + id, + 14, + "SubjectTerms_mfacet", + new String[] {"mathematical models", "mathematical analysis"}); + indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); + indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); + String[] vals = new String[100]; + for (int i = 0; i < 100; i++) { + vals[i] = "test " + i; + } + indexr(id, 17, "SubjectTerms_mfacet", vals); + + for (int i = 100; i < 150; i++) { + indexr(id, i); + } + + commit(); + + testTokenizedGrouping(); + testSortableTextFaceting(); + testSortableTextSorting(); + testSortableTextGrouping(); + + queryAndCompareShards( + params( + "q", "*:*", + "sort", "id desc", + "distrib", "false", + "sanity_check", "is_empty")); + + // random value sort + for (String f : fieldNames) { + query(false, new String[] {"q", "*:*", "sort", f + " desc"}); + query(false, new String[] {"q", "*:*", "sort", f + " asc"}); + } + + // these queries should be exactly ordered and scores should exactly match + query(false, new String[] {"q", "*:*", "sort", i1 + " desc"}); + query(false, new String[] {"q", "*:*", "sort", i1 + " asc"}); + query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"}); + query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"}); + query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"}); + handle.put("maxScore", SKIPVAL); + query( + false, + new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it. + // JavaBinCodec.writeSolrDocumentList() + // is agnostic of request params. + handle.remove("maxScore"); + query( + false, + new String[] { + "q", "{!func}" + i1, "fl", "*,score" + }); // even scores should match exactly here + + handle.put("highlighting", UNORDERED); + handle.put("response", UNORDERED); + + handle.put("maxScore", SKIPVAL); + query(false, new String[] {"q", "quick"}); + query(false, new String[] {"q", "all", "fl", "id", "start", "0"}); + query( + false, + new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs + query(false, new String[] {"q", "all", "fl", "id", "start", "100"}); + + handle.put("score", SKIPVAL); + query(false, new String[] {"q", "quick", "fl", "*,score"}); + query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"}); + query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"}); + + query( + false, + new String[] { + "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1 + }); + + query( + false, + new String[] { + "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1 + }); + + query(false, new String[] {"q", "matchesnothing", "fl", "*,score"}); + + query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1}); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "count" + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "count", + "facet.mincount", + 2 + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "index" + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "index", + "facet.mincount", + 2 + }); + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1 + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.query", + "quick", + "facet.query", + "all", + "facet.query", + "*:*" + }); + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1 + }); + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2 + }); + + // test faceting multiple things at once + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.query", + "quick", + "facet.query", + "all", + "facet.query", + "*:*", + "facet.field", + t1 + }); + + // test filter tagging, facet exclusion, and naming (multi-select facet support) + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.query", + "{!key=myquick}quick", + "facet.query", + "{!key=myall ex=a}all", + "facet.query", + "*:*", + "facet.field", + "{!key=mykey ex=a}" + t1, + "facet.field", + "{!key=other ex=b}" + t1, + "facet.field", + "{!key=again ex=a,b}" + t1, + "facet.field", + t1, + "fq", + "{!tag=a}id_i1:[1 TO 7]", + "fq", + "{!tag=b}id_i1:[3 TO 9]" + }); + query( + false, + new Object[] { + "q", + "*:*", + "facet", + "true", + "facet.field", + "{!ex=t1}SubjectTerms_mfacet", + "fq", + "{!tag=t1}SubjectTerms_mfacet:(test 1)", + "facet.limit", + "10", + "facet.mincount", + "1" + }); + + // test field that is valid in schema but missing in all shards + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2 + }); + // test field that is valid in schema and missing in some shards + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2 + }); + + query( + false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1}); + + /* TODO: the failure may come back in "exception" + try { + // test error produced for field that is invalid for schema + query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2); + TestCase.fail("SolrServerException expected for invalid field that is not in schema"); + } catch (SolrServerException ex) { + // expected + } + */ + + // Try to get better coverage for refinement queries by turning off over requesting. + // This makes it much more likely that we may not get the top facet values and hence + // we turn of that checking. + handle.put("facet_fields", SKIPVAL); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 0, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + 5, + "facet.shard.limit", + 5 + }); + // check a complex key name + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 0, + "facet", + "true", + "facet.field", + "{!key='a b/c \\' \\} foo'}" + t1, + "facet.limit", + 5, + "facet.shard.limit", + 5 + }); + handle.remove("facet_fields"); + + // index the same document to two servers and make sure things + // don't blow up. + if (clients.size() >= 2) { + index(id, 100, i1, 107, t1, "oh no, a duplicate!"); + for (int i = 0; i < clients.size(); i++) { + index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!"); + } + commit(); + query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1}); + query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1}); + query(false, new Object[] {"q", "*:*", "rows", 100}); + } + + // test debugging + handle.put("explain", SKIPVAL); + handle.put("debug", UNORDERED); + handle.put("time", SKIPVAL); + handle.put("track", SKIP); + query( + false, + new Object[] { + "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true" + }); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY}); + + // try add commitWithin + long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + for (SolrClient client : clients) { + assertEquals( + "unexpected pre-commitWithin document count on node: " + + ((HttpSolrClient) client).getBaseURL(), + before, + client.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("commitWithin", 10); + add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301))); + + newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); + + ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); + DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION); + + assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl); + + // try deleteById commitWithin + UpdateRequest deleteByIdReq = new UpdateRequest(); + deleteByIdReq.deleteById("300"); + deleteByIdReq.setCommitWithin(10); + deleteByIdReq.process(cloudClient); + + newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); + + assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl); + + // try deleteByQuery commitWithin + UpdateRequest deleteByQueryReq = new UpdateRequest(); + deleteByQueryReq.deleteByQuery("id:301"); + deleteByQueryReq.setCommitWithin(10); + deleteByQueryReq.process(cloudClient); + + newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); + + assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl); + + // TODO: This test currently fails because debug info is obtained only + // on shards with matches. + // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); + + // would be better if these where all separate tests - but much, much + // slower + doOptimisticLockingAndUpdating(); + testShardParamVariations(); + testMultipleCollections(); + testANewCollectionInOneInstance(); + testSearchByCollectionName(); + testUpdateByCollectionName(); + testANewCollectionInOneInstanceWithManualShardAssignement(); + testNumberOfCommitsWithCommitAfterAdd(); + + testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit"); + testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit"); + + testStopAndStartCoresInOneInstance(); + } + + private void testSortableTextFaceting() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.addFacetField(tsort); + query.setFacetMissing(false); + QueryResponse resp = queryRandomShard(query); + List ffs = resp.getFacetFields(); + for (FacetField ff : ffs) { + if (ff.getName().equals(tsort) == false) continue; + for (FacetField.Count count : ff.getValues()) { + long num = count.getCount(); + switch (count.getName()) { + case "all the kings horses and all the kings men": + case "An eye for eye only ends up making the whole world blind.": + case "Great works are performed, not by strength, but by perseverance.": + case "how now brown cow": + case "no eggs on wall, lesson learned": + case "now is the time for all good men": + case "this too shall pass": + case "to come to the aid of their country.": + assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num); + break; + case "the quick fox jumped over the lazy dog": + assertEquals("Should have 5 docs for the lazy dog", 5, num); + break; + default: + fail("No case for facet '" + ff.getName() + "'"); + } + } + } + } + + private void testSortableTextSorting() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.addSort(tsort, SolrQuery.ORDER.desc); + query.addField("*"); + query.addField("eoe_sortable"); + query.addField(tsort); + QueryResponse resp = queryRandomShard(query); + + SolrDocumentList docs = resp.getResults(); + + String title = docs.get(0).getFieldValue(tsort).toString(); + for (SolrDocument doc : docs) { + assertTrue( + "Docs should be back in sorted order, descending", + title.compareTo(doc.getFieldValue(tsort).toString()) >= 0); + title = doc.getFieldValue(tsort).toString(); + } + } + + private void testSortableTextGrouping() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.add("group", "true"); + query.add("group.field", tsort); + QueryResponse resp = queryRandomShard(query); + GroupResponse groupResp = resp.getGroupResponse(); + List grpCmds = groupResp.getValues(); + for (GroupCommand grpCmd : grpCmds) { + if (grpCmd.getName().equals(tsort) == false) continue; + for (Group grp : grpCmd.getValues()) { + long count = grp.getResult().getNumFound(); + if (grp.getGroupValue() == null) + continue; // Don't count the groups without an entry as the numnber is variable + switch (grp.getGroupValue()) { + case "all the kings horses and all the kings men": + case "An eye for eye only ends up making the whole world blind.": + case "Great works are performed, not by strength, but by perseverance.": + case "how now brown cow": + case "no eggs on wall, lesson learned": + case "now is the time for all good men": + case "this too shall pass": + case "to come to the aid of their country.": + assertEquals( + "Should have exactly one facet count for field " + grpCmd.getName(), 1, count); + break; + case "the quick fox jumped over the lazy dog": + assertEquals("Should have 5 docs for the lazy dog", 5, count); + break; + default: + fail("No case for facet '" + grpCmd.getName() + "'"); + } + } + } + } + + private void testTokenizedGrouping() throws Exception { + SolrException ex = + expectThrows( + SolrException.class, + () -> { + query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1}); + }); + assertTrue( + "Expected error from server that SortableTextFields are required", + ex.getMessage() + .contains( + "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode")); + } + + private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception { + long found = checkSlicesSameCounts(dColl); + + if (found != expected) { + // we get one do over in a bad race + Thread.sleep(1000); + found = checkSlicesSameCounts(dColl); + } + + assertEquals(msg, expected, checkSlicesSameCounts(dColl)); + } + + // Ensure that total docs found is the expected number. + private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage) + throws Exception { + AtomicLong total = new AtomicLong(-1); + try { + ZkStateReader.from(getCommonCloudSolrClient()) + .waitForState( + DEFAULT_COLLECTION, + waitMillis, + TimeUnit.MILLISECONDS, + (n, c) -> { + long docTotal; + try { + docTotal = checkSlicesSameCounts(c); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + total.set(docTotal); + if (docTotal == expectedNumFound) { + return true; + } + return false; + }); + } catch (TimeoutException | InterruptedException e) { + + } + // We could fail here if we broke out of the above because we exceeded the time allowed. + assertEquals(failureMessage, expectedNumFound, total.get()); + + // This should be redundant, but it caught a test error after all. + for (SolrClient client : clients) { + assertEquals( + failureMessage, + expectedNumFound, + client.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + } + + // Insure that counts are the same for all replicas in each shard + // Return the total doc count for the query. + private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException { + long docTotal = 0; // total number of documents found counting only one replica per slice. + for (Slice slice : dColl.getActiveSlices()) { + long sliceDocCount = -1; + for (Replica rep : slice.getReplicas()) { + try (SolrClient one = getHttpSolrClient(rep)) { + SolrQuery query = new SolrQuery("*:*"); + query.setDistrib(false); + QueryResponse resp = one.query(query); + long hits = resp.getResults().getNumFound(); + if (sliceDocCount == -1) { + sliceDocCount = hits; + docTotal += hits; + } else { + if (hits != sliceDocCount) { + return -1; + } + } + } + } + } + return docTotal; + } + + private void testShardParamVariations() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + Map shardCounts = new HashMap<>(); + + for (String shard : shardToJetty.keySet()) { + // every client should give the same numDocs for this shard + // shffle the clients in a diff order for each shard + List solrclients = new ArrayList<>(this.clients); + Collections.shuffle(solrclients, random()); + for (SolrClient client : solrclients) { + query.set("shards", shard); + long numDocs = client.query(query).getResults().getNumFound(); + assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs); + if (!shardCounts.containsKey(shard)) { + shardCounts.put(shard, numDocs); + } + assertEquals( + "inconsitent numDocs for shard " + shard + " via " + client, + shardCounts.get(shard).longValue(), + numDocs); + + List replicaJetties = new ArrayList<>(shardToJetty.get(shard)); + Collections.shuffle(replicaJetties, random()); + + // each replica should also give the same numDocs + ArrayList replicaAlts = new ArrayList<>(replicaJetties.size() * 2); + for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { + String replica = replicaJetty.url; + query.set("shards", replica); + + // replicas already shuffled, use this in the alternative check below + if (0 == random().nextInt(3) || replicaAlts.size() < 2) { + replicaAlts.add(replica); + } + + numDocs = client.query(query).getResults().getNumFound(); + assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs); + assertEquals( + "inconsitent numDocs for shard " + + shard + + " in replica " + + replica + + " via " + + client, + shardCounts.get(shard).longValue(), + numDocs); + } + + // any combination of replica alternatives should give same numDocs + String replicas = String.join("|", replicaAlts); + query.set("shards", replicas); + numDocs = client.query(query).getResults().getNumFound(); + assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs); + assertEquals( + "inconsitent numDocs for replicas " + replicas + " via " + client, + shardCounts.get(shard).longValue(), + numDocs); + } + } + + // sums of multiple shards should add up regardless of how we + // query those shards or which client we use + long randomShardCountsExpected = 0; + ArrayList randomShards = new ArrayList<>(shardCounts.size()); + for (Map.Entry shardData : shardCounts.entrySet()) { + if (random().nextBoolean() || randomShards.size() < 2) { + String shard = shardData.getKey(); + randomShardCountsExpected += shardData.getValue(); + if (random().nextBoolean()) { + // use shard id + randomShards.add(shard); + } else { + // use some set explicit replicas + ArrayList replicas = new ArrayList<>(7); + for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { + if (0 == random().nextInt(3) || 0 == replicas.size()) { + replicas.add(replicaJetty.url); + } + } + Collections.shuffle(replicas, random()); + randomShards.add(String.join("|", replicas)); + } + } + } + String randShards = String.join(",", randomShards); + query.set("shards", randShards); + for (SolrClient client : this.clients) { + assertEquals( + "numDocs for " + randShards + " via " + client, + randomShardCountsExpected, + client.query(query).getResults().getNumFound()); + } + + // total num docs must match sum of every shard's numDocs + query = new SolrQuery("*:*"); + long totalShardNumDocs = 0; + for (Long c : shardCounts.values()) { + totalShardNumDocs += c; + } + for (SolrClient client : clients) { + assertEquals( + "sum of shard numDocs on client: " + client, + totalShardNumDocs, + client.query(query).getResults().getNumFound()); + } + assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs); + } + + private void testStopAndStartCoresInOneInstance() throws Exception { + JettySolrRunner jetty = jettys.get(0); + try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) { + ThreadPoolExecutor executor = null; + try { + executor = + new ExecutorUtil.MDCAwareThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 5, + TimeUnit.SECONDS, + new SynchronousQueue(), + new SolrNamedThreadFactory("testExecutor")); + int cnt = 3; + + // create the cores + createCollectionInOneInstance( + httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt); + } finally { + if (executor != null) { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + } + } + + cloudJettys.get(0).jetty.stop(); + printLayout(); + + cloudJettys.get(0).jetty.start(); + ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2"); + try { + ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000); + } catch (SolrException e) { + printLayout(); + throw e; + } + + printLayout(); + } + + /** Create a collection in single node */ + public static void createCollectionInOneInstance( + final SolrClient client, + String nodeName, + ThreadPoolExecutor executor, + final String collection, + final int numShards, + int numReplicas) { + assertNotNull(nodeName); + try { + assertEquals( + 0, + CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) + .setCreateNodeSet("") + .process(client) + .getStatus()); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + for (int i = 0; i < numReplicas; i++) { + final int freezeI = i; + executor.execute( + () -> { + try { + assertTrue( + CollectionAdminRequest.addReplicaToShard( + collection, "shard" + ((freezeI % numShards) + 1)) + .setCoreName(collection + freezeI) + .setNode(nodeName) + .process(client) + .isSuccess()); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + }); + } + } + + @Override + protected CollectionAdminResponse createCollection( + Map> collectionInfos, + String collectionName, + String configSetName, + int numShards, + int numReplicas, + SolrClient client, + String createNodeSetStr) + throws SolrServerException, IOException { + // TODO: Use CollectionAdminRequest for this test + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + + params.set(CollectionHandlingUtils.NUM_SLICES, numShards); + params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas); + if (createNodeSetStr != null) + params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr); + + int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0; + List list = new ArrayList<>(); + list.add(numShards); + list.add(numReplicas); + if (collectionInfos != null) { + collectionInfos.put(collectionName, list); + } + params.set("name", collectionName); + params.set("collection.configName", configSetName); + QueryRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + + CollectionAdminResponse res = new CollectionAdminResponse(); + if (client == null) { + final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL(); + + try (SolrClient aClient = createNewSolrClient("", baseUrl)) { + res.setResponse(aClient.request(request)); + } + } else { + res.setResponse(client.request(request)); + } + return res; + } + + @Override + protected Replica getLeaderFromZk(String collection, String slice) { + ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); + Replica leader = clusterState.getCollection(collection).getLeader(slice); + if (leader == null) { + throw new RuntimeException("Could not find leader:" + collection + " " + slice); + } + return leader; + } + + /** + * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two + * (stored) string fields. + * + *

If the values are "double-doubled" or "not-doubled" then we know the processor was not run + * the appropriate number of times + */ + private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception { + + final String fieldA = "regex_dup_A_s"; + final String fieldB = "regex_dup_B_s"; + final String val = "x"; + final String expected = "x_x"; + final ModifiableSolrParams updateParams = new ModifiableSolrParams(); + updateParams.add(UpdateParams.UPDATE_CHAIN, chain); + + final int numLoops = atLeast(50); + + for (int i = 1; i < numLoops; i++) { + // add doc to random client + SolrClient updateClient = clients.get(random().nextInt(clients.size())); + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, id, i, fieldA, val, fieldB, val); + UpdateResponse ures = add(updateClient, updateParams, doc); + assertEquals(chain + ": update failed", 0, ures.getStatus()); + ures = updateClient.commit(); + assertEquals(chain + ": commit failed", 0, ures.getStatus()); + } + + // query for each doc, and check both fields to ensure the value is correct + for (int i = 1; i < numLoops; i++) { + final String query = id + ":" + i; + QueryResponse qres = queryRandomShard(new SolrQuery(query)); + assertEquals(chain + ": query failed: " + query, 0, qres.getStatus()); + assertEquals( + chain + ": didn't find correct # docs with query: " + query, + 1, + qres.getResults().getNumFound()); + SolrDocument doc = qres.getResults().get(0); + + for (String field : new String[] {fieldA, fieldB}) { + assertEquals( + chain + ": doc#" + i + " has wrong value for " + field, + expected, + doc.getFirstValue(field)); + } + } + } + + // cloud level test mainly needed just to make sure that versions and errors are propagated + // correctly + private void doOptimisticLockingAndUpdating() throws Exception { + log.info("### STARTING doOptimisticLockingAndUpdating"); + printLayout(); + + final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1); + indexDoc(sd); + + ignoreException("version conflict"); + for (SolrClient client : clients) { + SolrException e = expectThrows(SolrException.class, () -> client.add(sd)); + assertEquals(409, e.code()); + } + unIgnoreException("version conflict"); + + // TODO: test deletes. SolrJ needs a good way to pass version for delete... + + final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5); + clients.get(0).add(sd2); + + List expected = new ArrayList<>(); + int val = 0; + for (SolrClient client : clients) { + val += 10; + client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val)); + expected.add(val); + } + + QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000")); + for (SolrClient client : clients) { + val += 10; + NamedList rsp = client.request(qr); + String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected); + if (match != null) throw new RuntimeException(match); + } + } + + private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException { + log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd"); + long startCommits = getNumCommits((HttpSolrClient) clients.get(0)); + + NamedList result = + clients + .get(0) + .request( + new StreamingUpdateRequest( + "/update", getFile("books_numeric_ids.csv"), "application/csv") + .setCommitWithin(900000) + .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true)); + + long endCommits = getNumCommits((HttpSolrClient) clients.get(0)); + assertEquals(startCommits + 1L, endCommits); + } + + private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException { + String collection = sourceClient.getDefaultCollection(); + try (SolrClient client = + new HttpSolrClient.Builder(sourceClient.getBaseURL()) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(60000, TimeUnit.MILLISECONDS) + .build()) { + var req = + new GenericSolrRequest( + SolrRequest.METHOD.GET, + "/admin/metrics", + SolrRequest.SolrRequestType.ADMIN, + SolrParams.of("wt", "prometheus")); + req.setResponseParser(new InputStreamResponseParser("prometheus")); + + NamedList resp = client.request(req); + try (InputStream in = (InputStream) resp.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + String metricName = "solr_core_update_commit_ops"; + + return (long) + output + .lines() + .filter( + l -> + l.startsWith(metricName) + && l.contains("collection=\"" + collection + "\"") + && l.contains("ops=\"commits\"")) + .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" ")))) + .sum(); + } + } + } + + private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception { + log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement"); + assertEquals( + 0, + CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + + List collectionClients = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + CollectionAdminResponse resp = + CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1)) + .setNode(jettys.get(0).getNodeName()) + .process(cloudClient); + for (String coreName : resp.getCollectionCoresStatus().keySet()) { + collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); + } + } + + SolrClient client1 = collectionClients.get(0); + SolrClient client2 = collectionClients.get(1); + SolrClient client3 = collectionClients.get(2); + SolrClient client4 = collectionClients.get(3); + + // no one should be recovering + waitForRecoveriesToFinish( + oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true); + + assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient())); + + // printLayout(); + + // TODO: enable when we don't falsely get slice1... + // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000); + // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000); + client2.add(getDoc(id, "1")); + client3.add(getDoc(id, "2")); + client4.add(getDoc(id, "3")); + + client1.commit(); + SolrQuery query = new SolrQuery("*:*"); + query.set("distrib", false); + long oneDocs = client1.query(query).getResults().getNumFound(); + long twoDocs = client2.query(query).getResults().getNumFound(); + long threeDocs = client3.query(query).getResults().getNumFound(); + long fourDocs = client4.query(query).getResults().getNumFound(); + + query.set("collection", oneInstanceCollection2); + query.set("distrib", true); + long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + + // System.out.println("1:" + oneDocs); + // System.out.println("2:" + twoDocs); + // System.out.println("3:" + threeDocs); + // System.out.println("4:" + fourDocs); + // System.out.println("All Docs:" + allDocs); + + // assertEquals(oneDocs, threeDocs); + // assertEquals(twoDocs, fourDocs); + // assertNotSame(oneDocs, twoDocs); + assertEquals(3, allDocs); + + // we added a role of none on these creates - check for it + ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient()); + zkStateReader.forceUpdateCollection(oneInstanceCollection2); + Map slices = + zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap(); + assertNotNull(slices); + + Replica leader = + getCommonCloudSolrClient() + .getClusterState() + .getCollection(oneInstanceCollection2) + .getLeader("shard1"); + + // now test that unloading a core gets us a new leader + try (SolrClient unloadClient = + new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString()) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(60000, TimeUnit.MILLISECONDS) + .build()) { + Unload unloadCmd = new Unload(true); + unloadCmd.setCoreName(leader.getCoreName()); + + String leaderUrl = leader.getCoreUrl(); + + testExecutor.execute( + new Runnable() { + + @Override + public void run() { + try { + unloadClient.request(unloadCmd); + } catch (SolrServerException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + try { + ZkStateReader.from(getCommonCloudSolrClient()) + .waitForState( + oneInstanceCollection2, + 20000, + TimeUnit.MILLISECONDS, + (n, c) -> { + try { + if (leaderUrl.equals( + zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) { + return false; + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + }); + } catch (TimeoutException | InterruptedException e) { + fail("Leader never changed"); + } + } + + IOUtils.close(collectionClients); + } + + private void testSearchByCollectionName() throws SolrServerException, IOException { + log.info("### STARTING testSearchByCollectionName"); + SolrClient client = clients.get(0); + final String baseUrl = ((HttpSolrClient) client).getBaseURL(); + + // the cores each have different names, but if we add the collection name to the url + // we should get mapped to the right core + try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { + SolrQuery query = new SolrQuery("*:*"); + long oneDocs = client1.query(query).getResults().getNumFound(); + assertEquals(3, oneDocs); + } + } + + private void testUpdateByCollectionName() throws SolrServerException, IOException { + log.info("### STARTING testUpdateByCollectionName"); + SolrClient client = clients.get(0); + final String baseUrl = ((HttpSolrClient) client).getBaseURL(); + + // the cores each have different names, but if we add the collection name to the url + // we should get mapped to the right core + // test hitting an update url + try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { + client1.commit(); + } + } + + private void testANewCollectionInOneInstance() throws Exception { + log.info("### STARTING testANewCollectionInOneInstance"); + CollectionAdminResponse response = + CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2) + .setCreateNodeSet(jettys.get(0).getNodeName()) + .process(cloudClient); + assertEquals(0, response.getStatus()); + List collectionClients = new ArrayList<>(); + for (String coreName : response.getCollectionCoresStatus().keySet()) { + collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); + } + + SolrClient client1 = collectionClients.get(0); + SolrClient client2 = collectionClients.get(1); + SolrClient client3 = collectionClients.get(2); + SolrClient client4 = collectionClients.get(3); + + waitForRecoveriesToFinish( + oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false); + assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient())); + + client2.add(getDoc(id, "1")); + client3.add(getDoc(id, "2")); + client4.add(getDoc(id, "3")); + + client1.commit(); + SolrQuery query = new SolrQuery("*:*"); + query.set("distrib", false); + long oneDocs = client1.query(query).getResults().getNumFound(); + long twoDocs = client2.query(query).getResults().getNumFound(); + long threeDocs = client3.query(query).getResults().getNumFound(); + long fourDocs = client4.query(query).getResults().getNumFound(); + + query.set("collection", oneInstanceCollection); + query.set("distrib", true); + long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + + // System.out.println("1:" + oneDocs); + // System.out.println("2:" + twoDocs); + // System.out.println("3:" + threeDocs); + // System.out.println("4:" + fourDocs); + // System.out.println("All Docs:" + allDocs); + + assertEquals(3, allDocs); + IOUtils.close(collectionClients); + } + + private void createSolrCore( + final String collection, + List collectionClients, + final String baseUrl, + final int num, + final String shardId) { + Callable call = + () -> { + try (SolrClient client = getHttpSolrClient(baseUrl)) { + // client.setConnectionTimeout(15000); + Create createCmd = new Create(); + createCmd.setCoreName(collection + num); + createCmd.setCollection(collection); + + if (random().nextBoolean()) { + // sometimes we use an explicit core node name + createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet()); + } + + if (shardId == null) { + createCmd.setNumShards(2); + } + createCmd.setDataDir(getDataDir(createTempDir(collection).toString())); + if (shardId != null) { + createCmd.setShardId(shardId); + } + client.request(createCmd); + } catch (Exception e) { + log.error("error creating core", e); + // fail + } + return null; + }; + + pending.add(completionService.submit(call)); + + collectionClients.add(createNewSolrClient(collection + num, baseUrl)); + } + + private void testMultipleCollections() throws Exception { + log.info("### STARTING testMultipleCollections"); + // create another 2 collections and search across them + createNewCollection("collection2"); + createNewCollection("collection3"); + + while (pending != null && pending.size() > 0) { + + Future future = completionService.take(); + if (future == null) return; + pending.remove(future); + } + + indexDoc("collection2", getDoc(id, "10000000")); + indexDoc("collection2", getDoc(id, "10000001")); + indexDoc("collection2", getDoc(id, "10000003")); + + getSolrClient("collection2").add(getDoc(id, "10000004")); + + indexDoc("collection3", getDoc(id, "20000000")); + indexDoc("collection3", getDoc(id, "20000001")); + + getSolrClient("collection3").add(getDoc(id, "10000005")); + + otherCollectionClients.get("collection2").get(0).commit(); + otherCollectionClients.get("collection3").get(0).commit(); + + SolrClient clientForCollection1 = getSolrClient("collection1"); + long collection1Docs = + clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound(); + + long collection2Docs = + otherCollectionClients + .get("collection2") + .get(0) + .query(new SolrQuery("*:*")) + .getResults() + .getNumFound(); + + long collection3Docs = + otherCollectionClients + .get("collection3") + .get(0) + .query(new SolrQuery("*:*")) + .getResults() + .getNumFound(); + + SolrQuery query = new SolrQuery("*:*"); + query.set("collection", "collection2,collection3"); + long found = clients.get(0).query(query).getResults().getNumFound(); + assertEquals(collection2Docs + collection3Docs, found); + + query = new SolrQuery("*:*"); + query.set("collection", "collection1,collection2,collection3"); + found = clients.get(0).query(query).getResults().getNumFound(); + assertEquals(collection1Docs + collection2Docs + collection3Docs, found); + + // try to search multiple with cloud client + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection1Docs + collection2Docs + collection3Docs, found); + + query.set("collection", "collection2,collection3"); + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection2Docs + collection3Docs, found); + + query.set("collection", "collection3"); + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection3Docs, found); + + query.remove("collection"); + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection1Docs, found); + + assertEquals(collection3Docs, collection2Docs - 1); + } + + protected void indexDoc(String collection, SolrInputDocument doc) + throws IOException, SolrServerException { + List clients = otherCollectionClients.get(collection); + int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size(); + SolrClient client = clients.get(which); + client.add(doc); + } + + @SuppressWarnings({"unchecked"}) + private void createNewCollection(final String collection) throws InterruptedException { + try { + assertEquals( + 0, + CollectionAdminRequest.createCollection(collection, "conf1", 2, 1) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + } catch (Exception e) { + log.error("error creating collection", e); + // fails + } + final List collectionClients = new ArrayList<>(); + otherCollectionClients.put(collection, collectionClients); + int unique = 0; + for (final JettySolrRunner runner : jettys) { + unique++; + final int frozeUnique = unique; + Callable call = + () -> { + try { + assertTrue( + CollectionAdminRequest.addReplicaToShard( + collection, "shard" + ((frozeUnique % 2) + 1)) + .setNode(runner.getNodeName()) + .process(cloudClient) + .isSuccess()); + } catch (Exception e) { + log.error("error adding replica", e); + // fails + } + return null; + }; + + collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString())); + pending.add(completionService.submit(call)); + while (pending != null && pending.size() > 0) { + + Future future = completionService.take(); + if (future == null) return; + pending.remove(future); + } + } + } + + @Override + protected SolrClient createNewSolrClient(String collection, String baseUrl) { + + SolrClient client = getHttpSolrClient(baseUrl, collection); + + return client; + } + + /** + * @param collection the name of a collection or core to set as the "default" on the created + * client. + * @param baseUrl the "base" URL of a Solr node. Should not contain a collection or core + * name. + * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds + * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds + */ + protected SolrClient createNewSolrClient( + String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) { + + SolrClient client = + new HttpSolrClient.Builder(baseUrl) + .withDefaultCollection(collection) + .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS) + .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS) + .build(); + + return client; + } + + @Override + protected QueryResponse queryRandomShard(ModifiableSolrParams params) + throws SolrServerException, IOException { + + if (r.nextBoolean()) return super.queryRandomShard(params); + + if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION); + + QueryResponse rsp = getCommonCloudSolrClient().query(params); + return rsp; + } + + @Override + public void distribTearDown() throws Exception { + super.distribTearDown(); + if (otherCollectionClients != null) { + for (List clientList : otherCollectionClients.values()) { + IOUtils.close(clientList); + } + } + otherCollectionClients = null; + List tasks = executor.shutdownNow(); + assertTrue(tasks.isEmpty()); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 3fb1bbabc6e..15732a642d3 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -16,11 +16,325 @@ */ package org.apache.solr.cloud; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; -/* - * Implementation moved to AbstractChaosMonkeyNothingIsSafeTestBase.java as it is also - * used by the HDFS contrib tests. - */ +@LuceneTestCase.Nightly @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class ChaosMonkeyNothingIsSafeTest extends AbstractChaosMonkeyNothingIsSafeTestBase {} +public class ChaosMonkeyNothingIsSafeTest + extends AbstractFullDistribZkTestBase { + private static final int FAIL_TOLERANCE = 100; + + private static final Integer RUN_LENGTH = + Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); + + private final boolean onlyLeaderIndexes = random().nextBoolean(); + + @BeforeClass + public static void beforeSuperClass() { + schemaString = "schema15.xml"; // we need a string id + System.setProperty("solr.autoCommit.maxTime", "15000"); + System.clearProperty("solr.httpclient.retries"); + System.clearProperty("solr.retries.on.forward"); + System.clearProperty("solr.retries.to.followers"); + setErrorHook(); + } + + @AfterClass + public static void afterSuperClass() { + System.clearProperty("solr.autoCommit.maxTime"); + clearErrorHook(); + } + + @Override + protected void destroyServers() throws Exception { + + super.destroyServers(); + } + + protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; + protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; + + private int clientSoTimeout = 60000; + + private volatile FullThrottleStoppableIndexingThread ftIndexThread; + + private final boolean runFullThrottle; + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public RandVal[] getRandValues() { + return randVals; + } + + @Override + public void distribSetUp() throws Exception { + super.distribSetUp(); + // can help to hide this when testing and looking at logs + // ignoreException("shard update error"); + useFactory("solr.StandardDirectoryFactory"); + } + + @Override + public void distribTearDown() throws Exception { + try { + ftIndexThread.safeStop(); + } catch (NullPointerException e) { + // okay + } + super.distribTearDown(); + } + + public ChaosMonkeyNothingIsSafeTest() { + super(); + sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); + if (sliceCount == -1) { + sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; + } + + int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); + if (numShards == -1) { + // we make sure that there's at least one shard with more than one replica + // so that the ChaosMonkey has something to kill + numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; + } + fixShardCount(numShards); + + // TODO: we only do this sometimes so that we can sometimes compare against control, + // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer + runFullThrottle = random().nextBoolean(); + } + + @Override + protected boolean useTlogReplicas() { + return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug + // and it's TestInjection use + } + + @Override + protected CloudSolrClient createCloudClient(String defaultCollection) { + return this.createCloudClient(defaultCollection, this.clientSoTimeout); + } + + protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) { + + return getCloudSolrClient( + zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout); + } + + @Test + @SuppressWarnings({"try"}) + public void test() throws Exception { + // None of the operations used here are particularly costly, so this should work. + // Using this low timeout will also help us catch index stalling. + clientSoTimeout = 5000; + + boolean testSuccessful = false; + try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) { + handle.clear(); + handle.put("timestamp", SKIPVAL); + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + // make sure we have leaders for each shard + for (int j = 1; j < sliceCount; j++) { + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); + } // make sure we again have leaders for each shard + + waitForRecoveriesToFinish(false); + + // we cannot do delete by query + // as it's not supported for recovery + del("*:*"); + + List threads = new ArrayList<>(); + List indexTreads = new ArrayList<>(); + int threadCount = TEST_NIGHTLY ? 3 : 1; + int i = 0; + for (i = 0; i < threadCount; i++) { + StoppableIndexingThread indexThread = + new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true); + threads.add(indexThread); + indexTreads.add(indexThread); + indexThread.start(); + } + + threadCount = 1; + i = 0; + for (i = 0; i < threadCount; i++) { + StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient); + threads.add(searchThread); + searchThread.start(); + } + + if (runFullThrottle) { + ftIndexThread = + new FullThrottleStoppableIndexingThread( + ((CloudLegacySolrClient) cloudClient).getHttpClient(), + controlClient, + cloudClient, + clients, + "ft1", + true, + this.clientSoTimeout); + ftIndexThread.start(); + } + + chaosMonkey.startTheMonkey(true, 10000); + try { + long runLength; + if (RUN_LENGTH != -1) { + runLength = RUN_LENGTH; + } else { + int[] runTimes; + if (TEST_NIGHTLY) { + runTimes = + new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; + } else { + runTimes = new int[] {5000, 7000, 15000}; + } + runLength = runTimes[random().nextInt(runTimes.length - 1)]; + } + + Thread.sleep(runLength); + } finally { + chaosMonkey.stopTheMonkey(); + } + + // ideally this should go into chaosMonkey + restartZk(1000 * (5 + random().nextInt(4))); + + if (runFullThrottle) { + ftIndexThread.safeStop(); + } + + for (StoppableThread indexThread : threads) { + indexThread.safeStop(); + } + + // start any downed jetties to be sure we still will end up with a leader per shard... + + // wait for stop... + for (StoppableThread indexThread : threads) { + indexThread.join(); + } + + // try and wait for any replications and what not to finish... + + Thread.sleep(2000); + + // wait until there are no recoveries... + waitForThingsToLevelOut(); + + // make sure we again have leaders for each shard + for (int j = 1; j < sliceCount; j++) { + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000); + } + + commit(); + + // TODO: assert we didn't kill everyone + + zkStateReader.updateLiveNodes(); + assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0); + + // we expect full throttle fails, but cloud client should not easily fail + for (StoppableThread indexThread : threads) { + if (indexThread instanceof StoppableIndexingThread + && !(indexThread instanceof FullThrottleStoppableIndexingThread)) { + int failCount = ((StoppableIndexingThread) indexThread).getFailCount(); + assertFalse( + "There were too many update fails (" + + failCount + + " > " + + FAIL_TOLERANCE + + ") - we expect it can happen, but shouldn't easily", + failCount > FAIL_TOLERANCE); + } + } + + waitForThingsToLevelOut(20, TimeUnit.SECONDS); + + commit(); + + Set addFails = getAddFails(indexTreads); + Set deleteFails = getDeleteFails(indexTreads); + // full throttle thread can + // have request fails + checkShardConsistency(!runFullThrottle, true, addFails, deleteFails); + + long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + // ensure we have added more than 0 docs + long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0); + + if (VERBOSE) + System.out.println( + "control docs:" + + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + + "\n\n"); + + // try and make a collection to make sure the overseer has survived the expiration and session + // loss + + // sometimes we restart zookeeper as well + if (random().nextBoolean()) { + // restartZk(1000 * (5 + random().nextInt(4))); + } + + try (CloudSolrClient client = createCloudClient("collection1", 30000)) { + createCollection(null, "testcollection", 1, 1, client, null, "conf1"); + } + List numShardsNumReplicas = new ArrayList<>(2); + numShardsNumReplicas.add(1); + numShardsNumReplicas.add(1); + checkForCollection("testcollection", numShardsNumReplicas); + + testSuccessful = true; + } finally { + if (!testSuccessful) { + printLayout(); + } + } + } + + private Set getAddFails(List threads) { + Set addFails = new HashSet<>(); + for (StoppableIndexingThread thread : threads) { + addFails.addAll(thread.getAddFails()); + } + return addFails; + } + + private Set getDeleteFails(List threads) { + Set deleteFails = new HashSet<>(); + for (StoppableIndexingThread thread : threads) { + deleteFails.addAll(thread.getDeleteFails()); + } + return deleteFails; + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = getDoc(fields); + indexDoc(doc); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index a4c1ab267d1..3a35ee150fa 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -14,14 +14,218 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.solr.cloud; -/* - * Implementation moved to AbstractChaosMonkeySafeLeaderTestBase as is used by HDFS contrib module test - */ -public class ChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeaderTestBase { +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.common.SolrInputDocument; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@LuceneTestCase.Nightly +public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final Integer RUN_LENGTH = + Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); + + @BeforeClass + public static void beforeSuperClass() { + schemaString = "schema15.xml"; // we need a string id + System.setProperty("solr.autoCommit.maxTime", "15000"); + System.clearProperty("solr.httpclient.retries"); + System.clearProperty("solr.retries.on.forward"); + System.clearProperty("solr.retries.to.followers"); + setErrorHook(); + } + + @AfterClass + public static void afterSuperClass() { + System.clearProperty("solr.autoCommit.maxTime"); + clearErrorHook(); + } + + protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; + protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public RandVal[] getRandValues() { + return randVals; + } + + protected String getDirectoryFactory() { return "solr.StandardDirectoryFactory"; } + + @Override + public void distribSetUp() throws Exception { + useFactory(getDirectoryFactory()); + super.distribSetUp(); + } + + public ChaosMonkeySafeLeaderTest() { + super(); + sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); + if (sliceCount == -1) { + sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; + } + + int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); + if (numShards == -1) { + // we make sure that there's at least one shard with more than one replica + // so that the ChaosMonkey has something to kill + numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; + } + fixShardCount(numShards); + } + + @Test + public void test() throws Exception { + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + // randomly turn on 1 seconds 'soft' commit + randomlyEnableAutoSoftCommit(); + + tryDelete(); + + List threads = new ArrayList<>(); + int threadCount = 2; + int batchSize = 1; + if (random().nextBoolean()) { + batchSize = random().nextInt(98) + 2; + } + + boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true; + int maxUpdates = -1; + if (!pauseBetweenUpdates) { + maxUpdates = 1000 + random().nextInt(1000); + } else { + maxUpdates = 15000; + } + + for (int i = 0; i < threadCount; i++) { + StoppableIndexingThread indexThread = + new StoppableIndexingThread( + controlClient, + cloudClient, + Integer.toString(i), + true, + maxUpdates, + batchSize, + pauseBetweenUpdates); // random().nextInt(999) + 1 + threads.add(indexThread); + indexThread.start(); + } + + chaosMonkey.startTheMonkey(false, 500); + try { + long runLength; + if (RUN_LENGTH != -1) { + runLength = RUN_LENGTH; + } else { + int[] runTimes; + if (TEST_NIGHTLY) { + runTimes = + new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; + } else { + runTimes = new int[] {5000, 7000, 15000}; + } + runLength = runTimes[random().nextInt(runTimes.length - 1)]; + } + + Thread.sleep(runLength); + } finally { + chaosMonkey.stopTheMonkey(); + } + + for (StoppableIndexingThread indexThread : threads) { + indexThread.safeStop(); + } + + // wait for stop... + for (StoppableIndexingThread indexThread : threads) { + indexThread.join(); + } + + for (StoppableIndexingThread indexThread : threads) { + assertEquals(0, indexThread.getFailCount()); + } + + // try and wait for any replications and what not to finish... + + Thread.sleep(2000); + + waitForThingsToLevelOut(3, TimeUnit.MINUTES); + + // even if things were leveled out, a jetty may have just been stopped or something + // we wait again and wait to level out again to make sure the system is not still in flux + + Thread.sleep(3000); + + waitForThingsToLevelOut(3, TimeUnit.MINUTES); + + checkShardConsistency(batchSize == 1, true); + + if (VERBOSE) + System.out.println( + "control docs:" + + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + + "\n\n"); + + // try and make a collection to make sure the overseer has survived the expiration and session + // loss + + // sometimes we restart zookeeper as well + if (random().nextBoolean()) { + zkServer.shutdown(); + zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); + zkServer.run(false); + } + + try (CloudSolrClient client = createCloudClient("collection1")) { + createCollection(null, "testcollection", 1, 1, client, null, "conf1"); + } + List numShardsNumReplicas = new ArrayList<>(2); + numShardsNumReplicas.add(1); + numShardsNumReplicas.add(1); + checkForCollection("testcollection", numShardsNumReplicas); + } + + private void tryDelete() throws Exception { + long start = System.nanoTime(); + long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS); + while (System.nanoTime() < timeout) { + try { + del("*:*"); + break; + } catch (SolrServerException e) { + log.error("cluster may not be up yet", e); + } + Thread.sleep(100); + } + } + + // skip the randoms - they can deadlock... @Override - protected String getDirectoryFactory() { - return "solr.StandardDirectoryFactory"; + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index 265cbd4d1ea..d5dd04076fe 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -17,23 +17,394 @@ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.solr.client.api.model.CoreStatusResponse; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.IdUtils; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Implementation moved to AbstractMoveReplicaTestBase as it is referenced by HDFS contrib module - * tests. - */ -public class MoveReplicaTest extends AbstractMoveReplicaTestBase { +public class MoveReplicaTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + // used by MoveReplicaHDFSTest + protected boolean inPlaceMove = true; + protected boolean isCollectionApiDistributed = false; + + protected String getConfigSet() { + return "cloud-dynamic"; + } + + @Before + public void beforeTest() throws Exception { + inPlaceMove = true; + + configureCluster(4) + .addConfig("conf1", configset(getConfigSet())) + .addConfig("conf2", configset(getConfigSet())) + .withSolrXml(TEST_PATH().resolve("solr.xml")) + .configure(); + + // If Collection API is distributed let's not wait for Overseer. + isCollectionApiDistributed = + new CollectionAdminRequest.RequestApiDistributedProcessing() + .process(cluster.getSolrClient()) + .getIsCollectionApiDistributed(); + if (isCollectionApiDistributed) { + return; + } + + NamedList overSeerStatus = + cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); + JettySolrRunner overseerJetty = null; + String overseerLeader = (String) overSeerStatus.get("leader"); + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + if (jetty.getNodeName().equals(overseerLeader)) { + overseerJetty = jetty; + break; + } + } + if (overseerJetty == null) { + fail("no overseer leader!"); + } + } + + @After + public void afterTest() throws Exception { + try { + shutdownCluster(); + } finally { + super.tearDown(); + } + } - @Override @Test public void test() throws Exception { - super.test(); + String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove; + if (log.isInfoEnabled()) { + log.info("total_jettys: {}", cluster.getJettySolrRunners().size()); + } + int REPLICATION = 2; + + CloudSolrClient cloudClient = cluster.getSolrClient(); + + // random create tlog or pull type replicas with nrt + boolean isTlog = random().nextBoolean(); + CollectionAdminRequest.Create create = + CollectionAdminRequest.createCollection( + coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); + cloudClient.request(create); + + addDocs(coll, 100); + + Replica replica = getRandomReplica(coll, cloudClient); + Set liveNodes = cloudClient.getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + String targetNode = null; + for (String node : liveNodes) { + if (!replica.getNodeName().equals(node)) { + targetNode = node; + break; + } + } + assertNotNull(targetNode); + String shardId = null; + for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) { + if (slice.getReplicas().contains(replica)) { + shardId = slice.getName(); + } + } + + int sourceNumCores = + getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()); + int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()); + + CollectionAdminRequest.MoveReplica moveReplica = + createMoveReplicaRequest(coll, replica, targetNode); + moveReplica.setInPlaceMove(inPlaceMove); + String asyncId = IdUtils.randomId(); + moveReplica.processAsync(asyncId, cloudClient); + CollectionAdminRequest.RequestStatus requestStatus = + CollectionAdminRequest.requestStatus(asyncId); + // wait for async request success + boolean success = false; + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + success = true; + break; + } + assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED); + Thread.sleep(500); + } + assertTrue(success); + assertEquals( + "should be one less core on the source node!", + sourceNumCores - 1, + getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name())); + assertEquals( + "should be one more core on target node!", + targetNumCores + 1, + getNumOfCores(cloudClient, targetNode, coll, replica.getType().name())); + // wait for recovery + boolean recovered = false; + for (int i = 0; i < 300; i++) { + DocCollection collState = getCollectionState(coll); + log.debug("###### {}", collState); + Collection replicas = collState.getSlice(shardId).getReplicas(); + boolean allActive = true; + boolean hasLeaders = true; + if (replicas != null && !replicas.isEmpty()) { + for (Replica r : replicas) { + if (!r.getNodeName().equals(targetNode)) { + continue; + } + if (!r.isActive(Collections.singleton(targetNode))) { + log.info("Not active: {}", r); + allActive = false; + } + } + } else { + allActive = false; + } + for (Slice slice : collState.getSlices()) { + if (slice.getLeader() == null) { + hasLeaders = false; + } + } + if (allActive && hasLeaders) { + // check the number of active replicas + assertEquals("total number of replicas", REPLICATION, replicas.size()); + recovered = true; + break; + } else { + log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders); + Thread.sleep(1000); + } + } + assertTrue("replica never fully recovered", recovered); + + assertEquals( + 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); + + moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId); + moveReplica.setInPlaceMove(inPlaceMove); + moveReplica.process(cloudClient); + checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores); + // wait for recovery + recovered = false; + for (int i = 0; i < 300; i++) { + DocCollection collState = getCollectionState(coll); + log.debug("###### {}", collState); + Collection replicas = collState.getSlice(shardId).getReplicas(); + boolean allActive = true; + boolean hasLeaders = true; + if (replicas != null && !replicas.isEmpty()) { + for (Replica r : replicas) { + if (!r.getNodeName().equals(replica.getNodeName())) { + continue; + } + if (!r.isActive(Collections.singleton(replica.getNodeName()))) { + log.info("Not active yet: {}", r); + allActive = false; + } + } + } else { + allActive = false; + } + for (Slice slice : collState.getSlices()) { + if (slice.getLeader() == null) { + hasLeaders = false; + } + } + if (allActive && hasLeaders) { + assertEquals("total number of replicas", REPLICATION, replicas.size()); + recovered = true; + break; + } else { + Thread.sleep(1000); + } + } + assertTrue("replica never fully recovered", recovered); + + assertEquals( + 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); } - @Override @Test public void testFailedMove() throws Exception { - super.testFailedMove(); + String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove; + int REPLICATION = 2; + + CloudSolrClient cloudClient = cluster.getSolrClient(); + + // random create tlog or pull type replicas with nrt + boolean isTlog = random().nextBoolean(); + CollectionAdminRequest.Create create = + CollectionAdminRequest.createCollection( + coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); + cloudClient.request(create); + + addDocs(coll, 100); + + NamedList overSeerStatus = + cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); + String overseerLeader = (String) overSeerStatus.get("leader"); + + // don't kill overseer in this test + Replica replica; + int count = 10; + do { + replica = getRandomReplica(coll, cloudClient); + } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0); + assertNotNull("could not find non-overseer replica???", replica); + Set liveNodes = cloudClient.getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + String targetNode = null; + for (String node : liveNodes) { + if (!replica.getNodeName().equals(node) + && (isCollectionApiDistributed || !overseerLeader.equals(node))) { + targetNode = node; + break; + } + } + assertNotNull(targetNode); + CollectionAdminRequest.MoveReplica moveReplica = + createMoveReplicaRequest(coll, replica, targetNode); + moveReplica.setInPlaceMove(inPlaceMove); + // start moving + String asyncId = IdUtils.randomId(); + moveReplica.processAsync(asyncId, cloudClient); + // shut down target node + for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { + if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) { + JettySolrRunner j = cluster.stopJettySolrRunner(i); + cluster.waitForJettyToStop(j); + break; + } + } + CollectionAdminRequest.RequestStatus requestStatus = + CollectionAdminRequest.requestStatus(asyncId); + // wait for async request success + boolean success = true; + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + assertNotSame( + rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED); + if (rsp.getRequestStatus() == RequestStatusState.FAILED) { + success = false; + break; + } + Thread.sleep(500); + } + assertFalse(success); + + if (log.isInfoEnabled()) { + log.info( + "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll)); + } + assertEquals( + 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); + } + + private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( + String coll, Replica replica, String targetNode, String shardId) { + return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName()); + } + + private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( + String coll, Replica replica, String targetNode) { + return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode); + } + + private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException { + List replicas = cloudClient.getClusterState().getCollection(coll).getReplicas(); + Collections.shuffle(replicas, random()); + return replicas.get(0); + } + + private void checkNumOfCores( + CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores) + throws IOException, SolrServerException { + assertEquals( + nodeName + " does not have expected number of cores", + expectedCores, + getNumOfCores(cloudClient, nodeName, collectionName)); + } + + private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName) + throws IOException, SolrServerException { + return getNumOfCores(cloudClient, nodeName, collectionName, null); + } + + private int getNumOfCores( + CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType) + throws IOException, SolrServerException { + try (SolrClient coreclient = + getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) { + CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient); + if (status.getCoreStatus().size() == 0) { + return 0; + } + if (collectionName == null && replicaType == null) { + return status.getCoreStatus().size(); + } + // filter size by collection name + int size = 0; + for (Map.Entry coreStatusEntry : + status.getCoreStatus().entrySet()) { + if (collectionName != null) { + String coll = coreStatusEntry.getValue().cloud.collection; + if (!collectionName.equals(coll)) { + continue; + } + } + if (replicaType != null) { + String type = coreStatusEntry.getValue().cloud.replicaType; + if (!replicaType.equals(type)) { + continue; + } + } + size++; + } + return size; + } + } + + protected void addDocs(String collection, int numDocs) throws Exception { + SolrClient solrClient = cluster.getSolrClient(); + for (int docId = 1; docId <= numDocs; docId++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", docId); + solrClient.add(collection, doc); + } + solrClient.commit(collection); + Thread.sleep(5000); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java index b0188069562..4c0ad7af750 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java @@ -122,7 +122,7 @@ public void testOutOfBoxSolrZkClient() throws Exception { .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) .build(); try { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, true, true, true, true, true, true, true, true); } finally { zkClient.close(); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java index 5810c50fe17..2f3f2201ede 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java @@ -153,7 +153,7 @@ public void testNoCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() th try (SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -165,7 +165,7 @@ public void testWrongCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() new SolrZkClientFactoryUsingCompletelyNewProviders( "connectAndAllACLUsername", "connectAndAllACLPasswordWrong", null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -176,7 +176,7 @@ public void testAllCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() t new SolrZkClientFactoryUsingCompletelyNewProviders( "connectAndAllACLUsername", "connectAndAllACLPassword", null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, true, true, true, true, true, true, true, true); } } @@ -188,7 +188,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClientFactoryUsingCompletelyNewProviders( "readonlyACLUsername", "readonlyACLPassword", null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, false, false, false, false, false, false, false, false); } } @@ -204,7 +204,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -220,7 +220,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -236,7 +236,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, true, true, true, true, true, true, true, true); } } @@ -252,7 +252,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, false, false, false, false, false, false, false, false); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index 6394eb566e0..dcf97db350c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -16,16 +16,143 @@ */ package org.apache.solr.cloud; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.After; +import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/* - * Implementation moved to AbstractRecoveryZkTestBase as it is used by HDFS contrib tests. - */ -public class RecoveryZkTest extends AbstractRecoveryZkTestBase { +public class RecoveryZkTest extends SolrCloudTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() throws Exception { + cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure(); + } + + private final List threads = new ArrayList<>(); + + @After + public void stopThreads() throws InterruptedException { + for (StoppableIndexingThread t : threads) { + t.safeStop(); + } + for (StoppableIndexingThread t : threads) { + t.join(); + } + threads.clear(); + } @Test - @Override public void test() throws Exception { - super.test(); + + final String collection = "recoverytest"; + + CollectionAdminRequest.createCollection(collection, "conf", 1, 2) + .process(cluster.getSolrClient()); + waitForState( + "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2)); + + // start a couple indexing threads + + int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000}; + int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000}; + + int maxDoc; + if (!TEST_NIGHTLY) { + maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; + } else { + maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)]; + } + log.info("Indexing {} documents", maxDoc); + + try (SolrClient solrClient = + cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) { + final StoppableIndexingThread indexThread = + new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true); + threads.add(indexThread); + indexThread.start(); + + final StoppableIndexingThread indexThread2 = + new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true); + threads.add(indexThread2); + indexThread2.start(); + + // give some time to index... + int[] waitTimes = new int[] {200, 2000, 3000}; + Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); + + // bring shard replica down + DocCollection state = getCollectionState(collection); + Replica leader = state.getLeader("shard1"); + Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r)); + + JettySolrRunner jetty = cluster.getReplicaJetty(replica); + jetty.stop(); + + // wait a moment - lets allow some docs to be indexed so replication time is non 0 + Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); + + // bring shard replica up + jetty.start(); + + // make sure replication can start + Thread.sleep(3000); + + // stop indexing threads + indexThread.safeStop(); + indexThread2.safeStop(); + + indexThread.join(); + indexThread2.join(); + + new UpdateRequest().commit(solrClient, collection); + + cluster + .getZkStateReader() + .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2)); + + // test that leader and replica have same doc count + state = getCollectionState(collection); + assertShardConsistency(state.getSlice("shard1"), true); + } + } + + private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception { + List replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE); + long[] numCounts = new long[replicas.size()]; + int i = 0; + for (Replica replica : replicas) { + try (var client = + new HttpSolrClient.Builder(replica.getBaseUrl()) + .withDefaultCollection(replica.getCoreName()) + .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient()) + .build()) { + numCounts[i] = + client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound(); + i++; + } + } + for (int j = 1; j < replicas.size(); j++) { + if (numCounts[j] != numCounts[j - 1]) + fail("Mismatch in counts between replicas"); // TODO improve this! + if (numCounts[j] == 0 && expectDocs) + fail("Expected docs on shard " + shard.getName() + " but found none"); + } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java index bdff2a6a477..90533fa835d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java @@ -16,8 +16,285 @@ */ package org.apache.solr.cloud; -/** - * Test sync phase that occurs when Leader goes down and a new Leader is elected. Implementation - * moved to AbstractSyncSliceTestBase.java as it is also used by the HDFS contrib tests. - */ -public class SyncSliceTest extends AbstractSyncSliceTestBase {} +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.util.LogLevel; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */ +@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN") +public class SyncSliceTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private boolean success = false; + + @Override + public void distribTearDown() throws Exception { + if (!success) { + printLayoutOnTearDown = true; + } + super.distribTearDown(); + } + + public SyncSliceTest() { + super(); + sliceCount = 1; + fixShardCount(TEST_NIGHTLY ? 7 : 4); + } + + @Test + public void test() throws Exception { + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + waitForThingsToLevelOut(30, TimeUnit.SECONDS); + + del("*:*"); + List skipServers = new ArrayList<>(); + int docId = 0; + indexDoc( + skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); + + indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue."); + + skipServers.add(shardToJetty.get("shard1").get(1)); + + indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy."); + + skipServers.add(shardToJetty.get("shard1").get(2)); + + indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake"); + + commit(); + + waitForRecoveriesToFinish(false); + + // shard should be inconsistent + String shardFailMessage = checkShardConsistency("shard1", true, false); + assertNotNull(shardFailMessage); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.SYNCSHARD.toString()); + params.set("collection", "collection1"); + params.set("shard", "shard1"); + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params); + + String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString(); + + // we only set the connect timeout, not so timeout + try (SolrClient baseClient = + new HttpSolrClient.Builder(baseUrl) + .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + baseClient.request(request); + } + + waitForThingsToLevelOut(15, TimeUnit.SECONDS); + + checkShardConsistency(false, true); + + long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + assertEquals(4, cloudClientDocs); + + // kill the leader - new leader could have all the docs or be missing one + CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1"); + + skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader + + // this doc won't be on one node + indexDoc( + skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); + commit(); + + Set jetties = new HashSet<>(); + jetties.addAll(shardToJetty.get("shard1")); + jetties.remove(leaderJetty); + assertEquals(getShardCount() - 1, jetties.size()); + + leaderJetty.jetty.stop(); + + Thread.sleep(3000); + + waitForNoShardInconsistency(); + + Thread.sleep(1000); + + checkShardConsistency(false, true); + + cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + assertEquals(5, cloudClientDocs); + + CloudJettyRunner deadJetty = leaderJetty; + + // let's get the latest leader + while (deadJetty.equals(leaderJetty)) { + updateMappingsFromZk(this.jettys, this.clients); + leaderJetty = shardToLeaderJetty.get("shard1"); + } + + // bring back dead node + deadJetty.jetty.start(); // he is not the leader anymore + + waitTillAllNodesActive(); + + skipServers = getRandomOtherJetty(leaderJetty, deadJetty); + skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty)); + // skip list should be + + // System.out.println("leader:" + leaderJetty.url); + // System.out.println("dead:" + deadJetty.url); + // System.out.println("skip list:" + skipServers); + + // we are skipping 2 nodes + assertEquals(2, skipServers.size()); + + // more docs than can peer sync + for (int i = 0; i < 300; i++) { + indexDoc( + skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); + } + + commit(); + waitForRecoveriesToFinish(false); + + // shard should be inconsistent + shardFailMessage = waitTillInconsistent(); + assertNotNull( + "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:" + + leaderJetty.url + + " Dead Guy:" + + deadJetty.url + + "skip list:" + + skipServers, + shardFailMessage); + + // good place to test compareResults + boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient); + assertTrue("A test that compareResults is working correctly failed", shouldFail); + + jetties = new HashSet<>(); + jetties.addAll(shardToJetty.get("shard1")); + jetties.remove(leaderJetty); + assertEquals(getShardCount() - 1, jetties.size()); + + // kill the current leader + leaderJetty.jetty.stop(); + + waitForNoShardInconsistency(); + + checkShardConsistency(true, true); + + success = true; + } + + private void waitTillAllNodesActive() throws InterruptedException, TimeoutException { + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + + zkStateReader.waitForState( + "collection1", + 3, + TimeUnit.MINUTES, + (n, c) -> { + Collection replicas = c.getSlice("shard1").getReplicas(); + Set nodes = + replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet()); + return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals) + && n.containsAll(nodes); + }); + } + + private String waitTillInconsistent() throws Exception, InterruptedException { + String shardFailMessage = null; + + shardFailMessage = pollConsistency(shardFailMessage, 0); + shardFailMessage = pollConsistency(shardFailMessage, 3000); + shardFailMessage = pollConsistency(shardFailMessage, 5000); + shardFailMessage = pollConsistency(shardFailMessage, 15000); + + return shardFailMessage; + } + + private String pollConsistency(String shardFailMessage, int sleep) + throws InterruptedException, Exception { + try { + commit(); + } catch (Throwable t) { + log.error("commit error", t); + } + if (shardFailMessage == null) { + // try again + Thread.sleep(sleep); + shardFailMessage = checkShardConsistency("shard1", true, false); + } + return shardFailMessage; + } + + private List getRandomOtherJetty( + CloudJettyRunner leader, CloudJettyRunner down) { + List skipServers = new ArrayList<>(); + List candidates = new ArrayList<>(); + candidates.addAll(shardToJetty.get("shard1")); + + if (leader != null) { + candidates.remove(leader); + } + + if (down != null) { + candidates.remove(down); + } + + CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size())); + skipServers.add(cjetty); + return skipServers; + } + + protected void indexDoc(List skipServers, Object... fields) + throws IOException, SolrServerException { + SolrInputDocument doc = new SolrInputDocument(); + + addFields(doc, fields); + addFields(doc, "rnd_b", true); + + controlClient.add(doc); + + UpdateRequest ureq = new UpdateRequest(); + ureq.add(doc); + for (CloudJettyRunner skip : skipServers) { + ureq.getParams().add("test.distrib.skip.servers", skip.url + "/"); + } + ureq.process(cloudClient); + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java index e056c40a893..8872881f6d7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java @@ -16,26 +16,135 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; import org.apache.lucene.tests.util.LuceneTestCase.Nightly; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; -import org.junit.Test; @Nightly @SuppressSSL -/* - * Implementation moved to AbstractTlogReplayBufferedWhileIndexingTestBase as it is also used by HDFS contrib - * module tests - */ public class TlogReplayBufferedWhileIndexingTest - extends AbstractTlogReplayBufferedWhileIndexingTestBase { + extends AbstractFullDistribZkTestBase { + + private List threads; public TlogReplayBufferedWhileIndexingTest() throws Exception { super(); + sliceCount = 1; + fixShardCount(2); + schemaString = "schema15.xml"; // we need a string id + } + + @BeforeClass + public static void beforeRestartWhileUpdatingTest() throws Exception { + System.setProperty("leaderVoteWait", "300000"); + System.setProperty("solr.autoCommit.maxTime", "10000"); + System.setProperty("solr.autoSoftCommit.maxTime", "3000"); + TestInjection.updateLogReplayRandomPause = "true:10"; + TestInjection.updateRandomPause = "true:10"; + useFactory("solr.StandardDirectoryFactory"); + } + + @AfterClass + public static void afterRestartWhileUpdatingTest() { + System.clearProperty("leaderVoteWait"); + System.clearProperty("solr.autoCommit.maxTime"); + System.clearProperty("solr.autoSoftCommit.maxTime"); } @Test - @Override public void test() throws Exception { - super.test(); + handle.clear(); + handle.put("timestamp", SKIPVAL); + + waitForRecoveriesToFinish(false); + + int numThreads = 3; + + threads = new ArrayList<>(numThreads); + + ArrayList allJetty = new ArrayList<>(); + allJetty.addAll(jettys); + allJetty.remove(shardToLeaderJetty.get("shard1").jetty); + assert allJetty.size() == 1 : allJetty.size(); + allJetty.get(0).stop(); + + StoppableIndexingThread indexThread; + for (int i = 0; i < numThreads; i++) { + boolean pauseBetweenUpdates = random().nextBoolean(); + int batchSize = random().nextInt(4) + 1; + indexThread = + new StoppableIndexingThread( + controlClient, + cloudClient, + Integer.toString(i), + true, + 900, + batchSize, + pauseBetweenUpdates); + threads.add(indexThread); + indexThread.start(); + } + + Thread.sleep(2000); + + allJetty.get(0).start(); + + Thread.sleep(45000); + + waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while, + // especially when beasting this test + + Thread.sleep(2000); + + waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); + + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + + waitForThingsToLevelOut(30, TimeUnit.SECONDS); + + checkShardConsistency(false, false); + } + + @Override + protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { + cloudClient.add(doc); + } + + @Override + public void distribTearDown() throws Exception { + // make sure threads have been stopped... + if (threads != null) { + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + + for (StoppableIndexingThread thread : threads) { + thread.join(); + } + } + + super.distribTearDown(); + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 30d3f884b97..81e573aae54 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -16,12 +16,450 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrPaths; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. Implementation moved to AbstractUnloadDistributedZkTestBase as it is used by HDFS - * contrib module tests. + * expected. */ @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class UnloadDistributedZkTest extends AbstractUnloadDistributedZkTestBase {} +public class UnloadDistributedZkTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public UnloadDistributedZkTest() { + super(); + fixShardCount(4); // needs at least 4 servers + } + + @Override + protected String getSolrXml() { + return "solr.xml"; + } + + @Test + public void test() throws Exception { + jettys.forEach( + j -> { + Set allowPath = j.getCoreContainer().getAllowPaths(); + allowPath.clear(); + allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path + }); + log.info("###Starting testCoreUnloadAndLeaders"); + testCoreUnloadAndLeaders(); // long + log.info("###Starting testUnloadOfCores"); + testUnloadLotsOfCores(); // long + log.info("###Starting testUnloadShardAndCollection"); + testUnloadShardAndCollection(); + } + + /** + * @param url a Solr node base URL. Should not contain a core or collection name. + */ + private SolrClient newSolrClient(String url) { + return new HttpSolrClient.Builder(url) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(30000, TimeUnit.MILLISECONDS) + .build(); + } + + private void checkCoreNamePresenceAndSliceCount( + String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount) + throws Exception { + ZkStateReader reader = ZkStateReader.from(cloudClient); + try { + reader.waitForState( + collectionName, + 45, + TimeUnit.SECONDS, + c -> { + final Collection slices = (c != null) ? c.getSlices() : Collections.emptyList(); + if (expectedSliceCount == slices.size()) { + for (Slice slice : slices) { + for (Replica replica : slice.getReplicas()) { + if (coreName.equals(replica.get("core"))) { + return shouldBePresent; + } + } + } + return !shouldBePresent; + } else { + return false; + } + }); + } catch (TimeoutException e) { + printLayout(); + fail( + "checkCoreNamePresenceAndSliceCount failed:" + + " collection=" + + collectionName + + " CoreName=" + + coreName + + " shouldBePresent=" + + shouldBePresent); + } + } + + private void testUnloadShardAndCollection() throws Exception { + final int numShards = 2; + + final String collection = "test_unload_shard_and_collection"; + + final String coreName1 = collection + "_1"; + final String coreName2 = collection + "_2"; + + assertEquals( + 0, + CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard(collection, "shard1") + .setCoreName(coreName1) + .setNode(jettys.get(0).getNodeName()) + .process(cloudClient) + .isSuccess()); + + assertTrue( + CollectionAdminRequest.addReplicaToShard(collection, "shard2") + .setCoreName(coreName2) + .setNode(jettys.get(0).getNodeName()) + .process(cloudClient) + .isSuccess()); + + // does not mean they are active and up yet :* + waitForRecoveriesToFinish(collection, false); + + final boolean unloadInOrder = random().nextBoolean(); + final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2); + final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1); + + try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) { + // now unload one of the two + Unload unloadCmd = new Unload(false); + unloadCmd.setCoreName(unloadCmdCoreName1); + adminClient.request(unloadCmd); + + // there should still be two shards (as of SOLR-5209) + checkCoreNamePresenceAndSliceCount( + collection, + unloadCmdCoreName1, + false /* shouldBePresent */, + numShards /* expectedSliceCount */); + + // now unload one of the other + unloadCmd = new Unload(false); + unloadCmd.setCoreName(unloadCmdCoreName2); + adminClient.request(unloadCmd); + checkCoreNamePresenceAndSliceCount( + collection, + unloadCmdCoreName2, + false /* shouldBePresent */, + numShards /* expectedSliceCount */); + } + + // printLayout(); + // the collection should still be present (as of SOLR-5209 replica removal does not cascade to + // remove the slice and collection) + getCommonCloudSolrClient(); + assertTrue( + "No longer found collection " + collection, + cloudClient.getClusterState().hasCollection(collection)); + } + + protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) { + SolrCore solrCore = null; + for (SolrCore core : jetty.getCoreContainer().getCores()) { + if (core.getName().startsWith(collection)) { + solrCore = core; + } + } + return solrCore; + } + + /** + * @throws Exception on any problem + */ + private void testCoreUnloadAndLeaders() throws Exception { + JettySolrRunner jetty1 = jettys.get(0); + + assertEquals( + 0, + CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1) + .setCreateNodeSet(jetty1.getNodeName()) + .process(cloudClient) + .getStatus()); + getCommonCloudSolrClient(); + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + + zkStateReader.forceUpdateCollection("unloadcollection"); + + int slices = + zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); + assertEquals(1, slices); + SolrCore solrCore = getFirstCore("unloadcollection", jetty1); + String core1DataDir = solrCore.getDataDir(); + + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName("unloadcollection_shard1_replica2") + .setNode(jettys.get(1).getNodeName()) + .process(cloudClient) + .isSuccess()); + zkStateReader.forceUpdateCollection("unloadcollection"); + slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); + assertEquals(1, slices); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + Replica leader = getLeaderFromZk("unloadcollection", "shard1"); + + Random random = random(); + if (random.nextBoolean()) { + try (SolrClient collectionClient = + getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) { + // lets try and use the solrj client to index and retrieve a couple + // documents + SolrInputDocument doc1 = + getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); + SolrInputDocument doc2 = + getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls"); + SolrInputDocument doc3 = + getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled"); + collectionClient.add(doc1); + collectionClient.add(doc2); + collectionClient.add(doc3); + collectionClient.commit(); + } + } + + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName("unloadcollection_shard1_replica3") + .setNode(jettys.get(2).getNodeName()) + .process(cloudClient) + .isSuccess()); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + // so that we start with some versions when we reload... + TestInjection.skipIndexWriterCommitOnClose = true; + + try (SolrClient addClient = + new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) + .withDefaultCollection("unloadcollection_shard1_replica3") + .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + + // add a few docs + for (int x = 20; x < 100; x++) { + SolrInputDocument doc1 = + getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); + addClient.add(doc1); + } + } + // don't commit so they remain in the tran log + // collectionClient.commit(); + + // unload the leader + try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { + + Unload unloadCmd = new Unload(false); + unloadCmd.setCoreName(leader.getCoreName()); + ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams(); + + collectionClient.request(unloadCmd); + } + // Thread.currentThread().sleep(500); + // printLayout(); + + int tries = 50; + while (leader + .getCoreUrl() + .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { + Thread.sleep(100); + if (tries-- == 0) { + fail("Leader never changed"); + } + } + + // ensure there is a leader + zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); + + try (SolrClient addClient = + new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString()) + .withDefaultCollection("unloadcollection_shard1_replica2") + .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) + .withSocketTimeout(90000, TimeUnit.MILLISECONDS) + .build()) { + + // add a few docs while the leader is down + for (int x = 101; x < 200; x++) { + SolrInputDocument doc1 = + getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); + addClient.add(doc1); + } + } + + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName("unloadcollection_shard1_replica4") + .setNode(jettys.get(3).getNodeName()) + .process(cloudClient) + .isSuccess()); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + // unload the leader again + leader = getLeaderFromZk("unloadcollection", "shard1"); + try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { + + Unload unloadCmd = new Unload(false); + unloadCmd.setCoreName(leader.getCoreName()); + collectionClient.request(unloadCmd); + } + tries = 50; + while (leader + .getCoreUrl() + .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { + Thread.sleep(100); + if (tries-- == 0) { + fail("Leader never changed"); + } + } + + zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); + + TestInjection.skipIndexWriterCommitOnClose = false; // set this back + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName(leader.getCoreName()) + .setDataDir(core1DataDir) + .setNode(leader.getNodeName()) + .process(cloudClient) + .isSuccess()); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + long found1, found3; + + try (SolrClient adminClient = + newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) { + adminClient.commit(); + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + found1 = adminClient.query(q).getResults().getNumFound(); + } + + try (SolrClient adminClient = + new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) + .withDefaultCollection("unloadcollection_shard1_replica3") + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + adminClient.commit(); + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + found3 = adminClient.query(q).getResults().getNumFound(); + } + + try (SolrClient adminClient = + newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) { + adminClient.commit(); + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + long found4 = adminClient.query(q).getResults().getNumFound(); + + // all 3 shards should now have the same number of docs + assertEquals(found1, found3); + assertEquals(found3, found4); + } + } + + private void testUnloadLotsOfCores() throws Exception { + JettySolrRunner jetty = jettys.get(0); + int shards = TEST_NIGHTLY ? 2 : 1; + try (final SolrClient adminClient = jetty.newClient(15000, 60000)) { + int numReplicas = atLeast(3); + ThreadPoolExecutor executor = + new ExecutorUtil.MDCAwareThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 5, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new SolrNamedThreadFactory("testExecutor")); + try { + // create the cores + BasicDistributedZkTest.createCollectionInOneInstance( + adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas); + } finally { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + + if (TEST_NIGHTLY == false) { + // with nightly tests, we can try doing the unloads before the creates are done + // it still works, but takes much longer since we end up waiting for a timeout + waitForRecoveriesToFinish("multiunload", false); + } + + executor = + new ExecutorUtil.MDCAwareThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 5, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new SolrNamedThreadFactory("testExecutor")); + try { + for (int j = 0; j < numReplicas; j++) { + final int freezeJ = j; + executor.execute( + () -> { + Unload unloadCmd = new Unload(true); + unloadCmd.setCoreName("multiunload" + freezeJ); + try { + adminClient.request(unloadCmd); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + }); + Thread.sleep(random().nextInt(50)); + } + } finally { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java index 82b60068d83..aca50b7bd3a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java @@ -16,5 +16,552 @@ */ package org.apache.solr.cloud; -public class VMParamsZkACLAndCredentialsProvidersTest - extends AbstractDigestZkACLAndCredentialsProvidersTestBase {} +import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME; +import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE; + +import java.io.FileWriter; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.DigestZkACLProvider; +import org.apache.solr.common.cloud.DigestZkCredentialsProvider; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector; +import org.apache.solr.common.cloud.ZkCredentialsInjector; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException.NoAuthException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final Charset DATA_ENCODING = StandardCharsets.UTF_8; + + private static final String ALL_USERNAME = "connectAndAllACLUsername"; + private static final String ALL_PASSWORD = "connectAndAllACLPassword"; + private static final String READONLY_USERNAME = "readonlyACLUsername"; + private static final String READONLY_PASSWORD = "readonlyACLPassword"; + + protected ZkTestServer zkServer; + + protected Path zkDir; + + @BeforeClass + public static void beforeClass() { + System.setProperty("solrcloud.skip.autorecovery", "true"); + } + + @AfterClass + public static void afterClass() { + System.clearProperty("solrcloud.skip.autorecovery"); + } + + @Override + public void setUp() throws Exception { + // TODO: Does all of this setup need to happen for each test case, or can it be done once for + // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full + // startup and teardown of a new ZkTestServer in each cycle? + super.setUp(); + if (log.isInfoEnabled()) { + log.info("####SETUP_START {}", getTestName()); + } + createTempDir(); + + zkDir = createTempDir().resolve("zookeeper/server1/data"); + log.info("ZooKeeper dataDir:{}", zkDir); + setSecuritySystemProperties(); + zkServer = new ZkTestServer(zkDir); + zkServer.run(false); + + System.setProperty("zkHost", zkServer.getZkAddress()); + + setDigestZkSystemProps(); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); + + SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkHost()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + + zkClient.makePath("/solr", false, true); + zkClient.close(); + + zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + zkClient.create( + "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath( + "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + + zkClient.create( + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, + "content".getBytes(DATA_ENCODING), + CreateMode.PERSISTENT, + false); + zkClient.close(); + + clearSecuritySystemProperties(); + + zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + // Currently, no credentials on ZK connection, because those same VM-params are used for adding + // ACLs, and here we want + // no (or completely open) ACLs added. Therefore, hack your way into being authorized for + // creating anyway + zkClient + .getCuratorFramework() + .getZookeeperClient() + .getZooKeeper() + .addAuthInfo( + "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8)); + zkClient.create( + "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath( + "/unprotectedMakePathNode", + "content".getBytes(DATA_ENCODING), + CreateMode.PERSISTENT, + false); + zkClient.close(); + + setDigestZkSystemProps(); + if (log.isInfoEnabled()) { + log.info("####SETUP_END {}", getTestName()); + } + } + + private void setDigestZkSystemProps() { + System.setProperty( + SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + DigestZkCredentialsProvider.class.getName()); + System.setProperty( + SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); + } + + @Override + public void tearDown() throws Exception { + zkServer.shutdown(); + + clearSecuritySystemProperties(); + + super.tearDown(); + } + + @Test + public void testNoCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class), + new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class)); + + testInjectors( + testZkCredentialsInjectors, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false); + } + + @Test + public void testWrongCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class), + new TestZkCredentialsInjector( + VMParamsZkCredentialsInjector.class, + List.of( + DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), + List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong"))); + + testInjectors( + testZkCredentialsInjectors, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false); + } + + @Test + public void testAllCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class), + new TestZkCredentialsInjector( + VMParamsZkCredentialsInjector.class, + List.of( + DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), + List.of(ALL_USERNAME, ALL_PASSWORD))); + + testInjectors( + testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true); + } + + @Test + public void testReadonlyCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class), + new TestZkCredentialsInjector( + VMParamsZkCredentialsInjector.class, + List.of( + DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), + List.of(READONLY_USERNAME, READONLY_PASSWORD))); + testInjectors( + testZkCredentialsInjectors, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false); + } + + protected void testInjectors( + List testZkCredentialsInjectors, + boolean getData, + boolean list, + boolean create, + boolean setData, + boolean delete, + boolean secureGet, + boolean secureList, + boolean secureCreate, + boolean secureSet, + boolean secureDelete) + throws Exception { + for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) { + tearDown(); + setUp(); + testZkCredentialsInjector.setSystemProps(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + doTest( + zkClient, + getData, + list, + create, + setData, + delete, + secureGet, + secureList, + secureCreate, + secureSet, + secureDelete); + } + } + } + + @Test + public void testRepairACL() throws Exception { + clearSecuritySystemProperties(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + // Currently, no credentials on ZK connection, because those same VM-params are used for + // adding ACLs, and here we want + // no (or completely open) ACLs added. Therefore, hack your way into being authorized for + // creating anyway + zkClient + .getCuratorFramework() + .getZookeeperClient() + .getZooKeeper() + .addAuthInfo( + "digest", + ("connectAndAllACLUsername:connectAndAllACLPassword") + .getBytes(StandardCharsets.UTF_8)); + + zkClient.create( + "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false); + assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); + } + + setSecuritySystemProperties(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + ZkController.createClusterZkNodes(zkClient); + assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); + } + + useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class); + // useReadonlyCredentials(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + NoAuthException e = + assertThrows( + NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false)); + assertEquals("/solr/security.json", e.getPath()); + } + } + + private void useZkCredentialsInjector(Class zkCredentialsInjectorClass) { + clearSecuritySystemProperties(); + setDigestZkSystemProps(); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + zkCredentialsInjectorClass.getName()); + } + + private void setSecuritySystemProperties() { + System.setProperty( + SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + DigestZkCredentialsProvider.class.getName()); + System.setProperty( + SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); + } + + private void clearSecuritySystemProperties() { + System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME); + } + + public static void doTest( + SolrZkClient zkClient, + boolean getData, + boolean list, + boolean create, + boolean setData, + boolean delete, + boolean secureGet, + boolean secureList, + boolean secureCreate, + boolean secureSet, + boolean secureDelete) + throws Exception { + doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete); + doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete); + doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete); + doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete); + doTest( + zkClient, + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, + secureGet, + secureList, + secureCreate, + secureSet, + secureDelete); + } + + protected static void doTest( + SolrZkClient zkClient, + String path, + boolean getData, + boolean list, + boolean create, + boolean setData, + boolean delete) + throws Exception { + doTest(getData, () -> zkClient.getData(path, null, null, false)); + doTest(list, () -> zkClient.getChildren(path, null, false)); + + doTest( + create, + () -> { + zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false); + zkClient.delete(path + "/subnode", -1, false); + }); + doTest( + create, + () -> { + zkClient.makePath(path + "/subnode/subsubnode", false); + zkClient.delete(path + "/subnode/subsubnode", -1, false); + zkClient.delete(path + "/subnode", -1, false); + }); + + doTest(setData, () -> zkClient.setData(path, (byte[]) null, false)); + + // Actually about the ACLs on /solr, but that is protected + doTest(delete, () -> zkClient.delete(path, -1, false)); + } + + interface ExceptingRunnable { + void run() throws Exception; + } + + private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception { + if (shouldSucceed) { + action.run(); + } else { + expectThrows(NoAuthException.class, action::run); + } + } + + @Test + public void testVMParamsAllCredentialsFromFile() throws Exception { + useVMParamsAllCredentialsFromFile(); + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + doTest(zkClient, true, true, true, true, true, true, true, true, true, true); + } + } + + @Test + public void testVMParamsReadonlyCredentialsFromFile() throws Exception { + useVMParamsReadonlyCredentialsFromFile(); + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + doTest(zkClient, true, true, false, false, false, false, false, false, false, false); + } + } + + private void useVMParamsAllCredentialsFromFile() throws IOException { + useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword"); + } + + private void useVMParamsReadonlyCredentialsFromFile() throws IOException { + useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword"); + } + + private void useVMParamsCredentialsFromFile(String username, String password) throws IOException { + Properties props = new Properties(); + props.setProperty( + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username); + props.setProperty( + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password); + String credsFile = saveCredentialsFile(props); + + useZkCredentialsInjector(VMParamsZkCredentialsInjector.class); + System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile); + } + + private String saveCredentialsFile(Properties props) throws IOException { + Path tmp = createTempFile("zk-creds", "properties"); + try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) { + props.store(w, "test"); + } + return tmp.toAbsolutePath().toString(); + } + + public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return Collections.emptyList(); + } + } + + public static class AllAndReadonlyCredentialZkCredentialsInjector + implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of( + new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL), + new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ)); + } + } + + public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of( + new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL)); + } + } + + public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL)); + } + } + + public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of( + new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL)); + } + } + + class TestZkCredentialsInjector { + private final Class zkCredentialsInjectorClass; + private final List systemPropsKeys; + private final List systemPropsValues; + + public TestZkCredentialsInjector(Class zkCredentialsInjectorClass) { + this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList()); + } + + public TestZkCredentialsInjector( + Class zkCredentialsInjectorClass, + List systemPropsKeys, + List systemPropsValues) { + this.zkCredentialsInjectorClass = zkCredentialsInjectorClass; + this.systemPropsKeys = systemPropsKeys; + this.systemPropsValues = systemPropsValues; + } + + private void setSystemProps() { + clearSecuritySystemProperties(); + setDigestZkSystemProps(); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + zkCredentialsInjectorClass.getName()); + int i = 0; + for (String key : systemPropsKeys) { + System.setProperty(key, systemPropsValues.get(i++)); + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java index 11c04c976cd..2367c6cbb99 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java @@ -16,10 +16,67 @@ */ package org.apache.solr.cloud.api.collections; +import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.solr.client.api.model.CoreStatusResponse; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.TimeSource; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.apache.solr.util.TimeOut; +import org.junit.After; import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests the Cloud Collections API. + * + *

Because the different setups require distinct config-sets, we have to push down cluster + * creation to subclasses + */ +public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); -/** Tests the Cloud Collections API. */ -public class CollectionsAPIDistributedZkTest extends AbstractCollectionsAPIDistributedZkTestBase { @BeforeClass public static void createCluster() throws Exception { configureCluster(4) @@ -28,4 +85,625 @@ public static void createCluster() throws Exception { .withSolrXml(TEST_PATH().resolve("solr.xml")) .configure(); } + + @BeforeClass + public static void setupCluster() { + // we don't want this test to have zk timeouts + System.setProperty("zkClientTimeout", "60000"); + System.setProperty("createCollectionWaitTimeTillActive", "5"); + TestInjection.randomDelayInCoreCreation = "true:5"; + System.setProperty("validateAfterInactivity", "200"); + System.setProperty("solr.security.allow.paths", "*"); + } + + @Override + @After + public void tearDown() throws Exception { + cluster.deleteAllCollections(); + super.tearDown(); + } + + @Test + public void testCreationAndDeletion() throws Exception { + String collectionName = "created_and_deleted"; + + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) + .process(cluster.getSolrClient()); + assertTrue( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + assertFalse( + cluster + .getZkClient() + .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); + } + + @Test + public void deleteCollectionRemovesStaleZkCollectionsNode() throws Exception { + String collectionName = "out_of_sync_collection"; + + // manually create a collections zknode + cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + assertFalse( + cluster + .getZkClient() + .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); + } + + @Test + public void deletePartiallyCreatedCollection() throws Exception { + final String collectionName = "halfdeletedcollection"; + + assertEquals( + 0, + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()) + .getStatus()); + String dataDir = createTempDir().toString(); + // create a core that simulates something left over from a partially-deleted collection + assertTrue( + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .setDataDir(dataDir) + .process(cluster.getSolrClient()) + .isSuccess()); + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .process(cluster.getSolrClient()); + + assertTrue( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + } + + @Test + public void deleteCollectionOnlyInZk() throws Exception { + final String collectionName = "onlyinzk"; + + // create the collections node, but nothing else + cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); + + // delete via API - should remove collections node + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + // now creating that collection should work + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .process(cluster.getSolrClient()); + assertTrue( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + } + + @Test + public void testBadActionNames() { + // try a bad action + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "BADACTION"); + String collectionName = "badactioncollection"; + params.set("name", collectionName); + params.set("numShards", 2); + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testMissingRequiredParameters() { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("numShards", 2); + // missing required collection parameter + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testMissingNumShards() { + // No numShards should fail + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("name", "acollection"); + params.set(REPLICATION_FACTOR, 10); + params.set("collection.configName", "conf"); + + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testZeroNumShards() { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("name", "acollection"); + params.set(REPLICATION_FACTOR, 10); + params.set("numShards", 0); + params.set("collection.configName", "conf"); + + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testCreateShouldFailOnExistingCore() throws Exception { + String nn1 = cluster.getJettySolrRunner(0).getNodeName(); + String nn2 = cluster.getJettySolrRunner(1).getNodeName(); + + assertEquals( + 0, + CollectionAdminRequest.createCollection("halfcollectionblocker", "conf", 1, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard("halfcollectionblocker", "shard1") + .setNode(cluster.getJettySolrRunner(0).getNodeName()) + .setCoreName("halfcollection_shard1_replica_n1") + .process(cluster.getSolrClient()) + .isSuccess()); + + assertEquals( + 0, + CollectionAdminRequest.createCollection("halfcollectionblocker2", "conf", 1, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard("halfcollectionblocker2", "shard1") + .setNode(cluster.getJettySolrRunner(1).getNodeName()) + .setCoreName("halfcollection_shard1_replica_n1") + .process(cluster.getSolrClient()) + .isSuccess()); + + expectThrows( + SolrClient.RemoteSolrException.class, + () -> { + CollectionAdminRequest.createCollection("halfcollection", "conf", 1, 1) + .setCreateNodeSet(nn1 + "," + nn2) + .process(cluster.getSolrClient()); + }); + } + + @Test + public void testNoConfigSetExist() throws Exception { + expectThrows( + Exception.class, + () -> { + CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1) + .process(cluster.getSolrClient()); + }); + + TimeUnit.MILLISECONDS.sleep(1000); + // in both cases, the collection should have default to the core name + cluster.getZkStateReader().forceUpdateCollection("noconfig"); + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig")); + } + + @Test + public void testCoresAreDistributedAcrossNodes() throws Exception { + CollectionAdminRequest.createCollection("nodes_used_collection", "conf", 2, 2) + .process(cluster.getSolrClient()); + + Set liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes(); + + List createNodeList = new ArrayList<>(liveNodes); + + DocCollection collection = getCollectionState("nodes_used_collection"); + for (Slice slice : collection.getSlices()) { + for (Replica replica : slice.getReplicas()) { + createNodeList.remove(replica.getNodeName()); + } + } + + assertEquals(createNodeList.toString(), 0, createNodeList.size()); + } + + @Test + public void testDeleteNonExistentCollection() throws Exception { + + expectThrows( + SolrException.class, + () -> { + CollectionAdminRequest.deleteCollection("unknown_collection") + .process(cluster.getSolrClient()); + }); + + // create another collection should still work + CollectionAdminRequest.createCollection("acollectionafterbaddelete", "conf", 1, 2) + .process(cluster.getSolrClient()); + waitForState( + "Collection creation after a bad delete failed", + "acollectionafterbaddelete", + (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 2)); + } + + @Test + public void testSpecificConfigsets() throws Exception { + CollectionAdminRequest.createCollection("withconfigset2", "conf2", 1, 1) + .process(cluster.getSolrClient()); + String configName = + cluster + .getSolrClient() + .getClusterStateProvider() + .getCollection("withconfigset2") + .getConfigName(); + assertEquals("conf2", configName); + } + + @Test + public void testCreateNodeSet() throws Exception { + JettySolrRunner jetty1 = cluster.getRandomJetty(random()); + JettySolrRunner jetty2 = cluster.getRandomJetty(random()); + + List baseUrls = List.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString()); + + CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1) + .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1)) + .process(cluster.getSolrClient()); + + DocCollection collectionState = getCollectionState("nodeset_collection"); + for (Replica replica : collectionState.getReplicas()) { + String replicaUrl = replica.getCoreUrl(); + boolean matchingJetty = false; + for (String jettyUrl : baseUrls) { + if (replicaUrl.startsWith(jettyUrl)) { + matchingJetty = true; + } + } + if (matchingJetty == false) { + fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl); + } + } + } + + @Test + public void testCollectionsAPI() throws Exception { + + // create new collections rapid fire + int cnt = random().nextInt(TEST_NIGHTLY ? 3 : 1) + 1; + CollectionAdminRequest.Create[] createRequests = new CollectionAdminRequest.Create[cnt]; + + class Coll { + String name; + int numShards; + int replicationFactor; + } + + List colls = new ArrayList<>(); + + for (int i = 0; i < cnt; i++) { + + int numShards = TestUtil.nextInt(random(), 0, cluster.getJettySolrRunners().size()) + 1; + int replicationFactor = TestUtil.nextInt(random(), 0, 3) + 1; + + createRequests[i] = + CollectionAdminRequest.createCollection( + "awhollynewcollection_" + i, "conf2", numShards, replicationFactor); + createRequests[i].processAsync(cluster.getSolrClient()); + + Coll coll = new Coll(); + coll.name = "awhollynewcollection_" + i; + coll.numShards = numShards; + coll.replicationFactor = replicationFactor; + colls.add(coll); + } + + for (Coll coll : colls) { + cluster.waitForActiveCollection( + coll.name, coll.numShards, coll.numShards * coll.replicationFactor); + } + + waitForStable(cnt, createRequests); + + for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { + checkInstanceDirs(cluster.getJettySolrRunner(i)); + } + + String collectionName = + createRequests[random().nextInt(createRequests.length)].getCollectionName(); + + // TODO: we should not need this...beast test well when trying to fix + Thread.sleep(1000); + + cluster.getZkStateReader().forciblyRefreshAllClusterStateSlow(); + + new UpdateRequest() + .add("id", "6") + .add("id", "7") + .add("id", "8") + .commit(cluster.getSolrClient(), collectionName); + long numFound = 0; + TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); + while (!timeOut.hasTimedOut()) { + + numFound = + cluster + .getSolrClient() + .query(collectionName, new SolrQuery("*:*")) + .getResults() + .getNumFound(); + if (numFound == 3) { + break; + } + + Thread.sleep(500); + } + + if (timeOut.hasTimedOut()) { + fail( + "Timeout waiting to see 3 found, instead saw " + + numFound + + " for collection " + + collectionName); + } + + checkNoTwoShardsUseTheSameIndexDir(); + } + + private void waitForStable(int cnt, CollectionAdminRequest.Create[] createRequests) + throws InterruptedException { + for (int i = 0; i < cnt; i++) { + String collectionName = "awhollynewcollection_" + i; + final int j = i; + waitForState( + "Expected to see collection " + collectionName, + collectionName, + (n, c) -> { + CollectionAdminRequest.Create req = createRequests[j]; + return SolrCloudTestCase.replicasForCollectionAreFullyActive( + n, c, req.getNumShards(), req.getReplicationFactor()); + }); + + ZkStateReader zkStateReader = cluster.getZkStateReader(); + // make sure we have leaders for each shard + for (int z = 1; z < createRequests[j].getNumShards(); z++) { + zkStateReader.getLeaderRetry(collectionName, "shard" + z, 10000); + } // make sure we again have leaders for each shard + } + } + + @Test + public void testCollectionReload() throws Exception { + final String collectionName = "reloaded_collection"; + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) + .process(cluster.getSolrClient()); + + // get core open times + Map urlToTimeBefore = new HashMap<>(); + collectStartTimes(collectionName, urlToTimeBefore); + assertTrue(urlToTimeBefore.size() > 0); + + CollectionAdminRequest.reloadCollection(collectionName).processAsync(cluster.getSolrClient()); + + // reloads make take a short while + boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore); + assertTrue("some core start times did not change on reload", allTimesAreCorrect); + } + + private void checkInstanceDirs(JettySolrRunner jetty) throws IOException { + CoreContainer cores = jetty.getCoreContainer(); + Collection theCores = cores.getCores(); + for (SolrCore core : theCores) { + // look for core props file + Path instancedir = core.getInstancePath(); + assertTrue( + "Could not find expected core.properties file", + Files.exists(instancedir.resolve("core.properties"))); + + Path expected = Path.of(jetty.getSolrHome()).resolve(core.getName()); + + assertTrue( + "Expected: " + expected + "\nFrom core stats: " + instancedir, + Files.isSameFile(expected, instancedir)); + } + } + + private boolean waitForReloads(String collectionName, Map urlToTimeBefore) + throws SolrServerException, IOException { + TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME); + + boolean allTimesAreCorrect = false; + while (!timeout.hasTimedOut()) { + Map urlToTimeAfter = new HashMap<>(); + collectStartTimes(collectionName, urlToTimeAfter); + + boolean retry = false; + Set> entries = urlToTimeBefore.entrySet(); + for (Entry entry : entries) { + Long beforeTime = entry.getValue(); + Long afterTime = urlToTimeAfter.get(entry.getKey()); + assertNotNull(afterTime); + if (afterTime <= beforeTime) { + retry = true; + break; + } + } + if (!retry) { + allTimesAreCorrect = true; + break; + } + } + return allTimesAreCorrect; + } + + private void collectStartTimes(String collectionName, Map urlToTime) + throws SolrServerException, IOException { + + DocCollection collectionState = getCollectionState(collectionName); + if (collectionState != null) { + for (Slice shard : collectionState) { + for (Replica replica : shard) { + CoreStatusResponse.SingleCoreData coreStatus; + try (SolrClient server = getHttpSolrClient(replica.getBaseUrl())) { + coreStatus = CoreAdminRequest.getCoreStatus(replica.getCoreName(), false, server); + } + long before = coreStatus.startTime.getTime(); + urlToTime.put(replica.getCoreUrl(), before); + } + } + } else { + throw new IllegalArgumentException("Could not find collection " + collectionName); + } + } + + private void checkNoTwoShardsUseTheSameIndexDir() { + Map> indexDirToShardNamesMap = new HashMap<>(); + + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + CoreContainer coreContainer = jetty.getCoreContainer(); + List coreNames = coreContainer.getAllCoreNames(); + for (String coreName : coreNames) { + try (SolrCore core = coreContainer.getCore(coreName)) { + String indexDir = core.getIndexDir(); + String shardKey = jetty.getNodeName() + "." + coreName; + if (!indexDirToShardNamesMap.containsKey(indexDir)) { + indexDirToShardNamesMap.put(indexDir, new HashSet<>()); + } + indexDirToShardNamesMap.get(indexDir).add(shardKey); + } + } + } + + assertTrue( + "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of " + + SolrCore.class.getSimpleName() + + " : " + + indexDirToShardNamesMap, + indexDirToShardNamesMap.size() > 0); + + for (Map.Entry> entry : indexDirToShardNamesMap.entrySet()) { + if (entry.getValue().size() > 1) { + fail( + "We have shards using the same indexDir. E.g. shards " + + entry.getValue().toString() + + " all use indexDir " + + entry.getKey()); + } + } + } + + @Test + public void addReplicaTest() throws Exception { + String collectionName = "addReplicaColl"; + + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(collectionName, 2, 4); + + ArrayList nodeList = + new ArrayList<>(cluster.getSolrClient().getClusterState().getLiveNodes()); + Collections.shuffle(nodeList, random()); + + CollectionAdminResponse response = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .setNode(nodeList.get(0)) + .process(cluster.getSolrClient()); + Replica newReplica = grabNewReplica(response, getCollectionState(collectionName)); + + assertEquals( + "Replica should be created on the right node", + cluster.getZkStateReader().getBaseUrlForNodeName(nodeList.get(0)), + newReplica.getBaseUrl()); + + Path instancePath = createTempDir(); + response = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .withProperty(CoreAdminParams.INSTANCE_DIR, instancePath.toString()) + .process(cluster.getSolrClient()); + newReplica = grabNewReplica(response, getCollectionState(collectionName)); + assertNotNull(newReplica); + cluster.waitForActiveCollection(collectionName, 2, 6); + + try (SolrClient coreclient = getHttpSolrClient(newReplica.getBaseUrl())) { + CoreAdminResponse status = CoreAdminRequest.getStatus(newReplica.getStr("core"), coreclient); + final var coreStatus = status.getCoreStatus(newReplica.getStr("core")); + String instanceDirStr = coreStatus.instanceDir; + assertEquals(instanceDirStr, instancePath.toString()); + } + + // Test to make sure we can't create another replica with an existing core_name of that + // collection + String coreName = newReplica.getStr(CORE_NAME_PROP); + SolrException e = + expectThrows( + SolrException.class, + () -> { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "addreplica"); + params.set("collection", collectionName); + params.set("shard", "shard1"); + params.set("name", coreName); + var request = + new GenericSolrRequest( + METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + cluster.getSolrClient().request(request); + }); + + assertTrue( + e.getMessage() + .contains( + "Another replica with the same core name already exists for this collection")); + + // Check that specifying property.name works. DO NOT remove this when the "name" property is + // deprecated + // for ADDREPLICA, this is "property.name". See SOLR-7132 + response = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .withProperty(CoreAdminParams.NAME, "propertyDotName") + .process(cluster.getSolrClient()); + + newReplica = grabNewReplica(response, getCollectionState(collectionName)); + assertEquals( + "'core' should be 'propertyDotName' ", "propertyDotName", newReplica.getStr("core")); + cluster.waitForActiveCollection(collectionName, 2, 7); + } + + private Replica grabNewReplica(CollectionAdminResponse response, DocCollection docCollection) { + String replicaName = response.getCollectionCoresStatus().keySet().iterator().next(); + Optional optional = + docCollection.getReplicas().stream() + .filter(replica -> replicaName.equals(replica.getCoreName())) + .findAny(); + if (optional.isPresent()) { + return optional.get(); + } + throw new AssertionError("Can not find " + replicaName + " from " + docCollection); + } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java deleted file mode 100644 index 1ac793e852d..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.mockfile.FilterPath; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.handler.BackupStatusChecker; -import org.apache.solr.handler.ReplicationHandler; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. - */ -// Backups do checksum validation against a footer value not present in 'SimpleText' -@LuceneTestCase.SuppressCodecs({"SimpleText"}) -public abstract class AbstractBasicDistributedZk2TestBase extends AbstractFullDistribZkTestBase { - private static final String SHARD2 = "shard2"; - private static final String SHARD1 = "shard1"; - private static final String ONE_NODE_COLLECTION = "onenodecollection"; - private final boolean onlyLeaderIndexes = random().nextBoolean(); - - public AbstractBasicDistributedZk2TestBase() { - super(); - // we need DVs on point fields to compute stats & facets - if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) - System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); - - sliceCount = 2; - } - - @Override - protected boolean useTlogReplicas() { - return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug - // and it's TestInjection use - } - - @BeforeClass - public static void _setPrsDefault() { - configurePrsDefault(); - } - - @Test - @ShardsFixed(num = 4) - public void test() throws Exception { - boolean testFinished = false; - try { - handle.clear(); - handle.put("timestamp", SKIPVAL); - - testNodeWithoutCollectionForwarding(); - - indexr( - id, - 1, - i1, - 100, - tlong, - 100, - t1, - "now is the time for all good men", - "foo_f", - 1.414f, - "foo_b", - "true", - "foo_d", - 1.414d); - - commit(); - - // make sure we are in a steady state... - waitForRecoveriesToFinish(false); - - assertDocCounts(false); - - indexAbunchOfDocs(); - - // check again - waitForRecoveriesToFinish(false); - - commit(); - - assertDocCounts(VERBOSE); - checkQueries(); - - assertDocCounts(VERBOSE); - - query("q", "*:*", "sort", "n_tl1 desc"); - - bringDownShardIndexSomeDocsAndRecover(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - // test adding another replica to a shard - it should do a - // recovery/replication to pick up the index from the leader - addNewReplica(); - - long docId = testUpdateAndDelete(); - - // index a bad doc... - expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id")); - - // TODO: bring this to its own method? - // try indexing to a leader that has no replicas up - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2); - - String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP); - chaosMonkey.stopShardExcept(SHARD2, nodeName); - - SolrClient client = getClient(nodeName); - - index_specific(client, "id", docId + 1, t1, "what happens here?"); - - // expire a session... - CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0); - chaosMonkey.expireSession(cloudJetty.jetty); - // Wait until the jetty is reconnected, otherwise the following index command could fail - cloudJetty - .jetty - .getCoreContainer() - .getZkController() - .getZkClient() - .getCuratorFramework() - .blockUntilConnected(50, TimeUnit.MILLISECONDS); - - indexr("id", docId + 1, t1, "slip this doc in"); - - waitForRecoveriesToFinish(false); - - checkShardConsistency(SHARD1); - checkShardConsistency(SHARD2); - - testFinished = true; - } finally { - if (!testFinished) { - printLayoutOnTearDown = true; - } - } - } - - private void testNodeWithoutCollectionForwarding() throws Exception { - assertEquals( - 0, - CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1") - .setCoreName(ONE_NODE_COLLECTION + "core") - .process(cloudClient) - .isSuccess()); - - waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1); - waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false); - - ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000); - - int docs = 2; - for (JettySolrRunner jetty : jettys) { - final String clientUrl = getBaseUrl(jetty); - addAndQueryDocs(clientUrl, docs); - docs += 2; - } - } - - // 2 docs added every call - private void addAndQueryDocs(final String baseUrl, int docs) throws Exception { - - SolrQuery query = new SolrQuery("*:*"); - - try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) { - // add a doc - client.add(sdoc("id", docs)); - client.commit(); - - QueryResponse results = client.query(query); - assertEquals(docs - 1, results.getResults().getNumFound()); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", docs + 1); - client.add(doc); - client.commit(); - - query = new SolrQuery("*:*"); - query.set("rows", 0); - results = client.query(query); - assertEquals(docs, results.getResults().getNumFound()); - } - } - - private long testUpdateAndDelete() throws Exception { - long docId = 99999999L; - indexr("id", docId, t1, "originalcontent"); - - commit(); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add("q", t1 + ":originalcontent"); - QueryResponse results = clients.get(0).query(params); - assertEquals(1, results.getResults().getNumFound()); - - // update doc - indexr("id", docId, t1, "updatedcontent"); - - commit(); - - results = clients.get(0).query(params); - assertEquals(0, results.getResults().getNumFound()); - - params.set("q", t1 + ":updatedcontent"); - - results = clients.get(0).query(params); - assertEquals(1, results.getResults().getNumFound()); - - UpdateRequest uReq = new UpdateRequest(); - // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN); - uReq.deleteById(Long.toString(docId)).process(clients.get(0)); - - commit(); - - results = clients.get(0).query(params); - assertEquals(0, results.getResults().getNumFound()); - return docId; - } - - private void bringDownShardIndexSomeDocsAndRecover() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.set("distrib", false); - - commit(); - - long deadShardCount = - shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - int oldLiveNodes = - ZkStateReader.from(cloudClient) - .getZkClient() - .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true) - .size(); - - assertEquals(5, oldLiveNodes); - - // kill a shard - CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0); - - // ensure shard is dead - expectThrows( - SolrServerException.class, - "This server should be down and this update should have failed", - () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!")); - - commit(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - // long cloudClientDocs = cloudClient.query(new - // SolrQuery("*:*")).getResults().getNumFound(); - // System.out.println("clouddocs:" + cloudClientDocs); - - // try to index to a living shard at shard2 - - long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000); - - try { - index_specific( - shardToJetty.get(SHARD1).get(1).client.solrClient, - id, - 1000, - i1, - 108, - t1, - "specific doc!"); - } catch (Exception e) { - // wait and try again - Thread.sleep(4000); - index_specific( - shardToJetty.get(SHARD1).get(1).client.solrClient, - id, - 1000, - i1, - 108, - t1, - "specific doc!"); - } - - commit(); - - checkShardConsistency(true, false); - - query("q", "*:*", "sort", "n_tl1 desc"); - - long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - assertEquals(numFound1 + 1, numFound2); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", 1001); - - controlClient.add(doc); - - // try adding a doc with CloudSolrServer - UpdateRequest ureq = new UpdateRequest(); - ureq.add(doc); - // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); - - try { - ureq.process(cloudClient); - } catch (SolrServerException e) { - // try again - Thread.sleep(3500); - ureq.process(cloudClient); - } - - commit(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - // lets just check that the one doc since last commit made it in... - assertEquals(numFound2 + 1, numFound3); - - // test debugging - testDebugQueries(); - - if (VERBOSE) { - System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()); - - for (SolrClient client : clients) { - try { - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - System.err.println(client.query(q).getResults().getNumFound()); - } catch (Exception e) { - - } - } - } - // TODO: This test currently fails because debug info is obtained only - // on shards with matches. - // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); - - // this should trigger a recovery phase on deadShard - deadShard.jetty.start(); - - // make sure we have published we are recovering - Thread.sleep(1500); - - waitForRecoveriesToFinish(false); - - deadShardCount = - shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound(); - // if we properly recovered, we should now have the couple missing docs that - // came in while shard was down - checkShardConsistency(true, false); - - // recover over 100 docs so we do more than just peer sync (replicate recovery) - chaosMonkey.stopJetty(deadShard); - - for (int i = 0; i < 226; i++) { - doc = new SolrInputDocument(); - doc.addField("id", 2000 + i); - controlClient.add(doc); - ureq = new UpdateRequest(); - ureq.add(doc); - // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); - ureq.process(cloudClient); - } - commit(); - - Thread.sleep(1500); - - deadShard.jetty.start(); - - // make sure we have published we are recovering - Thread.sleep(1500); - - waitForThingsToLevelOut(1, TimeUnit.MINUTES); - - Thread.sleep(500); - - waitForRecoveriesToFinish(false); - - checkShardConsistency(true, false); - - // try a backup command - try (final SolrClient client = - getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) { - final String backupName = "the_backup"; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("qt", ReplicationHandler.PATH); - params.set("command", "backup"); - params.set("name", backupName); - final Path location = FilterPath.unwrap(createTempDir()).toRealPath(); - // Allow non-standard location outside SOLR_HOME - jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location)); - params.set("location", location.toString()); - - QueryRequest request = new QueryRequest(params); - client.request(request, DEFAULT_TEST_COLLECTION_NAME); - - final BackupStatusChecker backupStatus = - new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication"); - final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30); - assertTrue( - "Backup dir does not exist: " + backupDirName, - Files.exists(location.resolve(backupDirName))); - } - } - - private void addNewReplica() throws Exception { - - waitForRecoveriesToFinish(false); - - // new server should be part of first shard - // how many docs are on the new shard? - for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) { - if (VERBOSE) - System.err.println( - "shard1 total:" - + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) { - if (VERBOSE) - System.err.println( - "shard2 total:" - + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - - checkShardConsistency(SHARD1); - checkShardConsistency(SHARD2); - - assertDocCounts(VERBOSE); - } - - private void testDebugQueries() throws Exception { - handle.put("explain", SKIPVAL); - handle.put("debug", UNORDERED); - handle.put("time", SKIPVAL); - handle.put("track", SKIP); - query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java deleted file mode 100644 index 1dae465d515..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java +++ /dev/null @@ -1,1737 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.Future; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.lucene.util.IOUtils; -import org.apache.solr.JSONTestUtil; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.impl.InputStreamResponseParser; -import org.apache.solr.client.solrj.request.AbstractUpdateRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; -import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.StreamingUpdateRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.Group; -import org.apache.solr.client.solrj.response.GroupCommand; -import org.apache.solr.client.solrj.response.GroupResponse; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.apache.solr.util.TestInjection.Hook; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. - */ -public abstract class AbstractBasicDistributedZkTestBase extends AbstractFullDistribZkTestBase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String DEFAULT_COLLECTION = "collection1"; - - private final boolean onlyLeaderIndexes = random().nextBoolean(); - - String tsort = "t_sortable"; - - private Map> otherCollectionClients = new HashMap<>(); - - private String oneInstanceCollection = "oneInstanceCollection"; - private String oneInstanceCollection2 = "oneInstanceCollection2"; - - private AtomicInteger nodeCounter = new AtomicInteger(); - - CompletionService completionService; - Set> pending; - - private static Hook newSearcherHook = - new Hook() { - volatile CountDownLatch latch; - AtomicReference collection = new AtomicReference<>(); - - @Override - public void newSearcher(String collectionName) { - String c = collection.get(); - if (c != null && c.equals(collectionName)) { - log.info("Hook detected newSearcher"); - try { - latch.countDown(); - } catch (NullPointerException e) { - - } - } - } - - @Override - public void waitForSearcher( - String collection, int cnt, int timeoutms, boolean failOnTimeout) - throws InterruptedException { - latch = new CountDownLatch(cnt); - this.collection.set(collection); - boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS); - if (timeout && failOnTimeout) { - fail("timed out waiting for new searcher event " + latch.getCount()); - } - } - }; - - public AbstractBasicDistributedZkTestBase() { - // we need DVs on point fields to compute stats & facets - if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) - System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); - - sliceCount = 2; - completionService = new ExecutorCompletionService<>(executor); - pending = new HashSet<>(); - } - - @BeforeClass - public static void beforeBDZKTClass() { - TestInjection.newSearcherHook(newSearcherHook); - } - - @Override - protected boolean useTlogReplicas() { - // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's - // TestInjection use - return false; - } - - @Override - protected void setDistributedParams(ModifiableSolrParams params) { - - if (r.nextBoolean()) { - // don't set shards, let that be figured out from the cloud state - } else { - // use shard ids rather than physical locations - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < getShardCount(); i++) { - if (i > 0) sb.append(','); - sb.append("shard" + (i + 3)); - } - params.set("shards", sb.toString()); - } - } - - @Test - @ShardsFixed(num = 4) - protected void test() throws Exception { - // setLoggingLevel(null); - - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - // make sure we have leaders for each shard - for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); - } // make sure we again have leaders for each shard - - waitForRecoveriesToFinish(false); - - handle.clear(); - handle.put("timestamp", SKIPVAL); - - del("*:*"); - queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty")); - - // ask every individual replica of every shard to update+commit the same doc id - // with an incrementing counter on each update+commit - int foo_i_counter = 0; - for (SolrClient client : clients) { - foo_i_counter++; - indexDoc( - client, - params("commit", "true"), // SOLR-4923 - sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter)); - // after every update+commit, check all the shards consistency - queryAndCompareShards( - params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup")); - queryAndCompareShards( - params( - "q", "id:1", - "sanity_check", "distrib_id_1_lookup")); - } - - indexr( - id, - 1, - i1, - 100, - tlong, - 100, - t1, - "now is the time for all good men", - "foo_f", - 1.414f, - "foo_b", - "true", - "foo_d", - 1.414d, - tsort, - "now is the time for all good men"); - indexr( - id, - 2, - i1, - 50, - tlong, - 50, - t1, - "to come to the aid of their country.", - tsort, - "to come to the aid of their country."); - indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow"); - indexr( - id, - 4, - i1, - -100, - tlong, - 101, - t1, - "the quick fox jumped over the lazy dog", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 5, - i1, - 500, - tlong, - 500, - t1, - "the quick fox jumped way over the lazy dog", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 6, - i1, - -600, - tlong, - 600, - t1, - "humpty dumpy sat on a wall", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 7, - i1, - 123, - tlong, - 123, - t1, - "humpty dumpy had a great fall", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 8, - i1, - 876, - tlong, - 876, - t1, - "all the kings horses and all the kings men", - tsort, - "all the kings horses and all the kings men"); - indexr( - id, - 9, - i1, - 7, - tlong, - 7, - t1, - "couldn't put humpty together again", - tsort, - "the quick fox jumped over the lazy dog"); - indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass"); - indexr( - id, - 11, - i1, - -987, - tlong, - 987, - t1, - "An eye for eye only ends up making the whole world blind.", - tsort, - "An eye for eye only ends up making the whole world blind."); - indexr( - id, - 12, - i1, - 379, - tlong, - 379, - t1, - "Great works are performed, not by strength, but by perseverance.", - tsort, - "Great works are performed, not by strength, but by perseverance."); - indexr( - id, - 13, - i1, - 232, - tlong, - 232, - t1, - "no eggs on wall, lesson learned", - oddField, - "odd man out", - tsort, - "no eggs on wall, lesson learned"); - - indexr( - id, - 14, - "SubjectTerms_mfacet", - new String[] {"mathematical models", "mathematical analysis"}); - indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); - indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); - String[] vals = new String[100]; - for (int i = 0; i < 100; i++) { - vals[i] = "test " + i; - } - indexr(id, 17, "SubjectTerms_mfacet", vals); - - for (int i = 100; i < 150; i++) { - indexr(id, i); - } - - commit(); - - testTokenizedGrouping(); - testSortableTextFaceting(); - testSortableTextSorting(); - testSortableTextGrouping(); - - queryAndCompareShards( - params( - "q", "*:*", - "sort", "id desc", - "distrib", "false", - "sanity_check", "is_empty")); - - // random value sort - for (String f : fieldNames) { - query(false, new String[] {"q", "*:*", "sort", f + " desc"}); - query(false, new String[] {"q", "*:*", "sort", f + " asc"}); - } - - // these queries should be exactly ordered and scores should exactly match - query(false, new String[] {"q", "*:*", "sort", i1 + " desc"}); - query(false, new String[] {"q", "*:*", "sort", i1 + " asc"}); - query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"}); - query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"}); - query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"}); - handle.put("maxScore", SKIPVAL); - query( - false, - new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it. - // JavaBinCodec.writeSolrDocumentList() - // is agnostic of request params. - handle.remove("maxScore"); - query( - false, - new String[] { - "q", "{!func}" + i1, "fl", "*,score" - }); // even scores should match exactly here - - handle.put("highlighting", UNORDERED); - handle.put("response", UNORDERED); - - handle.put("maxScore", SKIPVAL); - query(false, new String[] {"q", "quick"}); - query(false, new String[] {"q", "all", "fl", "id", "start", "0"}); - query( - false, - new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs - query(false, new String[] {"q", "all", "fl", "id", "start", "100"}); - - handle.put("score", SKIPVAL); - query(false, new String[] {"q", "quick", "fl", "*,score"}); - query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"}); - query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"}); - - query( - false, - new String[] { - "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1 - }); - - query( - false, - new String[] { - "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1 - }); - - query(false, new String[] {"q", "matchesnothing", "fl", "*,score"}); - - query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1}); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "count" - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "count", - "facet.mincount", - 2 - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "index" - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "index", - "facet.mincount", - 2 - }); - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1 - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.query", - "quick", - "facet.query", - "all", - "facet.query", - "*:*" - }); - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1 - }); - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2 - }); - - // test faceting multiple things at once - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.query", - "quick", - "facet.query", - "all", - "facet.query", - "*:*", - "facet.field", - t1 - }); - - // test filter tagging, facet exclusion, and naming (multi-select facet support) - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.query", - "{!key=myquick}quick", - "facet.query", - "{!key=myall ex=a}all", - "facet.query", - "*:*", - "facet.field", - "{!key=mykey ex=a}" + t1, - "facet.field", - "{!key=other ex=b}" + t1, - "facet.field", - "{!key=again ex=a,b}" + t1, - "facet.field", - t1, - "fq", - "{!tag=a}id_i1:[1 TO 7]", - "fq", - "{!tag=b}id_i1:[3 TO 9]" - }); - query( - false, - new Object[] { - "q", - "*:*", - "facet", - "true", - "facet.field", - "{!ex=t1}SubjectTerms_mfacet", - "fq", - "{!tag=t1}SubjectTerms_mfacet:(test 1)", - "facet.limit", - "10", - "facet.mincount", - "1" - }); - - // test field that is valid in schema but missing in all shards - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2 - }); - // test field that is valid in schema and missing in some shards - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2 - }); - - query( - false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1}); - - /* TODO: the failure may come back in "exception" - try { - // test error produced for field that is invalid for schema - query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2); - TestCase.fail("SolrServerException expected for invalid field that is not in schema"); - } catch (SolrServerException ex) { - // expected - } - */ - - // Try to get better coverage for refinement queries by turning off over requesting. - // This makes it much more likely that we may not get the top facet values and hence - // we turn of that checking. - handle.put("facet_fields", SKIPVAL); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 0, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - 5, - "facet.shard.limit", - 5 - }); - // check a complex key name - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 0, - "facet", - "true", - "facet.field", - "{!key='a b/c \\' \\} foo'}" + t1, - "facet.limit", - 5, - "facet.shard.limit", - 5 - }); - handle.remove("facet_fields"); - - // index the same document to two servers and make sure things - // don't blow up. - if (clients.size() >= 2) { - index(id, 100, i1, 107, t1, "oh no, a duplicate!"); - for (int i = 0; i < clients.size(); i++) { - index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!"); - } - commit(); - query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1}); - query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1}); - query(false, new Object[] {"q", "*:*", "rows", 100}); - } - - // test debugging - handle.put("explain", SKIPVAL); - handle.put("debug", UNORDERED); - handle.put("time", SKIPVAL); - handle.put("track", SKIP); - query( - false, - new Object[] { - "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true" - }); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS}); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY}); - - // try add commitWithin - long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - for (SolrClient client : clients) { - assertEquals( - "unexpected pre-commitWithin document count on node: " - + ((HttpSolrClient) client).getBaseURL(), - before, - client.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("commitWithin", 10); - add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301))); - - newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); - - ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); - DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION); - - assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl); - - // try deleteById commitWithin - UpdateRequest deleteByIdReq = new UpdateRequest(); - deleteByIdReq.deleteById("300"); - deleteByIdReq.setCommitWithin(10); - deleteByIdReq.process(cloudClient); - - newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); - - assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl); - - // try deleteByQuery commitWithin - UpdateRequest deleteByQueryReq = new UpdateRequest(); - deleteByQueryReq.deleteByQuery("id:301"); - deleteByQueryReq.setCommitWithin(10); - deleteByQueryReq.process(cloudClient); - - newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); - - assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl); - - // TODO: This test currently fails because debug info is obtained only - // on shards with matches. - // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); - - // would be better if these where all separate tests - but much, much - // slower - doOptimisticLockingAndUpdating(); - testShardParamVariations(); - testMultipleCollections(); - testANewCollectionInOneInstance(); - testSearchByCollectionName(); - testUpdateByCollectionName(); - testANewCollectionInOneInstanceWithManualShardAssignement(); - testNumberOfCommitsWithCommitAfterAdd(); - - testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit"); - testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit"); - - testStopAndStartCoresInOneInstance(); - } - - private void testSortableTextFaceting() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.addFacetField(tsort); - query.setFacetMissing(false); - QueryResponse resp = queryRandomShard(query); - List ffs = resp.getFacetFields(); - for (FacetField ff : ffs) { - if (ff.getName().equals(tsort) == false) continue; - for (FacetField.Count count : ff.getValues()) { - long num = count.getCount(); - switch (count.getName()) { - case "all the kings horses and all the kings men": - case "An eye for eye only ends up making the whole world blind.": - case "Great works are performed, not by strength, but by perseverance.": - case "how now brown cow": - case "no eggs on wall, lesson learned": - case "now is the time for all good men": - case "this too shall pass": - case "to come to the aid of their country.": - assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num); - break; - case "the quick fox jumped over the lazy dog": - assertEquals("Should have 5 docs for the lazy dog", 5, num); - break; - default: - fail("No case for facet '" + ff.getName() + "'"); - } - } - } - } - - private void testSortableTextSorting() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.addSort(tsort, SolrQuery.ORDER.desc); - query.addField("*"); - query.addField("eoe_sortable"); - query.addField(tsort); - QueryResponse resp = queryRandomShard(query); - - SolrDocumentList docs = resp.getResults(); - - String title = docs.get(0).getFieldValue(tsort).toString(); - for (SolrDocument doc : docs) { - assertTrue( - "Docs should be back in sorted order, descending", - title.compareTo(doc.getFieldValue(tsort).toString()) >= 0); - title = doc.getFieldValue(tsort).toString(); - } - } - - private void testSortableTextGrouping() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.add("group", "true"); - query.add("group.field", tsort); - QueryResponse resp = queryRandomShard(query); - GroupResponse groupResp = resp.getGroupResponse(); - List grpCmds = groupResp.getValues(); - for (GroupCommand grpCmd : grpCmds) { - if (grpCmd.getName().equals(tsort) == false) continue; - for (Group grp : grpCmd.getValues()) { - long count = grp.getResult().getNumFound(); - if (grp.getGroupValue() == null) - continue; // Don't count the groups without an entry as the numnber is variable - switch (grp.getGroupValue()) { - case "all the kings horses and all the kings men": - case "An eye for eye only ends up making the whole world blind.": - case "Great works are performed, not by strength, but by perseverance.": - case "how now brown cow": - case "no eggs on wall, lesson learned": - case "now is the time for all good men": - case "this too shall pass": - case "to come to the aid of their country.": - assertEquals( - "Should have exactly one facet count for field " + grpCmd.getName(), 1, count); - break; - case "the quick fox jumped over the lazy dog": - assertEquals("Should have 5 docs for the lazy dog", 5, count); - break; - default: - fail("No case for facet '" + grpCmd.getName() + "'"); - } - } - } - } - - private void testTokenizedGrouping() throws Exception { - SolrException ex = - expectThrows( - SolrException.class, - () -> { - query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1}); - }); - assertTrue( - "Expected error from server that SortableTextFields are required", - ex.getMessage() - .contains( - "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode")); - } - - private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception { - long found = checkSlicesSameCounts(dColl); - - if (found != expected) { - // we get one do over in a bad race - Thread.sleep(1000); - found = checkSlicesSameCounts(dColl); - } - - assertEquals(msg, expected, checkSlicesSameCounts(dColl)); - } - - // Ensure that total docs found is the expected number. - private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage) - throws Exception { - AtomicLong total = new AtomicLong(-1); - try { - ZkStateReader.from(getCommonCloudSolrClient()) - .waitForState( - DEFAULT_COLLECTION, - waitMillis, - TimeUnit.MILLISECONDS, - (n, c) -> { - long docTotal; - try { - docTotal = checkSlicesSameCounts(c); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - total.set(docTotal); - if (docTotal == expectedNumFound) { - return true; - } - return false; - }); - } catch (TimeoutException | InterruptedException e) { - - } - // We could fail here if we broke out of the above because we exceeded the time allowed. - assertEquals(failureMessage, expectedNumFound, total.get()); - - // This should be redundant, but it caught a test error after all. - for (SolrClient client : clients) { - assertEquals( - failureMessage, - expectedNumFound, - client.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - } - - // Insure that counts are the same for all replicas in each shard - // Return the total doc count for the query. - private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException { - long docTotal = 0; // total number of documents found counting only one replica per slice. - for (Slice slice : dColl.getActiveSlices()) { - long sliceDocCount = -1; - for (Replica rep : slice.getReplicas()) { - try (SolrClient one = getHttpSolrClient(rep)) { - SolrQuery query = new SolrQuery("*:*"); - query.setDistrib(false); - QueryResponse resp = one.query(query); - long hits = resp.getResults().getNumFound(); - if (sliceDocCount == -1) { - sliceDocCount = hits; - docTotal += hits; - } else { - if (hits != sliceDocCount) { - return -1; - } - } - } - } - } - return docTotal; - } - - private void testShardParamVariations() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - Map shardCounts = new HashMap<>(); - - for (String shard : shardToJetty.keySet()) { - // every client should give the same numDocs for this shard - // shffle the clients in a diff order for each shard - List solrclients = new ArrayList<>(this.clients); - Collections.shuffle(solrclients, random()); - for (SolrClient client : solrclients) { - query.set("shards", shard); - long numDocs = client.query(query).getResults().getNumFound(); - assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs); - if (!shardCounts.containsKey(shard)) { - shardCounts.put(shard, numDocs); - } - assertEquals( - "inconsitent numDocs for shard " + shard + " via " + client, - shardCounts.get(shard).longValue(), - numDocs); - - List replicaJetties = new ArrayList<>(shardToJetty.get(shard)); - Collections.shuffle(replicaJetties, random()); - - // each replica should also give the same numDocs - ArrayList replicaAlts = new ArrayList<>(replicaJetties.size() * 2); - for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { - String replica = replicaJetty.url; - query.set("shards", replica); - - // replicas already shuffled, use this in the alternative check below - if (0 == random().nextInt(3) || replicaAlts.size() < 2) { - replicaAlts.add(replica); - } - - numDocs = client.query(query).getResults().getNumFound(); - assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs); - assertEquals( - "inconsitent numDocs for shard " - + shard - + " in replica " - + replica - + " via " - + client, - shardCounts.get(shard).longValue(), - numDocs); - } - - // any combination of replica alternatives should give same numDocs - String replicas = String.join("|", replicaAlts); - query.set("shards", replicas); - numDocs = client.query(query).getResults().getNumFound(); - assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs); - assertEquals( - "inconsitent numDocs for replicas " + replicas + " via " + client, - shardCounts.get(shard).longValue(), - numDocs); - } - } - - // sums of multiple shards should add up regardless of how we - // query those shards or which client we use - long randomShardCountsExpected = 0; - ArrayList randomShards = new ArrayList<>(shardCounts.size()); - for (Map.Entry shardData : shardCounts.entrySet()) { - if (random().nextBoolean() || randomShards.size() < 2) { - String shard = shardData.getKey(); - randomShardCountsExpected += shardData.getValue(); - if (random().nextBoolean()) { - // use shard id - randomShards.add(shard); - } else { - // use some set explicit replicas - ArrayList replicas = new ArrayList<>(7); - for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { - if (0 == random().nextInt(3) || 0 == replicas.size()) { - replicas.add(replicaJetty.url); - } - } - Collections.shuffle(replicas, random()); - randomShards.add(String.join("|", replicas)); - } - } - } - String randShards = String.join(",", randomShards); - query.set("shards", randShards); - for (SolrClient client : this.clients) { - assertEquals( - "numDocs for " + randShards + " via " + client, - randomShardCountsExpected, - client.query(query).getResults().getNumFound()); - } - - // total num docs must match sum of every shard's numDocs - query = new SolrQuery("*:*"); - long totalShardNumDocs = 0; - for (Long c : shardCounts.values()) { - totalShardNumDocs += c; - } - for (SolrClient client : clients) { - assertEquals( - "sum of shard numDocs on client: " + client, - totalShardNumDocs, - client.query(query).getResults().getNumFound()); - } - assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs); - } - - private void testStopAndStartCoresInOneInstance() throws Exception { - JettySolrRunner jetty = jettys.get(0); - try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) { - ThreadPoolExecutor executor = null; - try { - executor = - new ExecutorUtil.MDCAwareThreadPoolExecutor( - 0, - Integer.MAX_VALUE, - 5, - TimeUnit.SECONDS, - new SynchronousQueue(), - new SolrNamedThreadFactory("testExecutor")); - int cnt = 3; - - // create the cores - createCollectionInOneInstance( - httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt); - } finally { - if (executor != null) { - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - } - } - - cloudJettys.get(0).jetty.stop(); - printLayout(); - - cloudJettys.get(0).jetty.start(); - ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2"); - try { - ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000); - } catch (SolrException e) { - printLayout(); - throw e; - } - - printLayout(); - } - - /** Create a collection in single node */ - public static void createCollectionInOneInstance( - final SolrClient client, - String nodeName, - ThreadPoolExecutor executor, - final String collection, - final int numShards, - int numReplicas) { - assertNotNull(nodeName); - try { - assertEquals( - 0, - CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) - .setCreateNodeSet("") - .process(client) - .getStatus()); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - for (int i = 0; i < numReplicas; i++) { - final int freezeI = i; - executor.execute( - () -> { - try { - assertTrue( - CollectionAdminRequest.addReplicaToShard( - collection, "shard" + ((freezeI % numShards) + 1)) - .setCoreName(collection + freezeI) - .setNode(nodeName) - .process(client) - .isSuccess()); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - }); - } - } - - @Override - protected CollectionAdminResponse createCollection( - Map> collectionInfos, - String collectionName, - String configSetName, - int numShards, - int numReplicas, - SolrClient client, - String createNodeSetStr) - throws SolrServerException, IOException { - // TODO: Use CollectionAdminRequest for this test - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - - params.set(CollectionHandlingUtils.NUM_SLICES, numShards); - params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas); - if (createNodeSetStr != null) - params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr); - - int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0; - List list = new ArrayList<>(); - list.add(numShards); - list.add(numReplicas); - if (collectionInfos != null) { - collectionInfos.put(collectionName, list); - } - params.set("name", collectionName); - params.set("collection.configName", configSetName); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - - CollectionAdminResponse res = new CollectionAdminResponse(); - if (client == null) { - final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL(); - - try (SolrClient aClient = createNewSolrClient("", baseUrl)) { - res.setResponse(aClient.request(request)); - } - } else { - res.setResponse(client.request(request)); - } - return res; - } - - @Override - protected Replica getLeaderFromZk(String collection, String slice) { - ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); - Replica leader = clusterState.getCollection(collection).getLeader(slice); - if (leader == null) { - throw new RuntimeException("Could not find leader:" + collection + " " + slice); - } - return leader; - } - - /** - * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two - * (stored) string fields. - * - *

If the values are "double-doubled" or "not-doubled" then we know the processor was not run - * the appropriate number of times - */ - private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception { - - final String fieldA = "regex_dup_A_s"; - final String fieldB = "regex_dup_B_s"; - final String val = "x"; - final String expected = "x_x"; - final ModifiableSolrParams updateParams = new ModifiableSolrParams(); - updateParams.add(UpdateParams.UPDATE_CHAIN, chain); - - final int numLoops = atLeast(50); - - for (int i = 1; i < numLoops; i++) { - // add doc to random client - SolrClient updateClient = clients.get(random().nextInt(clients.size())); - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, id, i, fieldA, val, fieldB, val); - UpdateResponse ures = add(updateClient, updateParams, doc); - assertEquals(chain + ": update failed", 0, ures.getStatus()); - ures = updateClient.commit(); - assertEquals(chain + ": commit failed", 0, ures.getStatus()); - } - - // query for each doc, and check both fields to ensure the value is correct - for (int i = 1; i < numLoops; i++) { - final String query = id + ":" + i; - QueryResponse qres = queryRandomShard(new SolrQuery(query)); - assertEquals(chain + ": query failed: " + query, 0, qres.getStatus()); - assertEquals( - chain + ": didn't find correct # docs with query: " + query, - 1, - qres.getResults().getNumFound()); - SolrDocument doc = qres.getResults().get(0); - - for (String field : new String[] {fieldA, fieldB}) { - assertEquals( - chain + ": doc#" + i + " has wrong value for " + field, - expected, - doc.getFirstValue(field)); - } - } - } - - // cloud level test mainly needed just to make sure that versions and errors are propagated - // correctly - private void doOptimisticLockingAndUpdating() throws Exception { - log.info("### STARTING doOptimisticLockingAndUpdating"); - printLayout(); - - final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1); - indexDoc(sd); - - ignoreException("version conflict"); - for (SolrClient client : clients) { - SolrException e = expectThrows(SolrException.class, () -> client.add(sd)); - assertEquals(409, e.code()); - } - unIgnoreException("version conflict"); - - // TODO: test deletes. SolrJ needs a good way to pass version for delete... - - final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5); - clients.get(0).add(sd2); - - List expected = new ArrayList<>(); - int val = 0; - for (SolrClient client : clients) { - val += 10; - client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val)); - expected.add(val); - } - - QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000")); - for (SolrClient client : clients) { - val += 10; - NamedList rsp = client.request(qr); - String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected); - if (match != null) throw new RuntimeException(match); - } - } - - private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException { - log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd"); - long startCommits = getNumCommits((HttpSolrClient) clients.get(0)); - - NamedList result = - clients - .get(0) - .request( - new StreamingUpdateRequest( - "/update", getFile("books_numeric_ids.csv"), "application/csv") - .setCommitWithin(900000) - .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true)); - - long endCommits = getNumCommits((HttpSolrClient) clients.get(0)); - assertEquals(startCommits + 1L, endCommits); - } - - private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException { - String collection = sourceClient.getDefaultCollection(); - try (SolrClient client = - new HttpSolrClient.Builder(sourceClient.getBaseURL()) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(60000, TimeUnit.MILLISECONDS) - .build()) { - var req = - new GenericSolrRequest( - SolrRequest.METHOD.GET, - "/admin/metrics", - SolrRequest.SolrRequestType.ADMIN, - SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); - - NamedList resp = client.request(req); - try (InputStream in = (InputStream) resp.get("stream")) { - String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); - String metricName = "solr_core_update_commit_ops"; - - return (long) - output - .lines() - .filter( - l -> - l.startsWith(metricName) - && l.contains("collection=\"" + collection + "\"") - && l.contains("ops=\"commits\"")) - .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" ")))) - .sum(); - } - } - } - - private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception { - log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement"); - assertEquals( - 0, - CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - - List collectionClients = new ArrayList<>(); - for (int i = 0; i < 4; i++) { - CollectionAdminResponse resp = - CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1)) - .setNode(jettys.get(0).getNodeName()) - .process(cloudClient); - for (String coreName : resp.getCollectionCoresStatus().keySet()) { - collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); - } - } - - SolrClient client1 = collectionClients.get(0); - SolrClient client2 = collectionClients.get(1); - SolrClient client3 = collectionClients.get(2); - SolrClient client4 = collectionClients.get(3); - - // no one should be recovering - waitForRecoveriesToFinish( - oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true); - - assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient())); - - // printLayout(); - - // TODO: enable when we don't falsely get slice1... - // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000); - // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000); - client2.add(getDoc(id, "1")); - client3.add(getDoc(id, "2")); - client4.add(getDoc(id, "3")); - - client1.commit(); - SolrQuery query = new SolrQuery("*:*"); - query.set("distrib", false); - long oneDocs = client1.query(query).getResults().getNumFound(); - long twoDocs = client2.query(query).getResults().getNumFound(); - long threeDocs = client3.query(query).getResults().getNumFound(); - long fourDocs = client4.query(query).getResults().getNumFound(); - - query.set("collection", oneInstanceCollection2); - query.set("distrib", true); - long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - - // System.out.println("1:" + oneDocs); - // System.out.println("2:" + twoDocs); - // System.out.println("3:" + threeDocs); - // System.out.println("4:" + fourDocs); - // System.out.println("All Docs:" + allDocs); - - // assertEquals(oneDocs, threeDocs); - // assertEquals(twoDocs, fourDocs); - // assertNotSame(oneDocs, twoDocs); - assertEquals(3, allDocs); - - // we added a role of none on these creates - check for it - ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient()); - zkStateReader.forceUpdateCollection(oneInstanceCollection2); - Map slices = - zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap(); - assertNotNull(slices); - - Replica leader = - getCommonCloudSolrClient() - .getClusterState() - .getCollection(oneInstanceCollection2) - .getLeader("shard1"); - - // now test that unloading a core gets us a new leader - try (SolrClient unloadClient = - new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString()) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(60000, TimeUnit.MILLISECONDS) - .build()) { - Unload unloadCmd = new Unload(true); - unloadCmd.setCoreName(leader.getCoreName()); - - String leaderUrl = leader.getCoreUrl(); - - testExecutor.execute( - new Runnable() { - - @Override - public void run() { - try { - unloadClient.request(unloadCmd); - } catch (SolrServerException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - - try { - ZkStateReader.from(getCommonCloudSolrClient()) - .waitForState( - oneInstanceCollection2, - 20000, - TimeUnit.MILLISECONDS, - (n, c) -> { - try { - if (leaderUrl.equals( - zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) { - return false; - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return true; - }); - } catch (TimeoutException | InterruptedException e) { - fail("Leader never changed"); - } - } - - IOUtils.close(collectionClients); - } - - private void testSearchByCollectionName() throws SolrServerException, IOException { - log.info("### STARTING testSearchByCollectionName"); - SolrClient client = clients.get(0); - final String baseUrl = ((HttpSolrClient) client).getBaseURL(); - - // the cores each have different names, but if we add the collection name to the url - // we should get mapped to the right core - try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { - SolrQuery query = new SolrQuery("*:*"); - long oneDocs = client1.query(query).getResults().getNumFound(); - assertEquals(3, oneDocs); - } - } - - private void testUpdateByCollectionName() throws SolrServerException, IOException { - log.info("### STARTING testUpdateByCollectionName"); - SolrClient client = clients.get(0); - final String baseUrl = ((HttpSolrClient) client).getBaseURL(); - - // the cores each have different names, but if we add the collection name to the url - // we should get mapped to the right core - // test hitting an update url - try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { - client1.commit(); - } - } - - private void testANewCollectionInOneInstance() throws Exception { - log.info("### STARTING testANewCollectionInOneInstance"); - CollectionAdminResponse response = - CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2) - .setCreateNodeSet(jettys.get(0).getNodeName()) - .process(cloudClient); - assertEquals(0, response.getStatus()); - List collectionClients = new ArrayList<>(); - for (String coreName : response.getCollectionCoresStatus().keySet()) { - collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); - } - - SolrClient client1 = collectionClients.get(0); - SolrClient client2 = collectionClients.get(1); - SolrClient client3 = collectionClients.get(2); - SolrClient client4 = collectionClients.get(3); - - waitForRecoveriesToFinish( - oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false); - assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient())); - - client2.add(getDoc(id, "1")); - client3.add(getDoc(id, "2")); - client4.add(getDoc(id, "3")); - - client1.commit(); - SolrQuery query = new SolrQuery("*:*"); - query.set("distrib", false); - long oneDocs = client1.query(query).getResults().getNumFound(); - long twoDocs = client2.query(query).getResults().getNumFound(); - long threeDocs = client3.query(query).getResults().getNumFound(); - long fourDocs = client4.query(query).getResults().getNumFound(); - - query.set("collection", oneInstanceCollection); - query.set("distrib", true); - long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - - // System.out.println("1:" + oneDocs); - // System.out.println("2:" + twoDocs); - // System.out.println("3:" + threeDocs); - // System.out.println("4:" + fourDocs); - // System.out.println("All Docs:" + allDocs); - - assertEquals(3, allDocs); - IOUtils.close(collectionClients); - } - - private void createSolrCore( - final String collection, - List collectionClients, - final String baseUrl, - final int num, - final String shardId) { - Callable call = - () -> { - try (SolrClient client = getHttpSolrClient(baseUrl)) { - // client.setConnectionTimeout(15000); - Create createCmd = new Create(); - createCmd.setCoreName(collection + num); - createCmd.setCollection(collection); - - if (random().nextBoolean()) { - // sometimes we use an explicit core node name - createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet()); - } - - if (shardId == null) { - createCmd.setNumShards(2); - } - createCmd.setDataDir(getDataDir(createTempDir(collection).toString())); - if (shardId != null) { - createCmd.setShardId(shardId); - } - client.request(createCmd); - } catch (Exception e) { - log.error("error creating core", e); - // fail - } - return null; - }; - - pending.add(completionService.submit(call)); - - collectionClients.add(createNewSolrClient(collection + num, baseUrl)); - } - - private void testMultipleCollections() throws Exception { - log.info("### STARTING testMultipleCollections"); - // create another 2 collections and search across them - createNewCollection("collection2"); - createNewCollection("collection3"); - - while (pending != null && pending.size() > 0) { - - Future future = completionService.take(); - if (future == null) return; - pending.remove(future); - } - - indexDoc("collection2", getDoc(id, "10000000")); - indexDoc("collection2", getDoc(id, "10000001")); - indexDoc("collection2", getDoc(id, "10000003")); - - getSolrClient("collection2").add(getDoc(id, "10000004")); - - indexDoc("collection3", getDoc(id, "20000000")); - indexDoc("collection3", getDoc(id, "20000001")); - - getSolrClient("collection3").add(getDoc(id, "10000005")); - - otherCollectionClients.get("collection2").get(0).commit(); - otherCollectionClients.get("collection3").get(0).commit(); - - SolrClient clientForCollection1 = getSolrClient("collection1"); - long collection1Docs = - clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound(); - - long collection2Docs = - otherCollectionClients - .get("collection2") - .get(0) - .query(new SolrQuery("*:*")) - .getResults() - .getNumFound(); - - long collection3Docs = - otherCollectionClients - .get("collection3") - .get(0) - .query(new SolrQuery("*:*")) - .getResults() - .getNumFound(); - - SolrQuery query = new SolrQuery("*:*"); - query.set("collection", "collection2,collection3"); - long found = clients.get(0).query(query).getResults().getNumFound(); - assertEquals(collection2Docs + collection3Docs, found); - - query = new SolrQuery("*:*"); - query.set("collection", "collection1,collection2,collection3"); - found = clients.get(0).query(query).getResults().getNumFound(); - assertEquals(collection1Docs + collection2Docs + collection3Docs, found); - - // try to search multiple with cloud client - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection1Docs + collection2Docs + collection3Docs, found); - - query.set("collection", "collection2,collection3"); - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection2Docs + collection3Docs, found); - - query.set("collection", "collection3"); - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection3Docs, found); - - query.remove("collection"); - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection1Docs, found); - - assertEquals(collection3Docs, collection2Docs - 1); - } - - protected void indexDoc(String collection, SolrInputDocument doc) - throws IOException, SolrServerException { - List clients = otherCollectionClients.get(collection); - int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size(); - SolrClient client = clients.get(which); - client.add(doc); - } - - @SuppressWarnings({"unchecked"}) - private void createNewCollection(final String collection) throws InterruptedException { - try { - assertEquals( - 0, - CollectionAdminRequest.createCollection(collection, "conf1", 2, 1) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - } catch (Exception e) { - log.error("error creating collection", e); - // fails - } - final List collectionClients = new ArrayList<>(); - otherCollectionClients.put(collection, collectionClients); - int unique = 0; - for (final JettySolrRunner runner : jettys) { - unique++; - final int frozeUnique = unique; - Callable call = - () -> { - try { - assertTrue( - CollectionAdminRequest.addReplicaToShard( - collection, "shard" + ((frozeUnique % 2) + 1)) - .setNode(runner.getNodeName()) - .process(cloudClient) - .isSuccess()); - } catch (Exception e) { - log.error("error adding replica", e); - // fails - } - return null; - }; - - collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString())); - pending.add(completionService.submit(call)); - while (pending != null && pending.size() > 0) { - - Future future = completionService.take(); - if (future == null) return; - pending.remove(future); - } - } - } - - @Override - protected SolrClient createNewSolrClient(String collection, String baseUrl) { - - SolrClient client = getHttpSolrClient(baseUrl, collection); - - return client; - } - - /** - * @param collection the name of a collection or core to set as the "default" on the created - * client. - * @param baseUrl the "base" URL of a Solr node. Should not contain a collection or core - * name. - * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds - * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds - */ - protected SolrClient createNewSolrClient( - String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) { - - SolrClient client = - new HttpSolrClient.Builder(baseUrl) - .withDefaultCollection(collection) - .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS) - .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS) - .build(); - - return client; - } - - @Override - protected QueryResponse queryRandomShard(ModifiableSolrParams params) - throws SolrServerException, IOException { - - if (r.nextBoolean()) return super.queryRandomShard(params); - - if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION); - - QueryResponse rsp = getCommonCloudSolrClient().query(params); - return rsp; - } - - @Override - public void distribTearDown() throws Exception { - super.distribTearDown(); - if (otherCollectionClients != null) { - for (List clientList : otherCollectionClients.values()) { - IOUtils.close(clientList); - } - } - otherCollectionClients = null; - List tasks = executor.shutdownNow(); - assertTrue(tasks.isEmpty()); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java deleted file mode 100644 index fba15eb5416..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -@LuceneTestCase.Nightly -public abstract class AbstractChaosMonkeyNothingIsSafeTestBase - extends AbstractFullDistribZkTestBase { - private static final int FAIL_TOLERANCE = 100; - - private static final Integer RUN_LENGTH = - Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); - - private final boolean onlyLeaderIndexes = random().nextBoolean(); - - @BeforeClass - public static void beforeSuperClass() { - schemaString = "schema15.xml"; // we need a string id - System.setProperty("solr.autoCommit.maxTime", "15000"); - System.clearProperty("solr.httpclient.retries"); - System.clearProperty("solr.retries.on.forward"); - System.clearProperty("solr.retries.to.followers"); - setErrorHook(); - } - - @AfterClass - public static void afterSuperClass() { - System.clearProperty("solr.autoCommit.maxTime"); - clearErrorHook(); - } - - @Override - protected void destroyServers() throws Exception { - - super.destroyServers(); - } - - protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; - protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; - - private int clientSoTimeout = 60000; - - private volatile FullThrottleStoppableIndexingThread ftIndexThread; - - private final boolean runFullThrottle; - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public RandVal[] getRandValues() { - return randVals; - } - - @Override - public void distribSetUp() throws Exception { - super.distribSetUp(); - // can help to hide this when testing and looking at logs - // ignoreException("shard update error"); - useFactory("solr.StandardDirectoryFactory"); - } - - @Override - public void distribTearDown() throws Exception { - try { - ftIndexThread.safeStop(); - } catch (NullPointerException e) { - // okay - } - super.distribTearDown(); - } - - public AbstractChaosMonkeyNothingIsSafeTestBase() { - super(); - sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); - if (sliceCount == -1) { - sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; - } - - int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); - if (numShards == -1) { - // we make sure that there's at least one shard with more than one replica - // so that the ChaosMonkey has something to kill - numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; - } - fixShardCount(numShards); - - // TODO: we only do this sometimes so that we can sometimes compare against control, - // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer - runFullThrottle = random().nextBoolean(); - } - - @Override - protected boolean useTlogReplicas() { - return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug - // and it's TestInjection use - } - - @Override - protected CloudSolrClient createCloudClient(String defaultCollection) { - return this.createCloudClient(defaultCollection, this.clientSoTimeout); - } - - protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) { - - return getCloudSolrClient( - zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout); - } - - @Test - @SuppressWarnings({"try"}) - public void test() throws Exception { - // None of the operations used here are particularly costly, so this should work. - // Using this low timeout will also help us catch index stalling. - clientSoTimeout = 5000; - - boolean testSuccessful = false; - try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) { - handle.clear(); - handle.put("timestamp", SKIPVAL); - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - // make sure we have leaders for each shard - for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); - } // make sure we again have leaders for each shard - - waitForRecoveriesToFinish(false); - - // we cannot do delete by query - // as it's not supported for recovery - del("*:*"); - - List threads = new ArrayList<>(); - List indexTreads = new ArrayList<>(); - int threadCount = TEST_NIGHTLY ? 3 : 1; - int i = 0; - for (i = 0; i < threadCount; i++) { - StoppableIndexingThread indexThread = - new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true); - threads.add(indexThread); - indexTreads.add(indexThread); - indexThread.start(); - } - - threadCount = 1; - i = 0; - for (i = 0; i < threadCount; i++) { - StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient); - threads.add(searchThread); - searchThread.start(); - } - - if (runFullThrottle) { - ftIndexThread = - new FullThrottleStoppableIndexingThread( - ((CloudLegacySolrClient) cloudClient).getHttpClient(), - controlClient, - cloudClient, - clients, - "ft1", - true, - this.clientSoTimeout); - ftIndexThread.start(); - } - - chaosMonkey.startTheMonkey(true, 10000); - try { - long runLength; - if (RUN_LENGTH != -1) { - runLength = RUN_LENGTH; - } else { - int[] runTimes; - if (TEST_NIGHTLY) { - runTimes = - new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; - } else { - runTimes = new int[] {5000, 7000, 15000}; - } - runLength = runTimes[random().nextInt(runTimes.length - 1)]; - } - - Thread.sleep(runLength); - } finally { - chaosMonkey.stopTheMonkey(); - } - - // ideally this should go into chaosMonkey - restartZk(1000 * (5 + random().nextInt(4))); - - if (runFullThrottle) { - ftIndexThread.safeStop(); - } - - for (StoppableThread indexThread : threads) { - indexThread.safeStop(); - } - - // start any downed jetties to be sure we still will end up with a leader per shard... - - // wait for stop... - for (StoppableThread indexThread : threads) { - indexThread.join(); - } - - // try and wait for any replications and what not to finish... - - Thread.sleep(2000); - - // wait until there are no recoveries... - waitForThingsToLevelOut(); - - // make sure we again have leaders for each shard - for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000); - } - - commit(); - - // TODO: assert we didn't kill everyone - - zkStateReader.updateLiveNodes(); - assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0); - - // we expect full throttle fails, but cloud client should not easily fail - for (StoppableThread indexThread : threads) { - if (indexThread instanceof StoppableIndexingThread - && !(indexThread instanceof FullThrottleStoppableIndexingThread)) { - int failCount = ((StoppableIndexingThread) indexThread).getFailCount(); - assertFalse( - "There were too many update fails (" - + failCount - + " > " - + FAIL_TOLERANCE - + ") - we expect it can happen, but shouldn't easily", - failCount > FAIL_TOLERANCE); - } - } - - waitForThingsToLevelOut(20, TimeUnit.SECONDS); - - commit(); - - Set addFails = getAddFails(indexTreads); - Set deleteFails = getDeleteFails(indexTreads); - // full throttle thread can - // have request fails - checkShardConsistency(!runFullThrottle, true, addFails, deleteFails); - - long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - // ensure we have added more than 0 docs - long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0); - - if (VERBOSE) - System.out.println( - "control docs:" - + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() - + "\n\n"); - - // try and make a collection to make sure the overseer has survived the expiration and session - // loss - - // sometimes we restart zookeeper as well - if (random().nextBoolean()) { - // restartZk(1000 * (5 + random().nextInt(4))); - } - - try (CloudSolrClient client = createCloudClient("collection1", 30000)) { - createCollection(null, "testcollection", 1, 1, client, null, "conf1"); - } - List numShardsNumReplicas = new ArrayList<>(2); - numShardsNumReplicas.add(1); - numShardsNumReplicas.add(1); - checkForCollection("testcollection", numShardsNumReplicas); - - testSuccessful = true; - } finally { - if (!testSuccessful) { - printLayout(); - } - } - } - - private Set getAddFails(List threads) { - Set addFails = new HashSet<>(); - for (StoppableIndexingThread thread : threads) { - addFails.addAll(thread.getAddFails()); - } - return addFails; - } - - private Set getDeleteFails(List threads) { - Set deleteFails = new HashSet<>(); - for (StoppableIndexingThread thread : threads) { - deleteFails.addAll(thread.getDeleteFails()); - } - return deleteFails; - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = getDoc(fields); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java deleted file mode 100644 index ef371efcad4..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.common.SolrInputDocument; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@LuceneTestCase.Nightly -public abstract class AbstractChaosMonkeySafeLeaderTestBase extends AbstractFullDistribZkTestBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final Integer RUN_LENGTH = - Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); - - @BeforeClass - public static void beforeSuperClass() { - schemaString = "schema15.xml"; // we need a string id - System.setProperty("solr.autoCommit.maxTime", "15000"); - System.clearProperty("solr.httpclient.retries"); - System.clearProperty("solr.retries.on.forward"); - System.clearProperty("solr.retries.to.followers"); - setErrorHook(); - } - - @AfterClass - public static void afterSuperClass() { - System.clearProperty("solr.autoCommit.maxTime"); - clearErrorHook(); - } - - protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; - protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public RandVal[] getRandValues() { - return randVals; - } - - protected abstract String getDirectoryFactory(); - - @Override - public void distribSetUp() throws Exception { - useFactory(getDirectoryFactory()); - super.distribSetUp(); - } - - public AbstractChaosMonkeySafeLeaderTestBase() { - super(); - sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); - if (sliceCount == -1) { - sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; - } - - int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); - if (numShards == -1) { - // we make sure that there's at least one shard with more than one replica - // so that the ChaosMonkey has something to kill - numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; - } - fixShardCount(numShards); - } - - @Test - public void test() throws Exception { - - handle.clear(); - handle.put("timestamp", SKIPVAL); - - // randomly turn on 1 seconds 'soft' commit - randomlyEnableAutoSoftCommit(); - - tryDelete(); - - List threads = new ArrayList<>(); - int threadCount = 2; - int batchSize = 1; - if (random().nextBoolean()) { - batchSize = random().nextInt(98) + 2; - } - - boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true; - int maxUpdates = -1; - if (!pauseBetweenUpdates) { - maxUpdates = 1000 + random().nextInt(1000); - } else { - maxUpdates = 15000; - } - - for (int i = 0; i < threadCount; i++) { - StoppableIndexingThread indexThread = - new StoppableIndexingThread( - controlClient, - cloudClient, - Integer.toString(i), - true, - maxUpdates, - batchSize, - pauseBetweenUpdates); // random().nextInt(999) + 1 - threads.add(indexThread); - indexThread.start(); - } - - chaosMonkey.startTheMonkey(false, 500); - try { - long runLength; - if (RUN_LENGTH != -1) { - runLength = RUN_LENGTH; - } else { - int[] runTimes; - if (TEST_NIGHTLY) { - runTimes = - new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; - } else { - runTimes = new int[] {5000, 7000, 15000}; - } - runLength = runTimes[random().nextInt(runTimes.length - 1)]; - } - - Thread.sleep(runLength); - } finally { - chaosMonkey.stopTheMonkey(); - } - - for (StoppableIndexingThread indexThread : threads) { - indexThread.safeStop(); - } - - // wait for stop... - for (StoppableIndexingThread indexThread : threads) { - indexThread.join(); - } - - for (StoppableIndexingThread indexThread : threads) { - assertEquals(0, indexThread.getFailCount()); - } - - // try and wait for any replications and what not to finish... - - Thread.sleep(2000); - - waitForThingsToLevelOut(3, TimeUnit.MINUTES); - - // even if things were leveled out, a jetty may have just been stopped or something - // we wait again and wait to level out again to make sure the system is not still in flux - - Thread.sleep(3000); - - waitForThingsToLevelOut(3, TimeUnit.MINUTES); - - checkShardConsistency(batchSize == 1, true); - - if (VERBOSE) - System.out.println( - "control docs:" - + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() - + "\n\n"); - - // try and make a collection to make sure the overseer has survived the expiration and session - // loss - - // sometimes we restart zookeeper as well - if (random().nextBoolean()) { - zkServer.shutdown(); - zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); - zkServer.run(false); - } - - try (CloudSolrClient client = createCloudClient("collection1")) { - createCollection(null, "testcollection", 1, 1, client, null, "conf1"); - } - List numShardsNumReplicas = new ArrayList<>(2); - numShardsNumReplicas.add(1); - numShardsNumReplicas.add(1); - checkForCollection("testcollection", numShardsNumReplicas); - } - - private void tryDelete() throws Exception { - long start = System.nanoTime(); - long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS); - while (System.nanoTime() < timeout) { - try { - del("*:*"); - break; - } catch (SolrServerException e) { - log.error("cluster may not be up yet", e); - } - Thread.sleep(100); - } - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java deleted file mode 100644 index 31bfdb837c6..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java +++ /dev/null @@ -1,567 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME; -import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE; - -import java.io.FileWriter; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.cloud.DigestZkACLProvider; -import org.apache.solr.common.cloud.DigestZkCredentialsProvider; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector; -import org.apache.solr.common.cloud.ZkCredentialsInjector; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException.NoAuthException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AbstractDigestZkACLAndCredentialsProvidersTestBase extends SolrTestCaseJ4 { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final Charset DATA_ENCODING = StandardCharsets.UTF_8; - - private static final String ALL_USERNAME = "connectAndAllACLUsername"; - private static final String ALL_PASSWORD = "connectAndAllACLPassword"; - private static final String READONLY_USERNAME = "readonlyACLUsername"; - private static final String READONLY_PASSWORD = "readonlyACLPassword"; - - protected ZkTestServer zkServer; - - protected Path zkDir; - - @BeforeClass - public static void beforeClass() { - System.setProperty("solrcloud.skip.autorecovery", "true"); - } - - @AfterClass - public static void afterClass() { - System.clearProperty("solrcloud.skip.autorecovery"); - } - - @Override - public void setUp() throws Exception { - // TODO: Does all of this setup need to happen for each test case, or can it be done once for - // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full - // startup and teardown of a new ZkTestServer in each cycle? - super.setUp(); - if (log.isInfoEnabled()) { - log.info("####SETUP_START {}", getTestName()); - } - createTempDir(); - - zkDir = createTempDir().resolve("zookeeper/server1/data"); - log.info("ZooKeeper dataDir:{}", zkDir); - setSecuritySystemProperties(); - zkServer = new ZkTestServer(zkDir); - zkServer.run(false); - - System.setProperty("zkHost", zkServer.getZkAddress()); - - setDigestZkSystemProps(); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); - - SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkHost()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - zkClient.makePath("/solr", false, true); - zkClient.close(); - - zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - zkClient.create( - "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - zkClient.makePath( - "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - - zkClient.create( - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - zkClient.close(); - - clearSecuritySystemProperties(); - - zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - // Currently, no credentials on ZK connection, because those same VM-params are used for adding - // ACLs, and here we want - // no (or completely open) ACLs added. Therefore, hack your way into being authorized for - // creating anyway - zkClient - .getCuratorFramework() - .getZookeeperClient() - .getZooKeeper() - .addAuthInfo( - "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8)); - zkClient.create( - "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - zkClient.makePath( - "/unprotectedMakePathNode", - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - zkClient.close(); - - setDigestZkSystemProps(); - if (log.isInfoEnabled()) { - log.info("####SETUP_END {}", getTestName()); - } - } - - private void setDigestZkSystemProps() { - System.setProperty( - SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, - DigestZkCredentialsProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); - } - - @Override - public void tearDown() throws Exception { - zkServer.shutdown(); - - clearSecuritySystemProperties(); - - super.tearDown(); - } - - @Test - public void testNoCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class), - new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class)); - - testInjectors( - testZkCredentialsInjectors, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false); - } - - @Test - public void testWrongCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class), - new TestZkCredentialsInjector( - VMParamsZkCredentialsInjector.class, - List.of( - DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), - List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong"))); - - testInjectors( - testZkCredentialsInjectors, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false); - } - - @Test - public void testAllCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class), - new TestZkCredentialsInjector( - VMParamsZkCredentialsInjector.class, - List.of( - DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), - List.of(ALL_USERNAME, ALL_PASSWORD))); - - testInjectors( - testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true); - } - - @Test - public void testReadonlyCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class), - new TestZkCredentialsInjector( - VMParamsZkCredentialsInjector.class, - List.of( - DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), - List.of(READONLY_USERNAME, READONLY_PASSWORD))); - testInjectors( - testZkCredentialsInjectors, - true, - true, - false, - false, - false, - false, - false, - false, - false, - false); - } - - protected void testInjectors( - List testZkCredentialsInjectors, - boolean getData, - boolean list, - boolean create, - boolean setData, - boolean delete, - boolean secureGet, - boolean secureList, - boolean secureCreate, - boolean secureSet, - boolean secureDelete) - throws Exception { - for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) { - tearDown(); - setUp(); - testZkCredentialsInjector.setSystemProps(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - doTest( - zkClient, - getData, - list, - create, - setData, - delete, - secureGet, - secureList, - secureCreate, - secureSet, - secureDelete); - } - } - } - - @Test - public void testRepairACL() throws Exception { - clearSecuritySystemProperties(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - // Currently, no credentials on ZK connection, because those same VM-params are used for - // adding ACLs, and here we want - // no (or completely open) ACLs added. Therefore, hack your way into being authorized for - // creating anyway - zkClient - .getCuratorFramework() - .getZookeeperClient() - .getZooKeeper() - .addAuthInfo( - "digest", - ("connectAndAllACLUsername:connectAndAllACLPassword") - .getBytes(StandardCharsets.UTF_8)); - - zkClient.create( - "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false); - assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); - } - - setSecuritySystemProperties(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - ZkController.createClusterZkNodes(zkClient); - assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); - } - - useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class); - // useReadonlyCredentials(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - NoAuthException e = - assertThrows( - NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false)); - assertEquals("/solr/security.json", e.getPath()); - } - } - - private void useZkCredentialsInjector(Class zkCredentialsInjectorClass) { - clearSecuritySystemProperties(); - setDigestZkSystemProps(); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - zkCredentialsInjectorClass.getName()); - } - - private void setSecuritySystemProperties() { - System.setProperty( - SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, - DigestZkCredentialsProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); - } - - private void clearSecuritySystemProperties() { - System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); - System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); - System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME); - } - - public static void doTest( - SolrZkClient zkClient, - boolean getData, - boolean list, - boolean create, - boolean setData, - boolean delete, - boolean secureGet, - boolean secureList, - boolean secureCreate, - boolean secureSet, - boolean secureDelete) - throws Exception { - doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete); - doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete); - doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete); - doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete); - doTest( - zkClient, - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, - secureGet, - secureList, - secureCreate, - secureSet, - secureDelete); - } - - protected static void doTest( - SolrZkClient zkClient, - String path, - boolean getData, - boolean list, - boolean create, - boolean setData, - boolean delete) - throws Exception { - doTest(getData, () -> zkClient.getData(path, null, null, false)); - doTest(list, () -> zkClient.getChildren(path, null, false)); - - doTest( - create, - () -> { - zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false); - zkClient.delete(path + "/subnode", -1, false); - }); - doTest( - create, - () -> { - zkClient.makePath(path + "/subnode/subsubnode", false); - zkClient.delete(path + "/subnode/subsubnode", -1, false); - zkClient.delete(path + "/subnode", -1, false); - }); - - doTest(setData, () -> zkClient.setData(path, (byte[]) null, false)); - - // Actually about the ACLs on /solr, but that is protected - doTest(delete, () -> zkClient.delete(path, -1, false)); - } - - interface ExceptingRunnable { - void run() throws Exception; - } - - private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception { - if (shouldSucceed) { - action.run(); - } else { - expectThrows(NoAuthException.class, action::run); - } - } - - @Test - public void testVMParamsAllCredentialsFromFile() throws Exception { - useVMParamsAllCredentialsFromFile(); - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - doTest(zkClient, true, true, true, true, true, true, true, true, true, true); - } - } - - @Test - public void testVMParamsReadonlyCredentialsFromFile() throws Exception { - useVMParamsReadonlyCredentialsFromFile(); - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - doTest(zkClient, true, true, false, false, false, false, false, false, false, false); - } - } - - private void useVMParamsAllCredentialsFromFile() throws IOException { - useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword"); - } - - private void useVMParamsReadonlyCredentialsFromFile() throws IOException { - useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword"); - } - - private void useVMParamsCredentialsFromFile(String username, String password) throws IOException { - Properties props = new Properties(); - props.setProperty( - VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username); - props.setProperty( - VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password); - String credsFile = saveCredentialsFile(props); - - useZkCredentialsInjector(VMParamsZkCredentialsInjector.class); - System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile); - } - - private String saveCredentialsFile(Properties props) throws IOException { - Path tmp = createTempFile("zk-creds", "properties"); - try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) { - props.store(w, "test"); - } - return tmp.toAbsolutePath().toString(); - } - - public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return Collections.emptyList(); - } - } - - public static class AllAndReadonlyCredentialZkCredentialsInjector - implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of( - new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL), - new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ)); - } - } - - public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of( - new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL)); - } - } - - public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL)); - } - } - - public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of( - new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL)); - } - } - - class TestZkCredentialsInjector { - private final Class zkCredentialsInjectorClass; - private final List systemPropsKeys; - private final List systemPropsValues; - - public TestZkCredentialsInjector(Class zkCredentialsInjectorClass) { - this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList()); - } - - public TestZkCredentialsInjector( - Class zkCredentialsInjectorClass, - List systemPropsKeys, - List systemPropsValues) { - this.zkCredentialsInjectorClass = zkCredentialsInjectorClass; - this.systemPropsKeys = systemPropsKeys; - this.systemPropsValues = systemPropsValues; - } - - private void setSystemProps() { - clearSecuritySystemProperties(); - setDigestZkSystemProps(); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - zkCredentialsInjectorClass.getName()); - int i = 0; - for (String key : systemPropsKeys) { - System.setProperty(key, systemPropsValues.get(i++)); - } - } - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java deleted file mode 100644 index 2ca97dbbbe4..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.solr.client.api.model.CoreStatusResponse; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.IdUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractMoveReplicaTestBase extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - // used by MoveReplicaHDFSTest - protected boolean inPlaceMove = true; - protected boolean isCollectionApiDistributed = false; - - protected String getConfigSet() { - return "cloud-dynamic"; - } - - @Before - public void beforeTest() throws Exception { - inPlaceMove = true; - - configureCluster(4) - .addConfig("conf1", configset(getConfigSet())) - .addConfig("conf2", configset(getConfigSet())) - .withSolrXml(TEST_PATH().resolve("solr.xml")) - .configure(); - - // If Collection API is distributed let's not wait for Overseer. - isCollectionApiDistributed = - new CollectionAdminRequest.RequestApiDistributedProcessing() - .process(cluster.getSolrClient()) - .getIsCollectionApiDistributed(); - if (isCollectionApiDistributed) { - return; - } - - NamedList overSeerStatus = - cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - JettySolrRunner overseerJetty = null; - String overseerLeader = (String) overSeerStatus.get("leader"); - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - if (jetty.getNodeName().equals(overseerLeader)) { - overseerJetty = jetty; - break; - } - } - if (overseerJetty == null) { - fail("no overseer leader!"); - } - } - - @After - public void afterTest() throws Exception { - try { - shutdownCluster(); - } finally { - super.tearDown(); - } - } - - @Test - public void test() throws Exception { - String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove; - if (log.isInfoEnabled()) { - log.info("total_jettys: {}", cluster.getJettySolrRunners().size()); - } - int REPLICATION = 2; - - CloudSolrClient cloudClient = cluster.getSolrClient(); - - // random create tlog or pull type replicas with nrt - boolean isTlog = random().nextBoolean(); - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection( - coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); - cloudClient.request(create); - - addDocs(coll, 100); - - Replica replica = getRandomReplica(coll, cloudClient); - Set liveNodes = cloudClient.getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - Collections.shuffle(l, random()); - String targetNode = null; - for (String node : liveNodes) { - if (!replica.getNodeName().equals(node)) { - targetNode = node; - break; - } - } - assertNotNull(targetNode); - String shardId = null; - for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) { - if (slice.getReplicas().contains(replica)) { - shardId = slice.getName(); - } - } - - int sourceNumCores = - getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()); - int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()); - - CollectionAdminRequest.MoveReplica moveReplica = - createMoveReplicaRequest(coll, replica, targetNode); - moveReplica.setInPlaceMove(inPlaceMove); - String asyncId = IdUtils.randomId(); - moveReplica.processAsync(asyncId, cloudClient); - CollectionAdminRequest.RequestStatus requestStatus = - CollectionAdminRequest.requestStatus(asyncId); - // wait for async request success - boolean success = false; - for (int i = 0; i < 200; i++) { - CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); - if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { - success = true; - break; - } - assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED); - Thread.sleep(500); - } - assertTrue(success); - assertEquals( - "should be one less core on the source node!", - sourceNumCores - 1, - getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name())); - assertEquals( - "should be one more core on target node!", - targetNumCores + 1, - getNumOfCores(cloudClient, targetNode, coll, replica.getType().name())); - // wait for recovery - boolean recovered = false; - for (int i = 0; i < 300; i++) { - DocCollection collState = getCollectionState(coll); - log.debug("###### {}", collState); - Collection replicas = collState.getSlice(shardId).getReplicas(); - boolean allActive = true; - boolean hasLeaders = true; - if (replicas != null && !replicas.isEmpty()) { - for (Replica r : replicas) { - if (!r.getNodeName().equals(targetNode)) { - continue; - } - if (!r.isActive(Collections.singleton(targetNode))) { - log.info("Not active: {}", r); - allActive = false; - } - } - } else { - allActive = false; - } - for (Slice slice : collState.getSlices()) { - if (slice.getLeader() == null) { - hasLeaders = false; - } - } - if (allActive && hasLeaders) { - // check the number of active replicas - assertEquals("total number of replicas", REPLICATION, replicas.size()); - recovered = true; - break; - } else { - log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders); - Thread.sleep(1000); - } - } - assertTrue("replica never fully recovered", recovered); - - assertEquals( - 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); - - moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId); - moveReplica.setInPlaceMove(inPlaceMove); - moveReplica.process(cloudClient); - checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores); - // wait for recovery - recovered = false; - for (int i = 0; i < 300; i++) { - DocCollection collState = getCollectionState(coll); - log.debug("###### {}", collState); - Collection replicas = collState.getSlice(shardId).getReplicas(); - boolean allActive = true; - boolean hasLeaders = true; - if (replicas != null && !replicas.isEmpty()) { - for (Replica r : replicas) { - if (!r.getNodeName().equals(replica.getNodeName())) { - continue; - } - if (!r.isActive(Collections.singleton(replica.getNodeName()))) { - log.info("Not active yet: {}", r); - allActive = false; - } - } - } else { - allActive = false; - } - for (Slice slice : collState.getSlices()) { - if (slice.getLeader() == null) { - hasLeaders = false; - } - } - if (allActive && hasLeaders) { - assertEquals("total number of replicas", REPLICATION, replicas.size()); - recovered = true; - break; - } else { - Thread.sleep(1000); - } - } - assertTrue("replica never fully recovered", recovered); - - assertEquals( - 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); - } - - @Test - public void testFailedMove() throws Exception { - String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove; - int REPLICATION = 2; - - CloudSolrClient cloudClient = cluster.getSolrClient(); - - // random create tlog or pull type replicas with nrt - boolean isTlog = random().nextBoolean(); - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection( - coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); - cloudClient.request(create); - - addDocs(coll, 100); - - NamedList overSeerStatus = - cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - - // don't kill overseer in this test - Replica replica; - int count = 10; - do { - replica = getRandomReplica(coll, cloudClient); - } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0); - assertNotNull("could not find non-overseer replica???", replica); - Set liveNodes = cloudClient.getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - Collections.shuffle(l, random()); - String targetNode = null; - for (String node : liveNodes) { - if (!replica.getNodeName().equals(node) - && (isCollectionApiDistributed || !overseerLeader.equals(node))) { - targetNode = node; - break; - } - } - assertNotNull(targetNode); - CollectionAdminRequest.MoveReplica moveReplica = - createMoveReplicaRequest(coll, replica, targetNode); - moveReplica.setInPlaceMove(inPlaceMove); - // start moving - String asyncId = IdUtils.randomId(); - moveReplica.processAsync(asyncId, cloudClient); - // shut down target node - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - CollectionAdminRequest.RequestStatus requestStatus = - CollectionAdminRequest.requestStatus(asyncId); - // wait for async request success - boolean success = true; - for (int i = 0; i < 200; i++) { - CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); - assertNotSame( - rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED); - if (rsp.getRequestStatus() == RequestStatusState.FAILED) { - success = false; - break; - } - Thread.sleep(500); - } - assertFalse(success); - - if (log.isInfoEnabled()) { - log.info( - "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll)); - } - assertEquals( - 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); - } - - private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( - String coll, Replica replica, String targetNode, String shardId) { - return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName()); - } - - private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( - String coll, Replica replica, String targetNode) { - return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode); - } - - private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException { - List replicas = cloudClient.getClusterState().getCollection(coll).getReplicas(); - Collections.shuffle(replicas, random()); - return replicas.get(0); - } - - private void checkNumOfCores( - CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores) - throws IOException, SolrServerException { - assertEquals( - nodeName + " does not have expected number of cores", - expectedCores, - getNumOfCores(cloudClient, nodeName, collectionName)); - } - - private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName) - throws IOException, SolrServerException { - return getNumOfCores(cloudClient, nodeName, collectionName, null); - } - - private int getNumOfCores( - CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType) - throws IOException, SolrServerException { - try (SolrClient coreclient = - getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) { - CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient); - if (status.getCoreStatus().size() == 0) { - return 0; - } - if (collectionName == null && replicaType == null) { - return status.getCoreStatus().size(); - } - // filter size by collection name - int size = 0; - for (Map.Entry coreStatusEntry : - status.getCoreStatus().entrySet()) { - if (collectionName != null) { - String coll = coreStatusEntry.getValue().cloud.collection; - if (!collectionName.equals(coll)) { - continue; - } - } - if (replicaType != null) { - String type = coreStatusEntry.getValue().cloud.replicaType; - if (!replicaType.equals(type)) { - continue; - } - } - size++; - } - return size; - } - } - - protected void addDocs(String collection, int numDocs) throws Exception { - SolrClient solrClient = cluster.getSolrClient(); - for (int docId = 1; docId <= numDocs; docId++) { - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", docId); - solrClient.add(collection, doc); - } - solrClient.commit(collection); - Thread.sleep(5000); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java deleted file mode 100644 index 1e7aa064edd..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.embedded.JettySolrRunner; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractRecoveryZkTestBase extends SolrCloudTestCase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() throws Exception { - cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure(); - } - - private final List threads = new ArrayList<>(); - - @After - public void stopThreads() throws InterruptedException { - for (StoppableIndexingThread t : threads) { - t.safeStop(); - } - for (StoppableIndexingThread t : threads) { - t.join(); - } - threads.clear(); - } - - @Test - public void test() throws Exception { - - final String collection = "recoverytest"; - - CollectionAdminRequest.createCollection(collection, "conf", 1, 2) - .process(cluster.getSolrClient()); - waitForState( - "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2)); - - // start a couple indexing threads - - int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000}; - int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000}; - - int maxDoc; - if (!TEST_NIGHTLY) { - maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; - } else { - maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)]; - } - log.info("Indexing {} documents", maxDoc); - - try (SolrClient solrClient = - cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) { - final StoppableIndexingThread indexThread = - new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true); - threads.add(indexThread); - indexThread.start(); - - final StoppableIndexingThread indexThread2 = - new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true); - threads.add(indexThread2); - indexThread2.start(); - - // give some time to index... - int[] waitTimes = new int[] {200, 2000, 3000}; - Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); - - // bring shard replica down - DocCollection state = getCollectionState(collection); - Replica leader = state.getLeader("shard1"); - Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r)); - - JettySolrRunner jetty = cluster.getReplicaJetty(replica); - jetty.stop(); - - // wait a moment - lets allow some docs to be indexed so replication time is non 0 - Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); - - // bring shard replica up - jetty.start(); - - // make sure replication can start - Thread.sleep(3000); - - // stop indexing threads - indexThread.safeStop(); - indexThread2.safeStop(); - - indexThread.join(); - indexThread2.join(); - - new UpdateRequest().commit(solrClient, collection); - - cluster - .getZkStateReader() - .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2)); - - // test that leader and replica have same doc count - state = getCollectionState(collection); - assertShardConsistency(state.getSlice("shard1"), true); - } - } - - private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception { - List replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE); - long[] numCounts = new long[replicas.size()]; - int i = 0; - for (Replica replica : replicas) { - try (var client = - new HttpSolrClient.Builder(replica.getBaseUrl()) - .withDefaultCollection(replica.getCoreName()) - .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient()) - .build()) { - numCounts[i] = - client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound(); - i++; - } - } - for (int j = 1; j < replicas.size(); j++) { - if (numCounts[j] != numCounts[j - 1]) - fail("Mismatch in counts between replicas"); // TODO improve this! - if (numCounts[j] == 0 && expectDocs) - fail("Expected docs on shard " + shard.getName() + " but found none"); - } - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java deleted file mode 100644 index b3f254bbedc..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.util.LogLevel; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */ -@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN") -public abstract class AbstractSyncSliceTestBase extends AbstractFullDistribZkTestBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private boolean success = false; - - @Override - public void distribTearDown() throws Exception { - if (!success) { - printLayoutOnTearDown = true; - } - super.distribTearDown(); - } - - public AbstractSyncSliceTestBase() { - super(); - sliceCount = 1; - fixShardCount(TEST_NIGHTLY ? 7 : 4); - } - - @Test - public void test() throws Exception { - - handle.clear(); - handle.put("timestamp", SKIPVAL); - - waitForThingsToLevelOut(30, TimeUnit.SECONDS); - - del("*:*"); - List skipServers = new ArrayList<>(); - int docId = 0; - indexDoc( - skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); - - indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue."); - - skipServers.add(shardToJetty.get("shard1").get(1)); - - indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy."); - - skipServers.add(shardToJetty.get("shard1").get(2)); - - indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake"); - - commit(); - - waitForRecoveriesToFinish(false); - - // shard should be inconsistent - String shardFailMessage = checkShardConsistency("shard1", true, false); - assertNotNull(shardFailMessage); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.SYNCSHARD.toString()); - params.set("collection", "collection1"); - params.set("shard", "shard1"); - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params); - - String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString(); - - // we only set the connect timeout, not so timeout - try (SolrClient baseClient = - new HttpSolrClient.Builder(baseUrl) - .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - baseClient.request(request); - } - - waitForThingsToLevelOut(15, TimeUnit.SECONDS); - - checkShardConsistency(false, true); - - long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - assertEquals(4, cloudClientDocs); - - // kill the leader - new leader could have all the docs or be missing one - CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1"); - - skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader - - // this doc won't be on one node - indexDoc( - skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); - commit(); - - Set jetties = new HashSet<>(); - jetties.addAll(shardToJetty.get("shard1")); - jetties.remove(leaderJetty); - assertEquals(getShardCount() - 1, jetties.size()); - - leaderJetty.jetty.stop(); - - Thread.sleep(3000); - - waitForNoShardInconsistency(); - - Thread.sleep(1000); - - checkShardConsistency(false, true); - - cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - assertEquals(5, cloudClientDocs); - - CloudJettyRunner deadJetty = leaderJetty; - - // let's get the latest leader - while (deadJetty.equals(leaderJetty)) { - updateMappingsFromZk(this.jettys, this.clients); - leaderJetty = shardToLeaderJetty.get("shard1"); - } - - // bring back dead node - deadJetty.jetty.start(); // he is not the leader anymore - - waitTillAllNodesActive(); - - skipServers = getRandomOtherJetty(leaderJetty, deadJetty); - skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty)); - // skip list should be - - // System.out.println("leader:" + leaderJetty.url); - // System.out.println("dead:" + deadJetty.url); - // System.out.println("skip list:" + skipServers); - - // we are skipping 2 nodes - assertEquals(2, skipServers.size()); - - // more docs than can peer sync - for (int i = 0; i < 300; i++) { - indexDoc( - skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); - } - - commit(); - waitForRecoveriesToFinish(false); - - // shard should be inconsistent - shardFailMessage = waitTillInconsistent(); - assertNotNull( - "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:" - + leaderJetty.url - + " Dead Guy:" - + deadJetty.url - + "skip list:" - + skipServers, - shardFailMessage); - - // good place to test compareResults - boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient); - assertTrue("A test that compareResults is working correctly failed", shouldFail); - - jetties = new HashSet<>(); - jetties.addAll(shardToJetty.get("shard1")); - jetties.remove(leaderJetty); - assertEquals(getShardCount() - 1, jetties.size()); - - // kill the current leader - leaderJetty.jetty.stop(); - - waitForNoShardInconsistency(); - - checkShardConsistency(true, true); - - success = true; - } - - private void waitTillAllNodesActive() throws InterruptedException, TimeoutException { - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - - zkStateReader.waitForState( - "collection1", - 3, - TimeUnit.MINUTES, - (n, c) -> { - Collection replicas = c.getSlice("shard1").getReplicas(); - Set nodes = - replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet()); - return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals) - && n.containsAll(nodes); - }); - } - - private String waitTillInconsistent() throws Exception, InterruptedException { - String shardFailMessage = null; - - shardFailMessage = pollConsistency(shardFailMessage, 0); - shardFailMessage = pollConsistency(shardFailMessage, 3000); - shardFailMessage = pollConsistency(shardFailMessage, 5000); - shardFailMessage = pollConsistency(shardFailMessage, 15000); - - return shardFailMessage; - } - - private String pollConsistency(String shardFailMessage, int sleep) - throws InterruptedException, Exception { - try { - commit(); - } catch (Throwable t) { - log.error("commit error", t); - } - if (shardFailMessage == null) { - // try again - Thread.sleep(sleep); - shardFailMessage = checkShardConsistency("shard1", true, false); - } - return shardFailMessage; - } - - private List getRandomOtherJetty( - CloudJettyRunner leader, CloudJettyRunner down) { - List skipServers = new ArrayList<>(); - List candidates = new ArrayList<>(); - candidates.addAll(shardToJetty.get("shard1")); - - if (leader != null) { - candidates.remove(leader); - } - - if (down != null) { - candidates.remove(down); - } - - CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size())); - skipServers.add(cjetty); - return skipServers; - } - - protected void indexDoc(List skipServers, Object... fields) - throws IOException, SolrServerException { - SolrInputDocument doc = new SolrInputDocument(); - - addFields(doc, fields); - addFields(doc, "rnd_b", true); - - controlClient.add(doc); - - UpdateRequest ureq = new UpdateRequest(); - ureq.add(doc); - for (CloudJettyRunner skip : skipServers) { - ureq.getParams().add("test.distrib.skip.servers", skip.url + "/"); - } - ureq.process(cloudClient); - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java deleted file mode 100644 index ee73ea8e817..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public abstract class AbstractTlogReplayBufferedWhileIndexingTestBase - extends AbstractFullDistribZkTestBase { - - private List threads; - - public AbstractTlogReplayBufferedWhileIndexingTestBase() throws Exception { - super(); - sliceCount = 1; - fixShardCount(2); - schemaString = "schema15.xml"; // we need a string id - } - - @BeforeClass - public static void beforeRestartWhileUpdatingTest() throws Exception { - System.setProperty("leaderVoteWait", "300000"); - System.setProperty("solr.autoCommit.maxTime", "10000"); - System.setProperty("solr.autoSoftCommit.maxTime", "3000"); - TestInjection.updateLogReplayRandomPause = "true:10"; - TestInjection.updateRandomPause = "true:10"; - useFactory("solr.StandardDirectoryFactory"); - } - - @AfterClass - public static void afterRestartWhileUpdatingTest() { - System.clearProperty("leaderVoteWait"); - System.clearProperty("solr.autoCommit.maxTime"); - System.clearProperty("solr.autoSoftCommit.maxTime"); - } - - @Test - public void test() throws Exception { - handle.clear(); - handle.put("timestamp", SKIPVAL); - - waitForRecoveriesToFinish(false); - - int numThreads = 3; - - threads = new ArrayList<>(numThreads); - - ArrayList allJetty = new ArrayList<>(); - allJetty.addAll(jettys); - allJetty.remove(shardToLeaderJetty.get("shard1").jetty); - assert allJetty.size() == 1 : allJetty.size(); - allJetty.get(0).stop(); - - StoppableIndexingThread indexThread; - for (int i = 0; i < numThreads; i++) { - boolean pauseBetweenUpdates = random().nextBoolean(); - int batchSize = random().nextInt(4) + 1; - indexThread = - new StoppableIndexingThread( - controlClient, - cloudClient, - Integer.toString(i), - true, - 900, - batchSize, - pauseBetweenUpdates); - threads.add(indexThread); - indexThread.start(); - } - - Thread.sleep(2000); - - allJetty.get(0).start(); - - Thread.sleep(45000); - - waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while, - // especially when beasting this test - - Thread.sleep(2000); - - waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); - - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - - waitForThingsToLevelOut(30, TimeUnit.SECONDS); - - checkShardConsistency(false, false); - } - - @Override - protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { - cloudClient.add(doc); - } - - @Override - public void distribTearDown() throws Exception { - // make sure threads have been stopped... - if (threads != null) { - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - - for (StoppableIndexingThread thread : threads) { - thread.join(); - } - } - - super.distribTearDown(); - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java deleted file mode 100644 index 2ad56bdd2c6..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrPaths; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. - */ -public abstract class AbstractUnloadDistributedZkTestBase extends AbstractFullDistribZkTestBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public AbstractUnloadDistributedZkTestBase() { - super(); - fixShardCount(4); // needs at least 4 servers - } - - @Override - protected String getSolrXml() { - return "solr.xml"; - } - - @Test - public void test() throws Exception { - jettys.forEach( - j -> { - Set allowPath = j.getCoreContainer().getAllowPaths(); - allowPath.clear(); - allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path - }); - log.info("###Starting testCoreUnloadAndLeaders"); - testCoreUnloadAndLeaders(); // long - log.info("###Starting testUnloadOfCores"); - testUnloadLotsOfCores(); // long - log.info("###Starting testUnloadShardAndCollection"); - testUnloadShardAndCollection(); - } - - /** - * @param url a Solr node base URL. Should not contain a core or collection name. - */ - private SolrClient newSolrClient(String url) { - return new HttpSolrClient.Builder(url) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .build(); - } - - private void checkCoreNamePresenceAndSliceCount( - String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount) - throws Exception { - ZkStateReader reader = ZkStateReader.from(cloudClient); - try { - reader.waitForState( - collectionName, - 45, - TimeUnit.SECONDS, - c -> { - final Collection slices = (c != null) ? c.getSlices() : Collections.emptyList(); - if (expectedSliceCount == slices.size()) { - for (Slice slice : slices) { - for (Replica replica : slice.getReplicas()) { - if (coreName.equals(replica.get("core"))) { - return shouldBePresent; - } - } - } - return !shouldBePresent; - } else { - return false; - } - }); - } catch (TimeoutException e) { - printLayout(); - fail( - "checkCoreNamePresenceAndSliceCount failed:" - + " collection=" - + collectionName - + " CoreName=" - + coreName - + " shouldBePresent=" - + shouldBePresent); - } - } - - private void testUnloadShardAndCollection() throws Exception { - final int numShards = 2; - - final String collection = "test_unload_shard_and_collection"; - - final String coreName1 = collection + "_1"; - final String coreName2 = collection + "_2"; - - assertEquals( - 0, - CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard(collection, "shard1") - .setCoreName(coreName1) - .setNode(jettys.get(0).getNodeName()) - .process(cloudClient) - .isSuccess()); - - assertTrue( - CollectionAdminRequest.addReplicaToShard(collection, "shard2") - .setCoreName(coreName2) - .setNode(jettys.get(0).getNodeName()) - .process(cloudClient) - .isSuccess()); - - // does not mean they are active and up yet :* - waitForRecoveriesToFinish(collection, false); - - final boolean unloadInOrder = random().nextBoolean(); - final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2); - final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1); - - try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) { - // now unload one of the two - Unload unloadCmd = new Unload(false); - unloadCmd.setCoreName(unloadCmdCoreName1); - adminClient.request(unloadCmd); - - // there should still be two shards (as of SOLR-5209) - checkCoreNamePresenceAndSliceCount( - collection, - unloadCmdCoreName1, - false /* shouldBePresent */, - numShards /* expectedSliceCount */); - - // now unload one of the other - unloadCmd = new Unload(false); - unloadCmd.setCoreName(unloadCmdCoreName2); - adminClient.request(unloadCmd); - checkCoreNamePresenceAndSliceCount( - collection, - unloadCmdCoreName2, - false /* shouldBePresent */, - numShards /* expectedSliceCount */); - } - - // printLayout(); - // the collection should still be present (as of SOLR-5209 replica removal does not cascade to - // remove the slice and collection) - getCommonCloudSolrClient(); - assertTrue( - "No longer found collection " + collection, - cloudClient.getClusterState().hasCollection(collection)); - } - - protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) { - SolrCore solrCore = null; - for (SolrCore core : jetty.getCoreContainer().getCores()) { - if (core.getName().startsWith(collection)) { - solrCore = core; - } - } - return solrCore; - } - - /** - * @throws Exception on any problem - */ - private void testCoreUnloadAndLeaders() throws Exception { - JettySolrRunner jetty1 = jettys.get(0); - - assertEquals( - 0, - CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1) - .setCreateNodeSet(jetty1.getNodeName()) - .process(cloudClient) - .getStatus()); - getCommonCloudSolrClient(); - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - - zkStateReader.forceUpdateCollection("unloadcollection"); - - int slices = - zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); - assertEquals(1, slices); - SolrCore solrCore = getFirstCore("unloadcollection", jetty1); - String core1DataDir = solrCore.getDataDir(); - - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName("unloadcollection_shard1_replica2") - .setNode(jettys.get(1).getNodeName()) - .process(cloudClient) - .isSuccess()); - zkStateReader.forceUpdateCollection("unloadcollection"); - slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); - assertEquals(1, slices); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - Replica leader = getLeaderFromZk("unloadcollection", "shard1"); - - Random random = random(); - if (random.nextBoolean()) { - try (SolrClient collectionClient = - getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) { - // lets try and use the solrj client to index and retrieve a couple - // documents - SolrInputDocument doc1 = - getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); - SolrInputDocument doc2 = - getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls"); - SolrInputDocument doc3 = - getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled"); - collectionClient.add(doc1); - collectionClient.add(doc2); - collectionClient.add(doc3); - collectionClient.commit(); - } - } - - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName("unloadcollection_shard1_replica3") - .setNode(jettys.get(2).getNodeName()) - .process(cloudClient) - .isSuccess()); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - // so that we start with some versions when we reload... - TestInjection.skipIndexWriterCommitOnClose = true; - - try (SolrClient addClient = - new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) - .withDefaultCollection("unloadcollection_shard1_replica3") - .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - - // add a few docs - for (int x = 20; x < 100; x++) { - SolrInputDocument doc1 = - getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); - addClient.add(doc1); - } - } - // don't commit so they remain in the tran log - // collectionClient.commit(); - - // unload the leader - try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { - - Unload unloadCmd = new Unload(false); - unloadCmd.setCoreName(leader.getCoreName()); - ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams(); - - collectionClient.request(unloadCmd); - } - // Thread.currentThread().sleep(500); - // printLayout(); - - int tries = 50; - while (leader - .getCoreUrl() - .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { - Thread.sleep(100); - if (tries-- == 0) { - fail("Leader never changed"); - } - } - - // ensure there is a leader - zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); - - try (SolrClient addClient = - new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString()) - .withDefaultCollection("unloadcollection_shard1_replica2") - .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) - .withSocketTimeout(90000, TimeUnit.MILLISECONDS) - .build()) { - - // add a few docs while the leader is down - for (int x = 101; x < 200; x++) { - SolrInputDocument doc1 = - getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); - addClient.add(doc1); - } - } - - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName("unloadcollection_shard1_replica4") - .setNode(jettys.get(3).getNodeName()) - .process(cloudClient) - .isSuccess()); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - // unload the leader again - leader = getLeaderFromZk("unloadcollection", "shard1"); - try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { - - Unload unloadCmd = new Unload(false); - unloadCmd.setCoreName(leader.getCoreName()); - collectionClient.request(unloadCmd); - } - tries = 50; - while (leader - .getCoreUrl() - .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { - Thread.sleep(100); - if (tries-- == 0) { - fail("Leader never changed"); - } - } - - zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); - - TestInjection.skipIndexWriterCommitOnClose = false; // set this back - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName(leader.getCoreName()) - .setDataDir(core1DataDir) - .setNode(leader.getNodeName()) - .process(cloudClient) - .isSuccess()); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - long found1, found3; - - try (SolrClient adminClient = - newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) { - adminClient.commit(); - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - found1 = adminClient.query(q).getResults().getNumFound(); - } - - try (SolrClient adminClient = - new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) - .withDefaultCollection("unloadcollection_shard1_replica3") - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - adminClient.commit(); - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - found3 = adminClient.query(q).getResults().getNumFound(); - } - - try (SolrClient adminClient = - newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) { - adminClient.commit(); - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - long found4 = adminClient.query(q).getResults().getNumFound(); - - // all 3 shards should now have the same number of docs - assertEquals(found1, found3); - assertEquals(found3, found4); - } - } - - private void testUnloadLotsOfCores() throws Exception { - JettySolrRunner jetty = jettys.get(0); - int shards = TEST_NIGHTLY ? 2 : 1; - try (final SolrClient adminClient = jetty.newClient(15000, 60000)) { - int numReplicas = atLeast(3); - ThreadPoolExecutor executor = - new ExecutorUtil.MDCAwareThreadPoolExecutor( - 0, - Integer.MAX_VALUE, - 5, - TimeUnit.SECONDS, - new SynchronousQueue<>(), - new SolrNamedThreadFactory("testExecutor")); - try { - // create the cores - AbstractBasicDistributedZkTestBase.createCollectionInOneInstance( - adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas); - } finally { - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - - if (TEST_NIGHTLY == false) { - // with nightly tests, we can try doing the unloads before the creates are done - // it still works, but takes much longer since we end up waiting for a timeout - waitForRecoveriesToFinish("multiunload", false); - } - - executor = - new ExecutorUtil.MDCAwareThreadPoolExecutor( - 0, - Integer.MAX_VALUE, - 5, - TimeUnit.SECONDS, - new SynchronousQueue<>(), - new SolrNamedThreadFactory("testExecutor")); - try { - for (int j = 0; j < numReplicas; j++) { - final int freezeJ = j; - executor.execute( - () -> { - Unload unloadCmd = new Unload(true); - unloadCmd.setCoreName("multiunload" + freezeJ); - try { - adminClient.request(unloadCmd); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - }); - Thread.sleep(random().nextInt(50)); - } - } finally { - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - } - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java deleted file mode 100644 index 98edf7f3e87..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java +++ /dev/null @@ -1,700 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud.api.collections; - -import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.util.TestUtil; -import org.apache.solr.client.api.model.CoreStatusResponse; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest.METHOD; -import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrCore; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.apache.solr.util.TimeOut; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Tests the Cloud Collections API. - * - *

Because the different setups require distinct config-sets, we have to push down cluster - * creation to subclasses - */ -public abstract class AbstractCollectionsAPIDistributedZkTestBase extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() { - // we don't want this test to have zk timeouts - System.setProperty("zkClientTimeout", "60000"); - System.setProperty("createCollectionWaitTimeTillActive", "5"); - TestInjection.randomDelayInCoreCreation = "true:5"; - System.setProperty("validateAfterInactivity", "200"); - System.setProperty("solr.security.allow.paths", "*"); - } - - @Override - @After - public void tearDown() throws Exception { - cluster.deleteAllCollections(); - super.tearDown(); - } - - @Test - public void testCreationAndDeletion() throws Exception { - String collectionName = "created_and_deleted"; - - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) - .process(cluster.getSolrClient()); - assertTrue( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - assertFalse( - cluster - .getZkClient() - .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); - } - - @Test - public void deleteCollectionRemovesStaleZkCollectionsNode() throws Exception { - String collectionName = "out_of_sync_collection"; - - // manually create a collections zknode - cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); - - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - assertFalse( - cluster - .getZkClient() - .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); - } - - @Test - public void deletePartiallyCreatedCollection() throws Exception { - final String collectionName = "halfdeletedcollection"; - - assertEquals( - 0, - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) - .setCreateNodeSet("") - .process(cluster.getSolrClient()) - .getStatus()); - String dataDir = createTempDir().toString(); - // create a core that simulates something left over from a partially-deleted collection - assertTrue( - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .setDataDir(dataDir) - .process(cluster.getSolrClient()) - .isSuccess()); - - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) - .process(cluster.getSolrClient()); - - assertTrue( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - } - - @Test - public void deleteCollectionOnlyInZk() throws Exception { - final String collectionName = "onlyinzk"; - - // create the collections node, but nothing else - cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); - - // delete via API - should remove collections node - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - // now creating that collection should work - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) - .process(cluster.getSolrClient()); - assertTrue( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - } - - @Test - public void testBadActionNames() { - // try a bad action - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", "BADACTION"); - String collectionName = "badactioncollection"; - params.set("name", collectionName); - params.set("numShards", 2); - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testMissingRequiredParameters() { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("numShards", 2); - // missing required collection parameter - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testMissingNumShards() { - // No numShards should fail - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("name", "acollection"); - params.set(REPLICATION_FACTOR, 10); - params.set("collection.configName", "conf"); - - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testZeroNumShards() { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("name", "acollection"); - params.set(REPLICATION_FACTOR, 10); - params.set("numShards", 0); - params.set("collection.configName", "conf"); - - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testCreateShouldFailOnExistingCore() throws Exception { - String nn1 = cluster.getJettySolrRunner(0).getNodeName(); - String nn2 = cluster.getJettySolrRunner(1).getNodeName(); - - assertEquals( - 0, - CollectionAdminRequest.createCollection("halfcollectionblocker", "conf", 1, 1) - .setCreateNodeSet("") - .process(cluster.getSolrClient()) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard("halfcollectionblocker", "shard1") - .setNode(cluster.getJettySolrRunner(0).getNodeName()) - .setCoreName("halfcollection_shard1_replica_n1") - .process(cluster.getSolrClient()) - .isSuccess()); - - assertEquals( - 0, - CollectionAdminRequest.createCollection("halfcollectionblocker2", "conf", 1, 1) - .setCreateNodeSet("") - .process(cluster.getSolrClient()) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard("halfcollectionblocker2", "shard1") - .setNode(cluster.getJettySolrRunner(1).getNodeName()) - .setCoreName("halfcollection_shard1_replica_n1") - .process(cluster.getSolrClient()) - .isSuccess()); - - expectThrows( - SolrClient.RemoteSolrException.class, - () -> { - CollectionAdminRequest.createCollection("halfcollection", "conf", 1, 1) - .setCreateNodeSet(nn1 + "," + nn2) - .process(cluster.getSolrClient()); - }); - } - - @Test - public void testNoConfigSetExist() throws Exception { - expectThrows( - Exception.class, - () -> { - CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1) - .process(cluster.getSolrClient()); - }); - - TimeUnit.MILLISECONDS.sleep(1000); - // in both cases, the collection should have default to the core name - cluster.getZkStateReader().forceUpdateCollection("noconfig"); - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig")); - } - - @Test - public void testCoresAreDistributedAcrossNodes() throws Exception { - CollectionAdminRequest.createCollection("nodes_used_collection", "conf", 2, 2) - .process(cluster.getSolrClient()); - - Set liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes(); - - List createNodeList = new ArrayList<>(liveNodes); - - DocCollection collection = getCollectionState("nodes_used_collection"); - for (Slice slice : collection.getSlices()) { - for (Replica replica : slice.getReplicas()) { - createNodeList.remove(replica.getNodeName()); - } - } - - assertEquals(createNodeList.toString(), 0, createNodeList.size()); - } - - @Test - public void testDeleteNonExistentCollection() throws Exception { - - expectThrows( - SolrException.class, - () -> { - CollectionAdminRequest.deleteCollection("unknown_collection") - .process(cluster.getSolrClient()); - }); - - // create another collection should still work - CollectionAdminRequest.createCollection("acollectionafterbaddelete", "conf", 1, 2) - .process(cluster.getSolrClient()); - waitForState( - "Collection creation after a bad delete failed", - "acollectionafterbaddelete", - (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 2)); - } - - @Test - public void testSpecificConfigsets() throws Exception { - CollectionAdminRequest.createCollection("withconfigset2", "conf2", 1, 1) - .process(cluster.getSolrClient()); - String configName = - cluster - .getSolrClient() - .getClusterStateProvider() - .getCollection("withconfigset2") - .getConfigName(); - assertEquals("conf2", configName); - } - - @Test - public void testCreateNodeSet() throws Exception { - JettySolrRunner jetty1 = cluster.getRandomJetty(random()); - JettySolrRunner jetty2 = cluster.getRandomJetty(random()); - - List baseUrls = List.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString()); - - CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1) - .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1)) - .process(cluster.getSolrClient()); - - DocCollection collectionState = getCollectionState("nodeset_collection"); - for (Replica replica : collectionState.getReplicas()) { - String replicaUrl = replica.getCoreUrl(); - boolean matchingJetty = false; - for (String jettyUrl : baseUrls) { - if (replicaUrl.startsWith(jettyUrl)) { - matchingJetty = true; - } - } - if (matchingJetty == false) { - fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl); - } - } - } - - @Test - public void testCollectionsAPI() throws Exception { - - // create new collections rapid fire - int cnt = random().nextInt(TEST_NIGHTLY ? 3 : 1) + 1; - CollectionAdminRequest.Create[] createRequests = new CollectionAdminRequest.Create[cnt]; - - class Coll { - String name; - int numShards; - int replicationFactor; - } - - List colls = new ArrayList<>(); - - for (int i = 0; i < cnt; i++) { - - int numShards = TestUtil.nextInt(random(), 0, cluster.getJettySolrRunners().size()) + 1; - int replicationFactor = TestUtil.nextInt(random(), 0, 3) + 1; - - createRequests[i] = - CollectionAdminRequest.createCollection( - "awhollynewcollection_" + i, "conf2", numShards, replicationFactor); - createRequests[i].processAsync(cluster.getSolrClient()); - - Coll coll = new Coll(); - coll.name = "awhollynewcollection_" + i; - coll.numShards = numShards; - coll.replicationFactor = replicationFactor; - colls.add(coll); - } - - for (Coll coll : colls) { - cluster.waitForActiveCollection( - coll.name, coll.numShards, coll.numShards * coll.replicationFactor); - } - - waitForStable(cnt, createRequests); - - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - checkInstanceDirs(cluster.getJettySolrRunner(i)); - } - - String collectionName = - createRequests[random().nextInt(createRequests.length)].getCollectionName(); - - // TODO: we should not need this...beast test well when trying to fix - Thread.sleep(1000); - - cluster.getZkStateReader().forciblyRefreshAllClusterStateSlow(); - - new UpdateRequest() - .add("id", "6") - .add("id", "7") - .add("id", "8") - .commit(cluster.getSolrClient(), collectionName); - long numFound = 0; - TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); - while (!timeOut.hasTimedOut()) { - - numFound = - cluster - .getSolrClient() - .query(collectionName, new SolrQuery("*:*")) - .getResults() - .getNumFound(); - if (numFound == 3) { - break; - } - - Thread.sleep(500); - } - - if (timeOut.hasTimedOut()) { - fail( - "Timeout waiting to see 3 found, instead saw " - + numFound - + " for collection " - + collectionName); - } - - checkNoTwoShardsUseTheSameIndexDir(); - } - - private void waitForStable(int cnt, CollectionAdminRequest.Create[] createRequests) - throws InterruptedException { - for (int i = 0; i < cnt; i++) { - String collectionName = "awhollynewcollection_" + i; - final int j = i; - waitForState( - "Expected to see collection " + collectionName, - collectionName, - (n, c) -> { - CollectionAdminRequest.Create req = createRequests[j]; - return SolrCloudTestCase.replicasForCollectionAreFullyActive( - n, c, req.getNumShards(), req.getReplicationFactor()); - }); - - ZkStateReader zkStateReader = cluster.getZkStateReader(); - // make sure we have leaders for each shard - for (int z = 1; z < createRequests[j].getNumShards(); z++) { - zkStateReader.getLeaderRetry(collectionName, "shard" + z, 10000); - } // make sure we again have leaders for each shard - } - } - - @Test - public void testCollectionReload() throws Exception { - final String collectionName = "reloaded_collection"; - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) - .process(cluster.getSolrClient()); - - // get core open times - Map urlToTimeBefore = new HashMap<>(); - collectStartTimes(collectionName, urlToTimeBefore); - assertTrue(urlToTimeBefore.size() > 0); - - CollectionAdminRequest.reloadCollection(collectionName).processAsync(cluster.getSolrClient()); - - // reloads make take a short while - boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore); - assertTrue("some core start times did not change on reload", allTimesAreCorrect); - } - - private void checkInstanceDirs(JettySolrRunner jetty) throws IOException { - CoreContainer cores = jetty.getCoreContainer(); - Collection theCores = cores.getCores(); - for (SolrCore core : theCores) { - // look for core props file - Path instancedir = core.getInstancePath(); - assertTrue( - "Could not find expected core.properties file", - Files.exists(instancedir.resolve("core.properties"))); - - Path expected = Path.of(jetty.getSolrHome()).resolve(core.getName()); - - assertTrue( - "Expected: " + expected + "\nFrom core stats: " + instancedir, - Files.isSameFile(expected, instancedir)); - } - } - - private boolean waitForReloads(String collectionName, Map urlToTimeBefore) - throws SolrServerException, IOException { - TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME); - - boolean allTimesAreCorrect = false; - while (!timeout.hasTimedOut()) { - Map urlToTimeAfter = new HashMap<>(); - collectStartTimes(collectionName, urlToTimeAfter); - - boolean retry = false; - Set> entries = urlToTimeBefore.entrySet(); - for (Entry entry : entries) { - Long beforeTime = entry.getValue(); - Long afterTime = urlToTimeAfter.get(entry.getKey()); - assertNotNull(afterTime); - if (afterTime <= beforeTime) { - retry = true; - break; - } - } - if (!retry) { - allTimesAreCorrect = true; - break; - } - } - return allTimesAreCorrect; - } - - private void collectStartTimes(String collectionName, Map urlToTime) - throws SolrServerException, IOException { - - DocCollection collectionState = getCollectionState(collectionName); - if (collectionState != null) { - for (Slice shard : collectionState) { - for (Replica replica : shard) { - CoreStatusResponse.SingleCoreData coreStatus; - try (SolrClient server = getHttpSolrClient(replica.getBaseUrl())) { - coreStatus = CoreAdminRequest.getCoreStatus(replica.getCoreName(), false, server); - } - long before = coreStatus.startTime.getTime(); - urlToTime.put(replica.getCoreUrl(), before); - } - } - } else { - throw new IllegalArgumentException("Could not find collection " + collectionName); - } - } - - private void checkNoTwoShardsUseTheSameIndexDir() { - Map> indexDirToShardNamesMap = new HashMap<>(); - - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - CoreContainer coreContainer = jetty.getCoreContainer(); - List coreNames = coreContainer.getAllCoreNames(); - for (String coreName : coreNames) { - try (SolrCore core = coreContainer.getCore(coreName)) { - String indexDir = core.getIndexDir(); - String shardKey = jetty.getNodeName() + "." + coreName; - if (!indexDirToShardNamesMap.containsKey(indexDir)) { - indexDirToShardNamesMap.put(indexDir, new HashSet<>()); - } - indexDirToShardNamesMap.get(indexDir).add(shardKey); - } - } - } - - assertTrue( - "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of " - + SolrCore.class.getSimpleName() - + " : " - + indexDirToShardNamesMap, - indexDirToShardNamesMap.size() > 0); - - for (Map.Entry> entry : indexDirToShardNamesMap.entrySet()) { - if (entry.getValue().size() > 1) { - fail( - "We have shards using the same indexDir. E.g. shards " - + entry.getValue().toString() - + " all use indexDir " - + entry.getKey()); - } - } - } - - @Test - public void addReplicaTest() throws Exception { - String collectionName = "addReplicaColl"; - - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(collectionName, 2, 4); - - ArrayList nodeList = - new ArrayList<>(cluster.getSolrClient().getClusterState().getLiveNodes()); - Collections.shuffle(nodeList, random()); - - CollectionAdminResponse response = - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .setNode(nodeList.get(0)) - .process(cluster.getSolrClient()); - Replica newReplica = grabNewReplica(response, getCollectionState(collectionName)); - - assertEquals( - "Replica should be created on the right node", - cluster.getZkStateReader().getBaseUrlForNodeName(nodeList.get(0)), - newReplica.getBaseUrl()); - - Path instancePath = createTempDir(); - response = - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .withProperty(CoreAdminParams.INSTANCE_DIR, instancePath.toString()) - .process(cluster.getSolrClient()); - newReplica = grabNewReplica(response, getCollectionState(collectionName)); - assertNotNull(newReplica); - cluster.waitForActiveCollection(collectionName, 2, 6); - - try (SolrClient coreclient = getHttpSolrClient(newReplica.getBaseUrl())) { - CoreAdminResponse status = CoreAdminRequest.getStatus(newReplica.getStr("core"), coreclient); - final var coreStatus = status.getCoreStatus(newReplica.getStr("core")); - String instanceDirStr = coreStatus.instanceDir; - assertEquals(instanceDirStr, instancePath.toString()); - } - - // Test to make sure we can't create another replica with an existing core_name of that - // collection - String coreName = newReplica.getStr(CORE_NAME_PROP); - SolrException e = - expectThrows( - SolrException.class, - () -> { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", "addreplica"); - params.set("collection", collectionName); - params.set("shard", "shard1"); - params.set("name", coreName); - var request = - new GenericSolrRequest( - METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - cluster.getSolrClient().request(request); - }); - - assertTrue( - e.getMessage() - .contains( - "Another replica with the same core name already exists for this collection")); - - // Check that specifying property.name works. DO NOT remove this when the "name" property is - // deprecated - // for ADDREPLICA, this is "property.name". See SOLR-7132 - response = - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .withProperty(CoreAdminParams.NAME, "propertyDotName") - .process(cluster.getSolrClient()); - - newReplica = grabNewReplica(response, getCollectionState(collectionName)); - assertEquals( - "'core' should be 'propertyDotName' ", "propertyDotName", newReplica.getStr("core")); - cluster.waitForActiveCollection(collectionName, 2, 7); - } - - private Replica grabNewReplica(CollectionAdminResponse response, DocCollection docCollection) { - String replicaName = response.getCollectionCoresStatus().keySet().iterator().next(); - Optional optional = - docCollection.getReplicas().stream() - .filter(replica -> replicaName.equals(replica.getCoreName())) - .findAny(); - if (optional.isPresent()) { - return optional.get(); - } - throw new AssertionError("Can not find " + replicaName + " from " + docCollection); - } -} From 81aa45387817342b988fbee7f7c30963c3470650 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 13:38:39 +0000 Subject: [PATCH 04/16] Run gradlew tidy to format code Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../test/org/apache/solr/cloud/BasicDistributedZkTest.java | 2 +- .../apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 5 ++--- .../org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java | 4 +++- .../solr/cloud/TlogReplayBufferedWhileIndexingTest.java | 7 +++---- .../org/apache/solr/cloud/UnloadDistributedZkTest.java | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 57b71796da8..6d08c2053ed 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -41,6 +41,7 @@ import java.util.concurrent.atomic.AtomicReference; import org.apache.lucene.util.IOUtils; import org.apache.solr.JSONTestUtil; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; @@ -87,7 +88,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.solr.SolrTestCaseJ4.SuppressSSL; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 15732a642d3..3cff701db4e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -22,6 +22,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; @@ -30,12 +31,10 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.solr.SolrTestCaseJ4.SuppressSSL; @LuceneTestCase.Nightly @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class ChaosMonkeyNothingIsSafeTest - extends AbstractFullDistribZkTestBase { +public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase { private static final int FAIL_TOLERANCE = 100; private static final Integer RUN_LENGTH = diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index 3a35ee150fa..a4d227aaf0e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -68,7 +68,9 @@ public RandVal[] getRandValues() { return randVals; } - protected String getDirectoryFactory() { return "solr.StandardDirectoryFactory"; } + protected String getDirectoryFactory() { + return "solr.StandardDirectoryFactory"; + } @Override public void distribSetUp() throws Exception { diff --git a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java index 8872881f6d7..2fbd5128177 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java @@ -20,6 +20,8 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.LuceneTestCase.Nightly; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ZkStateReader; @@ -28,13 +30,10 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; -import org.apache.lucene.tests.util.LuceneTestCase.Nightly; -import org.apache.solr.SolrTestCaseJ4.SuppressSSL; @Nightly @SuppressSSL -public class TlogReplayBufferedWhileIndexingTest - extends AbstractFullDistribZkTestBase { +public class TlogReplayBufferedWhileIndexingTest extends AbstractFullDistribZkTestBase { private List threads; diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 81e573aae54..91ef7830206 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -27,6 +27,7 @@ import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; +import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; @@ -47,7 +48,6 @@ import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.apache.solr.SolrTestCaseJ4.SuppressSSL; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as From 31ec4de19798aa3e9ddca73b6b6f7ab34679ce54 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 08:58:45 -0500 Subject: [PATCH 05/16] a test that no longer does anything when HDFS is gone --- .../AbstractCloudBackupRestoreTestCase.java | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java index 87377bb02e5..b6fd4b45f3c 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java @@ -47,7 +47,6 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -230,22 +229,6 @@ public void testRestoreFailure() throws Exception { } } - /** - * This test validates the backup of collection configuration using {@linkplain - * CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}. - * - * @param configName The config name for the collection to be backed up. - * @param collectionName The name of the collection to be backed up. - * @throws Exception in case of errors. - */ - protected void testConfigBackupOnly(String configName, String collectionName) throws Exception { - // This is deliberately no-op since we want to run this test only for one of the backup - // repository - // implementation (mainly to avoid redundant test execution). Currently HDFS backup repository - // test - // implements this. - } - // This test verifies the system behavior when the backup location cluster property is configured // with an invalid // value for the specified repository (and the default backup location is not configured in From e99e5fcde3d088269357c8fd7640c168904cb574 Mon Sep 17 00:00:00 2001 From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com> Date: Mon, 1 Dec 2025 14:00:36 +0000 Subject: [PATCH 06/16] Merge AbstractDistribZkTestBase into AbstractFullDistribZkTestBase Co-authored-by: epugh <22395+epugh@users.noreply.github.com> --- .../apache/solr/cli/SolrCLIZkToolsTest.java | 20 +- .../apache/solr/cli/ZkSubcommandsTest.java | 4 +- .../solr/cloud/ChaosMonkeyShardSplitTest.java | 8 +- .../cloud/LeaderElectionContextKeyTest.java | 4 +- .../org/apache/solr/cloud/OverseerTest.java | 2 +- .../cloud/ParallelCommitExecutionTest.java | 2 +- ...stCloudPhrasesIdentificationComponent.java | 2 +- .../cloud/TestCloudPseudoReturnFields.java | 2 +- .../apache/solr/cloud/TestCloudRecovery2.java | 2 +- .../solr/cloud/TestRandomFlRTGCloud.java | 2 +- .../TestStressCloudBlindAtomicUpdates.java | 2 +- .../LocalFSCloudIncrementalBackupTest.java | 4 +- .../cloud/api/collections/ShardSplitTest.java | 70 +-- ...TestCollectionsAPIViaSolrCloudCluster.java | 6 +- .../snapshots/TestSolrCloudSnapshots.java | 4 +- .../CustomHighlightComponentTest.java | 4 +- .../SearchHandlerAppendsCloudTest.java | 4 +- .../handler/component/UpdateLogCloudTest.java | 8 +- .../TestSubQueryTransformerDistrib.java | 6 +- .../facet/TestCloudJSONFacetJoinDomain.java | 4 +- .../search/facet/TestCloudJSONFacetSKG.java | 4 +- .../facet/TestCloudJSONFacetSKGEquiv.java | 4 +- .../search/join/TestCloudNestedDocsSort.java | 4 +- .../solr/servlet/HttpSolrCallCloudTest.java | 4 +- .../solrj/io/graph/GraphExpressionTest.java | 4 +- .../solr/client/solrj/io/sql/JdbcTest.java | 4 +- .../solrj/io/stream/JDBCStreamTest.java | 4 +- .../solrj/io/stream/MathExpressionTest.java | 4 +- .../io/stream/SelectWithEvaluatorsTest.java | 4 +- .../solrj/io/stream/StreamDecoratorTest.java | 4 +- .../solrj/impl/CloudHttp2SolrClientTest.java | 6 +- .../solr/cloud/AbstractDistribZkTestBase.java | 402 ------------------ .../cloud/AbstractFullDistribZkTestBase.java | 329 +++++++++++++- .../solr/cloud/MultiSolrCloudTestCase.java | 2 +- .../AbstractCloudBackupRestoreTestCase.java | 4 +- .../AbstractIncrementalBackupTest.java | 8 +- .../solrj/apache/CloudSolrClientTest.java | 6 +- 37 files changed, 441 insertions(+), 516 deletions(-) delete mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java diff --git a/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java b/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java index c7101a0f493..d7c86595dbc 100644 --- a/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java +++ b/solr/core/src/test/org/apache/solr/cli/SolrCLIZkToolsTest.java @@ -27,7 +27,7 @@ import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkMaintenanceUtils; @@ -72,7 +72,7 @@ public void testUpconfig() throws Exception { Path configSet = TEST_PATH().resolve("configsets"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf"); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "upconfig1", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "upconfig1", zkAddr); // Now do we have that config up on ZK? verifyZkLocalPathsMatch(srcPathCheck, "/configs/upconfig1"); @@ -115,7 +115,7 @@ public void testDownconfig() throws Exception { Path configSet = TEST_PATH().resolve("configsets"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf"); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "downconfig1", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "downconfig1", zkAddr); // Now do we have that config up on ZK? verifyZkLocalPathsMatch(srcPathCheck, "/configs/downconfig1"); @@ -134,7 +134,7 @@ public void testDownconfig() throws Exception { Files.createFile(emptyFile); // Now copy it up and back and insure it's still a file in the new place - AbstractDistribZkTestBase.copyConfigUp(tmp.getParent(), "myconfset", "downconfig2", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(tmp.getParent(), "myconfset", "downconfig2", zkAddr); Path tmp2 = createTempDir("downConfigNewPlace2"); args = new String[] { @@ -157,7 +157,7 @@ public void testCp() throws Exception { Path configSet = TEST_PATH().resolve("configsets"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf"); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "cp1", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "cp1", zkAddr); // Now copy it somewhere else on ZK. String[] args = @@ -398,7 +398,7 @@ public void testMv() throws Exception { Path configSet = TEST_PATH().resolve("configsets"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf"); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "mv1", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "mv1", zkAddr); // Now move it somewhere else. String[] args = new String[] {"mv", "--zk-host", zkAddr, "zk:/configs/mv1", "zk:/mv2"}; @@ -467,7 +467,7 @@ public void testLs() throws Exception { Path configSet = TEST_PATH().resolve("configsets"); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "lister", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "lister", zkAddr); // Should only find a single level. String[] args = new String[] {"ls", "--zk-host", zkAddr, "/configs"}; @@ -540,8 +540,8 @@ public void testRm() throws Exception { Path configSet = TEST_PATH().resolve("configsets"); Path srcPathCheck = configSet.resolve("cloud-subdirs").resolve("conf"); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm1", zkAddr); - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm2", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm1", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm2", zkAddr); // Should fail if recursive not set. String[] args = new String[] {"rm", "--zk-host", zkAddr, "/configs/rm1"}; @@ -580,7 +580,7 @@ public void testRm() throws Exception { // This should silently just refuse to do anything to the / or /zookeeper args = new String[] {"rm", "--recursive", "--zk-host", zkAddr, "zk:/"}; - AbstractDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm3", zkAddr); + AbstractFullDistribZkTestBase.copyConfigUp(configSet, "cloud-subdirs", "rm3", zkAddr); res = CLITestHelper.runTool(args, ZkRmTool.class); assertNotEquals("Should fail when trying to remove /.", 0, res); } diff --git a/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java b/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java index 2dad25d6719..26cc304ef84 100644 --- a/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java +++ b/solr/core/src/test/org/apache/solr/cli/ZkSubcommandsTest.java @@ -28,7 +28,7 @@ import java.util.stream.Stream; import org.apache.lucene.tests.mockfile.FilterPath; import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.AbstractZkTestCase; import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.cloud.ZkTestServer; @@ -605,7 +605,7 @@ public void testUpdateAcls() throws Exception { new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout( - AbstractDistribZkTestBase.DEFAULT_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS) + AbstractFullDistribZkTestBase.DEFAULT_CONNECTION_TIMEOUT, TimeUnit.MILLISECONDS) .build()) { zkClient.getData("/", null, null, true); } catch (KeeperException.NoAuthException e) { diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java index 103efc7856c..ca017ca376b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyShardSplitTest.java @@ -67,9 +67,11 @@ public void test() throws Exception { ClusterState clusterState = cloudClient.getClusterState(); final DocRouter router = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); + clusterState.getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); Slice shard1 = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getSlice(SHARD1); + clusterState + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION) + .getSlice(SHARD1); DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange(); final List ranges = router.partitionRange(2, shard1Range); @@ -142,7 +144,7 @@ public void run() { killerThread.start(); killCounter.incrementAndGet(); - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null, null, false); + splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, null, null, false); log.info("Layout after split: \n"); printLayout(); diff --git a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java index ea48ae83a96..e8519f6382c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/LeaderElectionContextKeyTest.java @@ -59,9 +59,9 @@ public static void setupCluster() throws Exception { .process(cluster.getSolrClient()); } - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( "testCollection1", cluster.getZkStateReader(), false, true, 30); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( "testCollection2", cluster.getZkStateReader(), false, true, 30); } diff --git a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java index b1aaf9e457b..757da989ad9 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverseerTest.java @@ -16,7 +16,7 @@ */ package org.apache.solr.cloud; -import static org.apache.solr.cloud.AbstractDistribZkTestBase.verifyReplicaStatus; +import static org.apache.solr.cloud.AbstractFullDistribZkTestBase.verifyReplicaStatus; import static org.apache.zookeeper.WatchedEvent.NO_ZXID; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyBoolean; diff --git a/solr/core/src/test/org/apache/solr/cloud/ParallelCommitExecutionTest.java b/solr/core/src/test/org/apache/solr/cloud/ParallelCommitExecutionTest.java index 1f07d6fcf45..234640f3256 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ParallelCommitExecutionTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ParallelCommitExecutionTest.java @@ -100,7 +100,7 @@ public void testParallelOk() throws Exception { public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java index c0d79824b8b..1bd20b5db22 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPhrasesIdentificationComponent.java @@ -203,7 +203,7 @@ public static SolrClient getRandClient(Random rand) { public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java index 483c9b9ecf5..81969dae345 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudPseudoReturnFields.java @@ -1130,7 +1130,7 @@ public static SolrClient getRandClient(Random rand) { public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java index 3297529cfdf..84f8dbf3086 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestCloudRecovery2.java @@ -41,7 +41,7 @@ public static void setupCluster() throws Exception { CollectionAdminRequest.createCollection(COLLECTION, "config", 1, 2) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, 30); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java index 3cf825f02bd..15cfa0d2c0a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestRandomFlRTGCloud.java @@ -625,7 +625,7 @@ public static SolrClient getRandomClient(Random rand) { public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java index 72593d91e2f..8f692d3006f 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestStressCloudBlindAtomicUpdates.java @@ -525,7 +525,7 @@ public static SolrClient getRandClient(Random rand) { public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); ZkStateReader.from(client).forceUpdateCollection(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java index 93d95752670..7661e6ce71e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/LocalFSCloudIncrementalBackupTest.java @@ -25,7 +25,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.core.backup.repository.BackupRepository; import org.junit.BeforeClass; @@ -140,7 +140,7 @@ public void testCustomProperties() throws Exception { .setRepositoryName(BACKUP_REPO_NAME) .processAndWait(solrClient, 500); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( restoreCollectionName, ZkStateReader.from(solrClient), false, false, 3); assertEquals( numDocs, diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java index 9a827e2a8b7..f231974cc90 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/ShardSplitTest.java @@ -49,7 +49,7 @@ import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.BasicDistributedZkTest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.StoppableIndexingThread; @@ -134,7 +134,9 @@ private void doSplitStaticIndexReplication(SolrIndexSplitter.SplitMethod splitMe waitForThingsToLevelOut(15, TimeUnit.SECONDS); DocCollection defCol = - cloudClient.getClusterState().getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + cloudClient + .getClusterState() + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); Replica replica = defCol.getReplicas().get(0); String nodeName = replica.getNodeName(); @@ -351,7 +353,7 @@ public void testSplitAfterFailedSplit() throws Exception { private void splitAfterFailedSplit() throws KeeperException, InterruptedException { try { CollectionAdminRequest.SplitShard splitShard = - CollectionAdminRequest.splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + CollectionAdminRequest.splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); splitShard.setShardName(SHARD1); splitShard.process(cloudClient); fail("Shard split was not supposed to succeed after failure injection!"); @@ -361,9 +363,10 @@ private void splitAfterFailedSplit() throws KeeperException, InterruptedExceptio // assert that sub-shards cores exist and sub-shard is in construction state ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - zkStateReader.forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + zkStateReader.forceUpdateCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); ClusterState state = zkStateReader.getClusterState(); - DocCollection collection = state.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + DocCollection collection = + state.getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); // should be cleaned up Slice shard10 = collection.getSlice(SHARD1_0); @@ -376,7 +379,7 @@ private void splitAfterFailedSplit() throws KeeperException, InterruptedExceptio TestInjection.reset(); // let the split succeed try { CollectionAdminRequest.SplitShard splitShard = - CollectionAdminRequest.splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + CollectionAdminRequest.splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); splitShard.setShardName(SHARD1); splitShard.process(cloudClient); // Yay! @@ -516,7 +519,7 @@ public void testSplitWithChaosMonkey() throws Exception { () -> { ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); zkStateReader.registerCollectionStateWatcher( - AbstractDistribZkTestBase.DEFAULT_COLLECTION, + AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, (liveNodes, collectionState) -> { if (stop.get()) { return true; // abort and remove the watch @@ -527,7 +530,7 @@ public void testSplitWithChaosMonkey() throws Exception { if (killed.compareAndSet(false, true)) { log.info( "Monkey thread found 2 replicas for {} {}", - AbstractDistribZkTestBase.DEFAULT_COLLECTION, + AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1); CloudJettyRunner cjetty = shardToLeaderJetty.get(SHARD1); try { @@ -543,7 +546,7 @@ public void testSplitWithChaosMonkey() throws Exception { } log.info( "Monkey thread found only one replica for {} {}", - AbstractDistribZkTestBase.DEFAULT_COLLECTION, + AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1); return false; }); @@ -553,7 +556,7 @@ public void testSplitWithChaosMonkey() throws Exception { monkeyThread.start(); try { CollectionAdminRequest.SplitShard splitShard = - CollectionAdminRequest.splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + CollectionAdminRequest.splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); splitShard.setShardName(SHARD1); String asyncId = splitShard.processAsync(cloudClient); RequestStatusState splitStatus = null; @@ -590,23 +593,23 @@ public void testSplitWithChaosMonkey() throws Exception { } cjetty.jetty.start(); ZkStateReader.from(cloudClient) - .forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + .forceUpdateCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); if (log.isInfoEnabled()) { log.info( "Current collection state: {}", - printClusterStateInfo(AbstractDistribZkTestBase.DEFAULT_COLLECTION)); + printClusterStateInfo(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION)); } // true if sub-shard states switch to 'active' eventually AtomicBoolean areSubShardsActive = new AtomicBoolean(false); if (splitStatus == RequestStatusState.COMPLETED) { // all sub-shard replicas were created successfully so all cores must recover eventually - waitForRecoveriesToFinish(AbstractDistribZkTestBase.DEFAULT_COLLECTION, true); + waitForRecoveriesToFinish(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, true); // let's wait for the overseer to switch shard states CountDownLatch latch = new CountDownLatch(1); ZkStateReader.from(cloudClient) .registerCollectionStateWatcher( - AbstractDistribZkTestBase.DEFAULT_COLLECTION, + AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, (liveNodes, collectionState) -> { Slice parent = collectionState.getSlice(SHARD1); Slice slice10 = collectionState.getSlice(SHARD1_0); @@ -657,7 +660,7 @@ public void testSplitWithChaosMonkey() throws Exception { if (areSubShardsActive.get()) { ClusterState clusterState = cloudClient.getClusterState(); DocCollection collection = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + clusterState.getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); int numReplicasChecked = assertConsistentReplicas(collection.getSlice(SHARD1_0)); assertEquals( "We should have checked consistency for exactly 2 replicas of shard1_0", @@ -782,9 +785,11 @@ private void doSplitShardWithRule(SolrIndexSplitter.SplitMethod splitMethod) thr private void incompleteOrOverlappingCustomRangeTest() throws Exception { ClusterState clusterState = cloudClient.getClusterState(); final DocRouter router = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); + clusterState.getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); Slice shard1 = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getSlice(SHARD1); + clusterState + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION) + .getSlice(SHARD1); DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange(); @@ -794,7 +799,7 @@ private void incompleteOrOverlappingCustomRangeTest() throws Exception { // test with only one range subRanges.add(ranges.get(0)); try { - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); + splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); fail("Shard splitting with just one custom hash range should not succeed"); } catch (SolrClient.RemoteSolrException e) { log.info("Expected exception:", e); @@ -805,7 +810,7 @@ private void incompleteOrOverlappingCustomRangeTest() throws Exception { subRanges.add(ranges.get(3)); // order shouldn't matter subRanges.add(ranges.get(0)); try { - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); + splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); fail("Shard splitting with missing hashes in between given ranges should not succeed"); } catch (SolrClient.RemoteSolrException e) { log.info("Expected exception:", e); @@ -818,7 +823,7 @@ private void incompleteOrOverlappingCustomRangeTest() throws Exception { subRanges.add(ranges.get(2)); subRanges.add(new DocRouter.Range(ranges.get(3).min - 15, ranges.get(3).max)); try { - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); + splitShard(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); fail("Shard splitting with overlapping ranges should not succeed"); } catch (SolrClient.RemoteSolrException e) { log.info("Expected exception:", e); @@ -829,9 +834,11 @@ private void incompleteOrOverlappingCustomRangeTest() throws Exception { private void splitByUniqueKeyTest() throws Exception { ClusterState clusterState = cloudClient.getClusterState(); final DocRouter router = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); + clusterState.getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION).getRouter(); Slice shard1 = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getSlice(SHARD1); + clusterState + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION) + .getSlice(SHARD1); DocRouter.Range shard1Range = shard1.getRange() != null ? shard1.getRange() : router.fullRange(); List subRanges = new ArrayList<>(); @@ -849,12 +856,12 @@ private void splitByUniqueKeyTest() throws Exception { int numReplicas = shard1.getReplicas().size(); ZkStateReader.from(cloudClient) - .forceUpdateCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION); + .forceUpdateCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION); clusterState = cloudClient.getClusterState(); if (log.isDebugEnabled()) { log.debug( "-- COLLECTION: {}", - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION)); + clusterState.getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION)); } del("*:*"); for (int id = 0; id <= 100; id++) { @@ -908,7 +915,8 @@ private void splitByUniqueKeyTest() throws Exception { try { for (int i = 0; i < 3; i++) { try { - splitShard(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); + splitShard( + AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1, subRanges, null, false); log.info("Layout after split: \n"); printLayout(); break; @@ -1153,11 +1161,11 @@ protected void checkDocCountsAndShardStates( clusterState = zkStateReader.getClusterState(); slice1_0 = clusterState - .getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION) + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION) .getSlice("shard1_0"); slice1_1 = clusterState - .getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION) + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION) .getSlice("shard1_1"); if (slice1_0.getState() == Slice.State.ACTIVE && slice1_1.getState() == Slice.State.ACTIVE) { break; @@ -1189,7 +1197,7 @@ protected void checkDocCountsAndShardStates( SolrQuery query = new SolrQuery("*:*").setRows(1000).setFields("id", "_version_"); query.set("distrib", false); - Replica shard1_0 = getLeaderFromZk(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_0); + Replica shard1_0 = getLeaderFromZk(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_0); QueryResponse response; try (SolrClient shard1_0Client = getHttpSolrClient(shard1_0.getBaseUrl(), shard1_0.getCoreName())) { @@ -1197,7 +1205,7 @@ protected void checkDocCountsAndShardStates( } long shard10Count = response.getResults().getNumFound(); - Replica shard1_1 = getLeaderFromZk(AbstractDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_1); + Replica shard1_1 = getLeaderFromZk(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION, SHARD1_1); QueryResponse response2; try (SolrClient shard1_1Client = getHttpSolrClient(shard1_1.getBaseUrl(), shard1_1.getCoreName())) { @@ -1217,7 +1225,9 @@ protected void checkSubShardConsistency(String shard) throws SolrServerException ClusterState clusterState = cloudClient.getClusterState(); Slice slice = - clusterState.getCollection(AbstractDistribZkTestBase.DEFAULT_COLLECTION).getSlice(shard); + clusterState + .getCollection(AbstractFullDistribZkTestBase.DEFAULT_COLLECTION) + .getSlice(shard); long[] numFound = new long[slice.getReplicasMap().size()]; int c = 0; for (Replica replica : slice.getReplicas()) { diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java index 5b7efcfb940..a836b07401e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/TestCollectionsAPIViaSolrCloudCluster.java @@ -37,7 +37,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ClusterState; @@ -167,7 +167,7 @@ public void testCollectionCreateSearchDelete() throws Exception { assertEquals(nodeCount, cluster.getJettySolrRunners().size()); CollectionAdminRequest.deleteCollection(collectionName).process(client); - AbstractDistribZkTestBase.waitForCollectionToDisappear( + AbstractFullDistribZkTestBase.waitForCollectionToDisappear( collectionName, ZkStateReader.from(client), true, 330); // create it again @@ -206,7 +206,7 @@ public void testCollectionCreateWithoutCoresThenDelete() throws Exception { // delete the collection CollectionAdminRequest.deleteCollection(collectionName).process(client); - AbstractDistribZkTestBase.waitForCollectionToDisappear( + AbstractFullDistribZkTestBase.waitForCollectionToDisappear( collectionName, ZkStateReader.from(client), true, 330); } diff --git a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java index a6e04654eb6..9b23db09695 100644 --- a/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java +++ b/solr/core/src/test/org/apache/solr/core/snapshots/TestSolrCloudSnapshots.java @@ -33,7 +33,7 @@ import org.apache.solr.client.solrj.request.CoreAdminRequest.ListSnapshots; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; @@ -216,7 +216,7 @@ public void testSnapshots() throws Exception { } else { assertEquals(RequestStatusState.COMPLETED, restore.processAndWait(solrClient, 30)); // async } - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( restoreCollectionName, ZkStateReader.from(solrClient), log.isDebugEnabled(), true, 30); BackupRestoreUtils.verifyDocs(nDocs, solrClient, restoreCollectionName); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java index 3c7c79ca904..299940aff9a 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/CustomHighlightComponentTest.java @@ -27,7 +27,7 @@ import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.ConfigRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.util.NamedList; @@ -118,7 +118,7 @@ public static void setupCluster() throws Exception { // create an empty collection CollectionAdminRequest.createCollection(COLLECTION, "conf", numShards, numReplicas) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, DEFAULT_TIMEOUT); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerAppendsCloudTest.java b/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerAppendsCloudTest.java index 1d9c2ca96e1..cb6b60b2681 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerAppendsCloudTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/SearchHandlerAppendsCloudTest.java @@ -26,7 +26,7 @@ import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.ConfigRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.ZkStateReader; @@ -58,7 +58,7 @@ public static void setupCluster() throws Exception { // create an empty collection CollectionAdminRequest.createCollection(COLLECTION, "conf", NUM_SHARDS, NUM_REPLICAS) .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, ZkStateReader.from(cluster.getSolrClient()), false, true, DEFAULT_TIMEOUT); } diff --git a/solr/core/src/test/org/apache/solr/handler/component/UpdateLogCloudTest.java b/solr/core/src/test/org/apache/solr/handler/component/UpdateLogCloudTest.java index b4b7efe17d5..c36da38c8cd 100644 --- a/solr/core/src/test/org/apache/solr/handler/component/UpdateLogCloudTest.java +++ b/solr/core/src/test/org/apache/solr/handler/component/UpdateLogCloudTest.java @@ -25,7 +25,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.util.NamedList; import org.apache.solr.core.MockDirectoryFactory; @@ -68,7 +68,7 @@ public void beforeTest() throws Exception { // create an empty collection CollectionAdminRequest.createCollection(COLLECTION, "conf", NUM_SHARDS, NUM_REPLICAS) .processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, DEFAULT_TIMEOUT); } @@ -102,7 +102,7 @@ public void test() throws Exception { } cluster.getJettySolrRunner(specialIdx).stop(); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, DEFAULT_TIMEOUT); new UpdateRequest() @@ -112,7 +112,7 @@ public void test() throws Exception { .commit(cluster.getSolrClient(), COLLECTION); cluster.getJettySolrRunner(specialIdx).start(); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, DEFAULT_TIMEOUT); int idx = 0; diff --git a/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java b/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java index e4cb7ae836f..565c73dceec 100644 --- a/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java +++ b/solr/core/src/test/org/apache/solr/response/transform/TestSubQueryTransformerDistrib.java @@ -43,7 +43,7 @@ import org.apache.solr.client.solrj.request.ContentStreamUpdateRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -129,9 +129,9 @@ public static void setupCluster() throws Exception { CloudSolrClient client = cluster.getSolrClient(); ZkStateReader zkStateReader = ZkStateReader.from(client); - AbstractDistribZkTestBase.waitForRecoveriesToFinish(people, zkStateReader, true, true, 30); + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(people, zkStateReader, true, true, 30); - AbstractDistribZkTestBase.waitForRecoveriesToFinish(depts, zkStateReader, false, true, 30); + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish(depts, zkStateReader, false, true, 30); } @SuppressWarnings("serial") diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java index 594726be18e..0644631e7cb 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetJoinDomain.java @@ -35,7 +35,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.cloud.TestCloudPivotFacet; import org.apache.solr.common.SolrException; @@ -1063,7 +1063,7 @@ public static SolrClient getRandClient(Random rand) { public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } } diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java index 06ada43f108..f1046806863 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKG.java @@ -38,7 +38,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ZkStateReader; @@ -908,7 +908,7 @@ public static long getNumFound(final SolrParams req) throws SolrServerException, public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } diff --git a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java index c612b1a9277..2ce2bd10563 100644 --- a/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java +++ b/solr/core/src/test/org/apache/solr/search/facet/TestCloudJSONFacetSKGEquiv.java @@ -40,7 +40,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ZkStateReader; @@ -1316,7 +1316,7 @@ public static long getNumFound(final SolrParams req) throws SolrServerException, public static void waitForRecoveriesToFinish(CloudSolrClient client) throws Exception { assertNotNull(client.getDefaultCollection()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( client.getDefaultCollection(), ZkStateReader.from(client), true, true, 330); } diff --git a/solr/core/src/test/org/apache/solr/search/join/TestCloudNestedDocsSort.java b/solr/core/src/test/org/apache/solr/search/join/TestCloudNestedDocsSort.java index 2c0cea7e18b..0098aafc35d 100644 --- a/solr/core/src/test/org/apache/solr/search/join/TestCloudNestedDocsSort.java +++ b/solr/core/src/test/org/apache/solr/search/join/TestCloudNestedDocsSort.java @@ -29,7 +29,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrInputDocument; @@ -76,7 +76,7 @@ public static void setupCluster() throws Exception { client = cluster.basicSolrClientBuilder().withDefaultCollection("collection1").build(); ZkStateReader zkStateReader = ZkStateReader.from(client); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( "collection1", zkStateReader, true, true, 30); { int id = 42; diff --git a/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java b/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java index 447ce487506..c3db96528dd 100644 --- a/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java +++ b/solr/core/src/test/org/apache/solr/servlet/HttpSolrCallCloudTest.java @@ -29,7 +29,7 @@ import java.util.Set; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.util.SuppressForbidden; import org.apache.solr.embedded.JettySolrRunner; @@ -53,7 +53,7 @@ public static void setupCluster() throws Exception { CollectionAdminRequest.createCollection(COLLECTION, "config", NUM_SHARD, REPLICA_FACTOR) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, 30); } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java index 90a47751395..15b5d31e669 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/graph/GraphExpressionTest.java @@ -52,7 +52,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.params.ModifiableSolrParams; import org.apache.solr.common.util.NamedList; @@ -89,7 +89,7 @@ public static void setupCluster() throws Exception { CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COLLECTION, cluster.getZkStateReader(), false, true, TIMEOUT); } diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java index 1a182667a2e..7f719871a2c 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/sql/JdbcTest.java @@ -34,7 +34,7 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.Aliases; @@ -80,7 +80,7 @@ public static void setupCluster() throws Exception { cluster.waitForActiveCollection(collection, 2, 2); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( collection, cluster.getZkStateReader(), false, true, DEFAULT_TIMEOUT); if (useAlias) { CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java index b8e200da234..2dd63e664ba 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/JDBCStreamTest.java @@ -40,7 +40,7 @@ import org.apache.solr.client.solrj.io.stream.metrics.MinMetric; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.junit.AfterClass; import org.junit.Before; @@ -80,7 +80,7 @@ public static void setupCluster() throws Exception { } CollectionAdminRequest.createCollection(collection, "conf", 2, 1) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( collection, cluster.getZkStateReader(), false, true, TIMEOUT); if (useAlias) { CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java index 9b598c42755..2ad9eb9dbce 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/MathExpressionTest.java @@ -31,7 +31,7 @@ import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.cloud.ClusterState; import org.apache.solr.common.cloud.DocCollection; @@ -75,7 +75,7 @@ public static void setupCluster() throws Exception { CollectionAdminRequest.createCollection(collection, "conf", 2, 1) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( collection, cluster.getZkStateReader(), false, true, TIMEOUT); if (useAlias) { CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java index 2abfa0fad3d..15785b4101a 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/SelectWithEvaluatorsTest.java @@ -29,7 +29,7 @@ import org.apache.solr.client.solrj.io.stream.expr.StreamFactory; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.junit.Before; import org.junit.BeforeClass; @@ -70,7 +70,7 @@ public static void setupCluster() throws Exception { } CollectionAdminRequest.createCollection(collection, "conf", 2, 1) .process(cluster.getSolrClient()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( collection, cluster.getZkStateReader(), false, true, TIMEOUT); if (useAlias) { CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection) diff --git a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java index 64d0f9bfc1b..58f6d910ada 100644 --- a/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java +++ b/solr/solrj-streaming/src/test/org/apache/solr/client/solrj/io/stream/StreamDecoratorTest.java @@ -58,7 +58,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.cloud.ClusterState; @@ -109,7 +109,7 @@ public static void setupCluster() throws Exception { cluster.waitForActiveCollection(collection, 2, 2); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( collection, cluster.getZkStateReader(), false, true, TIMEOUT); if (useAlias) { CollectionAdminRequest.createAlias(COLLECTIONORALIAS, collection) diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java index 22d581bd4b7..d09f2ea9ba0 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/impl/CloudHttp2SolrClientTest.java @@ -55,7 +55,7 @@ import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -1063,7 +1063,7 @@ public void testRetryUpdatesWhenClusterStateIsStale() throws Exception { // change .process(cluster.getSolrClient()) .getStatus()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COL, cluster.getZkStateReader(), true, true, 330); // ...and delete our original leader. assertEquals( @@ -1074,7 +1074,7 @@ public void testRetryUpdatesWhenClusterStateIsStale() throws Exception { // change .process(cluster.getSolrClient()) .getStatus()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COL, cluster.getZkStateReader(), true, true, 330); // stale_client's collection state cache should now only point at a leader that no longer diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java deleted file mode 100644 index ef1760592c7..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDistribZkTestBase.java +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import static java.util.concurrent.TimeUnit.SECONDS; - -import java.lang.invoke.MethodHandles; -import java.nio.file.Path; -import java.util.Map; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import org.apache.commons.io.file.PathUtils; -import org.apache.solr.BaseDistributedSearchTestCase; -import org.apache.solr.cli.ConfigSetUploadTool; -import org.apache.solr.cli.DefaultToolRuntime; -import org.apache.solr.cli.SolrCLI; -import org.apache.solr.cli.ToolRuntime; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.core.Diagnostics; -import org.apache.solr.core.MockDirectoryFactory; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.zookeeper.KeeperException; -import org.junit.BeforeClass; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractDistribZkTestBase extends BaseDistributedSearchTestCase { - - private static final String REMOVE_VERSION_FIELD = "remove.version.field"; - private static final String ENABLE_UPDATE_LOG = "solr.index.updatelog.enabled"; - private static final String ZK_HOST = "zkHost"; - private static final String ZOOKEEPER_FORCE_SYNC = "zookeeper.forceSync"; - protected static final String DEFAULT_COLLECTION = "collection1"; - protected volatile ZkTestServer zkServer; - private final AtomicInteger homeCount = new AtomicInteger(); - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void beforeThisClass() throws Exception { - // Only For Manual Testing: this will force an fs based dir factory - // useFactory(null); - } - - @Override - public void distribSetUp() throws Exception { - super.distribSetUp(); - - Path zkDir = testDir.resolve("zookeeper/server1/data"); - zkServer = new ZkTestServer(zkDir); - zkServer.run(); - - System.setProperty(ZK_HOST, zkServer.getZkAddress()); - System.setProperty(ENABLE_UPDATE_LOG, "true"); - System.setProperty(REMOVE_VERSION_FIELD, "true"); - System.setProperty(ZOOKEEPER_FORCE_SYNC, "false"); - System.setProperty( - MockDirectoryFactory.SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE, "true"); - - String schema = getCloudSchemaFile(); - if (schema == null) schema = "schema.xml"; - zkServer.buildZooKeeper(getCloudSolrConfig(), schema); - - // set some system properties for use by tests - System.setProperty("solr.test.sys.prop1", "propone"); - System.setProperty("solr.test.sys.prop2", "proptwo"); - } - - protected String getCloudSolrConfig() { - return "solrconfig-tlog.xml"; - } - - protected String getCloudSchemaFile() { - return getSchemaFile(); - } - - @Override - protected void createServers(int numShards) throws Exception { - // give everyone there own solrhome - Path controlHome = getSolrHome().getParent().resolve("control" + homeCount.incrementAndGet()); - PathUtils.copyDirectory(getSolrHome(), controlHome); - setupJettySolrHome(controlHome); - - controlJetty = createJetty(controlHome, null); // let the shardId default to shard1 - controlJetty.start(); - controlClient = createNewSolrClient(controlJetty.getLocalPort()); - - assertTrue( - CollectionAdminRequest.createCollection("control_collection", 1, 1) - .setCreateNodeSet(controlJetty.getNodeName()) - .process(controlClient) - .isSuccess()); - - ZkStateReader zkStateReader = - jettys.get(0).getCoreContainer().getZkController().getZkStateReader(); - - waitForRecoveriesToFinish("control_collection", zkStateReader, false, true, 15); - - StringBuilder sb = new StringBuilder(); - for (int i = 1; i <= numShards; i++) { - if (sb.length() > 0) sb.append(','); - // give everyone there own solrhome - Path jettyHome = getSolrHome().getParent().resolve("jetty" + homeCount.incrementAndGet()); - setupJettySolrHome(jettyHome); - JettySolrRunner j = createJetty(jettyHome, null, "shard" + (i + 2)); - j.start(); - jettys.add(j); - clients.add(createNewSolrClient(j.getLocalPort())); - sb.append(buildUrl(j.getLocalPort())); - } - - shards = sb.toString(); - } - - protected void waitForRecoveriesToFinish( - String collection, ZkStateReader zkStateReader, boolean verbose) throws Exception { - waitForRecoveriesToFinish(collection, zkStateReader, verbose, true); - } - - protected void waitForRecoveriesToFinish( - String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout) - throws Exception { - waitForRecoveriesToFinish(collection, zkStateReader, verbose, failOnTimeout, 330, SECONDS); - } - - public static void waitForRecoveriesToFinish( - String collection, - ZkStateReader zkStateReader, - boolean verbose, - boolean failOnTimeout, - long timeoutSeconds) - throws Exception { - waitForRecoveriesToFinish( - collection, zkStateReader, verbose, failOnTimeout, timeoutSeconds, SECONDS); - } - - public static void waitForRecoveriesToFinish( - String collection, - ZkStateReader zkStateReader, - boolean verbose, - boolean failOnTimeout, - long timeout, - TimeUnit unit) - throws Exception { - log.info( - "Wait for recoveries to finish - collection:{} failOnTimeout:{} timeout:{}{}", - collection, - failOnTimeout, - timeout, - unit); - try { - zkStateReader.waitForState( - collection, - timeout, - unit, - (liveNodes, docCollection) -> { - if (docCollection == null) return false; - boolean sawLiveRecovering = false; - - Map slices = docCollection.getSlicesMap(); - assertNotNull("Could not find collection:" + collection, slices); - for (Map.Entry entry : slices.entrySet()) { - Slice slice = entry.getValue(); - if (slice.getState() - == Slice.State - .CONSTRUCTION) { // similar to replica recovering; pretend its the same - // thing - if (verbose) System.out.println("Found a slice in construction state; will wait."); - sawLiveRecovering = true; - } - Map shards = slice.getReplicasMap(); - for (Map.Entry shard : shards.entrySet()) { - if (verbose) - System.out.println( - "replica:" - + shard.getValue().getName() - + " rstate:" - + shard.getValue().getStr(ZkStateReader.STATE_PROP) - + " live:" - + liveNodes.contains(shard.getValue().getNodeName())); - final Replica.State state = shard.getValue().getState(); - if ((state == Replica.State.RECOVERING - || state == Replica.State.DOWN - || state == Replica.State.RECOVERY_FAILED) - && liveNodes.contains(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) { - return false; - } - } - } - if (!sawLiveRecovering) { - if (verbose) System.out.println("no one is recoverying"); - return true; - } else { - return false; - } - }); - } catch (TimeoutException | InterruptedException e) { - Diagnostics.logThreadDumps("Gave up waiting for recovery to finish. THREAD DUMP:"); - zkStateReader.getZkClient().printLayoutToStream(System.out); - fail("There are still nodes recovering - waited for " + timeout + unit); - } - - log.info("Recoveries finished - collection:{}", collection); - } - - public static void waitForCollectionToDisappear( - String collection, ZkStateReader zkStateReader, boolean failOnTimeout, int timeoutSeconds) - throws Exception { - log.info( - "Wait for collection to disappear - collection: {} failOnTimeout:{} timeout (sec):{}", - collection, - failOnTimeout, - timeoutSeconds); - - zkStateReader.waitForState( - collection, timeoutSeconds, TimeUnit.SECONDS, (docCollection) -> docCollection == null); - log.info("Collection has disappeared - collection:{}", collection); - } - - static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader) - throws Exception { - log.info("Will wait for a node to become leader for 15 secs"); - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - - long startNs = System.nanoTime(); - try { - zkStateReader.waitForState( - "collection1", - 15, - TimeUnit.SECONDS, - (docCollection) -> { - if (docCollection == null) return false; - - Slice slice = docCollection.getSlice(shardName); - if (slice != null - && slice.getLeader() != null - && !slice.getLeader().equals(oldLeader) - && slice.getLeader().getState() == Replica.State.ACTIVE) { - if (log.isInfoEnabled()) { - log.info( - "Old leader {}, new leader {}. New leader got elected in {} ms", - oldLeader, - slice.getLeader(), - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs)); - } - return true; - } - return false; - }); - } catch (TimeoutException e) { - // If we failed to get a new leader, print some diagnotics before the test fails - Diagnostics.logThreadDumps("Could not find new leader in specified timeout"); - zkStateReader.getZkClient().printLayoutToStream(System.out); - fail("Could not find new leader even after waiting for 15s"); - } - } - - public static void verifyReplicaStatus( - ZkStateReader reader, - String collection, - String shard, - String coreNodeName, - Replica.State expectedState) - throws InterruptedException, TimeoutException { - log.info("verifyReplicaStatus ({}) shard={} coreNodeName={}", collection, shard, coreNodeName); - reader.waitForState( - collection, - 15000, - TimeUnit.MILLISECONDS, - (collectionState) -> - collectionState != null - && collectionState.getSlice(shard) != null - && collectionState.getSlice(shard).getReplicasMap().get(coreNodeName) != null - && collectionState.getSlice(shard).getReplicasMap().get(coreNodeName).getState() - == expectedState); - } - - protected static void assertAllActive(String collection, ZkStateReader zkStateReader) - throws KeeperException, InterruptedException { - - zkStateReader.forceUpdateCollection(collection); - ClusterState clusterState = zkStateReader.getClusterState(); - final DocCollection docCollection = clusterState.getCollectionOrNull(collection); - if (docCollection == null || docCollection.getSlices() == null) { - throw new IllegalArgumentException("Cannot find collection:" + collection); - } - - Map slices = docCollection.getSlicesMap(); - for (Map.Entry entry : slices.entrySet()) { - Slice slice = entry.getValue(); - if (slice.getState() != Slice.State.ACTIVE) { - fail( - "Not all shards are ACTIVE - found a shard " - + slice.getName() - + " that is: " - + slice.getState()); - } - Map shards = slice.getReplicasMap(); - for (Map.Entry shard : shards.entrySet()) { - Replica replica = shard.getValue(); - if (replica.getState() != Replica.State.ACTIVE) { - fail( - "Not all replicas are ACTIVE - found a replica " - + replica.getName() - + " that is: " - + replica.getState()); - } - } - } - } - - @Override - public void distribTearDown() throws Exception { - resetExceptionIgnores(); - - try { - zkServer.shutdown(); - } catch (Exception e) { - throw new RuntimeException("Exception shutting down Zk Test Server.", e); - } finally { - try { - super.distribTearDown(); - } finally { - System.clearProperty(ZK_HOST); - System.clearProperty("collection"); - System.clearProperty(ENABLE_UPDATE_LOG); - System.clearProperty(REMOVE_VERSION_FIELD); - System.clearProperty("solr.directoryFactory"); - System.clearProperty("solr.test.sys.prop1"); - System.clearProperty("solr.test.sys.prop2"); - System.clearProperty(ZOOKEEPER_FORCE_SYNC); - System.clearProperty( - MockDirectoryFactory.SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE); - } - } - } - - protected void printLayout() throws Exception { - SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkHost()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - zkClient.printLayoutToStream(System.out); - zkClient.close(); - } - - protected void restartZk(int pauseMillis) throws Exception { - log.info("Restarting ZK with a pause of {}ms in between", pauseMillis); - zkServer.shutdown(); - // disconnect enough to test stalling, if things stall, then clientSoTimeout will be hit - Thread.sleep(pauseMillis); - zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); - zkServer.run(false); - } - - // Copy a configset up from some path on the local machine to ZK. - // Example usage: - // - // copyConfigUp(TEST_PATH().resolve("configsets"), "cloud-minimal", "configset-name", zk_address); - - public static void copyConfigUp( - Path configSetDir, String srcConfigSet, String dstConfigName, String zkAddr) - throws Exception { - - Path fullConfDir = configSetDir.resolve(srcConfigSet); - String[] args = - new String[] { - "--conf-name", dstConfigName, - "--conf-dir", fullConfDir.toAbsolutePath().toString(), - "-z", zkAddr - }; - - ToolRuntime runtime = new DefaultToolRuntime(); - ConfigSetUploadTool tool = new ConfigSetUploadTool(runtime); - - int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args)); - assertEquals("Tool should have returned 0 for success, returned: " + res, res, 0); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index cbac9ba4c69..3eed7a88099 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -16,6 +16,7 @@ */ package org.apache.solr.cloud; +import static java.util.concurrent.TimeUnit.SECONDS; import static org.apache.solr.common.cloud.ZkStateReader.HTTPS; import static org.apache.solr.common.cloud.ZkStateReader.URL_SCHEME; @@ -46,6 +47,11 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; import java.util.function.Consumer; +import org.apache.solr.BaseDistributedSearchTestCase; +import org.apache.solr.cli.ConfigSetUploadTool; +import org.apache.solr.cli.DefaultToolRuntime; +import org.apache.solr.cli.SolrCLI; +import org.apache.solr.cli.ToolRuntime; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; @@ -75,6 +81,7 @@ import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.SolrZkClient; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.params.CollectionParams; @@ -90,6 +97,7 @@ import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.Diagnostics; +import org.apache.solr.core.MockDirectoryFactory; import org.apache.solr.core.SolrCore; import org.apache.solr.embedded.JettyConfig; import org.apache.solr.embedded.JettySolrRunner; @@ -114,9 +122,24 @@ * TODO: we should still test this works as a custom update chain as well as what we test now - the * default update chain */ -public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTestBase { +public abstract class AbstractFullDistribZkTestBase extends BaseDistributedSearchTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + // Fields from AbstractFullDistribZkTestBase + private static final String REMOVE_VERSION_FIELD = "remove.version.field"; + private static final String ENABLE_UPDATE_LOG = "solr.index.updatelog.enabled"; + private static final String ZK_HOST = "zkHost"; + private static final String ZOOKEEPER_FORCE_SYNC = "zookeeper.forceSync"; + protected static final String DEFAULT_COLLECTION = "collection1"; + protected volatile ZkTestServer zkServer; + private final AtomicInteger homeCount = new AtomicInteger(); + + @BeforeClass + public static void beforeThisClass() throws Exception { + // Only For Manual Testing: this will force an fs based dir factory + // useFactory(null); + } + @BeforeClass public static void beforeFullSolrCloudTest() {} @@ -242,6 +265,28 @@ protected static void clearErrorHook() { @Override public void distribSetUp() throws Exception { super.distribSetUp(); + + // Setup from AbstractFullDistribZkTestBase + Path zkDir = testDir.resolve("zookeeper/server1/data"); + zkServer = new ZkTestServer(zkDir); + zkServer.run(); + + System.setProperty(ZK_HOST, zkServer.getZkAddress()); + System.setProperty(ENABLE_UPDATE_LOG, "true"); + System.setProperty(REMOVE_VERSION_FIELD, "true"); + System.setProperty(ZOOKEEPER_FORCE_SYNC, "false"); + System.setProperty( + MockDirectoryFactory.SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE, "true"); + + String schema = getCloudSchemaFile(); + if (schema == null) schema = "schema.xml"; + zkServer.buildZooKeeper(getCloudSolrConfig(), schema); + + // set some system properties for use by tests + System.setProperty("solr.test.sys.prop1", "propone"); + System.setProperty("solr.test.sys.prop2", "proptwo"); + // End setup from AbstractFullDistribZkTestBase + // ignoreException(".*"); cloudInit = false; @@ -285,6 +330,14 @@ public void distribSetUp() throws Exception { } } + protected String getCloudSolrConfig() { + return "solrconfig-tlog.xml"; + } + + protected String getCloudSchemaFile() { + return getSchemaFile(); + } + @BeforeClass public static void beforeClass() { System.setProperty("solrcloud.update.delay", "0"); @@ -1152,12 +1205,12 @@ protected void del(String q) throws Exception { protected void waitForRecoveriesToFinish(boolean verbose) throws Exception { ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - super.waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose); + waitForRecoveriesToFinish(DEFAULT_COLLECTION, zkStateReader, verbose); } protected void waitForRecoveriesToFinish(String collection, boolean verbose) throws Exception { ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - super.waitForRecoveriesToFinish(collection, zkStateReader, verbose); + waitForRecoveriesToFinish(collection, zkStateReader, verbose); } protected void waitForRecoveriesToFinish(boolean verbose, long timeoutSeconds) throws Exception { @@ -1949,16 +2002,37 @@ void doQuery(String expectedDocs, String... queryParams) throws Exception { public void distribTearDown() throws Exception { try { if (VERBOSE || printLayoutOnTearDown) { - super.printLayout(); + printLayout(); } closeRestTestHarnesses(); // TODO: close here or later? } finally { - super.distribTearDown(); + // Cleanup from AbstractFullDistribZkTestBase + resetExceptionIgnores(); - System.clearProperty("zkHost"); - System.clearProperty("numShards"); + try { + zkServer.shutdown(); + } catch (Exception e) { + throw new RuntimeException("Exception shutting down Zk Test Server.", e); + } finally { + try { + super.distribTearDown(); + } finally { + System.clearProperty(ZK_HOST); + System.clearProperty("collection"); + System.clearProperty(ENABLE_UPDATE_LOG); + System.clearProperty(REMOVE_VERSION_FIELD); + System.clearProperty("solr.directoryFactory"); + System.clearProperty("solr.test.sys.prop1"); + System.clearProperty("solr.test.sys.prop2"); + System.clearProperty(ZOOKEEPER_FORCE_SYNC); + System.clearProperty( + MockDirectoryFactory.SOLR_TESTS_ALLOW_READING_FILES_STILL_OPEN_FOR_WRITE); + System.clearProperty("zkHost"); + System.clearProperty("numShards"); + } + } } } @@ -2865,4 +2939,245 @@ protected RestTestHarness randomRestTestHarness(Random random) { protected void forAllRestTestHarnesses(Consumer op) { restTestHarnesses.forEach(op); } + + // Methods from AbstractFullDistribZkTestBase + protected void waitForRecoveriesToFinish( + String collection, ZkStateReader zkStateReader, boolean verbose) throws Exception { + waitForRecoveriesToFinish(collection, zkStateReader, verbose, true); + } + + protected void waitForRecoveriesToFinish( + String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout) + throws Exception { + waitForRecoveriesToFinish(collection, zkStateReader, verbose, failOnTimeout, 330, SECONDS); + } + + public static void waitForRecoveriesToFinish( + String collection, + ZkStateReader zkStateReader, + boolean verbose, + boolean failOnTimeout, + long timeoutSeconds) + throws Exception { + waitForRecoveriesToFinish( + collection, zkStateReader, verbose, failOnTimeout, timeoutSeconds, SECONDS); + } + + public static void waitForRecoveriesToFinish( + String collection, + ZkStateReader zkStateReader, + boolean verbose, + boolean failOnTimeout, + long timeout, + TimeUnit unit) + throws Exception { + log.info( + "Wait for recoveries to finish - collection:{} failOnTimeout:{} timeout:{}{}", + collection, + failOnTimeout, + timeout, + unit); + try { + zkStateReader.waitForState( + collection, + timeout, + unit, + (liveNodes, docCollection) -> { + if (docCollection == null) return false; + boolean sawLiveRecovering = false; + + Map slices = docCollection.getSlicesMap(); + assertNotNull("Could not find collection:" + collection, slices); + for (Map.Entry entry : slices.entrySet()) { + Slice slice = entry.getValue(); + if (slice.getState() + == Slice.State + .CONSTRUCTION) { // similar to replica recovering; pretend its the same + // thing + if (verbose) System.out.println("Found a slice in construction state; will wait."); + sawLiveRecovering = true; + } + Map shards = slice.getReplicasMap(); + for (Map.Entry shard : shards.entrySet()) { + if (verbose) + System.out.println( + "replica:" + + shard.getValue().getName() + + " rstate:" + + shard.getValue().getStr(ZkStateReader.STATE_PROP) + + " live:" + + liveNodes.contains(shard.getValue().getNodeName())); + final Replica.State state = shard.getValue().getState(); + if ((state == Replica.State.RECOVERING + || state == Replica.State.DOWN + || state == Replica.State.RECOVERY_FAILED) + && liveNodes.contains(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) { + return false; + } + } + } + if (!sawLiveRecovering) { + if (verbose) System.out.println("no one is recoverying"); + return true; + } else { + return false; + } + }); + } catch (TimeoutException | InterruptedException e) { + Diagnostics.logThreadDumps("Gave up waiting for recovery to finish. THREAD DUMP:"); + zkStateReader.getZkClient().printLayoutToStream(System.out); + fail("There are still nodes recovering - waited for " + timeout + unit); + } + + log.info("Recoveries finished - collection:{}", collection); + } + + public static void waitForCollectionToDisappear( + String collection, ZkStateReader zkStateReader, boolean failOnTimeout, int timeoutSeconds) + throws Exception { + log.info( + "Wait for collection to disappear - collection: {} failOnTimeout:{} timeout (sec):{}", + collection, + failOnTimeout, + timeoutSeconds); + + zkStateReader.waitForState( + collection, timeoutSeconds, TimeUnit.SECONDS, (docCollection) -> docCollection == null); + log.info("Collection has disappeared - collection:{}", collection); + } + + static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader) + throws Exception { + log.info("Will wait for a node to become leader for 15 secs"); + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + + long startNs = System.nanoTime(); + try { + zkStateReader.waitForState( + "collection1", + 15, + TimeUnit.SECONDS, + (docCollection) -> { + if (docCollection == null) return false; + + Slice slice = docCollection.getSlice(shardName); + if (slice != null + && slice.getLeader() != null + && !slice.getLeader().equals(oldLeader) + && slice.getLeader().getState() == Replica.State.ACTIVE) { + if (log.isInfoEnabled()) { + log.info( + "Old leader {}, new leader {}. New leader got elected in {} ms", + oldLeader, + slice.getLeader(), + TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs)); + } + return true; + } + return false; + }); + } catch (TimeoutException e) { + // If we failed to get a new leader, print some diagnotics before the test fails + Diagnostics.logThreadDumps("Could not find new leader in specified timeout"); + zkStateReader.getZkClient().printLayoutToStream(System.out); + fail("Could not find new leader even after waiting for 15s"); + } + } + + public static void verifyReplicaStatus( + ZkStateReader reader, + String collection, + String shard, + String coreNodeName, + Replica.State expectedState) + throws InterruptedException, TimeoutException { + log.info("verifyReplicaStatus ({}) shard={} coreNodeName={}", collection, shard, coreNodeName); + reader.waitForState( + collection, + 15000, + TimeUnit.MILLISECONDS, + (collectionState) -> + collectionState != null + && collectionState.getSlice(shard) != null + && collectionState.getSlice(shard).getReplicasMap().get(coreNodeName) != null + && collectionState.getSlice(shard).getReplicasMap().get(coreNodeName).getState() + == expectedState); + } + + protected static void assertAllActive(String collection, ZkStateReader zkStateReader) + throws KeeperException, InterruptedException { + + zkStateReader.forceUpdateCollection(collection); + ClusterState clusterState = zkStateReader.getClusterState(); + final DocCollection docCollection = clusterState.getCollectionOrNull(collection); + if (docCollection == null || docCollection.getSlices() == null) { + throw new IllegalArgumentException("Cannot find collection:" + collection); + } + + Map slices = docCollection.getSlicesMap(); + for (Map.Entry entry : slices.entrySet()) { + Slice slice = entry.getValue(); + if (slice.getState() != Slice.State.ACTIVE) { + fail( + "Not all shards are ACTIVE - found a shard " + + slice.getName() + + " that is: " + + slice.getState()); + } + Map shards = slice.getReplicasMap(); + for (Map.Entry shard : shards.entrySet()) { + Replica replica = shard.getValue(); + if (replica.getState() != Replica.State.ACTIVE) { + fail( + "Not all replicas are ACTIVE - found a replica " + + replica.getName() + + " that is: " + + replica.getState()); + } + } + } + } + + protected void printLayout() throws Exception { + SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkHost()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + zkClient.printLayoutToStream(System.out); + zkClient.close(); + } + + protected void restartZk(int pauseMillis) throws Exception { + log.info("Restarting ZK with a pause of {}ms in between", pauseMillis); + zkServer.shutdown(); + // disconnect enough to test stalling, if things stall, then clientSoTimeout will be hit + Thread.sleep(pauseMillis); + zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); + zkServer.run(false); + } + + // Copy a configset up from some path on the local machine to ZK. + // Example usage: + // + // copyConfigUp(TEST_PATH().resolve("configsets"), "cloud-minimal", "configset-name", zk_address); + + public static void copyConfigUp( + Path configSetDir, String srcConfigSet, String dstConfigName, String zkAddr) + throws Exception { + + Path fullConfDir = configSetDir.resolve(srcConfigSet); + String[] args = + new String[] { + "--conf-name", dstConfigName, + "--conf-dir", fullConfDir.toAbsolutePath().toString(), + "-z", zkAddr + }; + + ToolRuntime runtime = new DefaultToolRuntime(); + ConfigSetUploadTool tool = new ConfigSetUploadTool(runtime); + + int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args)); + assertEquals("Tool should have returned 0 for success, returned: " + res, res, 0); + } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java index fbe1fba5ec8..a621c04fbbb 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/MultiSolrCloudTestCase.java @@ -74,7 +74,7 @@ protected void doAccept(String collection, MiniSolrCloudCluster cluster) { CollectionAdminRequest.createCollection(collection, "conf", numShards, numReplicas) .processAndWait(cluster.getSolrClient(), SolrCloudTestCase.DEFAULT_TIMEOUT); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( collection, cluster.getZkStateReader(), false, true, SolrCloudTestCase.DEFAULT_TIMEOUT); } catch (Exception e) { throw new RuntimeException(e); diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java index b6fd4b45f3c..91476c27992 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java @@ -37,7 +37,7 @@ import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest.ClusterProp; import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrException; import org.apache.solr.common.SolrException.ErrorCode; @@ -388,7 +388,7 @@ private void testBackupAndRestore(String collectionName) throws Exception { assertNotNull(asyncId); CollectionAdminRequest.waitForAsyncRequest(asyncId, client, 60); } - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( restoreCollectionName, ZkStateReader.from(client), log.isDebugEnabled(), true, 30); // Check the number of results are the same diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java index 9cf6c15e0aa..7a59d490beb 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractIncrementalBackupTest.java @@ -54,7 +54,7 @@ import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.Replica; @@ -178,7 +178,7 @@ public void testSimple() throws Exception { timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t); log.info("Restored from backup, took {}ms", timeTaken); t = System.nanoTime(); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( restoreCollectionName, ZkStateReader.from(solrClient), log.isDebugEnabled(), false, 3); timeTaken = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - t); log.info("Restored collection healthy, took {}ms", timeTaken); @@ -409,7 +409,7 @@ public void testSkipConfigset() throws Exception { .setRepositoryName(BACKUP_REPO_NAME) .processAndWait(solrClient, 500); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( restoreCollectionName, ZkStateReader.from(solrClient), log.isDebugEnabled(), false, 3); assertEquals( numDocs, @@ -553,7 +553,7 @@ private void simpleRestoreAndCheckDocCount( .setRepositoryName(BACKUP_REPO_NAME) .process(solrClient); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( restoreCollectionName, ZkStateReader.from(solrClient), log.isDebugEnabled(), true, 30); // check num docs are the same diff --git a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/CloudSolrClientTest.java b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/CloudSolrClientTest.java index a989ba1ebe6..896b00ae6aa 100644 --- a/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/CloudSolrClientTest.java +++ b/solr/test-framework/src/test/org/apache/solr/client/solrj/apache/CloudSolrClientTest.java @@ -54,7 +54,7 @@ import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.client.solrj.response.SolrPingResponse; import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.AbstractDistribZkTestBase; +import org.apache.solr.cloud.AbstractFullDistribZkTestBase; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -953,7 +953,7 @@ public void testRetryUpdatesWhenClusterStateIsStale() throws Exception { // change .process(cluster.getSolrClient()) .getStatus()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COL, cluster.getZkStateReader(), true, true, 330); // ...and delete our original leader. assertEquals( @@ -964,7 +964,7 @@ public void testRetryUpdatesWhenClusterStateIsStale() throws Exception { // change .process(cluster.getSolrClient()) .getStatus()); - AbstractDistribZkTestBase.waitForRecoveriesToFinish( + AbstractFullDistribZkTestBase.waitForRecoveriesToFinish( COL, cluster.getZkStateReader(), true, true, 330); // stale_client's collection state cache should now only point at a leader that no longer From 8dbe540cb403a2e227462995ba2e105d511fa8b9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 09:03:06 -0500 Subject: [PATCH 07/16] Also remove the call to the method --- .../api/collections/AbstractCloudBackupRestoreTestCase.java | 1 - 1 file changed, 1 deletion(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java index b6fd4b45f3c..335ed187626 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java @@ -172,7 +172,6 @@ public void test() throws Exception { } testBackupAndRestore(getCollectionName()); - testConfigBackupOnly("conf1", getCollectionName()); testInvalidPath(getCollectionName()); } From 19c7340231d65bdb0bfb35992329e882ad7e2de4 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 09:16:29 -0500 Subject: [PATCH 08/16] inPlaceMove being true or false makes no impact So lets be random! Remove reference to no longer existing HDFS test --- solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index d5dd04076fe..00ff9f9b74d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -51,8 +51,7 @@ public class MoveReplicaTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - // used by MoveReplicaHDFSTest - protected boolean inPlaceMove = true; + protected boolean inPlaceMove = random().nextBoolean(); protected boolean isCollectionApiDistributed = false; protected String getConfigSet() { @@ -61,7 +60,6 @@ protected String getConfigSet() { @Before public void beforeTest() throws Exception { - inPlaceMove = true; configureCluster(4) .addConfig("conf1", configset(getConfigSet())) From db4860ad08db87611d515095e70c55ba7e19aab5 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 11:21:37 -0500 Subject: [PATCH 09/16] Back out the consolidation change till we can debug test failures. --- .../CollectionsAPIDistributedZkTest.java | 682 +---------------- ...ctCollectionsAPIDistributedZkTestBase.java | 700 ++++++++++++++++++ 2 files changed, 702 insertions(+), 680 deletions(-) create mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java diff --git a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java index 2367c6cbb99..11c04c976cd 100644 --- a/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/api/collections/CollectionsAPIDistributedZkTest.java @@ -16,67 +16,10 @@ */ package org.apache.solr.cloud.api.collections; -import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; -import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Map.Entry; -import java.util.Optional; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.util.TestUtil; -import org.apache.solr.client.api.model.CoreStatusResponse; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; -import org.apache.solr.client.solrj.SolrRequest.METHOD; -import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.cloud.SolrCloudTestCase; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.CoreAdminParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.TimeSource; -import org.apache.solr.core.CoreContainer; -import org.apache.solr.core.SolrCore; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.apache.solr.util.TimeOut; -import org.junit.After; import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Tests the Cloud Collections API. - * - *

Because the different setups require distinct config-sets, we have to push down cluster - * creation to subclasses - */ -public class CollectionsAPIDistributedZkTest extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); +/** Tests the Cloud Collections API. */ +public class CollectionsAPIDistributedZkTest extends AbstractCollectionsAPIDistributedZkTestBase { @BeforeClass public static void createCluster() throws Exception { configureCluster(4) @@ -85,625 +28,4 @@ public static void createCluster() throws Exception { .withSolrXml(TEST_PATH().resolve("solr.xml")) .configure(); } - - @BeforeClass - public static void setupCluster() { - // we don't want this test to have zk timeouts - System.setProperty("zkClientTimeout", "60000"); - System.setProperty("createCollectionWaitTimeTillActive", "5"); - TestInjection.randomDelayInCoreCreation = "true:5"; - System.setProperty("validateAfterInactivity", "200"); - System.setProperty("solr.security.allow.paths", "*"); - } - - @Override - @After - public void tearDown() throws Exception { - cluster.deleteAllCollections(); - super.tearDown(); - } - - @Test - public void testCreationAndDeletion() throws Exception { - String collectionName = "created_and_deleted"; - - CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) - .process(cluster.getSolrClient()); - assertTrue( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - assertFalse( - cluster - .getZkClient() - .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); - } - - @Test - public void deleteCollectionRemovesStaleZkCollectionsNode() throws Exception { - String collectionName = "out_of_sync_collection"; - - // manually create a collections zknode - cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); - - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - assertFalse( - cluster - .getZkClient() - .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); - } - - @Test - public void deletePartiallyCreatedCollection() throws Exception { - final String collectionName = "halfdeletedcollection"; - - assertEquals( - 0, - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) - .setCreateNodeSet("") - .process(cluster.getSolrClient()) - .getStatus()); - String dataDir = createTempDir().toString(); - // create a core that simulates something left over from a partially-deleted collection - assertTrue( - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .setDataDir(dataDir) - .process(cluster.getSolrClient()) - .isSuccess()); - - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) - .process(cluster.getSolrClient()); - - assertTrue( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - } - - @Test - public void deleteCollectionOnlyInZk() throws Exception { - final String collectionName = "onlyinzk"; - - // create the collections node, but nothing else - cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); - - // delete via API - should remove collections node - CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - - // now creating that collection should work - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) - .process(cluster.getSolrClient()); - assertTrue( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); - } - - @Test - public void testBadActionNames() { - // try a bad action - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", "BADACTION"); - String collectionName = "badactioncollection"; - params.set("name", collectionName); - params.set("numShards", 2); - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testMissingRequiredParameters() { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("numShards", 2); - // missing required collection parameter - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testMissingNumShards() { - // No numShards should fail - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("name", "acollection"); - params.set(REPLICATION_FACTOR, 10); - params.set("collection.configName", "conf"); - - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testZeroNumShards() { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - params.set("name", "acollection"); - params.set(REPLICATION_FACTOR, 10); - params.set("numShards", 0); - params.set("collection.configName", "conf"); - - var request = - new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - expectThrows( - Exception.class, - () -> { - cluster.getSolrClient().request(request); - }); - } - - @Test - public void testCreateShouldFailOnExistingCore() throws Exception { - String nn1 = cluster.getJettySolrRunner(0).getNodeName(); - String nn2 = cluster.getJettySolrRunner(1).getNodeName(); - - assertEquals( - 0, - CollectionAdminRequest.createCollection("halfcollectionblocker", "conf", 1, 1) - .setCreateNodeSet("") - .process(cluster.getSolrClient()) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard("halfcollectionblocker", "shard1") - .setNode(cluster.getJettySolrRunner(0).getNodeName()) - .setCoreName("halfcollection_shard1_replica_n1") - .process(cluster.getSolrClient()) - .isSuccess()); - - assertEquals( - 0, - CollectionAdminRequest.createCollection("halfcollectionblocker2", "conf", 1, 1) - .setCreateNodeSet("") - .process(cluster.getSolrClient()) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard("halfcollectionblocker2", "shard1") - .setNode(cluster.getJettySolrRunner(1).getNodeName()) - .setCoreName("halfcollection_shard1_replica_n1") - .process(cluster.getSolrClient()) - .isSuccess()); - - expectThrows( - SolrClient.RemoteSolrException.class, - () -> { - CollectionAdminRequest.createCollection("halfcollection", "conf", 1, 1) - .setCreateNodeSet(nn1 + "," + nn2) - .process(cluster.getSolrClient()); - }); - } - - @Test - public void testNoConfigSetExist() throws Exception { - expectThrows( - Exception.class, - () -> { - CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1) - .process(cluster.getSolrClient()); - }); - - TimeUnit.MILLISECONDS.sleep(1000); - // in both cases, the collection should have default to the core name - cluster.getZkStateReader().forceUpdateCollection("noconfig"); - assertFalse( - CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig")); - } - - @Test - public void testCoresAreDistributedAcrossNodes() throws Exception { - CollectionAdminRequest.createCollection("nodes_used_collection", "conf", 2, 2) - .process(cluster.getSolrClient()); - - Set liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes(); - - List createNodeList = new ArrayList<>(liveNodes); - - DocCollection collection = getCollectionState("nodes_used_collection"); - for (Slice slice : collection.getSlices()) { - for (Replica replica : slice.getReplicas()) { - createNodeList.remove(replica.getNodeName()); - } - } - - assertEquals(createNodeList.toString(), 0, createNodeList.size()); - } - - @Test - public void testDeleteNonExistentCollection() throws Exception { - - expectThrows( - SolrException.class, - () -> { - CollectionAdminRequest.deleteCollection("unknown_collection") - .process(cluster.getSolrClient()); - }); - - // create another collection should still work - CollectionAdminRequest.createCollection("acollectionafterbaddelete", "conf", 1, 2) - .process(cluster.getSolrClient()); - waitForState( - "Collection creation after a bad delete failed", - "acollectionafterbaddelete", - (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 2)); - } - - @Test - public void testSpecificConfigsets() throws Exception { - CollectionAdminRequest.createCollection("withconfigset2", "conf2", 1, 1) - .process(cluster.getSolrClient()); - String configName = - cluster - .getSolrClient() - .getClusterStateProvider() - .getCollection("withconfigset2") - .getConfigName(); - assertEquals("conf2", configName); - } - - @Test - public void testCreateNodeSet() throws Exception { - JettySolrRunner jetty1 = cluster.getRandomJetty(random()); - JettySolrRunner jetty2 = cluster.getRandomJetty(random()); - - List baseUrls = List.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString()); - - CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1) - .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1)) - .process(cluster.getSolrClient()); - - DocCollection collectionState = getCollectionState("nodeset_collection"); - for (Replica replica : collectionState.getReplicas()) { - String replicaUrl = replica.getCoreUrl(); - boolean matchingJetty = false; - for (String jettyUrl : baseUrls) { - if (replicaUrl.startsWith(jettyUrl)) { - matchingJetty = true; - } - } - if (matchingJetty == false) { - fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl); - } - } - } - - @Test - public void testCollectionsAPI() throws Exception { - - // create new collections rapid fire - int cnt = random().nextInt(TEST_NIGHTLY ? 3 : 1) + 1; - CollectionAdminRequest.Create[] createRequests = new CollectionAdminRequest.Create[cnt]; - - class Coll { - String name; - int numShards; - int replicationFactor; - } - - List colls = new ArrayList<>(); - - for (int i = 0; i < cnt; i++) { - - int numShards = TestUtil.nextInt(random(), 0, cluster.getJettySolrRunners().size()) + 1; - int replicationFactor = TestUtil.nextInt(random(), 0, 3) + 1; - - createRequests[i] = - CollectionAdminRequest.createCollection( - "awhollynewcollection_" + i, "conf2", numShards, replicationFactor); - createRequests[i].processAsync(cluster.getSolrClient()); - - Coll coll = new Coll(); - coll.name = "awhollynewcollection_" + i; - coll.numShards = numShards; - coll.replicationFactor = replicationFactor; - colls.add(coll); - } - - for (Coll coll : colls) { - cluster.waitForActiveCollection( - coll.name, coll.numShards, coll.numShards * coll.replicationFactor); - } - - waitForStable(cnt, createRequests); - - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - checkInstanceDirs(cluster.getJettySolrRunner(i)); - } - - String collectionName = - createRequests[random().nextInt(createRequests.length)].getCollectionName(); - - // TODO: we should not need this...beast test well when trying to fix - Thread.sleep(1000); - - cluster.getZkStateReader().forciblyRefreshAllClusterStateSlow(); - - new UpdateRequest() - .add("id", "6") - .add("id", "7") - .add("id", "8") - .commit(cluster.getSolrClient(), collectionName); - long numFound = 0; - TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); - while (!timeOut.hasTimedOut()) { - - numFound = - cluster - .getSolrClient() - .query(collectionName, new SolrQuery("*:*")) - .getResults() - .getNumFound(); - if (numFound == 3) { - break; - } - - Thread.sleep(500); - } - - if (timeOut.hasTimedOut()) { - fail( - "Timeout waiting to see 3 found, instead saw " - + numFound - + " for collection " - + collectionName); - } - - checkNoTwoShardsUseTheSameIndexDir(); - } - - private void waitForStable(int cnt, CollectionAdminRequest.Create[] createRequests) - throws InterruptedException { - for (int i = 0; i < cnt; i++) { - String collectionName = "awhollynewcollection_" + i; - final int j = i; - waitForState( - "Expected to see collection " + collectionName, - collectionName, - (n, c) -> { - CollectionAdminRequest.Create req = createRequests[j]; - return SolrCloudTestCase.replicasForCollectionAreFullyActive( - n, c, req.getNumShards(), req.getReplicationFactor()); - }); - - ZkStateReader zkStateReader = cluster.getZkStateReader(); - // make sure we have leaders for each shard - for (int z = 1; z < createRequests[j].getNumShards(); z++) { - zkStateReader.getLeaderRetry(collectionName, "shard" + z, 10000); - } // make sure we again have leaders for each shard - } - } - - @Test - public void testCollectionReload() throws Exception { - final String collectionName = "reloaded_collection"; - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) - .process(cluster.getSolrClient()); - - // get core open times - Map urlToTimeBefore = new HashMap<>(); - collectStartTimes(collectionName, urlToTimeBefore); - assertTrue(urlToTimeBefore.size() > 0); - - CollectionAdminRequest.reloadCollection(collectionName).processAsync(cluster.getSolrClient()); - - // reloads make take a short while - boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore); - assertTrue("some core start times did not change on reload", allTimesAreCorrect); - } - - private void checkInstanceDirs(JettySolrRunner jetty) throws IOException { - CoreContainer cores = jetty.getCoreContainer(); - Collection theCores = cores.getCores(); - for (SolrCore core : theCores) { - // look for core props file - Path instancedir = core.getInstancePath(); - assertTrue( - "Could not find expected core.properties file", - Files.exists(instancedir.resolve("core.properties"))); - - Path expected = Path.of(jetty.getSolrHome()).resolve(core.getName()); - - assertTrue( - "Expected: " + expected + "\nFrom core stats: " + instancedir, - Files.isSameFile(expected, instancedir)); - } - } - - private boolean waitForReloads(String collectionName, Map urlToTimeBefore) - throws SolrServerException, IOException { - TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME); - - boolean allTimesAreCorrect = false; - while (!timeout.hasTimedOut()) { - Map urlToTimeAfter = new HashMap<>(); - collectStartTimes(collectionName, urlToTimeAfter); - - boolean retry = false; - Set> entries = urlToTimeBefore.entrySet(); - for (Entry entry : entries) { - Long beforeTime = entry.getValue(); - Long afterTime = urlToTimeAfter.get(entry.getKey()); - assertNotNull(afterTime); - if (afterTime <= beforeTime) { - retry = true; - break; - } - } - if (!retry) { - allTimesAreCorrect = true; - break; - } - } - return allTimesAreCorrect; - } - - private void collectStartTimes(String collectionName, Map urlToTime) - throws SolrServerException, IOException { - - DocCollection collectionState = getCollectionState(collectionName); - if (collectionState != null) { - for (Slice shard : collectionState) { - for (Replica replica : shard) { - CoreStatusResponse.SingleCoreData coreStatus; - try (SolrClient server = getHttpSolrClient(replica.getBaseUrl())) { - coreStatus = CoreAdminRequest.getCoreStatus(replica.getCoreName(), false, server); - } - long before = coreStatus.startTime.getTime(); - urlToTime.put(replica.getCoreUrl(), before); - } - } - } else { - throw new IllegalArgumentException("Could not find collection " + collectionName); - } - } - - private void checkNoTwoShardsUseTheSameIndexDir() { - Map> indexDirToShardNamesMap = new HashMap<>(); - - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - CoreContainer coreContainer = jetty.getCoreContainer(); - List coreNames = coreContainer.getAllCoreNames(); - for (String coreName : coreNames) { - try (SolrCore core = coreContainer.getCore(coreName)) { - String indexDir = core.getIndexDir(); - String shardKey = jetty.getNodeName() + "." + coreName; - if (!indexDirToShardNamesMap.containsKey(indexDir)) { - indexDirToShardNamesMap.put(indexDir, new HashSet<>()); - } - indexDirToShardNamesMap.get(indexDir).add(shardKey); - } - } - } - - assertTrue( - "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of " - + SolrCore.class.getSimpleName() - + " : " - + indexDirToShardNamesMap, - indexDirToShardNamesMap.size() > 0); - - for (Map.Entry> entry : indexDirToShardNamesMap.entrySet()) { - if (entry.getValue().size() > 1) { - fail( - "We have shards using the same indexDir. E.g. shards " - + entry.getValue().toString() - + " all use indexDir " - + entry.getKey()); - } - } - } - - @Test - public void addReplicaTest() throws Exception { - String collectionName = "addReplicaColl"; - - CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) - .process(cluster.getSolrClient()); - cluster.waitForActiveCollection(collectionName, 2, 4); - - ArrayList nodeList = - new ArrayList<>(cluster.getSolrClient().getClusterState().getLiveNodes()); - Collections.shuffle(nodeList, random()); - - CollectionAdminResponse response = - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .setNode(nodeList.get(0)) - .process(cluster.getSolrClient()); - Replica newReplica = grabNewReplica(response, getCollectionState(collectionName)); - - assertEquals( - "Replica should be created on the right node", - cluster.getZkStateReader().getBaseUrlForNodeName(nodeList.get(0)), - newReplica.getBaseUrl()); - - Path instancePath = createTempDir(); - response = - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .withProperty(CoreAdminParams.INSTANCE_DIR, instancePath.toString()) - .process(cluster.getSolrClient()); - newReplica = grabNewReplica(response, getCollectionState(collectionName)); - assertNotNull(newReplica); - cluster.waitForActiveCollection(collectionName, 2, 6); - - try (SolrClient coreclient = getHttpSolrClient(newReplica.getBaseUrl())) { - CoreAdminResponse status = CoreAdminRequest.getStatus(newReplica.getStr("core"), coreclient); - final var coreStatus = status.getCoreStatus(newReplica.getStr("core")); - String instanceDirStr = coreStatus.instanceDir; - assertEquals(instanceDirStr, instancePath.toString()); - } - - // Test to make sure we can't create another replica with an existing core_name of that - // collection - String coreName = newReplica.getStr(CORE_NAME_PROP); - SolrException e = - expectThrows( - SolrException.class, - () -> { - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", "addreplica"); - params.set("collection", collectionName); - params.set("shard", "shard1"); - params.set("name", coreName); - var request = - new GenericSolrRequest( - METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); - cluster.getSolrClient().request(request); - }); - - assertTrue( - e.getMessage() - .contains( - "Another replica with the same core name already exists for this collection")); - - // Check that specifying property.name works. DO NOT remove this when the "name" property is - // deprecated - // for ADDREPLICA, this is "property.name". See SOLR-7132 - response = - CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") - .withProperty(CoreAdminParams.NAME, "propertyDotName") - .process(cluster.getSolrClient()); - - newReplica = grabNewReplica(response, getCollectionState(collectionName)); - assertEquals( - "'core' should be 'propertyDotName' ", "propertyDotName", newReplica.getStr("core")); - cluster.waitForActiveCollection(collectionName, 2, 7); - } - - private Replica grabNewReplica(CollectionAdminResponse response, DocCollection docCollection) { - String replicaName = response.getCollectionCoresStatus().keySet().iterator().next(); - Optional optional = - docCollection.getReplicas().stream() - .filter(replica -> replicaName.equals(replica.getCoreName())) - .findAny(); - if (optional.isPresent()) { - return optional.get(); - } - throw new AssertionError("Can not find " + replicaName + " from " + docCollection); - } } diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java new file mode 100644 index 00000000000..98edf7f3e87 --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCollectionsAPIDistributedZkTestBase.java @@ -0,0 +1,700 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud.api.collections; + +import static org.apache.solr.common.cloud.ZkStateReader.CORE_NAME_PROP; +import static org.apache.solr.common.cloud.ZkStateReader.REPLICATION_FACTOR; + +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.TestUtil; +import org.apache.solr.client.api.model.CoreStatusResponse; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrRequest.METHOD; +import org.apache.solr.client.solrj.SolrRequest.SolrRequestType; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.CoreAdminParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.TimeSource; +import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.SolrCore; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.apache.solr.util.TimeOut; +import org.junit.After; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Tests the Cloud Collections API. + * + *

Because the different setups require distinct config-sets, we have to push down cluster + * creation to subclasses + */ +public abstract class AbstractCollectionsAPIDistributedZkTestBase extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() { + // we don't want this test to have zk timeouts + System.setProperty("zkClientTimeout", "60000"); + System.setProperty("createCollectionWaitTimeTillActive", "5"); + TestInjection.randomDelayInCoreCreation = "true:5"; + System.setProperty("validateAfterInactivity", "200"); + System.setProperty("solr.security.allow.paths", "*"); + } + + @Override + @After + public void tearDown() throws Exception { + cluster.deleteAllCollections(); + super.tearDown(); + } + + @Test + public void testCreationAndDeletion() throws Exception { + String collectionName = "created_and_deleted"; + + CollectionAdminRequest.createCollection(collectionName, "conf", 1, 1) + .process(cluster.getSolrClient()); + assertTrue( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + assertFalse( + cluster + .getZkClient() + .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); + } + + @Test + public void deleteCollectionRemovesStaleZkCollectionsNode() throws Exception { + String collectionName = "out_of_sync_collection"; + + // manually create a collections zknode + cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + assertFalse( + cluster + .getZkClient() + .exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); + } + + @Test + public void deletePartiallyCreatedCollection() throws Exception { + final String collectionName = "halfdeletedcollection"; + + assertEquals( + 0, + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()) + .getStatus()); + String dataDir = createTempDir().toString(); + // create a core that simulates something left over from a partially-deleted collection + assertTrue( + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .setDataDir(dataDir) + .process(cluster.getSolrClient()) + .isSuccess()); + + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .process(cluster.getSolrClient()); + + assertTrue( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + } + + @Test + public void deleteCollectionOnlyInZk() throws Exception { + final String collectionName = "onlyinzk"; + + // create the collections node, but nothing else + cluster.getZkClient().makePath(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true); + + // delete via API - should remove collections node + CollectionAdminRequest.deleteCollection(collectionName).process(cluster.getSolrClient()); + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + + // now creating that collection should work + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 1) + .process(cluster.getSolrClient()); + assertTrue( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains(collectionName)); + } + + @Test + public void testBadActionNames() { + // try a bad action + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "BADACTION"); + String collectionName = "badactioncollection"; + params.set("name", collectionName); + params.set("numShards", 2); + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testMissingRequiredParameters() { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("numShards", 2); + // missing required collection parameter + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testMissingNumShards() { + // No numShards should fail + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("name", "acollection"); + params.set(REPLICATION_FACTOR, 10); + params.set("collection.configName", "conf"); + + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testZeroNumShards() { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + params.set("name", "acollection"); + params.set(REPLICATION_FACTOR, 10); + params.set("numShards", 0); + params.set("collection.configName", "conf"); + + var request = + new GenericSolrRequest(METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + expectThrows( + Exception.class, + () -> { + cluster.getSolrClient().request(request); + }); + } + + @Test + public void testCreateShouldFailOnExistingCore() throws Exception { + String nn1 = cluster.getJettySolrRunner(0).getNodeName(); + String nn2 = cluster.getJettySolrRunner(1).getNodeName(); + + assertEquals( + 0, + CollectionAdminRequest.createCollection("halfcollectionblocker", "conf", 1, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard("halfcollectionblocker", "shard1") + .setNode(cluster.getJettySolrRunner(0).getNodeName()) + .setCoreName("halfcollection_shard1_replica_n1") + .process(cluster.getSolrClient()) + .isSuccess()); + + assertEquals( + 0, + CollectionAdminRequest.createCollection("halfcollectionblocker2", "conf", 1, 1) + .setCreateNodeSet("") + .process(cluster.getSolrClient()) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard("halfcollectionblocker2", "shard1") + .setNode(cluster.getJettySolrRunner(1).getNodeName()) + .setCoreName("halfcollection_shard1_replica_n1") + .process(cluster.getSolrClient()) + .isSuccess()); + + expectThrows( + SolrClient.RemoteSolrException.class, + () -> { + CollectionAdminRequest.createCollection("halfcollection", "conf", 1, 1) + .setCreateNodeSet(nn1 + "," + nn2) + .process(cluster.getSolrClient()); + }); + } + + @Test + public void testNoConfigSetExist() throws Exception { + expectThrows( + Exception.class, + () -> { + CollectionAdminRequest.createCollection("noconfig", "conf123", 1, 1) + .process(cluster.getSolrClient()); + }); + + TimeUnit.MILLISECONDS.sleep(1000); + // in both cases, the collection should have default to the core name + cluster.getZkStateReader().forceUpdateCollection("noconfig"); + assertFalse( + CollectionAdminRequest.listCollections(cluster.getSolrClient()).contains("noconfig")); + } + + @Test + public void testCoresAreDistributedAcrossNodes() throws Exception { + CollectionAdminRequest.createCollection("nodes_used_collection", "conf", 2, 2) + .process(cluster.getSolrClient()); + + Set liveNodes = cluster.getSolrClient().getClusterState().getLiveNodes(); + + List createNodeList = new ArrayList<>(liveNodes); + + DocCollection collection = getCollectionState("nodes_used_collection"); + for (Slice slice : collection.getSlices()) { + for (Replica replica : slice.getReplicas()) { + createNodeList.remove(replica.getNodeName()); + } + } + + assertEquals(createNodeList.toString(), 0, createNodeList.size()); + } + + @Test + public void testDeleteNonExistentCollection() throws Exception { + + expectThrows( + SolrException.class, + () -> { + CollectionAdminRequest.deleteCollection("unknown_collection") + .process(cluster.getSolrClient()); + }); + + // create another collection should still work + CollectionAdminRequest.createCollection("acollectionafterbaddelete", "conf", 1, 2) + .process(cluster.getSolrClient()); + waitForState( + "Collection creation after a bad delete failed", + "acollectionafterbaddelete", + (n, c) -> SolrCloudTestCase.replicasForCollectionAreFullyActive(n, c, 1, 2)); + } + + @Test + public void testSpecificConfigsets() throws Exception { + CollectionAdminRequest.createCollection("withconfigset2", "conf2", 1, 1) + .process(cluster.getSolrClient()); + String configName = + cluster + .getSolrClient() + .getClusterStateProvider() + .getCollection("withconfigset2") + .getConfigName(); + assertEquals("conf2", configName); + } + + @Test + public void testCreateNodeSet() throws Exception { + JettySolrRunner jetty1 = cluster.getRandomJetty(random()); + JettySolrRunner jetty2 = cluster.getRandomJetty(random()); + + List baseUrls = List.of(jetty1.getBaseUrl().toString(), jetty2.getBaseUrl().toString()); + + CollectionAdminRequest.createCollection("nodeset_collection", "conf", 2, 1) + .setCreateNodeSet(baseUrls.get(0) + "," + baseUrls.get(1)) + .process(cluster.getSolrClient()); + + DocCollection collectionState = getCollectionState("nodeset_collection"); + for (Replica replica : collectionState.getReplicas()) { + String replicaUrl = replica.getCoreUrl(); + boolean matchingJetty = false; + for (String jettyUrl : baseUrls) { + if (replicaUrl.startsWith(jettyUrl)) { + matchingJetty = true; + } + } + if (matchingJetty == false) { + fail("Expected replica to be on " + baseUrls + " but was on " + replicaUrl); + } + } + } + + @Test + public void testCollectionsAPI() throws Exception { + + // create new collections rapid fire + int cnt = random().nextInt(TEST_NIGHTLY ? 3 : 1) + 1; + CollectionAdminRequest.Create[] createRequests = new CollectionAdminRequest.Create[cnt]; + + class Coll { + String name; + int numShards; + int replicationFactor; + } + + List colls = new ArrayList<>(); + + for (int i = 0; i < cnt; i++) { + + int numShards = TestUtil.nextInt(random(), 0, cluster.getJettySolrRunners().size()) + 1; + int replicationFactor = TestUtil.nextInt(random(), 0, 3) + 1; + + createRequests[i] = + CollectionAdminRequest.createCollection( + "awhollynewcollection_" + i, "conf2", numShards, replicationFactor); + createRequests[i].processAsync(cluster.getSolrClient()); + + Coll coll = new Coll(); + coll.name = "awhollynewcollection_" + i; + coll.numShards = numShards; + coll.replicationFactor = replicationFactor; + colls.add(coll); + } + + for (Coll coll : colls) { + cluster.waitForActiveCollection( + coll.name, coll.numShards, coll.numShards * coll.replicationFactor); + } + + waitForStable(cnt, createRequests); + + for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { + checkInstanceDirs(cluster.getJettySolrRunner(i)); + } + + String collectionName = + createRequests[random().nextInt(createRequests.length)].getCollectionName(); + + // TODO: we should not need this...beast test well when trying to fix + Thread.sleep(1000); + + cluster.getZkStateReader().forciblyRefreshAllClusterStateSlow(); + + new UpdateRequest() + .add("id", "6") + .add("id", "7") + .add("id", "8") + .commit(cluster.getSolrClient(), collectionName); + long numFound = 0; + TimeOut timeOut = new TimeOut(10, TimeUnit.SECONDS, TimeSource.NANO_TIME); + while (!timeOut.hasTimedOut()) { + + numFound = + cluster + .getSolrClient() + .query(collectionName, new SolrQuery("*:*")) + .getResults() + .getNumFound(); + if (numFound == 3) { + break; + } + + Thread.sleep(500); + } + + if (timeOut.hasTimedOut()) { + fail( + "Timeout waiting to see 3 found, instead saw " + + numFound + + " for collection " + + collectionName); + } + + checkNoTwoShardsUseTheSameIndexDir(); + } + + private void waitForStable(int cnt, CollectionAdminRequest.Create[] createRequests) + throws InterruptedException { + for (int i = 0; i < cnt; i++) { + String collectionName = "awhollynewcollection_" + i; + final int j = i; + waitForState( + "Expected to see collection " + collectionName, + collectionName, + (n, c) -> { + CollectionAdminRequest.Create req = createRequests[j]; + return SolrCloudTestCase.replicasForCollectionAreFullyActive( + n, c, req.getNumShards(), req.getReplicationFactor()); + }); + + ZkStateReader zkStateReader = cluster.getZkStateReader(); + // make sure we have leaders for each shard + for (int z = 1; z < createRequests[j].getNumShards(); z++) { + zkStateReader.getLeaderRetry(collectionName, "shard" + z, 10000); + } // make sure we again have leaders for each shard + } + } + + @Test + public void testCollectionReload() throws Exception { + final String collectionName = "reloaded_collection"; + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) + .process(cluster.getSolrClient()); + + // get core open times + Map urlToTimeBefore = new HashMap<>(); + collectStartTimes(collectionName, urlToTimeBefore); + assertTrue(urlToTimeBefore.size() > 0); + + CollectionAdminRequest.reloadCollection(collectionName).processAsync(cluster.getSolrClient()); + + // reloads make take a short while + boolean allTimesAreCorrect = waitForReloads(collectionName, urlToTimeBefore); + assertTrue("some core start times did not change on reload", allTimesAreCorrect); + } + + private void checkInstanceDirs(JettySolrRunner jetty) throws IOException { + CoreContainer cores = jetty.getCoreContainer(); + Collection theCores = cores.getCores(); + for (SolrCore core : theCores) { + // look for core props file + Path instancedir = core.getInstancePath(); + assertTrue( + "Could not find expected core.properties file", + Files.exists(instancedir.resolve("core.properties"))); + + Path expected = Path.of(jetty.getSolrHome()).resolve(core.getName()); + + assertTrue( + "Expected: " + expected + "\nFrom core stats: " + instancedir, + Files.isSameFile(expected, instancedir)); + } + } + + private boolean waitForReloads(String collectionName, Map urlToTimeBefore) + throws SolrServerException, IOException { + TimeOut timeout = new TimeOut(45, TimeUnit.SECONDS, TimeSource.NANO_TIME); + + boolean allTimesAreCorrect = false; + while (!timeout.hasTimedOut()) { + Map urlToTimeAfter = new HashMap<>(); + collectStartTimes(collectionName, urlToTimeAfter); + + boolean retry = false; + Set> entries = urlToTimeBefore.entrySet(); + for (Entry entry : entries) { + Long beforeTime = entry.getValue(); + Long afterTime = urlToTimeAfter.get(entry.getKey()); + assertNotNull(afterTime); + if (afterTime <= beforeTime) { + retry = true; + break; + } + } + if (!retry) { + allTimesAreCorrect = true; + break; + } + } + return allTimesAreCorrect; + } + + private void collectStartTimes(String collectionName, Map urlToTime) + throws SolrServerException, IOException { + + DocCollection collectionState = getCollectionState(collectionName); + if (collectionState != null) { + for (Slice shard : collectionState) { + for (Replica replica : shard) { + CoreStatusResponse.SingleCoreData coreStatus; + try (SolrClient server = getHttpSolrClient(replica.getBaseUrl())) { + coreStatus = CoreAdminRequest.getCoreStatus(replica.getCoreName(), false, server); + } + long before = coreStatus.startTime.getTime(); + urlToTime.put(replica.getCoreUrl(), before); + } + } + } else { + throw new IllegalArgumentException("Could not find collection " + collectionName); + } + } + + private void checkNoTwoShardsUseTheSameIndexDir() { + Map> indexDirToShardNamesMap = new HashMap<>(); + + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + CoreContainer coreContainer = jetty.getCoreContainer(); + List coreNames = coreContainer.getAllCoreNames(); + for (String coreName : coreNames) { + try (SolrCore core = coreContainer.getCore(coreName)) { + String indexDir = core.getIndexDir(); + String shardKey = jetty.getNodeName() + "." + coreName; + if (!indexDirToShardNamesMap.containsKey(indexDir)) { + indexDirToShardNamesMap.put(indexDir, new HashSet<>()); + } + indexDirToShardNamesMap.get(indexDir).add(shardKey); + } + } + } + + assertTrue( + "Something is broken in the assert for no shards using the same indexDir - probably something was changed in the attributes published in the MBean of " + + SolrCore.class.getSimpleName() + + " : " + + indexDirToShardNamesMap, + indexDirToShardNamesMap.size() > 0); + + for (Map.Entry> entry : indexDirToShardNamesMap.entrySet()) { + if (entry.getValue().size() > 1) { + fail( + "We have shards using the same indexDir. E.g. shards " + + entry.getValue().toString() + + " all use indexDir " + + entry.getKey()); + } + } + } + + @Test + public void addReplicaTest() throws Exception { + String collectionName = "addReplicaColl"; + + CollectionAdminRequest.createCollection(collectionName, "conf", 2, 2) + .process(cluster.getSolrClient()); + cluster.waitForActiveCollection(collectionName, 2, 4); + + ArrayList nodeList = + new ArrayList<>(cluster.getSolrClient().getClusterState().getLiveNodes()); + Collections.shuffle(nodeList, random()); + + CollectionAdminResponse response = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .setNode(nodeList.get(0)) + .process(cluster.getSolrClient()); + Replica newReplica = grabNewReplica(response, getCollectionState(collectionName)); + + assertEquals( + "Replica should be created on the right node", + cluster.getZkStateReader().getBaseUrlForNodeName(nodeList.get(0)), + newReplica.getBaseUrl()); + + Path instancePath = createTempDir(); + response = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .withProperty(CoreAdminParams.INSTANCE_DIR, instancePath.toString()) + .process(cluster.getSolrClient()); + newReplica = grabNewReplica(response, getCollectionState(collectionName)); + assertNotNull(newReplica); + cluster.waitForActiveCollection(collectionName, 2, 6); + + try (SolrClient coreclient = getHttpSolrClient(newReplica.getBaseUrl())) { + CoreAdminResponse status = CoreAdminRequest.getStatus(newReplica.getStr("core"), coreclient); + final var coreStatus = status.getCoreStatus(newReplica.getStr("core")); + String instanceDirStr = coreStatus.instanceDir; + assertEquals(instanceDirStr, instancePath.toString()); + } + + // Test to make sure we can't create another replica with an existing core_name of that + // collection + String coreName = newReplica.getStr(CORE_NAME_PROP); + SolrException e = + expectThrows( + SolrException.class, + () -> { + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", "addreplica"); + params.set("collection", collectionName); + params.set("shard", "shard1"); + params.set("name", coreName); + var request = + new GenericSolrRequest( + METHOD.GET, "/admin/collections", SolrRequestType.ADMIN, params); + cluster.getSolrClient().request(request); + }); + + assertTrue( + e.getMessage() + .contains( + "Another replica with the same core name already exists for this collection")); + + // Check that specifying property.name works. DO NOT remove this when the "name" property is + // deprecated + // for ADDREPLICA, this is "property.name". See SOLR-7132 + response = + CollectionAdminRequest.addReplicaToShard(collectionName, "shard1") + .withProperty(CoreAdminParams.NAME, "propertyDotName") + .process(cluster.getSolrClient()); + + newReplica = grabNewReplica(response, getCollectionState(collectionName)); + assertEquals( + "'core' should be 'propertyDotName' ", "propertyDotName", newReplica.getStr("core")); + cluster.waitForActiveCollection(collectionName, 2, 7); + } + + private Replica grabNewReplica(CollectionAdminResponse response, DocCollection docCollection) { + String replicaName = response.getCollectionCoresStatus().keySet().iterator().next(); + Optional optional = + docCollection.getReplicas().stream() + .filter(replica -> replicaName.equals(replica.getCoreName())) + .findAny(); + if (optional.isPresent()) { + return optional.get(); + } + throw new AssertionError("Can not find " + replicaName + " from " + docCollection); + } +} From 167e29d3b5066629e4c508a2a93a5f3411e24843 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 11:22:37 -0500 Subject: [PATCH 10/16] the blockcache code is being kept with expectation of use beyond HDFS. Give non hdfs centric names for this code base. --- .../solr/blockcache/BlockDirectory.java | 2 +- .../blockcache/CustomBufferedIndexInput.java | 2 +- .../org/apache/solr/blockcache/Metrics.java | 4 +-- .../solr/blockcache/BlockDirectoryTest.java | 32 ++++++++++--------- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java index fdd48350f7a..c1ef93b790e 100644 --- a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java +++ b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java @@ -39,7 +39,7 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final long BLOCK_SHIFT = Integer.getInteger("solr.hdfs.blockcache.blockshift", 13); + public static final long BLOCK_SHIFT = Integer.getInteger("solr.blockcache.blockshift", 13); public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT; public static final long BLOCK_MOD = BLOCK_SIZE - 1L; diff --git a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java index 79d44d0fc6b..df3f1a058b2 100644 --- a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java +++ b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java @@ -28,7 +28,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput { public static final int BUFFER_SIZE = - Integer.getInteger("solr.hdfs.readbuffer.size.default", 32768); + Integer.getInteger("solr.blockcache.readbuffer.size.default", 32768); private int bufferSize = BUFFER_SIZE; diff --git a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java index ed820fbb500..4070cd557c2 100644 --- a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java +++ b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java @@ -157,12 +157,12 @@ public void close() { @Override public String getName() { - return "hdfsBlockCache"; + return "blockCache"; } @Override public String getDescription() { - return "Provides metrics for the HdfsDirectoryFactory BlockCache."; + return "Provides metrics for the BlockCache."; } @Override diff --git a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java index dbd69bca4d9..7a1a0c16fa8 100644 --- a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java +++ b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java @@ -142,10 +142,10 @@ public void testEOF() throws IOException { String name = "test.eof"; createFile(name, fsDir, directory); long fsLength = fsDir.fileLength(name); - long hdfsLength = directory.fileLength(name); - assertEquals(fsLength, hdfsLength); + long blockLength = directory.fileLength(name); + assertEquals(fsLength, blockLength); testEof(name, fsDir, fsLength); - testEof(name, directory, hdfsLength); + testEof(name, directory, blockLength); fsDir.close(); } @@ -189,11 +189,12 @@ public void testRandomAccessWritesLargeCache() throws IOException { testRandomAccessWrites(); } - private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException { + private void assertInputsEquals(String name, Directory fsDir, Directory blockDirectory) + throws IOException { int reads = random.nextInt(MAX_NUMBER_OF_READS); IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT); - IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT); - assertEquals(fsInput.length(), hdfsInput.length()); + IndexInput blockInput = blockDirectory.openInput(name, IOContext.DEFAULT); + assertEquals(fsInput.length(), blockInput.length()); int fileLength = (int) fsInput.length(); for (int i = 0; i < reads; i++) { int rnd; @@ -204,7 +205,7 @@ private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) th } byte[] fsBuf = new byte[rnd + MIN_BUFFER_SIZE]; - byte[] hdfsBuf = new byte[fsBuf.length]; + byte[] blockBuf = new byte[fsBuf.length]; int offset = random.nextInt(fsBuf.length); int length = random.nextInt(fsBuf.length - offset); @@ -217,23 +218,24 @@ private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) th fsInput.seek(pos); fsInput.readBytes(fsBuf, offset, length); - hdfsInput.seek(pos); - hdfsInput.readBytes(hdfsBuf, offset, length); + blockInput.seek(pos); + blockInput.readBytes(blockBuf, offset, length); for (int f = offset; f < length; f++) { - if (fsBuf[f] != hdfsBuf[f]) { + if (fsBuf[f] != blockBuf[f]) { fail("read [" + i + "]"); } } } fsInput.close(); - hdfsInput.close(); + blockInput.close(); } - private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException { + private void createFile(String name, Directory fsDir, Directory blockDirectory) + throws IOException { int writes = random.nextInt(MAX_NUMBER_OF_WRITES); int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE; IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT); - IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT); + IndexOutput blockOutput = blockDirectory.createOutput(name, IOContext.DEFAULT); for (int i = 0; i < writes; i++) { byte[] buf = new byte @@ -243,10 +245,10 @@ private void createFile(String name, Directory fsDir, Directory hdfs) throws IOE int offset = random.nextInt(buf.length); int length = random.nextInt(buf.length - offset); fsOutput.writeBytes(buf, offset, length); - hdfsOutput.writeBytes(buf, offset, length); + blockOutput.writeBytes(buf, offset, length); } fsOutput.close(); - hdfsOutput.close(); + blockOutput.close(); } private String getName() { From d95e894d581fe7757c20323570cb4a38811e9260 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 11:23:05 -0500 Subject: [PATCH 11/16] Address legacy HDFS references. --- solr/core/src/java/org/apache/solr/update/UpdateLog.java | 5 +++-- .../solr/update/processor/DistributedZkUpdateProcessor.java | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 464c47cc53e..8ecd634b757 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -446,6 +446,7 @@ public void init(UpdateHandler uhandler, SolrCore core) { // `init(UpdateHandler, SolrCore` is never actually called concurrently in application code // (`TestHdfsUpdateLog.testFSThreadSafety()`, introduced by SOLR-7113, seems to be the only // place that requires true thread safety from this method?). + // HDFS was removed in Solr 10, and therefore the test referenced is gone as well. if (debug) { log.debug( "UpdateHandler init: tlogDir={}, next id={} this is a reopen or double init ... nothing else to do.", @@ -509,8 +510,8 @@ protected final void maybeClearLog(SolrCore core) { /** * Resolves any relative path wrt the highest core-scoped level (whatever that means for a - * particular implementation). For most filesystems, this will be the core instanceDir, but there - * are other cases; e.g., HdfsUpdateLog will resolve paths relative to the core dataDir. + * particular implementation). For most filesystems, this will be the core instanceDir, but that + * is not a hard and fast rule. * *

If the input path is already absolute, it will be returned unmodified. * diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index df0c8f98a2e..34532421bf5 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -1378,7 +1378,7 @@ private void zkCheck(UpdateCommand updateCommand) { // Streaming updates can delay shutdown and cause big update reorderings (new streams can't be // initiated, but existing streams carry on). This is why we check if the CC is shutdown. - // See SOLR-8203 and loop HdfsChaosMonkeyNothingIsSafeTest (and check for inconsistent shards) + // See SOLR-8203 and loop ChaosMonkeyNothingIsSafeTest (and check for inconsistent shards) // to test. if (req.getCoreContainer().isShutDown()) { throw new SolrException( From 4ddbb6c6a8502dda6828c08e6a7d599a2a9a234a Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 11:23:14 -0500 Subject: [PATCH 12/16] Fix visiblity to run test --- .../src/test/org/apache/solr/cloud/BasicDistributedZkTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 6d08c2053ed..7f568367c0e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -185,7 +185,7 @@ protected void setDistributedParams(ModifiableSolrParams params) { @Test @ShardsFixed(num = 4) - protected void test() throws Exception { + public void test() throws Exception { // setLoggingLevel(null); ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); From 3bec94fdfc29f30020ba46619173eb6b909d9eb9 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Mon, 1 Dec 2025 12:39:58 -0500 Subject: [PATCH 13/16] Without HDFS, there are no default Solr DirectoryFactory's that would use this method. Keeping for now. --- .../apache/solr/cloud/api/collections/MoveReplicaCmd.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 48f065f537e..9bcae03e812 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -154,8 +154,8 @@ private void moveReplica( replica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) && dataDir != null; if (isSharedFS && inPlaceMove) { - log.debug("-- moveHdfsReplica"); - moveHdfsReplica( + log.debug("-- moveSharedFsReplica"); + moveSharedFsReplica( clusterState, results, dataDir.toString(), @@ -179,7 +179,7 @@ private void moveReplica( } } - private void moveHdfsReplica( + private void moveSharedFsReplica( ClusterState clusterState, NamedList results, String dataDir, From bf879ea372e58773b94c97c966750931b362d2ad Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 3 Dec 2025 08:12:44 -0500 Subject: [PATCH 14/16] Fix merge went bad duplicating methods. --- .../cloud/AbstractFullDistribZkTestBase.java | 241 ------------------ 1 file changed, 241 deletions(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index 6f774a8043a..5a24a1f2f4d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -3176,245 +3176,4 @@ protected RestTestHarness randomRestTestHarness(Random random) { protected void forAllRestTestHarnesses(Consumer op) { restTestHarnesses.forEach(op); } - - // Methods from AbstractFullDistribZkTestBase - protected void waitForRecoveriesToFinish( - String collection, ZkStateReader zkStateReader, boolean verbose) throws Exception { - waitForRecoveriesToFinish(collection, zkStateReader, verbose, true); - } - - protected void waitForRecoveriesToFinish( - String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout) - throws Exception { - waitForRecoveriesToFinish(collection, zkStateReader, verbose, failOnTimeout, 330, SECONDS); - } - - public static void waitForRecoveriesToFinish( - String collection, - ZkStateReader zkStateReader, - boolean verbose, - boolean failOnTimeout, - long timeoutSeconds) - throws Exception { - waitForRecoveriesToFinish( - collection, zkStateReader, verbose, failOnTimeout, timeoutSeconds, SECONDS); - } - - public static void waitForRecoveriesToFinish( - String collection, - ZkStateReader zkStateReader, - boolean verbose, - boolean failOnTimeout, - long timeout, - TimeUnit unit) - throws Exception { - log.info( - "Wait for recoveries to finish - collection:{} failOnTimeout:{} timeout:{}{}", - collection, - failOnTimeout, - timeout, - unit); - try { - zkStateReader.waitForState( - collection, - timeout, - unit, - (liveNodes, docCollection) -> { - if (docCollection == null) return false; - boolean sawLiveRecovering = false; - - Map slices = docCollection.getSlicesMap(); - assertNotNull("Could not find collection:" + collection, slices); - for (Map.Entry entry : slices.entrySet()) { - Slice slice = entry.getValue(); - if (slice.getState() - == Slice.State - .CONSTRUCTION) { // similar to replica recovering; pretend its the same - // thing - if (verbose) System.out.println("Found a slice in construction state; will wait."); - sawLiveRecovering = true; - } - Map shards = slice.getReplicasMap(); - for (Map.Entry shard : shards.entrySet()) { - if (verbose) - System.out.println( - "replica:" - + shard.getValue().getName() - + " rstate:" - + shard.getValue().getStr(ZkStateReader.STATE_PROP) - + " live:" - + liveNodes.contains(shard.getValue().getNodeName())); - final Replica.State state = shard.getValue().getState(); - if ((state == Replica.State.RECOVERING - || state == Replica.State.DOWN - || state == Replica.State.RECOVERY_FAILED) - && liveNodes.contains(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) { - return false; - } - } - } - if (!sawLiveRecovering) { - if (verbose) System.out.println("no one is recoverying"); - return true; - } else { - return false; - } - }); - } catch (TimeoutException | InterruptedException e) { - Diagnostics.logThreadDumps("Gave up waiting for recovery to finish. THREAD DUMP:"); - zkStateReader.getZkClient().printLayoutToStream(System.out); - fail("There are still nodes recovering - waited for " + timeout + unit); - } - - log.info("Recoveries finished - collection:{}", collection); - } - - public static void waitForCollectionToDisappear( - String collection, ZkStateReader zkStateReader, boolean failOnTimeout, int timeoutSeconds) - throws Exception { - log.info( - "Wait for collection to disappear - collection: {} failOnTimeout:{} timeout (sec):{}", - collection, - failOnTimeout, - timeoutSeconds); - - zkStateReader.waitForState( - collection, timeoutSeconds, TimeUnit.SECONDS, (docCollection) -> docCollection == null); - log.info("Collection has disappeared - collection:{}", collection); - } - - static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader) - throws Exception { - log.info("Will wait for a node to become leader for 15 secs"); - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - - long startNs = System.nanoTime(); - try { - zkStateReader.waitForState( - "collection1", - 15, - TimeUnit.SECONDS, - (docCollection) -> { - if (docCollection == null) return false; - - Slice slice = docCollection.getSlice(shardName); - if (slice != null - && slice.getLeader() != null - && !slice.getLeader().equals(oldLeader) - && slice.getLeader().getState() == Replica.State.ACTIVE) { - if (log.isInfoEnabled()) { - log.info( - "Old leader {}, new leader {}. New leader got elected in {} ms", - oldLeader, - slice.getLeader(), - TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNs)); - } - return true; - } - return false; - }); - } catch (TimeoutException e) { - // If we failed to get a new leader, print some diagnotics before the test fails - Diagnostics.logThreadDumps("Could not find new leader in specified timeout"); - zkStateReader.getZkClient().printLayoutToStream(System.out); - fail("Could not find new leader even after waiting for 15s"); - } - } - - public static void verifyReplicaStatus( - ZkStateReader reader, - String collection, - String shard, - String coreNodeName, - Replica.State expectedState) - throws InterruptedException, TimeoutException { - log.info("verifyReplicaStatus ({}) shard={} coreNodeName={}", collection, shard, coreNodeName); - reader.waitForState( - collection, - 15000, - TimeUnit.MILLISECONDS, - (collectionState) -> - collectionState != null - && collectionState.getSlice(shard) != null - && collectionState.getSlice(shard).getReplicasMap().get(coreNodeName) != null - && collectionState.getSlice(shard).getReplicasMap().get(coreNodeName).getState() - == expectedState); - } - - protected static void assertAllActive(String collection, ZkStateReader zkStateReader) - throws KeeperException, InterruptedException { - - zkStateReader.forceUpdateCollection(collection); - ClusterState clusterState = zkStateReader.getClusterState(); - final DocCollection docCollection = clusterState.getCollectionOrNull(collection); - if (docCollection == null || docCollection.getSlices() == null) { - throw new IllegalArgumentException("Cannot find collection:" + collection); - } - - Map slices = docCollection.getSlicesMap(); - for (Map.Entry entry : slices.entrySet()) { - Slice slice = entry.getValue(); - if (slice.getState() != Slice.State.ACTIVE) { - fail( - "Not all shards are ACTIVE - found a shard " - + slice.getName() - + " that is: " - + slice.getState()); - } - Map shards = slice.getReplicasMap(); - for (Map.Entry shard : shards.entrySet()) { - Replica replica = shard.getValue(); - if (replica.getState() != Replica.State.ACTIVE) { - fail( - "Not all replicas are ACTIVE - found a replica " - + replica.getName() - + " that is: " - + replica.getState()); - } - } - } - } - - protected void printLayout() throws Exception { - SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkHost()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - zkClient.printLayoutToStream(System.out); - zkClient.close(); - } - - protected void restartZk(int pauseMillis) throws Exception { - log.info("Restarting ZK with a pause of {}ms in between", pauseMillis); - zkServer.shutdown(); - // disconnect enough to test stalling, if things stall, then clientSoTimeout will be hit - Thread.sleep(pauseMillis); - zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); - zkServer.run(false); - } - - // Copy a configset up from some path on the local machine to ZK. - // Example usage: - // - // copyConfigUp(TEST_PATH().resolve("configsets"), "cloud-minimal", "configset-name", zk_address); - - public static void copyConfigUp( - Path configSetDir, String srcConfigSet, String dstConfigName, String zkAddr) - throws Exception { - - Path fullConfDir = configSetDir.resolve(srcConfigSet); - String[] args = - new String[] { - "--conf-name", dstConfigName, - "--conf-dir", fullConfDir.toAbsolutePath().toString(), - "-z", zkAddr - }; - - ToolRuntime runtime = new DefaultToolRuntime(); - ConfigSetUploadTool tool = new ConfigSetUploadTool(runtime); - - int res = tool.runTool(SolrCLI.processCommandLineArgs(tool, args)); - assertEquals("Tool should have returned 0 for success, returned: " + res, res, 0); - } } From f9705d9db159fc9f45ad0244dd9f0fa7fe703c75 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 3 Dec 2025 08:13:02 -0500 Subject: [PATCH 15/16] Update import path to match main. --- .../test/org/apache/solr/cloud/BasicDistributedZk2Test.java | 2 +- .../test/org/apache/solr/cloud/BasicDistributedZkTest.java | 4 ++-- .../org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 2 +- .../test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java | 2 +- solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java | 2 +- solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java | 2 +- solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java | 2 +- .../test/org/apache/solr/cloud/UnloadDistributedZkTest.java | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index f975deca67c..45bc2b49e9a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -25,7 +25,7 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 7f568367c0e..1b05c383e0c 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -43,11 +43,11 @@ import org.apache.solr.JSONTestUtil; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.impl.InputStreamResponseParser; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 3cff701db4e..bab3a37c2b8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -23,7 +23,7 @@ import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.SolrInputDocument; diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index a4d227aaf0e..b1bc69741f8 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.common.SolrInputDocument; diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index 00ff9f9b74d..0c0b76f99a0 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -27,7 +27,7 @@ import java.util.Set; import org.apache.solr.client.api.model.CoreStatusResponse; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index dcf97db350c..8673556fc5e 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -21,7 +21,7 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.apache.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java index 90533fa835d..62df2ea506d 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java @@ -27,7 +27,7 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.HttpSolrClient; diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 91ef7830206..0c426f9dec7 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -29,7 +29,7 @@ import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; From 100ea606c0f7b7a521ee91bfc48cfec8fd33a242 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Wed, 3 Dec 2025 09:25:40 -0500 Subject: [PATCH 16/16] lint --- .../test/org/apache/solr/cloud/BasicDistributedZk2Test.java | 2 +- .../test/org/apache/solr/cloud/BasicDistributedZkTest.java | 4 ++-- .../org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java | 2 +- .../test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java | 2 +- solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java | 2 +- solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java | 2 +- solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java | 2 +- .../test/org/apache/solr/cloud/UnloadDistributedZkTest.java | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index 45bc2b49e9a..d4ad2628c32 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -25,10 +25,10 @@ import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.common.SolrException; diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 1b05c383e0c..f9779cfb551 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -43,17 +43,16 @@ import org.apache.solr.JSONTestUtil; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.request.AbstractUpdateRequest; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; import org.apache.solr.client.solrj.request.GenericSolrRequest; import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.StreamingUpdateRequest; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.client.solrj.response.CollectionAdminResponse; @@ -61,6 +60,7 @@ import org.apache.solr.client.solrj.response.Group; import org.apache.solr.client.solrj.response.GroupCommand; import org.apache.solr.client.solrj.response.GroupResponse; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; import org.apache.solr.client.solrj.response.QueryResponse; import org.apache.solr.client.solrj.response.UpdateResponse; import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index bab3a37c2b8..3f4ab451c85 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -23,9 +23,9 @@ import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.ZkStateReader; import org.junit.AfterClass; diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index b1bc69741f8..a75b637af22 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -22,9 +22,9 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.common.SolrInputDocument; import org.junit.AfterClass; import org.junit.BeforeClass; diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index 0c0b76f99a0..d6e00a202ee 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -27,11 +27,11 @@ import java.util.Set; import org.apache.solr.client.api.model.CoreStatusResponse; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.response.CoreAdminResponse; import org.apache.solr.client.solrj.response.RequestStatusState; import org.apache.solr.common.SolrInputDocument; diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index 8673556fc5e..5ea4794ff4b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -21,10 +21,10 @@ import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; import org.apache.solr.client.solrj.apache.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.cloud.DocCollection; import org.apache.solr.common.cloud.Replica; diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java index 62df2ea506d..8f49afba970 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java @@ -27,11 +27,11 @@ import java.util.concurrent.TimeoutException; import java.util.stream.Collectors; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.HttpSolrClient; import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.Replica; diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 0c426f9dec7..7e66787a4a5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -29,11 +29,11 @@ import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.apache.HttpSolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice;