diff --git a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java index fdd48350f7a..c1ef93b790e 100644 --- a/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java +++ b/solr/core/src/java/org/apache/solr/blockcache/BlockDirectory.java @@ -39,7 +39,7 @@ public class BlockDirectory extends FilterDirectory implements ShutdownAwareDirectory { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - public static final long BLOCK_SHIFT = Integer.getInteger("solr.hdfs.blockcache.blockshift", 13); + public static final long BLOCK_SHIFT = Integer.getInteger("solr.blockcache.blockshift", 13); public static final int BLOCK_SIZE = 1 << BLOCK_SHIFT; public static final long BLOCK_MOD = BLOCK_SIZE - 1L; diff --git a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java index 79d44d0fc6b..df3f1a058b2 100644 --- a/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java +++ b/solr/core/src/java/org/apache/solr/blockcache/CustomBufferedIndexInput.java @@ -28,7 +28,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput { public static final int BUFFER_SIZE = - Integer.getInteger("solr.hdfs.readbuffer.size.default", 32768); + Integer.getInteger("solr.blockcache.readbuffer.size.default", 32768); private int bufferSize = BUFFER_SIZE; diff --git a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java index ed820fbb500..4070cd557c2 100644 --- a/solr/core/src/java/org/apache/solr/blockcache/Metrics.java +++ b/solr/core/src/java/org/apache/solr/blockcache/Metrics.java @@ -157,12 +157,12 @@ public void close() { @Override public String getName() { - return "hdfsBlockCache"; + return "blockCache"; } @Override public String getDescription() { - return "Provides metrics for the HdfsDirectoryFactory BlockCache."; + return "Provides metrics for the BlockCache."; } @Override diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java index 48f065f537e..9bcae03e812 100644 --- a/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java +++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/MoveReplicaCmd.java @@ -154,8 +154,8 @@ private void moveReplica( replica.getBool(ZkStateReader.SHARED_STORAGE_PROP, false) && dataDir != null; if (isSharedFS && inPlaceMove) { - log.debug("-- moveHdfsReplica"); - moveHdfsReplica( + log.debug("-- moveSharedFsReplica"); + moveSharedFsReplica( clusterState, results, dataDir.toString(), @@ -179,7 +179,7 @@ private void moveReplica( } } - private void moveHdfsReplica( + private void moveSharedFsReplica( ClusterState clusterState, NamedList results, String dataDir, diff --git a/solr/core/src/java/org/apache/solr/update/UpdateLog.java b/solr/core/src/java/org/apache/solr/update/UpdateLog.java index 464c47cc53e..8ecd634b757 100644 --- a/solr/core/src/java/org/apache/solr/update/UpdateLog.java +++ b/solr/core/src/java/org/apache/solr/update/UpdateLog.java @@ -446,6 +446,7 @@ public void init(UpdateHandler uhandler, SolrCore core) { // `init(UpdateHandler, SolrCore` is never actually called concurrently in application code // (`TestHdfsUpdateLog.testFSThreadSafety()`, introduced by SOLR-7113, seems to be the only // place that requires true thread safety from this method?). + // HDFS was removed in Solr 10, and therefore the test referenced is gone as well. if (debug) { log.debug( "UpdateHandler init: tlogDir={}, next id={} this is a reopen or double init ... nothing else to do.", @@ -509,8 +510,8 @@ protected final void maybeClearLog(SolrCore core) { /** * Resolves any relative path wrt the highest core-scoped level (whatever that means for a - * particular implementation). For most filesystems, this will be the core instanceDir, but there - * are other cases; e.g., HdfsUpdateLog will resolve paths relative to the core dataDir. + * particular implementation). For most filesystems, this will be the core instanceDir, but that + * is not a hard and fast rule. * *

If the input path is already absolute, it will be returned unmodified. * diff --git a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java index df0c8f98a2e..34532421bf5 100644 --- a/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java +++ b/solr/core/src/java/org/apache/solr/update/processor/DistributedZkUpdateProcessor.java @@ -1378,7 +1378,7 @@ private void zkCheck(UpdateCommand updateCommand) { // Streaming updates can delay shutdown and cause big update reorderings (new streams can't be // initiated, but existing streams carry on). This is why we check if the CC is shutdown. - // See SOLR-8203 and loop HdfsChaosMonkeyNothingIsSafeTest (and check for inconsistent shards) + // See SOLR-8203 and loop ChaosMonkeyNothingIsSafeTest (and check for inconsistent shards) // to test. if (req.getCoreContainer().isShutDown()) { throw new SolrException( diff --git a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java index dbd69bca4d9..7a1a0c16fa8 100644 --- a/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java +++ b/solr/core/src/test/org/apache/solr/blockcache/BlockDirectoryTest.java @@ -142,10 +142,10 @@ public void testEOF() throws IOException { String name = "test.eof"; createFile(name, fsDir, directory); long fsLength = fsDir.fileLength(name); - long hdfsLength = directory.fileLength(name); - assertEquals(fsLength, hdfsLength); + long blockLength = directory.fileLength(name); + assertEquals(fsLength, blockLength); testEof(name, fsDir, fsLength); - testEof(name, directory, hdfsLength); + testEof(name, directory, blockLength); fsDir.close(); } @@ -189,11 +189,12 @@ public void testRandomAccessWritesLargeCache() throws IOException { testRandomAccessWrites(); } - private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) throws IOException { + private void assertInputsEquals(String name, Directory fsDir, Directory blockDirectory) + throws IOException { int reads = random.nextInt(MAX_NUMBER_OF_READS); IndexInput fsInput = fsDir.openInput(name, IOContext.DEFAULT); - IndexInput hdfsInput = hdfs.openInput(name, IOContext.DEFAULT); - assertEquals(fsInput.length(), hdfsInput.length()); + IndexInput blockInput = blockDirectory.openInput(name, IOContext.DEFAULT); + assertEquals(fsInput.length(), blockInput.length()); int fileLength = (int) fsInput.length(); for (int i = 0; i < reads; i++) { int rnd; @@ -204,7 +205,7 @@ private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) th } byte[] fsBuf = new byte[rnd + MIN_BUFFER_SIZE]; - byte[] hdfsBuf = new byte[fsBuf.length]; + byte[] blockBuf = new byte[fsBuf.length]; int offset = random.nextInt(fsBuf.length); int length = random.nextInt(fsBuf.length - offset); @@ -217,23 +218,24 @@ private void assertInputsEquals(String name, Directory fsDir, Directory hdfs) th fsInput.seek(pos); fsInput.readBytes(fsBuf, offset, length); - hdfsInput.seek(pos); - hdfsInput.readBytes(hdfsBuf, offset, length); + blockInput.seek(pos); + blockInput.readBytes(blockBuf, offset, length); for (int f = offset; f < length; f++) { - if (fsBuf[f] != hdfsBuf[f]) { + if (fsBuf[f] != blockBuf[f]) { fail("read [" + i + "]"); } } } fsInput.close(); - hdfsInput.close(); + blockInput.close(); } - private void createFile(String name, Directory fsDir, Directory hdfs) throws IOException { + private void createFile(String name, Directory fsDir, Directory blockDirectory) + throws IOException { int writes = random.nextInt(MAX_NUMBER_OF_WRITES); int fileLength = random.nextInt(MAX_FILE_SIZE - MIN_FILE_SIZE) + MIN_FILE_SIZE; IndexOutput fsOutput = fsDir.createOutput(name, IOContext.DEFAULT); - IndexOutput hdfsOutput = hdfs.createOutput(name, IOContext.DEFAULT); + IndexOutput blockOutput = blockDirectory.createOutput(name, IOContext.DEFAULT); for (int i = 0; i < writes; i++) { byte[] buf = new byte @@ -243,10 +245,10 @@ private void createFile(String name, Directory fsDir, Directory hdfs) throws IOE int offset = random.nextInt(buf.length); int length = random.nextInt(buf.length - offset); fsOutput.writeBytes(buf, offset, length); - hdfsOutput.writeBytes(buf, offset, length); + blockOutput.writeBytes(buf, offset, length); } fsOutput.close(); - hdfsOutput.close(); + blockOutput.close(); } private String getName() { diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java index b99330645f1..d4ad2628c32 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZk2Test.java @@ -16,21 +16,474 @@ */ package org.apache.solr.cloud; +import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.mockfile.FilterPath; +import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkNodeProps; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.handler.BackupStatusChecker; +import org.apache.solr.handler.ReplicationHandler; +import org.junit.BeforeClass; import org.junit.Test; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. Implementation moved to AbstractBasicDistributedZk2TestBase as it is used by HDFS - * contrib module tests. + * expected. */ +// Backups do checksum validation against a footer value not present in 'SimpleText' +@LuceneTestCase.SuppressCodecs({"SimpleText"}) @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class BasicDistributedZk2Test extends AbstractBasicDistributedZk2TestBase { +public class BasicDistributedZk2Test extends AbstractFullDistribZkTestBase { + private static final String SHARD2 = "shard2"; + private static final String SHARD1 = "shard1"; + private static final String ONE_NODE_COLLECTION = "onenodecollection"; + private final boolean onlyLeaderIndexes = random().nextBoolean(); + + public BasicDistributedZk2Test() { + super(); + // we need DVs on point fields to compute stats & facets + if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) + System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); + + sliceCount = 2; + } - @Test @Override + protected boolean useTlogReplicas() { + return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug + // and it's TestInjection use + } + + @BeforeClass + public static void _setPrsDefault() { + configurePrsDefault(); + } + + @Test @ShardsFixed(num = 4) public void test() throws Exception { - super.test(); + boolean testFinished = false; + try { + handle.clear(); + handle.put("timestamp", SKIPVAL); + + testNodeWithoutCollectionForwarding(); + + indexr( + id, + 1, + i1, + 100, + tlong, + 100, + t1, + "now is the time for all good men", + "foo_f", + 1.414f, + "foo_b", + "true", + "foo_d", + 1.414d); + + commit(); + + // make sure we are in a steady state... + waitForRecoveriesToFinish(false); + + assertDocCounts(false); + + indexAbunchOfDocs(); + + // check again + waitForRecoveriesToFinish(false); + + commit(); + + assertDocCounts(VERBOSE); + checkQueries(); + + assertDocCounts(VERBOSE); + + query("q", "*:*", "sort", "n_tl1 desc"); + + bringDownShardIndexSomeDocsAndRecover(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + // test adding another replica to a shard - it should do a + // recovery/replication to pick up the index from the leader + addNewReplica(); + + long docId = testUpdateAndDelete(); + + // index a bad doc... + expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id")); + + // TODO: bring this to its own method? + // try indexing to a leader that has no replicas up + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2); + + String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP); + chaosMonkey.stopShardExcept(SHARD2, nodeName); + + SolrClient client = getClient(nodeName); + + index_specific(client, "id", docId + 1, t1, "what happens here?"); + + // expire a session... + CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0); + chaosMonkey.expireSession(cloudJetty.jetty); + // Wait until the jetty is reconnected, otherwise the following index command could fail + cloudJetty + .jetty + .getCoreContainer() + .getZkController() + .getZkClient() + .getCuratorFramework() + .blockUntilConnected(50, TimeUnit.MILLISECONDS); + + indexr("id", docId + 1, t1, "slip this doc in"); + + waitForRecoveriesToFinish(false); + + checkShardConsistency(SHARD1); + checkShardConsistency(SHARD2); + + testFinished = true; + } finally { + if (!testFinished) { + printLayoutOnTearDown = true; + } + } + } + + private void testNodeWithoutCollectionForwarding() throws Exception { + assertEquals( + 0, + CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1") + .setCoreName(ONE_NODE_COLLECTION + "core") + .process(cloudClient) + .isSuccess()); + + waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1); + waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false); + + ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000); + + int docs = 2; + for (JettySolrRunner jetty : jettys) { + final String clientUrl = getBaseUrl(jetty); + addAndQueryDocs(clientUrl, docs); + docs += 2; + } + } + + // 2 docs added every call + private void addAndQueryDocs(final String baseUrl, int docs) throws Exception { + + SolrQuery query = new SolrQuery("*:*"); + + try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) { + // add a doc + client.add(sdoc("id", docs)); + client.commit(); + + QueryResponse results = client.query(query); + assertEquals(docs - 1, results.getResults().getNumFound()); + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", docs + 1); + client.add(doc); + client.commit(); + + query = new SolrQuery("*:*"); + query.set("rows", 0); + results = client.query(query); + assertEquals(docs, results.getResults().getNumFound()); + } + } + + private long testUpdateAndDelete() throws Exception { + long docId = 99999999L; + indexr("id", docId, t1, "originalcontent"); + + commit(); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.add("q", t1 + ":originalcontent"); + QueryResponse results = clients.get(0).query(params); + assertEquals(1, results.getResults().getNumFound()); + + // update doc + indexr("id", docId, t1, "updatedcontent"); + + commit(); + + results = clients.get(0).query(params); + assertEquals(0, results.getResults().getNumFound()); + + params.set("q", t1 + ":updatedcontent"); + + results = clients.get(0).query(params); + assertEquals(1, results.getResults().getNumFound()); + + UpdateRequest uReq = new UpdateRequest(); + // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN); + uReq.deleteById(Long.toString(docId)).process(clients.get(0)); + + commit(); + + results = clients.get(0).query(params); + assertEquals(0, results.getResults().getNumFound()); + return docId; + } + + private void bringDownShardIndexSomeDocsAndRecover() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.set("distrib", false); + + commit(); + + long deadShardCount = + shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + int oldLiveNodes = + ZkStateReader.from(cloudClient) + .getZkClient() + .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true) + .size(); + + assertEquals(5, oldLiveNodes); + + // kill a shard + CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0); + + // ensure shard is dead + expectThrows( + SolrServerException.class, + "This server should be down and this update should have failed", + () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!")); + + commit(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + // long cloudClientDocs = cloudClient.query(new + // SolrQuery("*:*")).getResults().getNumFound(); + // System.out.println("clouddocs:" + cloudClientDocs); + + // try to index to a living shard at shard2 + + long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000); + + try { + index_specific( + shardToJetty.get(SHARD1).get(1).client.solrClient, + id, + 1000, + i1, + 108, + t1, + "specific doc!"); + } catch (Exception e) { + // wait and try again + Thread.sleep(4000); + index_specific( + shardToJetty.get(SHARD1).get(1).client.solrClient, + id, + 1000, + i1, + 108, + t1, + "specific doc!"); + } + + commit(); + + checkShardConsistency(true, false); + + query("q", "*:*", "sort", "n_tl1 desc"); + + long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + assertEquals(numFound1 + 1, numFound2); + + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", 1001); + + controlClient.add(doc); + + // try adding a doc with CloudSolrServer + UpdateRequest ureq = new UpdateRequest(); + ureq.add(doc); + // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); + + try { + ureq.process(cloudClient); + } catch (SolrServerException e) { + // try again + Thread.sleep(3500); + ureq.process(cloudClient); + } + + commit(); + + query("q", "*:*", "sort", "n_tl1 desc"); + + long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + // lets just check that the one doc since last commit made it in... + assertEquals(numFound2 + 1, numFound3); + + // test debugging + testDebugQueries(); + + if (VERBOSE) { + System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()); + + for (SolrClient client : clients) { + try { + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + System.err.println(client.query(q).getResults().getNumFound()); + } catch (Exception e) { + + } + } + } + // TODO: This test currently fails because debug info is obtained only + // on shards with matches. + // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); + + // this should trigger a recovery phase on deadShard + deadShard.jetty.start(); + + // make sure we have published we are recovering + Thread.sleep(1500); + + waitForRecoveriesToFinish(false); + + deadShardCount = + shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound(); + // if we properly recovered, we should now have the couple missing docs that + // came in while shard was down + checkShardConsistency(true, false); + + // recover over 100 docs so we do more than just peer sync (replicate recovery) + chaosMonkey.stopJetty(deadShard); + + for (int i = 0; i < 226; i++) { + doc = new SolrInputDocument(); + doc.addField("id", 2000 + i); + controlClient.add(doc); + ureq = new UpdateRequest(); + ureq.add(doc); + // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); + ureq.process(cloudClient); + } + commit(); + + Thread.sleep(1500); + + deadShard.jetty.start(); + + // make sure we have published we are recovering + Thread.sleep(1500); + + waitForThingsToLevelOut(1, TimeUnit.MINUTES); + + Thread.sleep(500); + + waitForRecoveriesToFinish(false); + + checkShardConsistency(true, false); + + // try a backup command + try (final SolrClient client = + getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) { + final String backupName = "the_backup"; + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("qt", ReplicationHandler.PATH); + params.set("command", "backup"); + params.set("name", backupName); + final Path location = FilterPath.unwrap(createTempDir()).toRealPath(); + // Allow non-standard location outside SOLR_HOME + jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location)); + params.set("location", location.toString()); + + QueryRequest request = new QueryRequest(params); + client.request(request, DEFAULT_TEST_COLLECTION_NAME); + + final BackupStatusChecker backupStatus = + new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication"); + final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30); + assertTrue( + "Backup dir does not exist: " + backupDirName, + Files.exists(location.resolve(backupDirName))); + } + } + + private void addNewReplica() throws Exception { + + waitForRecoveriesToFinish(false); + + // new server should be part of first shard + // how many docs are on the new shard? + for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) { + if (VERBOSE) + System.err.println( + "shard1 total:" + + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) { + if (VERBOSE) + System.err.println( + "shard2 total:" + + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + + checkShardConsistency(SHARD1); + checkShardConsistency(SHARD2); + + assertDocCounts(VERBOSE); + } + + private void testDebugQueries() throws Exception { + handle.put("explain", SKIPVAL); + handle.put("debug", UNORDERED); + handle.put("time", SKIPVAL); + handle.put("track", SKIP); + query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); + query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java index 7975cfb68d3..f9779cfb551 100644 --- a/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/BasicDistributedZkTest.java @@ -16,21 +16,1724 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.io.InputStream; +import java.lang.invoke.MethodHandles; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.Callable; +import java.util.concurrent.CompletionService; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorCompletionService; +import java.util.concurrent.Future; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; +import org.apache.lucene.util.IOUtils; +import org.apache.solr.JSONTestUtil; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.AbstractUpdateRequest; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; +import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.StreamingUpdateRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.CollectionAdminResponse; +import org.apache.solr.client.solrj.response.FacetField; +import org.apache.solr.client.solrj.response.Group; +import org.apache.solr.client.solrj.response.GroupCommand; +import org.apache.solr.client.solrj.response.GroupResponse; +import org.apache.solr.client.solrj.response.InputStreamResponseParser; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.UpdateResponse; +import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; +import org.apache.solr.common.SolrDocument; +import org.apache.solr.common.SolrDocumentList; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ClusterState; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.common.params.UpdateParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.apache.solr.util.TestInjection.Hook; +import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. Implementation moved to AbstractBasicDistributedZkTestBase as it is used by many HDFS - * contrib tests. + * expected. */ @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class BasicDistributedZkTest extends AbstractBasicDistributedZkTestBase { +public class BasicDistributedZkTest extends AbstractFullDistribZkTestBase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final String DEFAULT_COLLECTION = "collection1"; + + private final boolean onlyLeaderIndexes = random().nextBoolean(); + + String tsort = "t_sortable"; + + private Map> otherCollectionClients = new HashMap<>(); + + private String oneInstanceCollection = "oneInstanceCollection"; + private String oneInstanceCollection2 = "oneInstanceCollection2"; + + private AtomicInteger nodeCounter = new AtomicInteger(); + + CompletionService completionService; + Set> pending; + + private static Hook newSearcherHook = + new Hook() { + volatile CountDownLatch latch; + AtomicReference collection = new AtomicReference<>(); + + @Override + public void newSearcher(String collectionName) { + String c = collection.get(); + if (c != null && c.equals(collectionName)) { + log.info("Hook detected newSearcher"); + try { + latch.countDown(); + } catch (NullPointerException e) { + + } + } + } + + @Override + public void waitForSearcher( + String collection, int cnt, int timeoutms, boolean failOnTimeout) + throws InterruptedException { + latch = new CountDownLatch(cnt); + this.collection.set(collection); + boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS); + if (timeout && failOnTimeout) { + fail("timed out waiting for new searcher event " + latch.getCount()); + } + } + }; + + public BasicDistributedZkTest() { + // we need DVs on point fields to compute stats & facets + if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) + System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); + + sliceCount = 2; + completionService = new ExecutorCompletionService<>(executor); + pending = new HashSet<>(); + } + + @BeforeClass + public static void beforeBDZKTClass() { + TestInjection.newSearcherHook(newSearcherHook); + } + + @Override + protected boolean useTlogReplicas() { + // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's + // TestInjection use + return false; + } - @Test @Override + protected void setDistributedParams(ModifiableSolrParams params) { + + if (r.nextBoolean()) { + // don't set shards, let that be figured out from the cloud state + } else { + // use shard ids rather than physical locations + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < getShardCount(); i++) { + if (i > 0) sb.append(','); + sb.append("shard" + (i + 3)); + } + params.set("shards", sb.toString()); + } + } + + @Test @ShardsFixed(num = 4) public void test() throws Exception { - super.test(); + // setLoggingLevel(null); + + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + // make sure we have leaders for each shard + for (int j = 1; j < sliceCount; j++) { + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); + } // make sure we again have leaders for each shard + + waitForRecoveriesToFinish(false); + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + del("*:*"); + queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty")); + + // ask every individual replica of every shard to update+commit the same doc id + // with an incrementing counter on each update+commit + int foo_i_counter = 0; + for (SolrClient client : clients) { + foo_i_counter++; + indexDoc( + client, + params("commit", "true"), // SOLR-4923 + sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter)); + // after every update+commit, check all the shards consistency + queryAndCompareShards( + params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup")); + queryAndCompareShards( + params( + "q", "id:1", + "sanity_check", "distrib_id_1_lookup")); + } + + indexr( + id, + 1, + i1, + 100, + tlong, + 100, + t1, + "now is the time for all good men", + "foo_f", + 1.414f, + "foo_b", + "true", + "foo_d", + 1.414d, + tsort, + "now is the time for all good men"); + indexr( + id, + 2, + i1, + 50, + tlong, + 50, + t1, + "to come to the aid of their country.", + tsort, + "to come to the aid of their country."); + indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow"); + indexr( + id, + 4, + i1, + -100, + tlong, + 101, + t1, + "the quick fox jumped over the lazy dog", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 5, + i1, + 500, + tlong, + 500, + t1, + "the quick fox jumped way over the lazy dog", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 6, + i1, + -600, + tlong, + 600, + t1, + "humpty dumpy sat on a wall", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 7, + i1, + 123, + tlong, + 123, + t1, + "humpty dumpy had a great fall", + tsort, + "the quick fox jumped over the lazy dog"); + indexr( + id, + 8, + i1, + 876, + tlong, + 876, + t1, + "all the kings horses and all the kings men", + tsort, + "all the kings horses and all the kings men"); + indexr( + id, + 9, + i1, + 7, + tlong, + 7, + t1, + "couldn't put humpty together again", + tsort, + "the quick fox jumped over the lazy dog"); + indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass"); + indexr( + id, + 11, + i1, + -987, + tlong, + 987, + t1, + "An eye for eye only ends up making the whole world blind.", + tsort, + "An eye for eye only ends up making the whole world blind."); + indexr( + id, + 12, + i1, + 379, + tlong, + 379, + t1, + "Great works are performed, not by strength, but by perseverance.", + tsort, + "Great works are performed, not by strength, but by perseverance."); + indexr( + id, + 13, + i1, + 232, + tlong, + 232, + t1, + "no eggs on wall, lesson learned", + oddField, + "odd man out", + tsort, + "no eggs on wall, lesson learned"); + + indexr( + id, + 14, + "SubjectTerms_mfacet", + new String[] {"mathematical models", "mathematical analysis"}); + indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); + indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); + String[] vals = new String[100]; + for (int i = 0; i < 100; i++) { + vals[i] = "test " + i; + } + indexr(id, 17, "SubjectTerms_mfacet", vals); + + for (int i = 100; i < 150; i++) { + indexr(id, i); + } + + commit(); + + testTokenizedGrouping(); + testSortableTextFaceting(); + testSortableTextSorting(); + testSortableTextGrouping(); + + queryAndCompareShards( + params( + "q", "*:*", + "sort", "id desc", + "distrib", "false", + "sanity_check", "is_empty")); + + // random value sort + for (String f : fieldNames) { + query(false, new String[] {"q", "*:*", "sort", f + " desc"}); + query(false, new String[] {"q", "*:*", "sort", f + " asc"}); + } + + // these queries should be exactly ordered and scores should exactly match + query(false, new String[] {"q", "*:*", "sort", i1 + " desc"}); + query(false, new String[] {"q", "*:*", "sort", i1 + " asc"}); + query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"}); + query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"}); + query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"}); + handle.put("maxScore", SKIPVAL); + query( + false, + new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it. + // JavaBinCodec.writeSolrDocumentList() + // is agnostic of request params. + handle.remove("maxScore"); + query( + false, + new String[] { + "q", "{!func}" + i1, "fl", "*,score" + }); // even scores should match exactly here + + handle.put("highlighting", UNORDERED); + handle.put("response", UNORDERED); + + handle.put("maxScore", SKIPVAL); + query(false, new String[] {"q", "quick"}); + query(false, new String[] {"q", "all", "fl", "id", "start", "0"}); + query( + false, + new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs + query(false, new String[] {"q", "all", "fl", "id", "start", "100"}); + + handle.put("score", SKIPVAL); + query(false, new String[] {"q", "quick", "fl", "*,score"}); + query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"}); + query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"}); + + query( + false, + new String[] { + "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1 + }); + + query( + false, + new String[] { + "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1 + }); + + query(false, new String[] {"q", "matchesnothing", "fl", "*,score"}); + + query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1}); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "count" + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "count", + "facet.mincount", + 2 + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "index" + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + -1, + "facet.sort", + "index", + "facet.mincount", + 2 + }); + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1 + }); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.query", + "quick", + "facet.query", + "all", + "facet.query", + "*:*" + }); + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1 + }); + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2 + }); + + // test faceting multiple things at once + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.query", + "quick", + "facet.query", + "all", + "facet.query", + "*:*", + "facet.field", + t1 + }); + + // test filter tagging, facet exclusion, and naming (multi-select facet support) + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 100, + "facet", + "true", + "facet.query", + "{!key=myquick}quick", + "facet.query", + "{!key=myall ex=a}all", + "facet.query", + "*:*", + "facet.field", + "{!key=mykey ex=a}" + t1, + "facet.field", + "{!key=other ex=b}" + t1, + "facet.field", + "{!key=again ex=a,b}" + t1, + "facet.field", + t1, + "fq", + "{!tag=a}id_i1:[1 TO 7]", + "fq", + "{!tag=b}id_i1:[3 TO 9]" + }); + query( + false, + new Object[] { + "q", + "*:*", + "facet", + "true", + "facet.field", + "{!ex=t1}SubjectTerms_mfacet", + "fq", + "{!tag=t1}SubjectTerms_mfacet:(test 1)", + "facet.limit", + "10", + "facet.mincount", + "1" + }); + + // test field that is valid in schema but missing in all shards + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2 + }); + // test field that is valid in schema and missing in some shards + query( + false, + new Object[] { + "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2 + }); + + query( + false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1}); + + /* TODO: the failure may come back in "exception" + try { + // test error produced for field that is invalid for schema + query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2); + TestCase.fail("SolrServerException expected for invalid field that is not in schema"); + } catch (SolrServerException ex) { + // expected + } + */ + + // Try to get better coverage for refinement queries by turning off over requesting. + // This makes it much more likely that we may not get the top facet values and hence + // we turn of that checking. + handle.put("facet_fields", SKIPVAL); + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 0, + "facet", + "true", + "facet.field", + t1, + "facet.limit", + 5, + "facet.shard.limit", + 5 + }); + // check a complex key name + query( + false, + new Object[] { + "q", + "*:*", + "rows", + 0, + "facet", + "true", + "facet.field", + "{!key='a b/c \\' \\} foo'}" + t1, + "facet.limit", + 5, + "facet.shard.limit", + 5 + }); + handle.remove("facet_fields"); + + // index the same document to two servers and make sure things + // don't blow up. + if (clients.size() >= 2) { + index(id, 100, i1, 107, t1, "oh no, a duplicate!"); + for (int i = 0; i < clients.size(); i++) { + index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!"); + } + commit(); + query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1}); + query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1}); + query(false, new Object[] {"q", "*:*", "rows", 100}); + } + + // test debugging + handle.put("explain", SKIPVAL); + handle.put("debug", UNORDERED); + handle.put("time", SKIPVAL); + handle.put("track", SKIP); + query( + false, + new Object[] { + "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true" + }); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS}); + query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY}); + + // try add commitWithin + long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + for (SolrClient client : clients) { + assertEquals( + "unexpected pre-commitWithin document count on node: " + + ((HttpSolrClient) client).getBaseURL(), + before, + client.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("commitWithin", 10); + add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301))); + + newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); + + ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); + DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION); + + assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl); + + // try deleteById commitWithin + UpdateRequest deleteByIdReq = new UpdateRequest(); + deleteByIdReq.deleteById("300"); + deleteByIdReq.setCommitWithin(10); + deleteByIdReq.process(cloudClient); + + newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); + + assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl); + + // try deleteByQuery commitWithin + UpdateRequest deleteByQueryReq = new UpdateRequest(); + deleteByQueryReq.deleteByQuery("id:301"); + deleteByQueryReq.setCommitWithin(10); + deleteByQueryReq.process(cloudClient); + + newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); + + assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl); + + // TODO: This test currently fails because debug info is obtained only + // on shards with matches. + // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); + + // would be better if these where all separate tests - but much, much + // slower + doOptimisticLockingAndUpdating(); + testShardParamVariations(); + testMultipleCollections(); + testANewCollectionInOneInstance(); + testSearchByCollectionName(); + testUpdateByCollectionName(); + testANewCollectionInOneInstanceWithManualShardAssignement(); + testNumberOfCommitsWithCommitAfterAdd(); + + testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit"); + testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit"); + + testStopAndStartCoresInOneInstance(); + } + + private void testSortableTextFaceting() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.addFacetField(tsort); + query.setFacetMissing(false); + QueryResponse resp = queryRandomShard(query); + List ffs = resp.getFacetFields(); + for (FacetField ff : ffs) { + if (ff.getName().equals(tsort) == false) continue; + for (FacetField.Count count : ff.getValues()) { + long num = count.getCount(); + switch (count.getName()) { + case "all the kings horses and all the kings men": + case "An eye for eye only ends up making the whole world blind.": + case "Great works are performed, not by strength, but by perseverance.": + case "how now brown cow": + case "no eggs on wall, lesson learned": + case "now is the time for all good men": + case "this too shall pass": + case "to come to the aid of their country.": + assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num); + break; + case "the quick fox jumped over the lazy dog": + assertEquals("Should have 5 docs for the lazy dog", 5, num); + break; + default: + fail("No case for facet '" + ff.getName() + "'"); + } + } + } + } + + private void testSortableTextSorting() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.addSort(tsort, SolrQuery.ORDER.desc); + query.addField("*"); + query.addField("eoe_sortable"); + query.addField(tsort); + QueryResponse resp = queryRandomShard(query); + + SolrDocumentList docs = resp.getResults(); + + String title = docs.get(0).getFieldValue(tsort).toString(); + for (SolrDocument doc : docs) { + assertTrue( + "Docs should be back in sorted order, descending", + title.compareTo(doc.getFieldValue(tsort).toString()) >= 0); + title = doc.getFieldValue(tsort).toString(); + } + } + + private void testSortableTextGrouping() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + query.add("group", "true"); + query.add("group.field", tsort); + QueryResponse resp = queryRandomShard(query); + GroupResponse groupResp = resp.getGroupResponse(); + List grpCmds = groupResp.getValues(); + for (GroupCommand grpCmd : grpCmds) { + if (grpCmd.getName().equals(tsort) == false) continue; + for (Group grp : grpCmd.getValues()) { + long count = grp.getResult().getNumFound(); + if (grp.getGroupValue() == null) + continue; // Don't count the groups without an entry as the numnber is variable + switch (grp.getGroupValue()) { + case "all the kings horses and all the kings men": + case "An eye for eye only ends up making the whole world blind.": + case "Great works are performed, not by strength, but by perseverance.": + case "how now brown cow": + case "no eggs on wall, lesson learned": + case "now is the time for all good men": + case "this too shall pass": + case "to come to the aid of their country.": + assertEquals( + "Should have exactly one facet count for field " + grpCmd.getName(), 1, count); + break; + case "the quick fox jumped over the lazy dog": + assertEquals("Should have 5 docs for the lazy dog", 5, count); + break; + default: + fail("No case for facet '" + grpCmd.getName() + "'"); + } + } + } + } + + private void testTokenizedGrouping() throws Exception { + SolrException ex = + expectThrows( + SolrException.class, + () -> { + query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1}); + }); + assertTrue( + "Expected error from server that SortableTextFields are required", + ex.getMessage() + .contains( + "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode")); + } + + private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception { + long found = checkSlicesSameCounts(dColl); + + if (found != expected) { + // we get one do over in a bad race + Thread.sleep(1000); + found = checkSlicesSameCounts(dColl); + } + + assertEquals(msg, expected, checkSlicesSameCounts(dColl)); + } + + // Ensure that total docs found is the expected number. + private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage) + throws Exception { + AtomicLong total = new AtomicLong(-1); + try { + ZkStateReader.from(getCommonCloudSolrClient()) + .waitForState( + DEFAULT_COLLECTION, + waitMillis, + TimeUnit.MILLISECONDS, + (n, c) -> { + long docTotal; + try { + docTotal = checkSlicesSameCounts(c); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + total.set(docTotal); + if (docTotal == expectedNumFound) { + return true; + } + return false; + }); + } catch (TimeoutException | InterruptedException e) { + + } + // We could fail here if we broke out of the above because we exceeded the time allowed. + assertEquals(failureMessage, expectedNumFound, total.get()); + + // This should be redundant, but it caught a test error after all. + for (SolrClient client : clients) { + assertEquals( + failureMessage, + expectedNumFound, + client.query(new SolrQuery("*:*")).getResults().getNumFound()); + } + } + + // Insure that counts are the same for all replicas in each shard + // Return the total doc count for the query. + private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException { + long docTotal = 0; // total number of documents found counting only one replica per slice. + for (Slice slice : dColl.getActiveSlices()) { + long sliceDocCount = -1; + for (Replica rep : slice.getReplicas()) { + try (SolrClient one = getHttpSolrClient(rep)) { + SolrQuery query = new SolrQuery("*:*"); + query.setDistrib(false); + QueryResponse resp = one.query(query); + long hits = resp.getResults().getNumFound(); + if (sliceDocCount == -1) { + sliceDocCount = hits; + docTotal += hits; + } else { + if (hits != sliceDocCount) { + return -1; + } + } + } + } + } + return docTotal; + } + + private void testShardParamVariations() throws Exception { + SolrQuery query = new SolrQuery("*:*"); + Map shardCounts = new HashMap<>(); + + for (String shard : shardToJetty.keySet()) { + // every client should give the same numDocs for this shard + // shffle the clients in a diff order for each shard + List solrclients = new ArrayList<>(this.clients); + Collections.shuffle(solrclients, random()); + for (SolrClient client : solrclients) { + query.set("shards", shard); + long numDocs = client.query(query).getResults().getNumFound(); + assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs); + if (!shardCounts.containsKey(shard)) { + shardCounts.put(shard, numDocs); + } + assertEquals( + "inconsitent numDocs for shard " + shard + " via " + client, + shardCounts.get(shard).longValue(), + numDocs); + + List replicaJetties = new ArrayList<>(shardToJetty.get(shard)); + Collections.shuffle(replicaJetties, random()); + + // each replica should also give the same numDocs + ArrayList replicaAlts = new ArrayList<>(replicaJetties.size() * 2); + for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { + String replica = replicaJetty.url; + query.set("shards", replica); + + // replicas already shuffled, use this in the alternative check below + if (0 == random().nextInt(3) || replicaAlts.size() < 2) { + replicaAlts.add(replica); + } + + numDocs = client.query(query).getResults().getNumFound(); + assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs); + assertEquals( + "inconsitent numDocs for shard " + + shard + + " in replica " + + replica + + " via " + + client, + shardCounts.get(shard).longValue(), + numDocs); + } + + // any combination of replica alternatives should give same numDocs + String replicas = String.join("|", replicaAlts); + query.set("shards", replicas); + numDocs = client.query(query).getResults().getNumFound(); + assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs); + assertEquals( + "inconsitent numDocs for replicas " + replicas + " via " + client, + shardCounts.get(shard).longValue(), + numDocs); + } + } + + // sums of multiple shards should add up regardless of how we + // query those shards or which client we use + long randomShardCountsExpected = 0; + ArrayList randomShards = new ArrayList<>(shardCounts.size()); + for (Map.Entry shardData : shardCounts.entrySet()) { + if (random().nextBoolean() || randomShards.size() < 2) { + String shard = shardData.getKey(); + randomShardCountsExpected += shardData.getValue(); + if (random().nextBoolean()) { + // use shard id + randomShards.add(shard); + } else { + // use some set explicit replicas + ArrayList replicas = new ArrayList<>(7); + for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { + if (0 == random().nextInt(3) || 0 == replicas.size()) { + replicas.add(replicaJetty.url); + } + } + Collections.shuffle(replicas, random()); + randomShards.add(String.join("|", replicas)); + } + } + } + String randShards = String.join(",", randomShards); + query.set("shards", randShards); + for (SolrClient client : this.clients) { + assertEquals( + "numDocs for " + randShards + " via " + client, + randomShardCountsExpected, + client.query(query).getResults().getNumFound()); + } + + // total num docs must match sum of every shard's numDocs + query = new SolrQuery("*:*"); + long totalShardNumDocs = 0; + for (Long c : shardCounts.values()) { + totalShardNumDocs += c; + } + for (SolrClient client : clients) { + assertEquals( + "sum of shard numDocs on client: " + client, + totalShardNumDocs, + client.query(query).getResults().getNumFound()); + } + assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs); + } + + private void testStopAndStartCoresInOneInstance() throws Exception { + JettySolrRunner jetty = jettys.get(0); + try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) { + ThreadPoolExecutor executor = null; + try { + executor = + new ExecutorUtil.MDCAwareThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 5, + TimeUnit.SECONDS, + new SynchronousQueue(), + new SolrNamedThreadFactory("testExecutor")); + int cnt = 3; + + // create the cores + createCollectionInOneInstance( + httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt); + } finally { + if (executor != null) { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + } + } + + cloudJettys.get(0).jetty.stop(); + printLayout(); + + cloudJettys.get(0).jetty.start(); + ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2"); + try { + ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000); + } catch (SolrException e) { + printLayout(); + throw e; + } + + printLayout(); + } + + /** Create a collection in single node */ + public static void createCollectionInOneInstance( + final SolrClient client, + String nodeName, + ThreadPoolExecutor executor, + final String collection, + final int numShards, + int numReplicas) { + assertNotNull(nodeName); + try { + assertEquals( + 0, + CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) + .setCreateNodeSet("") + .process(client) + .getStatus()); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + for (int i = 0; i < numReplicas; i++) { + final int freezeI = i; + executor.execute( + () -> { + try { + assertTrue( + CollectionAdminRequest.addReplicaToShard( + collection, "shard" + ((freezeI % numShards) + 1)) + .setCoreName(collection + freezeI) + .setNode(nodeName) + .process(client) + .isSuccess()); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + }); + } + } + + @Override + protected CollectionAdminResponse createCollection( + Map> collectionInfos, + String collectionName, + String configSetName, + int numShards, + int numReplicas, + SolrClient client, + String createNodeSetStr) + throws SolrServerException, IOException { + // TODO: Use CollectionAdminRequest for this test + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.CREATE.toString()); + + params.set(CollectionHandlingUtils.NUM_SLICES, numShards); + params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas); + if (createNodeSetStr != null) + params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr); + + int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0; + List list = new ArrayList<>(); + list.add(numShards); + list.add(numReplicas); + if (collectionInfos != null) { + collectionInfos.put(collectionName, list); + } + params.set("name", collectionName); + params.set("collection.configName", configSetName); + QueryRequest request = new QueryRequest(params); + request.setPath("/admin/collections"); + + CollectionAdminResponse res = new CollectionAdminResponse(); + if (client == null) { + final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL(); + + try (SolrClient aClient = createNewSolrClient("", baseUrl)) { + res.setResponse(aClient.request(request)); + } + } else { + res.setResponse(client.request(request)); + } + return res; + } + + @Override + protected Replica getLeaderFromZk(String collection, String slice) { + ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); + Replica leader = clusterState.getCollection(collection).getLeader(slice); + if (leader == null) { + throw new RuntimeException("Could not find leader:" + collection + " " + slice); + } + return leader; + } + + /** + * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two + * (stored) string fields. + * + *

If the values are "double-doubled" or "not-doubled" then we know the processor was not run + * the appropriate number of times + */ + private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception { + + final String fieldA = "regex_dup_A_s"; + final String fieldB = "regex_dup_B_s"; + final String val = "x"; + final String expected = "x_x"; + final ModifiableSolrParams updateParams = new ModifiableSolrParams(); + updateParams.add(UpdateParams.UPDATE_CHAIN, chain); + + final int numLoops = atLeast(50); + + for (int i = 1; i < numLoops; i++) { + // add doc to random client + SolrClient updateClient = clients.get(random().nextInt(clients.size())); + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, id, i, fieldA, val, fieldB, val); + UpdateResponse ures = add(updateClient, updateParams, doc); + assertEquals(chain + ": update failed", 0, ures.getStatus()); + ures = updateClient.commit(); + assertEquals(chain + ": commit failed", 0, ures.getStatus()); + } + + // query for each doc, and check both fields to ensure the value is correct + for (int i = 1; i < numLoops; i++) { + final String query = id + ":" + i; + QueryResponse qres = queryRandomShard(new SolrQuery(query)); + assertEquals(chain + ": query failed: " + query, 0, qres.getStatus()); + assertEquals( + chain + ": didn't find correct # docs with query: " + query, + 1, + qres.getResults().getNumFound()); + SolrDocument doc = qres.getResults().get(0); + + for (String field : new String[] {fieldA, fieldB}) { + assertEquals( + chain + ": doc#" + i + " has wrong value for " + field, + expected, + doc.getFirstValue(field)); + } + } + } + + // cloud level test mainly needed just to make sure that versions and errors are propagated + // correctly + private void doOptimisticLockingAndUpdating() throws Exception { + log.info("### STARTING doOptimisticLockingAndUpdating"); + printLayout(); + + final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1); + indexDoc(sd); + + ignoreException("version conflict"); + for (SolrClient client : clients) { + SolrException e = expectThrows(SolrException.class, () -> client.add(sd)); + assertEquals(409, e.code()); + } + unIgnoreException("version conflict"); + + // TODO: test deletes. SolrJ needs a good way to pass version for delete... + + final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5); + clients.get(0).add(sd2); + + List expected = new ArrayList<>(); + int val = 0; + for (SolrClient client : clients) { + val += 10; + client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val)); + expected.add(val); + } + + QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000")); + for (SolrClient client : clients) { + val += 10; + NamedList rsp = client.request(qr); + String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected); + if (match != null) throw new RuntimeException(match); + } + } + + private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException { + log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd"); + long startCommits = getNumCommits((HttpSolrClient) clients.get(0)); + + NamedList result = + clients + .get(0) + .request( + new StreamingUpdateRequest( + "/update", getFile("books_numeric_ids.csv"), "application/csv") + .setCommitWithin(900000) + .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true)); + + long endCommits = getNumCommits((HttpSolrClient) clients.get(0)); + assertEquals(startCommits + 1L, endCommits); + } + + private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException { + String collection = sourceClient.getDefaultCollection(); + try (SolrClient client = + new HttpSolrClient.Builder(sourceClient.getBaseURL()) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(60000, TimeUnit.MILLISECONDS) + .build()) { + var req = + new GenericSolrRequest( + SolrRequest.METHOD.GET, + "/admin/metrics", + SolrRequest.SolrRequestType.ADMIN, + SolrParams.of("wt", "prometheus")); + req.setResponseParser(new InputStreamResponseParser("prometheus")); + + NamedList resp = client.request(req); + try (InputStream in = (InputStream) resp.get("stream")) { + String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); + String metricName = "solr_core_update_commit_ops"; + + return (long) + output + .lines() + .filter( + l -> + l.startsWith(metricName) + && l.contains("collection=\"" + collection + "\"") + && l.contains("ops=\"commits\"")) + .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" ")))) + .sum(); + } + } + } + + private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception { + log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement"); + assertEquals( + 0, + CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + + List collectionClients = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + CollectionAdminResponse resp = + CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1)) + .setNode(jettys.get(0).getNodeName()) + .process(cloudClient); + for (String coreName : resp.getCollectionCoresStatus().keySet()) { + collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); + } + } + + SolrClient client1 = collectionClients.get(0); + SolrClient client2 = collectionClients.get(1); + SolrClient client3 = collectionClients.get(2); + SolrClient client4 = collectionClients.get(3); + + // no one should be recovering + waitForRecoveriesToFinish( + oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true); + + assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient())); + + // printLayout(); + + // TODO: enable when we don't falsely get slice1... + // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000); + // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000); + client2.add(getDoc(id, "1")); + client3.add(getDoc(id, "2")); + client4.add(getDoc(id, "3")); + + client1.commit(); + SolrQuery query = new SolrQuery("*:*"); + query.set("distrib", false); + long oneDocs = client1.query(query).getResults().getNumFound(); + long twoDocs = client2.query(query).getResults().getNumFound(); + long threeDocs = client3.query(query).getResults().getNumFound(); + long fourDocs = client4.query(query).getResults().getNumFound(); + + query.set("collection", oneInstanceCollection2); + query.set("distrib", true); + long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + + // System.out.println("1:" + oneDocs); + // System.out.println("2:" + twoDocs); + // System.out.println("3:" + threeDocs); + // System.out.println("4:" + fourDocs); + // System.out.println("All Docs:" + allDocs); + + // assertEquals(oneDocs, threeDocs); + // assertEquals(twoDocs, fourDocs); + // assertNotSame(oneDocs, twoDocs); + assertEquals(3, allDocs); + + // we added a role of none on these creates - check for it + ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient()); + zkStateReader.forceUpdateCollection(oneInstanceCollection2); + Map slices = + zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap(); + assertNotNull(slices); + + Replica leader = + getCommonCloudSolrClient() + .getClusterState() + .getCollection(oneInstanceCollection2) + .getLeader("shard1"); + + // now test that unloading a core gets us a new leader + try (SolrClient unloadClient = + new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString()) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(60000, TimeUnit.MILLISECONDS) + .build()) { + Unload unloadCmd = new Unload(true); + unloadCmd.setCoreName(leader.getCoreName()); + + String leaderUrl = leader.getCoreUrl(); + + testExecutor.execute( + new Runnable() { + + @Override + public void run() { + try { + unloadClient.request(unloadCmd); + } catch (SolrServerException e) { + throw new RuntimeException(e); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + }); + + try { + ZkStateReader.from(getCommonCloudSolrClient()) + .waitForState( + oneInstanceCollection2, + 20000, + TimeUnit.MILLISECONDS, + (n, c) -> { + try { + if (leaderUrl.equals( + zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) { + return false; + } + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return true; + }); + } catch (TimeoutException | InterruptedException e) { + fail("Leader never changed"); + } + } + + IOUtils.close(collectionClients); + } + + private void testSearchByCollectionName() throws SolrServerException, IOException { + log.info("### STARTING testSearchByCollectionName"); + SolrClient client = clients.get(0); + final String baseUrl = ((HttpSolrClient) client).getBaseURL(); + + // the cores each have different names, but if we add the collection name to the url + // we should get mapped to the right core + try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { + SolrQuery query = new SolrQuery("*:*"); + long oneDocs = client1.query(query).getResults().getNumFound(); + assertEquals(3, oneDocs); + } + } + + private void testUpdateByCollectionName() throws SolrServerException, IOException { + log.info("### STARTING testUpdateByCollectionName"); + SolrClient client = clients.get(0); + final String baseUrl = ((HttpSolrClient) client).getBaseURL(); + + // the cores each have different names, but if we add the collection name to the url + // we should get mapped to the right core + // test hitting an update url + try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { + client1.commit(); + } + } + + private void testANewCollectionInOneInstance() throws Exception { + log.info("### STARTING testANewCollectionInOneInstance"); + CollectionAdminResponse response = + CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2) + .setCreateNodeSet(jettys.get(0).getNodeName()) + .process(cloudClient); + assertEquals(0, response.getStatus()); + List collectionClients = new ArrayList<>(); + for (String coreName : response.getCollectionCoresStatus().keySet()) { + collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); + } + + SolrClient client1 = collectionClients.get(0); + SolrClient client2 = collectionClients.get(1); + SolrClient client3 = collectionClients.get(2); + SolrClient client4 = collectionClients.get(3); + + waitForRecoveriesToFinish( + oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false); + assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient())); + + client2.add(getDoc(id, "1")); + client3.add(getDoc(id, "2")); + client4.add(getDoc(id, "3")); + + client1.commit(); + SolrQuery query = new SolrQuery("*:*"); + query.set("distrib", false); + long oneDocs = client1.query(query).getResults().getNumFound(); + long twoDocs = client2.query(query).getResults().getNumFound(); + long threeDocs = client3.query(query).getResults().getNumFound(); + long fourDocs = client4.query(query).getResults().getNumFound(); + + query.set("collection", oneInstanceCollection); + query.set("distrib", true); + long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + + // System.out.println("1:" + oneDocs); + // System.out.println("2:" + twoDocs); + // System.out.println("3:" + threeDocs); + // System.out.println("4:" + fourDocs); + // System.out.println("All Docs:" + allDocs); + + assertEquals(3, allDocs); + IOUtils.close(collectionClients); + } + + private void createSolrCore( + final String collection, + List collectionClients, + final String baseUrl, + final int num, + final String shardId) { + Callable call = + () -> { + try (SolrClient client = getHttpSolrClient(baseUrl)) { + // client.setConnectionTimeout(15000); + Create createCmd = new Create(); + createCmd.setCoreName(collection + num); + createCmd.setCollection(collection); + + if (random().nextBoolean()) { + // sometimes we use an explicit core node name + createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet()); + } + + if (shardId == null) { + createCmd.setNumShards(2); + } + createCmd.setDataDir(getDataDir(createTempDir(collection).toString())); + if (shardId != null) { + createCmd.setShardId(shardId); + } + client.request(createCmd); + } catch (Exception e) { + log.error("error creating core", e); + // fail + } + return null; + }; + + pending.add(completionService.submit(call)); + + collectionClients.add(createNewSolrClient(collection + num, baseUrl)); + } + + private void testMultipleCollections() throws Exception { + log.info("### STARTING testMultipleCollections"); + // create another 2 collections and search across them + createNewCollection("collection2"); + createNewCollection("collection3"); + + while (pending != null && pending.size() > 0) { + + Future future = completionService.take(); + if (future == null) return; + pending.remove(future); + } + + indexDoc("collection2", getDoc(id, "10000000")); + indexDoc("collection2", getDoc(id, "10000001")); + indexDoc("collection2", getDoc(id, "10000003")); + + getSolrClient("collection2").add(getDoc(id, "10000004")); + + indexDoc("collection3", getDoc(id, "20000000")); + indexDoc("collection3", getDoc(id, "20000001")); + + getSolrClient("collection3").add(getDoc(id, "10000005")); + + otherCollectionClients.get("collection2").get(0).commit(); + otherCollectionClients.get("collection3").get(0).commit(); + + SolrClient clientForCollection1 = getSolrClient("collection1"); + long collection1Docs = + clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound(); + + long collection2Docs = + otherCollectionClients + .get("collection2") + .get(0) + .query(new SolrQuery("*:*")) + .getResults() + .getNumFound(); + + long collection3Docs = + otherCollectionClients + .get("collection3") + .get(0) + .query(new SolrQuery("*:*")) + .getResults() + .getNumFound(); + + SolrQuery query = new SolrQuery("*:*"); + query.set("collection", "collection2,collection3"); + long found = clients.get(0).query(query).getResults().getNumFound(); + assertEquals(collection2Docs + collection3Docs, found); + + query = new SolrQuery("*:*"); + query.set("collection", "collection1,collection2,collection3"); + found = clients.get(0).query(query).getResults().getNumFound(); + assertEquals(collection1Docs + collection2Docs + collection3Docs, found); + + // try to search multiple with cloud client + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection1Docs + collection2Docs + collection3Docs, found); + + query.set("collection", "collection2,collection3"); + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection2Docs + collection3Docs, found); + + query.set("collection", "collection3"); + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection3Docs, found); + + query.remove("collection"); + found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); + assertEquals(collection1Docs, found); + + assertEquals(collection3Docs, collection2Docs - 1); + } + + protected void indexDoc(String collection, SolrInputDocument doc) + throws IOException, SolrServerException { + List clients = otherCollectionClients.get(collection); + int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size(); + SolrClient client = clients.get(which); + client.add(doc); + } + + @SuppressWarnings({"unchecked"}) + private void createNewCollection(final String collection) throws InterruptedException { + try { + assertEquals( + 0, + CollectionAdminRequest.createCollection(collection, "conf1", 2, 1) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + } catch (Exception e) { + log.error("error creating collection", e); + // fails + } + final List collectionClients = new ArrayList<>(); + otherCollectionClients.put(collection, collectionClients); + int unique = 0; + for (final JettySolrRunner runner : jettys) { + unique++; + final int frozeUnique = unique; + Callable call = + () -> { + try { + assertTrue( + CollectionAdminRequest.addReplicaToShard( + collection, "shard" + ((frozeUnique % 2) + 1)) + .setNode(runner.getNodeName()) + .process(cloudClient) + .isSuccess()); + } catch (Exception e) { + log.error("error adding replica", e); + // fails + } + return null; + }; + + collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString())); + pending.add(completionService.submit(call)); + while (pending != null && pending.size() > 0) { + + Future future = completionService.take(); + if (future == null) return; + pending.remove(future); + } + } + } + + @Override + protected SolrClient createNewSolrClient(String collection, String baseUrl) { + + SolrClient client = getHttpSolrClient(baseUrl, collection); + + return client; + } + + /** + * @param collection the name of a collection or core to set as the "default" on the created + * client. + * @param baseUrl the "base" URL of a Solr node. Should not contain a collection or core + * name. + * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds + * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds + */ + protected SolrClient createNewSolrClient( + String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) { + + SolrClient client = + new HttpSolrClient.Builder(baseUrl) + .withDefaultCollection(collection) + .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS) + .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS) + .build(); + + return client; + } + + @Override + protected QueryResponse queryRandomShard(ModifiableSolrParams params) + throws SolrServerException, IOException { + + if (r.nextBoolean()) return super.queryRandomShard(params); + + if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION); + + QueryResponse rsp = getCommonCloudSolrClient().query(params); + return rsp; + } + + @Override + public void distribTearDown() throws Exception { + super.distribTearDown(); + if (otherCollectionClients != null) { + for (List clientList : otherCollectionClients.values()) { + IOUtils.close(clientList); + } + } + otherCollectionClients = null; + List tasks = executor.shutdownNow(); + assertTrue(tasks.isEmpty()); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java index 3fb1bbabc6e..3f4ab451c85 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeyNothingIsSafeTest.java @@ -16,11 +16,324 @@ */ package org.apache.solr.cloud; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.LuceneTestCase; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; -/* - * Implementation moved to AbstractChaosMonkeyNothingIsSafeTestBase.java as it is also - * used by the HDFS contrib tests. - */ +@LuceneTestCase.Nightly @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class ChaosMonkeyNothingIsSafeTest extends AbstractChaosMonkeyNothingIsSafeTestBase {} +public class ChaosMonkeyNothingIsSafeTest extends AbstractFullDistribZkTestBase { + private static final int FAIL_TOLERANCE = 100; + + private static final Integer RUN_LENGTH = + Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); + + private final boolean onlyLeaderIndexes = random().nextBoolean(); + + @BeforeClass + public static void beforeSuperClass() { + schemaString = "schema15.xml"; // we need a string id + System.setProperty("solr.autoCommit.maxTime", "15000"); + System.clearProperty("solr.httpclient.retries"); + System.clearProperty("solr.retries.on.forward"); + System.clearProperty("solr.retries.to.followers"); + setErrorHook(); + } + + @AfterClass + public static void afterSuperClass() { + System.clearProperty("solr.autoCommit.maxTime"); + clearErrorHook(); + } + + @Override + protected void destroyServers() throws Exception { + + super.destroyServers(); + } + + protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; + protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; + + private int clientSoTimeout = 60000; + + private volatile FullThrottleStoppableIndexingThread ftIndexThread; + + private final boolean runFullThrottle; + + @Override + public String[] getFieldNames() { + return fieldNames; + } + + @Override + public RandVal[] getRandValues() { + return randVals; + } + + @Override + public void distribSetUp() throws Exception { + super.distribSetUp(); + // can help to hide this when testing and looking at logs + // ignoreException("shard update error"); + useFactory("solr.StandardDirectoryFactory"); + } + + @Override + public void distribTearDown() throws Exception { + try { + ftIndexThread.safeStop(); + } catch (NullPointerException e) { + // okay + } + super.distribTearDown(); + } + + public ChaosMonkeyNothingIsSafeTest() { + super(); + sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); + if (sliceCount == -1) { + sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; + } + + int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); + if (numShards == -1) { + // we make sure that there's at least one shard with more than one replica + // so that the ChaosMonkey has something to kill + numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; + } + fixShardCount(numShards); + + // TODO: we only do this sometimes so that we can sometimes compare against control, + // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer + runFullThrottle = random().nextBoolean(); + } + + @Override + protected boolean useTlogReplicas() { + return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug + // and it's TestInjection use + } + + @Override + protected CloudSolrClient createCloudClient(String defaultCollection) { + return this.createCloudClient(defaultCollection, this.clientSoTimeout); + } + + protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) { + + return getCloudSolrClient( + zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout); + } + + @Test + @SuppressWarnings({"try"}) + public void test() throws Exception { + // None of the operations used here are particularly costly, so this should work. + // Using this low timeout will also help us catch index stalling. + clientSoTimeout = 5000; + + boolean testSuccessful = false; + try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) { + handle.clear(); + handle.put("timestamp", SKIPVAL); + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + // make sure we have leaders for each shard + for (int j = 1; j < sliceCount; j++) { + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); + } // make sure we again have leaders for each shard + + waitForRecoveriesToFinish(false); + + // we cannot do delete by query + // as it's not supported for recovery + del("*:*"); + + List threads = new ArrayList<>(); + List indexTreads = new ArrayList<>(); + int threadCount = TEST_NIGHTLY ? 3 : 1; + int i = 0; + for (i = 0; i < threadCount; i++) { + StoppableIndexingThread indexThread = + new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true); + threads.add(indexThread); + indexTreads.add(indexThread); + indexThread.start(); + } + + threadCount = 1; + i = 0; + for (i = 0; i < threadCount; i++) { + StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient); + threads.add(searchThread); + searchThread.start(); + } + + if (runFullThrottle) { + ftIndexThread = + new FullThrottleStoppableIndexingThread( + ((CloudLegacySolrClient) cloudClient).getHttpClient(), + controlClient, + cloudClient, + clients, + "ft1", + true, + this.clientSoTimeout); + ftIndexThread.start(); + } + + chaosMonkey.startTheMonkey(true, 10000); + try { + long runLength; + if (RUN_LENGTH != -1) { + runLength = RUN_LENGTH; + } else { + int[] runTimes; + if (TEST_NIGHTLY) { + runTimes = + new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; + } else { + runTimes = new int[] {5000, 7000, 15000}; + } + runLength = runTimes[random().nextInt(runTimes.length - 1)]; + } + + Thread.sleep(runLength); + } finally { + chaosMonkey.stopTheMonkey(); + } + + // ideally this should go into chaosMonkey + restartZk(1000 * (5 + random().nextInt(4))); + + if (runFullThrottle) { + ftIndexThread.safeStop(); + } + + for (StoppableThread indexThread : threads) { + indexThread.safeStop(); + } + + // start any downed jetties to be sure we still will end up with a leader per shard... + + // wait for stop... + for (StoppableThread indexThread : threads) { + indexThread.join(); + } + + // try and wait for any replications and what not to finish... + + Thread.sleep(2000); + + // wait until there are no recoveries... + waitForThingsToLevelOut(); + + // make sure we again have leaders for each shard + for (int j = 1; j < sliceCount; j++) { + zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000); + } + + commit(); + + // TODO: assert we didn't kill everyone + + zkStateReader.updateLiveNodes(); + assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0); + + // we expect full throttle fails, but cloud client should not easily fail + for (StoppableThread indexThread : threads) { + if (indexThread instanceof StoppableIndexingThread + && !(indexThread instanceof FullThrottleStoppableIndexingThread)) { + int failCount = ((StoppableIndexingThread) indexThread).getFailCount(); + assertFalse( + "There were too many update fails (" + + failCount + + " > " + + FAIL_TOLERANCE + + ") - we expect it can happen, but shouldn't easily", + failCount > FAIL_TOLERANCE); + } + } + + waitForThingsToLevelOut(20, TimeUnit.SECONDS); + + commit(); + + Set addFails = getAddFails(indexTreads); + Set deleteFails = getDeleteFails(indexTreads); + // full throttle thread can + // have request fails + checkShardConsistency(!runFullThrottle, true, addFails, deleteFails); + + long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + // ensure we have added more than 0 docs + long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + + assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0); + + if (VERBOSE) + System.out.println( + "control docs:" + + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + + "\n\n"); + + // try and make a collection to make sure the overseer has survived the expiration and session + // loss + + // sometimes we restart zookeeper as well + if (random().nextBoolean()) { + // restartZk(1000 * (5 + random().nextInt(4))); + } + + try (CloudSolrClient client = createCloudClient("collection1", 30000)) { + createCollection(null, "testcollection", 1, 1, client, null, "conf1"); + } + List numShardsNumReplicas = new ArrayList<>(2); + numShardsNumReplicas.add(1); + numShardsNumReplicas.add(1); + checkForCollection("testcollection", numShardsNumReplicas); + + testSuccessful = true; + } finally { + if (!testSuccessful) { + printLayout(); + } + } + } + + private Set getAddFails(List threads) { + Set addFails = new HashSet<>(); + for (StoppableIndexingThread thread : threads) { + addFails.addAll(thread.getAddFails()); + } + return addFails; + } + + private Set getDeleteFails(List threads) { + Set deleteFails = new HashSet<>(); + for (StoppableIndexingThread thread : threads) { + deleteFails.addAll(thread.getDeleteFails()); + } + return deleteFails; + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = getDoc(fields); + indexDoc(doc); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java index a4c1ab267d1..a75b637af22 100644 --- a/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/ChaosMonkeySafeLeaderTest.java @@ -14,14 +14,220 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.solr.cloud; -/* - * Implementation moved to AbstractChaosMonkeySafeLeaderTestBase as is used by HDFS contrib module test - */ -public class ChaosMonkeySafeLeaderTest extends AbstractChaosMonkeySafeLeaderTestBase { +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.lucene.tests.util.LuceneTestCase; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.common.SolrInputDocument; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +@LuceneTestCase.Nightly +public class ChaosMonkeySafeLeaderTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final Integer RUN_LENGTH = + Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); + + @BeforeClass + public static void beforeSuperClass() { + schemaString = "schema15.xml"; // we need a string id + System.setProperty("solr.autoCommit.maxTime", "15000"); + System.clearProperty("solr.httpclient.retries"); + System.clearProperty("solr.retries.on.forward"); + System.clearProperty("solr.retries.to.followers"); + setErrorHook(); + } + + @AfterClass + public static void afterSuperClass() { + System.clearProperty("solr.autoCommit.maxTime"); + clearErrorHook(); + } + + protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; + protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; + + @Override + public String[] getFieldNames() { + return fieldNames; + } + @Override + public RandVal[] getRandValues() { + return randVals; + } + protected String getDirectoryFactory() { return "solr.StandardDirectoryFactory"; } + + @Override + public void distribSetUp() throws Exception { + useFactory(getDirectoryFactory()); + super.distribSetUp(); + } + + public ChaosMonkeySafeLeaderTest() { + super(); + sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); + if (sliceCount == -1) { + sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; + } + + int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); + if (numShards == -1) { + // we make sure that there's at least one shard with more than one replica + // so that the ChaosMonkey has something to kill + numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; + } + fixShardCount(numShards); + } + + @Test + public void test() throws Exception { + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + // randomly turn on 1 seconds 'soft' commit + randomlyEnableAutoSoftCommit(); + + tryDelete(); + + List threads = new ArrayList<>(); + int threadCount = 2; + int batchSize = 1; + if (random().nextBoolean()) { + batchSize = random().nextInt(98) + 2; + } + + boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true; + int maxUpdates = -1; + if (!pauseBetweenUpdates) { + maxUpdates = 1000 + random().nextInt(1000); + } else { + maxUpdates = 15000; + } + + for (int i = 0; i < threadCount; i++) { + StoppableIndexingThread indexThread = + new StoppableIndexingThread( + controlClient, + cloudClient, + Integer.toString(i), + true, + maxUpdates, + batchSize, + pauseBetweenUpdates); // random().nextInt(999) + 1 + threads.add(indexThread); + indexThread.start(); + } + + chaosMonkey.startTheMonkey(false, 500); + try { + long runLength; + if (RUN_LENGTH != -1) { + runLength = RUN_LENGTH; + } else { + int[] runTimes; + if (TEST_NIGHTLY) { + runTimes = + new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; + } else { + runTimes = new int[] {5000, 7000, 15000}; + } + runLength = runTimes[random().nextInt(runTimes.length - 1)]; + } + + Thread.sleep(runLength); + } finally { + chaosMonkey.stopTheMonkey(); + } + + for (StoppableIndexingThread indexThread : threads) { + indexThread.safeStop(); + } + + // wait for stop... + for (StoppableIndexingThread indexThread : threads) { + indexThread.join(); + } + + for (StoppableIndexingThread indexThread : threads) { + assertEquals(0, indexThread.getFailCount()); + } + + // try and wait for any replications and what not to finish... + + Thread.sleep(2000); + + waitForThingsToLevelOut(3, TimeUnit.MINUTES); + + // even if things were leveled out, a jetty may have just been stopped or something + // we wait again and wait to level out again to make sure the system is not still in flux + + Thread.sleep(3000); + + waitForThingsToLevelOut(3, TimeUnit.MINUTES); + + checkShardConsistency(batchSize == 1, true); + + if (VERBOSE) + System.out.println( + "control docs:" + + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() + + "\n\n"); + + // try and make a collection to make sure the overseer has survived the expiration and session + // loss + + // sometimes we restart zookeeper as well + if (random().nextBoolean()) { + zkServer.shutdown(); + zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); + zkServer.run(false); + } + + try (CloudSolrClient client = createCloudClient("collection1")) { + createCollection(null, "testcollection", 1, 1, client, null, "conf1"); + } + List numShardsNumReplicas = new ArrayList<>(2); + numShardsNumReplicas.add(1); + numShardsNumReplicas.add(1); + checkForCollection("testcollection", numShardsNumReplicas); + } + + private void tryDelete() throws Exception { + long start = System.nanoTime(); + long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS); + while (System.nanoTime() < timeout) { + try { + del("*:*"); + break; + } catch (SolrServerException e) { + log.error("cluster may not be up yet", e); + } + Thread.sleep(100); + } + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java index 265cbd4d1ea..d6e00a202ee 100644 --- a/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/MoveReplicaTest.java @@ -17,23 +17,392 @@ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.apache.solr.client.api.model.CoreStatusResponse; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.impl.CloudSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.response.CoreAdminResponse; +import org.apache.solr.client.solrj.response.RequestStatusState; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.IdUtils; +import org.junit.After; +import org.junit.Before; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/** - * Implementation moved to AbstractMoveReplicaTestBase as it is referenced by HDFS contrib module - * tests. - */ -public class MoveReplicaTest extends AbstractMoveReplicaTestBase { +public class MoveReplicaTest extends SolrCloudTestCase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + protected boolean inPlaceMove = random().nextBoolean(); + protected boolean isCollectionApiDistributed = false; + + protected String getConfigSet() { + return "cloud-dynamic"; + } + + @Before + public void beforeTest() throws Exception { + + configureCluster(4) + .addConfig("conf1", configset(getConfigSet())) + .addConfig("conf2", configset(getConfigSet())) + .withSolrXml(TEST_PATH().resolve("solr.xml")) + .configure(); + + // If Collection API is distributed let's not wait for Overseer. + isCollectionApiDistributed = + new CollectionAdminRequest.RequestApiDistributedProcessing() + .process(cluster.getSolrClient()) + .getIsCollectionApiDistributed(); + if (isCollectionApiDistributed) { + return; + } + + NamedList overSeerStatus = + cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); + JettySolrRunner overseerJetty = null; + String overseerLeader = (String) overSeerStatus.get("leader"); + for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { + if (jetty.getNodeName().equals(overseerLeader)) { + overseerJetty = jetty; + break; + } + } + if (overseerJetty == null) { + fail("no overseer leader!"); + } + } + + @After + public void afterTest() throws Exception { + try { + shutdownCluster(); + } finally { + super.tearDown(); + } + } - @Override @Test public void test() throws Exception { - super.test(); + String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove; + if (log.isInfoEnabled()) { + log.info("total_jettys: {}", cluster.getJettySolrRunners().size()); + } + int REPLICATION = 2; + + CloudSolrClient cloudClient = cluster.getSolrClient(); + + // random create tlog or pull type replicas with nrt + boolean isTlog = random().nextBoolean(); + CollectionAdminRequest.Create create = + CollectionAdminRequest.createCollection( + coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); + cloudClient.request(create); + + addDocs(coll, 100); + + Replica replica = getRandomReplica(coll, cloudClient); + Set liveNodes = cloudClient.getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + String targetNode = null; + for (String node : liveNodes) { + if (!replica.getNodeName().equals(node)) { + targetNode = node; + break; + } + } + assertNotNull(targetNode); + String shardId = null; + for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) { + if (slice.getReplicas().contains(replica)) { + shardId = slice.getName(); + } + } + + int sourceNumCores = + getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()); + int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()); + + CollectionAdminRequest.MoveReplica moveReplica = + createMoveReplicaRequest(coll, replica, targetNode); + moveReplica.setInPlaceMove(inPlaceMove); + String asyncId = IdUtils.randomId(); + moveReplica.processAsync(asyncId, cloudClient); + CollectionAdminRequest.RequestStatus requestStatus = + CollectionAdminRequest.requestStatus(asyncId); + // wait for async request success + boolean success = false; + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { + success = true; + break; + } + assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED); + Thread.sleep(500); + } + assertTrue(success); + assertEquals( + "should be one less core on the source node!", + sourceNumCores - 1, + getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name())); + assertEquals( + "should be one more core on target node!", + targetNumCores + 1, + getNumOfCores(cloudClient, targetNode, coll, replica.getType().name())); + // wait for recovery + boolean recovered = false; + for (int i = 0; i < 300; i++) { + DocCollection collState = getCollectionState(coll); + log.debug("###### {}", collState); + Collection replicas = collState.getSlice(shardId).getReplicas(); + boolean allActive = true; + boolean hasLeaders = true; + if (replicas != null && !replicas.isEmpty()) { + for (Replica r : replicas) { + if (!r.getNodeName().equals(targetNode)) { + continue; + } + if (!r.isActive(Collections.singleton(targetNode))) { + log.info("Not active: {}", r); + allActive = false; + } + } + } else { + allActive = false; + } + for (Slice slice : collState.getSlices()) { + if (slice.getLeader() == null) { + hasLeaders = false; + } + } + if (allActive && hasLeaders) { + // check the number of active replicas + assertEquals("total number of replicas", REPLICATION, replicas.size()); + recovered = true; + break; + } else { + log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders); + Thread.sleep(1000); + } + } + assertTrue("replica never fully recovered", recovered); + + assertEquals( + 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); + + moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId); + moveReplica.setInPlaceMove(inPlaceMove); + moveReplica.process(cloudClient); + checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores); + // wait for recovery + recovered = false; + for (int i = 0; i < 300; i++) { + DocCollection collState = getCollectionState(coll); + log.debug("###### {}", collState); + Collection replicas = collState.getSlice(shardId).getReplicas(); + boolean allActive = true; + boolean hasLeaders = true; + if (replicas != null && !replicas.isEmpty()) { + for (Replica r : replicas) { + if (!r.getNodeName().equals(replica.getNodeName())) { + continue; + } + if (!r.isActive(Collections.singleton(replica.getNodeName()))) { + log.info("Not active yet: {}", r); + allActive = false; + } + } + } else { + allActive = false; + } + for (Slice slice : collState.getSlices()) { + if (slice.getLeader() == null) { + hasLeaders = false; + } + } + if (allActive && hasLeaders) { + assertEquals("total number of replicas", REPLICATION, replicas.size()); + recovered = true; + break; + } else { + Thread.sleep(1000); + } + } + assertTrue("replica never fully recovered", recovered); + + assertEquals( + 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); } - @Override @Test public void testFailedMove() throws Exception { - super.testFailedMove(); + String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove; + int REPLICATION = 2; + + CloudSolrClient cloudClient = cluster.getSolrClient(); + + // random create tlog or pull type replicas with nrt + boolean isTlog = random().nextBoolean(); + CollectionAdminRequest.Create create = + CollectionAdminRequest.createCollection( + coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); + cloudClient.request(create); + + addDocs(coll, 100); + + NamedList overSeerStatus = + cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); + String overseerLeader = (String) overSeerStatus.get("leader"); + + // don't kill overseer in this test + Replica replica; + int count = 10; + do { + replica = getRandomReplica(coll, cloudClient); + } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0); + assertNotNull("could not find non-overseer replica???", replica); + Set liveNodes = cloudClient.getClusterState().getLiveNodes(); + ArrayList l = new ArrayList<>(liveNodes); + Collections.shuffle(l, random()); + String targetNode = null; + for (String node : liveNodes) { + if (!replica.getNodeName().equals(node) + && (isCollectionApiDistributed || !overseerLeader.equals(node))) { + targetNode = node; + break; + } + } + assertNotNull(targetNode); + CollectionAdminRequest.MoveReplica moveReplica = + createMoveReplicaRequest(coll, replica, targetNode); + moveReplica.setInPlaceMove(inPlaceMove); + // start moving + String asyncId = IdUtils.randomId(); + moveReplica.processAsync(asyncId, cloudClient); + // shut down target node + for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { + if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) { + JettySolrRunner j = cluster.stopJettySolrRunner(i); + cluster.waitForJettyToStop(j); + break; + } + } + CollectionAdminRequest.RequestStatus requestStatus = + CollectionAdminRequest.requestStatus(asyncId); + // wait for async request success + boolean success = true; + for (int i = 0; i < 200; i++) { + CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); + assertNotSame( + rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED); + if (rsp.getRequestStatus() == RequestStatusState.FAILED) { + success = false; + break; + } + Thread.sleep(500); + } + assertFalse(success); + + if (log.isInfoEnabled()) { + log.info( + "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll)); + } + assertEquals( + 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); + } + + private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( + String coll, Replica replica, String targetNode, String shardId) { + return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName()); + } + + private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( + String coll, Replica replica, String targetNode) { + return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode); + } + + private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException { + List replicas = cloudClient.getClusterState().getCollection(coll).getReplicas(); + Collections.shuffle(replicas, random()); + return replicas.get(0); + } + + private void checkNumOfCores( + CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores) + throws IOException, SolrServerException { + assertEquals( + nodeName + " does not have expected number of cores", + expectedCores, + getNumOfCores(cloudClient, nodeName, collectionName)); + } + + private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName) + throws IOException, SolrServerException { + return getNumOfCores(cloudClient, nodeName, collectionName, null); + } + + private int getNumOfCores( + CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType) + throws IOException, SolrServerException { + try (SolrClient coreclient = + getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) { + CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient); + if (status.getCoreStatus().size() == 0) { + return 0; + } + if (collectionName == null && replicaType == null) { + return status.getCoreStatus().size(); + } + // filter size by collection name + int size = 0; + for (Map.Entry coreStatusEntry : + status.getCoreStatus().entrySet()) { + if (collectionName != null) { + String coll = coreStatusEntry.getValue().cloud.collection; + if (!collectionName.equals(coll)) { + continue; + } + } + if (replicaType != null) { + String type = coreStatusEntry.getValue().cloud.replicaType; + if (!replicaType.equals(type)) { + continue; + } + } + size++; + } + return size; + } + } + + protected void addDocs(String collection, int numDocs) throws Exception { + SolrClient solrClient = cluster.getSolrClient(); + for (int docId = 1; docId <= numDocs; docId++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", docId); + solrClient.add(collection, doc); + } + solrClient.commit(collection); + Thread.sleep(5000); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java index b0188069562..4c0ad7af750 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OutOfBoxZkACLAndCredentialsProvidersTest.java @@ -122,7 +122,7 @@ public void testOutOfBoxSolrZkClient() throws Exception { .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) .build(); try { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, true, true, true, true, true, true, true, true); } finally { zkClient.close(); diff --git a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java index 5810c50fe17..2f3f2201ede 100644 --- a/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/OverriddenZkACLAndCredentialsProvidersTest.java @@ -153,7 +153,7 @@ public void testNoCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() th try (SolrZkClient zkClient = new SolrZkClientFactoryUsingCompletelyNewProviders(null, null, null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -165,7 +165,7 @@ public void testWrongCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() new SolrZkClientFactoryUsingCompletelyNewProviders( "connectAndAllACLUsername", "connectAndAllACLPasswordWrong", null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -176,7 +176,7 @@ public void testAllCredentialsSolrZkClientFactoryUsingCompletelyNewProviders() t new SolrZkClientFactoryUsingCompletelyNewProviders( "connectAndAllACLUsername", "connectAndAllACLPassword", null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, true, true, true, true, true, true, true, true); } } @@ -188,7 +188,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClientFactoryUsingCompletelyNewProviders( "readonlyACLUsername", "readonlyACLPassword", null, null) .getSolrZkClient(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT)) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, false, false, false, false, false, false, false, false); } } @@ -204,7 +204,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -220,7 +220,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, false, false, false, false, false, false, false, false, false, false); } } @@ -236,7 +236,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, true, true, true, true, true, true, true, true); } } @@ -252,7 +252,7 @@ public void testReadonlyCredentialsSolrZkClientFactoryUsingCompletelyNewProvider new SolrZkClient.Builder() .withUrl(zkServer.getZkAddress()) .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS))) { - AbstractDigestZkACLAndCredentialsProvidersTestBase.doTest( + VMParamsZkACLAndCredentialsProvidersTest.doTest( zkClient, true, true, false, false, false, false, false, false, false, false); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java index 6394eb566e0..5ea4794ff4b 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RecoveryZkTest.java @@ -16,16 +16,143 @@ */ package org.apache.solr.cloud; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.cloud.DocCollection; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.embedded.JettySolrRunner; +import org.junit.After; +import org.junit.BeforeClass; import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; -/* - * Implementation moved to AbstractRecoveryZkTestBase as it is used by HDFS contrib tests. - */ -public class RecoveryZkTest extends AbstractRecoveryZkTestBase { +public class RecoveryZkTest extends SolrCloudTestCase { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + @BeforeClass + public static void setupCluster() throws Exception { + cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure(); + } + + private final List threads = new ArrayList<>(); + + @After + public void stopThreads() throws InterruptedException { + for (StoppableIndexingThread t : threads) { + t.safeStop(); + } + for (StoppableIndexingThread t : threads) { + t.join(); + } + threads.clear(); + } @Test - @Override public void test() throws Exception { - super.test(); + + final String collection = "recoverytest"; + + CollectionAdminRequest.createCollection(collection, "conf", 1, 2) + .process(cluster.getSolrClient()); + waitForState( + "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2)); + + // start a couple indexing threads + + int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000}; + int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000}; + + int maxDoc; + if (!TEST_NIGHTLY) { + maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; + } else { + maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)]; + } + log.info("Indexing {} documents", maxDoc); + + try (SolrClient solrClient = + cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) { + final StoppableIndexingThread indexThread = + new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true); + threads.add(indexThread); + indexThread.start(); + + final StoppableIndexingThread indexThread2 = + new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true); + threads.add(indexThread2); + indexThread2.start(); + + // give some time to index... + int[] waitTimes = new int[] {200, 2000, 3000}; + Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); + + // bring shard replica down + DocCollection state = getCollectionState(collection); + Replica leader = state.getLeader("shard1"); + Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r)); + + JettySolrRunner jetty = cluster.getReplicaJetty(replica); + jetty.stop(); + + // wait a moment - lets allow some docs to be indexed so replication time is non 0 + Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); + + // bring shard replica up + jetty.start(); + + // make sure replication can start + Thread.sleep(3000); + + // stop indexing threads + indexThread.safeStop(); + indexThread2.safeStop(); + + indexThread.join(); + indexThread2.join(); + + new UpdateRequest().commit(solrClient, collection); + + cluster + .getZkStateReader() + .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2)); + + // test that leader and replica have same doc count + state = getCollectionState(collection); + assertShardConsistency(state.getSlice("shard1"), true); + } + } + + private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception { + List replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE); + long[] numCounts = new long[replicas.size()]; + int i = 0; + for (Replica replica : replicas) { + try (var client = + new HttpSolrClient.Builder(replica.getBaseUrl()) + .withDefaultCollection(replica.getCoreName()) + .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient()) + .build()) { + numCounts[i] = + client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound(); + i++; + } + } + for (int j = 1; j < replicas.size(); j++) { + if (numCounts[j] != numCounts[j - 1]) + fail("Mismatch in counts between replicas"); // TODO improve this! + if (numCounts[j] == 0 && expectDocs) + fail("Expected docs on shard " + shard.getName() + " but found none"); + } } } diff --git a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java index 1bc74fff254..22b0084e989 100644 --- a/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/RestartWhileUpdatingTest.java @@ -16,23 +16,186 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase.Nightly; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; -/** - * Implementation moved to AbstractRestartWhileUpdatingTestBase because it is used by HDFS contrib - * module tests - */ @Nightly -public class RestartWhileUpdatingTest extends AbstractRestartWhileUpdatingTestBase { +public class RestartWhileUpdatingTest extends AbstractFullDistribZkTestBase { + + // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain"; + private List threads; + + private volatile boolean stopExpire = false; public RestartWhileUpdatingTest() throws Exception { super(); + sliceCount = 1; + fixShardCount(3); + schemaString = "schema15.xml"; // we need a string id + useFactory("solr.StandardDirectoryFactory"); } - @Test + public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; + public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; + @Override + protected String[] getFieldNames() { + return fieldNames; + } + + @Override + protected RandVal[] getRandValues() { + return randVals; + } + + @BeforeClass + public static void beforeRestartWhileUpdatingTest() { + System.setProperty("leaderVoteWait", "300000"); + System.setProperty("solr.autoCommit.maxTime", "30000"); + System.setProperty("solr.autoSoftCommit.maxTime", "3000"); + // SOLR-13212 // TestInjection.nonGracefullClose = "true:60"; + // SOLR-13189 // TestInjection.failReplicaRequests = "true:03"; + } + + @AfterClass + public static void afterRestartWhileUpdatingTest() { + System.clearProperty("leaderVoteWait"); + System.clearProperty("solr.autoCommit.maxTime"); + System.clearProperty("solr.autoSoftCommit.maxTime"); + } + + @Test public void test() throws Exception { - super.test(); + handle.clear(); + handle.put("timestamp", SKIPVAL); + + // start a couple indexing threads + + int[] maxDocList = new int[] {5000, 10000}; + + int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; + + int numThreads = random().nextInt(4) + 1; + + threads = new ArrayList<>(numThreads); + + Thread expireThread = + new Thread("expireThread") { + @Override + public void run() { + while (!stopExpire) { + try { + Thread.sleep(random().nextInt(15000)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + + // try { + // chaosMonkey.expireRandomSession(); + // } catch (KeeperException e) { + // throw new RuntimeException(e); + // } catch (InterruptedException e) { + // throw new RuntimeException(e); + // } + } + } + }; + + // Currently unused + // expireThread.start(); + + StoppableIndexingThread indexThread; + for (int i = 0; i < numThreads; i++) { + indexThread = + new StoppableIndexingThread( + controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true); + threads.add(indexThread); + indexThread.start(); + } + + Thread.sleep(2000); + + int restartTimes = 1; // random().nextInt(4) + 1;; + for (int i = 0; i < restartTimes; i++) { + Thread.sleep(random().nextInt(30000)); + stopAndStartAllReplicas(); + Thread.sleep(random().nextInt(30000)); + } + + Thread.sleep(2000); + + // stop indexing threads + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + stopExpire = true; + expireThread.join(); + + Thread.sleep(1000); + + waitForThingsToLevelOut(320, TimeUnit.SECONDS); + + Thread.sleep(2000); + + waitForThingsToLevelOut(30, TimeUnit.SECONDS); + + Thread.sleep(5000); + + waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); + + for (StoppableIndexingThread thread : threads) { + thread.join(); + } + + checkShardConsistency(false, false); + } + + public void stopAndStartAllReplicas() throws Exception, InterruptedException { + chaosMonkey.stopAll(random().nextInt(1)); + + if (random().nextBoolean()) { + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + } + Thread.sleep(1000); + + chaosMonkey.startAll(); + } + + @Override + protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { + cloudClient.add(doc); + } + + @Override + public void distribTearDown() throws Exception { + // make sure threads have been stopped... + if (threads != null) { + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + thread.safeStop(); + } + } + + super.distribTearDown(); + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java index bdff2a6a477..8f49afba970 100644 --- a/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/SyncSliceTest.java @@ -16,8 +16,285 @@ */ package org.apache.solr.cloud; -/** - * Test sync phase that occurs when Leader goes down and a new Leader is elected. Implementation - * moved to AbstractSyncSliceTestBase.java as it is also used by the HDFS contrib tests. - */ -public class SyncSliceTest extends AbstractSyncSliceTestBase {} +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.util.ArrayList; +import java.util.Collection; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; +import java.util.stream.Collectors; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.CollectionParams.CollectionAction; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.util.LogLevel; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */ +@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN") +public class SyncSliceTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private boolean success = false; + + @Override + public void distribTearDown() throws Exception { + if (!success) { + printLayoutOnTearDown = true; + } + super.distribTearDown(); + } + + public SyncSliceTest() { + super(); + sliceCount = 1; + fixShardCount(TEST_NIGHTLY ? 7 : 4); + } + + @Test + public void test() throws Exception { + + handle.clear(); + handle.put("timestamp", SKIPVAL); + + waitForThingsToLevelOut(30, TimeUnit.SECONDS); + + del("*:*"); + List skipServers = new ArrayList<>(); + int docId = 0; + indexDoc( + skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); + + indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue."); + + skipServers.add(shardToJetty.get("shard1").get(1)); + + indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy."); + + skipServers.add(shardToJetty.get("shard1").get(2)); + + indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake"); + + commit(); + + waitForRecoveriesToFinish(false); + + // shard should be inconsistent + String shardFailMessage = checkShardConsistency("shard1", true, false); + assertNotNull(shardFailMessage); + + ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("action", CollectionAction.SYNCSHARD.toString()); + params.set("collection", "collection1"); + params.set("shard", "shard1"); + final var request = + new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params); + + String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString(); + + // we only set the connect timeout, not so timeout + try (SolrClient baseClient = + new HttpSolrClient.Builder(baseUrl) + .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + baseClient.request(request); + } + + waitForThingsToLevelOut(15, TimeUnit.SECONDS); + + checkShardConsistency(false, true); + + long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + assertEquals(4, cloudClientDocs); + + // kill the leader - new leader could have all the docs or be missing one + CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1"); + + skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader + + // this doc won't be on one node + indexDoc( + skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); + commit(); + + Set jetties = new HashSet<>(); + jetties.addAll(shardToJetty.get("shard1")); + jetties.remove(leaderJetty); + assertEquals(getShardCount() - 1, jetties.size()); + + leaderJetty.jetty.stop(); + + Thread.sleep(3000); + + waitForNoShardInconsistency(); + + Thread.sleep(1000); + + checkShardConsistency(false, true); + + cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); + assertEquals(5, cloudClientDocs); + + CloudJettyRunner deadJetty = leaderJetty; + + // let's get the latest leader + while (deadJetty.equals(leaderJetty)) { + updateMappingsFromZk(this.jettys, this.clients); + leaderJetty = shardToLeaderJetty.get("shard1"); + } + + // bring back dead node + deadJetty.jetty.start(); // he is not the leader anymore + + waitTillAllNodesActive(); + + skipServers = getRandomOtherJetty(leaderJetty, deadJetty); + skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty)); + // skip list should be + + // System.out.println("leader:" + leaderJetty.url); + // System.out.println("dead:" + deadJetty.url); + // System.out.println("skip list:" + skipServers); + + // we are skipping 2 nodes + assertEquals(2, skipServers.size()); + + // more docs than can peer sync + for (int i = 0; i < 300; i++) { + indexDoc( + skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); + } + + commit(); + waitForRecoveriesToFinish(false); + + // shard should be inconsistent + shardFailMessage = waitTillInconsistent(); + assertNotNull( + "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:" + + leaderJetty.url + + " Dead Guy:" + + deadJetty.url + + "skip list:" + + skipServers, + shardFailMessage); + + // good place to test compareResults + boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient); + assertTrue("A test that compareResults is working correctly failed", shouldFail); + + jetties = new HashSet<>(); + jetties.addAll(shardToJetty.get("shard1")); + jetties.remove(leaderJetty); + assertEquals(getShardCount() - 1, jetties.size()); + + // kill the current leader + leaderJetty.jetty.stop(); + + waitForNoShardInconsistency(); + + checkShardConsistency(true, true); + + success = true; + } + + private void waitTillAllNodesActive() throws InterruptedException, TimeoutException { + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + + zkStateReader.waitForState( + "collection1", + 3, + TimeUnit.MINUTES, + (n, c) -> { + Collection replicas = c.getSlice("shard1").getReplicas(); + Set nodes = + replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet()); + return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals) + && n.containsAll(nodes); + }); + } + + private String waitTillInconsistent() throws Exception, InterruptedException { + String shardFailMessage = null; + + shardFailMessage = pollConsistency(shardFailMessage, 0); + shardFailMessage = pollConsistency(shardFailMessage, 3000); + shardFailMessage = pollConsistency(shardFailMessage, 5000); + shardFailMessage = pollConsistency(shardFailMessage, 15000); + + return shardFailMessage; + } + + private String pollConsistency(String shardFailMessage, int sleep) + throws InterruptedException, Exception { + try { + commit(); + } catch (Throwable t) { + log.error("commit error", t); + } + if (shardFailMessage == null) { + // try again + Thread.sleep(sleep); + shardFailMessage = checkShardConsistency("shard1", true, false); + } + return shardFailMessage; + } + + private List getRandomOtherJetty( + CloudJettyRunner leader, CloudJettyRunner down) { + List skipServers = new ArrayList<>(); + List candidates = new ArrayList<>(); + candidates.addAll(shardToJetty.get("shard1")); + + if (leader != null) { + candidates.remove(leader); + } + + if (down != null) { + candidates.remove(down); + } + + CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size())); + skipServers.add(cjetty); + return skipServers; + } + + protected void indexDoc(List skipServers, Object... fields) + throws IOException, SolrServerException { + SolrInputDocument doc = new SolrInputDocument(); + + addFields(doc, fields); + addFields(doc, "rnd_b", true); + + controlClient.add(doc); + + UpdateRequest ureq = new UpdateRequest(); + ureq.add(doc); + for (CloudJettyRunner skip : skipServers) { + ureq.getParams().add("test.distrib.skip.servers", skip.url + "/"); + } + ureq.process(cloudClient); + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java index e056c40a893..2fbd5128177 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/TlogReplayBufferedWhileIndexingTest.java @@ -16,26 +16,134 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.TimeUnit; import org.apache.lucene.tests.util.LuceneTestCase.Nightly; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; @Nightly @SuppressSSL -/* - * Implementation moved to AbstractTlogReplayBufferedWhileIndexingTestBase as it is also used by HDFS contrib - * module tests - */ -public class TlogReplayBufferedWhileIndexingTest - extends AbstractTlogReplayBufferedWhileIndexingTestBase { +public class TlogReplayBufferedWhileIndexingTest extends AbstractFullDistribZkTestBase { + + private List threads; public TlogReplayBufferedWhileIndexingTest() throws Exception { super(); + sliceCount = 1; + fixShardCount(2); + schemaString = "schema15.xml"; // we need a string id + } + + @BeforeClass + public static void beforeRestartWhileUpdatingTest() throws Exception { + System.setProperty("leaderVoteWait", "300000"); + System.setProperty("solr.autoCommit.maxTime", "10000"); + System.setProperty("solr.autoSoftCommit.maxTime", "3000"); + TestInjection.updateLogReplayRandomPause = "true:10"; + TestInjection.updateRandomPause = "true:10"; + useFactory("solr.StandardDirectoryFactory"); + } + + @AfterClass + public static void afterRestartWhileUpdatingTest() { + System.clearProperty("leaderVoteWait"); + System.clearProperty("solr.autoCommit.maxTime"); + System.clearProperty("solr.autoSoftCommit.maxTime"); } @Test - @Override public void test() throws Exception { - super.test(); + handle.clear(); + handle.put("timestamp", SKIPVAL); + + waitForRecoveriesToFinish(false); + + int numThreads = 3; + + threads = new ArrayList<>(numThreads); + + ArrayList allJetty = new ArrayList<>(); + allJetty.addAll(jettys); + allJetty.remove(shardToLeaderJetty.get("shard1").jetty); + assert allJetty.size() == 1 : allJetty.size(); + allJetty.get(0).stop(); + + StoppableIndexingThread indexThread; + for (int i = 0; i < numThreads; i++) { + boolean pauseBetweenUpdates = random().nextBoolean(); + int batchSize = random().nextInt(4) + 1; + indexThread = + new StoppableIndexingThread( + controlClient, + cloudClient, + Integer.toString(i), + true, + 900, + batchSize, + pauseBetweenUpdates); + threads.add(indexThread); + indexThread.start(); + } + + Thread.sleep(2000); + + allJetty.get(0).start(); + + Thread.sleep(45000); + + waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while, + // especially when beasting this test + + Thread.sleep(2000); + + waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); + + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + + waitForThingsToLevelOut(30, TimeUnit.SECONDS); + + checkShardConsistency(false, false); + } + + @Override + protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { + cloudClient.add(doc); + } + + @Override + public void distribTearDown() throws Exception { + // make sure threads have been stopped... + if (threads != null) { + for (StoppableIndexingThread thread : threads) { + thread.safeStop(); + } + + for (StoppableIndexingThread thread : threads) { + thread.join(); + } + } + + super.distribTearDown(); + } + + // skip the randoms - they can deadlock... + @Override + protected void indexr(Object... fields) throws Exception { + SolrInputDocument doc = new SolrInputDocument(); + addFields(doc, fields); + addFields(doc, "rnd_b", true); + indexDoc(doc); } } diff --git a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java index 30d3f884b97..7e66787a4a5 100644 --- a/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/UnloadDistributedZkTest.java @@ -16,12 +16,450 @@ */ package org.apache.solr.cloud; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.file.Path; +import java.util.Collection; +import java.util.Collections; +import java.util.Random; +import java.util.Set; +import java.util.concurrent.SynchronousQueue; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.TimeoutException; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.apache.HttpSolrClient; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; +import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.cloud.Replica; +import org.apache.solr.common.cloud.Slice; +import org.apache.solr.common.cloud.ZkStateReader; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.ExecutorUtil; +import org.apache.solr.common.util.SolrNamedThreadFactory; +import org.apache.solr.core.SolrCore; +import org.apache.solr.core.SolrPaths; +import org.apache.solr.embedded.JettySolrRunner; +import org.apache.solr.util.TestInjection; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. Implementation moved to AbstractUnloadDistributedZkTestBase as it is used by HDFS - * contrib module tests. + * expected. */ @SuppressSSL(bugUrl = "https://issues.apache.org/jira/browse/SOLR-5776") -public class UnloadDistributedZkTest extends AbstractUnloadDistributedZkTestBase {} +public class UnloadDistributedZkTest extends AbstractFullDistribZkTestBase { + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + public UnloadDistributedZkTest() { + super(); + fixShardCount(4); // needs at least 4 servers + } + + @Override + protected String getSolrXml() { + return "solr.xml"; + } + + @Test + public void test() throws Exception { + jettys.forEach( + j -> { + Set allowPath = j.getCoreContainer().getAllowPaths(); + allowPath.clear(); + allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path + }); + log.info("###Starting testCoreUnloadAndLeaders"); + testCoreUnloadAndLeaders(); // long + log.info("###Starting testUnloadOfCores"); + testUnloadLotsOfCores(); // long + log.info("###Starting testUnloadShardAndCollection"); + testUnloadShardAndCollection(); + } + + /** + * @param url a Solr node base URL. Should not contain a core or collection name. + */ + private SolrClient newSolrClient(String url) { + return new HttpSolrClient.Builder(url) + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(30000, TimeUnit.MILLISECONDS) + .build(); + } + + private void checkCoreNamePresenceAndSliceCount( + String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount) + throws Exception { + ZkStateReader reader = ZkStateReader.from(cloudClient); + try { + reader.waitForState( + collectionName, + 45, + TimeUnit.SECONDS, + c -> { + final Collection slices = (c != null) ? c.getSlices() : Collections.emptyList(); + if (expectedSliceCount == slices.size()) { + for (Slice slice : slices) { + for (Replica replica : slice.getReplicas()) { + if (coreName.equals(replica.get("core"))) { + return shouldBePresent; + } + } + } + return !shouldBePresent; + } else { + return false; + } + }); + } catch (TimeoutException e) { + printLayout(); + fail( + "checkCoreNamePresenceAndSliceCount failed:" + + " collection=" + + collectionName + + " CoreName=" + + coreName + + " shouldBePresent=" + + shouldBePresent); + } + } + + private void testUnloadShardAndCollection() throws Exception { + final int numShards = 2; + + final String collection = "test_unload_shard_and_collection"; + + final String coreName1 = collection + "_1"; + final String coreName2 = collection + "_2"; + + assertEquals( + 0, + CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) + .setCreateNodeSet("") + .process(cloudClient) + .getStatus()); + assertTrue( + CollectionAdminRequest.addReplicaToShard(collection, "shard1") + .setCoreName(coreName1) + .setNode(jettys.get(0).getNodeName()) + .process(cloudClient) + .isSuccess()); + + assertTrue( + CollectionAdminRequest.addReplicaToShard(collection, "shard2") + .setCoreName(coreName2) + .setNode(jettys.get(0).getNodeName()) + .process(cloudClient) + .isSuccess()); + + // does not mean they are active and up yet :* + waitForRecoveriesToFinish(collection, false); + + final boolean unloadInOrder = random().nextBoolean(); + final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2); + final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1); + + try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) { + // now unload one of the two + Unload unloadCmd = new Unload(false); + unloadCmd.setCoreName(unloadCmdCoreName1); + adminClient.request(unloadCmd); + + // there should still be two shards (as of SOLR-5209) + checkCoreNamePresenceAndSliceCount( + collection, + unloadCmdCoreName1, + false /* shouldBePresent */, + numShards /* expectedSliceCount */); + + // now unload one of the other + unloadCmd = new Unload(false); + unloadCmd.setCoreName(unloadCmdCoreName2); + adminClient.request(unloadCmd); + checkCoreNamePresenceAndSliceCount( + collection, + unloadCmdCoreName2, + false /* shouldBePresent */, + numShards /* expectedSliceCount */); + } + + // printLayout(); + // the collection should still be present (as of SOLR-5209 replica removal does not cascade to + // remove the slice and collection) + getCommonCloudSolrClient(); + assertTrue( + "No longer found collection " + collection, + cloudClient.getClusterState().hasCollection(collection)); + } + + protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) { + SolrCore solrCore = null; + for (SolrCore core : jetty.getCoreContainer().getCores()) { + if (core.getName().startsWith(collection)) { + solrCore = core; + } + } + return solrCore; + } + + /** + * @throws Exception on any problem + */ + private void testCoreUnloadAndLeaders() throws Exception { + JettySolrRunner jetty1 = jettys.get(0); + + assertEquals( + 0, + CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1) + .setCreateNodeSet(jetty1.getNodeName()) + .process(cloudClient) + .getStatus()); + getCommonCloudSolrClient(); + ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); + + zkStateReader.forceUpdateCollection("unloadcollection"); + + int slices = + zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); + assertEquals(1, slices); + SolrCore solrCore = getFirstCore("unloadcollection", jetty1); + String core1DataDir = solrCore.getDataDir(); + + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName("unloadcollection_shard1_replica2") + .setNode(jettys.get(1).getNodeName()) + .process(cloudClient) + .isSuccess()); + zkStateReader.forceUpdateCollection("unloadcollection"); + slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); + assertEquals(1, slices); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + Replica leader = getLeaderFromZk("unloadcollection", "shard1"); + + Random random = random(); + if (random.nextBoolean()) { + try (SolrClient collectionClient = + getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) { + // lets try and use the solrj client to index and retrieve a couple + // documents + SolrInputDocument doc1 = + getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); + SolrInputDocument doc2 = + getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls"); + SolrInputDocument doc3 = + getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled"); + collectionClient.add(doc1); + collectionClient.add(doc2); + collectionClient.add(doc3); + collectionClient.commit(); + } + } + + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName("unloadcollection_shard1_replica3") + .setNode(jettys.get(2).getNodeName()) + .process(cloudClient) + .isSuccess()); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + // so that we start with some versions when we reload... + TestInjection.skipIndexWriterCommitOnClose = true; + + try (SolrClient addClient = + new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) + .withDefaultCollection("unloadcollection_shard1_replica3") + .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + + // add a few docs + for (int x = 20; x < 100; x++) { + SolrInputDocument doc1 = + getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); + addClient.add(doc1); + } + } + // don't commit so they remain in the tran log + // collectionClient.commit(); + + // unload the leader + try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { + + Unload unloadCmd = new Unload(false); + unloadCmd.setCoreName(leader.getCoreName()); + ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams(); + + collectionClient.request(unloadCmd); + } + // Thread.currentThread().sleep(500); + // printLayout(); + + int tries = 50; + while (leader + .getCoreUrl() + .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { + Thread.sleep(100); + if (tries-- == 0) { + fail("Leader never changed"); + } + } + + // ensure there is a leader + zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); + + try (SolrClient addClient = + new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString()) + .withDefaultCollection("unloadcollection_shard1_replica2") + .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) + .withSocketTimeout(90000, TimeUnit.MILLISECONDS) + .build()) { + + // add a few docs while the leader is down + for (int x = 101; x < 200; x++) { + SolrInputDocument doc1 = + getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); + addClient.add(doc1); + } + } + + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName("unloadcollection_shard1_replica4") + .setNode(jettys.get(3).getNodeName()) + .process(cloudClient) + .isSuccess()); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + // unload the leader again + leader = getLeaderFromZk("unloadcollection", "shard1"); + try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { + + Unload unloadCmd = new Unload(false); + unloadCmd.setCoreName(leader.getCoreName()); + collectionClient.request(unloadCmd); + } + tries = 50; + while (leader + .getCoreUrl() + .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { + Thread.sleep(100); + if (tries-- == 0) { + fail("Leader never changed"); + } + } + + zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); + + TestInjection.skipIndexWriterCommitOnClose = false; // set this back + assertTrue( + CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") + .setCoreName(leader.getCoreName()) + .setDataDir(core1DataDir) + .setNode(leader.getNodeName()) + .process(cloudClient) + .isSuccess()); + + waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); + + long found1, found3; + + try (SolrClient adminClient = + newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) { + adminClient.commit(); + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + found1 = adminClient.query(q).getResults().getNumFound(); + } + + try (SolrClient adminClient = + new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) + .withDefaultCollection("unloadcollection_shard1_replica3") + .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) + .withSocketTimeout(30000, TimeUnit.MILLISECONDS) + .build()) { + adminClient.commit(); + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + found3 = adminClient.query(q).getResults().getNumFound(); + } + + try (SolrClient adminClient = + newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) { + adminClient.commit(); + SolrQuery q = new SolrQuery("*:*"); + q.set("distrib", false); + long found4 = adminClient.query(q).getResults().getNumFound(); + + // all 3 shards should now have the same number of docs + assertEquals(found1, found3); + assertEquals(found3, found4); + } + } + + private void testUnloadLotsOfCores() throws Exception { + JettySolrRunner jetty = jettys.get(0); + int shards = TEST_NIGHTLY ? 2 : 1; + try (final SolrClient adminClient = jetty.newClient(15000, 60000)) { + int numReplicas = atLeast(3); + ThreadPoolExecutor executor = + new ExecutorUtil.MDCAwareThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 5, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new SolrNamedThreadFactory("testExecutor")); + try { + // create the cores + BasicDistributedZkTest.createCollectionInOneInstance( + adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas); + } finally { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + + if (TEST_NIGHTLY == false) { + // with nightly tests, we can try doing the unloads before the creates are done + // it still works, but takes much longer since we end up waiting for a timeout + waitForRecoveriesToFinish("multiunload", false); + } + + executor = + new ExecutorUtil.MDCAwareThreadPoolExecutor( + 0, + Integer.MAX_VALUE, + 5, + TimeUnit.SECONDS, + new SynchronousQueue<>(), + new SolrNamedThreadFactory("testExecutor")); + try { + for (int j = 0; j < numReplicas; j++) { + final int freezeJ = j; + executor.execute( + () -> { + Unload unloadCmd = new Unload(true); + unloadCmd.setCoreName("multiunload" + freezeJ); + try { + adminClient.request(unloadCmd); + } catch (SolrServerException | IOException e) { + throw new RuntimeException(e); + } + }); + Thread.sleep(random().nextInt(50)); + } + } finally { + ExecutorUtil.shutdownAndAwaitTermination(executor); + } + } + } +} diff --git a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java index 82b60068d83..aca50b7bd3a 100644 --- a/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java +++ b/solr/core/src/test/org/apache/solr/cloud/VMParamsZkACLAndCredentialsProvidersTest.java @@ -16,5 +16,552 @@ */ package org.apache.solr.cloud; -public class VMParamsZkACLAndCredentialsProvidersTest - extends AbstractDigestZkACLAndCredentialsProvidersTestBase {} +import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME; +import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME; +import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE; + +import java.io.FileWriter; +import java.io.IOException; +import java.lang.invoke.MethodHandles; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.util.Collections; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.cloud.DigestZkACLProvider; +import org.apache.solr.common.cloud.DigestZkCredentialsProvider; +import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; +import org.apache.solr.common.cloud.SolrZkClient; +import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector; +import org.apache.solr.common.cloud.ZkCredentialsInjector; +import org.apache.zookeeper.CreateMode; +import org.apache.zookeeper.KeeperException.NoAuthException; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class VMParamsZkACLAndCredentialsProvidersTest extends SolrTestCaseJ4 { + + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); + + private static final Charset DATA_ENCODING = StandardCharsets.UTF_8; + + private static final String ALL_USERNAME = "connectAndAllACLUsername"; + private static final String ALL_PASSWORD = "connectAndAllACLPassword"; + private static final String READONLY_USERNAME = "readonlyACLUsername"; + private static final String READONLY_PASSWORD = "readonlyACLPassword"; + + protected ZkTestServer zkServer; + + protected Path zkDir; + + @BeforeClass + public static void beforeClass() { + System.setProperty("solrcloud.skip.autorecovery", "true"); + } + + @AfterClass + public static void afterClass() { + System.clearProperty("solrcloud.skip.autorecovery"); + } + + @Override + public void setUp() throws Exception { + // TODO: Does all of this setup need to happen for each test case, or can it be done once for + // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full + // startup and teardown of a new ZkTestServer in each cycle? + super.setUp(); + if (log.isInfoEnabled()) { + log.info("####SETUP_START {}", getTestName()); + } + createTempDir(); + + zkDir = createTempDir().resolve("zookeeper/server1/data"); + log.info("ZooKeeper dataDir:{}", zkDir); + setSecuritySystemProperties(); + zkServer = new ZkTestServer(zkDir); + zkServer.run(false); + + System.setProperty("zkHost", zkServer.getZkAddress()); + + setDigestZkSystemProps(); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); + + SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkHost()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + + zkClient.makePath("/solr", false, true); + zkClient.close(); + + zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + zkClient.create( + "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath( + "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + + zkClient.create( + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, + "content".getBytes(DATA_ENCODING), + CreateMode.PERSISTENT, + false); + zkClient.close(); + + clearSecuritySystemProperties(); + + zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build(); + // Currently, no credentials on ZK connection, because those same VM-params are used for adding + // ACLs, and here we want + // no (or completely open) ACLs added. Therefore, hack your way into being authorized for + // creating anyway + zkClient + .getCuratorFramework() + .getZookeeperClient() + .getZooKeeper() + .addAuthInfo( + "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8)); + zkClient.create( + "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); + zkClient.makePath( + "/unprotectedMakePathNode", + "content".getBytes(DATA_ENCODING), + CreateMode.PERSISTENT, + false); + zkClient.close(); + + setDigestZkSystemProps(); + if (log.isInfoEnabled()) { + log.info("####SETUP_END {}", getTestName()); + } + } + + private void setDigestZkSystemProps() { + System.setProperty( + SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + DigestZkCredentialsProvider.class.getName()); + System.setProperty( + SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); + } + + @Override + public void tearDown() throws Exception { + zkServer.shutdown(); + + clearSecuritySystemProperties(); + + super.tearDown(); + } + + @Test + public void testNoCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class), + new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class)); + + testInjectors( + testZkCredentialsInjectors, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false); + } + + @Test + public void testWrongCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class), + new TestZkCredentialsInjector( + VMParamsZkCredentialsInjector.class, + List.of( + DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), + List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong"))); + + testInjectors( + testZkCredentialsInjectors, + false, + false, + false, + false, + false, + false, + false, + false, + false, + false); + } + + @Test + public void testAllCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class), + new TestZkCredentialsInjector( + VMParamsZkCredentialsInjector.class, + List.of( + DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), + List.of(ALL_USERNAME, ALL_PASSWORD))); + + testInjectors( + testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true); + } + + @Test + public void testReadonlyCredentials() throws Exception { + List testZkCredentialsInjectors = + List.of( + new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class), + new TestZkCredentialsInjector( + VMParamsZkCredentialsInjector.class, + List.of( + DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), + List.of(READONLY_USERNAME, READONLY_PASSWORD))); + testInjectors( + testZkCredentialsInjectors, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false); + } + + protected void testInjectors( + List testZkCredentialsInjectors, + boolean getData, + boolean list, + boolean create, + boolean setData, + boolean delete, + boolean secureGet, + boolean secureList, + boolean secureCreate, + boolean secureSet, + boolean secureDelete) + throws Exception { + for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) { + tearDown(); + setUp(); + testZkCredentialsInjector.setSystemProps(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + doTest( + zkClient, + getData, + list, + create, + setData, + delete, + secureGet, + secureList, + secureCreate, + secureSet, + secureDelete); + } + } + } + + @Test + public void testRepairACL() throws Exception { + clearSecuritySystemProperties(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + // Currently, no credentials on ZK connection, because those same VM-params are used for + // adding ACLs, and here we want + // no (or completely open) ACLs added. Therefore, hack your way into being authorized for + // creating anyway + zkClient + .getCuratorFramework() + .getZookeeperClient() + .getZooKeeper() + .addAuthInfo( + "digest", + ("connectAndAllACLUsername:connectAndAllACLPassword") + .getBytes(StandardCharsets.UTF_8)); + + zkClient.create( + "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false); + assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); + } + + setSecuritySystemProperties(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + ZkController.createClusterZkNodes(zkClient); + assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); + } + + useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class); + // useReadonlyCredentials(); + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + NoAuthException e = + assertThrows( + NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false)); + assertEquals("/solr/security.json", e.getPath()); + } + } + + private void useZkCredentialsInjector(Class zkCredentialsInjectorClass) { + clearSecuritySystemProperties(); + setDigestZkSystemProps(); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + zkCredentialsInjectorClass.getName()); + } + + private void setSecuritySystemProperties() { + System.setProperty( + SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, + DigestZkCredentialsProvider.class.getName()); + System.setProperty( + SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); + } + + private void clearSecuritySystemProperties() { + System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); + System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME); + } + + public static void doTest( + SolrZkClient zkClient, + boolean getData, + boolean list, + boolean create, + boolean setData, + boolean delete, + boolean secureGet, + boolean secureList, + boolean secureCreate, + boolean secureSet, + boolean secureDelete) + throws Exception { + doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete); + doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete); + doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete); + doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete); + doTest( + zkClient, + SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, + secureGet, + secureList, + secureCreate, + secureSet, + secureDelete); + } + + protected static void doTest( + SolrZkClient zkClient, + String path, + boolean getData, + boolean list, + boolean create, + boolean setData, + boolean delete) + throws Exception { + doTest(getData, () -> zkClient.getData(path, null, null, false)); + doTest(list, () -> zkClient.getChildren(path, null, false)); + + doTest( + create, + () -> { + zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false); + zkClient.delete(path + "/subnode", -1, false); + }); + doTest( + create, + () -> { + zkClient.makePath(path + "/subnode/subsubnode", false); + zkClient.delete(path + "/subnode/subsubnode", -1, false); + zkClient.delete(path + "/subnode", -1, false); + }); + + doTest(setData, () -> zkClient.setData(path, (byte[]) null, false)); + + // Actually about the ACLs on /solr, but that is protected + doTest(delete, () -> zkClient.delete(path, -1, false)); + } + + interface ExceptingRunnable { + void run() throws Exception; + } + + private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception { + if (shouldSucceed) { + action.run(); + } else { + expectThrows(NoAuthException.class, action::run); + } + } + + @Test + public void testVMParamsAllCredentialsFromFile() throws Exception { + useVMParamsAllCredentialsFromFile(); + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + doTest(zkClient, true, true, true, true, true, true, true, true, true, true); + } + } + + @Test + public void testVMParamsReadonlyCredentialsFromFile() throws Exception { + useVMParamsReadonlyCredentialsFromFile(); + + try (SolrZkClient zkClient = + new SolrZkClient.Builder() + .withUrl(zkServer.getZkAddress()) + .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) + .build()) { + doTest(zkClient, true, true, false, false, false, false, false, false, false, false); + } + } + + private void useVMParamsAllCredentialsFromFile() throws IOException { + useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword"); + } + + private void useVMParamsReadonlyCredentialsFromFile() throws IOException { + useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword"); + } + + private void useVMParamsCredentialsFromFile(String username, String password) throws IOException { + Properties props = new Properties(); + props.setProperty( + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username); + props.setProperty( + VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password); + String credsFile = saveCredentialsFile(props); + + useZkCredentialsInjector(VMParamsZkCredentialsInjector.class); + System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile); + } + + private String saveCredentialsFile(Properties props) throws IOException { + Path tmp = createTempFile("zk-creds", "properties"); + try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) { + props.store(w, "test"); + } + return tmp.toAbsolutePath().toString(); + } + + public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return Collections.emptyList(); + } + } + + public static class AllAndReadonlyCredentialZkCredentialsInjector + implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of( + new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL), + new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ)); + } + } + + public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of( + new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL)); + } + } + + public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL)); + } + } + + public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector { + @Override + public List getZkCredentials() { + return List.of( + new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL)); + } + } + + class TestZkCredentialsInjector { + private final Class zkCredentialsInjectorClass; + private final List systemPropsKeys; + private final List systemPropsValues; + + public TestZkCredentialsInjector(Class zkCredentialsInjectorClass) { + this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList()); + } + + public TestZkCredentialsInjector( + Class zkCredentialsInjectorClass, + List systemPropsKeys, + List systemPropsValues) { + this.zkCredentialsInjectorClass = zkCredentialsInjectorClass; + this.systemPropsKeys = systemPropsKeys; + this.systemPropsValues = systemPropsValues; + } + + private void setSystemProps() { + clearSecuritySystemProperties(); + setDigestZkSystemProps(); + System.setProperty( + SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, + zkCredentialsInjectorClass.getName()); + int i = 0; + for (String key : systemPropsKeys) { + System.setProperty(key, systemPropsValues.get(i++)); + } + } + } +} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java deleted file mode 100644 index e758dbf26a7..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZk2TestBase.java +++ /dev/null @@ -1,487 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import static org.apache.solr.cloud.SolrCloudTestCase.configurePrsDefault; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.mockfile.FilterPath; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkNodeProps; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.handler.BackupStatusChecker; -import org.apache.solr.handler.ReplicationHandler; -import org.junit.BeforeClass; -import org.junit.Test; - -/** - * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. - */ -// Backups do checksum validation against a footer value not present in 'SimpleText' -@LuceneTestCase.SuppressCodecs({"SimpleText"}) -public abstract class AbstractBasicDistributedZk2TestBase extends AbstractFullDistribZkTestBase { - private static final String SHARD2 = "shard2"; - private static final String SHARD1 = "shard1"; - private static final String ONE_NODE_COLLECTION = "onenodecollection"; - private final boolean onlyLeaderIndexes = random().nextBoolean(); - - public AbstractBasicDistributedZk2TestBase() { - super(); - // we need DVs on point fields to compute stats & facets - if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) - System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); - - sliceCount = 2; - } - - @Override - protected boolean useTlogReplicas() { - return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug - // and it's TestInjection use - } - - @BeforeClass - public static void _setPrsDefault() { - configurePrsDefault(); - } - - @Test - @ShardsFixed(num = 4) - public void test() throws Exception { - boolean testFinished = false; - try { - handle.clear(); - handle.put("timestamp", SKIPVAL); - - testNodeWithoutCollectionForwarding(); - - indexr( - id, - 1, - i1, - 100, - tlong, - 100, - t1, - "now is the time for all good men", - "foo_f", - 1.414f, - "foo_b", - "true", - "foo_d", - 1.414d); - - commit(); - - // make sure we are in a steady state... - waitForRecoveriesToFinish(false); - - assertDocCounts(false); - - indexAbunchOfDocs(); - - // check again - waitForRecoveriesToFinish(false); - - commit(); - - assertDocCounts(VERBOSE); - checkQueries(); - - assertDocCounts(VERBOSE); - - query("q", "*:*", "sort", "n_tl1 desc"); - - bringDownShardIndexSomeDocsAndRecover(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - // test adding another replica to a shard - it should do a - // recovery/replication to pick up the index from the leader - addNewReplica(); - - long docId = testUpdateAndDelete(); - - // index a bad doc... - expectThrows(SolrException.class, () -> indexr(t1, "a doc with no id")); - - // TODO: bring this to its own method? - // try indexing to a leader that has no replicas up - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - ZkNodeProps leaderProps = zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, SHARD2); - - String nodeName = leaderProps.getStr(ZkStateReader.NODE_NAME_PROP); - chaosMonkey.stopShardExcept(SHARD2, nodeName); - - SolrClient client = getClient(nodeName); - - index_specific(client, "id", docId + 1, t1, "what happens here?"); - - // expire a session... - CloudJettyRunner cloudJetty = shardToJetty.get(SHARD1).get(0); - chaosMonkey.expireSession(cloudJetty.jetty); - // Wait until the jetty is reconnected, otherwise the following index command could fail - cloudJetty - .jetty - .getCoreContainer() - .getZkController() - .getZkClient() - .getCuratorFramework() - .blockUntilConnected(50, TimeUnit.MILLISECONDS); - - indexr("id", docId + 1, t1, "slip this doc in"); - - waitForRecoveriesToFinish(false); - - checkShardConsistency(SHARD1); - checkShardConsistency(SHARD2); - - testFinished = true; - } finally { - if (!testFinished) { - printLayoutOnTearDown = true; - } - } - } - - private void testNodeWithoutCollectionForwarding() throws Exception { - assertEquals( - 0, - CollectionAdminRequest.createCollection(ONE_NODE_COLLECTION, "conf1", 1, 1) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard(ONE_NODE_COLLECTION, "shard1") - .setCoreName(ONE_NODE_COLLECTION + "core") - .process(cloudClient) - .isSuccess()); - - waitForCollection(ZkStateReader.from(cloudClient), ONE_NODE_COLLECTION, 1); - waitForRecoveriesToFinish(ONE_NODE_COLLECTION, ZkStateReader.from(cloudClient), false); - - ZkStateReader.from(cloudClient).getLeaderRetry(ONE_NODE_COLLECTION, SHARD1, 30000); - - int docs = 2; - for (JettySolrRunner jetty : jettys) { - final String clientUrl = getBaseUrl(jetty); - addAndQueryDocs(clientUrl, docs); - docs += 2; - } - } - - // 2 docs added every call - private void addAndQueryDocs(final String baseUrl, int docs) throws Exception { - - SolrQuery query = new SolrQuery("*:*"); - - try (SolrClient client = getHttpSolrClient(baseUrl, "onenodecollection")) { - // add a doc - client.add(sdoc("id", docs)); - client.commit(); - - QueryResponse results = client.query(query); - assertEquals(docs - 1, results.getResults().getNumFound()); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", docs + 1); - client.add(doc); - client.commit(); - - query = new SolrQuery("*:*"); - query.set("rows", 0); - results = client.query(query); - assertEquals(docs, results.getResults().getNumFound()); - } - } - - private long testUpdateAndDelete() throws Exception { - long docId = 99999999L; - indexr("id", docId, t1, "originalcontent"); - - commit(); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.add("q", t1 + ":originalcontent"); - QueryResponse results = clients.get(0).query(params); - assertEquals(1, results.getResults().getNumFound()); - - // update doc - indexr("id", docId, t1, "updatedcontent"); - - commit(); - - results = clients.get(0).query(params); - assertEquals(0, results.getResults().getNumFound()); - - params.set("q", t1 + ":updatedcontent"); - - results = clients.get(0).query(params); - assertEquals(1, results.getResults().getNumFound()); - - UpdateRequest uReq = new UpdateRequest(); - // uReq.setParam(UpdateParams.UPDATE_CHAIN, DISTRIB_UPDATE_CHAIN); - uReq.deleteById(Long.toString(docId)).process(clients.get(0)); - - commit(); - - results = clients.get(0).query(params); - assertEquals(0, results.getResults().getNumFound()); - return docId; - } - - private void bringDownShardIndexSomeDocsAndRecover() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.set("distrib", false); - - commit(); - - long deadShardCount = - shardToJetty.get(SHARD2).get(0).client.solrClient.query(query).getResults().getNumFound(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - int oldLiveNodes = - ZkStateReader.from(cloudClient) - .getZkClient() - .getChildren(ZkStateReader.LIVE_NODES_ZKNODE, null, true) - .size(); - - assertEquals(5, oldLiveNodes); - - // kill a shard - CloudJettyRunner deadShard = chaosMonkey.stopShard(SHARD1, 0); - - // ensure shard is dead - expectThrows( - SolrServerException.class, - "This server should be down and this update should have failed", - () -> index_specific(deadShard.client.solrClient, id, 999, i1, 107, t1, "specific doc!")); - - commit(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - // long cloudClientDocs = cloudClient.query(new - // SolrQuery("*:*")).getResults().getNumFound(); - // System.out.println("clouddocs:" + cloudClientDocs); - - // try to index to a living shard at shard2 - - long numFound1 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - ZkStateReader.from(cloudClient).getLeaderRetry(DEFAULT_COLLECTION, SHARD1, 60000); - - try { - index_specific( - shardToJetty.get(SHARD1).get(1).client.solrClient, - id, - 1000, - i1, - 108, - t1, - "specific doc!"); - } catch (Exception e) { - // wait and try again - Thread.sleep(4000); - index_specific( - shardToJetty.get(SHARD1).get(1).client.solrClient, - id, - 1000, - i1, - 108, - t1, - "specific doc!"); - } - - commit(); - - checkShardConsistency(true, false); - - query("q", "*:*", "sort", "n_tl1 desc"); - - long numFound2 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - assertEquals(numFound1 + 1, numFound2); - - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", 1001); - - controlClient.add(doc); - - // try adding a doc with CloudSolrServer - UpdateRequest ureq = new UpdateRequest(); - ureq.add(doc); - // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); - - try { - ureq.process(cloudClient); - } catch (SolrServerException e) { - // try again - Thread.sleep(3500); - ureq.process(cloudClient); - } - - commit(); - - query("q", "*:*", "sort", "n_tl1 desc"); - - long numFound3 = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - // lets just check that the one doc since last commit made it in... - assertEquals(numFound2 + 1, numFound3); - - // test debugging - testDebugQueries(); - - if (VERBOSE) { - System.err.println(controlClient.query(new SolrQuery("*:*")).getResults().getNumFound()); - - for (SolrClient client : clients) { - try { - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - System.err.println(client.query(q).getResults().getNumFound()); - } catch (Exception e) { - - } - } - } - // TODO: This test currently fails because debug info is obtained only - // on shards with matches. - // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); - - // this should trigger a recovery phase on deadShard - deadShard.jetty.start(); - - // make sure we have published we are recovering - Thread.sleep(1500); - - waitForRecoveriesToFinish(false); - - deadShardCount = - shardToJetty.get(SHARD1).get(0).client.solrClient.query(query).getResults().getNumFound(); - // if we properly recovered, we should now have the couple missing docs that - // came in while shard was down - checkShardConsistency(true, false); - - // recover over 100 docs so we do more than just peer sync (replicate recovery) - chaosMonkey.stopJetty(deadShard); - - for (int i = 0; i < 226; i++) { - doc = new SolrInputDocument(); - doc.addField("id", 2000 + i); - controlClient.add(doc); - ureq = new UpdateRequest(); - ureq.add(doc); - // ureq.setParam("update.chain", DISTRIB_UPDATE_CHAIN); - ureq.process(cloudClient); - } - commit(); - - Thread.sleep(1500); - - deadShard.jetty.start(); - - // make sure we have published we are recovering - Thread.sleep(1500); - - waitForThingsToLevelOut(1, TimeUnit.MINUTES); - - Thread.sleep(500); - - waitForRecoveriesToFinish(false); - - checkShardConsistency(true, false); - - // try a backup command - try (final SolrClient client = - getHttpSolrClient((String) shardToJetty.get(SHARD2).get(0).info.get("base_url"))) { - final String backupName = "the_backup"; - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("qt", ReplicationHandler.PATH); - params.set("command", "backup"); - params.set("name", backupName); - final Path location = FilterPath.unwrap(createTempDir()).toRealPath(); - // Allow non-standard location outside SOLR_HOME - jettys.forEach(j -> j.getCoreContainer().getAllowPaths().add(location)); - params.set("location", location.toString()); - - QueryRequest request = new QueryRequest(params); - client.request(request, DEFAULT_TEST_COLLECTION_NAME); - - final BackupStatusChecker backupStatus = - new BackupStatusChecker(client, "/" + DEFAULT_TEST_COLLECTION_NAME + "/replication"); - final String backupDirName = backupStatus.waitForBackupSuccess(backupName, 30); - assertTrue( - "Backup dir does not exist: " + backupDirName, - Files.exists(location.resolve(backupDirName))); - } - } - - private void addNewReplica() throws Exception { - - waitForRecoveriesToFinish(false); - - // new server should be part of first shard - // how many docs are on the new shard? - for (CloudJettyRunner cjetty : shardToJetty.get(SHARD1)) { - if (VERBOSE) - System.err.println( - "shard1 total:" - + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - for (CloudJettyRunner cjetty : shardToJetty.get(SHARD2)) { - if (VERBOSE) - System.err.println( - "shard2 total:" - + cjetty.client.solrClient.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - - checkShardConsistency(SHARD1); - checkShardConsistency(SHARD2); - - assertDocCounts(VERBOSE); - } - - private void testDebugQueries() throws Exception { - handle.put("explain", SKIPVAL); - handle.put("debug", UNORDERED); - handle.put("time", SKIPVAL); - handle.put("track", SKIP); - query("q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true"); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS); - query("q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java deleted file mode 100644 index 24d0689c5a6..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractBasicDistributedZkTestBase.java +++ /dev/null @@ -1,1737 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.io.InputStream; -import java.lang.invoke.MethodHandles; -import java.nio.charset.StandardCharsets; -import java.util.ArrayList; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.Callable; -import java.util.concurrent.CompletionService; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorCompletionService; -import java.util.concurrent.Future; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; -import java.util.concurrent.atomic.AtomicReference; -import org.apache.lucene.util.IOUtils; -import org.apache.solr.JSONTestUtil; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.AbstractUpdateRequest; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest.Create; -import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.QueryRequest; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.request.StreamingUpdateRequest; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.client.solrj.response.CollectionAdminResponse; -import org.apache.solr.client.solrj.response.FacetField; -import org.apache.solr.client.solrj.response.Group; -import org.apache.solr.client.solrj.response.GroupCommand; -import org.apache.solr.client.solrj.response.GroupResponse; -import org.apache.solr.client.solrj.response.InputStreamResponseParser; -import org.apache.solr.client.solrj.response.QueryResponse; -import org.apache.solr.client.solrj.response.UpdateResponse; -import org.apache.solr.cloud.api.collections.CollectionHandlingUtils; -import org.apache.solr.common.SolrDocument; -import org.apache.solr.common.SolrDocumentList; -import org.apache.solr.common.SolrException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ClusterState; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.CommonParams; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.params.SolrParams; -import org.apache.solr.common.params.UpdateParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.apache.solr.util.TestInjection.Hook; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. - */ -public abstract class AbstractBasicDistributedZkTestBase extends AbstractFullDistribZkTestBase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final String DEFAULT_COLLECTION = "collection1"; - - private final boolean onlyLeaderIndexes = random().nextBoolean(); - - String tsort = "t_sortable"; - - private Map> otherCollectionClients = new HashMap<>(); - - private String oneInstanceCollection = "oneInstanceCollection"; - private String oneInstanceCollection2 = "oneInstanceCollection2"; - - private AtomicInteger nodeCounter = new AtomicInteger(); - - CompletionService completionService; - Set> pending; - - private static Hook newSearcherHook = - new Hook() { - volatile CountDownLatch latch; - AtomicReference collection = new AtomicReference<>(); - - @Override - public void newSearcher(String collectionName) { - String c = collection.get(); - if (c != null && c.equals(collectionName)) { - log.info("Hook detected newSearcher"); - try { - latch.countDown(); - } catch (NullPointerException e) { - - } - } - } - - @Override - public void waitForSearcher( - String collection, int cnt, int timeoutms, boolean failOnTimeout) - throws InterruptedException { - latch = new CountDownLatch(cnt); - this.collection.set(collection); - boolean timeout = !latch.await(timeoutms, TimeUnit.MILLISECONDS); - if (timeout && failOnTimeout) { - fail("timed out waiting for new searcher event " + latch.getCount()); - } - } - }; - - public AbstractBasicDistributedZkTestBase() { - // we need DVs on point fields to compute stats & facets - if (Boolean.getBoolean(NUMERIC_POINTS_SYSPROP)) - System.setProperty(NUMERIC_DOCVALUES_SYSPROP, "true"); - - sliceCount = 2; - completionService = new ExecutorCompletionService<>(executor); - pending = new HashSet<>(); - } - - @BeforeClass - public static void beforeBDZKTClass() { - TestInjection.newSearcherHook(newSearcherHook); - } - - @Override - protected boolean useTlogReplicas() { - // TODO: tlog replicas makes commits take way to long due to what is likely a bug and it's - // TestInjection use - return false; - } - - @Override - protected void setDistributedParams(ModifiableSolrParams params) { - - if (r.nextBoolean()) { - // don't set shards, let that be figured out from the cloud state - } else { - // use shard ids rather than physical locations - StringBuilder sb = new StringBuilder(); - for (int i = 0; i < getShardCount(); i++) { - if (i > 0) sb.append(','); - sb.append("shard" + (i + 3)); - } - params.set("shards", sb.toString()); - } - } - - @Test - @ShardsFixed(num = 4) - protected void test() throws Exception { - // setLoggingLevel(null); - - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - // make sure we have leaders for each shard - for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); - } // make sure we again have leaders for each shard - - waitForRecoveriesToFinish(false); - - handle.clear(); - handle.put("timestamp", SKIPVAL); - - del("*:*"); - queryAndCompareShards(params("q", "*:*", "distrib", "false", "sanity_check", "is_empty")); - - // ask every individual replica of every shard to update+commit the same doc id - // with an incrementing counter on each update+commit - int foo_i_counter = 0; - for (SolrClient client : clients) { - foo_i_counter++; - indexDoc( - client, - params("commit", "true"), // SOLR-4923 - sdoc(id, 1, i1, 100, tlong, 100, "foo_i", foo_i_counter)); - // after every update+commit, check all the shards consistency - queryAndCompareShards( - params("q", "id:1", "distrib", "false", "sanity_check", "non_distrib_id_1_lookup")); - queryAndCompareShards( - params( - "q", "id:1", - "sanity_check", "distrib_id_1_lookup")); - } - - indexr( - id, - 1, - i1, - 100, - tlong, - 100, - t1, - "now is the time for all good men", - "foo_f", - 1.414f, - "foo_b", - "true", - "foo_d", - 1.414d, - tsort, - "now is the time for all good men"); - indexr( - id, - 2, - i1, - 50, - tlong, - 50, - t1, - "to come to the aid of their country.", - tsort, - "to come to the aid of their country."); - indexr(id, 3, i1, 2, tlong, 2, t1, "how now brown cow", tsort, "how now brown cow"); - indexr( - id, - 4, - i1, - -100, - tlong, - 101, - t1, - "the quick fox jumped over the lazy dog", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 5, - i1, - 500, - tlong, - 500, - t1, - "the quick fox jumped way over the lazy dog", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 6, - i1, - -600, - tlong, - 600, - t1, - "humpty dumpy sat on a wall", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 7, - i1, - 123, - tlong, - 123, - t1, - "humpty dumpy had a great fall", - tsort, - "the quick fox jumped over the lazy dog"); - indexr( - id, - 8, - i1, - 876, - tlong, - 876, - t1, - "all the kings horses and all the kings men", - tsort, - "all the kings horses and all the kings men"); - indexr( - id, - 9, - i1, - 7, - tlong, - 7, - t1, - "couldn't put humpty together again", - tsort, - "the quick fox jumped over the lazy dog"); - indexr(id, 10, i1, 4321, tlong, 4321, t1, "this too shall pass", tsort, "this too shall pass"); - indexr( - id, - 11, - i1, - -987, - tlong, - 987, - t1, - "An eye for eye only ends up making the whole world blind.", - tsort, - "An eye for eye only ends up making the whole world blind."); - indexr( - id, - 12, - i1, - 379, - tlong, - 379, - t1, - "Great works are performed, not by strength, but by perseverance.", - tsort, - "Great works are performed, not by strength, but by perseverance."); - indexr( - id, - 13, - i1, - 232, - tlong, - 232, - t1, - "no eggs on wall, lesson learned", - oddField, - "odd man out", - tsort, - "no eggs on wall, lesson learned"); - - indexr( - id, - 14, - "SubjectTerms_mfacet", - new String[] {"mathematical models", "mathematical analysis"}); - indexr(id, 15, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); - indexr(id, 16, "SubjectTerms_mfacet", new String[] {"test 1", "test 2", "test3"}); - String[] vals = new String[100]; - for (int i = 0; i < 100; i++) { - vals[i] = "test " + i; - } - indexr(id, 17, "SubjectTerms_mfacet", vals); - - for (int i = 100; i < 150; i++) { - indexr(id, i); - } - - commit(); - - testTokenizedGrouping(); - testSortableTextFaceting(); - testSortableTextSorting(); - testSortableTextGrouping(); - - queryAndCompareShards( - params( - "q", "*:*", - "sort", "id desc", - "distrib", "false", - "sanity_check", "is_empty")); - - // random value sort - for (String f : fieldNames) { - query(false, new String[] {"q", "*:*", "sort", f + " desc"}); - query(false, new String[] {"q", "*:*", "sort", f + " asc"}); - } - - // these queries should be exactly ordered and scores should exactly match - query(false, new String[] {"q", "*:*", "sort", i1 + " desc"}); - query(false, new String[] {"q", "*:*", "sort", i1 + " asc"}); - query(false, new String[] {"q", "*:*", "sort", i1 + " desc", "fl", "*,score"}); - query(false, new String[] {"q", "*:*", "sort", "n_tl1 asc", "fl", "*,score"}); - query(false, new String[] {"q", "*:*", "sort", "n_tl1 desc"}); - handle.put("maxScore", SKIPVAL); - query( - false, - new String[] {"q", "{!func}" + i1}); // does not expect maxScore. So if it comes ,ignore it. - // JavaBinCodec.writeSolrDocumentList() - // is agnostic of request params. - handle.remove("maxScore"); - query( - false, - new String[] { - "q", "{!func}" + i1, "fl", "*,score" - }); // even scores should match exactly here - - handle.put("highlighting", UNORDERED); - handle.put("response", UNORDERED); - - handle.put("maxScore", SKIPVAL); - query(false, new String[] {"q", "quick"}); - query(false, new String[] {"q", "all", "fl", "id", "start", "0"}); - query( - false, - new String[] {"q", "all", "fl", "foofoofoo", "start", "0"}); // no fields in returned docs - query(false, new String[] {"q", "all", "fl", "id", "start", "100"}); - - handle.put("score", SKIPVAL); - query(false, new String[] {"q", "quick", "fl", "*,score"}); - query(false, new String[] {"q", "all", "fl", "*,score", "start", "1"}); - query(false, new String[] {"q", "all", "fl", "*,score", "start", "100"}); - - query( - false, - new String[] { - "q", "now their fox sat had put", "fl", "*,score", "hl", "true", "hl.fl", t1 - }); - - query( - false, - new String[] { - "q", "now their fox sat had put", "fl", "foofoofoo", "hl", "true", "hl.fl", t1 - }); - - query(false, new String[] {"q", "matchesnothing", "fl", "*,score"}); - - query(false, new Object[] {"q", "*:*", "rows", 100, "facet", "true", "facet.field", t1}); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "count" - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "count", - "facet.mincount", - 2 - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "index" - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - -1, - "facet.sort", - "index", - "facet.mincount", - 2 - }); - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.limit", 1 - }); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.query", - "quick", - "facet.query", - "all", - "facet.query", - "*:*" - }); - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.offset", 1 - }); - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", t1, "facet.mincount", 2 - }); - - // test faceting multiple things at once - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.query", - "quick", - "facet.query", - "all", - "facet.query", - "*:*", - "facet.field", - t1 - }); - - // test filter tagging, facet exclusion, and naming (multi-select facet support) - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 100, - "facet", - "true", - "facet.query", - "{!key=myquick}quick", - "facet.query", - "{!key=myall ex=a}all", - "facet.query", - "*:*", - "facet.field", - "{!key=mykey ex=a}" + t1, - "facet.field", - "{!key=other ex=b}" + t1, - "facet.field", - "{!key=again ex=a,b}" + t1, - "facet.field", - t1, - "fq", - "{!tag=a}id_i1:[1 TO 7]", - "fq", - "{!tag=b}id_i1:[3 TO 9]" - }); - query( - false, - new Object[] { - "q", - "*:*", - "facet", - "true", - "facet.field", - "{!ex=t1}SubjectTerms_mfacet", - "fq", - "{!tag=t1}SubjectTerms_mfacet:(test 1)", - "facet.limit", - "10", - "facet.mincount", - "1" - }); - - // test field that is valid in schema but missing in all shards - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", missingField, "facet.mincount", 2 - }); - // test field that is valid in schema and missing in some shards - query( - false, - new Object[] { - "q", "*:*", "rows", 100, "facet", "true", "facet.field", oddField, "facet.mincount", 2 - }); - - query( - false, new Object[] {"q", "*:*", "sort", i1 + " desc", "stats", "true", "stats.field", i1}); - - /* TODO: the failure may come back in "exception" - try { - // test error produced for field that is invalid for schema - query("q","*:*", "rows",100, "facet","true", "facet.field",invalidField, "facet.mincount",2); - TestCase.fail("SolrServerException expected for invalid field that is not in schema"); - } catch (SolrServerException ex) { - // expected - } - */ - - // Try to get better coverage for refinement queries by turning off over requesting. - // This makes it much more likely that we may not get the top facet values and hence - // we turn of that checking. - handle.put("facet_fields", SKIPVAL); - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 0, - "facet", - "true", - "facet.field", - t1, - "facet.limit", - 5, - "facet.shard.limit", - 5 - }); - // check a complex key name - query( - false, - new Object[] { - "q", - "*:*", - "rows", - 0, - "facet", - "true", - "facet.field", - "{!key='a b/c \\' \\} foo'}" + t1, - "facet.limit", - 5, - "facet.shard.limit", - 5 - }); - handle.remove("facet_fields"); - - // index the same document to two servers and make sure things - // don't blow up. - if (clients.size() >= 2) { - index(id, 100, i1, 107, t1, "oh no, a duplicate!"); - for (int i = 0; i < clients.size(); i++) { - index_specific(i, id, 100, i1, 107, t1, "oh no, a duplicate!"); - } - commit(); - query(false, new Object[] {"q", "duplicate", "hl", "true", "hl.fl", t1}); - query(false, new Object[] {"q", "fox duplicate horses", "hl", "true", "hl.fl", t1}); - query(false, new Object[] {"q", "*:*", "rows", 100}); - } - - // test debugging - handle.put("explain", SKIPVAL); - handle.put("debug", UNORDERED); - handle.put("time", SKIPVAL); - handle.put("track", SKIP); - query( - false, - new Object[] { - "q", "now their fox sat had put", "fl", "*,score", CommonParams.DEBUG_QUERY, "true" - }); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG_QUERY, "true"}); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.TIMING}); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.RESULTS}); - query(false, new Object[] {"q", "id_i1:[1 TO 5]", CommonParams.DEBUG, CommonParams.QUERY}); - - // try add commitWithin - long before = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - for (SolrClient client : clients) { - assertEquals( - "unexpected pre-commitWithin document count on node: " - + ((HttpSolrClient) client).getBaseURL(), - before, - client.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("commitWithin", 10); - add(cloudClient, params, List.of(getDoc("id", 300), getDoc("id", 301))); - - newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); - - ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); - DocCollection dColl = clusterState.getCollection(DEFAULT_COLLECTION); - - assertSliceCounts("should have found 2 docs, 300 and 301", before + 2, dColl); - - // try deleteById commitWithin - UpdateRequest deleteByIdReq = new UpdateRequest(); - deleteByIdReq.deleteById("300"); - deleteByIdReq.setCommitWithin(10); - deleteByIdReq.process(cloudClient); - - newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); - - assertSliceCounts("deleteById commitWithin did not work", before + 1, dColl); - - // try deleteByQuery commitWithin - UpdateRequest deleteByQueryReq = new UpdateRequest(); - deleteByQueryReq.deleteByQuery("id:301"); - deleteByQueryReq.setCommitWithin(10); - deleteByQueryReq.process(cloudClient); - - newSearcherHook.waitForSearcher(DEFAULT_COLLECTION, 2, 20000, false); - - assertSliceCounts("deleteByQuery commitWithin did not work", before, dColl); - - // TODO: This test currently fails because debug info is obtained only - // on shards with matches. - // query("q","matchesnothing","fl","*,score", "debugQuery", "true"); - - // would be better if these where all separate tests - but much, much - // slower - doOptimisticLockingAndUpdating(); - testShardParamVariations(); - testMultipleCollections(); - testANewCollectionInOneInstance(); - testSearchByCollectionName(); - testUpdateByCollectionName(); - testANewCollectionInOneInstanceWithManualShardAssignement(); - testNumberOfCommitsWithCommitAfterAdd(); - - testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-explicit"); - testUpdateProcessorsRunOnlyOnce("distrib-dup-test-chain-implicit"); - - testStopAndStartCoresInOneInstance(); - } - - private void testSortableTextFaceting() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.addFacetField(tsort); - query.setFacetMissing(false); - QueryResponse resp = queryRandomShard(query); - List ffs = resp.getFacetFields(); - for (FacetField ff : ffs) { - if (ff.getName().equals(tsort) == false) continue; - for (FacetField.Count count : ff.getValues()) { - long num = count.getCount(); - switch (count.getName()) { - case "all the kings horses and all the kings men": - case "An eye for eye only ends up making the whole world blind.": - case "Great works are performed, not by strength, but by perseverance.": - case "how now brown cow": - case "no eggs on wall, lesson learned": - case "now is the time for all good men": - case "this too shall pass": - case "to come to the aid of their country.": - assertEquals("Should have exactly one facet count for field " + ff.getName(), 1, num); - break; - case "the quick fox jumped over the lazy dog": - assertEquals("Should have 5 docs for the lazy dog", 5, num); - break; - default: - fail("No case for facet '" + ff.getName() + "'"); - } - } - } - } - - private void testSortableTextSorting() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.addSort(tsort, SolrQuery.ORDER.desc); - query.addField("*"); - query.addField("eoe_sortable"); - query.addField(tsort); - QueryResponse resp = queryRandomShard(query); - - SolrDocumentList docs = resp.getResults(); - - String title = docs.get(0).getFieldValue(tsort).toString(); - for (SolrDocument doc : docs) { - assertTrue( - "Docs should be back in sorted order, descending", - title.compareTo(doc.getFieldValue(tsort).toString()) >= 0); - title = doc.getFieldValue(tsort).toString(); - } - } - - private void testSortableTextGrouping() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - query.add("group", "true"); - query.add("group.field", tsort); - QueryResponse resp = queryRandomShard(query); - GroupResponse groupResp = resp.getGroupResponse(); - List grpCmds = groupResp.getValues(); - for (GroupCommand grpCmd : grpCmds) { - if (grpCmd.getName().equals(tsort) == false) continue; - for (Group grp : grpCmd.getValues()) { - long count = grp.getResult().getNumFound(); - if (grp.getGroupValue() == null) - continue; // Don't count the groups without an entry as the numnber is variable - switch (grp.getGroupValue()) { - case "all the kings horses and all the kings men": - case "An eye for eye only ends up making the whole world blind.": - case "Great works are performed, not by strength, but by perseverance.": - case "how now brown cow": - case "no eggs on wall, lesson learned": - case "now is the time for all good men": - case "this too shall pass": - case "to come to the aid of their country.": - assertEquals( - "Should have exactly one facet count for field " + grpCmd.getName(), 1, count); - break; - case "the quick fox jumped over the lazy dog": - assertEquals("Should have 5 docs for the lazy dog", 5, count); - break; - default: - fail("No case for facet '" + grpCmd.getName() + "'"); - } - } - } - } - - private void testTokenizedGrouping() throws Exception { - SolrException ex = - expectThrows( - SolrException.class, - () -> { - query(false, new String[] {"q", "*:*", "group", "true", "group.field", t1}); - }); - assertTrue( - "Expected error from server that SortableTextFields are required", - ex.getMessage() - .contains( - "Sorting on a tokenized field that is not a SortableTextField is not supported in cloud mode")); - } - - private void assertSliceCounts(String msg, long expected, DocCollection dColl) throws Exception { - long found = checkSlicesSameCounts(dColl); - - if (found != expected) { - // we get one do over in a bad race - Thread.sleep(1000); - found = checkSlicesSameCounts(dColl); - } - - assertEquals(msg, expected, checkSlicesSameCounts(dColl)); - } - - // Ensure that total docs found is the expected number. - private void waitForDocCount(long expectedNumFound, long waitMillis, String failureMessage) - throws Exception { - AtomicLong total = new AtomicLong(-1); - try { - ZkStateReader.from(getCommonCloudSolrClient()) - .waitForState( - DEFAULT_COLLECTION, - waitMillis, - TimeUnit.MILLISECONDS, - (n, c) -> { - long docTotal; - try { - docTotal = checkSlicesSameCounts(c); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - total.set(docTotal); - if (docTotal == expectedNumFound) { - return true; - } - return false; - }); - } catch (TimeoutException | InterruptedException e) { - - } - // We could fail here if we broke out of the above because we exceeded the time allowed. - assertEquals(failureMessage, expectedNumFound, total.get()); - - // This should be redundant, but it caught a test error after all. - for (SolrClient client : clients) { - assertEquals( - failureMessage, - expectedNumFound, - client.query(new SolrQuery("*:*")).getResults().getNumFound()); - } - } - - // Insure that counts are the same for all replicas in each shard - // Return the total doc count for the query. - private long checkSlicesSameCounts(DocCollection dColl) throws SolrServerException, IOException { - long docTotal = 0; // total number of documents found counting only one replica per slice. - for (Slice slice : dColl.getActiveSlices()) { - long sliceDocCount = -1; - for (Replica rep : slice.getReplicas()) { - try (SolrClient one = getHttpSolrClient(rep)) { - SolrQuery query = new SolrQuery("*:*"); - query.setDistrib(false); - QueryResponse resp = one.query(query); - long hits = resp.getResults().getNumFound(); - if (sliceDocCount == -1) { - sliceDocCount = hits; - docTotal += hits; - } else { - if (hits != sliceDocCount) { - return -1; - } - } - } - } - } - return docTotal; - } - - private void testShardParamVariations() throws Exception { - SolrQuery query = new SolrQuery("*:*"); - Map shardCounts = new HashMap<>(); - - for (String shard : shardToJetty.keySet()) { - // every client should give the same numDocs for this shard - // shffle the clients in a diff order for each shard - List solrclients = new ArrayList<>(this.clients); - Collections.shuffle(solrclients, random()); - for (SolrClient client : solrclients) { - query.set("shards", shard); - long numDocs = client.query(query).getResults().getNumFound(); - assertTrue("numDocs < 0 for shard " + shard + " via " + client, 0 <= numDocs); - if (!shardCounts.containsKey(shard)) { - shardCounts.put(shard, numDocs); - } - assertEquals( - "inconsitent numDocs for shard " + shard + " via " + client, - shardCounts.get(shard).longValue(), - numDocs); - - List replicaJetties = new ArrayList<>(shardToJetty.get(shard)); - Collections.shuffle(replicaJetties, random()); - - // each replica should also give the same numDocs - ArrayList replicaAlts = new ArrayList<>(replicaJetties.size() * 2); - for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { - String replica = replicaJetty.url; - query.set("shards", replica); - - // replicas already shuffled, use this in the alternative check below - if (0 == random().nextInt(3) || replicaAlts.size() < 2) { - replicaAlts.add(replica); - } - - numDocs = client.query(query).getResults().getNumFound(); - assertTrue("numDocs < 0 for replica " + replica + " via " + client, 0 <= numDocs); - assertEquals( - "inconsitent numDocs for shard " - + shard - + " in replica " - + replica - + " via " - + client, - shardCounts.get(shard).longValue(), - numDocs); - } - - // any combination of replica alternatives should give same numDocs - String replicas = String.join("|", replicaAlts); - query.set("shards", replicas); - numDocs = client.query(query).getResults().getNumFound(); - assertTrue("numDocs < 0 for replicas " + replicas + " via " + client, 0 <= numDocs); - assertEquals( - "inconsitent numDocs for replicas " + replicas + " via " + client, - shardCounts.get(shard).longValue(), - numDocs); - } - } - - // sums of multiple shards should add up regardless of how we - // query those shards or which client we use - long randomShardCountsExpected = 0; - ArrayList randomShards = new ArrayList<>(shardCounts.size()); - for (Map.Entry shardData : shardCounts.entrySet()) { - if (random().nextBoolean() || randomShards.size() < 2) { - String shard = shardData.getKey(); - randomShardCountsExpected += shardData.getValue(); - if (random().nextBoolean()) { - // use shard id - randomShards.add(shard); - } else { - // use some set explicit replicas - ArrayList replicas = new ArrayList<>(7); - for (CloudJettyRunner replicaJetty : shardToJetty.get(shard)) { - if (0 == random().nextInt(3) || 0 == replicas.size()) { - replicas.add(replicaJetty.url); - } - } - Collections.shuffle(replicas, random()); - randomShards.add(String.join("|", replicas)); - } - } - } - String randShards = String.join(",", randomShards); - query.set("shards", randShards); - for (SolrClient client : this.clients) { - assertEquals( - "numDocs for " + randShards + " via " + client, - randomShardCountsExpected, - client.query(query).getResults().getNumFound()); - } - - // total num docs must match sum of every shard's numDocs - query = new SolrQuery("*:*"); - long totalShardNumDocs = 0; - for (Long c : shardCounts.values()) { - totalShardNumDocs += c; - } - for (SolrClient client : clients) { - assertEquals( - "sum of shard numDocs on client: " + client, - totalShardNumDocs, - client.query(query).getResults().getNumFound()); - } - assertTrue("total numDocs <= 0, WTF? Test is useless", 0 < totalShardNumDocs); - } - - private void testStopAndStartCoresInOneInstance() throws Exception { - JettySolrRunner jetty = jettys.get(0); - try (final SolrClient httpSolrClient = (HttpSolrClient) jetty.newClient(15000, 60000)) { - ThreadPoolExecutor executor = null; - try { - executor = - new ExecutorUtil.MDCAwareThreadPoolExecutor( - 0, - Integer.MAX_VALUE, - 5, - TimeUnit.SECONDS, - new SynchronousQueue(), - new SolrNamedThreadFactory("testExecutor")); - int cnt = 3; - - // create the cores - createCollectionInOneInstance( - httpSolrClient, jetty.getNodeName(), executor, "multiunload2", 1, cnt); - } finally { - if (executor != null) { - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - } - } - - cloudJettys.get(0).jetty.stop(); - printLayout(); - - cloudJettys.get(0).jetty.start(); - ZkStateReader.from(cloudClient).forceUpdateCollection("multiunload2"); - try { - ZkStateReader.from(cloudClient).getLeaderRetry("multiunload2", "shard1", 30000); - } catch (SolrException e) { - printLayout(); - throw e; - } - - printLayout(); - } - - /** Create a collection in single node */ - public static void createCollectionInOneInstance( - final SolrClient client, - String nodeName, - ThreadPoolExecutor executor, - final String collection, - final int numShards, - int numReplicas) { - assertNotNull(nodeName); - try { - assertEquals( - 0, - CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) - .setCreateNodeSet("") - .process(client) - .getStatus()); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - for (int i = 0; i < numReplicas; i++) { - final int freezeI = i; - executor.execute( - () -> { - try { - assertTrue( - CollectionAdminRequest.addReplicaToShard( - collection, "shard" + ((freezeI % numShards) + 1)) - .setCoreName(collection + freezeI) - .setNode(nodeName) - .process(client) - .isSuccess()); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - }); - } - } - - @Override - protected CollectionAdminResponse createCollection( - Map> collectionInfos, - String collectionName, - String configSetName, - int numShards, - int numReplicas, - SolrClient client, - String createNodeSetStr) - throws SolrServerException, IOException { - // TODO: Use CollectionAdminRequest for this test - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.CREATE.toString()); - - params.set(CollectionHandlingUtils.NUM_SLICES, numShards); - params.set(ZkStateReader.REPLICATION_FACTOR, numReplicas); - if (createNodeSetStr != null) - params.set(CollectionHandlingUtils.CREATE_NODE_SET, createNodeSetStr); - - int clientIndex = clients.size() > 1 ? random().nextInt(2) : 0; - List list = new ArrayList<>(); - list.add(numShards); - list.add(numReplicas); - if (collectionInfos != null) { - collectionInfos.put(collectionName, list); - } - params.set("name", collectionName); - params.set("collection.configName", configSetName); - QueryRequest request = new QueryRequest(params); - request.setPath("/admin/collections"); - - CollectionAdminResponse res = new CollectionAdminResponse(); - if (client == null) { - final String baseUrl = ((HttpSolrClient) clients.get(clientIndex)).getBaseURL(); - - try (SolrClient aClient = createNewSolrClient("", baseUrl)) { - res.setResponse(aClient.request(request)); - } - } else { - res.setResponse(client.request(request)); - } - return res; - } - - @Override - protected Replica getLeaderFromZk(String collection, String slice) { - ClusterState clusterState = getCommonCloudSolrClient().getClusterState(); - Replica leader = clusterState.getCollection(collection).getLeader(slice); - if (leader == null) { - throw new RuntimeException("Could not find leader:" + collection + " " + slice); - } - return leader; - } - - /** - * Expects a RegexReplaceProcessorFactories in the chain which will "double up" the values in two - * (stored) string fields. - * - *

If the values are "double-doubled" or "not-doubled" then we know the processor was not run - * the appropriate number of times - */ - private void testUpdateProcessorsRunOnlyOnce(final String chain) throws Exception { - - final String fieldA = "regex_dup_A_s"; - final String fieldB = "regex_dup_B_s"; - final String val = "x"; - final String expected = "x_x"; - final ModifiableSolrParams updateParams = new ModifiableSolrParams(); - updateParams.add(UpdateParams.UPDATE_CHAIN, chain); - - final int numLoops = atLeast(50); - - for (int i = 1; i < numLoops; i++) { - // add doc to random client - SolrClient updateClient = clients.get(random().nextInt(clients.size())); - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, id, i, fieldA, val, fieldB, val); - UpdateResponse ures = add(updateClient, updateParams, doc); - assertEquals(chain + ": update failed", 0, ures.getStatus()); - ures = updateClient.commit(); - assertEquals(chain + ": commit failed", 0, ures.getStatus()); - } - - // query for each doc, and check both fields to ensure the value is correct - for (int i = 1; i < numLoops; i++) { - final String query = id + ":" + i; - QueryResponse qres = queryRandomShard(new SolrQuery(query)); - assertEquals(chain + ": query failed: " + query, 0, qres.getStatus()); - assertEquals( - chain + ": didn't find correct # docs with query: " + query, - 1, - qres.getResults().getNumFound()); - SolrDocument doc = qres.getResults().get(0); - - for (String field : new String[] {fieldA, fieldB}) { - assertEquals( - chain + ": doc#" + i + " has wrong value for " + field, - expected, - doc.getFirstValue(field)); - } - } - } - - // cloud level test mainly needed just to make sure that versions and errors are propagated - // correctly - private void doOptimisticLockingAndUpdating() throws Exception { - log.info("### STARTING doOptimisticLockingAndUpdating"); - printLayout(); - - final SolrInputDocument sd = sdoc("id", 1000, "_version_", -1); - indexDoc(sd); - - ignoreException("version conflict"); - for (SolrClient client : clients) { - SolrException e = expectThrows(SolrException.class, () -> client.add(sd)); - assertEquals(409, e.code()); - } - unIgnoreException("version conflict"); - - // TODO: test deletes. SolrJ needs a good way to pass version for delete... - - final SolrInputDocument sd2 = sdoc("id", 1000, "foo_i", 5); - clients.get(0).add(sd2); - - List expected = new ArrayList<>(); - int val = 0; - for (SolrClient client : clients) { - val += 10; - client.add(sdoc("id", 1000, "val_i", map("add", val), "foo_i", val)); - expected.add(val); - } - - QueryRequest qr = new QueryRequest(params("qt", "/get", "id", "1000")); - for (SolrClient client : clients) { - val += 10; - NamedList rsp = client.request(qr); - String match = JSONTestUtil.matchObj("/val_i", rsp.get("doc"), expected); - if (match != null) throw new RuntimeException(match); - } - } - - private void testNumberOfCommitsWithCommitAfterAdd() throws SolrServerException, IOException { - log.info("### STARTING testNumberOfCommitsWithCommitAfterAdd"); - long startCommits = getNumCommits((HttpSolrClient) clients.get(0)); - - NamedList result = - clients - .get(0) - .request( - new StreamingUpdateRequest( - "/update", getFile("books_numeric_ids.csv"), "application/csv") - .setCommitWithin(900000) - .setAction(AbstractUpdateRequest.ACTION.COMMIT, true, true)); - - long endCommits = getNumCommits((HttpSolrClient) clients.get(0)); - assertEquals(startCommits + 1L, endCommits); - } - - private Long getNumCommits(HttpSolrClient sourceClient) throws SolrServerException, IOException { - String collection = sourceClient.getDefaultCollection(); - try (SolrClient client = - new HttpSolrClient.Builder(sourceClient.getBaseURL()) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(60000, TimeUnit.MILLISECONDS) - .build()) { - var req = - new GenericSolrRequest( - SolrRequest.METHOD.GET, - "/admin/metrics", - SolrRequest.SolrRequestType.ADMIN, - SolrParams.of("wt", "prometheus")); - req.setResponseParser(new InputStreamResponseParser("prometheus")); - - NamedList resp = client.request(req); - try (InputStream in = (InputStream) resp.get("stream")) { - String output = new String(in.readAllBytes(), StandardCharsets.UTF_8); - String metricName = "solr_core_update_commit_ops"; - - return (long) - output - .lines() - .filter( - l -> - l.startsWith(metricName) - && l.contains("collection=\"" + collection + "\"") - && l.contains("ops=\"commits\"")) - .mapToDouble(s -> Double.parseDouble(s.substring(s.lastIndexOf(" ")))) - .sum(); - } - } - } - - private void testANewCollectionInOneInstanceWithManualShardAssignement() throws Exception { - log.info("### STARTING testANewCollectionInOneInstanceWithManualShardAssignement"); - assertEquals( - 0, - CollectionAdminRequest.createCollection(oneInstanceCollection2, "conf1", 2, 2) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - - List collectionClients = new ArrayList<>(); - for (int i = 0; i < 4; i++) { - CollectionAdminResponse resp = - CollectionAdminRequest.addReplicaToShard(oneInstanceCollection2, "shard" + ((i % 2) + 1)) - .setNode(jettys.get(0).getNodeName()) - .process(cloudClient); - for (String coreName : resp.getCollectionCoresStatus().keySet()) { - collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); - } - } - - SolrClient client1 = collectionClients.get(0); - SolrClient client2 = collectionClients.get(1); - SolrClient client3 = collectionClients.get(2); - SolrClient client4 = collectionClients.get(3); - - // no one should be recovering - waitForRecoveriesToFinish( - oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient()), false, true); - - assertAllActive(oneInstanceCollection2, ZkStateReader.from(getCommonCloudSolrClient())); - - // printLayout(); - - // TODO: enable when we don't falsely get slice1... - // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice1", 30000); - // solrj.getZkStateReader().getLeaderUrl(oneInstanceCollection2, "slice2", 30000); - client2.add(getDoc(id, "1")); - client3.add(getDoc(id, "2")); - client4.add(getDoc(id, "3")); - - client1.commit(); - SolrQuery query = new SolrQuery("*:*"); - query.set("distrib", false); - long oneDocs = client1.query(query).getResults().getNumFound(); - long twoDocs = client2.query(query).getResults().getNumFound(); - long threeDocs = client3.query(query).getResults().getNumFound(); - long fourDocs = client4.query(query).getResults().getNumFound(); - - query.set("collection", oneInstanceCollection2); - query.set("distrib", true); - long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - - // System.out.println("1:" + oneDocs); - // System.out.println("2:" + twoDocs); - // System.out.println("3:" + threeDocs); - // System.out.println("4:" + fourDocs); - // System.out.println("All Docs:" + allDocs); - - // assertEquals(oneDocs, threeDocs); - // assertEquals(twoDocs, fourDocs); - // assertNotSame(oneDocs, twoDocs); - assertEquals(3, allDocs); - - // we added a role of none on these creates - check for it - ZkStateReader zkStateReader = ZkStateReader.from(getCommonCloudSolrClient()); - zkStateReader.forceUpdateCollection(oneInstanceCollection2); - Map slices = - zkStateReader.getClusterState().getCollection(oneInstanceCollection2).getSlicesMap(); - assertNotNull(slices); - - Replica leader = - getCommonCloudSolrClient() - .getClusterState() - .getCollection(oneInstanceCollection2) - .getLeader("shard1"); - - // now test that unloading a core gets us a new leader - try (SolrClient unloadClient = - new HttpSolrClient.Builder(jettys.get(0).getBaseUrl().toString()) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(60000, TimeUnit.MILLISECONDS) - .build()) { - Unload unloadCmd = new Unload(true); - unloadCmd.setCoreName(leader.getCoreName()); - - String leaderUrl = leader.getCoreUrl(); - - testExecutor.execute( - new Runnable() { - - @Override - public void run() { - try { - unloadClient.request(unloadCmd); - } catch (SolrServerException e) { - throw new RuntimeException(e); - } catch (IOException e) { - throw new RuntimeException(e); - } - } - }); - - try { - ZkStateReader.from(getCommonCloudSolrClient()) - .waitForState( - oneInstanceCollection2, - 20000, - TimeUnit.MILLISECONDS, - (n, c) -> { - try { - if (leaderUrl.equals( - zkStateReader.getLeaderUrl(oneInstanceCollection2, "shard1", 10000))) { - return false; - } - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - return true; - }); - } catch (TimeoutException | InterruptedException e) { - fail("Leader never changed"); - } - } - - IOUtils.close(collectionClients); - } - - private void testSearchByCollectionName() throws SolrServerException, IOException { - log.info("### STARTING testSearchByCollectionName"); - SolrClient client = clients.get(0); - final String baseUrl = ((HttpSolrClient) client).getBaseURL(); - - // the cores each have different names, but if we add the collection name to the url - // we should get mapped to the right core - try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { - SolrQuery query = new SolrQuery("*:*"); - long oneDocs = client1.query(query).getResults().getNumFound(); - assertEquals(3, oneDocs); - } - } - - private void testUpdateByCollectionName() throws SolrServerException, IOException { - log.info("### STARTING testUpdateByCollectionName"); - SolrClient client = clients.get(0); - final String baseUrl = ((HttpSolrClient) client).getBaseURL(); - - // the cores each have different names, but if we add the collection name to the url - // we should get mapped to the right core - // test hitting an update url - try (SolrClient client1 = createNewSolrClient(oneInstanceCollection, baseUrl)) { - client1.commit(); - } - } - - private void testANewCollectionInOneInstance() throws Exception { - log.info("### STARTING testANewCollectionInOneInstance"); - CollectionAdminResponse response = - CollectionAdminRequest.createCollection(oneInstanceCollection, "conf1", 2, 2) - .setCreateNodeSet(jettys.get(0).getNodeName()) - .process(cloudClient); - assertEquals(0, response.getStatus()); - List collectionClients = new ArrayList<>(); - for (String coreName : response.getCollectionCoresStatus().keySet()) { - collectionClients.add(createNewSolrClient(coreName, jettys.get(0).getBaseUrl().toString())); - } - - SolrClient client1 = collectionClients.get(0); - SolrClient client2 = collectionClients.get(1); - SolrClient client3 = collectionClients.get(2); - SolrClient client4 = collectionClients.get(3); - - waitForRecoveriesToFinish( - oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient()), false); - assertAllActive(oneInstanceCollection, ZkStateReader.from(getCommonCloudSolrClient())); - - client2.add(getDoc(id, "1")); - client3.add(getDoc(id, "2")); - client4.add(getDoc(id, "3")); - - client1.commit(); - SolrQuery query = new SolrQuery("*:*"); - query.set("distrib", false); - long oneDocs = client1.query(query).getResults().getNumFound(); - long twoDocs = client2.query(query).getResults().getNumFound(); - long threeDocs = client3.query(query).getResults().getNumFound(); - long fourDocs = client4.query(query).getResults().getNumFound(); - - query.set("collection", oneInstanceCollection); - query.set("distrib", true); - long allDocs = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - - // System.out.println("1:" + oneDocs); - // System.out.println("2:" + twoDocs); - // System.out.println("3:" + threeDocs); - // System.out.println("4:" + fourDocs); - // System.out.println("All Docs:" + allDocs); - - assertEquals(3, allDocs); - IOUtils.close(collectionClients); - } - - private void createSolrCore( - final String collection, - List collectionClients, - final String baseUrl, - final int num, - final String shardId) { - Callable call = - () -> { - try (SolrClient client = getHttpSolrClient(baseUrl)) { - // client.setConnectionTimeout(15000); - Create createCmd = new Create(); - createCmd.setCoreName(collection + num); - createCmd.setCollection(collection); - - if (random().nextBoolean()) { - // sometimes we use an explicit core node name - createCmd.setCoreNodeName("anode" + nodeCounter.incrementAndGet()); - } - - if (shardId == null) { - createCmd.setNumShards(2); - } - createCmd.setDataDir(getDataDir(createTempDir(collection).toString())); - if (shardId != null) { - createCmd.setShardId(shardId); - } - client.request(createCmd); - } catch (Exception e) { - log.error("error creating core", e); - // fail - } - return null; - }; - - pending.add(completionService.submit(call)); - - collectionClients.add(createNewSolrClient(collection + num, baseUrl)); - } - - private void testMultipleCollections() throws Exception { - log.info("### STARTING testMultipleCollections"); - // create another 2 collections and search across them - createNewCollection("collection2"); - createNewCollection("collection3"); - - while (pending != null && pending.size() > 0) { - - Future future = completionService.take(); - if (future == null) return; - pending.remove(future); - } - - indexDoc("collection2", getDoc(id, "10000000")); - indexDoc("collection2", getDoc(id, "10000001")); - indexDoc("collection2", getDoc(id, "10000003")); - - getSolrClient("collection2").add(getDoc(id, "10000004")); - - indexDoc("collection3", getDoc(id, "20000000")); - indexDoc("collection3", getDoc(id, "20000001")); - - getSolrClient("collection3").add(getDoc(id, "10000005")); - - otherCollectionClients.get("collection2").get(0).commit(); - otherCollectionClients.get("collection3").get(0).commit(); - - SolrClient clientForCollection1 = getSolrClient("collection1"); - long collection1Docs = - clientForCollection1.query(new SolrQuery("*:*")).getResults().getNumFound(); - - long collection2Docs = - otherCollectionClients - .get("collection2") - .get(0) - .query(new SolrQuery("*:*")) - .getResults() - .getNumFound(); - - long collection3Docs = - otherCollectionClients - .get("collection3") - .get(0) - .query(new SolrQuery("*:*")) - .getResults() - .getNumFound(); - - SolrQuery query = new SolrQuery("*:*"); - query.set("collection", "collection2,collection3"); - long found = clients.get(0).query(query).getResults().getNumFound(); - assertEquals(collection2Docs + collection3Docs, found); - - query = new SolrQuery("*:*"); - query.set("collection", "collection1,collection2,collection3"); - found = clients.get(0).query(query).getResults().getNumFound(); - assertEquals(collection1Docs + collection2Docs + collection3Docs, found); - - // try to search multiple with cloud client - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection1Docs + collection2Docs + collection3Docs, found); - - query.set("collection", "collection2,collection3"); - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection2Docs + collection3Docs, found); - - query.set("collection", "collection3"); - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection3Docs, found); - - query.remove("collection"); - found = getCommonCloudSolrClient().query(query).getResults().getNumFound(); - assertEquals(collection1Docs, found); - - assertEquals(collection3Docs, collection2Docs - 1); - } - - protected void indexDoc(String collection, SolrInputDocument doc) - throws IOException, SolrServerException { - List clients = otherCollectionClients.get(collection); - int which = (doc.getField(id).toString().hashCode() & 0x7fffffff) % clients.size(); - SolrClient client = clients.get(which); - client.add(doc); - } - - @SuppressWarnings({"unchecked"}) - private void createNewCollection(final String collection) throws InterruptedException { - try { - assertEquals( - 0, - CollectionAdminRequest.createCollection(collection, "conf1", 2, 1) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - } catch (Exception e) { - log.error("error creating collection", e); - // fails - } - final List collectionClients = new ArrayList<>(); - otherCollectionClients.put(collection, collectionClients); - int unique = 0; - for (final JettySolrRunner runner : jettys) { - unique++; - final int frozeUnique = unique; - Callable call = - () -> { - try { - assertTrue( - CollectionAdminRequest.addReplicaToShard( - collection, "shard" + ((frozeUnique % 2) + 1)) - .setNode(runner.getNodeName()) - .process(cloudClient) - .isSuccess()); - } catch (Exception e) { - log.error("error adding replica", e); - // fails - } - return null; - }; - - collectionClients.add(createNewSolrClient(collection, runner.getBaseUrl().toString())); - pending.add(completionService.submit(call)); - while (pending != null && pending.size() > 0) { - - Future future = completionService.take(); - if (future == null) return; - pending.remove(future); - } - } - } - - @Override - protected SolrClient createNewSolrClient(String collection, String baseUrl) { - - SolrClient client = getHttpSolrClient(baseUrl, collection); - - return client; - } - - /** - * @param collection the name of a collection or core to set as the "default" on the created - * client. - * @param baseUrl the "base" URL of a Solr node. Should not contain a collection or core - * name. - * @param connectionTimeoutMillis the HTTP connection timeout in milliseconds - * @param socketTimeoutMillis the HTTP socket-read timeout in milliseconds - */ - protected SolrClient createNewSolrClient( - String collection, String baseUrl, int connectionTimeoutMillis, int socketTimeoutMillis) { - - SolrClient client = - new HttpSolrClient.Builder(baseUrl) - .withDefaultCollection(collection) - .withConnectionTimeout(connectionTimeoutMillis, TimeUnit.MILLISECONDS) - .withSocketTimeout(socketTimeoutMillis, TimeUnit.MILLISECONDS) - .build(); - - return client; - } - - @Override - protected QueryResponse queryRandomShard(ModifiableSolrParams params) - throws SolrServerException, IOException { - - if (r.nextBoolean()) return super.queryRandomShard(params); - - if (r.nextBoolean()) params.set("collection", DEFAULT_COLLECTION); - - QueryResponse rsp = getCommonCloudSolrClient().query(params); - return rsp; - } - - @Override - public void distribTearDown() throws Exception { - super.distribTearDown(); - if (otherCollectionClients != null) { - for (List clientList : otherCollectionClients.values()) { - IOUtils.close(clientList); - } - } - otherCollectionClients = null; - List tasks = executor.shutdownNow(); - assertTrue(tasks.isEmpty()); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java deleted file mode 100644 index 6d55d1f560b..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeyNothingIsSafeTestBase.java +++ /dev/null @@ -1,338 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -@LuceneTestCase.Nightly -public abstract class AbstractChaosMonkeyNothingIsSafeTestBase - extends AbstractFullDistribZkTestBase { - private static final int FAIL_TOLERANCE = 100; - - private static final Integer RUN_LENGTH = - Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); - - private final boolean onlyLeaderIndexes = random().nextBoolean(); - - @BeforeClass - public static void beforeSuperClass() { - schemaString = "schema15.xml"; // we need a string id - System.setProperty("solr.autoCommit.maxTime", "15000"); - System.clearProperty("solr.httpclient.retries"); - System.clearProperty("solr.retries.on.forward"); - System.clearProperty("solr.retries.to.followers"); - setErrorHook(); - } - - @AfterClass - public static void afterSuperClass() { - System.clearProperty("solr.autoCommit.maxTime"); - clearErrorHook(); - } - - @Override - protected void destroyServers() throws Exception { - - super.destroyServers(); - } - - protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; - protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; - - private int clientSoTimeout = 60000; - - private volatile FullThrottleStoppableIndexingThread ftIndexThread; - - private final boolean runFullThrottle; - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public RandVal[] getRandValues() { - return randVals; - } - - @Override - public void distribSetUp() throws Exception { - super.distribSetUp(); - // can help to hide this when testing and looking at logs - // ignoreException("shard update error"); - useFactory("solr.StandardDirectoryFactory"); - } - - @Override - public void distribTearDown() throws Exception { - try { - ftIndexThread.safeStop(); - } catch (NullPointerException e) { - // okay - } - super.distribTearDown(); - } - - public AbstractChaosMonkeyNothingIsSafeTestBase() { - super(); - sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); - if (sliceCount == -1) { - sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; - } - - int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); - if (numShards == -1) { - // we make sure that there's at least one shard with more than one replica - // so that the ChaosMonkey has something to kill - numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; - } - fixShardCount(numShards); - - // TODO: we only do this sometimes so that we can sometimes compare against control, - // it's currently hard to know what requests failed when using ConcurrentSolrUpdateServer - runFullThrottle = random().nextBoolean(); - } - - @Override - protected boolean useTlogReplicas() { - return false; // TODO: tlog replicas makes commits take way to long due to what is likely a bug - // and it's TestInjection use - } - - @Override - protected CloudSolrClient createCloudClient(String defaultCollection) { - return this.createCloudClient(defaultCollection, this.clientSoTimeout); - } - - protected CloudSolrClient createCloudClient(String defaultCollection, int socketTimeout) { - - return getCloudSolrClient( - zkServer.getZkAddress(), defaultCollection, random().nextBoolean(), 30000, socketTimeout); - } - - @Test - @SuppressWarnings({"try"}) - public void test() throws Exception { - // None of the operations used here are particularly costly, so this should work. - // Using this low timeout will also help us catch index stalling. - clientSoTimeout = 5000; - - boolean testSuccessful = false; - try (CloudSolrClient ourCloudClient = createCloudClient(DEFAULT_COLLECTION)) { - handle.clear(); - handle.put("timestamp", SKIPVAL); - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - // make sure we have leaders for each shard - for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 10000); - } // make sure we again have leaders for each shard - - waitForRecoveriesToFinish(false); - - // we cannot do delete by query - // as it's not supported for recovery - del("*:*"); - - List threads = new ArrayList<>(); - List indexTreads = new ArrayList<>(); - int threadCount = TEST_NIGHTLY ? 3 : 1; - int i = 0; - for (i = 0; i < threadCount; i++) { - StoppableIndexingThread indexThread = - new StoppableIndexingThread(controlClient, cloudClient, Integer.toString(i), true); - threads.add(indexThread); - indexTreads.add(indexThread); - indexThread.start(); - } - - threadCount = 1; - i = 0; - for (i = 0; i < threadCount; i++) { - StoppableSearchThread searchThread = new StoppableSearchThread(cloudClient); - threads.add(searchThread); - searchThread.start(); - } - - if (runFullThrottle) { - ftIndexThread = - new FullThrottleStoppableIndexingThread( - ((CloudLegacySolrClient) cloudClient).getHttpClient(), - controlClient, - cloudClient, - clients, - "ft1", - true, - this.clientSoTimeout); - ftIndexThread.start(); - } - - chaosMonkey.startTheMonkey(true, 10000); - try { - long runLength; - if (RUN_LENGTH != -1) { - runLength = RUN_LENGTH; - } else { - int[] runTimes; - if (TEST_NIGHTLY) { - runTimes = - new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; - } else { - runTimes = new int[] {5000, 7000, 15000}; - } - runLength = runTimes[random().nextInt(runTimes.length - 1)]; - } - - Thread.sleep(runLength); - } finally { - chaosMonkey.stopTheMonkey(); - } - - // ideally this should go into chaosMonkey - restartZk(1000 * (5 + random().nextInt(4))); - - if (runFullThrottle) { - ftIndexThread.safeStop(); - } - - for (StoppableThread indexThread : threads) { - indexThread.safeStop(); - } - - // start any downed jetties to be sure we still will end up with a leader per shard... - - // wait for stop... - for (StoppableThread indexThread : threads) { - indexThread.join(); - } - - // try and wait for any replications and what not to finish... - - Thread.sleep(2000); - - // wait until there are no recoveries... - waitForThingsToLevelOut(); - - // make sure we again have leaders for each shard - for (int j = 1; j < sliceCount; j++) { - zkStateReader.getLeaderRetry(DEFAULT_COLLECTION, "shard" + j, 30000); - } - - commit(); - - // TODO: assert we didn't kill everyone - - zkStateReader.updateLiveNodes(); - assertTrue(zkStateReader.getClusterState().getLiveNodes().size() > 0); - - // we expect full throttle fails, but cloud client should not easily fail - for (StoppableThread indexThread : threads) { - if (indexThread instanceof StoppableIndexingThread - && !(indexThread instanceof FullThrottleStoppableIndexingThread)) { - int failCount = ((StoppableIndexingThread) indexThread).getFailCount(); - assertFalse( - "There were too many update fails (" - + failCount - + " > " - + FAIL_TOLERANCE - + ") - we expect it can happen, but shouldn't easily", - failCount > FAIL_TOLERANCE); - } - } - - waitForThingsToLevelOut(20, TimeUnit.SECONDS); - - commit(); - - Set addFails = getAddFails(indexTreads); - Set deleteFails = getDeleteFails(indexTreads); - // full throttle thread can - // have request fails - checkShardConsistency(!runFullThrottle, true, addFails, deleteFails); - - long ctrlDocs = controlClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - // ensure we have added more than 0 docs - long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - - assertTrue("Found " + ctrlDocs + " control docs", cloudClientDocs > 0); - - if (VERBOSE) - System.out.println( - "control docs:" - + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() - + "\n\n"); - - // try and make a collection to make sure the overseer has survived the expiration and session - // loss - - // sometimes we restart zookeeper as well - if (random().nextBoolean()) { - // restartZk(1000 * (5 + random().nextInt(4))); - } - - try (CloudSolrClient client = createCloudClient("collection1", 30000)) { - createCollection(null, "testcollection", 1, 1, client, null, "conf1"); - } - List numShardsNumReplicas = new ArrayList<>(2); - numShardsNumReplicas.add(1); - numShardsNumReplicas.add(1); - checkForCollection("testcollection", numShardsNumReplicas); - - testSuccessful = true; - } finally { - if (!testSuccessful) { - printLayout(); - } - } - } - - private Set getAddFails(List threads) { - Set addFails = new HashSet<>(); - for (StoppableIndexingThread thread : threads) { - addFails.addAll(thread.getAddFails()); - } - return addFails; - } - - private Set getDeleteFails(List threads) { - Set deleteFails = new HashSet<>(); - for (StoppableIndexingThread thread : threads) { - deleteFails.addAll(thread.getDeleteFails()); - } - return deleteFails; - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = getDoc(fields); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java deleted file mode 100644 index 5d94eae05e3..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractChaosMonkeySafeLeaderTestBase.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.lucene.tests.util.LuceneTestCase; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.common.SolrInputDocument; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -@LuceneTestCase.Nightly -public abstract class AbstractChaosMonkeySafeLeaderTestBase extends AbstractFullDistribZkTestBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final Integer RUN_LENGTH = - Integer.parseInt(System.getProperty("solr.tests.cloud.cm.runlength", "-1")); - - @BeforeClass - public static void beforeSuperClass() { - schemaString = "schema15.xml"; // we need a string id - System.setProperty("solr.autoCommit.maxTime", "15000"); - System.clearProperty("solr.httpclient.retries"); - System.clearProperty("solr.retries.on.forward"); - System.clearProperty("solr.retries.to.followers"); - setErrorHook(); - } - - @AfterClass - public static void afterSuperClass() { - System.clearProperty("solr.autoCommit.maxTime"); - clearErrorHook(); - } - - protected static final String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; - protected static final RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; - - @Override - public String[] getFieldNames() { - return fieldNames; - } - - @Override - public RandVal[] getRandValues() { - return randVals; - } - - protected abstract String getDirectoryFactory(); - - @Override - public void distribSetUp() throws Exception { - useFactory(getDirectoryFactory()); - super.distribSetUp(); - } - - public AbstractChaosMonkeySafeLeaderTestBase() { - super(); - sliceCount = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.slicecount", "-1")); - if (sliceCount == -1) { - sliceCount = random().nextInt(TEST_NIGHTLY ? 5 : 3) + 1; - } - - int numShards = Integer.parseInt(System.getProperty("solr.tests.cloud.cm.shardcount", "-1")); - if (numShards == -1) { - // we make sure that there's at least one shard with more than one replica - // so that the ChaosMonkey has something to kill - numShards = sliceCount + random().nextInt(TEST_NIGHTLY ? 12 : 2) + 1; - } - fixShardCount(numShards); - } - - @Test - public void test() throws Exception { - - handle.clear(); - handle.put("timestamp", SKIPVAL); - - // randomly turn on 1 seconds 'soft' commit - randomlyEnableAutoSoftCommit(); - - tryDelete(); - - List threads = new ArrayList<>(); - int threadCount = 2; - int batchSize = 1; - if (random().nextBoolean()) { - batchSize = random().nextInt(98) + 2; - } - - boolean pauseBetweenUpdates = TEST_NIGHTLY ? random().nextBoolean() : true; - int maxUpdates = -1; - if (!pauseBetweenUpdates) { - maxUpdates = 1000 + random().nextInt(1000); - } else { - maxUpdates = 15000; - } - - for (int i = 0; i < threadCount; i++) { - StoppableIndexingThread indexThread = - new StoppableIndexingThread( - controlClient, - cloudClient, - Integer.toString(i), - true, - maxUpdates, - batchSize, - pauseBetweenUpdates); // random().nextInt(999) + 1 - threads.add(indexThread); - indexThread.start(); - } - - chaosMonkey.startTheMonkey(false, 500); - try { - long runLength; - if (RUN_LENGTH != -1) { - runLength = RUN_LENGTH; - } else { - int[] runTimes; - if (TEST_NIGHTLY) { - runTimes = - new int[] {5000, 6000, 10000, 15000, 25000, 30000, 30000, 45000, 90000, 120000}; - } else { - runTimes = new int[] {5000, 7000, 15000}; - } - runLength = runTimes[random().nextInt(runTimes.length - 1)]; - } - - Thread.sleep(runLength); - } finally { - chaosMonkey.stopTheMonkey(); - } - - for (StoppableIndexingThread indexThread : threads) { - indexThread.safeStop(); - } - - // wait for stop... - for (StoppableIndexingThread indexThread : threads) { - indexThread.join(); - } - - for (StoppableIndexingThread indexThread : threads) { - assertEquals(0, indexThread.getFailCount()); - } - - // try and wait for any replications and what not to finish... - - Thread.sleep(2000); - - waitForThingsToLevelOut(3, TimeUnit.MINUTES); - - // even if things were leveled out, a jetty may have just been stopped or something - // we wait again and wait to level out again to make sure the system is not still in flux - - Thread.sleep(3000); - - waitForThingsToLevelOut(3, TimeUnit.MINUTES); - - checkShardConsistency(batchSize == 1, true); - - if (VERBOSE) - System.out.println( - "control docs:" - + controlClient.query(new SolrQuery("*:*")).getResults().getNumFound() - + "\n\n"); - - // try and make a collection to make sure the overseer has survived the expiration and session - // loss - - // sometimes we restart zookeeper as well - if (random().nextBoolean()) { - zkServer.shutdown(); - zkServer = new ZkTestServer(zkServer.getZkDir(), zkServer.getPort()); - zkServer.run(false); - } - - try (CloudSolrClient client = createCloudClient("collection1")) { - createCollection(null, "testcollection", 1, 1, client, null, "conf1"); - } - List numShardsNumReplicas = new ArrayList<>(2); - numShardsNumReplicas.add(1); - numShardsNumReplicas.add(1); - checkForCollection("testcollection", numShardsNumReplicas); - } - - private void tryDelete() throws Exception { - long start = System.nanoTime(); - long timeout = start + TimeUnit.NANOSECONDS.convert(10, TimeUnit.SECONDS); - while (System.nanoTime() < timeout) { - try { - del("*:*"); - break; - } catch (SolrServerException e) { - log.error("cluster may not be up yet", e); - } - Thread.sleep(100); - } - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java deleted file mode 100644 index 31bfdb837c6..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractDigestZkACLAndCredentialsProvidersTestBase.java +++ /dev/null @@ -1,567 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_FILE_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME; -import static org.apache.solr.common.cloud.VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME; -import static org.apache.zookeeper.ZooDefs.Ids.OPEN_ACL_UNSAFE; - -import java.io.FileWriter; -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.charset.Charset; -import java.nio.charset.StandardCharsets; -import java.nio.file.Path; -import java.util.Collections; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.TimeUnit; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.common.cloud.DigestZkACLProvider; -import org.apache.solr.common.cloud.DigestZkCredentialsProvider; -import org.apache.solr.common.cloud.SecurityAwareZkACLProvider; -import org.apache.solr.common.cloud.SolrZkClient; -import org.apache.solr.common.cloud.VMParamsZkCredentialsInjector; -import org.apache.solr.common.cloud.ZkCredentialsInjector; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.KeeperException.NoAuthException; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public class AbstractDigestZkACLAndCredentialsProvidersTestBase extends SolrTestCaseJ4 { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private static final Charset DATA_ENCODING = StandardCharsets.UTF_8; - - private static final String ALL_USERNAME = "connectAndAllACLUsername"; - private static final String ALL_PASSWORD = "connectAndAllACLPassword"; - private static final String READONLY_USERNAME = "readonlyACLUsername"; - private static final String READONLY_PASSWORD = "readonlyACLPassword"; - - protected ZkTestServer zkServer; - - protected Path zkDir; - - @BeforeClass - public static void beforeClass() { - System.setProperty("solrcloud.skip.autorecovery", "true"); - } - - @AfterClass - public static void afterClass() { - System.clearProperty("solrcloud.skip.autorecovery"); - } - - @Override - public void setUp() throws Exception { - // TODO: Does all of this setup need to happen for each test case, or can it be done once for - // the class? (i.e. @BeforeClass) and maybe some minor reset logic in setup, instead of full - // startup and teardown of a new ZkTestServer in each cycle? - super.setUp(); - if (log.isInfoEnabled()) { - log.info("####SETUP_START {}", getTestName()); - } - createTempDir(); - - zkDir = createTempDir().resolve("zookeeper/server1/data"); - log.info("ZooKeeper dataDir:{}", zkDir); - setSecuritySystemProperties(); - zkServer = new ZkTestServer(zkDir); - zkServer.run(false); - - System.setProperty("zkHost", zkServer.getZkAddress()); - - setDigestZkSystemProps(); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); - - SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkHost()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .withConnTimeOut(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - - zkClient.makePath("/solr", false, true); - zkClient.close(); - - zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - zkClient.create( - "/protectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - zkClient.makePath( - "/protectedMakePathNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - - zkClient.create( - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - zkClient.close(); - - clearSecuritySystemProperties(); - - zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build(); - // Currently, no credentials on ZK connection, because those same VM-params are used for adding - // ACLs, and here we want - // no (or completely open) ACLs added. Therefore, hack your way into being authorized for - // creating anyway - zkClient - .getCuratorFramework() - .getZookeeperClient() - .getZooKeeper() - .addAuthInfo( - "digest", (ALL_USERNAME + ":" + ALL_PASSWORD).getBytes(StandardCharsets.UTF_8)); - zkClient.create( - "/unprotectedCreateNode", "content".getBytes(DATA_ENCODING), CreateMode.PERSISTENT, false); - zkClient.makePath( - "/unprotectedMakePathNode", - "content".getBytes(DATA_ENCODING), - CreateMode.PERSISTENT, - false); - zkClient.close(); - - setDigestZkSystemProps(); - if (log.isInfoEnabled()) { - log.info("####SETUP_END {}", getTestName()); - } - } - - private void setDigestZkSystemProps() { - System.setProperty( - SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, - DigestZkCredentialsProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); - } - - @Override - public void tearDown() throws Exception { - zkServer.shutdown(); - - clearSecuritySystemProperties(); - - super.tearDown(); - } - - @Test - public void testNoCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(NoCredentialZkCredentialsInjector.class), - new TestZkCredentialsInjector(VMParamsZkCredentialsInjector.class)); - - testInjectors( - testZkCredentialsInjectors, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false); - } - - @Test - public void testWrongCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(WrongAllCredentialZkCredentialsInjector.class), - new TestZkCredentialsInjector( - VMParamsZkCredentialsInjector.class, - List.of( - DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), - List.of(ALL_USERNAME, ALL_PASSWORD + "Wrong"))); - - testInjectors( - testZkCredentialsInjectors, - false, - false, - false, - false, - false, - false, - false, - false, - false, - false); - } - - @Test - public void testAllCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(AllCredentialZkCredentialsInjector.class), - new TestZkCredentialsInjector( - VMParamsZkCredentialsInjector.class, - List.of( - DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), - List.of(ALL_USERNAME, ALL_PASSWORD))); - - testInjectors( - testZkCredentialsInjectors, true, true, true, true, true, true, true, true, true, true); - } - - @Test - public void testReadonlyCredentials() throws Exception { - List testZkCredentialsInjectors = - List.of( - new TestZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class), - new TestZkCredentialsInjector( - VMParamsZkCredentialsInjector.class, - List.of( - DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME), - List.of(READONLY_USERNAME, READONLY_PASSWORD))); - testInjectors( - testZkCredentialsInjectors, - true, - true, - false, - false, - false, - false, - false, - false, - false, - false); - } - - protected void testInjectors( - List testZkCredentialsInjectors, - boolean getData, - boolean list, - boolean create, - boolean setData, - boolean delete, - boolean secureGet, - boolean secureList, - boolean secureCreate, - boolean secureSet, - boolean secureDelete) - throws Exception { - for (TestZkCredentialsInjector testZkCredentialsInjector : testZkCredentialsInjectors) { - tearDown(); - setUp(); - testZkCredentialsInjector.setSystemProps(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - doTest( - zkClient, - getData, - list, - create, - setData, - delete, - secureGet, - secureList, - secureCreate, - secureSet, - secureDelete); - } - } - } - - @Test - public void testRepairACL() throws Exception { - clearSecuritySystemProperties(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - // Currently, no credentials on ZK connection, because those same VM-params are used for - // adding ACLs, and here we want - // no (or completely open) ACLs added. Therefore, hack your way into being authorized for - // creating anyway - zkClient - .getCuratorFramework() - .getZookeeperClient() - .getZooKeeper() - .addAuthInfo( - "digest", - ("connectAndAllACLUsername:connectAndAllACLPassword") - .getBytes(StandardCharsets.UTF_8)); - - zkClient.create( - "/security.json", "{}".getBytes(StandardCharsets.UTF_8), CreateMode.PERSISTENT, false); - assertEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); - } - - setSecuritySystemProperties(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - ZkController.createClusterZkNodes(zkClient); - assertNotEquals(OPEN_ACL_UNSAFE, zkClient.getACL("/security.json", null, false)); - } - - useZkCredentialsInjector(ConnectWithReadonlyCredsInjector.class); - // useReadonlyCredentials(); - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - NoAuthException e = - assertThrows( - NoAuthException.class, () -> zkClient.getData("/security.json", null, null, false)); - assertEquals("/solr/security.json", e.getPath()); - } - } - - private void useZkCredentialsInjector(Class zkCredentialsInjectorClass) { - clearSecuritySystemProperties(); - setDigestZkSystemProps(); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - zkCredentialsInjectorClass.getName()); - } - - private void setSecuritySystemProperties() { - System.setProperty( - SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME, - DigestZkCredentialsProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME, DigestZkACLProvider.class.getName()); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - AllAndReadonlyCredentialZkCredentialsInjector.class.getName()); - } - - private void clearSecuritySystemProperties() { - System.clearProperty(SolrZkClient.ZK_CRED_PROVIDER_CLASS_NAME_VM_PARAM_NAME); - System.clearProperty(SolrZkClient.ZK_ACL_PROVIDER_CLASS_NAME_VM_PARAM_NAME); - System.clearProperty(SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME); - } - - public static void doTest( - SolrZkClient zkClient, - boolean getData, - boolean list, - boolean create, - boolean setData, - boolean delete, - boolean secureGet, - boolean secureList, - boolean secureCreate, - boolean secureSet, - boolean secureDelete) - throws Exception { - doTest(zkClient, "/protectedCreateNode", getData, list, create, setData, delete); - doTest(zkClient, "/protectedMakePathNode", getData, list, create, setData, delete); - doTest(zkClient, "/unprotectedCreateNode", true, true, true, true, delete); - doTest(zkClient, "/unprotectedMakePathNode", true, true, true, true, delete); - doTest( - zkClient, - SecurityAwareZkACLProvider.SECURITY_ZNODE_PATH, - secureGet, - secureList, - secureCreate, - secureSet, - secureDelete); - } - - protected static void doTest( - SolrZkClient zkClient, - String path, - boolean getData, - boolean list, - boolean create, - boolean setData, - boolean delete) - throws Exception { - doTest(getData, () -> zkClient.getData(path, null, null, false)); - doTest(list, () -> zkClient.getChildren(path, null, false)); - - doTest( - create, - () -> { - zkClient.create(path + "/subnode", null, CreateMode.PERSISTENT, false); - zkClient.delete(path + "/subnode", -1, false); - }); - doTest( - create, - () -> { - zkClient.makePath(path + "/subnode/subsubnode", false); - zkClient.delete(path + "/subnode/subsubnode", -1, false); - zkClient.delete(path + "/subnode", -1, false); - }); - - doTest(setData, () -> zkClient.setData(path, (byte[]) null, false)); - - // Actually about the ACLs on /solr, but that is protected - doTest(delete, () -> zkClient.delete(path, -1, false)); - } - - interface ExceptingRunnable { - void run() throws Exception; - } - - private static void doTest(boolean shouldSucceed, ExceptingRunnable action) throws Exception { - if (shouldSucceed) { - action.run(); - } else { - expectThrows(NoAuthException.class, action::run); - } - } - - @Test - public void testVMParamsAllCredentialsFromFile() throws Exception { - useVMParamsAllCredentialsFromFile(); - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - doTest(zkClient, true, true, true, true, true, true, true, true, true, true); - } - } - - @Test - public void testVMParamsReadonlyCredentialsFromFile() throws Exception { - useVMParamsReadonlyCredentialsFromFile(); - - try (SolrZkClient zkClient = - new SolrZkClient.Builder() - .withUrl(zkServer.getZkAddress()) - .withTimeout(AbstractZkTestCase.TIMEOUT, TimeUnit.MILLISECONDS) - .build()) { - doTest(zkClient, true, true, false, false, false, false, false, false, false, false); - } - } - - private void useVMParamsAllCredentialsFromFile() throws IOException { - useVMParamsCredentialsFromFile("connectAndAllACLUsername", "connectAndAllACLPassword"); - } - - private void useVMParamsReadonlyCredentialsFromFile() throws IOException { - useVMParamsCredentialsFromFile("readonlyACLUsername", "readonlyACLPassword"); - } - - private void useVMParamsCredentialsFromFile(String username, String password) throws IOException { - Properties props = new Properties(); - props.setProperty( - VMParamsZkCredentialsInjector.DEFAULT_DIGEST_USERNAME_VM_PARAM_NAME, username); - props.setProperty( - VMParamsZkCredentialsInjector.DEFAULT_DIGEST_PASSWORD_VM_PARAM_NAME, password); - String credsFile = saveCredentialsFile(props); - - useZkCredentialsInjector(VMParamsZkCredentialsInjector.class); - System.setProperty(DEFAULT_DIGEST_FILE_VM_PARAM_NAME, credsFile); - } - - private String saveCredentialsFile(Properties props) throws IOException { - Path tmp = createTempFile("zk-creds", "properties"); - try (FileWriter w = new FileWriter(tmp.toFile(), StandardCharsets.UTF_8)) { - props.store(w, "test"); - } - return tmp.toAbsolutePath().toString(); - } - - public static class NoCredentialZkCredentialsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return Collections.emptyList(); - } - } - - public static class AllAndReadonlyCredentialZkCredentialsInjector - implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of( - new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL), - new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.READ)); - } - } - - public static class ConnectWithReadonlyCredsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of( - new ZkCredential(READONLY_USERNAME, READONLY_PASSWORD, ZkCredential.Perms.ALL)); - } - } - - public static class AllCredentialZkCredentialsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of(new ZkCredential(ALL_USERNAME, ALL_PASSWORD, ZkCredential.Perms.ALL)); - } - } - - public static class WrongAllCredentialZkCredentialsInjector implements ZkCredentialsInjector { - @Override - public List getZkCredentials() { - return List.of( - new ZkCredential(ALL_USERNAME, ALL_PASSWORD + "Wrong", ZkCredential.Perms.ALL)); - } - } - - class TestZkCredentialsInjector { - private final Class zkCredentialsInjectorClass; - private final List systemPropsKeys; - private final List systemPropsValues; - - public TestZkCredentialsInjector(Class zkCredentialsInjectorClass) { - this(zkCredentialsInjectorClass, Collections.emptyList(), Collections.emptyList()); - } - - public TestZkCredentialsInjector( - Class zkCredentialsInjectorClass, - List systemPropsKeys, - List systemPropsValues) { - this.zkCredentialsInjectorClass = zkCredentialsInjectorClass; - this.systemPropsKeys = systemPropsKeys; - this.systemPropsValues = systemPropsValues; - } - - private void setSystemProps() { - clearSecuritySystemProperties(); - setDigestZkSystemProps(); - System.setProperty( - SolrZkClient.ZK_CREDENTIALS_INJECTOR_CLASS_NAME_VM_PARAM_NAME, - zkCredentialsInjectorClass.getName()); - int i = 0; - for (String key : systemPropsKeys) { - System.setProperty(key, systemPropsValues.get(i++)); - } - } - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java index e123fa6d53f..5a24a1f2f4d 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java @@ -133,6 +133,12 @@ public abstract class AbstractFullDistribZkTestBase extends BaseDistributedSearc protected volatile ZkTestServer zkServer; private final AtomicInteger homeCount = new AtomicInteger(); + @BeforeClass + public static void beforeThisClass() throws Exception { + // Only For Manual Testing: this will force an fs based dir factory + // useFactory(null); + } + @BeforeClass public static void beforeFullSolrCloudTest() {} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java deleted file mode 100644 index da9ed3da1fe..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractMoveReplicaTestBase.java +++ /dev/null @@ -1,410 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.apache.solr.client.api.model.CoreStatusResponse; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.response.CoreAdminResponse; -import org.apache.solr.client.solrj.response.RequestStatusState; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.util.NamedList; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.IdUtils; -import org.junit.After; -import org.junit.Before; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractMoveReplicaTestBase extends SolrCloudTestCase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - // used by MoveReplicaHDFSTest - protected boolean inPlaceMove = true; - protected boolean isCollectionApiDistributed = false; - - protected String getConfigSet() { - return "cloud-dynamic"; - } - - @Before - public void beforeTest() throws Exception { - inPlaceMove = true; - - configureCluster(4) - .addConfig("conf1", configset(getConfigSet())) - .addConfig("conf2", configset(getConfigSet())) - .withSolrXml(TEST_PATH().resolve("solr.xml")) - .configure(); - - // If Collection API is distributed let's not wait for Overseer. - isCollectionApiDistributed = - new CollectionAdminRequest.RequestApiDistributedProcessing() - .process(cluster.getSolrClient()) - .getIsCollectionApiDistributed(); - if (isCollectionApiDistributed) { - return; - } - - NamedList overSeerStatus = - cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - JettySolrRunner overseerJetty = null; - String overseerLeader = (String) overSeerStatus.get("leader"); - for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { - if (jetty.getNodeName().equals(overseerLeader)) { - overseerJetty = jetty; - break; - } - } - if (overseerJetty == null) { - fail("no overseer leader!"); - } - } - - @After - public void afterTest() throws Exception { - try { - shutdownCluster(); - } finally { - super.tearDown(); - } - } - - @Test - public void test() throws Exception { - String coll = getTestClass().getSimpleName() + "_coll_" + inPlaceMove; - if (log.isInfoEnabled()) { - log.info("total_jettys: {}", cluster.getJettySolrRunners().size()); - } - int REPLICATION = 2; - - CloudSolrClient cloudClient = cluster.getSolrClient(); - - // random create tlog or pull type replicas with nrt - boolean isTlog = random().nextBoolean(); - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection( - coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); - cloudClient.request(create); - - addDocs(coll, 100); - - Replica replica = getRandomReplica(coll, cloudClient); - Set liveNodes = cloudClient.getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - Collections.shuffle(l, random()); - String targetNode = null; - for (String node : liveNodes) { - if (!replica.getNodeName().equals(node)) { - targetNode = node; - break; - } - } - assertNotNull(targetNode); - String shardId = null; - for (Slice slice : cloudClient.getClusterState().getCollection(coll).getSlices()) { - if (slice.getReplicas().contains(replica)) { - shardId = slice.getName(); - } - } - - int sourceNumCores = - getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name()); - int targetNumCores = getNumOfCores(cloudClient, targetNode, coll, replica.getType().name()); - - CollectionAdminRequest.MoveReplica moveReplica = - createMoveReplicaRequest(coll, replica, targetNode); - moveReplica.setInPlaceMove(inPlaceMove); - String asyncId = IdUtils.randomId(); - moveReplica.processAsync(asyncId, cloudClient); - CollectionAdminRequest.RequestStatus requestStatus = - CollectionAdminRequest.requestStatus(asyncId); - // wait for async request success - boolean success = false; - for (int i = 0; i < 200; i++) { - CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); - if (rsp.getRequestStatus() == RequestStatusState.COMPLETED) { - success = true; - break; - } - assertNotSame(rsp.getRequestStatus(), RequestStatusState.FAILED); - Thread.sleep(500); - } - assertTrue(success); - assertEquals( - "should be one less core on the source node!", - sourceNumCores - 1, - getNumOfCores(cloudClient, replica.getNodeName(), coll, replica.getType().name())); - assertEquals( - "should be one more core on target node!", - targetNumCores + 1, - getNumOfCores(cloudClient, targetNode, coll, replica.getType().name())); - // wait for recovery - boolean recovered = false; - for (int i = 0; i < 300; i++) { - DocCollection collState = getCollectionState(coll); - log.debug("###### {}", collState); - Collection replicas = collState.getSlice(shardId).getReplicas(); - boolean allActive = true; - boolean hasLeaders = true; - if (replicas != null && !replicas.isEmpty()) { - for (Replica r : replicas) { - if (!r.getNodeName().equals(targetNode)) { - continue; - } - if (!r.isActive(Collections.singleton(targetNode))) { - log.info("Not active: {}", r); - allActive = false; - } - } - } else { - allActive = false; - } - for (Slice slice : collState.getSlices()) { - if (slice.getLeader() == null) { - hasLeaders = false; - } - } - if (allActive && hasLeaders) { - // check the number of active replicas - assertEquals("total number of replicas", REPLICATION, replicas.size()); - recovered = true; - break; - } else { - log.info("--- waiting, allActive={}, hasLeaders={}", allActive, hasLeaders); - Thread.sleep(1000); - } - } - assertTrue("replica never fully recovered", recovered); - - assertEquals( - 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); - - moveReplica = createMoveReplicaRequest(coll, replica, targetNode, shardId); - moveReplica.setInPlaceMove(inPlaceMove); - moveReplica.process(cloudClient); - checkNumOfCores(cloudClient, replica.getNodeName(), coll, sourceNumCores); - // wait for recovery - recovered = false; - for (int i = 0; i < 300; i++) { - DocCollection collState = getCollectionState(coll); - log.debug("###### {}", collState); - Collection replicas = collState.getSlice(shardId).getReplicas(); - boolean allActive = true; - boolean hasLeaders = true; - if (replicas != null && !replicas.isEmpty()) { - for (Replica r : replicas) { - if (!r.getNodeName().equals(replica.getNodeName())) { - continue; - } - if (!r.isActive(Collections.singleton(replica.getNodeName()))) { - log.info("Not active yet: {}", r); - allActive = false; - } - } - } else { - allActive = false; - } - for (Slice slice : collState.getSlices()) { - if (slice.getLeader() == null) { - hasLeaders = false; - } - } - if (allActive && hasLeaders) { - assertEquals("total number of replicas", REPLICATION, replicas.size()); - recovered = true; - break; - } else { - Thread.sleep(1000); - } - } - assertTrue("replica never fully recovered", recovered); - - assertEquals( - 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); - } - - @Test - public void testFailedMove() throws Exception { - String coll = getTestClass().getSimpleName() + "_failed_coll_" + inPlaceMove; - int REPLICATION = 2; - - CloudSolrClient cloudClient = cluster.getSolrClient(); - - // random create tlog or pull type replicas with nrt - boolean isTlog = random().nextBoolean(); - CollectionAdminRequest.Create create = - CollectionAdminRequest.createCollection( - coll, "conf1", 2, 1, isTlog ? 1 : 0, !isTlog ? 1 : 0); - cloudClient.request(create); - - addDocs(coll, 100); - - NamedList overSeerStatus = - cluster.getSolrClient().request(CollectionAdminRequest.getOverseerStatus()); - String overseerLeader = (String) overSeerStatus.get("leader"); - - // don't kill overseer in this test - Replica replica; - int count = 10; - do { - replica = getRandomReplica(coll, cloudClient); - } while (!replica.getNodeName().equals(overseerLeader) && count-- > 0); - assertNotNull("could not find non-overseer replica???", replica); - Set liveNodes = cloudClient.getClusterState().getLiveNodes(); - ArrayList l = new ArrayList<>(liveNodes); - Collections.shuffle(l, random()); - String targetNode = null; - for (String node : liveNodes) { - if (!replica.getNodeName().equals(node) - && (isCollectionApiDistributed || !overseerLeader.equals(node))) { - targetNode = node; - break; - } - } - assertNotNull(targetNode); - CollectionAdminRequest.MoveReplica moveReplica = - createMoveReplicaRequest(coll, replica, targetNode); - moveReplica.setInPlaceMove(inPlaceMove); - // start moving - String asyncId = IdUtils.randomId(); - moveReplica.processAsync(asyncId, cloudClient); - // shut down target node - for (int i = 0; i < cluster.getJettySolrRunners().size(); i++) { - if (cluster.getJettySolrRunner(i).getNodeName().equals(targetNode)) { - JettySolrRunner j = cluster.stopJettySolrRunner(i); - cluster.waitForJettyToStop(j); - break; - } - } - CollectionAdminRequest.RequestStatus requestStatus = - CollectionAdminRequest.requestStatus(asyncId); - // wait for async request success - boolean success = true; - for (int i = 0; i < 200; i++) { - CollectionAdminRequest.RequestStatusResponse rsp = requestStatus.process(cloudClient); - assertNotSame( - rsp.getRequestStatus().toString(), rsp.getRequestStatus(), RequestStatusState.COMPLETED); - if (rsp.getRequestStatus() == RequestStatusState.FAILED) { - success = false; - break; - } - Thread.sleep(500); - } - assertFalse(success); - - if (log.isInfoEnabled()) { - log.info( - "--- current collection state: {}", cloudClient.getClusterState().getCollection(coll)); - } - assertEquals( - 100, cluster.getSolrClient().query(coll, new SolrQuery("*:*")).getResults().getNumFound()); - } - - private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( - String coll, Replica replica, String targetNode, String shardId) { - return new CollectionAdminRequest.MoveReplica(coll, shardId, targetNode, replica.getNodeName()); - } - - private CollectionAdminRequest.MoveReplica createMoveReplicaRequest( - String coll, Replica replica, String targetNode) { - return new CollectionAdminRequest.MoveReplica(coll, replica.getName(), targetNode); - } - - private Replica getRandomReplica(String coll, CloudSolrClient cloudClient) throws IOException { - List replicas = cloudClient.getClusterState().getCollection(coll).getReplicas(); - Collections.shuffle(replicas, random()); - return replicas.get(0); - } - - private void checkNumOfCores( - CloudSolrClient cloudClient, String nodeName, String collectionName, int expectedCores) - throws IOException, SolrServerException { - assertEquals( - nodeName + " does not have expected number of cores", - expectedCores, - getNumOfCores(cloudClient, nodeName, collectionName)); - } - - private int getNumOfCores(CloudSolrClient cloudClient, String nodeName, String collectionName) - throws IOException, SolrServerException { - return getNumOfCores(cloudClient, nodeName, collectionName, null); - } - - private int getNumOfCores( - CloudSolrClient cloudClient, String nodeName, String collectionName, String replicaType) - throws IOException, SolrServerException { - try (SolrClient coreclient = - getHttpSolrClient(ZkStateReader.from(cloudClient).getBaseUrlForNodeName(nodeName))) { - CoreAdminResponse status = CoreAdminRequest.getStatus(null, coreclient); - if (status.getCoreStatus().size() == 0) { - return 0; - } - if (collectionName == null && replicaType == null) { - return status.getCoreStatus().size(); - } - // filter size by collection name - int size = 0; - for (Map.Entry coreStatusEntry : - status.getCoreStatus().entrySet()) { - if (collectionName != null) { - String coll = coreStatusEntry.getValue().cloud.collection; - if (!collectionName.equals(coll)) { - continue; - } - } - if (replicaType != null) { - String type = coreStatusEntry.getValue().cloud.replicaType; - if (!replicaType.equals(type)) { - continue; - } - } - size++; - } - return size; - } - } - - protected void addDocs(String collection, int numDocs) throws Exception { - SolrClient solrClient = cluster.getSolrClient(); - for (int docId = 1; docId <= numDocs; docId++) { - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", docId); - solrClient.add(collection, doc); - } - solrClient.commit(collection); - Thread.sleep(5000); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java deleted file mode 100644 index 545a9486c41..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRecoveryZkTestBase.java +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.apache.CloudLegacySolrClient; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.cloud.DocCollection; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.embedded.JettySolrRunner; -import org.junit.After; -import org.junit.BeforeClass; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class AbstractRecoveryZkTestBase extends SolrCloudTestCase { - - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - @BeforeClass - public static void setupCluster() throws Exception { - cluster = configureCluster(2).addConfig("conf", configset("cloud-minimal")).configure(); - } - - private final List threads = new ArrayList<>(); - - @After - public void stopThreads() throws InterruptedException { - for (StoppableIndexingThread t : threads) { - t.safeStop(); - } - for (StoppableIndexingThread t : threads) { - t.join(); - } - threads.clear(); - } - - @Test - public void test() throws Exception { - - final String collection = "recoverytest"; - - CollectionAdminRequest.createCollection(collection, "conf", 1, 2) - .process(cluster.getSolrClient()); - waitForState( - "Expected a collection with one shard and two replicas", collection, clusterShape(1, 2)); - - // start a couple indexing threads - - int[] maxDocList = new int[] {300, 700, 1200, 1350, 3000}; - int[] maxDocNightlyList = new int[] {3000, 7000, 12000, 30000, 45000, 60000}; - - int maxDoc; - if (!TEST_NIGHTLY) { - maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; - } else { - maxDoc = maxDocNightlyList[random().nextInt(maxDocList.length - 1)]; - } - log.info("Indexing {} documents", maxDoc); - - try (SolrClient solrClient = - cluster.basicSolrClientBuilder().withDefaultCollection(collection).build(); ) { - final StoppableIndexingThread indexThread = - new StoppableIndexingThread(null, solrClient, "1", true, maxDoc, 1, true); - threads.add(indexThread); - indexThread.start(); - - final StoppableIndexingThread indexThread2 = - new StoppableIndexingThread(null, solrClient, "2", true, maxDoc, 1, true); - threads.add(indexThread2); - indexThread2.start(); - - // give some time to index... - int[] waitTimes = new int[] {200, 2000, 3000}; - Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); - - // bring shard replica down - DocCollection state = getCollectionState(collection); - Replica leader = state.getLeader("shard1"); - Replica replica = getRandomReplica(state.getSlice("shard1"), (r) -> !leader.equals(r)); - - JettySolrRunner jetty = cluster.getReplicaJetty(replica); - jetty.stop(); - - // wait a moment - lets allow some docs to be indexed so replication time is non 0 - Thread.sleep(waitTimes[random().nextInt(waitTimes.length - 1)]); - - // bring shard replica up - jetty.start(); - - // make sure replication can start - Thread.sleep(3000); - - // stop indexing threads - indexThread.safeStop(); - indexThread2.safeStop(); - - indexThread.join(); - indexThread2.join(); - - new UpdateRequest().commit(solrClient, collection); - - cluster - .getZkStateReader() - .waitForState(collection, 120, TimeUnit.SECONDS, clusterShape(1, 2)); - - // test that leader and replica have same doc count - state = getCollectionState(collection); - assertShardConsistency(state.getSlice("shard1"), true); - } - } - - private void assertShardConsistency(Slice shard, boolean expectDocs) throws Exception { - List replicas = shard.getReplicas(r -> r.getState() == Replica.State.ACTIVE); - long[] numCounts = new long[replicas.size()]; - int i = 0; - for (Replica replica : replicas) { - try (var client = - new HttpSolrClient.Builder(replica.getBaseUrl()) - .withDefaultCollection(replica.getCoreName()) - .withHttpClient(((CloudLegacySolrClient) cluster.getSolrClient()).getHttpClient()) - .build()) { - numCounts[i] = - client.query(new SolrQuery("*:*").add("distrib", "false")).getResults().getNumFound(); - i++; - } - } - for (int j = 1; j < replicas.size(); j++) { - if (numCounts[j] != numCounts[j - 1]) - fail("Mismatch in counts between replicas"); // TODO improve this! - if (numCounts[j] == 0 && expectDocs) - fail("Expected docs on shard " + shard.getName() + " but found none"); - } - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java deleted file mode 100644 index 38733b30fbe..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractRestartWhileUpdatingTestBase.java +++ /dev/null @@ -1,199 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public abstract class AbstractRestartWhileUpdatingTestBase extends AbstractFullDistribZkTestBase { - - // private static final String DISTRIB_UPDATE_CHAIN = "distrib-update-chain"; - private List threads; - - private volatile boolean stopExpire = false; - - public AbstractRestartWhileUpdatingTestBase() throws Exception { - super(); - sliceCount = 1; - fixShardCount(3); - schemaString = "schema15.xml"; // we need a string id - useFactory("solr.StandardDirectoryFactory"); - } - - public static String[] fieldNames = new String[] {"f_i", "f_f", "f_d", "f_l", "f_dt"}; - public static RandVal[] randVals = new RandVal[] {rint, rfloat, rdouble, rlong, rdate}; - - @Override - protected String[] getFieldNames() { - return fieldNames; - } - - @Override - protected RandVal[] getRandValues() { - return randVals; - } - - @BeforeClass - public static void beforeRestartWhileUpdatingTest() { - System.setProperty("leaderVoteWait", "300000"); - System.setProperty("solr.autoCommit.maxTime", "30000"); - System.setProperty("solr.autoSoftCommit.maxTime", "3000"); - // SOLR-13212 // TestInjection.nonGracefullClose = "true:60"; - // SOLR-13189 // TestInjection.failReplicaRequests = "true:03"; - } - - @AfterClass - public static void afterRestartWhileUpdatingTest() { - System.clearProperty("leaderVoteWait"); - System.clearProperty("solr.autoCommit.maxTime"); - System.clearProperty("solr.autoSoftCommit.maxTime"); - } - - @Test - public void test() throws Exception { - handle.clear(); - handle.put("timestamp", SKIPVAL); - - // start a couple indexing threads - - int[] maxDocList = new int[] {5000, 10000}; - - int maxDoc = maxDocList[random().nextInt(maxDocList.length - 1)]; - - int numThreads = random().nextInt(4) + 1; - - threads = new ArrayList<>(numThreads); - - Thread expireThread = - new Thread("expireThread") { - @Override - public void run() { - while (!stopExpire) { - try { - Thread.sleep(random().nextInt(15000)); - } catch (InterruptedException e) { - throw new RuntimeException(e); - } - - // try { - // chaosMonkey.expireRandomSession(); - // } catch (KeeperException e) { - // throw new RuntimeException(e); - // } catch (InterruptedException e) { - // throw new RuntimeException(e); - // } - } - } - }; - - // Currently unused - // expireThread.start(); - - StoppableIndexingThread indexThread; - for (int i = 0; i < numThreads; i++) { - indexThread = - new StoppableIndexingThread( - controlClient, cloudClient, Integer.toString(i), true, maxDoc, 1, true); - threads.add(indexThread); - indexThread.start(); - } - - Thread.sleep(2000); - - int restartTimes = 1; // random().nextInt(4) + 1;; - for (int i = 0; i < restartTimes; i++) { - Thread.sleep(random().nextInt(30000)); - stopAndStartAllReplicas(); - Thread.sleep(random().nextInt(30000)); - } - - Thread.sleep(2000); - - // stop indexing threads - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - stopExpire = true; - expireThread.join(); - - Thread.sleep(1000); - - waitForThingsToLevelOut(320, TimeUnit.SECONDS); - - Thread.sleep(2000); - - waitForThingsToLevelOut(30, TimeUnit.SECONDS); - - Thread.sleep(5000); - - waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); - - for (StoppableIndexingThread thread : threads) { - thread.join(); - } - - checkShardConsistency(false, false); - } - - public void stopAndStartAllReplicas() throws Exception, InterruptedException { - chaosMonkey.stopAll(random().nextInt(1)); - - if (random().nextBoolean()) { - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - } - Thread.sleep(1000); - - chaosMonkey.startAll(); - } - - @Override - protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { - cloudClient.add(doc); - } - - @Override - public void distribTearDown() throws Exception { - // make sure threads have been stopped... - if (threads != null) { - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - thread.safeStop(); - } - } - - super.distribTearDown(); - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java deleted file mode 100644 index c0c41f2eb22..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractSyncSliceTestBase.java +++ /dev/null @@ -1,300 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import java.util.stream.Collectors; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrRequest; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.GenericSolrRequest; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.client.solrj.request.UpdateRequest; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionParams.CollectionAction; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.util.LogLevel; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** Test sync phase that occurs when Leader goes down and a new Leader is elected. */ -@LogLevel("org.apache.solr.update.processor.DistributedZkUpdateProcessor=WARN") -public abstract class AbstractSyncSliceTestBase extends AbstractFullDistribZkTestBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - private boolean success = false; - - @Override - public void distribTearDown() throws Exception { - if (!success) { - printLayoutOnTearDown = true; - } - super.distribTearDown(); - } - - public AbstractSyncSliceTestBase() { - super(); - sliceCount = 1; - fixShardCount(TEST_NIGHTLY ? 7 : 4); - } - - @Test - public void test() throws Exception { - - handle.clear(); - handle.put("timestamp", SKIPVAL); - - waitForThingsToLevelOut(30, TimeUnit.SECONDS); - - del("*:*"); - List skipServers = new ArrayList<>(); - int docId = 0; - indexDoc( - skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); - - indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "old haven was blue."); - - skipServers.add(shardToJetty.get("shard1").get(1)); - - indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "but the song was fancy."); - - skipServers.add(shardToJetty.get("shard1").get(2)); - - indexDoc(skipServers, id, docId++, i1, 50, tlong, 50, t1, "under the moon and over the lake"); - - commit(); - - waitForRecoveriesToFinish(false); - - // shard should be inconsistent - String shardFailMessage = checkShardConsistency("shard1", true, false); - assertNotNull(shardFailMessage); - - ModifiableSolrParams params = new ModifiableSolrParams(); - params.set("action", CollectionAction.SYNCSHARD.toString()); - params.set("collection", "collection1"); - params.set("shard", "shard1"); - final var request = - new GenericSolrRequest(SolrRequest.METHOD.GET, "/admin/collections", params); - - String baseUrl = shardToJetty.get(SHARD1).get(2).jetty.getBaseUrl().toString(); - - // we only set the connect timeout, not so timeout - try (SolrClient baseClient = - new HttpSolrClient.Builder(baseUrl) - .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - baseClient.request(request); - } - - waitForThingsToLevelOut(15, TimeUnit.SECONDS); - - checkShardConsistency(false, true); - - long cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - assertEquals(4, cloudClientDocs); - - // kill the leader - new leader could have all the docs or be missing one - CloudJettyRunner leaderJetty = shardToLeaderJetty.get("shard1"); - - skipServers = getRandomOtherJetty(leaderJetty, null); // but not the leader - - // this doc won't be on one node - indexDoc( - skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); - commit(); - - Set jetties = new HashSet<>(); - jetties.addAll(shardToJetty.get("shard1")); - jetties.remove(leaderJetty); - assertEquals(getShardCount() - 1, jetties.size()); - - leaderJetty.jetty.stop(); - - Thread.sleep(3000); - - waitForNoShardInconsistency(); - - Thread.sleep(1000); - - checkShardConsistency(false, true); - - cloudClientDocs = cloudClient.query(new SolrQuery("*:*")).getResults().getNumFound(); - assertEquals(5, cloudClientDocs); - - CloudJettyRunner deadJetty = leaderJetty; - - // let's get the latest leader - while (deadJetty.equals(leaderJetty)) { - updateMappingsFromZk(this.jettys, this.clients); - leaderJetty = shardToLeaderJetty.get("shard1"); - } - - // bring back dead node - deadJetty.jetty.start(); // he is not the leader anymore - - waitTillAllNodesActive(); - - skipServers = getRandomOtherJetty(leaderJetty, deadJetty); - skipServers.addAll(getRandomOtherJetty(leaderJetty, deadJetty)); - // skip list should be - - // System.out.println("leader:" + leaderJetty.url); - // System.out.println("dead:" + deadJetty.url); - // System.out.println("skip list:" + skipServers); - - // we are skipping 2 nodes - assertEquals(2, skipServers.size()); - - // more docs than can peer sync - for (int i = 0; i < 300; i++) { - indexDoc( - skipServers, id, docId++, i1, 50, tlong, 50, t1, "to come to the aid of their country."); - } - - commit(); - waitForRecoveriesToFinish(false); - - // shard should be inconsistent - shardFailMessage = waitTillInconsistent(); - assertNotNull( - "Test Setup Failure: shard1 should have just been set up to be inconsistent - but it's still consistent. Leader:" - + leaderJetty.url - + " Dead Guy:" - + deadJetty.url - + "skip list:" - + skipServers, - shardFailMessage); - - // good place to test compareResults - boolean shouldFail = CloudInspectUtil.compareResults(controlClient, cloudClient); - assertTrue("A test that compareResults is working correctly failed", shouldFail); - - jetties = new HashSet<>(); - jetties.addAll(shardToJetty.get("shard1")); - jetties.remove(leaderJetty); - assertEquals(getShardCount() - 1, jetties.size()); - - // kill the current leader - leaderJetty.jetty.stop(); - - waitForNoShardInconsistency(); - - checkShardConsistency(true, true); - - success = true; - } - - private void waitTillAllNodesActive() throws InterruptedException, TimeoutException { - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - - zkStateReader.waitForState( - "collection1", - 3, - TimeUnit.MINUTES, - (n, c) -> { - Collection replicas = c.getSlice("shard1").getReplicas(); - Set nodes = - replicas.stream().map(Replica::getNodeName).collect(Collectors.toSet()); - return replicas.stream().map(Replica::getState).allMatch(Replica.State.ACTIVE::equals) - && n.containsAll(nodes); - }); - } - - private String waitTillInconsistent() throws Exception, InterruptedException { - String shardFailMessage = null; - - shardFailMessage = pollConsistency(shardFailMessage, 0); - shardFailMessage = pollConsistency(shardFailMessage, 3000); - shardFailMessage = pollConsistency(shardFailMessage, 5000); - shardFailMessage = pollConsistency(shardFailMessage, 15000); - - return shardFailMessage; - } - - private String pollConsistency(String shardFailMessage, int sleep) - throws InterruptedException, Exception { - try { - commit(); - } catch (Throwable t) { - log.error("commit error", t); - } - if (shardFailMessage == null) { - // try again - Thread.sleep(sleep); - shardFailMessage = checkShardConsistency("shard1", true, false); - } - return shardFailMessage; - } - - private List getRandomOtherJetty( - CloudJettyRunner leader, CloudJettyRunner down) { - List skipServers = new ArrayList<>(); - List candidates = new ArrayList<>(); - candidates.addAll(shardToJetty.get("shard1")); - - if (leader != null) { - candidates.remove(leader); - } - - if (down != null) { - candidates.remove(down); - } - - CloudJettyRunner cjetty = candidates.get(random().nextInt(candidates.size())); - skipServers.add(cjetty); - return skipServers; - } - - protected void indexDoc(List skipServers, Object... fields) - throws IOException, SolrServerException { - SolrInputDocument doc = new SolrInputDocument(); - - addFields(doc, fields); - addFields(doc, "rnd_b", true); - - controlClient.add(doc); - - UpdateRequest ureq = new UpdateRequest(); - ureq.add(doc); - for (CloudJettyRunner skip : skipServers) { - ureq.getParams().add("test.distrib.skip.servers", skip.url + "/"); - } - ureq.process(cloudClient); - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java deleted file mode 100644 index ee73ea8e817..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractTlogReplayBufferedWhileIndexingTestBase.java +++ /dev/null @@ -1,146 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.junit.AfterClass; -import org.junit.BeforeClass; -import org.junit.Test; - -public abstract class AbstractTlogReplayBufferedWhileIndexingTestBase - extends AbstractFullDistribZkTestBase { - - private List threads; - - public AbstractTlogReplayBufferedWhileIndexingTestBase() throws Exception { - super(); - sliceCount = 1; - fixShardCount(2); - schemaString = "schema15.xml"; // we need a string id - } - - @BeforeClass - public static void beforeRestartWhileUpdatingTest() throws Exception { - System.setProperty("leaderVoteWait", "300000"); - System.setProperty("solr.autoCommit.maxTime", "10000"); - System.setProperty("solr.autoSoftCommit.maxTime", "3000"); - TestInjection.updateLogReplayRandomPause = "true:10"; - TestInjection.updateRandomPause = "true:10"; - useFactory("solr.StandardDirectoryFactory"); - } - - @AfterClass - public static void afterRestartWhileUpdatingTest() { - System.clearProperty("leaderVoteWait"); - System.clearProperty("solr.autoCommit.maxTime"); - System.clearProperty("solr.autoSoftCommit.maxTime"); - } - - @Test - public void test() throws Exception { - handle.clear(); - handle.put("timestamp", SKIPVAL); - - waitForRecoveriesToFinish(false); - - int numThreads = 3; - - threads = new ArrayList<>(numThreads); - - ArrayList allJetty = new ArrayList<>(); - allJetty.addAll(jettys); - allJetty.remove(shardToLeaderJetty.get("shard1").jetty); - assert allJetty.size() == 1 : allJetty.size(); - allJetty.get(0).stop(); - - StoppableIndexingThread indexThread; - for (int i = 0; i < numThreads; i++) { - boolean pauseBetweenUpdates = random().nextBoolean(); - int batchSize = random().nextInt(4) + 1; - indexThread = - new StoppableIndexingThread( - controlClient, - cloudClient, - Integer.toString(i), - true, - 900, - batchSize, - pauseBetweenUpdates); - threads.add(indexThread); - indexThread.start(); - } - - Thread.sleep(2000); - - allJetty.get(0).start(); - - Thread.sleep(45000); - - waitForThingsToLevelOut(); // we can insert random update delays, so this can take a while, - // especially when beasting this test - - Thread.sleep(2000); - - waitForRecoveriesToFinish(DEFAULT_COLLECTION, ZkStateReader.from(cloudClient), false, true); - - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - - waitForThingsToLevelOut(30, TimeUnit.SECONDS); - - checkShardConsistency(false, false); - } - - @Override - protected void indexDoc(SolrInputDocument doc) throws IOException, SolrServerException { - cloudClient.add(doc); - } - - @Override - public void distribTearDown() throws Exception { - // make sure threads have been stopped... - if (threads != null) { - for (StoppableIndexingThread thread : threads) { - thread.safeStop(); - } - - for (StoppableIndexingThread thread : threads) { - thread.join(); - } - } - - super.distribTearDown(); - } - - // skip the randoms - they can deadlock... - @Override - protected void indexr(Object... fields) throws Exception { - SolrInputDocument doc = new SolrInputDocument(); - addFields(doc, fields); - addFields(doc, "rnd_b", true); - indexDoc(doc); - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java deleted file mode 100644 index 6477af7aac6..00000000000 --- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractUnloadDistributedZkTestBase.java +++ /dev/null @@ -1,463 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.solr.cloud; - -import java.io.IOException; -import java.lang.invoke.MethodHandles; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Collections; -import java.util.Random; -import java.util.Set; -import java.util.concurrent.SynchronousQueue; -import java.util.concurrent.ThreadPoolExecutor; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; -import org.apache.solr.client.solrj.SolrClient; -import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.apache.HttpSolrClient; -import org.apache.solr.client.solrj.request.CollectionAdminRequest; -import org.apache.solr.client.solrj.request.CoreAdminRequest.Unload; -import org.apache.solr.client.solrj.request.SolrQuery; -import org.apache.solr.common.SolrInputDocument; -import org.apache.solr.common.cloud.Replica; -import org.apache.solr.common.cloud.Slice; -import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.ExecutorUtil; -import org.apache.solr.common.util.SolrNamedThreadFactory; -import org.apache.solr.core.SolrCore; -import org.apache.solr.core.SolrPaths; -import org.apache.solr.embedded.JettySolrRunner; -import org.apache.solr.util.TestInjection; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * This test simply does a bunch of basic things in solrcloud mode and asserts things work as - * expected. - */ -public abstract class AbstractUnloadDistributedZkTestBase extends AbstractFullDistribZkTestBase { - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - - public AbstractUnloadDistributedZkTestBase() { - super(); - fixShardCount(4); // needs at least 4 servers - } - - @Override - protected String getSolrXml() { - return "solr.xml"; - } - - @Test - public void test() throws Exception { - jettys.forEach( - j -> { - Set allowPath = j.getCoreContainer().getAllowPaths(); - allowPath.clear(); - allowPath.add(SolrPaths.ALL_PATH); // Allow non-standard core instance path - }); - log.info("###Starting testCoreUnloadAndLeaders"); - testCoreUnloadAndLeaders(); // long - log.info("###Starting testUnloadOfCores"); - testUnloadLotsOfCores(); // long - log.info("###Starting testUnloadShardAndCollection"); - testUnloadShardAndCollection(); - } - - /** - * @param url a Solr node base URL. Should not contain a core or collection name. - */ - private SolrClient newSolrClient(String url) { - return new HttpSolrClient.Builder(url) - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .build(); - } - - private void checkCoreNamePresenceAndSliceCount( - String collectionName, String coreName, boolean shouldBePresent, int expectedSliceCount) - throws Exception { - ZkStateReader reader = ZkStateReader.from(cloudClient); - try { - reader.waitForState( - collectionName, - 45, - TimeUnit.SECONDS, - c -> { - final Collection slices = (c != null) ? c.getSlices() : Collections.emptyList(); - if (expectedSliceCount == slices.size()) { - for (Slice slice : slices) { - for (Replica replica : slice.getReplicas()) { - if (coreName.equals(replica.get("core"))) { - return shouldBePresent; - } - } - } - return !shouldBePresent; - } else { - return false; - } - }); - } catch (TimeoutException e) { - printLayout(); - fail( - "checkCoreNamePresenceAndSliceCount failed:" - + " collection=" - + collectionName - + " CoreName=" - + coreName - + " shouldBePresent=" - + shouldBePresent); - } - } - - private void testUnloadShardAndCollection() throws Exception { - final int numShards = 2; - - final String collection = "test_unload_shard_and_collection"; - - final String coreName1 = collection + "_1"; - final String coreName2 = collection + "_2"; - - assertEquals( - 0, - CollectionAdminRequest.createCollection(collection, "conf1", numShards, 1) - .setCreateNodeSet("") - .process(cloudClient) - .getStatus()); - assertTrue( - CollectionAdminRequest.addReplicaToShard(collection, "shard1") - .setCoreName(coreName1) - .setNode(jettys.get(0).getNodeName()) - .process(cloudClient) - .isSuccess()); - - assertTrue( - CollectionAdminRequest.addReplicaToShard(collection, "shard2") - .setCoreName(coreName2) - .setNode(jettys.get(0).getNodeName()) - .process(cloudClient) - .isSuccess()); - - // does not mean they are active and up yet :* - waitForRecoveriesToFinish(collection, false); - - final boolean unloadInOrder = random().nextBoolean(); - final String unloadCmdCoreName1 = (unloadInOrder ? coreName1 : coreName2); - final String unloadCmdCoreName2 = (unloadInOrder ? coreName2 : coreName1); - - try (SolrClient adminClient = getHttpSolrClient(buildUrl(jettys.get(0).getLocalPort()))) { - // now unload one of the two - Unload unloadCmd = new Unload(false); - unloadCmd.setCoreName(unloadCmdCoreName1); - adminClient.request(unloadCmd); - - // there should still be two shards (as of SOLR-5209) - checkCoreNamePresenceAndSliceCount( - collection, - unloadCmdCoreName1, - false /* shouldBePresent */, - numShards /* expectedSliceCount */); - - // now unload one of the other - unloadCmd = new Unload(false); - unloadCmd.setCoreName(unloadCmdCoreName2); - adminClient.request(unloadCmd); - checkCoreNamePresenceAndSliceCount( - collection, - unloadCmdCoreName2, - false /* shouldBePresent */, - numShards /* expectedSliceCount */); - } - - // printLayout(); - // the collection should still be present (as of SOLR-5209 replica removal does not cascade to - // remove the slice and collection) - getCommonCloudSolrClient(); - assertTrue( - "No longer found collection " + collection, - cloudClient.getClusterState().hasCollection(collection)); - } - - protected SolrCore getFirstCore(String collection, JettySolrRunner jetty) { - SolrCore solrCore = null; - for (SolrCore core : jetty.getCoreContainer().getCores()) { - if (core.getName().startsWith(collection)) { - solrCore = core; - } - } - return solrCore; - } - - /** - * @throws Exception on any problem - */ - private void testCoreUnloadAndLeaders() throws Exception { - JettySolrRunner jetty1 = jettys.get(0); - - assertEquals( - 0, - CollectionAdminRequest.createCollection("unloadcollection", "conf1", 1, 1) - .setCreateNodeSet(jetty1.getNodeName()) - .process(cloudClient) - .getStatus()); - getCommonCloudSolrClient(); - ZkStateReader zkStateReader = ZkStateReader.from(cloudClient); - - zkStateReader.forceUpdateCollection("unloadcollection"); - - int slices = - zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); - assertEquals(1, slices); - SolrCore solrCore = getFirstCore("unloadcollection", jetty1); - String core1DataDir = solrCore.getDataDir(); - - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName("unloadcollection_shard1_replica2") - .setNode(jettys.get(1).getNodeName()) - .process(cloudClient) - .isSuccess()); - zkStateReader.forceUpdateCollection("unloadcollection"); - slices = zkStateReader.getClusterState().getCollection("unloadcollection").getSlices().size(); - assertEquals(1, slices); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - Replica leader = getLeaderFromZk("unloadcollection", "shard1"); - - Random random = random(); - if (random.nextBoolean()) { - try (SolrClient collectionClient = - getHttpSolrClient(leader.getBaseUrl(), leader.getCoreName())) { - // lets try and use the solrj client to index and retrieve a couple - // documents - SolrInputDocument doc1 = - getDoc(id, 6, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); - SolrInputDocument doc2 = - getDoc(id, 7, i1, -600, tlong, 600, t1, "humpty dumpy3 sat on a walls"); - SolrInputDocument doc3 = - getDoc(id, 8, i1, -600, tlong, 600, t1, "humpty dumpy2 sat on a walled"); - collectionClient.add(doc1); - collectionClient.add(doc2); - collectionClient.add(doc3); - collectionClient.commit(); - } - } - - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName("unloadcollection_shard1_replica3") - .setNode(jettys.get(2).getNodeName()) - .process(cloudClient) - .isSuccess()); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - // so that we start with some versions when we reload... - TestInjection.skipIndexWriterCommitOnClose = true; - - try (SolrClient addClient = - new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) - .withDefaultCollection("unloadcollection_shard1_replica3") - .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - - // add a few docs - for (int x = 20; x < 100; x++) { - SolrInputDocument doc1 = - getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); - addClient.add(doc1); - } - } - // don't commit so they remain in the tran log - // collectionClient.commit(); - - // unload the leader - try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { - - Unload unloadCmd = new Unload(false); - unloadCmd.setCoreName(leader.getCoreName()); - ModifiableSolrParams p = (ModifiableSolrParams) unloadCmd.getParams(); - - collectionClient.request(unloadCmd); - } - // Thread.currentThread().sleep(500); - // printLayout(); - - int tries = 50; - while (leader - .getCoreUrl() - .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { - Thread.sleep(100); - if (tries-- == 0) { - fail("Leader never changed"); - } - } - - // ensure there is a leader - zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); - - try (SolrClient addClient = - new HttpSolrClient.Builder(jettys.get(1).getBaseUrl().toString()) - .withDefaultCollection("unloadcollection_shard1_replica2") - .withConnectionTimeout(30000, TimeUnit.MILLISECONDS) - .withSocketTimeout(90000, TimeUnit.MILLISECONDS) - .build()) { - - // add a few docs while the leader is down - for (int x = 101; x < 200; x++) { - SolrInputDocument doc1 = - getDoc(id, x, i1, -600, tlong, 600, t1, "humpty dumpy sat on a wall"); - addClient.add(doc1); - } - } - - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName("unloadcollection_shard1_replica4") - .setNode(jettys.get(3).getNodeName()) - .process(cloudClient) - .isSuccess()); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - // unload the leader again - leader = getLeaderFromZk("unloadcollection", "shard1"); - try (SolrClient collectionClient = newSolrClient(leader.getBaseUrl())) { - - Unload unloadCmd = new Unload(false); - unloadCmd.setCoreName(leader.getCoreName()); - collectionClient.request(unloadCmd); - } - tries = 50; - while (leader - .getCoreUrl() - .equals(zkStateReader.getLeaderUrl("unloadcollection", "shard1", 15000))) { - Thread.sleep(100); - if (tries-- == 0) { - fail("Leader never changed"); - } - } - - zkStateReader.getLeaderRetry("unloadcollection", "shard1", 15000); - - TestInjection.skipIndexWriterCommitOnClose = false; // set this back - assertTrue( - CollectionAdminRequest.addReplicaToShard("unloadcollection", "shard1") - .setCoreName(leader.getCoreName()) - .setDataDir(core1DataDir) - .setNode(leader.getNodeName()) - .process(cloudClient) - .isSuccess()); - - waitForRecoveriesToFinish("unloadcollection", zkStateReader, false); - - long found1, found3; - - try (SolrClient adminClient = - newSolrClient((jettys.get(1).getBaseUrl() + "/unloadcollection_shard1_replica2"))) { - adminClient.commit(); - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - found1 = adminClient.query(q).getResults().getNumFound(); - } - - try (SolrClient adminClient = - new HttpSolrClient.Builder(jettys.get(2).getBaseUrl().toString()) - .withDefaultCollection("unloadcollection_shard1_replica3") - .withConnectionTimeout(15000, TimeUnit.MILLISECONDS) - .withSocketTimeout(30000, TimeUnit.MILLISECONDS) - .build()) { - adminClient.commit(); - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - found3 = adminClient.query(q).getResults().getNumFound(); - } - - try (SolrClient adminClient = - newSolrClient((jettys.get(3).getBaseUrl() + "/unloadcollection_shard1_replica4"))) { - adminClient.commit(); - SolrQuery q = new SolrQuery("*:*"); - q.set("distrib", false); - long found4 = adminClient.query(q).getResults().getNumFound(); - - // all 3 shards should now have the same number of docs - assertEquals(found1, found3); - assertEquals(found3, found4); - } - } - - private void testUnloadLotsOfCores() throws Exception { - JettySolrRunner jetty = jettys.get(0); - int shards = TEST_NIGHTLY ? 2 : 1; - try (final SolrClient adminClient = jetty.newClient(15000, 60000)) { - int numReplicas = atLeast(3); - ThreadPoolExecutor executor = - new ExecutorUtil.MDCAwareThreadPoolExecutor( - 0, - Integer.MAX_VALUE, - 5, - TimeUnit.SECONDS, - new SynchronousQueue<>(), - new SolrNamedThreadFactory("testExecutor")); - try { - // create the cores - AbstractBasicDistributedZkTestBase.createCollectionInOneInstance( - adminClient, jetty.getNodeName(), executor, "multiunload", shards, numReplicas); - } finally { - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - - if (TEST_NIGHTLY == false) { - // with nightly tests, we can try doing the unloads before the creates are done - // it still works, but takes much longer since we end up waiting for a timeout - waitForRecoveriesToFinish("multiunload", false); - } - - executor = - new ExecutorUtil.MDCAwareThreadPoolExecutor( - 0, - Integer.MAX_VALUE, - 5, - TimeUnit.SECONDS, - new SynchronousQueue<>(), - new SolrNamedThreadFactory("testExecutor")); - try { - for (int j = 0; j < numReplicas; j++) { - final int freezeJ = j; - executor.execute( - () -> { - Unload unloadCmd = new Unload(true); - unloadCmd.setCoreName("multiunload" + freezeJ); - try { - adminClient.request(unloadCmd); - } catch (SolrServerException | IOException e) { - throw new RuntimeException(e); - } - }); - Thread.sleep(random().nextInt(50)); - } - } finally { - ExecutorUtil.shutdownAndAwaitTermination(executor); - } - } - } -} diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java index 79dc6bde071..5dc62e94ed2 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractCloudBackupRestoreTestCase.java @@ -47,7 +47,6 @@ import org.apache.solr.common.cloud.Replica; import org.apache.solr.common.cloud.Slice; import org.apache.solr.common.cloud.ZkStateReader; -import org.apache.solr.common.params.CollectionAdminParams; import org.apache.solr.common.params.CoreAdminParams; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -173,7 +172,6 @@ public void test() throws Exception { } testBackupAndRestore(getCollectionName()); - testConfigBackupOnly("conf1", getCollectionName()); testInvalidPath(getCollectionName()); } @@ -230,22 +228,6 @@ public void testRestoreFailure() throws Exception { } } - /** - * This test validates the backup of collection configuration using {@linkplain - * CollectionAdminParams#NO_INDEX_BACKUP_STRATEGY}. - * - * @param configName The config name for the collection to be backed up. - * @param collectionName The name of the collection to be backed up. - * @throws Exception in case of errors. - */ - protected void testConfigBackupOnly(String configName, String collectionName) throws Exception { - // This is deliberately no-op since we want to run this test only for one of the backup - // repository - // implementation (mainly to avoid redundant test execution). Currently HDFS backup repository - // test - // implements this. - } - // This test verifies the system behavior when the backup location cluster property is configured // with an invalid // value for the specified repository (and the default backup location is not configured in