From c27d1377535678be399068bb28e4667c34d33b9e Mon Sep 17 00:00:00 2001 From: "Christopher L. Shannon" Date: Fri, 16 Aug 2024 15:47:18 -0400 Subject: [PATCH] Configure test server classes before starting MAC Update a few ITs that were stopping a normal server instance and starting up a test impl by taking advantage of changes in #4643 to set the server instance type before start up. This simplifies the tests and also prevents weird behavior by having servers start up and be shut down quickly. This closes #4644 --- .../accumulo/test/ScanServerShutdownIT.java | 13 ++++------- .../compaction/ExternalCompaction2BaseIT.java | 2 ++ .../compaction/ExternalCompaction_2_IT.java | 4 ---- .../FlakyExternalCompaction2IT.java | 20 +++++++++------- .../test/functional/MemoryStarvedMajCIT.java | 23 +++---------------- 5 files changed, 21 insertions(+), 41 deletions(-) diff --git a/test/src/main/java/org/apache/accumulo/test/ScanServerShutdownIT.java b/test/src/main/java/org/apache/accumulo/test/ScanServerShutdownIT.java index 023b6610fd2..ce33695e73c 100644 --- a/test/src/main/java/org/apache/accumulo/test/ScanServerShutdownIT.java +++ b/test/src/main/java/org/apache/accumulo/test/ScanServerShutdownIT.java @@ -57,7 +57,7 @@ private static class ScanServerShutdownITConfiguration public void configureMiniCluster(MiniAccumuloConfigImpl cfg, org.apache.hadoop.conf.Configuration coreSite) { - cfg.getClusterServerConfiguration().setNumDefaultScanServers(0); + cfg.getClusterServerConfiguration().setNumDefaultScanServers(1); // Timeout scan sessions after being idle for 3 seconds cfg.setProperty(Property.TSERV_SESSION_MAXIDLE, "3s"); @@ -65,6 +65,9 @@ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, // Configure the scan server to only have 1 scan executor thread. This means // that the scan server will run scans serially, not concurrently. cfg.setProperty(Property.SSERV_SCAN_EXECUTORS_DEFAULT_THREADS, "1"); + + // Set our custom implementation that shuts down after 3 batch scans + cfg.setServerClass(ServerType.SCAN_SERVER, SelfStoppingScanServer.class); } } @@ -87,14 +90,6 @@ public void testRefRemovalOnShutdown() throws Exception { ZooReaderWriter zrw = ctx.getZooReaderWriter(); String scanServerRoot = zooRoot + Constants.ZSSERVERS; - Wait.waitFor(() -> zrw.getChildren(scanServerRoot).size() == 0); - - // Stop normal ScanServers so that we can start our custom implementation - // that shuts down after 3 batch scans - getCluster().getConfig().getClusterServerConfiguration().setNumDefaultScanServers(1); - getCluster().getClusterControl().start(ServerType.SCAN_SERVER, null, 1, - SelfStoppingScanServer.class); - // Wait for the ScanServer to register in ZK Wait.waitFor(() -> zrw.getChildren(scanServerRoot).size() == 1); diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2BaseIT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2BaseIT.java index 26cdd4329e2..fd1238f8004 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2BaseIT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction2BaseIT.java @@ -59,6 +59,7 @@ import org.apache.accumulo.core.metadata.schema.TabletsMetadata; import org.apache.accumulo.harness.MiniClusterConfigurationCallback; import org.apache.accumulo.harness.SharedMiniClusterBase; +import org.apache.accumulo.minicluster.ServerType; import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.server.ServerContext; import org.apache.accumulo.server.util.FindCompactionTmpFiles; @@ -75,6 +76,7 @@ static class ExternalCompaction2Config implements MiniClusterConfigurationCallba @Override public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) { ExternalCompactionTestUtils.configureMiniCluster(cfg, coreSite); + cfg.setServerClass(ServerType.COMPACTOR, ExternalDoNothingCompactor.class); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java index e3132d820bd..4442fd7e6ce 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/ExternalCompaction_2_IT.java @@ -18,7 +18,6 @@ */ package org.apache.accumulo.test.compaction; -import org.apache.accumulo.minicluster.ServerType; import org.junit.jupiter.api.BeforeAll; public class ExternalCompaction_2_IT extends ExternalCompaction2BaseIT { @@ -26,8 +25,5 @@ public class ExternalCompaction_2_IT extends ExternalCompaction2BaseIT { @BeforeAll public static void beforeTests() throws Exception { startMiniClusterWithConfig(new ExternalCompaction2Config()); - getCluster().getClusterControl().stop(ServerType.COMPACTOR); - getCluster().getClusterControl().start(ServerType.COMPACTOR, null, 1, - ExternalDoNothingCompactor.class); } } diff --git a/test/src/main/java/org/apache/accumulo/test/compaction/FlakyExternalCompaction2IT.java b/test/src/main/java/org/apache/accumulo/test/compaction/FlakyExternalCompaction2IT.java index cff34b18b74..f4c5476fd3e 100644 --- a/test/src/main/java/org/apache/accumulo/test/compaction/FlakyExternalCompaction2IT.java +++ b/test/src/main/java/org/apache/accumulo/test/compaction/FlakyExternalCompaction2IT.java @@ -20,9 +20,11 @@ import org.apache.accumulo.harness.SharedMiniClusterBase; import org.apache.accumulo.minicluster.ServerType; +import org.apache.accumulo.miniclusterImpl.MiniAccumuloConfigImpl; import org.apache.accumulo.test.ample.FlakyAmpleManager; import org.apache.accumulo.test.ample.FlakyAmpleServerContext; import org.apache.accumulo.test.ample.FlakyAmpleTserver; +import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -37,16 +39,18 @@ */ public class FlakyExternalCompaction2IT extends ExternalCompaction2BaseIT { - @BeforeAll - public static void setup() throws Exception { - SharedMiniClusterBase.startMiniClusterWithConfig((cfg, coreSite) -> { - ExternalCompactionTestUtils.configureMiniCluster(cfg, coreSite); + static class FlakyExternalCompaction2Config extends ExternalCompaction2Config { + @Override + public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreSite) { + super.configureMiniCluster(cfg, coreSite); cfg.setServerClass(ServerType.MANAGER, FlakyAmpleManager.class); cfg.setServerClass(ServerType.TABLET_SERVER, FlakyAmpleTserver.class); - }); - getCluster().getClusterControl().stop(ServerType.COMPACTOR); - getCluster().getClusterControl().start(ServerType.COMPACTOR, null, 1, - ExternalDoNothingCompactor.class); + } + } + + @BeforeAll + public static void setup() throws Exception { + startMiniClusterWithConfig(new FlakyExternalCompaction2Config()); } @AfterAll diff --git a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java index 637a71eca8c..6bc1b5f109e 100644 --- a/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java +++ b/test/src/main/java/org/apache/accumulo/test/functional/MemoryStarvedMajCIT.java @@ -22,7 +22,6 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNull; import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.junit.jupiter.api.Assertions.fail; import java.util.ArrayList; import java.util.List; @@ -83,6 +82,9 @@ public void configureMiniCluster(MiniAccumuloConfigImpl cfg, Configuration coreS Map sysProps = Map.of(TestStatsDRegistryFactory.SERVER_HOST, "127.0.0.1", TestStatsDRegistryFactory.SERVER_PORT, Integer.toString(sink.getPort())); cfg.setSystemProperties(sysProps); + + // Set a compactor that will consume and free memory when we need it to + cfg.setServerClass(ServerType.COMPACTOR, MemoryConsumingCompactor.class); } } @@ -137,25 +139,6 @@ public void testMajCPauses() throws Exception { ClientContext ctx = (ClientContext) client; - // Kill the normal compactors and wait until their addresses in ZK are cleared - getCluster().getConfig().getClusterServerConfiguration().getCompactorConfiguration().keySet() - .forEach(resourceGroup -> { - List procs = getCluster().getClusterControl().getCompactors(resourceGroup); - for (int i = 0; i < procs.size(); i++) { - LOG.info("Stopping compactor process: {}", procs.get(i).pid()); - try { - procs.get(i).destroyForcibly().waitFor(); - } catch (InterruptedException e) { - fail("Interrupted trying to stop compactor process"); - } - } - getCluster().getClusterControl().getCompactors(resourceGroup).clear(); - }); - Wait.waitFor(() -> ExternalCompactionUtil.getCompactorAddrs(ctx).size() == 0, 60_000); - - // Start the Compactors that will consume and free memory when we need it to - getCluster().getClusterControl().start(ServerType.COMPACTOR, null, 1, - MemoryConsumingCompactor.class); Wait.waitFor(() -> ExternalCompactionUtil.getCompactorAddrs(ctx).size() == 1, 60_000); Wait.waitFor(() -> ExternalCompactionUtil.getCompactorAddrs(ctx) .get(Constants.DEFAULT_RESOURCE_GROUP_NAME).size() == 1, 60_000);