CASSSIDECAR-277 Support testing of post-TCM Cassandra

.github/workflows/ci.yml

@@ -166,14 +166,19 @@ jobs:
         cassandra: ["4.0", "4.1", "5.0", "5.1"]
         include:
           - cassandra: "4.0"
-            dtestVersion: "4.0.16"
+            dtestVersion: "4.0.20"
           - cassandra: "4.1"
-            dtestVersion: "4.1.6"
+            dtestVersion: "4.1.11"
           - cassandra: "5.0"
-            dtestVersion: "5.0.3"
+            dtestVersion: "5.0.6"
           - cassandra: "5.1"
             dtestVersion: "5.1"
-
+        # Exclude jdk17 for 4.0/4.1 as they don't support 17
+        exclude:
+          - cassandra: "4.0"
+            java: "17"
+          - cassandra: "4.1"
+            java: "17"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
@@ -199,6 +204,7 @@ jobs:
         run: |
           for i in {2..20}; do
             sudo ip addr add 127.0.0.$i/8 dev lo
+            sudo echo 127.0.0.${i} localhost${i} | sudo tee -a /etc/hosts
           done
 
       - name: Download dtest jars
@@ -257,7 +263,12 @@ jobs:
             dtestVersion: "5.0.3"
           - cassandra: "5.1"
             dtestVersion: "5.1"
-
+        # Exclude jdk17 for 4.0/4.1 as they don't support 17
+        exclude:
+          - cassandra: "4.0"
+            java: "17"
+          - cassandra: "4.1"
+            java: "17"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v5
@@ -283,7 +294,9 @@ jobs:
         run: |
           for i in {2..20}; do
             sudo ip addr add 127.0.0.$i/8 dev lo
+            sudo echo 127.0.0.${i} localhost${i} | sudo tee -a /etc/hosts
           done
+          cat /etc/hosts
 
       - name: Download dtest jars
         uses: actions/download-artifact@v5

README.md

@@ -61,6 +61,8 @@ To get up and running, create a temporary alias for every node except the first:
  for i in {2..20}; do sudo ifconfig lo0 alias "127.0.0.${i}"; done
 ```
 
+Note that this does not persist across reboots, so you'll have to run it every time you restart.
+
 Getting started: Running The Sidecar
 --------------------------------------
 

adapters/adapters-base/src/main/java/org/apache/cassandra/sidecar/adapters/base/CassandraStorageOperations.java

@@ -40,6 +40,7 @@
 import org.apache.cassandra.sidecar.common.server.dns.DnsResolver;
 import org.apache.cassandra.sidecar.common.server.exceptions.NodeBootstrappingException;
 import org.apache.cassandra.sidecar.common.server.exceptions.SnapshotAlreadyExistsException;
+import org.apache.cassandra.sidecar.common.server.utils.ThrowableUtils;
 import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
@@ -132,10 +133,19 @@ protected void takeSnapshotInternal(@NotNull String tag,
         }
         catch (IOException e)
         {
+            // post-5.0, IOExceptions are thrown when previously something else like
+            // an IllegalArgumentException was thrown. First, try to unwrap the IOException and throw
+            // the original cause, which we will process correctly in CreateSnapshotHandler
+            // if the exception was an IllegalArgumentException
+            IllegalArgumentException iex = ThrowableUtils.getCause(e, IllegalArgumentException.class);
+            if (iex != null) {
+                throw iex;
+            }
             String errorMessage = e.getMessage();
             if (errorMessage != null)
             {
-                if (errorMessage.contains("Snapshot " + tag + " already exists"))
+                if (errorMessage.contains("Snapshot " + tag + " already exists") ||
+                    errorMessage.contains("Snapshot " + tag + " for " + keyspace + "." + table + " already exists"))
                 {
                     throw new SnapshotAlreadyExistsException(e);
                 }

client-common/src/main/java/org/apache/cassandra/sidecar/common/utils/StringUtils.java

@@ -18,6 +18,8 @@
 
 package org.apache.cassandra.sidecar.common.utils;
 
+import java.util.Objects;
+import org.jetbrains.annotations.NotNull;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -46,4 +48,12 @@ public static boolean isNotEmpty(@Nullable String string)
     {
         return !isNullOrEmpty(string);
     }
+
+    public static boolean contains(@NotNull String string, @NotNull String target) {
+        if (isNullOrEmpty(string) || isNullOrEmpty(target))
+        {
+            return false;
+        }
+        return string.contains(target);
+    }
 }

gradle/common/java11Options.gradle

@@ -23,12 +23,22 @@ project.ext.JDK11_OPTIONS = ['-Djdk.attach.allowAttachSelf=true',
                              '--add-exports', 'java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED',
                              '--add-exports', 'java.rmi/sun.rmi.registry=ALL-UNNAMED',
                              '--add-exports', 'java.rmi/sun.rmi.server=ALL-UNNAMED',
+                             '--add-exports', 'java.rmi/sun.rmi.transport=ALL-UNNAMED',
+                             '--add-exports', 'java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED',
                              '--add-exports', 'java.sql/java.sql=ALL-UNNAMED',
+                             '--add-opens', 'java.base/java.io=ALL-UNNAMED',
+                             '--add-opens', 'java.base/sun.nio.ch=ALL-UNNAMED',
+                             '--add-opens', 'java.base/java.lang=ALL-UNNAMED',
                              '--add-opens', 'java.base/java.lang.module=ALL-UNNAMED',
+                             '--add-opens', 'java.base/java.lang.reflect=ALL-UNNAMED',
+                             '--add-opens', 'java.base/java.util=ALL-UNNAMED',
+                             '--add-opens', 'java.base/java.util.concurrent=ALL-UNNAMED',
+                             '--add-opens', 'java.base/java.util.concurrent.atomic=ALL-UNNAMED',
                              '--add-opens', 'java.base/jdk.internal.loader=ALL-UNNAMED',
                              '--add-opens', 'java.base/jdk.internal.ref=ALL-UNNAMED',
                              '--add-opens', 'java.base/jdk.internal.reflect=ALL-UNNAMED',
                              '--add-opens', 'java.base/jdk.internal.math=ALL-UNNAMED',
                              '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED',
                              '--add-opens', 'java.base/jdk.internal.util.jar=ALL-UNNAMED',
+                             '--add-opens', 'java.rmi/sun.rmi.transport.tcp=ALL-UNNAMED',
                              '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED']

integration-framework/src/main/java/org/apache/cassandra/sidecar/testing/SharedClusterIntegrationTestBase.java

@@ -88,6 +88,7 @@
 import org.apache.cassandra.sidecar.common.server.utils.SecondBoundConfiguration;
 import org.apache.cassandra.sidecar.common.server.utils.SidecarVersionProvider;
 import org.apache.cassandra.sidecar.common.server.utils.ThrowableUtils;
+import org.apache.cassandra.sidecar.common.utils.StringUtils;
 import org.apache.cassandra.sidecar.config.JmxConfiguration;
 import org.apache.cassandra.sidecar.config.KeyStoreConfiguration;
 import org.apache.cassandra.sidecar.config.S3ClientConfiguration;
@@ -218,9 +219,8 @@ private IClusterExtension<? extends IInstance> provisionClusterWithRetries(TestV
             }
             catch (RuntimeException runtimeException)
             {
-                boolean addressAlreadyInUse = ThrowableUtils.getCause(runtimeException, ex -> ex instanceof BindException &&
-                                                                                              ex.getMessage() != null &&
-                                                                                              ex.getMessage().contains("Address already in use")) != null;
+                boolean addressAlreadyInUse =
+                ThrowableUtils.getCause(runtimeException, SharedClusterIntegrationTestBase::portNotAvailableToBind) != null;
                 if (addressAlreadyInUse)
                 {
                     logger.warn("Failed to provision cluster after {} retries", retry, runtimeException);
@@ -234,6 +234,14 @@ private IClusterExtension<? extends IInstance> provisionClusterWithRetries(TestV
         throw new RuntimeException("Unable to provision cluster after " + MAX_CLUSTER_PROVISION_RETRIES + " retries");
     }
 
+    private static boolean portNotAvailableToBind(Throwable cause)
+    {
+        return (cause instanceof BindException && StringUtils.contains(cause.getMessage(), "Address already in use")) ||
+               // InboundConnectionInitiator in Cassandra throws a ConfigurationException with this string
+               StringUtils.contains(cause.getMessage(), "is in use by another process");
+    }
+
+
     @AfterAll
     protected void tearDown() throws Exception
     {

integration-framework/src/main/java/org/apache/cassandra/testing/TestUtils.java

@@ -123,9 +123,11 @@ public static CassandraVersionProvider cassandraVersionProvider(DnsResolver dnsR
      */
     public static void configureDefaultDTestJarProperties()
     {
+        // NOTE: `cassandra.consistent.rangemovement` is no longer supported in Cassandra post-TCM.
+        // While this sped up tests, enabling it will make it easier to "accidentally" make tests that don't
+        // work in a post-TCM world.
         // Settings to reduce the test setup delay incurred if gossip is enabled
         System.setProperty("cassandra.ring_delay_ms", "5000"); // down from 30s default
-        System.setProperty("cassandra.consistent.rangemovement", "false");
         System.setProperty("cassandra.consistent.simultaneousmoves.allow", "true");
         // End gossip delay settings
         // Set the location of dtest jars

integration-tests/src/integrationTest/org/apache/cassandra/sidecar/routes/RoutesIntegrationTest.java

@@ -68,9 +68,15 @@ void retrieveGossipInfo()
         assertThat(gossipInfo.heartbeat()).isNotNull();
         assertThat(gossipInfo.hostId()).isNotNull();
         String releaseVersion = cluster.getFirstRunningInstance().getReleaseVersionString();
+        releaseVersion = stripSnapshot(releaseVersion);
         assertThat(gossipInfo.releaseVersion()).startsWith(releaseVersion);
     }
 
+    private String stripSnapshot(String version)
+    {
+        return version.replace("-SNAPSHOT", "");
+    }
+
     @Test
     void testGossipHealth()
     {

scripts/build-dtest-jars.sh

@@ -19,11 +19,10 @@
 
 set -xe
 CANDIDATE_BRANCHES=(
-  "cassandra-4.0:64b8d6b9add607b80752cd1a8fbce51839af9ec4"
-  "cassandra-4.1:044727aabafeab2f6fef74c52d349d55c8732ef5"
-  "cassandra-5.0:a0d58a9ce8814d096c1bd8a0440e8e28d8ea15a9"
-  # note the trunk hash cannot be advanced beyond ae0842372ff6dd1437d026f82968a3749f555ff4 (TCM), which breaks integration test
-  "trunk:2a5e1b77c9f8a205dbec1afdea3f4ed1eaf6a4eb"
+  "cassandra-4.0:aa0e2f1631ae343e35334e5419b193a9a1cfa0a6"
+  "cassandra-4.1:c988b609b0239e37f37e3b764728d960220cc3e8"
+  "cassandra-5.0:b4dcef78419c29584937b44aa484cf0c13cf37e0"
+  "trunk:9142d0c8519944e02b3d449b21c3b42ab80caeb6"
 )
 BRANCHES=( ${BRANCHES:-cassandra-4.0 cassandra-4.1 cassandra-5.0 trunk} )
 echo ${BRANCHES[*]}

scripts/relocate-dtest-dependencies.pom

@@ -93,8 +93,7 @@
             <plugin>
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-shade-plugin</artifactId>
-                <version>3.4.1</version>
-
+                <version>3.6.1</version>
                 <executions>
                     <execution>
                         <phase>package</phase>
@@ -182,6 +181,11 @@
 
                             </filters>
                             <outputFile>${outputFilePath}</outputFile>
+                            <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+                                <manifestEntries>
+                                    <Multi-Release>true</Multi-Release>
+                                </manifestEntries>
+                            </transformer>
                         </configuration>
                     </execution>
                 </executions>

server-common/build.gradle

@@ -26,11 +26,16 @@ plugins {
     id 'java-test-fixtures'
 }
 apply from: "$rootDir/gradle/common/publishing.gradle"
+apply from: "${project.rootDir}/gradle/common/java11Options.gradle"
 
 sourceCompatibility = JavaVersion.VERSION_11
 
 test {
     useJUnitPlatform()
+    if (JavaVersion.current().isJava11Compatible()) {
+        jvmArgs(project.ext.JDK11_OPTIONS)
+        println("JVM arguments for $project.name are $allJvmArgs")
+    }
     maxParallelForks = Runtime.runtime.availableProcessors().intdiv(2) ?: 1
     reports {
         junitXml.setRequired(true)

server/src/main/java/org/apache/cassandra/sidecar/concurrent/AsyncConcurrentTaskExecutor.java

@@ -130,7 +130,7 @@ List<Promise<T>> getTaskPromises()
      * futures for all tasks. As running tasks complete, pending tasks are automatically
      * triggered to maintain the concurrency level.</p>
      *
-     * <h3>Execution Flow Diagram:</h3>
+     * Execution Flow Diagram:
      * <pre>
      *            index
      *             +
@@ -250,7 +250,7 @@ void triggerNextTask()
      * Currently running tasks are not interrupted, but all pending tasks will be
      * cancelled with a {@link CancellationException}.</p>
      *
-     * <h3>Cancellation Diagram:</h3>
+     * Cancellation Diagram:
      * <pre>
      *                                          index
      *                                          +

server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationApiEnableDisableHandler.java

@@ -34,7 +34,7 @@
 /**
  * Handler for enabling or disabling live migration APIs based on instance role and migration status.
  * <p>
- * <h3>Handler Methods:</h3>
+ * <h2>Handler Methods:</h2>
  * <ul>
  *   <li>{@link #isSource(RoutingContext)} - Allows access only if instance is a migration source</li>
  *   <li>{@link #isDestination(RoutingContext)} - Allows access only if instance is a migration destination</li>

server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationStatusClearHandler.java

@@ -49,7 +49,7 @@
  * migrated again in the future. Clearing the status is necessary to prevent blocking
  * future migrations of the same destination instance.
  *
- * <h3>Usage Safety:</h3>
+ * <h2>Usage Safety:</h2>
  * IMPORTANT: This endpoint should ONLY be called after completing the live migration
  * process and the instance entry has been removed from the Live Migration map
  * Calling this endpoint prematurely may lead to inconsistent migration state.

server/src/main/java/org/apache/cassandra/sidecar/handlers/livemigration/LiveMigrationStatusCompleteHandler.java

@@ -50,7 +50,7 @@
  * been migrated already. To ensure safety, other live migration endpoints such as file streaming
  * or data-copy requests will automatically stop serving to requests.
  *
- * <h3>Usage Safety:</h3>
+ * <h2>Usage Safety:</h2>
  * This endpoint should only be called when the live migration process has genuinely completed
  * all data transfer and validation steps. Premature completion can lead to data inconsistency
  * or loss during cluster operations.

server/src/main/java/org/apache/cassandra/sidecar/server/Server.java

@@ -80,6 +80,7 @@ public class Server
     protected final List<ServerVerticle> deployedServerVerticles = new CopyOnWriteArrayList<>();
     // Keeps track of all the Cassandra instance identifiers where CQL is ready
     private final Set<Integer> cqlReadyInstanceIds = Collections.synchronizedSet(new HashSet<>());
+    private volatile Future<Void> closeFuture;
 
     @Inject
     public Server(Vertx vertx,
@@ -152,6 +153,21 @@ public Future<Void> stop(String deploymentId)
      * @return a future completed with the result
      */
     public Future<Void> close()
+    {
+        if (closeFuture == null)
+        {
+            synchronized (this)
+            {
+                if (closeFuture == null)
+                {
+                    setCloseFuture();
+                }
+            }
+        }
+        return closeFuture;
+    }
+
+    private void setCloseFuture()
     {
         LOGGER.info("Stopping Cassandra Sidecar");
         deployedServerVerticles.clear();
@@ -182,11 +198,11 @@ public Future<Void> close()
             closingFutures.add(closingFutureForInstance.future());
         });
 
-        return Future.all(closingFutures)
+        closeFuture = (Future.all(closingFutures)
                      .onSuccess(ignored -> LOGGER.debug("Closed Cassandra adapters"))
                      .transform(v -> {
-                        LOGGER.debug("Closing PeriodicTaskExecutor");
-                        return periodicTaskExecutor.close();
+                         LOGGER.debug("Closing PeriodicTaskExecutor");
+                         return periodicTaskExecutor.close();
                      })
                      .transform(v -> {
                          LOGGER.debug("Closing executor pools");
@@ -197,7 +213,8 @@ public Future<Void> close()
                          return vertx.close();
                      })
                      .onFailure(t -> LOGGER.error("Failed to gracefully shutdown Cassandra Sidecar", t))
-                     .onSuccess(f -> LOGGER.info("Successfully stopped Cassandra Sidecar"));
+                     .onSuccess(f -> LOGGER.info("Successfully stopped Cassandra Sidecar"))
+        );
     }
 
     /**

server/src/test/integration/org/apache/cassandra/sidecar/common/CQLSessionProviderTest.java

@@ -47,7 +47,9 @@
 public class CQLSessionProviderTest extends IntegrationTestBase
 {
 
-    public static final String OK_KEYSPACE_RESPONSE_START = "{\"schema\":\"CREATE KEYSPACE ";
+    // Cassandra has started including virtual keyspace information at the beginning of the schema response.
+    // Therefore, the first entry no longer includes `CREATE KEYSPACE`. Just look for the schema start tag instead.
+    public static final String OK_KEYSPACE_RESPONSE_START = "{\"schema\":";
     public static final String KEYSPACE_FAILED_RESPONSE_START = "{\"status\":\"Service Unavailable\",";
 
     @Override

server/src/test/integration/org/apache/cassandra/sidecar/common/DelegateIntegrationTest.java

@@ -207,7 +207,7 @@ void testStoppingAnInstance(VertxTestContext context)
     }
 
     @Timeout(value = 2, timeUnit = TimeUnit.MINUTES)
-    @CassandraIntegrationTest(nodesPerDc = 2, newNodesPerDc = 1, startCluster = false)
+    @CassandraIntegrationTest(nodesPerDc = 2, newNodesPerDc = 1, startCluster = false, network = true)
     public void testChangingClusterSize(VertxTestContext context) throws InterruptedException
     {
         // assume the sidecar has 3 managed instances, even though the cluster only starts with 2 instances initially

server/src/test/integration/org/apache/cassandra/sidecar/db/SystemAuthDatabaseAccessorIntTest.java

@@ -32,7 +32,7 @@ class SystemAuthDatabaseAccessorIntTest extends IntegrationTestBase
     @CassandraIntegrationTest(authMode = AuthMode.PASSWORD)
     void testCrudOperations()
     {
-        waitForSchemaReady(10, TimeUnit.SECONDS);
+        waitForSchemaReady(20, TimeUnit.SECONDS);
 
         createRole("super_user_role", true);
         createRole("non_super_user_role", false);

server/src/test/integration/org/apache/cassandra/sidecar/routes/NodeDecommissionIntegrationTest.java

@@ -50,7 +50,7 @@
 @ExtendWith(VertxExtension.class)
 public class NodeDecommissionIntegrationTest extends IntegrationTestBase
 {
-    @CassandraIntegrationTest(nodesPerDc = 2)
+    @CassandraIntegrationTest(nodesPerDc = 2, network = true)
     void decommissionNodeDefault(VertxTestContext context)
     {
         final AtomicReference<String> jobId = new AtomicReference<>();