Merge pull request #940 from ie3-institute/sp/#755-couchbase-it-fails

Fixing sometimes failing tests

Author: danielfeismann

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 ### Fixed
+- Fixed Couchbase integration tests that randomly failed [#755](https://github.com/ie3-institute/PowerSystemDataModel/issues/755)
 
 ### Changed
 - Changing from comparing strings to comparing uuids in `EntitySource.findFirstEntityByUuid` [#829](https://github.com/ie3-institute/PowerSystemDataModel/issues/829)

src/main/java/edu/ie3/datamodel/io/connectors/CouchbaseConnector.java

@@ -8,10 +8,12 @@
 import com.couchbase.client.core.diagnostics.PingResult;
 import com.couchbase.client.java.AsyncCollection;
 import com.couchbase.client.java.Cluster;
+import com.couchbase.client.java.ClusterOptions;
 import com.couchbase.client.java.Collection;
 import com.couchbase.client.java.kv.GetResult;
 import com.couchbase.client.java.kv.MutationResult;
 import com.couchbase.client.java.query.QueryResult;
+import java.time.Duration;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 
@@ -36,6 +38,24 @@ public CouchbaseConnector(String url, String bucketName, String username, String
     cluster = Cluster.connect(url, username, password);
   }
 
+  /**
+   * Initializes a new CouchbaseConnector with given KV timeout
+   *
+   * @param url the url to the cluster
+   * @param bucketName the name of the bucket to connect to
+   * @param username the user name
+   * @param password the user password
+   * @param kvTimeout the key-value access timeout
+   */
+  public CouchbaseConnector(
+      String url, String bucketName, String username, String password, Duration kvTimeout) {
+    this.bucketName = bucketName;
+    ClusterOptions clusterOptions =
+        ClusterOptions.clusterOptions(username, password)
+            .environment(env -> env.timeoutConfig(to -> to.kvTimeout(kvTimeout)));
+    cluster = Cluster.connect(url, clusterOptions);
+  }
+
   /**
    * Return the couchbase java sdk equivalent of a session - a collection - to the previously set
    * bucket

src/test/groovy/edu/ie3/datamodel/io/factory/result/SystemParticipantResultFactoryTest.groovy

@@ -156,13 +156,13 @@ class SystemParticipantResultFactoryTest extends Specification implements Factor
       "p"         : "2",
       "q"         : "2",
     ]
-    expect: "that the factory should not need more than 2 seconds for processing 100.000 entities"
+    expect: "that the factory should not need more than 3 seconds for processing 10.000 entities"
     Long startTime = System.currentTimeMillis()
     10000.times {
       resultFactory.get(new SimpleEntityData(parameter, StorageResult))
     }
     BigDecimal elapsedTime = (System
         .currentTimeMillis() - startTime) / 1000.0
-    elapsedTime < 2
+    elapsedTime < 3
   }
 }

src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceCosmoIT.groovy

@@ -23,6 +23,7 @@ import org.testcontainers.utility.MountableFile
 import spock.lang.Shared
 import spock.lang.Specification
 
+import java.time.Duration
 import java.time.ZoneId
 
 @Testcontainers
@@ -32,9 +33,10 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain
   BucketDefinition bucketDefinition = new BucketDefinition("ie3_in")
 
   @Shared
-  CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2")
+  CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.6.0")
   .withBucket(bucketDefinition)
   .withExposedPorts(8091, 8092, 8093, 8094, 11210)
+  .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early
 
   @Shared
   CouchbaseWeatherSource source
@@ -63,7 +65,13 @@ class CouchbaseWeatherSourceCosmoIT extends Specification implements TestContain
         "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%",
         "--dataset", "file:///home/weather_cosmo.json")
 
-    def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password)
+    // increased timeout to deal with CI under high load
+    def connector = new CouchbaseConnector(
+        couchbaseContainer.connectionString,
+        bucketDefinition.name,
+        couchbaseContainer.username,
+        couchbaseContainer.password,
+        Duration.ofSeconds(20))
     def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx"
     def weatherFactory = new CosmoTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern))
     source = new CouchbaseWeatherSource(connector, CosmoWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern)

src/test/groovy/edu/ie3/datamodel/io/source/couchbase/CouchbaseWeatherSourceIconIT.groovy

@@ -20,6 +20,7 @@ import org.testcontainers.spock.Testcontainers
 import spock.lang.Shared
 import spock.lang.Specification
 
+import java.time.Duration
 import java.time.ZoneId
 
 @Testcontainers
@@ -29,8 +30,10 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine
   BucketDefinition bucketDefinition = new BucketDefinition("ie3_in")
 
   @Shared
-  CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.0.2").withBucket(bucketDefinition)
+  CouchbaseContainer couchbaseContainer = new CouchbaseContainer("couchbase/server:6.6.0")
+  .withBucket(bucketDefinition)
   .withExposedPorts(8091, 8092, 8093, 8094, 11210)
+  .withStartupAttempts(3) // 3 attempts because startup (node renaming) sometimes fails when executed too early
 
   @Shared
   CouchbaseWeatherSource source
@@ -59,7 +62,13 @@ class CouchbaseWeatherSourceIconIT extends Specification implements TestContaine
         "--generate-key", "weather::%" + coordinateIdColumnName + "%::%time%",
         "--dataset", "file:///home/weather_icon.json")
 
-    def connector = new CouchbaseConnector(couchbaseContainer.connectionString, bucketDefinition.name, couchbaseContainer.username, couchbaseContainer.password)
+    // increased timeout to deal with CI under high load
+    def connector = new CouchbaseConnector(
+        couchbaseContainer.connectionString,
+        bucketDefinition.name,
+        couchbaseContainer.username,
+        couchbaseContainer.password,
+        Duration.ofSeconds(20))
     def dtfPattern = "yyyy-MM-dd'T'HH:mm:ssxxx"
     def weatherFactory = new IconTimeBasedWeatherValueFactory(new TimeUtil(ZoneId.of("UTC"), Locale.GERMANY, dtfPattern))
     source = new CouchbaseWeatherSource(connector, IconWeatherTestData.coordinateSource, coordinateIdColumnName, weatherFactory, dtfPattern)