diff --git a/.travis.yml b/.travis.yml index 26d5959200..149649b58e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,7 +40,10 @@ env: - MODULE='hbase-parent/janusgraph-hbase-10' ARGS='-Dtest=**/graphdb/hbase/*' - MODULE='hbase-parent/janusgraph-hbase-098' ARGS='-Dtest=**/diskstorage/hbase/*' - MODULE='hbase-parent/janusgraph-hbase-098' ARGS='-Dtest=**/graphdb/hbase/*' - - MODULE='cql' + - MODULE='cql' ARGS='-Dtest=**/diskstorage/cql/* -Dtest.skip.murmur=true' + - MODULE='cql' ARGS='-Dtest=**/diskstorage/cql/* -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true -Dtest.skip.murmur-ssl=true' + - MODULE='cql' ARGS='-Dtest=**/graphdb/cql/* -Dtest.skip.murmur=true' + - MODULE='cql' ARGS='-Dtest=**/graphdb/cql/* -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true -Dtest.skip.murmur-ssl=true' - COVERITY_ONLY=true matrix: @@ -56,7 +59,10 @@ matrix: - env: MODULE='cassandra' ARGS='-Dtest=**/graphdb/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' - env: MODULE='cassandra' ARGS='-Dtest=**/graphdb/thrift/* -Dtest.skip.ordered=true -Dtest.skip.ssl=true -Dtest.skip.serial=true' - env: MODULE='cassandra' ARGS='-Dtest=**/graphdb/thrift/* -Dtest.skip.unordered=true -Dtest.skip.ordered=true' - - env: MODULE='cql' + - env: MODULE='cql' ARGS='-Dtest=**/diskstorage/cql/* -Dtest.skip.murmur=true' + - env: MODULE='cql' ARGS='-Dtest=**/diskstorage/cql/* -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true -Dtest.skip.murmur-ssl=true' + - env: MODULE='cql' ARGS='-Dtest=**/graphdb/cql/* -Dtest.skip.murmur=true' + - env: MODULE='cql' ARGS='-Dtest=**/graphdb/cql/* -Dtest.skip.byteorderedpartitioner=true -Dtest.skip.murmur-serial=true -Dtest.skip.murmur-ssl=true' addons: coverity_scan: diff --git a/janusgraph-cql/pom.xml b/janusgraph-cql/pom.xml index fa2e4af351..0899b498f8 100644 --- a/janusgraph-cql/pom.xml +++ b/janusgraph-cql/pom.xml @@ -191,10 +191,9 @@ ${test.jvm.opts} 1 false - classes - true - 2 - random + none + false + 1 file:${project.build.testOutputDirectory}/log4j.properties diff --git a/janusgraph-cql/src/main/java/org/janusgraph/diskstorage/cql/CQLStoreManager.java b/janusgraph-cql/src/main/java/org/janusgraph/diskstorage/cql/CQLStoreManager.java index 38c1a8f52a..9f839e7b39 100644 --- a/janusgraph-cql/src/main/java/org/janusgraph/diskstorage/cql/CQLStoreManager.java +++ b/janusgraph-cql/src/main/java/org/janusgraph/diskstorage/cql/CQLStoreManager.java @@ -15,7 +15,7 @@ package org.janusgraph.diskstorage.cql; import static com.datastax.driver.core.schemabuilder.SchemaBuilder.createKeyspace; -import static com.datastax.driver.core.schemabuilder.SchemaBuilder.dropKeyspace; +import static com.datastax.driver.core.querybuilder.QueryBuilder.truncate; import static io.vavr.API.$; import static io.vavr.API.Case; import static io.vavr.API.Match; @@ -115,7 +115,7 @@ public class CQLStoreManager extends DistributedStoreManager implements KeyColum private final int batchSize; private final boolean atomicBatch; - private final ExecutorService executorService; + final ExecutorService executorService; private final Cluster cluster; private final Session session; @@ -189,7 +189,7 @@ public CQLStoreManager(final Configuration configuration) throws BackendExceptio this.openStores = new ConcurrentHashMap<>(); } - private Cluster initializeCluster() throws PermanentBackendException { + Cluster initializeCluster() throws PermanentBackendException { final Configuration configuration = getStorageConfig(); final List contactPoints; @@ -249,7 +249,7 @@ private Cluster initializeCluster() throws PermanentBackendException { return builder.build(); } - private Session initializeSession(final String keyspaceName) { + Session initializeSession(final String keyspaceName) { final Configuration configuration = getStorageConfig(); final Map replication = Match(configuration.get(REPLICATION_STRATEGY)).of( Case($("SimpleStrategy"), strategy -> HashMap. of("class", strategy, "replication_factor", configuration.get(REPLICATION_FACTOR))), @@ -328,7 +328,10 @@ public StoreTransaction beginTransaction(final BaseTransactionConfig config) thr @Override public void clearStorage() throws BackendException { - this.session.execute(dropKeyspace(this.keyspace)); + final Future> result = Future.sequence( + Iterator.ofAll(this.cluster.getMetadata().getKeyspace(this.keyspace).getTables()) + .map(table -> Future.fromJavaFuture(this.session.executeAsync(truncate(this.keyspace, table.getName()))))); + result.await(); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java index 62240e8d64..f7e105f19c 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLDistributedStoreManagerTest.java @@ -35,7 +35,7 @@ public static void startCassandra() { @Before public void setUp() throws BackendException { - manager = new CQLStoreManager(CassandraStorageSetup.getCQLConfiguration(this.getClass().getSimpleName())); + manager = new CachingCQLStoreManager(CassandraStorageSetup.getCQLConfiguration(this.getClass().getSimpleName())); store = manager.openDatabase("distributedcf"); } diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLIDAuthorityTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLIDAuthorityTest.java index b3cc8da5a7..0897c221c0 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLIDAuthorityTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLIDAuthorityTest.java @@ -39,7 +39,7 @@ private ModifiableConfiguration getBaseStorageConfiguration() { } private CQLStoreManager openStorageManager(final Configuration c) throws BackendException { - return new CQLStoreManager(c); + return new CachingCQLStoreManager(c); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLockStoreTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLockStoreTest.java index 46ea8b16e7..fa28e79c20 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLockStoreTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLockStoreTest.java @@ -34,7 +34,7 @@ private ModifiableConfiguration getBaseStorageConfiguration() { } private CQLStoreManager openStorageManager(final Configuration c) throws BackendException { - return new CQLStoreManager(c); + return new CachingCQLStoreManager(c); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLogTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLogTest.java index 62ba66c73f..1275b50083 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLogTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLLogTest.java @@ -37,7 +37,7 @@ private ModifiableConfiguration getBaseStorageConfiguration() { } private CQLStoreManager openStorageManager(final Configuration c) throws BackendException { - return new CQLStoreManager(c); + return new CachingCQLStoreManager(c); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLMultiWriteStoreTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLMultiWriteStoreTest.java index c35aba87da..f1eb1716cc 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLMultiWriteStoreTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLMultiWriteStoreTest.java @@ -34,7 +34,7 @@ private ModifiableConfiguration getBaseStorageConfiguration() { } private CQLStoreManager openStorageManager(final Configuration c) throws BackendException { - return new CQLStoreManager(c); + return new CachingCQLStoreManager(c); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLSSLStoreTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLSSLStoreTest.java index 77df189631..c213ed6970 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLSSLStoreTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLSSLStoreTest.java @@ -37,7 +37,7 @@ protected ModifiableConfiguration getBaseStorageConfiguration() { } private CQLStoreManager openStorageManager(final Configuration c) throws BackendException { - return new CQLStoreManager(c); + return new CachingCQLStoreManager(c); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java index 31916f9095..16338427be 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CQLStoreTest.java @@ -53,7 +53,7 @@ protected ModifiableConfiguration getBaseStorageConfiguration() { } private CQLStoreManager openStorageManager(final Configuration c) throws BackendException { - return new CQLStoreManager(c); + return new CachingCQLStoreManager(c); } @Override diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CachingCQLStoreManager.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CachingCQLStoreManager.java new file mode 100644 index 0000000000..3f0c6cad27 --- /dev/null +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CachingCQLStoreManager.java @@ -0,0 +1,42 @@ +package org.janusgraph.diskstorage.cql; + +import com.datastax.driver.core.Cluster; +import com.datastax.driver.core.Session; +import org.janusgraph.diskstorage.BackendException; +import org.janusgraph.diskstorage.PermanentBackendException; +import org.janusgraph.diskstorage.configuration.Configuration; + +import java.util.HashMap; +import java.util.Map; + +public class CachingCQLStoreManager extends CQLStoreManager { + + private static Cluster cluster; + private static Map sessions = new HashMap<>(); + + public CachingCQLStoreManager(final Configuration configuration) throws BackendException { + super(configuration); + } + + @Override + Cluster initializeCluster() throws PermanentBackendException { + if (cluster == null || cluster.isClosed()) { + cluster = super.initializeCluster(); + } + return cluster; + } + + @Override + Session initializeSession(final String keyspaceName) { + if (!sessions.containsKey(keyspaceName)) { + sessions.put(keyspaceName, super.initializeSession(keyspaceName)); + } + return sessions.get(keyspaceName); + } + + @Override + public void close() { + this.executorService.shutdownNow(); + } + +} diff --git a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CassandraStorageSetup.java b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CassandraStorageSetup.java index b2dc286426..fad272bce5 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CassandraStorageSetup.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/diskstorage/cql/CassandraStorageSetup.java @@ -27,9 +27,14 @@ import java.io.File; import java.io.IOException; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.time.Duration; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.apache.commons.io.FileUtils; +import org.janusgraph.diskstorage.StandardStoreManager; import org.janusgraph.diskstorage.cassandra.utils.CassandraDaemonWrapper; import org.janusgraph.diskstorage.configuration.ConfigElement; import org.janusgraph.diskstorage.configuration.ModifiableConfiguration; @@ -47,6 +52,10 @@ public class CassandraStorageSetup { public static final String DATADIR_SYSPROP = "test.cassandra.datadir"; public static final String HOSTNAME = System.getProperty(ConfigElement.getPath(STORAGE_HOSTS)); + static { + setWrapperStoreManager(); + } + private static volatile Paths paths; /** @@ -61,6 +70,7 @@ public static void startCleanEmbedded() { if (!CassandraDaemonWrapper.isStarted()) { try { FileUtils.deleteDirectory(new File(p.dataPath)); + FileUtils.deleteQuietly(new File((new File(p.dataPath)).getParent() + File.separator + "commitlog")); } catch (final IOException e) { throw new RuntimeException(e); } @@ -134,6 +144,29 @@ private static String cleanKeyspaceName(final String raw) { } } + private static void setWrapperStoreManager() { + try { + final Field modifiersField = Field.class.getDeclaredField("modifiers"); + modifiersField.setAccessible(true); + + Field field = StandardStoreManager.class.getDeclaredField("managerClass"); + field.setAccessible(true); + field.set(StandardStoreManager.CQL, CachingCQLStoreManager.class.getCanonicalName()); + + field = StandardStoreManager.class.getDeclaredField("ALL_SHORTHANDS"); + field.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + field.set(null, ImmutableList.copyOf(StandardStoreManager.CQL.getShorthands())); + + field = StandardStoreManager.class.getDeclaredField("ALL_MANAGER_CLASSES"); + field.setAccessible(true); + modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL); + field.set(null, ImmutableMap.of(StandardStoreManager.CQL.getShorthands().get(0), StandardStoreManager.CQL.getManagerClass())); + } catch (ReflectiveOperationException e) { + throw new RuntimeException("Unable to set wrapper CQL store manager", e); + } + } + private static class Paths { private final String yamlPath; diff --git a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphIterativeTest.java b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphIterativeTest.java index f97cec795c..36b654944e 100644 --- a/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphIterativeTest.java +++ b/janusgraph-cql/src/test/java/org/janusgraph/graphdb/cql/CQLGraphIterativeTest.java @@ -17,7 +17,7 @@ import org.janusgraph.diskstorage.BackendException; import org.janusgraph.diskstorage.configuration.BasicConfiguration; import org.janusgraph.diskstorage.configuration.WriteConfiguration; -import org.janusgraph.diskstorage.cql.CQLStoreManager; +import org.janusgraph.diskstorage.cql.CachingCQLStoreManager; import org.janusgraph.diskstorage.cql.CassandraStorageSetup; import org.janusgraph.diskstorage.keycolumnvalue.KeyColumnValueStoreManager; import org.janusgraph.graphdb.JanusGraphIterativeBenchmark; @@ -33,7 +33,7 @@ public WriteConfiguration getConfiguration() { @Override public KeyColumnValueStoreManager openStorageManager() throws BackendException { - return new CQLStoreManager(new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS,getConfiguration(), BasicConfiguration.Restriction.NONE)); + return new CachingCQLStoreManager(new BasicConfiguration(GraphDatabaseConfiguration.ROOT_NS,getConfiguration(), BasicConfiguration.Restriction.NONE)); } diff --git a/janusgraph-cql/src/test/resources/log4j.properties b/janusgraph-cql/src/test/resources/log4j.properties index 3f01de576a..bdf413081e 100644 --- a/janusgraph-cql/src/test/resources/log4j.properties +++ b/janusgraph-cql/src/test/resources/log4j.properties @@ -1,15 +1,16 @@ -log4j.appender.FILE=org.apache.log4j.FileAppender -log4j.appender.FILE.File=${project.build.directory}/test.log -log4j.appender.FILE.Threshold=TRACE -log4j.appender.FILE.layout=org.apache.log4j.PatternLayout -log4j.appender.FILE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} [%20.20t] %-5p %30.30c: %m%n +log4j.appender.A1=org.apache.log4j.FileAppender +log4j.appender.A1.File=${project.build.directory}/test.log +log4j.appender.A1.Threshold=ALL +log4j.appender.A1.layout=org.apache.log4j.PatternLayout +log4j.appender.A1.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} [%20.20t] %-5p %30.30c: %m%n -log4j.appender.CONSOLE=org.apache.log4j.ConsoleAppender -log4j.appender.CONSOLE.Threshold=INFO -log4j.appender.CONSOLE.layout=org.apache.log4j.PatternLayout -log4j.appender.CONSOLE.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} [%20.20t] %-5p %30.30c: %m%n +log4j.appender.A2=org.apache.log4j.ConsoleAppender +log4j.appender.A2.Threshold=ALL +log4j.appender.A2.layout=org.apache.log4j.PatternLayout +log4j.appender.A2.layout.ConversionPattern=%d{yyyy-MM-dd HH:mm:ss.SSS} [%20.20t] %-5p %30.30c: %m%n -log4j.rootLogger=INFO,FILE,CONSOLE +#log4j.rootLogger=INFO, A1, A2 +log4j.rootLogger=ERROR, A1 log4j.logger.org.apache.cassandra=WARN -log4j.logger.com.datastax.driver=WARN \ No newline at end of file +log4j.logger.com.datastax.driver=WARN