From d5d51fe7e356a514b10fe4ed12ca572a7e625137 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 8 Jun 2021 23:19:06 -0500 Subject: [PATCH 01/30] First sketch of an idea, still lots to flesh out yet --- python/adelphi/bin/run-adelphi-tests.py | 51 +++++++++++ python/adelphi/test-requirements.txt | 2 + python/adelphi/tests/integration/__init__.py | 95 ++------------------ python/adelphi/tests/util/cassandra_util.py | 49 ++++++++++ python/adelphi/tests/util/schemadiff.py | 1 - python/adelphi/tox.ini | 2 - python/adelphi/tox.ini.dev | 2 - 7 files changed, 108 insertions(+), 94 deletions(-) create mode 100644 python/adelphi/bin/run-adelphi-tests.py create mode 100644 python/adelphi/test-requirements.txt create mode 100644 python/adelphi/tests/util/cassandra_util.py diff --git a/python/adelphi/bin/run-adelphi-tests.py b/python/adelphi/bin/run-adelphi-tests.py new file mode 100644 index 0000000..02ffd2c --- /dev/null +++ b/python/adelphi/bin/run-adelphi-tests.py @@ -0,0 +1,51 @@ +# Run test suite for some set of Cassandra versions. +# +# We implement this as a front-end script because the tox/pytest/unittest chain +# isn't great about iterating over a suite of test fixtures and re-running the +# collected tests for each of them. Rather than fight with the frameworks we +# manage the fixture iteration manually in this script. This also has the +# nice side effect of moving a lot of C* checking/session code out of the test +# suite, which in turn should allow us to write simpler tests. +import logging +import os + +import tox + +import docker +from tenacity import retry, stop_after_attempt, wait_fixed + +# Default C* versions to include in all integration tests +CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-beta4"] + +logging.basicConfig(filename="adelphi-tests.log", level=logging.INFO) +log = logging.getLogger('adelphi') + + +@retry(stop=stop_after_attempt(3), wait=wait_fixed(2)) +def getContainer(client, version): + return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042: 9042}, image="cassandra:{}".format(version)) + + +def getCassandraVersions(): + if "CASSANDRA_VERSIONS" in os.environ: + return [s.strip() for s in os.environ["CASSANDRA_VERSIONS"].split(',')] + else: + return CASSANDRA_VERSIONS + + +if __name__ == '__main__': + client = docker.from_env() + for version in getCassandraVersions(): + + log.info("Running test suite for Cassandra version {}".format(version)) + container = getContainer(client, version) + + try: + tox.cmdline() + except Exception as exc: + log.error("Exception running tests for Cassandra version {}".format(version), exc_info=exc) + finally: + if "KEEP_CONTAINER" in os.environ: + log.info("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'") + else: + container.stop() diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt new file mode 100644 index 0000000..a479f87 --- /dev/null +++ b/python/adelphi/test-requirements.txt @@ -0,0 +1,2 @@ +docker ~= 4.4 +tenacity ~= 7.0 diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 2357b7a..5a406a0 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -1,21 +1,12 @@ import logging import os import shutil -import sys import tempfile -import time from collections import namedtuple -from cassandra.cluster import Cluster +from tests.util.cassandra_util import connectToLocalCassandra,createSchema -import docker -from tenacity import retry, stop_after_attempt, wait_fixed - -# Default C* versions to include in all integration tests -CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-beta4"] - -logging.basicConfig(filename="adelphi.log", level=logging.INFO) log = logging.getLogger('adelphi') TempDirs = namedtuple('TempDirs', 'basePath, outputDirPath') @@ -45,96 +36,22 @@ def makeTempDirs(self): self.dirs = TempDirs(base, outputDir) - def connectToLocalCassandra(self): - session = None - while not session: - try: - cluster = Cluster(["127.0.0.1"],port=9042) - session = cluster.connect() - - # Confirm that the session is actually functioning before calling things good - rs = session.execute("select * from system.local") - log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) - log.info("Cassandra cluster ready") - return (cluster,session) - except: - log.info("Couldn't quite connect yet, will retry") - time.sleep(1) - - - def createSchema(self, session=None): - schemaPath = self.getBaseSchemaPath() - log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) - with open(schemaPath) as schema: - buff = "" - for line in schema: - realLine = line.strip() - if len(realLine) == 0: - log.debug("Skipping empty statement") - continue - if realLine.startswith("//"): - log.debug("Skipping commented statement {}".format(stmt)) - continue - buff += (" " if len(buff) > 0 else "") - buff += realLine - if realLine.endswith(';'): - log.debug("Executing statement {}".format(buff)) - try: - session.execute(buff) - except: - log.error("Exception executing statement: {}".format(buff), exc_info=sys.exc_info()[0]) - self.fail("Exception executing statement: {}, check log for details".format(buff)) - buff = "" - - - @retry(stop=stop_after_attempt(3), wait=wait_fixed(2)) - def getContainer(self, client, version): - return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042:9042}, image="cassandra:{}".format(version)) - - - def runTestForVersion(self, version=None): + def testAdelphi(self, version=None): log.info("Testing Cassandra version {}".format(version)) - client = docker.from_env() - container = self.getContainer(client, version) - self.makeTempDirs() - - (cluster,session) = (None,None) try: - (cluster,session) = self.connectToLocalCassandra() - self.createSchema(session) + (_,session) = connectToLocalCassandra() + createSchema(session, self.getBaseSchemaPath) log.info("Running Adelphi") self.runAdelphi(version) log.info("Adelphi run completed, evaluating Adelphi output(s)") self.evalAdelphiOutput(version) - except: - log.error("Exception running test for version {}".format(version), exc_info=sys.exc_info()[0]) - self.fail("Exception running test for version {}, check log for details".format(version)) finally: - if cluster: - cluster.shutdown() - - if "KEEP_CONTAINER" in os.environ: - log.info("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'") - else: - container.stop() - - self.cleanUpVersion(version) - - - def testVersions(self): - versions = CASSANDRA_VERSIONS - if "CASSANDRA_VERSIONS" in os.environ: - versions = [s.strip() for s in os.environ["CASSANDRA_VERSIONS"].split(',')] - - log.info("Testing the following Cassandra versions: {}".format(versions)) - - for version in versions: - self.runTestForVersion(version) + self.cleanUpTempDirs(version) - def cleanUpVersion(self, version): + def cleanUpTempDirs(self, version): if "KEEP_LOGS" in os.environ: log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath)) else: diff --git a/python/adelphi/tests/util/cassandra_util.py b/python/adelphi/tests/util/cassandra_util.py new file mode 100644 index 0000000..1f8acb1 --- /dev/null +++ b/python/adelphi/tests/util/cassandra_util.py @@ -0,0 +1,49 @@ +# Using "c8" rather than "cassandra" to avoid any conflicts with "cassandra" package in +# cassandra-driver +import logging +import time + +from cassandra.cluster import Cluster + +log = logging.getLogger('adelphi') + +def connectToLocalCassandra(): + session = None + while not session: + try: + cluster = Cluster(["127.0.0.1"], port=9042) + session = cluster.connect() + + # Confirm that the session is actually functioning before calling things good + rs = session.execute("select * from system.local") + log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) + log.info("Cassandra cluster ready") + return (cluster, session) + except: + log.info("Couldn't quite connect yet, will retry") + time.sleep(1) + + +def createSchema(session, schemaPathProvider): + schemaPath = schemaPathProvider() + log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) + with open(schemaPath) as schema: + buff = "" + for line in schema: + realLine = line.strip() + if len(realLine) == 0: + log.debug("Skipping empty statement") + continue + # TODO: this fails completely for CQL statements which span lines + if realLine.startswith("//"): + log.debug("Skipping commented statement") + continue + buff += (" " if len(buff) > 0 else "") + buff += realLine + if realLine.endswith(';'): + log.debug("Executing statement {}".format(buff)) + try: + session.execute(buff) + except Exception as exc: + log.error("Exception executing statement: {}".format(buff), exc_info=exc) + buff = "" diff --git a/python/adelphi/tests/util/schemadiff.py b/python/adelphi/tests/util/schemadiff.py index 948b379..f821faf 100644 --- a/python/adelphi/tests/util/schemadiff.py +++ b/python/adelphi/tests/util/schemadiff.py @@ -1,6 +1,5 @@ import base64 import hashlib -import logging # Imports required by the __main__ case below import sys diff --git a/python/adelphi/tox.ini b/python/adelphi/tox.ini index 43d5e5f..db08b16 100644 --- a/python/adelphi/tox.ini +++ b/python/adelphi/tox.ini @@ -3,7 +3,5 @@ envlist = py2,py3 [testenv] deps = pytest - docker ~= 4.4 subprocess32 ~= 3.5 - tenacity ~= 7.0 commands = pytest {posargs} \ No newline at end of file diff --git a/python/adelphi/tox.ini.dev b/python/adelphi/tox.ini.dev index 06b04ea..981f045 100644 --- a/python/adelphi/tox.ini.dev +++ b/python/adelphi/tox.ini.dev @@ -3,9 +3,7 @@ envlist = py2,py3 [testenv] deps = pytest - docker ~= 4.4 subprocess32 ~= 3.5 - tenacity ~= 7.0 setenv = KEEP_LOGS=1 CASSANDRA_VERSIONS=4.0-rc1 From c99f0c0543b8ad761a0dba37aa7c4717c9a47dc8 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 8 Jun 2021 23:49:10 -0500 Subject: [PATCH 02/30] Add automated generation of tox.ini --- python/adelphi/bin/run-adelphi-tests.py | 18 ++++++++++++++++++ python/adelphi/tests/integration/__init__.py | 4 +++- python/adelphi/tox.ini | 7 ------- python/adelphi/tox.ini.dev | 10 ---------- 4 files changed, 21 insertions(+), 18 deletions(-) delete mode 100644 python/adelphi/tox.ini delete mode 100644 python/adelphi/tox.ini.dev diff --git a/python/adelphi/bin/run-adelphi-tests.py b/python/adelphi/bin/run-adelphi-tests.py index 02ffd2c..7bd64eb 100644 --- a/python/adelphi/bin/run-adelphi-tests.py +++ b/python/adelphi/bin/run-adelphi-tests.py @@ -6,6 +6,7 @@ # manage the fixture iteration manually in this script. This also has the # nice side effect of moving a lot of C* checking/session code out of the test # suite, which in turn should allow us to write simpler tests. +import configparser import logging import os @@ -17,6 +18,10 @@ # Default C* versions to include in all integration tests CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-beta4"] +TOX_DEPENDENCIES = """pytest + subprocess32 ~= 3.5""" +TOX_CONFIG = "tox.ini" + logging.basicConfig(filename="adelphi-tests.log", level=logging.INFO) log = logging.getLogger('adelphi') @@ -33,6 +38,17 @@ def getCassandraVersions(): return CASSANDRA_VERSIONS +def writeToxIni(version): + config = configparser.ConfigParser() + config["tox"] = { "envlist": "py2, py3" } + envs = {"CASSANDRA_VERSION": version} + config["testenv"] = {"deps": TOX_DEPENDENCIES, \ + "commands": "pytest {posargs}", \ + "setenv": "CASSANDRA_VERSION = {}".format(version)} + with open(TOX_CONFIG, 'w') as configfile: + config.write(configfile) + + if __name__ == '__main__': client = docker.from_env() for version in getCassandraVersions(): @@ -41,6 +57,8 @@ def getCassandraVersions(): container = getContainer(client, version) try: + os.remove(TOX_CONFIG) + writeToxIni(version) tox.cmdline() except Exception as exc: log.error("Exception running tests for Cassandra version {}".format(version), exc_info=exc) diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 5a406a0..038b2fe 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -36,7 +36,9 @@ def makeTempDirs(self): self.dirs = TempDirs(base, outputDir) - def testAdelphi(self, version=None): + def testAdelphi(self): + # This should be set in the tox config + version = os.environ["CASSANDRA_VERSION"] log.info("Testing Cassandra version {}".format(version)) self.makeTempDirs() diff --git a/python/adelphi/tox.ini b/python/adelphi/tox.ini deleted file mode 100644 index db08b16..0000000 --- a/python/adelphi/tox.ini +++ /dev/null @@ -1,7 +0,0 @@ -[tox] -envlist = py2,py3 - -[testenv] -deps = pytest - subprocess32 ~= 3.5 -commands = pytest {posargs} \ No newline at end of file diff --git a/python/adelphi/tox.ini.dev b/python/adelphi/tox.ini.dev deleted file mode 100644 index 981f045..0000000 --- a/python/adelphi/tox.ini.dev +++ /dev/null @@ -1,10 +0,0 @@ -[tox] -envlist = py2,py3 - -[testenv] -deps = pytest - subprocess32 ~= 3.5 -setenv = - KEEP_LOGS=1 - CASSANDRA_VERSIONS=4.0-rc1 -commands = pytest {posargs} \ No newline at end of file From 2419d8cce142521597550cb26e229d457a71167b Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 14:38:06 -0500 Subject: [PATCH 03/30] Single file nb tests now working --- python/adelphi/bin/run-adelphi-tests.py | 5 +- python/adelphi/tests/integration/__init__.py | 34 +++++------ .../adelphi/tests/integration/nb/test_nb.py | 56 +++++++++++++++++++ .../nb/test_nb_export_outputdir.py | 43 -------------- .../integration/nb/test_nb_export_stdout.py | 37 ------------ .../integration/resources/nb-base-schema.cql | 2 +- python/adelphi/tests/util/cassandra_util.py | 11 ++-- 7 files changed, 82 insertions(+), 106 deletions(-) create mode 100644 python/adelphi/tests/integration/nb/test_nb.py delete mode 100644 python/adelphi/tests/integration/nb/test_nb_export_outputdir.py delete mode 100644 python/adelphi/tests/integration/nb/test_nb_export_stdout.py diff --git a/python/adelphi/bin/run-adelphi-tests.py b/python/adelphi/bin/run-adelphi-tests.py index 7bd64eb..f765cbb 100644 --- a/python/adelphi/bin/run-adelphi-tests.py +++ b/python/adelphi/bin/run-adelphi-tests.py @@ -16,7 +16,7 @@ from tenacity import retry, stop_after_attempt, wait_fixed # Default C* versions to include in all integration tests -CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-beta4"] +CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-rc1"] TOX_DEPENDENCIES = """pytest subprocess32 ~= 3.5""" @@ -57,7 +57,8 @@ def writeToxIni(version): container = getContainer(client, version) try: - os.remove(TOX_CONFIG) + if os.path.exists(TOX_CONFIG): + os.remove(TOX_CONFIG) writeToxIni(version) tox.cmdline() except Exception as exc: diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 038b2fe..3ad37f9 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -3,6 +3,11 @@ import shutil import tempfile +try: + import unittest2 as unittest +except ImportError: + import unittest + from collections import namedtuple from tests.util.cassandra_util import connectToLocalCassandra,createSchema @@ -11,7 +16,7 @@ TempDirs = namedtuple('TempDirs', 'basePath, outputDirPath') -class SchemaTestMixin: +class SchemaTestMixin(unittest.TestCase): def basePath(self, name): return os.path.join(self.dirs.basePath, name) @@ -36,25 +41,20 @@ def makeTempDirs(self): self.dirs = TempDirs(base, outputDir) - def testAdelphi(self): + def baseTestSetup(self): # This should be set in the tox config - version = os.environ["CASSANDRA_VERSION"] - log.info("Testing Cassandra version {}".format(version)) - + self.version = os.environ["CASSANDRA_VERSION"] + log.info("Testing Cassandra version {}".format(self.version)) self.makeTempDirs() - try: - (_,session) = connectToLocalCassandra() - createSchema(session, self.getBaseSchemaPath) - log.info("Running Adelphi") - self.runAdelphi(version) - log.info("Adelphi run completed, evaluating Adelphi output(s)") - self.evalAdelphiOutput(version) - finally: - self.cleanUpTempDirs(version) - - - def cleanUpTempDirs(self, version): + + + def baseTestTeardown(self): if "KEEP_LOGS" in os.environ: log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath)) else: shutil.rmtree(self.dirs.basePath) + + + def setupSchema(self, schemaPath): + (_,session) = connectToLocalCassandra() + createSchema(session, schemaPath) diff --git a/python/adelphi/tests/integration/nb/test_nb.py b/python/adelphi/tests/integration/nb/test_nb.py new file mode 100644 index 0000000..6357de0 --- /dev/null +++ b/python/adelphi/tests/integration/nb/test_nb.py @@ -0,0 +1,56 @@ +import glob +import logging +import os +import sys +import yaml + +if os.name == 'posix' and sys.version_info[0] < 3: + import subprocess32 as subprocess +else: + import subprocess + +from tests.integration import SchemaTestMixin + +log = logging.getLogger('adelphi') + +NB_SCHEMA_PATH = "tests/integration/resources/nb-base-schema.cql" +NB_REFERENCE_SCHEMA_PATH = "tests/integration/resources/nb-schemas/{}.yaml" + +class TestNb(SchemaTestMixin): + + def setUp(self): + super(TestNb, self).setUp() + self.baseTestSetup() + self.setupSchema(NB_SCHEMA_PATH) + + + def tearDown(self): + super(TestNb, self).tearDown() + self.baseTestTeardown() + + + def compareToReferenceYaml(self, comparePath, version=None): + referencePath = NB_REFERENCE_SCHEMA_PATH.format(version) + # Loader specification here to avoid a deprecation warning... see https://msg.pyyaml.org/load + referenceYaml = yaml.load(open(referencePath), Loader=yaml.FullLoader) + compareYaml = yaml.load(open(comparePath), Loader=yaml.FullLoader) + self.assertEqual(referenceYaml, compareYaml) + + + def test_stdout(self): + stdoutPath = self.stdoutPath(self.version) + stderrPath = self.stderrPath(self.version) + subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) + self.compareToReferenceYaml(self.stdoutPath(self.version), self.version) + + + def test_outputdir(self): + stderrPath = self.stderrPath(self.version) + outputDirPath = self.outputDirPath(self.version) + os.mkdir(outputDirPath) + subprocess.run("adelphi --output-dir={} export-nb 2>> {}".format(outputDirPath, stderrPath), shell=True) + + outputDirPath = self.outputDirPath(self.version) + outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) + self.assertEqual(len(outputSchemas), 1, "Export of nosqlbench config only supports a single keyspace") + self.compareToReferenceYaml(outputSchemas[0], self.version) diff --git a/python/adelphi/tests/integration/nb/test_nb_export_outputdir.py b/python/adelphi/tests/integration/nb/test_nb_export_outputdir.py deleted file mode 100644 index 2a239ca..0000000 --- a/python/adelphi/tests/integration/nb/test_nb_export_outputdir.py +++ /dev/null @@ -1,43 +0,0 @@ -import glob -import logging -import os -import sys -import yaml - -try: - import unittest2 as unittest -except ImportError: - import unittest - -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - -from tests.integration import SchemaTestMixin -from tests.integration.nb import ExportNbMixin - -log = logging.getLogger('adelphi') - -class TestNbExportOutputDir(unittest.TestCase, SchemaTestMixin, ExportNbMixin): - - def setUp(self): - super(TestNbExportOutputDir, self).setUp() - - - def getBaseSchemaPath(self): - return "tests/integration/resources/nb-base-schema.cql" - - - def runAdelphi(self, version=None): - stderrPath = self.stderrPath(version) - outputDirPath = self.outputDirPath(version) - os.mkdir(outputDirPath) - subprocess.run("adelphi --output-dir={} export-nb 2>> {}".format(outputDirPath, stderrPath), shell=True) - - - def evalAdelphiOutput(self, version=None): - outputDirPath = self.outputDirPath(version) - outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) - self.assertEqual(len(outputSchemas), 1, "Export of nosqlbench config only supports a single keyspace") - self.compareToReferenceYaml(outputSchemas[0], version) diff --git a/python/adelphi/tests/integration/nb/test_nb_export_stdout.py b/python/adelphi/tests/integration/nb/test_nb_export_stdout.py deleted file mode 100644 index 1365c0b..0000000 --- a/python/adelphi/tests/integration/nb/test_nb_export_stdout.py +++ /dev/null @@ -1,37 +0,0 @@ -import logging -import os -import sys - -try: - import unittest2 as unittest -except ImportError: - import unittest - -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - -from tests.integration import SchemaTestMixin -from tests.integration.nb import ExportNbMixin - -log = logging.getLogger('adelphi') - -class TestNbExportStdout(unittest.TestCase, SchemaTestMixin, ExportNbMixin): - - def setUp(self): - super(TestNbExportStdout, self).setUp() - - - def getBaseSchemaPath(self): - return "tests/integration/resources/nb-base-schema.cql" - - - def runAdelphi(self, version=None): - stdoutPath = self.stdoutPath(version) - stderrPath = self.stderrPath(version) - subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) - - - def evalAdelphiOutput(self, version=None): - self.compareToReferenceYaml(self.stdoutPath(version), version) diff --git a/python/adelphi/tests/integration/resources/nb-base-schema.cql b/python/adelphi/tests/integration/resources/nb-base-schema.cql index 412d521..dfbf7e3 100644 --- a/python/adelphi/tests/integration/resources/nb-base-schema.cql +++ b/python/adelphi/tests/integration/resources/nb-base-schema.cql @@ -1,4 +1,4 @@ -CREATE KEYSPACE testkeyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true; +CREATE KEYSPACE testkeyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true; CREATE TABLE testkeyspace.testtable ( key1 text, diff --git a/python/adelphi/tests/util/cassandra_util.py b/python/adelphi/tests/util/cassandra_util.py index 1f8acb1..7451f11 100644 --- a/python/adelphi/tests/util/cassandra_util.py +++ b/python/adelphi/tests/util/cassandra_util.py @@ -21,11 +21,10 @@ def connectToLocalCassandra(): return (cluster, session) except: log.info("Couldn't quite connect yet, will retry") - time.sleep(1) + time.sleep(3) -def createSchema(session, schemaPathProvider): - schemaPath = schemaPathProvider() +def createSchema(session, schemaPath): log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) with open(schemaPath) as schema: buff = "" @@ -34,16 +33,16 @@ def createSchema(session, schemaPathProvider): if len(realLine) == 0: log.debug("Skipping empty statement") continue - # TODO: this fails completely for CQL statements which span lines - if realLine.startswith("//"): + if realLine.startswith("//") or realLine.startswith("--"): log.debug("Skipping commented statement") continue buff += (" " if len(buff) > 0 else "") buff += realLine - if realLine.endswith(';'): + if buff.endswith(';'): log.debug("Executing statement {}".format(buff)) try: session.execute(buff) except Exception as exc: log.error("Exception executing statement: {}".format(buff), exc_info=exc) + finally: buff = "" From 98345e3352608f5c680a91b09baed6093d9728ab Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 14:50:10 -0500 Subject: [PATCH 04/30] Forgot to clean out tests.integration.nb package init --- python/adelphi/tests/integration/nb/__init__.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/python/adelphi/tests/integration/nb/__init__.py b/python/adelphi/tests/integration/nb/__init__.py index 0fffb1a..e69de29 100644 --- a/python/adelphi/tests/integration/nb/__init__.py +++ b/python/adelphi/tests/integration/nb/__init__.py @@ -1,10 +0,0 @@ -import yaml - -class ExportNbMixin: - - def compareToReferenceYaml(self, comparePath, version=None): - referencePath = "tests/integration/resources/nb-schemas/{}.yaml".format(version) - # Loader specification here to avoid a deprecation warning... see https://msg.pyyaml.org/load - referenceYaml = yaml.load(open(referencePath), Loader=yaml.FullLoader) - compareYaml = yaml.load(open(comparePath), Loader=yaml.FullLoader) - self.assertEqual(referenceYaml, compareYaml) From 47869f95591b07c53a01efeef2c2395f7fb3209d Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 15:54:58 -0500 Subject: [PATCH 05/30] First cut at a consolidated test_cql.py --- .../adelphi/tests/integration/cql/__init__.py | 46 ------ .../adelphi/tests/integration/cql/test_cql.py | 134 ++++++++++++++++++ .../cql/test_cql_export_outputdir.py | 53 ------- ...est_cql_export_outputdir_some_keyspaces.py | 44 ------ .../integration/cql/test_cql_export_stdout.py | 36 ----- .../test_cql_export_stdout_some_keyspaces.py | 36 ----- .../adelphi/tests/integration/nb/test_nb.py | 3 + 7 files changed, 137 insertions(+), 215 deletions(-) create mode 100644 python/adelphi/tests/integration/cql/test_cql.py delete mode 100644 python/adelphi/tests/integration/cql/test_cql_export_outputdir.py delete mode 100644 python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py delete mode 100644 python/adelphi/tests/integration/cql/test_cql_export_stdout.py delete mode 100644 python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py diff --git a/python/adelphi/tests/integration/cql/__init__.py b/python/adelphi/tests/integration/cql/__init__.py index 4b7ce1a..e69de29 100644 --- a/python/adelphi/tests/integration/cql/__init__.py +++ b/python/adelphi/tests/integration/cql/__init__.py @@ -1,46 +0,0 @@ -import logging - -from tests.util.schemadiff import cqlDigestGenerator -from tests.util.schema_util import get_schema - -def digestSet(schemaFile): - rv = set() - for (_, digest) in cqlDigestGenerator(schemaFile): - rv.add(digest) - return rv - - -def logCqlDigest(schemaFile, digestSet): - for (cql, digest) in cqlDigestGenerator(schemaFile): - if digest in digestSet: - log.info("Digest: {}, CQL: {}".format(digest,cql)) - -log = logging.getLogger('adelphi') - -class ExportCqlMixin: - - # TODO: Note that we currently have to disable support for SASI indexes when creating - # a schema since the base config for the Cassandra Docker images doesn't enable it. - # https://github.com/datastax/adelphi/issues/105 aims to fix this problem but until - # that's in place we simply exclude SASI indexes from testing. - def getBaseSchemaPath(self): - baseSchemaPath = self.basePath("base-schema.cql") - with open(baseSchemaPath, "w") as f: - f.write("\n\n".join(ks.export_as_string() for ks in get_schema(sasi=False).keyspaces)) - return baseSchemaPath - - - def compareToReferenceCql(self, referencePath, comparePath): - referenceSet = digestSet(referencePath) - compareSet = digestSet(comparePath) - - refOnlySet = referenceSet - compareSet - if len(refOnlySet) > 0: - log.info("Statements in reference file {} but not in compare file {}:".format(referencePath, comparePath)) - logCqlDigest(referencePath, refOnlySet) - compareOnlySet = compareSet - referenceSet - if len(compareOnlySet) > 0: - log.info("Statements in compare file {} but not in reference file {}:".format(comparePath, referencePath)) - logCqlDigest(comparePath, compareOnlySet) - - self.assertEqual(referenceSet, compareSet) diff --git a/python/adelphi/tests/integration/cql/test_cql.py b/python/adelphi/tests/integration/cql/test_cql.py new file mode 100644 index 0000000..c372682 --- /dev/null +++ b/python/adelphi/tests/integration/cql/test_cql.py @@ -0,0 +1,134 @@ +import glob +import logging +import os +import shutil +import sys + +if os.name == 'posix' and sys.version_info[0] < 3: + import subprocess32 as subprocess +else: + import subprocess + +from tests.integration import SchemaTestMixin +from tests.util.schemadiff import cqlDigestGenerator +from tests.util.schema_util import get_schema + +log = logging.getLogger('adelphi') + +CQL_REFERENCE_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}.cql" +CQL_REFERENCE_KS0_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}-ks0.cql" + +def digestSet(schemaFile): + rv = set() + for (_, digest) in cqlDigestGenerator(schemaFile): + rv.add(digest) + return rv + + +def logCqlDigest(schemaFile, digestSet): + for (cql, digest) in cqlDigestGenerator(schemaFile): + if digest in digestSet: + log.info("Digest: {}, CQL: {}".format(digest,cql)) + + +class TestCql(SchemaTestMixin): + + # ========================== Unittest infrastructure ========================== + def setUp(self): + super(TestCql, self).setUp() + self.baseTestSetup() + self.setupSchema(self.buildSchema()) + + + def tearDown(self): + super(TestCql, self).tearDown() + self.baseTestTeardown() + + + # ========================== Helper functions ========================== + # TODO: Note that we currently have to disable support for SASI indexes when creating + # a schema since the base config for the Cassandra Docker images doesn't enable it. + # https://github.com/datastax/adelphi/issues/105 aims to fix this problem but until + # that's in place we simply exclude SASI indexes from testing. + def buildSchema(self): + baseSchemaPath = self.basePath("base-schema.cql") + with open(baseSchemaPath, "w") as f: + f.write("\n\n".join(ks.export_as_string() for ks in get_schema(sasi=False).keyspaces)) + return baseSchemaPath + + + def compareToReferenceCql(self, referencePath, comparePath): + referenceSet = digestSet(referencePath) + compareSet = digestSet(comparePath) + + refOnlySet = referenceSet - compareSet + if len(refOnlySet) > 0: + log.info("Statements in reference file {} but not in compare file {}:".format(referencePath, comparePath)) + logCqlDigest(referencePath, refOnlySet) + compareOnlySet = compareSet - referenceSet + if len(compareOnlySet) > 0: + log.info("Statements in compare file {} but not in reference file {}:".format(comparePath, referencePath)) + logCqlDigest(comparePath, compareOnlySet) + + self.assertEqual(referenceSet, compareSet) + + + # ========================== Test functions ========================== + def test_stdout(self): + stdoutPath = self.stdoutPath(self.version) + stderrPath = self.stderrPath(self.version) + subprocess.run("adelphi export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) + + self.compareToReferenceCql( + CQL_REFERENCE_SCHEMA_PATH.format(self.version), + self.stdoutPath(self.version)) + + + def test_outputdir(self): + stderrPath = self.stderrPath(self.version) + outputDirPath = self.outputDirPath(self.version) + os.mkdir(outputDirPath) + subprocess.run("adelphi --output-dir={} export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True) + + # Basic idea here is to find all schemas written to the output dir and aggregate them into a single schema + # file. We then compare this aggregated file to the reference schema. Ordering is important here but + # the current keyspace names hash to something that causes individual keyspaces to be discovered in the + # correct order. + outputDirPath = self.outputDirPath(self.version) + allOutputFileName = "{}-all".format(self.version) + allOutputPath = self.outputDirPath(allOutputFileName) + + outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) + self.assertGreater(len(outputSchemas), 0) + with open(allOutputPath, "w+") as allOutputFile: + for outputSchema in outputSchemas: + with open(outputSchema) as outputSchemaFile: + shutil.copyfileobj(outputSchemaFile, allOutputFile) + allOutputFile.write("\n") + self.compareToReferenceCql( + CQL_REFERENCE_SCHEMA_PATH.format(self.version), + allOutputPath) + + + def test_some_keyspaces_stdout(self): + stdoutPath = self.stdoutPath(self.version) + stderrPath = self.stderrPath(self.version) + subprocess.run("adelphi --keyspaces=my_ks_0 export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) + + self.compareToReferenceCql( + CQL_REFERENCE_KS0_SCHEMA_PATH.format(self.version), + self.stdoutPath(self.version)) + + + def test_some_keyspaces_outputdir(self): + stderrPath = self.stderrPath(self.version) + outputDirPath = self.outputDirPath(self.version) + os.mkdir(outputDirPath) + subprocess.run("adelphi --output-dir={} --keyspaces=my_ks_0 export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True) + + outputDirPath = self.outputDirPath(self.version) + outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) + self.assertEqual(len(outputSchemas), 1) + self.compareToReferenceCql( + CQL_REFERENCE_KS0_SCHEMA_PATH.format(self.version), + outputSchemas[0]) diff --git a/python/adelphi/tests/integration/cql/test_cql_export_outputdir.py b/python/adelphi/tests/integration/cql/test_cql_export_outputdir.py deleted file mode 100644 index cbdcb44..0000000 --- a/python/adelphi/tests/integration/cql/test_cql_export_outputdir.py +++ /dev/null @@ -1,53 +0,0 @@ -import glob -import logging -import os -import shutil -import sys - -try: - import unittest2 as unittest -except ImportError: - import unittest - -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - -from tests.integration import SchemaTestMixin -from tests.integration.cql import ExportCqlMixin - -log = logging.getLogger('adelphi') - -class TestCqlExportOutputDir(unittest.TestCase, SchemaTestMixin, ExportCqlMixin): - - def setUp(self): - super(TestCqlExportOutputDir, self).setUp() - - - def runAdelphi(self, version): - stderrPath = self.stderrPath(version) - outputDirPath = self.outputDirPath(version) - os.mkdir(outputDirPath) - subprocess.run("adelphi --output-dir={} export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True) - - - def evalAdelphiOutput(self, version): - referencePath = "tests/integration/resources/cql-schemas/{}.cql".format(version) - - # Basic idea here is to find all schemas written to the output dir and aggregate them into a single schema - # file. We then compare this aggregated file to the reference schema. Ordering is important here but - # the current keyspace names hash to something that causes individual keyspaces to be discovered in the - # correct order. - outputDirPath = self.outputDirPath(version) - allOutputFileName = "{}-all".format(version) - allOutputPath = self.outputDirPath(allOutputFileName) - - outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) - self.assertGreater(len(outputSchemas), 0) - with open(allOutputPath, "w+") as allOutputFile: - for outputSchema in outputSchemas: - with open(outputSchema) as outputSchemaFile: - shutil.copyfileobj(outputSchemaFile, allOutputFile) - allOutputFile.write("\n") - self.compareToReferenceCql(referencePath, allOutputPath) diff --git a/python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py b/python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py deleted file mode 100644 index e9b79da..0000000 --- a/python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py +++ /dev/null @@ -1,44 +0,0 @@ -import glob -import logging -import os -import shutil -import sys - -try: - import unittest2 as unittest -except ImportError: - import unittest - -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - -from tests.integration import SchemaTestMixin -from tests.integration.cql import ExportCqlMixin - -log = logging.getLogger('adelphi') - -# Implemented for https://github.com/datastax/adelphi/issues/106. -# -# We should see exactly one keyspace schema written to the output dir and that file -# should match up exactly to the requested keyspace -class TestCqlExportOutputDirSomeKeyspaces(unittest.TestCase, SchemaTestMixin, ExportCqlMixin): - - def setUp(self): - super(TestCqlExportOutputDirSomeKeyspaces, self).setUp() - - - def runAdelphi(self, version): - stderrPath = self.stderrPath(version) - outputDirPath = self.outputDirPath(version) - os.mkdir(outputDirPath) - subprocess.run("adelphi --output-dir={} --keyspaces=my_ks_0 export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True) - - - def evalAdelphiOutput(self, version): - referencePath = "tests/integration/resources/cql-schemas/{}-ks0.cql".format(version) - outputDirPath = self.outputDirPath(version) - outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) - self.assertEqual(len(outputSchemas), 1) - self.compareToReferenceCql(referencePath, outputSchemas[0]) diff --git a/python/adelphi/tests/integration/cql/test_cql_export_stdout.py b/python/adelphi/tests/integration/cql/test_cql_export_stdout.py deleted file mode 100644 index 9bf5ee5..0000000 --- a/python/adelphi/tests/integration/cql/test_cql_export_stdout.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging -import os -import sys - -try: - import unittest2 as unittest -except ImportError: - import unittest - -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - -from tests.integration import SchemaTestMixin -from tests.integration.cql import ExportCqlMixin - -log = logging.getLogger('adelphi') - -class TestCqlExportStdout(unittest.TestCase, SchemaTestMixin, ExportCqlMixin): - - def setUp(self): - super(TestCqlExportStdout, self).setUp() - - - def runAdelphi(self, version): - stdoutPath = self.stdoutPath(version) - stderrPath = self.stderrPath(version) - subprocess.run("adelphi export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) - - - def evalAdelphiOutput(self, version): - referencePath = "tests/integration/resources/cql-schemas/{}.cql".format(version) - self.compareToReferenceCql(referencePath, self.stdoutPath(version)) - - diff --git a/python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py b/python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py deleted file mode 100644 index 2448b66..0000000 --- a/python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py +++ /dev/null @@ -1,36 +0,0 @@ -import logging -import os -import sys - -try: - import unittest2 as unittest -except ImportError: - import unittest - -if os.name == 'posix' and sys.version_info[0] < 3: - import subprocess32 as subprocess -else: - import subprocess - -from tests.integration import SchemaTestMixin -from tests.integration.cql import ExportCqlMixin - -log = logging.getLogger('adelphi') - -class TestCqlExportStdoutSomeKeyspaces(unittest.TestCase, SchemaTestMixin, ExportCqlMixin): - - def setUp(self): - super(TestCqlExportStdoutSomeKeyspaces, self).setUp() - - - def runAdelphi(self, version): - stdoutPath = self.stdoutPath(version) - stderrPath = self.stderrPath(version) - subprocess.run("adelphi --keyspaces=my_ks_0 export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) - - - def evalAdelphiOutput(self, version): - referencePath = "tests/integration/resources/cql-schemas/{}-ks0.cql".format(version) - self.compareToReferenceCql(referencePath, self.stdoutPath(version)) - - diff --git a/python/adelphi/tests/integration/nb/test_nb.py b/python/adelphi/tests/integration/nb/test_nb.py index 6357de0..72a7a95 100644 --- a/python/adelphi/tests/integration/nb/test_nb.py +++ b/python/adelphi/tests/integration/nb/test_nb.py @@ -18,6 +18,7 @@ class TestNb(SchemaTestMixin): + # ========================== Unittest infrastructure ========================== def setUp(self): super(TestNb, self).setUp() self.baseTestSetup() @@ -29,6 +30,7 @@ def tearDown(self): self.baseTestTeardown() + # ========================== Helper functions ========================== def compareToReferenceYaml(self, comparePath, version=None): referencePath = NB_REFERENCE_SCHEMA_PATH.format(version) # Loader specification here to avoid a deprecation warning... see https://msg.pyyaml.org/load @@ -37,6 +39,7 @@ def compareToReferenceYaml(self, comparePath, version=None): self.assertEqual(referenceYaml, compareYaml) + # ========================== Test functions ========================== def test_stdout(self): stdoutPath = self.stdoutPath(self.version) stderrPath = self.stderrPath(self.version) From 5dfe8b1fd558e6fb0021247ac640f579b20fdebf Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 16:27:33 -0500 Subject: [PATCH 06/30] Tests appear to be passing on py2 and py3 again --- python/adelphi/tests/integration/__init__.py | 28 +++++++++++++------ .../adelphi/tests/integration/cql/test_cql.py | 13 +++++---- .../adelphi/tests/integration/nb/test_nb.py | 10 +++---- 3 files changed, 32 insertions(+), 19 deletions(-) diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 3ad37f9..8f24617 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -10,13 +10,24 @@ from collections import namedtuple -from tests.util.cassandra_util import connectToLocalCassandra,createSchema +from tests.util.cassandra_util import connectToLocalCassandra, createSchema log = logging.getLogger('adelphi') TempDirs = namedtuple('TempDirs', 'basePath, outputDirPath') -class SchemaTestMixin(unittest.TestCase): + +def setupSchema(schemaPath): + (_,session) = connectToLocalCassandra() + createSchema(session, schemaPath) + + +def dropKeyspace(keyspace): + (_,session) = connectToLocalCassandra() + session.execute("drop keyspace {}".format(keyspace)) + + +class SchemaTestCase(unittest.TestCase): def basePath(self, name): return os.path.join(self.dirs.basePath, name) @@ -41,20 +52,19 @@ def makeTempDirs(self): self.dirs = TempDirs(base, outputDir) - def baseTestSetup(self): + def setUp(self): + super(SchemaTestCase, self).setUp() + # This should be set in the tox config self.version = os.environ["CASSANDRA_VERSION"] log.info("Testing Cassandra version {}".format(self.version)) self.makeTempDirs() - def baseTestTeardown(self): + def tearDown(self): + super(SchemaTestCase, self).tearDown() + if "KEEP_LOGS" in os.environ: log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath)) else: shutil.rmtree(self.dirs.basePath) - - - def setupSchema(self, schemaPath): - (_,session) = connectToLocalCassandra() - createSchema(session, schemaPath) diff --git a/python/adelphi/tests/integration/cql/test_cql.py b/python/adelphi/tests/integration/cql/test_cql.py index c372682..e633beb 100644 --- a/python/adelphi/tests/integration/cql/test_cql.py +++ b/python/adelphi/tests/integration/cql/test_cql.py @@ -9,7 +9,7 @@ else: import subprocess -from tests.integration import SchemaTestMixin +from tests.integration import SchemaTestCase, setupSchema, dropKeyspace from tests.util.schemadiff import cqlDigestGenerator from tests.util.schema_util import get_schema @@ -31,18 +31,21 @@ def logCqlDigest(schemaFile, digestSet): log.info("Digest: {}, CQL: {}".format(digest,cql)) -class TestCql(SchemaTestMixin): +class TestCql(SchemaTestCase): # ========================== Unittest infrastructure ========================== def setUp(self): super(TestCql, self).setUp() - self.baseTestSetup() - self.setupSchema(self.buildSchema()) + log.warning("Creating schema") + setupSchema(self.buildSchema()) def tearDown(self): super(TestCql, self).tearDown() - self.baseTestTeardown() + log.warning("Dropping keyspace testkeyspace my_ks_0") + dropKeyspace("my_ks_0") + log.warning("Dropping keyspace testkeyspace my_ks_1") + dropKeyspace("my_ks_1") # ========================== Helper functions ========================== diff --git a/python/adelphi/tests/integration/nb/test_nb.py b/python/adelphi/tests/integration/nb/test_nb.py index 72a7a95..288a62f 100644 --- a/python/adelphi/tests/integration/nb/test_nb.py +++ b/python/adelphi/tests/integration/nb/test_nb.py @@ -9,25 +9,25 @@ else: import subprocess -from tests.integration import SchemaTestMixin +from tests.integration import SchemaTestCase, setupSchema, dropKeyspace log = logging.getLogger('adelphi') NB_SCHEMA_PATH = "tests/integration/resources/nb-base-schema.cql" NB_REFERENCE_SCHEMA_PATH = "tests/integration/resources/nb-schemas/{}.yaml" -class TestNb(SchemaTestMixin): +class TestNb(SchemaTestCase): # ========================== Unittest infrastructure ========================== def setUp(self): super(TestNb, self).setUp() - self.baseTestSetup() - self.setupSchema(NB_SCHEMA_PATH) + setupSchema(NB_SCHEMA_PATH) def tearDown(self): super(TestNb, self).tearDown() - self.baseTestTeardown() + log.warning("Dropping keyspace testkeyspace") + dropKeyspace("testkeyspace") # ========================== Helper functions ========================== From ed51b6701a70c92611096b7a11b5fed7c1e58953 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 16:32:56 -0500 Subject: [PATCH 07/30] Now that export-specific functionality is consolidated within a single test we don't really need export-specific packages --- python/adelphi/tests/integration/cql/__init__.py | 0 python/adelphi/tests/integration/nb/__init__.py | 0 python/adelphi/tests/integration/{cql => }/test_cql.py | 0 python/adelphi/tests/integration/{nb => }/test_nb.py | 0 4 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 python/adelphi/tests/integration/cql/__init__.py delete mode 100644 python/adelphi/tests/integration/nb/__init__.py rename python/adelphi/tests/integration/{cql => }/test_cql.py (100%) rename python/adelphi/tests/integration/{nb => }/test_nb.py (100%) diff --git a/python/adelphi/tests/integration/cql/__init__.py b/python/adelphi/tests/integration/cql/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/adelphi/tests/integration/nb/__init__.py b/python/adelphi/tests/integration/nb/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/python/adelphi/tests/integration/cql/test_cql.py b/python/adelphi/tests/integration/test_cql.py similarity index 100% rename from python/adelphi/tests/integration/cql/test_cql.py rename to python/adelphi/tests/integration/test_cql.py diff --git a/python/adelphi/tests/integration/nb/test_nb.py b/python/adelphi/tests/integration/test_nb.py similarity index 100% rename from python/adelphi/tests/integration/nb/test_nb.py rename to python/adelphi/tests/integration/test_nb.py From 4002710c7c5ca69fa204962eff32d77217fd36fe Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 16:45:32 -0500 Subject: [PATCH 08/30] Would like to keep unittests runnable on their own as well --- python/adelphi/tests/integration/test_cql.py | 9 +++++++++ python/adelphi/tests/integration/test_nb.py | 9 +++++++++ 2 files changed, 18 insertions(+) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index e633beb..6392bd9 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -4,6 +4,11 @@ import shutil import sys +try: + import unittest2 as unittest +except ImportError: + import unittest + if os.name == 'posix' and sys.version_info[0] < 3: import subprocess32 as subprocess else: @@ -135,3 +140,7 @@ def test_some_keyspaces_outputdir(self): self.compareToReferenceCql( CQL_REFERENCE_KS0_SCHEMA_PATH.format(self.version), outputSchemas[0]) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file diff --git a/python/adelphi/tests/integration/test_nb.py b/python/adelphi/tests/integration/test_nb.py index 288a62f..5e44c53 100644 --- a/python/adelphi/tests/integration/test_nb.py +++ b/python/adelphi/tests/integration/test_nb.py @@ -4,6 +4,11 @@ import sys import yaml +try: + import unittest2 as unittest +except ImportError: + import unittest + if os.name == 'posix' and sys.version_info[0] < 3: import subprocess32 as subprocess else: @@ -57,3 +62,7 @@ def test_outputdir(self): outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) self.assertEqual(len(outputSchemas), 1, "Export of nosqlbench config only supports a single keyspace") self.compareToReferenceYaml(outputSchemas[0], self.version) + + +if __name__ == '__main__': + unittest.main() \ No newline at end of file From 2adabcedbe2517614b053df2c3e9969f8c87e8e3 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 17:05:07 -0500 Subject: [PATCH 09/30] Fix tox invocation to avoid exiting after first run completes --- python/adelphi/bin/run-adelphi-tests.py | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/python/adelphi/bin/run-adelphi-tests.py b/python/adelphi/bin/run-adelphi-tests.py index f765cbb..666c5df 100644 --- a/python/adelphi/bin/run-adelphi-tests.py +++ b/python/adelphi/bin/run-adelphi-tests.py @@ -22,9 +22,6 @@ subprocess32 ~= 3.5""" TOX_CONFIG = "tox.ini" -logging.basicConfig(filename="adelphi-tests.log", level=logging.INFO) -log = logging.getLogger('adelphi') - @retry(stop=stop_after_attempt(3), wait=wait_fixed(2)) def getContainer(client, version): @@ -44,7 +41,8 @@ def writeToxIni(version): envs = {"CASSANDRA_VERSION": version} config["testenv"] = {"deps": TOX_DEPENDENCIES, \ "commands": "pytest {posargs}", \ - "setenv": "CASSANDRA_VERSION = {}".format(version)} + "setenv": """CASSANDRA_VERSION = {} + KEEP_LOGS=1""".format(version)} with open(TOX_CONFIG, 'w') as configfile: config.write(configfile) @@ -53,18 +51,23 @@ def writeToxIni(version): client = docker.from_env() for version in getCassandraVersions(): - log.info("Running test suite for Cassandra version {}".format(version)) + print("Running test suite for Cassandra version {}".format(version)) container = getContainer(client, version) try: if os.path.exists(TOX_CONFIG): os.remove(TOX_CONFIG) writeToxIni(version) - tox.cmdline() + # cmdline() will raise SystemExit when it's done so trap that here to avoid + # exiting all the things + try: + tox.cmdline() + except SystemExit: + pass except Exception as exc: - log.error("Exception running tests for Cassandra version {}".format(version), exc_info=exc) + print("Exception running tests for Cassandra version {}".format(version), exc) finally: if "KEEP_CONTAINER" in os.environ: - log.info("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'") + print("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'") else: container.stop() From fbc8476c8b282cbfc350b0b0e15c026a18e22191 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 9 Jun 2021 17:17:07 -0500 Subject: [PATCH 10/30] test_nb doesn't need method-specific data for init so let's make schema creation a class-level op --- python/adelphi/tests/integration/test_nb.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/adelphi/tests/integration/test_nb.py b/python/adelphi/tests/integration/test_nb.py index 5e44c53..ce68a18 100644 --- a/python/adelphi/tests/integration/test_nb.py +++ b/python/adelphi/tests/integration/test_nb.py @@ -24,13 +24,12 @@ class TestNb(SchemaTestCase): # ========================== Unittest infrastructure ========================== - def setUp(self): - super(TestNb, self).setUp() + @classmethod + def setUpClass(cls): setupSchema(NB_SCHEMA_PATH) - - def tearDown(self): - super(TestNb, self).tearDown() + @classmethod + def tearDownClass(cls): log.warning("Dropping keyspace testkeyspace") dropKeyspace("testkeyspace") From 21ef1c2cfd0b022a4bb9a61bcf7d0bfae3906e77 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 10 Jun 2021 00:27:12 -0500 Subject: [PATCH 11/30] Add click for handling CLI args --- python/adelphi/bin/run-adelphi-tests.py | 42 ++++++++++++++----------- python/adelphi/test-requirements.txt | 1 + 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/python/adelphi/bin/run-adelphi-tests.py b/python/adelphi/bin/run-adelphi-tests.py index 666c5df..0445131 100644 --- a/python/adelphi/bin/run-adelphi-tests.py +++ b/python/adelphi/bin/run-adelphi-tests.py @@ -7,16 +7,16 @@ # nice side effect of moving a lot of C* checking/session code out of the test # suite, which in turn should allow us to write simpler tests. import configparser -import logging import os +import click import tox import docker from tenacity import retry, stop_after_attempt, wait_fixed # Default C* versions to include in all integration tests -CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-rc1"] +DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-rc1"] TOX_DEPENDENCIES = """pytest subprocess32 ~= 3.5""" @@ -28,28 +28,31 @@ def getContainer(client, version): return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042: 9042}, image="cassandra:{}".format(version)) -def getCassandraVersions(): - if "CASSANDRA_VERSIONS" in os.environ: - return [s.strip() for s in os.environ["CASSANDRA_VERSIONS"].split(',')] - else: - return CASSANDRA_VERSIONS - - def writeToxIni(version): config = configparser.ConfigParser() config["tox"] = { "envlist": "py2, py3" } envs = {"CASSANDRA_VERSION": version} config["testenv"] = {"deps": TOX_DEPENDENCIES, \ "commands": "pytest {posargs}", \ - "setenv": """CASSANDRA_VERSION = {} - KEEP_LOGS=1""".format(version)} + "setenv": "CASSANDRA_VERSION = {}".format(version)} with open(TOX_CONFIG, 'w') as configfile: config.write(configfile) - -if __name__ == '__main__': +@click.command() +@click.option('--cassandra', '-c', multiple=True, type=str) +@click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False)) +@click.option("--pytest", "-t", type=str, help="Arguments to be passed to pytest") +def runtests(cassandra, python, pytest): client = docker.from_env() - for version in getCassandraVersions(): + tox_args = ["-e {}".format(py) for py in python] if python else [] + if pytest: + tox_args.append("--") + tox_args.append(pytest) + print("Full tox args: {}".format(tox_args)) + + cassandra_versions = cassandra or DEFAULT_CASSANDRA_VERSIONS + print("Cassandra versions to test: {}".format(','.join(cassandra_versions))) + for version in cassandra_versions: print("Running test suite for Cassandra version {}".format(version)) container = getContainer(client, version) @@ -61,13 +64,14 @@ def writeToxIni(version): # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things try: - tox.cmdline() + tox.cmdline(tox_args) except SystemExit: pass except Exception as exc: print("Exception running tests for Cassandra version {}".format(version), exc) finally: - if "KEEP_CONTAINER" in os.environ: - print("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'") - else: - container.stop() + container.stop() + + +if __name__ == '__main__': + runtests(obj={}) diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt index a479f87..2bedca4 100644 --- a/python/adelphi/test-requirements.txt +++ b/python/adelphi/test-requirements.txt @@ -1,2 +1,3 @@ docker ~= 4.4 tenacity ~= 7.0 +click ~= 7.1 From 8a38d9a8e5751e2a10f61ea56fdf7926b73ae38d Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 10 Jun 2021 00:51:38 -0500 Subject: [PATCH 12/30] Comment updates --- python/adelphi/tests/util/cassandra_util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/python/adelphi/tests/util/cassandra_util.py b/python/adelphi/tests/util/cassandra_util.py index 7451f11..1383b59 100644 --- a/python/adelphi/tests/util/cassandra_util.py +++ b/python/adelphi/tests/util/cassandra_util.py @@ -1,5 +1,4 @@ -# Using "c8" rather than "cassandra" to avoid any conflicts with "cassandra" package in -# cassandra-driver +# A few utility methods for interacting with Cassandra from within tests import logging import time @@ -15,6 +14,9 @@ def connectToLocalCassandra(): session = cluster.connect() # Confirm that the session is actually functioning before calling things good + # + # TODO: Might be worth seeing if we can move some part of this validation step + # to tenacity rather than implementing it manually. rs = session.execute("select * from system.local") log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) log.info("Cassandra cluster ready") From e707cf840961374ecfe8a2cc260628ddc187638c Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 10 Jun 2021 10:45:32 -0500 Subject: [PATCH 13/30] Rename + making test app executable --- python/adelphi/bin/{run-adelphi-tests.py => test-adelphi} | 2 ++ 1 file changed, 2 insertions(+) rename python/adelphi/bin/{run-adelphi-tests.py => test-adelphi} (99%) diff --git a/python/adelphi/bin/run-adelphi-tests.py b/python/adelphi/bin/test-adelphi similarity index 99% rename from python/adelphi/bin/run-adelphi-tests.py rename to python/adelphi/bin/test-adelphi index 0445131..1970814 100644 --- a/python/adelphi/bin/run-adelphi-tests.py +++ b/python/adelphi/bin/test-adelphi @@ -1,3 +1,5 @@ +#!python + # Run test suite for some set of Cassandra versions. # # We implement this as a front-end script because the tox/pytest/unittest chain From 4298a8e941cab998ef18faa0a1d453e38d8667b5 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Thu, 10 Jun 2021 15:42:45 -0500 Subject: [PATCH 14/30] Update keyspace drop logic to be more automatic --- python/adelphi/bin/test-adelphi | 17 ++++++--- python/adelphi/test-requirements.txt | 3 +- python/adelphi/tests/integration/__init__.py | 29 ++++++++++++---- python/adelphi/tests/integration/test_cql.py | 10 +++--- python/adelphi/tests/integration/test_nb.py | 11 ++++-- python/adelphi/tests/util/cassandra_util.py | 36 +++++++++++--------- 6 files changed, 68 insertions(+), 38 deletions(-) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index 1970814..e090c25 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -11,22 +11,25 @@ import configparser import os -import click -import tox +from tests.util.cassandra_util import connectToLocalCassandra +import click import docker from tenacity import retry, stop_after_attempt, wait_fixed +import tox + # Default C* versions to include in all integration tests DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-rc1"] TOX_DEPENDENCIES = """pytest - subprocess32 ~= 3.5""" + subprocess32 ~= 3.5 + tenacity ~= 7.0""" TOX_CONFIG = "tox.ini" @retry(stop=stop_after_attempt(3), wait=wait_fixed(2)) -def getContainer(client, version): +def runCassandraContainer(client, version): return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042: 9042}, image="cassandra:{}".format(version)) @@ -57,12 +60,16 @@ def runtests(cassandra, python, pytest): for version in cassandra_versions: print("Running test suite for Cassandra version {}".format(version)) - container = getContainer(client, version) + container = runCassandraContainer(client, version) + + print("Validating connection to local Cassandra") + connectToLocalCassandra() try: if os.path.exists(TOX_CONFIG): os.remove(TOX_CONFIG) writeToxIni(version) + # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things try: diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt index 2bedca4..7156e2e 100644 --- a/python/adelphi/test-requirements.txt +++ b/python/adelphi/test-requirements.txt @@ -1,3 +1,4 @@ +click ~= 7.1 +cassandra-driver ~= 3.24 docker ~= 4.4 tenacity ~= 7.0 -click ~= 7.1 diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 8f24617..14abec5 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -10,21 +10,33 @@ from collections import namedtuple -from tests.util.cassandra_util import connectToLocalCassandra, createSchema +from tests.util.cassandra_util import callWithCassandra, createSchema log = logging.getLogger('adelphi') TempDirs = namedtuple('TempDirs', 'basePath, outputDirPath') +def __keyspacesForCluster(cluster): + return set(cluster.metadata.keyspaces.keys()) + + def setupSchema(schemaPath): - (_,session) = connectToLocalCassandra() - createSchema(session, schemaPath) + return callWithCassandra(lambda _,s: createSchema(s, schemaPath)) + +def getAllKeyspaces(): + return callWithCassandra(lambda c,s: __keyspacesForCluster(c)) -def dropKeyspace(keyspace): - (_,session) = connectToLocalCassandra() - session.execute("drop keyspace {}".format(keyspace)) + +def dropNewKeyspaces(origKeyspaces): + def dropFn(cluster, session): + currentKeyspaces = __keyspacesForCluster(cluster) + droppingKeyspaces = currentKeyspaces - origKeyspaces + log.info("Dropping the following keyspaes created by this test: {}".format(",".join(droppingKeyspaces))) + for keyspace in droppingKeyspaces: + session.execute("drop keyspace {}".format(keyspace)) + return callWithCassandra(dropFn) class SchemaTestCase(unittest.TestCase): @@ -53,17 +65,22 @@ def makeTempDirs(self): def setUp(self): + # Invoking for completeness; for unittest base setUp/tearDown impls are no-ops super(SchemaTestCase, self).setUp() # This should be set in the tox config self.version = os.environ["CASSANDRA_VERSION"] log.info("Testing Cassandra version {}".format(self.version)) + self.makeTempDirs() def tearDown(self): super(SchemaTestCase, self).tearDown() + # TODO: Note that there's no easy way to access this from test-adelphi unless we modify the + # ini generation code... and I'm not completely sure that's worth it. Might want to think + # about just deleting this outright... or making it a CLI option that can be easily accessed. if "KEEP_LOGS" in os.environ: log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath)) else: diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index 6392bd9..70d9e76 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -14,7 +14,7 @@ else: import subprocess -from tests.integration import SchemaTestCase, setupSchema, dropKeyspace +from tests.integration import SchemaTestCase, setupSchema, getAllKeyspaces, dropNewKeyspaces from tests.util.schemadiff import cqlDigestGenerator from tests.util.schema_util import get_schema @@ -41,16 +41,14 @@ class TestCql(SchemaTestCase): # ========================== Unittest infrastructure ========================== def setUp(self): super(TestCql, self).setUp() - log.warning("Creating schema") + self.origKeyspaces = getAllKeyspaces() + log.info("Creating schema") setupSchema(self.buildSchema()) def tearDown(self): super(TestCql, self).tearDown() - log.warning("Dropping keyspace testkeyspace my_ks_0") - dropKeyspace("my_ks_0") - log.warning("Dropping keyspace testkeyspace my_ks_1") - dropKeyspace("my_ks_1") + dropNewKeyspaces(self.origKeyspaces) # ========================== Helper functions ========================== diff --git a/python/adelphi/tests/integration/test_nb.py b/python/adelphi/tests/integration/test_nb.py index ce68a18..66edab4 100644 --- a/python/adelphi/tests/integration/test_nb.py +++ b/python/adelphi/tests/integration/test_nb.py @@ -14,7 +14,7 @@ else: import subprocess -from tests.integration import SchemaTestCase, setupSchema, dropKeyspace +from tests.integration import SchemaTestCase, setupSchema, getAllKeyspaces, dropNewKeyspaces log = logging.getLogger('adelphi') @@ -23,15 +23,19 @@ class TestNb(SchemaTestCase): + origKeyspaces = None + # ========================== Unittest infrastructure ========================== @classmethod def setUpClass(cls): + TestNb.origKeyspaces = getAllKeyspaces() + log.info("Creating schema") setupSchema(NB_SCHEMA_PATH) + @classmethod def tearDownClass(cls): - log.warning("Dropping keyspace testkeyspace") - dropKeyspace("testkeyspace") + dropNewKeyspaces(TestNb.origKeyspaces) # ========================== Helper functions ========================== @@ -45,6 +49,7 @@ def compareToReferenceYaml(self, comparePath, version=None): # ========================== Test functions ========================== def test_stdout(self): + print("All keyspaces: {}".format(getAllKeyspaces())) stdoutPath = self.stdoutPath(self.version) stderrPath = self.stderrPath(self.version) subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) diff --git a/python/adelphi/tests/util/cassandra_util.py b/python/adelphi/tests/util/cassandra_util.py index 1383b59..02a40b2 100644 --- a/python/adelphi/tests/util/cassandra_util.py +++ b/python/adelphi/tests/util/cassandra_util.py @@ -4,26 +4,19 @@ from cassandra.cluster import Cluster +from tenacity import retry, wait_fixed + log = logging.getLogger('adelphi') +@retry(wait=wait_fixed(3)) def connectToLocalCassandra(): - session = None - while not session: - try: - cluster = Cluster(["127.0.0.1"], port=9042) - session = cluster.connect() - - # Confirm that the session is actually functioning before calling things good - # - # TODO: Might be worth seeing if we can move some part of this validation step - # to tenacity rather than implementing it manually. - rs = session.execute("select * from system.local") - log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) - log.info("Cassandra cluster ready") - return (cluster, session) - except: - log.info("Couldn't quite connect yet, will retry") - time.sleep(3) + cluster = Cluster(["127.0.0.1"], port=9042) + session = cluster.connect() + + # Confirm that the session is actually functioning before calling things good + rs = session.execute("select * from system.local") + log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) + return (cluster, session) def createSchema(session, schemaPath): @@ -48,3 +41,12 @@ def createSchema(session, schemaPath): log.error("Exception executing statement: {}".format(buff), exc_info=exc) finally: buff = "" + + +def callWithCassandra(someFn): + cluster = None + try: + (cluster,session) = connectToLocalCassandra() + return someFn(cluster, session) + finally: + cluster.shutdown() From 69f3287c404f8307093f16841cec4de25d26bbda Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 11 Jun 2021 15:35:21 -0500 Subject: [PATCH 15/30] Forgot to add tox to the test dependencies --- python/adelphi/test-requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt index 7156e2e..087a0ae 100644 --- a/python/adelphi/test-requirements.txt +++ b/python/adelphi/test-requirements.txt @@ -2,3 +2,4 @@ click ~= 7.1 cassandra-driver ~= 3.24 docker ~= 4.4 tenacity ~= 7.0 +tox ~= 3.22 From 011f0b437e6dd8285ba04d42e7ae799974a20c6b Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 11 Jun 2021 15:40:54 -0500 Subject: [PATCH 16/30] Just testing some things out --- .github/workflows/adelphi-python.yaml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/adelphi-python.yaml diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml new file mode 100644 index 0000000..7270afc --- /dev/null +++ b/.github/workflows/adelphi-python.yaml @@ -0,0 +1,24 @@ +name: adelphi Python + +on: + push: + branches: [ $default-branch ] + pull_request: + branches: [ $default-branch ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + cd python/adelphi + pip install -r test-requirements.txt + python bin/test-adelphi -c 2.1.22 -p py2 From 7a9e911f8cd576128f156cea0b63ad7d1cc511f3 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 11 Jun 2021 15:49:01 -0500 Subject: [PATCH 17/30] Try specifying a branch --- .github/workflows/adelphi-python.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml index 7270afc..a320d41 100644 --- a/.github/workflows/adelphi-python.yaml +++ b/.github/workflows/adelphi-python.yaml @@ -2,9 +2,9 @@ name: adelphi Python on: push: - branches: [ $default-branch ] + branches: [ 115-breadth-first-testing ] pull_request: - branches: [ $default-branch ] + branches: [ 115-breadth-first-testing ] jobs: build: From f1592085e7d24474e0ec08420588072bca19af99 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 14 Jun 2021 16:30:38 -0500 Subject: [PATCH 18/30] Moved tests.util.cassandra_util into adelphi.store so that necessary fns are available after installing adelphi Python package. Also cleaned up some unused imports and some excessive logging configs. --- python/adelphi/adelphi/anonymize.py | 1 - python/adelphi/adelphi/gh.py | 1 - python/adelphi/adelphi/store.py | 56 +++++++++++++++----- python/adelphi/bin/test-adelphi | 10 ++-- python/adelphi/tests/integration/__init__.py | 13 +++-- python/adelphi/tests/util/cassandra_util.py | 52 ------------------ 6 files changed, 58 insertions(+), 75 deletions(-) delete mode 100644 python/adelphi/tests/util/cassandra_util.py diff --git a/python/adelphi/adelphi/anonymize.py b/python/adelphi/adelphi/anonymize.py index f2622ba..13264ed 100644 --- a/python/adelphi/adelphi/anonymize.py +++ b/python/adelphi/adelphi/anonymize.py @@ -15,7 +15,6 @@ # Functions and constants related to the anonymization process from adelphi.store import get_standard_columns_from_table_metadata -import re # default prefixes for the anonymized names KEYSPACE_PREFIX = "ks" diff --git a/python/adelphi/adelphi/gh.py b/python/adelphi/adelphi/gh.py index a6a14b7..4610053 100644 --- a/python/adelphi/adelphi/gh.py +++ b/python/adelphi/adelphi/gh.py @@ -22,7 +22,6 @@ from github import Github -logging.basicConfig(level=logging.INFO) log = logging.getLogger('adelphi') # We're assuming any storage repo will be created after the conversion to "main" diff --git a/python/adelphi/adelphi/store.py b/python/adelphi/adelphi/store.py index b3f2d69..4b250e7 100644 --- a/python/adelphi/adelphi/store.py +++ b/python/adelphi/adelphi/store.py @@ -27,7 +27,8 @@ from cassandra.cluster import Cluster, ExecutionProfile, EXEC_PROFILE_DEFAULT, default_lbp_factory from cassandra.auth import PlainTextAuthProvider -logging.basicConfig(level=logging.INFO) +from tenacity import retry + log = logging.getLogger('adelphi') system_keyspaces = set(["system", @@ -39,20 +40,26 @@ "system_views"]) def build_auth_provider(username = None,password = None): - # instantiate auth provider if credentials have been provided - auth_provider = None - if username is not None and password is not None: - auth_provider = PlainTextAuthProvider(username=username, password=password) - return auth_provider + """Instantiate auth provider if credentials have been provided""" + if username is None or password is None: + return None + return PlainTextAuthProvider(username=username, password=password) +@retry def with_cluster(cluster_fn, hosts, port, username = None, password = None): ep = ExecutionProfile(load_balancing_policy=default_lbp_factory()) cluster = Cluster(hosts, port=port, auth_provider=build_auth_provider(username,password), execution_profiles={EXEC_PROFILE_DEFAULT: ep}) - cluster.connect() - rv = cluster_fn(cluster) - cluster.shutdown() - return rv + try: + cluster.connect() + return cluster_fn(cluster) + finally: + cluster.shutdown() + + +@retry +def with_local_cluster(cluster_fn): + return with_cluster(cluster_fn, ["127.0.0.1"], port=9042) def build_keyspace_objects(keyspaces, metadata): @@ -71,9 +78,7 @@ def partition(pred, iterable): def get_standard_columns_from_table_metadata(table_metadata): - """ - Return the standard columns and ensure to exclude pk and ck ones. - """ + """Return the standard columns and ensure to exclude pk and ck ones""" partition_column_names = [c.name for c in table_metadata.partition_key] clustering_column_names = [c.name for c in table_metadata.clustering_key] standard_columns = [] @@ -96,3 +101,28 @@ def set_replication_factor(selected_keyspaces, factor): log.debug("Replication for keyspace " + ks.name+ ": " + str(ks.replication_strategy)) strategy = ks.replication_strategy strategy.replication_factor_info = factor + + +def create_schema(session, schemaPath): + """Read schema CQL document and apply CQL commands to cluster""" + log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) + with open(schemaPath) as schema: + buff = "" + for line in schema: + realLine = line.strip() + if len(realLine) == 0: + log.debug("Skipping empty statement") + continue + if realLine.startswith("//") or realLine.startswith("--"): + log.debug("Skipping commented statement") + continue + buff += (" " if len(buff) > 0 else "") + buff += realLine + if buff.endswith(';'): + log.debug("Executing statement {}".format(buff)) + try: + session.execute(buff) + except Exception as exc: + log.error("Exception executing statement: {}".format(buff), exc_info=exc) + finally: + buff = "" diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index e090c25..d7dc9fd 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -11,7 +11,7 @@ import configparser import os -from tests.util.cassandra_util import connectToLocalCassandra +from adelphi.store import with_local_cluster import click import docker @@ -36,7 +36,6 @@ def runCassandraContainer(client, version): def writeToxIni(version): config = configparser.ConfigParser() config["tox"] = { "envlist": "py2, py3" } - envs = {"CASSANDRA_VERSION": version} config["testenv"] = {"deps": TOX_DEPENDENCIES, \ "commands": "pytest {posargs}", \ "setenv": "CASSANDRA_VERSION = {}".format(version)} @@ -63,7 +62,12 @@ def runtests(cassandra, python, pytest): container = runCassandraContainer(client, version) print("Validating connection to local Cassandra") - connectToLocalCassandra() + def validationFn(cluster): + session = cluster.connect() + rs = session.execute("select * from system.local") + print("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) + return (cluster, session) + with_local_cluster.retry_with(stop=stop_after_attempt(5), wait=wait_fixed(3))(validationFn) try: if os.path.exists(TOX_CONFIG): diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 14abec5..87da0e2 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -10,7 +10,7 @@ from collections import namedtuple -from tests.util.cassandra_util import callWithCassandra, createSchema +from adelphi.store import with_local_cluster, create_schema log = logging.getLogger('adelphi') @@ -22,21 +22,24 @@ def __keyspacesForCluster(cluster): def setupSchema(schemaPath): - return callWithCassandra(lambda _,s: createSchema(s, schemaPath)) + def schemaFn(cluster): + return create_schema(cluster.connect(), schemaPath) + return with_local_cluster(schemaFn) def getAllKeyspaces(): - return callWithCassandra(lambda c,s: __keyspacesForCluster(c)) + return with_local_cluster(__keyspacesForCluster) def dropNewKeyspaces(origKeyspaces): - def dropFn(cluster, session): + def dropFn(cluster): currentKeyspaces = __keyspacesForCluster(cluster) droppingKeyspaces = currentKeyspaces - origKeyspaces log.info("Dropping the following keyspaes created by this test: {}".format(",".join(droppingKeyspaces))) + session = cluster.connect() for keyspace in droppingKeyspaces: session.execute("drop keyspace {}".format(keyspace)) - return callWithCassandra(dropFn) + return with_local_cluster(dropFn) class SchemaTestCase(unittest.TestCase): diff --git a/python/adelphi/tests/util/cassandra_util.py b/python/adelphi/tests/util/cassandra_util.py deleted file mode 100644 index 02a40b2..0000000 --- a/python/adelphi/tests/util/cassandra_util.py +++ /dev/null @@ -1,52 +0,0 @@ -# A few utility methods for interacting with Cassandra from within tests -import logging -import time - -from cassandra.cluster import Cluster - -from tenacity import retry, wait_fixed - -log = logging.getLogger('adelphi') - -@retry(wait=wait_fixed(3)) -def connectToLocalCassandra(): - cluster = Cluster(["127.0.0.1"], port=9042) - session = cluster.connect() - - # Confirm that the session is actually functioning before calling things good - rs = session.execute("select * from system.local") - log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) - return (cluster, session) - - -def createSchema(session, schemaPath): - log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) - with open(schemaPath) as schema: - buff = "" - for line in schema: - realLine = line.strip() - if len(realLine) == 0: - log.debug("Skipping empty statement") - continue - if realLine.startswith("//") or realLine.startswith("--"): - log.debug("Skipping commented statement") - continue - buff += (" " if len(buff) > 0 else "") - buff += realLine - if buff.endswith(';'): - log.debug("Executing statement {}".format(buff)) - try: - session.execute(buff) - except Exception as exc: - log.error("Exception executing statement: {}".format(buff), exc_info=exc) - finally: - buff = "" - - -def callWithCassandra(someFn): - cluster = None - try: - (cluster,session) = connectToLocalCassandra() - return someFn(cluster, session) - finally: - cluster.shutdown() From 80a2ea36a0fab2ae1e09c05b6362080acc28e69a Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 14 Jun 2021 16:56:23 -0500 Subject: [PATCH 19/30] Promote test-adelphi to a full bin script within the package; should simplify the import of functionality from adelphi packages. Also simplify the dependency specification; test-requirements.txt really should be testing-only stuff now. --- .github/workflows/adelphi-python.yaml | 6 +++--- python/adelphi/setup.py | 5 +++-- python/adelphi/test-requirements.txt | 3 --- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml index a320d41..d7d1e33 100644 --- a/.github/workflows/adelphi-python.yaml +++ b/.github/workflows/adelphi-python.yaml @@ -19,6 +19,6 @@ jobs: python-version: '3.x' - name: Install dependencies run: | - cd python/adelphi - pip install -r test-requirements.txt - python bin/test-adelphi -c 2.1.22 -p py2 + cd python + pip install ./adelphi + test-adelphi -c 2.1.22 -p py2 diff --git a/python/adelphi/setup.py b/python/adelphi/setup.py index 79bc65c..9e30846 100644 --- a/python/adelphi/setup.py +++ b/python/adelphi/setup.py @@ -8,7 +8,8 @@ 'cassandra-driver ~= 3.24', 'click ~= 7.1', 'PyGithub ~= 1.45', - 'PyYAML ~= 5.4' + 'PyYAML ~= 5.4', + 'tenacity ~= 7.0' ] if not PY3: @@ -48,6 +49,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules' ], packages=['adelphi'], - scripts=['bin/adelphi'], + scripts=['bin/adelphi','bin/test-adelphi'], install_requires=dependencies, ) diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt index 087a0ae..9c59653 100644 --- a/python/adelphi/test-requirements.txt +++ b/python/adelphi/test-requirements.txt @@ -1,5 +1,2 @@ -click ~= 7.1 -cassandra-driver ~= 3.24 docker ~= 4.4 -tenacity ~= 7.0 tox ~= 3.22 From 6e2a38bc53c04604d85608233e650a4e5e9ef826 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 14 Jun 2021 17:08:37 -0500 Subject: [PATCH 20/30] Forgot to include pip install of test dependencies as well --- .github/workflows/adelphi-python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml index d7d1e33..fe92cff 100644 --- a/.github/workflows/adelphi-python.yaml +++ b/.github/workflows/adelphi-python.yaml @@ -21,4 +21,5 @@ jobs: run: | cd python pip install ./adelphi + pip install -r ./adelphi/test-requirements.txt test-adelphi -c 2.1.22 -p py2 From 4bf717405a15d2592917152bc32144a9038d1312 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 14 Jun 2021 17:25:48 -0500 Subject: [PATCH 21/30] Make sure we're in the directory with setup.py when we invoke test-adelphi (since this will in turn be required for tox) --- .github/workflows/adelphi-python.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml index fe92cff..7c43944 100644 --- a/.github/workflows/adelphi-python.yaml +++ b/.github/workflows/adelphi-python.yaml @@ -21,5 +21,6 @@ jobs: run: | cd python pip install ./adelphi - pip install -r ./adelphi/test-requirements.txt + cd adelphi + pip install -r ./test-requirements.txt test-adelphi -c 2.1.22 -p py2 From fea0e52b2863dd7323fd121b74137f6b2d8f7b58 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 15 Jun 2021 10:46:40 -0500 Subject: [PATCH 22/30] Explicit step for running tests + removal of constraints around C*/Python versions --- .github/workflows/adelphi-python.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml index 7c43944..d8524ad 100644 --- a/.github/workflows/adelphi-python.yaml +++ b/.github/workflows/adelphi-python.yaml @@ -23,4 +23,6 @@ jobs: pip install ./adelphi cd adelphi pip install -r ./test-requirements.txt - test-adelphi -c 2.1.22 -p py2 + - name: Execute tests + run: | + test-adelphi From bd78e57f01eb8e21d14828d5a73833eda928d1ab Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 15 Jun 2021 10:56:23 -0500 Subject: [PATCH 23/30] Separate cd for the separate command --- .github/workflows/adelphi-python.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml index d8524ad..5e11473 100644 --- a/.github/workflows/adelphi-python.yaml +++ b/.github/workflows/adelphi-python.yaml @@ -25,4 +25,5 @@ jobs: pip install -r ./test-requirements.txt - name: Execute tests run: | + cd python/adelphi test-adelphi From 5221b7450c7bafdc8b5c584598d4c0fb93de1abd Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 15 Jun 2021 15:25:41 -0500 Subject: [PATCH 24/30] Breaking 2.1 CQL integration tests to confirm that GH actions tests correctly report failure --- .../adelphi/tests/integration/resources/cql-schemas/2.1.22.cql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql b/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql index 26d7af8..f253d2b 100644 --- a/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql +++ b/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql @@ -2,7 +2,7 @@ CREATE KEYSPACE IF NOT EXISTS ks_0 WITH replication = {'class': 'SimpleStrategy' CREATE TYPE IF NOT EXISTS ks_0.udt_0 ( fld_0 text, - fld_1 int + fld_2 int ); CREATE TYPE IF NOT EXISTS ks_0.udt_1 ( From ddc889d0c1821d6c34615afd3f578c5d30822274 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 16 Jun 2021 14:33:40 -0500 Subject: [PATCH 25/30] Fix CLI logic to handle specification of partial C* version numbers --- python/adelphi/bin/test-adelphi | 36 +++++++++++++++++++-- python/adelphi/tests/integration/test_nb.py | 1 - 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index d7dc9fd..0c1a7c0 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -10,6 +10,7 @@ # suite, which in turn should allow us to write simpler tests. import configparser import os +import re from adelphi.store import with_local_cluster @@ -42,6 +43,34 @@ def writeToxIni(version): with open(TOX_CONFIG, 'w') as configfile: config.write(configfile) + +def buildVersionMap(): + assert sorted(DEFAULT_CASSANDRA_VERSIONS) + rv = {v:v for v in DEFAULT_CASSANDRA_VERSIONS} + majorMinorPattern = re.compile(r"(\d\.\d).*") + for v in DEFAULT_CASSANDRA_VERSIONS: + majorMinorMatch = majorMinorPattern.match(v) + if majorMinorMatch: + majorMinor = majorMinorMatch.group(1) + rv[majorMinor] = v + # The reason we needed to asset that our input was sorted; we want the last + # major version entry we discover in the list to map to the major version. + # So "2" => "2.2.whatever" rather than "2.1.whatever" + rv[majorMinor.split('.')[0]] = v + return rv + + +def resolveCassandraVersions(cassandra_versions): + if not cassandra_versions: + return DEFAULT_CASSANDRA_VERSIONS + versionMap = buildVersionMap() + computedVersions = [x for x in [versionMap.get(v) for v in cassandra_versions] if x is not None] + if not computedVersions: + print("Could not compute valid Cassandra versions based on args, using defaults") + return DEFAULT_CASSANDRA_VERSIONS + return computedVersions + + @click.command() @click.option('--cassandra', '-c', multiple=True, type=str) @click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False)) @@ -54,9 +83,9 @@ def runtests(cassandra, python, pytest): tox_args.append(pytest) print("Full tox args: {}".format(tox_args)) - cassandra_versions = cassandra or DEFAULT_CASSANDRA_VERSIONS + cassandra_versions = resolveCassandraVersions(cassandra) print("Cassandra versions to test: {}".format(','.join(cassandra_versions))) - for version in cassandra_versions: + for version in resolveCassandraVersions(cassandra_versions): print("Running test suite for Cassandra version {}".format(version)) container = runCassandraContainer(client, version) @@ -77,7 +106,8 @@ def runtests(cassandra, python, pytest): # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things try: - tox.cmdline(tox_args) + status = tox.cmdline(tox_args) + print("tox status: {}".format(status)) except SystemExit: pass except Exception as exc: diff --git a/python/adelphi/tests/integration/test_nb.py b/python/adelphi/tests/integration/test_nb.py index 66edab4..7c711f5 100644 --- a/python/adelphi/tests/integration/test_nb.py +++ b/python/adelphi/tests/integration/test_nb.py @@ -49,7 +49,6 @@ def compareToReferenceYaml(self, comparePath, version=None): # ========================== Test functions ========================== def test_stdout(self): - print("All keyspaces: {}".format(getAllKeyspaces())) stdoutPath = self.stdoutPath(self.version) stderrPath = self.stderrPath(self.version) subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) From a2818d1727ba46c23ab08b5b47588f2fac9f224c Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 16 Jun 2021 15:52:17 -0500 Subject: [PATCH 26/30] Set test-adelphi exit status code based on the number of tox invocations which returned a non-zero status code themselves --- python/adelphi/bin/test-adelphi | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index 0c1a7c0..df44dd6 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -11,6 +11,7 @@ import configparser import os import re +import sys from adelphi.store import with_local_cluster @@ -85,6 +86,7 @@ def runtests(cassandra, python, pytest): cassandra_versions = resolveCassandraVersions(cassandra) print("Cassandra versions to test: {}".format(','.join(cassandra_versions))) + exitCodes = [] for version in resolveCassandraVersions(cassandra_versions): print("Running test suite for Cassandra version {}".format(version)) @@ -108,13 +110,13 @@ def runtests(cassandra, python, pytest): try: status = tox.cmdline(tox_args) print("tox status: {}".format(status)) - except SystemExit: - pass + except SystemExit as exc: + exitCodes.append(exc.code) except Exception as exc: print("Exception running tests for Cassandra version {}".format(version), exc) finally: container.stop() - + sys.exit(sum(1 for x in exitCodes if x != 0)) if __name__ == '__main__': runtests(obj={}) From 2990f2c53fe6f8d940cff74849a3bb1edaacdd71 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 16 Jun 2021 16:54:18 -0500 Subject: [PATCH 27/30] Revert "Breaking 2.1 CQL integration tests to confirm that GH actions tests correctly report failure" This reverts commit 5221b7450c7bafdc8b5c584598d4c0fb93de1abd. --- .../adelphi/tests/integration/resources/cql-schemas/2.1.22.cql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql b/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql index f253d2b..26d7af8 100644 --- a/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql +++ b/python/adelphi/tests/integration/resources/cql-schemas/2.1.22.cql @@ -2,7 +2,7 @@ CREATE KEYSPACE IF NOT EXISTS ks_0 WITH replication = {'class': 'SimpleStrategy' CREATE TYPE IF NOT EXISTS ks_0.udt_0 ( fld_0 text, - fld_2 int + fld_1 int ); CREATE TYPE IF NOT EXISTS ks_0.udt_1 ( From 7b87fd728133b60244a5af8db09ceefba805a442 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 16 Jun 2021 17:09:01 -0500 Subject: [PATCH 28/30] Removing original (and incorrect) code to get tox exit codes --- python/adelphi/bin/test-adelphi | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index df44dd6..dc4bbde 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -108,8 +108,7 @@ def runtests(cassandra, python, pytest): # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things try: - status = tox.cmdline(tox_args) - print("tox status: {}".format(status)) + tox.cmdline(tox_args) except SystemExit as exc: exitCodes.append(exc.code) except Exception as exc: From 7ed1cc0b19fc13f8ab95384a19f1363f184d3bd3 Mon Sep 17 00:00:00 2001 From: Jeff DiNoto Date: Thu, 17 Jun 2021 15:13:58 -0400 Subject: [PATCH 29/30] Update .gitignore to exclude generated tox file (#156) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index aeae72d..70324f0 100644 --- a/.gitignore +++ b/.gitignore @@ -56,6 +56,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ +python/adelphi/tox.ini # Translations *.mo From 98fa005cef22aa752596672a21af6f4622e7b486 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 21 Jun 2021 06:47:49 -0500 Subject: [PATCH 30/30] Bump 3.11.x C* version to 3.11.10 --- python/adelphi/bin/test-adelphi | 2 +- .../resources/cql-schemas/{3.11.9-ks0.cql => 3.11.10-ks0.cql} | 0 .../resources/cql-schemas/{3.11.9.cql => 3.11.10.cql} | 0 .../resources/nb-schemas/{3.11.9.yaml => 3.11.10.yaml} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename python/adelphi/tests/integration/resources/cql-schemas/{3.11.9-ks0.cql => 3.11.10-ks0.cql} (100%) rename python/adelphi/tests/integration/resources/cql-schemas/{3.11.9.cql => 3.11.10.cql} (100%) rename python/adelphi/tests/integration/resources/nb-schemas/{3.11.9.yaml => 3.11.10.yaml} (100%) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index e090c25..63e3891 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -20,7 +20,7 @@ import tox # Default C* versions to include in all integration tests -DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-rc1"] +DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.10", "4.0-rc1"] TOX_DEPENDENCIES = """pytest subprocess32 ~= 3.5 diff --git a/python/adelphi/tests/integration/resources/cql-schemas/3.11.9-ks0.cql b/python/adelphi/tests/integration/resources/cql-schemas/3.11.10-ks0.cql similarity index 100% rename from python/adelphi/tests/integration/resources/cql-schemas/3.11.9-ks0.cql rename to python/adelphi/tests/integration/resources/cql-schemas/3.11.10-ks0.cql diff --git a/python/adelphi/tests/integration/resources/cql-schemas/3.11.9.cql b/python/adelphi/tests/integration/resources/cql-schemas/3.11.10.cql similarity index 100% rename from python/adelphi/tests/integration/resources/cql-schemas/3.11.9.cql rename to python/adelphi/tests/integration/resources/cql-schemas/3.11.10.cql diff --git a/python/adelphi/tests/integration/resources/nb-schemas/3.11.9.yaml b/python/adelphi/tests/integration/resources/nb-schemas/3.11.10.yaml similarity index 100% rename from python/adelphi/tests/integration/resources/nb-schemas/3.11.9.yaml rename to python/adelphi/tests/integration/resources/nb-schemas/3.11.10.yaml