From 3f2efc5f724dc983aebc702146c233330481cb99 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 11 Jun 2021 14:21:53 -0500 Subject: [PATCH 01/13] Logging CQL statements rather than digests when we observe mismatches --- python/adelphi/tests/integration/test_cql.py | 35 +++++++------------ .../util/{schemadiff.py => schema_diff.py} | 6 ++-- 2 files changed, 16 insertions(+), 25 deletions(-) rename python/adelphi/tests/util/{schemadiff.py => schema_diff.py} (82%) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index 70d9e76..51ed957 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -15,7 +15,7 @@ import subprocess from tests.integration import SchemaTestCase, setupSchema, getAllKeyspaces, dropNewKeyspaces -from tests.util.schemadiff import cqlDigestGenerator +from tests.util.schema_diff import cqlDigestGenerator from tests.util.schema_util import get_schema log = logging.getLogger('adelphi') @@ -23,19 +23,6 @@ CQL_REFERENCE_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}.cql" CQL_REFERENCE_KS0_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}-ks0.cql" -def digestSet(schemaFile): - rv = set() - for (_, digest) in cqlDigestGenerator(schemaFile): - rv.add(digest) - return rv - - -def logCqlDigest(schemaFile, digestSet): - for (cql, digest) in cqlDigestGenerator(schemaFile): - if digest in digestSet: - log.info("Digest: {}, CQL: {}".format(digest,cql)) - - class TestCql(SchemaTestCase): # ========================== Unittest infrastructure ========================== @@ -64,19 +51,23 @@ def buildSchema(self): def compareToReferenceCql(self, referencePath, comparePath): - referenceSet = digestSet(referencePath) - compareSet = digestSet(comparePath) + referenceDict = dict(cqlDigestGenerator(referencePath)) + compareDict = dict(cqlDigestGenerator(comparePath)) + referenceDigests = set(referenceDict.keys()) + compareDigests =set(compareDict.keys()) - refOnlySet = referenceSet - compareSet - if len(refOnlySet) > 0: + referenceOnlySet = referenceDigests - compareDigests + if len(referenceOnlySet) > 0: log.info("Statements in reference file {} but not in compare file {}:".format(referencePath, comparePath)) - logCqlDigest(referencePath, refOnlySet) - compareOnlySet = compareSet - referenceSet + for digest in referenceOnlySet: + log.info(referenceDict[digest]) + compareOnlySet = compareDigests - referenceDigests if len(compareOnlySet) > 0: log.info("Statements in compare file {} but not in reference file {}:".format(comparePath, referencePath)) - logCqlDigest(comparePath, compareOnlySet) + for digest in compareOnlySet: + log.info(compareDict[digest]) - self.assertEqual(referenceSet, compareSet) + self.assertEqual(set(referenceDict.values()), set(compareDict.values())) # ========================== Test functions ========================== diff --git a/python/adelphi/tests/util/schemadiff.py b/python/adelphi/tests/util/schema_diff.py similarity index 82% rename from python/adelphi/tests/util/schemadiff.py rename to python/adelphi/tests/util/schema_diff.py index f821faf..725f546 100644 --- a/python/adelphi/tests/util/schemadiff.py +++ b/python/adelphi/tests/util/schema_diff.py @@ -13,16 +13,16 @@ def cqlDigestGenerator(schemaPath): continue buff += (" " if len(buff) > 0 else "") buff += realLine - if realLine.endswith(';'): + if buff.endswith(';'): m = hashlib.sha256() m.update(buff.encode('utf-8')) # We want to return a string containing the base64 representation # so that the user doesn't have to mess around with bytestrings - yield (buff, base64.b64encode(m.digest()).decode('utf-8')) + yield (base64.b64encode(m.digest()).decode('utf-8'), buff) buff = "" if __name__ == "__main__": """Preserving this for validation of the logic above""" - for (cql, digest) in cqlDigestGenerator(sys.argv[1]): + for (digest, cql) in cqlDigestGenerator(sys.argv[1]): print("Digest: {}, CQL: {}".format(digest,cql)) From e5dd14f3b3feb14454f75b09b272d28f6a3dfab6 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 30 Jul 2021 15:47:44 -0500 Subject: [PATCH 02/13] Move to difflib for outputting differences. Also make sure all arrays of lines fed into difflib include a trailing newline to avoid trivial mismatches. Shell redirection won't always give us a trailing newline so we manually make sure it's there. --- python/adelphi/tests/integration/test_cql.py | 41 +++++++++++--------- python/adelphi/tests/util/schema_diff.py | 28 ------------- 2 files changed, 22 insertions(+), 47 deletions(-) delete mode 100644 python/adelphi/tests/util/schema_diff.py diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index 51ed957..d84f3f9 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -1,3 +1,4 @@ +import difflib import glob import logging import os @@ -15,7 +16,6 @@ import subprocess from tests.integration import SchemaTestCase, setupSchema, getAllKeyspaces, dropNewKeyspaces -from tests.util.schema_diff import cqlDigestGenerator from tests.util.schema_util import get_schema log = logging.getLogger('adelphi') @@ -23,6 +23,13 @@ CQL_REFERENCE_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}.cql" CQL_REFERENCE_KS0_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}-ks0.cql" +def linesWithNewline(f): + rv = f.readlines() + lastLine = rv[-1] + if not lastLine.endswith("\n"): + rv[-1] = lastLine + "\n" + return rv + class TestCql(SchemaTestCase): # ========================== Unittest infrastructure ========================== @@ -51,24 +58,20 @@ def buildSchema(self): def compareToReferenceCql(self, referencePath, comparePath): - referenceDict = dict(cqlDigestGenerator(referencePath)) - compareDict = dict(cqlDigestGenerator(comparePath)) - referenceDigests = set(referenceDict.keys()) - compareDigests =set(compareDict.keys()) - - referenceOnlySet = referenceDigests - compareDigests - if len(referenceOnlySet) > 0: - log.info("Statements in reference file {} but not in compare file {}:".format(referencePath, comparePath)) - for digest in referenceOnlySet: - log.info(referenceDict[digest]) - compareOnlySet = compareDigests - referenceDigests - if len(compareOnlySet) > 0: - log.info("Statements in compare file {} but not in reference file {}:".format(comparePath, referencePath)) - for digest in compareOnlySet: - log.info(compareDict[digest]) - - self.assertEqual(set(referenceDict.values()), set(compareDict.values())) - + with open(referencePath) as referenceFile, open(comparePath) as compareFile: + diffGen = difflib.unified_diff( \ + linesWithNewline(compareFile), \ + linesWithNewline(referenceFile), \ + fromfile=os.path.basename(comparePath), \ + tofile=os.path.basename(referencePath)) + + log.info("Diff of generated file against reference file") + diffEmpty = True + for line in diffGen: + diffEmpty = False + log.info(line) + + self.assertTrue(diffEmpty) # ========================== Test functions ========================== def test_stdout(self): diff --git a/python/adelphi/tests/util/schema_diff.py b/python/adelphi/tests/util/schema_diff.py deleted file mode 100644 index 725f546..0000000 --- a/python/adelphi/tests/util/schema_diff.py +++ /dev/null @@ -1,28 +0,0 @@ -import base64 -import hashlib - -# Imports required by the __main__ case below -import sys - -def cqlDigestGenerator(schemaPath): - buff = "" - with open(schemaPath) as schema: - for line in schema: - realLine = line.strip() - if len(realLine) == 0 or realLine.isspace(): - continue - buff += (" " if len(buff) > 0 else "") - buff += realLine - if buff.endswith(';'): - m = hashlib.sha256() - m.update(buff.encode('utf-8')) - # We want to return a string containing the base64 representation - # so that the user doesn't have to mess around with bytestrings - yield (base64.b64encode(m.digest()).decode('utf-8'), buff) - buff = "" - - -if __name__ == "__main__": - """Preserving this for validation of the logic above""" - for (digest, cql) in cqlDigestGenerator(sys.argv[1]): - print("Digest: {}, CQL: {}".format(digest,cql)) From 14e1d29502598bdeea1ee7f212fa081f9fc2202d Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 16 Aug 2021 16:57:43 -0500 Subject: [PATCH 03/13] Fix false test failures + clean up the dispaly a bit --- python/adelphi/tests/integration/test_cql.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index d84f3f9..be7ca05 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -59,19 +59,24 @@ def buildSchema(self): def compareToReferenceCql(self, referencePath, comparePath): with open(referencePath) as referenceFile, open(comparePath) as compareFile: + compareLines = linesWithNewline(compareFile) + referenceLines = linesWithNewline(referenceFile) + diffGen = difflib.unified_diff( \ - linesWithNewline(compareFile), \ - linesWithNewline(referenceFile), \ + compareLines, \ + referenceLines, \ fromfile=os.path.basename(comparePath), \ tofile=os.path.basename(referencePath)) - log.info("Diff of generated file against reference file") diffEmpty = True for line in diffGen: + if diffEmpty: + print("Diff of generated file ({}) against reference file ({})".format(comparePath, referencePath)) diffEmpty = False - log.info(line) + print(line.strip()) - self.assertTrue(diffEmpty) + if not diffEmpty: + self.fail() # ========================== Test functions ========================== def test_stdout(self): From 63e9150a91338f03678448c32a52b0fc32bd92ff Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Mon, 16 Aug 2021 17:11:20 -0500 Subject: [PATCH 04/13] Another tweak to diff display format --- python/adelphi/tests/integration/test_cql.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index be7ca05..b57756a 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -65,13 +65,15 @@ def compareToReferenceCql(self, referencePath, comparePath): diffGen = difflib.unified_diff( \ compareLines, \ referenceLines, \ - fromfile=os.path.basename(comparePath), \ - tofile=os.path.basename(referencePath)) + fromfile=os.path.abspath(comparePath), \ + tofile=os.path.abspath(referencePath)) diffEmpty = True for line in diffGen: if diffEmpty: - print("Diff of generated file ({}) against reference file ({})".format(comparePath, referencePath)) + print("Diff of generated file ({}) against reference file ({})".format( \ + os.path.basename(comparePath), \ + os.path.basename(referencePath))) diffEmpty = False print(line.strip()) From 8eff34c769162ae9b5020a10b507ad60710f60bb Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 17 Aug 2021 16:32:16 -0500 Subject: [PATCH 05/13] Some minor cleanups --- python/adelphi/tests/integration/test_cql.py | 46 ++++++++++---------- python/adelphi/tests/util/schema_util.py | 13 +++--- 2 files changed, 30 insertions(+), 29 deletions(-) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index b57756a..14f4415 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -30,6 +30,7 @@ def linesWithNewline(f): rv[-1] = lastLine + "\n" return rv + class TestCql(SchemaTestCase): # ========================== Unittest infrastructure ========================== @@ -62,17 +63,17 @@ def compareToReferenceCql(self, referencePath, comparePath): compareLines = linesWithNewline(compareFile) referenceLines = linesWithNewline(referenceFile) - diffGen = difflib.unified_diff( \ - compareLines, \ - referenceLines, \ - fromfile=os.path.abspath(comparePath), \ + diffGen = difflib.unified_diff( + compareLines, + referenceLines, + fromfile=os.path.abspath(comparePath), tofile=os.path.abspath(referencePath)) diffEmpty = True for line in diffGen: if diffEmpty: - print("Diff of generated file ({}) against reference file ({})".format( \ - os.path.basename(comparePath), \ + print("Diff of generated file ({}) against reference file ({})".format( + os.path.basename(comparePath), os.path.basename(referencePath))) diffEmpty = False print(line.strip()) @@ -80,6 +81,22 @@ def compareToReferenceCql(self, referencePath, comparePath): if not diffEmpty: self.fail() + + def combineSchemas(self): + outputDirPath = self.outputDirPath(self.version) + allOutputFileName = "{}-all".format(self.version) + allOutputPath = self.outputDirPath(allOutputFileName) + + outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) + self.assertGreater(len(outputSchemas), 0) + with open(allOutputPath, "w+") as allOutputFile: + for outputSchema in outputSchemas: + with open(outputSchema) as outputSchemaFile: + shutil.copyfileobj(outputSchemaFile, allOutputFile) + allOutputFile.write("\n") + + return allOutputPath + # ========================== Test functions ========================== def test_stdout(self): stdoutPath = self.stdoutPath(self.version) @@ -97,24 +114,9 @@ def test_outputdir(self): os.mkdir(outputDirPath) subprocess.run("adelphi --output-dir={} export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True) - # Basic idea here is to find all schemas written to the output dir and aggregate them into a single schema - # file. We then compare this aggregated file to the reference schema. Ordering is important here but - # the current keyspace names hash to something that causes individual keyspaces to be discovered in the - # correct order. - outputDirPath = self.outputDirPath(self.version) - allOutputFileName = "{}-all".format(self.version) - allOutputPath = self.outputDirPath(allOutputFileName) - - outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) - self.assertGreater(len(outputSchemas), 0) - with open(allOutputPath, "w+") as allOutputFile: - for outputSchema in outputSchemas: - with open(outputSchema) as outputSchemaFile: - shutil.copyfileobj(outputSchemaFile, allOutputFile) - allOutputFile.write("\n") self.compareToReferenceCql( CQL_REFERENCE_SCHEMA_PATH.format(self.version), - allOutputPath) + self.combineSchemas()) def test_some_keyspaces_stdout(self): diff --git a/python/adelphi/tests/util/schema_util.py b/python/adelphi/tests/util/schema_util.py index 0905c90..f5d1216 100644 --- a/python/adelphi/tests/util/schema_util.py +++ b/python/adelphi/tests/util/schema_util.py @@ -92,14 +92,13 @@ def get_keyspace(name, durable_writes, strategy_class, strategy_options, sasi=Tr return keyspace def get_schema(sasi=True): - # build a couple of keyspaces - keyspaces = [] - for k in range(2): - keyspace = get_keyspace("my_ks_%s" % k, True, "SimpleStrategy", {"replication_factor": 1}, sasi=sasi) - keyspaces.append(keyspace) - schema = Metadata() - schema.keyspaces = keyspaces + schema.keyspaces = [get_keyspace( + "my_ks_%s" % k, + True, + "SimpleStrategy", + {"replication_factor": 1}, + sasi=sasi) for k in range(2)] return schema if __name__ == "__main__": From 055ac4412907bb69190a0f4a59b0d5ba8fbc811e Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 17 Aug 2021 16:58:07 -0500 Subject: [PATCH 06/13] Some other minor cleanup --- python/adelphi/tests/util/schema_util.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/python/adelphi/tests/util/schema_util.py b/python/adelphi/tests/util/schema_util.py index f5d1216..6b2f30a 100644 --- a/python/adelphi/tests/util/schema_util.py +++ b/python/adelphi/tests/util/schema_util.py @@ -1,9 +1,10 @@ +from functools import partial + from cassandra.metadata import Metadata,\ KeyspaceMetadata,\ TableMetadata,\ ColumnMetadata,\ IndexMetadata,\ - SimpleStrategy, \ UserType # types compatible with C* 2.1+ @@ -93,12 +94,13 @@ def get_keyspace(name, durable_writes, strategy_class, strategy_options, sasi=Tr def get_schema(sasi=True): schema = Metadata() - schema.keyspaces = [get_keyspace( - "my_ks_%s" % k, - True, - "SimpleStrategy", - {"replication_factor": 1}, - sasi=sasi) for k in range(2)] + buildKs = partial( + get_keyspace, + durable_writes=True, + strategy_class="SimpleStrategy", + strategy_options={"replication_factor": 1}, + sasi=sasi) + schema.keyspaces = [buildKs("my_ks_%s" % k) for k in range(2)] return schema if __name__ == "__main__": From bbb13af56ce535548208cc139ced9838d09a6163 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 17 Aug 2021 17:13:40 -0500 Subject: [PATCH 07/13] Another minor cleanup --- python/adelphi/tests/util/schema_util.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/python/adelphi/tests/util/schema_util.py b/python/adelphi/tests/util/schema_util.py index 6b2f30a..956b151 100644 --- a/python/adelphi/tests/util/schema_util.py +++ b/python/adelphi/tests/util/schema_util.py @@ -104,10 +104,6 @@ def get_schema(sasi=True): return schema if __name__ == "__main__": - """ - Use this to print the test schema. - The output can be used in the integration tests too. - """ - # As discussed elsewhere SASI support is disabled until https://github.com/datastax/adelphi/issues/105 - # is completed - print("\n\n".join(ks.export_as_string() for ks in get_schema(sasi=False).keyspaces)) + # As discussed elsewhere SASI support is disabled until https://github.com/datastax/adelphi/issues/105 + # is completed + print("\n\n".join(ks.export_as_string() for ks in get_schema(sasi=False).keyspaces)) From dc61fabf8636712eead03438f02b1f3a83e5bcf0 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Tue, 17 Aug 2021 17:35:10 -0500 Subject: [PATCH 08/13] Upgrade C* 4.0-rc1 to 4.0.0 --- python/adelphi/bin/test-adelphi | 2 +- .../resources/cql-schemas/{4.0-rc1-ks0.cql => 4.0.0-ks0.cql} | 0 .../resources/cql-schemas/{4.0-rc1.cql => 4.0.0.cql} | 0 .../resources/nb-schemas/{4.0-rc1.yaml => 4.0.0.yaml} | 0 4 files changed, 1 insertion(+), 1 deletion(-) rename python/adelphi/tests/integration/resources/cql-schemas/{4.0-rc1-ks0.cql => 4.0.0-ks0.cql} (100%) rename python/adelphi/tests/integration/resources/cql-schemas/{4.0-rc1.cql => 4.0.0.cql} (100%) rename python/adelphi/tests/integration/resources/nb-schemas/{4.0-rc1.yaml => 4.0.0.yaml} (100%) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index 63e3891..a474237 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -20,7 +20,7 @@ import tox # Default C* versions to include in all integration tests -DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.10", "4.0-rc1"] +DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.10", "4.0.0"] TOX_DEPENDENCIES = """pytest subprocess32 ~= 3.5 diff --git a/python/adelphi/tests/integration/resources/cql-schemas/4.0-rc1-ks0.cql b/python/adelphi/tests/integration/resources/cql-schemas/4.0.0-ks0.cql similarity index 100% rename from python/adelphi/tests/integration/resources/cql-schemas/4.0-rc1-ks0.cql rename to python/adelphi/tests/integration/resources/cql-schemas/4.0.0-ks0.cql diff --git a/python/adelphi/tests/integration/resources/cql-schemas/4.0-rc1.cql b/python/adelphi/tests/integration/resources/cql-schemas/4.0.0.cql similarity index 100% rename from python/adelphi/tests/integration/resources/cql-schemas/4.0-rc1.cql rename to python/adelphi/tests/integration/resources/cql-schemas/4.0.0.cql diff --git a/python/adelphi/tests/integration/resources/nb-schemas/4.0-rc1.yaml b/python/adelphi/tests/integration/resources/nb-schemas/4.0.0.yaml similarity index 100% rename from python/adelphi/tests/integration/resources/nb-schemas/4.0-rc1.yaml rename to python/adelphi/tests/integration/resources/nb-schemas/4.0.0.yaml From a3061d165bf7c26a2fbe42d30e7d7a15745a457c Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 18 Aug 2021 14:54:59 -0500 Subject: [PATCH 09/13] Numerous fixes here: * Shift to cleanup functions rather than an explicit tearDown * Add a few validation fns to linesWithNewline * Add test-adelphi CLI flag for preserving temp dirs --- python/adelphi/adelphi/cql.py | 2 +- python/adelphi/adelphi/export.py | 5 +- python/adelphi/bin/adelphi | 2 - python/adelphi/bin/test-adelphi | 13 ++-- python/adelphi/tests/integration/__init__.py | 23 ++++--- python/adelphi/tests/integration/test_cql.py | 64 ++++++++++---------- 6 files changed, 57 insertions(+), 52 deletions(-) diff --git a/python/adelphi/adelphi/cql.py b/python/adelphi/adelphi/cql.py index f8fb2dd..d6e668d 100644 --- a/python/adelphi/adelphi/cql.py +++ b/python/adelphi/adelphi/cql.py @@ -49,5 +49,5 @@ def export_schema(self, keyspace=None): def each_keyspace(self, ks_fn): - for (ks_name, ks_tuple) in self.keyspaces.items(): + for (_, ks_tuple) in self.keyspaces.items(): ks_fn(ks_tuple.ks_obj, ks_tuple.ks_id) diff --git a/python/adelphi/adelphi/export.py b/python/adelphi/adelphi/export.py index 20e83c9..1522b89 100644 --- a/python/adelphi/adelphi/export.py +++ b/python/adelphi/adelphi/export.py @@ -15,7 +15,7 @@ import hashlib import logging from base64 import urlsafe_b64encode -from collections import namedtuple +from collections import namedtuple, OrderedDict from datetime import datetime, tzinfo, timedelta try: @@ -95,7 +95,8 @@ def make_tuple(ks): if props['anonymize']: anonymize_keyspace(ks) return KsTuple(ids[orig_name], ks) - return {t.ks_obj.name : t for t in [make_tuple(ks) for ks in keyspaces]} + tuples = sorted([make_tuple(ks) for ks in keyspaces], key=lambda ks: ks.ks_obj.name) + return OrderedDict([(t.ks_obj.name,t) for t in tuples]) def get_cluster_metadata(self, cluster): diff --git a/python/adelphi/bin/adelphi b/python/adelphi/bin/adelphi index 20f3e32..6ffa7a5 100644 --- a/python/adelphi/bin/adelphi +++ b/python/adelphi/bin/adelphi @@ -14,11 +14,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -import argparse import json import logging import os -import os.path import sys from functools import partial diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index a474237..9ce8748 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -33,13 +33,17 @@ def runCassandraContainer(client, version): return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042: 9042}, image="cassandra:{}".format(version)) -def writeToxIni(version): +def writeToxIni(version, keep_tmpdirs = False): config = configparser.ConfigParser() config["tox"] = { "envlist": "py2, py3" } envs = {"CASSANDRA_VERSION": version} + if keep_tmpdirs: + print("Preserving temporary directories") + envs["KEEP_LOGS"] = True + envStr = "\n ".join(["{} = {}".format(k,v) for (k,v) in envs.items()]) config["testenv"] = {"deps": TOX_DEPENDENCIES, \ "commands": "pytest {posargs}", \ - "setenv": "CASSANDRA_VERSION = {}".format(version)} + "setenv": envStr} with open(TOX_CONFIG, 'w') as configfile: config.write(configfile) @@ -47,7 +51,8 @@ def writeToxIni(version): @click.option('--cassandra', '-c', multiple=True, type=str) @click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False)) @click.option("--pytest", "-t", type=str, help="Arguments to be passed to pytest") -def runtests(cassandra, python, pytest): +@click.option('--keep-tmpdirs', help="Preserve temporary directories", is_flag=True) +def runtests(cassandra, python, pytest, keep_tmpdirs): client = docker.from_env() tox_args = ["-e {}".format(py) for py in python] if python else [] if pytest: @@ -68,7 +73,7 @@ def runtests(cassandra, python, pytest): try: if os.path.exists(TOX_CONFIG): os.remove(TOX_CONFIG) - writeToxIni(version) + writeToxIni(version, keep_tmpdirs) # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 14abec5..2ad9a12 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -62,6 +62,17 @@ def makeTempDirs(self): outputDir = os.path.join(base, "outputDir") os.mkdir(outputDir) self.dirs = TempDirs(base, outputDir) + self.addCleanup(self.cleanUpTempDirs) + + + def cleanUpTempDirs(self): + # TODO: Note that there's no easy way to access this from test-adelphi unless we modify the + # ini generation code... and I'm not completely sure that's worth it. Might want to think + # about just deleting this outright... or making it a CLI option that can be easily accessed. + if "KEEP_LOGS" in os.environ: + log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath)) + else: + shutil.rmtree(self.dirs.basePath) def setUp(self): @@ -73,15 +84,3 @@ def setUp(self): log.info("Testing Cassandra version {}".format(self.version)) self.makeTempDirs() - - - def tearDown(self): - super(SchemaTestCase, self).tearDown() - - # TODO: Note that there's no easy way to access this from test-adelphi unless we modify the - # ini generation code... and I'm not completely sure that's worth it. Might want to think - # about just deleting this outright... or making it a CLI option that can be easily accessed. - if "KEEP_LOGS" in os.environ: - log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath)) - else: - shutil.rmtree(self.dirs.basePath) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index 14f4415..83299d2 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -23,12 +23,18 @@ CQL_REFERENCE_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}.cql" CQL_REFERENCE_KS0_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}-ks0.cql" -def linesWithNewline(f): - rv = f.readlines() - lastLine = rv[-1] - if not lastLine.endswith("\n"): - rv[-1] = lastLine + "\n" - return rv +def linesWithNewline(fpath): + if not os.path.exists(fpath): + print("File {} does not exist") + if os.path.getsize(fpath) <= 0: + print("File {} is empty") + print("Reading lines for file {}".format(fpath)) + with open(fpath) as f: + rv = f.readlines() + lastLine = rv[-1] + if not lastLine.endswith("\n"): + rv[-1] = lastLine + "\n" + return rv class TestCql(SchemaTestCase): @@ -39,11 +45,7 @@ def setUp(self): self.origKeyspaces = getAllKeyspaces() log.info("Creating schema") setupSchema(self.buildSchema()) - - - def tearDown(self): - super(TestCql, self).tearDown() - dropNewKeyspaces(self.origKeyspaces) + self.addCleanup(dropNewKeyspaces, self.origKeyspaces) # ========================== Helper functions ========================== @@ -59,27 +61,26 @@ def buildSchema(self): def compareToReferenceCql(self, referencePath, comparePath): - with open(referencePath) as referenceFile, open(comparePath) as compareFile: - compareLines = linesWithNewline(compareFile) - referenceLines = linesWithNewline(referenceFile) + compareLines = linesWithNewline(comparePath) + referenceLines = linesWithNewline(referencePath) - diffGen = difflib.unified_diff( - compareLines, - referenceLines, - fromfile=os.path.abspath(comparePath), - tofile=os.path.abspath(referencePath)) + diffGen = difflib.unified_diff( + compareLines, + referenceLines, + fromfile=os.path.abspath(comparePath), + tofile=os.path.abspath(referencePath)) - diffEmpty = True - for line in diffGen: - if diffEmpty: - print("Diff of generated file ({}) against reference file ({})".format( - os.path.basename(comparePath), - os.path.basename(referencePath))) - diffEmpty = False - print(line.strip()) + diffEmpty = True + for line in diffGen: + if diffEmpty: + print("Diff of generated file ({}) against reference file ({})".format( + os.path.basename(comparePath), + os.path.basename(referencePath))) + diffEmpty = False + print(line.strip()) - if not diffEmpty: - self.fail() + if not diffEmpty: + self.fail() def combineSchemas(self): @@ -97,6 +98,7 @@ def combineSchemas(self): return allOutputPath + # ========================== Test functions ========================== def test_stdout(self): stdoutPath = self.stdoutPath(self.version) @@ -105,7 +107,7 @@ def test_stdout(self): self.compareToReferenceCql( CQL_REFERENCE_SCHEMA_PATH.format(self.version), - self.stdoutPath(self.version)) + stdoutPath) def test_outputdir(self): @@ -126,7 +128,7 @@ def test_some_keyspaces_stdout(self): self.compareToReferenceCql( CQL_REFERENCE_KS0_SCHEMA_PATH.format(self.version), - self.stdoutPath(self.version)) + stdoutPath) def test_some_keyspaces_outputdir(self): From d4cd8707d5d1817e8f2de58a83943c6148fc91ae Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 18 Aug 2021 16:01:19 -0500 Subject: [PATCH 10/13] When compiling per-keyspace outputDir data into a single CQL file make sure we take the keyspaces in order --- python/adelphi/tests/integration/test_cql.py | 29 ++++++++++++++------ 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py index 83299d2..7207c56 100644 --- a/python/adelphi/tests/integration/test_cql.py +++ b/python/adelphi/tests/integration/test_cql.py @@ -2,6 +2,7 @@ import glob import logging import os +import re import shutil import sys @@ -23,11 +24,13 @@ CQL_REFERENCE_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}.cql" CQL_REFERENCE_KS0_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}-ks0.cql" +KEYSPACE_LINE_REGEX = re.compile(r'\s*CREATE KEYSPACE IF NOT EXISTS (\w+) ') + def linesWithNewline(fpath): if not os.path.exists(fpath): - print("File {} does not exist") + print("File {} does not exist".format(fpath)) if os.path.getsize(fpath) <= 0: - print("File {} is empty") + print("File {} is empty".format(fpath)) print("Reading lines for file {}".format(fpath)) with open(fpath) as f: rv = f.readlines() @@ -37,6 +40,15 @@ def linesWithNewline(fpath): return rv +def extractKeyspaceName(schemaPath): + with open(schemaPath) as schemaFile: + for line in schemaFile: + matcher = KEYSPACE_LINE_REGEX.match(line) + if matcher: + return matcher.group(1) + return None + + class TestCql(SchemaTestCase): # ========================== Unittest infrastructure ========================== @@ -88,13 +100,14 @@ def combineSchemas(self): allOutputFileName = "{}-all".format(self.version) allOutputPath = self.outputDirPath(allOutputFileName) - outputSchemas = glob.glob("{}/*/schema".format(outputDirPath)) - self.assertGreater(len(outputSchemas), 0) + schemaPaths = glob.glob("{}/*/schema".format(outputDirPath)) + self.assertGreater(len(schemaPaths), 0) + schemas = { extractKeyspaceName(p) : p for p in schemaPaths} + sortedKeyspaces = sorted(schemas.keys()) + with open(allOutputPath, "w+") as allOutputFile: - for outputSchema in outputSchemas: - with open(outputSchema) as outputSchemaFile: - shutil.copyfileobj(outputSchemaFile, allOutputFile) - allOutputFile.write("\n") + cqlStr = "\n\n".join(open(schemas[ks]).read() for ks in sortedKeyspaces) + allOutputFile.write(cqlStr) return allOutputPath From 5073fed1fc279dcf4e69a46eb07746de497ce17d Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 18 Aug 2021 16:32:32 -0500 Subject: [PATCH 11/13] Impose an ordering on tables within keyspace objects as well --- python/adelphi/adelphi/store.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/python/adelphi/adelphi/store.py b/python/adelphi/adelphi/store.py index b3f2d69..20f9301 100644 --- a/python/adelphi/adelphi/store.py +++ b/python/adelphi/adelphi/store.py @@ -16,6 +16,7 @@ # Functions to facilitate interactions with the underlying data store import logging +from collections import OrderedDict from itertools import tee # Account for name change in itertools as of py3k @@ -59,6 +60,10 @@ def build_keyspace_objects(keyspaces, metadata): """Build a list of cassandra.metadata.KeyspaceMetadata objects from a list of strings and a c.m.Metadata instance. System keyspaces will be excluded.""" all_keyspace_objs = [metadata.keyspaces[ks] for ks in keyspaces] if keyspaces is not None else metadata.keyspaces.values() + # Make sure tables are ordered (by table name) + for ks_obj in all_keyspace_objs: + ks_obj.tables = OrderedDict(sorted(ks_obj.tables.items(), key=lambda item: item[0])) + # Borrowed from itertools def partition(pred, iterable): t1, t2 = tee(iterable) From be94399a763d25e306ed8528435c73370e3d5cd3 Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Wed, 18 Aug 2021 16:49:07 -0500 Subject: [PATCH 12/13] Add CLI flag to disable generation of tox.ini --- python/adelphi/bin/test-adelphi | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index 9ce8748..3b871be 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -52,7 +52,8 @@ def writeToxIni(version, keep_tmpdirs = False): @click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False)) @click.option("--pytest", "-t", type=str, help="Arguments to be passed to pytest") @click.option('--keep-tmpdirs', help="Preserve temporary directories", is_flag=True) -def runtests(cassandra, python, pytest, keep_tmpdirs): +@click.option('--no-gen-tox', help="Do not generate a tox.ini file", is_flag=True) +def runtests(cassandra, python, pytest, keep_tmpdirs, no_gen_tox): client = docker.from_env() tox_args = ["-e {}".format(py) for py in python] if python else [] if pytest: @@ -71,9 +72,11 @@ def runtests(cassandra, python, pytest, keep_tmpdirs): connectToLocalCassandra() try: - if os.path.exists(TOX_CONFIG): - os.remove(TOX_CONFIG) - writeToxIni(version, keep_tmpdirs) + if not no_gen_tox: + print("Generating tox.ini file") + if os.path.exists(TOX_CONFIG): + os.remove(TOX_CONFIG) + writeToxIni(version, keep_tmpdirs) # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things From 2e8e9c9c67f9ad944dd3a2a43638fc5c2b65f28c Mon Sep 17 00:00:00 2001 From: Bret McGuire Date: Fri, 20 Aug 2021 14:32:47 -0500 Subject: [PATCH 13/13] Revert "Add CLI flag to disable generation of tox.ini" This reverts commit be94399a763d25e306ed8528435c73370e3d5cd3. --- python/adelphi/bin/test-adelphi | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index 3b871be..9ce8748 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -52,8 +52,7 @@ def writeToxIni(version, keep_tmpdirs = False): @click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False)) @click.option("--pytest", "-t", type=str, help="Arguments to be passed to pytest") @click.option('--keep-tmpdirs', help="Preserve temporary directories", is_flag=True) -@click.option('--no-gen-tox', help="Do not generate a tox.ini file", is_flag=True) -def runtests(cassandra, python, pytest, keep_tmpdirs, no_gen_tox): +def runtests(cassandra, python, pytest, keep_tmpdirs): client = docker.from_env() tox_args = ["-e {}".format(py) for py in python] if python else [] if pytest: @@ -72,11 +71,9 @@ def runtests(cassandra, python, pytest, keep_tmpdirs, no_gen_tox): connectToLocalCassandra() try: - if not no_gen_tox: - print("Generating tox.ini file") - if os.path.exists(TOX_CONFIG): - os.remove(TOX_CONFIG) - writeToxIni(version, keep_tmpdirs) + if os.path.exists(TOX_CONFIG): + os.remove(TOX_CONFIG) + writeToxIni(version, keep_tmpdirs) # cmdline() will raise SystemExit when it's done so trap that here to avoid # exiting all the things