diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml new file mode 100644 index 0000000..b62c4f2 --- /dev/null +++ b/.github/workflows/adelphi-python.yaml @@ -0,0 +1,29 @@ +name: adelphi Python + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.x + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - name: Install dependencies + run: | + cd python + pip install ./adelphi + cd adelphi + pip install -r ./test-requirements.txt + - name: Execute tests + run: | + cd python/adelphi + test-adelphi diff --git a/python/adelphi/adelphi/anonymize.py b/python/adelphi/adelphi/anonymize.py index f2622ba..13264ed 100644 --- a/python/adelphi/adelphi/anonymize.py +++ b/python/adelphi/adelphi/anonymize.py @@ -15,7 +15,6 @@ # Functions and constants related to the anonymization process from adelphi.store import get_standard_columns_from_table_metadata -import re # default prefixes for the anonymized names KEYSPACE_PREFIX = "ks" diff --git a/python/adelphi/adelphi/gh.py b/python/adelphi/adelphi/gh.py index a6a14b7..4610053 100644 --- a/python/adelphi/adelphi/gh.py +++ b/python/adelphi/adelphi/gh.py @@ -22,7 +22,6 @@ from github import Github -logging.basicConfig(level=logging.INFO) log = logging.getLogger('adelphi') # We're assuming any storage repo will be created after the conversion to "main" diff --git a/python/adelphi/adelphi/store.py b/python/adelphi/adelphi/store.py index b3f2d69..4b250e7 100644 --- a/python/adelphi/adelphi/store.py +++ b/python/adelphi/adelphi/store.py @@ -27,7 +27,8 @@ from cassandra.cluster import Cluster, ExecutionProfile, EXEC_PROFILE_DEFAULT, default_lbp_factory from cassandra.auth import PlainTextAuthProvider -logging.basicConfig(level=logging.INFO) +from tenacity import retry + log = logging.getLogger('adelphi') system_keyspaces = set(["system", @@ -39,20 +40,26 @@ "system_views"]) def build_auth_provider(username = None,password = None): - # instantiate auth provider if credentials have been provided - auth_provider = None - if username is not None and password is not None: - auth_provider = PlainTextAuthProvider(username=username, password=password) - return auth_provider + """Instantiate auth provider if credentials have been provided""" + if username is None or password is None: + return None + return PlainTextAuthProvider(username=username, password=password) +@retry def with_cluster(cluster_fn, hosts, port, username = None, password = None): ep = ExecutionProfile(load_balancing_policy=default_lbp_factory()) cluster = Cluster(hosts, port=port, auth_provider=build_auth_provider(username,password), execution_profiles={EXEC_PROFILE_DEFAULT: ep}) - cluster.connect() - rv = cluster_fn(cluster) - cluster.shutdown() - return rv + try: + cluster.connect() + return cluster_fn(cluster) + finally: + cluster.shutdown() + + +@retry +def with_local_cluster(cluster_fn): + return with_cluster(cluster_fn, ["127.0.0.1"], port=9042) def build_keyspace_objects(keyspaces, metadata): @@ -71,9 +78,7 @@ def partition(pred, iterable): def get_standard_columns_from_table_metadata(table_metadata): - """ - Return the standard columns and ensure to exclude pk and ck ones. - """ + """Return the standard columns and ensure to exclude pk and ck ones""" partition_column_names = [c.name for c in table_metadata.partition_key] clustering_column_names = [c.name for c in table_metadata.clustering_key] standard_columns = [] @@ -96,3 +101,28 @@ def set_replication_factor(selected_keyspaces, factor): log.debug("Replication for keyspace " + ks.name+ ": " + str(ks.replication_strategy)) strategy = ks.replication_strategy strategy.replication_factor_info = factor + + +def create_schema(session, schemaPath): + """Read schema CQL document and apply CQL commands to cluster""" + log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) + with open(schemaPath) as schema: + buff = "" + for line in schema: + realLine = line.strip() + if len(realLine) == 0: + log.debug("Skipping empty statement") + continue + if realLine.startswith("//") or realLine.startswith("--"): + log.debug("Skipping commented statement") + continue + buff += (" " if len(buff) > 0 else "") + buff += realLine + if buff.endswith(';'): + log.debug("Executing statement {}".format(buff)) + try: + session.execute(buff) + except Exception as exc: + log.error("Exception executing statement: {}".format(buff), exc_info=exc) + finally: + buff = "" diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi index 63e3891..82d02e3 100644 --- a/python/adelphi/bin/test-adelphi +++ b/python/adelphi/bin/test-adelphi @@ -10,8 +10,10 @@ # suite, which in turn should allow us to write simpler tests. import configparser import os +import re +import sys -from tests.util.cassandra_util import connectToLocalCassandra +from adelphi.store import with_local_cluster import click import docker @@ -36,13 +38,40 @@ def runCassandraContainer(client, version): def writeToxIni(version): config = configparser.ConfigParser() config["tox"] = { "envlist": "py2, py3" } - envs = {"CASSANDRA_VERSION": version} config["testenv"] = {"deps": TOX_DEPENDENCIES, \ "commands": "pytest {posargs}", \ "setenv": "CASSANDRA_VERSION = {}".format(version)} with open(TOX_CONFIG, 'w') as configfile: config.write(configfile) + +def buildVersionMap(): + assert sorted(DEFAULT_CASSANDRA_VERSIONS) + rv = {v:v for v in DEFAULT_CASSANDRA_VERSIONS} + majorMinorPattern = re.compile(r"(\d\.\d).*") + for v in DEFAULT_CASSANDRA_VERSIONS: + majorMinorMatch = majorMinorPattern.match(v) + if majorMinorMatch: + majorMinor = majorMinorMatch.group(1) + rv[majorMinor] = v + # The reason we needed to asset that our input was sorted; we want the last + # major version entry we discover in the list to map to the major version. + # So "2" => "2.2.whatever" rather than "2.1.whatever" + rv[majorMinor.split('.')[0]] = v + return rv + + +def resolveCassandraVersions(cassandra_versions): + if not cassandra_versions: + return DEFAULT_CASSANDRA_VERSIONS + versionMap = buildVersionMap() + computedVersions = [x for x in [versionMap.get(v) for v in cassandra_versions] if x is not None] + if not computedVersions: + print("Could not compute valid Cassandra versions based on args, using defaults") + return DEFAULT_CASSANDRA_VERSIONS + return computedVersions + + @click.command() @click.option('--cassandra', '-c', multiple=True, type=str) @click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False)) @@ -55,15 +84,21 @@ def runtests(cassandra, python, pytest): tox_args.append(pytest) print("Full tox args: {}".format(tox_args)) - cassandra_versions = cassandra or DEFAULT_CASSANDRA_VERSIONS + cassandra_versions = resolveCassandraVersions(cassandra) print("Cassandra versions to test: {}".format(','.join(cassandra_versions))) - for version in cassandra_versions: + exitCodes = [] + for version in resolveCassandraVersions(cassandra_versions): print("Running test suite for Cassandra version {}".format(version)) container = runCassandraContainer(client, version) print("Validating connection to local Cassandra") - connectToLocalCassandra() + def validationFn(cluster): + session = cluster.connect() + rs = session.execute("select * from system.local") + print("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) + return (cluster, session) + with_local_cluster.retry_with(stop=stop_after_attempt(5), wait=wait_fixed(3))(validationFn) try: if os.path.exists(TOX_CONFIG): @@ -74,13 +109,13 @@ def runtests(cassandra, python, pytest): # exiting all the things try: tox.cmdline(tox_args) - except SystemExit: - pass + except SystemExit as exc: + exitCodes.append(exc.code) except Exception as exc: print("Exception running tests for Cassandra version {}".format(version), exc) finally: container.stop() - + sys.exit(sum(1 for x in exitCodes if x != 0)) if __name__ == '__main__': runtests(obj={}) diff --git a/python/adelphi/setup.py b/python/adelphi/setup.py index 79bc65c..9e30846 100644 --- a/python/adelphi/setup.py +++ b/python/adelphi/setup.py @@ -8,7 +8,8 @@ 'cassandra-driver ~= 3.24', 'click ~= 7.1', 'PyGithub ~= 1.45', - 'PyYAML ~= 5.4' + 'PyYAML ~= 5.4', + 'tenacity ~= 7.0' ] if not PY3: @@ -48,6 +49,6 @@ 'Topic :: Software Development :: Libraries :: Python Modules' ], packages=['adelphi'], - scripts=['bin/adelphi'], + scripts=['bin/adelphi','bin/test-adelphi'], install_requires=dependencies, ) diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt index 087a0ae..9c59653 100644 --- a/python/adelphi/test-requirements.txt +++ b/python/adelphi/test-requirements.txt @@ -1,5 +1,2 @@ -click ~= 7.1 -cassandra-driver ~= 3.24 docker ~= 4.4 -tenacity ~= 7.0 tox ~= 3.22 diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py index 14abec5..87da0e2 100644 --- a/python/adelphi/tests/integration/__init__.py +++ b/python/adelphi/tests/integration/__init__.py @@ -10,7 +10,7 @@ from collections import namedtuple -from tests.util.cassandra_util import callWithCassandra, createSchema +from adelphi.store import with_local_cluster, create_schema log = logging.getLogger('adelphi') @@ -22,21 +22,24 @@ def __keyspacesForCluster(cluster): def setupSchema(schemaPath): - return callWithCassandra(lambda _,s: createSchema(s, schemaPath)) + def schemaFn(cluster): + return create_schema(cluster.connect(), schemaPath) + return with_local_cluster(schemaFn) def getAllKeyspaces(): - return callWithCassandra(lambda c,s: __keyspacesForCluster(c)) + return with_local_cluster(__keyspacesForCluster) def dropNewKeyspaces(origKeyspaces): - def dropFn(cluster, session): + def dropFn(cluster): currentKeyspaces = __keyspacesForCluster(cluster) droppingKeyspaces = currentKeyspaces - origKeyspaces log.info("Dropping the following keyspaes created by this test: {}".format(",".join(droppingKeyspaces))) + session = cluster.connect() for keyspace in droppingKeyspaces: session.execute("drop keyspace {}".format(keyspace)) - return callWithCassandra(dropFn) + return with_local_cluster(dropFn) class SchemaTestCase(unittest.TestCase): diff --git a/python/adelphi/tests/integration/test_nb.py b/python/adelphi/tests/integration/test_nb.py index 66edab4..7c711f5 100644 --- a/python/adelphi/tests/integration/test_nb.py +++ b/python/adelphi/tests/integration/test_nb.py @@ -49,7 +49,6 @@ def compareToReferenceYaml(self, comparePath, version=None): # ========================== Test functions ========================== def test_stdout(self): - print("All keyspaces: {}".format(getAllKeyspaces())) stdoutPath = self.stdoutPath(self.version) stderrPath = self.stderrPath(self.version) subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True) diff --git a/python/adelphi/tests/util/cassandra_util.py b/python/adelphi/tests/util/cassandra_util.py deleted file mode 100644 index 02a40b2..0000000 --- a/python/adelphi/tests/util/cassandra_util.py +++ /dev/null @@ -1,52 +0,0 @@ -# A few utility methods for interacting with Cassandra from within tests -import logging -import time - -from cassandra.cluster import Cluster - -from tenacity import retry, wait_fixed - -log = logging.getLogger('adelphi') - -@retry(wait=wait_fixed(3)) -def connectToLocalCassandra(): - cluster = Cluster(["127.0.0.1"], port=9042) - session = cluster.connect() - - # Confirm that the session is actually functioning before calling things good - rs = session.execute("select * from system.local") - log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one())) - return (cluster, session) - - -def createSchema(session, schemaPath): - log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath)) - with open(schemaPath) as schema: - buff = "" - for line in schema: - realLine = line.strip() - if len(realLine) == 0: - log.debug("Skipping empty statement") - continue - if realLine.startswith("//") or realLine.startswith("--"): - log.debug("Skipping commented statement") - continue - buff += (" " if len(buff) > 0 else "") - buff += realLine - if buff.endswith(';'): - log.debug("Executing statement {}".format(buff)) - try: - session.execute(buff) - except Exception as exc: - log.error("Exception executing statement: {}".format(buff), exc_info=exc) - finally: - buff = "" - - -def callWithCassandra(someFn): - cluster = None - try: - (cluster,session) = connectToLocalCassandra() - return someFn(cluster, session) - finally: - cluster.shutdown()