diff --git a/.github/workflows/adelphi-python.yaml b/.github/workflows/adelphi-python.yaml
new file mode 100644
index 0000000..5e11473
--- /dev/null
+++ b/.github/workflows/adelphi-python.yaml
@@ -0,0 +1,29 @@
+name: adelphi Python
+
+on:
+  push:
+    branches: [ 115-breadth-first-testing ]
+  pull_request:
+    branches: [ 115-breadth-first-testing ]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.x
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.x'
+    - name: Install dependencies
+      run: |
+        cd python
+        pip install ./adelphi
+        cd adelphi
+        pip install -r ./test-requirements.txt
+    - name: Execute tests
+      run: |
+        cd python/adelphi
+        test-adelphi
diff --git a/.gitignore b/.gitignore
index aeae72d..70324f0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,6 +56,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+python/adelphi/tox.ini
 
 # Translations
 *.mo
diff --git a/python/adelphi/adelphi/anonymize.py b/python/adelphi/adelphi/anonymize.py
index f2622ba..13264ed 100644
--- a/python/adelphi/adelphi/anonymize.py
+++ b/python/adelphi/adelphi/anonymize.py
@@ -15,7 +15,6 @@
 
 # Functions and constants related to the anonymization process
 from adelphi.store import get_standard_columns_from_table_metadata
-import re
 
 # default prefixes for the anonymized names
 KEYSPACE_PREFIX = "ks"
diff --git a/python/adelphi/adelphi/gh.py b/python/adelphi/adelphi/gh.py
index a6a14b7..4610053 100644
--- a/python/adelphi/adelphi/gh.py
+++ b/python/adelphi/adelphi/gh.py
@@ -22,7 +22,6 @@
 
 from github import Github
 
-logging.basicConfig(level=logging.INFO)
 log = logging.getLogger('adelphi')
 
 # We're assuming any storage repo will be created after the conversion to "main"
diff --git a/python/adelphi/adelphi/store.py b/python/adelphi/adelphi/store.py
index b3f2d69..4b250e7 100644
--- a/python/adelphi/adelphi/store.py
+++ b/python/adelphi/adelphi/store.py
@@ -27,7 +27,8 @@
 from cassandra.cluster import Cluster, ExecutionProfile, EXEC_PROFILE_DEFAULT, default_lbp_factory
 from cassandra.auth import PlainTextAuthProvider
 
-logging.basicConfig(level=logging.INFO)
+from tenacity import retry
+
 log = logging.getLogger('adelphi')
 
 system_keyspaces = set(["system",
@@ -39,20 +40,26 @@
                     "system_views"])
 
 def build_auth_provider(username = None,password = None):
-    # instantiate auth provider if credentials have been provided
-    auth_provider = None
-    if username is not None and password is not None:
-        auth_provider = PlainTextAuthProvider(username=username, password=password)
-    return auth_provider
+    """Instantiate auth provider if credentials have been provided"""
+    if username is None or password is None:
+        return None
+    return PlainTextAuthProvider(username=username, password=password)
 
 
+@retry
 def with_cluster(cluster_fn, hosts, port, username = None, password = None):
     ep = ExecutionProfile(load_balancing_policy=default_lbp_factory())
     cluster = Cluster(hosts, port=port, auth_provider=build_auth_provider(username,password), execution_profiles={EXEC_PROFILE_DEFAULT: ep})
-    cluster.connect()
-    rv = cluster_fn(cluster)
-    cluster.shutdown()
-    return rv
+    try:
+        cluster.connect()
+        return cluster_fn(cluster)
+    finally:
+        cluster.shutdown()
+
+
+@retry
+def with_local_cluster(cluster_fn):
+    return with_cluster(cluster_fn, ["127.0.0.1"], port=9042)
 
 
 def build_keyspace_objects(keyspaces, metadata):
@@ -71,9 +78,7 @@ def partition(pred, iterable):
 
 
 def get_standard_columns_from_table_metadata(table_metadata):
-    """
-    Return the standard columns and ensure to exclude pk and ck ones.
-    """
+    """Return the standard columns and ensure to exclude pk and ck ones"""
     partition_column_names = [c.name for c in table_metadata.partition_key]
     clustering_column_names = [c.name for c in table_metadata.clustering_key]
     standard_columns = []
@@ -96,3 +101,28 @@ def set_replication_factor(selected_keyspaces, factor):
             log.debug("Replication for keyspace " + ks.name+ ": " + str(ks.replication_strategy))
             strategy = ks.replication_strategy
             strategy.replication_factor_info = factor
+
+
+def create_schema(session, schemaPath):
+    """Read schema CQL document and apply CQL commands to cluster"""
+    log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath))
+    with open(schemaPath) as schema:
+        buff = ""
+        for line in schema:
+            realLine = line.strip()
+            if len(realLine) == 0:
+                log.debug("Skipping empty statement")
+                continue
+            if realLine.startswith("//") or realLine.startswith("--"):
+                log.debug("Skipping commented statement")
+                continue
+            buff += (" " if len(buff) > 0 else "")
+            buff += realLine
+            if buff.endswith(';'):
+                log.debug("Executing statement {}".format(buff))
+                try:
+                    session.execute(buff)
+                except Exception as exc:
+                    log.error("Exception executing statement: {}".format(buff), exc_info=exc)
+                finally:
+                    buff = ""
diff --git a/python/adelphi/bin/test-adelphi b/python/adelphi/bin/test-adelphi
new file mode 100644
index 0000000..82d02e3
--- /dev/null
+++ b/python/adelphi/bin/test-adelphi
@@ -0,0 +1,121 @@
+#!python
+
+# Run test suite for some set of Cassandra versions.
+#
+# We implement this as a front-end script because the tox/pytest/unittest chain
+# isn't great about iterating over a suite of test fixtures and re-running the
+# collected tests for each of them.  Rather than fight with the frameworks we
+# manage the fixture iteration manually in this script.  This also has the
+# nice side effect of moving a lot of C* checking/session code out of the test
+# suite, which in turn should allow us to write simpler tests.
+import configparser
+import os
+import re
+import sys
+
+from adelphi.store import with_local_cluster
+
+import click
+import docker
+from tenacity import retry, stop_after_attempt, wait_fixed
+import tox
+
+
+# Default C* versions to include in all integration tests
+DEFAULT_CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.10", "4.0-rc1"]
+
+TOX_DEPENDENCIES = """pytest
+    subprocess32 ~= 3.5
+    tenacity ~= 7.0"""
+TOX_CONFIG = "tox.ini"
+
+
+@retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
+def runCassandraContainer(client, version):
+    return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042: 9042}, image="cassandra:{}".format(version))
+
+
+def writeToxIni(version):
+    config = configparser.ConfigParser()
+    config["tox"] = { "envlist": "py2, py3" }
+    config["testenv"] = {"deps": TOX_DEPENDENCIES, \
+        "commands": "pytest {posargs}", \
+        "setenv": "CASSANDRA_VERSION = {}".format(version)}
+    with open(TOX_CONFIG, 'w') as configfile:
+        config.write(configfile)
+
+
+def buildVersionMap():
+    assert sorted(DEFAULT_CASSANDRA_VERSIONS)
+    rv = {v:v for v in DEFAULT_CASSANDRA_VERSIONS}
+    majorMinorPattern = re.compile(r"(\d\.\d).*")
+    for v in DEFAULT_CASSANDRA_VERSIONS:
+        majorMinorMatch = majorMinorPattern.match(v)
+        if majorMinorMatch:
+            majorMinor = majorMinorMatch.group(1)
+            rv[majorMinor] = v
+            # The reason we needed to asset that our input was sorted; we want the last
+            # major version entry we discover in the list to map to the major version.
+            # So "2" => "2.2.whatever" rather than "2.1.whatever"
+            rv[majorMinor.split('.')[0]] = v
+    return rv
+
+
+def resolveCassandraVersions(cassandra_versions):
+    if not cassandra_versions:
+        return DEFAULT_CASSANDRA_VERSIONS
+    versionMap = buildVersionMap()
+    computedVersions = [x for x in [versionMap.get(v) for v in cassandra_versions] if x is not None]
+    if not computedVersions:
+        print("Could not compute valid Cassandra versions based on args, using defaults")
+        return DEFAULT_CASSANDRA_VERSIONS
+    return computedVersions
+
+
+@click.command()
+@click.option('--cassandra', '-c', multiple=True, type=str)
+@click.option('--python', '-p', multiple=True, type=click.Choice(["py2","py3"], case_sensitive = False))
+@click.option("--pytest", "-t", type=str, help="Arguments to be passed to pytest")
+def runtests(cassandra, python, pytest):
+    client = docker.from_env()
+    tox_args = ["-e {}".format(py) for py in python] if python else []
+    if pytest:
+        tox_args.append("--")
+        tox_args.append(pytest)
+    print("Full tox args: {}".format(tox_args))
+
+    cassandra_versions = resolveCassandraVersions(cassandra)
+    print("Cassandra versions to test: {}".format(','.join(cassandra_versions)))
+    exitCodes = []
+    for version in resolveCassandraVersions(cassandra_versions):
+
+        print("Running test suite for Cassandra version {}".format(version))
+        container = runCassandraContainer(client, version)
+
+        print("Validating connection to local Cassandra")
+        def validationFn(cluster):
+            session = cluster.connect()
+            rs = session.execute("select * from system.local")
+            print("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one()))
+            return (cluster, session)
+        with_local_cluster.retry_with(stop=stop_after_attempt(5), wait=wait_fixed(3))(validationFn)
+
+        try:
+            if os.path.exists(TOX_CONFIG):
+                os.remove(TOX_CONFIG)
+            writeToxIni(version)
+
+            # cmdline() will raise SystemExit when it's done so trap that here to avoid
+            # exiting all the things
+            try:
+                tox.cmdline(tox_args)
+            except SystemExit as exc:
+                exitCodes.append(exc.code)
+        except Exception as exc:
+            print("Exception running tests for Cassandra version {}".format(version), exc)
+        finally:
+            container.stop()
+    sys.exit(sum(1 for x in exitCodes if x != 0))
+
+if __name__ == '__main__':
+    runtests(obj={})
diff --git a/python/adelphi/setup.py b/python/adelphi/setup.py
index 79bc65c..9e30846 100644
--- a/python/adelphi/setup.py
+++ b/python/adelphi/setup.py
@@ -8,7 +8,8 @@
     'cassandra-driver ~= 3.24',
     'click ~= 7.1',
     'PyGithub ~= 1.45',
-    'PyYAML ~= 5.4'
+    'PyYAML ~= 5.4',
+    'tenacity ~= 7.0'
     ]
 
 if not PY3:
@@ -48,6 +49,6 @@
         'Topic :: Software Development :: Libraries :: Python Modules'
    ],
    packages=['adelphi'],
-   scripts=['bin/adelphi'],
+   scripts=['bin/adelphi','bin/test-adelphi'],
    install_requires=dependencies,
 )
diff --git a/python/adelphi/test-requirements.txt b/python/adelphi/test-requirements.txt
new file mode 100644
index 0000000..9c59653
--- /dev/null
+++ b/python/adelphi/test-requirements.txt
@@ -0,0 +1,2 @@
+docker ~= 4.4
+tox ~= 3.22
diff --git a/python/adelphi/tests/integration/__init__.py b/python/adelphi/tests/integration/__init__.py
index 2357b7a..87da0e2 100644
--- a/python/adelphi/tests/integration/__init__.py
+++ b/python/adelphi/tests/integration/__init__.py
@@ -1,26 +1,48 @@
 import logging
 import os
 import shutil
-import sys
 import tempfile
-import time
 
-from collections import namedtuple
-
-from cassandra.cluster import Cluster
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
 
-import docker
-from tenacity import retry, stop_after_attempt, wait_fixed
+from collections import namedtuple
 
-# Default C* versions to include in all integration tests
-CASSANDRA_VERSIONS = ["2.1.22", "2.2.19", "3.0.23", "3.11.9", "4.0-beta4"]
+from adelphi.store import with_local_cluster, create_schema
 
-logging.basicConfig(filename="adelphi.log", level=logging.INFO)
 log = logging.getLogger('adelphi')
 
 TempDirs = namedtuple('TempDirs', 'basePath, outputDirPath')
 
-class SchemaTestMixin:
+
+def __keyspacesForCluster(cluster):
+    return set(cluster.metadata.keyspaces.keys())
+
+
+def setupSchema(schemaPath):
+    def schemaFn(cluster):
+        return create_schema(cluster.connect(), schemaPath)
+    return with_local_cluster(schemaFn)
+
+
+def getAllKeyspaces():
+    return with_local_cluster(__keyspacesForCluster)
+
+
+def dropNewKeyspaces(origKeyspaces):
+    def dropFn(cluster):
+        currentKeyspaces = __keyspacesForCluster(cluster)
+        droppingKeyspaces = currentKeyspaces - origKeyspaces
+        log.info("Dropping the following keyspaes created by this test: {}".format(",".join(droppingKeyspaces)))
+        session = cluster.connect()
+        for keyspace in droppingKeyspaces:
+            session.execute("drop keyspace {}".format(keyspace))
+    return with_local_cluster(dropFn)
+
+
+class SchemaTestCase(unittest.TestCase):
 
     def basePath(self, name):
         return os.path.join(self.dirs.basePath, name)
@@ -45,96 +67,23 @@ def makeTempDirs(self):
         self.dirs = TempDirs(base, outputDir)
 
 
-    def connectToLocalCassandra(self):
-        session = None
-        while not session:
-            try:
-                cluster = Cluster(["127.0.0.1"],port=9042)
-                session = cluster.connect()
-
-                # Confirm that the session is actually functioning before calling things good
-                rs = session.execute("select * from system.local")
-                log.info("Connected to Cassandra cluster, first row of system.local: {}".format(rs.one()))
-                log.info("Cassandra cluster ready")
-                return (cluster,session)
-            except:
-                log.info("Couldn't quite connect yet, will retry")
-                time.sleep(1)
-
-
-    def createSchema(self, session=None):
-        schemaPath = self.getBaseSchemaPath()
-        log.info("Creating schema on Cassandra cluster from file {}".format(schemaPath))
-        with open(schemaPath) as schema:
-            buff = ""
-            for line in schema:
-                realLine = line.strip()
-                if len(realLine) == 0:
-                    log.debug("Skipping empty statement")
-                    continue
-                if realLine.startswith("//"):
-                    log.debug("Skipping commented statement {}".format(stmt))
-                    continue
-                buff += (" " if len(buff) > 0 else "")
-                buff += realLine
-                if realLine.endswith(';'):
-                    log.debug("Executing statement {}".format(buff))
-                    try:
-                        session.execute(buff)
-                    except:
-                        log.error("Exception executing statement: {}".format(buff), exc_info=sys.exc_info()[0])
-                        self.fail("Exception executing statement: {}, check log for details".format(buff))
-                    buff = ""
-
-
-    @retry(stop=stop_after_attempt(3), wait=wait_fixed(2))
-    def getContainer(self, client, version):
-        return client.containers.run(name="adelphi", remove=True, detach=True, ports={9042:9042}, image="cassandra:{}".format(version))
-
-
-    def runTestForVersion(self, version=None):
-        log.info("Testing Cassandra version {}".format(version))
-
-        client = docker.from_env()
-        container = self.getContainer(client, version)
-
-        self.makeTempDirs()
-
-        (cluster,session) = (None,None)
-        try:
-            (cluster,session) = self.connectToLocalCassandra()
-            self.createSchema(session)
-            log.info("Running Adelphi")
-            self.runAdelphi(version)
-            log.info("Adelphi run completed, evaluating Adelphi output(s)")
-            self.evalAdelphiOutput(version)
-        except:
-            log.error("Exception running test for version {}".format(version), exc_info=sys.exc_info()[0])
-            self.fail("Exception running test for version {}, check log for details".format(version))
-        finally:
-            if cluster:
-                cluster.shutdown()
+    def setUp(self):
+        # Invoking for completeness; for unittest base setUp/tearDown impls are no-ops
+        super(SchemaTestCase, self).setUp()
 
-            if "KEEP_CONTAINER" in os.environ:
-                log.info("KEEP_CONTAINER env var set, preserving Cassandra container 'adelphi'")
-            else:
-                container.stop()
+        # This should be set in the tox config
+        self.version = os.environ["CASSANDRA_VERSION"]
+        log.info("Testing Cassandra version {}".format(self.version))
 
-            self.cleanUpVersion(version)
-
-
-    def testVersions(self):
-        versions = CASSANDRA_VERSIONS
-        if "CASSANDRA_VERSIONS" in os.environ:
-            versions = [s.strip() for s in os.environ["CASSANDRA_VERSIONS"].split(',')]
-
-        log.info("Testing the following Cassandra versions: {}".format(versions))
+        self.makeTempDirs()
 
-        for version in versions:
-            self.runTestForVersion(version)
 
+    def tearDown(self):
+        super(SchemaTestCase, self).tearDown()
 
-    def cleanUpVersion(self, version):
+        # TODO: Note that there's no easy way to access this from test-adelphi unless we modify the
+        # ini generation code... and I'm not completely sure that's worth it.  Might want to think
+        # about just deleting this outright... or making it a CLI option that can be easily accessed.
         if "KEEP_LOGS" in os.environ:
             log.info("KEEP_LOGS env var set, preserving logs/output at {}".format(self.dirs.basePath))
         else:
diff --git a/python/adelphi/tests/integration/cql/__init__.py b/python/adelphi/tests/integration/cql/__init__.py
deleted file mode 100644
index 4b7ce1a..0000000
--- a/python/adelphi/tests/integration/cql/__init__.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import logging
-
-from tests.util.schemadiff import cqlDigestGenerator
-from tests.util.schema_util import get_schema
-
-def digestSet(schemaFile):
-    rv = set()
-    for (_, digest) in cqlDigestGenerator(schemaFile):
-        rv.add(digest)
-    return rv
-
-
-def logCqlDigest(schemaFile, digestSet):
-    for (cql, digest) in cqlDigestGenerator(schemaFile):
-        if digest in digestSet:
-            log.info("Digest: {}, CQL: {}".format(digest,cql))
-
-log = logging.getLogger('adelphi')
-
-class ExportCqlMixin:
-
-    # TODO: Note that we currently have to disable support for SASI indexes when creating
-    # a schema since the base config for the Cassandra Docker images doesn't enable it.
-    # https://github.com/datastax/adelphi/issues/105 aims to fix this problem but until
-    # that's in place we simply exclude SASI indexes from testing.
-    def getBaseSchemaPath(self):
-        baseSchemaPath = self.basePath("base-schema.cql")
-        with open(baseSchemaPath, "w") as f:
-            f.write("\n\n".join(ks.export_as_string() for ks in get_schema(sasi=False).keyspaces))
-        return baseSchemaPath
-
-
-    def compareToReferenceCql(self, referencePath, comparePath):
-        referenceSet = digestSet(referencePath)
-        compareSet = digestSet(comparePath)
-
-        refOnlySet = referenceSet - compareSet
-        if len(refOnlySet) > 0:
-            log.info("Statements in reference file {} but not in compare file {}:".format(referencePath, comparePath))
-            logCqlDigest(referencePath, refOnlySet)
-        compareOnlySet = compareSet - referenceSet
-        if len(compareOnlySet) > 0:
-            log.info("Statements in compare file {} but not in reference file {}:".format(comparePath, referencePath))
-            logCqlDigest(comparePath, compareOnlySet)
-
-        self.assertEqual(referenceSet, compareSet)
diff --git a/python/adelphi/tests/integration/cql/test_cql_export_outputdir.py b/python/adelphi/tests/integration/cql/test_cql_export_outputdir.py
deleted file mode 100644
index cbdcb44..0000000
--- a/python/adelphi/tests/integration/cql/test_cql_export_outputdir.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import glob
-import logging
-import os
-import shutil
-import sys
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-if os.name == 'posix' and sys.version_info[0] < 3:
-    import subprocess32 as subprocess
-else:
-    import subprocess
-
-from tests.integration import SchemaTestMixin
-from tests.integration.cql import ExportCqlMixin
-
-log = logging.getLogger('adelphi')
-
-class TestCqlExportOutputDir(unittest.TestCase, SchemaTestMixin, ExportCqlMixin):
-
-    def setUp(self):
-        super(TestCqlExportOutputDir, self).setUp()
-
-
-    def runAdelphi(self, version):
-        stderrPath = self.stderrPath(version)
-        outputDirPath = self.outputDirPath(version)
-        os.mkdir(outputDirPath)
-        subprocess.run("adelphi --output-dir={} export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True)
-
-
-    def evalAdelphiOutput(self, version):
-        referencePath = "tests/integration/resources/cql-schemas/{}.cql".format(version)
-
-        # Basic idea here is to find all schemas written to the output dir and aggregate them into a single schema
-        # file.  We then compare this aggregated file to the reference schema.  Ordering is important here but
-        # the current keyspace names hash to something that causes individual keyspaces to be discovered in the
-        # correct order.
-        outputDirPath = self.outputDirPath(version)
-        allOutputFileName = "{}-all".format(version)
-        allOutputPath = self.outputDirPath(allOutputFileName)
-
-        outputSchemas = glob.glob("{}/*/schema".format(outputDirPath))
-        self.assertGreater(len(outputSchemas), 0)
-        with open(allOutputPath, "w+") as allOutputFile:
-            for outputSchema in outputSchemas:
-                with open(outputSchema) as outputSchemaFile:
-                    shutil.copyfileobj(outputSchemaFile, allOutputFile)
-                    allOutputFile.write("\n")
-        self.compareToReferenceCql(referencePath, allOutputPath)
diff --git a/python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py b/python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py
deleted file mode 100644
index e9b79da..0000000
--- a/python/adelphi/tests/integration/cql/test_cql_export_outputdir_some_keyspaces.py
+++ /dev/null
@@ -1,44 +0,0 @@
-import glob
-import logging
-import os
-import shutil
-import sys
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-if os.name == 'posix' and sys.version_info[0] < 3:
-    import subprocess32 as subprocess
-else:
-    import subprocess
-
-from tests.integration import SchemaTestMixin
-from tests.integration.cql import ExportCqlMixin
-
-log = logging.getLogger('adelphi')
-
-# Implemented for https://github.com/datastax/adelphi/issues/106.
-#
-# We should see exactly one keyspace schema written to the output dir and that file
-# should match up exactly to the requested keyspace
-class TestCqlExportOutputDirSomeKeyspaces(unittest.TestCase, SchemaTestMixin, ExportCqlMixin):
-
-    def setUp(self):
-        super(TestCqlExportOutputDirSomeKeyspaces, self).setUp()
-
-
-    def runAdelphi(self, version):
-        stderrPath = self.stderrPath(version)
-        outputDirPath = self.outputDirPath(version)
-        os.mkdir(outputDirPath)
-        subprocess.run("adelphi --output-dir={} --keyspaces=my_ks_0 export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True)
-
-
-    def evalAdelphiOutput(self, version):
-        referencePath = "tests/integration/resources/cql-schemas/{}-ks0.cql".format(version)
-        outputDirPath = self.outputDirPath(version)
-        outputSchemas = glob.glob("{}/*/schema".format(outputDirPath))
-        self.assertEqual(len(outputSchemas), 1)
-        self.compareToReferenceCql(referencePath, outputSchemas[0])
diff --git a/python/adelphi/tests/integration/cql/test_cql_export_stdout.py b/python/adelphi/tests/integration/cql/test_cql_export_stdout.py
deleted file mode 100644
index 9bf5ee5..0000000
--- a/python/adelphi/tests/integration/cql/test_cql_export_stdout.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import logging
-import os
-import sys
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-if os.name == 'posix' and sys.version_info[0] < 3:
-    import subprocess32 as subprocess
-else:
-    import subprocess
-
-from tests.integration import SchemaTestMixin
-from tests.integration.cql import ExportCqlMixin
-
-log = logging.getLogger('adelphi')
-
-class TestCqlExportStdout(unittest.TestCase, SchemaTestMixin, ExportCqlMixin):
-
-    def setUp(self):
-        super(TestCqlExportStdout, self).setUp()
-
-
-    def runAdelphi(self, version):
-        stdoutPath = self.stdoutPath(version)
-        stderrPath = self.stderrPath(version)
-        subprocess.run("adelphi export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True)
-
-
-    def evalAdelphiOutput(self, version):
-        referencePath = "tests/integration/resources/cql-schemas/{}.cql".format(version)
-        self.compareToReferenceCql(referencePath, self.stdoutPath(version))
-
-
diff --git a/python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py b/python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py
deleted file mode 100644
index 2448b66..0000000
--- a/python/adelphi/tests/integration/cql/test_cql_export_stdout_some_keyspaces.py
+++ /dev/null
@@ -1,36 +0,0 @@
-import logging
-import os
-import sys
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-if os.name == 'posix' and sys.version_info[0] < 3:
-    import subprocess32 as subprocess
-else:
-    import subprocess
-
-from tests.integration import SchemaTestMixin
-from tests.integration.cql import ExportCqlMixin
-
-log = logging.getLogger('adelphi')
-
-class TestCqlExportStdoutSomeKeyspaces(unittest.TestCase, SchemaTestMixin, ExportCqlMixin):
-
-    def setUp(self):
-        super(TestCqlExportStdoutSomeKeyspaces, self).setUp()
-
-
-    def runAdelphi(self, version):
-        stdoutPath = self.stdoutPath(version)
-        stderrPath = self.stderrPath(version)
-        subprocess.run("adelphi --keyspaces=my_ks_0 export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True)
-
-
-    def evalAdelphiOutput(self, version):
-        referencePath = "tests/integration/resources/cql-schemas/{}-ks0.cql".format(version)
-        self.compareToReferenceCql(referencePath, self.stdoutPath(version))
-
-
diff --git a/python/adelphi/tests/integration/nb/__init__.py b/python/adelphi/tests/integration/nb/__init__.py
deleted file mode 100644
index 0fffb1a..0000000
--- a/python/adelphi/tests/integration/nb/__init__.py
+++ /dev/null
@@ -1,10 +0,0 @@
-import yaml
-
-class ExportNbMixin:
-
-    def compareToReferenceYaml(self, comparePath, version=None):
-        referencePath = "tests/integration/resources/nb-schemas/{}.yaml".format(version)
-        # Loader specification here to avoid a deprecation warning... see https://msg.pyyaml.org/load
-        referenceYaml = yaml.load(open(referencePath), Loader=yaml.FullLoader)
-        compareYaml = yaml.load(open(comparePath), Loader=yaml.FullLoader)
-        self.assertEqual(referenceYaml, compareYaml)
diff --git a/python/adelphi/tests/integration/nb/test_nb_export_outputdir.py b/python/adelphi/tests/integration/nb/test_nb_export_outputdir.py
deleted file mode 100644
index 2a239ca..0000000
--- a/python/adelphi/tests/integration/nb/test_nb_export_outputdir.py
+++ /dev/null
@@ -1,43 +0,0 @@
-import glob
-import logging
-import os
-import sys
-import yaml
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-if os.name == 'posix' and sys.version_info[0] < 3:
-    import subprocess32 as subprocess
-else:
-    import subprocess
-
-from tests.integration import SchemaTestMixin
-from tests.integration.nb import ExportNbMixin
-
-log = logging.getLogger('adelphi')
-
-class TestNbExportOutputDir(unittest.TestCase, SchemaTestMixin, ExportNbMixin):
-
-    def setUp(self):
-        super(TestNbExportOutputDir, self).setUp()
-
-
-    def getBaseSchemaPath(self):
-        return "tests/integration/resources/nb-base-schema.cql"
-
-
-    def runAdelphi(self, version=None):
-        stderrPath = self.stderrPath(version)
-        outputDirPath = self.outputDirPath(version)
-        os.mkdir(outputDirPath)
-        subprocess.run("adelphi --output-dir={} export-nb 2>> {}".format(outputDirPath, stderrPath), shell=True)
-
-
-    def evalAdelphiOutput(self, version=None):
-        outputDirPath = self.outputDirPath(version)
-        outputSchemas = glob.glob("{}/*/schema".format(outputDirPath))
-        self.assertEqual(len(outputSchemas), 1, "Export of nosqlbench config only supports a single keyspace")
-        self.compareToReferenceYaml(outputSchemas[0], version)
diff --git a/python/adelphi/tests/integration/nb/test_nb_export_stdout.py b/python/adelphi/tests/integration/nb/test_nb_export_stdout.py
deleted file mode 100644
index 1365c0b..0000000
--- a/python/adelphi/tests/integration/nb/test_nb_export_stdout.py
+++ /dev/null
@@ -1,37 +0,0 @@
-import logging
-import os
-import sys
-
-try:
-    import unittest2 as unittest
-except ImportError:
-    import unittest
-
-if os.name == 'posix' and sys.version_info[0] < 3:
-    import subprocess32 as subprocess
-else:
-    import subprocess
-
-from tests.integration import SchemaTestMixin
-from tests.integration.nb import ExportNbMixin
-
-log = logging.getLogger('adelphi')
-
-class TestNbExportStdout(unittest.TestCase, SchemaTestMixin, ExportNbMixin):
-
-    def setUp(self):
-        super(TestNbExportStdout, self).setUp()
-
-
-    def getBaseSchemaPath(self):
-        return "tests/integration/resources/nb-base-schema.cql"
-
-
-    def runAdelphi(self, version=None):
-        stdoutPath = self.stdoutPath(version)
-        stderrPath = self.stderrPath(version)
-        subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True)
-
-
-    def evalAdelphiOutput(self, version=None):
-        self.compareToReferenceYaml(self.stdoutPath(version), version)
diff --git a/python/adelphi/tests/integration/resources/cql-schemas/3.11.9-ks0.cql b/python/adelphi/tests/integration/resources/cql-schemas/3.11.10-ks0.cql
similarity index 100%
rename from python/adelphi/tests/integration/resources/cql-schemas/3.11.9-ks0.cql
rename to python/adelphi/tests/integration/resources/cql-schemas/3.11.10-ks0.cql
diff --git a/python/adelphi/tests/integration/resources/cql-schemas/3.11.9.cql b/python/adelphi/tests/integration/resources/cql-schemas/3.11.10.cql
similarity index 100%
rename from python/adelphi/tests/integration/resources/cql-schemas/3.11.9.cql
rename to python/adelphi/tests/integration/resources/cql-schemas/3.11.10.cql
diff --git a/python/adelphi/tests/integration/resources/nb-base-schema.cql b/python/adelphi/tests/integration/resources/nb-base-schema.cql
index 412d521..dfbf7e3 100644
--- a/python/adelphi/tests/integration/resources/nb-base-schema.cql
+++ b/python/adelphi/tests/integration/resources/nb-base-schema.cql
@@ -1,4 +1,4 @@
-CREATE KEYSPACE testkeyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}  AND durable_writes = true;
+CREATE KEYSPACE testkeyspace WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'} AND durable_writes = true;
 
 CREATE TABLE testkeyspace.testtable (
     key1 text,
diff --git a/python/adelphi/tests/integration/resources/nb-schemas/3.11.9.yaml b/python/adelphi/tests/integration/resources/nb-schemas/3.11.10.yaml
similarity index 100%
rename from python/adelphi/tests/integration/resources/nb-schemas/3.11.9.yaml
rename to python/adelphi/tests/integration/resources/nb-schemas/3.11.10.yaml
diff --git a/python/adelphi/tests/integration/test_cql.py b/python/adelphi/tests/integration/test_cql.py
new file mode 100644
index 0000000..70d9e76
--- /dev/null
+++ b/python/adelphi/tests/integration/test_cql.py
@@ -0,0 +1,144 @@
+import glob
+import logging
+import os
+import shutil
+import sys
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess
+else:
+    import subprocess
+
+from tests.integration import SchemaTestCase, setupSchema, getAllKeyspaces, dropNewKeyspaces
+from tests.util.schemadiff import cqlDigestGenerator
+from tests.util.schema_util import get_schema
+
+log = logging.getLogger('adelphi')
+
+CQL_REFERENCE_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}.cql"
+CQL_REFERENCE_KS0_SCHEMA_PATH = "tests/integration/resources/cql-schemas/{}-ks0.cql"
+
+def digestSet(schemaFile):
+    rv = set()
+    for (_, digest) in cqlDigestGenerator(schemaFile):
+        rv.add(digest)
+    return rv
+
+
+def logCqlDigest(schemaFile, digestSet):
+    for (cql, digest) in cqlDigestGenerator(schemaFile):
+        if digest in digestSet:
+            log.info("Digest: {}, CQL: {}".format(digest,cql))
+
+
+class TestCql(SchemaTestCase):
+
+    # ========================== Unittest infrastructure ==========================
+    def setUp(self):
+        super(TestCql, self).setUp()
+        self.origKeyspaces = getAllKeyspaces()
+        log.info("Creating schema")
+        setupSchema(self.buildSchema())
+
+
+    def tearDown(self):
+        super(TestCql, self).tearDown()
+        dropNewKeyspaces(self.origKeyspaces)
+
+
+    # ========================== Helper functions ==========================
+    # TODO: Note that we currently have to disable support for SASI indexes when creating
+    # a schema since the base config for the Cassandra Docker images doesn't enable it.
+    # https://github.com/datastax/adelphi/issues/105 aims to fix this problem but until
+    # that's in place we simply exclude SASI indexes from testing.
+    def buildSchema(self):
+        baseSchemaPath = self.basePath("base-schema.cql")
+        with open(baseSchemaPath, "w") as f:
+            f.write("\n\n".join(ks.export_as_string() for ks in get_schema(sasi=False).keyspaces))
+        return baseSchemaPath
+
+
+    def compareToReferenceCql(self, referencePath, comparePath):
+        referenceSet = digestSet(referencePath)
+        compareSet = digestSet(comparePath)
+
+        refOnlySet = referenceSet - compareSet
+        if len(refOnlySet) > 0:
+            log.info("Statements in reference file {} but not in compare file {}:".format(referencePath, comparePath))
+            logCqlDigest(referencePath, refOnlySet)
+        compareOnlySet = compareSet - referenceSet
+        if len(compareOnlySet) > 0:
+            log.info("Statements in compare file {} but not in reference file {}:".format(comparePath, referencePath))
+            logCqlDigest(comparePath, compareOnlySet)
+
+        self.assertEqual(referenceSet, compareSet)
+
+
+    # ========================== Test functions ==========================
+    def test_stdout(self):
+        stdoutPath = self.stdoutPath(self.version)
+        stderrPath = self.stderrPath(self.version)
+        subprocess.run("adelphi export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True)
+
+        self.compareToReferenceCql(
+            CQL_REFERENCE_SCHEMA_PATH.format(self.version), 
+            self.stdoutPath(self.version))
+
+
+    def test_outputdir(self):
+        stderrPath = self.stderrPath(self.version)
+        outputDirPath = self.outputDirPath(self.version)
+        os.mkdir(outputDirPath)
+        subprocess.run("adelphi --output-dir={} export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True)
+
+        # Basic idea here is to find all schemas written to the output dir and aggregate them into a single schema
+        # file.  We then compare this aggregated file to the reference schema.  Ordering is important here but
+        # the current keyspace names hash to something that causes individual keyspaces to be discovered in the
+        # correct order.
+        outputDirPath = self.outputDirPath(self.version)
+        allOutputFileName = "{}-all".format(self.version)
+        allOutputPath = self.outputDirPath(allOutputFileName)
+
+        outputSchemas = glob.glob("{}/*/schema".format(outputDirPath))
+        self.assertGreater(len(outputSchemas), 0)
+        with open(allOutputPath, "w+") as allOutputFile:
+            for outputSchema in outputSchemas:
+                with open(outputSchema) as outputSchemaFile:
+                    shutil.copyfileobj(outputSchemaFile, allOutputFile)
+                    allOutputFile.write("\n")
+        self.compareToReferenceCql(
+            CQL_REFERENCE_SCHEMA_PATH.format(self.version), 
+            allOutputPath)
+
+
+    def test_some_keyspaces_stdout(self):
+        stdoutPath = self.stdoutPath(self.version)
+        stderrPath = self.stderrPath(self.version)
+        subprocess.run("adelphi --keyspaces=my_ks_0 export-cql --no-metadata > {} 2>> {}".format(stdoutPath, stderrPath), shell=True)
+
+        self.compareToReferenceCql(
+            CQL_REFERENCE_KS0_SCHEMA_PATH.format(self.version), 
+            self.stdoutPath(self.version))
+
+
+    def test_some_keyspaces_outputdir(self):
+        stderrPath = self.stderrPath(self.version)
+        outputDirPath = self.outputDirPath(self.version)
+        os.mkdir(outputDirPath)
+        subprocess.run("adelphi --output-dir={} --keyspaces=my_ks_0 export-cql --no-metadata 2>> {}".format(outputDirPath, stderrPath), shell=True)
+
+        outputDirPath = self.outputDirPath(self.version)
+        outputSchemas = glob.glob("{}/*/schema".format(outputDirPath))
+        self.assertEqual(len(outputSchemas), 1)
+        self.compareToReferenceCql(
+            CQL_REFERENCE_KS0_SCHEMA_PATH.format(self.version), 
+            outputSchemas[0])
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/python/adelphi/tests/integration/test_nb.py b/python/adelphi/tests/integration/test_nb.py
new file mode 100644
index 0000000..7c711f5
--- /dev/null
+++ b/python/adelphi/tests/integration/test_nb.py
@@ -0,0 +1,71 @@
+import glob
+import logging
+import os
+import sys
+import yaml
+
+try:
+    import unittest2 as unittest
+except ImportError:
+    import unittest
+
+if os.name == 'posix' and sys.version_info[0] < 3:
+    import subprocess32 as subprocess
+else:
+    import subprocess
+
+from tests.integration import SchemaTestCase, setupSchema, getAllKeyspaces, dropNewKeyspaces
+
+log = logging.getLogger('adelphi')
+
+NB_SCHEMA_PATH = "tests/integration/resources/nb-base-schema.cql"
+NB_REFERENCE_SCHEMA_PATH = "tests/integration/resources/nb-schemas/{}.yaml"
+
+class TestNb(SchemaTestCase):
+
+    origKeyspaces = None
+
+    # ========================== Unittest infrastructure ==========================
+    @classmethod
+    def setUpClass(cls):
+        TestNb.origKeyspaces = getAllKeyspaces()
+        log.info("Creating schema")
+        setupSchema(NB_SCHEMA_PATH)
+
+
+    @classmethod
+    def tearDownClass(cls):
+        dropNewKeyspaces(TestNb.origKeyspaces)
+
+
+    # ========================== Helper functions ==========================
+    def compareToReferenceYaml(self, comparePath, version=None):
+        referencePath = NB_REFERENCE_SCHEMA_PATH.format(version)
+        # Loader specification here to avoid a deprecation warning... see https://msg.pyyaml.org/load
+        referenceYaml = yaml.load(open(referencePath), Loader=yaml.FullLoader)
+        compareYaml = yaml.load(open(comparePath), Loader=yaml.FullLoader)
+        self.assertEqual(referenceYaml, compareYaml)
+
+
+    # ========================== Test functions ==========================
+    def test_stdout(self):
+        stdoutPath = self.stdoutPath(self.version)
+        stderrPath = self.stderrPath(self.version)
+        subprocess.run("adelphi export-nb > {} 2>> {}".format(stdoutPath, stderrPath), shell=True)
+        self.compareToReferenceYaml(self.stdoutPath(self.version), self.version)
+
+
+    def test_outputdir(self):
+        stderrPath = self.stderrPath(self.version)
+        outputDirPath = self.outputDirPath(self.version)
+        os.mkdir(outputDirPath)
+        subprocess.run("adelphi --output-dir={} export-nb 2>> {}".format(outputDirPath, stderrPath), shell=True)
+
+        outputDirPath = self.outputDirPath(self.version)
+        outputSchemas = glob.glob("{}/*/schema".format(outputDirPath))
+        self.assertEqual(len(outputSchemas), 1, "Export of nosqlbench config only supports a single keyspace")
+        self.compareToReferenceYaml(outputSchemas[0], self.version)
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file
diff --git a/python/adelphi/tests/util/schemadiff.py b/python/adelphi/tests/util/schemadiff.py
index 948b379..f821faf 100644
--- a/python/adelphi/tests/util/schemadiff.py
+++ b/python/adelphi/tests/util/schemadiff.py
@@ -1,6 +1,5 @@
 import base64
 import hashlib
-import logging
 
 # Imports required by the __main__ case below
 import sys
diff --git a/python/adelphi/tox.ini b/python/adelphi/tox.ini
deleted file mode 100644
index 43d5e5f..0000000
--- a/python/adelphi/tox.ini
+++ /dev/null
@@ -1,9 +0,0 @@
-[tox]
-envlist = py2,py3
-
-[testenv]
-deps = pytest
-       docker ~= 4.4
-       subprocess32 ~= 3.5
-       tenacity ~= 7.0
-commands = pytest {posargs}
\ No newline at end of file
diff --git a/python/adelphi/tox.ini.dev b/python/adelphi/tox.ini.dev
deleted file mode 100644
index 06b04ea..0000000
--- a/python/adelphi/tox.ini.dev
+++ /dev/null
@@ -1,12 +0,0 @@
-[tox]
-envlist = py2,py3
-
-[testenv]
-deps = pytest
-       docker ~= 4.4
-       subprocess32 ~= 3.5
-       tenacity ~= 7.0
-setenv =
-       KEEP_LOGS=1
-       CASSANDRA_VERSIONS=4.0-rc1
-commands = pytest {posargs}
\ No newline at end of file