DM-49446: Improvements to diff tool

docs/changes/DM-49446.feature.rst

@@ -0,0 +1,6 @@
+The ``diff`` module was completely overhauled based on best practices for using the ``deepdiff`` library.
+The formatted differences are now displayed in a JSON format, which is more readable and structured than the previous print outs.
+A number of bugs and issues with the comparison logic and the display of the diffs were also fixed, improving the accuracy of the schema diffs.
+The error handling in the database diff class was improved so that exception messages should be more informative and helpful.
+A command line option has been added for writing the diff output to a file.
+Finally, an additional command line option allows filtering only on specified tables in the two schemas being compared.

docs/documenteer.toml

@@ -34,3 +34,4 @@ python_api_dir = "dev/internals"
 python = "https://docs.python.org/3"
 sqlalchemy = "https://docs.sqlalchemy.org/en/latest"
 lsst = "https://pipelines.lsst.io/v/weekly"
+click = "https://click.palletsprojects.com"

python/felis/__init__.py

@@ -19,9 +19,21 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
-from .datamodel import Schema
+from .datamodel import (
+    CheckConstraint,
+    Column,
+    ColumnGroup,
+    Constraint,
+    DataType,
+    ForeignKeyConstraint,
+    Index,
+    Schema,
+    SchemaVersion,
+    Table,
+    UniqueConstraint,
+)
 from .db.schema import create_database
-from .diff import DatabaseDiff, FormattedSchemaDiff, SchemaDiff
+from .diff import DatabaseDiff, FormattedSchemaDiff
 from .metadata import MetaDataBuilder
 
 from importlib.metadata import PackageNotFoundError, version

python/felis/cli.py

@@ -431,13 +431,23 @@ def validate(
     default="deepdiff",
 )
 @click.option("-E", "--error-on-change", is_flag=True, help="Exit with error code if schemas are different")
+@click.option("--table", "tables", multiple=True, help="Table names to filter on.")
+@click.option(
+    "--output-file",
+    "-o",
+    type=click.File(mode="w"),
+    help="Write diff output to a file insteading of printing",
+    default=None,
+)
 @click.argument("files", nargs=-1, type=click.File())
 @click.pass_context
 def diff(
     ctx: click.Context,
     engine_url: str | None,
     comparator: str,
     error_on_change: bool,
+    tables: list[str],
+    output_file: IO[str] | None,
     files: Iterable[IO[str]],
 ) -> None:
     schemas = [
@@ -447,20 +457,24 @@ def diff(
     diff: SchemaDiff
     if len(schemas) == 2 and engine_url is None:
         if comparator == "alembic":
+            if tables:
+                raise click.ClickException("Table filtering is not supported for Alembic comparator")
             db_context = create_database(schemas[0])
             assert isinstance(db_context.engine, Engine)
             diff = DatabaseDiff(schemas[1], db_context.engine)
         else:
             diff = FormattedSchemaDiff(schemas[0], schemas[1])
     elif len(schemas) == 1 and engine_url is not None:
+        if tables:
+            raise click.ClickException("Table filtering is not supported for database comparison")
         engine = create_engine(engine_url)
         diff = DatabaseDiff(schemas[0], engine)
     else:
         raise click.ClickException(
             "Invalid arguments - provide two schemas or a schema and a database engine URL"
         )
 
-    diff.print()
+    diff.print(output_file)
 
     if diff.has_changes and error_on_change:
         raise click.ClickException("Schema was changed")