From 54832c15ede9a305f58e0e1e5a7d78e9dcbdface Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 14:36:46 -0700 Subject: [PATCH 1/9] fix: lock formatter versions to avoid style drift Signed-off-by: maximsmol --- pyproject.toml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b205563..1b9be11 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [tool.poetry] name = "schemainspect" version = "3.1" -authors = [ "Robert Lechte ",] +authors = ["Robert Lechte "] license = "Unlicense" readme = "README.md" description = "Schema inspection for PostgreSQL (and possibly others)" @@ -21,15 +21,13 @@ pytest-cov = "*" pytest-clarity = "*" psycopg2-binary = "*" flake8 = "*" -isort = "*" +isort = "5.10.1" migra = "*" -black = "*" +black = "22.3.0" toml = "*" [tool.poetry.scripts] schemainspect = 'schemainspect:do_command' [tool.isort] -multi_line_output = 3 -include_trailing_comma = true -line_length = 88 +profile = "black" From 3cabd594d385a3c67fd49a035558c754509f4fa9 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 14:36:57 -0700 Subject: [PATCH 2/9] feat: add pre-commit Signed-off-by: maximsmol --- .pre-commit-config.yaml | 47 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..0b5f9cd --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,47 @@ +repos: + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.2.0 + hooks: + - id: check-added-large-files + - id: check-ast + - id: check-builtin-literals + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-json + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + - id: check-symlinks + - id: check-toml + - id: check-vcs-permalinks + - id: check-xml + - id: check-yaml + - id: debug-statements + - id: destroyed-symlinks + - id: detect-aws-credentials + args: [--allow-missing-credentials] + - id: detect-private-key + # - id: double-quote-string-fixer + - id: end-of-file-fixer + - id: fix-byte-order-marker + - id: fix-encoding-pragma + args: [--remove] + # - id: file-contents-sorter + # - id: forbid-new-submodules + - id: mixed-line-ending + args: ["--fix", "no"] + # - id: name-tests-test + # - id: no-commit-to-branch + # - id: pretty-format-json + # - id: requirements-txt-fixer + # - id: sort-simple-yaml + - id: trailing-whitespace + args: [--markdown-linebreak-ext=md] + - repo: https://github.com/psf/black + rev: 22.3.0 + hooks: + - id: black + - repo: https://github.com/PyCQA/isort + rev: 5.10.1 + hooks: + - id: isort From 0cabcc4ad57745aee10ffe0e1752598a53b13721 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 14:37:15 -0700 Subject: [PATCH 3/9] fix: trim trailing whitespace, add new lines at end of file Signed-off-by: maximsmol --- schemainspect/pg/sql/constraints.sql | 6 +++--- schemainspect/pg/sql/deps.sql | 2 +- schemainspect/pg/sql/domains.sql | 4 ++-- schemainspect/pg/sql/relations.sql | 2 +- schemainspect/pg/sql/schemas.sql | 2 +- 5 files changed, 8 insertions(+), 8 deletions(-) diff --git a/schemainspect/pg/sql/constraints.sql b/schemainspect/pg/sql/constraints.sql index 9c49791..4834359 100644 --- a/schemainspect/pg/sql/constraints.sql +++ b/schemainspect/pg/sql/constraints.sql @@ -60,7 +60,7 @@ select pg_attribute ta join unnest(conkey) with ordinality c(cn, rn) - on + on ta.attrelid = conrelid and ta.attnum = c.cn ) else null end as fk_columns_local, @@ -72,7 +72,7 @@ select pg_attribute ta join unnest(confkey) with ordinality c(cn, rn) - on + on ta.attrelid = confrelid and ta.attnum = c.cn ) else null end as fk_columns_foreign, @@ -102,4 +102,4 @@ from where true -- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast', 'pg_temp_1', 'pg_toast_temp_1') -- SKIP_INTERNAL and e.objid is null and er.objid is null and cr.objid is null -order by 1, 3, 2; \ No newline at end of file +order by 1, 3, 2; diff --git a/schemainspect/pg/sql/deps.sql b/schemainspect/pg/sql/deps.sql index 3995e03..09e8561 100644 --- a/schemainspect/pg/sql/deps.sql +++ b/schemainspect/pg/sql/deps.sql @@ -84,4 +84,4 @@ combined as ( select * from combined order by schema, name, identity_arguments, kind_dependent_on, -schema_dependent_on, name_dependent_on, identity_arguments_dependent_on \ No newline at end of file +schema_dependent_on, name_dependent_on, identity_arguments_dependent_on diff --git a/schemainspect/pg/sql/domains.sql b/schemainspect/pg/sql/domains.sql index ce2538b..8e81b2c 100644 --- a/schemainspect/pg/sql/domains.sql +++ b/schemainspect/pg/sql/domains.sql @@ -5,7 +5,7 @@ with extension_oids as ( pg_depend d WHERE d.refclassid = 'pg_extension'::regclass and - d.classid = 'pg_type'::regclass + d.classid = 'pg_type'::regclass ) SELECT n.nspname as "schema", t.typname as "name", @@ -26,4 +26,4 @@ WHERE t.typtype = 'd' AND n.nspname <> 'information_schema' AND pg_catalog.pg_type_is_visible(t.oid) and t.oid not in (select * from extension_oids) -ORDER BY 1, 2; \ No newline at end of file +ORDER BY 1, 2; diff --git a/schemainspect/pg/sql/relations.sql b/schemainspect/pg/sql/relations.sql index a099155..92f5f94 100644 --- a/schemainspect/pg/sql/relations.sql +++ b/schemainspect/pg/sql/relations.sql @@ -5,7 +5,7 @@ with extension_oids as ( pg_depend d WHERE d.refclassid = 'pg_extension'::regclass and - d.classid = 'pg_class'::regclass + d.classid = 'pg_class'::regclass ), enums as ( SELECT diff --git a/schemainspect/pg/sql/schemas.sql b/schemainspect/pg/sql/schemas.sql index 32701a1..bb657e0 100644 --- a/schemainspect/pg/sql/schemas.sql +++ b/schemainspect/pg/sql/schemas.sql @@ -12,7 +12,7 @@ from pg_catalog.pg_namespace left outer join extension_oids e on e.objid = oid --- SKIP_INTERNAL where nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') +-- SKIP_INTERNAL where nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' -- SKIP_INTERNAL and e.objid is null order by 1; From ae4d124826b78ed6a1de2e3a03a8411b8b34efd6 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 14:54:56 -0700 Subject: [PATCH 4/9] feat: add pre-commit to circle ci Signed-off-by: maximsmol --- .circleci/config.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 21bb73f..cc3cec7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -272,8 +272,34 @@ jobs: poetry config pypi-token.pypi $PYPI_PASSWORD poetry publish + pre-commit: + working_directory: ~/circleci + docker: + - image: cimg/python:3.10 + steps: + - setup_remote_docker + - checkout + - run: + name: Install pre-commit + command: | + python -m pip install -U pip pre-commit + - restore_cache: + keys: + - v1-pre-commit-env-{{ checksum ".pre-commit-config.yaml" }} + - v1-pre-commit-env- + - run: + name: Run pre-commit + command: | + pre-commit run --show-diff-on-failure --color=always + - save_cache: + key: v1-pre-commit-env-{{ checksum ".pre-commit-config.yaml" }} + paths: ~/.cache/pre-commit + workflows: version: 2 + lint: + jobs: + - pre-commit build-then-publish: jobs: - build-pg14 From f752deaf8252b7cbee1143ecf3f45665601dc8f1 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 15:15:18 -0700 Subject: [PATCH 5/9] fix: use python-m to run pre-commit Signed-off-by: maximsmol --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cc3cec7..ff923b1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,7 +290,7 @@ jobs: - run: name: Run pre-commit command: | - pre-commit run --show-diff-on-failure --color=always + python -m pre_commit run --show-diff-on-failure --color=always - save_cache: key: v1-pre-commit-env-{{ checksum ".pre-commit-config.yaml" }} paths: ~/.cache/pre-commit From 4198d8f6877cfb98356dadef01df0b605534f310 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 15:18:08 -0700 Subject: [PATCH 6/9] fix: pre-commit should run on all files Signed-off-by: maximsmol --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ff923b1..ffa5302 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -290,7 +290,7 @@ jobs: - run: name: Run pre-commit command: | - python -m pre_commit run --show-diff-on-failure --color=always + python -m pre_commit run --show-diff-on-failure --color=always --all-files - save_cache: key: v1-pre-commit-env-{{ checksum ".pre-commit-config.yaml" }} paths: ~/.cache/pre-commit From a62600b1b691cec326bb5626e85cbb2f65e5dbde Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 15:22:24 -0700 Subject: [PATCH 7/9] feat: cache pre-commit dependencies too Signed-off-by: maximsmol --- .circleci/config.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index ffa5302..f106e98 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -279,10 +279,18 @@ jobs: steps: - setup_remote_docker - checkout + - restore_cache: + keys: + - v1-pre-commit-pip - run: name: Install pre-commit command: | python -m pip install -U pip pre-commit + - save_cache: + key: v1-pre-commit-pip + paths: + - .venv + - ~/.cache/pip - restore_cache: keys: - v1-pre-commit-env-{{ checksum ".pre-commit-config.yaml" }} From 3f356464c384cafc192c9ef204cab2202340d3e8 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 15:59:23 -0700 Subject: [PATCH 8/9] fix: drop python 2 shims since we don't support it anyway Signed-off-by: maximsmol --- pyproject.toml | 1 - schemainspect/misc.py | 4 +--- schemainspect/pg/obj.py | 4 ---- schemainspect/tableformat.py | 7 +------ tests/test_deps_fk.py | 5 ----- 5 files changed, 2 insertions(+), 19 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1b9be11..3fe0463 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,7 +11,6 @@ homepage = "https://github.com/djrobstep/schemainspect" [tool.poetry.dependencies] python = ">=3.7,<4" -six = "*" sqlalchemy = "*" [tool.poetry.dev-dependencies] diff --git a/schemainspect/misc.py b/schemainspect/misc.py index e2aa28c..26e16ca 100644 --- a/schemainspect/misc.py +++ b/schemainspect/misc.py @@ -1,7 +1,6 @@ import inspect from reprlib import recursive_repr -import six from pkg_resources import resource_stream as pkg_resource_stream @@ -17,8 +16,7 @@ def connection_from_s_or_c(s_or_c): # pragma: no cover return s_or_c -@six.python_2_unicode_compatible -class AutoRepr(object): # pragma: no cover +class AutoRepr: # pragma: no cover @recursive_repr() def __repr__(self): done = set() diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index 6ebbc6d..eaf6485 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -1,4 +1,3 @@ -import sys from collections import OrderedDict as od from itertools import groupby @@ -1238,9 +1237,6 @@ def dependency_order( enums=True, include_fk_deps=False, ): - if sys.version_info < (3, 0): - raise NotImplementedError - from schemainspect import TopologicalSorter graph, things = {}, {} diff --git a/schemainspect/tableformat.py b/schemainspect/tableformat.py index 7c49ca2..02b94b7 100644 --- a/schemainspect/tableformat.py +++ b/schemainspect/tableformat.py @@ -1,9 +1,4 @@ -import sys - -if sys.version_info >= (3, 0): - from itertools import zip_longest -else: - from itertools import izip_longest as zip_longest # noqa +from itertools import zip_longest def transposed(in_data): diff --git a/tests/test_deps_fk.py b/tests/test_deps_fk.py index 9126a9c..0053cdf 100644 --- a/tests/test_deps_fk.py +++ b/tests/test_deps_fk.py @@ -1,5 +1,3 @@ -import sys - from sqlbag import S from schemainspect import get_inspector @@ -83,9 +81,6 @@ def test_dep_order(db): i = get_inspector(s) - # dependency_order doesn't work in py2 - if sys.version_info < (3, 0): - return create_order = i.dependency_order( include_fk_deps=True, ) From 427ad0bb2a99f067d036583810164d41f65b45e9 Mon Sep 17 00:00:00 2001 From: maximsmol Date: Wed, 13 Apr 2022 16:31:18 -0700 Subject: [PATCH 9/9] feat: add typing info to graphlib Signed-off-by: maximsmol --- schemainspect/graphlib/__init__.py | 29 +++++++++++++++++------------ 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/schemainspect/graphlib/__init__.py b/schemainspect/graphlib/__init__.py index 95b861c..c57370c 100644 --- a/schemainspect/graphlib/__init__.py +++ b/schemainspect/graphlib/__init__.py @@ -1,5 +1,7 @@ __all__ = ["TopologicalSorter", "CycleError"] +from typing import Dict, List, Optional, Set, TypeAlias + _NODE_OUT = -1 _NODE_DONE = -2 @@ -7,7 +9,7 @@ class _NodeInfo: __slots__ = "node", "npredecessors", "successors" - def __init__(self, node): + def __init__(self, node: str): # The node this class is augmenting. self.node = node @@ -18,7 +20,7 @@ def __init__(self, node): # List of successor nodes. The list can contain duplicated elements as # long as they're all reflected in the successor's npredecessors attribute). - self.successors = [] + self.successors: List[str] = [] class CycleError(ValueError): @@ -35,12 +37,15 @@ class CycleError(ValueError): pass +Graph: TypeAlias = Dict[str, List[str]] + + class TopologicalSorter: """Provides functionality to topologically sort a graph of hashable nodes""" - def __init__(self, graph=None): - self._node2info = {} - self._ready_nodes = None + def __init__(self, graph: Optional[Graph] = None): + self._node2info: Dict[str, _NodeInfo] = {} + self._ready_nodes: Optional[List[str]] = None self._npassedout = 0 self._nfinished = 0 @@ -48,14 +53,14 @@ def __init__(self, graph=None): for node, predecessors in graph.items(): self.add(node, *predecessors) - def _get_nodeinfo(self, node): + def _get_nodeinfo(self, node: str): result = self._node2info.get(node) if result is None: self._node2info[node] = result = _NodeInfo(node) return result - def add(self, node, *predecessors): + def add(self, node: str, *predecessors: str): """Add a new node and its predecessors to the graph. Both the *node* and all elements in *predecessors* must be hashable. @@ -147,7 +152,7 @@ def is_active(self): def __bool__(self): return self.is_active() - def done(self, *nodes): + def done(self, *nodes: str): """Marks a set of nodes returned by "get_ready" as processed. This method unblocks any successor of each node in *nodes* for being returned @@ -204,10 +209,10 @@ def done(self, *nodes): def _find_cycle(self): n2i = self._node2info - stack = [] - itstack = [] - seen = set() - node2stacki = {} + stack: List[str] = [] + itstack: List[str] = [] + seen: Set[str] = set() + node2stacki: Dict[str, int] = {} for node in n2i: if node in seen: