diff --git a/.circleci/config.yml b/.circleci/config.yml index a04894b..3770547 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -47,40 +47,6 @@ jobs: - store_artifacts: path: test-reports/ destination: tr1 - build-py2: - working_directory: ~/circleci - docker: - - image: circleci/python:2.7.18 - - image: circleci/postgres:12 - environment: - POSTGRES_USER: circleci - POSTGRES_DB: circleci - POSTGRES_HOST_AUTH_METHOD: trust - steps: - - checkout - - restore_cache: - key: deps1-{{ .Branch }}-{{ checksum "pyproject.toml" }} - - run: - name: Wait for db - command: dockerize -wait tcp://localhost:5432 -timeout 1m - - run: sudo apt-get install -y postgresql-client - - run: - name: create postgres user - command: psql postgresql://@localhost/circleci -c 'create role postgres' - - run: - name: Install poetry - command: | - sudo pip install poetry autovenv - poetry config virtualenvs.create false - - run: - command: | - poetry install - - run: - command: | - make test - - store_artifacts: - path: test-reports/ - destination: tr1 build-pg11: working_directory: ~/circleci @@ -257,7 +223,6 @@ workflows: build-then-publish: jobs: - build - - build-py2 - build-pg11 - build-pg10 - build-pg9 diff --git a/Makefile b/Makefile index a582f55..4edf143 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ clean: find . -name \*.pyc -delete fmt: - isort -rc . + isort . black . lint: diff --git a/pyproject.toml b/pyproject.toml index 35f1953..f23c471 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "schemainspect" -version = "0.1" +version = "3.0" authors = [ "Robert Lechte ",] license = "Unlicense" readme = "README.md" @@ -10,18 +10,18 @@ repository = "https://github.com/djrobstep/schemainspect" homepage = "https://github.com/djrobstep/schemainspect" [tool.poetry.dependencies] -python = "*" +python = ">=3.6,<4" six = "*" sqlalchemy = "*" [tool.poetry.dev-dependencies] -sqlbag = "*" -pytest = "*" +sqlbag = ">=0.1.1616028516" +pytest = {version="*", python=">=3.5,<4"} pytest-cov = "*" -pytest-clarity = ">=0.3.0-alpha.0" +pytest-clarity = {version=">=0.3.0-alpha.0", python=">=3.5,<4"} psycopg2-binary = "*" flake8 = "*" -isort = "*" +isort = {version=">=5", python=">=3.6,<4"} migra = "*" black = { version = ">=19.10b0", python=">=3.6" } diff --git a/schemainspect/get.py b/schemainspect/get.py index 2184474..fdff900 100644 --- a/schemainspect/get.py +++ b/schemainspect/get.py @@ -5,7 +5,9 @@ SUPPORTED = {"postgresql": PostgreSQL} -def get_inspector(x, schema=None): +def get_inspector(x, schema=None, exclude_schema=None): + if schema and exclude_schema: + raise ValueError("Cannot provide both schema and exclude_schema") if x is None: return NullInspector() @@ -18,4 +20,6 @@ def get_inspector(x, schema=None): inspected = ic(c) if schema: inspected.one_schema(schema) + elif exclude_schema: + inspected.exclude_schema(exclude_schema) return inspected diff --git a/schemainspect/inspected.py b/schemainspect/inspected.py index e7c0990..8ee185b 100644 --- a/schemainspect/inspected.py +++ b/schemainspect/inspected.py @@ -53,6 +53,7 @@ def __init__( is_identity=False, is_identity_always=False, is_generated=False, + is_inherited=False, ): self.name = name or "" self.dbtype = dbtype @@ -66,6 +67,7 @@ def __init__( self.is_identity = is_identity self.is_identity_always = is_identity_always self.is_generated = is_generated + self.is_inherited = is_inherited def __eq__(self, other): return ( @@ -80,6 +82,7 @@ def __eq__(self, other): and self.is_identity == other.is_identity and self.is_identity_always == other.is_identity_always and self.is_generated == other.is_generated + and self.is_inherited == other.is_inherited ) def alter_clauses(self, other): diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index e7eeb38..e75518e 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -1,5 +1,6 @@ from __future__ import absolute_import, division, print_function, unicode_literals +import inspect import sys from collections import OrderedDict as od from itertools import groupby @@ -164,7 +165,9 @@ def is_table(self): @property def is_alterable(self): - return self.is_table and not self.parent_table + return self.is_table and ( + not self.parent_table or self.is_inheritance_child_table + ) @property def contains_data(self): @@ -400,6 +403,8 @@ def __init__( algorithm, definition=None, constraint=None, + index_columns=None, + included_columns=None, ): self.name = name self.schema = schema @@ -418,6 +423,8 @@ def __init__( self.partial_predicate = partial_predicate self.algorithm = algorithm self.constraint = constraint + self.index_columns = index_columns + self.included_columns = included_columns @property def drop_statement(self): @@ -429,14 +436,18 @@ def create_statement(self): def __eq__(self, other): """ - :type other: InspectedIndex + :type other: Optional[InspectedIndex] :rtype: bool """ + if other is None: # sometimes the other index doesn't exist + return False + equalities = ( self.name == other.name, self.schema == other.schema, self.table_name == other.table_name, self.key_columns == other.key_columns, + self.included_columns == other.included_columns, self.key_options == other.key_options, self.num_att == other.num_att, self.is_unique == other.is_unique, @@ -635,6 +646,14 @@ def create_statement(self): def drop_statement(self): return "drop schema if exists {};".format(self.quoted_schema) + @property + def quoted_full_name(self): + return self.quoted_name + + @property + def quoted_name(self): + return quoted_identifier(self.schema) + def __eq__(self, other): return self.schema == other.schema @@ -827,22 +846,33 @@ def drop_statement(self): self.quoted_full_table_name, self.quoted_name ) + @property + def deferrable_subclause(self): + # [ DEFERRABLE | NOT DEFERRABLE ] [ INITIALLY DEFERRED | INITIALLY IMMEDIATE ] + + if not self.is_deferrable: + return "" + + else: + clause = " DEFERRABLE" + + if self.initially_deferred: + clause += " INITIALLY DEFERRED" + + return clause + @property def create_statement(self): - USING = "alter table {} add constraint {} {} using index {};" - NOT_USING = "alter table {} add constraint {} {};" if self.index: - return USING.format( - self.quoted_full_table_name, - self.quoted_name, - self.constraint_type, - self.quoted_name, + using_clause = "{} using index {}{}".format( + self.constraint_type, self.quoted_name, self.deferrable_subclause ) - else: - return NOT_USING.format( - self.quoted_full_table_name, self.quoted_name, self.definition - ) + using_clause = self.definition + + USING = "alter table {} add constraint {} {};" + + return USING.format(self.quoted_full_table_name, self.quoted_name, using_clause) @property def quoted_full_name(self): @@ -865,6 +895,8 @@ def __eq__(self, other): self.table_name == other.table_name, self.definition == other.definition, self.index == other.index, + self.is_deferrable == other.is_deferrable, + self.initially_deferred == other.initially_deferred, ) return all(equalities) @@ -993,6 +1025,9 @@ def __eq__(self, other): return all(equalities) +PROPS = "schemas relations tables views functions selectables sequences constraints indexes enums extensions privileges collations triggers" + + class PostgreSQL(DBInspector): def __init__(self, c, include_internal=False): pg_version = c.dialect.server_version_info[0] @@ -1150,6 +1185,11 @@ def load_deps(self): r.dependent_on.append(e_sig) c.enum.dependents.append(k) + if r.parent_table: + pt = self.relations[r.parent_table] + r.dependent_on.append(r.parent_table) + pt.dependents.append(r.signature) + def get_dependency_by_signature(self, signature): things = [self.selectables, self.enums, self.triggers] @@ -1197,6 +1237,11 @@ def dependency_order( things.update(self.triggers) for k, x in things.items(): + dependent_on = list(x.dependent_on) + + if k in self.tables and x.parent_table: + dependent_on.append(x.parent_table) + graph[k] = list(x.dependent_on) if include_fk_deps: @@ -1213,7 +1258,19 @@ def dependency_order( graph.update(fk_deps) ts = TopologicalSorter(graph) - ordering = list(ts.static_order()) + + ordering = [] + + ts.prepare() + + while ts.is_active(): + items = ts.get_ready() + + itemslist = list(items) + + # itemslist.sort() + ordering += itemslist + ts.done(*items) if drop_order: ordering.reverse() @@ -1318,6 +1375,14 @@ def get_enum(name, schema): } att = getattr(self, RELATIONTYPES[f.relationtype]) att[s.quoted_full_name] = s + + for k, t in self.tables.items(): + if t.is_inheritance_child_table: + parent_table = self.tables[t.parent_table] + for cname, c in t.columns.items(): + if cname in parent_table.columns: + c.is_inherited = True + self.relations = od() for x in (self.tables, self.views, self.materialized_views): self.relations.update(x) @@ -1329,6 +1394,8 @@ def get_enum(name, schema): definition=i.definition, table_name=i.table_name, key_columns=i.key_columns, + index_columns=i.index_columns, + included_columns=i.included_columns, key_options=i.key_options, num_att=i.num_att, is_unique=i.is_unique, @@ -1367,13 +1434,13 @@ def get_enum(name, schema): constraint_type=i.constraint_type, table_name=i.table_name, definition=i.definition, - index=i.index, + index=i['index'], is_fk=i.is_fk, is_deferrable=i.is_deferrable, initially_deferred=i.initially_deferred, ) if constraint.index: - index_name = quoted_identifier(i.index, schema=i.schema) + index_name = quoted_identifier(constraint.index, schema=i.schema) index = self.indexes[index_name] index.constraint = constraint constraint.index = index @@ -1518,13 +1585,60 @@ def col(defn): ] # type: list[InspectedType] self.domains = od((t.signature, t) for t in domains) - def one_schema(self, schema): - props = "schemas relations tables views functions selectables sequences constraints indexes enums extensions privileges collations triggers" - for prop in props.split(): + def filter_schema(self, schema=None, exclude_schema=None): + if schema and exclude_schema: + raise ValueError("Can only have schema or exclude schema, not both") + + def equal_to_schema(x): + return x.schema == schema + + def not_equal_to_exclude_schema(x): + return x.schema != exclude_schema + + if schema: + comparator = equal_to_schema + elif exclude_schema: + comparator = not_equal_to_exclude_schema + else: + raise ValueError("schema or exclude_schema must be not be none") + + for prop in PROPS.split(): att = getattr(self, prop) - filtered = {k: v for k, v in att.items() if v.schema == schema} + filtered = {k: v for k, v in att.items() if comparator(v)} setattr(self, prop, filtered) + def _as_dicts(self): + def obj_to_d(x): + if isinstance(x, dict): + return {k: obj_to_d(v) for k, v in x.items()} + + elif isinstance(x, (ColumnInfo, Inspected)): + return { + k: obj_to_d(getattr(x, k)) + for k in dir(x) + if not k.startswith("_") and not inspect.ismethod(getattr(x, k)) + } + else: + + return str(x) + + d = {} + + for prop in PROPS.split(): + att = getattr(self, prop) + + _d = {k: obj_to_d(v) for k, v in att.items()} + + d[prop] = _d + + return d + + def one_schema(self, schema): + self.filter_schema(schema=schema) + + def exclude_schema(self, schema): + self.filter_schema(exclude_schema=schema) + def __eq__(self, other): """ :type other: PostgreSQL diff --git a/schemainspect/pg/sql/constraints.sql b/schemainspect/pg/sql/constraints.sql index f263094..5de3119 100644 --- a/schemainspect/pg/sql/constraints.sql +++ b/schemainspect/pg/sql/constraints.sql @@ -6,6 +6,14 @@ with extension_oids as ( WHERE d.refclassid = 'pg_extension'::regclass and d.classid = 'pg_constraint'::regclass +), extension_rels as ( + select + objid + from + pg_depend d + WHERE + d.refclassid = 'pg_extension'::regclass + and d.classid = 'pg_class'::regclass ), indexes as ( select schemaname as schema, @@ -36,10 +44,13 @@ select ON c.relnamespace = ns.oid WHERE c.oid = confrelid::regclass ) - end as foreign_table_schema, case when tc.constraint_type = 'FOREIGN KEY' then - confrelid::regclass + ( + select relname + from pg_catalog.pg_class c + where c.oid = confrelid::regclass + ) end as foreign_table_name, case when tc.constraint_type = 'FOREIGN KEY' then ( @@ -76,7 +87,11 @@ from and relname = i.table_name left outer join extension_oids e on pg_class.oid = e.objid + left outer join extension_rels er + on er.objid = conrelid + left outer join extension_rels cr + on cr.objid = confrelid where true -- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast', 'pg_temp_1', 'pg_toast_temp_1') - -- SKIP_INTERNAL and e.objid is null -order by 1, 3, 2; + -- SKIP_INTERNAL and e.objid is null and er.objid is null and cr.objid is null +order by 1, 3, 2; \ No newline at end of file diff --git a/schemainspect/pg/sql/deps.sql b/schemainspect/pg/sql/deps.sql index edb4c55..3995e03 100644 --- a/schemainspect/pg/sql/deps.sql +++ b/schemainspect/pg/sql/deps.sql @@ -28,6 +28,14 @@ extension_objids as ( pg_depend d WHERE d.refclassid = 'pg_extension'::regclass + union + select + t.typrelid as extension_objid + from + pg_depend d + join pg_type t on t.oid = d.objid + where + d.refclassid = 'pg_extension'::regclass ), things as ( select diff --git a/schemainspect/pg/sql/functions.sql b/schemainspect/pg/sql/functions.sql index bb7c418..471de77 100644 --- a/schemainspect/pg/sql/functions.sql +++ b/schemainspect/pg/sql/functions.sql @@ -1,25 +1,108 @@ -with r1 as ( - SELECT - r.routine_name as name, - r.routine_schema as schema, - r.specific_name as specific_name, - r.data_type, - r.type_udt_schema, - r.type_udt_name, - r.external_language, - r.routine_definition as definition - FROM information_schema.routines r - -- SKIP_INTERNAL where r.external_language not in ('C', 'INTERNAL') - -- SKIP_INTERNAL and r.routine_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') - -- SKIP_INTERNAL and r.routine_schema not like 'pg_temp_%' and r.routine_schema not like 'pg_toast_temp_%' - order by - r.specific_name +with extension_oids as ( + select + objid + from + pg_depend d + WHERE + d.refclassid = 'pg_extension'::regclass + and d.classid = 'pg_proc'::regclass + ), + pg_proc_pre as ( + select + pp.*, + -- 11_AND_LATER pp.oid as p_oid + -- 10_AND_EARLIER pp.oid as p_oid, case when pp.proisagg then 'a' else 'f' end as prokind + from pg_proc pp ), +routines as ( + SELECT current_database()::information_schema.sql_identifier AS specific_catalog, + n.nspname::information_schema.sql_identifier AS specific_schema, + --nameconcatoid(p.proname, p.oid)::information_schema.sql_identifier AS specific_name, + current_database()::information_schema.sql_identifier AS routine_catalog, + n.nspname::information_schema.sql_identifier AS schema, + p.proname::information_schema.sql_identifier AS name, + CASE p.prokind + WHEN 'f'::"char" THEN 'FUNCTION'::text + WHEN 'p'::"char" THEN 'PROCEDURE'::text + ELSE NULL::text + END::information_schema.character_data AS routine_type, + CASE + WHEN p.prokind = 'p'::"char" THEN NULL::text + WHEN t.typelem <> 0::oid AND t.typlen = '-1'::integer THEN 'ARRAY'::text + WHEN nt.nspname = 'pg_catalog'::name THEN format_type(t.oid, NULL::integer) + ELSE 'USER-DEFINED'::text + END::information_schema.character_data AS data_type, + + CASE + WHEN nt.nspname IS NOT NULL THEN current_database() + ELSE NULL::name + END::information_schema.sql_identifier AS type_udt_catalog, + nt.nspname::information_schema.sql_identifier AS type_udt_schema, + t.typname::information_schema.sql_identifier AS type_udt_name, + CASE + WHEN p.prokind <> 'p'::"char" THEN 0 + ELSE NULL::integer + END::information_schema.sql_identifier AS dtd_identifier, + CASE + WHEN l.lanname = 'sql'::name THEN 'SQL'::text + ELSE 'EXTERNAL'::text + END::information_schema.character_data AS routine_body, + CASE + WHEN pg_has_role(p.proowner, 'USAGE'::text) THEN p.prosrc + ELSE NULL::text + END::information_schema.character_data AS definition, + CASE + WHEN l.lanname = 'c'::name THEN p.prosrc + ELSE NULL::text + END::information_schema.character_data AS external_name, + upper(l.lanname::text)::information_schema.character_data AS external_language, + 'GENERAL'::character varying::information_schema.character_data AS parameter_style, + CASE + WHEN p.provolatile = 'i'::"char" THEN 'YES'::text + ELSE 'NO'::text + END::information_schema.yes_or_no AS is_deterministic, + 'MODIFIES'::character varying::information_schema.character_data AS sql_data_access, + CASE + WHEN p.prokind <> 'p'::"char" THEN + CASE + WHEN p.proisstrict THEN 'YES'::text + ELSE 'NO'::text + END + ELSE NULL::text + END::information_schema.yes_or_no AS is_null_call, + 'YES'::character varying::information_schema.yes_or_no AS schema_level_routine, + 0::information_schema.cardinal_number AS max_dynamic_result_sets, + CASE + WHEN p.prosecdef THEN 'DEFINER'::text + ELSE 'INVOKER'::text + END::information_schema.character_data AS security_type, + 'NO'::character varying::information_schema.yes_or_no AS as_locator, + 'NO'::character varying::information_schema.yes_or_no AS is_udt_dependent, + p.p_oid as oid, + p.proisstrict, + p.prosecdef, + p.provolatile, + p.proargtypes, + p.proallargtypes, + p.proargnames, + p.proargdefaults, + p.proargmodes, + p.proowner, + p.prokind as kind + FROM pg_namespace n + JOIN pg_proc_pre p ON n.oid = p.pronamespace + JOIN pg_language l ON p.prolang = l.oid + LEFT JOIN (pg_type t + JOIN pg_namespace nt ON t.typnamespace = nt.oid) ON p.prorettype = t.oid AND p.prokind <> 'p'::"char" + WHERE pg_has_role(p.proowner, 'USAGE'::text) OR has_function_privilege(p.p_oid, 'EXECUTE'::text) + +), pgproc as ( select - nspname as schema, - proname as name, + schema, + name, p.oid as oid, + e.objid as extension_oid, case proisstrict when true then 'RETURNS NULL ON NULL INPUT' else @@ -47,51 +130,27 @@ with r1 as ( p.proargmodes, p.proowner, COALESCE(p.proallargtypes, p.proargtypes::oid[]) as procombinedargtypes, - -- 11_AND_LATER p.prokind as kind - -- 10_AND_EARLIER case when p.proisagg then 'a' else 'f' end as kind + p.kind, + p.type_udt_schema, + p.type_udt_name, + p.definition, + p.external_language + from - pg_proc p - INNER JOIN pg_namespace n - ON n.oid=p.pronamespace + routines p + left outer join extension_oids e + on p.oid = e.objid where true - -- 11_AND_LATER and p.prokind != 'a' - -- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') - -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' - ), - extension_oids as ( - select - objid - from - pg_depend d - WHERE - d.refclassid = 'pg_extension'::regclass - and d.classid = 'pg_proc'::regclass - ), - r as ( - select - r1.*, - pp.volatility, - pp.strictness, - pp.security_type, - pp.oid, - pp.kind, - e.objid as extension_oid - from r1 - left outer join pgproc pp - on r1.schema = pp.schema - and r1.specific_name = pp.name || '_' || pp.oid - left outer join extension_oids e - on pp.oid = e.objid - -- SKIP_INTERNAL where e.objid is null + -- 11_AND_LATER and p.kind != 'a' + -- SKIP_INTERNAL and schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') + -- SKIP_INTERNAL and schema not like 'pg_temp_%' and schema not like 'pg_toast_temp_%' + -- SKIP_INTERNAL and e.objid is null + -- SKIP_INTERNAL and p.external_language not in ('C', 'INTERNAL') ), unnested as ( select - p.oid as p_oid, - --schema, - --name, + p.*, pname as parameter_name, - --pdatatype, - --pargtype as data_type, pnum as position_number, CASE WHEN pargmode IS NULL THEN null @@ -124,40 +183,33 @@ unnested as ( ), pre as ( SELECT - r.schema as schema, - r.name as name, - case when r.data_type = 'USER-DEFINED' then - '"' || r.type_udt_schema || '"."' || r.type_udt_name || '"' + p.schema as schema, + p.name as name, + case when p.data_type = 'USER-DEFINED' then + '"' || p.type_udt_schema || '"."' || p.type_udt_name || '"' else - r.data_type + p.data_type end as returntype, - r.data_type = 'USER-DEFINED' as has_user_defined_returntype, + p.data_type = 'USER-DEFINED' as has_user_defined_returntype, p.parameter_name as parameter_name, p.data_type as data_type, p.parameter_mode as parameter_mode, p.parameter_default as parameter_default, p.position_number as position_number, - r.definition as definition, - pg_get_functiondef(r.oid) as full_definition, - r.external_language as language, - r.strictness as strictness, - r.security_type as security_type, - r.volatility as volatility, - r.kind as kind, - r.oid as oid, - r.extension_oid as extension_oid, - pg_get_function_result(r.oid) as result_string, - pg_get_function_identity_arguments(r.oid) as identity_arguments, - pg_catalog.obj_description(r.oid) as comment - FROM r - left join unnested p - on r.oid = p.p_oid - -- LEFT JOIN information_schema.parameters p ON - -- r.specific_name=p.specific_name - - - order by - name, parameter_mode, position_number, parameter_name + p.definition as definition, + pg_get_functiondef(p.oid) as full_definition, + p.external_language as language, + p.strictness as strictness, + p.security_type as security_type, + p.volatility as volatility, + p.kind as kind, + p.oid as oid, + p.extension_oid as extension_oid, + pg_get_function_result(p.oid) as result_string, + pg_get_function_identity_arguments(p.oid) as identity_arguments, + pg_catalog.obj_description(p.oid) as comment + FROM + unnested p ) select * diff --git a/schemainspect/pg/sql/indexes.sql b/schemainspect/pg/sql/indexes.sql index cc8b579..be76492 100644 --- a/schemainspect/pg/sql/indexes.sql +++ b/schemainspect/pg/sql/indexes.sql @@ -1,21 +1,52 @@ with extension_oids as ( + select + objid, + classid::regclass::text as classid + from + pg_depend d + WHERE + d.refclassid = 'pg_extension'::regclass and + d.classid = 'pg_index'::regclass +), +extension_relations as ( select objid from pg_depend d WHERE d.refclassid = 'pg_extension'::regclass and - d.classid = 'pg_index'::regclass -) SELECT n.nspname AS schema, + d.classid = 'pg_class'::regclass +), pre as ( + SELECT n.nspname AS schema, c.relname AS table_name, i.relname AS name, i.oid as oid, e.objid as extension_oid, pg_get_indexdef(i.oid) AS definition, - (select string_agg(attname, ' ' order by attname) from pg_attribute where attnum = any(string_to_array(x.indkey::text, ' ')::int[]) and attrelid = x.indrelid) key_columns, - indoption key_options, indnatts num_att, indisunique is_unique, - indisprimary is_pk, indisexclusion is_exclusion, indimmediate is_immediate, - indisclustered is_clustered, indcollation key_collations, + ( + select + array_agg(attname order by ik.n) + from + unnest(x.indkey) with ordinality ik(i, n) + join pg_attribute aa + on + aa.attrelid = x.indrelid + and ik.i = aa.attnum + ) + index_columns, + indoption key_options, + indnatts total_column_count, + -- 11_AND_LATER indnkeyatts key_column_count, + -- 10_AND_EARLIER indnatts key_column_count, + indnatts num_att, + -- 11_AND_LATER indnatts - indnkeyatts included_column_count, + -- 10_AND_EARLIER 0 included_column_count, + indisunique is_unique, + indisprimary is_pk, + indisexclusion is_exclusion, + indimmediate is_immediate, + indisclustered is_clustered, + indcollation key_collations, pg_get_expr(indexprs, indrelid) key_expressions, pg_get_expr(indpred, indrelid) partial_predicate, amname algorithm @@ -24,10 +55,19 @@ with extension_oids as ( JOIN pg_class i ON i.oid = x.indexrelid JOIN pg_am am ON i.relam = am.oid LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - left join extension_oids e - on c.oid = e.objid or i.oid = e.objid - WHERE c.relkind in ('r', 'm', 'p') AND i.relkind in ('i', 'I') + left join extension_oids e + on i.oid = e.objid + left join extension_relations er + on c.oid = er.objid +WHERE + x.indislive + and c.relkind in ('r', 'm', 'p') AND i.relkind in ('i', 'I') -- SKIP_INTERNAL and nspname not in ('pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' - -- SKIP_INTERNAL and e.objid is null + -- SKIP_INTERNAL and e.objid is null and er.objid is null +) +select * , +index_columns[1\:key_column_count] as key_columns, +index_columns[key_column_count+1\:array_length(index_columns, 1)] as included_columns +from pre order by 1, 2, 3; diff --git a/schemainspect/pg/sql/schemas.sql b/schemainspect/pg/sql/schemas.sql index ffc5f68..32701a1 100644 --- a/schemainspect/pg/sql/schemas.sql +++ b/schemainspect/pg/sql/schemas.sql @@ -1,7 +1,18 @@ -select +with extension_oids as ( + select + objid + from + pg_depend d + WHERE + d.refclassid = 'pg_extension'::regclass + and d.classid = 'pg_namespace'::regclass +) select nspname as schema from pg_catalog.pg_namespace + left outer join extension_oids e + on e.objid = oid -- SKIP_INTERNAL where nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' +-- SKIP_INTERNAL and e.objid is null order by 1; diff --git a/tests/conftest.py b/tests/conftest.py index 45b6e0c..e1cc3d8 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -6,3 +6,22 @@ def db(): with temporary_database(host="localhost") as dburi: yield dburi + + +def pytest_addoption(parser): + parser.addoption( + "--timescale", action="store_true", help="Test with Timescale extension" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "timescale: mark timescale specific tests") + + +def pytest_collection_modifyitems(config, items): + skip_timescale = pytest.mark.skip(reason="need --timescale option to run") + if not config.getoption("--timescale", default=False): + # no option given in cli: skip the timescale tests + for item in items: + if "timescale" in item.keywords: + item.add_marker(skip_timescale) diff --git a/tests/test_all.py b/tests/test_all.py index cd58a0a..88b8f18 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -148,7 +148,9 @@ def test_postgres_objects(): name="name", schema="schema", table_name="table", - key_columns="y", + key_columns=["y"], + index_columns=["y"], + included_columns=[], key_options="0", num_att="1", is_unique=False, @@ -324,7 +326,7 @@ def n(name, schema="public"): return quoted_identifier(name, schema=schema) -def asserts_pg(i): +def asserts_pg(i, has_timescale=False): # schemas assert list(i.schemas.keys()) == ["otherschema", "public"] otherschema = i.schemas["otherschema"] @@ -405,10 +407,13 @@ def asserts_pg(i): assert f2.comment is None # extensions - assert [e.quoted_full_name for e in i.extensions.values()] == [ + ext = [ n("plpgsql", schema="pg_catalog"), n("pg_trgm"), ] + if has_timescale: + ext.append(n("timescaledb")) + assert [e.quoted_full_name for e in i.extensions.values()] == ext # constraints cons = i.constraints['"public"."films"."firstkey"'] @@ -613,11 +618,25 @@ def test_sequences(db): assert owned.quoted_table_and_column_name == '"public"."t"."id"' -def test_postgres_inspect(db): +def test_postgres_inspect(db, pytestconfig): + if pytestconfig.getoption("timescale"): + pytest.skip("--timescale was specified") + else: + assert_postgres_inspect(db) + + +@pytest.mark.timescale +def test_timescale_inspect(db): + assert_postgres_inspect(db, has_timescale=True) + + +def assert_postgres_inspect(db, has_timescale=False): with S(db) as s: + if has_timescale: + s.execute("create extension if not exists timescaledb;") setup_pg_schema(s) i = get_inspector(s) - asserts_pg(i) + asserts_pg(i, has_timescale) assert i == i == get_inspector(s) diff --git a/tests/test_deps_fk.py b/tests/test_deps_fk.py index 1f8f763..ec5e2e4 100644 --- a/tests/test_deps_fk.py +++ b/tests/test_deps_fk.py @@ -28,25 +28,26 @@ """ CREATES_FK = """ +create schema other; create type emptype as enum('a', 'b', 'c'); -CREATE TABLE emp ( +CREATE TABLE other.emp ( id bigint primary key, empname text, category emptype ); create table salary ( - emp_id bigint unique references emp(id), + emp_id bigint unique references other.emp(id), salary bigint not null ); create view empview as ( select * - from emp + from other.emp join salary on emp.id = salary.emp_id ); @@ -64,13 +65,9 @@ END; $emp_stamp$ LANGUAGE plpgsql; -CREATE TRIGGER emp_stamp BEFORE INSERT OR UPDATE ON emp +CREATE TRIGGER emp_stamp BEFORE INSERT OR UPDATE ON other.emp FOR EACH ROW EXECUTE FUNCTION emp(); - - - - """ @@ -109,3 +106,20 @@ def test_dep_order(db): create = thing.create_statement s.execute(create) + + +def test_fk_info(db): + with S(db) as s: + i = get_inspector(s) + + if i.pg_version <= 10: + return + + s.execute(CREATES_FK) + + i = get_inspector(s) + + fk = i.constraints['"public"."salary"."salary_emp_id_fkey"'] + + assert fk.is_fk is True + assert fk.quoted_full_foreign_table_name == '"other"."emp"' diff --git a/tests/test_excludeschema.py b/tests/test_excludeschema.py new file mode 100644 index 0000000..dca6497 --- /dev/null +++ b/tests/test_excludeschema.py @@ -0,0 +1,32 @@ +from sqlbag import S + +from schemainspect import get_inspector + +from .test_all import setup_pg_schema + + +def asserts_pg_excludedschema(i, schema_names, excludedschema_name): + schemas = set() + for ( + prop + ) in "schemas relations tables views functions selectables sequences enums constraints".split(): + att = getattr(i, prop) + for k, v in att.items(): + assert v.schema != excludedschema_name + schemas.add(v.schema) + assert schemas == set(schema_names) + + +def test_postgres_inspect_excludeschema(db): + with S(db) as s: + setup_pg_schema(s) + s.execute("create schema thirdschema;") + s.execute("create schema forthschema;") + i = get_inspector(s, exclude_schema="otherschema") + asserts_pg_excludedschema( + i, ["public", "forthschema", "thirdschema"], "otherschema" + ) + i = get_inspector(s, exclude_schema="forthschema") + asserts_pg_excludedschema( + i, ["public", "otherschema", "thirdschema"], "forthschema" + ) diff --git a/tests/test_funcs.py b/tests/test_funcs.py index b507589..c8cb38e 100644 --- a/tests/test_funcs.py +++ b/tests/test_funcs.py @@ -45,3 +45,23 @@ def test_kinds(db): p.drop_statement == 'drop procedure if exists "public"."proc"(a integer, b integer);' ) + + +def test_long_identifiers(db): + with S(db) as s: + + for i in range(50, 70): + ident = "x" * i + truncated = "x" * min(i, 63) + + func = FUNC.replace("ordinary_f", ident) + + s.execute(func) + + i = get_inspector(s) + + expected_sig = '"public"."{}"(t text)'.format(truncated) + + f = i.functions[expected_sig] + + assert f.full_definition is not None diff --git a/tests/test_indexes.py b/tests/test_indexes.py index 1a58c8e..5ccbb6c 100644 --- a/tests/test_indexes.py +++ b/tests/test_indexes.py @@ -53,3 +53,60 @@ def test_constraints(db): indexes_keys = list(i.indexes.keys()) assert indexes_keys == ['"public"."t_pkey"'] + + +INDEX_DEFS = """ + +create schema s; + +CREATE TABLE s.t ( + id uuid NOT NULL, + a int4 NULL, + b int4 NULL, + CONSTRAINT pk PRIMARY KEY (id) +); + +CREATE UNIQUE INDEX i ON s.t USING btree (a); + +CREATE UNIQUE INDEX iii ON s.t USING btree (b, a) include (id); + +CREATE UNIQUE INDEX iii_exp ON s.t((lower(id::text))); + +""" + + +def test_index_defs(db): + with S(db) as s: + ii = get_inspector(s) + + if ii.pg_version <= 10: + return + s.execute(INDEX_DEFS) + + ii = get_inspector(s) + + indexes_keys = list(ii.indexes.keys()) + + EXPECTED = ['"s"."i"', '"s"."iii"', '"s"."iii_exp"', '"s"."pk"'] + assert indexes_keys == EXPECTED + + i = ii.indexes['"s"."i"'] + + assert i.index_columns == ["a"] + assert i.key_columns == ["a"] + assert i.included_columns == [] + assert i.key_expressions is None + + i = ii.indexes['"s"."iii"'] + + assert i.index_columns == ["b", "a", "id"] + assert i.key_columns == ["b", "a"] + assert i.included_columns == ["id"] + assert i.key_expressions is None + + i = ii.indexes['"s"."iii_exp"'] + + assert i.index_columns is None + assert i.key_columns is None + assert i.included_columns is None + assert i.key_expressions == "lower((id)::text)" diff --git a/tests/test_inheritance.py b/tests/test_inheritance.py new file mode 100644 index 0000000..020ce79 --- /dev/null +++ b/tests/test_inheritance.py @@ -0,0 +1,67 @@ +from sqlbag import S + +from schemainspect import get_inspector + +INHERITANCE = """ + +create table normal (id integer); + +create table parent (t timestamp); + +create table child (a integer, b integer) inherits (parent); +""" + + +def test_inheritance(db): + with S(db) as s: + s.execute(INHERITANCE) + + ii = get_inspector(s) + + normal = ii.tables['"public"."normal"'] + parent = ii.tables['"public"."parent"'] + child = ii.tables['"public"."child"'] + + assert list(normal.columns) == ["id"] + assert list(parent.columns) == ["t"] + assert list(child.columns) == "t a b".split() + + assert normal.columns["id"].is_inherited is False + assert parent.columns["t"].is_inherited is False + + if ii.pg_version <= 9: + return + + # uncertain why this fails on <=pg9 only + assert child.columns["t"].is_inherited is True + + for c in "a b".split(): + child.columns[c].is_inherited is False + + assert parent.dependents == ['"public"."child"'] + assert child.dependent_on == ['"public"."parent"'] + + +def test_table_dependency_order(db): + with S(db) as s: + i = get_inspector(s) + + if i.pg_version <= 9: + return + s.execute(INHERITANCE) + + ii = get_inspector(s) + + dep_order = ii.dependency_order() + + assert list(ii.tables.keys()) == [ + '"public"."child"', + '"public"."normal"', + '"public"."parent"', + ] + + assert dep_order == [ + '"public"."parent"', + '"public"."normal"', + '"public"."child"', + ]