From 578a04622edd63997e9524d6faa8bc90a9e9bc6f Mon Sep 17 00:00:00 2001 From: mgbiotech <84165050+mgbiotech@users.noreply.github.com> Date: Fri, 14 May 2021 12:53:51 +1000 Subject: [PATCH 01/26] Include VIEWS in privileges (aka role, permission) Currently only table privileges are handled --- schemainspect/pg/sql/privileges.sql | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index f2198e0..398b044 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -1,16 +1,18 @@ select - table_schema as schema, - table_name as name, - 'table' as object_type, - grantee as user, - privilege_type as privilege -from information_schema.role_table_grants -where grantee != ( - select tableowner - from pg_tables - where schemaname = table_schema - and tablename = table_name -) + table_schema as schema, + table_name as name, + 'table' as object_type, + grantee as "user", + privilege_type as privilege +from + information_schema.role_table_grants r + left join pg_tables t + on t.schemaname = r.table_schema and t.tablename = r.table_name + left join pg_views v + on v.schemaname = r.table_schema and v.viewname = r.table_name +where + r.grantee is distinct from t.tableowner + and r.grantee is distinct from v.viewowner -- SKIP_INTERNAL and table_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and table_schema not like 'pg_temp_%' and table_schema not like 'pg_toast_temp_%' order by schema, name, user; From bccde5d643850f2d8e0e3e3fbc274c172532cc77 Mon Sep 17 00:00:00 2001 From: mgbiotech <84165050+mgbiotech@users.noreply.github.com> Date: Fri, 14 May 2021 13:14:40 +1000 Subject: [PATCH 02/26] Update privileges.sql --- schemainspect/pg/sql/privileges.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 398b044..9e52844 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -7,12 +7,13 @@ select from information_schema.role_table_grants r left join pg_tables t - on t.schemaname = r.table_schema and t.tablename = r.table_name + on t.schemaname = r.table_schema and t.tablename = r.table_name left join pg_views v - on v.schemaname = r.table_schema and v.viewname = r.table_name + on v.schemaname = r.table_schema and v.viewname = r.table_name where r.grantee is distinct from t.tableowner and r.grantee is distinct from v.viewowner -- SKIP_INTERNAL and table_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and table_schema not like 'pg_temp_%' and table_schema not like 'pg_toast_temp_%' order by schema, name, user; + From e32ab287b8caeb17fa83e5afbb6ea60efbd5148c Mon Sep 17 00:00:00 2001 From: mgbiotech <84165050+mgbiotech@users.noreply.github.com> Date: Fri, 14 May 2021 19:02:06 +1000 Subject: [PATCH 03/26] Also include SEQUENCE & COLUMN privileges --- schemainspect/pg/sql/privileges.sql | 40 +++++++++++++++++------------ 1 file changed, 24 insertions(+), 16 deletions(-) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 9e52844..6fcb3d6 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -1,19 +1,27 @@ select - table_schema as schema, - table_name as name, - 'table' as object_type, - grantee as "user", - privilege_type as privilege + n.nspname as schema, + c.relname as name, + case + when c.relkind in ('r', 'v', 'm', 'f', 'p') then 'table' + when c.relkind = 'S' then 'sequence' + else null end as object_type, + pg_get_userbyid(acl.grantee) as "user", + acl.privilege from - information_schema.role_table_grants r - left join pg_tables t - on t.schemaname = r.table_schema and t.tablename = r.table_name - left join pg_views v - on v.schemaname = r.table_schema and v.viewname = r.table_name + pg_catalog.pg_class c + join pg_catalog.pg_namespace n + on n.oid = c.relnamespace, + lateral (select aclx.*, privilege_type as privilege + from aclexplode(c.relacl) aclx + union + select aclx.*, privilege_type || '(' || a.attname || ')' as privilege + from + pg_catalog.pg_attribute a + cross join aclexplode(a.attacl) aclx + where attrelid = c.oid and not attisdropped and attacl is not null ) acl where - r.grantee is distinct from t.tableowner - and r.grantee is distinct from v.viewowner --- SKIP_INTERNAL and table_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') --- SKIP_INTERNAL and table_schema not like 'pg_temp_%' and table_schema not like 'pg_toast_temp_%' -order by schema, name, user; - + acl.grantee != acl.grantor + and c.relkind in ('r', 'v', 'm', 'S', 'f', 'p') +-- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') +-- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' +order by schema, name, "user"; From 8816e8c3087ec8b34f1783880c648c6b8049bade Mon Sep 17 00:00:00 2001 From: mgbiotech <84165050+mgbiotech@users.noreply.github.com> Date: Fri, 14 May 2021 20:09:00 +1000 Subject: [PATCH 04/26] Add support for function priveleges --- schemainspect/pg/sql/privileges.sql | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 6fcb3d6..3cdfd51 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -22,6 +22,19 @@ from where acl.grantee != acl.grantor and c.relkind in ('r', 'v', 'm', 'S', 'f', 'p') --- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') --- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' +-- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') +-- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' +union +select + routine_schema as schema, + routine_name as name, + 'function' as object_type, + grantee as "user", + privilege_type as privilege +from information_schema.role_routine_grants +where + grantor != grantee + and grantee != 'PUBLIC' +-- SKIP_INTERNAL and routine_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') +-- SKIP_INTERNAL and routine_schema not like 'pg_temp_%' and routine_schema not like 'pg_toast_temp_%' order by schema, name, "user"; From e6ba49ca8a3065402b9874e1c490461a895f3ab0 Mon Sep 17 00:00:00 2001 From: biodevc <84165050+biodevc@users.noreply.github.com> Date: Thu, 28 Oct 2021 14:37:35 +1100 Subject: [PATCH 05/26] Add support for schema priveleges As per @jld3103 --- schemainspect/pg/sql/privileges.sql | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 3cdfd51..060e0cf 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -37,4 +37,26 @@ where and grantee != 'PUBLIC' -- SKIP_INTERNAL and routine_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and routine_schema not like 'pg_temp_%' and routine_schema not like 'pg_toast_temp_%' +union +select + '' as schema, + n.nspname as name, + 'schema' as object_type, + pg_get_userbyid(acl.grantee) as "user", + privilege +from pg_catalog.pg_namespace n, + lateral (select aclx.*, privilege_type as privilege + from aclexplode(n.nspacl) aclx + union + select aclx.*, privilege_type || '(' || a.attname || ')' as privilege + from + pg_catalog.pg_attribute a + cross join aclexplode(a.attacl) aclx + where attrelid = n.oid and not attisdropped and attacl is not null ) acl +where + privilege != 'CREATE' + and acl.grantor != acl.grantee + and acl.grantee != 0 +-- SKIP_INTERNAL and n.nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') +-- SKIP_INTERNAL and n.nspname not like 'pg_temp_%' and n.nspname not like 'pg_toast_temp_%' order by schema, name, "user"; From d04ca6860f613f8750d9bb4c6dd86d91a7b92017 Mon Sep 17 00:00:00 2001 From: biodevc <84165050+biodevc@users.noreply.github.com> Date: Thu, 28 Oct 2021 14:39:04 +1100 Subject: [PATCH 06/26] Add support for schema privileges As per @jld3103 --- schemainspect/inspected.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schemainspect/inspected.py b/schemainspect/inspected.py index 8ee185b..64e554e 100644 --- a/schemainspect/inspected.py +++ b/schemainspect/inspected.py @@ -6,6 +6,8 @@ class Inspected(AutoRepr): @property def quoted_full_name(self): + if self.schema == '': + return quoted_identifier(self.name) return "{}.{}".format( quoted_identifier(self.schema), quoted_identifier(self.name) ) From e01a4eed2415df1beba72502fb374b571c7ff05f Mon Sep 17 00:00:00 2001 From: biodevc <84165050+biodevc@users.noreply.github.com> Date: Thu, 23 Jun 2022 10:15:25 +1000 Subject: [PATCH 07/26] Update privileges.sql --- schemainspect/pg/sql/privileges.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 060e0cf..7d06c34 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -21,6 +21,7 @@ from where attrelid = c.oid and not attisdropped and attacl is not null ) acl where acl.grantee != acl.grantor + and acl.grantee != 0 and c.relkind in ('r', 'v', 'm', 'S', 'f', 'p') -- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' From 3689c453607271b74967ced31c0f4b8d243af3b3 Mon Sep 17 00:00:00 2001 From: biodevc <84165050+biodevc@users.noreply.github.com> Date: Thu, 23 Jun 2022 10:52:00 +1000 Subject: [PATCH 08/26] Update inspected.py --- schemainspect/inspected.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/schemainspect/inspected.py b/schemainspect/inspected.py index 8a1eb5b..24d18aa 100644 --- a/schemainspect/inspected.py +++ b/schemainspect/inspected.py @@ -6,6 +6,8 @@ class Inspected(AutoRepr): @property def quoted_full_name(self): + if self.schema == "": + return quoted_identifier(self.name) return quoted_identifier(self.name, schema=self.schema) @property From 6402a3aaa54c405b50c312c2369de0bf1c06dde5 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 29 Jun 2022 08:39:21 +1000 Subject: [PATCH 09/26] pre-commit fixes --- pyproject.toml | 5 +++++ schemainspect/inspected.py | 2 +- schemainspect/pg/sql/privileges.sql | 12 ++++++------ 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 3fe0463..6918bd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -24,9 +24,14 @@ isort = "5.10.1" migra = "*" black = "22.3.0" toml = "*" +pre-commit = "*" [tool.poetry.scripts] schemainspect = 'schemainspect:do_command' [tool.isort] profile = "black" + +[build-system] +requires = ["poetry-core>=1.0.0"] +build-backend = "poetry.core.masonry.api" diff --git a/schemainspect/inspected.py b/schemainspect/inspected.py index 24d18aa..ea0feaa 100644 --- a/schemainspect/inspected.py +++ b/schemainspect/inspected.py @@ -7,7 +7,7 @@ class Inspected(AutoRepr): @property def quoted_full_name(self): if self.schema == "": - return quoted_identifier(self.name) + return quoted_identifier(self.name) return quoted_identifier(self.name, schema=self.schema) @property diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 7d06c34..afdbf58 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -1,11 +1,11 @@ select - n.nspname as schema, + n.nspname as schema, c.relname as name, case when c.relkind in ('r', 'v', 'm', 'f', 'p') then 'table' when c.relkind = 'S' then 'sequence' - else null end as object_type, - pg_get_userbyid(acl.grantee) as "user", + else null end as object_type, + pg_get_userbyid(acl.grantee) as "user", acl.privilege from pg_catalog.pg_class c @@ -21,7 +21,7 @@ from where attrelid = c.oid and not attisdropped and attacl is not null ) acl where acl.grantee != acl.grantor - and acl.grantee != 0 + and acl.grantee != 0 and c.relkind in ('r', 'v', 'm', 'S', 'f', 'p') -- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' @@ -39,7 +39,7 @@ where -- SKIP_INTERNAL and routine_schema not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and routine_schema not like 'pg_temp_%' and routine_schema not like 'pg_toast_temp_%' union -select +select '' as schema, n.nspname as name, 'schema' as object_type, @@ -54,7 +54,7 @@ from pg_catalog.pg_namespace n, pg_catalog.pg_attribute a cross join aclexplode(a.attacl) aclx where attrelid = n.oid and not attisdropped and attacl is not null ) acl -where +where privilege != 'CREATE' and acl.grantor != acl.grantee and acl.grantee != 0 From 1ffaf5864ada1875f5e898ad4726c105dec21d9f Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 6 Jul 2022 09:57:16 +1100 Subject: [PATCH 10/26] include defaults in create function sig --- schemainspect/pg/obj.py | 12 ++++++++++-- schemainspect/pg/sql/functions.sql | 1 + 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index fe99108..eb2c911 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -246,6 +246,7 @@ def __init__( strictness, security_type, identity_arguments, + function_arguments, result_string, language, full_definition, @@ -254,6 +255,7 @@ def __init__( kind, ): self.identity_arguments = identity_arguments + self.function_arguments = function_arguments self.result_string = result_string self.language = language self.volatility = volatility @@ -280,6 +282,10 @@ def returntype_is_table(self): @property def signature(self): + return "{}({})".format(self.quoted_full_name, self.function_arguments) + + @property + def identity_signature(self): return "{}({})".format(self.quoted_full_name, self.identity_arguments) @property @@ -304,11 +310,12 @@ def thing(self): @property def drop_statement(self): - return "drop {} if exists {};".format(self.thing, self.signature) + return "drop {} if exists {};".format(self.thing, self.identity_signature) def __eq__(self, other): return ( - self.signature == other.signature + self.identity_signature == other.identity_signature + and self.signature == other.signature and self.result_string == other.result_string and self.definition == other.definition and self.language == other.language @@ -1561,6 +1568,7 @@ def load_functions(self): columns=od((c.name, c) for c in columns), inputs=plist, identity_arguments=f.identity_arguments, + function_arguments=f.function_arguments, result_string=f.result_string, language=f.language, definition=f.definition, diff --git a/schemainspect/pg/sql/functions.sql b/schemainspect/pg/sql/functions.sql index 471de77..57ad9ab 100644 --- a/schemainspect/pg/sql/functions.sql +++ b/schemainspect/pg/sql/functions.sql @@ -207,6 +207,7 @@ unnested as ( p.extension_oid as extension_oid, pg_get_function_result(p.oid) as result_string, pg_get_function_identity_arguments(p.oid) as identity_arguments, + pg_get_function_arguments(p.oid) as function_arguments, pg_catalog.obj_description(p.oid) as comment FROM unnested p From 14a1d7a6a00dee203033da31461e619b31f2e3b3 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Fri, 29 Jul 2022 10:41:31 +1000 Subject: [PATCH 11/26] support for function privileges --- schemainspect/pg/obj.py | 20 ++++++++++++++++++-- schemainspect/pg/sql/privileges.sql | 12 ++++++++---- tests/test_all.py | 2 +- tests/test_privileges.py | 16 +++++++++------- 4 files changed, 36 insertions(+), 14 deletions(-) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index eb2c911..6132006 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -926,12 +926,20 @@ def __eq__(self, other): class InspectedPrivilege(Inspected): - def __init__(self, object_type, schema, name, privilege, target_user): + def __init__(self, object_type, schema, name, privilege, target_user, postfix): self.schema = schema self.object_type = object_type self.name = name self.privilege = privilege.lower() self.target_user = target_user + self.postfix = postfix + + @property + def quoted_full_name(self): + full_name = super().quoted_full_name + if self.postfix: + full_name += self.postfix + return full_name @property def quoted_target_user(self): @@ -962,12 +970,19 @@ def __eq__(self, other): self.name == other.name, self.privilege == other.privilege, self.target_user == other.target_user, + self.postfix == other.postfix, ) return all(equalities) @property def key(self): - return self.object_type, self.quoted_full_name, self.target_user, self.privilege + return ( + self.object_type, + self.quoted_full_name, + self.target_user, + self.privilege, + self.postfix, + ) RLS_POLICY_CREATE = """create policy {name} @@ -1191,6 +1206,7 @@ def load_privileges(self): name=i.name, privilege=i.privilege, target_user=i.user, + postfix=i.postfix, ) for i in q ] diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index afdbf58..2e04828 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -6,7 +6,8 @@ select when c.relkind = 'S' then 'sequence' else null end as object_type, pg_get_userbyid(acl.grantee) as "user", - acl.privilege + acl.privilege, + NULL as postfix from pg_catalog.pg_class c join pg_catalog.pg_namespace n @@ -31,8 +32,10 @@ select routine_name as name, 'function' as object_type, grantee as "user", - privilege_type as privilege -from information_schema.role_routine_grants + privilege_type as privilege, + FORMAT('(%s)', PG_GET_FUNCTION_IDENTITY_ARGUMENTS(oid)) as postfix + FROM information_schema.role_routine_grants g + JOIN pg_catalog.pg_proc p ON g.specific_schema::REGNAMESPACE = p.pronamespace::REGNAMESPACE AND g.specific_name = FORMAT('%s_%s', p.proname, p.oid) where grantor != grantee and grantee != 'PUBLIC' @@ -44,7 +47,8 @@ select n.nspname as name, 'schema' as object_type, pg_get_userbyid(acl.grantee) as "user", - privilege + privilege, + NULL as postfix from pg_catalog.pg_namespace n, lateral (select aclx.*, privilege_type as privilege from aclexplode(n.nspacl) aclx diff --git a/tests/test_all.py b/tests/test_all.py index 89c66f8..892a0df 100644 --- a/tests/test_all.py +++ b/tests/test_all.py @@ -456,7 +456,7 @@ def asserts_pg(i, has_timescale=False): assert n("films_title_idx") in t.indexes # privileges - g = InspectedPrivilege("table", "public", "films", "select", "postgres") + g = InspectedPrivilege("table", "public", "films", "select", "postgres", None) g = i.privileges[g.key] assert g.create_statement == 'grant select on table {} to "postgres";'.format( t_films diff --git a/tests/test_privileges.py b/tests/test_privileges.py index 2368758..da9b989 100644 --- a/tests/test_privileges.py +++ b/tests/test_privileges.py @@ -2,13 +2,15 @@ def test_inspected_privilege(): - a = InspectedPrivilege("table", "public", "test_table", "select", "test_user") - a2 = InspectedPrivilege("table", "public", "test_table", "select", "test_user") + a = InspectedPrivilege("table", "public", "test_table", "select", "test_user", None) + a2 = InspectedPrivilege( + "table", "public", "test_table", "select", "test_user", None + ) b = InspectedPrivilege( - "function", "schema", "test_function", "execute", "test_user" + "function", "schema", "test_function", "execute", "test_user", "(int,int)" ) b2 = InspectedPrivilege( - "function", "schema", "test_function", "modify", "test_user" + "function", "schema", "test_function", "modify", "test_user", "(int,int)" ) assert a == a2 assert a == a @@ -16,10 +18,10 @@ def test_inspected_privilege(): assert b != b2 assert ( b2.create_statement - == 'grant modify on function "schema"."test_function" to "test_user";' + == 'grant modify on function "schema"."test_function"(int,int) to "test_user";' ) assert ( b.drop_statement - == 'revoke execute on function "schema"."test_function" from "test_user";' + == 'revoke execute on function "schema"."test_function"(int,int) from "test_user";' ) - assert a.key == ("table", '"public"."test_table"', "test_user", "select") + assert a.key == ("table", '"public"."test_table"', "test_user", "select", None) From 9d79b0d8ade2149dd36686f5567d5f3edaf46755 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Tue, 4 Oct 2022 10:42:22 +1100 Subject: [PATCH 12/26] add returntype to func comparison --- schemainspect/pg/obj.py | 1 + 1 file changed, 1 insertion(+) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index cfa06a9..a168842 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -316,6 +316,7 @@ def __eq__(self, other): self.identity_signature == other.identity_signature and self.signature == other.signature and self.result_string == other.result_string + and self.returntype == other.returntype and self.definition == other.definition and self.language == other.language and self.volatility == other.volatility From 2df4dfdae65e2ab5ddfe9e2dc1d5037bed830220 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 12 Apr 2023 16:42:50 +1000 Subject: [PATCH 13/26] handle comments and array dependencies --- pyproject.toml | 2 +- schemainspect/pg/obj.py | 57 +++++++++++++++++++++++++++++++ schemainspect/pg/sql/comments.sql | 45 ++++++++++++++++++++++++ schemainspect/pg/sql/deps.sql | 28 ++++++++++++++- 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 schemainspect/pg/sql/comments.sql diff --git a/pyproject.toml b/pyproject.toml index 6918bd6..c5b730a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ repository = "https://github.com/djrobstep/schemainspect" homepage = "https://github.com/djrobstep/schemainspect" [tool.poetry.dependencies] -python = ">=3.7,<4" +python = ">=3.9,<4" sqlalchemy = "*" [tool.poetry.dev-dependencies] diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index a168842..09a76ea 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -34,6 +34,7 @@ COLLATIONS_QUERY = resource_text("sql/collations.sql") COLLATIONS_QUERY_9 = resource_text("sql/collations9.sql") RLSPOLICIES_QUERY = resource_text("sql/rlspolicies.sql") +COMMENTS_QUERY = resource_text("sql/comments.sql") class InspectedSelectable(BaseInspectedSelectable): @@ -1017,6 +1018,54 @@ def key(self): ) +class InspectedComment(Inspected): + def __init__( + self, + object_type: str, + object_addr: list[str], + object_args: list[str], + comment, + create_statement, + drop_statement, + ): + self.object_type = object_type + self.object_addr = tuple(object_addr) + self.object_args = tuple(object_args) + self.comment = comment + self._create_statement = create_statement + self._drop_statement = drop_statement + self.name = object_addr[-1] + self.schema = object_addr[0] + + @property + def quoted_full_name(self): + if self.object_type == "type": + return + return "{}.{}".format( + quoted_identifier(self.schema), quoted_identifier(self.name) + ) + + @property + def drop_statement(self): + return self._drop_statement + + @property + def create_statement(self): + return self._create_statement + + def __eq__(self, other): + equalities = ( + self.object_type == other.object_type, + self.object_addr == other.object_addr, + self.object_args == other.object_args, + ) + return all(equalities) + + @property + def key(self): + return self.object_type, self.object_addr, self.object_args + + RLS_POLICY_CREATE = """create policy {name} on {table_name} as {permissiveness} @@ -1157,6 +1206,7 @@ def processed(q): self.SCHEMAS_QUERY = processed(SCHEMAS_QUERY) self.PRIVILEGES_QUERY = processed(PRIVILEGES_QUERY) self.TRIGGERS_QUERY = processed(TRIGGERS_QUERY) + self.COMMENTS_QUERY = processed(COMMENTS_QUERY) super(PostgreSQL, self).__init__(c, include_internal) @@ -1187,6 +1237,13 @@ def load_all(self): self.load_deps() self.load_deps_all() + self.load_comments() + + def load_comments(self): + q = self.execute(self.COMMENTS_QUERY) + comments = [InspectedComment(**each) for each in q] + self.comments = od((comment.key, comment) for comment in comments) + def load_schemas(self): q = self.execute(self.SCHEMAS_QUERY) schemas = [InspectedSchema(schema=each.schema) for each in q] diff --git a/schemainspect/pg/sql/comments.sql b/schemainspect/pg/sql/comments.sql new file mode 100644 index 0000000..1bca72b --- /dev/null +++ b/schemainspect/pg/sql/comments.sql @@ -0,0 +1,45 @@ +SELECT obj_address.type AS object_type + , obj_address.object_names AS object_addr + , obj_address.object_args AS object_args + , d.description AS comment + , 'comment on ' || pg_describe_object(d.classoid, d.objoid, d.objsubid) AS object_description + , CASE obj_address.type + WHEN 'function' + THEN 'COMMENT ON FUNCTION ' || ARRAY_TO_STRING( + (SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || '(' || + ARRAY_TO_STRING(object_args, ', ') || ') IS ' || QUOTE_LITERAL(d.description) || ';' + WHEN 'table column' + THEN 'COMMENT ON COLUMN ' || + ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || + ' IS ' || QUOTE_LITERAL(d.description) || ';' + WHEN 'type' + THEN 'COMMENT ON TYPE ' || + (SELECT QUOTE_IDENT(typnamespace::REGNAMESPACE::TEXT) || '.' || QUOTE_IDENT(typname) + FROM pg_type + WHERE oid = obj_address.object_names[1]::REGTYPE) || ' IS ' || QUOTE_LITERAL(d.description) || ';' + ELSE 'COMMENT ON ' || UPPER(obj_address.type) || ' ' || + ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || + ' IS ' || QUOTE_LITERAL(d.description) || ';' + END AS create_statement + , CASE obj_address.type + WHEN 'function' + THEN 'COMMENT ON FUNCTION ' || + ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || + '(' || ARRAY_TO_STRING(object_args, ', ') || ') IS NULL;' + WHEN 'table column' + THEN 'COMMENT ON COLUMN ' || + ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || + ' IS NULL;' + WHEN 'type' + THEN 'COMMENT ON TYPE ' || + (SELECT QUOTE_IDENT(typnamespace::REGNAMESPACE::TEXT) || '.' || QUOTE_IDENT(typname) + FROM pg_type + WHERE oid = obj_address.object_names[1]::REGTYPE) || ' IS NULL;' + ELSE 'COMMENT ON ' || UPPER(obj_address.type) || ' ' || + ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || + ' IS NULL;' + END AS drop_statement +FROM pg_description d +JOIN LATERAL PG_IDENTIFY_OBJECT_AS_ADDRESS(d.classoid, d.objoid, d.objsubid) AS obj_address ON TRUE +WHERE obj_address.object_names[1] NOT LIKE 'pg_%' + AND obj_address.object_names[1] != 'information_schema'; diff --git a/schemainspect/pg/sql/deps.sql b/schemainspect/pg/sql/deps.sql index 09e8561..2682f5f 100644 --- a/schemainspect/pg/sql/deps.sql +++ b/schemainspect/pg/sql/deps.sql @@ -1,4 +1,3 @@ - with things1 as ( select oid as objid, @@ -55,6 +54,15 @@ things as ( and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' and extension_objids.extension_objid is null ), +array_dependencies as ( + select + att.attrelid as objid, + att.attname as column_name, + tbl.typrelid as objid_dependent_on + from pg_attribute att + join pg_type tbl on tbl.oid = att.atttypid + where tbl.typcategory = 'A' +), combined as ( select distinct t.objid, @@ -80,6 +88,24 @@ combined as ( d.deptype in ('n') and rw.rulename = '_RETURN' + union all + select + t.objid, + t.schema, + t.name, + t.identity_arguments, + t.kind, + things_dependent_on.objid as objid_dependent_on, + things_dependent_on.schema as schema_dependent_on, + things_dependent_on.name as name_dependent_on, + things_dependent_on.identity_arguments as identity_arguments_dependent_on, + things_dependent_on.kind as kind_dependent_on + FROM + array_dependencies ad + inner join things things_dependent_on + on ad.objid_dependent_on = things_dependent_on.objid + inner join things t + on ad.objid = t.objid ) select * from combined order by From a0bfc917070b4218cdc9656c49ff02d8f35dafb0 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 12 Apr 2023 17:25:10 +1000 Subject: [PATCH 14/26] fix comment --- schemainspect/pg/obj.py | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index 09a76ea..094b17f 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -1025,6 +1025,7 @@ def __init__( object_addr: list[str], object_args: list[str], comment, + object_description, create_statement, drop_statement, ): @@ -1032,6 +1033,7 @@ def __init__( self.object_addr = tuple(object_addr) self.object_args = tuple(object_args) self.comment = comment + self.object_description = object_description self._create_statement = create_statement self._drop_statement = drop_statement self.name = object_addr[-1] @@ -1039,11 +1041,7 @@ def __init__( @property def quoted_full_name(self): - if self.object_type == "type": - return - return "{}.{}".format( - quoted_identifier(self.schema), quoted_identifier(self.name) - ) + return self.object_description @property def drop_statement(self): @@ -1241,7 +1239,7 @@ def load_all(self): def load_comments(self): q = self.execute(self.COMMENTS_QUERY) - comments = [InspectedComment(**each) for each in q] + comments = [InspectedComment(*each) for each in q] self.comments = od((comment.key, comment) for comment in comments) def load_schemas(self): From 8fe1b389337537882f8aa983a297b5f3e188ee2f Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 12 Apr 2023 20:20:57 +1000 Subject: [PATCH 15/26] improve dependency query --- schemainspect/pg/sql/comments.sql | 30 +++++++++++++++--------------- schemainspect/pg/sql/deps.sql | 15 +++++++++++++-- 2 files changed, 28 insertions(+), 17 deletions(-) diff --git a/schemainspect/pg/sql/comments.sql b/schemainspect/pg/sql/comments.sql index 1bca72b..f5f5c8f 100644 --- a/schemainspect/pg/sql/comments.sql +++ b/schemainspect/pg/sql/comments.sql @@ -1,18 +1,18 @@ -SELECT obj_address.type AS object_type - , obj_address.object_names AS object_addr - , obj_address.object_args AS object_args - , d.description AS comment - , 'comment on ' || pg_describe_object(d.classoid, d.objoid, d.objsubid) AS object_description - , CASE obj_address.type - WHEN 'function' +SELECT obj_address.type AS object_type + , obj_address.object_names AS object_addr + , obj_address.object_args AS object_args + , d.description AS comment + , 'comment on ' || PG_DESCRIBE_OBJECT(d.classoid, d.objoid, d.objsubid) AS object_description + , CASE + WHEN obj_address.type = 'function' THEN 'COMMENT ON FUNCTION ' || ARRAY_TO_STRING( (SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || '(' || ARRAY_TO_STRING(object_args, ', ') || ') IS ' || QUOTE_LITERAL(d.description) || ';' - WHEN 'table column' + WHEN obj_address.type LIKE '% column' THEN 'COMMENT ON COLUMN ' || ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || ' IS ' || QUOTE_LITERAL(d.description) || ';' - WHEN 'type' + WHEN obj_address.type = 'type' THEN 'COMMENT ON TYPE ' || (SELECT QUOTE_IDENT(typnamespace::REGNAMESPACE::TEXT) || '.' || QUOTE_IDENT(typname) FROM pg_type @@ -20,17 +20,17 @@ SELECT obj_address.type AS object_type ELSE 'COMMENT ON ' || UPPER(obj_address.type) || ' ' || ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || ' IS ' || QUOTE_LITERAL(d.description) || ';' - END AS create_statement - , CASE obj_address.type - WHEN 'function' + END AS create_statement + , CASE + WHEN obj_address.type = 'function' THEN 'COMMENT ON FUNCTION ' || ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || '(' || ARRAY_TO_STRING(object_args, ', ') || ') IS NULL;' - WHEN 'table column' + WHEN obj_address.type LIKE '% column' THEN 'COMMENT ON COLUMN ' || ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || ' IS NULL;' - WHEN 'type' + WHEN obj_address.type = 'type' THEN 'COMMENT ON TYPE ' || (SELECT QUOTE_IDENT(typnamespace::REGNAMESPACE::TEXT) || '.' || QUOTE_IDENT(typname) FROM pg_type @@ -38,7 +38,7 @@ SELECT obj_address.type AS object_type ELSE 'COMMENT ON ' || UPPER(obj_address.type) || ' ' || ARRAY_TO_STRING((SELECT ARRAY_AGG(QUOTE_IDENT(o)) FROM UNNEST(obj_address.object_names) o), '.') || ' IS NULL;' - END AS drop_statement + END AS drop_statement FROM pg_description d JOIN LATERAL PG_IDENTIFY_OBJECT_AS_ADDRESS(d.classoid, d.objoid, d.objsubid) AS obj_address ON TRUE WHERE obj_address.object_names[1] NOT LIKE 'pg_%' diff --git a/schemainspect/pg/sql/deps.sql b/schemainspect/pg/sql/deps.sql index 2682f5f..dbd0e09 100644 --- a/schemainspect/pg/sql/deps.sql +++ b/schemainspect/pg/sql/deps.sql @@ -19,6 +19,15 @@ with things1 as ( where oid not in ( select ftrelid from pg_foreign_table ) + union + select + oid, + typnamespace as namespace, + typname as name, + null as identity_arguments, + 'c' as kind + from pg_type + where typrelid != 0 ), extension_objids as ( select @@ -58,9 +67,11 @@ array_dependencies as ( select att.attrelid as objid, att.attname as column_name, - tbl.typrelid as objid_dependent_on + tbl.typelem as composite_type_oid, + comp_tbl.typrelid as objid_dependent_on from pg_attribute att join pg_type tbl on tbl.oid = att.atttypid + join pg_type comp_tbl on tbl.typelem = comp_tbl.oid where tbl.typcategory = 'A' ), combined as ( @@ -110,4 +121,4 @@ combined as ( select * from combined order by schema, name, identity_arguments, kind_dependent_on, -schema_dependent_on, name_dependent_on, identity_arguments_dependent_on +schema_dependent_on, name_dependent_on, identity_arguments_dependent_on; From a434855feeb85411e2a8e7b231c72d404f3dddbd Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 12 Apr 2023 20:57:33 +1000 Subject: [PATCH 16/26] more dependency handling --- schemainspect/pg/sql/deps.sql | 42 +++++++++++++++++++++++++++-------- 1 file changed, 33 insertions(+), 9 deletions(-) diff --git a/schemainspect/pg/sql/deps.sql b/schemainspect/pg/sql/deps.sql index dbd0e09..5aa2d89 100644 --- a/schemainspect/pg/sql/deps.sql +++ b/schemainspect/pg/sql/deps.sql @@ -4,7 +4,8 @@ with things1 as ( pronamespace as namespace, proname as name, pg_get_function_identity_arguments(oid) as identity_arguments, - 'f' as kind + 'f' as kind, + null::oid as composite_type_oid from pg_proc -- 11_AND_LATER where pg_proc.prokind != 'a' -- 10_AND_EARLIER where pg_proc.proisagg is False @@ -14,7 +15,8 @@ with things1 as ( relnamespace as namespace, relname as name, null as identity_arguments, - relkind as kind + relkind as kind, + null::oid as composite_type_oid from pg_class where oid not in ( select ftrelid from pg_foreign_table @@ -25,7 +27,8 @@ with things1 as ( typnamespace as namespace, typname as name, null as identity_arguments, - 'c' as kind + 'c' as kind, + typrelid::oid as composite_type_oid from pg_type where typrelid != 0 ), @@ -51,7 +54,8 @@ things as ( kind, n.nspname as schema, name, - identity_arguments + identity_arguments, + t.composite_type_oid from things1 t inner join pg_namespace n on t.namespace = n.oid @@ -76,11 +80,11 @@ array_dependencies as ( ), combined as ( select distinct - t.objid, + coalesce(t.composite_type_oid, t.objid), t.schema, t.name, t.identity_arguments, - t.kind, + case when t.composite_type_oid is not null then 'r' ELSE t.kind end, things_dependent_on.objid as objid_dependent_on, things_dependent_on.schema as schema_dependent_on, things_dependent_on.name as name_dependent_on, @@ -100,12 +104,32 @@ combined as ( and rw.rulename = '_RETURN' union all + select distinct + coalesce(t.composite_type_oid, t.objid), + t.schema, + t.name, + t.identity_arguments, + case when t.composite_type_oid is not null then 'r' ELSE t.kind end, + things_dependent_on.objid as objid_dependent_on, + things_dependent_on.schema as schema_dependent_on, + things_dependent_on.name as name_dependent_on, + things_dependent_on.identity_arguments as identity_arguments_dependent_on, + things_dependent_on.kind as kind_dependent_on + FROM + pg_depend d + inner join things things_dependent_on + on d.refobjid = things_dependent_on.objid + inner join things t + on d.objid = t.objid + where + d.deptype in ('n') + union all select - t.objid, + coalesce(t.composite_type_oid, t.objid), t.schema, t.name, t.identity_arguments, - t.kind, + case when t.composite_type_oid is not null then 'r' ELSE t.kind end, things_dependent_on.objid as objid_dependent_on, things_dependent_on.schema as schema_dependent_on, things_dependent_on.name as name_dependent_on, @@ -121,4 +145,4 @@ combined as ( select * from combined order by schema, name, identity_arguments, kind_dependent_on, -schema_dependent_on, name_dependent_on, identity_arguments_dependent_on; +schema_dependent_on, name_dependent_on, identity_arguments_dependent_on From 0a0ca10de4d3264693e0e5a3e0a0d5f46b8e2e52 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Mon, 28 Aug 2023 11:11:59 +1000 Subject: [PATCH 17/26] Add exclusion of partition child tables in triggers.sql In the file 'triggers.sql', a subquery was added to exclude triggers related to partition child tables. The decision to exclude these triggers is based on the assumption that they inherit the triggers of their parent tables, so it is unnecessary to list them separately. This gives a clearer and more concise output from the triggers query, reducing redundancy and potential confusion. --- schemainspect/pg/sql/triggers.sql | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/schemainspect/pg/sql/triggers.sql b/schemainspect/pg/sql/triggers.sql index 11ed062..40ce1cf 100644 --- a/schemainspect/pg/sql/triggers.sql +++ b/schemainspect/pg/sql/triggers.sql @@ -6,6 +6,12 @@ with extension_oids as ( WHERE d.refclassid = 'pg_extension'::regclass and d.classid = 'pg_trigger'::regclass +), +partition_child_tables as ( + select + inhrelid + from + pg_inherits ) select tg.tgname "name", @@ -22,5 +28,6 @@ join pg_namespace nsp on nsp.oid = cls.relnamespace join pg_proc proc on proc.oid = tg.tgfoid join pg_namespace nspp on nspp.oid = proc.pronamespace where not tg.tgisinternal + and cls.oid not in (select * from partition_child_tables) -- Exclude partition child tables -- SKIP_INTERNAL and not tg.oid in (select * from extension_oids) order by schema, table_name, name; From 1fe185bf35790f45b1ded9f3eda96a317d99d897 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Thu, 7 Sep 2023 11:29:32 +1000 Subject: [PATCH 18/26] Add exclusion of partition child tables in privileges.sql The code is modified to exclude the partition child tables when checking for table privileges in the `privileges.sql` file. Earlier, the privileges check was including all the tables. With this change, the partitioned child tables, which generally do not require independent privileges, are excluded from the privileges check. This enhances the accuracy of the privilege validation process. --- schemainspect/pg/sql/privileges.sql | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/schemainspect/pg/sql/privileges.sql b/schemainspect/pg/sql/privileges.sql index 2e04828..43dc7ae 100644 --- a/schemainspect/pg/sql/privileges.sql +++ b/schemainspect/pg/sql/privileges.sql @@ -1,3 +1,9 @@ +WITH partition_child_tables as ( + select + inhrelid + from + pg_inherits +) select n.nspname as schema, c.relname as name, @@ -24,6 +30,8 @@ where acl.grantee != acl.grantor and acl.grantee != 0 and c.relkind in ('r', 'v', 'm', 'S', 'f', 'p') + -- and table is not a partition child table + and c.oid not in (select inhrelid from partition_child_tables) -- SKIP_INTERNAL and nspname not in ('pg_internal', 'pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and nspname not like 'pg_temp_%' and nspname not like 'pg_toast_temp_%' union From 51d47d64021818b5c5e8fc4f85f25aaa6df81571 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Thu, 7 Sep 2023 11:30:42 +1000 Subject: [PATCH 19/26] Add support for inspecting aggregate functions This commit adds new functionality to the codebase to allow inspection of aggregate database functions. The `InspectedAggFunction` class was added to `schemainspect/pg/obj.py`, with properties needed to create, describe, and delete aggregate functions. Moreover, `load_aggregate_functions` method was added to the `PostgreSQL` class in the same file to facilitate loading of these functions. Associated queries for getting this information are stored in `agg_functions.sql`. This was done to enhance the library's capabilities and cover more aspects of database inspecting. --- schemainspect/pg/obj.py | 118 ++++++++++++++++++++++++- schemainspect/pg/sql/agg_functions.sql | 28 ++++++ 2 files changed, 145 insertions(+), 1 deletion(-) create mode 100644 schemainspect/pg/sql/agg_functions.sql diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index 094b17f..ef4e0d4 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -24,6 +24,7 @@ SEQUENCES_QUERY = resource_text("sql/sequences.sql") CONSTRAINTS_QUERY = resource_text("sql/constraints.sql") FUNCTIONS_QUERY = resource_text("sql/functions.sql") +AGG_FUNCTIONS_QUERY = resource_text("sql/agg_functions.sql") TYPES_QUERY = resource_text("sql/types.sql") DOMAINS_QUERY = resource_text("sql/domains.sql") EXTENSIONS_QUERY = resource_text("sql/extensions.sql") @@ -326,6 +327,88 @@ def __eq__(self, other): and self.kind == other.kind ) +class InspectedAggFunction(InspectedSelectable): + def __init__(self, object_type, object_addr, object_args, schema, name, function_arguments, + function_identity_arguments, aggtransfn, aggfinalfn, aggmtransfn, aggmfinalfn, aggtransspace, + agginitval, aggminitval, state_type): + + self.object_type = object_type + self.object_addr = tuple(object_addr) + self.object_args = tuple(object_args) + self.schema = schema + self.name = name + self.function_arguments = function_arguments + self.function_identity_arguments = function_identity_arguments + + self.aggtransfn = aggtransfn + self.aggfinalfn = aggfinalfn + self.aggmtransfn = aggmtransfn + self.aggmfinalfn = aggmfinalfn + self.aggtransspace = aggtransspace + self.agginitval = agginitval + self.aggminitval = aggminitval + self.state_type = state_type + + super(InspectedAggFunction, self).__init__( + name=name, + schema=schema, + columns=None, + inputs=self.object_args, + definition="", + relationtype="a", + comment=None, + ) + + @property + def signature(self): + return "{}({})".format(self.quoted_full_name, self.function_arguments) + + @property + def identity_signature(self): + return "{}({})".format(self.quoted_full_name, self.function_identity_arguments) + + @property + def create_statement(self): + ddl = f"CREATE AGGREGATE {self.quoted_full_name} (" + + # Add arguments + ddl += f"{self.function_arguments}) (\n" + + # Add options + options = [] + if self.aggtransfn: + options.append(f" SFUNC = {self.aggtransfn}") + if self.aggfinalfn: + options.append(f" FINALFUNC = {self.aggfinalfn}") + if self.aggmtransfn: + options.append(f" MSFUNC = {self.aggmtransfn}") + if self.aggmfinalfn: + options.append(f" MFINALFUNC = {self.aggmfinalfn}") + if self.state_type: + options.append(f" STYPE = {self.state_type}") + if self.agginitval: + options.append(f" INITCOND = '{self.agginitval}'") + if self.aggminitval: + options.append(f" MINITCOND = '{self.aggminitval}'") + + ddl += ",\n".join(options) + ddl += "\n);" + + return ddl + + + @property + def drop_statement(self): + return "drop aggregate if exists {};".format(self.identity_signature) + + def __eq__(self, other): + return ( + self.object_type == other.object_type + and self.object_addr == other.object_addr + and self.object_args == other.object_args + ) + + class InspectedTrigger(Inspected): def __init__( @@ -1143,7 +1226,7 @@ def __eq__(self, other): return all(equalities) -PROPS = "schemas relations tables views functions selectables sequences constraints indexes enums extensions privileges collations triggers rlspolicies" +PROPS = "schemas relations tables views functions aggregate_functions selectables sequences constraints indexes enums extensions privileges collations triggers rlspolicies" class PostgreSQL(DBInspector): @@ -1196,6 +1279,7 @@ def processed(q): self.SEQUENCES_QUERY = processed(SEQUENCES_QUERY) self.CONSTRAINTS_QUERY = processed(CONSTRAINTS_QUERY) self.FUNCTIONS_QUERY = processed(FUNCTIONS_QUERY) + self.AGG_FUNCTIONS_QUERY = processed(AGG_FUNCTIONS_QUERY) self.TYPES_QUERY = processed(TYPES_QUERY) self.DOMAINS_QUERY = processed(DOMAINS_QUERY) self.EXTENSIONS_QUERY = processed(EXTENSIONS_QUERY) @@ -1220,10 +1304,12 @@ def load_all(self): self.load_schemas() self.load_all_relations() self.load_functions() + self.load_aggregate_functions() self.selectables = od() self.selectables.update(self.relations) self.selectables.update(self.composite_types) self.selectables.update(self.functions) + self.selectables.update(self.aggregate_functions) self.load_privileges() self.load_triggers() @@ -1695,6 +1781,35 @@ def load_functions(self): identity_arguments = "({})".format(s.identity_arguments) self.functions[s.quoted_full_name + identity_arguments] = s + def load_aggregate_functions(self): + q = self.execute(self.AGG_FUNCTIONS_QUERY) + agg_functions = [ + InspectedAggFunction( + object_type=i.object_type, + object_addr=i.object_addr, + object_args=i.object_args, + schema=i.schema, + name=i.name, + function_arguments=i.function_arguments, + function_identity_arguments=i.function_identity_arguments, + aggtransfn=i.aggtransfn, + aggfinalfn=i.aggfinalfn, + aggmtransfn=i.aggmtransfn, + aggmfinalfn=i.aggmfinalfn, + aggtransspace=i.aggtransspace, + agginitval=i.agginitval, + aggminitval=i.aggminitval, + state_type=i.state_type + ) + for i in q + ] + self.aggregate_functions = od( + (i.identity_signature , i) for i in agg_functions + ) + + + + def load_triggers(self): q = self.execute(self.TRIGGERS_QUERY) triggers = [ @@ -1842,6 +1957,7 @@ def __eq__(self, other): and self.constraints == other.constraints and self.extensions == other.extensions and self.functions == other.functions + and self.agg_functions == other.agg_functions and self.triggers == other.triggers and self.collations == other.collations and self.rlspolicies == other.rlspolicies diff --git a/schemainspect/pg/sql/agg_functions.sql b/schemainspect/pg/sql/agg_functions.sql new file mode 100644 index 0000000..c57afeb --- /dev/null +++ b/schemainspect/pg/sql/agg_functions.sql @@ -0,0 +1,28 @@ +SELECT obj_address.type AS object_type + , obj_address.object_names AS object_addr + , obj_address.object_args AS object_args + , nsp.nspname AS schema + , p.proname AS name + , pg_catalog.PG_GET_FUNCTION_ARGUMENTS(p.oid) AS function_arguments + , pg_catalog.pg_get_function_identity_arguments(p.oid) AS function_identity_arguments + , pg_catalog.PG_GET_FUNCTION_RESULT(p.oid) AS result_type + , NULLIF(a.aggtransfn::TEXT, '-')::REGPROC AS aggtransfn + , NULLIF(a.aggfinalfn::TEXT, '-')::REGPROC AS aggfinalfn + , NULLIF(a.aggmtransfn::TEXT, '-')::REGPROC AS aggmtransfn + , NULLIF(a.aggmfinalfn::TEXT, '-')::REGPROC AS aggmfinalfn + , a.aggtransspace AS aggtransspace + , a.aggmtransspace AS aggmtransspace + , a.agginitval AS agginitval + , a.aggminitval AS aggminitval + , a.aggkind AS aggkind + , a.aggnumdirectargs AS aggnumdirectargs + , tt.oid::REGTYPE AS state_type + , ttf.oid::REGTYPE AS final_type +FROM pg_catalog.pg_proc p +JOIN LATERAL PG_IDENTIFY_OBJECT_AS_ADDRESS('pg_proc'::REGCLASS, p.oid, 0) AS obj_address ON TRUE +JOIN pg_catalog.pg_namespace nsp ON p.pronamespace = nsp.oid +JOIN pg_catalog.pg_aggregate a ON a.aggfnoid = p.oid +LEFT JOIN pg_catalog.pg_type tt ON tt.oid = a.aggtranstype +LEFT JOIN pg_catalog.pg_type ttf ON ttf.oid = p.prorettype +WHERE nsp.nspname NOT IN ('pg_catalog', 'information_schema') +ORDER BY schema, name; From 79b3fbdb9c22252574578e8f83dcb7f85ebaf69a Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Fri, 15 Mar 2024 15:50:41 +1100 Subject: [PATCH 20/26] revert python version minimum change --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c5b730a..6918bd6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -10,7 +10,7 @@ repository = "https://github.com/djrobstep/schemainspect" homepage = "https://github.com/djrobstep/schemainspect" [tool.poetry.dependencies] -python = ">=3.9,<4" +python = ">=3.7,<4" sqlalchemy = "*" [tool.poetry.dev-dependencies] From a11324674ac759812f0627b4eebc46bdc8b5cf45 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Fri, 15 Mar 2024 16:46:27 +1100 Subject: [PATCH 21/26] update isort in .pre-commit-config.yaml to fix ci check --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0b5f9cd..baaec26 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -42,6 +42,6 @@ repos: hooks: - id: black - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 5.13.2 hooks: - id: isort From 044462de79b8c407ba180c1c1188dbfb9b600bcb Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Fri, 15 Mar 2024 16:50:47 +1100 Subject: [PATCH 22/26] run black --- schemainspect/pg/obj.py | 33 +++++++++++++++++++++------------ 1 file changed, 21 insertions(+), 12 deletions(-) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index ef4e0d4..ef2a6c1 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -327,10 +327,26 @@ def __eq__(self, other): and self.kind == other.kind ) + class InspectedAggFunction(InspectedSelectable): - def __init__(self, object_type, object_addr, object_args, schema, name, function_arguments, - function_identity_arguments, aggtransfn, aggfinalfn, aggmtransfn, aggmfinalfn, aggtransspace, - agginitval, aggminitval, state_type): + def __init__( + self, + object_type, + object_addr, + object_args, + schema, + name, + function_arguments, + function_identity_arguments, + aggtransfn, + aggfinalfn, + aggmtransfn, + aggmfinalfn, + aggtransspace, + agginitval, + aggminitval, + state_type, + ): self.object_type = object_type self.object_addr = tuple(object_addr) @@ -396,7 +412,6 @@ def create_statement(self): return ddl - @property def drop_statement(self): return "drop aggregate if exists {};".format(self.identity_signature) @@ -409,7 +424,6 @@ def __eq__(self, other): ) - class InspectedTrigger(Inspected): def __init__( self, name, schema, table_name, proc_schema, proc_name, enabled, full_definition @@ -1799,16 +1813,11 @@ def load_aggregate_functions(self): aggtransspace=i.aggtransspace, agginitval=i.agginitval, aggminitval=i.aggminitval, - state_type=i.state_type + state_type=i.state_type, ) for i in q ] - self.aggregate_functions = od( - (i.identity_signature , i) for i in agg_functions - ) - - - + self.aggregate_functions = od((i.identity_signature, i) for i in agg_functions) def load_triggers(self): q = self.execute(self.TRIGGERS_QUERY) From c72c8877170aec5a47cdd8c3b2eb0b2ee7c8849f Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 17 Apr 2024 21:47:15 +1000 Subject: [PATCH 23/26] Add return type to function identity in schemainspect Modified the function identifier in schemainspect to include return type. Previously, function identification was only based on the function's full name and identity arguments. This change accommodates functions with identical arguments but different return types. (cherry picked from commit 467a12bb729725b19df57a066e196264be12ccd4) --- schemainspect/pg/obj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index ef2a6c1..e2a07f5 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -1793,7 +1793,7 @@ def load_functions(self): ) identity_arguments = "({})".format(s.identity_arguments) - self.functions[s.quoted_full_name + identity_arguments] = s + self.functions[s.quoted_full_name + identity_arguments + s.returntype] = s def load_aggregate_functions(self): q = self.execute(self.AGG_FUNCTIONS_QUERY) From 1d110529f3373f74118124aab823ca91c8b7ba50 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 17 Apr 2024 21:56:17 +1000 Subject: [PATCH 24/26] handle void return (cherry picked from commit 621e7b4fcb2f52d7d0f5d4a30ecae182ff4a2ff0) --- schemainspect/pg/obj.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index e2a07f5..65cf29b 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -1793,7 +1793,7 @@ def load_functions(self): ) identity_arguments = "({})".format(s.identity_arguments) - self.functions[s.quoted_full_name + identity_arguments + s.returntype] = s + self.functions[s.quoted_full_name + identity_arguments + str(s.returntype)] = s def load_aggregate_functions(self): q = self.execute(self.AGG_FUNCTIONS_QUERY) From cb1f6f26705b987fa1f878b01eb2e070d57e737a Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 17 Apr 2024 22:36:57 +1000 Subject: [PATCH 25/26] Add function result to identifiers and dependencies Added the function result to the quoted and unquoted identifiers, as well as to the dependencies. This additional parameter was included in multiple functions and SQL queries, and it allows to better identify and manage dependencies in the 'schemainspect' Python module. (cherry picked from commit 91fe044711901cac6bf3c6e41bde116f72417412) --- schemainspect/command.py | 3 ++- schemainspect/misc.py | 8 ++++---- schemainspect/pg/obj.py | 7 ++++--- schemainspect/pg/sql/deps.sql | 14 ++++++++++++-- 4 files changed, 22 insertions(+), 10 deletions(-) diff --git a/schemainspect/command.py b/schemainspect/command.py index be06c5f..c4a9b66 100644 --- a/schemainspect/command.py +++ b/schemainspect/command.py @@ -30,11 +30,12 @@ def do_deps(db_url): deps = i.deps def process_row(dep): - depends_on = quoted_identifier(dep.name, dep.schema, dep.identity_arguments) + depends_on = quoted_identifier(dep.name, dep.schema, dep.identity_arguments, dep.result) thing = quoted_identifier( dep.name_dependent_on, dep.schema_dependent_on, dep.identity_arguments_dependent_on, + dep.result_dependent_on, ) return dict( diff --git a/schemainspect/misc.py b/schemainspect/misc.py index 26e16ca..d98117b 100644 --- a/schemainspect/misc.py +++ b/schemainspect/misc.py @@ -42,25 +42,25 @@ def __ne__(self, other): return not self == other -def unquoted_identifier(identifier, *, schema=None, identity_arguments=None): +def unquoted_identifier(identifier, *, schema=None, identity_arguments=None, return_type=None): if identifier is None and schema is not None: return schema s = "{}".format(identifier) if schema: s = "{}.{}".format(schema, s) if identity_arguments is not None: - s = "{}({})".format(s, identity_arguments) + s = "{}({}) RETURNS {}".format(s, identity_arguments, return_type) return s -def quoted_identifier(identifier, schema=None, identity_arguments=None): +def quoted_identifier(identifier, schema=None, identity_arguments=None, return_type=None): if identifier is None and schema is not None: return '"{}"'.format(schema.replace('"', '""')) s = '"{}"'.format(identifier.replace('"', '""')) if schema: s = '"{}".{}'.format(schema.replace('"', '""'), s) if identity_arguments is not None: - s = "{}({})".format(s, identity_arguments) + s = "{}({}) RETURNS {}".format(s, identity_arguments, return_type) return s diff --git a/schemainspect/pg/obj.py b/schemainspect/pg/obj.py index 65cf29b..ceb0a0d 100644 --- a/schemainspect/pg/obj.py +++ b/schemainspect/pg/obj.py @@ -1407,11 +1407,12 @@ def load_deps(self): self.deps = list(q) for dep in self.deps: - x = quoted_identifier(dep.name, dep.schema, dep.identity_arguments) + x = quoted_identifier(dep.name, dep.schema, dep.identity_arguments, dep.result) x_dependent_on = quoted_identifier( dep.name_dependent_on, dep.schema_dependent_on, dep.identity_arguments_dependent_on, + dep.result_dependent_on, ) self.selectables[x].dependent_on.append(x_dependent_on) self.selectables[x].dependent_on.sort() @@ -1792,8 +1793,8 @@ def load_functions(self): kind=f.kind, ) - identity_arguments = "({})".format(s.identity_arguments) - self.functions[s.quoted_full_name + identity_arguments + str(s.returntype)] = s + identity_arguments = "({}) RETURNS {}".format(s.identity_arguments, s.result_string) + self.functions[s.quoted_full_name + identity_arguments] = s def load_aggregate_functions(self): q = self.execute(self.AGG_FUNCTIONS_QUERY) diff --git a/schemainspect/pg/sql/deps.sql b/schemainspect/pg/sql/deps.sql index 5aa2d89..b621b3b 100644 --- a/schemainspect/pg/sql/deps.sql +++ b/schemainspect/pg/sql/deps.sql @@ -4,6 +4,7 @@ with things1 as ( pronamespace as namespace, proname as name, pg_get_function_identity_arguments(oid) as identity_arguments, + pg_get_function_result(oid) as result, 'f' as kind, null::oid as composite_type_oid from pg_proc @@ -15,6 +16,7 @@ with things1 as ( relnamespace as namespace, relname as name, null as identity_arguments, + null as result, relkind as kind, null::oid as composite_type_oid from pg_class @@ -27,6 +29,7 @@ with things1 as ( typnamespace as namespace, typname as name, null as identity_arguments, + null as result, 'c' as kind, typrelid::oid as composite_type_oid from pg_type @@ -55,6 +58,7 @@ things as ( n.nspname as schema, name, identity_arguments, + result, t.composite_type_oid from things1 t inner join pg_namespace n @@ -84,11 +88,13 @@ combined as ( t.schema, t.name, t.identity_arguments, + t.result, case when t.composite_type_oid is not null then 'r' ELSE t.kind end, things_dependent_on.objid as objid_dependent_on, things_dependent_on.schema as schema_dependent_on, things_dependent_on.name as name_dependent_on, things_dependent_on.identity_arguments as identity_arguments_dependent_on, + things_dependent_on.result as result_dependent_on, things_dependent_on.kind as kind_dependent_on FROM pg_depend d @@ -109,11 +115,13 @@ combined as ( t.schema, t.name, t.identity_arguments, + t.result, case when t.composite_type_oid is not null then 'r' ELSE t.kind end, things_dependent_on.objid as objid_dependent_on, things_dependent_on.schema as schema_dependent_on, things_dependent_on.name as name_dependent_on, things_dependent_on.identity_arguments as identity_arguments_dependent_on, + things_dependent_on.result as result_dependent_on, things_dependent_on.kind as kind_dependent_on FROM pg_depend d @@ -129,11 +137,13 @@ combined as ( t.schema, t.name, t.identity_arguments, + t.result, case when t.composite_type_oid is not null then 'r' ELSE t.kind end, things_dependent_on.objid as objid_dependent_on, things_dependent_on.schema as schema_dependent_on, things_dependent_on.name as name_dependent_on, things_dependent_on.identity_arguments as identity_arguments_dependent_on, + things_dependent_on.result as result_dependent_on, things_dependent_on.kind as kind_dependent_on FROM array_dependencies ad @@ -144,5 +154,5 @@ combined as ( ) select * from combined order by -schema, name, identity_arguments, kind_dependent_on, -schema_dependent_on, name_dependent_on, identity_arguments_dependent_on +schema, name, identity_arguments, result, kind_dependent_on, +schema_dependent_on, name_dependent_on, identity_arguments_dependent_on, result_dependent_on From eeda5c18d26fce006a826cfe63f8683b4490a321 Mon Sep 17 00:00:00 2001 From: Josha Inglis Date: Wed, 24 Apr 2024 14:29:45 +1000 Subject: [PATCH 26/26] handle case where a relation has no columns This can happen if a column is of a custom type and the type is deleted with a cascade (cherry picked from commit f2f5dffdc398037a2d41f1592500df4d60433e4d) --- schemainspect/pg/sql/relations.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/schemainspect/pg/sql/relations.sql b/schemainspect/pg/sql/relations.sql index 92f5f94..2929a65 100644 --- a/schemainspect/pg/sql/relations.sql +++ b/schemainspect/pg/sql/relations.sql @@ -97,12 +97,12 @@ FROM r left join pg_catalog.pg_attribute a on r.oid = a.attrelid and a.attnum > 0 + and not a.attisdropped left join pg_catalog.pg_attrdef ad on a.attrelid = ad.adrelid and a.attnum = ad.adnum left join enums e on a.atttypid = e.enum_oid -where a.attisdropped is not true --- SKIP_INTERNAL and r.schema not in ('pg_catalog', 'information_schema', 'pg_toast') +-- SKIP_INTERNAL WHERE r.schema not in ('pg_catalog', 'information_schema', 'pg_toast') -- SKIP_INTERNAL and r.schema not like 'pg_temp_%' and r.schema not like 'pg_toast_temp_%' order by relationtype, r.schema, r.name, position_number;