Skip to content

Commit 1d80ade

Browse files
jfrochesamrose
jfroche
and
samrose
authored
feat: support multiple versions of the wal2json extension (#1875)
* feat: support multiple versions of the wal2json extension Build multiple versions of the wal2json extension on different PostgreSQL versions. Add test for the extension and their upgrade on PostgreSQL 15 and 17. * chore: update release suffix for testing * feat: improve pg_regress to validate `pg_logical_slot_get_changes` parameters used in supabase/realtime * chore: bump to release * chore: bump suffix to test * chore: bump to release --------- Co-authored-by: Sam Rose <samuel@supabase.io>
1 parent 79cbab5 commit 1d80ade

File tree

9 files changed

+120
-34
lines changed

9 files changed

+120
-34
lines changed

‎ansible/vars.yml‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,9 @@ postgres_major:
1010

1111
# Full version strings for each major version
1212
postgres_release:
13-
postgresorioledb-17: "17.5.1.065-orioledb"
14-
postgres17: "17.6.1.044"
15-
postgres15: "15.14.1.044"
13+
postgresorioledb-17: "17.5.1.066-orioledb"
14+
postgres17: "17.6.1.045"
15+
postgres15: "15.14.1.045"
1616

1717
# Non Postgres Extensions
1818
pgbouncer_release: 1.19.0

‎nix/ext/tests/default.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -224,6 +224,7 @@ builtins.listToAttrs (
224224
"pgaudit"
225225
"pg_tle"
226226
"vector"
227+
"wal2json"
227228
"wrappers"
228229
]
229230
)

‎nix/ext/versions.json‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -757,5 +757,14 @@
757757
],
758758
"hash": "sha256-crxj5R9jblIv0h8lpqddAoYe2UqgUlnvbOajKTzVces="
759759
}
760+
},
761+
"wal2json": {
762+
"2.6": {
763+
"postgresql": [
764+
"15",
765+
"17"
766+
],
767+
"hash": "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM="
768+
}
760769
}
761770
}

‎nix/ext/wal2json.nix‎

Lines changed: 90 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,106 @@
11
{
2+
pkgs,
23
lib,
34
stdenv,
45
fetchFromGitHub,
56
postgresql,
7+
makeWrapper,
68
}:
79

8-
stdenv.mkDerivation rec {
10+
let
911
pname = "wal2json";
10-
version = "2_6";
12+
build =
13+
version: rev: hash:
14+
stdenv.mkDerivation rec {
15+
inherit version pname;
1116

12-
src = fetchFromGitHub {
13-
owner = "eulerto";
14-
repo = "wal2json";
15-
rev = "wal2json_${builtins.replaceStrings [ "." ] [ "_" ] version}";
16-
hash = "sha256-+QoACPCKiFfuT2lJfSUmgfzC5MXf75KpSoc2PzPxKyM=";
17-
};
17+
src = fetchFromGitHub {
18+
owner = "eulerto";
19+
repo = "wal2json";
20+
rev = "wal2json_${builtins.replaceStrings [ "." ] [ "_" ] version}";
21+
inherit hash;
22+
};
23+
24+
buildInputs = [ postgresql ];
25+
26+
makeFlags = [ "USE_PGXS=1" ];
27+
28+
installPhase = ''
29+
runHook preInstall
30+
31+
mkdir -p $out/share/postgresql/extension
32+
33+
# Install versioned library
34+
install -Dm755 ${pname}${postgresql.dlSuffix} $out/lib/${pname}-${version}${postgresql.dlSuffix}
35+
if [[ "${version}" == "${latestVersion}" ]]; then
36+
cp sql/*.sql $out/share/postgresql/extension/
37+
fi
38+
39+
touch $out/share/postgresql/extension/${pname}--${version}.control
40+
touch $out/share/postgresql/extension/${pname}--${version}.sql
41+
42+
runHook postInstall
43+
'';
44+
45+
meta = with lib; {
46+
description = "PostgreSQL JSON output plugin for changeset extraction";
47+
homepage = "https://github.com/eulerto/wal2json";
48+
changelog = "https://github.com/eulerto/wal2json/releases/";
49+
platforms = postgresql.meta.platforms;
50+
license = licenses.bsd3;
51+
};
52+
};
53+
allVersions = (builtins.fromJSON (builtins.readFile ./versions.json)).wal2json;
54+
supportedVersions = lib.filterAttrs (
55+
_: value: builtins.elem (lib.versions.major postgresql.version) value.postgresql
56+
) allVersions;
57+
versions = lib.naturalSort (lib.attrNames supportedVersions);
58+
latestVersion = lib.last versions;
59+
numberOfVersions = builtins.length versions;
60+
packages = builtins.attrValues (
61+
lib.mapAttrs (name: value: build name value.rev value.hash) supportedVersions
62+
);
63+
in
64+
pkgs.buildEnv {
65+
name = pname;
66+
paths = packages;
67+
nativeBuildInputs = [ makeWrapper ];
68+
pathsToLink = [
69+
"/lib"
70+
"/share/postgresql/extension"
71+
];
72+
postBuild = ''
73+
ln -sfn ${pname}-${latestVersion}${postgresql.dlSuffix} $out/lib/${pname}${postgresql.dlSuffix}
1874
19-
buildInputs = [ postgresql ];
75+
{
76+
echo "default_version = '${latestVersion}'"
77+
} > $out/share/postgresql/extension/${pname}.control
2078
21-
makeFlags = [ "USE_PGXS=1" ];
79+
# Create empty upgrade files between consecutive versions
80+
# plpgsql_check ships without upgrade scripts - extensions are backward-compatible
81+
previous_version=""
82+
for ver in ${lib.concatStringsSep " " versions}; do
83+
if [[ -n "$previous_version" ]]; then
84+
touch $out/share/postgresql/extension/${pname}--''${previous_version}--''${ver}.sql
85+
fi
86+
previous_version=$ver
87+
done
2288
23-
installPhase = ''
24-
install -D -t $out/lib *${postgresql.dlSuffix}
25-
install -D -t $out/share/postgresql/extension sql/*.sql
89+
# checks
90+
(set -x
91+
test "$(ls -A $out/lib/${pname}*${postgresql.dlSuffix} | wc -l)" = "${
92+
toString (numberOfVersions + 1)
93+
}"
94+
)
2695
'';
2796

28-
meta = with lib; {
29-
description = "PostgreSQL JSON output plugin for changeset extraction";
30-
homepage = "https://github.com/eulerto/wal2json";
31-
changelog = "https://github.com/eulerto/wal2json/releases/tag/wal2json_${version}";
32-
platforms = postgresql.meta.platforms;
33-
license = licenses.bsd3;
97+
passthru = {
98+
inherit versions numberOfVersions;
99+
pname = "${pname}-all";
100+
version =
101+
"multi-" + lib.concatStringsSep "-" (map (v: lib.replaceStrings [ "." ] [ "-" ] v) versions);
102+
defaultSettings = {
103+
wal_level = "logical";
104+
};
34105
};
35106
}

‎nix/tests/expected/wal2json.out‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,16 +19,17 @@ from
1919
'reg_test',
2020
null,
2121
null,
22-
'include-pk', '1',
22+
'include-pk', 'true',
2323
'include-transaction', 'false',
2424
'include-timestamp', 'false',
25-
'include-type-oids', 'false',
25+
'include-type-oids', 'true',
2626
'format-version', '2',
27-
'actions', 'insert,update,delete'
27+
'actions', 'insert,update,delete',
28+
'add-tables', 'v.foo'
2829
) x;
29-
data
30-
--------------------------------------------------------------------------------------------------------------------------------------
31-
{"action":"I","schema":"v","table":"foo","columns":[{"name":"id","type":"integer","value":1}],"pk":[{"name":"id","type":"integer"}]}
30+
data
31+
----------------------------------------------------------------------------------------------------------------------------------------------------------------
32+
{"action":"I","schema":"v","table":"foo","columns":[{"name":"id","type":"integer","typeoid":23,"value":1}],"pk":[{"name":"id","type":"integer","typeoid":23}]}
3233
(1 row)
3334

3435
select

‎nix/tests/expected/z_15_ext_interface.out‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,8 @@ order by
3232
pg_cron
3333
pgjwt
3434
tsm_system_time
35-
(3 rows)
35+
wal2json
36+
(4 rows)
3637

3738
/*
3839

‎nix/tests/expected/z_17_ext_interface.out‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ order by
2626
pgjwt
2727
postgis_tiger_geocoder
2828
tsm_system_time
29-
(4 rows)
29+
wal2json
30+
(5 rows)
3031

3132
/*
3233

‎nix/tests/expected/z_orioledb-17_ext_interface.out‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,8 @@ order by
2626
pgjwt
2727
postgis_tiger_geocoder
2828
tsm_system_time
29-
(4 rows)
29+
wal2json
30+
(5 rows)
3031

3132
/*
3233

‎nix/tests/sql/wal2json.sql‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,12 +18,13 @@ from
1818
'reg_test',
1919
null,
2020
null,
21-
'include-pk', '1',
21+
'include-pk', 'true',
2222
'include-transaction', 'false',
2323
'include-timestamp', 'false',
24-
'include-type-oids', 'false',
24+
'include-type-oids', 'true',
2525
'format-version', '2',
26-
'actions', 'insert,update,delete'
26+
'actions', 'insert,update,delete',
27+
'add-tables', 'v.foo'
2728
) x;
2829

2930
select

0 commit comments

Comments
 (0)