From 321911ad528c7a7c0ab355fd344b5cd649e755c8 Mon Sep 17 00:00:00 2001 From: Steven Niu Date: Tue, 10 Feb 2026 17:35:10 +0800 Subject: [PATCH 1/3] add wal2json doc --- CN/modules/ROOT/nav.adoc | 1 + .../master/ecosystem_components/wal2json.adoc | 137 ++++++++++++++++++ EN/modules/ROOT/nav.adoc | 1 + .../master/ecosystem_components/wal2json.adoc | 135 +++++++++++++++++ 4 files changed, 274 insertions(+) create mode 100644 CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc create mode 100644 EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc diff --git a/CN/modules/ROOT/nav.adoc b/CN/modules/ROOT/nav.adoc index 2062603..887698a 100644 --- a/CN/modules/ROOT/nav.adoc +++ b/CN/modules/ROOT/nav.adoc @@ -35,6 +35,7 @@ **** xref:master/ecosystem_components/pgaudit.adoc[pgaudit] **** xref:master/ecosystem_components/pgrouting.adoc[pgrouting] **** xref:master/ecosystem_components/system_stats.adoc[system_stats] +**** xref:master/ecosystem_components/wal2json.adoc[wal2json] ** IvorySQL架构设计 *** 查询处理 **** xref:master/architecture/dual_parser.adoc[双parser] diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc new file mode 100644 index 0000000..e255c6e --- /dev/null +++ b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -0,0 +1,137 @@ + +:sectnums: +:sectnumlevels: 5 + += wal2json + +== 概述 +wal2json 是一个用于 PostgreSQL 逻辑解码的输出插件，这个插件为每个事务生成一个JSON对象。 + +== 安装 + +[TIP] +源码安装环境为 Ubuntu 24.04(x86_64)，环境中已经安装了IvorySQL5及以上版本，安装路径为/usr/ivory-5 + +=== 源码安装 + +[literal] +---- +# 从 https://github.com/eulerto/wal2json/releases/tag/wal2json_2_6 下载 2.6 的源码包 wal2json_2_6.zip + +unzip wal2json_2_6.zip +cd wal2json_2_6 + +# 编译安装插件 +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install + +---- + +[TIP] +如果出现找不到xlocale.h的错误，需要手动修改 /usr/ivory-5/include/postgresql/server/pg_config.h +删除或者注释掉 #define HAVE_XLOCALE_H 1 这一行 + +=== 修改数据库配置文件 + +修改 postgresql.conf 文件 +启用wal_level为logical，并设置最大复制槽数和最大wal发送进程数 + +[literal] +---- +wal_level = logical +max_replication_slots = 10 +max_wal_senders = 10 +---- + +pg_hba.conf不需要修改，有以下内容即可（仅限本地链接测试）： +[literal] +---- + local replication all trust + host replication all 127.0.0.1/32 trust + host replication all ::1/128 trust +---- + +重启数据库后配置生效。 + +== 使用 + +在第一个终端中执行命令： +[literal] +---- +sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json + +启动监听，实时输出变更的JSON格式 +sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +---- + +在第二个终端中连接数据库： +[literal] +---- +bin/psql -d postgres -p 1521 +---- + +执行下面的SQL语句： +[literal] +---- +CREATE TABLE test_cdc (id int primary key, name varchar(50)); +INSERT INTO test_cdc VALUES (1, 'test1'); +UPDATE test_cdc SET name = 'test1_update' WHERE id = 1; +DELETE FROM test_cdc WHERE id = 1; +DROP TABLE test_cdc; +---- + +此时在第一个终端上可以看到下面的输出： +[literal] +---- +{ + "change": [ + ] +} +{ + "change": [ + { + "kind": "insert", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1"] + } + ] +} +{ + "change": [ + { + "kind": "update", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1_update"], + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + { + "kind": "delete", + "schema": "public", + "table": "test_cdc", + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + ] +} +---- \ No newline at end of file diff --git a/EN/modules/ROOT/nav.adoc b/EN/modules/ROOT/nav.adoc index 97ababa..0d85011 100644 --- a/EN/modules/ROOT/nav.adoc +++ b/EN/modules/ROOT/nav.adoc @@ -34,6 +34,7 @@ *** xref:master/ecosystem_components/pgaudit.adoc[pgaudit] *** xref:master/ecosystem_components/pgrouting.adoc[pgrouting] *** xref:master/ecosystem_components/system_stats.adoc[system_stats] +*** xref:master/ecosystem_components/wal2json.adoc[wal2json] * IvorySQL Architecture Design ** Query Processing *** xref:master/architecture/dual_parser.adoc[Dual Parser] diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc new file mode 100644 index 0000000..09bf16b --- /dev/null +++ b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -0,0 +1,135 @@ + +:sectnums: +:sectnumlevels: 5 + += wal2json + +== Overview +wal2json is an output plugin for logical decoding. It generates JSON object for each transaction. + +== Installation + +[TIP] +The source code installation environment is Ubuntu 24.04 (x86_64), in which IvorySQL 5 or a later version has been installed. The installation path is /usr/ivory-5. + +=== Source Code Installation + +[literal] +---- +# download source code package from: https://github.com/eulerto/wal2json/releases/tag/wal2json_2_6 + +unzip wal2json_2_6.zip +cd wal2json_2_6 + +# compile and install the extension +make PG_CONFIG=/usr/ivory-5/bin/pg_config +make PG_CONFIG=/usr/ivory-5/bin/pg_config install +---- + +[TIP] +If there is error "xlocale.h: No such file or directory" during compilation, user should remove the +line of "#define HAVE_XLOCALE_H 1" from file /usr/ivory-5/include/postgresql/server/pg_config.h. + +=== Modify the configuration file + +Modify the postgresql.conf file to set wal_level as "logical", and set max_replication_slots/max_wal_senders. +[literal] +---- +wal_level = logical +max_replication_slots = 10 +max_wal_senders = 10 +---- + +Make sure the following content to be in pg_hba.conf. +[literal] +---- + local replication all trust + host replication all 127.0.0.1/32 trust + host replication all ::1/128 trust +---- + +Then restart the database. + +== Use + +Open the first terminal and execute command: + +[literal] +---- +sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json + +Start monitoring, output the changes in JSON format and in real time. +sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +---- + +Connect database in the second terminal: +[literal] +---- +bin/psql -d postgres -p 1521 +---- + +Execute the following SQL statement: +[literal] +---- +CREATE TABLE test_cdc (id int primary key, name varchar(50)); +INSERT INTO test_cdc VALUES (1, 'test1'); +UPDATE test_cdc SET name = 'test1_update' WHERE id = 1; +DELETE FROM test_cdc WHERE id = 1; +DROP TABLE test_cdc; +---- + +The following output will appear in the first terminal: +[literal] +---- +{ + "change": [ + ] +} +{ + "change": [ + { + "kind": "insert", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1"] + } + ] +} +{ + "change": [ + { + "kind": "update", + "schema": "public", + "table": "test_cdc", + "columnnames": ["id", "name"], + "columntypes": ["integer", "sys.oravarcharbyte(50)"], + "columnvalues": [1, "test1_update"], + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + { + "kind": "delete", + "schema": "public", + "table": "test_cdc", + "oldkeys": { + "keynames": ["id"], + "keytypes": ["integer"], + "keyvalues": [1] + } + } + ] +} +{ + "change": [ + ] +} +---- From 5b042c043211c06853bb423dd622b1aaae36996f Mon Sep 17 00:00:00 2001 From: Steven Niu Date: Tue, 3 Mar 2026 15:12:52 +0800 Subject: [PATCH 2/3] use ivorysql to replace highgo --- .../ROOT/pages/master/ecosystem_components/wal2json.adoc | 6 +++--- .../ROOT/pages/master/ecosystem_components/wal2json.adoc | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc index e255c6e..eb5b4f4 100644 --- a/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc +++ b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -58,10 +58,10 @@ pg_hba.conf不需要修改，有以下内容即可（仅限本地链接测试） 在第一个终端中执行命令： [literal] ---- -sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json +sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json 启动监听，实时输出变更的JSON格式 -sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - ---- 在第二个终端中连接数据库： @@ -134,4 +134,4 @@ DROP TABLE test_cdc; "change": [ ] } ----- \ No newline at end of file +---- diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc index 09bf16b..d6bf9b1 100644 --- a/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc +++ b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -56,10 +56,10 @@ Open the first terminal and execute command: [literal] ---- -sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json +sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json Start monitoring, output the changes in JSON format and in real time. -sudo -u highgo /home/highgo/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - ---- Connect database in the second terminal: From 57b0e78312fafaa0a5dcc7e29a69011bf70022d1 Mon Sep 17 00:00:00 2001 From: Steven Niu Date: Tue, 3 Mar 2026 15:22:56 +0800 Subject: [PATCH 3/3] align the path names --- .../ROOT/pages/master/ecosystem_components/wal2json.adoc | 6 +++--- .../ROOT/pages/master/ecosystem_components/wal2json.adoc | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc index eb5b4f4..10d4c8c 100644 --- a/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc +++ b/CN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -58,16 +58,16 @@ pg_hba.conf不需要修改，有以下内容即可（仅限本地链接测试） 在第一个终端中执行命令： [literal] ---- -sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json +sudo -u ivorysql /usr/ivory-5/bin/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json 启动监听，实时输出变更的JSON格式 -sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +sudo -u ivorysql /usr/ivory-5/bin/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - ---- 在第二个终端中连接数据库： [literal] ---- -bin/psql -d postgres -p 1521 +/usr/ivory-5/bin/psql -d postgres -p 1521 ---- 执行下面的SQL语句： diff --git a/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc index d6bf9b1..2fb85dc 100644 --- a/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc +++ b/EN/modules/ROOT/pages/master/ecosystem_components/wal2json.adoc @@ -56,16 +56,16 @@ Open the first terminal and execute command: [literal] ---- -sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json +sudo -u ivorysql /usr/ivory-5/bin/pg_recvlogical -d postgres --slot wal2json_slot --create-slot -P wal2json Start monitoring, output the changes in JSON format and in real time. -sudo -u ivorysql /home/ivorysql/ivy/inst/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - +sudo -u ivorysql /usr/ivory-5/bin/pg_recvlogical -d postgres --slot wal2json_slot --start -o pretty-print=1 -f - ---- Connect database in the second terminal: [literal] ---- -bin/psql -d postgres -p 1521 +/usr/ivory-5/bin/psql -d postgres -p 1521 ---- Execute the following SQL statement: