From 32a078906d9376022cbb75a5b77480386e12e084 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Fri, 10 Apr 2026 16:48:44 +0100 Subject: [PATCH 1/7] Add shadow linking lab for Kubernetes disaster recovery Add a new lab demonstrating Redpanda's shadow linking feature for disaster recovery on Kubernetes. The lab deploys two single-node clusters and configures continuous replication of topics, consumer group offsets, and Schema Registry data. Includes three demo scenarios: - Basic topic shadowing - Schema Registry shadowing - Consumer group failover Based on https://github.com/pmw-rp/shadowing-demo --- .../kubernetes/pages/shadow-linking.adoc | 1 + kubernetes/shadow-linking/README.adoc | 233 ++++++++++++++++++ kubernetes/shadow-linking/config | 4 + .../1-create-topic-on-source.sh | 8 + .../2-produce-record-to-source.sh | 8 + .../3-list-topics-on-shadow.sh | 9 + .../4-consume-messages-on-both.sh | 13 + .../2-schema-shadowing/1-register-schema.sh | 8 + .../2-schema-shadowing/2-create-topic.sh | 8 + .../2-schema-shadowing/3-produce-records.sh | 8 + .../4-list-schemas-on-both.sh | 13 + .../5-consume-raw-messages-from-shadow.sh | 9 + .../6-consume-decoded-messages-from-shadow.sh | 9 + .../2-schema-shadowing/resources/syslog.avsc | 78 ++++++ .../2-schema-shadowing/resources/syslogs.json | 10 + .../3-failover/1-create-topic-on-source.sh | 8 + .../2-produce-and-consume-on-source.sh | 11 + .../3-failover/3-show-consumer-groups.sh | 13 + .../demos/3-failover/4-failover.sh | 11 + .../5-produce-and-consume-on-shadow.sh | 11 + .../6-show-consumer-group-on-shadow.sh | 10 + .../resources/shadow-cluster.yaml | 43 ++++ .../shadow-linking/resources/shadow-link.yaml | 46 ++++ .../resources/source-cluster.yaml | 43 ++++ kubernetes/shadow-linking/setup.sh | 50 ++++ 25 files changed, 665 insertions(+) create mode 120000 docs/modules/kubernetes/pages/shadow-linking.adoc create mode 100644 kubernetes/shadow-linking/README.adoc create mode 100644 kubernetes/shadow-linking/config create mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh create mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh create mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh create mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh create mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh create mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh create mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh create mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh create mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh create mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh create mode 100644 kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc create mode 100644 kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json create mode 100755 kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh create mode 100755 kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh create mode 100755 kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh create mode 100755 kubernetes/shadow-linking/demos/3-failover/4-failover.sh create mode 100755 kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh create mode 100755 kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh create mode 100644 kubernetes/shadow-linking/resources/shadow-cluster.yaml create mode 100644 kubernetes/shadow-linking/resources/shadow-link.yaml create mode 100644 kubernetes/shadow-linking/resources/source-cluster.yaml create mode 100755 kubernetes/shadow-linking/setup.sh diff --git a/docs/modules/kubernetes/pages/shadow-linking.adoc b/docs/modules/kubernetes/pages/shadow-linking.adoc new file mode 120000 index 00000000..715333b9 --- /dev/null +++ b/docs/modules/kubernetes/pages/shadow-linking.adoc @@ -0,0 +1 @@ +../../../../kubernetes/shadow-linking/README.adoc \ No newline at end of file diff --git a/kubernetes/shadow-linking/README.adoc b/kubernetes/shadow-linking/README.adoc new file mode 100644 index 00000000..8929acd9 --- /dev/null +++ b/kubernetes/shadow-linking/README.adoc @@ -0,0 +1,233 @@ += Explore Shadow Linking for Disaster Recovery +:env-kubernetes: true +:page-categories: Disaster Recovery, Replication +:description: Deploy two Redpanda clusters on Kubernetes and configure shadow linking to continuously replicate topics, consumer group offsets, and Schema Registry data. +:page-layout: lab +// Set the image path for GitHub rendering +ifndef::env-site[] +:imagesdir: ../../docs/modules/kubernetes/images/ +endif::[] + +Shadow linking is a built-in disaster recovery mechanism that continuously replicates topics, consumer group offsets, and Schema Registry data from a source cluster to a shadow cluster running on Kubernetes. This lab demonstrates how shadow linking works and how you can use it to fail over to a shadow cluster. + +In this lab, you deploy two single-node Redpanda clusters on Kubernetes and configure a shadow link between them. You then run three demo scenarios to explore basic topic shadowing, Schema Registry shadowing, and consumer group failover. + +== Prerequisites + +You must have the following: + +* A Kubernetes cluster with adequate resources (at least 4 CPUs and 8GB memory) +* https://kubernetes.io/docs/tasks/tools/[kubectl] CLI installed +* https://helm.sh/docs/intro/install/[Helm 3+] installed +* https://docs.redpanda.com/current/get-started/rpk-install/[rpk] CLI installed + +== What gets deployed + +The setup script installs the following components: + +* *cert-manager* (namespace: `cert-manager`): Manages TLS certificates +* *Redpanda Operator* (namespace: `rp-operator`): Manages Redpanda cluster lifecycle +* *Source cluster* (namespace: `source`): Primary cluster where you write data +* *Shadow cluster* (namespace: `shadow`): Replication target cluster +* *ShadowLink resource*: Defines the replication relationship between clusters + +The shadow link configuration mirrors all topics using wildcard filtering, replicates consumer group offsets, and synchronizes the Schema Registry, all at 5-second intervals. + +== Run the lab + +. Clone this repository: ++ +[,bash] +---- +git clone https://github.com/redpanda-data/redpanda-labs.git +cd redpanda-labs/kubernetes/shadow-linking +---- + +. Run the setup script to deploy both clusters and configure shadow linking: ++ +[,bash] +---- +./setup.sh +---- ++ +The script does the following: + +.. Installs cert-manager for TLS certificate management +.. Deploys the Redpanda Operator +.. Creates the source and shadow Redpanda clusters +.. Configures the shadow link between clusters +.. Sets up port forwarding for local access +.. Creates rpk profiles for both clusters + +. Verify both clusters are healthy: ++ +[,bash] +---- +rpk --profile source cluster health +rpk --profile shadow cluster health +---- + +== Explore the demos + +This lab includes three demo scenarios that build on each other. Run them in order to understand shadow linking capabilities. + +=== Demo 1: Basic topic shadowing + +This demo shows how topics and messages automatically replicate from the source to the shadow cluster. + +. Create a topic on the source cluster: ++ +[,bash] +---- +./demos/1-basic-shadowing/1-create-topic-on-source.sh +---- + +. Produce a message to the source cluster: ++ +[,bash] +---- +./demos/1-basic-shadowing/2-produce-record-to-source.sh +---- + +. List topics on the shadow cluster to verify replication: ++ +[,bash] +---- +./demos/1-basic-shadowing/3-list-topics-on-shadow.sh +---- ++ +You should see the `basic` topic appear on the shadow cluster. + +. Consume messages from both clusters to verify data replication: ++ +[,bash] +---- +./demos/1-basic-shadowing/4-consume-messages-on-both.sh +---- ++ +The same message appears on both clusters. + +=== Demo 2: Schema Registry shadowing + +This demo shows how Avro schemas registered on the source cluster replicate to the shadow cluster, allowing shadow-side message deserialization. + +. Register an Avro schema on the source cluster: ++ +[,bash] +---- +./demos/2-schema-shadowing/1-register-schema.sh +---- + +. Create a topic for schema-encoded messages: ++ +[,bash] +---- +./demos/2-schema-shadowing/2-create-topic.sh +---- + +. Produce Avro-encoded syslog records to the source cluster: ++ +[,bash] +---- +./demos/2-schema-shadowing/3-produce-records.sh +---- + +. List schemas on both clusters to verify schema replication: ++ +[,bash] +---- +./demos/2-schema-shadowing/4-list-schemas-on-both.sh +---- ++ +The `syslog-value` schema appears on both clusters. + +. Consume raw (encoded) messages from the shadow cluster: ++ +[,bash] +---- +./demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh +---- + +. Consume decoded messages from the shadow cluster using the replicated schema: ++ +[,bash] +---- +./demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh +---- ++ +The shadow cluster can deserialize messages using the replicated schema. + +=== Demo 3: Consumer group failover + +This demo shows how consumer group offsets replicate to the shadow cluster, allowing consumers to resume from their exact position after a failover. + +. Create a topic on the source cluster: ++ +[,bash] +---- +./demos/3-failover/1-create-topic-on-source.sh +---- + +. Produce records and consume them on the source cluster: ++ +[,bash] +---- +./demos/3-failover/2-produce-and-consume-on-source.sh +---- ++ +This creates a consumer group and processes some messages. + +. View consumer group offsets on both clusters: ++ +[,bash] +---- +./demos/3-failover/3-show-consumer-groups.sh +---- ++ +The consumer group and its offsets are replicated to the shadow cluster. + +. Trigger a failover to the shadow cluster: ++ +[,bash] +---- +./demos/3-failover/4-failover.sh +---- ++ +This command promotes the shadow cluster to become writable. + +. Produce more records and consume on the shadow cluster: ++ +[,bash] +---- +./demos/3-failover/5-produce-and-consume-on-shadow.sh +---- ++ +The consumer resumes from the last committed offset. + +. Verify the consumer group offset on the shadow cluster: ++ +[,bash] +---- +./demos/3-failover/6-show-consumer-group-on-shadow.sh +---- + +== Clean up + +To stop port forwarding, terminate the background processes: + +[,bash] +---- +kill $(cat local-port-forward.pid) +---- + +To delete all resources, remove the namespaces: + +[,bash] +---- +kubectl delete namespace source shadow rp-operator cert-manager +---- + +== Suggested reading + +* https://docs.redpanda.com/current/manage/kubernetes/k-shadow-linking/[Shadow linking documentation] +* https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/kubernetes/[Redpanda on Kubernetes] diff --git a/kubernetes/shadow-linking/config b/kubernetes/shadow-linking/config new file mode 100644 index 00000000..02a2618f --- /dev/null +++ b/kubernetes/shadow-linking/config @@ -0,0 +1,4 @@ +export CERT_MANAGER_NAMESPACE=cert-manager +export OPERATOR_NAMESPACE=rp-operator +export SOURCE_REDPANDA_NAMESPACE=source +export SHADOW_REDPANDA_NAMESPACE=shadow diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh new file mode 100755 index 00000000..f6be8fa6 --- /dev/null +++ b/kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Create a topic +rpk --profile source topic create basic -p1 -r1 + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh new file mode 100755 index 00000000..756b210c --- /dev/null +++ b/kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Produce a record +echo "Hello, world" | rpk --profile source topic produce basic + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh new file mode 100755 index 00000000..ca1f6592 --- /dev/null +++ b/kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh @@ -0,0 +1,9 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +rpk --profile shadow topic list + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh new file mode 100755 index 00000000..3bfb1cb7 --- /dev/null +++ b/kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh @@ -0,0 +1,13 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +echo On source cluster: +rpk --profile source topic consume basic -n1 + +echo On shadow cluster: +rpk --profile shadow topic consume basic -n1 + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh new file mode 100755 index 00000000..4c65961e --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Register the schema +rpk --profile source registry schema create syslog-value --schema ./resources/syslog.avsc --type avro + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh new file mode 100755 index 00000000..c646459a --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Create a topic +rpk --profile source topic create syslog -p1 -r1 + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh new file mode 100755 index 00000000..bdaa1ba1 --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Produce some records +cat ./resources/syslogs.json | rpk --profile source topic produce syslog --schema-id=topic + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh new file mode 100755 index 00000000..3f7df1aa --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh @@ -0,0 +1,13 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +echo On source cluster: +rpk --profile source registry subject list + +echo On shadow cluster: +rpk --profile shadow registry subject list + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh new file mode 100755 index 00000000..507067a0 --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh @@ -0,0 +1,9 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +rpk --profile shadow topic consume syslog -o0 -n 1 + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh new file mode 100755 index 00000000..6a291709 --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh @@ -0,0 +1,9 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +rpk --profile shadow topic consume syslog -o0 -n1 --use-schema-registry -f'%v\n' | jq + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc b/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc new file mode 100644 index 00000000..c2127dce --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc @@ -0,0 +1,78 @@ +{ + "type": "record", + "name": "syslog", + "fields": [ + { + "name": "name", + "type": "string" + }, + { + "name": "type", + "type": "string" + }, + { + "name": "message", + "type": "string" + }, + { + "name": "host", + "type": "string" + }, + { + "name": "version", + "type": "string" + }, + { + "name": "tag", + "type": "string" + }, + { + "name": "level", + "type": "int" + }, + { + "name": "facility", + "type": "string" + }, + { + "name": "severity", + "type": "int" + }, + { + "name": "appName", + "type": "string" + }, + { + "name": "remoteAddress", + "type": "string" + }, + { + "name": "rawMessage", + "type": "string" + }, + { + "name": "processId", + "type": "string" + }, + { + "name": "messageId", + "type": "string" + }, + { + "name": "deviceVendor", + "type": "string" + }, + { + "name": "deviceProduct", + "type": "string" + }, + { + "name": "deviceVersion", + "type": "string" + }, + { + "name": "ts", + "type": "int" + } + ] +} diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json b/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json new file mode 100644 index 00000000..503c44e2 --- /dev/null +++ b/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json @@ -0,0 +1,10 @@ +{ "name" : "bxDQHOGoKgwWaSqH", "type" : "UNKNOWN", "message" : "jfrkrvsrzg xdo", "host" : "192.0.2.1", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 36, "facility" : "syslog", "severity" : 6, "appName" : "NMDFQKIB", "remoteAddress" : "198.51.100.1", "rawMessage" : "wypcdigeqmqna", "processId" : "4065031", "messageId" : "91007", "deviceVendor" : "qgrdce", "deviceProduct" : "qtboqr", "deviceVersion" : "4.0", "ts" : 1} +{ "name" : "rHiHaQSukGYKC", "type" : "RFC5424", "message" : "qeailsfermffvdp", "host" : "192.0.2.2", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 41, "facility" : "cron", "severity" : 2, "appName" : "ACDWLYUM", "remoteAddress" : "198.51.100.2", "rawMessage" : "vjqis aaksxtm", "processId" : "751456541", "messageId" : "5926", "deviceVendor" : "zaiwvm", "deviceProduct" : "ntrxut", "deviceVersion" : "5.0", "ts" : 101} +{ "name" : "lQBbDoBSupsafcWhN", "type" : "CEF", "message" : "ztbvb cvvpgmoe", "host" : "192.0.2.3", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 2, "facility" : "authpriv", "severity" : 0, "appName" : "LKXXWTF", "remoteAddress" : "198.51.100.3", "rawMessage" : "mfnntbdbnbba", "processId" : "48791776", "messageId" : "92749", "deviceVendor" : "wppmhp", "deviceProduct" : "tvefu", "deviceVersion" : "5.0", "ts" : 201} +{ "name" : "ybiyfb", "type" : "CEF", "message" : "mrmtdau svkgs", "host" : "192.0.2.4", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 95, "facility" : "authpriv", "severity" : 4, "appName" : "BNBKAC", "remoteAddress" : "198.51.100.4", "rawMessage" : "elmpzpsjywa ", "processId" : "33157604", "messageId" : "987", "deviceVendor" : "ttmlnx", "deviceProduct" : "cqbrd", "deviceVersion" : "3.0", "ts" : 301} +{ "name" : "ScVGe", "type" : "RFC5424", "message" : "beaqygdkwyuyb", "host" : "192.0.2.5", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 118, "facility" : "authpriv", "severity" : 1, "appName" : "OURNH", "remoteAddress" : "198.51.100.5", "rawMessage" : "txtuaxthfex", "processId" : "584158260", "messageId" : "1755", "deviceVendor" : "unwtb", "deviceProduct" : "zmogds", "deviceVersion" : "5.0", "ts" : 401} +{ "name" : "RYWkuWsBiJ", "type" : "CEF", "message" : "xymvfwrcuyqo", "host" : "192.0.2.6", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 112, "facility" : "syslog", "severity" : 4, "appName" : "GENQI", "remoteAddress" : "198.51.100.6", "rawMessage" : "tpjlcpzjrgu", "processId" : "87033144", "messageId" : "491", "deviceVendor" : "nllkm", "deviceProduct" : "cqonw", "deviceVersion" : "4.0", "ts" : 501} +{ "name" : "ojajvZAMPETMB", "type" : "CEF", "message" : "ecrxleeyakltnh", "host" : "192.0.2.7", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 138, "facility" : "syslog", "severity" : 5, "appName" : "OICRHZZH", "remoteAddress" : "198.51.100.7", "rawMessage" : "chxwvqwfhd", "processId" : "44548987", "messageId" : "91172", "deviceVendor" : "nieyv", "deviceProduct" : "cbeqlc", "deviceVersion" : "5.0", "ts" : 601} +{ "name" : "NVrfOrYOhzeLjPl", "type" : "UNKNOWN", "message" : "lgsfjbezere", "host" : "192.0.2.8", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 12, "facility" : "syslog", "severity" : 1, "appName" : "XNGJP", "remoteAddress" : "198.51.100.8", "rawMessage" : "bdyxmgodfpdkcfz", "processId" : "157600899", "messageId" : "48518", "deviceVendor" : "dxvsrk", "deviceProduct" : "nwdyoe", "deviceVersion" : "2.0", "ts" : 701} +{ "name" : "zPUw", "type" : "CEF", "message" : "wduhd lkvfz", "host" : "192.0.2.9", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 57, "facility" : "authpriv", "severity" : 5, "appName" : "PGTEZJTC", "remoteAddress" : "198.51.100.9", "rawMessage" : "c exsvqslvevsm", "processId" : "2664427", "messageId" : "2314", "deviceVendor" : "nyyls", "deviceProduct" : "hgbbdo", "deviceVersion" : "4.0", "ts" : 801} +{ "name" : "CTLutuxyi", "type" : "RFC3164", "message" : "zazlbjduaqov", "host" : "192.0.2.10", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 74, "facility" : "authpriv", "severity" : 6, "appName" : "JDYKJ", "remoteAddress" : "198.51.100.10", "rawMessage" : "wqbyxckvjhw", "processId" : "743558242", "messageId" : "3758", "deviceVendor" : "gjmvyn", "deviceProduct" : "vfibq", "deviceVersion" : "4.0", "ts" : 901} diff --git a/kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh b/kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh new file mode 100755 index 00000000..aecdea17 --- /dev/null +++ b/kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh @@ -0,0 +1,8 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Create a topic +rpk --profile source topic create foo -p1 -r1 + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh b/kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh new file mode 100755 index 00000000..7365cb55 --- /dev/null +++ b/kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh @@ -0,0 +1,11 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Produce some records to the source +seq 0 2 | rpk --profile source topic produce foo + +# Consume the source cluster for 10 seconds +timeout 10s rpk --profile source topic consume foo -g consumer-group-foo || true + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh b/kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh new file mode 100755 index 00000000..a8102be7 --- /dev/null +++ b/kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh @@ -0,0 +1,13 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +echo On source cluster: +rpk --profile source group describe consumer-group-foo + +echo On shadow cluster: +rpk --profile shadow group describe consumer-group-foo + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/4-failover.sh b/kubernetes/shadow-linking/demos/3-failover/4-failover.sh new file mode 100755 index 00000000..a0108cac --- /dev/null +++ b/kubernetes/shadow-linking/demos/3-failover/4-failover.sh @@ -0,0 +1,11 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +rpk --profile shadow shadow failover shadow-link --all --no-confirm +sleep 5 +rpk --profile shadow shadow status shadow-link + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh b/kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh new file mode 100755 index 00000000..0b8823ca --- /dev/null +++ b/kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh @@ -0,0 +1,11 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +# Produce some records to the shadow +seq 3 5 | rpk --profile shadow topic produce foo + +# Consume the shadow cluster for 10 seconds +timeout 10s rpk --profile shadow topic consume foo -g consumer-group-foo || true + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh b/kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh new file mode 100755 index 00000000..433510db --- /dev/null +++ b/kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh @@ -0,0 +1,10 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null + +source ../../config + +echo On shadow cluster: +rpk --profile shadow group describe consumer-group-foo + +popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/resources/shadow-cluster.yaml b/kubernetes/shadow-linking/resources/shadow-cluster.yaml new file mode 100644 index 00000000..b7b7e3fd --- /dev/null +++ b/kubernetes/shadow-linking/resources/shadow-cluster.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: shadow +--- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda + namespace: shadow +spec: + clusterSpec: + image: + tag: v25.3.4 + external: + enabled: true + service: + enabled: false + addresses: + - localhost + listeners: + kafka: + external: + default: + enabled: true + port: 9094 + advertisedPorts: + - 29094 + statefulset: + replicas: 1 + config: + cluster: + default_topic_replications: 1 + enable_shadow_linking: true + storage: + tiered: + config: + cloud_storage_enabled: false + tls: + enabled: false + auth: + sasl: + enabled: false diff --git a/kubernetes/shadow-linking/resources/shadow-link.yaml b/kubernetes/shadow-linking/resources/shadow-link.yaml new file mode 100644 index 00000000..1aedf808 --- /dev/null +++ b/kubernetes/shadow-linking/resources/shadow-link.yaml @@ -0,0 +1,46 @@ +apiVersion: cluster.redpanda.com/v1alpha2 +kind: ShadowLink +metadata: + name: shadow-link + namespace: shadow +spec: + + # What cluster are we writing to? + shadowCluster: + staticConfiguration: + kafka: + brokers: ["redpanda.shadow.svc.cluster.local:9093"] + admin: + urls: ["http://redpanda.shadow.svc.cluster.local:9644/"] + schemaRegistry: + urls: ["http://redpanda.shadow.svc.cluster.local:8081/"] + + # What cluster are we reading from? + sourceCluster: + staticConfiguration: + kafka: + brokers: ["redpanda.source.svc.cluster.local:9093"] + admin: + urls: ["http://redpanda.source.svc.cluster.local:9644/"] + schemaRegistry: + urls: ["http://redpanda.source.svc.cluster.local:8081/"] + + # Replicate all topics (except for any that are pre-filtered) + topicMetadataSyncOptions: + interval: 5s + autoCreateShadowTopicFilters: + - name: '*' + filterType: include + patternType: literal + + # Replicate all consumer groups + consumerOffsetSyncOptions: + interval: 5s + groupFilters: + - patternType: literal + filterType: include + name: '*' + + # Replicate schema registry (using _schemas topic replication) + schemaRegistrySyncOptions: + schema_registry_shadowing_mode: "topic" diff --git a/kubernetes/shadow-linking/resources/source-cluster.yaml b/kubernetes/shadow-linking/resources/source-cluster.yaml new file mode 100644 index 00000000..31291050 --- /dev/null +++ b/kubernetes/shadow-linking/resources/source-cluster.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: source +--- +apiVersion: cluster.redpanda.com/v1alpha2 +kind: Redpanda +metadata: + name: redpanda + namespace: source +spec: + clusterSpec: + image: + tag: v25.3.4 + external: + enabled: true + service: + enabled: false + addresses: + - localhost + listeners: + kafka: + external: + default: + enabled: true + port: 9094 + advertisedPorts: + - 19094 + statefulset: + replicas: 1 + config: + cluster: + default_topic_replications: 1 + enable_shadow_linking: true + storage: + tiered: + config: + cloud_storage_enabled: false + tls: + enabled: false + auth: + sasl: + enabled: false diff --git a/kubernetes/shadow-linking/setup.sh b/kubernetes/shadow-linking/setup.sh new file mode 100755 index 00000000..03f48fc1 --- /dev/null +++ b/kubernetes/shadow-linking/setup.sh @@ -0,0 +1,50 @@ +#!/bin/bash +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +pushd $SCRIPT_DIR + +source ./config + +# Install Cert Manager + +helm repo add jetstack https://charts.jetstack.io +helm repo update +helm install cert-manager jetstack/cert-manager \ + --set crds.enabled=true \ + --namespace ${CERT_MANAGER_NAMESPACE} \ + --create-namespace + +# Install Redpanda Operator + +helm repo add redpanda https://charts.redpanda.com +helm repo update +helm upgrade --install redpanda-controller redpanda/operator \ + --namespace ${OPERATOR_NAMESPACE} \ + --create-namespace \ + --version v25.3.1 \ + --set crds.enabled=true + +# Install Source Cluster + +kubectl apply -f resources/source-cluster.yaml +kubectl wait -n ${SOURCE_REDPANDA_NAMESPACE} redpanda/redpanda --for=condition=Ready --timeout=600s + +# Install Shadow Cluster + +kubectl apply -f resources/shadow-cluster.yaml +kubectl wait -n ${SHADOW_REDPANDA_NAMESPACE} redpanda/redpanda --for=condition=Ready --timeout=600s + +# Create Shadow Link + +kubectl apply -f resources/shadow-link.yaml + +kubectl port-forward pod/redpanda-0 -n $SOURCE_REDPANDA_NAMESPACE 19094:9094 19644:9644 18081:8081 1>/dev/null 2>/dev/null & +echo $! > local-port-forward.pid +rpk profile create source -s brokers=localhost:19094 -s admin.hosts=localhost:19644 -s registry.hosts=http://localhost:18081/ 1>/dev/null 2>/dev/null || rpk profile use source +rpk cluster health + +kubectl port-forward pod/redpanda-0 -n $SHADOW_REDPANDA_NAMESPACE 29094:9094 29644:9644 28081:8081 1>/dev/null 2>/dev/null & +echo $! >> local-port-forward.pid +rpk profile create shadow -s brokers=localhost:29094 -s admin.hosts=localhost:29644 -s registry.hosts=http://localhost:28081/ 1>/dev/null 2>/dev/null || rpk profile use shadow +rpk cluster health + +popd From 97288217225907b4fcbc1e759f1200d1c2a7c0cf Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 13 Apr 2026 11:16:47 +0100 Subject: [PATCH 2/7] Simplify lab by removing demo scripts Replace bash wrapper scripts with direct rpk commands in the README. Users can now copy commands directly from the documentation instead of running numbered scripts. This makes the lab clearer and more educational. --- kubernetes/shadow-linking/README.adoc | 87 +++++++------------ .../1-create-topic-on-source.sh | 8 -- .../2-produce-record-to-source.sh | 8 -- .../3-list-topics-on-shadow.sh | 9 -- .../4-consume-messages-on-both.sh | 13 --- .../2-schema-shadowing/1-register-schema.sh | 8 -- .../2-schema-shadowing/2-create-topic.sh | 8 -- .../2-schema-shadowing/3-produce-records.sh | 8 -- .../4-list-schemas-on-both.sh | 13 --- .../5-consume-raw-messages-from-shadow.sh | 9 -- .../6-consume-decoded-messages-from-shadow.sh | 9 -- .../2-schema-shadowing/resources/syslog.avsc | 78 ----------------- .../2-schema-shadowing/resources/syslogs.json | 10 --- .../3-failover/1-create-topic-on-source.sh | 8 -- .../2-produce-and-consume-on-source.sh | 11 --- .../3-failover/3-show-consumer-groups.sh | 13 --- .../demos/3-failover/4-failover.sh | 11 --- .../5-produce-and-consume-on-shadow.sh | 11 --- .../6-show-consumer-group-on-shadow.sh | 10 --- .../shadow-linking/resources/syslog.avsc | 24 +++++ .../shadow-linking/resources/syslogs.json | 3 + 21 files changed, 59 insertions(+), 300 deletions(-) delete mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh delete mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh delete mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh delete mode 100755 kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh delete mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh delete mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh delete mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh delete mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh delete mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh delete mode 100755 kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh delete mode 100644 kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc delete mode 100644 kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json delete mode 100755 kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh delete mode 100755 kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh delete mode 100755 kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh delete mode 100755 kubernetes/shadow-linking/demos/3-failover/4-failover.sh delete mode 100755 kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh delete mode 100755 kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh create mode 100644 kubernetes/shadow-linking/resources/syslog.avsc create mode 100644 kubernetes/shadow-linking/resources/syslogs.json diff --git a/kubernetes/shadow-linking/README.adoc b/kubernetes/shadow-linking/README.adoc index 8929acd9..b190d612 100644 --- a/kubernetes/shadow-linking/README.adoc +++ b/kubernetes/shadow-linking/README.adoc @@ -3,14 +3,10 @@ :page-categories: Disaster Recovery, Replication :description: Deploy two Redpanda clusters on Kubernetes and configure shadow linking to continuously replicate topics, consumer group offsets, and Schema Registry data. :page-layout: lab -// Set the image path for GitHub rendering -ifndef::env-site[] -:imagesdir: ../../docs/modules/kubernetes/images/ -endif::[] Shadow linking is a built-in disaster recovery mechanism that continuously replicates topics, consumer group offsets, and Schema Registry data from a source cluster to a shadow cluster running on Kubernetes. This lab demonstrates how shadow linking works and how you can use it to fail over to a shadow cluster. -In this lab, you deploy two single-node Redpanda clusters on Kubernetes and configure a shadow link between them. You then run three demo scenarios to explore basic topic shadowing, Schema Registry shadowing, and consumer group failover. +In this lab, you deploy two single-node Redpanda clusters on Kubernetes and configure a shadow link between them. You then explore basic topic shadowing, Schema Registry shadowing, and consumer group failover. == Prerequisites @@ -50,14 +46,7 @@ cd redpanda-labs/kubernetes/shadow-linking ./setup.sh ---- + -The script does the following: - -.. Installs cert-manager for TLS certificate management -.. Deploys the Redpanda Operator -.. Creates the source and shadow Redpanda clusters -.. Configures the shadow link between clusters -.. Sets up port forwarding for local access -.. Creates rpk profiles for both clusters +The script installs cert-manager, deploys the Redpanda Operator, creates the source and shadow clusters, configures the shadow link, sets up port forwarding for local access, and creates rpk profiles for both clusters. . Verify both clusters are healthy: + @@ -67,33 +56,29 @@ rpk --profile source cluster health rpk --profile shadow cluster health ---- -== Explore the demos - -This lab includes three demo scenarios that build on each other. Run them in order to understand shadow linking capabilities. +== Explore basic topic shadowing -=== Demo 1: Basic topic shadowing - -This demo shows how topics and messages automatically replicate from the source to the shadow cluster. +This section shows how topics and messages automatically replicate from the source to the shadow cluster. . Create a topic on the source cluster: + [,bash] ---- -./demos/1-basic-shadowing/1-create-topic-on-source.sh +rpk --profile source topic create basic -p1 -r1 ---- . Produce a message to the source cluster: + [,bash] ---- -./demos/1-basic-shadowing/2-produce-record-to-source.sh +echo "Hello, world" | rpk --profile source topic produce basic ---- -. List topics on the shadow cluster to verify replication: +. Wait a few seconds for replication, then list topics on the shadow cluster: + [,bash] ---- -./demos/1-basic-shadowing/3-list-topics-on-shadow.sh +rpk --profile shadow topic list ---- + You should see the `basic` topic appear on the shadow cluster. @@ -102,86 +87,83 @@ You should see the `basic` topic appear on the shadow cluster. + [,bash] ---- -./demos/1-basic-shadowing/4-consume-messages-on-both.sh +rpk --profile source topic consume basic -n1 +rpk --profile shadow topic consume basic -n1 ---- + The same message appears on both clusters. -=== Demo 2: Schema Registry shadowing +== Explore Schema Registry shadowing -This demo shows how Avro schemas registered on the source cluster replicate to the shadow cluster, allowing shadow-side message deserialization. +This section shows how Avro schemas registered on the source cluster replicate to the shadow cluster, allowing shadow-side message deserialization. . Register an Avro schema on the source cluster: + [,bash] ---- -./demos/2-schema-shadowing/1-register-schema.sh +rpk --profile source registry schema create syslog-value --schema resources/syslog.avsc --type avro ---- . Create a topic for schema-encoded messages: + [,bash] ---- -./demos/2-schema-shadowing/2-create-topic.sh +rpk --profile source topic create syslog -p1 -r1 ---- . Produce Avro-encoded syslog records to the source cluster: + [,bash] ---- -./demos/2-schema-shadowing/3-produce-records.sh +cat resources/syslogs.json | rpk --profile source topic produce syslog --schema-id=topic ---- -. List schemas on both clusters to verify schema replication: +. Wait a few seconds, then list schemas on both clusters: + [,bash] ---- -./demos/2-schema-shadowing/4-list-schemas-on-both.sh +rpk --profile source registry subject list +rpk --profile shadow registry subject list ---- + The `syslog-value` schema appears on both clusters. -. Consume raw (encoded) messages from the shadow cluster: -+ -[,bash] ----- -./demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh ----- - . Consume decoded messages from the shadow cluster using the replicated schema: + [,bash] ---- -./demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh +rpk --profile shadow topic consume syslog -n1 --use-schema-registry -f'%v\n' | jq ---- + The shadow cluster can deserialize messages using the replicated schema. -=== Demo 3: Consumer group failover +== Explore consumer group failover -This demo shows how consumer group offsets replicate to the shadow cluster, allowing consumers to resume from their exact position after a failover. +This section shows how consumer group offsets replicate to the shadow cluster, allowing consumers to resume from their exact position after a failover. . Create a topic on the source cluster: + [,bash] ---- -./demos/3-failover/1-create-topic-on-source.sh +rpk --profile source topic create foo -p1 -r1 ---- . Produce records and consume them on the source cluster: + [,bash] ---- -./demos/3-failover/2-produce-and-consume-on-source.sh +seq 0 2 | rpk --profile source topic produce foo +rpk --profile source topic consume foo -n3 -g consumer-group-foo ---- + -This creates a consumer group and processes some messages. +This creates a consumer group and processes the messages. . View consumer group offsets on both clusters: + [,bash] ---- -./demos/3-failover/3-show-consumer-groups.sh +rpk --profile source group describe consumer-group-foo +rpk --profile shadow group describe consumer-group-foo ---- + The consumer group and its offsets are replicated to the shadow cluster. @@ -190,7 +172,8 @@ The consumer group and its offsets are replicated to the shadow cluster. + [,bash] ---- -./demos/3-failover/4-failover.sh +rpk --profile shadow shadow failover shadow-link --all --no-confirm +rpk --profile shadow shadow status shadow-link ---- + This command promotes the shadow cluster to become writable. @@ -199,17 +182,11 @@ This command promotes the shadow cluster to become writable. + [,bash] ---- -./demos/3-failover/5-produce-and-consume-on-shadow.sh +seq 3 5 | rpk --profile shadow topic produce foo +rpk --profile shadow topic consume foo -n3 -g consumer-group-foo ---- + -The consumer resumes from the last committed offset. - -. Verify the consumer group offset on the shadow cluster: -+ -[,bash] ----- -./demos/3-failover/6-show-consumer-group-on-shadow.sh ----- +The consumer resumes from the last committed offset (3) and consumes the new messages. == Clean up diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh deleted file mode 100755 index f6be8fa6..00000000 --- a/kubernetes/shadow-linking/demos/1-basic-shadowing/1-create-topic-on-source.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Create a topic -rpk --profile source topic create basic -p1 -r1 - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh deleted file mode 100755 index 756b210c..00000000 --- a/kubernetes/shadow-linking/demos/1-basic-shadowing/2-produce-record-to-source.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Produce a record -echo "Hello, world" | rpk --profile source topic produce basic - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh deleted file mode 100755 index ca1f6592..00000000 --- a/kubernetes/shadow-linking/demos/1-basic-shadowing/3-list-topics-on-shadow.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -rpk --profile shadow topic list - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh b/kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh deleted file mode 100755 index 3bfb1cb7..00000000 --- a/kubernetes/shadow-linking/demos/1-basic-shadowing/4-consume-messages-on-both.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -echo On source cluster: -rpk --profile source topic consume basic -n1 - -echo On shadow cluster: -rpk --profile shadow topic consume basic -n1 - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh deleted file mode 100755 index 4c65961e..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/1-register-schema.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Register the schema -rpk --profile source registry schema create syslog-value --schema ./resources/syslog.avsc --type avro - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh deleted file mode 100755 index c646459a..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/2-create-topic.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Create a topic -rpk --profile source topic create syslog -p1 -r1 - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh deleted file mode 100755 index bdaa1ba1..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/3-produce-records.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Produce some records -cat ./resources/syslogs.json | rpk --profile source topic produce syslog --schema-id=topic - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh deleted file mode 100755 index 3f7df1aa..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/4-list-schemas-on-both.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -echo On source cluster: -rpk --profile source registry subject list - -echo On shadow cluster: -rpk --profile shadow registry subject list - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh deleted file mode 100755 index 507067a0..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/5-consume-raw-messages-from-shadow.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -rpk --profile shadow topic consume syslog -o0 -n 1 - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh b/kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh deleted file mode 100755 index 6a291709..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/6-consume-decoded-messages-from-shadow.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -rpk --profile shadow topic consume syslog -o0 -n1 --use-schema-registry -f'%v\n' | jq - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc b/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc deleted file mode 100644 index c2127dce..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslog.avsc +++ /dev/null @@ -1,78 +0,0 @@ -{ - "type": "record", - "name": "syslog", - "fields": [ - { - "name": "name", - "type": "string" - }, - { - "name": "type", - "type": "string" - }, - { - "name": "message", - "type": "string" - }, - { - "name": "host", - "type": "string" - }, - { - "name": "version", - "type": "string" - }, - { - "name": "tag", - "type": "string" - }, - { - "name": "level", - "type": "int" - }, - { - "name": "facility", - "type": "string" - }, - { - "name": "severity", - "type": "int" - }, - { - "name": "appName", - "type": "string" - }, - { - "name": "remoteAddress", - "type": "string" - }, - { - "name": "rawMessage", - "type": "string" - }, - { - "name": "processId", - "type": "string" - }, - { - "name": "messageId", - "type": "string" - }, - { - "name": "deviceVendor", - "type": "string" - }, - { - "name": "deviceProduct", - "type": "string" - }, - { - "name": "deviceVersion", - "type": "string" - }, - { - "name": "ts", - "type": "int" - } - ] -} diff --git a/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json b/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json deleted file mode 100644 index 503c44e2..00000000 --- a/kubernetes/shadow-linking/demos/2-schema-shadowing/resources/syslogs.json +++ /dev/null @@ -1,10 +0,0 @@ -{ "name" : "bxDQHOGoKgwWaSqH", "type" : "UNKNOWN", "message" : "jfrkrvsrzg xdo", "host" : "192.0.2.1", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 36, "facility" : "syslog", "severity" : 6, "appName" : "NMDFQKIB", "remoteAddress" : "198.51.100.1", "rawMessage" : "wypcdigeqmqna", "processId" : "4065031", "messageId" : "91007", "deviceVendor" : "qgrdce", "deviceProduct" : "qtboqr", "deviceVersion" : "4.0", "ts" : 1} -{ "name" : "rHiHaQSukGYKC", "type" : "RFC5424", "message" : "qeailsfermffvdp", "host" : "192.0.2.2", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 41, "facility" : "cron", "severity" : 2, "appName" : "ACDWLYUM", "remoteAddress" : "198.51.100.2", "rawMessage" : "vjqis aaksxtm", "processId" : "751456541", "messageId" : "5926", "deviceVendor" : "zaiwvm", "deviceProduct" : "ntrxut", "deviceVersion" : "5.0", "ts" : 101} -{ "name" : "lQBbDoBSupsafcWhN", "type" : "CEF", "message" : "ztbvb cvvpgmoe", "host" : "192.0.2.3", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 2, "facility" : "authpriv", "severity" : 0, "appName" : "LKXXWTF", "remoteAddress" : "198.51.100.3", "rawMessage" : "mfnntbdbnbba", "processId" : "48791776", "messageId" : "92749", "deviceVendor" : "wppmhp", "deviceProduct" : "tvefu", "deviceVersion" : "5.0", "ts" : 201} -{ "name" : "ybiyfb", "type" : "CEF", "message" : "mrmtdau svkgs", "host" : "192.0.2.4", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 95, "facility" : "authpriv", "severity" : 4, "appName" : "BNBKAC", "remoteAddress" : "198.51.100.4", "rawMessage" : "elmpzpsjywa ", "processId" : "33157604", "messageId" : "987", "deviceVendor" : "ttmlnx", "deviceProduct" : "cqbrd", "deviceVersion" : "3.0", "ts" : 301} -{ "name" : "ScVGe", "type" : "RFC5424", "message" : "beaqygdkwyuyb", "host" : "192.0.2.5", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 118, "facility" : "authpriv", "severity" : 1, "appName" : "OURNH", "remoteAddress" : "198.51.100.5", "rawMessage" : "txtuaxthfex", "processId" : "584158260", "messageId" : "1755", "deviceVendor" : "unwtb", "deviceProduct" : "zmogds", "deviceVersion" : "5.0", "ts" : 401} -{ "name" : "RYWkuWsBiJ", "type" : "CEF", "message" : "xymvfwrcuyqo", "host" : "192.0.2.6", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 112, "facility" : "syslog", "severity" : 4, "appName" : "GENQI", "remoteAddress" : "198.51.100.6", "rawMessage" : "tpjlcpzjrgu", "processId" : "87033144", "messageId" : "491", "deviceVendor" : "nllkm", "deviceProduct" : "cqonw", "deviceVersion" : "4.0", "ts" : 501} -{ "name" : "ojajvZAMPETMB", "type" : "CEF", "message" : "ecrxleeyakltnh", "host" : "192.0.2.7", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 138, "facility" : "syslog", "severity" : 5, "appName" : "OICRHZZH", "remoteAddress" : "198.51.100.7", "rawMessage" : "chxwvqwfhd", "processId" : "44548987", "messageId" : "91172", "deviceVendor" : "nieyv", "deviceProduct" : "cbeqlc", "deviceVersion" : "5.0", "ts" : 601} -{ "name" : "NVrfOrYOhzeLjPl", "type" : "UNKNOWN", "message" : "lgsfjbezere", "host" : "192.0.2.8", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 12, "facility" : "syslog", "severity" : 1, "appName" : "XNGJP", "remoteAddress" : "198.51.100.8", "rawMessage" : "bdyxmgodfpdkcfz", "processId" : "157600899", "messageId" : "48518", "deviceVendor" : "dxvsrk", "deviceProduct" : "nwdyoe", "deviceVersion" : "2.0", "ts" : 701} -{ "name" : "zPUw", "type" : "CEF", "message" : "wduhd lkvfz", "host" : "192.0.2.9", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 57, "facility" : "authpriv", "severity" : 5, "appName" : "PGTEZJTC", "remoteAddress" : "198.51.100.9", "rawMessage" : "c exsvqslvevsm", "processId" : "2664427", "messageId" : "2314", "deviceVendor" : "nyyls", "deviceProduct" : "hgbbdo", "deviceVersion" : "4.0", "ts" : 801} -{ "name" : "CTLutuxyi", "type" : "RFC3164", "message" : "zazlbjduaqov", "host" : "192.0.2.10", "version" : "3.25.1", "tag" : ".source.s_src", "level" : 74, "facility" : "authpriv", "severity" : 6, "appName" : "JDYKJ", "remoteAddress" : "198.51.100.10", "rawMessage" : "wqbyxckvjhw", "processId" : "743558242", "messageId" : "3758", "deviceVendor" : "gjmvyn", "deviceProduct" : "vfibq", "deviceVersion" : "4.0", "ts" : 901} diff --git a/kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh b/kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh deleted file mode 100755 index aecdea17..00000000 --- a/kubernetes/shadow-linking/demos/3-failover/1-create-topic-on-source.sh +++ /dev/null @@ -1,8 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Create a topic -rpk --profile source topic create foo -p1 -r1 - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh b/kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh deleted file mode 100755 index 7365cb55..00000000 --- a/kubernetes/shadow-linking/demos/3-failover/2-produce-and-consume-on-source.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Produce some records to the source -seq 0 2 | rpk --profile source topic produce foo - -# Consume the source cluster for 10 seconds -timeout 10s rpk --profile source topic consume foo -g consumer-group-foo || true - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh b/kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh deleted file mode 100755 index a8102be7..00000000 --- a/kubernetes/shadow-linking/demos/3-failover/3-show-consumer-groups.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -echo On source cluster: -rpk --profile source group describe consumer-group-foo - -echo On shadow cluster: -rpk --profile shadow group describe consumer-group-foo - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/4-failover.sh b/kubernetes/shadow-linking/demos/3-failover/4-failover.sh deleted file mode 100755 index a0108cac..00000000 --- a/kubernetes/shadow-linking/demos/3-failover/4-failover.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -rpk --profile shadow shadow failover shadow-link --all --no-confirm -sleep 5 -rpk --profile shadow shadow status shadow-link - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh b/kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh deleted file mode 100755 index 0b8823ca..00000000 --- a/kubernetes/shadow-linking/demos/3-failover/5-produce-and-consume-on-shadow.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -# Produce some records to the shadow -seq 3 5 | rpk --profile shadow topic produce foo - -# Consume the shadow cluster for 10 seconds -timeout 10s rpk --profile shadow topic consume foo -g consumer-group-foo || true - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh b/kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh deleted file mode 100755 index 433510db..00000000 --- a/kubernetes/shadow-linking/demos/3-failover/6-show-consumer-group-on-shadow.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR 1>/dev/null 2>/dev/null - -source ../../config - -echo On shadow cluster: -rpk --profile shadow group describe consumer-group-foo - -popd 1>/dev/null 2>/dev/null diff --git a/kubernetes/shadow-linking/resources/syslog.avsc b/kubernetes/shadow-linking/resources/syslog.avsc new file mode 100644 index 00000000..b9c875f2 --- /dev/null +++ b/kubernetes/shadow-linking/resources/syslog.avsc @@ -0,0 +1,24 @@ +{ + "type": "record", + "name": "syslog", + "fields": [ + {"name": "name", "type": "string"}, + {"name": "type", "type": "string"}, + {"name": "message", "type": "string"}, + {"name": "host", "type": "string"}, + {"name": "version", "type": "string"}, + {"name": "tag", "type": "string"}, + {"name": "level", "type": "int"}, + {"name": "facility", "type": "string"}, + {"name": "severity", "type": "int"}, + {"name": "appName", "type": "string"}, + {"name": "remoteAddress", "type": "string"}, + {"name": "rawMessage", "type": "string"}, + {"name": "processId", "type": "string"}, + {"name": "messageId", "type": "string"}, + {"name": "deviceVendor", "type": "string"}, + {"name": "deviceProduct", "type": "string"}, + {"name": "deviceVersion", "type": "string"}, + {"name": "ts", "type": "int"} + ] +} diff --git a/kubernetes/shadow-linking/resources/syslogs.json b/kubernetes/shadow-linking/resources/syslogs.json new file mode 100644 index 00000000..e2da0c82 --- /dev/null +++ b/kubernetes/shadow-linking/resources/syslogs.json @@ -0,0 +1,3 @@ +{"name":"log-001","type":"RFC5424","message":"System startup complete","host":"192.0.2.10","version":"3.25.1","tag":".source.s_src","level":6,"facility":"syslog","severity":6,"appName":"SYSTEM","remoteAddress":"198.51.100.1","rawMessage":"System startup complete","processId":"1001","messageId":"10001","deviceVendor":"example","deviceProduct":"server","deviceVersion":"1.0","ts":1} +{"name":"log-002","type":"RFC5424","message":"User authentication successful","host":"192.0.2.11","version":"3.25.1","tag":".source.s_src","level":6,"facility":"authpriv","severity":6,"appName":"AUTH","remoteAddress":"198.51.100.2","rawMessage":"User authentication successful","processId":"1002","messageId":"10002","deviceVendor":"example","deviceProduct":"server","deviceVersion":"1.0","ts":2} +{"name":"log-003","type":"RFC5424","message":"Configuration updated","host":"192.0.2.12","version":"3.25.1","tag":".source.s_src","level":5,"facility":"syslog","severity":5,"appName":"CONFIG","remoteAddress":"198.51.100.3","rawMessage":"Configuration updated","processId":"1003","messageId":"10003","deviceVendor":"example","deviceProduct":"server","deviceVersion":"1.0","ts":3} From 42cb5c39757aa3a6c276cd1e8f0ca13147ccc407 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 13 Apr 2026 12:16:11 +0100 Subject: [PATCH 3/7] Fix xrefs and categories for Antora build - Use correct xref path for shadow linking docs - Add ^ to external links for new tab behavior - Fix invalid category: Replication -> Data Replication --- kubernetes/shadow-linking/README.adoc | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/kubernetes/shadow-linking/README.adoc b/kubernetes/shadow-linking/README.adoc index b190d612..c60f25d4 100644 --- a/kubernetes/shadow-linking/README.adoc +++ b/kubernetes/shadow-linking/README.adoc @@ -1,6 +1,6 @@ = Explore Shadow Linking for Disaster Recovery :env-kubernetes: true -:page-categories: Disaster Recovery, Replication +:page-categories: Disaster Recovery, Data Replication :description: Deploy two Redpanda clusters on Kubernetes and configure shadow linking to continuously replicate topics, consumer group offsets, and Schema Registry data. :page-layout: lab @@ -13,9 +13,9 @@ In this lab, you deploy two single-node Redpanda clusters on Kubernetes and conf You must have the following: * A Kubernetes cluster with adequate resources (at least 4 CPUs and 8GB memory) -* https://kubernetes.io/docs/tasks/tools/[kubectl] CLI installed -* https://helm.sh/docs/intro/install/[Helm 3+] installed -* https://docs.redpanda.com/current/get-started/rpk-install/[rpk] CLI installed +* https://kubernetes.io/docs/tasks/tools/[kubectl^] CLI installed +* https://helm.sh/docs/intro/install/[Helm 3+^] installed +* xref:ROOT:get-started:rpk-install.adoc[rpk] CLI installed == What gets deployed @@ -206,5 +206,5 @@ kubectl delete namespace source shadow rp-operator cert-manager == Suggested reading -* https://docs.redpanda.com/current/manage/kubernetes/k-shadow-linking/[Shadow linking documentation] -* https://docs.redpanda.com/current/deploy/deployment-option/self-hosted/kubernetes/[Redpanda on Kubernetes] +* xref:ROOT:manage:disaster-recovery/shadowing/index.adoc[Shadowing for Disaster Recovery] +* xref:ROOT:deploy:deployment-option/self-hosted/kubernetes/index.adoc[Redpanda on Kubernetes] From f56674689df47d267b8d7d71b105aa210e0acb3b Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 13 Apr 2026 12:19:07 +0100 Subject: [PATCH 4/7] Add kind cluster setup instructions - Add kind and Docker to prerequisites - Add step to create kind cluster before running setup - Update clean up to delete kind cluster - Add .gitignore for pid files --- kubernetes/shadow-linking/.gitignore | 1 + kubernetes/shadow-linking/README.adoc | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) create mode 100644 kubernetes/shadow-linking/.gitignore diff --git a/kubernetes/shadow-linking/.gitignore b/kubernetes/shadow-linking/.gitignore new file mode 100644 index 00000000..9fd4f9b7 --- /dev/null +++ b/kubernetes/shadow-linking/.gitignore @@ -0,0 +1 @@ +*.pid diff --git a/kubernetes/shadow-linking/README.adoc b/kubernetes/shadow-linking/README.adoc index c60f25d4..6961f412 100644 --- a/kubernetes/shadow-linking/README.adoc +++ b/kubernetes/shadow-linking/README.adoc @@ -12,10 +12,11 @@ In this lab, you deploy two single-node Redpanda clusters on Kubernetes and conf You must have the following: -* A Kubernetes cluster with adequate resources (at least 4 CPUs and 8GB memory) * https://kubernetes.io/docs/tasks/tools/[kubectl^] CLI installed * https://helm.sh/docs/intro/install/[Helm 3+^] installed * xref:ROOT:get-started:rpk-install.adoc[rpk] CLI installed +* https://kind.sigs.k8s.io/docs/user/quick-start/#installation[kind^] installed (for local testing) +* https://docs.docker.com/get-docker/[Docker^] installed and running == What gets deployed @@ -39,6 +40,13 @@ git clone https://github.com/redpanda-data/redpanda-labs.git cd redpanda-labs/kubernetes/shadow-linking ---- +. Create a local Kubernetes cluster using kind: ++ +[,bash] +---- +kind create cluster --name redpanda-shadow +---- + . Run the setup script to deploy both clusters and configure shadow linking: + [,bash] @@ -197,11 +205,11 @@ To stop port forwarding, terminate the background processes: kill $(cat local-port-forward.pid) ---- -To delete all resources, remove the namespaces: +To delete the kind cluster and all resources: [,bash] ---- -kubectl delete namespace source shadow rp-operator cert-manager +kind delete cluster --name redpanda-shadow ---- == Suggested reading From 56d61657e56b0d9733e90bdb6c7be996d6bfbd29 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 13 Apr 2026 12:29:17 +0100 Subject: [PATCH 5/7] Clean up setup script output - Add progress messages for each step - Suppress helm notes, repo updates, and K8s warnings - Add set -e to exit on errors - Show next steps on completion --- kubernetes/shadow-linking/setup.sh | 64 +++++++++++++++++------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/kubernetes/shadow-linking/setup.sh b/kubernetes/shadow-linking/setup.sh index 03f48fc1..e55552b1 100755 --- a/kubernetes/shadow-linking/setup.sh +++ b/kubernetes/shadow-linking/setup.sh @@ -1,50 +1,58 @@ #!/bin/bash +set -e + SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) -pushd $SCRIPT_DIR +pushd $SCRIPT_DIR > /dev/null source ./config -# Install Cert Manager - -helm repo add jetstack https://charts.jetstack.io -helm repo update +echo "Installing cert-manager..." +helm repo add jetstack https://charts.jetstack.io --force-update > /dev/null 2>&1 helm install cert-manager jetstack/cert-manager \ --set crds.enabled=true \ --namespace ${CERT_MANAGER_NAMESPACE} \ - --create-namespace - -# Install Redpanda Operator + --create-namespace \ + --wait \ + --quiet 2>/dev/null -helm repo add redpanda https://charts.redpanda.com -helm repo update +echo "Installing Redpanda Operator..." +helm repo add redpanda https://charts.redpanda.com --force-update > /dev/null 2>&1 helm upgrade --install redpanda-controller redpanda/operator \ --namespace ${OPERATOR_NAMESPACE} \ --create-namespace \ --version v25.3.1 \ - --set crds.enabled=true + --set crds.enabled=true \ + --wait \ + --quiet 2>/dev/null -# Install Source Cluster +echo "Deploying source cluster (this may take a few minutes)..." +kubectl apply -f resources/source-cluster.yaml > /dev/null +kubectl wait -n ${SOURCE_REDPANDA_NAMESPACE} redpanda/redpanda --for=condition=Ready --timeout=600s 2>/dev/null -kubectl apply -f resources/source-cluster.yaml -kubectl wait -n ${SOURCE_REDPANDA_NAMESPACE} redpanda/redpanda --for=condition=Ready --timeout=600s +echo "Deploying shadow cluster (this may take a few minutes)..." +kubectl apply -f resources/shadow-cluster.yaml > /dev/null +kubectl wait -n ${SHADOW_REDPANDA_NAMESPACE} redpanda/redpanda --for=condition=Ready --timeout=600s 2>/dev/null -# Install Shadow Cluster +echo "Creating shadow link..." +kubectl apply -f resources/shadow-link.yaml > /dev/null -kubectl apply -f resources/shadow-cluster.yaml -kubectl wait -n ${SHADOW_REDPANDA_NAMESPACE} redpanda/redpanda --for=condition=Ready --timeout=600s +echo "Setting up port forwarding and rpk profiles..." +kubectl port-forward pod/redpanda-0 -n $SOURCE_REDPANDA_NAMESPACE 19094:9094 19644:9644 18081:8081 > /dev/null 2>&1 & +echo $! > local-port-forward.pid -# Create Shadow Link +kubectl port-forward pod/redpanda-0 -n $SHADOW_REDPANDA_NAMESPACE 29094:9094 29644:9644 28081:8081 > /dev/null 2>&1 & +echo $! >> local-port-forward.pid -kubectl apply -f resources/shadow-link.yaml +sleep 2 -kubectl port-forward pod/redpanda-0 -n $SOURCE_REDPANDA_NAMESPACE 19094:9094 19644:9644 18081:8081 1>/dev/null 2>/dev/null & -echo $! > local-port-forward.pid -rpk profile create source -s brokers=localhost:19094 -s admin.hosts=localhost:19644 -s registry.hosts=http://localhost:18081/ 1>/dev/null 2>/dev/null || rpk profile use source -rpk cluster health +rpk profile create source -s brokers=localhost:19094 -s admin.hosts=localhost:19644 -s registry.hosts=http://localhost:18081/ > /dev/null 2>&1 || rpk profile use source > /dev/null 2>&1 +rpk profile create shadow -s brokers=localhost:29094 -s admin.hosts=localhost:29644 -s registry.hosts=http://localhost:28081/ > /dev/null 2>&1 || rpk profile use shadow > /dev/null 2>&1 -kubectl port-forward pod/redpanda-0 -n $SHADOW_REDPANDA_NAMESPACE 29094:9094 29644:9644 28081:8081 1>/dev/null 2>/dev/null & -echo $! >> local-port-forward.pid -rpk profile create shadow -s brokers=localhost:29094 -s admin.hosts=localhost:29644 -s registry.hosts=http://localhost:28081/ 1>/dev/null 2>/dev/null || rpk profile use shadow -rpk cluster health +popd > /dev/null -popd +echo "" +echo "Setup complete! Both clusters are ready." +echo "" +echo "Verify cluster health:" +echo " rpk --profile source cluster health" +echo " rpk --profile shadow cluster health" From 5ec15fddffe585192d1fcf6da3bf324095bad2cb Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 13 Apr 2026 12:30:35 +0100 Subject: [PATCH 6/7] Make cert-manager install idempotent Use helm upgrade --install so script can be re-run --- kubernetes/shadow-linking/setup.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kubernetes/shadow-linking/setup.sh b/kubernetes/shadow-linking/setup.sh index e55552b1..2b5d8493 100755 --- a/kubernetes/shadow-linking/setup.sh +++ b/kubernetes/shadow-linking/setup.sh @@ -8,7 +8,7 @@ source ./config echo "Installing cert-manager..." helm repo add jetstack https://charts.jetstack.io --force-update > /dev/null 2>&1 -helm install cert-manager jetstack/cert-manager \ +helm upgrade --install cert-manager jetstack/cert-manager \ --set crds.enabled=true \ --namespace ${CERT_MANAGER_NAMESPACE} \ --create-namespace \ From 8204aad6086ffc0da00569d23b85bbadc9ec8565 Mon Sep 17 00:00:00 2001 From: JakeSCahill Date: Mon, 13 Apr 2026 13:05:40 +0100 Subject: [PATCH 7/7] Fix script --- kubernetes/shadow-linking/setup.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/kubernetes/shadow-linking/setup.sh b/kubernetes/shadow-linking/setup.sh index 2b5d8493..1528f7a2 100755 --- a/kubernetes/shadow-linking/setup.sh +++ b/kubernetes/shadow-linking/setup.sh @@ -12,8 +12,7 @@ helm upgrade --install cert-manager jetstack/cert-manager \ --set crds.enabled=true \ --namespace ${CERT_MANAGER_NAMESPACE} \ --create-namespace \ - --wait \ - --quiet 2>/dev/null + --wait > /dev/null 2>&1 echo "Installing Redpanda Operator..." helm repo add redpanda https://charts.redpanda.com --force-update > /dev/null 2>&1 @@ -22,8 +21,7 @@ helm upgrade --install redpanda-controller redpanda/operator \ --create-namespace \ --version v25.3.1 \ --set crds.enabled=true \ - --wait \ - --quiet 2>/dev/null + --wait > /dev/null 2>&1 echo "Deploying source cluster (this may take a few minutes)..." kubectl apply -f resources/source-cluster.yaml > /dev/null