diff --git a/docker-compose/README.adoc b/docker-compose/README.adoc index 031f5b73..ae048b4a 100644 --- a/docker-compose/README.adoc +++ b/docker-compose/README.adoc @@ -7,4 +7,5 @@ - link:./three-brokers/README.adoc[Three brokers] - link:./owl-shop/README.adoc[Owl shop streaming application] - link:./console-plain-login/README.adoc[Redpanda Console with Plain Login Authentication] -- link:./cdc/README.adoc[Change data capture (CDC) with Debezium] \ No newline at end of file +- link:./cdc/README.adoc[Change data capture (CDC) with Debezium] +- link:./redpanda-shadow-demo/README.md[Shadow linking for disaster recovery and data replication] \ No newline at end of file diff --git a/docker-compose/redpanda-shadow-demo/.gitignore b/docker-compose/redpanda-shadow-demo/.gitignore new file mode 100644 index 00000000..7be59e24 --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/.gitignore @@ -0,0 +1,16 @@ +# Docker volumes +*.log + +# Temporary files +*.tmp +*.swp +*~ + +# OS files +.DS_Store +Thumbs.db + +# IDE +.vscode/ +.idea/ +*.iml diff --git a/docker-compose/redpanda-shadow-demo/Makefile b/docker-compose/redpanda-shadow-demo/Makefile new file mode 100644 index 00000000..e4c0707d --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/Makefile @@ -0,0 +1,63 @@ +.PHONY: help start stop restart logs status setup demo check clean + +help: + @echo "Redpanda Shadow Linking Demo - Available Commands" + @echo "" + @echo " make start - Start all services" + @echo " make stop - Stop all services" + @echo " make restart - Restart all services" + @echo " make logs - View logs from all services" + @echo " make status - Check service status" + @echo " make setup - Initialize shadow link" + @echo " make demo - Produce demo data" + @echo " make check - Check replication status" + @echo " make clean - Stop and remove all containers and volumes" + @echo "" + @echo "Quick Start:" + @echo " 1. make start" + @echo " 2. make setup" + @echo " 3. make demo" + @echo " 4. make check" + +start: + @echo "Starting Redpanda shadow demo environment..." + docker compose up -d + @echo "" + @echo "Waiting for services to be ready..." + @sleep 5 + @docker compose ps + @echo "" + @echo "Services started!" + @echo " Source Console: http://localhost:8080" + @echo " Shadow Console: http://localhost:8081" + +stop: + @echo "Stopping services..." + docker compose stop + +restart: + @echo "Restarting services..." + docker compose restart + +logs: + docker compose logs -f + +status: + @docker compose ps + +setup: + @echo "Initializing shadow link..." + docker exec rpk-client /scripts/setup-shadow-link.sh + +demo: + @echo "Producing demo data..." + docker exec rpk-client /scripts/demo-produce.sh + +check: + @echo "Checking replication status..." + docker exec rpk-client /scripts/check-replication.sh + +clean: + @echo "Stopping and removing all containers and volumes..." + docker compose down -v + @echo "Cleanup complete!" diff --git a/docker-compose/redpanda-shadow-demo/README.md b/docker-compose/redpanda-shadow-demo/README.md new file mode 100644 index 00000000..6a55684d --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/README.md @@ -0,0 +1,442 @@ +# Redpanda Shadow Linking Demo + +This demo environment showcases Redpanda's shadow linking feature for disaster recovery and data replication between two clusters. + +## Architecture + +The demo includes: + +- **Source Cluster** (redpanda-source) + - Kafka API: localhost:19092 + - Schema Registry: localhost:18081 + - Admin API: localhost:19644 + - Console UI: http://localhost:8080 + +- **Shadow Cluster** (redpanda-shadow) + - Kafka API: localhost:29092 + - Schema Registry: localhost:28081 + - Admin API: localhost:29644 + - Console UI: http://localhost:8081 + +- **RPK Client Container** - Pre-configured client for running commands + +## Prerequisites + +- Docker and Docker Compose installed +- Redpanda Enterprise Edition license (required for shadow linking) + - Both clusters are configured with `enable_shadow_linking=true` + - For production use, you'll need valid enterprise licenses + +## Quick Start + +```bash +# 1. Start all services +make start + +# 2. Initialize shadow link and create demo topics +make setup + +# 3. Produce sample data to source cluster +make demo + +# 4. Verify replication to shadow cluster +make check +``` + +**Console URLs:** +- Source: http://localhost:8080 +- Shadow: http://localhost:8081 + +That's it! The shadow link is now replicating data from source to shadow cluster. + +## Detailed Walkthrough + +### 1. Start the Environment + +```bash +make start +``` + +**What happens:** +- Starts all Docker containers (2 Redpanda clusters, 2 consoles, 1 rpk client) +- Waits for services to initialize +- Displays service status and console URLs + +### 2. Initialize Shadow Link + +```bash +make setup +``` + +**What happens under the covers:** + +1. **Check cluster health** - Waits for both clusters to be ready: + ```bash + rpk cluster health -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + rpk cluster health -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + *Verifies that both Redpanda clusters are healthy and accepting connections* + +2. **Verify shadow linking is enabled** - Checks cluster configuration: + ```bash + rpk cluster config get enable_shadow_linking -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + rpk cluster config get enable_shadow_linking -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + *Confirms that shadow linking feature is enabled on both clusters (Enterprise feature)* + +3. **Create demo topics** - Creates topics on the source cluster: + ```bash + rpk topic create demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --partitions 3 --replicas 1 + rpk topic create demo-metrics -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --partitions 1 --replicas 1 + ``` + *Creates two topics: demo-events (3 partitions) and demo-metrics (1 partition)* + +4. **List topics** - Displays created topics: + ```bash + rpk topic list -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + ``` + *Shows all topics on the source cluster* + +5. **Create shadow link** - Establishes replication from shadow to source: + ```bash + rpk shadow create --config-file shadow-link.yaml --no-confirm -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + *Creates a shadow link named "demo-shadow-link" that pulls data from source cluster to shadow cluster based on the configuration in shadow-link.yaml* + +6. **Check shadow link status** - Verifies the link is active: + ```bash + rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + *Displays the current status of the shadow link including active tasks and replication progress* + +### 3. Produce Sample Data + +```bash +make demo +``` + +**What happens under the covers:** + +1. **Produce events to demo-events** - Writes 10 JSON messages: + ```bash + echo '{"event_id": 1, "timestamp": "...", "message": "Sample event 1"}' | \ + rpk topic produce demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + ``` + *Produces 10 sample event messages to the demo-events topic on the source cluster (one per second)* + +2. **Produce metrics to demo-metrics** - Writes 5 JSON messages: + ```bash + echo '{"metric_id": 1, "value": 42, "timestamp": "..."}' | \ + rpk topic produce demo-metrics -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + ``` + *Produces 5 sample metric messages to the demo-metrics topic on the source cluster (one per second)* + +The shadow link automatically replicates this data to the shadow cluster. + +### 4. Verify Replication + +```bash +make check +``` + +**What happens under the covers:** + +1. **Check shadow link status** - Displays detailed replication status: + ```bash + rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + *Shows shadow link health, active replication tasks, and any lag or errors* + +2. **List topics on both clusters** - Compare topic lists: + ```bash + rpk topic list -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + rpk topic list -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + *Lists all topics on source and shadow clusters to verify topics have been replicated* + +3. **Compare message counts** - Verify data has been replicated: + ```bash + rpk topic describe demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --print-partitions + rpk topic describe demo-events -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 --print-partitions + ``` + *Gets high watermark (message count) for each partition on both clusters and compares them to verify data is in sync* + +## Additional Make Commands + +### View Logs + +```bash +make logs +``` +*Streams logs from all containers (both Redpanda clusters and consoles) - useful for debugging* + +### Check Service Status + +```bash +make status +``` +*Shows Docker container status for all services including health status and port mappings* + +### Stop Services + +```bash +make stop +``` +*Stops all running containers without removing them - data is preserved* + +### Restart Services + +```bash +make restart +``` +*Restarts all containers - useful after configuration changes* + +### Clean Up Everything + +```bash +make clean +``` +*Stops and removes all containers and volumes - completely resets the demo environment* + +## Manual RPK Commands + +All commands below can be run from your host machine using: +```bash +docker exec rpk-client +``` + +### Check Shadow Link Status + +```bash +rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 +``` +*Displays overview, task status, and detailed topic replication status for the shadow link* + +### List Topics on Each Cluster + +Source cluster: +```bash +rpk topic list -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 +``` +*Lists all topics on the source cluster* + +Shadow cluster: +```bash +rpk topic list -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 +``` +*Lists all topics on the shadow cluster (should match source topics based on filters)* + +### Describe Topic Details + +```bash +rpk topic describe demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --print-partitions +``` +*Shows partition details including leader, replicas, log start offset, and high watermark* + +### Consume Messages + +From source: +```bash +rpk topic consume demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --num 10 +``` +*Reads the first 10 messages from the demo-events topic on the source cluster* + +From shadow: +```bash +rpk topic consume demo-events -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 --num 10 +``` +*Reads the first 10 messages from the demo-events topic on the shadow cluster (should match source)* + +### Produce Additional Messages + +```bash +echo '{"test": "message", "timestamp": "2024-01-01T00:00:00Z"}' | \ + rpk topic produce demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 +``` +*Produces a single message to the source cluster - will be automatically replicated to shadow* + +### Delete Shadow Link + +```bash +rpk shadow delete demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 +``` +*Removes the shadow link - stops replication but preserves existing replicated data* + +## Using Redpanda Console + +Access the web UIs to visualize topics, messages, and shadow link status: + +- **Source Console**: http://localhost:8080 +- **Shadow Console**: http://localhost:8081 + +In each console, you can: +- Browse topics and view messages in real-time +- Monitor consumer groups and lag +- View cluster health and broker status +- Inspect topic configurations +- See replication lag (on shadow cluster) + +## Shadow Link Configuration + +The shadow link configuration is in `config/shadow-link.yaml`. Key settings: + +```yaml +name: demo-shadow-link + +client_options: + bootstrap_servers: + - redpanda-source:9092 + +topic_metadata_sync_options: + interval: 10s + auto_create_shadow_topic_filters: + - pattern_type: PREFIX + filter_type: INCLUDE + name: demo- + start_at_earliest: {} + +consumer_offset_sync_options: + interval: 10s + group_filters: + - pattern_type: LITERAL + filter_type: INCLUDE + name: '*' + +schema_registry_sync_options: + shadow_schema_registry_topic: {} +``` + +**Configuration breakdown:** +- **name**: Identifier for this shadow link +- **bootstrap_servers**: Source cluster connection details +- **topic_metadata_sync_options**: Controls which topics are replicated (PREFIX filter for "demo-*" topics) +- **start_at_earliest**: Replicates from the beginning of topics +- **consumer_offset_sync_options**: Replicates consumer group offsets +- **schema_registry_sync_options**: Enables schema registry replication + +### Modifying the Configuration + +To change what gets replicated: + +1. Edit `config/shadow-link.yaml` +2. Delete the existing shadow link: + ```bash + docker exec rpk-client rpk shadow delete demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` +3. Recreate with new config: + ```bash + docker exec rpk-client rpk shadow create --config-file /config/shadow-link.yaml --no-confirm -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + +### Generate Custom Configuration + +To generate a new configuration template: +```bash +docker exec rpk-client rpk shadow config generate -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 +``` + +## Demonstrating Disaster Recovery + +### Scenario: Source Cluster Failure + +1. Produce data to source cluster: + ```bash + make demo + ``` + +2. Verify replication to shadow: + ```bash + make check + ``` + +3. Simulate source failure: + ```bash + docker compose stop redpanda-source + ``` + +4. Applications can now read from shadow cluster at `localhost:29092`: + ```bash + docker exec rpk-client rpk topic consume demo-events -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 --num 10 + ``` + +5. Restore source cluster: + ```bash + docker compose start redpanda-source + ``` + +### Scenario: Testing Lag and Catch-Up + +1. Stop the shadow cluster: + ```bash + docker compose stop redpanda-shadow + ``` + +2. Produce large amounts of data to source: + ```bash + for i in {1..100}; do + echo "{\"id\": $i}" | docker exec -i rpk-client rpk topic produce demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + done + ``` + +3. Restart shadow and watch it catch up: + ```bash + docker compose start redpanda-shadow + sleep 10 + make check + ``` + +## Troubleshooting + +### Shadow link creation fails + +- Verify both clusters are healthy: `make status` +- Check logs: `docker compose logs redpanda-shadow` +- Ensure `enable_shadow_linking=true` on both clusters: + ```bash + docker exec rpk-client rpk cluster config get enable_shadow_linking -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + ``` + +### Replication not working + +1. Check shadow link status: + ```bash + docker exec rpk-client rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + ``` + +2. Check shadow cluster logs: + ```bash + docker compose logs -f redpanda-shadow + ``` + +3. Verify network connectivity between clusters: + ```bash + docker exec redpanda-shadow nc -zv redpanda-source 9092 + ``` + +### Topics not appearing on shadow + +- Verify topic names match the filter pattern in `shadow-link.yaml` (must start with "demo-") +- Check that topics exist on source before creating shadow link +- Topics are created automatically on shadow when they match filters + +### Console not accessible + +- Verify consoles are running: `make status` +- Check console logs: `docker compose logs console-source console-shadow` +- Ensure ports 8080 and 8081 are not already in use + +## Additional Resources + +- [Redpanda Shadow Linking Documentation](https://docs.redpanda.com/current/manage/disaster-recovery/shadowing/setup/) +- [Redpanda Enterprise Features](https://docs.redpanda.com/current/get-started/licenses/) +- [RPK Shadow Commands](https://docs.redpanda.com/current/reference/rpk/rpk-shadow/) +- [RPK Command Reference](https://docs.redpanda.com/current/reference/rpk/) + +## Notes + +- This demo uses dev-container mode with single brokers for simplicity +- For production use, configure proper replication factors (typically 3) and cluster sizing +- Shadow linking is pull-based replication (shadow pulls from source) +- Network connectivity must allow shadow cluster to reach source cluster +- Authentication and TLS can be added to the shadow-link.yaml configuration +- The `-X brokers=` and `-X admin.hosts=` flags override rpk configuration for connecting to specific clusters diff --git a/docker-compose/redpanda-shadow-demo/config/console-shadow.yml b/docker-compose/redpanda-shadow-demo/config/console-shadow.yml new file mode 100644 index 00000000..9dcd833a --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/config/console-shadow.yml @@ -0,0 +1,14 @@ +kafka: + brokers: + - redpanda-shadow:9092 + +schemaRegistry: + enabled: true + urls: + - http://redpanda-shadow:8081 + +redpanda: + adminApi: + enabled: true + urls: + - http://redpanda-shadow:9644 diff --git a/docker-compose/redpanda-shadow-demo/config/console-source.yml b/docker-compose/redpanda-shadow-demo/config/console-source.yml new file mode 100644 index 00000000..9bc080a1 --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/config/console-source.yml @@ -0,0 +1,14 @@ +kafka: + brokers: + - redpanda-source:9092 + +schemaRegistry: + enabled: true + urls: + - http://redpanda-source:8081 + +redpanda: + adminApi: + enabled: true + urls: + - http://redpanda-source:9644 diff --git a/docker-compose/redpanda-shadow-demo/config/shadow-link.yaml b/docker-compose/redpanda-shadow-demo/config/shadow-link.yaml new file mode 100644 index 00000000..b1db6238 --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/config/shadow-link.yaml @@ -0,0 +1,44 @@ +name: demo-shadow-link + +# Source cluster connection +client_options: + bootstrap_servers: + - redpanda-source:9092 + +# Topic replication settings +topic_metadata_sync_options: + interval: 10s + auto_create_shadow_topic_filters: + - pattern_type: PREFIX + filter_type: INCLUDE + name: demo- + synced_shadow_topic_properties: + - retention.ms + - segment.ms + start_at_earliest: {} + +# Consumer group offset replication +consumer_offset_sync_options: + interval: 10s + group_filters: + - pattern_type: LITERAL + filter_type: INCLUDE + name: '*' + +# ACL replication (optional) +security_sync_options: + interval: 30s + acl_filters: + - resource_filter: + resource_type: TOPIC + pattern_type: PREFIXED + name: demo- + access_filter: + principal: '*' + operation: ANY + permission_type: ALLOW + host: '*' + +# Schema registry replication +schema_registry_sync_options: + shadow_schema_registry_topic: {} diff --git a/docker-compose/redpanda-shadow-demo/docker-compose.yml b/docker-compose/redpanda-shadow-demo/docker-compose.yml new file mode 100644 index 00000000..bd2d9e18 --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/docker-compose.yml @@ -0,0 +1,131 @@ +services: + # Source Cluster + redpanda-source: + image: docker.redpanda.com/redpandadata/redpanda:v25.3.1 + container_name: redpanda-source + hostname: redpanda-source + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092 + - --advertise-kafka-addr internal://redpanda-source:9092,external://localhost:19092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082 + - --advertise-pandaproxy-addr internal://redpanda-source:8082,external://localhost:18082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081 + - --rpc-addr redpanda-source:33145 + - --advertise-rpc-addr redpanda-source:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + - --set redpanda.enable_shadow_linking=true + ports: + - "19092:19092" # Kafka API + - "18081:18081" # Schema Registry + - "18082:18082" # HTTP Proxy + - "19644:9644" # Admin API + networks: + - redpanda-network + volumes: + - redpanda-source-data:/var/lib/redpanda/data + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s + + console-source: + image: docker.redpanda.com/redpandadata/console:v3.3.2 + container_name: console-source + depends_on: + - redpanda-source + ports: + - "8080:8080" + networks: + - redpanda-network + environment: + CONFIG_FILEPATH: /tmp/config.yml + KAFKA_BROKERS: redpanda-source:9092 + KAFKA_SCHEMAREGISTRY_ENABLED: "true" + KAFKA_SCHEMAREGISTRY_URLS: http://redpanda-source:8081 + REDPANDA_ADMINAPI_ENABLED: "true" + REDPANDA_ADMINAPI_URLS: http://redpanda-source:9644 + volumes: + - ./config/console-source.yml:/tmp/config.yml + + # Shadow Cluster + redpanda-shadow: + image: docker.redpanda.com/redpandadata/redpanda:v25.3.1 + container_name: redpanda-shadow + hostname: redpanda-shadow + command: + - redpanda + - start + - --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092 + - --advertise-kafka-addr internal://redpanda-shadow:9092,external://localhost:29092 + - --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082 + - --advertise-pandaproxy-addr internal://redpanda-shadow:8082,external://localhost:28082 + - --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081 + - --rpc-addr redpanda-shadow:33145 + - --advertise-rpc-addr redpanda-shadow:33145 + - --mode dev-container + - --smp 1 + - --default-log-level=info + - --set redpanda.enable_shadow_linking=true + ports: + - "29092:29092" # Kafka API + - "28081:28081" # Schema Registry + - "28082:28082" # HTTP Proxy + - "29644:9644" # Admin API + networks: + - redpanda-network + volumes: + - redpanda-shadow-data:/var/lib/redpanda/data + healthcheck: + test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"] + interval: 15s + timeout: 10s + retries: 5 + start_period: 30s + + console-shadow: + image: docker.redpanda.com/redpandadata/console:v3.3.2 + container_name: console-shadow + depends_on: + - redpanda-shadow + ports: + - "8081:8080" + networks: + - redpanda-network + environment: + CONFIG_FILEPATH: /tmp/config.yml + KAFKA_BROKERS: redpanda-shadow:9092 + KAFKA_SCHEMAREGISTRY_ENABLED: "true" + KAFKA_SCHEMAREGISTRY_URLS: http://redpanda-shadow:8081 + REDPANDA_ADMINAPI_ENABLED: "true" + REDPANDA_ADMINAPI_URLS: http://redpanda-shadow:9644 + volumes: + - ./config/console-shadow.yml:/tmp/config.yml + + # Client container for testing and setup + rpk-client: + image: docker.redpanda.com/redpandadata/redpanda:v25.3.1 + container_name: rpk-client + depends_on: + - redpanda-source + - redpanda-shadow + networks: + - redpanda-network + volumes: + - ./config:/config + - ./scripts:/scripts + entrypoint: /bin/bash + command: -c "tail -f /dev/null" + +networks: + redpanda-network: + driver: bridge + +volumes: + redpanda-source-data: + redpanda-shadow-data: diff --git a/docker-compose/redpanda-shadow-demo/scripts/check-replication.sh b/docker-compose/redpanda-shadow-demo/scripts/check-replication.sh new file mode 100755 index 00000000..49c2debf --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/scripts/check-replication.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +echo "===================================" +echo "Shadow Link Replication Status" +echo "===================================" + +# Check shadow link status +echo "Shadow link details:" +rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + +echo "" +echo "===================================" +echo "Topic Comparison" +echo "===================================" + +# Compare topics on both clusters +echo "" +echo "Topics on SOURCE cluster:" +rpk topic list -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + +echo "" +echo "Topics on SHADOW cluster:" +rpk topic list -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + +echo "" +echo "===================================" +echo "Message Count Comparison" +echo "===================================" + +# Check message counts +for topic in demo-events demo-metrics; do + echo "" + echo "Topic: $topic" + + SOURCE_COUNT=$(rpk topic describe $topic -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --print-partitions 2>/dev/null | grep -v "PARTITION" | awk '{sum += $6} END {print sum}') + SHADOW_COUNT=$(rpk topic describe $topic -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 --print-partitions 2>/dev/null | grep -v "PARTITION" | awk '{sum += $6} END {print sum}') + + echo " Source messages: ${SOURCE_COUNT:-0}" + echo " Shadow messages: ${SHADOW_COUNT:-0}" + + if [ "${SOURCE_COUNT:-0}" -eq "${SHADOW_COUNT:-0}" ]; then + echo " Status: ✓ In sync" + else + echo " Status: ⚠ Replication in progress or lag detected" + fi +done + +echo "" +echo "===================================" diff --git a/docker-compose/redpanda-shadow-demo/scripts/demo-produce.sh b/docker-compose/redpanda-shadow-demo/scripts/demo-produce.sh new file mode 100755 index 00000000..cda95193 --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/scripts/demo-produce.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +echo "===================================" +echo "Producing sample data to source cluster" +echo "===================================" + +# Produce some sample events +echo "Producing events to demo-events topic..." +for i in {1..10}; do + echo "{\"event_id\": $i, \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\", \"message\": \"Sample event $i\"}" | \ + rpk topic produce demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + sleep 1 +done + +echo "" +echo "Producing metrics to demo-metrics topic..." +for i in {1..5}; do + echo "{\"metric_id\": $i, \"value\": $((RANDOM % 100)), \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"}" | \ + rpk topic produce demo-metrics -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + sleep 1 +done + +echo "" +echo "===================================" +echo "Data production complete!" +echo "===================================" +echo "" +echo "Check data on source cluster:" +echo " docker exec rpk-client rpk topic consume demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --num 10" +echo "" +echo "Check data replicated to shadow cluster:" +echo " docker exec rpk-client rpk topic consume demo-events -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 --num 10" +echo "" +echo "View in consoles:" +echo " Source: http://localhost:8080/topics/demo-events" +echo " Shadow: http://localhost:8081/topics/demo-events" +echo "===================================" diff --git a/docker-compose/redpanda-shadow-demo/scripts/setup-shadow-link.sh b/docker-compose/redpanda-shadow-demo/scripts/setup-shadow-link.sh new file mode 100755 index 00000000..8685a89c --- /dev/null +++ b/docker-compose/redpanda-shadow-demo/scripts/setup-shadow-link.sh @@ -0,0 +1,71 @@ +#!/bin/bash + +set -e + +echo "===================================" +echo "Shadow Link Setup Script" +echo "===================================" + +# Wait for clusters to be ready +echo "Waiting for source cluster to be ready..." +until rpk cluster health -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 2>/dev/null | grep -q "Healthy.*true"; do + echo " Waiting for source cluster..." + sleep 2 +done +echo "Source cluster is healthy!" + +echo "Waiting for shadow cluster to be ready..." +until rpk cluster health -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 2>/dev/null | grep -q "Healthy.*true"; do + echo " Waiting for shadow cluster..." + sleep 2 +done +echo "Shadow cluster is healthy!" + +# Verify shadow linking is enabled on both clusters +echo "" +echo "Verifying shadow linking configuration..." +SOURCE_ENABLED=$(rpk cluster config get enable_shadow_linking -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644) +SHADOW_ENABLED=$(rpk cluster config get enable_shadow_linking -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644) + +echo "Source cluster shadow linking enabled: $SOURCE_ENABLED" +echo "Shadow cluster shadow linking enabled: $SHADOW_ENABLED" + +# Create some demo topics on the source cluster +echo "" +echo "Creating demo topics on source cluster..." +rpk topic create demo-events -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --partitions 3 --replicas 1 2>/dev/null || echo " demo-events already exists" +rpk topic create demo-metrics -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 --partitions 1 --replicas 1 2>/dev/null || echo " demo-metrics already exists" + +echo "" +echo "Topics on source cluster:" +rpk topic list -X brokers=redpanda-source:9092 -X admin.hosts=redpanda-source:9644 + +# Create the shadow link from the shadow cluster +echo "" +echo "Creating shadow link from shadow cluster..." +cd /config +if rpk shadow create --config-file shadow-link.yaml --no-confirm -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644; then + echo "" + echo "Shadow link created successfully!" +else + echo "" + echo "Shadow link already exists or creation failed - checking status..." +fi +echo "" +echo "Verifying shadow link status..." +rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644 + +echo "" +echo "===================================" +echo "Setup Complete!" +echo "===================================" +echo "Source cluster: redpanda-source:9092 (external: localhost:19092)" +echo "Shadow cluster: redpanda-shadow:9092 (external: localhost:29092)" +echo "" +echo "Console URLs:" +echo " Source: http://localhost:8080" +echo " Shadow: http://localhost:8081" +echo "" +echo "To check shadow link status:" +echo " docker exec rpk-client rpk shadow status demo-shadow-link -X brokers=redpanda-shadow:9092 -X admin.hosts=redpanda-shadow:9644" +echo "==================================="