From ecf1af0f0424feacdd62359877fff2fdf7f2647b Mon Sep 17 00:00:00 2001 From: ulya-sidorina Date: Fri, 28 Nov 2025 12:22:20 +0100 Subject: [PATCH] feat(deployment): support db schema initialization and migration via goose --- .../integration/scripts/create_test_tables.sh | 3 --- docker-compose.yaml | 27 ++++++++++++++++--- .../connectors/db/yql/schema/drop_tables.yql | 3 --- .../connectors/db/yql/schema/fill_tables.yql | 14 ---------- .../yql/20251126000000_init_schema.sql | 18 ++++++++++--- .../20251127120000_add_root_path_columns.sql | 14 ++++++++++ 6 files changed, 53 insertions(+), 26 deletions(-) rename init_db/create_tables.sh => cmd/integration/scripts/create_test_tables.sh (86%) delete mode 100644 internal/connectors/db/yql/schema/drop_tables.yql delete mode 100644 internal/connectors/db/yql/schema/fill_tables.yql rename internal/connectors/db/yql/schema/create_tables.yql => migrations/yql/20251126000000_init_schema.sql (85%) create mode 100644 migrations/yql/20251127120000_add_root_path_columns.sql diff --git a/init_db/create_tables.sh b/cmd/integration/scripts/create_test_tables.sh similarity index 86% rename from init_db/create_tables.sh rename to cmd/integration/scripts/create_test_tables.sh index 3bb0505a..eb0cc5c4 100755 --- a/init_db/create_tables.sh +++ b/cmd/integration/scripts/create_test_tables.sh @@ -1,6 +1,3 @@ -# create ydbcp tables -./ydb -e ${YDB_ENDPOINT} -d /local scripting yql -f init_db/schema/create_tables.yql - # create and fill user table kv_test ./ydb -e ${YDB_ENDPOINT} -d /local workload kv init ./ydb -e ${YDB_ENDPOINT} -d /local workload kv run upsert --rows 100 diff --git a/docker-compose.yaml b/docker-compose.yaml index 742eebdf..bcb64fd3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -21,20 +21,41 @@ services: networks: - ydbcp-net + init_ydb_schema: + image: golang:1.23-alpine + platform: linux/amd64 + container_name: ${YDB_NAME}-init-schema + volumes: + - ./migrations:/migrations + environment: + - YDB_ENDPOINT=grpc://${YDB_NAME}:2136 + - YDB_DATABASE=/local + - GOOSE_YDB_PARAMS=go_query_mode=scripting&go_fake_tx=scripting&go_query_bind=declare,numeric + depends_on: + ydb: + condition: service_healthy + restart: "no" + command: > + sh -c "go install github.com/pressly/goose/v3/cmd/goose@latest && + goose -dir /migrations/yql ydb \"$$YDB_ENDPOINT/$$YDB_DATABASE?$$GOOSE_YDB_PARAMS\" up" + networks: + - ydbcp-net + setup_ydb: image: cr.yandex/yc/yandex-docker-local-ydb:24.1 platform: linux/amd64 container_name: ${YDB_NAME}-setup volumes: - - ./init_db:/init_db - - ./internal/connectors/db/yql/schema:/init_db/schema + - ./cmd/integration/scripts:/scripts environment: - YDB_ENDPOINT=grpc://${YDB_NAME}:2136 depends_on: ydb: condition: service_healthy + init_ydb_schema: + condition: service_completed_successfully restart: "no" - command: bash -c "chmod +x ./init_db/create_tables.sh && ./init_db/create_tables.sh" + command: bash -c "chmod +x ./scripts/create_test_tables.sh && ./scripts/create_test_tables.sh" networks: - ydbcp-net diff --git a/internal/connectors/db/yql/schema/drop_tables.yql b/internal/connectors/db/yql/schema/drop_tables.yql deleted file mode 100644 index 74619965..00000000 --- a/internal/connectors/db/yql/schema/drop_tables.yql +++ /dev/null @@ -1,3 +0,0 @@ -DROP TABLE Backups; -DROP TABLE OperationTypes; -DROP TABLE Operations; \ No newline at end of file diff --git a/internal/connectors/db/yql/schema/fill_tables.yql b/internal/connectors/db/yql/schema/fill_tables.yql deleted file mode 100644 index 85517250..00000000 --- a/internal/connectors/db/yql/schema/fill_tables.yql +++ /dev/null @@ -1,14 +0,0 @@ -UPSERT INTO `OperationTypes` (code, description, is_cancellable) VALUES - ('TB', 'Take backup', True), - ('RB', 'Restore backup', True), - ('RM', 'Remove backup', False); - ---for testing purposes -UPSERT INTO `Backups` (id, container_id, database, status, endpoint, initiated, created_at, completed_at) VALUES - ('12345678-1234-5678-1234-567812345678', '', '', 'PENDING', '', 'author', CurrentUtcTimestamp(), CurrentUtcTimestamp()); - -UPSERT INTO `Operations` (id, container_id, database, type, status, operation_id, backup_id, created_at, endpoint) VALUES - ('11111111-1111-1111-1111-111111111111', '', '', 'TB', 'DONE', '', '11111111-1111-1111-1111-111111111112', CurrentUtcTimestamp(), ''); - -UPSERT INTO `Operations` (id, container_id, database, type, status, operation_id, backup_id, created_at, endpoint) VALUES - ('11111111-1111-1111-1111-111111111112', '', '', 'TB', 'DONE', '', '12345678-1234-5678-1234-567812345678', CurrentUtcTimestamp(), ''); diff --git a/internal/connectors/db/yql/schema/create_tables.yql b/migrations/yql/20251126000000_init_schema.sql similarity index 85% rename from internal/connectors/db/yql/schema/create_tables.yql rename to migrations/yql/20251126000000_init_schema.sql index 20c8e1c3..f0446216 100644 --- a/internal/connectors/db/yql/schema/create_tables.yql +++ b/migrations/yql/20251126000000_init_schema.sql @@ -1,3 +1,4 @@ +-- +goose Up CREATE TABLE Backups ( id String NOT NULL, container_id String NOT NULL, @@ -51,7 +52,6 @@ CREATE TABLE Operations ( status String, message String, - root_path String, paths String, paths_to_exclude String, operation_id String, @@ -80,7 +80,6 @@ CREATE TABLE BackupSchedules ( crontab String NOT NULL, ttl Interval, - root_path String, paths String, paths_to_exclude String, @@ -91,4 +90,17 @@ CREATE TABLE BackupSchedules ( next_launch Timestamp, PRIMARY KEY (id) -) +); + +UPSERT INTO OperationTypes (code, description, is_cancellable) VALUES + ('TB', 'Take backup', True), + ('RB', 'Restore backup', True), + ('DB', 'Delete backup', False), + ('TBWR', 'Take backup with retries', True); + +-- +goose Down +DROP TABLE BackupSchedules; +DROP TABLE Operations; +DROP TABLE OperationTypes; +DROP TABLE Backups; + diff --git a/migrations/yql/20251127120000_add_root_path_columns.sql b/migrations/yql/20251127120000_add_root_path_columns.sql new file mode 100644 index 00000000..c66f906f --- /dev/null +++ b/migrations/yql/20251127120000_add_root_path_columns.sql @@ -0,0 +1,14 @@ +-- +goose Up +ALTER TABLE Operations + ADD COLUMN root_path String; + +ALTER TABLE BackupSchedules + ADD COLUMN root_path String; + +-- +goose Down +ALTER TABLE BackupSchedules + DROP COLUMN root_path; + +ALTER TABLE Operations + DROP COLUMN root_path; +