From 3c928844ecee553672a6fe153eee96ed88f8ff42 Mon Sep 17 00:00:00 2001 From: Jonathan Gramain Date: Thu, 18 Sep 2025 15:11:05 +0200 Subject: [PATCH] WKBCH-7 metadata migration support Add support for the metadata migration components, when explicitly enabled in the features yaml. It adds: - a standalone deployment for the migration bucketd and map - a container with the migration tools worker and API server --- cmd/config.go | 77 +++++++++++++++++----- cmd/configure.go | 12 ++++ cmd/util.go | 4 ++ templates/global/defaults.env | 1 + templates/global/docker-compose.yaml | 27 ++++++++ templates/global/values.yaml | 6 ++ templates/metadata/config.json | 19 +++++- templates/migration-tools/env | 0 templates/migration-tools/migration.yml | 21 ++++++ templates/migration-tools/supervisord.conf | 49 ++++++++++++++ 10 files changed, 196 insertions(+), 20 deletions(-) create mode 100644 templates/migration-tools/env create mode 100644 templates/migration-tools/migration.yml create mode 100644 templates/migration-tools/supervisord.conf diff --git a/cmd/config.go b/cmd/config.go index 41832a4..fff5f72 100644 --- a/cmd/config.go +++ b/cmd/config.go @@ -44,18 +44,19 @@ func RuntimeConfigFromFlags(envDir, envName string) RuntimeConfig { } type EnvironmentConfig struct { - Global GlobalConfig `yaml:"global"` - Features FeatureConfig `yaml:"features"` - Cloudserver CloudserverConfig `yaml:"cloudserver"` - S3Metadata MetadataConfig `yaml:"s3_metadata"` - Backbeat BackbeatConfig `yaml:"backbeat"` - Vault VaultConfig `yaml:"vault"` - Scuba ScubaConfig `yaml:"scuba"` - ScubaMetadata MetadataConfig `yaml:"scuba_metadata"` - Kafka KafkaConfig `yaml:"kafka"` - Zookeeper ZookeeperConfig `yaml:"zookeeper"` - Redis RedisConfig `yaml:"redis"` - Utapi UtapiConfig `yaml:"utapi"` + Global GlobalConfig `yaml:"global"` + Features FeatureConfig `yaml:"features"` + Cloudserver CloudserverConfig `yaml:"cloudserver"` + S3Metadata MetadataConfig `yaml:"s3_metadata"` + Backbeat BackbeatConfig `yaml:"backbeat"` + Vault VaultConfig `yaml:"vault"` + Scuba ScubaConfig `yaml:"scuba"` + ScubaMetadata MetadataConfig `yaml:"scuba_metadata"` + Kafka KafkaConfig `yaml:"kafka"` + Zookeeper ZookeeperConfig `yaml:"zookeeper"` + Redis RedisConfig `yaml:"redis"` + Utapi UtapiConfig `yaml:"utapi"` + MigrationTools MigrationToolsConfig `yaml:"migration_tools"` } type GlobalConfig struct { @@ -67,6 +68,7 @@ type FeatureConfig struct { Scuba ScubaFeatureConfig `yaml:"scuba"` BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"` Utapi UtapiFeatureConfig `yaml:"utapi"` + Migration MigrationFeatureConfig `yaml:"migration"` } type ScubaFeatureConfig struct { @@ -87,6 +89,10 @@ type UtapiFeatureConfig struct { Enabled bool `yaml:"enabled"` } +type MigrationFeatureConfig struct { + Enabled bool `yaml:"enabled"` +} + type CloudserverConfig struct { Image string `yaml:"image"` EnableNullVersionCompatMode bool `yaml:"enableNullVersionCompatMode"` @@ -108,6 +114,11 @@ type UtapiConfig struct { LogLevel string `yaml:"log_level"` } +type MigrationToolsConfig struct { + Image string `yaml:"image"` + LogLevel string `yaml:"log_level"` +} + type VFormat string func (vf VFormat) String() string { @@ -157,11 +168,12 @@ const ( ) type MetadataConfig struct { - Image string `yaml:"image"` - RaftSessions int `yaml:"raft_sessions"` - BasePorts MdPortConfig `yaml:"base_ports"` - LogLevel string `yaml:"log_level"` - VFormat VFormat `yaml:"vformat"` + Image string `yaml:"image"` + RaftSessions int `yaml:"raft_sessions"` + BasePorts MdPortConfig `yaml:"base_ports"` + LogLevel string `yaml:"log_level"` + VFormat VFormat `yaml:"vformat"` + Migration *MigrationConfig `yaml:"migration"` } type MdPortConfig struct { @@ -170,6 +182,11 @@ type MdPortConfig struct { RepdAdmin uint16 `yaml:"repdAdmin"` } +type MigrationConfig struct { + Deploy bool `yaml:"deploy"` + BasePorts MdPortConfig `yaml:"base_ports"` +} + type ScubaConfig struct { Image string `yaml:"image"` LogLevel string `yaml:"log_level"` @@ -209,6 +226,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig { Utapi: UtapiFeatureConfig{ Enabled: false, }, + Migration: MigrationFeatureConfig{ + Enabled: false, + }, }, Cloudserver: CloudserverConfig{}, S3Metadata: MetadataConfig{ @@ -220,6 +240,14 @@ func DefaultEnvironmentConfig() EnvironmentConfig { }, RaftSessions: 3, // LogLevel: "info", + Migration: &MigrationConfig{ + Deploy: false, + BasePorts: MdPortConfig{ + Bucketd: 9001, + Repd: 4700, + RepdAdmin: 4750, + }, + }, }, Backbeat: BackbeatConfig{}, Vault: VaultConfig{}, @@ -234,7 +262,8 @@ func DefaultEnvironmentConfig() EnvironmentConfig { RaftSessions: 1, // LogLevel: "info", }, - Utapi: UtapiConfig{}, + Utapi: UtapiConfig{}, + MigrationTools: MigrationToolsConfig{}, } } @@ -270,6 +299,10 @@ func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) { if cfg.S3Metadata.LogLevel == "" { cfg.S3Metadata.LogLevel = cfg.Global.LogLevel } + // deploy the migration metadata map and bucketds if enabled + if cfg.Features.Migration.Enabled { + cfg.S3Metadata.Migration.Deploy = true + } if cfg.Backbeat.LogLevel == "" { cfg.Backbeat.LogLevel = cfg.Global.LogLevel @@ -299,5 +332,13 @@ func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) { cfg.Redis.LogLevel = cfg.Global.LogLevel } + if cfg.Utapi.LogLevel == "" { + cfg.Utapi.LogLevel = cfg.Global.LogLevel + } + + if cfg.MigrationTools.LogLevel == "" { + cfg.MigrationTools.LogLevel = cfg.Global.LogLevel + } + return cfg, nil } diff --git a/cmd/configure.go b/cmd/configure.go index 6f39040..9374052 100644 --- a/cmd/configure.go +++ b/cmd/configure.go @@ -39,6 +39,7 @@ func createLogDirectories(envDir string) error { filepath.Join(envDir, "logs"), filepath.Join(envDir, "logs", "scuba"), filepath.Join(envDir, "logs", "backbeat"), + filepath.Join(envDir, "logs", "migration-tools"), } for _, dir := range logDirs { @@ -69,6 +70,7 @@ func configureEnv(cfg EnvironmentConfig, envDir string) error { generateScubaMetadataConfig, generateKafkaConfig, generateUtapiConfig, + generateMigrationToolsConfig, } configDir := filepath.Join(envDir, "config") @@ -159,3 +161,13 @@ func generateKafkaConfig(cfg EnvironmentConfig, path string) error { func generateUtapiConfig(cfg EnvironmentConfig, path string) error { return renderTemplateToFile(getTemplates(), "templates/utapi/config.json", cfg, filepath.Join(path, "utapi", "config.json")) } + +func generateMigrationToolsConfig(cfg EnvironmentConfig, path string) error { + templates := []string{ + "supervisord.conf", + "migration.yml", + "env", + } + + return renderTemplates(cfg, "templates/migration-tools", filepath.Join(path, "migration-tools"), templates) +} diff --git a/cmd/util.go b/cmd/util.go index f31ff63..922fd82 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -78,6 +78,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string { profiles = append(profiles, "feature-utapi") } + if cfg.Features.Migration.Enabled { + profiles = append(profiles, "feature-migration") + } + return profiles } diff --git a/templates/global/defaults.env b/templates/global/defaults.env index 53f79e4..91f249f 100644 --- a/templates/global/defaults.env +++ b/templates/global/defaults.env @@ -8,6 +8,7 @@ VAULT_IMAGE="{{ .Vault.Image }}" SCUBA_IMAGE="{{ .Scuba.Image }}" BACKBEAT_IMAGE="{{ .Backbeat.Image }}" UTAPI_IMAGE="{{ .Utapi.Image }}" +MIGRATION_TOOLS_IMAGE="{{ .MigrationTools.Image }}" METADATA_S3_DB_VERSION="{{ .S3Metadata.VFormat }}" CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE="{{ .Cloudserver.EnableNullVersionCompatMode }}" diff --git a/templates/global/docker-compose.yaml b/templates/global/docker-compose.yaml index f6806fa..616ed19 100644 --- a/templates/global/docker-compose.yaml +++ b/templates/global/docker-compose.yaml @@ -161,10 +161,17 @@ services: image: ${REDIS_IMAGE} container_name: workbench-redis network_mode: host + healthcheck: + test: ["CMD-SHELL", "redis-cli ping | grep PONG"] + interval: 10s + retries: 10 + start_period: 30s + timeout: 10s profiles: - feature-crr - feature-notifications - feature-utapi + - feature-migration zookeeper: build: @@ -254,3 +261,23 @@ services: - ./config/utapi/config.json:/conf/config.json:ro environment: UTAPI_CONFIG_FILE: /conf/config.json + + migration-tools: + image: ${MIGRATION_TOOLS_IMAGE} + container_name: workbench-migration-tools + network_mode: host + depends_on: + metadata-s3: + condition: service_healthy + redis: + condition: service_healthy + environment: + SUPERVISORD_CONF: supervisord.conf + MIGRATION_CONFIG_FILE: /conf/migration.yml + volumes: + - ./config/migration-tools/supervisord.conf:/conf/supervisord.conf:ro + - ./config/migration-tools/migration.yml:/conf/migration.yml:ro + - ./config/migration-tools/env:/conf/env:ro + - ./logs/migration-tools:/logs:rw + profiles: + - feature-migration diff --git a/templates/global/values.yaml b/templates/global/values.yaml index dee8319..a253a88 100644 --- a/templates/global/values.yaml +++ b/templates/global/values.yaml @@ -16,6 +16,9 @@ features: utapi: enabled: false + migration: + enabled: false + cloudserver: image: ghcr.io/scality/cloudserver:7.70.77 @@ -45,3 +48,6 @@ redis: utapi: image: ghcr.io/scality/utapi:7.70.9 + +migration_tools: + image: ghcr.io/scality/metadata-migration:1.4.0-svc-base diff --git a/templates/metadata/config.json b/templates/metadata/config.json index 908f03f..fcd5014 100644 --- a/templates/metadata/config.json +++ b/templates/metadata/config.json @@ -8,8 +8,23 @@ "repd": {{ .BasePorts.Repd }}, "repdAdmin": {{ .BasePorts.RepdAdmin }} }, - "logLevel": "{{ .LogLevel }}", "env": { "METADATA_NEW_BUCKETS_VFORMAT": "{{ .VFormat }}" - } + }, + {{ if .Migration }} + "migration": { + "deploy": {{ .Migration.Deploy }}, + "raftSessions": 0, + "raftMembers": 5, + "bucketdCount": 1, + "bucketdWorkers": 1, + "basePorts": { + "bucketd": {{ .Migration.BasePorts.Bucketd }}, + "repd": {{ .Migration.BasePorts.Repd }}, + "repdAdmin": {{ .Migration.BasePorts.RepdAdmin }} + }, + "logLevel": "info" + }, + {{ end }} + "logLevel": "{{ .LogLevel }}" } diff --git a/templates/migration-tools/env b/templates/migration-tools/env new file mode 100644 index 0000000..e69de29 diff --git a/templates/migration-tools/migration.yml b/templates/migration-tools/migration.yml new file mode 100644 index 0000000..582a6ec --- /dev/null +++ b/templates/migration-tools/migration.yml @@ -0,0 +1,21 @@ +--- +backend: + production-bucketd-url: 'http://localhost:9000' + migration-bucketd-url: 'http://localhost:9001' + redis: + url: "redis://localhost:6379" +worker: + switch: + all-production-bucketd-urls: + - 'http://localhost:9000' +api-server: + listen-address: '0.0.0.0' + listen-port: 9180 + metrics-server: + enabled: true + listen-address: '0.0.0.0' + listen-port: 9181 + expose-job-specific-metrics: true + authorized-client-ips: + - 127.0.0.1 + - ::1 diff --git a/templates/migration-tools/supervisord.conf b/templates/migration-tools/supervisord.conf new file mode 100644 index 0000000..71bc90b --- /dev/null +++ b/templates/migration-tools/supervisord.conf @@ -0,0 +1,49 @@ +[supervisord] +nodaemon = true +loglevel = info +stdout_logfile_maxbytes = 20MB +stdout_logfile_backups = 2 +logfile = %(ENV_LOG_DIR)s/supervisord.log +pidfile = %(ENV_SUP_RUN_DIR)s/supervisord.pid + +[unix_http_server] +file = %(ENV_SUP_RUN_DIR)s/supervisor.sock + +[rpcinterface:supervisor] +supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface + +[supervisorctl] +serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock + +[program:migration-worker] +command = migration-worker +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +[program:migration-api-server] +command = migration-api-server +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true + +[program:asynqmon] +command = asynqmon -redis-url 'redis://localhost:6379' -port 9102 +stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log +stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log +stdout_logfile_maxbytes=100MB +stdout_logfile_backups=7 +stderr_logfile_maxbytes=100MB +stderr_logfile_backups=7 +autorestart = true +autostart = true