WKBCH-7 metadata migration support

cmd/config.go

@@ -44,18 +44,19 @@ func RuntimeConfigFromFlags(envDir, envName string) RuntimeConfig {
 }
 
 type EnvironmentConfig struct {
-	Global        GlobalConfig      `yaml:"global"`
-	Features      FeatureConfig     `yaml:"features"`
-	Cloudserver   CloudserverConfig `yaml:"cloudserver"`
-	S3Metadata    MetadataConfig    `yaml:"s3_metadata"`
-	Backbeat      BackbeatConfig    `yaml:"backbeat"`
-	Vault         VaultConfig       `yaml:"vault"`
-	Scuba         ScubaConfig       `yaml:"scuba"`
-	ScubaMetadata MetadataConfig    `yaml:"scuba_metadata"`
-	Kafka         KafkaConfig       `yaml:"kafka"`
-	Zookeeper     ZookeeperConfig   `yaml:"zookeeper"`
-	Redis         RedisConfig       `yaml:"redis"`
-	Utapi         UtapiConfig       `yaml:"utapi"`
+	Global         GlobalConfig         `yaml:"global"`
+	Features       FeatureConfig        `yaml:"features"`
+	Cloudserver    CloudserverConfig    `yaml:"cloudserver"`
+	S3Metadata     MetadataConfig       `yaml:"s3_metadata"`
+	Backbeat       BackbeatConfig       `yaml:"backbeat"`
+	Vault          VaultConfig          `yaml:"vault"`
+	Scuba          ScubaConfig          `yaml:"scuba"`
+	ScubaMetadata  MetadataConfig       `yaml:"scuba_metadata"`
+	Kafka          KafkaConfig          `yaml:"kafka"`
+	Zookeeper      ZookeeperConfig      `yaml:"zookeeper"`
+	Redis          RedisConfig          `yaml:"redis"`
+	Utapi          UtapiConfig          `yaml:"utapi"`
+	MigrationTools MigrationToolsConfig `yaml:"migration_tools"`
 }
 
 type GlobalConfig struct {
@@ -67,6 +68,7 @@ type FeatureConfig struct {
 	Scuba               ScubaFeatureConfig               `yaml:"scuba"`
 	BucketNotifications BucketNotificationsFeatureConfig `yaml:"bucket_notifications"`
 	Utapi               UtapiFeatureConfig               `yaml:"utapi"`
+	Migration           MigrationFeatureConfig           `yaml:"migration"`
 }
 
 type ScubaFeatureConfig struct {
@@ -87,6 +89,10 @@ type UtapiFeatureConfig struct {
 	Enabled bool `yaml:"enabled"`
 }
 
+type MigrationFeatureConfig struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type CloudserverConfig struct {
 	Image                       string `yaml:"image"`
 	EnableNullVersionCompatMode bool   `yaml:"enableNullVersionCompatMode"`
@@ -108,6 +114,11 @@ type UtapiConfig struct {
 	LogLevel string `yaml:"log_level"`
 }
 
+type MigrationToolsConfig struct {
+	Image    string `yaml:"image"`
+	LogLevel string `yaml:"log_level"`
+}
+
 type VFormat string
 
 func (vf VFormat) String() string {
@@ -157,11 +168,12 @@ const (
 )
 
 type MetadataConfig struct {
-	Image        string       `yaml:"image"`
-	RaftSessions int          `yaml:"raft_sessions"`
-	BasePorts    MdPortConfig `yaml:"base_ports"`
-	LogLevel     string       `yaml:"log_level"`
-	VFormat      VFormat      `yaml:"vformat"`
+	Image        string           `yaml:"image"`
+	RaftSessions int              `yaml:"raft_sessions"`
+	BasePorts    MdPortConfig     `yaml:"base_ports"`
+	LogLevel     string           `yaml:"log_level"`
+	VFormat      VFormat          `yaml:"vformat"`
+	Migration    *MigrationConfig `yaml:"migration"`
 }
 
 type MdPortConfig struct {
@@ -170,6 +182,11 @@ type MdPortConfig struct {
 	RepdAdmin uint16 `yaml:"repdAdmin"`
 }
 
+type MigrationConfig struct {
+	Deploy    bool         `yaml:"deploy"`
+	BasePorts MdPortConfig `yaml:"base_ports"`
+}
+
 type ScubaConfig struct {
 	Image    string `yaml:"image"`
 	LogLevel string `yaml:"log_level"`
@@ -209,6 +226,9 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
 			Utapi: UtapiFeatureConfig{
 				Enabled: false,
 			},
+			Migration: MigrationFeatureConfig{
+				Enabled: false,
+			},
 		},
 		Cloudserver: CloudserverConfig{},
 		S3Metadata: MetadataConfig{
@@ -220,6 +240,14 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
 			},
 			RaftSessions: 3,
 			// LogLevel:     "info",
+			Migration: &MigrationConfig{
+				Deploy: false,
+				BasePorts: MdPortConfig{
+					Bucketd:   9001,
+					Repd:      4700,
+					RepdAdmin: 4750,
+				},
+			},
 		},
 		Backbeat: BackbeatConfig{},
 		Vault:    VaultConfig{},
@@ -234,7 +262,8 @@ func DefaultEnvironmentConfig() EnvironmentConfig {
 			RaftSessions: 1,
 			// LogLevel:     "info",
 		},
-		Utapi: UtapiConfig{},
+		Utapi:          UtapiConfig{},
+		MigrationTools: MigrationToolsConfig{},
 	}
 }
 
@@ -270,6 +299,10 @@ func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) {
 	if cfg.S3Metadata.LogLevel == "" {
 		cfg.S3Metadata.LogLevel = cfg.Global.LogLevel
 	}
+	// deploy the migration metadata map and bucketds if enabled
+	if cfg.Features.Migration.Enabled {
+		cfg.S3Metadata.Migration.Deploy = true
+	}
 
 	if cfg.Backbeat.LogLevel == "" {
 		cfg.Backbeat.LogLevel = cfg.Global.LogLevel
@@ -299,5 +332,13 @@ func LoadEnvironmentConfig(path string) (EnvironmentConfig, error) {
 		cfg.Redis.LogLevel = cfg.Global.LogLevel
 	}
 
+	if cfg.Utapi.LogLevel == "" {
+		cfg.Utapi.LogLevel = cfg.Global.LogLevel
+	}
+
+	if cfg.MigrationTools.LogLevel == "" {
+		cfg.MigrationTools.LogLevel = cfg.Global.LogLevel
+	}
+
 	return cfg, nil
 }

cmd/configure.go

@@ -39,6 +39,7 @@ func createLogDirectories(envDir string) error {
 		filepath.Join(envDir, "logs"),
 		filepath.Join(envDir, "logs", "scuba"),
 		filepath.Join(envDir, "logs", "backbeat"),
+		filepath.Join(envDir, "logs", "migration-tools"),
 	}
 
 	for _, dir := range logDirs {
@@ -69,6 +70,7 @@ func configureEnv(cfg EnvironmentConfig, envDir string) error {
 		generateScubaMetadataConfig,
 		generateKafkaConfig,
 		generateUtapiConfig,
+		generateMigrationToolsConfig,
 	}
 
 	configDir := filepath.Join(envDir, "config")
@@ -159,3 +161,13 @@ func generateKafkaConfig(cfg EnvironmentConfig, path string) error {
 func generateUtapiConfig(cfg EnvironmentConfig, path string) error {
 	return renderTemplateToFile(getTemplates(), "templates/utapi/config.json", cfg, filepath.Join(path, "utapi", "config.json"))
 }
+
+func generateMigrationToolsConfig(cfg EnvironmentConfig, path string) error {
+	templates := []string{
+		"supervisord.conf",
+		"migration.yml",
+		"env",
+	}
+
+	return renderTemplates(cfg, "templates/migration-tools", filepath.Join(path, "migration-tools"), templates)
+}

cmd/util.go

@@ -78,6 +78,10 @@ func getComposeProfiles(cfg EnvironmentConfig) []string {
 		profiles = append(profiles, "feature-utapi")
 	}
 
+	if cfg.Features.Migration.Enabled {
+		profiles = append(profiles, "feature-migration")
+	}
+
 	return profiles
 }
 

templates/global/defaults.env

@@ -8,6 +8,7 @@ VAULT_IMAGE="{{ .Vault.Image }}"
 SCUBA_IMAGE="{{ .Scuba.Image }}"
 BACKBEAT_IMAGE="{{ .Backbeat.Image }}"
 UTAPI_IMAGE="{{ .Utapi.Image }}"
+MIGRATION_TOOLS_IMAGE="{{ .MigrationTools.Image }}"
 
 METADATA_S3_DB_VERSION="{{ .S3Metadata.VFormat }}"
 CLOUDSERVER_ENABLE_NULL_VERSION_COMPAT_MODE="{{ .Cloudserver.EnableNullVersionCompatMode }}"

templates/global/docker-compose.yaml

@@ -161,10 +161,17 @@ services:
     image: ${REDIS_IMAGE}
     container_name: workbench-redis
     network_mode: host
+    healthcheck:
+      test: ["CMD-SHELL", "redis-cli ping | grep PONG"]
+      interval: 10s
+      retries: 10
+      start_period: 30s
+      timeout: 10s
     profiles:
       - feature-crr
       - feature-notifications
       - feature-utapi
+      - feature-migration
 
   zookeeper:
     build:
@@ -254,3 +261,23 @@ services:
       - ./config/utapi/config.json:/conf/config.json:ro
     environment:
       UTAPI_CONFIG_FILE: /conf/config.json
+
+  migration-tools:
+    image: ${MIGRATION_TOOLS_IMAGE}
+    container_name: workbench-migration-tools
+    network_mode: host
+    depends_on:
+      metadata-s3:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    environment:
+      SUPERVISORD_CONF: supervisord.conf
+      MIGRATION_CONFIG_FILE: /conf/migration.yml
+    volumes:
+      - ./config/migration-tools/supervisord.conf:/conf/supervisord.conf:ro
+      - ./config/migration-tools/migration.yml:/conf/migration.yml:ro
+      - ./config/migration-tools/env:/conf/env:ro
+      - ./logs/migration-tools:/logs:rw
+    profiles:
+      - feature-migration

templates/global/values.yaml

@@ -16,6 +16,9 @@ features:
   utapi:
     enabled: false
 
+  migration:
+    enabled: false
+
 cloudserver:
   image: ghcr.io/scality/cloudserver:7.70.77
 
@@ -45,3 +48,6 @@ redis:
 
 utapi:
   image: ghcr.io/scality/utapi:7.70.9
+
+migration_tools:
+  image: ghcr.io/scality/metadata-migration:1.4.0-svc-base

templates/metadata/config.json

@@ -8,8 +8,23 @@
         "repd": {{ .BasePorts.Repd }},
         "repdAdmin": {{ .BasePorts.RepdAdmin }}
     },
-    "logLevel": "{{ .LogLevel }}",
     "env": {
         "METADATA_NEW_BUCKETS_VFORMAT": "{{ .VFormat }}"
-    }
+    },
+    {{ if .Migration }}
+    "migration": {
+        "deploy": {{ .Migration.Deploy }},
+        "raftSessions": 0,
+        "raftMembers": 5,
+        "bucketdCount": 1,
+        "bucketdWorkers": 1,
+        "basePorts": {
+            "bucketd": {{ .Migration.BasePorts.Bucketd }},
+            "repd": {{ .Migration.BasePorts.Repd }},
+            "repdAdmin": {{ .Migration.BasePorts.RepdAdmin }}
+        },
+        "logLevel": "info"
+    },
+    {{ end }}
+    "logLevel": "{{ .LogLevel }}"
 }

templates/migration-tools/migration.yml

@@ -0,0 +1,21 @@
+---
+backend:
+  production-bucketd-url: 'http://localhost:9000'
+  migration-bucketd-url: 'http://localhost:9001'
+  redis:
+    url: "redis://localhost:6379"
+worker:
+  switch:
+    all-production-bucketd-urls:
+      - 'http://localhost:9000'
+api-server:
+  listen-address: '0.0.0.0'
+  listen-port: 9180
+  metrics-server:
+    enabled: true
+    listen-address: '0.0.0.0'
+    listen-port: 9181
+    expose-job-specific-metrics: true
+  authorized-client-ips:
+    - 127.0.0.1
+    - ::1

templates/migration-tools/supervisord.conf

@@ -0,0 +1,49 @@
+[supervisord]
+nodaemon = true
+loglevel = info
+stdout_logfile_maxbytes = 20MB
+stdout_logfile_backups = 2
+logfile  = %(ENV_LOG_DIR)s/supervisord.log
+pidfile  = %(ENV_SUP_RUN_DIR)s/supervisord.pid
+
+[unix_http_server]
+file = %(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[rpcinterface:supervisor]
+supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
+
+[supervisorctl]
+serverurl = unix://%(ENV_SUP_RUN_DIR)s/supervisor.sock
+
+[program:migration-worker]
+command = migration-worker
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+[program:migration-api-server]
+command = migration-api-server
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true
+
+[program:asynqmon]
+command = asynqmon -redis-url 'redis://localhost:6379' -port 9102
+stdout_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s.log
+stderr_logfile = %(ENV_LOG_DIR)s/%(program_name)s-%(process_num)s-stderr.log
+stdout_logfile_maxbytes=100MB
+stdout_logfile_backups=7
+stderr_logfile_maxbytes=100MB
+stderr_logfile_backups=7
+autorestart = true
+autostart = true