feat: add 'downloads add-url' command for custom tarball URLs (#59)

cmd/downloads.go

@@ -596,6 +596,117 @@ func addTarballToCollectionFromStdin(cmd *cobra.Command, args []string) {
 	ops.DisplayTarball(tarballDesc)
 }
 
+func addTarballToCollectionFromUrl(cmd *cobra.Command, args []string) error {
+	if len(args) < 1 {
+		return fmt.Errorf("command 'add-url' requires a URL")
+	}
+	tarballUrl := args[0]
+	if !common.IsUrl(tarballUrl) {
+		return fmt.Errorf("argument %q does not look like a valid URL (must start with http:// or https://)", tarballUrl)
+	}
+
+	flags := cmd.Flags()
+	overrideOS, _ := flags.GetString(globals.OSLabel)
+	overrideArch, _ := flags.GetString(globals.ArchLabel)
+	overrideFlavor, _ := flags.GetString(globals.FlavorLabel)
+	overrideVersion, _ := flags.GetString(globals.VersionLabel)
+	overrideShortVersion, _ := flags.GetString(globals.ShortVersionLabel)
+	overrideMinimal, _ := flags.GetBool(globals.MinimalLabel)
+	overwrite, _ := flags.GetBool(globals.OverwriteLabel)
+	skipVerify, _ := flags.GetBool(globals.SkipVerifyUrlLabel)
+
+	// Parse the filename from the URL
+	tarballDesc, err := downloads.ParseTarballUrlInfo(tarballUrl)
+	if err != nil {
+		return fmt.Errorf("error parsing tarball URL: %s\n"+
+			"Use --version, --OS, --arch, --flavor flags to provide metadata manually", err)
+	}
+
+	// Apply any user overrides
+	if overrideOS != "" {
+		tarballDesc.OperatingSystem = overrideOS
+	}
+	if overrideArch != "" {
+		tarballDesc.Arch = overrideArch
+	}
+	if overrideFlavor != "" {
+		tarballDesc.Flavor = overrideFlavor
+	}
+	if overrideVersion != "" {
+		tarballDesc.Version = overrideVersion
+	}
+	if overrideShortVersion != "" {
+		tarballDesc.ShortVersion = overrideShortVersion
+	}
+	if overrideMinimal {
+		tarballDesc.Minimal = true
+	}
+
+	// Validate required fields
+	if tarballDesc.Version == "" {
+		return fmt.Errorf("could not detect version from filename; use --version to specify it")
+	}
+	if tarballDesc.ShortVersion == "" {
+		return fmt.Errorf("could not detect short version from filename; use --short-version to specify it")
+	}
+	if tarballDesc.OperatingSystem == "" {
+		return fmt.Errorf("could not detect OS from filename; use --OS to specify it")
+	}
+	if tarballDesc.Arch == "" {
+		return fmt.Errorf("could not detect architecture from filename; use --arch to specify it")
+	}
+	if tarballDesc.Flavor == "" {
+		return fmt.Errorf("could not detect flavor from filename; use --flavor to specify it")
+	}
+
+	// Verify the URL is accessible
+	if !skipVerify {
+		fmt.Printf("Verifying URL accessibility: %s\n", tarballUrl)
+		size, err := downloads.CheckRemoteUrl(tarballUrl)
+		if err != nil {
+			return fmt.Errorf("URL is not accessible: %s\nUse --skip-verify-url to bypass this check", err)
+		}
+		tarballDesc.Size = size
+		if size > 0 {
+			fmt.Printf("URL is accessible (size: %s)\n", humanize.Bytes(uint64(size)))
+		} else {
+			fmt.Printf("URL is accessible (size unknown)\n")
+		}
+	}
+
+	tarballDesc.Notes = fmt.Sprintf("added with version %s", common.VersionDef)
+	tarballDesc.DateAdded = time.Now().Format("2006-01-02 15:04")
+
+	// Load existing collection and add the new entry
+	var tarballCollection = downloads.DefaultTarballRegistry
+
+	existingTarball, findErr := downloads.FindTarballByName(tarballDesc.Name)
+	if findErr == nil {
+		if overwrite {
+			var newList []downloads.TarballDescription
+			newList, err = downloads.DeleteTarball(tarballCollection.Tarballs, tarballDesc.Name)
+			if err != nil {
+				return fmt.Errorf("error removing existing tarball %s from list: %s", tarballDesc.Name, err)
+			}
+			tarballCollection.Tarballs = newList
+		} else {
+			ops.DisplayTarball(existingTarball)
+			fmt.Println()
+			return fmt.Errorf("tarball %s already in the list; use --overwrite to replace it", tarballDesc.Name)
+		}
+	}
+
+	tarballCollection.Tarballs = append(tarballCollection.Tarballs, tarballDesc)
+
+	err = downloads.WriteTarballFileInfo(tarballCollection)
+	if err != nil {
+		return fmt.Errorf("error writing tarball list: %s", err)
+	}
+	fmt.Printf("Tarball below added to %s\n", downloads.TarballFileRegistry)
+	ops.DisplayTarball(tarballDesc)
+	return nil
+}
+
 func removeTarballFromCollection(cmd *cobra.Command, args []string) {
 	if len(args) < 1 {
 		common.Exit(1, "command 'delete' requires a tarball name")
@@ -753,6 +864,27 @@ var downloadsAddStdinCmd = &cobra.Command{
 	Run: addTarballToCollectionFromStdin,
 }
 
+var downloadsAddUrlCmd = &cobra.Command{
+	Use:   "add-url URL",
+	Short: "Adds a remote tarball to the list by URL",
+	Long: `Adds a tarball entry to the local registry using a direct URL.
+
+The filename is parsed to auto-detect version, OS, architecture, flavor, and
+whether the tarball is minimal. Use the override flags if auto-detection fails
+or produces wrong results.
+
+The URL is validated with an HTTP HEAD request before the entry is saved.
+Use --skip-verify-url to bypass this check.
+`,
+	Example: `
+$ dbdeployer downloads add-url https://example.com/mysql-8.4.8-linux-glibc2.17-x86_64.tar.xz
+$ dbdeployer downloads add-url https://example.com/Percona-Server-8.0.35-27-Linux.x86_64.glibc2.17-minimal.tar.gz
+$ dbdeployer downloads add-url https://example.com/mysql-8.4.8-macos15-arm64.tar.gz --OS=darwin --arch=arm64
+$ dbdeployer downloads add-url https://example.com/mysql-8.4.8-linux-glibc2.17-x86_64-minimal.tar.xz --overwrite
+`,
+	RunE: addTarballToCollectionFromUrl,
+}
+
 var downloadsCmd = &cobra.Command{
 	Use:   "downloads",
 	Short: "Manages remote tarballs",
@@ -788,6 +920,7 @@ func init() {
 	downloadsCmd.AddCommand(downloadsAddStdinCmd)
 	downloadsCmd.AddCommand(downloadsDeleteCmd)
 	downloadsCmd.AddCommand(downloadsTreeCmd)
+	downloadsCmd.AddCommand(downloadsAddUrlCmd)
 
 	downloadsListCmd.Flags().BoolP(globals.ShowUrlLabel, "", false, "Show the URL")
 	downloadsListCmd.Flags().String(globals.FlavorLabel, "", "Which flavor will be listed")
@@ -837,6 +970,15 @@ func init() {
 
 	downloadsAddStdinCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
 
+	downloadsAddUrlCmd.Flags().String(globals.OSLabel, "", "Override the detected OS (e.g. linux, darwin)")
+	downloadsAddUrlCmd.Flags().String(globals.ArchLabel, "", "Override the detected architecture (e.g. amd64, arm64)")
+	downloadsAddUrlCmd.Flags().String(globals.FlavorLabel, "", "Override the detected flavor (e.g. mysql, percona)")
+	downloadsAddUrlCmd.Flags().String(globals.VersionLabel, "", "Override the detected version (e.g. 8.4.8)")
+	downloadsAddUrlCmd.Flags().String(globals.ShortVersionLabel, "", "Override the detected short version (e.g. 8.4)")
+	downloadsAddUrlCmd.Flags().BoolP(globals.MinimalLabel, "", false, "Mark the tarball as minimal")
+	downloadsAddUrlCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry")
+	downloadsAddUrlCmd.Flags().BoolP(globals.SkipVerifyUrlLabel, "", false, "Skip URL accessibility check")
+
 	downloadsExportCmd.Flags().BoolP(globals.AddEmptyItemLabel, "", false, "Add an empty item to the tarballs list")
 
 	downloadsImportCmd.Flags().Int64P(globals.RetriesOnFailureLabel, "", 0, "How many times retry a download if a failure occurs on first try")

downloads/remote_registry.go

@@ -567,6 +567,95 @@ func checkRemoteUrl(remoteUrl string) (int64, error) {
 	return size, nil
 }
 
+// CheckRemoteUrl validates that a URL is accessible and returns its content size.
+// It uses an HTTP HEAD request first; if that fails (some servers don't support HEAD),
+// it falls back to a GET request.
+func CheckRemoteUrl(remoteUrl string) (int64, error) {
+	// Try HEAD first to avoid downloading the file
+	// #nosec G107
+	resp, err := http.Head(remoteUrl)
+	if err == nil {
+		defer func(Body io.ReadCloser) {
+			_ = Body.Close()
+		}(resp.Body)
+		if resp.StatusCode == http.StatusOK {
+			var size int64
+			for key := range resp.Header {
+				if strings.EqualFold(key, "Content-Length") && len(resp.Header[key]) > 0 {
+					size, _ = strconv.ParseInt(resp.Header[key][0], 10, 0)
+				}
+			}
+			return size, nil
+		}
+	}
+	// Fall back to GET if HEAD failed or returned non-200
+	return checkRemoteUrl(remoteUrl)
+}
+
+// ParseTarballUrlInfo parses a tarball URL/filename and returns a partially-filled
+// TarballDescription with auto-detected fields. The caller should override any
+// fields that were incorrectly detected.
+func ParseTarballUrlInfo(tarballUrl string) (TarballDescription, error) {
+	fileName := common.BaseName(tarballUrl)
+	if fileName == "" {
+		return TarballDescription{}, fmt.Errorf("could not determine filename from URL: %s", tarballUrl)
+	}
+
+	flavor, version, shortVersion, err := common.FindTarballInfo(fileName)
+	if err != nil {
+		return TarballDescription{}, fmt.Errorf("could not parse version from filename %q: %s", fileName, err)
+	}
+
+	OS, arch := detectOSArchFromFilename(fileName)
+	minimal := strings.Contains(strings.ToLower(fileName), "minimal")
+
+	return TarballDescription{
+		Name:         fileName,
+		Url:          tarballUrl,
+		Flavor:       flavor,
+		Version:      version,
+		ShortVersion: shortVersion,
+		OperatingSystem: OS,
+		Arch:         arch,
+		Minimal:      minimal,
+	}, nil
+}
+
+// detectOSArchFromFilename attempts to detect the OS and architecture from a tarball filename.
+// It handles common MySQL/Percona/MariaDB naming conventions.
+func detectOSArchFromFilename(fileName string) (OS, arch string) {
+	lower := strings.ToLower(fileName)
+
+	// Detect OS
+	switch {
+	case strings.Contains(lower, "linux") || strings.Contains(lower, "glibc"):
+		OS = "Linux"
+	case strings.Contains(lower, "macos") || strings.Contains(lower, "osx") || strings.Contains(lower, "darwin"):
+		OS = "Darwin"
+	case strings.Contains(lower, "windows") || strings.Contains(lower, "winx64"):
+		OS = "Windows"
+	default:
+		OS = runtime.GOOS
+		if OS == "darwin" {
+			OS = "Darwin"
+		} else if OS == "linux" {
+			OS = "Linux"
+		}
+	}
+
+	// Detect architecture
+	switch {
+	case strings.Contains(lower, "arm64") || strings.Contains(lower, "aarch64"):
+		arch = "arm64"
+	case strings.Contains(lower, "x86_64") || strings.Contains(lower, "x86-64") || strings.Contains(lower, "amd64"):
+		arch = "amd64"
+	default:
+		arch = runtime.GOARCH
+	}
+
+	return OS, arch
+}
+
 func CheckTarballList(tarballList []TarballDescription) error {
 	uniqueNames := make(map[string]bool)
 	uniqueCombinations := make(map[string]bool)

downloads/remote_registry_test.go

@@ -216,3 +216,105 @@ func TestMergeCollection(t *testing.T) {
 		})
 	}
 }
+
+func TestParseTarballUrlInfo(t *testing.T) {
+	tests := []struct {
+		name        string
+		url         string
+		wantName    string
+		wantVersion string
+		wantShort   string
+		wantOS      string
+		wantArch    string
+		wantFlavor  string
+		wantMinimal bool
+		wantErr     bool
+	}{
+		{
+			name:        "mysql-linux-amd64",
+			url:         "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.8-linux-glibc2.17-x86_64.tar.xz",
+			wantName:    "mysql-8.4.8-linux-glibc2.17-x86_64.tar.xz",
+			wantVersion: "8.4.8",
+			wantShort:   "8.4",
+			wantOS:      "Linux",
+			wantArch:    "amd64",
+			wantFlavor:  "mysql",
+			wantMinimal: false,
+		},
+		{
+			name:        "mysql-linux-amd64-minimal",
+			url:         "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.8-linux-glibc2.17-x86_64-minimal.tar.xz",
+			wantName:    "mysql-8.4.8-linux-glibc2.17-x86_64-minimal.tar.xz",
+			wantVersion: "8.4.8",
+			wantShort:   "8.4",
+			wantOS:      "Linux",
+			wantArch:    "amd64",
+			wantFlavor:  "mysql",
+			wantMinimal: true,
+		},
+		{
+			name:        "mysql-macos-arm64",
+			url:         "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.8-macos15-arm64.tar.gz",
+			wantName:    "mysql-8.4.8-macos15-arm64.tar.gz",
+			wantVersion: "8.4.8",
+			wantShort:   "8.4",
+			wantOS:      "Darwin",
+			wantArch:    "arm64",
+			wantFlavor:  "mysql",
+			wantMinimal: false,
+		},
+		{
+			name:        "percona-linux-amd64-minimal",
+			url:         "https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.35-27/binary/tarball/Percona-Server-8.0.35-27-Linux.x86_64.glibc2.17-minimal.tar.gz",
+			wantName:    "Percona-Server-8.0.35-27-Linux.x86_64.glibc2.17-minimal.tar.gz",
+			wantVersion: "8.0.35",
+			wantShort:   "8.0",
+			wantOS:      "Linux",
+			wantArch:    "amd64",
+			wantFlavor:  "percona",
+			wantMinimal: true,
+		},
+		{
+			name:    "invalid-no-version",
+			url:     "https://example.com/some-tarball-without-version.tar.gz",
+			wantErr: true,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ParseTarballUrlInfo(tt.url)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("ParseTarballUrlInfo() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if tt.wantErr {
+				return
+			}
+			if got.Name != tt.wantName {
+				t.Errorf("Name = %q, want %q", got.Name, tt.wantName)
+			}
+			if got.Version != tt.wantVersion {
+				t.Errorf("Version = %q, want %q", got.Version, tt.wantVersion)
+			}
+			if got.ShortVersion != tt.wantShort {
+				t.Errorf("ShortVersion = %q, want %q", got.ShortVersion, tt.wantShort)
+			}
+			if got.OperatingSystem != tt.wantOS {
+				t.Errorf("OperatingSystem = %q, want %q", got.OperatingSystem, tt.wantOS)
+			}
+			if got.Arch != tt.wantArch {
+				t.Errorf("Arch = %q, want %q", got.Arch, tt.wantArch)
+			}
+			if got.Flavor != tt.wantFlavor {
+				t.Errorf("Flavor = %q, want %q", got.Flavor, tt.wantFlavor)
+			}
+			if got.Minimal != tt.wantMinimal {
+				t.Errorf("Minimal = %v, want %v", got.Minimal, tt.wantMinimal)
+			}
+			if got.Url != tt.url {
+				t.Errorf("Url = %q, want %q", got.Url, tt.url)
+			}
+		})
+	}
+}

globals/globals.go

@@ -149,6 +149,7 @@ const (
 	DeleteAfterUnpackLabel = "delete-after-unpack"
 	MaxItemsLabel          = "max-items"
 	ChangeUserAgentLabel   = "change-user-agent"
+	SkipVerifyUrlLabel     = "skip-verify-url"
 
 	// Instantiated in cmd/admin.go
 	VerboseLabel = "verbose"