diff --git a/cmd/downloads.go b/cmd/downloads.go index 1da3050..b2aec89 100644 --- a/cmd/downloads.go +++ b/cmd/downloads.go @@ -596,6 +596,117 @@ func addTarballToCollectionFromStdin(cmd *cobra.Command, args []string) { ops.DisplayTarball(tarballDesc) } +func addTarballToCollectionFromUrl(cmd *cobra.Command, args []string) error { + if len(args) < 1 { + return fmt.Errorf("command 'add-url' requires a URL") + } + tarballUrl := args[0] + if !common.IsUrl(tarballUrl) { + return fmt.Errorf("argument %q does not look like a valid URL (must start with http:// or https://)", tarballUrl) + } + + flags := cmd.Flags() + overrideOS, _ := flags.GetString(globals.OSLabel) + overrideArch, _ := flags.GetString(globals.ArchLabel) + overrideFlavor, _ := flags.GetString(globals.FlavorLabel) + overrideVersion, _ := flags.GetString(globals.VersionLabel) + overrideShortVersion, _ := flags.GetString(globals.ShortVersionLabel) + overrideMinimal, _ := flags.GetBool(globals.MinimalLabel) + overwrite, _ := flags.GetBool(globals.OverwriteLabel) + skipVerify, _ := flags.GetBool(globals.SkipVerifyUrlLabel) + + // Parse the filename from the URL + tarballDesc, err := downloads.ParseTarballUrlInfo(tarballUrl) + if err != nil { + return fmt.Errorf("error parsing tarball URL: %s\n"+ + "Use --version, --OS, --arch, --flavor flags to provide metadata manually", err) + } + + // Apply any user overrides + if overrideOS != "" { + tarballDesc.OperatingSystem = overrideOS + } + if overrideArch != "" { + tarballDesc.Arch = overrideArch + } + if overrideFlavor != "" { + tarballDesc.Flavor = overrideFlavor + } + if overrideVersion != "" { + tarballDesc.Version = overrideVersion + } + if overrideShortVersion != "" { + tarballDesc.ShortVersion = overrideShortVersion + } + if overrideMinimal { + tarballDesc.Minimal = true + } + + // Validate required fields + if tarballDesc.Version == "" { + return fmt.Errorf("could not detect version from filename; use --version to specify it") + } + if tarballDesc.ShortVersion == "" { + return fmt.Errorf("could not detect short version from filename; use --short-version to specify it") + } + if tarballDesc.OperatingSystem == "" { + return fmt.Errorf("could not detect OS from filename; use --OS to specify it") + } + if tarballDesc.Arch == "" { + return fmt.Errorf("could not detect architecture from filename; use --arch to specify it") + } + if tarballDesc.Flavor == "" { + return fmt.Errorf("could not detect flavor from filename; use --flavor to specify it") + } + + // Verify the URL is accessible + if !skipVerify { + fmt.Printf("Verifying URL accessibility: %s\n", tarballUrl) + size, err := downloads.CheckRemoteUrl(tarballUrl) + if err != nil { + return fmt.Errorf("URL is not accessible: %s\nUse --skip-verify-url to bypass this check", err) + } + tarballDesc.Size = size + if size > 0 { + fmt.Printf("URL is accessible (size: %s)\n", humanize.Bytes(uint64(size))) + } else { + fmt.Printf("URL is accessible (size unknown)\n") + } + } + + tarballDesc.Notes = fmt.Sprintf("added with version %s", common.VersionDef) + tarballDesc.DateAdded = time.Now().Format("2006-01-02 15:04") + + // Load existing collection and add the new entry + var tarballCollection = downloads.DefaultTarballRegistry + + existingTarball, findErr := downloads.FindTarballByName(tarballDesc.Name) + if findErr == nil { + if overwrite { + var newList []downloads.TarballDescription + newList, err = downloads.DeleteTarball(tarballCollection.Tarballs, tarballDesc.Name) + if err != nil { + return fmt.Errorf("error removing existing tarball %s from list: %s", tarballDesc.Name, err) + } + tarballCollection.Tarballs = newList + } else { + ops.DisplayTarball(existingTarball) + fmt.Println() + return fmt.Errorf("tarball %s already in the list; use --overwrite to replace it", tarballDesc.Name) + } + } + + tarballCollection.Tarballs = append(tarballCollection.Tarballs, tarballDesc) + + err = downloads.WriteTarballFileInfo(tarballCollection) + if err != nil { + return fmt.Errorf("error writing tarball list: %s", err) + } + fmt.Printf("Tarball below added to %s\n", downloads.TarballFileRegistry) + ops.DisplayTarball(tarballDesc) + return nil +} + func removeTarballFromCollection(cmd *cobra.Command, args []string) { if len(args) < 1 { common.Exit(1, "command 'delete' requires a tarball name") @@ -753,6 +864,27 @@ var downloadsAddStdinCmd = &cobra.Command{ Run: addTarballToCollectionFromStdin, } +var downloadsAddUrlCmd = &cobra.Command{ + Use: "add-url URL", + Short: "Adds a remote tarball to the list by URL", + Long: `Adds a tarball entry to the local registry using a direct URL. + +The filename is parsed to auto-detect version, OS, architecture, flavor, and +whether the tarball is minimal. Use the override flags if auto-detection fails +or produces wrong results. + +The URL is validated with an HTTP HEAD request before the entry is saved. +Use --skip-verify-url to bypass this check. +`, + Example: ` +$ dbdeployer downloads add-url https://example.com/mysql-8.4.8-linux-glibc2.17-x86_64.tar.xz +$ dbdeployer downloads add-url https://example.com/Percona-Server-8.0.35-27-Linux.x86_64.glibc2.17-minimal.tar.gz +$ dbdeployer downloads add-url https://example.com/mysql-8.4.8-macos15-arm64.tar.gz --OS=darwin --arch=arm64 +$ dbdeployer downloads add-url https://example.com/mysql-8.4.8-linux-glibc2.17-x86_64-minimal.tar.xz --overwrite +`, + RunE: addTarballToCollectionFromUrl, +} + var downloadsCmd = &cobra.Command{ Use: "downloads", Short: "Manages remote tarballs", @@ -788,6 +920,7 @@ func init() { downloadsCmd.AddCommand(downloadsAddStdinCmd) downloadsCmd.AddCommand(downloadsDeleteCmd) downloadsCmd.AddCommand(downloadsTreeCmd) + downloadsCmd.AddCommand(downloadsAddUrlCmd) downloadsListCmd.Flags().BoolP(globals.ShowUrlLabel, "", false, "Show the URL") downloadsListCmd.Flags().String(globals.FlavorLabel, "", "Which flavor will be listed") @@ -837,6 +970,15 @@ func init() { downloadsAddStdinCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry") + downloadsAddUrlCmd.Flags().String(globals.OSLabel, "", "Override the detected OS (e.g. linux, darwin)") + downloadsAddUrlCmd.Flags().String(globals.ArchLabel, "", "Override the detected architecture (e.g. amd64, arm64)") + downloadsAddUrlCmd.Flags().String(globals.FlavorLabel, "", "Override the detected flavor (e.g. mysql, percona)") + downloadsAddUrlCmd.Flags().String(globals.VersionLabel, "", "Override the detected version (e.g. 8.4.8)") + downloadsAddUrlCmd.Flags().String(globals.ShortVersionLabel, "", "Override the detected short version (e.g. 8.4)") + downloadsAddUrlCmd.Flags().BoolP(globals.MinimalLabel, "", false, "Mark the tarball as minimal") + downloadsAddUrlCmd.Flags().BoolP(globals.OverwriteLabel, "", false, "Overwrite existing entry") + downloadsAddUrlCmd.Flags().BoolP(globals.SkipVerifyUrlLabel, "", false, "Skip URL accessibility check") + downloadsExportCmd.Flags().BoolP(globals.AddEmptyItemLabel, "", false, "Add an empty item to the tarballs list") downloadsImportCmd.Flags().Int64P(globals.RetriesOnFailureLabel, "", 0, "How many times retry a download if a failure occurs on first try") diff --git a/downloads/remote_registry.go b/downloads/remote_registry.go index 94125b4..6e383da 100644 --- a/downloads/remote_registry.go +++ b/downloads/remote_registry.go @@ -567,6 +567,95 @@ func checkRemoteUrl(remoteUrl string) (int64, error) { return size, nil } +// CheckRemoteUrl validates that a URL is accessible and returns its content size. +// It uses an HTTP HEAD request first; if that fails (some servers don't support HEAD), +// it falls back to a GET request. +func CheckRemoteUrl(remoteUrl string) (int64, error) { + // Try HEAD first to avoid downloading the file + // #nosec G107 + resp, err := http.Head(remoteUrl) + if err == nil { + defer func(Body io.ReadCloser) { + _ = Body.Close() + }(resp.Body) + if resp.StatusCode == http.StatusOK { + var size int64 + for key := range resp.Header { + if strings.EqualFold(key, "Content-Length") && len(resp.Header[key]) > 0 { + size, _ = strconv.ParseInt(resp.Header[key][0], 10, 0) + } + } + return size, nil + } + } + // Fall back to GET if HEAD failed or returned non-200 + return checkRemoteUrl(remoteUrl) +} + +// ParseTarballUrlInfo parses a tarball URL/filename and returns a partially-filled +// TarballDescription with auto-detected fields. The caller should override any +// fields that were incorrectly detected. +func ParseTarballUrlInfo(tarballUrl string) (TarballDescription, error) { + fileName := common.BaseName(tarballUrl) + if fileName == "" { + return TarballDescription{}, fmt.Errorf("could not determine filename from URL: %s", tarballUrl) + } + + flavor, version, shortVersion, err := common.FindTarballInfo(fileName) + if err != nil { + return TarballDescription{}, fmt.Errorf("could not parse version from filename %q: %s", fileName, err) + } + + OS, arch := detectOSArchFromFilename(fileName) + minimal := strings.Contains(strings.ToLower(fileName), "minimal") + + return TarballDescription{ + Name: fileName, + Url: tarballUrl, + Flavor: flavor, + Version: version, + ShortVersion: shortVersion, + OperatingSystem: OS, + Arch: arch, + Minimal: minimal, + }, nil +} + +// detectOSArchFromFilename attempts to detect the OS and architecture from a tarball filename. +// It handles common MySQL/Percona/MariaDB naming conventions. +func detectOSArchFromFilename(fileName string) (OS, arch string) { + lower := strings.ToLower(fileName) + + // Detect OS + switch { + case strings.Contains(lower, "linux") || strings.Contains(lower, "glibc"): + OS = "Linux" + case strings.Contains(lower, "macos") || strings.Contains(lower, "osx") || strings.Contains(lower, "darwin"): + OS = "Darwin" + case strings.Contains(lower, "windows") || strings.Contains(lower, "winx64"): + OS = "Windows" + default: + OS = runtime.GOOS + if OS == "darwin" { + OS = "Darwin" + } else if OS == "linux" { + OS = "Linux" + } + } + + // Detect architecture + switch { + case strings.Contains(lower, "arm64") || strings.Contains(lower, "aarch64"): + arch = "arm64" + case strings.Contains(lower, "x86_64") || strings.Contains(lower, "x86-64") || strings.Contains(lower, "amd64"): + arch = "amd64" + default: + arch = runtime.GOARCH + } + + return OS, arch +} + func CheckTarballList(tarballList []TarballDescription) error { uniqueNames := make(map[string]bool) uniqueCombinations := make(map[string]bool) diff --git a/downloads/remote_registry_test.go b/downloads/remote_registry_test.go index 882dcbb..44b730d 100644 --- a/downloads/remote_registry_test.go +++ b/downloads/remote_registry_test.go @@ -216,3 +216,105 @@ func TestMergeCollection(t *testing.T) { }) } } + +func TestParseTarballUrlInfo(t *testing.T) { + tests := []struct { + name string + url string + wantName string + wantVersion string + wantShort string + wantOS string + wantArch string + wantFlavor string + wantMinimal bool + wantErr bool + }{ + { + name: "mysql-linux-amd64", + url: "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.8-linux-glibc2.17-x86_64.tar.xz", + wantName: "mysql-8.4.8-linux-glibc2.17-x86_64.tar.xz", + wantVersion: "8.4.8", + wantShort: "8.4", + wantOS: "Linux", + wantArch: "amd64", + wantFlavor: "mysql", + wantMinimal: false, + }, + { + name: "mysql-linux-amd64-minimal", + url: "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.8-linux-glibc2.17-x86_64-minimal.tar.xz", + wantName: "mysql-8.4.8-linux-glibc2.17-x86_64-minimal.tar.xz", + wantVersion: "8.4.8", + wantShort: "8.4", + wantOS: "Linux", + wantArch: "amd64", + wantFlavor: "mysql", + wantMinimal: true, + }, + { + name: "mysql-macos-arm64", + url: "https://cdn.mysql.com/Downloads/MySQL-8.4/mysql-8.4.8-macos15-arm64.tar.gz", + wantName: "mysql-8.4.8-macos15-arm64.tar.gz", + wantVersion: "8.4.8", + wantShort: "8.4", + wantOS: "Darwin", + wantArch: "arm64", + wantFlavor: "mysql", + wantMinimal: false, + }, + { + name: "percona-linux-amd64-minimal", + url: "https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.35-27/binary/tarball/Percona-Server-8.0.35-27-Linux.x86_64.glibc2.17-minimal.tar.gz", + wantName: "Percona-Server-8.0.35-27-Linux.x86_64.glibc2.17-minimal.tar.gz", + wantVersion: "8.0.35", + wantShort: "8.0", + wantOS: "Linux", + wantArch: "amd64", + wantFlavor: "percona", + wantMinimal: true, + }, + { + name: "invalid-no-version", + url: "https://example.com/some-tarball-without-version.tar.gz", + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ParseTarballUrlInfo(tt.url) + if (err != nil) != tt.wantErr { + t.Errorf("ParseTarballUrlInfo() error = %v, wantErr %v", err, tt.wantErr) + return + } + if tt.wantErr { + return + } + if got.Name != tt.wantName { + t.Errorf("Name = %q, want %q", got.Name, tt.wantName) + } + if got.Version != tt.wantVersion { + t.Errorf("Version = %q, want %q", got.Version, tt.wantVersion) + } + if got.ShortVersion != tt.wantShort { + t.Errorf("ShortVersion = %q, want %q", got.ShortVersion, tt.wantShort) + } + if got.OperatingSystem != tt.wantOS { + t.Errorf("OperatingSystem = %q, want %q", got.OperatingSystem, tt.wantOS) + } + if got.Arch != tt.wantArch { + t.Errorf("Arch = %q, want %q", got.Arch, tt.wantArch) + } + if got.Flavor != tt.wantFlavor { + t.Errorf("Flavor = %q, want %q", got.Flavor, tt.wantFlavor) + } + if got.Minimal != tt.wantMinimal { + t.Errorf("Minimal = %v, want %v", got.Minimal, tt.wantMinimal) + } + if got.Url != tt.url { + t.Errorf("Url = %q, want %q", got.Url, tt.url) + } + }) + } +} diff --git a/globals/globals.go b/globals/globals.go index 307dc3a..790306e 100644 --- a/globals/globals.go +++ b/globals/globals.go @@ -149,6 +149,7 @@ const ( DeleteAfterUnpackLabel = "delete-after-unpack" MaxItemsLabel = "max-items" ChangeUserAgentLabel = "change-user-agent" + SkipVerifyUrlLabel = "skip-verify-url" // Instantiated in cmd/admin.go VerboseLabel = "verbose"