From fccd880e93bd7731b6111ed435e343b3ade30c61 Mon Sep 17 00:00:00 2001 From: PARTHIV PRATIM SAIKIA Date: Fri, 12 Dec 2025 16:06:10 +0530 Subject: [PATCH 1/2] fix: handle rate limits and resource leaks in Artifact Hub sync Signed-off-by: PARTHIV PRATIM SAIKIA --- generators/artifacthub/package.go | 38 +++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index c2dda39b..16131774 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -142,7 +142,7 @@ func (pkg *AhPackage) Validator() { } -// GetAllAhHelmPackages returns a list of all AhPackages and is super slow to avoid rate limits. +// GetAllAhHelmPackages returns a list of all AhPackages, using exponential backoff to handle rate limits. func GetAllAhHelmPackages() ([]AhPackage, error) { pkgs := make([]AhPackage, 0) resp, err := http.Get(AhHelmExporterEndpoint) @@ -154,35 +154,59 @@ func GetAllAhHelmPackages() ([]AhPackage, error) { return nil, ErrGetAllHelmPackages(err) } defer resp.Body.Close() + var res []map[string]interface{} err = json.NewDecoder(resp.Body).Decode(&res) if err != nil { return nil, err } + for _, p := range res { name := p["name"].(string) repo := p["repository"].(map[string]interface{})["name"].(string) url := fmt.Sprintf("https://artifacthub.io/api/v1/packages/helm/%s/%s", repo, name) - resp, err := http.Get(url) + + var resp *http.Response + var err error + + for i := 0; i < 5; i++ { + resp, err = http.Get(url) + if err != nil { + break + } + if resp.StatusCode == 429 { + resp.Body.Close() + time.Sleep(time.Duration(1< Date: Mon, 22 Dec 2025 09:21:13 +0530 Subject: [PATCH 2/2] refactor: improve logging and readability Signed-off-by: PARTHIV PRATIM SAIKIA --- generators/artifacthub/package.go | 39 ++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index 00ef926f..a288739c 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -3,17 +3,17 @@ package artifacthub import ( "encoding/json" "fmt" - "net/http" - "strings" - "time" - "github.com/meshery/meshkit/utils" "github.com/meshery/meshkit/utils/component" "github.com/meshery/meshkit/utils/manifests" "github.com/meshery/schemas/models/v1beta1/category" _component "github.com/meshery/schemas/models/v1beta1/component" "github.com/meshery/schemas/models/v1beta1/model" + "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "net/http" + "strings" + "time" ) const ArtifactHubAPIEndpoint = "https://artifacthub.io/api/v1" @@ -133,7 +133,7 @@ func (pkg *AhPackage) UpdatePackageData() error { if !strings.HasSuffix(pkg.RepoUrl, "/") { pkg.RepoUrl = pkg.RepoUrl + "/" } - chartUrl = fmt.Sprintf("%s%s", pkg.RepoUrl, chartUrl) + pkg.ChartUrl = fmt.Sprintf("%s%s", pkg.RepoUrl, chartUrl) } pkg.ChartUrl = chartUrl return nil @@ -162,15 +162,31 @@ func GetAllAhHelmPackages() ([]AhPackage, error) { return nil, err } + const maxRetries = 5 + for _, p := range res { - name := p["name"].(string) - repo := p["repository"].(map[string]interface{})["name"].(string) + name, ok := p["name"].(string) + if !ok { + logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid name") + continue + } + repoMap, ok := p["repository"].(map[string]interface{}) + if !ok { + logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid repository") + continue + } + + repo, ok := repoMap["name"].(string) + if !ok { + logrus.WithFields(logrus.Fields{"package_data": p}).Warn("Skipping package due to missing or invalid repository name") + continue + } url := fmt.Sprintf("https://artifacthub.io/api/v1/packages/helm/%s/%s", repo, name) var resp *http.Response var err error - for i := 0; i < 5; i++ { + for i := 0; i < maxRetries; i++ { resp, err = http.Get(url) if err != nil { break @@ -184,13 +200,13 @@ func GetAllAhHelmPackages() ([]AhPackage, error) { } if err != nil { - fmt.Println(err) + logrus.WithError(err).WithField("url", url).Error("Failed to fetch package") continue } if resp.StatusCode != 200 { err = fmt.Errorf("status code %d for %s", resp.StatusCode, url) - fmt.Println(err) + logrus.WithError(err).WithField("url", url).Error("Failed to fetch package") resp.Body.Close() continue } @@ -200,7 +216,7 @@ func GetAllAhHelmPackages() ([]AhPackage, error) { resp.Body.Close() if err != nil { - fmt.Println(err) + logrus.WithError(err).WithField("url", url).Error("Failed to decode package response") continue } @@ -250,3 +266,4 @@ func parseArtifacthubResponse(response map[string]interface{}) *AhPackage { Official: official, } } +