From f8a7a7cd42e8ebccb4f8fd2ca51f71970d4e14da Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 3 Feb 2026 11:00:31 +1100 Subject: [PATCH 1/2] fix: Remove private gomod handling --- internal/strategy/gomod/fetcher.go | 55 ---- internal/strategy/gomod/gomod.go | 79 +---- internal/strategy/gomod/matcher.go | 32 -- internal/strategy/gomod/matcher_test.go | 123 -------- internal/strategy/gomod/private_fetcher.go | 334 --------------------- 5 files changed, 13 insertions(+), 610 deletions(-) delete mode 100644 internal/strategy/gomod/fetcher.go delete mode 100644 internal/strategy/gomod/matcher.go delete mode 100644 internal/strategy/gomod/matcher_test.go delete mode 100644 internal/strategy/gomod/private_fetcher.go diff --git a/internal/strategy/gomod/fetcher.go b/internal/strategy/gomod/fetcher.go deleted file mode 100644 index a67020b..0000000 --- a/internal/strategy/gomod/fetcher.go +++ /dev/null @@ -1,55 +0,0 @@ -package gomod - -import ( - "context" - "io" - "time" - - "github.com/alecthomas/errors" - "github.com/goproxy/goproxy" -) - -type compositeFetcher struct { - publicFetcher goproxy.Fetcher - privateFetcher goproxy.Fetcher - matcher *ModulePathMatcher -} - -func newCompositeFetcher( - publicFetcher goproxy.Fetcher, - privateFetcher goproxy.Fetcher, - patterns []string, -) *compositeFetcher { - return &compositeFetcher{ - publicFetcher: publicFetcher, - privateFetcher: privateFetcher, - matcher: NewModulePathMatcher(patterns), - } -} - -func (c *compositeFetcher) Query(ctx context.Context, path, query string) (version string, t time.Time, err error) { - if c.matcher.IsPrivate(path) { - v, tm, err := c.privateFetcher.Query(ctx, path, query) - return v, tm, errors.Wrap(err, "private fetcher query") - } - v, tm, err := c.publicFetcher.Query(ctx, path, query) - return v, tm, errors.Wrap(err, "public fetcher query") -} - -func (c *compositeFetcher) List(ctx context.Context, path string) (versions []string, err error) { - if c.matcher.IsPrivate(path) { - v, err := c.privateFetcher.List(ctx, path) - return v, errors.Wrap(err, "private fetcher list") - } - v, err := c.publicFetcher.List(ctx, path) - return v, errors.Wrap(err, "public fetcher list") -} - -func (c *compositeFetcher) Download(ctx context.Context, path, version string) (info, mod, zip io.ReadSeekCloser, err error) { - if c.matcher.IsPrivate(path) { - i, m, z, err := c.privateFetcher.Download(ctx, path, version) - return i, m, z, errors.Wrap(err, "private fetcher download") - } - i, m, z, err := c.publicFetcher.Download(ctx, path, version) - return i, m, z, errors.Wrap(err, "public fetcher download") -} diff --git a/internal/strategy/gomod/gomod.go b/internal/strategy/gomod/gomod.go index 2f610c0..3fc91b3 100644 --- a/internal/strategy/gomod/gomod.go +++ b/internal/strategy/gomod/gomod.go @@ -6,15 +6,10 @@ import ( "log/slog" "net/http" "net/url" - "os" - "path/filepath" - "time" - "github.com/alecthomas/errors" "github.com/goproxy/goproxy" "github.com/block/cachew/internal/cache" - "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy" @@ -25,21 +20,15 @@ func init() { } type Config struct { - Proxy string `hcl:"proxy,optional" help:"Upstream Go module proxy URL (defaults to proxy.golang.org)" default:"https://proxy.golang.org"` - PrivatePaths []string `hcl:"private-paths,optional" help:"Module path patterns for private repositories"` - MirrorRoot string `hcl:"mirror-root,optional" help:"Directory to store git clones for private repos." default:""` - FetchInterval time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream for private repos." default:"15m"` - RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks for private repos." default:"10s"` - CloneDepth int `hcl:"clone-depth,optional" help:"Depth for shallow clones of private repos. 0 means full clone." default:"0"` + Proxy string `hcl:"proxy,optional" help:"Upstream Go module proxy URL (defaults to proxy.golang.org)" default:"https://proxy.golang.org"` } type Strategy struct { - config Config - cache cache.Cache - logger *slog.Logger - proxy *url.URL - goproxy *goproxy.Goproxy - cloneManager *gitclone.Manager // Manager for cloning private repositories + config Config + cache cache.Cache + logger *slog.Logger + proxy *url.URL + goproxy *goproxy.Goproxy } var _ strategy.Strategy = (*Strategy)(nil) @@ -57,56 +46,14 @@ func New(ctx context.Context, config Config, _ jobscheduler.Scheduler, cache cac proxy: parsedURL, } - publicFetcher := &goproxy.GoFetcher{ - Env: []string{ - "GOPROXY=" + config.Proxy, - "GOSUMDB=off", // Disable checksum database validation in fetcher, to prevent unneccessary double validation - }, - } - - var fetcher goproxy.Fetcher = publicFetcher - - if len(config.PrivatePaths) > 0 { - // Set default mirror root if not specified - mirrorRoot := config.MirrorRoot - if mirrorRoot == "" { - homeDir, err := os.UserHomeDir() - if err != nil { - return nil, errors.Wrap(err, "get user home directory") - } - mirrorRoot = filepath.Join(homeDir, ".cache", "cachew", "gomod-git-mirrors") - } - - // Create gitclone manager for private repositories - cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{ - RootDir: mirrorRoot, - FetchInterval: config.FetchInterval, - RefCheckInterval: config.RefCheckInterval, - CloneDepth: config.CloneDepth, - GitConfig: gitclone.DefaultGitTuningConfig(), - }) - if err != nil { - return nil, errors.Wrap(err, "create clone manager for private repos") - } - s.cloneManager = cloneManager - - // Discover existing clones - if err := cloneManager.DiscoverExisting(ctx); err != nil { - s.logger.WarnContext(ctx, "Failed to discover existing clones for private repos", - slog.String("error", err.Error())) - } - - privateFetcher := newPrivateFetcher(s, cloneManager) - fetcher = newCompositeFetcher(publicFetcher, privateFetcher, config.PrivatePaths) - - s.logger.InfoContext(ctx, "Configured private module support", - slog.Any("private_paths", config.PrivatePaths), - slog.String("mirror_root", mirrorRoot)) - } - s.goproxy = &goproxy.Goproxy{ - Logger: s.logger, - Fetcher: fetcher, + Logger: s.logger, + Fetcher: &goproxy.GoFetcher{ + Env: []string{ + "GOPROXY=" + config.Proxy, + "GOSUMDB=off", // Disable checksum database validation in fetcher, to prevent unneccessary double validation + }, + }, Cacher: &goproxyCacher{ cache: cache, }, diff --git a/internal/strategy/gomod/matcher.go b/internal/strategy/gomod/matcher.go deleted file mode 100644 index 3a163f9..0000000 --- a/internal/strategy/gomod/matcher.go +++ /dev/null @@ -1,32 +0,0 @@ -package gomod - -import ( - "path" - "strings" -) - -// ModulePathMatcher matches module paths against patterns. -type ModulePathMatcher struct { - patterns []string -} - -// NewModulePathMatcher creates a new matcher with the given patterns. -func NewModulePathMatcher(patterns []string) *ModulePathMatcher { - return &ModulePathMatcher{patterns: patterns} -} - -// IsPrivate checks if a module path matches any private pattern. -func (m *ModulePathMatcher) IsPrivate(modulePath string) bool { - for _, pattern := range m.patterns { - matched, err := path.Match(pattern, modulePath) - if err == nil && matched { - return true - } - - if strings.HasPrefix(modulePath, pattern+"/") || modulePath == pattern { - return true - } - } - - return false -} diff --git a/internal/strategy/gomod/matcher_test.go b/internal/strategy/gomod/matcher_test.go deleted file mode 100644 index d1ce7b0..0000000 --- a/internal/strategy/gomod/matcher_test.go +++ /dev/null @@ -1,123 +0,0 @@ -package gomod_test - -import ( - "testing" - - "github.com/block/cachew/internal/strategy/gomod" -) - -func TestModulePathMatcher(t *testing.T) { - tests := []struct { - name string - patterns []string - modulePath string - want bool - }{ - { - name: "exact match single pattern", - patterns: []string{"github.com/squareup"}, - modulePath: "github.com/squareup", - want: true, - }, - { - name: "exact match with multiple patterns", - patterns: []string{"github.com/org1", "github.com/squareup", "github.com/org2"}, - modulePath: "github.com/squareup", - want: true, - }, - { - name: "prefix match - one level deep", - patterns: []string{"github.com/squareup"}, - modulePath: "github.com/squareup/repo", - want: true, - }, - { - name: "prefix match - two levels deep", - patterns: []string{"github.com/squareup"}, - modulePath: "github.com/squareup/repo/submodule", - want: true, - }, - { - name: "prefix match with multiple patterns", - patterns: []string{"github.com/org1", "github.com/squareup"}, - modulePath: "github.com/squareup/repo", - want: true, - }, - { - name: "wildcard match", - patterns: []string{"github.com/squareup/*"}, - modulePath: "github.com/squareup/repo", - want: true, - }, - { - name: "wildcard match - multiple levels", - patterns: []string{"github.com/*/*"}, - modulePath: "github.com/squareup/repo", - want: true, - }, - { - name: "no match - different org", - patterns: []string{"github.com/squareup"}, - modulePath: "github.com/other/repo", - want: false, - }, - { - name: "no match - different host", - patterns: []string{"github.com/squareup"}, - modulePath: "gitlab.com/squareup/repo", - want: false, - }, - { - name: "no match - prefix without slash", - patterns: []string{"github.com/square"}, - modulePath: "github.com/squareup/repo", - want: false, - }, - { - name: "no match - empty patterns", - patterns: []string{}, - modulePath: "github.com/squareup/repo", - want: false, - }, - { - name: "empty module path", - patterns: []string{"github.com/squareup"}, - modulePath: "", - want: false, - }, - { - name: "multiple patterns with no match", - patterns: []string{"github.com/org1", "github.com/org2", "github.com/org3"}, - modulePath: "github.com/squareup/repo", - want: false, - }, - { - name: "pattern with trailing slash", - patterns: []string{"github.com/squareup/"}, - modulePath: "github.com/squareup/repo", - want: false, - }, - { - name: "gopkg.in pattern", - patterns: []string{"gopkg.in/square"}, - modulePath: "gopkg.in/square/go-jose.v2", - want: true, - }, - { - name: "nested GitHub org pattern", - patterns: []string{"github.com/squareup/internal"}, - modulePath: "github.com/squareup/internal/auth", - want: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - matcher := gomod.NewModulePathMatcher(tt.patterns) - got := matcher.IsPrivate(tt.modulePath) - if got != tt.want { - t.Errorf("IsPrivate() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/internal/strategy/gomod/private_fetcher.go b/internal/strategy/gomod/private_fetcher.go deleted file mode 100644 index 862cf4e..0000000 --- a/internal/strategy/gomod/private_fetcher.go +++ /dev/null @@ -1,334 +0,0 @@ -package gomod - -import ( - "bytes" - "context" - "fmt" - "io" - "io/fs" - "log/slog" - "os/exec" - "sort" - "strings" - "time" - - "github.com/alecthomas/errors" - "golang.org/x/mod/semver" - - "github.com/block/cachew/internal/gitclone" -) - -type privateFetcher struct { - gomod *Strategy - cloneManager *gitclone.Manager -} - -func newPrivateFetcher(gomod *Strategy, cloneManager *gitclone.Manager) *privateFetcher { - return &privateFetcher{ - gomod: gomod, - cloneManager: cloneManager, - } -} - -func (p *privateFetcher) Query(ctx context.Context, path, query string) (version string, t time.Time, err error) { - logger := p.gomod.logger.With(slog.String("module", path), slog.String("query", query)) - logger.DebugContext(ctx, "Private fetcher: Query") - - gitURL := p.modulePathToGitURL(path) - - repo, err := p.cloneManager.GetOrCreate(ctx, gitURL) - if err != nil { - return "", time.Time{}, errors.Wrapf(err, "get or create clone for %s", path) - } - - if err := p.ensureCloneReady(ctx, repo); err != nil { - return "", time.Time{}, errors.Wrapf(err, "ensure clone for %s", path) - } - - resolvedVersion, commitTime, err := p.resolveVersionQuery(ctx, repo, query) - if err != nil { - return "", time.Time{}, errors.Wrapf(err, "resolve version query %s", query) - } - - return resolvedVersion, commitTime, nil -} - -func (p *privateFetcher) List(ctx context.Context, path string) (versions []string, err error) { - logger := p.gomod.logger.With(slog.String("module", path)) - logger.DebugContext(ctx, "Private fetcher: List") - - gitURL := p.modulePathToGitURL(path) - repo, err := p.cloneManager.GetOrCreate(ctx, gitURL) - if err != nil { - return nil, errors.Wrapf(err, "get or create clone for %s", path) - } - - if err := p.ensureCloneReady(ctx, repo); err != nil { - return nil, errors.Wrapf(err, "ensure clone for %s", path) - } - - versions, err = p.listVersions(ctx, repo) - if err != nil { - return nil, errors.Wrap(err, "list versions") - } - - return versions, nil -} - -func (p *privateFetcher) Download(ctx context.Context, path, version string) (info, mod, zip io.ReadSeekCloser, err error) { - logger := p.gomod.logger.With(slog.String("module", path), slog.String("version", version)) - logger.DebugContext(ctx, "Private fetcher: Download") - - gitURL := p.modulePathToGitURL(path) - repo, err := p.cloneManager.GetOrCreate(ctx, gitURL) - if err != nil { - return nil, nil, nil, errors.Wrapf(err, "get or create clone for %s", path) - } - - if err := p.ensureCloneReady(ctx, repo); err != nil { - return nil, nil, nil, errors.Wrapf(err, "ensure clone for %s", path) - } - - infoReader, err := p.generateInfo(ctx, repo, version) - if err != nil { - return nil, nil, nil, errors.Wrap(err, "generate info") - } - - modReader := p.generateMod(ctx, repo, path, version) - - zipReader, err := p.generateZip(ctx, repo, path, version) - if err != nil { - _ = infoReader.Close() - _ = modReader.Close() - return nil, nil, nil, errors.Wrap(err, "generate zip") - } - - return infoReader, modReader, zipReader, nil -} - -func (p *privateFetcher) modulePathToGitURL(modulePath string) string { - return "https://" + modulePath -} - -// ensureCloneReady ensures the repository is cloned and ready to use. -// It handles the cloning state machine and waits for the clone to complete if necessary. -func (p *privateFetcher) ensureCloneReady(ctx context.Context, repo *gitclone.Repository) error { - state := repo.State() - - switch state { - case gitclone.StateEmpty: - // Need to clone - gitcloneConfig := gitclone.Config{ - RootDir: p.gomod.config.MirrorRoot, - FetchInterval: p.gomod.config.FetchInterval, - RefCheckInterval: p.gomod.config.RefCheckInterval, - CloneDepth: p.gomod.config.CloneDepth, - GitConfig: gitclone.DefaultGitTuningConfig(), - } - if err := repo.Clone(ctx, gitcloneConfig); err != nil { - return errors.Wrap(err, "clone repository") - } - - case gitclone.StateCloning: - // Wait for clone to complete - for { - time.Sleep(100 * time.Millisecond) - currentState := repo.State() - - if currentState == gitclone.StateReady { - break - } - if currentState == gitclone.StateEmpty { - return errors.New("clone failed") - } - - select { - case <-ctx.Done(): - return errors.Wrap(ctx.Err(), "context cancelled waiting for clone") - default: - } - } - - case gitclone.StateReady: - // Maybe fetch if needed - if repo.NeedsFetch(p.gomod.config.FetchInterval) { - gitcloneConfig := gitclone.Config{ - RootDir: p.gomod.config.MirrorRoot, - FetchInterval: p.gomod.config.FetchInterval, - RefCheckInterval: p.gomod.config.RefCheckInterval, - CloneDepth: p.gomod.config.CloneDepth, - GitConfig: gitclone.DefaultGitTuningConfig(), - } - if err := repo.Fetch(ctx, gitcloneConfig); err != nil { - p.gomod.logger.WarnContext(ctx, "Failed to fetch updates", - slog.String("upstream", repo.UpstreamURL()), - slog.String("error", err.Error())) - } - } - } - - return nil -} - -// resolveVersionQuery resolves a version query (like "latest" or "v1.2.3") to a specific version. -func (p *privateFetcher) resolveVersionQuery(ctx context.Context, repo *gitclone.Repository, query string) (string, time.Time, error) { - if query == "latest" { - versions, err := p.listVersions(ctx, repo) - if err != nil || len(versions) == 0 { - return p.getDefaultBranchVersion(ctx, repo) - } - - latestVersion := versions[len(versions)-1] - commitTime, err := p.getCommitTime(ctx, repo, latestVersion) - if err != nil { - return "", time.Time{}, err - } - return latestVersion, commitTime, nil - } - - if semver.IsValid(query) { - commitTime, err := p.getCommitTime(ctx, repo, query) - if err != nil { - return "", time.Time{}, fs.ErrNotExist - } - return query, commitTime, nil - } - - return p.getDefaultBranchVersion(ctx, repo) -} - -func (p *privateFetcher) listVersions(ctx context.Context, repo *gitclone.Repository) ([]string, error) { - var output []byte - var err error - - repo.WithReadLock(func() { - // #nosec G204 - repo.Path() is controlled by us - cmd := exec.CommandContext(ctx, "git", "-C", repo.Path(), "tag", "-l", "v*") - output, err = cmd.CombinedOutput() - }) - - if err != nil { - return nil, errors.Wrapf(err, "git tag failed: %s", string(output)) - } - - var versions []string - for line := range strings.Lines(string(output)) { - line = strings.TrimSpace(line) - if line != "" && semver.IsValid(line) { - versions = append(versions, line) - } - } - - sort.Slice(versions, func(i, j int) bool { - return semver.Compare(versions[i], versions[j]) < 0 - }) - - return versions, nil -} - -func (p *privateFetcher) getCommitTime(ctx context.Context, repo *gitclone.Repository, ref string) (time.Time, error) { - var output []byte - var err error - - repo.WithReadLock(func() { - // #nosec G204 - repo.Path() and ref are controlled by us - cmd := exec.CommandContext(ctx, "git", "-C", repo.Path(), "log", "-1", "--format=%cI", ref) - output, err = cmd.CombinedOutput() - }) - - if err != nil { - return time.Time{}, errors.Wrapf(err, "git log failed: %s", string(output)) - } - - timeStr := strings.TrimSpace(string(output)) - t, err := time.Parse(time.RFC3339, timeStr) - return t, errors.Wrap(err, "parse commit time") -} - -func (p *privateFetcher) getDefaultBranchVersion(ctx context.Context, repo *gitclone.Repository) (string, time.Time, error) { - var output []byte - var err error - - repo.WithReadLock(func() { - // #nosec G204 - repo.Path() is controlled by us - cmd := exec.CommandContext(ctx, "git", "-C", repo.Path(), "rev-parse", "HEAD") - output, err = cmd.CombinedOutput() - }) - - if err != nil { - return "", time.Time{}, errors.Wrapf(err, "git rev-parse failed: %s", string(output)) - } - - commitHash := strings.TrimSpace(string(output)) - commitTime, err := p.getCommitTime(ctx, repo, "HEAD") - if err != nil { - return "", time.Time{}, err - } - - pseudoVersion := fmt.Sprintf("v0.0.0-%s-%s", - commitTime.UTC().Format("20060102150405"), - commitHash[:12]) - - return pseudoVersion, commitTime, nil -} - -func (p *privateFetcher) generateInfo(ctx context.Context, repo *gitclone.Repository, version string) (io.ReadSeekCloser, error) { - commitTime, err := p.getCommitTime(ctx, repo, version) - if err != nil { - return nil, err - } - - info := fmt.Sprintf(`{"Version":"%s","Time":"%s"}`, version, commitTime.Format(time.RFC3339)) - return newReadSeekCloser(bytes.NewReader([]byte(info))), nil -} - -func (p *privateFetcher) generateMod(ctx context.Context, repo *gitclone.Repository, modulePath, version string) io.ReadSeekCloser { - var output []byte - var err error - - repo.WithReadLock(func() { - // #nosec G204 - version and repo.Path() are controlled by this package, not user input - cmd := exec.CommandContext(ctx, "git", "-C", repo.Path(), "show", fmt.Sprintf("%s:go.mod", version)) - output, err = cmd.CombinedOutput() - }) - - if err != nil { - minimal := fmt.Sprintf("module %s\n\ngo 1.21\n", modulePath) - return newReadSeekCloser(bytes.NewReader([]byte(minimal))) - } - - return newReadSeekCloser(bytes.NewReader(output)) -} - -func (p *privateFetcher) generateZip(ctx context.Context, repo *gitclone.Repository, modulePath, version string) (io.ReadSeekCloser, error) { - prefix := fmt.Sprintf("%s@%s/", modulePath, version) - var output []byte - var err error - - repo.WithReadLock(func() { - // #nosec G204 - version and repo.Path() are controlled by this package, not user input - cmd := exec.CommandContext(ctx, "git", "-C", repo.Path(), "archive", - "--format=zip", - fmt.Sprintf("--prefix=%s", prefix), - version) - output, err = cmd.CombinedOutput() - }) - - if err != nil { - return nil, errors.Wrapf(err, "git archive failed: %s", string(output)) - } - - return newReadSeekCloser(bytes.NewReader(output)), nil -} - -type readSeekCloser struct { - *bytes.Reader -} - -func newReadSeekCloser(r *bytes.Reader) io.ReadSeekCloser { - return &readSeekCloser{Reader: r} -} - -func (r *readSeekCloser) Close() error { - return nil -} From 69628bf3b0aae267de93ed67da09b2a8dc047955 Mon Sep 17 00:00:00 2001 From: John Murphy Date: Tue, 3 Feb 2026 11:04:18 +1100 Subject: [PATCH 2/2] fix: gomod tidy --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index ee59a97..ca8d1a0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/lmittmann/tint v1.1.2 github.com/minio/minio-go/v7 v7.0.97 go.etcd.io/bbolt v1.4.3 - golang.org/x/mod v0.31.0 ) require ( @@ -31,6 +30,7 @@ require ( github.com/stretchr/testify v1.11.1 // indirect github.com/tinylib/msgp v1.3.0 // indirect golang.org/x/crypto v0.44.0 // indirect + golang.org/x/mod v0.31.0 // indirect golang.org/x/net v0.47.0 // indirect golang.org/x/sys v0.38.0 // indirect golang.org/x/text v0.32.0 // indirect