diff --git a/.golangci.yml b/.golangci.yml index 72bd61a..fd6d20c 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -69,6 +69,7 @@ linters: - funcorder # checks the order of functions, methods, and constructors - funlen # tool for detection of long functions - gocheckcompilerdirectives # validates go compiler directive comments (//go:) + - gochecknoglobals - gochecksumtype # checks exhaustiveness on Go "sum types" - goconst # finds repeated strings that could be replaced by a constant - gocritic # provides diagnostics that check for bugs, performance and style issues diff --git a/cachew.hcl b/cachew.hcl index 6cfa596..107ecae 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -12,8 +12,11 @@ log { level = "debug" } +git-clone { + mirror-root = "./state/git-mirrors" +} + git { - mirror-root = "./state/git-mirrors" bundle-interval = "24h" snapshot-interval = "24h" } diff --git a/cmd/cachewd/main.go b/cmd/cachewd/main.go index be34db2..789820d 100644 --- a/cmd/cachewd/main.go +++ b/cmd/cachewd/main.go @@ -19,6 +19,7 @@ import ( "github.com/block/cachew/internal/cache" "github.com/block/cachew/internal/config" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/httputil" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" @@ -34,9 +35,10 @@ type GlobalConfig struct { SchedulerConfig jobscheduler.Config `embed:"" hcl:"scheduler,block" prefix:"scheduler-"` LoggingConfig logging.Config `embed:"" hcl:"log,block" prefix:"log-"` MetricsConfig metrics.Config `embed:"" hcl:"metrics,block" prefix:"metrics-"` + GitCloneConfig gitclone.Config `embed:"" hcl:"git-clone,block" prefix:"git-clone-"` } -var cli struct { +var cli struct { //nolint:gochecknoglobals Schema bool `help:"Print the configuration file schema." xor:"command"` Config kong.ConfigFlag `hcl:"-" help:"Configuration file path." placeholder:"PATH" required:"" default:"cachew.hcl"` @@ -60,9 +62,12 @@ func main() { ctx := context.Background() logger, ctx := logging.Configure(ctx, cli.LoggingConfig) + // Start initialising + managerProvider := gitclone.NewManagerProvider(ctx, cli.GitCloneConfig) + scheduler := jobscheduler.New(ctx, cli.SchedulerConfig) - cr, sr := newRegistries(scheduler) + cr, sr := newRegistries(scheduler, managerProvider) // Commands switch { //nolint:gocritic @@ -93,7 +98,7 @@ func main() { kctx.FatalIfErrorf(err) } -func newRegistries(scheduler jobscheduler.Scheduler) (*cache.Registry, *strategy.Registry) { +func newRegistries(scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider) (*cache.Registry, *strategy.Registry) { cr := cache.NewRegistry() cache.RegisterMemory(cr) cache.RegisterDisk(cr) @@ -105,8 +110,8 @@ func newRegistries(scheduler jobscheduler.Scheduler) (*cache.Registry, *strategy strategy.RegisterGitHubReleases(sr) strategy.RegisterHermit(sr, cli.URL) strategy.RegisterHost(sr) - git.Register(sr, scheduler) - gomod.Register(sr) + git.Register(sr, scheduler, cloneManagerProvider) + gomod.Register(sr, cloneManagerProvider) return cr, sr } diff --git a/internal/cache/disk_metadb.go b/internal/cache/disk_metadb.go index 0cc8979..9349aa7 100644 --- a/internal/cache/disk_metadb.go +++ b/internal/cache/disk_metadb.go @@ -10,6 +10,7 @@ import ( "go.etcd.io/bbolt" ) +//nolint:gochecknoglobals var ( ttlBucketName = []byte("ttl") headersBucketName = []byte("headers") diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index f332a2d..3df5bec 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -13,25 +13,10 @@ import ( "time" "github.com/alecthomas/errors" -) -var ( - sharedManager *Manager - sharedManagerMu sync.RWMutex + "github.com/block/cachew/internal/logging" ) -func SetShared(m *Manager) { - sharedManagerMu.Lock() - defer sharedManagerMu.Unlock() - sharedManager = m -} - -func GetShared() *Manager { - sharedManagerMu.RLock() - defer sharedManagerMu.RUnlock() - return sharedManager -} - type State int const ( @@ -68,14 +53,14 @@ func DefaultGitTuningConfig() GitTuningConfig { } type Config struct { - RootDir string - FetchInterval time.Duration - RefCheckInterval time.Duration - GitConfig GitTuningConfig + MirrorRoot string `hcl:"mirror-root" help:"Directory to store git clones."` + FetchInterval time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream in minutes." default:"15m"` + RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"` } type Repository struct { mu sync.RWMutex + config Config state State path string upstreamURL string @@ -86,26 +71,54 @@ type Repository struct { } type Manager struct { - config Config - clones map[string]*Repository - clonesMu sync.RWMutex + config Config + gitTuningConfig GitTuningConfig + clones map[string]*Repository + clonesMu sync.RWMutex +} + +// ManagerProvider is a function that lazily creates a singleton Manager. +type ManagerProvider func() (*Manager, error) + +func NewManagerProvider(ctx context.Context, config Config) ManagerProvider { + return sync.OnceValues(func() (*Manager, error) { + return NewManager(ctx, config) + }) } -func NewManager(_ context.Context, config Config) (*Manager, error) { - if config.RootDir == "" { - return nil, errors.New("RootDir is required") +func NewManager(ctx context.Context, config Config) (*Manager, error) { + if config.MirrorRoot == "" { + return nil, errors.New("mirror-root is required") } - if err := os.MkdirAll(config.RootDir, 0o750); err != nil { + if config.FetchInterval == 0 { + config.FetchInterval = 15 * time.Minute + } + + if config.RefCheckInterval == 0 { + config.RefCheckInterval = 10 * time.Second + } + + if err := os.MkdirAll(config.MirrorRoot, 0o750); err != nil { return nil, errors.Wrap(err, "create root directory") } + logging.FromContext(ctx).InfoContext(ctx, "Git clone manager initialised", + "mirror_root", config.MirrorRoot, + "fetch_interval", config.FetchInterval, + "ref_check_interval", config.RefCheckInterval) + return &Manager{ - config: config, - clones: make(map[string]*Repository), + config: config, + gitTuningConfig: DefaultGitTuningConfig(), + clones: make(map[string]*Repository), }, nil } +func (m *Manager) Config() Config { + return m.config +} + func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repository, error) { m.clonesMu.RLock() repo, exists := m.clones[upstreamURL] @@ -126,6 +139,7 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor repo = &Repository{ state: StateEmpty, + config: m.config, path: clonePath, upstreamURL: upstreamURL, fetchSem: make(chan struct{}, 1), @@ -148,13 +162,9 @@ func (m *Manager) Get(upstreamURL string) *Repository { return m.clones[upstreamURL] } -func (m *Manager) Config() Config { - return m.config -} - func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { var discovered []*Repository - err := filepath.Walk(m.config.RootDir, func(path string, info os.FileInfo, err error) error { + err := filepath.Walk(m.config.MirrorRoot, func(path string, info os.FileInfo, err error) error { if err != nil { return err } @@ -178,7 +188,7 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { return errors.Wrap(statErr, "stat HEAD file") } - relPath, err := filepath.Rel(m.config.RootDir, path) + relPath, err := filepath.Rel(m.config.MirrorRoot, path) if err != nil { return errors.Wrap(err, "get relative path") } @@ -221,11 +231,11 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { func (m *Manager) clonePathForURL(upstreamURL string) string { parsed, err := url.Parse(upstreamURL) if err != nil { - return filepath.Join(m.config.RootDir, "unknown") + return filepath.Join(m.config.MirrorRoot, "unknown") } repoPath := strings.TrimSuffix(parsed.Path, ".git") - return filepath.Join(m.config.RootDir, parsed.Host, repoPath) + return filepath.Join(m.config.MirrorRoot, parsed.Host, repoPath) } func (r *Repository) State() State { @@ -266,7 +276,7 @@ func WithReadLockReturn[T any](repo *Repository, fn func() (T, error)) (T, error return fn() } -func (r *Repository) Clone(ctx context.Context, config Config) error { +func (r *Repository) Clone(ctx context.Context) error { r.mu.Lock() if r.state != StateEmpty { r.mu.Unlock() @@ -275,7 +285,7 @@ func (r *Repository) Clone(ctx context.Context, config Config) error { r.state = StateCloning r.mu.Unlock() - err := r.executeClone(ctx, config) + err := r.executeClone(ctx) r.mu.Lock() if err != nil { @@ -290,17 +300,18 @@ func (r *Repository) Clone(ctx context.Context, config Config) error { return nil } -func (r *Repository) executeClone(ctx context.Context, config Config) error { +func (r *Repository) executeClone(ctx context.Context) error { if err := os.MkdirAll(filepath.Dir(r.path), 0o750); err != nil { return errors.Wrap(err, "create clone directory") } + config := DefaultGitTuningConfig() // #nosec G204 - r.upstreamURL and r.path are controlled by us args := []string{ "clone", - "-c", "http.postBuffer=" + strconv.Itoa(config.GitConfig.PostBuffer), - "-c", "http.lowSpeedLimit=" + strconv.Itoa(config.GitConfig.LowSpeedLimit), - "-c", "http.lowSpeedTime=" + strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), + "-c", "http.postBuffer=" + strconv.Itoa(config.PostBuffer), + "-c", "http.lowSpeedLimit=" + strconv.Itoa(config.LowSpeedLimit), + "-c", "http.lowSpeedTime=" + strconv.Itoa(int(config.LowSpeedTime.Seconds())), r.upstreamURL, r.path, } @@ -321,9 +332,9 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error { } cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path, - "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), - "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), - "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), + "-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer), + "-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit), + "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())), "fetch", "--all") if err != nil { return errors.Wrap(err, "create git command for fetch") @@ -336,7 +347,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error { return nil } -func (r *Repository) Fetch(ctx context.Context, config Config) error { +func (r *Repository) Fetch(ctx context.Context) error { select { case <-r.fetchSem: defer func() { @@ -356,11 +367,13 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error { r.mu.Lock() + config := DefaultGitTuningConfig() + // #nosec G204 - r.path is controlled by us cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path, - "-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer), - "-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit), - "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())), + "-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer), + "-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit), + "-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())), "remote", "update", "--prune") if err != nil { return errors.Wrap(err, "create git command") @@ -376,9 +389,9 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error { return nil } -func (r *Repository) EnsureRefsUpToDate(ctx context.Context, config Config) error { +func (r *Repository) EnsureRefsUpToDate(ctx context.Context) error { r.mu.Lock() - if r.refCheckValid && time.Since(r.lastRefCheck) < config.RefCheckInterval { + if r.refCheckValid && time.Since(r.lastRefCheck) < r.config.RefCheckInterval { r.mu.Unlock() return nil } @@ -419,7 +432,7 @@ func (r *Repository) EnsureRefsUpToDate(ctx context.Context, config Config) erro return nil } - err = r.Fetch(ctx, config) + err = r.Fetch(ctx) if err != nil { r.mu.Lock() r.refCheckValid = false diff --git a/internal/gitclone/manager_test.go b/internal/gitclone/manager_test.go index f538ae9..0793e30 100644 --- a/internal/gitclone/manager_test.go +++ b/internal/gitclone/manager_test.go @@ -2,6 +2,7 @@ package gitclone //nolint:testpackage // white-box testing required for unexport import ( "context" + "log/slog" "os" "os/exec" "path/filepath" @@ -9,45 +10,48 @@ import ( "time" "github.com/alecthomas/assert/v2" + + "github.com/block/cachew/internal/logging" ) func TestNewManager(t *testing.T) { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) tmpDir := t.TempDir() config := Config{ - RootDir: tmpDir, + MirrorRoot: tmpDir, FetchInterval: 15 * time.Minute, RefCheckInterval: 10 * time.Second, - GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(ctx, config) assert.NoError(t, err) assert.NotZero(t, manager) - assert.Equal(t, tmpDir, manager.config.RootDir) + assert.Equal(t, tmpDir, manager.config.MirrorRoot) } func TestNewManager_RequiresRootDir(t *testing.T) { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) config := Config{ FetchInterval: 15 * time.Minute, RefCheckInterval: 10 * time.Second, } - _, err := NewManager(context.Background(), config) + _, err := NewManager(ctx, config) assert.Error(t, err) - assert.Contains(t, err.Error(), "RootDir is required") + assert.Contains(t, err.Error(), "mirror-root is required") } func TestManager_GetOrCreate(t *testing.T) { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) tmpDir := t.TempDir() config := Config{ - RootDir: tmpDir, + MirrorRoot: tmpDir, FetchInterval: 15 * time.Minute, RefCheckInterval: 10 * time.Second, - GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(ctx, config) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -65,15 +69,15 @@ func TestManager_GetOrCreate(t *testing.T) { } func TestManager_GetOrCreate_ExistingClone(t *testing.T) { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) tmpDir := t.TempDir() config := Config{ - RootDir: tmpDir, + MirrorRoot: tmpDir, FetchInterval: 15 * time.Minute, RefCheckInterval: 10 * time.Second, - GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(ctx, config) assert.NoError(t, err) repoPath := filepath.Join(tmpDir, "github.com", "user", "repo") @@ -90,15 +94,15 @@ func TestManager_GetOrCreate_ExistingClone(t *testing.T) { } func TestManager_Get(t *testing.T) { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) tmpDir := t.TempDir() config := Config{ - RootDir: tmpDir, + MirrorRoot: tmpDir, FetchInterval: 15 * time.Minute, RefCheckInterval: 10 * time.Second, - GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(ctx, config) assert.NoError(t, err) upstreamURL := "https://github.com/user/repo" @@ -115,15 +119,15 @@ func TestManager_Get(t *testing.T) { } func TestManager_DiscoverExisting(t *testing.T) { + _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) tmpDir := t.TempDir() config := Config{ - RootDir: tmpDir, + MirrorRoot: tmpDir, FetchInterval: 15 * time.Minute, RefCheckInterval: 10 * time.Second, - GitConfig: DefaultGitTuningConfig(), } - manager, err := NewManager(context.Background(), config) + manager, err := NewManager(ctx, config) assert.NoError(t, err) repos := []string{ @@ -248,15 +252,10 @@ func TestRepository_Clone_StateVisibleDuringClone(t *testing.T) { } repo.fetchSem <- struct{}{} - config := Config{ - RootDir: tmpDir, - GitConfig: DefaultGitTuningConfig(), - } - // Start clone in background cloneDone := make(chan error, 1) go func() { - cloneDone <- repo.Clone(ctx, config) + cloneDone <- repo.Clone(ctx) }() // Poll until we observe StateCloning (should not block) diff --git a/internal/strategy/git/backend.go b/internal/strategy/git/backend.go index ff43f87..9b0dd8d 100644 --- a/internal/strategy/git/backend.go +++ b/internal/strategy/git/backend.go @@ -28,7 +28,7 @@ func (s *Strategy) serveFromBackend(w http.ResponseWriter, r *http.Request, repo return } - absRoot, err := filepath.Abs(s.config.MirrorRoot) + absRoot, err := filepath.Abs(s.cloneManager.Config().MirrorRoot) if err != nil { httputil.ErrorResponse(w, r, http.StatusInternalServerError, "failed to get absolute path") return @@ -87,13 +87,7 @@ func (s *Strategy) serveFromBackend(w http.ResponseWriter, r *http.Request, repo } func (s *Strategy) ensureRefsUpToDate(ctx context.Context, repo *gitclone.Repository) error { - gitcloneConfig := gitclone.Config{ - RootDir: s.config.MirrorRoot, - FetchInterval: s.config.FetchInterval, - RefCheckInterval: s.config.RefCheckInterval, - GitConfig: gitclone.DefaultGitTuningConfig(), - } - if err := repo.EnsureRefsUpToDate(ctx, gitcloneConfig); err != nil { + if err := repo.EnsureRefsUpToDate(ctx); err != nil { return errors.Wrap(err, "ensure refs up to date") } return nil diff --git a/internal/strategy/git/bundle_test.go b/internal/strategy/git/bundle_test.go index 70ca1f6..5d21fdf 100644 --- a/internal/strategy/git/bundle_test.go +++ b/internal/strategy/git/bundle_test.go @@ -10,6 +10,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" @@ -19,14 +20,17 @@ func TestBundleHTTPEndpoint(t *testing.T) { _, ctx := logging.Configure(context.Background(), logging.Config{}) tmpDir := t.TempDir() + cloneManager := gitclone.NewManagerProvider(ctx, gitclone.Config{ + MirrorRoot: tmpDir, + }) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{}) assert.NoError(t, err) mux := newTestMux() _, err = git.New(ctx, git.Config{ - MirrorRoot: tmpDir, BundleInterval: 24 * time.Hour, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cloneManager) assert.NoError(t, err) // Create a fake bundle in the cache @@ -99,10 +103,13 @@ func TestBundleInterval(t *testing.T) { assert.NoError(t, err) mux := newTestMux() + cloneManager := gitclone.NewManagerProvider(ctx, gitclone.Config{ + MirrorRoot: tmpDir, + }) + s, err := git.New(ctx, git.Config{ - MirrorRoot: tmpDir, BundleInterval: tt.bundleInterval, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cloneManager) assert.NoError(t, err) assert.NotZero(t, s) diff --git a/internal/strategy/git/git.go b/internal/strategy/git/git.go index 8c2fb6e..17de96e 100644 --- a/internal/strategy/git/git.go +++ b/internal/strategy/git/git.go @@ -26,16 +26,13 @@ import ( "github.com/block/cachew/internal/strategy" ) -func Register(r *strategy.Registry, scheduler jobscheduler.Scheduler) { +func Register(r *strategy.Registry, scheduler jobscheduler.Scheduler, cloneManager gitclone.ManagerProvider) { strategy.Register(r, "git", "Caches Git repositories, including bundle and tarball snapshots.", func(ctx context.Context, config Config, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { - return New(ctx, config, scheduler, cache, mux) + return New(ctx, config, scheduler, cache, mux, cloneManager) }) } type Config struct { - MirrorRoot string `hcl:"mirror-root" help:"Directory to store git clones." required:""` - FetchInterval time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream in minutes." default:"15m"` - RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"` BundleInterval time.Duration `hcl:"bundle-interval,optional" help:"How often to generate bundles. 0 disables bundling." default:"0"` SnapshotInterval time.Duration `hcl:"snapshot-interval,optional" help:"How often to generate tar.zstd snapshots. 0 disables snapshots." default:"0"` } @@ -52,34 +49,21 @@ type Strategy struct { spools map[string]*RepoSpools } -func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { +func New( + ctx context.Context, + config Config, + scheduler jobscheduler.Scheduler, + cache cache.Cache, + mux strategy.Mux, + cloneManagerProvider gitclone.ManagerProvider, +) (*Strategy, error) { logger := logging.FromContext(ctx) - if config.MirrorRoot == "" { - return nil, errors.New("mirror-root is required") - } - - if config.FetchInterval == 0 { - config.FetchInterval = 15 * time.Minute - } - - if config.RefCheckInterval == 0 { - config.RefCheckInterval = 10 * time.Second - } - - cloneManager, err := gitclone.NewManager(ctx, gitclone.Config{ - RootDir: config.MirrorRoot, - FetchInterval: config.FetchInterval, - RefCheckInterval: config.RefCheckInterval, - GitConfig: gitclone.DefaultGitTuningConfig(), - }) + cloneManager, err := cloneManagerProvider() if err != nil { - return nil, errors.Wrap(err, "create clone manager") + return nil, errors.Wrap(err, "failed to create clone manager") } - - gitclone.SetShared(cloneManager) - - if err := os.RemoveAll(filepath.Join(config.MirrorRoot, ".spools")); err != nil { + if err := os.RemoveAll(filepath.Join(cloneManager.Config().MirrorRoot, ".spools")); err != nil { return nil, errors.Wrap(err, "clean up stale spools") } @@ -125,9 +109,6 @@ func New(ctx context.Context, config Config, scheduler jobscheduler.Scheduler, c mux.Handle("POST /git/{host}/{path...}", http.HandlerFunc(s.handleRequest)) logger.InfoContext(ctx, "Git strategy initialized", - "mirror_root", config.MirrorRoot, - "fetch_interval", config.FetchInterval, - "ref_check_interval", config.RefCheckInterval, "bundle_interval", config.BundleInterval, "snapshot_interval", config.SnapshotInterval) @@ -247,7 +228,7 @@ func (s *Strategy) getOrCreateRepoSpools(upstreamURL string) *RepoSpools { defer s.spoolsMu.Unlock() rp, exists := s.spools[upstreamURL] if !exists { - dir := spoolDirForURL(s.config.MirrorRoot, upstreamURL) + dir := spoolDirForURL(s.cloneManager.Config().MirrorRoot, upstreamURL) rp = NewRepoSpools(dir) s.spools[upstreamURL] = rp } @@ -391,14 +372,7 @@ func (s *Strategy) startClone(ctx context.Context, repo *gitclone.Repository) { slog.String("upstream", repo.UpstreamURL()), slog.String("path", repo.Path())) - gitcloneConfig := gitclone.Config{ - RootDir: s.config.MirrorRoot, - FetchInterval: s.config.FetchInterval, - RefCheckInterval: s.config.RefCheckInterval, - GitConfig: gitclone.DefaultGitTuningConfig(), - } - - err := repo.Clone(ctx, gitcloneConfig) + err := repo.Clone(ctx) // Clean up spools regardless of clone success or failure, so that subsequent // requests either serve from the local backend or go directly to upstream. @@ -425,7 +399,7 @@ func (s *Strategy) startClone(ctx context.Context, repo *gitclone.Repository) { } func (s *Strategy) maybeBackgroundFetch(repo *gitclone.Repository) { - if !repo.NeedsFetch(s.config.FetchInterval) { + if !repo.NeedsFetch(s.cloneManager.Config().FetchInterval) { return } @@ -438,7 +412,7 @@ func (s *Strategy) maybeBackgroundFetch(repo *gitclone.Repository) { func (s *Strategy) backgroundFetch(ctx context.Context, repo *gitclone.Repository) { logger := logging.FromContext(ctx) - if !repo.NeedsFetch(s.config.FetchInterval) { + if !repo.NeedsFetch(s.cloneManager.Config().FetchInterval) { return } @@ -446,14 +420,7 @@ func (s *Strategy) backgroundFetch(ctx context.Context, repo *gitclone.Repositor slog.String("upstream", repo.UpstreamURL()), slog.String("path", repo.Path())) - gitcloneConfig := gitclone.Config{ - RootDir: s.config.MirrorRoot, - FetchInterval: s.config.FetchInterval, - RefCheckInterval: s.config.RefCheckInterval, - GitConfig: gitclone.DefaultGitTuningConfig(), - } - - if err := repo.Fetch(ctx, gitcloneConfig); err != nil { + if err := repo.Fetch(ctx); err != nil { logger.ErrorContext(ctx, "Fetch failed", slog.String("upstream", repo.UpstreamURL()), slog.String("error", err.Error())) diff --git a/internal/strategy/git/git_test.go b/internal/strategy/git/git_test.go index a56f118..03200db 100644 --- a/internal/strategy/git/git_test.go +++ b/internal/strategy/git/git_test.go @@ -38,26 +38,26 @@ func TestNew(t *testing.T) { tests := []struct { name string - config git.Config + config gitclone.Config wantError string }{ { name: "ValidConfig", - config: git.Config{ + config: gitclone.Config{ MirrorRoot: filepath.Join(tmpDir, "clones"), FetchInterval: 15, }, }, { name: "MissingClonesRoot", - config: git.Config{ + config: gitclone.Config{ FetchInterval: 15, }, wantError: "mirror-root is required", }, { name: "DefaultFetchInterval", - config: git.Config{ + config: gitclone.Config{ MirrorRoot: filepath.Join(tmpDir, "clones2"), }, }, @@ -66,7 +66,8 @@ func TestNew(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { mux := newTestMux() - s, err := git.New(ctx, tt.config, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux) + cm := gitclone.NewManagerProvider(ctx, tt.config) + s, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm) if tt.wantError != "" { assert.Error(t, err) assert.Contains(t, err.Error(), tt.wantError) @@ -146,10 +147,11 @@ func TestNewWithExistingCloneOnDisk(t *testing.T) { assert.NoError(t, err) mux := newTestMux() - s, err := git.New(ctx, git.Config{ + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, FetchInterval: 15, - }, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux) + }) + s, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm) assert.NoError(t, err) assert.NotZero(t, s) } @@ -169,10 +171,11 @@ func TestIntegrationWithMockUpstream(t *testing.T) { // Create strategy - it will register handlers mux := newTestMux() - _, err := git.New(ctx, git.Config{ + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ MirrorRoot: tmpDir, FetchInterval: 15, - }, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux) + }) + _, err := git.New(ctx, git.Config{}, jobscheduler.New(ctx, jobscheduler.Config{}), nil, mux, cm) assert.NoError(t, err) // Verify handlers exist diff --git a/internal/strategy/git/snapshot_test.go b/internal/strategy/git/snapshot_test.go index cf53709..594971a 100644 --- a/internal/strategy/git/snapshot_test.go +++ b/internal/strategy/git/snapshot_test.go @@ -10,6 +10,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/jobscheduler" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/git" @@ -23,10 +24,12 @@ func TestSnapshotHTTPEndpoint(t *testing.T) { assert.NoError(t, err) mux := newTestMux() + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ + MirrorRoot: tmpDir, + }) _, err = git.New(ctx, git.Config{ - MirrorRoot: tmpDir, SnapshotInterval: 24 * time.Hour, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cm) assert.NoError(t, err) // Create a fake snapshot in the cache @@ -95,10 +98,12 @@ func TestSnapshotInterval(t *testing.T) { assert.NoError(t, err) mux := newTestMux() + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ + MirrorRoot: tmpDir, + }) s, err := git.New(ctx, git.Config{ - MirrorRoot: tmpDir, SnapshotInterval: tt.snapshotInterval, - }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + }, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux, cm) assert.NoError(t, err) assert.NotZero(t, s) }) diff --git a/internal/strategy/github_releases_test.go b/internal/strategy/github_releases_test.go index 7518a04..1671481 100644 --- a/internal/strategy/github_releases_test.go +++ b/internal/strategy/github_releases_test.go @@ -20,7 +20,7 @@ import ( // httpTransportMutex ensures GitHub release tests don't run in parallel // since they modify the global http.DefaultTransport -var httpTransportMutex sync.Mutex +var httpTransportMutex sync.Mutex //nolint:gochecknoglobals type mockGitHubServer struct { server *httptest.Server diff --git a/internal/strategy/gomod/gomod.go b/internal/strategy/gomod/gomod.go index 36ff279..0a07d14 100644 --- a/internal/strategy/gomod/gomod.go +++ b/internal/strategy/gomod/gomod.go @@ -7,7 +7,6 @@ import ( "net/http" "net/url" - "github.com/alecthomas/errors" "github.com/goproxy/goproxy" "github.com/block/cachew/internal/cache" @@ -16,8 +15,10 @@ import ( "github.com/block/cachew/internal/strategy" ) -func Register(r *strategy.Registry) { - strategy.Register(r, "gomod", "Caches Go module proxy requests.", New) +func Register(r *strategy.Registry, cloneManager gitclone.ManagerProvider) { + strategy.Register(r, "gomod", "Caches Go module proxy requests.", func(ctx context.Context, config Config, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { + return New(ctx, config, cache, mux, cloneManager) + }) } type Config struct { @@ -36,17 +37,23 @@ type Strategy struct { var _ strategy.Strategy = (*Strategy)(nil) -func New(ctx context.Context, config Config, cache cache.Cache, mux strategy.Mux) (*Strategy, error) { +func New(ctx context.Context, config Config, cache cache.Cache, mux strategy.Mux, cloneManagerProvider gitclone.ManagerProvider) (*Strategy, error) { parsedURL, err := url.Parse(config.Proxy) if err != nil { return nil, fmt.Errorf("invalid proxy URL: %w", err) } + cloneManager, err := cloneManagerProvider() + if err != nil { + return nil, fmt.Errorf("failed to create clone manager: %w", err) + } + s := &Strategy{ - config: config, - cache: cache, - logger: logging.FromContext(ctx), - proxy: parsedURL, + config: config, + cache: cache, + logger: logging.FromContext(ctx), + proxy: parsedURL, + cloneManager: cloneManager, } publicFetcher := &goproxy.GoFetcher{ @@ -59,11 +66,6 @@ func New(ctx context.Context, config Config, cache cache.Cache, mux strategy.Mux var fetcher goproxy.Fetcher = publicFetcher if len(config.PrivatePaths) > 0 { - cloneManager := gitclone.GetShared() - if cloneManager == nil { - return nil, errors.New("private-paths configured but git strategy not initialized - git strategy with mirror-root is required for private module support") - } - s.cloneManager = cloneManager privateFetcher := newPrivateFetcher(s.logger, cloneManager) fetcher = NewCompositeFetcher(publicFetcher, privateFetcher, config.PrivatePaths) diff --git a/internal/strategy/gomod/gomod_test.go b/internal/strategy/gomod/gomod_test.go index cfa23ca..a89436a 100644 --- a/internal/strategy/gomod/gomod_test.go +++ b/internal/strategy/gomod/gomod_test.go @@ -15,6 +15,7 @@ import ( "github.com/alecthomas/assert/v2" "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/gitclone" "github.com/block/cachew/internal/logging" "github.com/block/cachew/internal/strategy/gomod" ) @@ -174,10 +175,14 @@ func setupGoModTest(t *testing.T) (*mockGoModServer, *http.ServeMux, context.Con assert.NoError(t, err) t.Cleanup(func() { _ = memCache.Close() }) + cm := gitclone.NewManagerProvider(ctx, gitclone.Config{ + MirrorRoot: t.TempDir(), + }) + assert.NoError(t, err) mux := http.NewServeMux() _, err = gomod.New(ctx, gomod.Config{ Proxy: mock.server.URL, - }, memCache, mux) + }, memCache, mux, cm) assert.NoError(t, err) return mock, mux, ctx diff --git a/internal/strategy/gomod/private_fetcher.go b/internal/strategy/gomod/private_fetcher.go index 0536db1..955a979 100644 --- a/internal/strategy/gomod/private_fetcher.go +++ b/internal/strategy/gomod/private_fetcher.go @@ -121,8 +121,7 @@ func (p *privateFetcher) ensureReady(ctx context.Context, repo *gitclone.Reposit return nil } - config := p.cloneManager.Config() - if err := repo.Clone(ctx, config); err != nil { + if err := repo.Clone(ctx); err != nil { return errors.Wrap(err, "clone repository") } diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index 3d9b0ae..55f7355 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -18,7 +18,7 @@ import ( // httpTransportMutexHermit ensures hermit tests don't run in parallel // since they modify the global http.DefaultTransport -var httpTransportMutexHermit sync.Mutex +var httpTransportMutexHermit sync.Mutex //nolint:gochecknoglobals func setupHermitTest(t *testing.T) (*http.ServeMux, context.Context, cache.Cache) { t.Helper()