refactor: simplify gitclone

.golangci.yml

@@ -69,6 +69,7 @@ linters:
     - funcorder # checks the order of functions, methods, and constructors
     - funlen # tool for detection of long functions
     - gocheckcompilerdirectives # validates go compiler directive comments (//go:)
+    - gochecknoglobals
     - gochecksumtype # checks exhaustiveness on Go "sum types"
     - goconst # finds repeated strings that could be replaced by a constant
     - gocritic # provides diagnostics that check for bugs, performance and style issues

cachew.hcl

@@ -12,8 +12,11 @@ log {
   level = "debug"
 }
 
+git-clone {
+    mirror-root = "./state/git-mirrors"
+}
+
 git {
-  mirror-root = "./state/git-mirrors"
   bundle-interval = "24h"
   snapshot-interval = "24h"
 }

cmd/cachewd/main.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/block/cachew/internal/cache"
 	"github.com/block/cachew/internal/config"
+	"github.com/block/cachew/internal/gitclone"
 	"github.com/block/cachew/internal/httputil"
 	"github.com/block/cachew/internal/jobscheduler"
 	"github.com/block/cachew/internal/logging"
@@ -34,9 +35,10 @@ type GlobalConfig struct {
 	SchedulerConfig jobscheduler.Config `embed:"" hcl:"scheduler,block" prefix:"scheduler-"`
 	LoggingConfig   logging.Config      `embed:"" hcl:"log,block" prefix:"log-"`
 	MetricsConfig   metrics.Config      `embed:"" hcl:"metrics,block" prefix:"metrics-"`
+	GitCloneConfig  gitclone.Config     `embed:"" hcl:"git-clone,block" prefix:"git-clone-"`
 }
 
-var cli struct {
+var cli struct { //nolint:gochecknoglobals
 	Schema bool `help:"Print the configuration file schema." xor:"command"`
 
 	Config kong.ConfigFlag `hcl:"-" help:"Configuration file path." placeholder:"PATH" required:"" default:"cachew.hcl"`
@@ -60,9 +62,12 @@ func main() {
 	ctx := context.Background()
 	logger, ctx := logging.Configure(ctx, cli.LoggingConfig)
 
+	// Start initialising
+	managerProvider := gitclone.NewManagerProvider(ctx, cli.GitCloneConfig)
+
 	scheduler := jobscheduler.New(ctx, cli.SchedulerConfig)
 
-	cr, sr := newRegistries(scheduler)
+	cr, sr := newRegistries(scheduler, managerProvider)
 
 	// Commands
 	switch { //nolint:gocritic
@@ -93,7 +98,7 @@ func main() {
 	kctx.FatalIfErrorf(err)
 }
 
-func newRegistries(scheduler jobscheduler.Scheduler) (*cache.Registry, *strategy.Registry) {
+func newRegistries(scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider) (*cache.Registry, *strategy.Registry) {
 	cr := cache.NewRegistry()
 	cache.RegisterMemory(cr)
 	cache.RegisterDisk(cr)
@@ -105,8 +110,8 @@ func newRegistries(scheduler jobscheduler.Scheduler) (*cache.Registry, *strategy
 	strategy.RegisterGitHubReleases(sr)
 	strategy.RegisterHermit(sr, cli.URL)
 	strategy.RegisterHost(sr)
-	git.Register(sr, scheduler)
-	gomod.Register(sr)
+	git.Register(sr, scheduler, cloneManagerProvider)
+	gomod.Register(sr, cloneManagerProvider)
 
 	return cr, sr
 }

internal/cache/disk_metadb.go

@@ -10,6 +10,7 @@ import (
 	"go.etcd.io/bbolt"
 )
 
+//nolint:gochecknoglobals
 var (
 	ttlBucketName     = []byte("ttl")
 	headersBucketName = []byte("headers")

internal/gitclone/manager.go

@@ -13,25 +13,10 @@ import (
 	"time"
 
 	"github.com/alecthomas/errors"
-)
 
-var (
-	sharedManager   *Manager
-	sharedManagerMu sync.RWMutex
+	"github.com/block/cachew/internal/logging"
 )
 
-func SetShared(m *Manager) {
-	sharedManagerMu.Lock()
-	defer sharedManagerMu.Unlock()
-	sharedManager = m
-}
-
-func GetShared() *Manager {
-	sharedManagerMu.RLock()
-	defer sharedManagerMu.RUnlock()
-	return sharedManager
-}
-
 type State int
 
 const (
@@ -68,14 +53,14 @@ func DefaultGitTuningConfig() GitTuningConfig {
 }
 
 type Config struct {
-	RootDir          string
-	FetchInterval    time.Duration
-	RefCheckInterval time.Duration
-	GitConfig        GitTuningConfig
+	MirrorRoot       string        `hcl:"mirror-root" help:"Directory to store git clones."`
+	FetchInterval    time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream in minutes." default:"15m"`
+	RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"`
 }
 
 type Repository struct {
 	mu            sync.RWMutex
+	config        Config
 	state         State
 	path          string
 	upstreamURL   string
@@ -86,26 +71,54 @@ type Repository struct {
 }
 
 type Manager struct {
-	config   Config
-	clones   map[string]*Repository
-	clonesMu sync.RWMutex
+	config          Config
+	gitTuningConfig GitTuningConfig
+	clones          map[string]*Repository
+	clonesMu        sync.RWMutex
+}
+
+// ManagerProvider is a function that lazily creates a singleton Manager.
+type ManagerProvider func() (*Manager, error)
+
+func NewManagerProvider(ctx context.Context, config Config) ManagerProvider {
+	return sync.OnceValues(func() (*Manager, error) {
+		return NewManager(ctx, config)
+	})
 }
 
-func NewManager(_ context.Context, config Config) (*Manager, error) {
-	if config.RootDir == "" {
-		return nil, errors.New("RootDir is required")
+func NewManager(ctx context.Context, config Config) (*Manager, error) {
+	if config.MirrorRoot == "" {
+		return nil, errors.New("mirror-root is required")
 	}
 
-	if err := os.MkdirAll(config.RootDir, 0o750); err != nil {
+	if config.FetchInterval == 0 {
+		config.FetchInterval = 15 * time.Minute
+	}
+
+	if config.RefCheckInterval == 0 {
+		config.RefCheckInterval = 10 * time.Second
+	}
+
+	if err := os.MkdirAll(config.MirrorRoot, 0o750); err != nil {
 		return nil, errors.Wrap(err, "create root directory")
 	}
 
+	logging.FromContext(ctx).InfoContext(ctx, "Git clone manager initialised",
+		"mirror_root", config.MirrorRoot,
+		"fetch_interval", config.FetchInterval,
+		"ref_check_interval", config.RefCheckInterval)
+
 	return &Manager{
-		config: config,
-		clones: make(map[string]*Repository),
+		config:          config,
+		gitTuningConfig: DefaultGitTuningConfig(),
+		clones:          make(map[string]*Repository),
 	}, nil
 }
 
+func (m *Manager) Config() Config {
+	return m.config
+}
+
 func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repository, error) {
 	m.clonesMu.RLock()
 	repo, exists := m.clones[upstreamURL]
@@ -126,6 +139,7 @@ func (m *Manager) GetOrCreate(_ context.Context, upstreamURL string) (*Repositor
 
 	repo = &Repository{
 		state:       StateEmpty,
+		config:      m.config,
 		path:        clonePath,
 		upstreamURL: upstreamURL,
 		fetchSem:    make(chan struct{}, 1),
@@ -148,13 +162,9 @@ func (m *Manager) Get(upstreamURL string) *Repository {
 	return m.clones[upstreamURL]
 }
 
-func (m *Manager) Config() Config {
-	return m.config
-}
-
 func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
 	var discovered []*Repository
-	err := filepath.Walk(m.config.RootDir, func(path string, info os.FileInfo, err error) error {
+	err := filepath.Walk(m.config.MirrorRoot, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -178,7 +188,7 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
 			return errors.Wrap(statErr, "stat HEAD file")
 		}
 
-		relPath, err := filepath.Rel(m.config.RootDir, path)
+		relPath, err := filepath.Rel(m.config.MirrorRoot, path)
 		if err != nil {
 			return errors.Wrap(err, "get relative path")
 		}
@@ -221,11 +231,11 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
 func (m *Manager) clonePathForURL(upstreamURL string) string {
 	parsed, err := url.Parse(upstreamURL)
 	if err != nil {
-		return filepath.Join(m.config.RootDir, "unknown")
+		return filepath.Join(m.config.MirrorRoot, "unknown")
 	}
 
 	repoPath := strings.TrimSuffix(parsed.Path, ".git")
-	return filepath.Join(m.config.RootDir, parsed.Host, repoPath)
+	return filepath.Join(m.config.MirrorRoot, parsed.Host, repoPath)
 }
 
 func (r *Repository) State() State {
@@ -266,7 +276,7 @@ func WithReadLockReturn[T any](repo *Repository, fn func() (T, error)) (T, error
 	return fn()
 }
 
-func (r *Repository) Clone(ctx context.Context, config Config) error {
+func (r *Repository) Clone(ctx context.Context) error {
 	r.mu.Lock()
 	if r.state != StateEmpty {
 		r.mu.Unlock()
@@ -275,7 +285,7 @@ func (r *Repository) Clone(ctx context.Context, config Config) error {
 	r.state = StateCloning
 	r.mu.Unlock()
 
-	err := r.executeClone(ctx, config)
+	err := r.executeClone(ctx)
 
 	r.mu.Lock()
 	if err != nil {
@@ -290,17 +300,18 @@ func (r *Repository) Clone(ctx context.Context, config Config) error {
 	return nil
 }
 
-func (r *Repository) executeClone(ctx context.Context, config Config) error {
+func (r *Repository) executeClone(ctx context.Context) error {
 	if err := os.MkdirAll(filepath.Dir(r.path), 0o750); err != nil {
 		return errors.Wrap(err, "create clone directory")
 	}
 
+	config := DefaultGitTuningConfig()
 	// #nosec G204 - r.upstreamURL and r.path are controlled by us
 	args := []string{
 		"clone",
-		"-c", "http.postBuffer=" + strconv.Itoa(config.GitConfig.PostBuffer),
-		"-c", "http.lowSpeedLimit=" + strconv.Itoa(config.GitConfig.LowSpeedLimit),
-		"-c", "http.lowSpeedTime=" + strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())),
+		"-c", "http.postBuffer=" + strconv.Itoa(config.PostBuffer),
+		"-c", "http.lowSpeedLimit=" + strconv.Itoa(config.LowSpeedLimit),
+		"-c", "http.lowSpeedTime=" + strconv.Itoa(int(config.LowSpeedTime.Seconds())),
 		r.upstreamURL, r.path,
 	}
 
@@ -321,9 +332,9 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error {
 	}
 
 	cmd, err = gitCommand(ctx, r.upstreamURL, "-C", r.path,
-		"-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer),
-		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit),
-		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())),
+		"-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer),
+		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit),
+		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())),
 		"fetch", "--all")
 	if err != nil {
 		return errors.Wrap(err, "create git command for fetch")
@@ -336,7 +347,7 @@ func (r *Repository) executeClone(ctx context.Context, config Config) error {
 	return nil
 }
 
-func (r *Repository) Fetch(ctx context.Context, config Config) error {
+func (r *Repository) Fetch(ctx context.Context) error {
 	select {
 	case <-r.fetchSem:
 		defer func() {
@@ -356,11 +367,13 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error {
 
 	r.mu.Lock()
 
+	config := DefaultGitTuningConfig()
+
 	// #nosec G204 - r.path is controlled by us
 	cmd, err := gitCommand(ctx, r.upstreamURL, "-C", r.path,
-		"-c", "http.postBuffer="+strconv.Itoa(config.GitConfig.PostBuffer),
-		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.GitConfig.LowSpeedLimit),
-		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.GitConfig.LowSpeedTime.Seconds())),
+		"-c", "http.postBuffer="+strconv.Itoa(config.PostBuffer),
+		"-c", "http.lowSpeedLimit="+strconv.Itoa(config.LowSpeedLimit),
+		"-c", "http.lowSpeedTime="+strconv.Itoa(int(config.LowSpeedTime.Seconds())),
 		"remote", "update", "--prune")
 	if err != nil {
 		return errors.Wrap(err, "create git command")
@@ -376,9 +389,9 @@ func (r *Repository) Fetch(ctx context.Context, config Config) error {
 	return nil
 }
 
-func (r *Repository) EnsureRefsUpToDate(ctx context.Context, config Config) error {
+func (r *Repository) EnsureRefsUpToDate(ctx context.Context) error {
 	r.mu.Lock()
-	if r.refCheckValid && time.Since(r.lastRefCheck) < config.RefCheckInterval {
+	if r.refCheckValid && time.Since(r.lastRefCheck) < r.config.RefCheckInterval {
 		r.mu.Unlock()
 		return nil
 	}
@@ -419,7 +432,7 @@ func (r *Repository) EnsureRefsUpToDate(ctx context.Context, config Config) erro
 		return nil
 	}
 
-	err = r.Fetch(ctx, config)
+	err = r.Fetch(ctx)
 	if err != nil {
 		r.mu.Lock()
 		r.refCheckValid = false