From 2d9261c555aef7795698d1f050b0e9ef3a6290a1 Mon Sep 17 00:00:00 2001 From: Alec Thomas Date: Fri, 20 Feb 2026 12:56:27 +1100 Subject: [PATCH] feat: apply mirror git config for serving performance (#124) Add configureMirror() helper that applies git config settings to mirror clones for optimised upload-pack serving: - Protocol v2, allowFilter, allowReachableSHA1InWant - Bitmap settings for faster pack generation - Commit graph (without Bloom filters) - Multi-pack-index to avoid full repack on fetch - Unpack limits to keep fetched objects as packs - Disable auto GC (maintenance will be explicit) - Pack performance tuning (threads, deltaCacheSize, windowMemory) Called after git clone --mirror succeeds and for repos discovered by DiscoverExisting() on startup, so pre-existing repos get updated. --- internal/gitclone/manager.go | 55 ++++++++++++++-- internal/gitclone/manager_test.go | 100 ++++++++++++++++-------------- 2 files changed, 102 insertions(+), 53 deletions(-) diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index 9632671..4f4c41d 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -181,7 +181,7 @@ func (m *Manager) Get(upstreamURL string) *Repository { return m.clones[upstreamURL] } -func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { +func (m *Manager) DiscoverExisting(ctx context.Context) ([]*Repository, error) { var discovered []*Repository err := filepath.Walk(m.config.MirrorRoot, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -226,6 +226,10 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) { } repo.fetchSem <- struct{}{} + if err := configureMirror(ctx, path); err != nil { + return errors.Wrapf(err, "configure mirror for %s", upstreamURL) + } + m.clonesMu.Lock() m.clones[upstreamURL] = repo m.clonesMu.Unlock() @@ -314,6 +318,48 @@ func (r *Repository) Clone(ctx context.Context) error { return nil } +// mirrorConfigSettings returns git config key-value pairs applied to mirror +// clones to optimise upload-pack serving performance. +func mirrorConfigSettings() [][2]string { + return [][2]string{ + // Protocol + {"protocol.version", "2"}, + {"uploadpack.allowFilter", "true"}, + {"uploadpack.allowReachableSHA1InWant", "true"}, + // Bitmaps + {"repack.writeBitmaps", "true"}, + {"pack.useBitmaps", "true"}, + {"pack.useBitmapBoundaryTraversal", "true"}, + // Commit graph + {"core.commitGraph", "true"}, + {"gc.writeCommitGraph", "true"}, + {"fetch.writeCommitGraph", "true"}, + // Multi-pack-index + {"core.multiPackIndex", "true"}, + // Keep fetched objects as packs + {"transfer.unpackLimit", "1"}, + {"fetch.unpackLimit", "1"}, + // Disable auto GC + {"gc.auto", "0"}, + // Pack performance + {"pack.threads", "0"}, + {"pack.deltaCacheSize", "512m"}, + {"pack.windowMemory", "1g"}, + } +} + +func configureMirror(ctx context.Context, repoPath string) error { + for _, kv := range mirrorConfigSettings() { + // #nosec G204 - repoPath and config values are controlled by us + cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "config", kv[0], kv[1]) + output, err := cmd.CombinedOutput() + if err != nil { + return errors.Wrapf(err, "configure %s: %s", kv[0], string(output)) + } + } + return nil +} + func (r *Repository) executeClone(ctx context.Context) error { if err := os.MkdirAll(filepath.Dir(r.path), 0o750); err != nil { return errors.Wrap(err, "create clone directory") @@ -338,11 +384,8 @@ func (r *Repository) executeClone(ctx context.Context) error { return errors.Wrapf(err, "git clone --mirror: %s", string(output)) } - // Enable partial clone support (e.g. --filter=blob:none) when serving via git http-backend. - cmd = exec.CommandContext(ctx, "git", "-C", r.path, "config", "uploadpack.allowFilter", "true") // #nosec G204 - output, err = cmd.CombinedOutput() - if err != nil { - return errors.Wrapf(err, "configure uploadpack.allowFilter: %s", string(output)) + if err := configureMirror(ctx, r.path); err != nil { + return errors.Wrap(err, "configure mirror") } return nil diff --git a/internal/gitclone/manager_test.go b/internal/gitclone/manager_test.go index 9768b7c..1444da4 100644 --- a/internal/gitclone/manager_test.go +++ b/internal/gitclone/manager_test.go @@ -15,6 +15,32 @@ import ( "github.com/block/cachew/internal/logging" ) +// createBareRepo creates a bare git repository at the given path, suitable for +// use as an upstream or as a mirror clone target. +func createBareRepo(t *testing.T, dir string) string { + t.Helper() + workPath := filepath.Join(dir, "work") + barePath := filepath.Join(dir, "upstream.git") + assert.NoError(t, os.MkdirAll(workPath, 0o755)) + + for _, args := range [][]string{ + {"git", "-C", workPath, "init"}, + {"git", "-C", workPath, "config", "user.email", "test@example.com"}, + {"git", "-C", workPath, "config", "user.name", "Test"}, + } { + assert.NoError(t, exec.Command(args[0], args[1:]...).Run()) + } + assert.NoError(t, os.WriteFile(filepath.Join(workPath, "f.txt"), []byte("x"), 0o644)) + for _, args := range [][]string{ + {"git", "-C", workPath, "add", "."}, + {"git", "-C", workPath, "commit", "-m", "init"}, + {"git", "clone", "--bare", workPath, barePath}, + } { + assert.NoError(t, exec.Command(args[0], args[1:]...).Run()) + } + return barePath +} + func TestNewManager(t *testing.T) { _, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError}) tmpDir := t.TempDir() @@ -130,15 +156,18 @@ func TestManager_DiscoverExisting(t *testing.T) { manager, err := NewManager(ctx, config, nil) assert.NoError(t, err) - repos := []string{ + // Create a real bare repo as a source, then clone it into the mirror paths. + upstreamPath := createBareRepo(t, t.TempDir()) + + repoPaths := []string{ filepath.Join(tmpDir, "github.com", "user1", "repo1"), filepath.Join(tmpDir, "github.com", "user2", "repo2"), filepath.Join(tmpDir, "gitlab.com", "org", "project"), } - - for _, repoPath := range repos { - assert.NoError(t, os.MkdirAll(repoPath, 0o755)) - assert.NoError(t, os.WriteFile(filepath.Join(repoPath, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644)) + for _, repoPath := range repoPaths { + assert.NoError(t, os.MkdirAll(filepath.Dir(repoPath), 0o755)) + cmd := exec.Command("git", "clone", "--bare", upstreamPath, repoPath) + assert.NoError(t, cmd.Run()) } discovered, err := manager.DiscoverExisting(context.Background()) @@ -156,6 +185,16 @@ func TestManager_DiscoverExisting(t *testing.T) { repo3 := manager.Get("https://gitlab.com/org/project") assert.NotZero(t, repo3) assert.Equal(t, StateReady, repo3.State()) + + // Verify mirror config was applied to discovered repos. + for _, repoPath := range repoPaths { + for _, kv := range mirrorConfigSettings() { + cmd := exec.Command("git", "-C", repoPath, "config", "--get", kv[0]) + output, err := cmd.Output() + assert.NoError(t, err, "config key %s in %s", kv[0], repoPath) + assert.Equal(t, kv[1], strings.TrimSpace(string(output)), "config key %s in %s", kv[0], repoPath) + } + } } func TestRepository_StateTransitions(t *testing.T) { @@ -222,25 +261,7 @@ func TestState_String(t *testing.T) { func TestRepository_Clone_StateVisibleDuringClone(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() - - // Create a bare upstream repo to clone from - upstreamPath := filepath.Join(tmpDir, "upstream.git") - workPath := filepath.Join(tmpDir, "work") - assert.NoError(t, os.MkdirAll(workPath, 0o755)) - - cmd := exec.Command("git", "-C", workPath, "init") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "-C", workPath, "config", "user.email", "test@example.com") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "-C", workPath, "config", "user.name", "Test") - assert.NoError(t, cmd.Run()) - assert.NoError(t, os.WriteFile(filepath.Join(workPath, "f.txt"), []byte("x"), 0o644)) - cmd = exec.Command("git", "-C", workPath, "add", ".") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "-C", workPath, "commit", "-m", "init") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "clone", "--bare", workPath, upstreamPath) - assert.NoError(t, cmd.Run()) + upstreamPath := createBareRepo(t, tmpDir) clonePath := filepath.Join(tmpDir, "clone") repo := &Repository{ @@ -278,27 +299,10 @@ func TestRepository_Clone_StateVisibleDuringClone(t *testing.T) { assert.Equal(t, StateReady, repo.State()) } -func TestRepository_CloneSetsAllowFilter(t *testing.T) { +func TestRepository_CloneSetsMirrorConfig(t *testing.T) { ctx := context.Background() tmpDir := t.TempDir() - - upstreamPath := filepath.Join(tmpDir, "upstream.git") - workPath := filepath.Join(tmpDir, "work") - assert.NoError(t, os.MkdirAll(workPath, 0o755)) - - cmd := exec.Command("git", "-C", workPath, "init") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "-C", workPath, "config", "user.email", "test@example.com") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "-C", workPath, "config", "user.name", "Test") - assert.NoError(t, cmd.Run()) - assert.NoError(t, os.WriteFile(filepath.Join(workPath, "f.txt"), []byte("x"), 0o644)) - cmd = exec.Command("git", "-C", workPath, "add", ".") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "-C", workPath, "commit", "-m", "init") - assert.NoError(t, cmd.Run()) - cmd = exec.Command("git", "clone", "--bare", workPath, upstreamPath) - assert.NoError(t, cmd.Run()) + upstreamPath := createBareRepo(t, tmpDir) clonePath := filepath.Join(tmpDir, "clone") repo := &Repository{ @@ -312,10 +316,12 @@ func TestRepository_CloneSetsAllowFilter(t *testing.T) { assert.NoError(t, repo.Clone(ctx)) assert.Equal(t, StateReady, repo.State()) - cmd = exec.Command("git", "-C", clonePath, "config", "uploadpack.allowFilter") - output, err := cmd.Output() - assert.NoError(t, err) - assert.Equal(t, "true", strings.TrimSpace(string(output))) + for _, kv := range mirrorConfigSettings() { + cmd := exec.Command("git", "-C", clonePath, "config", "--get", kv[0]) + output, err := cmd.Output() + assert.NoError(t, err, "config key %s", kv[0]) + assert.Equal(t, kv[1], strings.TrimSpace(string(output)), "config key %s", kv[0]) + } } func TestRepository_HasCommit(t *testing.T) {