Skip to content

feat: apply mirror git config for serving performance #130

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
alecthomas merged 1 commit into from
Feb 20, 2026
Merged

feat: apply mirror git config for serving performance #130

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
55 changes: 49 additions & 6 deletions internal/gitclone/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,7 @@ func (m *Manager) Get(upstreamURL string) *Repository {
return m.clones[upstreamURL]
}

func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
func (m *Manager) DiscoverExisting(ctx context.Context) ([]*Repository, error) {
var discovered []*Repository
err := filepath.Walk(m.config.MirrorRoot, func(path string, info os.FileInfo, err error) error {
if err != nil {
Expand Down Expand Up @@ -226,6 +226,10 @@ func (m *Manager) DiscoverExisting(_ context.Context) ([]*Repository, error) {
}
repo.fetchSem <- struct{}{}

if err := configureMirror(ctx, path); err != nil {
return errors.Wrapf(err, "configure mirror for %s", upstreamURL)
}

m.clonesMu.Lock()
m.clones[upstreamURL] = repo
m.clonesMu.Unlock()
Expand Down Expand Up @@ -314,6 +318,48 @@ func (r *Repository) Clone(ctx context.Context) error {
return nil
}

// mirrorConfigSettings returns git config key-value pairs applied to mirror
// clones to optimise upload-pack serving performance.
func mirrorConfigSettings() [][2]string {
return [][2]string{
// Protocol
{"protocol.version", "2"},
{"uploadpack.allowFilter", "true"},
{"uploadpack.allowReachableSHA1InWant", "true"},
// Bitmaps
{"repack.writeBitmaps", "true"},
{"pack.useBitmaps", "true"},
{"pack.useBitmapBoundaryTraversal", "true"},
// Commit graph
{"core.commitGraph", "true"},
{"gc.writeCommitGraph", "true"},
{"fetch.writeCommitGraph", "true"},
// Multi-pack-index
{"core.multiPackIndex", "true"},
// Keep fetched objects as packs
{"transfer.unpackLimit", "1"},
{"fetch.unpackLimit", "1"},
// Disable auto GC
{"gc.auto", "0"},
// Pack performance
{"pack.threads", "0"},
{"pack.deltaCacheSize", "512m"},
{"pack.windowMemory", "1g"},
}
}

func configureMirror(ctx context.Context, repoPath string) error {
for _, kv := range mirrorConfigSettings() {
// #nosec G204 - repoPath and config values are controlled by us
cmd := exec.CommandContext(ctx, "git", "-C", repoPath, "config", kv[0], kv[1])
output, err := cmd.CombinedOutput()
if err != nil {
return errors.Wrapf(err, "configure %s: %s", kv[0], string(output))
}
}
return nil
}

func (r *Repository) executeClone(ctx context.Context) error {
if err := os.MkdirAll(filepath.Dir(r.path), 0o750); err != nil {
return errors.Wrap(err, "create clone directory")
Expand All @@ -338,11 +384,8 @@ func (r *Repository) executeClone(ctx context.Context) error {
return errors.Wrapf(err, "git clone --mirror: %s", string(output))
}

// Enable partial clone support (e.g. --filter=blob:none) when serving via git http-backend.
cmd = exec.CommandContext(ctx, "git", "-C", r.path, "config", "uploadpack.allowFilter", "true") // #nosec G204
output, err = cmd.CombinedOutput()
if err != nil {
return errors.Wrapf(err, "configure uploadpack.allowFilter: %s", string(output))
if err := configureMirror(ctx, r.path); err != nil {
return errors.Wrap(err, "configure mirror")
}

return nil
Expand Down
100 changes: 53 additions & 47 deletions internal/gitclone/manager_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,32 @@ import (
"github.com/block/cachew/internal/logging"
)

// createBareRepo creates a bare git repository at the given path, suitable for
// use as an upstream or as a mirror clone target.
func createBareRepo(t *testing.T, dir string) string {
t.Helper()
workPath := filepath.Join(dir, "work")
barePath := filepath.Join(dir, "upstream.git")
assert.NoError(t, os.MkdirAll(workPath, 0o755))

for _, args := range [][]string{
{"git", "-C", workPath, "init"},
{"git", "-C", workPath, "config", "user.email", "test@example.com"},
{"git", "-C", workPath, "config", "user.name", "Test"},
} {
assert.NoError(t, exec.Command(args[0], args[1:]...).Run())
}
assert.NoError(t, os.WriteFile(filepath.Join(workPath, "f.txt"), []byte("x"), 0o644))
for _, args := range [][]string{
{"git", "-C", workPath, "add", "."},
{"git", "-C", workPath, "commit", "-m", "init"},
{"git", "clone", "--bare", workPath, barePath},
} {
assert.NoError(t, exec.Command(args[0], args[1:]...).Run())
}
return barePath
}

func TestNewManager(t *testing.T) {
_, ctx := logging.Configure(t.Context(), logging.Config{Level: slog.LevelError})
tmpDir := t.TempDir()
Expand Down Expand Up @@ -130,15 +156,18 @@ func TestManager_DiscoverExisting(t *testing.T) {
manager, err := NewManager(ctx, config, nil)
assert.NoError(t, err)

repos := []string{
// Create a real bare repo as a source, then clone it into the mirror paths.
upstreamPath := createBareRepo(t, t.TempDir())

repoPaths := []string{
filepath.Join(tmpDir, "github.com", "user1", "repo1"),
filepath.Join(tmpDir, "github.com", "user2", "repo2"),
filepath.Join(tmpDir, "gitlab.com", "org", "project"),
}

for _, repoPath := range repos {
assert.NoError(t, os.MkdirAll(repoPath, 0o755))
assert.NoError(t, os.WriteFile(filepath.Join(repoPath, "HEAD"), []byte("ref: refs/heads/main\n"), 0o644))
for _, repoPath := range repoPaths {
assert.NoError(t, os.MkdirAll(filepath.Dir(repoPath), 0o755))
cmd := exec.Command("git", "clone", "--bare", upstreamPath, repoPath)
assert.NoError(t, cmd.Run())
}

discovered, err := manager.DiscoverExisting(context.Background())
Expand All @@ -156,6 +185,16 @@ func TestManager_DiscoverExisting(t *testing.T) {
repo3 := manager.Get("https://gitlab.com/org/project")
assert.NotZero(t, repo3)
assert.Equal(t, StateReady, repo3.State())

// Verify mirror config was applied to discovered repos.
for _, repoPath := range repoPaths {
for _, kv := range mirrorConfigSettings() {
cmd := exec.Command("git", "-C", repoPath, "config", "--get", kv[0])
output, err := cmd.Output()
assert.NoError(t, err, "config key %s in %s", kv[0], repoPath)
assert.Equal(t, kv[1], strings.TrimSpace(string(output)), "config key %s in %s", kv[0], repoPath)
}
}
}

func TestRepository_StateTransitions(t *testing.T) {
Expand Down Expand Up @@ -222,25 +261,7 @@ func TestState_String(t *testing.T) {
func TestRepository_Clone_StateVisibleDuringClone(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()

// Create a bare upstream repo to clone from
upstreamPath := filepath.Join(tmpDir, "upstream.git")
workPath := filepath.Join(tmpDir, "work")
assert.NoError(t, os.MkdirAll(workPath, 0o755))

cmd := exec.Command("git", "-C", workPath, "init")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "-C", workPath, "config", "user.email", "test@example.com")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "-C", workPath, "config", "user.name", "Test")
assert.NoError(t, cmd.Run())
assert.NoError(t, os.WriteFile(filepath.Join(workPath, "f.txt"), []byte("x"), 0o644))
cmd = exec.Command("git", "-C", workPath, "add", ".")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "-C", workPath, "commit", "-m", "init")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "clone", "--bare", workPath, upstreamPath)
assert.NoError(t, cmd.Run())
upstreamPath := createBareRepo(t, tmpDir)

clonePath := filepath.Join(tmpDir, "clone")
repo := &Repository{
Expand Down Expand Up @@ -278,27 +299,10 @@ func TestRepository_Clone_StateVisibleDuringClone(t *testing.T) {
assert.Equal(t, StateReady, repo.State())
}

func TestRepository_CloneSetsAllowFilter(t *testing.T) {
func TestRepository_CloneSetsMirrorConfig(t *testing.T) {
ctx := context.Background()
tmpDir := t.TempDir()

upstreamPath := filepath.Join(tmpDir, "upstream.git")
workPath := filepath.Join(tmpDir, "work")
assert.NoError(t, os.MkdirAll(workPath, 0o755))

cmd := exec.Command("git", "-C", workPath, "init")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "-C", workPath, "config", "user.email", "test@example.com")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "-C", workPath, "config", "user.name", "Test")
assert.NoError(t, cmd.Run())
assert.NoError(t, os.WriteFile(filepath.Join(workPath, "f.txt"), []byte("x"), 0o644))
cmd = exec.Command("git", "-C", workPath, "add", ".")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "-C", workPath, "commit", "-m", "init")
assert.NoError(t, cmd.Run())
cmd = exec.Command("git", "clone", "--bare", workPath, upstreamPath)
assert.NoError(t, cmd.Run())
upstreamPath := createBareRepo(t, tmpDir)

clonePath := filepath.Join(tmpDir, "clone")
repo := &Repository{
Expand All @@ -312,10 +316,12 @@ func TestRepository_CloneSetsAllowFilter(t *testing.T) {
assert.NoError(t, repo.Clone(ctx))
assert.Equal(t, StateReady, repo.State())

cmd = exec.Command("git", "-C", clonePath, "config", "uploadpack.allowFilter")
output, err := cmd.Output()
assert.NoError(t, err)
assert.Equal(t, "true", strings.TrimSpace(string(output)))
for _, kv := range mirrorConfigSettings() {
cmd := exec.Command("git", "-C", clonePath, "config", "--get", kv[0])
output, err := cmd.Output()
assert.NoError(t, err, "config key %s", kv[0])
assert.Equal(t, kv[1], strings.TrimSpace(string(output)), "config key %s", kv[0])
}
}

func TestRepository_HasCommit(t *testing.T) {
Expand Down