From 64cad6d308199a6be9109ec9503f86b07006773d Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Tue, 3 Feb 2026 16:12:31 +1100 Subject: [PATCH 01/10] feat: Implement hermit caching strategy --- README.md | 8 +- internal/cache/noop.go | 62 +++++++ internal/strategy/hermit.go | 154 +++++++++++++++++ internal/strategy/hermit_test.go | 279 +++++++++++++++++++++++++++++++ 4 files changed, 502 insertions(+), 1 deletion(-) create mode 100644 internal/cache/noop.go create mode 100644 internal/strategy/hermit.go create mode 100644 internal/strategy/hermit_test.go diff --git a/README.md b/README.md index 7945b70..4668690 100644 --- a/README.md +++ b/README.md @@ -29,4 +29,10 @@ As Git itself isn't aware of the snapshots, Git-specific code in the Cachew CLI ## Hermit -Hermit +Caches Hermit package downloads from all sources (golang.org, npm, GitHub releases, etc.). + +**URL pattern:** `/hermit/{host}/{path...}` + +Example: `GET /hermit/golang.org/dl/go1.21.0.tar.gz` + +GitHub releases are automatically redirected to the `github-releases` strategy. diff --git a/internal/cache/noop.go b/internal/cache/noop.go new file mode 100644 index 0000000..ad645ab --- /dev/null +++ b/internal/cache/noop.go @@ -0,0 +1,62 @@ +package cache + +import ( + "context" + "io" + "net/http" + "os" + "time" +) + +// noOpCache is a cache implementation that doesn't cache anything. +// It always returns cache misses and discards writes. +// Useful for pass-through handlers that shouldn't cache. +type noOpCache struct{} + +// NoOpCache returns a cache that doesn't cache anything. +// All Open() calls return os.ErrNotExist (cache miss). +// All Create() calls return a writer that discards data. +func NoOpCache() Cache { + return &noOpCache{} +} + +func (n *noOpCache) String() string { return "noop" } + +func (n *noOpCache) Stat(_ context.Context, _ Key) (http.Header, error) { + return nil, os.ErrNotExist +} + +func (n *noOpCache) Open(_ context.Context, _ Key) (io.ReadCloser, http.Header, error) { + return nil, nil, os.ErrNotExist +} + +func (n *noOpCache) Create(_ context.Context, _ Key, _ http.Header, _ time.Duration) (io.WriteCloser, error) { + // Return a discard writer that does nothing + return &noOpWriter{}, nil +} + +func (n *noOpCache) Delete(_ context.Context, _ Key) error { + return nil +} + +func (n *noOpCache) Stats(_ context.Context) (Stats, error) { + return Stats{}, ErrStatsUnavailable +} + +func (n *noOpCache) Close() error { + return nil +} + +// noOpWriter is a writer that discards all data. +type noOpWriter struct{} + +func (n *noOpWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +func (n *noOpWriter) Close() error { + return nil +} + +var _ Cache = (*noOpCache)(nil) +var _ io.WriteCloser = (*noOpWriter)(nil) diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go new file mode 100644 index 0000000..0ee7df1 --- /dev/null +++ b/internal/strategy/hermit.go @@ -0,0 +1,154 @@ +package strategy + +import ( + "context" + "log/slog" + "net/http" + "net/url" + "strings" + + "github.com/alecthomas/errors" + + "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/jobscheduler" + "github.com/block/cachew/internal/logging" + "github.com/block/cachew/internal/strategy/handler" +) + +func init() { + Register("hermit", "Caches Hermit package downloads.", NewHermit) +} + +// HermitConfig for the Hermit strategy. +type HermitConfig struct { + // Future configuration can be added here +} + +// Hermit implements caching for Hermit package downloads. +// This strategy acts as a router that: +// 1. Detects GitHub release URLs and redirects to github-releases strategy +// 2. Handles all other URLs directly with simple HTTP GET. +type Hermit struct { + cache cache.Cache + client *http.Client + logger *slog.Logger + mux Mux +} + +var _ Strategy = (*Hermit)(nil) + +// NewHermit creates a new Hermit caching strategy. +func NewHermit(ctx context.Context, _ HermitConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*Hermit, error) { + logger := logging.FromContext(ctx) + + s := &Hermit{ + cache: cache, + client: http.DefaultClient, + logger: logger, + mux: mux, + } + + // Mount at /hermit/{host}/{path...} + // Example: /hermit/go.dev/dl/go1.21.0.linux-amd64.tar.gz + // Example: /hermit/github.com/squareup/repo/releases/download/v1.0/file.tar.gz + mux.Handle("GET /hermit/{host}/{path...}", http.HandlerFunc(s.handleRequest)) + + logger.InfoContext(ctx, "Hermit strategy initialized") + + return s, nil +} + +func (s *Hermit) String() string { return "hermit" } + +// handleRequest routes the request to the appropriate handler. +// GitHub releases are redirected to the github-releases strategy. +// Everything else is handled directly. +func (s *Hermit) handleRequest(w http.ResponseWriter, r *http.Request) { + host := r.PathValue("host") + path := r.PathValue("path") + + // Check if this is a GitHub release URL + if host == "github.com" && strings.Contains(path, "/releases/download/") { + // Redirect to github-releases strategy using handler pattern with no-cache + s.redirectToGitHubReleases(w, r, path) + return + } + + // Not a GitHub release, handle directly with handler pattern + s.handleNonGitHub(w, r, host, path) +} + +// redirectToGitHubReleases redirects to the github-releases strategy without caching. +// Uses handler pattern with NoOpCache to avoid double caching. +// The github-releases strategy will handle caching. +func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request, path string) { + newPath := "/github.com/" + path + + s.logger.DebugContext(r.Context(), "Redirecting to github-releases strategy", + slog.String("original_path", r.URL.Path), + slog.String("redirect_path", newPath)) + + // Use handler pattern with NoOpCache (no caching for the redirect) + // The github-releases strategy will handle caching + h := handler.New(s.client, cache.NoOpCache()). + Transform(func(r *http.Request) (*http.Request, error) { + // Create internal request to github-releases strategy + // Build internal URL from the incoming request + internalURL := &url.URL{ + Scheme: "http", + Host: r.Host, + Path: newPath, + RawQuery: r.URL.RawQuery, + } + + // Use https if original request was TLS + if r.TLS != nil { + internalURL.Scheme = "https" + } + + req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL.String(), nil) + if err != nil { + return nil, errors.Wrap(err, "create internal redirect request") + } + + // Copy headers from original request + req.Header = r.Header.Clone() + + return req, nil + }) + + h.ServeHTTP(w, r) +} + +// handleNonGitHub handles non-GitHub release downloads using the handler pattern. +func (s *Hermit) handleNonGitHub(w http.ResponseWriter, r *http.Request, host, path string) { + h := handler.New(s.client, s.cache). + CacheKey(func(r *http.Request) string { + // Cache key is the original URL with https:// scheme + return buildOriginalURL(host, path, r.URL.RawQuery) + }). + Transform(func(r *http.Request) (*http.Request, error) { + // Build original URL with https:// and create HTTP GET request + originalURL := buildOriginalURL(host, path, r.URL.RawQuery) + + s.logger.DebugContext(r.Context(), "Fetching Hermit package", + slog.String("url", originalURL)) + + return http.NewRequestWithContext(r.Context(), http.MethodGet, originalURL, nil) + }) + + h.ServeHTTP(w, r) +} + +// buildOriginalURL reconstructs the original URL from the host and path. +// Example: host="go.dev", path="dl/go1.21.0.tar.gz" → https://go.dev/dl/go1.21.0.tar.gz +func buildOriginalURL(host, path, query string) string { + // Use url.URL for proper URL construction (handles encoding, edge cases) + u := &url.URL{ + Scheme: "https", + Host: host, + Path: "/" + path, + RawQuery: query, + } + return u.String() +} diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go new file mode 100644 index 0000000..28a1f7d --- /dev/null +++ b/internal/strategy/hermit_test.go @@ -0,0 +1,279 @@ +package strategy_test + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/alecthomas/assert/v2" + + "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/jobscheduler" + "github.com/block/cachew/internal/logging" + "github.com/block/cachew/internal/strategy" +) + +// httpTransportMutexHermit ensures hermit tests don't run in parallel +// since they modify the global http.DefaultTransport +var httpTransportMutexHermit sync.Mutex + +func TestHermitNonGitHubCaching(t *testing.T) { + // Lock to prevent parallel execution since we modify http.DefaultTransport + httpTransportMutexHermit.Lock() + defer httpTransportMutexHermit.Unlock() + + callCount := 0 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + callCount++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("go-binary-content")) + })) + defer backend.Close() + + // Override http.DefaultTransport to redirect to our mock server + originalTransport := http.DefaultTransport + defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign + http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign + + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + defer memCache.Close() + + mux := http.NewServeMux() + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + // First request - cache miss + req1 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) + w1 := httptest.NewRecorder() + mux.ServeHTTP(w1, req1) + + assert.Equal(t, http.StatusOK, w1.Code) + assert.Equal(t, "go-binary-content", w1.Body.String()) + assert.Equal(t, 1, callCount, "first request should fetch from upstream") + + // Second request - cache hit + req2 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) + w2 := httptest.NewRecorder() + mux.ServeHTTP(w2, req2) + + assert.Equal(t, http.StatusOK, w2.Code) + assert.Equal(t, "go-binary-content", w2.Body.String()) + assert.Equal(t, 1, callCount, "second request should be served from cache") +} + +// mockTransport redirects all HTTP requests to the mock backend server +type mockTransport struct { + backend *httptest.Server + originalTransport http.RoundTripper +} + +func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + // Redirect all requests to our mock server + newReq := req.Clone(req.Context()) + newReq.URL.Scheme = "http" + newReq.URL.Host = m.backend.Listener.Addr().String() + newReq.RequestURI = "" + return m.originalTransport.RoundTrip(newReq) +} + +func TestHermitGitHubRelease(t *testing.T) { + // Mock GitHub server + githubCallCount := 0 + githubServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + githubCallCount++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("github-binary-content")) + })) + defer githubServer.Close() + + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + defer memCache.Close() + + mux := http.NewServeMux() + + // Create hermit strategy + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + // Also create github-releases strategy for redirect + _, err = strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + // Test GitHub release URL - should redirect to github-releases strategy + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + // Note: This will fail to fetch from real GitHub, but we're testing the redirect logic + // In a real test, we'd mock the GitHub server + assert.True(t, w.Code == http.StatusOK || w.Code == http.StatusBadGateway || w.Code == http.StatusNotFound, + "should attempt to fetch from GitHub (may fail without mock)") +} + +func TestHermitNonOKStatus(t *testing.T) { + // Lock to prevent parallel execution since we modify http.DefaultTransport + httpTransportMutexHermit.Lock() + defer httpTransportMutexHermit.Unlock() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) + })) + defer backend.Close() + + // Override http.DefaultTransport to redirect to our mock server + originalTransport := http.DefaultTransport + defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign + http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign + + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + defer memCache.Close() + + mux := http.NewServeMux() + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/example.com/missing.tar.gz", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.Equal(t, "not found", w.Body.String()) + + // Verify non-OK responses are not cached + key := cache.NewKey("https://example.com/missing.tar.gz") + _, _, err = memCache.Open(context.Background(), key) + assert.Error(t, err, "non-OK responses should not be cached") +} + +func TestHermitDifferentSources(t *testing.T) { + tests := []struct { + name string + path string + wantHost string + }{ + { + name: "golang.org", + path: "/hermit/golang.org/dl/go1.21.0.tar.gz", + wantHost: "golang.org", + }, + { + name: "npm registry", + path: "/hermit/registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + wantHost: "registry.npmjs.org", + }, + { + name: "HashiCorp", + path: "/hermit/releases.hashicorp.com/terraform/1.5.0/terraform_1.5.0_linux_amd64.zip", + wantHost: "releases.hashicorp.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Lock to prevent parallel execution since we modify http.DefaultTransport + httpTransportMutexHermit.Lock() + defer httpTransportMutexHermit.Unlock() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + // Verify the upstream request is correct + // Note: r.Host will be the mock server host, not the original host + // This is expected behavior with mockTransport + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("content")) + })) + defer backend.Close() + + // Override http.DefaultTransport to redirect to our mock server + originalTransport := http.DefaultTransport + defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign + http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign + + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + defer memCache.Close() + + mux := http.NewServeMux() + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + // Should successfully fetch from mock server + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "content", w.Body.String()) + }) + } +} + +func TestHermitString(t *testing.T) { + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + defer memCache.Close() + + mux := http.NewServeMux() + hermit, err := strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + assert.Equal(t, "hermit", hermit.String()) +} + +func TestHermitCacheKeyGeneration(t *testing.T) { + tests := []struct { + name string + path string + wantKey string + }{ + { + name: "golang.org", + path: "/hermit/golang.org/dl/go1.21.0.tar.gz", + wantKey: "https://golang.org/dl/go1.21.0.tar.gz", + }, + { + name: "npm registry with scope", + path: "/hermit/registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + wantKey: "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + }, + { + name: "GitHub release", + path: "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", + wantKey: "https://github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // We can't easily test the cache key directly without exposing internals, + // but we can verify the URL pattern is correct by checking the request + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + defer memCache.Close() + + mux := http.NewServeMux() + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + // The cache key should be the wantKey (we verify this indirectly through caching behavior) + // A more thorough test would mock the cache and verify the key + }) + } +} From ef2e7880af2a0b33ee921453de801048be3265ab Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 09:08:09 +1100 Subject: [PATCH 02/10] chore: Remove unncessary comments --- internal/strategy/hermit.go | 39 +++++--------------------------- internal/strategy/hermit_test.go | 29 +----------------------- 2 files changed, 7 insertions(+), 61 deletions(-) diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index 0ee7df1..f26b95f 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -19,15 +19,11 @@ func init() { Register("hermit", "Caches Hermit package downloads.", NewHermit) } -// HermitConfig for the Hermit strategy. -type HermitConfig struct { - // Future configuration can be added here -} +type HermitConfig struct{} -// Hermit implements caching for Hermit package downloads. -// This strategy acts as a router that: -// 1. Detects GitHub release URLs and redirects to github-releases strategy -// 2. Handles all other URLs directly with simple HTTP GET. +// Hermit caches Hermit package downloads. +// Acts as a smart router: GitHub releases redirect to github-releases strategy, +// all other sources are handled directly. type Hermit struct { cache cache.Cache client *http.Client @@ -37,7 +33,6 @@ type Hermit struct { var _ Strategy = (*Hermit)(nil) -// NewHermit creates a new Hermit caching strategy. func NewHermit(ctx context.Context, _ HermitConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*Hermit, error) { logger := logging.FromContext(ctx) @@ -48,9 +43,6 @@ func NewHermit(ctx context.Context, _ HermitConfig, _ jobscheduler.Scheduler, ca mux: mux, } - // Mount at /hermit/{host}/{path...} - // Example: /hermit/go.dev/dl/go1.21.0.linux-amd64.tar.gz - // Example: /hermit/github.com/squareup/repo/releases/download/v1.0/file.tar.gz mux.Handle("GET /hermit/{host}/{path...}", http.HandlerFunc(s.handleRequest)) logger.InfoContext(ctx, "Hermit strategy initialized") @@ -60,27 +52,20 @@ func NewHermit(ctx context.Context, _ HermitConfig, _ jobscheduler.Scheduler, ca func (s *Hermit) String() string { return "hermit" } -// handleRequest routes the request to the appropriate handler. -// GitHub releases are redirected to the github-releases strategy. -// Everything else is handled directly. func (s *Hermit) handleRequest(w http.ResponseWriter, r *http.Request) { host := r.PathValue("host") path := r.PathValue("path") - // Check if this is a GitHub release URL if host == "github.com" && strings.Contains(path, "/releases/download/") { - // Redirect to github-releases strategy using handler pattern with no-cache s.redirectToGitHubReleases(w, r, path) return } - // Not a GitHub release, handle directly with handler pattern s.handleNonGitHub(w, r, host, path) } -// redirectToGitHubReleases redirects to the github-releases strategy without caching. -// Uses handler pattern with NoOpCache to avoid double caching. -// The github-releases strategy will handle caching. +// redirectToGitHubReleases delegates to github-releases strategy using NoOpCache +// to avoid double caching (github-releases will cache the actual response). func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request, path string) { newPath := "/github.com/" + path @@ -88,12 +73,8 @@ func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request slog.String("original_path", r.URL.Path), slog.String("redirect_path", newPath)) - // Use handler pattern with NoOpCache (no caching for the redirect) - // The github-releases strategy will handle caching h := handler.New(s.client, cache.NoOpCache()). Transform(func(r *http.Request) (*http.Request, error) { - // Create internal request to github-releases strategy - // Build internal URL from the incoming request internalURL := &url.URL{ Scheme: "http", Host: r.Host, @@ -101,7 +82,6 @@ func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request RawQuery: r.URL.RawQuery, } - // Use https if original request was TLS if r.TLS != nil { internalURL.Scheme = "https" } @@ -111,7 +91,6 @@ func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request return nil, errors.Wrap(err, "create internal redirect request") } - // Copy headers from original request req.Header = r.Header.Clone() return req, nil @@ -120,15 +99,12 @@ func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request h.ServeHTTP(w, r) } -// handleNonGitHub handles non-GitHub release downloads using the handler pattern. func (s *Hermit) handleNonGitHub(w http.ResponseWriter, r *http.Request, host, path string) { h := handler.New(s.client, s.cache). CacheKey(func(r *http.Request) string { - // Cache key is the original URL with https:// scheme return buildOriginalURL(host, path, r.URL.RawQuery) }). Transform(func(r *http.Request) (*http.Request, error) { - // Build original URL with https:// and create HTTP GET request originalURL := buildOriginalURL(host, path, r.URL.RawQuery) s.logger.DebugContext(r.Context(), "Fetching Hermit package", @@ -140,10 +116,7 @@ func (s *Hermit) handleNonGitHub(w http.ResponseWriter, r *http.Request, host, p h.ServeHTTP(w, r) } -// buildOriginalURL reconstructs the original URL from the host and path. -// Example: host="go.dev", path="dl/go1.21.0.tar.gz" → https://go.dev/dl/go1.21.0.tar.gz func buildOriginalURL(host, path, query string) string { - // Use url.URL for proper URL construction (handles encoding, edge cases) u := &url.URL{ Scheme: "https", Host: host, diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index 28a1f7d..382719a 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -22,7 +22,6 @@ import ( var httpTransportMutexHermit sync.Mutex func TestHermitNonGitHubCaching(t *testing.T) { - // Lock to prevent parallel execution since we modify http.DefaultTransport httpTransportMutexHermit.Lock() defer httpTransportMutexHermit.Unlock() @@ -34,7 +33,6 @@ func TestHermitNonGitHubCaching(t *testing.T) { })) defer backend.Close() - // Override http.DefaultTransport to redirect to our mock server originalTransport := http.DefaultTransport defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign @@ -48,16 +46,14 @@ func TestHermitNonGitHubCaching(t *testing.T) { _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) assert.NoError(t, err) - // First request - cache miss req1 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) w1 := httptest.NewRecorder() mux.ServeHTTP(w1, req1) assert.Equal(t, http.StatusOK, w1.Code) assert.Equal(t, "go-binary-content", w1.Body.String()) - assert.Equal(t, 1, callCount, "first request should fetch from upstream") + assert.Equal(t, 1, callCount) - // Second request - cache hit req2 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) w2 := httptest.NewRecorder() mux.ServeHTTP(w2, req2) @@ -67,14 +63,12 @@ func TestHermitNonGitHubCaching(t *testing.T) { assert.Equal(t, 1, callCount, "second request should be served from cache") } -// mockTransport redirects all HTTP requests to the mock backend server type mockTransport struct { backend *httptest.Server originalTransport http.RoundTripper } func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { - // Redirect all requests to our mock server newReq := req.Clone(req.Context()) newReq.URL.Scheme = "http" newReq.URL.Host = m.backend.Listener.Addr().String() @@ -83,7 +77,6 @@ func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { } func TestHermitGitHubRelease(t *testing.T) { - // Mock GitHub server githubCallCount := 0 githubServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { githubCallCount++ @@ -98,28 +91,21 @@ func TestHermitGitHubRelease(t *testing.T) { defer memCache.Close() mux := http.NewServeMux() - - // Create hermit strategy _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) assert.NoError(t, err) - // Also create github-releases strategy for redirect _, err = strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) assert.NoError(t, err) - // Test GitHub release URL - should redirect to github-releases strategy req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", nil) w := httptest.NewRecorder() mux.ServeHTTP(w, req) - // Note: This will fail to fetch from real GitHub, but we're testing the redirect logic - // In a real test, we'd mock the GitHub server assert.True(t, w.Code == http.StatusOK || w.Code == http.StatusBadGateway || w.Code == http.StatusNotFound, "should attempt to fetch from GitHub (may fail without mock)") } func TestHermitNonOKStatus(t *testing.T) { - // Lock to prevent parallel execution since we modify http.DefaultTransport httpTransportMutexHermit.Lock() defer httpTransportMutexHermit.Unlock() @@ -129,7 +115,6 @@ func TestHermitNonOKStatus(t *testing.T) { })) defer backend.Close() - // Override http.DefaultTransport to redirect to our mock server originalTransport := http.DefaultTransport defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign @@ -150,7 +135,6 @@ func TestHermitNonOKStatus(t *testing.T) { assert.Equal(t, http.StatusNotFound, w.Code) assert.Equal(t, "not found", w.Body.String()) - // Verify non-OK responses are not cached key := cache.NewKey("https://example.com/missing.tar.gz") _, _, err = memCache.Open(context.Background(), key) assert.Error(t, err, "non-OK responses should not be cached") @@ -181,20 +165,15 @@ func TestHermitDifferentSources(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // Lock to prevent parallel execution since we modify http.DefaultTransport httpTransportMutexHermit.Lock() defer httpTransportMutexHermit.Unlock() backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { - // Verify the upstream request is correct - // Note: r.Host will be the mock server host, not the original host - // This is expected behavior with mockTransport w.WriteHeader(http.StatusOK) _, _ = w.Write([]byte("content")) })) defer backend.Close() - // Override http.DefaultTransport to redirect to our mock server originalTransport := http.DefaultTransport defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign @@ -212,7 +191,6 @@ func TestHermitDifferentSources(t *testing.T) { w := httptest.NewRecorder() mux.ServeHTTP(w, req) - // Should successfully fetch from mock server assert.Equal(t, http.StatusOK, w.Code) assert.Equal(t, "content", w.Body.String()) }) @@ -257,8 +235,6 @@ func TestHermitCacheKeyGeneration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - // We can't easily test the cache key directly without exposing internals, - // but we can verify the URL pattern is correct by checking the request _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) assert.NoError(t, err) @@ -271,9 +247,6 @@ func TestHermitCacheKeyGeneration(t *testing.T) { req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) w := httptest.NewRecorder() mux.ServeHTTP(w, req) - - // The cache key should be the wantKey (we verify this indirectly through caching behavior) - // A more thorough test would mock the cache and verify the key }) } } From b8e31bc4e549d0dc083dcab3f14f014f855649af Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 10:23:43 +1100 Subject: [PATCH 03/10] refactor: explicity set redirect url for hermit --- cachew.hcl | 4 ++ internal/strategy/hermit.go | 25 ++++++------ internal/strategy/hermit_test.go | 66 +++++++++++--------------------- 3 files changed, 39 insertions(+), 56 deletions(-) diff --git a/cachew.hcl b/cachew.hcl index 369bc13..112cacd 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -33,3 +33,7 @@ github-releases { gomod { proxy = "https://proxy.golang.org" } + +hermit { + base-url = "${CACHEW_URL}" +} \ No newline at end of file diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index f26b95f..0d79b1e 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -19,12 +19,15 @@ func init() { Register("hermit", "Caches Hermit package downloads.", NewHermit) } -type HermitConfig struct{} +type HermitConfig struct { + BaseURL string `hcl:"base-url" help:"Base URL for internal redirects to github-releases strategy (e.g., http://localhost:8080)."` +} // Hermit caches Hermit package downloads. // Acts as a smart router: GitHub releases redirect to github-releases strategy, // all other sources are handled directly. type Hermit struct { + config HermitConfig cache cache.Cache client *http.Client logger *slog.Logger @@ -33,10 +36,11 @@ type Hermit struct { var _ Strategy = (*Hermit)(nil) -func NewHermit(ctx context.Context, _ HermitConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*Hermit, error) { +func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*Hermit, error) { logger := logging.FromContext(ctx) s := &Hermit{ + config: config, cache: cache, client: http.DefaultClient, logger: logger, @@ -45,7 +49,8 @@ func NewHermit(ctx context.Context, _ HermitConfig, _ jobscheduler.Scheduler, ca mux.Handle("GET /hermit/{host}/{path...}", http.HandlerFunc(s.handleRequest)) - logger.InfoContext(ctx, "Hermit strategy initialized") + logger.InfoContext(ctx, "Hermit strategy initialized", + slog.String("base_url", config.BaseURL)) return s, nil } @@ -75,18 +80,12 @@ func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request h := handler.New(s.client, cache.NoOpCache()). Transform(func(r *http.Request) (*http.Request, error) { - internalURL := &url.URL{ - Scheme: "http", - Host: r.Host, - Path: newPath, - RawQuery: r.URL.RawQuery, - } - - if r.TLS != nil { - internalURL.Scheme = "https" + internalURL := s.config.BaseURL + newPath + if r.URL.RawQuery != "" { + internalURL += "?" + r.URL.RawQuery } - req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL.String(), nil) + req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL, nil) if err != nil { return nil, errors.Wrap(err, "create internal redirect request") } diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index 382719a..50b73a5 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -21,6 +21,21 @@ import ( // since they modify the global http.DefaultTransport var httpTransportMutexHermit sync.Mutex +func setupHermitTest(t *testing.T) (*http.ServeMux, context.Context, cache.Cache) { + t.Helper() + + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + t.Cleanup(func() { memCache.Close() }) + + mux := http.NewServeMux() + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{BaseURL: "http://localhost:8080"}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + return mux, ctx, memCache +} + func TestHermitNonGitHubCaching(t *testing.T) { httpTransportMutexHermit.Lock() defer httpTransportMutexHermit.Unlock() @@ -37,14 +52,7 @@ func TestHermitNonGitHubCaching(t *testing.T) { defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign - _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) - memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) - assert.NoError(t, err) - defer memCache.Close() - - mux := http.NewServeMux() - _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) - assert.NoError(t, err) + mux, ctx, _ := setupHermitTest(t) req1 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) w1 := httptest.NewRecorder() @@ -85,16 +93,9 @@ func TestHermitGitHubRelease(t *testing.T) { })) defer githubServer.Close() - _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) - memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) - assert.NoError(t, err) - defer memCache.Close() - - mux := http.NewServeMux() - _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) - assert.NoError(t, err) + mux, ctx, memCache := setupHermitTest(t) - _, err = strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + _, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) assert.NoError(t, err) req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", nil) @@ -119,14 +120,7 @@ func TestHermitNonOKStatus(t *testing.T) { defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign - _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) - memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) - assert.NoError(t, err) - defer memCache.Close() - - mux := http.NewServeMux() - _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) - assert.NoError(t, err) + mux, ctx, memCache := setupHermitTest(t) req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/example.com/missing.tar.gz", nil) w := httptest.NewRecorder() @@ -136,7 +130,7 @@ func TestHermitNonOKStatus(t *testing.T) { assert.Equal(t, "not found", w.Body.String()) key := cache.NewKey("https://example.com/missing.tar.gz") - _, _, err = memCache.Open(context.Background(), key) + _, _, err := memCache.Open(context.Background(), key) assert.Error(t, err, "non-OK responses should not be cached") } @@ -178,14 +172,7 @@ func TestHermitDifferentSources(t *testing.T) { defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign - _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) - memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) - assert.NoError(t, err) - defer memCache.Close() - - mux := http.NewServeMux() - _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) - assert.NoError(t, err) + mux, ctx, _ := setupHermitTest(t) req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) w := httptest.NewRecorder() @@ -204,7 +191,7 @@ func TestHermitString(t *testing.T) { defer memCache.Close() mux := http.NewServeMux() - hermit, err := strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + hermit, err := strategy.NewHermit(ctx, strategy.HermitConfig{BaseURL: "http://localhost:8080"}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) assert.NoError(t, err) assert.Equal(t, "hermit", hermit.String()) @@ -235,14 +222,7 @@ func TestHermitCacheKeyGeneration(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) - memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) - assert.NoError(t, err) - defer memCache.Close() - - mux := http.NewServeMux() - _, err = strategy.NewHermit(ctx, strategy.HermitConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) - assert.NoError(t, err) + mux, ctx, _ := setupHermitTest(t) req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) w := httptest.NewRecorder() From d5f0960505a5b5d79da324b7dcaffdf4f0af6566 Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 10:26:29 +1100 Subject: [PATCH 04/10] fix: prevent double slashes --- internal/strategy/hermit.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index 0d79b1e..d24d08e 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -116,10 +116,13 @@ func (s *Hermit) handleNonGitHub(w http.ResponseWriter, r *http.Request, host, p } func buildOriginalURL(host, path, query string) string { + if !strings.HasPrefix(path, "/") { + path = "/" + path + } u := &url.URL{ Scheme: "https", Host: host, - Path: "/" + path, + Path: path, RawQuery: query, } return u.String() From a314d74dfbfeb2c79c28c4f5798f497d524e6c27 Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 10:35:45 +1100 Subject: [PATCH 05/10] chore: add hermit to cachew-local.hcl --- cachew-local.hcl | 6 +++++- cachew.hcl | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/cachew-local.hcl b/cachew-local.hcl index 57595f3..7df0d40 100644 --- a/cachew-local.hcl +++ b/cachew-local.hcl @@ -29,4 +29,8 @@ disk { gomod { proxy = "https://proxy.golang.org" -} \ No newline at end of file +} + +hermit { + base-url = "${CACHEW_URL}" +} diff --git a/cachew.hcl b/cachew.hcl index 112cacd..abed293 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -36,4 +36,4 @@ gomod { hermit { base-url = "${CACHEW_URL}" -} \ No newline at end of file +} From f087b77571a895db54dfaf370b41b3ee436494d7 Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 15:52:34 +1100 Subject: [PATCH 06/10] refactor: set base_url to CACHEW_URL as a default --- cachew-local.hcl | 4 +--- cachew.hcl | 4 +--- internal/config/config.go | 2 +- internal/strategy/api.go | 33 ++++++++++++++++++++++++++++++--- internal/strategy/hermit.go | 9 ++++++++- 5 files changed, 41 insertions(+), 11 deletions(-) diff --git a/cachew-local.hcl b/cachew-local.hcl index 7df0d40..35bd5b9 100644 --- a/cachew-local.hcl +++ b/cachew-local.hcl @@ -31,6 +31,4 @@ gomod { proxy = "https://proxy.golang.org" } -hermit { - base-url = "${CACHEW_URL}" -} +hermit { } diff --git a/cachew.hcl b/cachew.hcl index abed293..6edb0d7 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -34,6 +34,4 @@ gomod { proxy = "https://proxy.golang.org" } -hermit { - base-url = "${CACHEW_URL}" -} +hermit { } diff --git a/internal/config/config.go b/internal/config/config.go index 933f2b4..22fa650 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -88,7 +88,7 @@ func Load(ctx context.Context, cr *cache.Registry, r io.Reader, scheduler jobsch for _, block := range strategyCandidates { logger := logger.With("strategy", block.Name) mlog := &loggingMux{logger: logger, mux: mux} - _, err := strategy.Create(ctx, block.Name, block, scheduler.WithQueuePrefix(block.Name), cache, mlog) + _, err := strategy.Create(ctx, block.Name, block, scheduler.WithQueuePrefix(block.Name), cache, mlog, vars) if err != nil { return errors.Errorf("%s: %w", block.Pos, err) } diff --git a/internal/strategy/api.go b/internal/strategy/api.go index c802a83..9b140de 100644 --- a/internal/strategy/api.go +++ b/internal/strategy/api.go @@ -4,6 +4,8 @@ package strategy import ( "context" "net/http" + "os" + "reflect" "github.com/alecthomas/errors" "github.com/alecthomas/hcl/v2" @@ -22,7 +24,7 @@ type Mux interface { type registryEntry struct { schema *hcl.Block - factory func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux) (Strategy, error) + factory func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, vars map[string]string) (Strategy, error) } var registry = map[string]registryEntry{} @@ -40,11 +42,13 @@ func Register[Config any, S Strategy](id, description string, factory Factory[Co block.Comments = hcl.CommentList{description} registry[id] = registryEntry{ schema: block, - factory: func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux) (Strategy, error) { + factory: func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, vars map[string]string) (Strategy, error) { var cfg Config if err := hcl.UnmarshalBlock(config, &cfg, hcl.AllowExtra(false)); err != nil { return nil, errors.WithStack(err) } + // Expand environment variables in string fields + expandStructStrings(&cfg, vars) return factory(ctx, cfg, scheduler, cache, mux) }, } @@ -69,9 +73,10 @@ func Create( scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, + vars map[string]string, ) (Strategy, error) { if entry, ok := registry[name]; ok { - return errors.WithStack2(entry.factory(ctx, config, scheduler.WithQueuePrefix(name), cache, mux)) + return errors.WithStack2(entry.factory(ctx, config, scheduler.WithQueuePrefix(name), cache, mux, vars)) } return nil, errors.Errorf("%s: %w", name, ErrNotFound) } @@ -79,3 +84,25 @@ func Create( type Strategy interface { String() string } + +// expandStructStrings expands environment variables in string fields of a struct. +func expandStructStrings(v any, vars map[string]string) { + rv := reflect.ValueOf(v) + if rv.Kind() != reflect.Ptr || rv.Elem().Kind() != reflect.Struct { + return + } + + rv = rv.Elem() + for i := range rv.NumField() { + field := rv.Field(i) + if field.Kind() == reflect.String && field.CanSet() { + expanded := os.Expand(field.String(), func(key string) string { + if val, ok := vars[key]; ok { + return val + } + return os.Getenv(key) + }) + field.SetString(expanded) + } + } +} diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index d24d08e..562e729 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -20,7 +20,7 @@ func init() { } type HermitConfig struct { - BaseURL string `hcl:"base-url" help:"Base URL for internal redirects to github-releases strategy (e.g., http://localhost:8080)."` + BaseURL string `hcl:"base-url" help:"Base URL for internal redirects to github-releases strategy" default:"${CACHEW_URL}"` } // Hermit caches Hermit package downloads. @@ -39,6 +39,9 @@ var _ Strategy = (*Hermit)(nil) func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*Hermit, error) { logger := logging.FromContext(ctx) + logger.DebugContext(ctx, "Hermit strategy config received", + slog.String("base_url_raw", config.BaseURL)) + s := &Hermit{ config: config, cache: cache, @@ -85,6 +88,10 @@ func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request internalURL += "?" + r.URL.RawQuery } + s.logger.DebugContext(r.Context(), "Creating internal redirect request", + slog.String("base_url", s.config.BaseURL), + slog.String("internal_url", internalURL)) + req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL, nil) if err != nil { return nil, errors.Wrap(err, "create internal redirect request") From 4108752dd00dccd9a9afd6a7c99466594ecfcf10 Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 16:03:56 +1100 Subject: [PATCH 07/10] refactor: only create handlers once instead of every request --- internal/strategy/hermit.go | 120 ++++++++++++++++++------------------ 1 file changed, 59 insertions(+), 61 deletions(-) diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index 562e729..b70e43c 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -27,16 +27,18 @@ type HermitConfig struct { // Acts as a smart router: GitHub releases redirect to github-releases strategy, // all other sources are handled directly. type Hermit struct { - config HermitConfig - cache cache.Cache - client *http.Client - logger *slog.Logger - mux Mux + config HermitConfig + cache cache.Cache + client *http.Client + logger *slog.Logger + mux Mux + redirectHandler http.Handler + directHandler http.Handler } var _ Strategy = (*Hermit)(nil) -func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Scheduler, cache cache.Cache, mux Mux) (*Hermit, error) { +func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Scheduler, c cache.Cache, mux Mux) (*Hermit, error) { logger := logging.FromContext(ctx) logger.DebugContext(ctx, "Hermit strategy config received", @@ -44,93 +46,89 @@ func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Schedule s := &Hermit{ config: config, - cache: cache, + cache: c, client: http.DefaultClient, logger: logger, mux: mux, } - mux.Handle("GET /hermit/{host}/{path...}", http.HandlerFunc(s.handleRequest)) + s.directHandler = s.createDirectHandler(c) + mux.Handle("GET /hermit/{host}/{path...}", s.directHandler) - logger.InfoContext(ctx, "Hermit strategy initialized", - slog.String("base_url", config.BaseURL)) + if config.BaseURL != "" { + s.redirectHandler = s.createRedirectHandler() + mux.Handle("GET /hermit/github.com/{path...}", s.redirectHandler) + logger.InfoContext(ctx, "Hermit strategy initialized", slog.String("base_url", config.BaseURL)) + } else { + logger.WarnContext(ctx, "Hermit strategy initialized without base-url - GitHub releases will fail") + } return s, nil } func (s *Hermit) String() string { return "hermit" } -func (s *Hermit) handleRequest(w http.ResponseWriter, r *http.Request) { - host := r.PathValue("host") - path := r.PathValue("path") - - if host == "github.com" && strings.Contains(path, "/releases/download/") { - s.redirectToGitHubReleases(w, r, path) - return - } - - s.handleNonGitHub(w, r, host, path) +func (s *Hermit) createDirectHandler(c cache.Cache) http.Handler { + return handler.New(s.client, c). + CacheKey(func(r *http.Request) string { + return s.buildOriginalURL(r) + }). + Transform(func(r *http.Request) (*http.Request, error) { + return s.buildDirectRequest(r) + }) } -// redirectToGitHubReleases delegates to github-releases strategy using NoOpCache -// to avoid double caching (github-releases will cache the actual response). -func (s *Hermit) redirectToGitHubReleases(w http.ResponseWriter, r *http.Request, path string) { - newPath := "/github.com/" + path - - s.logger.DebugContext(r.Context(), "Redirecting to github-releases strategy", - slog.String("original_path", r.URL.Path), - slog.String("redirect_path", newPath)) - - h := handler.New(s.client, cache.NoOpCache()). +func (s *Hermit) createRedirectHandler() http.Handler { + return handler.New(s.client, cache.NoOpCache()). Transform(func(r *http.Request) (*http.Request, error) { - internalURL := s.config.BaseURL + newPath - if r.URL.RawQuery != "" { - internalURL += "?" + r.URL.RawQuery - } - - s.logger.DebugContext(r.Context(), "Creating internal redirect request", - slog.String("base_url", s.config.BaseURL), - slog.String("internal_url", internalURL)) + return s.buildRedirectRequest(r) + }) +} - req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL, nil) - if err != nil { - return nil, errors.Wrap(err, "create internal redirect request") - } +func (s *Hermit) buildRedirectRequest(r *http.Request) (*http.Request, error) { + path := r.PathValue("path") + newPath := "/github.com/" + path - req.Header = r.Header.Clone() + internalURL := s.config.BaseURL + newPath + if r.URL.RawQuery != "" { + internalURL += "?" + r.URL.RawQuery + } - return req, nil - }) + req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL, nil) + if err != nil { + return nil, errors.Wrap(err, "create internal redirect request") + } - h.ServeHTTP(w, r) + req.Header = r.Header.Clone() + return req, nil } -func (s *Hermit) handleNonGitHub(w http.ResponseWriter, r *http.Request, host, path string) { - h := handler.New(s.client, s.cache). - CacheKey(func(r *http.Request) string { - return buildOriginalURL(host, path, r.URL.RawQuery) - }). - Transform(func(r *http.Request) (*http.Request, error) { - originalURL := buildOriginalURL(host, path, r.URL.RawQuery) +func (s *Hermit) buildDirectRequest(r *http.Request) (*http.Request, error) { + originalURL := s.buildOriginalURL(r) - s.logger.DebugContext(r.Context(), "Fetching Hermit package", - slog.String("url", originalURL)) + s.logger.DebugContext(r.Context(), "Fetching Hermit package", + slog.String("url", originalURL)) - return http.NewRequestWithContext(r.Context(), http.MethodGet, originalURL, nil) - }) - - h.ServeHTTP(w, r) + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, originalURL, nil) + if err != nil { + return nil, errors.Wrap(err, "create request") + } + return req, nil } -func buildOriginalURL(host, path, query string) string { +func (s *Hermit) buildOriginalURL(r *http.Request) string { + host := r.PathValue("host") + path := r.PathValue("path") + if !strings.HasPrefix(path, "/") { path = "/" + path } + u := &url.URL{ Scheme: "https", Host: host, Path: path, - RawQuery: query, + RawQuery: r.URL.RawQuery, } return u.String() } From 57f71b1c0d256c20fbb4d473135b82cf3e450b7b Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 16:05:48 +1100 Subject: [PATCH 08/10] test: remove unnessary test --- internal/strategy/hermit_test.go | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index 50b73a5..855afef 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -184,19 +184,6 @@ func TestHermitDifferentSources(t *testing.T) { } } -func TestHermitString(t *testing.T) { - _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) - memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) - assert.NoError(t, err) - defer memCache.Close() - - mux := http.NewServeMux() - hermit, err := strategy.NewHermit(ctx, strategy.HermitConfig{BaseURL: "http://localhost:8080"}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) - assert.NoError(t, err) - - assert.Equal(t, "hermit", hermit.String()) -} - func TestHermitCacheKeyGeneration(t *testing.T) { tests := []struct { name string From 3a4408407b0a035368929e5c6d8bd3f4da182750 Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Wed, 4 Feb 2026 18:35:11 +1100 Subject: [PATCH 09/10] chore: Configurable redirect target for GitHub releases - Defaults to ${CACHEW_URL}/github.com (uses github-releases strategy) - Can override to https://github.com (direct to real GitHub) --- internal/strategy/hermit.go | 68 ++++++++++++++++++++------------ internal/strategy/hermit_test.go | 2 +- 2 files changed, 44 insertions(+), 26 deletions(-) diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index b70e43c..f68b301 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -5,6 +5,7 @@ import ( "log/slog" "net/http" "net/url" + "os" "strings" "github.com/alecthomas/errors" @@ -20,7 +21,7 @@ func init() { } type HermitConfig struct { - BaseURL string `hcl:"base-url" help:"Base URL for internal redirects to github-releases strategy" default:"${CACHEW_URL}"` + GitHubBaseURL string `hcl:"github-base-url,optional" help:"Base URL for GitHub release redirects" default:"${CACHEW_URL}/github.com"` } // Hermit caches Hermit package downloads. @@ -41,9 +42,6 @@ var _ Strategy = (*Hermit)(nil) func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Scheduler, c cache.Cache, mux Mux) (*Hermit, error) { logger := logging.FromContext(ctx) - logger.DebugContext(ctx, "Hermit strategy config received", - slog.String("base_url_raw", config.BaseURL)) - s := &Hermit{ config: config, cache: c, @@ -55,12 +53,15 @@ func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Schedule s.directHandler = s.createDirectHandler(c) mux.Handle("GET /hermit/{host}/{path...}", s.directHandler) - if config.BaseURL != "" { - s.redirectHandler = s.createRedirectHandler() + if config.GitHubBaseURL != "" { + isInternalRedirect := strings.Contains(config.GitHubBaseURL, os.Getenv("CACHEW_URL")) + s.redirectHandler = s.createRedirectHandler(isInternalRedirect, c) mux.Handle("GET /hermit/github.com/{path...}", s.redirectHandler) - logger.InfoContext(ctx, "Hermit strategy initialized", slog.String("base_url", config.BaseURL)) + logger.InfoContext(ctx, "Hermit strategy initialized", + slog.String("github_base_url", config.GitHubBaseURL), + slog.Bool("internal_redirect", isInternalRedirect)) } else { - logger.WarnContext(ctx, "Hermit strategy initialized without base-url - GitHub releases will fail") + logger.InfoContext(ctx, "Hermit strategy initialized") } return s, nil @@ -78,23 +79,36 @@ func (s *Hermit) createDirectHandler(c cache.Cache) http.Handler { }) } -func (s *Hermit) createRedirectHandler() http.Handler { - return handler.New(s.client, cache.NoOpCache()). +func (s *Hermit) createRedirectHandler(isInternalRedirect bool, c cache.Cache) http.Handler { + var cacheBackend cache.Cache + if isInternalRedirect { + cacheBackend = cache.NoOpCache() + } else { + cacheBackend = c + } + + return handler.New(s.client, cacheBackend). + CacheKey(func(r *http.Request) string { + return s.buildGitHubURL(r) + }). Transform(func(r *http.Request) (*http.Request, error) { + s.logger.DebugContext(r.Context(), "Redirect handler called for GitHub release") return s.buildRedirectRequest(r) }) } -func (s *Hermit) buildRedirectRequest(r *http.Request) (*http.Request, error) { - path := r.PathValue("path") - newPath := "/github.com/" + path +func (s *Hermit) buildGitHubURL(r *http.Request) string { + return buildURL("https", "github.com", r.PathValue("path"), r.URL.RawQuery) +} - internalURL := s.config.BaseURL + newPath +func (s *Hermit) buildRedirectRequest(r *http.Request) (*http.Request, error) { + path := ensureLeadingSlash(r.PathValue("path")) + redirectURL := s.config.GitHubBaseURL + path if r.URL.RawQuery != "" { - internalURL += "?" + r.URL.RawQuery + redirectURL += "?" + r.URL.RawQuery } - req, err := http.NewRequestWithContext(r.Context(), r.Method, internalURL, nil) + req, err := http.NewRequestWithContext(r.Context(), r.Method, redirectURL, nil) if err != nil { return nil, errors.Wrap(err, "create internal redirect request") } @@ -117,18 +131,22 @@ func (s *Hermit) buildDirectRequest(r *http.Request) (*http.Request, error) { } func (s *Hermit) buildOriginalURL(r *http.Request) string { - host := r.PathValue("host") - path := r.PathValue("path") - - if !strings.HasPrefix(path, "/") { - path = "/" + path - } + return buildURL("https", r.PathValue("host"), r.PathValue("path"), r.URL.RawQuery) +} +func buildURL(scheme, host, path, query string) string { u := &url.URL{ - Scheme: "https", + Scheme: scheme, Host: host, - Path: path, - RawQuery: r.URL.RawQuery, + Path: ensureLeadingSlash(path), + RawQuery: query, } return u.String() } + +func ensureLeadingSlash(path string) string { + if !strings.HasPrefix(path, "/") { + return "/" + path + } + return path +} diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go index 855afef..02494cb 100644 --- a/internal/strategy/hermit_test.go +++ b/internal/strategy/hermit_test.go @@ -30,7 +30,7 @@ func setupHermitTest(t *testing.T) (*http.ServeMux, context.Context, cache.Cache t.Cleanup(func() { memCache.Close() }) mux := http.NewServeMux() - _, err = strategy.NewHermit(ctx, strategy.HermitConfig{BaseURL: "http://localhost:8080"}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{GitHubBaseURL: "http://localhost:8080"}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) assert.NoError(t, err) return mux, ctx, memCache From fa8bc7c7b20ea13071f8827960ced0a397fceaf4 Mon Sep 17 00:00:00 2001 From: Sutina Wipawiwat Date: Thu, 5 Feb 2026 10:23:34 +1100 Subject: [PATCH 10/10] chore: Enable var expandsion --- go.mod | 2 +- go.sum | 4 ++-- internal/strategy/api.go | 30 ++++-------------------------- internal/strategy/hermit.go | 2 +- 4 files changed, 8 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index d3bce86..270f666 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/block/cachew go 1.25.5 require ( - github.com/alecthomas/hcl/v2 v2.3.1 + github.com/alecthomas/hcl/v2 v2.4.0 github.com/alecthomas/kong v1.13.0 github.com/goproxy/goproxy v0.25.0 github.com/lmittmann/tint v1.1.2 diff --git a/go.sum b/go.sum index fe9319a..2dd5ffe 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= github.com/alecthomas/errors v0.9.1 h1:JNXtU30rtMNARCkW41OTZ4yL6Lyocq20xIJgIw2raqI= github.com/alecthomas/errors v0.9.1/go.mod h1:l8mjMEHMGUdIWPMNtvDyRYPVS1fQFXHFXc/iVCCLGkI= -github.com/alecthomas/hcl/v2 v2.3.1 h1:Nkj0svGJawz920nQyWUhD2PYmD47p7BB9vc2e3kft1o= -github.com/alecthomas/hcl/v2 v2.3.1/go.mod h1:4UUp66q8ony5j8tm2bANErujUpZ3GgHBLgaKxTUQlQI= +github.com/alecthomas/hcl/v2 v2.4.0 h1:j7sPnff/f6FLAPTZmpFzHS2ENwE/dHj6K40bRb9nk4g= +github.com/alecthomas/hcl/v2 v2.4.0/go.mod h1:4UUp66q8ony5j8tm2bANErujUpZ3GgHBLgaKxTUQlQI= github.com/alecthomas/kong v1.13.0 h1:5e/7XC3ugvhP1DQBmTS+WuHtCbcv44hsohMgcvVxSrA= github.com/alecthomas/kong v1.13.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= diff --git a/internal/strategy/api.go b/internal/strategy/api.go index 9b140de..b7979da 100644 --- a/internal/strategy/api.go +++ b/internal/strategy/api.go @@ -5,7 +5,6 @@ import ( "context" "net/http" "os" - "reflect" "github.com/alecthomas/errors" "github.com/alecthomas/hcl/v2" @@ -44,11 +43,12 @@ func Register[Config any, S Strategy](id, description string, factory Factory[Co schema: block, factory: func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, vars map[string]string) (Strategy, error) { var cfg Config - if err := hcl.UnmarshalBlock(config, &cfg, hcl.AllowExtra(false)); err != nil { + transformer := func(defaultValue string) string { + return os.Expand(defaultValue, func(key string) string { return vars[key] }) + } + if err := hcl.UnmarshalBlock(config, &cfg, hcl.AllowExtra(false), hcl.WithDefaultTransformer(transformer)); err != nil { return nil, errors.WithStack(err) } - // Expand environment variables in string fields - expandStructStrings(&cfg, vars) return factory(ctx, cfg, scheduler, cache, mux) }, } @@ -84,25 +84,3 @@ func Create( type Strategy interface { String() string } - -// expandStructStrings expands environment variables in string fields of a struct. -func expandStructStrings(v any, vars map[string]string) { - rv := reflect.ValueOf(v) - if rv.Kind() != reflect.Ptr || rv.Elem().Kind() != reflect.Struct { - return - } - - rv = rv.Elem() - for i := range rv.NumField() { - field := rv.Field(i) - if field.Kind() == reflect.String && field.CanSet() { - expanded := os.Expand(field.String(), func(key string) string { - if val, ok := vars[key]; ok { - return val - } - return os.Getenv(key) - }) - field.SetString(expanded) - } - } -} diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go index f68b301..ad5517e 100644 --- a/internal/strategy/hermit.go +++ b/internal/strategy/hermit.go @@ -21,7 +21,7 @@ func init() { } type HermitConfig struct { - GitHubBaseURL string `hcl:"github-base-url,optional" help:"Base URL for GitHub release redirects" default:"${CACHEW_URL}/github.com"` + GitHubBaseURL string `hcl:"github-base-url" help:"Base URL for GitHub release redirects" default:"${CACHEW_URL}/github.com"` } // Hermit caches Hermit package downloads.