diff --git a/README.md b/README.md index 7945b70..4668690 100644 --- a/README.md +++ b/README.md @@ -29,4 +29,10 @@ As Git itself isn't aware of the snapshots, Git-specific code in the Cachew CLI ## Hermit -Hermit +Caches Hermit package downloads from all sources (golang.org, npm, GitHub releases, etc.). + +**URL pattern:** `/hermit/{host}/{path...}` + +Example: `GET /hermit/golang.org/dl/go1.21.0.tar.gz` + +GitHub releases are automatically redirected to the `github-releases` strategy. diff --git a/cachew-local.hcl b/cachew-local.hcl index 57595f3..35bd5b9 100644 --- a/cachew-local.hcl +++ b/cachew-local.hcl @@ -29,4 +29,6 @@ disk { gomod { proxy = "https://proxy.golang.org" -} \ No newline at end of file +} + +hermit { } diff --git a/cachew.hcl b/cachew.hcl index 369bc13..6edb0d7 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -33,3 +33,5 @@ github-releases { gomod { proxy = "https://proxy.golang.org" } + +hermit { } diff --git a/go.mod b/go.mod index d3bce86..270f666 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/block/cachew go 1.25.5 require ( - github.com/alecthomas/hcl/v2 v2.3.1 + github.com/alecthomas/hcl/v2 v2.4.0 github.com/alecthomas/kong v1.13.0 github.com/goproxy/goproxy v0.25.0 github.com/lmittmann/tint v1.1.2 diff --git a/go.sum b/go.sum index fe9319a..2dd5ffe 100644 --- a/go.sum +++ b/go.sum @@ -4,8 +4,8 @@ github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o= github.com/alecthomas/errors v0.9.1 h1:JNXtU30rtMNARCkW41OTZ4yL6Lyocq20xIJgIw2raqI= github.com/alecthomas/errors v0.9.1/go.mod h1:l8mjMEHMGUdIWPMNtvDyRYPVS1fQFXHFXc/iVCCLGkI= -github.com/alecthomas/hcl/v2 v2.3.1 h1:Nkj0svGJawz920nQyWUhD2PYmD47p7BB9vc2e3kft1o= -github.com/alecthomas/hcl/v2 v2.3.1/go.mod h1:4UUp66q8ony5j8tm2bANErujUpZ3GgHBLgaKxTUQlQI= +github.com/alecthomas/hcl/v2 v2.4.0 h1:j7sPnff/f6FLAPTZmpFzHS2ENwE/dHj6K40bRb9nk4g= +github.com/alecthomas/hcl/v2 v2.4.0/go.mod h1:4UUp66q8ony5j8tm2bANErujUpZ3GgHBLgaKxTUQlQI= github.com/alecthomas/kong v1.13.0 h1:5e/7XC3ugvhP1DQBmTS+WuHtCbcv44hsohMgcvVxSrA= github.com/alecthomas/kong v1.13.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I= github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U= diff --git a/internal/cache/noop.go b/internal/cache/noop.go new file mode 100644 index 0000000..ad645ab --- /dev/null +++ b/internal/cache/noop.go @@ -0,0 +1,62 @@ +package cache + +import ( + "context" + "io" + "net/http" + "os" + "time" +) + +// noOpCache is a cache implementation that doesn't cache anything. +// It always returns cache misses and discards writes. +// Useful for pass-through handlers that shouldn't cache. +type noOpCache struct{} + +// NoOpCache returns a cache that doesn't cache anything. +// All Open() calls return os.ErrNotExist (cache miss). +// All Create() calls return a writer that discards data. +func NoOpCache() Cache { + return &noOpCache{} +} + +func (n *noOpCache) String() string { return "noop" } + +func (n *noOpCache) Stat(_ context.Context, _ Key) (http.Header, error) { + return nil, os.ErrNotExist +} + +func (n *noOpCache) Open(_ context.Context, _ Key) (io.ReadCloser, http.Header, error) { + return nil, nil, os.ErrNotExist +} + +func (n *noOpCache) Create(_ context.Context, _ Key, _ http.Header, _ time.Duration) (io.WriteCloser, error) { + // Return a discard writer that does nothing + return &noOpWriter{}, nil +} + +func (n *noOpCache) Delete(_ context.Context, _ Key) error { + return nil +} + +func (n *noOpCache) Stats(_ context.Context) (Stats, error) { + return Stats{}, ErrStatsUnavailable +} + +func (n *noOpCache) Close() error { + return nil +} + +// noOpWriter is a writer that discards all data. +type noOpWriter struct{} + +func (n *noOpWriter) Write(p []byte) (int, error) { + return len(p), nil +} + +func (n *noOpWriter) Close() error { + return nil +} + +var _ Cache = (*noOpCache)(nil) +var _ io.WriteCloser = (*noOpWriter)(nil) diff --git a/internal/config/config.go b/internal/config/config.go index 933f2b4..22fa650 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -88,7 +88,7 @@ func Load(ctx context.Context, cr *cache.Registry, r io.Reader, scheduler jobsch for _, block := range strategyCandidates { logger := logger.With("strategy", block.Name) mlog := &loggingMux{logger: logger, mux: mux} - _, err := strategy.Create(ctx, block.Name, block, scheduler.WithQueuePrefix(block.Name), cache, mlog) + _, err := strategy.Create(ctx, block.Name, block, scheduler.WithQueuePrefix(block.Name), cache, mlog, vars) if err != nil { return errors.Errorf("%s: %w", block.Pos, err) } diff --git a/internal/strategy/api.go b/internal/strategy/api.go index c802a83..b7979da 100644 --- a/internal/strategy/api.go +++ b/internal/strategy/api.go @@ -4,6 +4,7 @@ package strategy import ( "context" "net/http" + "os" "github.com/alecthomas/errors" "github.com/alecthomas/hcl/v2" @@ -22,7 +23,7 @@ type Mux interface { type registryEntry struct { schema *hcl.Block - factory func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux) (Strategy, error) + factory func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, vars map[string]string) (Strategy, error) } var registry = map[string]registryEntry{} @@ -40,9 +41,12 @@ func Register[Config any, S Strategy](id, description string, factory Factory[Co block.Comments = hcl.CommentList{description} registry[id] = registryEntry{ schema: block, - factory: func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux) (Strategy, error) { + factory: func(ctx context.Context, config *hcl.Block, scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, vars map[string]string) (Strategy, error) { var cfg Config - if err := hcl.UnmarshalBlock(config, &cfg, hcl.AllowExtra(false)); err != nil { + transformer := func(defaultValue string) string { + return os.Expand(defaultValue, func(key string) string { return vars[key] }) + } + if err := hcl.UnmarshalBlock(config, &cfg, hcl.AllowExtra(false), hcl.WithDefaultTransformer(transformer)); err != nil { return nil, errors.WithStack(err) } return factory(ctx, cfg, scheduler, cache, mux) @@ -69,9 +73,10 @@ func Create( scheduler jobscheduler.Scheduler, cache cache.Cache, mux Mux, + vars map[string]string, ) (Strategy, error) { if entry, ok := registry[name]; ok { - return errors.WithStack2(entry.factory(ctx, config, scheduler.WithQueuePrefix(name), cache, mux)) + return errors.WithStack2(entry.factory(ctx, config, scheduler.WithQueuePrefix(name), cache, mux, vars)) } return nil, errors.Errorf("%s: %w", name, ErrNotFound) } diff --git a/internal/strategy/hermit.go b/internal/strategy/hermit.go new file mode 100644 index 0000000..ad5517e --- /dev/null +++ b/internal/strategy/hermit.go @@ -0,0 +1,152 @@ +package strategy + +import ( + "context" + "log/slog" + "net/http" + "net/url" + "os" + "strings" + + "github.com/alecthomas/errors" + + "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/jobscheduler" + "github.com/block/cachew/internal/logging" + "github.com/block/cachew/internal/strategy/handler" +) + +func init() { + Register("hermit", "Caches Hermit package downloads.", NewHermit) +} + +type HermitConfig struct { + GitHubBaseURL string `hcl:"github-base-url" help:"Base URL for GitHub release redirects" default:"${CACHEW_URL}/github.com"` +} + +// Hermit caches Hermit package downloads. +// Acts as a smart router: GitHub releases redirect to github-releases strategy, +// all other sources are handled directly. +type Hermit struct { + config HermitConfig + cache cache.Cache + client *http.Client + logger *slog.Logger + mux Mux + redirectHandler http.Handler + directHandler http.Handler +} + +var _ Strategy = (*Hermit)(nil) + +func NewHermit(ctx context.Context, config HermitConfig, _ jobscheduler.Scheduler, c cache.Cache, mux Mux) (*Hermit, error) { + logger := logging.FromContext(ctx) + + s := &Hermit{ + config: config, + cache: c, + client: http.DefaultClient, + logger: logger, + mux: mux, + } + + s.directHandler = s.createDirectHandler(c) + mux.Handle("GET /hermit/{host}/{path...}", s.directHandler) + + if config.GitHubBaseURL != "" { + isInternalRedirect := strings.Contains(config.GitHubBaseURL, os.Getenv("CACHEW_URL")) + s.redirectHandler = s.createRedirectHandler(isInternalRedirect, c) + mux.Handle("GET /hermit/github.com/{path...}", s.redirectHandler) + logger.InfoContext(ctx, "Hermit strategy initialized", + slog.String("github_base_url", config.GitHubBaseURL), + slog.Bool("internal_redirect", isInternalRedirect)) + } else { + logger.InfoContext(ctx, "Hermit strategy initialized") + } + + return s, nil +} + +func (s *Hermit) String() string { return "hermit" } + +func (s *Hermit) createDirectHandler(c cache.Cache) http.Handler { + return handler.New(s.client, c). + CacheKey(func(r *http.Request) string { + return s.buildOriginalURL(r) + }). + Transform(func(r *http.Request) (*http.Request, error) { + return s.buildDirectRequest(r) + }) +} + +func (s *Hermit) createRedirectHandler(isInternalRedirect bool, c cache.Cache) http.Handler { + var cacheBackend cache.Cache + if isInternalRedirect { + cacheBackend = cache.NoOpCache() + } else { + cacheBackend = c + } + + return handler.New(s.client, cacheBackend). + CacheKey(func(r *http.Request) string { + return s.buildGitHubURL(r) + }). + Transform(func(r *http.Request) (*http.Request, error) { + s.logger.DebugContext(r.Context(), "Redirect handler called for GitHub release") + return s.buildRedirectRequest(r) + }) +} + +func (s *Hermit) buildGitHubURL(r *http.Request) string { + return buildURL("https", "github.com", r.PathValue("path"), r.URL.RawQuery) +} + +func (s *Hermit) buildRedirectRequest(r *http.Request) (*http.Request, error) { + path := ensureLeadingSlash(r.PathValue("path")) + redirectURL := s.config.GitHubBaseURL + path + if r.URL.RawQuery != "" { + redirectURL += "?" + r.URL.RawQuery + } + + req, err := http.NewRequestWithContext(r.Context(), r.Method, redirectURL, nil) + if err != nil { + return nil, errors.Wrap(err, "create internal redirect request") + } + + req.Header = r.Header.Clone() + return req, nil +} + +func (s *Hermit) buildDirectRequest(r *http.Request) (*http.Request, error) { + originalURL := s.buildOriginalURL(r) + + s.logger.DebugContext(r.Context(), "Fetching Hermit package", + slog.String("url", originalURL)) + + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, originalURL, nil) + if err != nil { + return nil, errors.Wrap(err, "create request") + } + return req, nil +} + +func (s *Hermit) buildOriginalURL(r *http.Request) string { + return buildURL("https", r.PathValue("host"), r.PathValue("path"), r.URL.RawQuery) +} + +func buildURL(scheme, host, path, query string) string { + u := &url.URL{ + Scheme: scheme, + Host: host, + Path: ensureLeadingSlash(path), + RawQuery: query, + } + return u.String() +} + +func ensureLeadingSlash(path string) string { + if !strings.HasPrefix(path, "/") { + return "/" + path + } + return path +} diff --git a/internal/strategy/hermit_test.go b/internal/strategy/hermit_test.go new file mode 100644 index 0000000..02494cb --- /dev/null +++ b/internal/strategy/hermit_test.go @@ -0,0 +1,219 @@ +package strategy_test + +import ( + "context" + "log/slog" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/alecthomas/assert/v2" + + "github.com/block/cachew/internal/cache" + "github.com/block/cachew/internal/jobscheduler" + "github.com/block/cachew/internal/logging" + "github.com/block/cachew/internal/strategy" +) + +// httpTransportMutexHermit ensures hermit tests don't run in parallel +// since they modify the global http.DefaultTransport +var httpTransportMutexHermit sync.Mutex + +func setupHermitTest(t *testing.T) (*http.ServeMux, context.Context, cache.Cache) { + t.Helper() + + _, ctx := logging.Configure(context.Background(), logging.Config{Level: slog.LevelError}) + memCache, err := cache.NewMemory(ctx, cache.MemoryConfig{MaxTTL: time.Hour}) + assert.NoError(t, err) + t.Cleanup(func() { memCache.Close() }) + + mux := http.NewServeMux() + _, err = strategy.NewHermit(ctx, strategy.HermitConfig{GitHubBaseURL: "http://localhost:8080"}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + return mux, ctx, memCache +} + +func TestHermitNonGitHubCaching(t *testing.T) { + httpTransportMutexHermit.Lock() + defer httpTransportMutexHermit.Unlock() + + callCount := 0 + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + callCount++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("go-binary-content")) + })) + defer backend.Close() + + originalTransport := http.DefaultTransport + defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign + http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign + + mux, ctx, _ := setupHermitTest(t) + + req1 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) + w1 := httptest.NewRecorder() + mux.ServeHTTP(w1, req1) + + assert.Equal(t, http.StatusOK, w1.Code) + assert.Equal(t, "go-binary-content", w1.Body.String()) + assert.Equal(t, 1, callCount) + + req2 := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/golang.org/dl/go1.21.0.tar.gz", nil) + w2 := httptest.NewRecorder() + mux.ServeHTTP(w2, req2) + + assert.Equal(t, http.StatusOK, w2.Code) + assert.Equal(t, "go-binary-content", w2.Body.String()) + assert.Equal(t, 1, callCount, "second request should be served from cache") +} + +type mockTransport struct { + backend *httptest.Server + originalTransport http.RoundTripper +} + +func (m *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) { + newReq := req.Clone(req.Context()) + newReq.URL.Scheme = "http" + newReq.URL.Host = m.backend.Listener.Addr().String() + newReq.RequestURI = "" + return m.originalTransport.RoundTrip(newReq) +} + +func TestHermitGitHubRelease(t *testing.T) { + githubCallCount := 0 + githubServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + githubCallCount++ + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("github-binary-content")) + })) + defer githubServer.Close() + + mux, ctx, memCache := setupHermitTest(t) + + _, err := strategy.NewGitHubReleases(ctx, strategy.GitHubReleasesConfig{}, jobscheduler.New(ctx, jobscheduler.Config{}), memCache, mux) + assert.NoError(t, err) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + assert.True(t, w.Code == http.StatusOK || w.Code == http.StatusBadGateway || w.Code == http.StatusNotFound, + "should attempt to fetch from GitHub (may fail without mock)") +} + +func TestHermitNonOKStatus(t *testing.T) { + httpTransportMutexHermit.Lock() + defer httpTransportMutexHermit.Unlock() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not found")) + })) + defer backend.Close() + + originalTransport := http.DefaultTransport + defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign + http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign + + mux, ctx, memCache := setupHermitTest(t) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/hermit/example.com/missing.tar.gz", nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + assert.Equal(t, http.StatusNotFound, w.Code) + assert.Equal(t, "not found", w.Body.String()) + + key := cache.NewKey("https://example.com/missing.tar.gz") + _, _, err := memCache.Open(context.Background(), key) + assert.Error(t, err, "non-OK responses should not be cached") +} + +func TestHermitDifferentSources(t *testing.T) { + tests := []struct { + name string + path string + wantHost string + }{ + { + name: "golang.org", + path: "/hermit/golang.org/dl/go1.21.0.tar.gz", + wantHost: "golang.org", + }, + { + name: "npm registry", + path: "/hermit/registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + wantHost: "registry.npmjs.org", + }, + { + name: "HashiCorp", + path: "/hermit/releases.hashicorp.com/terraform/1.5.0/terraform_1.5.0_linux_amd64.zip", + wantHost: "releases.hashicorp.com", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + httpTransportMutexHermit.Lock() + defer httpTransportMutexHermit.Unlock() + + backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte("content")) + })) + defer backend.Close() + + originalTransport := http.DefaultTransport + defer func() { http.DefaultTransport = originalTransport }() //nolint:reassign + http.DefaultTransport = &mockTransport{backend: backend, originalTransport: originalTransport} //nolint:reassign + + mux, ctx, _ := setupHermitTest(t) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "content", w.Body.String()) + }) + } +} + +func TestHermitCacheKeyGeneration(t *testing.T) { + tests := []struct { + name string + path string + wantKey string + }{ + { + name: "golang.org", + path: "/hermit/golang.org/dl/go1.21.0.tar.gz", + wantKey: "https://golang.org/dl/go1.21.0.tar.gz", + }, + { + name: "npm registry with scope", + path: "/hermit/registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + wantKey: "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.25.0.tgz", + }, + { + name: "GitHub release", + path: "/hermit/github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", + wantKey: "https://github.com/alecthomas/chroma/releases/download/v2.14.0/chroma-2.14.0-linux-amd64.tar.gz", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mux, ctx, _ := setupHermitTest(t) + + req := httptest.NewRequestWithContext(ctx, http.MethodGet, tt.path, nil) + w := httptest.NewRecorder() + mux.ServeHTTP(w, req) + }) + } +}