From 47b0e1a4bc241adc7a2bd90f8f3e63c5d0be223e Mon Sep 17 00:00:00 2001 From: Test User Date: Fri, 20 Feb 2026 17:31:45 +1100 Subject: [PATCH] feat: add global state directory config Add a top-level `state` config field (default: `./state`) that gets injected as CACHEW_STATE for expansion in plugin config defaults. Mirror root and disk cache root now default to `${CACHEW_STATE}/git-mirrors` and `${CACHEW_STATE}/cache`. Co-Authored-By: Claude Opus 4.6 --- cachew.hcl | 9 +++--- cmd/cachewd/main.go | 53 ++++++++++++++++++++++++++++++------ internal/cache/api.go | 14 ++++++---- internal/cache/disk.go | 15 ++++++---- internal/config/config.go | 2 +- internal/gitclone/manager.go | 2 +- 6 files changed, 69 insertions(+), 26 deletions(-) diff --git a/cachew.hcl b/cachew.hcl index 492de12..cc217cf 100644 --- a/cachew.hcl +++ b/cachew.hcl @@ -3,14 +3,13 @@ # target = "https://example.jfrog.io" # } +state = "./state" url = "http://127.0.0.1:8080" log { level = "debug" } -git-clone { - mirror-root = "./state/git-mirrors" -} +git-clone {} github-app { # Uncomment and add: @@ -24,7 +23,8 @@ metrics {} git { #bundle-interval = "24h" - #snapshot-interval = "24h" + snapshot-interval = "1h" + repack-interval = "1h" } host "https://w3.org" {} @@ -35,7 +35,6 @@ github-releases { } disk { - root = "./state/cache" limit-mb = 250000 max-ttl = "8h" } diff --git a/cmd/cachewd/main.go b/cmd/cachewd/main.go index 59efcbc..8bab8d6 100644 --- a/cmd/cachewd/main.go +++ b/cmd/cachewd/main.go @@ -31,6 +31,7 @@ import ( ) type GlobalConfig struct { + State string `hcl:"state" default:"./state" help:"Base directory for all state (git mirrors, cache, etc.)."` Bind string `hcl:"bind" default:"127.0.0.1:8080" help:"Bind address for the server."` URL string `hcl:"url" default:"http://127.0.0.1:8080/" help:"Base URL for cachewd."` SchedulerConfig jobscheduler.Config `hcl:"scheduler,block"` @@ -56,12 +57,7 @@ func main() { globalConfigHCL, providersConfigHCL := config.Split[GlobalConfig](ast) - // Load global config. - var globalConfig GlobalConfig - globalSchema, err := hcl.Schema(&globalConfig) - kctx.FatalIfErrorf(err) - config.InjectEnvars(globalSchema, globalConfigHCL, "CACHEW", parseEnvars()) - err = hcl.UnmarshalAST(globalConfigHCL, &globalConfig, hcl.HydratedImplicitBlocks(true)) + globalConfig, envars, err := loadGlobalConfig(globalConfigHCL) kctx.FatalIfErrorf(err) ctx := context.Background() @@ -84,7 +80,7 @@ func main() { return } - mux, err := newMux(ctx, cr, sr, providersConfigHCL) + mux, err := newMux(ctx, cr, sr, providersConfigHCL, envars) kctx.FatalIfErrorf(err) metricsClient, err := metrics.New(ctx, globalConfig.MetricsConfig) @@ -137,7 +133,7 @@ func printSchema(kctx *kong.Context, cr *cache.Registry, sr *strategy.Registry) } } -func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfigHCL *hcl.AST) (*http.ServeMux, error) { +func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfigHCL *hcl.AST, vars map[string]string) (*http.ServeMux, error) { mux := http.NewServeMux() mux.HandleFunc("GET /_liveness", func(w http.ResponseWriter, _ *http.Request) { @@ -150,7 +146,7 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov _, _ = w.Write([]byte("OK")) //nolint:errcheck }) - if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, parseEnvars()); err != nil { + if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, vars); err != nil { return nil, fmt.Errorf("load config: %w", err) } @@ -200,6 +196,45 @@ func newServer(ctx context.Context, mux *http.ServeMux, bind string, metricsConf } } +// loadGlobalConfig unmarshals the global config from HCL, using a two-pass +// approach so that the "state" field is resolved first and then injected as +// CACHEW_STATE for expansion in other defaults (e.g. mirror-root, disk root). +func loadGlobalConfig(ast *hcl.AST) (GlobalConfig, map[string]string, error) { + var cfg GlobalConfig + schema, err := hcl.Schema(&cfg) + if err != nil { + return cfg, nil, fmt.Errorf("global config schema: %w", err) + } + envars := parseEnvars() + config.InjectEnvars(schema, ast, "CACHEW", envars) + + // First pass: preserve unknown ${VAR} references so we can extract "state". + preserving := hcl.WithDefaultTransformer(func(s string) string { + return os.Expand(s, func(key string) string { + if v, ok := envars[key]; ok { + return v + } + return "${" + key + "}" + }) + }) + if err := hcl.UnmarshalAST(ast, &cfg, hcl.HydratedImplicitBlocks(true), preserving); err != nil { + return cfg, nil, fmt.Errorf("load global config: %w", err) + } + + // Inject state directory as CACHEW_STATE for provider config expansion. + envars["CACHEW_STATE"] = cfg.State + + // Second pass: re-expand now that CACHEW_STATE is available. + cfg = GlobalConfig{} + expanding := hcl.WithDefaultTransformer(func(s string) string { + return os.Expand(s, func(key string) string { return envars[key] }) + }) + if err := hcl.UnmarshalAST(ast, &cfg, hcl.HydratedImplicitBlocks(true), expanding); err != nil { + return cfg, nil, fmt.Errorf("load global config: %w", err) + } + return cfg, envars, nil +} + func parseEnvars() map[string]string { envars := map[string]string{} for _, env := range os.Environ() { diff --git a/internal/cache/api.go b/internal/cache/api.go index 1ad68ad..6f13121 100644 --- a/internal/cache/api.go +++ b/internal/cache/api.go @@ -7,6 +7,7 @@ import ( "encoding/hex" "io" "net/http" + "os" "time" "github.com/alecthomas/errors" @@ -21,7 +22,7 @@ var ErrStatsUnavailable = errors.New("stats unavailable") type registryEntry struct { schema *hcl.Block - factory func(ctx context.Context, config *hcl.Block) (Cache, error) + factory func(ctx context.Context, config *hcl.Block, vars map[string]string) (Cache, error) } type Registry struct { @@ -48,9 +49,12 @@ func Register[Config any, C Cache](r *Registry, id, description string, factory block.Comments = hcl.CommentList{description} r.registry[id] = registryEntry{ schema: block, - factory: func(ctx context.Context, config *hcl.Block) (Cache, error) { + factory: func(ctx context.Context, config *hcl.Block, vars map[string]string) (Cache, error) { var cfg Config - if err := hcl.UnmarshalBlock(config, &cfg); err != nil { + transformer := func(defaultValue string) string { + return os.Expand(defaultValue, func(key string) string { return vars[key] }) + } + if err := hcl.UnmarshalBlock(config, &cfg, hcl.WithDefaultTransformer(transformer)); err != nil { return nil, errors.WithStack(err) } return factory(ctx, cfg) @@ -75,9 +79,9 @@ func (r *Registry) Exists(name string) bool { // Create a new cache instance from the given name and configuration. // // Will return "ErrNotFound" if the cache backend is not found. -func (r *Registry) Create(ctx context.Context, name string, config *hcl.Block) (Cache, error) { +func (r *Registry) Create(ctx context.Context, name string, config *hcl.Block, vars map[string]string) (Cache, error) { if entry, ok := r.registry[name]; ok { - return errors.WithStack2(entry.factory(ctx, config)) + return errors.WithStack2(entry.factory(ctx, config, vars)) } return nil, errors.Errorf("%s: %w", name, ErrNotFound) } diff --git a/internal/cache/disk.go b/internal/cache/disk.go index 10ca9e2..7d28ba0 100644 --- a/internal/cache/disk.go +++ b/internal/cache/disk.go @@ -14,7 +14,6 @@ import ( "time" "github.com/alecthomas/errors" - "github.com/alecthomas/kong" "github.com/block/cachew/internal/logging" ) @@ -30,7 +29,7 @@ func RegisterDisk(r *Registry) { } type DiskConfig struct { - Root string `hcl:"root" help:"Root directory for the disk storage."` + Root string `hcl:"root,optional" help:"Root directory for the disk storage." default:"${CACHEW_STATE}/cache"` LimitMB int `hcl:"limit-mb,optional" help:"Maximum size of the disk cache in megabytes (defaults to 10GB)." default:"10240"` MaxTTL time.Duration `hcl:"max-ttl,optional" help:"Maximum time-to-live for entries in the disk cache (defaults to 1 hour)." default:"1h"` EvictInterval time.Duration `hcl:"evict-interval,optional" help:"Interval at which to check files for eviction (defaults to 1 minute)." default:"1m"` @@ -61,10 +60,16 @@ func NewDisk(ctx context.Context, config DiskConfig) (*Disk, error) { if config.Root == "" { return nil, errors.New("root directory is required") } - err := kong.ApplyDefaults(&config) - if err != nil { - return nil, errors.Errorf("failed to apply defaults: %w", err) + if config.LimitMB == 0 { + config.LimitMB = 10240 + } + if config.MaxTTL == 0 { + config.MaxTTL = time.Hour + } + if config.EvictInterval == 0 { + config.EvictInterval = time.Minute } + var err error config.Root, err = filepath.Abs(config.Root) if err != nil { return nil, errors.Errorf("failed to get absolute path for cache root: %w", err) diff --git a/internal/config/config.go b/internal/config/config.go index 52762cf..98241fa 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -109,7 +109,7 @@ func Load( for _, node := range ast.Entries { switch node := node.(type) { case *hcl.Block: - c, err := cr.Create(ctx, node.Name, node) + c, err := cr.Create(ctx, node.Name, node, vars) if errors.Is(err, cache.ErrNotFound) { strategyCandidates = append(strategyCandidates, node) continue diff --git a/internal/gitclone/manager.go b/internal/gitclone/manager.go index 192163e..ed9633b 100644 --- a/internal/gitclone/manager.go +++ b/internal/gitclone/manager.go @@ -53,7 +53,7 @@ func DefaultGitTuningConfig() GitTuningConfig { } type Config struct { - MirrorRoot string `hcl:"mirror-root" help:"Directory to store git clones."` + MirrorRoot string `hcl:"mirror-root,optional" help:"Directory to store git clones." default:"${CACHEW_STATE}/git-mirrors"` FetchInterval time.Duration `hcl:"fetch-interval,optional" help:"How often to fetch from upstream in minutes." default:"15m"` RefCheckInterval time.Duration `hcl:"ref-check-interval,optional" help:"How long to cache ref checks." default:"10s"` Maintenance bool `hcl:"maintenance,optional" help:"Enable git maintenance scheduling for mirror repos." default:"false"`