refactor: move all global config into HCL only (no flags)

Procfile

@@ -1 +1 @@
-cachewd **/*.go !**/*_test.go ready=http:8080/_readiness=200: cachewd --log-level=debug
+cachewd **/*.go !**/*_test.go debounce=2s ready=http:8080/_readiness=200: CACHEW_LOG_LEVEL=debug cachewd

cachew.hcl

@@ -16,6 +16,8 @@ git-clone {
     mirror-root = "./state/git-mirrors"
 }
 
+metrics {}
+
 git {
   bundle-interval = "24h"
   snapshot-interval = "24h"

cmd/cachewd/main.go

@@ -7,7 +7,6 @@ import (
 	"net"
 	"net/http"
 	"os"
-	"slices"
 	"strings"
 	"time"
 
@@ -32,42 +31,45 @@ import (
 type GlobalConfig struct {
 	Bind            string              `hcl:"bind" default:"127.0.0.1:8080" help:"Bind address for the server."`
 	URL             string              `hcl:"url" default:"http://127.0.0.1:8080/" help:"Base URL for cachewd."`
-	SchedulerConfig jobscheduler.Config `embed:"" hcl:"scheduler,block" prefix:"scheduler-"`
-	LoggingConfig   logging.Config      `embed:"" hcl:"log,block" prefix:"log-"`
-	MetricsConfig   metrics.Config      `embed:"" hcl:"metrics,block" prefix:"metrics-"`
-	GitCloneConfig  gitclone.Config     `embed:"" hcl:"git-clone,block" prefix:"git-clone-"`
+	SchedulerConfig jobscheduler.Config `hcl:"scheduler,block"`
+	LoggingConfig   logging.Config      `hcl:"log,block"`
+	MetricsConfig   metrics.Config      `hcl:"metrics,block"`
+	GitCloneConfig  gitclone.Config     `hcl:"git-clone,block"`
 }
 
-var cli struct { //nolint:gochecknoglobals
+type CLI struct {
 	Schema bool `help:"Print the configuration file schema." xor:"command"`
 
-	Config kong.ConfigFlag `hcl:"-" help:"Configuration file path." placeholder:"PATH" required:"" default:"cachew.hcl"`
-
-	// GlobalConfig accepts command-line, but can also be parsed from HCL.
-	GlobalConfig
+	Config *os.File `hcl:"-" help:"Configuration file path." required:"" default:"cachew.hcl"`
 }
 
 func main() {
-	kctx := kong.Parse(&cli, kong.DefaultEnvars("CACHEW"), kong.Configuration(config.KongLoader[GlobalConfig], "cachew.hcl"))
+	var cli CLI
+	kctx := kong.Parse(&cli, kong.DefaultEnvars("CACHEW"))
 
-	configReader, err := os.Open(string(cli.Config))
+	defer cli.Config.Close()
+	ast, err := hcl.Parse(cli.Config)
 	kctx.FatalIfErrorf(err)
-	defer configReader.Close()
 
-	ast, err := hcl.Parse(configReader)
-	kctx.FatalIfErrorf(err)
+	globalConfigHCL, providersConfigHCL := config.Split[GlobalConfig](ast)
 
-	_, providersConfig := config.Split[GlobalConfig](ast)
+	// Load global config.
+	var globalConfig GlobalConfig
+	globalSchema, err := hcl.Schema(&globalConfig)
+	kctx.FatalIfErrorf(err)
+	config.InjectEnvars(globalSchema, globalConfigHCL, "CACHEW", parseEnvars())
+	err = hcl.UnmarshalAST(globalConfigHCL, &globalConfig, hcl.HydratedImplicitBlocks(true))
+	kctx.FatalIfErrorf(err)
 
 	ctx := context.Background()
-	logger, ctx := logging.Configure(ctx, cli.LoggingConfig)
+	logger, ctx := logging.Configure(ctx, globalConfig.LoggingConfig)
 
 	// Start initialising
-	managerProvider := gitclone.NewManagerProvider(ctx, cli.GitCloneConfig)
+	managerProvider := gitclone.NewManagerProvider(ctx, globalConfig.GitCloneConfig)
 
-	scheduler := jobscheduler.New(ctx, cli.SchedulerConfig)
+	scheduler := jobscheduler.New(ctx, globalConfig.SchedulerConfig)
 
-	cr, sr := newRegistries(scheduler, managerProvider)
+	cr, sr := newRegistries(globalConfig.URL, scheduler, managerProvider)
 
 	// Commands
 	switch { //nolint:gocritic
@@ -76,10 +78,10 @@ func main() {
 		return
 	}
 
-	mux, err := newMux(ctx, cr, sr, providersConfig)
+	mux, err := newMux(ctx, cr, sr, providersConfigHCL)
 	kctx.FatalIfErrorf(err)
 
-	metricsClient, err := metrics.New(ctx, cli.MetricsConfig)
+	metricsClient, err := metrics.New(ctx, globalConfig.MetricsConfig)
 	kctx.FatalIfErrorf(err, "failed to create metrics client")
 	defer func() {
 		if err := metricsClient.Close(); err != nil {
@@ -91,14 +93,14 @@ func main() {
 		kctx.FatalIfErrorf(err, "failed to start metrics server")
 	}
 
-	logger.InfoContext(ctx, "Starting cachewd", slog.String("bind", cli.Bind))
+	logger.InfoContext(ctx, "Starting cachewd", slog.String("bind", globalConfig.Bind))
 
-	server := newServer(ctx, logger, mux)
+	server := newServer(ctx, mux, globalConfig.Bind, globalConfig.MetricsConfig)
 	err = server.ListenAndServe()
 	kctx.FatalIfErrorf(err)
 }
 
-func newRegistries(scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider) (*cache.Registry, *strategy.Registry) {
+func newRegistries(cachewURL string, scheduler jobscheduler.Scheduler, cloneManagerProvider gitclone.ManagerProvider) (*cache.Registry, *strategy.Registry) {
 	cr := cache.NewRegistry()
 	cache.RegisterMemory(cr)
 	cache.RegisterDisk(cr)
@@ -108,7 +110,7 @@ func newRegistries(scheduler jobscheduler.Scheduler, cloneManagerProvider gitclo
 	strategy.RegisterAPIV1(sr)
 	strategy.RegisterArtifactory(sr)
 	strategy.RegisterGitHubReleases(sr)
-	strategy.RegisterHermit(sr, cli.URL)
+	strategy.RegisterHermit(sr, cachewURL)
 	strategy.RegisterHost(sr)
 	git.Register(sr, scheduler, cloneManagerProvider)
 	gomod.Register(sr, cloneManagerProvider)
@@ -118,9 +120,6 @@ func newRegistries(scheduler jobscheduler.Scheduler, cloneManagerProvider gitclo
 
 func printSchema(kctx *kong.Context, cr *cache.Registry, sr *strategy.Registry) {
 	schema := config.Schema[GlobalConfig](cr, sr)
-	slices.SortStableFunc(schema.Entries, func(a, b hcl.Entry) int {
-		return strings.Compare(a.EntryKey(), b.EntryKey())
-	})
 	text, err := hcl.MarshalAST(schema)
 	kctx.FatalIfErrorf(err)
 
@@ -132,7 +131,7 @@ func printSchema(kctx *kong.Context, cr *cache.Registry, sr *strategy.Registry)
 	}
 }
 
-func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfig *hcl.AST) (*http.ServeMux, error) {
+func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, providersConfigHCL *hcl.AST) (*http.ServeMux, error) {
 	mux := http.NewServeMux()
 
 	mux.HandleFunc("GET /_liveness", func(w http.ResponseWriter, _ *http.Request) {
@@ -145,25 +144,26 @@ func newMux(ctx context.Context, cr *cache.Registry, sr *strategy.Registry, prov
 		_, _ = w.Write([]byte("OK")) //nolint:errcheck
 	})
 
-	if err := config.Load(ctx, cr, sr, providersConfig, mux, parseEnvars()); err != nil {
+	if err := config.Load(ctx, cr, sr, providersConfigHCL, mux, parseEnvars()); err != nil {
 		return nil, fmt.Errorf("load config: %w", err)
 	}
 
 	return mux, nil
 }
 
-func newServer(ctx context.Context, logger *slog.Logger, mux *http.ServeMux) *http.Server {
+func newServer(ctx context.Context, mux *http.ServeMux, bind string, metricsConfig metrics.Config) *http.Server {
+	logger := logging.FromContext(ctx)
 	var handler http.Handler = mux
 
-	handler = otelhttp.NewMiddleware(cli.MetricsConfig.ServiceName,
+	handler = otelhttp.NewMiddleware(metricsConfig.ServiceName,
 		otelhttp.WithMeterProvider(otel.GetMeterProvider()),
 		otelhttp.WithTracerProvider(otel.GetTracerProvider()),
 	)(handler)
 
 	handler = httputil.LoggingMiddleware(handler)
 
 	return &http.Server{
-		Addr:              cli.Bind,
+		Addr:              bind,
 		Handler:           handler,
 		ReadTimeout:       30 * time.Minute,
 		WriteTimeout:      30 * time.Minute,

docker/Justfile

@@ -2,6 +2,7 @@ set positional-arguments := true
 set shell := ["bash", "-c"]
 
 # Configuration
+
 ROOT := `git rev-parse --show-toplevel 2>/dev/null || echo "."`
 TAG := `git rev-parse --short HEAD 2>/dev/null || echo "dev"`
 VERSION := `git describe --tags --always --dirty 2>/dev/null || echo "dev"`
@@ -41,7 +42,7 @@ build-multi:
 run log_level="info":
     @just build
     @echo "→ Starting cachew at http://localhost:8080 (log-level={{ log_level }})"
-    @docker run --rm -it -p 8080:8080 -v {{ ROOT }}/state:/app/state --name cachew cachew:local --log-level={{ log_level }}
+    @docker run --rm -it -p 8080:8080 -e CACHEW_LOG_LEVEL={{ log_level }} -v {{ ROOT }}/state:/app/state --name cachew cachew:local
 
 # Clean up Docker images
 clean:

go.mod

@@ -3,7 +3,7 @@ module github.com/block/cachew
 go 1.25.5
 
 require (
-	github.com/alecthomas/hcl/v2 v2.5.0
+	github.com/alecthomas/hcl/v2 v2.6.0
 	github.com/alecthomas/kong v1.13.0
 	github.com/goproxy/goproxy v0.25.0
 	github.com/lmittmann/tint v1.1.2

go.sum

@@ -4,8 +4,8 @@ github.com/alecthomas/chroma/v2 v2.23.1 h1:nv2AVZdTyClGbVQkIzlDm/rnhk1E9bU9nXwmZ
 github.com/alecthomas/chroma/v2 v2.23.1/go.mod h1:NqVhfBR0lte5Ouh3DcthuUCTUpDC9cxBOfyMbMQPs3o=
 github.com/alecthomas/errors v0.9.1 h1:JNXtU30rtMNARCkW41OTZ4yL6Lyocq20xIJgIw2raqI=
 github.com/alecthomas/errors v0.9.1/go.mod h1:l8mjMEHMGUdIWPMNtvDyRYPVS1fQFXHFXc/iVCCLGkI=
-github.com/alecthomas/hcl/v2 v2.5.0 h1:0L0oGrZPHokiXaKtsEcLa3hBjfVrRLUUK3u5vXQSybg=
-github.com/alecthomas/hcl/v2 v2.5.0/go.mod h1:4UUp66q8ony5j8tm2bANErujUpZ3GgHBLgaKxTUQlQI=
+github.com/alecthomas/hcl/v2 v2.6.0 h1:5+3FFpVFP0PAKVmP66a32U5mVVa6C8VRx+w8HLqmGpo=
+github.com/alecthomas/hcl/v2 v2.6.0/go.mod h1:4UUp66q8ony5j8tm2bANErujUpZ3GgHBLgaKxTUQlQI=
 github.com/alecthomas/kong v1.13.0 h1:5e/7XC3ugvhP1DQBmTS+WuHtCbcv44hsohMgcvVxSrA=
 github.com/alecthomas/kong v1.13.0/go.mod h1:wrlbXem1CWqUV5Vbmss5ISYhsVPkBb1Yo7YKJghju2I=
 github.com/alecthomas/participle/v2 v2.1.4 h1:W/H79S8Sat/krZ3el6sQMvMaahJ+XcM9WSI2naI7w2U=

internal/config/config.go

@@ -4,8 +4,12 @@ package config
 import (
 	"context"
 	"log/slog"
+	"math/big"
 	"net/http"
 	"os"
+	"slices"
+	"strconv"
+	"strings"
 
 	"github.com/alecthomas/errors"
 	"github.com/alecthomas/hcl/v2"
@@ -152,3 +156,120 @@ func expandVars(ast *hcl.AST, vars map[string]string) {
 		return next()
 	})
 }
+
+// InjectEnvars walks the schema and for each attribute not present in the config,
+// checks for a corresponding environment variable and injects it.
+//
+// Environment variable names are derived from the path to the attribute:
+// prefix + block names + attr name, joined with "_", uppercased, hyphens replaced with "_".
+// e.g. prefix="CACHEW", path=["scheduler", "concurrency"] -> "CACHEW_SCHEDULER_CONCURRENCY".
+func InjectEnvars(schema *hcl.AST, config *hcl.AST, prefix string, vars map[string]string) {
+	container := &entryContainer{ast: config}
+	injectEntries(schema.Entries, container, []string{prefix}, vars)
+	_ = hcl.AddParentRefs(config) //nolint:errcheck
+}
+
+// entryContainer abstracts over AST (top-level) and Block (nested) for inserting entries.
+type entryContainer struct {
+	ast   *hcl.AST
+	block *hcl.Block
+}
+
+func (c *entryContainer) entries() hcl.Entries {
+	if c.block != nil {
+		return c.block.Body
+	}
+	return c.ast.Entries
+}
+
+func (c *entryContainer) append(entry hcl.Entry) {
+	if c.block != nil {
+		c.block.Body = append(c.block.Body, entry)
+	} else {
+		c.ast.Entries = append(c.ast.Entries, entry)
+	}
+}
+
+func (c *entryContainer) findBlock(name string) *entryContainer {
+	for _, e := range c.entries() {
+		if block, ok := e.(*hcl.Block); ok && block.Name == name {
+			return &entryContainer{ast: c.ast, block: block}
+		}
+	}
+	return nil
+}
+
+func injectEntries(schemaEntries hcl.Entries, container *entryContainer, path []string, vars map[string]string) {
+	for _, entry := range schemaEntries {
+		switch entry := entry.(type) {
+		case *hcl.Attribute:
+			typ, ok := entry.Value.(*hcl.Type)
+			if !ok {
+				continue
+			}
+			envarName := pathToEnvar(append(slices.Clone(path), entry.Key))
+			val, ok := vars[envarName]
+			if !ok {
+				continue
+			}
+			if hasAttr(container.entries(), entry.Key) {
+				continue
+			}
+			hclVal, err := parseValue(val, typ.Type)
+			if err != nil {
+				continue
+			}
+			container.append(&hcl.Attribute{Key: entry.Key, Value: hclVal})
+
+		case *hcl.Block:
+			child := container.findBlock(entry.Name)
+			if child == nil {
+				// Create a temporary container; only add the block to the
+				// config if at least one envar populated it.
+				tmp := &entryContainer{ast: container.ast, block: &hcl.Block{Name: entry.Name}}
+				injectEntries(entry.Body, tmp, append(path, entry.Name), vars)
+				if len(tmp.block.Body) > 0 {
+					container.append(tmp.block)
+				}
+			} else {
+				injectEntries(entry.Body, child, append(path, entry.Name), vars)
+			}
+		}
+	}
+}
+
+func pathToEnvar(path []string) string {
+	s := strings.Join(path, "_")
+	s = strings.ReplaceAll(s, "-", "_")
+	return strings.ToUpper(s)
+}
+
+func hasAttr(entries hcl.Entries, key string) bool {
+	for _, e := range entries {
+		if attr, ok := e.(*hcl.Attribute); ok && attr.Key == key {
+			return true
+		}
+	}
+	return false
+}
+
+func parseValue(raw string, typ string) (hcl.Value, error) {
+	switch typ {
+	case "string":
+		return &hcl.String{Str: raw}, nil
+	case "number":
+		f, _, err := big.ParseFloat(raw, 10, 256, big.ToNearestEven)
+		if err != nil {
+			return nil, errors.Wrap(err, raw)
+		}
+		return &hcl.Number{Float: f}, nil
+	case "boolean":
+		b, err := strconv.ParseBool(raw)
+		if err != nil {
+			return nil, errors.Wrap(err, raw)
+		}
+		return &hcl.Bool{Bool: b}, nil
+	default:
+		return nil, errors.Errorf("unsupported type %q", typ)
+	}
+}