Clean the docker runtime (#254)

This PR cleans a few things on the docker runtime:
- It moves the application of the overrides (user can override which
images/binaries use during execution) out of the runtime since they only
modify the manifest
([here](https://github.com/flashbots/builder-playground/pull/254/files#diff-b7bbff9c5fe2d0c8165900e164ed7b2471fc8f3d886ccee8786e1734901ebfa4L124)).
- Fixes the component unit tests. The way it was done before, the inputs
were not modified since the test framework would apply the default
variables.
([here](https://github.com/flashbots/builder-playground/pull/254/files#diff-35602f032417ec14bb4f3092bc1cd29cced0b7a6b1a52b108f39a68227998219R29)).
- Moved the logic for the interactive mode out of the docker runtime
using the callback interface from #232
([here](https://github.com/flashbots/builder-playground/pull/254/files#diff-b7bbff9c5fe2d0c8165900e164ed7b2471fc8f3d886ccee8786e1734901ebfa4L235)).
It does not make sense that the Docker runtime owns the component that
deals with the visualization on the terminal.

Author: ferranbt

main.go

@@ -5,7 +5,6 @@ import (
 	"fmt"
 	"log"
 	"os"
-	"os/exec"
 	"sort"
 	"strings"
 	"time"
@@ -228,28 +227,28 @@ func runIt(recipe playground.Recipe) error {
 		return nil
 	}
 
-	// validate that override is being applied to a service in the manifest
-	for k := range overrides {
-		if _, ok := svcManager.GetService(k); !ok {
-			return fmt.Errorf("service '%s' in override not found in manifest", k)
-		}
+	if err := svcManager.ApplyOverrides(overrides); err != nil {
+		return err
 	}
 
 	cfg := &playground.RunnerConfig{
 		Out:                  artifacts.Out,
 		Manifest:             svcManager,
-		Overrides:            overrides,
-		Interactive:          interactive,
 		BindHostPortsLocally: !bindExternal,
 		NetworkName:          networkName,
 		Labels:               labels,
 		LogInternally:        !disableLogs,
 		Platform:             platform,
 	}
 
+	if interactive {
+		i := playground.NewInteractiveDisplay(svcManager)
+		cfg.Callback = i.HandleUpdate
+	}
+
 	// Add callback to log service updates in debug mode
 	if logLevel == playground.LevelDebug {
-		cfg.Callback = func(serviceName, update string) {
+		cfg.Callback = func(serviceName string, update playground.TaskStatus) {
 			log.Printf("[DEBUG] [%s] %s\n", serviceName, update)
 		}
 	}
@@ -294,7 +293,7 @@ func runIt(recipe playground.Recipe) error {
 
 	fmt.Printf("\nWaiting for network to be ready for transactions...\n")
 	networkReadyStart := time.Now()
-	if err := playground.CompleteReady(ctx, dockerRunner.Instances()); err != nil {
+	if err := playground.CompleteReady(ctx, svcManager.Services); err != nil {
 		dockerRunner.Stop()
 		return fmt.Errorf("network not ready: %w", err)
 	}
@@ -316,7 +315,7 @@ func runIt(recipe playground.Recipe) error {
 	watchdogErr := make(chan error, 1)
 	if watchdog {
 		go func() {
-			if err := playground.RunWatchdog(artifacts.Out, dockerRunner.Instances()); err != nil {
+			if err := playground.RunWatchdog(artifacts.Out, svcManager.Services); err != nil {
 				watchdogErr <- fmt.Errorf("watchdog failed: %w", err)
 			}
 		}()
@@ -343,26 +342,3 @@ func runIt(recipe playground.Recipe) error {
 	}
 	return nil
 }
-
-func isExecutableValid(path string) error {
-	// First check if file exists
-	_, err := os.Stat(path)
-	if err != nil {
-		return fmt.Errorf("file does not exist or is inaccessible: %w", err)
-	}
-
-	// Try to execute with a harmless flag or in a way that won't run the main program
-	cmd := exec.Command(path, "--version")
-	// Redirect output to /dev/null
-	cmd.Stdout = nil
-	cmd.Stderr = nil
-
-	if err := cmd.Start(); err != nil {
-		return fmt.Errorf("cannot start executable: %w", err)
-	}
-
-	// Immediately kill the process since we just want to test if it starts
-	cmd.Process.Kill()
-
-	return nil
-}

playground/components.go

@@ -397,13 +397,13 @@ func (o *OpGeth) Apply(manifest *Manifest) {
 		})
 }
 
-func opGethReadyFn(ctx context.Context, instance *instance) error {
-	opGethURL := fmt.Sprintf("http://localhost:%d", instance.service.MustGetPort("http").HostPort)
+func opGethReadyFn(ctx context.Context, service *Service) error {
+	opGethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
 	return waitForFirstBlock(ctx, opGethURL, 60*time.Second)
 }
 
-func opGethWatchdogFn(out io.Writer, instance *instance, ctx context.Context) error {
-	gethURL := fmt.Sprintf("http://localhost:%d", instance.service.MustGetPort("http").HostPort)
+func opGethWatchdogFn(out io.Writer, service *Service, ctx context.Context) error {
+	gethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
 	return watchChainHead(out, gethURL, 2*time.Second)
 }
 
@@ -456,7 +456,7 @@ func (r *RethEL) Apply(manifest *Manifest) {
 			"--chain", "/data/genesis.json",
 			"--datadir", "/data_reth",
 			"--color", "never",
-			"--ipcpath", "/data_reth/reth.ipc",
+			"--ipcdisable",
 			"--addr", "0.0.0.0",
 			"--port", `{{Port "rpc" 30303}}`,
 			// "--disable-discovery",
@@ -480,12 +480,12 @@ func (r *RethEL) Apply(manifest *Manifest) {
 			logLevelToRethVerbosity(manifest.ctx.LogLevel),
 		).
 		WithRelease(rethELRelease).
-		WithWatchdog(func(out io.Writer, instance *instance, ctx context.Context) error {
-			rethURL := fmt.Sprintf("http://localhost:%d", instance.service.MustGetPort("http").HostPort)
+		WithWatchdog(func(out io.Writer, service *Service, ctx context.Context) error {
+			rethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
 			return watchChainHead(out, rethURL, 12*time.Second)
 		}).
-		WithReadyFn(func(ctx context.Context, instance *instance) error {
-			elURL := fmt.Sprintf("http://localhost:%d", instance.service.MustGetPort("http").HostPort)
+		WithReadyFn(func(ctx context.Context, service *Service) error {
+			elURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
 			return waitForFirstBlock(ctx, elURL, 60*time.Second)
 		}).
 		WithArtifact("/data/genesis.json", "genesis.json").
@@ -627,8 +627,8 @@ func (m *MevBoostRelay) Apply(manifest *Manifest) {
 	}
 }
 
-func mevboostRelayWatchdogFn(out io.Writer, instance *instance, ctx context.Context) error {
-	beaconNodeURL := fmt.Sprintf("http://localhost:%d", instance.service.MustGetPort("http").HostPort)
+func mevboostRelayWatchdogFn(out io.Writer, service *Service, ctx context.Context) error {
+	beaconNodeURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
 
 	watchGroup := newWatchGroup()
 	watchGroup.watch(func() error {
@@ -681,8 +681,8 @@ func (o *OpReth) Apply(manifest *Manifest) {
 			"--addr", "0.0.0.0",
 			"--port", `{{Port "rpc" 30303}}`).
 		WithRelease(opRethRelease).
-		WithWatchdog(func(out io.Writer, instance *instance, ctx context.Context) error {
-			rethURL := fmt.Sprintf("http://localhost:%d", instance.service.MustGetPort("http").HostPort)
+		WithWatchdog(func(out io.Writer, service *Service, ctx context.Context) error {
+			rethURL := fmt.Sprintf("http://localhost:%d", service.MustGetPort("http").HostPort)
 			return watchChainHead(out, rethURL, 2*time.Second)
 		}).
 		WithArtifact("/data/jwtsecret", "jwtsecret").

playground/components_test.go

@@ -19,15 +19,15 @@ func TestRecipeOpstackSimple(t *testing.T) {
 	tt := newTestFramework(t)
 	defer tt.Close()
 
-	tt.test(&OpRecipe{})
+	tt.test(&OpRecipe{}, nil)
 }
 
 func TestRecipeOpstackExternalBuilder(t *testing.T) {
 	tt := newTestFramework(t)
 	defer tt.Close()
 
-	tt.test(&OpRecipe{
-		externalBuilder: "http://host.docker.internal:4444",
+	tt.test(&OpRecipe{}, []string{
+		"--external-builder", "http://host.docker.internal:4444",
 	})
 }
 
@@ -36,8 +36,8 @@ func TestRecipeOpstackEnableForkAfter(t *testing.T) {
 	defer tt.Close()
 
 	forkTime := uint64(10)
-	manifest := tt.test(&OpRecipe{
-		enableLatestFork: &forkTime,
+	manifest := tt.test(&OpRecipe{}, []string{
+		"--enable-latest-fork", "10",
 	})
 
 	elService := manifest.MustGetService("op-geth")
@@ -49,23 +49,23 @@ func TestRecipeL1Simple(t *testing.T) {
 	tt := newTestFramework(t)
 	defer tt.Close()
 
-	tt.test(&L1Recipe{})
+	tt.test(&L1Recipe{}, nil)
 }
 
 func TestRecipeL1UseNativeReth(t *testing.T) {
 	tt := newTestFramework(t)
 	defer tt.Close()
 
-	tt.test(&L1Recipe{
-		useNativeReth: true,
+	tt.test(&L1Recipe{}, []string{
+		"--use-native-reth",
 	})
 }
 
 func TestComponentBuilderHub(t *testing.T) {
 	tt := newTestFramework(t)
 	defer tt.Close()
 
-	tt.test(&BuilderHub{})
+	tt.test(&BuilderHub{}, nil)
 
 	// TODO: Calling the port directly on the host machine will not work once we have multiple
 	// tests running in parallel
@@ -83,7 +83,7 @@ func newTestFramework(t *testing.T) *testFramework {
 	return &testFramework{t: t}
 }
 
-func (tt *testFramework) test(s ServiceGen) *Manifest {
+func (tt *testFramework) test(s ServiceGen, args []string) *Manifest {
 	t := tt.t
 
 	// use the name of the repo and the current timestamp to generate
@@ -104,13 +104,14 @@ func (tt *testFramework) test(s ServiceGen) *Manifest {
 	}
 
 	o := &output{
-		dst: e2eTestDir,
+		dst:     e2eTestDir,
+		homeDir: filepath.Join(e2eTestDir, "artifacts"),
 	}
 
 	if recipe, ok := s.(Recipe); ok {
 		// We have to parse the flags since they are used to set the
 		// default values for the recipe inputs
-		err := recipe.Flags().Parse([]string{})
+		err := recipe.Flags().Parse(args)
 		require.NoError(t, err)
 
 		_, err = recipe.Artifacts().OutputDir(e2eTestDir).Build()
@@ -142,7 +143,7 @@ func (tt *testFramework) test(s ServiceGen) *Manifest {
 	require.NoError(t, err)
 
 	require.NoError(t, dockerRunner.WaitForReady(context.Background(), 20*time.Second))
-	require.NoError(t, CompleteReady(context.Background(), dockerRunner.Instances()))
+	require.NoError(t, CompleteReady(context.Background(), svcManager.Services))
 
 	return svcManager
 }

playground/interactive.go

@@ -0,0 +1,112 @@
+package playground
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/charmbracelet/bubbles/spinner"
+	"github.com/charmbracelet/lipgloss"
+)
+
+type InteractiveDisplay struct {
+	manifest     *Manifest
+	taskUpdateCh chan struct{}
+	status       sync.Map
+}
+
+type taskUI struct {
+	tasks    map[string]string
+	spinners map[string]spinner.Model
+	style    lipgloss.Style
+}
+
+func NewInteractiveDisplay(manifest *Manifest) *InteractiveDisplay {
+	i := &InteractiveDisplay{
+		manifest:     manifest,
+		taskUpdateCh: make(chan struct{}),
+	}
+
+	go i.printStatus()
+	return i
+}
+
+func (i *InteractiveDisplay) HandleUpdate(serviceName string, status TaskStatus) {
+	i.status.Store(serviceName, status)
+
+	select {
+	case i.taskUpdateCh <- struct{}{}:
+	default:
+	}
+}
+
+func (i *InteractiveDisplay) printStatus() {
+	fmt.Print("\033[s")
+	lineOffset := 0
+
+	// Get ordered service names from manifest
+	orderedServices := make([]string, 0, len(i.manifest.Services))
+	for _, svc := range i.manifest.Services {
+		orderedServices = append(orderedServices, svc.Name)
+	}
+
+	// Initialize UI state
+	ui := taskUI{
+		tasks:    make(map[string]string),
+		spinners: make(map[string]spinner.Model),
+		style:    lipgloss.NewStyle(),
+	}
+
+	// Initialize spinners for each service
+	for _, name := range orderedServices {
+		sp := spinner.New()
+		sp.Spinner = spinner.Dot
+		ui.spinners[name] = sp
+	}
+
+	tickSpinner := func(name string) spinner.Model {
+		sp := ui.spinners[name]
+		sp.Tick()
+		ui.spinners[name] = sp
+		return sp
+	}
+
+	for {
+		select {
+		case <-i.taskUpdateCh:
+			// Clear the previous lines and move cursor up
+			if lineOffset > 0 {
+				fmt.Printf("\033[%dA", lineOffset)
+				fmt.Print("\033[J")
+			}
+
+			lineOffset = 0
+			// Use ordered services instead of ranging over map
+			for _, name := range orderedServices {
+				status, ok := i.status.Load(name)
+				if !ok {
+					status = TaskStatusPending
+				}
+
+				var statusLine string
+				switch status {
+				case TaskStatusStarted, TaskStatusHealthy:
+					sp := tickSpinner(name)
+					statusLine = ui.style.Foreground(lipgloss.Color("2")).Render(fmt.Sprintf("%s [%s] Running", sp.View(), name))
+				case TaskStatusDie:
+					statusLine = ui.style.Foreground(lipgloss.Color("1")).Render(fmt.Sprintf("✗ [%s] Failed", name))
+				case TaskStatusPulled, TaskStatusPending:
+					sp := tickSpinner(name)
+					statusLine = ui.style.Foreground(lipgloss.Color("3")).Render(fmt.Sprintf("%s [%s] Pending", sp.View(), name))
+				case TaskStatusPulling:
+					sp := tickSpinner(name)
+					statusLine = ui.style.Foreground(lipgloss.Color("3")).Render(fmt.Sprintf("%s [%s] Pulling", sp.View(), name))
+				default:
+					panic(fmt.Sprintf("BUG: status '%s' not handled", name))
+				}
+
+				fmt.Println(statusLine)
+				lineOffset++
+			}
+		}
+	}
+}