feat/ add log forwarding in HadesLogManager and ContinueOnError step support

.env.example

@@ -24,6 +24,10 @@ WATCH_NAMESPACE="hades"
 DELETE_ON_COMPLETE="true"
 MAX_PARALLELISM="100"
 
+### HadesLogManager Options ###
+HADESLOGMANAGER_API_PORT=8081 # Port for the Log Manager to listen on (default: 8081)
+ARTEMIS_ADAPTER_URL=http://localhost:8082/adapter/logs # URL for the Artemis Adapter to send logs to (default: http://localhost:8082/adapter/logs)
+
 ### DockerScheduler Options ###
 DOCKER_HOST="unix:///var/run/docker.sock" # Docker host to connect to (default: unix:///var/run/docker.sock)
 DOCKER_SCRIPT_EXECUTOR="/bin/bash -c" # Path to the script executor to use (default: /bin/sh)

.github/workflows/ci.yml

@@ -75,11 +75,29 @@ jobs:
       docker-file: ./HadesScheduler/HadesOperator/Dockerfile
     secrets: inherit
 
+  build-log-manager:
+    needs: test
+    if: github.actor != 'dependabot[bot]'
+    uses: ls1intum/.github/.github/workflows/build-and-push-docker-image.yml@v1.2.0
+    with:
+      image-name: ls1intum/hades/hades-log-manager
+      docker-file: ./HadesLogManager/Dockerfile
+    secrets: inherit
+
+  deploy-traefik:
+    needs: [build-api, build-scheduler]
+    uses: ls1intum/.github/.github/workflows/deploy-docker-traefik.yml@main
+    with:
+      environment: hades-test
+      image-tag: v3.6
+    secrets: inherit
+
   deploy:
     needs: [build-api, build-scheduler]
     runs-on: ubuntu-latest
     environment: hades-test
     steps:
+
       - name: Checkout repository
         uses: actions/checkout@v6
 
@@ -105,7 +123,7 @@ jobs:
           proxy_username: ${{ vars.DEPLOYMENT_GATEWAY_USER }}
           proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
           proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
-          source: "docker-compose.test.yml,traefik/traefik.yml"
+          source: "docker-compose.test.yml"
           target: /opt/hades
           strip_components: 0
 
@@ -120,8 +138,6 @@ jobs:
           proxy_key: ${{ secrets.DEPLOYMENT_GATEWAY_SSH_KEY }}
           proxy_port: ${{ vars.DEPLOYMENT_GATEWAY_PORT }}
           script: |
-            touch /opt/hades/traefik/acme.json
-            chmod 600 /opt/hades/traefik/acme.json
             install -m 600 /dev/null /opt/hades/.env
             cat > /opt/hades/.env <<'EOF'
             IMAGE_TAG=${{ needs.build-api.outputs.image_tag }}

HadesLogManager/Dockerfile

@@ -0,0 +1,30 @@
+# Use an official Go runtime as a parent image
+FROM golang:1.25-alpine AS builder
+
+# Set the working directory inside the container
+WORKDIR /app
+
+# Copy go.mod and go.sum files
+COPY ./HadesLogManager/go.mod ./HadesLogManager/go.sum ./HadesLogManager/
+COPY ./shared/go.mod ./shared/go.sum ./shared/
+RUN cd HadesLogManager && go mod download
+
+# Copy the Go application source code into the container
+COPY ./HadesLogManager ./HadesLogManager
+COPY ./shared ./shared
+
+# Build the Go application
+WORKDIR /app/HadesLogManager
+RUN CGO_ENABLED=0 go build -o hades-log-manager .
+
+# Start a new stage for the minimal runtime container
+FROM gcr.io/distroless/static-debian12
+
+# Set the working directory inside the minimal runtime container
+WORKDIR /app
+
+# Copy the built binary from the builder container into the minimal runtime container
+COPY --from=builder /app/HadesLogManager/hades-log-manager . 
+
+# Run your Go application
+CMD ["/app/hades-log-manager"]

HadesLogManager/go.mod

@@ -1,16 +1,16 @@
-module HadesAdapter
+module github.com/ls1intum/hades/hadesLogManager
 
 go 1.24.4
 
 require (
 	github.com/gin-gonic/gin v1.11.0
-	github.com/nats-io/nats.go v1.48.0
+	github.com/nats-io/nats.go v1.49.0
 )
 
 require (
 	github.com/bytedance/sonic v1.14.0 // indirect
 	github.com/bytedance/sonic/loader v0.3.0 // indirect
-	github.com/caarlos0/env/v11 v11.3.1 // indirect
+	github.com/caarlos0/env/v11 v11.4.0 // indirect
 	github.com/cloudwego/base64x v0.1.6 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.9 // indirect
 	github.com/gin-contrib/sse v1.1.0 // indirect
@@ -34,15 +34,15 @@ require (
 	github.com/ugorji/go/codec v1.3.0 // indirect
 	golang.org/x/arch v0.20.0 // indirect
 	golang.org/x/net v0.47.0 // indirect
-	golang.org/x/text v0.31.0 // indirect
+	golang.org/x/text v0.32.0 // indirect
 	google.golang.org/protobuf v1.36.9 // indirect
 )
 
 require (
-	github.com/klauspost/compress v1.18.0 // indirect
-	github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e
-	github.com/nats-io/nkeys v0.4.11 // indirect
+	github.com/klauspost/compress v1.18.2 // indirect
+	github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398
+	github.com/nats-io/nkeys v0.4.12 // indirect
 	github.com/nats-io/nuid v1.0.1 // indirect
-	golang.org/x/crypto v0.45.0 // indirect
-	golang.org/x/sys v0.38.0 // indirect
+	golang.org/x/crypto v0.46.0 // indirect
+	golang.org/x/sys v0.39.0 // indirect
 )

HadesLogManager/go.sum

@@ -2,8 +2,8 @@ github.com/bytedance/sonic v1.14.0 h1:/OfKt8HFw0kh2rj8N0F6C/qPGRESq0BbaNZgcNXXzQ
 github.com/bytedance/sonic v1.14.0/go.mod h1:WoEbx8WTcFJfzCe0hbmyTGrfjt8PzNEBdxlNUO24NhA=
 github.com/bytedance/sonic/loader v0.3.0 h1:dskwH8edlzNMctoruo8FPTJDF3vLtDT0sXZwvZJyqeA=
 github.com/bytedance/sonic/loader v0.3.0/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI=
-github.com/caarlos0/env/v11 v11.3.1 h1:cArPWC15hWmEt+gWk7YBi7lEXTXCvpaSdCiZE2X5mCA=
-github.com/caarlos0/env/v11 v11.3.1/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
+github.com/caarlos0/env/v11 v11.4.0 h1:Kcb6t5kIIr4XkoQC9AF2j+8E1Jsrl3Wz/hhm1LtoGAc=
+github.com/caarlos0/env/v11 v11.4.0/go.mod h1:qupehSf/Y0TUTsxKywqRt/vJjN5nz6vauiYEUUr8P4U=
 github.com/cloudwego/base64x v0.1.6 h1:t11wG9AECkCDk5fMSoxmufanudBtJ+/HemLstXDLI2M=
 github.com/cloudwego/base64x v0.1.6/go.mod h1:OFcloc187FXDaYHvrNIjxSe8ncn0OOM8gEHfghB2IPU=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -37,14 +37,14 @@ github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
-github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
-github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
+github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ=
 github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI=
-github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e h1:7wC0iDQ4m/gcLn2gYdxmKAMxIXxasKY5PJ0DJVAnLDM=
-github.com/ls1intum/hades/shared v0.0.0-20251217145134-8e471923356e/go.mod h1:1uOGnAwXXd3UTaO+jOEcbTv21KXFMMwE70gD1057miM=
+github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398 h1:hAVvbiD1m/NMZywecQdDR2pJ6sx2f9Nza5NgKgVxz7A=
+github.com/ls1intum/hades/shared v0.0.0-20260211120818-0675529ef398/go.mod h1:dtZoDaLxyhyNlMNGZ/zHJ2pEwvVcv62uPZD4f4y1jKY=
 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -53,10 +53,10 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee h1:W5t00kpgFdJifH4BDsTlE89Zl93FEloxaWZfGcifgq8=
 github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
-github.com/nats-io/nats.go v1.48.0 h1:pSFyXApG+yWU/TgbKCjmm5K4wrHu86231/w84qRVR+U=
-github.com/nats-io/nats.go v1.48.0/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g=
-github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0=
-github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE=
+github.com/nats-io/nats.go v1.49.0 h1:yh/WvY59gXqYpgl33ZI+XoVPKyut/IcEaqtsiuTJpoE=
+github.com/nats-io/nats.go v1.49.0/go.mod h1:fDCn3mN5cY8HooHwE2ukiLb4p4G4ImmzvXyJt+tGwdw=
+github.com/nats-io/nkeys v0.4.12 h1:nssm7JKOG9/x4J8II47VWCL1Ds29avyiQDRn0ckMvDc=
+github.com/nats-io/nkeys v0.4.12/go.mod h1:MT59A1HYcjIcyQDJStTfaOY6vhy9XTUjOFo+SVsvpBg=
 github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
 github.com/pelletier/go-toml/v2 v2.2.4 h1:mye9XuhQ6gvn5h28+VilKrrPoQVanw5PMw/TB0t5Ec4=
@@ -85,15 +85,15 @@ go.uber.org/mock v0.5.2 h1:LbtPTcP8A5k9WPXj54PPPbjcI4Y6lhyOZXn+VS7wNko=
 go.uber.org/mock v0.5.2/go.mod h1:wLlUxC2vVTPTaE3UD51E0BGOAElKrILxhVSDYQLld5o=
 golang.org/x/arch v0.20.0 h1:dx1zTU0MAE98U+TQ8BLl7XsJbgze2WnNKF/8tGp/Q6c=
 golang.org/x/arch v0.20.0/go.mod h1:bdwinDaKcfZUGpH09BB7ZmOfhalA8lQdzl62l8gGWsk=
-golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
-golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
+golang.org/x/crypto v0.46.0 h1:cKRW/pmt1pKAfetfu+RCEvjvZkA9RimPbh7bhFjGVBU=
+golang.org/x/crypto v0.46.0/go.mod h1:Evb/oLKmMraqjZ2iQTwDwvCtJkczlDuTmdJXoZVzqU0=
 golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
 golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
-golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
+golang.org/x/sys v0.39.0 h1:CvCKL8MeisomCi6qNZ+wbb0DN9E5AATixKsvNtMoMFk=
+golang.org/x/sys v0.39.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
+golang.org/x/text v0.32.0 h1:ZD01bjUt1FQ9WJ0ClOL5vxgxOI/sVCNgX1YtKwcY0mU=
+golang.org/x/text v0.32.0/go.mod h1:o/rUWzghvpD5TXrTIBuJU77MTaN0ljMWE47kxGJQ7jY=
 golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
 golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7Iw=

HadesLogManager/log_subscriber.go

@@ -37,6 +37,7 @@ type DynamicLogManager struct {
 type watcherState struct {
 	ctx    context.Context
 	cancel context.CancelFunc
+	wg     *sync.WaitGroup
 }
 
 // NewDynamicLogManager creates a new DynamicLogManager instance with the provided dependencies.
@@ -169,23 +170,28 @@ func (dlm *DynamicLogManager) cleanupSubscriptions(subs []*nats.Subscription) {
 func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID string) {
 	// Create new context for this job outside the lock
 	jobCtx, cancel := context.WithCancel(ctx)
+	wg := &sync.WaitGroup{}
+	wg.Add(1)
 
 	// Minimize critical section - only lock for map operations
 	dlm.mu.Lock()
 	oldWatcher, exists := dlm.watchers[jobID]
 	dlm.watchers[jobID] = watcherState{
 		ctx:    jobCtx,
 		cancel: cancel,
+		wg:     wg,
 	}
 	dlm.mu.Unlock()
 
 	// Cancel old watcher outside the lock to avoid potential deadlock
 	if exists {
 		oldWatcher.cancel()
+		oldWatcher.wg.Wait()
 	}
 
 	// Start watching logs for this job
 	go func() {
+		defer wg.Done()
 		defer func() {
 			// Use a more efficient cleanup check
 			dlm.mu.Lock()
@@ -223,13 +229,20 @@ func (dlm *DynamicLogManager) startWatchingJobLogs(ctx context.Context, jobID st
 //   - jobID: Unique identifier of the job to stop watching logs for
 func (dlm *DynamicLogManager) stopWatchingJobLogs(jobID string) {
 	dlm.mu.Lock()
-	defer dlm.mu.Unlock()
-
-	if watcher, exists := dlm.watchers[jobID]; exists {
+	watcher, exists := dlm.watchers[jobID]
+	if exists {
 		delete(dlm.watchers, jobID)
+	}
+	dlm.mu.Unlock()
+
+	if exists {
 		slog.Info("Stopping log watch", "job_id", jobID)
 		watcher.cancel()
+		watcher.wg.Wait() // Wait outside the lock
 
 		dlm.logAggregator.MarkJobCompleted(jobID)
+		if err := dlm.logAggregator.SendJobLogs(jobID); err != nil {
+			slog.Error("Failed to send job logs", "job_id", jobID, "error", err)
+		}
 	}
 }

HadesLogManager/main.go

@@ -26,7 +26,7 @@ const (
 // HadesLogManagerConfig holds the configuration for the log manager
 type HadesLogManagerConfig struct {
 	NatsConfig hadesnats.ConnectionConfig
-	APIPort    string `env:"API_PORT" envDefault:"8081"`
+	APIPort    string `env:"HADESLOGMANAGER_API_PORT" envDefault:"8081"`
 }
 
 func main() {
@@ -87,12 +87,7 @@ func connectNATS(config hadesnats.ConnectionConfig) (*nats.Conn, error) {
 }
 
 // runWithGracefulShutdown starts services and handles graceful shutdown
-func runWithGracefulShutdown(
-	ctx context.Context,
-	cancel context.CancelFunc,
-	cfg HadesLogManagerConfig,
-	dynamicManager LogManager,
-	logAggregator LogAggregator,
+func runWithGracefulShutdown(ctx context.Context, cancel context.CancelFunc, cfg HadesLogManagerConfig, dynamicManager LogManager, logAggregator LogAggregator,
 ) error {
 	var wg sync.WaitGroup
 	errChan := make(chan error, 2)
@@ -135,12 +130,7 @@ func runWithGracefulShutdown(
 }
 
 // waitForShutdown waits for OS signal or error and performs graceful shutdown
-func waitForShutdown(
-	ctx context.Context,
-	cancel context.CancelFunc,
-	server *http.Server,
-	wg *sync.WaitGroup,
-	errChan chan error,
+func waitForShutdown(ctx context.Context, cancel context.CancelFunc, server *http.Server, wg *sync.WaitGroup, errChan chan error,
 ) error {
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
@@ -179,6 +169,13 @@ func waitForShutdown(
 	return shutdownErr
 }
 
+// setupAPIRoute creates and configures the Gin router with all log manager endpoints.
+// It registers the following routes:
+//
+//   - GET /jobs/:jobId/logs   — returns all aggregated log entries for the given job ID. (Used for testing purposes)
+//   - GET /jobs/:jobId/status — returns the current build status for the given job ID, or 404 if the job is not found.
+//   - GET /jobs               — returns a list of all known job IDs (active and completed).
+//   - GET /health             — liveness probe returning a static OK response.
 func setupAPIRoute(aggregator LogAggregator) *gin.Engine {
 	r := gin.Default()
 	jobs := r.Group("/jobs")