feat: service separation

.vscode/settings.json

@@ -0,0 +1,7 @@
+{
+  "protoc": {
+    "options": [
+      "--proto_path=${workspaceRoot}/packages/grpc/proto"
+    ]
+  }
+}

README.md

@@ -38,7 +38,7 @@ The objective of this project is to provide nothing more than a basic interface
 - `DATABASE_URL` Postgres connection URL. Example: `postgresql://user:pass@host/db`
 - `JOBBER_NAME` The name of your jobber instance, should be unique per host.
 - `MANAGER_PORT` Port that runner-manager server operates on. Default: 5211
-- `MANAGER_HOST` Host that runner-manager server operates on. Default: hostname()
+- `MANAGER_GRPC_HOST` Host that runner-manager server operates on. Default: hostname()
 - `STARTUP_USERNAME` The administrator account username. Created at every startup. Has full permissions. If you change this after a previous start, it will create a NEW account, not update the previous account.
 - `STARTUP_PASSWORD` The administrator account password.
 <hr>

docker-compose.yaml

@@ -20,6 +20,7 @@ services:
       - "jobber-db"
     volumes:
       - /var/run/docker.sock:/var/run/docker.sock
+      - /tmp/jobber-env:/tmp/jobber-env
       - jobber-data:/app/config
     environment:
       DATABASE_URL: postgresql://pg-user:pg-pass@jobber-db/database

docker/gateway.Dockerfile

@@ -0,0 +1,22 @@
+FROM node:24-slim AS base
+ENV PNPM_HOME="/pnpm"
+ENV PATH="$PNPM_HOME:$PATH"
+WORKDIR /app
+RUN corepack enable && corepack prepare pnpm@10.15.1 --activate
+
+
+FROM base AS build
+COPY . /repo
+WORKDIR /repo
+RUN apt update \
+  && apt install protobuf-compiler --no-install-recommends -y \
+  && pnpm install --frozen-lockfile \
+  && pnpm run -r build \
+  && pnpm --prod --filter=@jobber/gateway --node-linker hoisted deploy /app
+
+
+
+FROM base
+WORKDIR /app
+COPY --from=build /app /app
+ENTRYPOINT ["node", "./dist/index.js"]

docker/node-20.Dockerfile

@@ -13,12 +13,14 @@ RUN apt update \
 FROM base AS build
 COPY . /repo
 WORKDIR /repo
-RUN pnpm install --frozen-lockfile \
+RUN apt update \
+  && apt install protobuf-compiler --no-install-recommends -y \
+  && pnpm install --frozen-lockfile \
   && pnpm run -r build \
   && pnpm --prod --filter=@jobber/runner-node-entrypoint --node-linker hoisted deploy /app
 
 
 
 FROM base
 WORKDIR /app
-COPY --from=build /app/dist/index.js /app/jobber-entrypoint.js
+COPY --from=build /app/dist/esm/ /app/

docker/node-22.Dockerfile

@@ -13,12 +13,14 @@ RUN apt update \
 FROM base AS build
 COPY . /repo
 WORKDIR /repo
-RUN pnpm install --frozen-lockfile \
+RUN apt update \
+  && apt install protobuf-compiler --no-install-recommends -y \
+  && pnpm install --frozen-lockfile \
   && pnpm run -r build \
   && pnpm --prod --filter=@jobber/runner-node-entrypoint --node-linker hoisted deploy /app
 
 
 
 FROM base
 WORKDIR /app
-COPY --from=build /app/dist/index.js /app/jobber-entrypoint.js
+COPY --from=build /app/dist/esm/ /app/

docker/node-24.Dockerfile

@@ -13,12 +13,14 @@ RUN apt update \
 FROM base AS build
 COPY . /repo
 WORKDIR /repo
-RUN pnpm install --frozen-lockfile \
+RUN apt update \
+  && apt install protobuf-compiler --no-install-recommends -y \
+  && pnpm install --frozen-lockfile \
   && pnpm run -r build \
   && pnpm --prod --filter=@jobber/runner-node-entrypoint --node-linker hoisted deploy /app
 
 
 
 FROM base
 WORKDIR /app
-COPY --from=build /app/dist/index.js /app/jobber-entrypoint.js
+COPY --from=build /app/dist/esm/ /app/

docker/server.Dockerfile

@@ -27,7 +27,9 @@ FROM base AS build
 COPY . /repo
 WORKDIR /repo
 
-RUN pnpm install --frozen-lockfile \
+RUN apt update \
+  && apt install protobuf-compiler --no-install-recommends -y \
+  && pnpm install --frozen-lockfile \
   && pnpm run -r build \
   && pnpm --prod --filter=@jobber/server --node-linker hoisted deploy /app \
   && mkdir /app/public/ \

docs/environment-variables.md

@@ -28,7 +28,7 @@ This document describes all environment variables used to configure Jobber.
 - `MANAGER_PORT` - Port for the runner-manager server  
   **Default:** `5211`
 
-- `MANAGER_HOST` - Host address for the runner-manager server  
+- `MANAGER_GRPC_HOST` - Host address for the runner-manager server  
   **Default:** `hostname()`
 
 ## Authentication
@@ -58,7 +58,7 @@ This document describes all environment variables used to configure Jobber.
   **Default:** eithan1231/runner-node-20:latest
 
 - `RUNNER_CONTAINER_DOCKER_NETWORK` - Docker network for runner containers  
-  **Note:** Must have access to `MANAGER_HOST`
+  **Note:** Must have access to `MANAGER_GRPC_HOST`
 
 - `RUNNER_ALLOW_DOCKER_ARGUMENT_TYPES` - Permitted Docker argument types for projects  
   **Values:** `volumes`, `networks`, `labels`, `memoryLimit`, `directPassthroughArguments`  

docs/permissions.md

@@ -28,7 +28,7 @@ Example resource pattern: `job/*/actions`
 
 #### Job -> Store
 
-- `job/:jobId/store/:storeId` READ/DELETE
+- `job/:jobId/store` READ/WRITE/DELETE
 
 #### Job -> Triggers
 

e2e/config/nanomq.conf

@@ -0,0 +1,3 @@
+listeners.tcp {
+  bind = "0.0.0.0:1883"
+}

e2e/docker-compose.yaml

@@ -0,0 +1,106 @@
+networks:
+  internal:
+    driver: bridge
+  runner:
+    driver: bridge
+  mqtt-network:
+    driver: bridge
+
+name: "jobber-test"
+
+services:
+  postgres:
+    image: postgres:14
+    restart: unless-stopped
+    networks:
+      - internal
+    environment:
+      - POSTGRES_USER=jobber-username
+      - POSTGRES_PASSWORD=jobber-password
+      - POSTGRES_DB=jobber-database
+
+  mqtt:
+    image: emqx/nanomq:0.24.6-slim
+    restart: unless-stopped
+    networks:
+      - mqtt-network
+    volumes:
+      - ./config/nanomq.conf:/etc/nanomq.conf:ro
+
+  server:
+    depends_on:
+      - postgres
+      - mqtt
+    labels:
+      - "jobber-discovery=server"
+    image: jobber-e2e-server
+    build:
+      context: ../
+      dockerfile: docker/server.Dockerfile
+    restart: unless-stopped
+    ports:
+      - 5000:5000
+    networks:
+      - internal
+      - runner
+      - mqtt-network
+    volumes:
+      - /tmp/jobber-env:/tmp/jobber-env
+      - /var/run/docker.sock:/var/run/docker.sock
+    environment:
+      SECRET_PASSPHRASE: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+
+      JOBBER_NAME: "JobberE2E"
+
+      ALLOWED_HOSTS: "localhost,localhost:5000,127.0.0.1:5000,server:5000"
+
+      DATABASE_URL: "postgresql://jobber-username:jobber-password@postgres:5432/jobber-database"
+
+      ALLOW_PUBLIC_REGISTRATION: "true"
+      AUTH_PUBLIC_LOGIN_ENABLED: "true"
+
+      OAUTH_ISSUER: "http://localhost:5000"
+
+      DEBUG_HTTP: "true"
+      DEBUG_RUNNER: "true"
+
+      API_PORT: "5000"
+
+      MANAGER_GRPC_PORT: "5001"
+      MANAGER_GRPC_HOST: "server"
+
+      # "{compose_name}_{network_name}" is the convention for compose networks
+      RUNNER_CONTAINER_DOCKER_NETWORK: "jobber-test_runner"
+      RUNNER_ALLOW_DOCKER_ARGUMENT_TYPES: "volumes,networks,labels,memoryLimit,directPassthroughArguments"
+      RUNNER_ALLOW_ARGUMENT_DIRECT_PASSTHROUGH: "true"
+
+      RUNNER_IMAGE_NODE20_URL: "jobber-e2e-runner:20-latest"
+      RUNNER_IMAGE_NODE22_URL: "jobber-e2e-runner:22-latest"
+      RUNNER_IMAGE_NODE24_URL: "jobber-e2e-runner:24-latest"
+
+      # Seed data required for e2e tests
+      # NOTE: This will be INSECURE! Do not use this format for production!
+      SEED: '{"oauth-clients": {"clientId": "e2e-client", "clientSecret": "secret-secret-secret-secret-secret"}, "api-tokens": [{"token": "super-power-anonymous-token", "permissions": { "type": "all" }}]}'
+
+  gateway:
+    depends_on:
+      - server
+    image: jobber-e2e-gateway
+    build:
+      context: ../
+      dockerfile: docker/gateway.Dockerfile
+    restart: unless-stopped
+    networks:
+      - runner
+    environment:
+      PORT: "5002"
+
+      GRPC_ENDPOINT: "https://server:5001"
+
+      OIDC_ISSUER_URL: "http://localhost:5000"
+      OIDC_DISCOVERY_URL: "http://server:5000/.well-known/openid-configuration"
+
+      OAUTH_CLIENT_ID: "e2e-client"
+      OAUTH_CLIENT_SECRET: "secret-secret-secret-secret-secret"
+    ports:
+      - 5002:5002

e2e/test.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+
+sudo docker build -f docker/node-20.Dockerfile -t jobber-e2e-runner:20-latest .
+sudo docker build -f docker/node-22.Dockerfile -t jobber-e2e-runner:22-latest .
+sudo docker build -f docker/node-24.Dockerfile -t jobber-e2e-runner:24-latest .
+
+docker compose -f e2e/docker-compose.yaml up -d --build
+
+# sleep a few seconds to allow processes to startup
+sleep 5
+
+# Run tests
+bash e2e/tests/test-runner-basics.sh
+bash e2e/tests/test-common-js.sh
+bash e2e/tests/test-run-once.sh
+
+docker compose -f e2e/docker-compose.yaml down

e2e/tests/test-common-js.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd examples/http-javascript-cjs
+./publish.sh super-power-anonymous-token http://localhost:5000
+cd ../../
+
+sleep 3
+
+RESPONSE=$(curl -s "http://localhost:5002/http-javascript-cjs")
+if echo "$RESPONSE" | grep -q "path.join example from commonjs"; then
+  echo "PASS: Successfully received expected response from http-javascript-cjs job"
+else
+  echo "FAIL: Failed to receive expected response from http-javascript-cjs job"
+  echo "Actual response: $RESPONSE"
+  exit 1
+fi

e2e/tests/test-run-once.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+cd examples/http-javascript-run-once
+./publish.sh super-power-anonymous-token http://localhost:5000
+cd ../../
+
+sleep 3
+
+RESPONSE=$(curl -s "http://localhost:5002/http-javascript-run-once")
+if echo "$RESPONSE" | grep -q "run-once-response"; then
+  echo "PASS: Successfully received expected response from http-javascript-run-once job"
+else
+  echo "FAIL: Failed to receive expected response from http-javascript-run-once job"
+  echo "Actual response: $RESPONSE"
+  exit 1
+fi

e2e/tests/test-runner-basics.sh

@@ -0,0 +1,65 @@
+#!/bin/bash
+
+cd examples/e2e
+./publish.sh super-power-anonymous-token http://localhost:5000
+cd ../../
+
+sleep 3
+
+curl -s "http://localhost:5002/e2e?action=set-state&value=my-test-value" > /dev/null
+GET_STORE_RESPONSE=$(curl -s "http://localhost:5002/e2e?action=get-state")
+if echo "$GET_STORE_RESPONSE" | grep -q "my-test-value"; then
+  echo "PASS: Successfully set and got state value"
+else
+  echo "FAIL: Failed to set and get state value"
+  echo "Actual response: $GET_STORE_RESPONSE"
+  exit 1
+fi
+
+
+sleep 1
+
+curl -s "http://localhost:5002/e2e?action=mqtt" > /dev/null
+
+sleep 1
+
+
+RESPONSE=$(curl -s "http://localhost:5002/e2e")
+
+# Check that bootstrap is true
+
+if echo "$RESPONSE" | grep -q '"bootstrap":true'; then
+  echo "PASS: Bootstrap is true"
+else
+  echo "FAIL: Bootstrap is not true"
+  echo "Actual response: $RESPONSE"
+  exit 1
+fi
+
+if echo "$RESPONSE" | grep -q '"lastScheduleRecent":true'; then
+  echo "PASS: Schedule Recent is true"
+else
+  echo "FAIL: Schedule Recent is not true"
+  echo "Actual response: $RESPONSE"
+  exit 1
+fi
+
+
+if echo "$RESPONSE" | grep -q '"lastMqttRecent":true'; then
+  echo "PASS: MQTT Recent is true"
+else
+  echo "FAIL: MQTT Recent is not true"
+  echo "Actual response: $RESPONSE"
+  exit 1
+fi
+
+
+# hang response status code should be 204
+HANG_RESPONSE=$(curl -s "http://localhost:5002/e2e?action=hang")
+if [ -z "$HANG_RESPONSE" ]; then
+  echo "PASS: Hang response is empty as expected"
+else
+  echo "FAIL: Hang response is not empty"
+  echo "Actual response: $HANG_RESPONSE"
+  exit 1
+fi