From 100f20ed51ddb465dcce58b65daf238759974f49 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Mon, 5 Jan 2026 20:10:25 -0300 Subject: [PATCH 01/15] fix: added Math.min to the dispatcher and removed redundant checks --- .../core/src/transitions/snooze-transition.ts | 4 +- .../engine/src/execution/dispatcher.test.ts | 44 ++++++++++++ packages/engine/src/execution/dispatcher.ts | 6 +- .../src/execution/executor-manager.test.ts | 40 ----------- .../engine/src/execution/executor-manager.ts | 71 +++++++++---------- 5 files changed, 85 insertions(+), 80 deletions(-) diff --git a/packages/core/src/transitions/snooze-transition.ts b/packages/core/src/transitions/snooze-transition.ts index 639a03fb..8bcd5c54 100644 --- a/packages/core/src/transitions/snooze-transition.ts +++ b/packages/core/src/transitions/snooze-transition.ts @@ -10,7 +10,7 @@ import { JobTransition } from "./transition"; * If the job is currently running, it will decrement the attempt count. * This allows the job to be retried after the delay. * - * Only jobs in "waiting" or "claimed" or "running" state can be snoozed. + * Only jobs in "waiting" or "running" state can be snoozed. */ export class SnoozeTransition extends JobTransition { /** The delay in milliseconds. */ @@ -40,6 +40,6 @@ export class SnoozeTransition extends JobTransition { } shouldRun(job: JobData): boolean { - return ["waiting", "claimed", "running"].includes(job.state); + return ["waiting", "running"].includes(job.state); } } diff --git a/packages/engine/src/execution/dispatcher.test.ts b/packages/engine/src/execution/dispatcher.test.ts index c13276dd..ff1a9aea 100644 --- a/packages/engine/src/execution/dispatcher.test.ts +++ b/packages/engine/src/execution/dispatcher.test.ts @@ -120,5 +120,49 @@ describe("Dispatcher", () => { await dispatcher.stop(); }); + + sidequestTest( + "claims min(availableSlots, globalSlots) jobs when queue has more slots than global", + async ({ backend }) => { + // Setup: Queue has concurrency of 10, but global has only 3 slots + const configWithHighQueueConcurrency: EngineConfig = { + backend: { driver: "@sidequest/sqlite-backend" }, + queues: [{ name: "default", concurrency: 10 }], + maxConcurrentJobs: 3, + }; + + // Create 5 jobs to ensure there are enough jobs to claim + await createJob(backend, "default"); + await createJob(backend, "default"); + await createJob(backend, "default"); + await createJob(backend, "default"); + + expect(await backend.listJobs({ state: "waiting" })).toHaveLength(5); + + const mockClaim = vi.spyOn(backend, "claimPendingJob"); + + const dispatcher = new Dispatcher( + backend, + new QueueManager(backend, configWithHighQueueConcurrency.queues!), + new ExecutorManager(backend, configWithHighQueueConcurrency as NonNullableEngineConfig), + 100, + ); + dispatcher.start(); + + runMock.mockImplementation(() => { + return { type: "completed", result: "foo", __is_job_transition__: true } as CompletedResult; + }); + + // Wait for the first claim to happen + await vi.waitUntil(() => mockClaim.mock.calls.length > 0); + + // Verify that claimPendingJob was called with Math.min(availableSlots, globalSlots) + // Queue has 10 slots available, global has 3 slots available + // So it should claim min(10, 3) = 3 jobs + expect(mockClaim).toHaveBeenCalledWith("default", 3); + + await dispatcher.stop(); + }, + ); }); }); diff --git a/packages/engine/src/execution/dispatcher.ts b/packages/engine/src/execution/dispatcher.ts index f364c371..493fe82c 100644 --- a/packages/engine/src/execution/dispatcher.ts +++ b/packages/engine/src/execution/dispatcher.ts @@ -48,7 +48,7 @@ export class Dispatcher { continue; } - const jobs: JobData[] = await this.backend.claimPendingJob(queue.name, availableSlots); + const jobs: JobData[] = await this.backend.claimPendingJob(queue.name, Math.min(availableSlots, globalSlots)); if (jobs.length > 0) { // if a job was found on any queue do not sleep @@ -56,6 +56,10 @@ export class Dispatcher { } for (const job of jobs) { + // adds jobs to active sets before execution to avoid race conditions + // because the execution is not awaited. This way we ensure that available slots + // are correctly calculated. + this.executorManager.queueJob(queue, job); // does not await for job execution. void this.executorManager.execute(queue, job); } diff --git a/packages/engine/src/execution/executor-manager.test.ts b/packages/engine/src/execution/executor-manager.test.ts index b0168384..f3cf691c 100644 --- a/packages/engine/src/execution/executor-manager.test.ts +++ b/packages/engine/src/execution/executor-manager.test.ts @@ -171,46 +171,6 @@ describe("ExecutorManager", () => { await executorManager.destroy(); }); - - sidequestTest("snoozes job when queue is full", async ({ backend, config }) => { - const queryConfig = await grantQueueConfig(backend, { name: "default", concurrency: 1 }); - const executorManager = new ExecutorManager(backend, config); - - vi.spyOn(executorManager, "availableSlotsByQueue").mockReturnValue(0); - - // Set up job in claimed state (as it would be when passed to execute) - jobData = await backend.updateJob({ ...jobData, state: "claimed", claimed_at: new Date() }); - - await executorManager.execute(queryConfig, jobData); - - // Verify the job runner was NOT called since the job was snoozed - expect(runMock).not.toHaveBeenCalled(); - - // Verify slots remain unchanged (no job was actually executed) - expect(executorManager.availableSlotsByQueue(queryConfig)).toEqual(0); - expect(executorManager.totalActiveWorkers()).toEqual(0); - await executorManager.destroy(); - }); - - sidequestTest("snoozes job when global slots are full", async ({ backend, config }) => { - const queryConfig = await grantQueueConfig(backend, { name: "default", concurrency: 5 }); - const executorManager = new ExecutorManager(backend, { ...config, maxConcurrentJobs: 1 }); - - vi.spyOn(executorManager, "availableSlotsGlobal").mockReturnValue(0); - - // Set up job in claimed state - jobData = await backend.updateJob({ ...jobData, state: "claimed", claimed_at: new Date() }); - - await executorManager.execute(queryConfig, jobData); - - // Verify the job runner was NOT called - expect(runMock).not.toHaveBeenCalled(); - - // Verify global slots show as full - expect(executorManager.availableSlotsGlobal()).toEqual(0); - expect(executorManager.totalActiveWorkers()).toEqual(0); - await executorManager.destroy(); - }); }); describe("availableSlotsByQueue", () => { diff --git a/packages/engine/src/execution/executor-manager.ts b/packages/engine/src/execution/executor-manager.ts index 0c7553d7..f97b357d 100644 --- a/packages/engine/src/execution/executor-manager.ts +++ b/packages/engine/src/execution/executor-manager.ts @@ -1,13 +1,5 @@ import { Backend } from "@sidequest/backend"; -import { - JobData, - JobTransitionFactory, - logger, - QueueConfig, - RetryTransition, - RunTransition, - SnoozeTransition, -} from "@sidequest/core"; +import { JobData, JobTransitionFactory, logger, QueueConfig, RetryTransition, RunTransition } from "@sidequest/core"; import EventEmitter from "events"; import { inspect } from "util"; import { NonNullableEngineConfig } from "../engine"; @@ -77,44 +69,49 @@ export class ExecutorManager { } /** - * Executes a job in the given queue. + * Prepares a job for execution by marking it as active and adding it to a queue slot. * @param queueConfig The queue configuration. - * @param job The job data to execute. + * @param job The job data. */ - async execute(queueConfig: QueueConfig, job: JobData): Promise { - logger("Executor Manager").debug(`Submitting job ${job.id} for execution in queue ${queueConfig.name}`); + queueJob(queueConfig: QueueConfig, job: JobData) { if (!this.activeByQueue[queueConfig.name]) { this.activeByQueue[queueConfig.name] = new Set(); } - - if (this.availableSlotsByQueue(queueConfig) <= 0 || this.availableSlotsGlobal() <= 0) { - logger("Executor Manager").debug(`No available slots for job ${job.id} in queue ${queueConfig.name}`); - await JobTransitioner.apply(this.backend, job, new SnoozeTransition(0)); - return; - } - this.activeByQueue[queueConfig.name].add(job.id); this.activeJobs.add(job.id); + } - job = await JobTransitioner.apply(this.backend, job, new RunTransition()); - - const signal = new EventEmitter(); - let isRunning = true; - const cancellationCheck = async () => { - while (isRunning) { - const watchedJob = await this.backend.getJob(job.id); - if (watchedJob!.state === "canceled") { - logger("Executor Manager").debug(`Emitting abort signal for job ${job.id}`); - signal.emit("abort"); - isRunning = false; - return; + /** + * Executes a job in the given queue. + * @param queueConfig The queue configuration. + * @param job The job data to execute. + */ + async execute(queueConfig: QueueConfig, job: JobData): Promise { + let isRunning = false; + try { + logger("Executor Manager").debug(`Submitting job ${job.id} for execution in queue ${queueConfig.name}`); + // We call prepareJob here again to make sure the jobs are in the queues. + // This might not be necessary, but for the sake of consistency we do it. + this.queueJob(queueConfig, job); + + job = await JobTransitioner.apply(this.backend, job, new RunTransition()); + + isRunning = true; + const signal = new EventEmitter(); + const cancellationCheck = async () => { + while (isRunning) { + const watchedJob = await this.backend.getJob(job.id); + if (watchedJob!.state === "canceled") { + logger("Executor Manager").debug(`Emitting abort signal for job ${job.id}`); + signal.emit("abort"); + isRunning = false; + return; + } + await new Promise((r) => setTimeout(r, 1000)); } - await new Promise((r) => setTimeout(r, 1000)); - } - }; - void cancellationCheck(); + }; + void cancellationCheck(); - try { logger("Executor Manager").debug(`Running job ${job.id} in queue ${queueConfig.name}`); const runPromise = this.runnerPool.run(job, signal); From b56f57e340616d65312b96d925d16e11f62025e5 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Mon, 5 Jan 2026 20:13:52 -0300 Subject: [PATCH 02/15] fix: update test to create 4 jobs for claiming logic --- packages/engine/src/execution/dispatcher.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine/src/execution/dispatcher.test.ts b/packages/engine/src/execution/dispatcher.test.ts index ff1a9aea..528cd9f0 100644 --- a/packages/engine/src/execution/dispatcher.test.ts +++ b/packages/engine/src/execution/dispatcher.test.ts @@ -131,7 +131,7 @@ describe("Dispatcher", () => { maxConcurrentJobs: 3, }; - // Create 5 jobs to ensure there are enough jobs to claim + // Create 4 jobs to ensure there are enough jobs to claim await createJob(backend, "default"); await createJob(backend, "default"); await createJob(backend, "default"); From 9aef0ceaf4cc96906c07b9cfb79efc74b0e297d1 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Tue, 6 Jan 2026 11:31:37 -0300 Subject: [PATCH 03/15] fix: refactor CI workflow to use setup action and streamline job steps --- .github/actions/setup/action.yml | 31 ++++++++++++ .github/workflows/pull-request.yml | 76 +++++++++++++++++++++--------- 2 files changed, 84 insertions(+), 23 deletions(-) create mode 100644 .github/actions/setup/action.yml diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml new file mode 100644 index 00000000..936e5522 --- /dev/null +++ b/.github/actions/setup/action.yml @@ -0,0 +1,31 @@ +name: Setup Project +description: Checkout code, setup Node.js, restore build cache, and setup Yarn + +inputs: + node-version: + description: 'Node.js version to use' + required: false + default: '22' + +runs: + using: composite + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Node.js + uses: actions/setup-node@v4 + with: + node-version: ${{ inputs.node-version }} + + - name: Restore build outputs + uses: actions/cache@v4 + with: + path: | + **/dist + key: ${{ runner.os }}-build-${{ hashFiles('**/yarn.lock') }}-${{ github.sha }} + + - name: Setup Latest Yarn + uses: threeal/setup-yarn-action@v2.0.0 + with: + version: berry diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index ba824784..3ca74ae1 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -15,9 +15,38 @@ concurrency: cancel-in-progress: true jobs: - build-and-test: + build: runs-on: ubuntu-latest + steps: + - name: Setup project + uses: ./.github/actions/setup + + - name: Build project + run: yarn build + + format-check: + runs-on: ubuntu-latest + needs: build + steps: + - name: Setup project + uses: ./.github/actions/setup + + - name: Run format checker + run: yarn format:check + lint: + runs-on: ubuntu-latest + needs: build + steps: + - name: Setup project + uses: ./.github/actions/setup + + - name: Run linter + run: yarn lint + + test: + runs-on: ubuntu-latest + needs: build services: postgres: image: postgres:latest @@ -56,28 +85,8 @@ jobs: --health-timeout=5s --health-retries=5 steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Set up Node.js - uses: actions/setup-node@v4 - with: - node-version: "22" - - - name: Setup Latest Yarn - uses: threeal/setup-yarn-action@v2.0.0 - with: - version: berry - cache: false - - - name: Build project - run: yarn build - - - name: Run format checker - run: yarn format:check - - - name: Run linter - run: yarn lint + - name: Setup project + uses: ./.github/actions/setup - name: Run tests env: @@ -86,6 +95,27 @@ jobs: MONGODB_URL: mongodb://127.0.0.1:27017/test run: yarn test:all:ci + integration-test: + runs-on: ubuntu-latest + needs: build + services: + postgres: + image: postgres:latest + ports: + - 5432:5432 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=10s + --health-timeout=5s + --health-retries=5 + steps: + - name: Setup project + uses: ./.github/actions/setup + - name: Run integration tests env: POSTGRES_URL: postgresql://postgres:postgres@localhost:5432/postgres From 0000e2a4babbf7ad13000ec763a543b542dda295 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Tue, 6 Jan 2026 11:34:16 -0300 Subject: [PATCH 04/15] fix: add checkout step to pull request workflow and update action description --- .github/actions/setup/action.yml | 5 +---- .github/workflows/pull-request.yml | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 936e5522..7fcfab5f 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -1,5 +1,5 @@ name: Setup Project -description: Checkout code, setup Node.js, restore build cache, and setup Yarn +description: Setup Node.js, restore build cache, and setup Yarn inputs: node-version: @@ -10,9 +10,6 @@ inputs: runs: using: composite steps: - - name: Checkout repository - uses: actions/checkout@v4 - - name: Set up Node.js uses: actions/setup-node@v4 with: diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index 3ca74ae1..99f5d27a 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -18,6 +18,9 @@ jobs: build: runs-on: ubuntu-latest steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup project uses: ./.github/actions/setup @@ -28,6 +31,9 @@ jobs: runs-on: ubuntu-latest needs: build steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup project uses: ./.github/actions/setup @@ -38,6 +44,9 @@ jobs: runs-on: ubuntu-latest needs: build steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup project uses: ./.github/actions/setup @@ -85,6 +94,9 @@ jobs: --health-timeout=5s --health-retries=5 steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup project uses: ./.github/actions/setup @@ -113,6 +125,9 @@ jobs: --health-timeout=5s --health-retries=5 steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Setup project uses: ./.github/actions/setup From 32f0416b3f6c54c448bcdce2815f6915fa55692f Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Tue, 6 Jan 2026 11:37:54 -0300 Subject: [PATCH 05/15] fix: add missing cache paths for Yarn and Node.js modules --- .github/actions/setup/action.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7fcfab5f..a0b1b81b 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -19,6 +19,9 @@ runs: uses: actions/cache@v4 with: path: | + .yarn + node_modules + **/node_modules **/dist key: ${{ runner.os }}-build-${{ hashFiles('**/yarn.lock') }}-${{ github.sha }} From d761869038ce17064668c50145bddaa74da5de5b Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:08:36 -0300 Subject: [PATCH 06/15] fix: implement dependency registry for managing engine dependencies --- packages/engine/src/dependency-registry.ts | 72 ++++++++++++++++++ packages/engine/src/engine.ts | 86 ++++++++++------------ tests/fixture.ts | 2 + 3 files changed, 113 insertions(+), 47 deletions(-) create mode 100644 packages/engine/src/dependency-registry.ts diff --git a/packages/engine/src/dependency-registry.ts b/packages/engine/src/dependency-registry.ts new file mode 100644 index 00000000..f1b2a98b --- /dev/null +++ b/packages/engine/src/dependency-registry.ts @@ -0,0 +1,72 @@ +import { Backend } from "@sidequest/backend"; +import { NonNullableEngineConfig } from "../dist"; + +/** + * Enumeration of available dependency tokens for the dependency registry. + * Used as keys to register and retrieve dependencies throughout the engine. + */ +export enum Dependency { + /** Engine configuration */ + Config = "config", + /** Backend instance */ + Backend = "backend", +} + +/** + * Type mapping interface that associates each dependency token with its corresponding type. + * This ensures type safety when registering and retrieving dependencies. + */ +interface DependencyRegistryTypes { + [Dependency.Config]: NonNullableEngineConfig; + [Dependency.Backend]: Backend; +} + +/** + * Union type of all valid dependency registry keys. + */ +type DependencyRegistryKey = keyof DependencyRegistryTypes; + +/** + * A type-safe dependency injection container for managing core engine dependencies. + * Provides methods to register, retrieve, and clear dependencies used throughout the engine lifecycle. + */ +class DependencyRegistry { + /** + * Internal storage for registered dependencies. + */ + private registry = new Map(); + + /** + * Retrieves a registered dependency by its token. + * @param token - The dependency token to look up + * @returns The registered dependency instance, or undefined if not found + */ + get(token: T): DependencyRegistryTypes[T] | undefined { + return this.registry.get(token) as DependencyRegistryTypes[T] | undefined; + } + + /** + * Registers a dependency instance with the specified token. + * @param token - The dependency token to register under + * @param instance - The dependency instance to register + * @returns The registered instance + */ + register(token: T, instance: DependencyRegistryTypes[T]) { + this.registry.set(token, instance); + return instance; + } + + /** + * Clears all registered dependencies from the registry. + * Useful for cleanup and testing scenarios. + */ + clear() { + this.registry.clear(); + } +} + +/** + * Singleton instance of the dependency registry. + * This is the main container used throughout the engine to manage dependencies. + */ +export const dependencyRegistry = new DependencyRegistry(); diff --git a/packages/engine/src/engine.ts b/packages/engine/src/engine.ts index 6338e099..bebfd120 100644 --- a/packages/engine/src/engine.ts +++ b/packages/engine/src/engine.ts @@ -1,4 +1,4 @@ -import { Backend, BackendConfig, LazyBackend, MISC_FALLBACK, NewQueueData, QUEUE_FALLBACK } from "@sidequest/backend"; +import { BackendConfig, LazyBackend, MISC_FALLBACK, NewQueueData, QUEUE_FALLBACK } from "@sidequest/backend"; import { configureLogger, JobClassType, logger, LoggerOptions } from "@sidequest/core"; import { ChildProcess, fork } from "child_process"; import { existsSync } from "fs"; @@ -6,6 +6,7 @@ import { cpus } from "os"; import { fileURLToPath } from "url"; import { inspect } from "util"; import { DEFAULT_WORKER_PATH } from "./constants"; +import { Dependency, dependencyRegistry } from "./dependency-registry"; import { JOB_BUILDER_FALLBACK } from "./job/constants"; import { ScheduledJobRegistry } from "./job/cron-registry"; import { JobBuilder, JobBuilderDefaults } from "./job/job-builder"; @@ -116,19 +117,6 @@ export type NonNullableEngineConfig = { * The main engine for managing job queues and workers in Sidequest. */ export class Engine { - /** - * Backend instance used by the engine. - * This is initialized when the engine is configured or started. - */ - private backend?: Backend; - - /** - * Current configuration of the engine. - * This is set when the engine is configured or started. - * It contains all the necessary settings for the engine to operate, such as backend, queues, logger options, and job defaults. - */ - private config?: NonNullableEngineConfig; - /** * Main worker process that runs the Sidequest engine. * This is created when the engine is started and handles job processing. @@ -147,11 +135,11 @@ export class Engine { * @returns The resolved configuration. */ async configure(config?: EngineConfig): Promise { - if (this.config) { + if (this.getConfig()) { logger("Engine").debug("Sidequest already configured"); - return this.config; + return this.getConfig()!; } - this.config = { + const nonNullConfig: NonNullableEngineConfig = { queues: config?.queues ?? [], backend: { driver: config?.backend?.driver ?? "@sidequest/sqlite-backend", @@ -190,21 +178,22 @@ export class Engine { jobsFilePath: config?.jobsFilePath?.trim() ?? "", jobPollingInterval: config?.jobPollingInterval ?? 100, }; + dependencyRegistry.register(Dependency.Config, nonNullConfig); this.validateConfig(); - logger("Engine").debug(`Configuring Sidequest engine: ${inspect(this.config)}`); + logger("Engine").debug(`Configuring Sidequest engine: ${inspect(nonNullConfig)}`); - if (this.config.logger) { - configureLogger(this.config.logger); + if (nonNullConfig.logger) { + configureLogger(nonNullConfig.logger); } - this.backend = new LazyBackend(this.config.backend); - if (!this.config.skipMigration) { - await this.backend.migrate(); + const backend = dependencyRegistry.register(Dependency.Backend, new LazyBackend(nonNullConfig.backend)); + if (!nonNullConfig.skipMigration) { + await backend.migrate(); } - return this.config; + return nonNullConfig; } /** @@ -224,18 +213,19 @@ export class Engine { * - Logs the resolved jobs file path when using manual job resolution */ validateConfig() { - if (this.config!.maxConcurrentJobs !== undefined && this.config!.maxConcurrentJobs < 1) { + const config = this.getConfig(); + if (config!.maxConcurrentJobs !== undefined && config!.maxConcurrentJobs < 1) { throw new Error(`Invalid "maxConcurrentJobs" value: must be at least 1.`); } - if (this.config!.manualJobResolution) { - if (this.config!.jobsFilePath) { - const scriptUrl = resolveScriptPath(this.config!.jobsFilePath); + if (config!.manualJobResolution) { + if (config!.jobsFilePath) { + const scriptUrl = resolveScriptPath(config!.jobsFilePath); if (!existsSync(fileURLToPath(scriptUrl))) { throw new Error(`The specified jobsFilePath does not exist. Resolved to: ${scriptUrl}`); } - logger("Engine").info(`Using manual jobs file at: ${this.config!.jobsFilePath}`); - this.config!.jobsFilePath = scriptUrl; + logger("Engine").info(`Using manual jobs file at: ${config!.jobsFilePath}`); + config!.jobsFilePath = scriptUrl; } else { // This should throw an error if not found findSidequestJobsScriptInParentDirs(); @@ -253,13 +243,13 @@ export class Engine { return; } - await this.configure(config); + const nonNullConfig = await this.configure(config); - logger("Engine").info(`Starting Sidequest using backend ${this.config!.backend.driver}`); + logger("Engine").info(`Starting Sidequest using backend ${nonNullConfig.backend.driver}`); - if (this.config!.queues) { - for (const queue of this.config!.queues) { - await grantQueueConfig(this.backend!, queue, this.config!.queueDefaults, true); + if (nonNullConfig.queues) { + for (const queue of nonNullConfig.queues) { + await grantQueueConfig(dependencyRegistry.get(Dependency.Backend)!, queue, nonNullConfig.queueDefaults, true); } } @@ -276,7 +266,7 @@ export class Engine { this.mainWorker.on("message", (msg) => { if (msg === "ready") { logger("Engine").debug("Main worker is ready"); - this.mainWorker?.send({ type: "start", sidequestConfig: this.config! }); + this.mainWorker?.send({ type: "start", sidequestConfig: nonNullConfig }); clearTimeout(timeout); resolve(); } @@ -291,7 +281,7 @@ export class Engine { }; runWorker(); - gracefulShutdown(this.close.bind(this), "Engine", this.config!.gracefulShutdown); + gracefulShutdown(this.close.bind(this), "Engine", nonNullConfig.gracefulShutdown); } }); } @@ -301,7 +291,7 @@ export class Engine { * @returns The current configuration, if set. */ getConfig() { - return this.config; + return dependencyRegistry.get(Dependency.Config); } /** @@ -309,7 +299,7 @@ export class Engine { * @returns The backend instance, if set. */ getBackend() { - return this.backend; + return dependencyRegistry.get(Dependency.Backend); } /** @@ -331,18 +321,18 @@ export class Engine { await promise; } try { - await this.backend?.close(); + await dependencyRegistry.get(Dependency.Backend)?.close(); } catch (error) { logger("Engine").error("Error closing backend:", error); } - this.config = undefined; - this.backend = undefined; this.mainWorker = undefined; // Reset the shutting down flag after closing // This allows the engine to be reconfigured or restarted later clearGracefulShutdown(); logger("Engine").debug("Sidequest engine closed."); this.shuttingDown = false; + // Clear the dependency registry to allow fresh configuration later + dependencyRegistry.clear(); } } @@ -352,7 +342,9 @@ export class Engine { * @returns A new JobBuilder instance for the job class. */ build(JobClass: T) { - if (!this.config || !this.backend) { + const backend = this.getBackend(); + const config = this.getConfig(); + if (!config || !backend) { throw new Error("Engine not configured. Call engine.configure() or engine.start() first."); } if (this.shuttingDown) { @@ -360,15 +352,15 @@ export class Engine { } logger("Engine").debug(`Building job for class: ${JobClass.name}`); return new JobBuilder( - this.backend, + backend, JobClass, { - ...this.config.jobDefaults, + ...config.jobDefaults, // We need to do this check again because available at is a getter. It needs to be set at job creation time. // If not set, it will use the fallback value which is outdated from config. - availableAt: this.config.jobDefaults.availableAt ?? JOB_BUILDER_FALLBACK.availableAt!, + availableAt: config.jobDefaults.availableAt ?? JOB_BUILDER_FALLBACK.availableAt!, }, - this.config.manualJobResolution, + config.manualJobResolution, ); } } diff --git a/tests/fixture.ts b/tests/fixture.ts index 550d8ab5..16a2d868 100644 --- a/tests/fixture.ts +++ b/tests/fixture.ts @@ -1,6 +1,7 @@ import { test as baseTest } from "vitest"; import { Backend } from "../packages/backends/backend/src"; import { Engine, NonNullableEngineConfig } from "../packages/engine/src"; +import { dependencyRegistry } from "../packages/engine/src/dependency-registry"; export interface SidequestTestFixture { engine: Engine; @@ -29,6 +30,7 @@ export const sidequestTest = baseTest.extend({ } await engine.close(); + dependencyRegistry.clear(); }, backend: async ({ engine }, use) => { From b64b7e593abc100ce31aa8adc45b6b41fda59949 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:12:30 -0300 Subject: [PATCH 07/15] fix: update import path for NonNullableEngineConfig in dependency registry --- packages/engine/src/dependency-registry.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/engine/src/dependency-registry.ts b/packages/engine/src/dependency-registry.ts index f1b2a98b..044b1109 100644 --- a/packages/engine/src/dependency-registry.ts +++ b/packages/engine/src/dependency-registry.ts @@ -1,5 +1,5 @@ import { Backend } from "@sidequest/backend"; -import { NonNullableEngineConfig } from "../dist"; +import { NonNullableEngineConfig } from "./engine"; /** * Enumeration of available dependency tokens for the dependency registry. From b6fcf17d9ef8a6348e0123d6fc72611285187f88 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:15:20 -0300 Subject: [PATCH 08/15] refactor: simplify dependency registry key handling by removing union type --- packages/engine/src/dependency-registry.ts | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/packages/engine/src/dependency-registry.ts b/packages/engine/src/dependency-registry.ts index 044b1109..002c253d 100644 --- a/packages/engine/src/dependency-registry.ts +++ b/packages/engine/src/dependency-registry.ts @@ -21,11 +21,6 @@ interface DependencyRegistryTypes { [Dependency.Backend]: Backend; } -/** - * Union type of all valid dependency registry keys. - */ -type DependencyRegistryKey = keyof DependencyRegistryTypes; - /** * A type-safe dependency injection container for managing core engine dependencies. * Provides methods to register, retrieve, and clear dependencies used throughout the engine lifecycle. @@ -34,14 +29,14 @@ class DependencyRegistry { /** * Internal storage for registered dependencies. */ - private registry = new Map(); + private registry = new Map(); /** * Retrieves a registered dependency by its token. * @param token - The dependency token to look up * @returns The registered dependency instance, or undefined if not found */ - get(token: T): DependencyRegistryTypes[T] | undefined { + get(token: T): DependencyRegistryTypes[T] | undefined { return this.registry.get(token) as DependencyRegistryTypes[T] | undefined; } @@ -51,7 +46,7 @@ class DependencyRegistry { * @param instance - The dependency instance to register * @returns The registered instance */ - register(token: T, instance: DependencyRegistryTypes[T]) { + register(token: T, instance: DependencyRegistryTypes[T]) { this.registry.set(token, instance); return instance; } From 286ea548d4691ad1ffcca2bb9d6a52cd8439f0ec Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:24:11 -0300 Subject: [PATCH 09/15] fix: disable cache for Yarn setup action --- .github/actions/setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index a0b1b81b..7ea6d3a1 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -29,3 +29,4 @@ runs: uses: threeal/setup-yarn-action@v2.0.0 with: version: berry + cache: false From d108fc6c57a1afc27b781f475b6075a07a6e09d9 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:29:43 -0300 Subject: [PATCH 10/15] fix: add conditional check for Yarn setup based on cache hit --- .github/actions/setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7ea6d3a1..0c953754 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -26,6 +26,7 @@ runs: key: ${{ runner.os }}-build-${{ hashFiles('**/yarn.lock') }}-${{ github.sha }} - name: Setup Latest Yarn + if: steps.cache.outputs.cache-hit != 'true' uses: threeal/setup-yarn-action@v2.0.0 with: version: berry From 819f7641653a09362c5abd259d1d3b86a84c752b Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:35:20 -0300 Subject: [PATCH 11/15] fix: add id to cache step in action.yml for better reference --- .github/actions/setup/action.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 0c953754..7f6bab51 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -16,6 +16,7 @@ runs: node-version: ${{ inputs.node-version }} - name: Restore build outputs + id: cache uses: actions/cache@v4 with: path: | From d917a4be5e050b11977ae3d315d1605959e4fbe9 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:39:34 -0300 Subject: [PATCH 12/15] refactor: streamline Yarn setup by enabling Corepack and installing Yarn Berry directly --- .github/actions/setup/action.yml | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 7f6bab51..633c7757 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -15,6 +15,14 @@ runs: with: node-version: ${{ inputs.node-version }} + - name: Enable Corepack + shell: bash + run: corepack enable + + - name: Install Yarn Berry + shell: bash + run: yarn set version berry + - name: Restore build outputs id: cache uses: actions/cache@v4 @@ -26,9 +34,7 @@ runs: **/dist key: ${{ runner.os }}-build-${{ hashFiles('**/yarn.lock') }}-${{ github.sha }} - - name: Setup Latest Yarn + - name: Install Dependencies if: steps.cache.outputs.cache-hit != 'true' - uses: threeal/setup-yarn-action@v2.0.0 - with: - version: berry - cache: false + shell: bash + run: yarn install --frozen-lockfile From 835c4f76987b409bbadb4348e4f08d124f35c99c Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 09:43:22 -0300 Subject: [PATCH 13/15] refactor: reorder Corepack and Yarn Berry setup steps for improved clarity --- .github/actions/setup/action.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/actions/setup/action.yml b/.github/actions/setup/action.yml index 633c7757..89f70de8 100644 --- a/.github/actions/setup/action.yml +++ b/.github/actions/setup/action.yml @@ -15,14 +15,6 @@ runs: with: node-version: ${{ inputs.node-version }} - - name: Enable Corepack - shell: bash - run: corepack enable - - - name: Install Yarn Berry - shell: bash - run: yarn set version berry - - name: Restore build outputs id: cache uses: actions/cache@v4 @@ -34,6 +26,14 @@ runs: **/dist key: ${{ runner.os }}-build-${{ hashFiles('**/yarn.lock') }}-${{ github.sha }} + - name: Enable Corepack + shell: bash + run: corepack enable + + - name: Install Yarn Berry + shell: bash + run: yarn set version berry + - name: Install Dependencies if: steps.cache.outputs.cache-hit != 'true' shell: bash From 3388dfb7326bf46ee0bfd74bb75e3f376182aa03 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 20:10:50 -0300 Subject: [PATCH 14/15] refactor: update job state checks to allow for more flexible job completion criteria --- sidequest.jobs.js | 5 +++++ tests/integration/shared-test-suite.js | 6 +++--- tests/integration/sidequest.jobs.js | 5 +++++ 3 files changed, 13 insertions(+), 3 deletions(-) create mode 100644 sidequest.jobs.js create mode 100644 tests/integration/sidequest.jobs.js diff --git a/sidequest.jobs.js b/sidequest.jobs.js new file mode 100644 index 00000000..a1e44283 --- /dev/null +++ b/sidequest.jobs.js @@ -0,0 +1,5 @@ + +import { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob } from "./tests/integration/jobs/test-jobs.js"; + +export { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob }; + \ No newline at end of file diff --git a/tests/integration/shared-test-suite.js b/tests/integration/shared-test-suite.js index a89f7cdb..80efc09c 100644 --- a/tests/integration/shared-test-suite.js +++ b/tests/integration/shared-test-suite.js @@ -556,7 +556,7 @@ export function createIntegrationTestSuite(Sidequest, jobs, moduleType = "ESM") // Wait for the two scheduled executions await vi.waitUntil(async () => { const currentJobs = await Sidequest.job.list(); - return currentJobs.length === 2 && currentJobs.every((job) => job.state === "completed"); + return currentJobs.length >= 2 && currentJobs.every((job) => job.state === "completed"); }, 5000); }); @@ -609,7 +609,7 @@ export function createIntegrationTestSuite(Sidequest, jobs, moduleType = "ESM") let currentJobs; await vi.waitUntil(async () => { currentJobs = await Sidequest.job.list(); - return currentJobs.length === 3 && currentJobs.every((job) => job.state === "completed"); + return currentJobs.length >= 3 && currentJobs.every((job) => job.state === "completed"); }, 5000); const job1Executions = currentJobs.filter((job) => job.args[0] === "job-1"); @@ -634,7 +634,7 @@ export function createIntegrationTestSuite(Sidequest, jobs, moduleType = "ESM") let currentJobs; await vi.waitUntil(async () => { currentJobs = await Sidequest.job.list(); - return currentJobs.length === 2 && currentJobs.every((job) => job.state === "completed"); + return currentJobs.length >= 2 && currentJobs.every((job) => job.state === "completed"); }, 5000); expect(currentJobs.length).toBe(2); diff --git a/tests/integration/sidequest.jobs.js b/tests/integration/sidequest.jobs.js new file mode 100644 index 00000000..78e359a5 --- /dev/null +++ b/tests/integration/sidequest.jobs.js @@ -0,0 +1,5 @@ + +import { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob } from "./jobs/test-jobs.js"; + +export { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob }; + \ No newline at end of file From dbca3787499b61097796e6b4efc07dd82ef79439 Mon Sep 17 00:00:00 2001 From: Giovani Guizzo Date: Wed, 7 Jan 2026 20:14:27 -0300 Subject: [PATCH 15/15] refactor: standardize import and export order for job modules --- sidequest.jobs.js | 12 ++++++++---- tests/integration/sidequest.jobs.js | 6 ++---- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/sidequest.jobs.js b/sidequest.jobs.js index a1e44283..fb064e2c 100644 --- a/sidequest.jobs.js +++ b/sidequest.jobs.js @@ -1,5 +1,9 @@ +import { + EnqueueFromWithinJob, + FailingJob, + RetryJob, + SuccessJob, + TimeoutJob, +} from "./tests/integration/jobs/test-jobs.js"; -import { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob } from "./tests/integration/jobs/test-jobs.js"; - -export { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob }; - \ No newline at end of file +export { EnqueueFromWithinJob, FailingJob, RetryJob, SuccessJob, TimeoutJob }; diff --git a/tests/integration/sidequest.jobs.js b/tests/integration/sidequest.jobs.js index 78e359a5..68cd8c60 100644 --- a/tests/integration/sidequest.jobs.js +++ b/tests/integration/sidequest.jobs.js @@ -1,5 +1,3 @@ +import { EnqueueFromWithinJob, FailingJob, RetryJob, SuccessJob, TimeoutJob } from "./jobs/test-jobs.js"; -import { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob } from "./jobs/test-jobs.js"; - -export { SuccessJob, RetryJob, FailingJob, TimeoutJob, EnqueueFromWithinJob }; - \ No newline at end of file +export { EnqueueFromWithinJob, FailingJob, RetryJob, SuccessJob, TimeoutJob };