diff --git a/_bmad-output/implementation-artifacts/tech-spec-frontend-pipeline-polling.md b/_bmad-output/implementation-artifacts/tech-spec-frontend-pipeline-polling.md new file mode 100644 index 0000000..819ccfd --- /dev/null +++ b/_bmad-output/implementation-artifacts/tech-spec-frontend-pipeline-polling.md @@ -0,0 +1,329 @@ +--- +title: 'Frontend Pipeline Polling' +slug: 'frontend-pipeline-polling' +created: '2026-03-31' +status: 'implementation-complete' +stepsCompleted: [1, 2, 3, 4] +tech_stack: + ['NestJS', 'MikroORM', 'PostgreSQL', 'BullMQ', 'Zod', 'Jest', 'Passport JWT'] +files_to_modify: + - 'src/entities/analysis-pipeline.entity.ts' + - 'src/modules/analysis/dto/pipeline-status.dto.ts' + - 'src/modules/analysis/services/pipeline-orchestrator.service.ts' + - 'src/modules/analysis/services/pipeline-orchestrator.service.spec.ts' + - 'src/modules/analysis/analysis.controller.spec.ts' +code_patterns: + - 'PascalCase public service methods' + - 'Zod schemas for response DTOs' + - 'EntityManager.fork() for isolated DB context' + - 'CustomBaseEntity provides id, createdAt, updatedAt, deletedAt' +test_patterns: + - 'Test.createTestingModule with useValue mocks' + - 'jest.fn() for service/repo method mocks' + - 'makeMockPipeline() factory in controller spec; service spec builds mocks inline' + - 'Tests co-located with source as .spec.ts' +--- + +# Tech-Spec: Frontend Pipeline Polling + +**Created:** 2026-03-31 + +## Overview + +### Problem Statement + +The frontend has no way to track analysis pipeline progress in real-time. There is no existing pipeline status UI, and the current `GET /analysis/pipelines/:id/status` response isn't optimized for polling — it lacks consistent field presence, per-stage progress tracking, and a retryable flag for failed states. + +### Solution + +Reshape the existing status endpoint response into a lean, polling-friendly DTO with consistent field presence (`null` over omission), per-stage progress counts (real for sentiment, `null` for binary stages), and a top-level `retryable` flag. The frontend uses React Query's `refetchInterval` for 3-second polling that auto-stops on terminal states. No new endpoints, no WebSockets, no denormalization. + +### Scope + +**In Scope:** + +- Reshape status endpoint response DTO for polling consistency +- Add `progress: { current, total } | null` per stage (sentiment gets real counts via result row count) +- Add `retryable: boolean` on pipeline-level failures +- Ensure `updatedAt` is always accurate on pipeline entity +- Document the frontend polling contract (React Query + Axios pattern) + +**Out of Scope:** + +- Frontend UI implementation (stepper component, animations) +- WebSocket/SSE infrastructure +- Batch multi-pipeline status endpoint +- Denormalization of stage status onto pipeline entity +- ETag / conditional request support + +## Context for Development + +### Codebase Patterns + +- **Response DTOs use Zod schemas** — `pipeline-status.dto.ts` defines `pipelineStatusSchema` and `stageStatusSchema` with Zod, then exports inferred TypeScript types. Changes to the response shape must update both the Zod schema and the service that constructs the response. +- **PascalCase public methods** — `GetPipelineStatus()`, `CreatePipeline()`, etc. +- **EntityManager.fork()** — `GetPipelineStatus()` forks the EM at the start for an isolated DB context, then runs all queries on the fork. +- **Stage status derivation** — stage statuses are computed from run entity statuses (or defaulted to `'pending'` if no run exists). There is a private `getEmbeddingStageStatus()` method for embedding-specific logic. +- **Current `StageStatus` shape uses optional fields** — `total?`, `completed?`, `processed?`, `included?`, `excluded?`. These fields appear/disappear depending on the stage, which is the core polling consistency problem to fix. + +### Enums + +- **`PipelineStatus`** (9 values): `AWAITING_CONFIRMATION`, `EMBEDDING_CHECK`, `SENTIMENT_ANALYSIS`, `SENTIMENT_GATE`, `TOPIC_MODELING`, `GENERATING_RECOMMENDATIONS`, `COMPLETED`, `FAILED`, `CANCELLED` +- **`RunStatus`** (4 values): `PENDING`, `PROCESSING`, `COMPLETED`, `FAILED` +- Terminal statuses: `COMPLETED`, `FAILED`, `CANCELLED` + +### Files to Reference + +| File | Purpose | +| --------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `src/modules/analysis/dto/pipeline-status.dto.ts` | Zod schema for status response — `pipelineStatusSchema`, `stageStatusSchema`, `PipelineStatusResponse` type | +| `src/entities/base.entity.ts` | `CustomBaseEntity` — defines shared `updatedAt` without `onUpdate` hook (Task 0 overrides on pipeline entity instead) | +| `src/modules/analysis/services/pipeline-orchestrator.service.ts` | `GetPipelineStatus()` at ~line 438 — constructs response from pipeline + run entities (up to 7 DB queries after adding sentiment COUNT; some conditional) | +| `src/modules/analysis/analysis.controller.ts` | Status endpoint at line 69 — `GET pipelines/:id/status`, delegates to orchestrator | +| `src/entities/analysis-pipeline.entity.ts` | Pipeline entity — has `updatedAt` (inherited from CustomBaseEntity), `status`, `commentCount`, `sentimentGateIncluded/Excluded` | +| `src/entities/sentiment-run.entity.ts` | SentimentRun — `status: RunStatus`, `submissionCount`, has `results` collection | +| `src/entities/sentiment-result.entity.ts` | Individual result per submission — COUNT of these gives sentiment progress | +| `src/entities/topic-model-run.entity.ts` | TopicModelRun — `status: RunStatus` | +| `src/entities/recommendation-run.entity.ts` | RecommendationRun — `status: RunStatus` | +| `src/modules/analysis/enums/pipeline-status.enum.ts` | `PipelineStatus` enum definition | +| `src/modules/analysis/enums/run-status.enum.ts` | `RunStatus` enum definition | +| `src/modules/analysis/services/pipeline-orchestrator.service.spec.ts` | Service tests — builds mocks inline, mocked EM fork | + +### Technical Decisions + +- **Single endpoint, no new routes** — reshape existing `GET /analysis/pipelines/:id/status` response rather than adding a separate polling endpoint. Avoids coordination complexity on the frontend. +- **No denormalization** — query run tables directly on each poll. With only a handful of privileged users polling concurrently, the join cost is negligible. +- **No WebSocket/SSE** — pipelines run for minutes. A 3-second REST polling interval via React Query is simpler, more resilient, and operationally trivial. +- **`null` over omission** — every field in the response is always present. Use `null` for absent optional values. This ensures React Query's referential comparison works correctly and avoids unnecessary re-renders from shape changes. +- **Sentiment gets real progress, others are binary** — derive sentiment `progress.current` from `SentimentResult` row count vs `progress.total` from `sentimentRun.submissionCount`. Topic modeling and recommendations are single-batch, so progress is effectively `null` (binary processing/done). +- **`retryable` flag** — top-level boolean on failed pipelines so the frontend knows whether to show "Retry" vs "Contact admin". Currently equivalent to `status === FAILED` but provides intent signaling for future error categorization. +- **Pre-existing DTO issue (out of scope)**: The top-level `status` field in `pipelineStatusSchema` is `z.string()` instead of `z.nativeEnum(PipelineStatus)`. This predates this spec and should be addressed separately. + +## Implementation Plan + +### Tasks + +- [x] Task 0: Fix `AnalysisPipeline.updatedAt` — add `onUpdate` hook (scoped to pipeline only) + - File: `src/entities/analysis-pipeline.entity.ts` + - Action: Override `updatedAt` on `AnalysisPipeline` with `onUpdate`: + ```typescript + @Property({ onUpdate: () => new Date() }) + override updatedAt: Date & Opt = new Date(); + ``` + - Notes: This is a prerequisite for the entire spec. Without it, `updatedAt` is set once at creation and never updated. Scoped to `AnalysisPipeline` only (not `CustomBaseEntity`) to avoid cross-cutting impact — other entities like `Enrollment` use `updatedAt` for sync tracking and could be affected by unrelated cascade flushes. Promoting `onUpdate` to `CustomBaseEntity` can be done as a separate, properly-analyzed change. No migration needed — `onUpdate` is an ORM-layer hook, not a DB schema change. + +- [x] Task 1: Reshape `stageStatusSchema` for consistent field presence + - File: `src/modules/analysis/dto/pipeline-status.dto.ts` + - Action: Replace all optional fields in `stageStatusSchema` with consistent, always-present fields using `null` for absent values: + ```typescript + const stageStatusSchema = z.object({ + status: z.enum([ + 'pending', + 'processing', + 'completed', + 'failed', + 'skipped', + ]), + progress: z + .object({ + current: z.number().int(), + total: z.number().int(), + }) + .nullable(), + startedAt: z.string().datetime().nullable(), + completedAt: z.string().datetime().nullable(), + }); + ``` + - Notes: This removes the old `total`, `completed`, `processed`, `included`, `excluded` optional fields. The `progress` object is either fully present or `null`. `startedAt` and `completedAt` give the frontend elapsed-time signal. **Preserve all existing top-level fields** (`confirmedAt`, `completedAt`, `createdAt`, `scope`, `coverage`, `warnings`). Only the `stages` shape, `errorMessage` optionality, and new fields (`retryable`, `updatedAt`) change. + +- [x] Task 2: Add `retryable` and `updatedAt` to `pipelineStatusSchema` + - File: `src/modules/analysis/dto/pipeline-status.dto.ts` + - Action: Add fields and normalize existing ones in `pipelineStatusSchema`: + - Add `retryable: z.boolean()` — after `errorMessage` + - Add `updatedAt: z.string().datetime()` — after `createdAt` + - Change `errorMessage: z.string().nullable().optional()` to `z.string().nullable()` — enforce `null` over omission principle + - Notes: `retryable` is `true` only when status is `FAILED`. `updatedAt` comes from `pipeline.updatedAt` (requires Task 0's `onUpdate` hook to be accurate). + +- [x] Task 3: Add sentiment gate fields to `pipelineStatusSchema` stages + - File: `src/modules/analysis/dto/pipeline-status.dto.ts` + - Action: The sentiment gate's `included`/`excluded` counts no longer fit in the generic `stageStatusSchema` (which now uses `progress`). Add a dedicated `sentimentGate` schema: + ```typescript + const sentimentGateSchema = stageStatusSchema.extend({ + included: z.number().int().nullable(), + excluded: z.number().int().nullable(), + }); + ``` + - Update `stages` in `pipelineStatusSchema` — change the `sentimentGate` field from `stageStatusSchema` to `sentimentGateSchema`: + ```typescript + stages: z.object({ + embeddings: stageStatusSchema, + sentiment: stageStatusSchema, + sentimentGate: sentimentGateSchema, // ← changed from stageStatusSchema + topicModeling: stageStatusSchema, + recommendations: stageStatusSchema, + }), + ``` + +- [x] Task 4: Update `GetPipelineStatus()` — add sentiment progress count query + - File: `src/modules/analysis/services/pipeline-orchestrator.service.ts` + - Action: After the existing `sentimentRun` query (~line 461), add a conditional `COUNT` query: + ```typescript + let sentimentCompleted = 0; + if (sentimentRun && sentimentRun.status !== RunStatus.PENDING) { + sentimentCompleted = await fork.count(SentimentResult, { + run: sentimentRun, + }); + } + ``` + - Notes: Only queries when a run exists and has started. Returns 0 when pending. This is a cheap indexed COUNT on `sentiment_result.run_id` (index confirmed on entity). If pipelines scale beyond ~5K comments, consider caching the count on `SentimentRun.completedCount` — out of scope for now. + +- [x] Task 5: Update `GetPipelineStatus()` — reshape return object to match new DTO + - File: `src/modules/analysis/services/pipeline-orchestrator.service.ts` + - **Ordering dependency**: Tasks 1-3 (DTO changes) must be completed before Tasks 4-5 (service changes). Do not implement in isolation — the intermediate state where the schema is updated but the service is not (or vice versa) will produce a compile error. + - Action: Replace the `stageStatus` helper, update `getEmbeddingStageStatus()`, and reshape the return object. Key changes: + 1. **First**, update `getEmbeddingStageStatus()` (currently at line 937): change return type from `{ status }` object to bare status string. Verify it has only one call site (line 518) by grepping for `getEmbeddingStageStatus`. This must be done before step 4 below, which consumes it as a string. + 2. Remove the `stageStatus()` and `getRunStageStatus()` helpers. Replace with a new helper: + ```typescript + const buildStage = ( + status: 'pending' | 'processing' | 'completed' | 'failed' | 'skipped', + run: { createdAt: Date; completedAt?: Date | null } | null, + progress: { current: number; total: number } | null = null, + ) => ({ + status, + progress, + startedAt: run?.createdAt?.toISOString() ?? null, + completedAt: run?.completedAt?.toISOString() ?? null, + }); + ``` + 3. Add a `getRunStatus` helper with a comment about enum coupling: + ```typescript + // If RunStatus gains new values, update this mapping to match the stage status union + const getRunStatus = ( + run: SentimentRun | TopicModelRun | RecommendationRun | null, + ) => + run + ? (run.status.toLowerCase() as + | 'pending' + | 'processing' + | 'completed' + | 'failed') + : 'pending'; + ``` + 4. Derive `gateStatus` before building the return block (handles terminal pipeline states): + ```typescript + // Gate either completed (has data) or didn't. Top-level pipeline.status + // handles failure attribution — no per-stage failure tracking for MVP. + // FAILED pipelines: gate shows 'pending' (never completed) which is correct. + // CANCELLED pipelines: gate shows 'skipped' (explicitly not attempted). + const gateStatus = + pipeline.sentimentGateIncluded != null + ? 'completed' + : pipeline.status === PipelineStatus.SENTIMENT_GATE + ? 'processing' + : pipeline.status === PipelineStatus.CANCELLED + ? 'skipped' + : 'pending'; + ``` + 5. Update each stage in the return block: + - `embeddings`: `buildStage(this.getEmbeddingStageStatus(pipeline), null)` — no run entity, no progress. Note: `startedAt` and `completedAt` will always be `null` for this stage since there is no run entity. Frontend should use `pipeline.confirmedAt` as a proxy if timing is needed. + - `sentiment`: when `sentimentRun` exists → `buildStage(getRunStatus(sentimentRun), sentimentRun, { current: sentimentCompleted, total: sentimentRun.submissionCount })` — real progress using run's own `submissionCount` as total. When `sentimentRun` is `null` → `buildStage('pending', null)` — no progress (consistent with binary "not started" pattern) + - `sentimentGate`: `{ ...buildStage(gateStatus, null), included: pipeline.sentimentGateIncluded ?? null, excluded: pipeline.sentimentGateExcluded ?? null }` — note: `startedAt`/`completedAt` always `null` (no run entity, same as embeddings) + - `topicModeling`: `buildStage(getRunStatus(topicModelRun), topicModelRun)` + - `recommendations`: `buildStage(getRunStatus(recommendationRun), recommendationRun)` + 6. Add to the top-level return: `updatedAt: pipeline.updatedAt.toISOString()` and `retryable: pipeline.status === PipelineStatus.FAILED` + 7. Optional runtime safety net — add `return pipelineStatusSchema.parse(response)` at the end to catch schema/implementation drift at runtime (the Zod schema is currently only used for type inference via `z.infer`, not runtime validation) + - Notes: + - For `retryable`, all current failure modes are transient (worker timeouts, network errors), so `status === FAILED` is sufficient. The flag provides intent signaling — when error categories are added later, `retryable` becomes meaningful without a frontend change. Add a code comment: `// Intent signal for future error categorization — currently equivalent to status === FAILED` + - `startedAt` uses `run.createdAt` as a proxy — this is the best available approximation since run entities don't have a dedicated `startedAt` field. Runs are created at enqueue time, which is close to processing start for the polling UI's purposes. + - **Commit strategy**: Split this task into two commits for reviewability: (a) replace helpers + reshape return block, (b) add `updatedAt` + `retryable` top-level fields. + +- [x] Task 6: Update service tests for reshaped response + - File: `src/modules/analysis/services/pipeline-orchestrator.service.spec.ts` + - Action: Update the `GetPipelineStatus` test cases: + 1. Add mock for `fork.count(SentimentResult, ...)` returning a number (e.g., `47`) + 2. Update expected response shape in assertions to match new DTO: `progress` objects instead of `total`/`completed`, `startedAt`/`completedAt` fields, `retryable`, `updatedAt` + 3. Add a test case for `retryable: true` when pipeline status is `FAILED` + 4. Add a test case for `retryable: false` when pipeline status is not `FAILED` + 5. Add a test case verifying sentiment `progress.current` equals the mocked count + 6. Add a test case for sentiment start: `sentimentRun` exists with status `PROCESSING` and zero results → `progress: { current: 0, total: N }` + +- [x] Task 7: Update controller tests for reshaped response + - File: `src/modules/analysis/analysis.controller.spec.ts` + - Action: Update the `GetPipelineStatus` test's `mockStatus` object to match the new response shape (add `retryable`, `updatedAt`, update stage shapes). The controller test is a pass-through, but the `PipelineStatusResponse` type change will cause TypeScript compile errors in mock data. + - Notes: Required because `PipelineStatusResponse` (inferred from the reshaped Zod schema) is used to type mock data in the controller spec. + +### Acceptance Criteria + +- [ ] AC 1: Given a pipeline in `SENTIMENT_ANALYSIS` status with a `SentimentRun` where `submissionCount` is 120 and 47 `SentimentResult` rows exist for that run, when `GET /analysis/pipelines/:id/status` is called, then `stages.sentiment.progress` is `{ current: 47, total: 120 }` and `stages.sentiment.status` is `"processing"`. + +- [ ] AC 2: Given a pipeline in `TOPIC_MODELING` status, when `GET /analysis/pipelines/:id/status` is called, then `stages.topicModeling.progress` is `null` and `stages.topicModeling.status` is `"processing"`. + +- [ ] AC 3: Given a pipeline in `FAILED` status, when `GET /analysis/pipelines/:id/status` is called, then `retryable` is `true`. + +- [ ] AC 4: Given a pipeline in `COMPLETED` status, when `GET /analysis/pipelines/:id/status` is called, then `retryable` is `false`. + +- [ ] AC 5: Given any pipeline status, when `GET /analysis/pipelines/:id/status` is called, then every stage object contains `status`, `progress`, `startedAt`, and `completedAt` fields (no missing keys — values may be `null`). + +- [ ] AC 6: Given any pipeline status, when `GET /analysis/pipelines/:id/status` is called, then the response contains `updatedAt` as an ISO 8601 datetime string. + +- [ ] AC 7: Given a pipeline in `SENTIMENT_ANALYSIS` status with a `SentimentRun` that has `createdAt` set, when the status is polled, then `stages.sentiment.startedAt` is the ISO 8601 representation of that run's `createdAt`. + +- [ ] AC 8: Given a pipeline with completed sentiment gate (included=80, excluded=40), when `GET /analysis/pipelines/:id/status` is called, then `stages.sentimentGate.included` is `80`, `stages.sentimentGate.excluded` is `40`, and `stages.sentimentGate.status` is `"completed"`. + +- [ ] AC 9: Given a pipeline that does not exist, when `GET /analysis/pipelines/:id/status` is called, then a `404 Not Found` is returned (existing behavior preserved). + +- [ ] AC 10: Given a pipeline in `CANCELLED` status with `sentimentGateIncluded` as `null` (gate never completed), when `GET /analysis/pipelines/:id/status` is called, then `retryable` is `false`, `stages.embeddings.status` is `"skipped"`, and `stages.sentimentGate.status` is `"skipped"`. (Note: if a pipeline is cancelled _after_ the gate completed, `sentimentGate.status` would correctly be `"completed"` since the gate data is real.) + +## Additional Context + +### Dependencies + +- No new packages required. All changes use existing NestJS, MikroORM, and Zod infrastructure. +- Sentiment progress count requires a `COUNT` query on `SentimentResult` for the active run — this is a new query addition to `GetPipelineStatus()` (up to 7 DB queries per poll; fewer when conditional queries are skipped, e.g., no courseIds → enrollment query skipped, no sentimentRun → COUNT skipped). The COUNT scopes to the latest run via `{ run: sentimentRun }`, so progress cannot regress across retries (new runs get new result rows). +- **Task 0 is a prerequisite**: `CustomBaseEntity.updatedAt` currently has no `onUpdate` hook — it is set once at creation and never updated. Task 0 fixes this by overriding `updatedAt` on `AnalysisPipeline` with `onUpdate: () => new Date()`. Scoped to pipeline only to avoid cross-cutting impact on entities like `Enrollment` whose `updatedAt` is used for sync tracking. No migration needed. + +### Testing Strategy + +**Unit Tests (Jest):** + +- `pipeline-orchestrator.service.spec.ts`: + - Test reshaped response has all fields present (no undefined keys) + - Test sentiment `progress.current` matches mocked `SentimentResult` count + - Test sentiment `progress.total` matches `sentimentRun.submissionCount` (falls back to `pipeline.commentCount` if no run) + - Test binary stages (embedding, topicModeling, recommendations) have `progress: null` + - Test `retryable: true` when `FAILED`, `false` otherwise + - Test `updatedAt` is present and matches pipeline entity + - Test `startedAt`/`completedAt` populated from run entities + - Test sentiment gate `included`/`excluded` still present +- `analysis.controller.spec.ts`: + - Update mock data shape to match new DTO + - Verify pass-through behavior unchanged + +**Manual Testing:** + +- Start a pipeline via `POST /analysis/pipelines` + confirm +- Poll `GET /analysis/pipelines/:id/status` and verify response shape consistency across stage transitions +- Verify sentiment progress increments as results are processed +- **Transition test**: Poll across a stage transition (e.g., `SENTIMENT_ANALYSIS` → `SENTIMENT_GATE`) and verify that the previous stage flips to `completed` and the next stage updates correctly in the subsequent poll response + +### Notes + +- **Breaking change**: The response shape of `GET /analysis/pipelines/:id/status` changes. The old `total`, `completed`, `processed` fields on stages are replaced with a `progress: { current, total } | null` object, and `startedAt`/`completedAt` are added. **Atomic deploy required** — backend and frontend must deploy in the same release window. Deploying backend first will break active polling sessions immediately. +- **No migration needed**: No database schema changes. Task 0 overrides `updatedAt` on `AnalysisPipeline` only (ORM-layer `onUpdate` hook), and the rest are DTO/service layer changes. +- **Frontend null-safety**: The frontend must null-check `progress` before accessing `.current`/`.total`. Consider sharing the Zod schema or generating OpenAPI types to keep the contract in sync. +- **Staleness detection (frontend concern)**: The frontend should compute staleness from `updatedAt` and `stages.*.startedAt` — e.g., if a stage has been `processing` for >10 minutes with no `updatedAt` change, display a "pipeline may be stuck" warning. No backend changes needed for this. +- **Embedding and sentiment gate timing**: Both the embedding stage and sentiment gate stage have no run entity, so `startedAt` and `completedAt` will always be `null` for these stages. The frontend should use `confirmedAt` from the top-level response as a proxy if timing info is needed. +- **`startedAt` approximation**: For stages with run entities, `startedAt` uses `run.createdAt` (entity creation time) as a proxy since runs don't have a dedicated `startedAt` field. This is close to processing start and sufficient for the polling UI. +- **Future scaling**: If pipeline comment counts grow beyond ~5K, the per-poll `SentimentResult` COUNT query should be revisited (cache on `SentimentRun.completedCount`). Current volumes don't warrant this. +- Frontend polling pattern (illustrative only, not a contract — frontend implementation is out of scope): + ```typescript + const { data } = useQuery({ + queryKey: ['pipeline-status', pipelineId], + queryFn: () => axios.get(`/analysis/pipelines/${pipelineId}/status`), + refetchInterval: (query) => { + const status = query.state.data?.data.status; + const isTerminal = ['COMPLETED', 'FAILED', 'CANCELLED'].includes(status); + return isTerminal ? false : 3000; + }, + }); + ``` diff --git a/package-lock.json b/package-lock.json index 63ab862..e9617ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,11 +10,11 @@ "license": "UNLICENSED", "dependencies": { "@keyv/redis": "^5.1.6", - "@mikro-orm/core": "^6.6.6", - "@mikro-orm/migrations": "^6.6.6", - "@mikro-orm/nestjs": "^6.1.1", - "@mikro-orm/postgresql": "^6.6.6", - "@mikro-orm/seeder": "^6.6.6", + "@mikro-orm/core": "^6.6.11", + "@mikro-orm/migrations": "^6.6.11", + "@mikro-orm/nestjs": "^6.1.2", + "@mikro-orm/postgresql": "^6.6.11", + "@mikro-orm/seeder": "^6.6.11", "@nest-lab/throttler-storage-redis": "^1.2.0", "@nestjs/bullmq": "^11.0.4", "@nestjs/cache-manager": "^3.1.0", @@ -58,7 +58,7 @@ "devDependencies": { "@eslint/eslintrc": "^3.2.0", "@eslint/js": "^9.18.0", - "@mikro-orm/cli": "^6.6.6", + "@mikro-orm/cli": "^6.6.11", "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", "@nestjs/testing": "^11.0.1", @@ -1881,9 +1881,9 @@ } }, "node_modules/@jest/reporters/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -2194,15 +2194,15 @@ "license": "MIT" }, "node_modules/@mikro-orm/cli": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/@mikro-orm/cli/-/cli-6.6.7.tgz", - "integrity": "sha512-6xqkC/Kr0ZkUeCpPEdNxNcxomkwKULDVrgxXnj9YxwCxkaGoZDOjR1gdPM4ey3Irwviq1cbBtFcvOHpax94A9w==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/@mikro-orm/cli/-/cli-6.6.11.tgz", + "integrity": "sha512-E+zB3cB5EbemeRvh/6TaX822xxm/jcVTsSNdObQsccXtCEsbYFzcqmB2pIj/5oPPCPU2jaPsZFtWaxLgUxzoJw==", "dev": true, "license": "MIT", "dependencies": { "@jercle/yargonaut": "1.1.5", - "@mikro-orm/core": "6.6.7", - "@mikro-orm/knex": "6.6.7", + "@mikro-orm/core": "6.6.11", + "@mikro-orm/knex": "6.6.11", "fs-extra": "11.3.3", "tsconfig-paths": "4.2.0", "yargs": "17.7.2" @@ -2216,17 +2216,17 @@ } }, "node_modules/@mikro-orm/core": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/@mikro-orm/core/-/core-6.6.7.tgz", - "integrity": "sha512-VuL9WK6Z1Op5Lg5FCDOfFeVQdfpCrtEDQXEMHnlb0mRL7WnNz2vUu8AJ96t7iOIxkIBJUXrlzpkaHPdrV9lmkA==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/@mikro-orm/core/-/core-6.6.11.tgz", + "integrity": "sha512-+edc3ctapRi0lyb2B0+QfUpoWkNmXOcaApDT6RhBxyFo74bpoU/tEb9aMobemN86VhAt/rjM1KDKbJYLM9lxTg==", "license": "MIT", "dependencies": { "dataloader": "2.2.3", - "dotenv": "17.2.3", + "dotenv": "17.3.1", "esprima": "4.0.1", "fs-extra": "11.3.3", "globby": "11.1.0", - "mikro-orm": "6.6.7", + "mikro-orm": "6.6.11", "reflect-metadata": "0.2.2" }, "engines": { @@ -2236,26 +2236,14 @@ "url": "https://github.com/sponsors/b4nan" } }, - "node_modules/@mikro-orm/core/node_modules/dotenv": { - "version": "17.2.3", - "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", - "integrity": "sha512-JVUnt+DUIzu87TABbhPmNfVdBDt18BLOWjMUFJMSi/Qqg7NTYtabbvSNJGOJ7afbRuv9D/lngizHtP7QyLQ+9w==", - "license": "BSD-2-Clause", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://dotenvx.com" - } - }, "node_modules/@mikro-orm/knex": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/@mikro-orm/knex/-/knex-6.6.7.tgz", - "integrity": "sha512-/EfSu3D1A5OrV3vyHSILbFrV0B4FFbHn4Fa3qc1wKf8Dl5adZlPe7jj+R4c87V1+oLo6VzST1sT4Rhp7NWArdw==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/@mikro-orm/knex/-/knex-6.6.11.tgz", + "integrity": "sha512-MUxqw+3COpcM06DC3ufW4Aov5RZWpW1Rv/kMfJkHQX+bO81jPdinXkRtx1l8EVWFRiLJEB+3MNhptFQRlmJNXA==", "license": "MIT", "dependencies": { "fs-extra": "11.3.3", - "knex": "3.1.0", + "knex": "3.2.8", "sqlstring": "2.3.3" }, "engines": { @@ -2280,12 +2268,12 @@ } }, "node_modules/@mikro-orm/migrations": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/@mikro-orm/migrations/-/migrations-6.6.7.tgz", - "integrity": "sha512-OFVcOwD5pUwWNjmoUSoCSAOAlepib0KfYPkM6nvBokRiMO3H6VPUitAPMPygPx5cPSVzTaAvW7gT6sS48VTwxA==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/@mikro-orm/migrations/-/migrations-6.6.11.tgz", + "integrity": "sha512-pTcH/pr/Ch4nXs8jJnX3fJfodbA0VFZCp1hOnE6M0vDbDw7EsP66y7JuZH6vOs4r4rpIbjguSnMSqIACLx3esQ==", "license": "MIT", "dependencies": { - "@mikro-orm/knex": "6.6.7", + "@mikro-orm/knex": "6.6.11", "fs-extra": "11.3.3", "umzug": "3.8.2" }, @@ -2297,9 +2285,9 @@ } }, "node_modules/@mikro-orm/nestjs": { - "version": "6.1.1", - "resolved": "https://registry.npmjs.org/@mikro-orm/nestjs/-/nestjs-6.1.1.tgz", - "integrity": "sha512-aluD3eTeuCvIePDk5UBanHIhu1zAJQXqWAg47MZdHJmFkNuXn62DCXbD2c4X5TCpKW/m0zjba22ilyZ/AFG9qg==", + "version": "6.1.2", + "resolved": "https://registry.npmjs.org/@mikro-orm/nestjs/-/nestjs-6.1.2.tgz", + "integrity": "sha512-rITNvXusRVAB7PjEftT5ooDg6U2FZHwpn0u9f5RJyc9gVJGL2XroSJ6iX7VbDXasY8GUT1agdeeKmImx+MThdw==", "license": "MIT", "engines": { "node": ">= 18.12.0" @@ -2311,13 +2299,13 @@ } }, "node_modules/@mikro-orm/postgresql": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/@mikro-orm/postgresql/-/postgresql-6.6.7.tgz", - "integrity": "sha512-2LR33f/+PrnA09iomhVraH5N9BcYmziasB06HCf+aFBtql5PXyTen8bQu+bZ1M7etkJ+Tt7E/pA8dU/ylnIqdg==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/@mikro-orm/postgresql/-/postgresql-6.6.11.tgz", + "integrity": "sha512-YIQroXsAPXRJc3ruk8M5ynbQEQtGUO0Swjb/MMtjn5o9qypqmPBoq4ANCwUY9P2jVlmheQM1O5VK/1OBm7/EVg==", "license": "MIT", "dependencies": { - "@mikro-orm/knex": "6.6.7", - "pg": "8.16.3", + "@mikro-orm/knex": "6.6.11", + "pg": "8.20.0", "postgres-array": "3.0.4", "postgres-date": "2.1.0", "postgres-interval": "4.0.2" @@ -2330,9 +2318,9 @@ } }, "node_modules/@mikro-orm/seeder": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/@mikro-orm/seeder/-/seeder-6.6.7.tgz", - "integrity": "sha512-7qWWqqBn3r49xO901/Xa8r8IVSS/dLscv68pUHedMhwiLPHkI6T880DhCP3FTR0NLJ78d4ZV8YVPXfHuOoLJdg==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/@mikro-orm/seeder/-/seeder-6.6.11.tgz", + "integrity": "sha512-7GAaE2RuYjT+D+i+7Aa46ZE+ss6DbwESa8QotSMUGv8r9XC8hdFaIGJW7vKDrzkGaqhf0VZCPOtV2rfbypejlA==", "license": "MIT", "dependencies": { "fs-extra": "11.3.3", @@ -4189,9 +4177,9 @@ } }, "node_modules/@typescript-eslint/typescript-estree/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -4936,9 +4924,9 @@ } }, "node_modules/anymatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "dev": true, "license": "MIT", "engines": { @@ -5399,9 +5387,9 @@ } }, "node_modules/brace-expansion": { - "version": "1.1.12", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", - "integrity": "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg==", + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.13.tgz", + "integrity": "sha512-9ZLprWS6EENmhEOpjCYW2c8VkmOvckIJZfkr7rBW6dObmfgJ/L1GpSYW5Hpo9lDz4D1+n0Ckz8rU7FwHDQiG/w==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0", @@ -7931,16 +7919,16 @@ } }, "node_modules/glob/node_modules/brace-expansion": { - "version": "5.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.2.tgz", - "integrity": "sha512-Pdk8c9poy+YhOgVWw1JNN22/HcivgKWwpxKq04M/jTmHyCZn12WPJebZxdjSa5TmBqISrUSgNYU3eRORljfCCw==", + "version": "5.0.5", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-5.0.5.tgz", + "integrity": "sha512-VZznLgtwhn+Mact9tfiwx64fA9erHH/MCXEUfB/0bX/6Fz6ny5EGTXYltMocqg4xFAQZtnO3DHWWXi8RiuN7cQ==", "dev": true, "license": "MIT", "dependencies": { "balanced-match": "^4.0.2" }, "engines": { - "node": "20 || >=22" + "node": "18 || 20 || >=22" } }, "node_modules/glob/node_modules/jackspeak": { @@ -8027,9 +8015,9 @@ "license": "ISC" }, "node_modules/handlebars": { - "version": "4.7.8", - "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.8.tgz", - "integrity": "sha512-vafaFqs8MZkRrSX7sFVUdo3ap/eNiLnb4IakshzvP56X5Nr1iGKAIqdX6tMlm6HcNRIkr6AxO5jFEoJzzpT8aQ==", + "version": "4.7.9", + "resolved": "https://registry.npmjs.org/handlebars/-/handlebars-4.7.9.tgz", + "integrity": "sha512-4E71E0rpOaQuJR2A3xDZ+GM1HyWYv1clR58tC8emQNeQe3RH7MAzSbat+V0wG78LQBo6m6bzSG/L4pBuCsgnUQ==", "dev": true, "license": "MIT", "dependencies": { @@ -8837,9 +8825,9 @@ } }, "node_modules/jest-config/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -9245,9 +9233,9 @@ } }, "node_modules/jest-runtime/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "dev": true, "license": "MIT", "dependencies": { @@ -9670,9 +9658,9 @@ } }, "node_modules/knex": { - "version": "3.1.0", - "resolved": "https://registry.npmjs.org/knex/-/knex-3.1.0.tgz", - "integrity": "sha512-GLoII6hR0c4ti243gMs5/1Rb3B+AjwMOfjYm97pu0FOQa7JH56hgBxYf5WK2525ceSbBY1cjeZ9yk99GPMB6Kw==", + "version": "3.2.8", + "resolved": "https://registry.npmjs.org/knex/-/knex-3.2.8.tgz", + "integrity": "sha512-ElXXxu9Nq+5hWYdBUddYIWIT5yKKs5KNCsmKGbJSHPyaMpAABp3xs4L55GgdQoAs6QQ7dv72ai3M4pxYQ8utEg==", "license": "MIT", "dependencies": { "colorette": "2.0.19", @@ -9696,6 +9684,9 @@ "engines": { "node": ">=16" }, + "peerDependencies": { + "pg-query-stream": "^4.14.0" + }, "peerDependenciesMeta": { "better-sqlite3": { "optional": true @@ -9712,6 +9703,9 @@ "pg-native": { "optional": true }, + "pg-query-stream": { + "optional": true + }, "sqlite3": { "optional": true }, @@ -10409,9 +10403,9 @@ } }, "node_modules/micromatch/node_modules/picomatch": { - "version": "2.3.1", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", - "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", + "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", "license": "MIT", "engines": { "node": ">=8.6" @@ -10421,9 +10415,9 @@ } }, "node_modules/mikro-orm": { - "version": "6.6.7", - "resolved": "https://registry.npmjs.org/mikro-orm/-/mikro-orm-6.6.7.tgz", - "integrity": "sha512-Iw8BC2qMeyqgU6lQS86Ht+yzxjK0DKfmXkGQC2wRzDLYiUQj/CEn5ne8Q+5yIrZdIr/y53KqUNyUWDSup+ZT5w==", + "version": "6.6.11", + "resolved": "https://registry.npmjs.org/mikro-orm/-/mikro-orm-6.6.11.tgz", + "integrity": "sha512-8z1pS5IfKGys0OR0m5bWDLbmCu7n86DXvozL9v7BYcqW6O3GbsioghmNobzl7PraOOIRy260rS+mO6Z1jLduDQ==", "license": "MIT", "engines": { "node": ">= 18.12.0" @@ -11231,9 +11225,9 @@ } }, "node_modules/path-to-regexp": { - "version": "8.3.0", - "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.3.0.tgz", - "integrity": "sha512-7jdwVIRtsP8MYpdXSwOS0YdD0Du+qOoF/AEPIt88PcCFrZCzx41oxku1jD88hZBwbNUIEfpqvuhjFaMAqMTWnA==", + "version": "8.4.1", + "resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-8.4.1.tgz", + "integrity": "sha512-fvU78fIjZ+SBM9YwCknCvKOUKkLVqtWDVctl0s7xIqfmfb38t2TT4ZU2gHm+Z8xGwgW+QWEU3oQSAzIbo89Ggw==", "license": "MIT", "funding": { "type": "opencollective", @@ -11256,14 +11250,14 @@ "peer": true }, "node_modules/pg": { - "version": "8.16.3", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.16.3.tgz", - "integrity": "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw==", + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.20.0.tgz", + "integrity": "sha512-ldhMxz2r8fl/6QkXnBD3CR9/xg694oT6DZQ2s6c/RI28OjtSOpxnPrUCGOBJ46RCUxcWdx3p6kw/xnDHjKvaRA==", "license": "MIT", "dependencies": { - "pg-connection-string": "^2.9.1", - "pg-pool": "^3.10.1", - "pg-protocol": "^1.10.3", + "pg-connection-string": "^2.12.0", + "pg-pool": "^3.13.0", + "pg-protocol": "^1.13.0", "pg-types": "2.2.0", "pgpass": "1.0.5" }, @@ -11271,7 +11265,7 @@ "node": ">= 16.0.0" }, "optionalDependencies": { - "pg-cloudflare": "^1.2.7" + "pg-cloudflare": "^1.3.0" }, "peerDependencies": { "pg-native": ">=3.0.1" @@ -11305,18 +11299,18 @@ } }, "node_modules/pg-pool": { - "version": "3.11.0", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.11.0.tgz", - "integrity": "sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==", + "version": "3.13.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.13.0.tgz", + "integrity": "sha512-gB+R+Xud1gLFuRD/QgOIgGOBE2KCQPaPwkzBBGC9oG69pHTkhQeIuejVIk3/cnDyX39av2AxomQiyPT13WKHQA==", "license": "MIT", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.11.0.tgz", - "integrity": "sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g==", + "version": "1.13.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.13.0.tgz", + "integrity": "sha512-zzdvXfS6v89r6v7OcFCHfHlyG/wvry1ALxZo4LqgUoy7W9xhBDMaqOuMiF3qEV45VqsN6rdlcehHrfDtlCPc8w==", "license": "MIT" }, "node_modules/pg-types": { @@ -11366,9 +11360,9 @@ } }, "node_modules/pg/node_modules/pg-connection-string": { - "version": "2.11.0", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", - "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "version": "2.12.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.12.0.tgz", + "integrity": "sha512-U7qg+bpswf3Cs5xLzRqbXbQl85ng0mfSV/J0nnA31MCLgvEaAo7CIhmeyrmJpOr7o+zm0rXK+hNnT5l9RHkCkQ==", "license": "MIT" }, "node_modules/pgpass": { @@ -11397,9 +11391,9 @@ "license": "ISC" }, "node_modules/picomatch": { - "version": "4.0.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.2.tgz", - "integrity": "sha512-M7BAV6Rlcy5u+m6oPhAPFgJTzAioX/6B0DxyvDlo9l8+T3nLKbrczg2WLUyzd45L8RqfUMyGPzekbMvX2Ldkwg==", + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.4.tgz", + "integrity": "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A==", "dev": true, "license": "MIT", "engines": { @@ -11887,9 +11881,9 @@ } }, "node_modules/readdir-glob/node_modules/brace-expansion": { - "version": "2.0.2", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.2.tgz", - "integrity": "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ==", + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.3.tgz", + "integrity": "sha512-MCV/fYJEbqx68aE58kv2cA/kiky1G8vux3OR6/jbS+jIMe/6fJWa0DTzJU7dqijOWYwHi1t29FlfYI9uytqlpA==", "license": "MIT", "dependencies": { "balanced-match": "^1.0.0" @@ -13193,19 +13187,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/tinyglobby/node_modules/picomatch": { - "version": "4.0.3", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-4.0.3.tgz", - "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/tmp": { "version": "0.2.5", "resolved": "https://registry.npmjs.org/tmp/-/tmp-0.2.5.tgz", @@ -14314,9 +14295,9 @@ "license": "ISC" }, "node_modules/yaml": { - "version": "2.8.2", - "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.2.tgz", - "integrity": "sha512-mplynKqc1C2hTVYxd0PU2xQAc22TI1vShAYGksCCfxbn/dFwnHTNi1bvYsBTkhdUNtGIf5xNOg938rrSSYvS9A==", + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", "dev": true, "license": "ISC", "bin": { diff --git a/package.json b/package.json index 317f406..1068272 100644 --- a/package.json +++ b/package.json @@ -32,11 +32,11 @@ }, "dependencies": { "@keyv/redis": "^5.1.6", - "@mikro-orm/core": "^6.6.6", - "@mikro-orm/migrations": "^6.6.6", - "@mikro-orm/nestjs": "^6.1.1", - "@mikro-orm/postgresql": "^6.6.6", - "@mikro-orm/seeder": "^6.6.6", + "@mikro-orm/core": "^6.6.11", + "@mikro-orm/migrations": "^6.6.11", + "@mikro-orm/nestjs": "^6.1.2", + "@mikro-orm/postgresql": "^6.6.11", + "@mikro-orm/seeder": "^6.6.11", "@nest-lab/throttler-storage-redis": "^1.2.0", "@nestjs/bullmq": "^11.0.4", "@nestjs/cache-manager": "^3.1.0", @@ -80,7 +80,7 @@ "devDependencies": { "@eslint/eslintrc": "^3.2.0", "@eslint/js": "^9.18.0", - "@mikro-orm/cli": "^6.6.6", + "@mikro-orm/cli": "^6.6.11", "@nestjs/cli": "^11.0.0", "@nestjs/schematics": "^11.0.0", "@nestjs/testing": "^11.0.1", @@ -116,7 +116,18 @@ }, "@rushstack/node-core-library": { "ajv": "8.18.0" - } + }, + "brace-expansion@^1.1.7": "1.1.13", + "brace-expansion@^2.0.1": "2.0.3", + "brace-expansion@^2.0.2": "2.0.3", + "brace-expansion@^5.0.2": "5.0.5", + "handlebars": "^4.7.9", + "picomatch@^2.0.4": "2.3.2", + "picomatch@^2.3.1": "2.3.2", + "picomatch@^4.0.2": "4.0.4", + "picomatch@^4.0.3": "4.0.4", + "path-to-regexp": "8.4.1", + "yaml": "2.8.3" }, "jest": { "moduleFileExtensions": [ diff --git a/src/entities/analysis-pipeline.entity.ts b/src/entities/analysis-pipeline.entity.ts index 0495f14..f86da3e 100644 --- a/src/entities/analysis-pipeline.entity.ts +++ b/src/entities/analysis-pipeline.entity.ts @@ -25,6 +25,9 @@ import { RecommendationRun } from './recommendation-run.entity'; @Entity({ repository: () => AnalysisPipelineRepository }) @Index({ properties: ['semester', 'status'] }) export class AnalysisPipeline extends CustomBaseEntity { + @Property({ onUpdate: () => new Date() }) + override updatedAt: Date & Opt = new Date(); + @ManyToOne(() => Semester) semester!: Semester; diff --git a/src/modules/analysis/analysis.controller.spec.ts b/src/modules/analysis/analysis.controller.spec.ts index 9279332..9c02a55 100644 --- a/src/modules/analysis/analysis.controller.spec.ts +++ b/src/modules/analysis/analysis.controller.spec.ts @@ -143,6 +143,42 @@ describe('AnalysisController', () => { status: PipelineStatus.SENTIMENT_ANALYSIS, scope: { semester: 'S2026' }, coverage: { totalEnrolled: 100, submissionCount: 50 }, + stages: { + embeddings: { + status: 'completed', + progress: null, + startedAt: null, + completedAt: null, + }, + sentiment: { + status: 'processing', + progress: { current: 10, total: 50 }, + startedAt: '2026-03-13T10:00:00.000Z', + completedAt: null, + }, + sentimentGate: { + status: 'pending', + progress: null, + startedAt: null, + completedAt: null, + included: null, + excluded: null, + }, + topicModeling: { + status: 'pending', + progress: null, + startedAt: null, + completedAt: null, + }, + recommendations: { + status: 'pending', + progress: null, + startedAt: null, + completedAt: null, + }, + }, + retryable: false, + updatedAt: '2026-03-13T12:00:00.000Z', }; mockOrchestrator.GetPipelineStatus.mockResolvedValue(mockStatus); diff --git a/src/modules/analysis/dto/pipeline-status.dto.ts b/src/modules/analysis/dto/pipeline-status.dto.ts index 2a21c7e..a45df99 100644 --- a/src/modules/analysis/dto/pipeline-status.dto.ts +++ b/src/modules/analysis/dto/pipeline-status.dto.ts @@ -2,11 +2,19 @@ import { z } from 'zod'; const stageStatusSchema = z.object({ status: z.enum(['pending', 'processing', 'completed', 'failed', 'skipped']), - total: z.number().int().optional(), - completed: z.number().int().optional(), - processed: z.number().int().optional(), - included: z.number().int().nullable().optional(), - excluded: z.number().int().nullable().optional(), + progress: z + .object({ + current: z.number().int(), + total: z.number().int(), + }) + .nullable(), + startedAt: z.string().datetime().nullable(), + completedAt: z.string().datetime().nullable(), +}); + +const sentimentGateSchema = stageStatusSchema.extend({ + included: z.number().int().nullable(), + excluded: z.number().int().nullable(), }); export const pipelineStatusSchema = z.object({ @@ -31,13 +39,15 @@ export const pipelineStatusSchema = z.object({ stages: z.object({ embeddings: stageStatusSchema, sentiment: stageStatusSchema, - sentimentGate: stageStatusSchema, + sentimentGate: sentimentGateSchema, topicModeling: stageStatusSchema, recommendations: stageStatusSchema, }), warnings: z.array(z.string()), - errorMessage: z.string().nullable().optional(), + errorMessage: z.string().nullable(), + retryable: z.boolean(), createdAt: z.string().datetime(), + updatedAt: z.string().datetime(), confirmedAt: z.string().datetime().nullable(), completedAt: z.string().datetime().nullable(), }); diff --git a/src/modules/analysis/services/pipeline-orchestrator.service.spec.ts b/src/modules/analysis/services/pipeline-orchestrator.service.spec.ts index 5f78494..a1f6019 100644 --- a/src/modules/analysis/services/pipeline-orchestrator.service.spec.ts +++ b/src/modules/analysis/services/pipeline-orchestrator.service.spec.ts @@ -454,6 +454,32 @@ describe('PipelineOrchestratorService', () => { }); describe('GetPipelineStatus', () => { + const basePipeline = { + id: 'p1', + status: PipelineStatus.SENTIMENT_ANALYSIS, + semester: { id: 's1', code: 'S2026' }, + faculty: null, + questionnaireVersion: null, + department: { code: 'CCS' }, + program: null, + campus: null, + course: null, + totalEnrolled: 100, + submissionCount: 50, + commentCount: 40, + responseRate: 0.5, + warnings: [], + errorMessage: null, + sentimentGateIncluded: null, + sentimentGateExcluded: null, + createdAt: new Date('2026-03-13'), + updatedAt: new Date('2026-03-13T12:00:00Z'), + confirmedAt: new Date('2026-03-13'), + completedAt: null, + }; + + const sentimentRunCreatedAt = new Date('2026-03-13T10:00:00Z'); + it('should throw NotFoundException if pipeline not found', async () => { mockFork.findOne.mockResolvedValue(null); @@ -462,37 +488,25 @@ describe('PipelineOrchestratorService', () => { ); }); - it('should return composed pipeline status', async () => { - const pipeline = { - id: 'p1', - status: PipelineStatus.SENTIMENT_ANALYSIS, - semester: { id: 's1', code: 'S2026' }, - faculty: null, - questionnaireVersion: null, - department: { code: 'CCS' }, - program: null, - campus: null, - course: null, - totalEnrolled: 100, - submissionCount: 50, - commentCount: 40, - responseRate: 0.5, - warnings: [], - errorMessage: null, - sentimentGateIncluded: null, - sentimentGateExcluded: null, - createdAt: new Date('2026-03-13'), - confirmedAt: new Date('2026-03-13'), + it('should return reshaped pipeline status with progress and timestamps', async () => { + const pipeline = { ...basePipeline }; + const sentimentRun = { + status: RunStatus.PROCESSING, + submissionCount: 120, + createdAt: sentimentRunCreatedAt, completedAt: null, }; mockFork.findOne .mockResolvedValueOnce(pipeline) - .mockResolvedValueOnce({ status: RunStatus.PROCESSING }) + .mockResolvedValueOnce(sentimentRun) // sentiment run .mockResolvedValueOnce(null) // topic model run .mockResolvedValueOnce(null) // recommendation run .mockResolvedValueOnce({ updatedAt: new Date() }); // enrollment for lastSyncAt + // count: sentiment results + mockFork.count.mockResolvedValueOnce(47); + // find: submissions for course scoping mockFork.find.mockResolvedValueOnce([{ course: { id: 'c1' } }]); @@ -503,8 +517,187 @@ describe('PipelineOrchestratorService', () => { expect(status.scope.semester).toBe('S2026'); expect(status.scope.department).toBe('CCS'); expect(status.coverage.totalEnrolled).toBe(100); + expect(status.updatedAt).toBe('2026-03-13T12:00:00.000Z'); + + // Sentiment stage: real progress expect(status.stages.sentiment.status).toBe('processing'); + expect(status.stages.sentiment.progress).toEqual({ + current: 47, + total: 120, + }); + expect(status.stages.sentiment.startedAt).toBe( + sentimentRunCreatedAt.toISOString(), + ); + expect(status.stages.sentiment.completedAt).toBeNull(); + + // Binary stages: null progress expect(status.stages.topicModeling.status).toBe('pending'); + expect(status.stages.topicModeling.progress).toBeNull(); + expect(status.stages.embeddings.progress).toBeNull(); + expect(status.stages.recommendations.progress).toBeNull(); + + // All stage fields present (no undefined) + for (const stage of Object.values(status.stages)) { + expect(stage).toHaveProperty('status'); + expect(stage).toHaveProperty('progress'); + expect(stage).toHaveProperty('startedAt'); + expect(stage).toHaveProperty('completedAt'); + } + }); + + it('should return retryable: true when pipeline FAILED', async () => { + const pipeline = { + ...basePipeline, + status: PipelineStatus.FAILED, + errorMessage: 'Worker crashed', + }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(null) // sentiment run + .mockResolvedValueOnce(null) // topic model run + .mockResolvedValueOnce(null); // recommendation run + + // find: submissions for course scoping (no courses) + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.retryable).toBe(true); + }); + + it('should return retryable: false when pipeline not FAILED', async () => { + const pipeline = { ...basePipeline, status: PipelineStatus.COMPLETED }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(null) // sentiment run + .mockResolvedValueOnce(null) // topic model run + .mockResolvedValueOnce(null); // recommendation run + + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.retryable).toBe(false); + }); + + it('should return sentiment progress.current matching result count', async () => { + const pipeline = { ...basePipeline }; + const sentimentRun = { + status: RunStatus.PROCESSING, + submissionCount: 120, + createdAt: sentimentRunCreatedAt, + completedAt: null, + }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(sentimentRun) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null); + + mockFork.count.mockResolvedValueOnce(47); + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.stages.sentiment.progress).toEqual({ + current: 47, + total: 120, + }); + }); + + it('should return zero progress when sentiment run is PROCESSING with no results yet', async () => { + const pipeline = { ...basePipeline }; + const sentimentRun = { + status: RunStatus.PROCESSING, + submissionCount: 80, + createdAt: sentimentRunCreatedAt, + completedAt: null, + }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(sentimentRun) + .mockResolvedValueOnce(null) + .mockResolvedValueOnce(null); + + mockFork.count.mockResolvedValueOnce(0); + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.stages.sentiment.progress).toEqual({ + current: 0, + total: 80, + }); + }); + + it('should return skipped stages and retryable: false for CANCELLED pipeline', async () => { + const pipeline = { + ...basePipeline, + status: PipelineStatus.CANCELLED, + sentimentGateIncluded: null, + sentimentGateExcluded: null, + }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(null) // sentiment run + .mockResolvedValueOnce(null) // topic model run + .mockResolvedValueOnce(null); // recommendation run + + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.retryable).toBe(false); + expect(status.stages.embeddings.status).toBe('skipped'); + expect(status.stages.sentimentGate.status).toBe('skipped'); + }); + + it('should return embeddings failed when pipeline FAILED with no sentiment run', async () => { + const pipeline = { + ...basePipeline, + status: PipelineStatus.FAILED, + errorMessage: 'Embedding check failed', + }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(null) // sentiment run (none = embedding likely failed) + .mockResolvedValueOnce(null) // topic model run + .mockResolvedValueOnce(null); // recommendation run + + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.stages.embeddings.status).toBe('failed'); + }); + + it('should return sentiment gate included/excluded with completed status', async () => { + const pipeline = { + ...basePipeline, + status: PipelineStatus.TOPIC_MODELING, + sentimentGateIncluded: 80, + sentimentGateExcluded: 40, + }; + + mockFork.findOne + .mockResolvedValueOnce(pipeline) + .mockResolvedValueOnce(null) // sentiment run + .mockResolvedValueOnce(null) // topic model run + .mockResolvedValueOnce(null); // recommendation run + + mockFork.find.mockResolvedValueOnce([]); + + const status = await service.GetPipelineStatus('p1'); + + expect(status.stages.sentimentGate.status).toBe('completed'); + expect(status.stages.sentimentGate.included).toBe(80); + expect(status.stages.sentimentGate.excluded).toBe(40); }); }); }); diff --git a/src/modules/analysis/services/pipeline-orchestrator.service.ts b/src/modules/analysis/services/pipeline-orchestrator.service.ts index 6d41ae7..25b77c1 100644 --- a/src/modules/analysis/services/pipeline-orchestrator.service.ts +++ b/src/modules/analysis/services/pipeline-orchestrator.service.ts @@ -470,6 +470,14 @@ export class PipelineOrchestratorService { { orderBy: { createdAt: 'DESC' } }, ); + // Sentiment progress count + let sentimentCompleted = 0; + if (sentimentRun && sentimentRun.status !== RunStatus.PENDING) { + sentimentCompleted = await fork.count(SentimentResult, { + run: sentimentRun, + }); + } + // Compute lastEnrollmentSyncAt by scoping through courses in submission scope const scope = buildSubmissionScope(pipeline); let lastEnrollmentSyncAt: Date | null = null; @@ -497,25 +505,41 @@ export class PipelineOrchestratorService { } } - const stageStatus = (status: string, extras?: Record) => ({ - status: status as - | 'pending' - | 'processing' - | 'completed' - | 'failed' - | 'skipped', - ...extras, + const buildStage = ( + status: 'pending' | 'processing' | 'completed' | 'failed' | 'skipped', + run: { createdAt: Date; completedAt?: Date | null } | null, + progress: { current: number; total: number } | null = null, + ) => ({ + status, + progress, + startedAt: run?.createdAt?.toISOString() ?? null, + completedAt: run?.completedAt?.toISOString() ?? null, }); - const getRunStageStatus = ( + // If RunStatus gains new values, update this mapping to match the stage status union + const getRunStatus = ( run: SentimentRun | TopicModelRun | RecommendationRun | null, - ) => { - if (!run) return stageStatus('pending'); - return stageStatus(run.status.toLowerCase()); - }; - - // Determine embedding stage status based on pipeline status - const embeddingStatus = this.getEmbeddingStageStatus(pipeline); + ) => + run + ? (run.status.toLowerCase() as + | 'pending' + | 'processing' + | 'completed' + | 'failed') + : 'pending'; + + // Gate either completed (has data) or didn't. Top-level pipeline.status + // handles failure attribution — no per-stage failure tracking for MVP. + // FAILED pipelines: gate shows 'pending' (never completed) which is correct. + // CANCELLED pipelines: gate shows 'skipped' (explicitly not attempted). + const gateStatus = + pipeline.sentimentGateIncluded != null + ? 'completed' + : pipeline.status === PipelineStatus.SENTIMENT_GATE + ? 'processing' + : pipeline.status === PipelineStatus.CANCELLED + ? 'skipped' + : 'pending'; return { id: pipeline.id, @@ -537,27 +561,36 @@ export class PipelineOrchestratorService { lastEnrollmentSyncAt: lastEnrollmentSyncAt?.toISOString() || null, }, stages: { - embeddings: embeddingStatus, - sentiment: { - ...getRunStageStatus(sentimentRun), - total: pipeline.commentCount, + embeddings: buildStage( + this.getEmbeddingStageStatus(pipeline, sentimentRun), + null, + ), + sentiment: sentimentRun + ? buildStage(getRunStatus(sentimentRun), sentimentRun, { + current: Math.min( + sentimentCompleted, + sentimentRun.submissionCount, + ), + total: sentimentRun.submissionCount, + }) + : buildStage('pending', null), + sentimentGate: { + ...buildStage(gateStatus, null), + included: pipeline.sentimentGateIncluded ?? null, + excluded: pipeline.sentimentGateExcluded ?? null, }, - sentimentGate: stageStatus( - pipeline.sentimentGateIncluded !== null && - pipeline.sentimentGateIncluded !== undefined - ? 'completed' - : 'pending', - { - included: pipeline.sentimentGateIncluded ?? null, - excluded: pipeline.sentimentGateExcluded ?? null, - }, + topicModeling: buildStage(getRunStatus(topicModelRun), topicModelRun), + recommendations: buildStage( + getRunStatus(recommendationRun), + recommendationRun, ), - topicModeling: getRunStageStatus(topicModelRun), - recommendations: getRunStageStatus(recommendationRun), }, warnings: pipeline.warnings, - errorMessage: pipeline.errorMessage || null, + errorMessage: pipeline.errorMessage ?? null, + // Intent signal for future error categorization — currently equivalent to status === FAILED + retryable: pipeline.status === PipelineStatus.FAILED, createdAt: pipeline.createdAt.toISOString(), + updatedAt: pipeline.updatedAt.toISOString(), confirmedAt: pipeline.confirmedAt?.toISOString() || null, completedAt: pipeline.completedAt?.toISOString() || null, }; @@ -934,19 +967,24 @@ export class PipelineOrchestratorService { this.logger.error(`Pipeline ${pipeline.id} failed: ${error}`); } - private getEmbeddingStageStatus(pipeline: AnalysisPipeline): { - status: 'pending' | 'processing' | 'completed' | 'failed' | 'skipped'; - } { + private getEmbeddingStageStatus( + pipeline: AnalysisPipeline, + sentimentRun: SentimentRun | null, + ): 'pending' | 'processing' | 'completed' | 'failed' | 'skipped' { if (pipeline.status === PipelineStatus.EMBEDDING_CHECK) { - return { status: 'processing' }; + return 'processing'; } if (pipeline.status === PipelineStatus.AWAITING_CONFIRMATION) { - return { status: 'pending' }; + return 'pending'; } if (pipeline.status === PipelineStatus.CANCELLED) { - return { status: 'skipped' }; + return 'skipped'; + } + // If pipeline failed and never created a sentiment run, embedding is the likely failure point + if (pipeline.status === PipelineStatus.FAILED && !sentimentRun) { + return 'failed'; } // Past embedding check (sentiment, topic modeling, recommendations, completed, or failed later) - return { status: 'completed' }; + return 'completed'; } }