From 720c0d2fb3beeeb34d9a00b26cf0bca2c389b43c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 11:25:18 +0000 Subject: [PATCH 01/30] allow workflow to be executed directly through workspace --- packages/cli/src/execute/command.ts | 3 + packages/cli/src/options.ts | 4 ++ packages/cli/src/util/load-plan.ts | 67 +++++++--------------- packages/cli/src/util/validate-adaptors.ts | 11 ++-- packages/cli/turtle-power/output.json | 3 + packages/project/src/Workspace.ts | 7 +++ 6 files changed, 41 insertions(+), 54 deletions(-) create mode 100644 packages/cli/turtle-power/output.json diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts index 71cdc65cd..9c5e3c812 100644 --- a/packages/cli/src/execute/command.ts +++ b/packages/cli/src/execute/command.ts @@ -1,6 +1,7 @@ import yargs from 'yargs'; import { build, ensure, override } from '../util/command-builders'; import * as o from '../options'; +import * as po from '../projects/options'; import type { Opts } from '../options'; @@ -75,6 +76,8 @@ const options = [ o.timeout, o.trace, o.useAdaptorsMonorepo, + + po.workspace, ]; const executeCommand: yargs.CommandModule = { diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index acf6403ba..575c93c1f 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -67,6 +67,7 @@ export type Opts = { trace?: boolean; useAdaptorsMonorepo?: boolean; workflow: string; + workflowName?: string; // deprecated workflowPath?: string; @@ -377,7 +378,10 @@ export const inputPath: CLIOption = { opts.planPath = basePath; } else if (basePath?.endsWith('.js')) { opts.expressionPath = basePath; + } else if (!opts.expressionPath) { + opts.workflowName = basePath; } else { + // Ok, so we should stop defaulting to job.js const base = getBaseDir(opts); setDefaultValue(opts, 'expressionPath', nodePath.join(base, 'job.js')); } diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index 1bfe3a682..eb80b2a38 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises'; import path, { dirname } from 'node:path'; import { isPath } from '@openfn/compiler'; -import Project, { yamlToJson } from '@openfn/project'; +import Project, { Workspace, yamlToJson } from '@openfn/project'; import abort from './abort'; import expandAdaptors from './expand-adaptors'; @@ -18,6 +18,7 @@ const loadPlan = async ( | 'expressionPath' | 'planPath' | 'workflowPath' + | 'workflowName' | 'adaptors' | 'baseDir' | 'expandAdaptors' @@ -25,14 +26,29 @@ const loadPlan = async ( | 'globals' > & { workflow?: Opts['workflow']; + workspace?: string; // from project opts }, logger: Logger ): Promise => { // TODO all these paths probably need rethinkng now that we're supporting // so many more input formats - const { workflowPath, planPath, expressionPath } = options; + const { workflowPath, planPath, expressionPath, workflowName } = options; let workflowObj; + + if (workflowName || options.workflow) { + logger.debug( + 'Loading workflow from active project in workspace at ', + options.workspace + ); + const workspace = new Workspace(options.workspace); + // ah - difficult in this approach is that it loads the yaml, + // not the file system + // mayeb that's a bug in getActiveProject though? + const proj = await workspace.getCheckedOutProject(); + workflowObj = proj?.getWorkflow(workflowName || options.workflow); + } + if (options.path && /ya?ml$/.test(options.path)) { const content = await fs.readFile(path.resolve(options.path), 'utf-8'); options.baseDir = dirname(options.path); @@ -44,26 +60,6 @@ const loadPlan = async ( workflowObj = { workflow: rest, options: o }; } } - // Run a workflow from a project, with a path and workflow name - else if (options.path && options.workflow) { - options.baseDir = options.path; - return fromProject(options.path, options.workflow, options, logger); - } - - // Run a workflow from a project in the current working dir - // (no expression or workflow path, and no file extension) - if ( - !workflowObj && - !expressionPath && - !workflowPath && - !/\.(js|json|yaml)+$/.test(options.path || '') && - !options.workflow - ) { - // If the path has no extension - // Run a workflow from a project in the working dir - const workflow = options.path; - return fromProject(path.resolve('.'), workflow!, options, logger); - } if (!workflowObj && expressionPath) { return loadExpression(options, logger); @@ -71,8 +67,8 @@ const loadPlan = async ( const jsonPath = planPath || workflowPath; - if (!options.baseDir) { - options.baseDir = path.dirname(jsonPath!); + if (jsonPath && !options.baseDir) { + options.baseDir = path.dirname(jsonPath); } workflowObj = workflowObj ?? (await loadJson(jsonPath!, logger)); @@ -98,29 +94,6 @@ const loadPlan = async ( export default loadPlan; -const fromProject = async ( - rootDir: string, - workflowName: string, - options: Partial, - logger: Logger -): Promise => { - logger.debug('Loading Repo from ', path.resolve(rootDir)); - const project = await Project.from('fs', { root: rootDir }); - logger.debug('Loading workflow ', workflowName); - const workflow = project.getWorkflow(workflowName); - if (!workflow) { - throw new Error(`Workflow "${workflowName}" not found`); - } - return loadXPlan({ workflow }, options, logger); -}; - -// load a workflow from a repo -// if you do `openfn wf1` then we use this - you've asked for a workflow name, which we'll find -// const loadRepo = () => {}; - -// Load a workflow straight from yaml -// const loadYaml = () => {}; - const loadJson = async (workflowPath: string, logger: Logger): Promise => { let text: string; diff --git a/packages/cli/src/util/validate-adaptors.ts b/packages/cli/src/util/validate-adaptors.ts index 54dcb0f2d..bb1a55ad4 100644 --- a/packages/cli/src/util/validate-adaptors.ts +++ b/packages/cli/src/util/validate-adaptors.ts @@ -10,6 +10,7 @@ const validateAdaptors = async ( | 'repoDir' | 'workflowPath' | 'planPath' + | 'expressionPath' > & { workflow?: Opts['workflow']; }, @@ -18,11 +19,9 @@ const validateAdaptors = async ( if (options.skipAdaptorValidation) { return; } - const isPlan = options.planPath || options.workflowPath || options.workflow; - const hasDeclaredAdaptors = options.adaptors && options.adaptors.length > 0; - if (isPlan && hasDeclaredAdaptors) { + if (!options.expressionPath && hasDeclaredAdaptors) { logger.error('ERROR: adaptor and workflow provided'); logger.error( 'This is probably not what you meant to do. A workflow should declare an adaptor for each job.' @@ -30,10 +29,8 @@ const validateAdaptors = async ( throw new Error('adaptor and workflow provided'); } - // If no adaptor is specified, pass a warning - // (The runtime is happy to run without) - // This can be overriden from options - if (!isPlan && !hasDeclaredAdaptors) { + // If running a .js file directly and no adaptor is specified, send a warning + if (!options.expressionPath && !hasDeclaredAdaptors) { logger.warn('WARNING: No adaptor provided!'); logger.warn( 'This job will probably fail. Pass an adaptor with the -a flag, eg:' diff --git a/packages/cli/turtle-power/output.json b/packages/cli/turtle-power/output.json new file mode 100644 index 000000000..3699bcced --- /dev/null +++ b/packages/cli/turtle-power/output.json @@ -0,0 +1,3 @@ +{ + "x": 1 +} \ No newline at end of file diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index 2e06265fc..4911ef386 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -22,6 +22,8 @@ export class Workspace { // TODO activeProject should be the actual project activeProject?: l.ProjectMeta; + root: string; + private projects: Project[] = []; private projectPaths = new Map(); private isValid: boolean = false; @@ -30,6 +32,7 @@ export class Workspace { // Set validate to false to suppress warnings if a Workspace doesn't exist // This is appropriate if, say, fetching a project for the first time constructor(workspacePath: string, logger?: Logger, validate = true) { + this.root = workspacePath; this.logger = logger ?? createLogger('Workspace', { level: 'info' }); let context = { workspace: undefined, project: undefined }; @@ -112,6 +115,10 @@ export class Workspace { ); } + getCheckedOutProject() { + return Project.from('fs', { root: this.root }); + } + // TODO this needs to return default values // We should always rely on the workspace to load these values getConfig(): Partial { From 2fde3f2653fd8252fd0868a1e54e8c1ccee114e9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 11:54:15 +0000 Subject: [PATCH 02/30] types --- packages/cli/src/util/load-plan.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index eb80b2a38..2af84ddea 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -1,7 +1,7 @@ import fs from 'node:fs/promises'; import path, { dirname } from 'node:path'; import { isPath } from '@openfn/compiler'; -import Project, { Workspace, yamlToJson } from '@openfn/project'; +import { Workspace, yamlToJson } from '@openfn/project'; import abort from './abort'; import expandAdaptors from './expand-adaptors'; @@ -41,12 +41,9 @@ const loadPlan = async ( 'Loading workflow from active project in workspace at ', options.workspace ); - const workspace = new Workspace(options.workspace); - // ah - difficult in this approach is that it loads the yaml, - // not the file system - // mayeb that's a bug in getActiveProject though? + const workspace = new Workspace(options.workspace!); const proj = await workspace.getCheckedOutProject(); - workflowObj = proj?.getWorkflow(workflowName || options.workflow); + workflowObj = proj?.getWorkflow(workflowName || options.workflow!); } if (options.path && /ya?ml$/.test(options.path)) { From f5af8622bb28c97b94992a4393679fafd893931c Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 12:05:53 +0000 Subject: [PATCH 03/30] remove default job.js --- packages/cli/src/options.ts | 4 ---- packages/cli/test/execute/options.test.ts | 6 ------ 2 files changed, 10 deletions(-) diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index 575c93c1f..65f3d0d8b 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -380,10 +380,6 @@ export const inputPath: CLIOption = { opts.expressionPath = basePath; } else if (!opts.expressionPath) { opts.workflowName = basePath; - } else { - // Ok, so we should stop defaulting to job.js - const base = getBaseDir(opts); - setDefaultValue(opts, 'expressionPath', nodePath.join(base, 'job.js')); } }, }; diff --git a/packages/cli/test/execute/options.test.ts b/packages/cli/test/execute/options.test.ts index f6d3bc56b..038d58950 100644 --- a/packages/cli/test/execute/options.test.ts +++ b/packages/cli/test/execute/options.test.ts @@ -75,12 +75,6 @@ test('enable immutability', (t) => { t.true(options.immutable); }); -test('default job path', (t) => { - const options = parse('execute /tmp/my-job/ --immutable'); - t.is(options.path, '/tmp/my-job/'); - t.is(options.expressionPath, '/tmp/my-job/job.js'); -}); - test('enable json logging', (t) => { const options = parse('execute job.js --log-json'); t.true(options.logJson); From 3b7d4545e69f2f01af639f7d6aaa7d251b3d10d9 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 12:06:27 +0000 Subject: [PATCH 04/30] changeset --- .changeset/curly-laws-walk.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/curly-laws-walk.md diff --git a/.changeset/curly-laws-walk.md b/.changeset/curly-laws-walk.md new file mode 100644 index 000000000..74916cb78 --- /dev/null +++ b/.changeset/curly-laws-walk.md @@ -0,0 +1,5 @@ +--- +'@openfn/cli': patch +--- + +When executing jobs, the CLI no longer defaults the path t job.js From 181813c9ade4e22c85780870a78f175d2814e0b2 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 16:10:15 +0000 Subject: [PATCH 05/30] integration test --- integration-tests/cli/test/project-v2.test.ts | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index 295ea83d9..129c6e5cc 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -26,6 +26,7 @@ options: retention_policy: retain_all workflows: - name: Hello Workflow + start: trigger steps: - id: trigger type: webhook @@ -40,8 +41,8 @@ workflows: uuid: add150e9-8616-48ca-844e-8aaa489c7a10 - id: transform-data name: Transform data - expression: |- - // TODO + expression: | + fn(() => ({ x: 1})) adaptor: "@openfn/language-dhis2@8.0.4" openfn: uuid: a9f64216-7974-469d-8415-d6d9baf2f92e @@ -81,6 +82,7 @@ options: retention_policy: retain_all workflows: - name: Hello Workflow + start: trigger steps: - id: trigger type: webhook @@ -95,7 +97,7 @@ workflows: uuid: f34146b5-de43-4b05-ac00-3b4f327e62ec - id: transform-data name: Transform data - expression: |- + expression: | fn() adaptor: "@openfn/language-dhis2@8.0.4" openfn: @@ -146,6 +148,7 @@ test.serial('Checkout a project', async (t) => { workflowYaml, `id: hello-workflow name: Hello Workflow +start: trigger options: {} steps: - id: trigger @@ -176,8 +179,6 @@ test.serial('merge a project', async (t) => { 'utf8' ).then((str) => str.trim()); - await run(`openfn checkout sandboxing-simple -w ${projectsPath}`); - // assert the initial step code const initial = await readStep(); t.is(initial, '// TODO'); @@ -191,3 +192,17 @@ test.serial('merge a project', async (t) => { const merged = await readStep(); t.is(merged, 'fn()'); }); + +test.serial('execute a workflow from the checked out project', async (t) => { + // cheeky bonus test of checkout by alias + await run(`openfn checkout main --log debug -w ${projectsPath}`); + + // execute a workflow + await run( + `openfn hello-workflow -o /tmp/output.json --workspace ${projectsPath}` + ); + + const output = await readFile('/tmp/output.json', 'utf8'); + const finalState = JSON.parse(output); + t.deepEqual(finalState, { x: 1 }); +}); From 57b0571770d7db2a2340350a82a057ce2c6733a6 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 16:12:37 +0000 Subject: [PATCH 06/30] another integration test --- integration-tests/cli/test/project-v1.test.ts | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index 1f584d0ae..e5444122d 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -35,7 +35,7 @@ workflows: jobs: - name: Transform data body: | - // TODO + fn(() => ({ x: 1})) adaptor: "@openfn/language-common@latest" id: b8b780f3-98dd-4244-880b-e534d8f24547 project_credential_id: null @@ -151,7 +151,7 @@ steps: path.resolve(projectsPath, 'workflows/my-workflow/transform-data.js'), 'utf8' ); - t.is(expr.trim(), '// TODO'); + t.is(expr.trim(), 'fn(() => ({ x: 1}))'); }); // requires the prior test to run @@ -164,7 +164,7 @@ test.serial('merge a project', async (t) => { // assert the initial step code const initial = await readStep(); - t.is(initial, '// TODO'); + t.is(initial, 'fn(() => ({ x: 1}))'); // Run the merge await run(`openfn merge hello-world-staging -w ${projectsPath} --force`); @@ -173,3 +173,17 @@ test.serial('merge a project', async (t) => { const merged = await readStep(); t.is(merged, "log('hello world')"); }); + +test.serial('execute a workflow from the checked out project', async (t) => { + // cheeky bonus test of checkout by alias + await run(`openfn checkout main --log debug -w ${projectsPath}`); + + // execute a workflow + await run( + `openfn hello-workflow -o /tmp/output.json --workspace ${projectsPath}` + ); + + const output = await readFile('/tmp/output.json', 'utf8'); + const finalState = JSON.parse(output); + t.deepEqual(finalState, { x: 1 }); +}); From aab2b99d5adaa3542ac1a76d149f988b7fcdfe06 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 16:21:35 +0000 Subject: [PATCH 07/30] support credentials on workspace config --- packages/lexicon/core.d.ts | 1 + packages/project/src/Workspace.ts | 4 ++++ packages/project/src/util/config.ts | 1 + packages/project/test/parse/from-fs.test.ts | 2 ++ packages/project/test/parse/from-path.test.ts | 1 + packages/project/test/parse/from-project.test.ts | 1 + packages/project/test/serialize/to-fs.test.ts | 1 + packages/project/test/util/config.test.ts | 1 + packages/project/test/workspace.test.ts | 1 + 9 files changed, 13 insertions(+) diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index f25e05601..e2d6256ab 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -87,6 +87,7 @@ export interface WorkspaceFile { } export interface WorkspaceConfig { + credentials?: string; dirs: { workflows: string; projects: string; diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index 4911ef386..6641c8f07 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -119,6 +119,10 @@ export class Workspace { return Project.from('fs', { root: this.root }); } + getCredentialMap() { + return this.config.credentials; + } + // TODO this needs to return default values // We should always rely on the workspace to load these values getConfig(): Partial { diff --git a/packages/project/src/util/config.ts b/packages/project/src/util/config.ts index 47dd1494b..67f7e9fec 100644 --- a/packages/project/src/util/config.ts +++ b/packages/project/src/util/config.ts @@ -8,6 +8,7 @@ import Project from '../Project'; // Initialize and default Workspace (and Project) config export const buildConfig = (config: Partial = {}) => ({ + credentials: 'credentials.yaml', ...config, dirs: { projects: config.dirs?.projects ?? '.projects', diff --git a/packages/project/test/parse/from-fs.test.ts b/packages/project/test/parse/from-fs.test.ts index 1a3d1e103..3b9eaf99e 100644 --- a/packages/project/test/parse/from-fs.test.ts +++ b/packages/project/test/parse/from-fs.test.ts @@ -40,6 +40,7 @@ test.serial('should load workspace config from json', async (t) => { t.deepEqual(project.config, { x: 1, + credentials: 'credentials.yaml', dirs: { projects: '.projects', workflows: 'workflows' }, formats: { openfn: 'json', project: 'json', workflow: 'json' }, }); @@ -62,6 +63,7 @@ test.serial('should load workspace config from yaml', async (t) => { const project = await parseProject({ root: '/ws' }); t.deepEqual(project.config, { + credentials: 'credentials.yaml', x: 1, dirs: { projects: '.projects', workflows: 'workflows' }, formats: { openfn: 'yaml', project: 'yaml', workflow: 'yaml' }, diff --git a/packages/project/test/parse/from-path.test.ts b/packages/project/test/parse/from-path.test.ts index 9d31f65bf..dd47a5e56 100644 --- a/packages/project/test/parse/from-path.test.ts +++ b/packages/project/test/parse/from-path.test.ts @@ -76,6 +76,7 @@ test.serial('should use workspace config', async (t) => { t.is(project.name, proj.name); t.deepEqual(project.config, { + credentials: 'credentials.yaml', dirs: { projects: 'p', workflows: 'w', diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts index 4da3625ae..288c80828 100644 --- a/packages/project/test/parse/from-project.test.ts +++ b/packages/project/test/parse/from-project.test.ts @@ -188,6 +188,7 @@ test('import with custom config', async (t) => { // note that alias should have been removed from config t.deepEqual(proj.config, { + credentials: 'credentials.yaml', dirs: { projects: 'p', workflows: 'w', diff --git a/packages/project/test/serialize/to-fs.test.ts b/packages/project/test/serialize/to-fs.test.ts index eb4063592..12ce41f43 100644 --- a/packages/project/test/serialize/to-fs.test.ts +++ b/packages/project/test/serialize/to-fs.test.ts @@ -216,6 +216,7 @@ test('toFs: extract a project with 1 workflow and 1 step', (t) => { const config = JSON.parse(files['openfn.json']); t.deepEqual(config, { workspace: { + credentials: 'credentials.yaml', formats: { openfn: 'json', project: 'yaml', workflow: 'json' }, dirs: { projects: '.projects', workflows: 'workflows' }, }, diff --git a/packages/project/test/util/config.test.ts b/packages/project/test/util/config.test.ts index cb336ad36..82cde69a4 100644 --- a/packages/project/test/util/config.test.ts +++ b/packages/project/test/util/config.test.ts @@ -176,6 +176,7 @@ test('generate openfn.yaml', (t) => { uuid: 1234 id: my-project workspace: + credentials: credentials.yaml formats: openfn: yaml project: yaml diff --git a/packages/project/test/workspace.test.ts b/packages/project/test/workspace.test.ts index 141b3cb96..0410c0cd9 100644 --- a/packages/project/test/workspace.test.ts +++ b/packages/project/test/workspace.test.ts @@ -260,6 +260,7 @@ test('load from custom path', (t) => { test('load config', (t) => { const ws = new Workspace('/ws'); t.deepEqual(ws.config, { + credentials: 'credentials.yaml', formats: { openfn: 'yaml', project: 'yaml', From 80b4ab540c8d238992ff1d971ce56dcdc79608d1 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Fri, 9 Jan 2026 16:28:56 +0000 Subject: [PATCH 08/30] remove mock --- packages/cli/src/util/load-plan.ts | 5 +++++ packages/cli/test/execute/execute.test.ts | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index 2af84ddea..6a3857465 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -24,6 +24,7 @@ const loadPlan = async ( | 'expandAdaptors' | 'path' | 'globals' + | 'credentials' > & { workflow?: Opts['workflow']; workspace?: string; // from project opts @@ -44,6 +45,10 @@ const loadPlan = async ( const workspace = new Workspace(options.workspace!); const proj = await workspace.getCheckedOutProject(); workflowObj = proj?.getWorkflow(workflowName || options.workflow!); + + if (!options.credentials) { + options.credentials = workspace.getConfig().credentials; + } } if (options.path && /ya?ml$/.test(options.path)) { diff --git a/packages/cli/test/execute/execute.test.ts b/packages/cli/test/execute/execute.test.ts index 1e28d46d4..e36e40336 100644 --- a/packages/cli/test/execute/execute.test.ts +++ b/packages/cli/test/execute/execute.test.ts @@ -130,7 +130,7 @@ test.serial('run a workflow with a JSON credential map', async (t) => { t.is(result.b, 'b'); }); -test.serial.skip('run a workflow with a YAML credential map', async (t) => { +test.serial('run a workflow with a YAML credential map', async (t) => { const workflow = { workflow: { steps: [ From 85cde896b5cf3726214513684854b56affcdae95 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Sun, 11 Jan 2026 12:27:24 +0000 Subject: [PATCH 09/30] project: better handling of start in workflow.yaml --- packages/project/src/Workflow.ts | 5 ++- packages/project/src/Workspace.ts | 2 +- packages/project/src/parse/from-fs.ts | 25 +++++++++++-- packages/project/test/parse/from-fs.test.ts | 41 ++++++++++++++++++++- 4 files changed, 65 insertions(+), 8 deletions(-) diff --git a/packages/project/src/Workflow.ts b/packages/project/src/Workflow.ts index 28b8c982d..0f57dc94d 100644 --- a/packages/project/src/Workflow.ts +++ b/packages/project/src/Workflow.ts @@ -37,7 +37,8 @@ class Workflow { steps, history, start: _start, - ...options + options, + ...rest } = workflow; if (!(id || name)) { throw new Error('A Workflow MUST have a name or id'); @@ -53,7 +54,7 @@ class Workflow { } this.openfn = openfn; - this.options = options; + this.options = Object.assign({}, options, rest); this._buildIndex(); } diff --git a/packages/project/src/Workspace.ts b/packages/project/src/Workspace.ts index 6641c8f07..16289bc34 100644 --- a/packages/project/src/Workspace.ts +++ b/packages/project/src/Workspace.ts @@ -116,7 +116,7 @@ export class Workspace { } getCheckedOutProject() { - return Project.from('fs', { root: this.root }); + return Project.from('fs', { root: this.root, config: this.config }); } getCredentialMap() { diff --git a/packages/project/src/parse/from-fs.ts b/packages/project/src/parse/from-fs.ts index 612c2b838..d1ff194d2 100644 --- a/packages/project/src/parse/from-fs.ts +++ b/packages/project/src/parse/from-fs.ts @@ -15,6 +15,7 @@ import { Logger } from '@openfn/logger'; export type FromFsConfig = { root: string; + config?: Partial; logger?: Logger; }; @@ -27,7 +28,7 @@ export const parseProject = async (options: FromFsConfig) => { const { type, content } = findWorkspaceFile(root); const context = loadWorkspaceFile(content, type as any); - const config = buildConfig(context.workspace); + const config = buildConfig(options.config ?? context.workspace); const proj: any = { id: context.project?.id, @@ -44,7 +45,7 @@ export const parseProject = async (options: FromFsConfig) => { const workflowDir = (config as any).workflowRoot ?? config.dirs?.workflows ?? 'workflows'; const fileType = config.formats?.workflow ?? 'yaml'; - const pattern = `${root}/${workflowDir}/*/*.${fileType}`; + const pattern = path.resolve(root, workflowDir) + `/**/*.${fileType}`; const candidateWfs = await glob(pattern, { ignore: ['**node_modules/**', '**tmp**'], }); @@ -52,8 +53,24 @@ export const parseProject = async (options: FromFsConfig) => { for (const filePath of candidateWfs) { const candidate = await fs.readFile(filePath, 'utf-8'); try { - const wf = + let wf = fileType === 'yaml' ? yamlToJson(candidate) : JSON.parse(candidate); + + if (wf.workflow) { + // Support the { workflow, options } workflow format + // TODO Would like to remove this on the next major + if (wf.options) { + const { start, ...rest } = wf.options; + if (start) { + wf.workflow.start = start; + } + if (rest) { + wf.workflow.options = Object.assign({}, wf.workflow.options, rest); + } + } + wf = wf.workflow; + } + if (wf.id && Array.isArray(wf.steps)) { //logger?.log('Loading workflow at ', filePath); // TODO logger.debug for (const step of wf.steps) { @@ -71,7 +88,7 @@ export const parseProject = async (options: FromFsConfig) => { } } - // Now track UUIDs for edges against state + // convert edge conditions for (const target in step.next || {}) { if (typeof step.next[target] === 'boolean') { const bool = step.next[target]; diff --git a/packages/project/test/parse/from-fs.test.ts b/packages/project/test/parse/from-fs.test.ts index 3b9eaf99e..c5fa400a2 100644 --- a/packages/project/test/parse/from-fs.test.ts +++ b/packages/project/test/parse/from-fs.test.ts @@ -70,7 +70,7 @@ test.serial('should load workspace config from yaml', async (t) => { }); }); -test.serial('should load single workflow', async (t) => { +test.serial('should load single workflow in new flat format', async (t) => { mockFile('/ws/openfn.yaml', buildConfig()); mockFile('/ws/workflows/my-workflow/my-workflow.yaml', { @@ -82,6 +82,7 @@ test.serial('should load single workflow', async (t) => { expression: 'job.js', }, ], + start: 'a', }); mockFile('/ws/workflows/my-workflow/job.js', `fn(s => s)`); @@ -94,8 +95,46 @@ test.serial('should load single workflow', async (t) => { t.truthy(wf); t.is(wf.id, 'my-workflow'); t.is(wf.name, 'My Workflow'); + t.is(wf.start, 'a'); }); +// hmm, maybe I shouldn't support this, because it puts some wierd stuff in the code +// and new CLI will just use the new format +test.serial( + 'should load single workflow in old { workflow, options } format', + async (t) => { + mockFile('/ws/openfn.yaml', buildConfig()); + + mockFile('/ws/workflows/my-workflow/my-workflow.yaml', { + workflow: { + id: 'my-workflow', + name: 'My Workflow', + steps: [ + { + id: 'a', + expression: 'job.js', + }, + ], + }, + options: { + start: 'a', + }, + }); + + mockFile('/ws/workflows/my-workflow/job.js', `fn(s => s)`); + + const project = await parseProject({ root: '/ws' }); + + t.is(project.workflows.length, 1); + + const wf = project.getWorkflow('my-workflow'); + t.truthy(wf); + t.is(wf.id, 'my-workflow'); + t.is(wf.name, 'My Workflow'); + t.is(wf.start, 'a'); + } +); + test.serial('should load single workflow from json', async (t) => { mockFile( '/ws/openfn.yaml', From 5104b0bc1a695f6736c53959ee644379138f8131 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Sun, 11 Jan 2026 12:29:06 +0000 Subject: [PATCH 10/30] apply credentials map --- packages/cli/src/util/load-plan.ts | 5 +- packages/cli/test/util/load-plan.test.ts | 72 ++++++++++++++++++++++++ packages/lexicon/core.d.ts | 3 + 3 files changed, 79 insertions(+), 1 deletion(-) diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index 6a3857465..9342089f2 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -44,7 +44,10 @@ const loadPlan = async ( ); const workspace = new Workspace(options.workspace!); const proj = await workspace.getCheckedOutProject(); - workflowObj = proj?.getWorkflow(workflowName || options.workflow!); + // TODO - throw if workflow not found! + workflowObj = { + workflow: proj?.getWorkflow(workflowName || options.workflow!)?.toJSON(), + }; if (!options.credentials) { options.credentials = workspace.getConfig().credentials; diff --git a/packages/cli/test/util/load-plan.test.ts b/packages/cli/test/util/load-plan.test.ts index 720e27348..98e9dc89d 100644 --- a/packages/cli/test/util/load-plan.test.ts +++ b/packages/cli/test/util/load-plan.test.ts @@ -518,3 +518,75 @@ options: t.truthy(plan); t.deepEqual(plan, sampleXPlan); }); + +test.serial('xplan: load a workflow through a Workspace', async (t) => { + mock({ + '/tmp/workflows/wf.yaml': ` +id: wf +steps: + - id: a + expression: x() +`, + '/tmp/openfn.yaml': ` +dirs: + workflows: /tmp/workflows +`, + }); + + const opts = { + // TODO is worked out through yargs via the inputPath option + workflowName: 'wf', + workspace: '/tmp', + }; + + const plan = await loadPlan(opts, logger); + t.truthy(plan); + t.deepEqual(plan, { + workflow: { + id: 'wf', + steps: [{ id: 'a', expression: 'x()', adaptors: [] }], + history: [], + }, + options: {}, + }); +}); + +test.serial( + 'xplan: load a workflow through a project .yaml and apply the credentials map by default', + async (t) => { + mock({ + '/tmp/workflows/wf.yaml': ` +id: wf +steps: + - id: a + expression: x() +start: a +`, + '/tmp/openfn.yaml': ` +credentials: /creds.yaml +dirs: + workflows: /tmp/workflows +`, + '/creds.yaml': `x: y`, + }); + const opts = { + workflowName: 'wf', + workspace: '/tmp', + }; + + const plan = await loadPlan(opts, logger); + + t.truthy(plan); + t.deepEqual(plan, { + workflow: { + id: 'wf', + steps: [{ id: 'a', expression: 'x()', adaptors: [] }], + history: [], + start: 'a', + }, + options: {}, + }); + + t.is(opts.credentials, '/creds.yaml'); + } +); diff --git a/packages/lexicon/core.d.ts b/packages/lexicon/core.d.ts index e2d6256ab..5e596d5e5 100644 --- a/packages/lexicon/core.d.ts +++ b/packages/lexicon/core.d.ts @@ -165,6 +165,9 @@ export type Workflow = { /** The default start node - the one the workflow was designed for (the trigger) */ start?: string; + + /** extra options from the app. Not really used */ + options?: any; }; export type StepId = string; From 7620d615e44508adf589fad17d26622aff1b2134 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Sun, 11 Jan 2026 12:58:20 +0000 Subject: [PATCH 11/30] add integration test for execute + credetial map --- integration-tests/cli/test/project-v2.test.ts | 54 +++++++++++++++++++ packages/cli/src/execute/command.ts | 2 +- packages/cli/src/execute/handler.ts | 2 +- 3 files changed, 56 insertions(+), 2 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index 129c6e5cc..1f4e83044 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -206,3 +206,57 @@ test.serial('execute a workflow from the checked out project', async (t) => { const finalState = JSON.parse(output); t.deepEqual(finalState, { x: 1 }); }); + +test.serial( + 'execute a workflow from the checked out project with a credential map', + async (t) => { + await run(`openfn checkout main --log debug -w ${projectsPath}`); + + // Modify the checked out workflow code + await writeFile( + 'tmp/project/workflows/hello-workflow/transform-data.js', + `fn(s => ({ user: s.configuration.username }))` + ); + + // Modify the checked out workflow to add a credential + await writeFile( + 'tmp/project/workflows/hello-workflow/hello-workflow.yaml', + `id: hello-workflow +name: Hello Workflow +start: trigger +options: {} +steps: + - id: trigger + type: webhook + next: + transform-data: + disabled: false + condition: true + - id: transform-data + name: Transform data + configuration: 1234 + adaptor: "@openfn/language-dhis2@8.0.4" + expression: ./transform-data.js +` + ); + + // add the credential map to the yaml + await writeFile('tmp/project/openfn.yaml', `credentials: creds.yaml`); + + // write the credential map + await writeFile( + 'tmp/project/creds.yaml', + `1234: +username: pparker` + ); + + // finally execute the workflow + const { stdout } = await run( + `openfn hello-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` + ); + console.log(stdout); + const output = await readFile('/tmp/output.json', 'utf8'); + const finalState = JSON.parse(output); + t.deepEqual(finalState, { user: 'pparker' }); + } +); diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts index 9c5e3c812..5107aee40 100644 --- a/packages/cli/src/execute/command.ts +++ b/packages/cli/src/execute/command.ts @@ -5,7 +5,7 @@ import * as po from '../projects/options'; import type { Opts } from '../options'; -export type ExecuteOptions = Required< +export type ExecuteOptions = { workspace?: string } & Required< Pick< Opts, | 'apiKey' diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts index 66f0d5fcc..a61d61807 100644 --- a/packages/cli/src/execute/handler.ts +++ b/packages/cli/src/execute/handler.ts @@ -57,7 +57,7 @@ const loadAndApplyCredentialMap = async ( if (options.credentials) { try { const credsRaw = await readFile( - path.resolve(options.credentials), + path.resolve(options.workspace!, options.credentials), 'utf8' ); if (options.credentials.endsWith('.json')) { From 41b39722e0d73a1b4a631f01b4240a8829cda6ba Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Sun, 11 Jan 2026 14:00:42 +0000 Subject: [PATCH 12/30] fix test --- packages/cli/test/projects/checkout.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cli/test/projects/checkout.test.ts b/packages/cli/test/projects/checkout.test.ts index 3ffd1df28..93ecc56f9 100644 --- a/packages/cli/test/projects/checkout.test.ts +++ b/packages/cli/test/projects/checkout.test.ts @@ -430,6 +430,7 @@ test.serial('respect openfn.yaml settings', async (t) => { uuid: id: staging workspace: + credentials: credentials.yaml dirs: projects: p workflows: w From 6228c826e63ba12f4f6e59f389fed812fdca3252 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 11:54:13 +0000 Subject: [PATCH 13/30] remove unused test --- packages/cli/test/compile/options.test.ts | 6 ---- .../cli/test/options/ensure/inputPath.test.ts | 30 ------------------- 2 files changed, 36 deletions(-) diff --git a/packages/cli/test/compile/options.test.ts b/packages/cli/test/compile/options.test.ts index e3d896f8d..a43b76b6e 100644 --- a/packages/cli/test/compile/options.test.ts +++ b/packages/cli/test/compile/options.test.ts @@ -49,12 +49,6 @@ test("don't expand adaptors if --no-expand-adaptors is set", (t) => { t.deepEqual(options.adaptors, ['common']); }); -test('default job path', (t) => { - const options = parse('compile /tmp/my-job/ --immutable'); - t.is(options.path, '/tmp/my-job/'); - t.is(options.expressionPath, '/tmp/my-job/job.js'); -}); - test('enable json logging', (t) => { const options = parse('compile job.js --log-json'); t.true(options.logJson); diff --git a/packages/cli/test/options/ensure/inputPath.test.ts b/packages/cli/test/options/ensure/inputPath.test.ts index e62a827b7..4083203ea 100644 --- a/packages/cli/test/options/ensure/inputPath.test.ts +++ b/packages/cli/test/options/ensure/inputPath.test.ts @@ -10,33 +10,3 @@ test('sets expressionPath using path', (t) => { t.is(opts.expressionPath, 'jam.js'); }); - -test('sets expressionPath to path/job.js if path is a folder', (t) => { - const opts = { - path: '/jam', - } as Opts; - - inputPath.ensure!(opts); - - t.is(opts.expressionPath, '/jam/job.js'); -}); - -test('sets expressionPath to path/job.js if path is a folder (trailing slash)', (t) => { - const opts = { - path: '/jam/', - } as Opts; - - inputPath.ensure!(opts); - - t.is(opts.expressionPath, '/jam/job.js'); -}); - -test.skip('set workflowPath if path ends in json', (t) => { - const opts = { - path: 'workflow.json', - } as Opts; - - inputPath.ensure!(opts); - - t.is(opts.workflowPath, 'workflow.json'); -}); From 7184a5aaa2b181b5499ddc40e55c52e5d1ac56db Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 12:05:47 +0000 Subject: [PATCH 14/30] format --- packages/cli/test/compile/compile.test.ts | 142 +++++++++++++--------- 1 file changed, 83 insertions(+), 59 deletions(-) diff --git a/packages/cli/test/compile/compile.test.ts b/packages/cli/test/compile/compile.test.ts index af5b5556b..158644d2d 100644 --- a/packages/cli/test/compile/compile.test.ts +++ b/packages/cli/test/compile/compile.test.ts @@ -34,7 +34,7 @@ test.serial('compile from source string', async (t) => { const opts = {} as CompileOptions; - const result = await compile(job, opts, mockLog) + const result = await compile(job, opts, mockLog); const expected = 'export default [x()];'; t.is(result.code, expected); @@ -92,53 +92,68 @@ test.serial('throw an AbortError if a job is uncompilable', async (t) => { t.assert(logger._find('error', /critical error: aborting command/i)); }); -test.serial('throw an AbortError if an xplan contains an uncompilable job', async (t) => { - const plan: ExecutionPlan = { - workflow: { - steps: [{ id: 'a', expression: 'x b' }], - }, - options: {}, - }; - - const opts = {} as CompileOptions; - - const logger = createMockLogger(); - await t.throwsAsync(() => compile(plan, opts, logger), { - message: 'Failed to compile job a', - }); +test.serial( + 'throw an AbortError if an xplan contains an uncompilable job', + async (t) => { + const plan: ExecutionPlan = { + workflow: { + steps: [{ id: 'a', expression: 'x b' }], + }, + options: {}, + }; + + const opts = {} as CompileOptions; + + const logger = createMockLogger(); + await t.throwsAsync(() => compile(plan, opts, logger), { + message: 'Failed to compile job a', + }); - t.assert(logger._find('error', /unexpected token/i)); - t.assert(logger._find('always', /check the syntax of the job expression/i)); - t.assert(logger._find('error', /critical error: aborting command/i)); -}); + t.assert(logger._find('error', /unexpected token/i)); + t.assert(logger._find('always', /check the syntax of the job expression/i)); + t.assert(logger._find('error', /critical error: aborting command/i)); + } +); -test.serial('stripVersionSpecifier: remove version specifier from @openfn', (t) => { - const specifier = '@openfn/language-common@3.0.0-rc2'; - const transformed = stripVersionSpecifier(specifier); - const expected = '@openfn/language-common'; - t.assert(transformed == expected); -}); +test.serial( + 'stripVersionSpecifier: remove version specifier from @openfn', + (t) => { + const specifier = '@openfn/language-common@3.0.0-rc2'; + const transformed = stripVersionSpecifier(specifier); + const expected = '@openfn/language-common'; + t.assert(transformed == expected); + } +); -test.serial('stripVersionSpecifier: remove version specifier from arbitrary package', (t) => { - const specifier = 'ava@1.0.0'; - const transformed = stripVersionSpecifier(specifier); - const expected = 'ava'; - t.assert(transformed == expected); -}); +test.serial( + 'stripVersionSpecifier: remove version specifier from arbitrary package', + (t) => { + const specifier = 'ava@1.0.0'; + const transformed = stripVersionSpecifier(specifier); + const expected = 'ava'; + t.assert(transformed == expected); + } +); -test.serial('stripVersionSpecifier: remove version specifier from arbitrary namespaced package', (t) => { - const specifier = '@ava/some-pkg@^1'; - const transformed = stripVersionSpecifier(specifier); - const expected = '@ava/some-pkg'; - t.assert(transformed == expected); -}); +test.serial( + 'stripVersionSpecifier: remove version specifier from arbitrary namespaced package', + (t) => { + const specifier = '@ava/some-pkg@^1'; + const transformed = stripVersionSpecifier(specifier); + const expected = '@ava/some-pkg'; + t.assert(transformed == expected); + } +); -test.serial("stripVersionSpecifier: do nothing if there's no specifier", (t) => { - const specifier = '@openfn/language-common'; - const transformed = stripVersionSpecifier(specifier); - const expected = '@openfn/language-common'; - t.assert(transformed == expected); -}); +test.serial( + "stripVersionSpecifier: do nothing if there's no specifier", + (t) => { + const specifier = '@openfn/language-common'; + const transformed = stripVersionSpecifier(specifier); + const expected = '@openfn/language-common'; + t.assert(transformed == expected); + } +); test.serial('loadTransformOptions: do nothing', async (t) => { const opts = {}; @@ -157,31 +172,40 @@ test.serial( } ); -test.serial('resolveSpecifierPath: return a relative path if passed', async (t) => { - const path = await resolveSpecifierPath('pkg=./a', '/repo', mockLog); - t.assert(path === './a'); -}); +test.serial( + 'resolveSpecifierPath: return a relative path if passed', + async (t) => { + const path = await resolveSpecifierPath('pkg=./a', '/repo', mockLog); + t.assert(path === './a'); + } +); -test.serial('resolveSpecifierPath: return an absolute path if passed', async (t) => { - const path = await resolveSpecifierPath('pkg=/a', '/repo', mockLog); - t.assert(path === '/a'); -}); +test.serial( + 'resolveSpecifierPath: return an absolute path if passed', + async (t) => { + const path = await resolveSpecifierPath('pkg=/a', '/repo', mockLog); + t.assert(path === '/a'); + } +); test.serial('resolveSpecifierPath: return a path if passed', async (t) => { const path = await resolveSpecifierPath('pkg=a/b/c', '/repo', mockLog); t.assert(path === 'a/b/c'); }); -test.serial('resolveSpecifierPath: basically return anything after the =', async (t) => { - const path = await resolveSpecifierPath('pkg=a', '/repo', mockLog); - t.assert(path === 'a'); +test.serial( + 'resolveSpecifierPath: basically return anything after the =', + async (t) => { + const path = await resolveSpecifierPath('pkg=a', '/repo', mockLog); + t.assert(path === 'a'); - const path2 = await resolveSpecifierPath('pkg=@', '/repo', mockLog); - t.assert(path2 === '@'); + const path2 = await resolveSpecifierPath('pkg=@', '/repo', mockLog); + t.assert(path2 === '@'); - const path3 = await resolveSpecifierPath('pkg=!', '/repo', mockLog); - t.assert(path3 === '!'); -}); + const path3 = await resolveSpecifierPath('pkg=!', '/repo', mockLog); + t.assert(path3 === '!'); + } +); test.serial( 'resolveSpecifierPath: return a path to the repo if the module is found', From 8c3fd82c5c0fe11a11737efb96de5bbe2ceb583d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 14:21:55 +0000 Subject: [PATCH 15/30] go deep on input tests --- packages/cli/src/execute/command.ts | 1 + packages/cli/src/options.ts | 1 + packages/cli/src/util/validate-adaptors.ts | 2 +- packages/cli/test/execute/options.test.ts | 39 ++++++++++++++++++ .../cli/test/util/validate-adaptors.test.ts | 41 ++++++++++++++++--- 5 files changed, 77 insertions(+), 7 deletions(-) diff --git a/packages/cli/src/execute/command.ts b/packages/cli/src/execute/command.ts index 5107aee40..77e859203 100644 --- a/packages/cli/src/execute/command.ts +++ b/packages/cli/src/execute/command.ts @@ -39,6 +39,7 @@ export type ExecuteOptions = { workspace?: string } & Required< | 'trace' | 'useAdaptorsMonorepo' | 'workflowPath' + | 'workflowName' | 'globals' > > & diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index 65f3d0d8b..be34b44f0 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -367,6 +367,7 @@ export const projectId: CLIOption = { }; // Input path covers expressionPath and workflowPath +// TODO this needs unit testing! export const inputPath: CLIOption = { name: 'input-path', yargs: { diff --git a/packages/cli/src/util/validate-adaptors.ts b/packages/cli/src/util/validate-adaptors.ts index bb1a55ad4..ea71dc3f6 100644 --- a/packages/cli/src/util/validate-adaptors.ts +++ b/packages/cli/src/util/validate-adaptors.ts @@ -30,7 +30,7 @@ const validateAdaptors = async ( } // If running a .js file directly and no adaptor is specified, send a warning - if (!options.expressionPath && !hasDeclaredAdaptors) { + if (options.expressionPath && !hasDeclaredAdaptors) { logger.warn('WARNING: No adaptor provided!'); logger.warn( 'This job will probably fail. Pass an adaptor with the -a flag, eg:' diff --git a/packages/cli/test/execute/options.test.ts b/packages/cli/test/execute/options.test.ts index 038d58950..96a87016a 100644 --- a/packages/cli/test/execute/options.test.ts +++ b/packages/cli/test/execute/options.test.ts @@ -26,6 +26,45 @@ test('correct default options', (t) => { t.falsy(options.useAdaptorsMonorepo); }); +test('inputPath: expression -> expressionPath', (t) => { + const options = parse('execute job.js'); + t.is(options.expressionPath, 'job.js'); +}); + +test('inputPath: json workflow -> planPath', (t) => { + const filename = parse('execute wf.json'); + t.is(filename.planPath, 'wf.json'); + + const rel = parse('execute ./wf.json'); + t.is(rel.planPath, './wf.json'); + + const abs = parse('execute /wf.json'); + t.is(abs.planPath, '/wf.json'); +}); + +test('inputPath: yaml workflow -> planPath', (t) => { + const filename = parse('execute wf.yaml'); + t.is(filename.planPath, 'wf.yaml'); + + // .yml extension works too! + const rel = parse('execute ./wf.yml'); + t.is(rel.planPath, './wf.yml'); + + const abs = parse('execute /wf.yml'); + t.is(abs.planPath, '/wf.yml'); +}); + +test('inputPath: workflow name -> workflowName', (t) => { + const simple = parse('execute workflow'); + t.is(simple.workflowName, 'workflow'); + + const hyphenated = parse('execute my-workflow'); + t.is(hyphenated.workflowName, 'my-workflow'); + + const dotted = parse('execute my.workflow'); + t.is(dotted.workflowName, 'my.workflow'); +}); + test('pass an adaptor (longform)', (t) => { const options = parse('execute job.js --adaptor @openfn/language-common'); t.deepEqual(options.adaptors, ['@openfn/language-common']); diff --git a/packages/cli/test/util/validate-adaptors.test.ts b/packages/cli/test/util/validate-adaptors.test.ts index 38ce2bff7..2e7a209c2 100644 --- a/packages/cli/test/util/validate-adaptors.test.ts +++ b/packages/cli/test/util/validate-adaptors.test.ts @@ -10,8 +10,8 @@ test.afterEach(() => { mockfs.restore(); }); -test.serial('should warn if no adaptor is passed', async (t) => { - await validateAdaptors({ adaptors: [] }, logger); +test.serial('should warn if expression passed with no adaptor', async (t) => { + await validateAdaptors({ expressionPath: 'job.js', adaptors: [] }, logger); t.assert(logger._history.length > 1); const { message, level } = logger._parse(logger._history[0]); t.is(level, 'warn'); @@ -19,18 +19,47 @@ test.serial('should warn if no adaptor is passed', async (t) => { }); test.serial( - 'should not warn if no adaptor is passed but skip-adaptor-warning is set', + 'should NOT warn if no adaptor ifskip-adaptor-warning is set', async (t) => { await validateAdaptors( - { adaptors: [], skipAdaptorValidation: true }, + { expressionPath: 'job.js', adaptors: [], skipAdaptorValidation: true }, logger ); t.is(logger._history.length, 0); } ); -test.serial('should not warn if a workflow is being used', async (t) => { - await validateAdaptors({ adaptors: [], workflowPath: 'wf.json' }, logger); +test.serial( + 'should NOT warn if a workflow json is being used (workflow path)', + async (t) => { + await validateAdaptors({ adaptors: [], workflowPath: 'wf.json' }, logger); + t.is(logger._history.length, 0); + } +); + +test.serial( + 'should NOT warn if a workflow yaml is being used (workflow path)', + async (t) => { + await validateAdaptors({ adaptors: [], workflowPath: 'wf.yaml' }, logger); + t.is(logger._history.length, 0); + } +); + +test.serial('should NOT warn if a workflow json is being used', async (t) => { + await validateAdaptors({ adaptors: [], planPath: 'wf.json' }, logger); + t.is(logger._history.length, 0); +}); + +test.serial('should NOT warn if a workflow yaml is being used', async (t) => { + await validateAdaptors({ adaptors: [], planPath: 'wf.yaml' }, logger); + t.is(logger._history.length, 0); +}); + +test.serial('should NOT warn if a workflow name is used', async (t) => { + await validateAdaptors( + { adaptors: [], workflowName: 'my-workflow' } as any, + logger + ); t.is(logger._history.length, 0); }); From e0c8f098a0a21b5d809ab1d8426227bf9c8f330d Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 14:51:20 +0000 Subject: [PATCH 16/30] throw if workflow not found --- integration-tests/cli/test/project-v1.test.ts | 42 +++++++++++-------- packages/cli/src/util/load-plan.ts | 10 ++++- packages/cli/test/util/load-plan.test.ts | 24 +++++++++++ 3 files changed, 56 insertions(+), 20 deletions(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index e5444122d..dd36b26bd 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -3,6 +3,8 @@ import { rm, mkdir, writeFile, readFile } from 'node:fs/promises'; import path from 'node:path'; import run from '../src/run'; +const TMP_DIR = 'tmp/project'; + // These tests use the legacy v1 yaml structure const mainYaml = ` @@ -98,13 +100,13 @@ workflows: const projectsPath = path.resolve('tmp/project'); test.before(async () => { - // await rm('tmp/project', { recursive: true }); - await mkdir('tmp/project/.projects', { recursive: true }); + // await rm(TMP_DIR, { recursive: true }); + await mkdir(`${TMP_DIR}/.projects`, { recursive: true }); - await writeFile('tmp/project/openfn.yaml', ''); - await writeFile('tmp/project/.projects/main@app.openfn.org.yaml', mainYaml); + await writeFile(`${TMP_DIR}/openfn.yaml`, ''); + await writeFile(`${TMP_DIR}/.projects/main@app.openfn.org.yaml`, mainYaml); await writeFile( - 'tmp/project/.projects/staging@app.openfn.org.yaml', + `${TMP_DIR}/.projects/staging@app.openfn.org.yaml`, stagingYaml ); }); @@ -174,16 +176,20 @@ test.serial('merge a project', async (t) => { t.is(merged, "log('hello world')"); }); -test.serial('execute a workflow from the checked out project', async (t) => { - // cheeky bonus test of checkout by alias - await run(`openfn checkout main --log debug -w ${projectsPath}`); - - // execute a workflow - await run( - `openfn hello-workflow -o /tmp/output.json --workspace ${projectsPath}` - ); - - const output = await readFile('/tmp/output.json', 'utf8'); - const finalState = JSON.parse(output); - t.deepEqual(finalState, { x: 1 }); -}); +test.serial.only( + 'execute a workflow from the checked out project', + async (t) => { + // cheeky bonus test of checkout by alias + await run(`openfn checkout main -w ${projectsPath}`); + + // execute a workflow + const { stdout } = await run( + `openfn hello-workflow -o /tmp/output.json --workspace ${projectsPath}` + ); + console.log(stdout); + + const output = await readFile('/tmp/output.json', 'utf8'); + const finalState = JSON.parse(output); + t.deepEqual(finalState, { x: 1 }); + } +); diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index 9342089f2..e79e546ac 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -44,9 +44,15 @@ const loadPlan = async ( ); const workspace = new Workspace(options.workspace!); const proj = await workspace.getCheckedOutProject(); - // TODO - throw if workflow not found! + + const name = workflowName || options.workflow!; + const workflow = proj?.getWorkflow(name); + if (!workflow) { + throw new Error(`Could not find Workflow "${name}"`); + } + workflowObj = { - workflow: proj?.getWorkflow(workflowName || options.workflow!)?.toJSON(), + workflow: workflow.toJSON(), }; if (!options.credentials) { diff --git a/packages/cli/test/util/load-plan.test.ts b/packages/cli/test/util/load-plan.test.ts index 98e9dc89d..c17163a31 100644 --- a/packages/cli/test/util/load-plan.test.ts +++ b/packages/cli/test/util/load-plan.test.ts @@ -551,6 +551,30 @@ dirs: }); }); +test.serial('xplan: throw if a named workflow does not exist', async (t) => { + mock({ + '/tmp/workflows/wf.yaml': ` +id: wf +steps: + - id: a + expression: x() +`, + '/tmp/openfn.yaml': ` +dirs: + workflows: /tmp/workflows +`, + }); + + const opts = { + workflowName: 'JAM', + workspace: '/tmp', + }; + + await t.throwsAsync(() => loadPlan(opts, logger), { + message: /could not find workflow "jam"/i, + }); +}); + test.serial( 'xplan: load a workflow through a project .yaml and apply the credentials map by default', async (t) => { From 265357737fc9e94fe320e92ec84adb716c88c3d7 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 15:44:22 +0000 Subject: [PATCH 17/30] project: fix an issue loading alias for a v1 project --- packages/project/src/parse/from-project.ts | 22 +++---- .../project/test/parse/from-project.test.ts | 64 ++++++++++++++----- 2 files changed, 58 insertions(+), 28 deletions(-) diff --git a/packages/project/src/parse/from-project.ts b/packages/project/src/parse/from-project.ts index 68399abe1..e99eabcd1 100644 --- a/packages/project/src/parse/from-project.ts +++ b/packages/project/src/parse/from-project.ts @@ -3,7 +3,7 @@ import * as l from '@openfn/lexicon'; import Project from '../Project'; import ensureJson from '../util/ensure-json'; import { Provisioner } from '@openfn/lexicon/lightning'; -import fromAppState from './from-app-state'; +import fromAppState, { fromAppStateConfig } from './from-app-state'; import { WithMeta } from '../Workflow'; // Load a project from any JSON or yaml representation @@ -32,25 +32,23 @@ export default ( // first ensure the data is in JSON format let rawJson = ensureJson(data); - let json; if (rawJson.cli?.version ?? rawJson.version /*deprecated*/) { // If there's any version key at all, its at least v2 - json = from_v2(rawJson as SerializedProject); - } else { - json = from_v1(rawJson as Provisioner.Project); + return new Project(from_v2(rawJson as SerializedProject), config); } - return new Project(json, config); + return from_v1(rawJson as Provisioner.Project, config as fromAppStateConfig); }; -const from_v1 = (data: Provisioner.Project) => { - // TODO is there any way to look up the config file? - // But we have no notion of a working dir here - // Maybe there are optional options that can be provided - // by from fs or from path - return fromAppState(data); +// TODO test that config (alias) works +const from_v1 = ( + data: Provisioner.Project, + config: fromAppStateConfig = {} +) => { + return fromAppState(data, {}, config); }; +// TODO this should return a Project really! const from_v2 = (data: SerializedProject) => { // nothing to do // (When we add v3, we'll ned to migrate through this) diff --git a/packages/project/test/parse/from-project.test.ts b/packages/project/test/parse/from-project.test.ts index 288c80828..21911f0e4 100644 --- a/packages/project/test/parse/from-project.test.ts +++ b/packages/project/test/parse/from-project.test.ts @@ -3,20 +3,7 @@ import state_v1 from '../fixtures/sample-v1-project'; import Project from '../../src/Project'; import * as v2 from '../fixtures/sample-v2-project'; -test('import from a v1 state as JSON', async (t) => { - const proj = await Project.from('project', state_v1, {}); - - // make a few basic assertions about the project - t.is(proj.id, 'my-workflow'); - t.is(proj.name, 'My Workflow'); - t.is(proj.openfn.uuid, 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00'); - t.is(proj.options.retention_policy, 'retain_all'); - - t.is(proj.workflows.length, 1); -}); - -test('import from a v1 state as YAML', async (t) => { - const yaml = `id: '1234' +const v1_yaml = `id: '1234' name: aaa description: a project project_credentials: [] @@ -57,7 +44,21 @@ workflows: source_trigger_id: 4a06289c-15aa-4662-8dc6-f0aaacd8a058 condition_type: always `; - const proj = await Project.from('project', yaml, {}); + +test('import from a v1 state as JSON', async (t) => { + const proj = await Project.from('project', state_v1, {}); + + // make a few basic assertions about the project + t.is(proj.id, 'my-workflow'); + t.is(proj.name, 'My Workflow'); + t.is(proj.openfn.uuid, 'e16c5f09-f0cb-4ba7-a4c2-73fcb2f29d00'); + t.is(proj.options.retention_policy, 'retain_all'); + + t.is(proj.workflows.length, 1); +}); + +test('import from a v1 state as YAML', async (t) => { + const proj = await Project.from('project', v1_yaml, {}); // make a few basic assertions about the project t.is(proj.id, 'aaa'); @@ -172,7 +173,38 @@ test('import from a v2 project as YAML', async (t) => { }); }); -test('import with custom config', async (t) => { +test('import v1 with custom config', async (t) => { + const config = { + x: 1234, + dirs: { + projects: 'p', + workflows: 'w', + }, + alias: 'staging', + format: 'yaml', + }; + const proj = await Project.from('project', v1_yaml, config); + t.is(proj.id, 'aaa'); + + t.is(proj.cli.alias, 'staging'); + + // note that alias and format should have been removed from config + t.deepEqual(proj.config, { + credentials: 'credentials.yaml', + dirs: { + projects: 'p', + workflows: 'w', + }, + formats: { + openfn: 'yaml', + project: 'yaml', + workflow: 'yaml', + }, + x: 1234, + }); +}); + +test('import v2 with custom config', async (t) => { const config = { x: 1234, dirs: { From 0394ecf5b4a2a76a39eea52733675fefa24ffdda Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 15:51:58 +0000 Subject: [PATCH 18/30] update v1 integration test --- integration-tests/cli/test/project-v1.test.ts | 35 +++++++++---------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index dd36b26bd..a55d1eae8 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -3,7 +3,7 @@ import { rm, mkdir, writeFile, readFile } from 'node:fs/promises'; import path from 'node:path'; import run from '../src/run'; -const TMP_DIR = 'tmp/project'; +const TMP_DIR = 'tmp/project-v1'; // These tests use the legacy v1 yaml structure @@ -97,7 +97,7 @@ workflows: source_trigger_id: 7bb476cc-0292-4573-89d0-b13417bc648e condition_type: always `; -const projectsPath = path.resolve('tmp/project'); +const projectsPath = path.resolve(TMP_DIR); test.before(async () => { // await rm(TMP_DIR, { recursive: true }); @@ -176,20 +176,17 @@ test.serial('merge a project', async (t) => { t.is(merged, "log('hello world')"); }); -test.serial.only( - 'execute a workflow from the checked out project', - async (t) => { - // cheeky bonus test of checkout by alias - await run(`openfn checkout main -w ${projectsPath}`); - - // execute a workflow - const { stdout } = await run( - `openfn hello-workflow -o /tmp/output.json --workspace ${projectsPath}` - ); - console.log(stdout); - - const output = await readFile('/tmp/output.json', 'utf8'); - const finalState = JSON.parse(output); - t.deepEqual(finalState, { x: 1 }); - } -); +// TODO why do both projects have alias main?? +test.serial('execute a workflow from the checked out project', async (t) => { + // cheeky bonus test of checkout by alias + await run(`openfn checkout main -w ${projectsPath}`); + + // execute a workflow + const { stdout } = await run( + `openfn my-workflow -o /tmp/output.json --workspace ${projectsPath}` + ); + + const output = await readFile('/tmp/output.json', 'utf8'); + const finalState = JSON.parse(output); + t.deepEqual(finalState, { x: 1 }); +}); From dd1af0e033b36a20c717f8fcb7f947fe66830216 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 16:05:38 +0000 Subject: [PATCH 19/30] fix integration test --- integration-tests/cli/test/project-v1.test.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index a55d1eae8..2f28d11e3 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -176,17 +176,16 @@ test.serial('merge a project', async (t) => { t.is(merged, "log('hello world')"); }); -// TODO why do both projects have alias main?? test.serial('execute a workflow from the checked out project', async (t) => { // cheeky bonus test of checkout by alias await run(`openfn checkout main -w ${projectsPath}`); // execute a workflow const { stdout } = await run( - `openfn my-workflow -o /tmp/output.json --workspace ${projectsPath}` + `openfn my-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` ); const output = await readFile('/tmp/output.json', 'utf8'); const finalState = JSON.parse(output); - t.deepEqual(finalState, { x: 1 }); + t.deepEqual(finalState, { data: {}, x: 1 }); }); From 2abc4a58584f2af9c3e46887381f1493894f49e8 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 16:18:38 +0000 Subject: [PATCH 20/30] better credential map handling --- packages/cli/src/execute/handler.ts | 27 +++++++++++++------ packages/cli/src/options.ts | 15 +++++++++++ packages/cli/src/util/load-plan.ts | 4 ++- packages/cli/test/execute/execute.test.ts | 32 +++++++++++++++++++++++ 4 files changed, 69 insertions(+), 9 deletions(-) diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts index a61d61807..34b50a480 100644 --- a/packages/cli/src/execute/handler.ts +++ b/packages/cli/src/execute/handler.ts @@ -56,7 +56,7 @@ const loadAndApplyCredentialMap = async ( let creds = {}; if (options.credentials) { try { - const credsRaw = await readFile( + let credsRaw = await readFile( path.resolve(options.workspace!, options.credentials), 'utf8' ); @@ -65,14 +65,25 @@ const loadAndApplyCredentialMap = async ( } else { creds = yamlToJson(credsRaw); } - } catch (e) { - logger.error('Error processing credential map:'); - logger.error(e); - // probably want to exist if the credential map is invalid - process.exitCode = 1; - return; + logger.info('Credential map loaded '); + } catch (e: any) { + // If we get here, the credential map failed to load + // That could mean 3 things: + // 1. The user passed --credentials to the CLI with an invalid path. + // 2. The user ran through a Project and the default credential map was not found + // 3. The user ran through a Project and an explicitly set credential map was not found + // The case of 1 is handled by opts.ensure(), which validates the path passed to the CLI + // For 2 we should continue executing but log a warning. For 3 we should probably error + // But because it's hard to recognise the case, we'll just log. + if (e?.message?.match(/ENOENT/)) { + logger.debug('Credential map not found at', options.credentials); + } else { + logger.error('Error processing credential map:'); + // probably want to exit if the credential map is invalid + process.exitCode = 1; + throw e; + } } - logger.info('Credential map loaded '); } return applyCredentialMap(plan, creds, logger); }; diff --git a/packages/cli/src/options.ts b/packages/cli/src/options.ts index be34b44f0..cc3f5917a 100644 --- a/packages/cli/src/options.ts +++ b/packages/cli/src/options.ts @@ -8,6 +8,7 @@ import { ensureLogOpts, LogLevel, } from './util'; +import { existsSync } from 'node:fs'; // Central type definition for the main options // This represents the types coming out of yargs, @@ -271,6 +272,20 @@ export const credentials: CLIOption = { alias: ['creds'], description: 'A path which points to a credential map', }, + ensure(opts) { + if (opts.credentials) { + const mapPath = nodePath.resolve( + (opts as any).workspace ?? '', + opts.credentials + ); + // Throw if a user-provided credential map not found + if (!existsSync(mapPath)) { + const e = new Error('Credential map not found at ' + mapPath); + delete e.stack; + throw e; + } + } + }, }; export const describe: CLIOption = { diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index e79e546ac..4abb62473 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -48,7 +48,9 @@ const loadPlan = async ( const name = workflowName || options.workflow!; const workflow = proj?.getWorkflow(name); if (!workflow) { - throw new Error(`Could not find Workflow "${name}"`); + const e = new Error(`Could not find Workflow "${name}"`); + delete e.stack; + throw e; } workflowObj = { diff --git a/packages/cli/test/execute/execute.test.ts b/packages/cli/test/execute/execute.test.ts index e36e40336..7fcbbff85 100644 --- a/packages/cli/test/execute/execute.test.ts +++ b/packages/cli/test/execute/execute.test.ts @@ -167,6 +167,38 @@ B: t.is(result.b, 'b'); }); +// Note that the execute function only logs if the credential map isn't found, +// which is what will happen when auto-loading the credential map +// The CLI will throw earlier through ensure() if an explicitly provided map file +// is not found. See loadAndApplyCredentialMap +test.serial( + 'Log if the credential map is not found (through Project map)', + async (t) => { + const logger = createMockLogger(undefined, { level: 'debug' }); + const workflow = { + workflow: { + steps: [ + { + id: 'a', + }, + ], + }, + }; + mockFs({ + '/workflow.json': JSON.stringify(workflow), + }); + + const options = { + ...defaultOptions, + workflowPath: '/workflow.json', + credentials: '/creds.json', + }; + + await handler(options, logger); + t.truthy(logger._find('debug', /credential map not found/i)); + } +); + test.serial('run a workflow with state', async (t) => { const workflow = { workflow: { From a6ab16996f5d69abae0d8a9cce516492c502ea42 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 16:33:43 +0000 Subject: [PATCH 21/30] fix test It was secretly failing all along --- packages/cli/test/projects/list.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/test/projects/list.test.ts b/packages/cli/test/projects/list.test.ts index dbc4e7e4a..6b4b2d05e 100644 --- a/packages/cli/test/projects/list.test.ts +++ b/packages/cli/test/projects/list.test.ts @@ -163,7 +163,7 @@ main | my-project (active) workflows: - simple-workflow -main | my-project +staging | my-project workflows: - simple-workflow From b032fda5ba061bcbb9b4063fe661d22c521acc8a Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 18:21:48 +0000 Subject: [PATCH 22/30] ensure collections and credential map can both be set --- integration-tests/cli/test/project-v2.test.ts | 89 +++++++++++++++++-- .../cli/src/execute/apply-credential-map.ts | 37 +++++--- packages/cli/src/types.ts | 4 +- packages/cli/src/util/load-plan.ts | 16 +++- .../test/execute/apply-credential-map.test.ts | 16 +++- packages/cli/test/util/load-plan.test.ts | 42 +++++++++ 6 files changed, 181 insertions(+), 23 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index 1f4e83044..a2bd56b08 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -2,6 +2,9 @@ import test from 'ava'; import { rm, mkdir, writeFile, readFile } from 'node:fs/promises'; import path from 'node:path'; import run from '../src/run'; +import createLightningServer from '@openfn/lightning-mock'; + +const TMP_DIR = 'tmp/project-v2'; const mainYaml = ` id: sandboxing-simple @@ -114,16 +117,16 @@ workflows: id: hello-workflow history: [] `; -const projectsPath = path.resolve('tmp/project'); +const projectsPath = path.resolve(TMP_DIR); test.before(async () => { - await rm('tmp/project', { recursive: true }); - await mkdir('tmp/project/.projects', { recursive: true }); + // await rm(TMP_DIR, { recursive: true }); + await mkdir(`${TMP_DIR}/.projects`, { recursive: true }); - await writeFile('tmp/project/openfn.yaml', ''); - await writeFile('tmp/project/.projects/main@app.openfn.org.yaml', mainYaml); + await writeFile(`${TMP_DIR}/openfn.yaml`, ''); + await writeFile(`${TMP_DIR}/.projects/main@app.openfn.org.yaml`, mainYaml); await writeFile( - 'tmp/project/.projects/staging@app.openfn.org.yaml', + `${TMP_DIR}/.projects/staging@app.openfn.org.yaml`, stagingYaml ); }); @@ -254,9 +257,81 @@ username: pparker` const { stdout } = await run( `openfn hello-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` ); - console.log(stdout); const output = await readFile('/tmp/output.json', 'utf8'); const finalState = JSON.parse(output); t.deepEqual(finalState, { user: 'pparker' }); } ); + +test.serial.only( + 'execute a workflow from the checked out project with credentials and collections', + async (t) => { + const server = await createLightningServer({ port: 1234 }); + server.collections.createCollection('stuff'); + // Important: the collection value MUST be as string + server.collections.upsert('stuff', 'x', JSON.stringify({ id: 'x' })); + + await run(`openfn checkout main --log debug -w ${projectsPath}`); + + // Modify the checked out workflow code + await writeFile( + `${TMP_DIR}/workflows/hello-workflow/transform-data.js`, + ` +fn(s => ({ ...s, user: s.configuration.username })); +collections.get('stuff', 'x')` + ); + + // Modify the checked out workflow to add a credential + await writeFile( + `${TMP_DIR}/workflows/hello-workflow/hello-workflow.yaml`, + `id: hello-workflow +name: Hello Workflow +start: trigger +options: {} +steps: + - id: trigger + type: webhook + next: + transform-data: + disabled: false + condition: true + - id: transform-data + name: Transform data + configuration: '1234' + adaptor: "@openfn/language-dhis2@8.0.4" + expression: ./transform-data.js +` + ); + + // add the credential map to the yaml + await writeFile( + `${TMP_DIR}/openfn.yaml`, + ` +project: + endpoint: http://localhost:1234 +workspace: + credentials: creds.yaml` + ); + + // write the credential map + await writeFile( + `${TMP_DIR}/creds.yaml`, + `1234: + username: pparker` + ); + + const { stdout } = await run( + `openfn hello-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` + ); + + const output = await readFile('/tmp/output.json', 'utf8'); + const finalState = JSON.parse(output); + + t.deepEqual(finalState, { + data: { id: 'x' }, + user: 'pparker', + references: [], + }); + server.destroy(); + } +); diff --git a/packages/cli/src/execute/apply-credential-map.ts b/packages/cli/src/execute/apply-credential-map.ts index e0c75691d..a4a0cf174 100644 --- a/packages/cli/src/execute/apply-credential-map.ts +++ b/packages/cli/src/execute/apply-credential-map.ts @@ -10,6 +10,8 @@ type JobId = string; export type CredentialMap = Record; +export const CREDENTIALS_KEY = '$CREDENTIALS$'; + const applyCredentialMap = ( plan: ExecutionPlan, map: CredentialMap = {}, @@ -17,22 +19,37 @@ const applyCredentialMap = ( ) => { const stepsWithCredentialIds = plan.workflow.steps.filter( (step: any) => - typeof step.configuration === 'string' && - !step.configuration.endsWith('.json') + (typeof step.configuration === 'string' && + !step.configuration.endsWith('.json')) || + step.configuration?.[CREDENTIALS_KEY] ) as { configuration: string; name?: string; id: string }[]; const unmapped: Record = {}; for (const step of stepsWithCredentialIds) { - if (map[step.configuration]) { - logger?.debug( - `Applying credential ${step.configuration} to "${step.name ?? step.id}"` - ); - step.configuration = map[step.configuration]; + if (typeof step.configuration === 'string') { + const configId = step.configuration; + if (configId in map) { + step.configuration = map[configId]; + } else { + unmapped[configId] = true; + // @ts-ignore + delete step.configuration; + } } else { - unmapped[step.configuration] = true; - // @ts-ignore - delete step.configuration; + const configId = step.configuration[CREDENTIALS_KEY]; + delete step.configuration[CREDENTIALS_KEY]; + if (configId in map) { + Object.assign(step.configuration, map[configId]); + } else { + unmapped[configId] = true; + } + + if (!(configId in unmapped)) { + logger?.debug( + `Applied credential ${configId} to "${step.name ?? step.id}"` + ); + } } } diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 00ccbe3f0..60e89ab0e 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -13,7 +13,9 @@ export type OldCLIWorkflow = { export type CLIExecutionPlan = { id?: UUID; - options?: WorkflowOptions; + options?: WorkflowOptions & { + collectionsEndpoint?: string; + }; workflow: { id?: string; name?: string; diff --git a/packages/cli/src/util/load-plan.ts b/packages/cli/src/util/load-plan.ts index 4abb62473..de773cc4d 100644 --- a/packages/cli/src/util/load-plan.ts +++ b/packages/cli/src/util/load-plan.ts @@ -11,6 +11,7 @@ import type { Opts } from '../options'; import type { Logger } from './logger'; import type { CLIExecutionPlan, CLIJobNode, OldCLIWorkflow } from '../types'; import resolvePath from './resolve-path'; +import { CREDENTIALS_KEY } from '../execute/apply-credential-map'; const loadPlan = async ( options: Pick< @@ -25,6 +26,7 @@ const loadPlan = async ( | 'path' | 'globals' | 'credentials' + | 'collectionsEndpoint' > & { workflow?: Opts['workflow']; workspace?: string; // from project opts @@ -57,9 +59,8 @@ const loadPlan = async ( workflow: workflow.toJSON(), }; - if (!options.credentials) { - options.credentials = workspace.getConfig().credentials; - } + options.credentials ??= workspace.getConfig().credentials; + options.collectionsEndpoint ??= proj.openfn?.endpoint; } if (options.path && /ya?ml$/.test(options.path)) { @@ -343,13 +344,14 @@ type ensureCollectionsOptions = { const ensureCollections = ( plan: CLIExecutionPlan, { - endpoint = 'https://app.openfn.org', + endpoint, version = 'latest', apiKey = 'null', }: ensureCollectionsOptions = {}, logger?: Logger ) => { let collectionsFound = false; + endpoint ??= plan.options?.collectionsEndpoint ?? 'https://app.openfn.org'; Object.values(plan.workflow.steps) .filter((step) => (step as any).expression?.match(/(collections\.)/)) @@ -365,6 +367,12 @@ const ensureCollections = ( job.adaptors.push( `@openfn/language-collections@${version || 'latest'}` ); + if (typeof job.configuration === 'string') { + // If the config is a string credential ID, write it to a special value + job.configuration = { + [CREDENTIALS_KEY]: job.configuration, + }; + } job.configuration = Object.assign({}, job.configuration, { collections_endpoint: `${endpoint}/collections`, diff --git a/packages/cli/test/execute/apply-credential-map.test.ts b/packages/cli/test/execute/apply-credential-map.test.ts index e4eb72c10..0c8f8af08 100644 --- a/packages/cli/test/execute/apply-credential-map.test.ts +++ b/packages/cli/test/execute/apply-credential-map.test.ts @@ -1,5 +1,7 @@ import test from 'ava'; -import applyCredentialMap from '../../src/execute/apply-credential-map'; +import applyCredentialMap, { + CREDENTIALS_KEY, +} from '../../src/execute/apply-credential-map'; import { createMockLogger } from '@openfn/logger/dist'; const fn = `const fn = (fn) => (s) => fn(s); @@ -52,6 +54,18 @@ test('apply a credential to a single step', (t) => { t.deepEqual(wf.workflow.steps[0].configuration, map.A); }); +test('apply a credential to a single step which already has config', (t) => { + const wf = createWorkflow(); + wf.workflow.steps[0].configuration = { x: 1, [CREDENTIALS_KEY]: 'A' }; + const map = { + A: { user: 'Anne Arnold' }, + }; + + applyCredentialMap(wf, map); + + t.deepEqual(wf.workflow.steps[0].configuration, { ...map.A, x: 1 }); +}); + test('apply a credential to several steps', (t) => { const wf = createWorkflow([ { id: 'a', configuration: 'A' }, diff --git a/packages/cli/test/util/load-plan.test.ts b/packages/cli/test/util/load-plan.test.ts index c17163a31..388495100 100644 --- a/packages/cli/test/util/load-plan.test.ts +++ b/packages/cli/test/util/load-plan.test.ts @@ -443,6 +443,48 @@ test.serial('xplan: append collections', async (t) => { }); }); +test.serial( + 'xplan: append collections to existing credential object', + async (t) => { + const opts = { + workflowPath: 'test/wf.json', + collectionsVersion: '1.1.1', + collectionsEndpoint: 'https://localhost:4000/', + apiKey: 'abc', + }; + + const plan = createPlan([ + { + id: 'a', + expression: 'collections.get()', + adaptors: ['@openfn/language-common@1.0.0'], + configuration: { + x: 1, + }, + }, + ]); + + mock({ + 'test/wf.json': JSON.stringify(plan), + }); + + const result = await loadPlan(opts, logger); + t.truthy(result); + + const step = result.workflow.steps[0] as Job; + t.deepEqual(step.adaptors, [ + '@openfn/language-common@1.0.0', + '@openfn/language-collections@1.1.1', + ]); + + t.deepEqual(step.configuration, { + collections_endpoint: `${opts.collectionsEndpoint}/collections`, + collections_token: opts.apiKey, + x: 1, + }); + } +); + test.serial( 'xplan: load a workflow.yaml without top workflow key', async (t) => { From 4a748c52cfd0a4f718d50be7fa7f9c666609f099 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 18:29:38 +0000 Subject: [PATCH 23/30] changesets --- .changeset/all-cloths-reply.md | 17 +++++++++++++++++ .changeset/curly-laws-walk.md | 2 +- .changeset/pretty-stars-play.md | 5 +++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 .changeset/all-cloths-reply.md create mode 100644 .changeset/pretty-stars-play.md diff --git a/.changeset/all-cloths-reply.md b/.changeset/all-cloths-reply.md new file mode 100644 index 000000000..55a245903 --- /dev/null +++ b/.changeset/all-cloths-reply.md @@ -0,0 +1,17 @@ +--- +'@openfn/cli': minor +--- + +When running `execute` inside a Workspace (a folder with an `openfn.yaml` file), allow workflows to be run directly. Ie: + +```bash +openfn process-patients +``` + +Instead of: + +``` +openfn ./workflows/process-patients/process-patients.yaml +``` + +When running through a Workspace, credential maps and collections endpoints are automatically applied for you. diff --git a/.changeset/curly-laws-walk.md b/.changeset/curly-laws-walk.md index 74916cb78..759d7f5f0 100644 --- a/.changeset/curly-laws-walk.md +++ b/.changeset/curly-laws-walk.md @@ -2,4 +2,4 @@ '@openfn/cli': patch --- -When executing jobs, the CLI no longer defaults the path t job.js +When executing jobs, the CLI no longer defaults the path to job.js diff --git a/.changeset/pretty-stars-play.md b/.changeset/pretty-stars-play.md new file mode 100644 index 000000000..c86f953cd --- /dev/null +++ b/.changeset/pretty-stars-play.md @@ -0,0 +1,5 @@ +--- +'@openfn/project': minor +--- + +Allow workflows to be run directly through a project From bb36848b68c979ef2dda2a46664917e76b851c04 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 18:36:58 +0000 Subject: [PATCH 24/30] test tweak --- integration-tests/cli/test/project-v1.test.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index 2f28d11e3..3fd12d569 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -3,7 +3,7 @@ import { rm, mkdir, writeFile, readFile } from 'node:fs/promises'; import path from 'node:path'; import run from '../src/run'; -const TMP_DIR = 'tmp/project-v1'; +const TMP_DIR = path.resolve('tmp/project-v1'); // These tests use the legacy v1 yaml structure @@ -182,10 +182,10 @@ test.serial('execute a workflow from the checked out project', async (t) => { // execute a workflow const { stdout } = await run( - `openfn my-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` + `openfn my-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` ); - const output = await readFile('/tmp/output.json', 'utf8'); + const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); - t.deepEqual(finalState, { data: {}, x: 1 }); + t.deepEqual(finalState, { x: 1 }); }); From d8ce47359f17191b2ed8046b36f97b9316742644 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Mon, 12 Jan 2026 18:38:39 +0000 Subject: [PATCH 25/30] more integration test tweaks --- integration-tests/cli/test/project-v2.test.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index a2bd56b08..adda8b33a 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -202,10 +202,10 @@ test.serial('execute a workflow from the checked out project', async (t) => { // execute a workflow await run( - `openfn hello-workflow -o /tmp/output.json --workspace ${projectsPath}` + `openfn hello-workflow ${TMP_DIR}/output.json --workspace ${projectsPath}` ); - const output = await readFile('/tmp/output.json', 'utf8'); + const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); t.deepEqual(finalState, { x: 1 }); }); @@ -255,9 +255,9 @@ username: pparker` // finally execute the workflow const { stdout } = await run( - `openfn hello-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` + `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` ); - const output = await readFile('/tmp/output.json', 'utf8'); + const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); t.deepEqual(finalState, { user: 'pparker' }); } @@ -321,10 +321,10 @@ workspace: ); const { stdout } = await run( - `openfn hello-workflow -o /tmp/output.json --log debug --workspace ${projectsPath}` + `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` ); - const output = await readFile('/tmp/output.json', 'utf8'); + const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); t.deepEqual(finalState, { From 309526a33c1ad2d984f4f78eaf9ed436220f3313 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 13 Jan 2026 09:11:27 +0000 Subject: [PATCH 26/30] fix integration tests yet again Turns out test order is important, so it was passing with .only but failing en masse --- integration-tests/cli/test/project-v1.test.ts | 29 ++++++++++--------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/integration-tests/cli/test/project-v1.test.ts b/integration-tests/cli/test/project-v1.test.ts index 3fd12d569..078ca0e42 100644 --- a/integration-tests/cli/test/project-v1.test.ts +++ b/integration-tests/cli/test/project-v1.test.ts @@ -156,6 +156,21 @@ steps: t.is(expr.trim(), 'fn(() => ({ x: 1}))'); }); +// note: order of tests is important here +test.serial('execute a workflow from the checked out project', async (t) => { + // cheeky bonus test of checkout by alias + await run(`openfn checkout main -w ${projectsPath}`); + + // execute a workflow + const { stdout } = await run( + `openfn my-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` + ); + + const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); + const finalState = JSON.parse(output); + t.deepEqual(finalState, { x: 1 }); +}); + // requires the prior test to run test.serial('merge a project', async (t) => { const readStep = () => @@ -175,17 +190,3 @@ test.serial('merge a project', async (t) => { const merged = await readStep(); t.is(merged, "log('hello world')"); }); - -test.serial('execute a workflow from the checked out project', async (t) => { - // cheeky bonus test of checkout by alias - await run(`openfn checkout main -w ${projectsPath}`); - - // execute a workflow - const { stdout } = await run( - `openfn my-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` - ); - - const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); - const finalState = JSON.parse(output); - t.deepEqual(finalState, { x: 1 }); -}); From 8b88291e123dc5ee09a619fc665f61054fff9e43 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 13 Jan 2026 09:20:53 +0000 Subject: [PATCH 27/30] change v2 test order --- integration-tests/cli/test/project-v2.test.ts | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index adda8b33a..742a3e5b9 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -174,28 +174,6 @@ steps: t.is(expr.trim(), 'fn()'); }); -// requires the prior test to run -test.serial('merge a project', async (t) => { - const readStep = () => - readFile( - path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'), - 'utf8' - ).then((str) => str.trim()); - - // assert the initial step code - const initial = await readStep(); - t.is(initial, '// TODO'); - - // Run the merge - const { stdout } = await run( - `openfn merge staging -w ${projectsPath} --force` - ); - - // Check the step is updated - const merged = await readStep(); - t.is(merged, 'fn()'); -}); - test.serial('execute a workflow from the checked out project', async (t) => { // cheeky bonus test of checkout by alias await run(`openfn checkout main --log debug -w ${projectsPath}`); @@ -335,3 +313,25 @@ workspace: server.destroy(); } ); + +// requires the prior test to run +test.serial('merge a project', async (t) => { + const readStep = () => + readFile( + path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'), + 'utf8' + ).then((str) => str.trim()); + + // assert the initial step code + const initial = await readStep(); + t.is(initial, '// TODO'); + + // Run the merge + const { stdout } = await run( + `openfn merge staging -w ${projectsPath} --force` + ); + + // Check the step is updated + const merged = await readStep(); + t.is(merged, 'fn()'); +}); From 0b100abc20499c6677d98a2a40843fe50d822b41 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 13 Jan 2026 09:25:08 +0000 Subject: [PATCH 28/30] let -> const --- packages/cli/src/execute/handler.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cli/src/execute/handler.ts b/packages/cli/src/execute/handler.ts index 34b50a480..e31d439be 100644 --- a/packages/cli/src/execute/handler.ts +++ b/packages/cli/src/execute/handler.ts @@ -56,7 +56,7 @@ const loadAndApplyCredentialMap = async ( let creds = {}; if (options.credentials) { try { - let credsRaw = await readFile( + const credsRaw = await readFile( path.resolve(options.workspace!, options.credentials), 'utf8' ); From 66dbe0546d8f6ecad78d1de483f426f1e5fb47b3 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 13 Jan 2026 11:55:58 +0000 Subject: [PATCH 29/30] update v2 test configuration uuids as numbers break things --- integration-tests/cli/test/project-v2.test.ts | 61 +++++++++---------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index 742a3e5b9..f2644db94 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -4,7 +4,7 @@ import path from 'node:path'; import run from '../src/run'; import createLightningServer from '@openfn/lightning-mock'; -const TMP_DIR = 'tmp/project-v2'; +const TMP_DIR = path.resolve('tmp/project-v2'); const mainYaml = ` id: sandboxing-simple @@ -117,7 +117,6 @@ workflows: id: hello-workflow history: [] `; -const projectsPath = path.resolve(TMP_DIR); test.before(async () => { // await rm(TMP_DIR, { recursive: true }); @@ -132,7 +131,7 @@ test.before(async () => { }); test.serial('list available projects', async (t) => { - const { stdout } = await run(`openfn projects -w ${projectsPath}`); + const { stdout } = await run(`openfn projects -w ${TMP_DIR}`); t.regex(stdout, /sandboxing-simple/); t.regex(stdout, /a272a529-716a-4de7-a01c-a082916c6d23/); t.regex(stdout, /staging/); @@ -140,11 +139,11 @@ test.serial('list available projects', async (t) => { }); test.serial('Checkout a project', async (t) => { - await run(`openfn checkout staging -w ${projectsPath}`); + await run(`openfn checkout staging -w ${TMP_DIR}`); // check workflow.yaml const workflowYaml = await readFile( - path.resolve(projectsPath, 'workflows/hello-workflow/hello-workflow.yaml'), + path.resolve(TMP_DIR, 'workflows/hello-workflow/hello-workflow.yaml'), 'utf8' ); t.is( @@ -168,7 +167,7 @@ steps: ); const expr = await readFile( - path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'), + path.resolve(TMP_DIR, 'workflows/hello-workflow/transform-data.js'), 'utf8' ); t.is(expr.trim(), 'fn()'); @@ -176,11 +175,11 @@ steps: test.serial('execute a workflow from the checked out project', async (t) => { // cheeky bonus test of checkout by alias - await run(`openfn checkout main --log debug -w ${projectsPath}`); + await run(`openfn checkout main -w ${TMP_DIR}`); // execute a workflow await run( - `openfn hello-workflow ${TMP_DIR}/output.json --workspace ${projectsPath}` + `openfn hello-workflow -o ${TMP_DIR}/output.json --workspace ${TMP_DIR}` ); const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); @@ -188,20 +187,20 @@ test.serial('execute a workflow from the checked out project', async (t) => { t.deepEqual(finalState, { x: 1 }); }); -test.serial( +test.serial.only( 'execute a workflow from the checked out project with a credential map', async (t) => { - await run(`openfn checkout main --log debug -w ${projectsPath}`); + await run(`openfn checkout main --log debug -w ${TMP_DIR}`); // Modify the checked out workflow code await writeFile( - 'tmp/project/workflows/hello-workflow/transform-data.js', - `fn(s => ({ user: s.configuration.username }))` + `${TMP_DIR}/workflows/hello-workflow/transform-data.js`, + `fn(s => ({ user: s.configuration.username }))` ); // Modify the checked out workflow to add a credential await writeFile( - 'tmp/project/workflows/hello-workflow/hello-workflow.yaml', + `${TMP_DIR}/workflows/hello-workflow/hello-workflow.yaml`, `id: hello-workflow name: Hello Workflow start: trigger @@ -215,33 +214,34 @@ steps: condition: true - id: transform-data name: Transform data - configuration: 1234 - adaptor: "@openfn/language-dhis2@8.0.4" + configuration: abcd + adaptor: "@openfn/language-common@3.2.1" expression: ./transform-data.js ` ); // add the credential map to the yaml - await writeFile('tmp/project/openfn.yaml', `credentials: creds.yaml`); + await writeFile(`${TMP_DIR}/openfn.yaml`, `credentials: creds.yaml`); // write the credential map await writeFile( - 'tmp/project/creds.yaml', - `1234: -username: pparker` + `${TMP_DIR}/creds.yaml`, + `abcd: + username: pparker` ); // finally execute the workflow const { stdout } = await run( - `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` + `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${TMP_DIR}` ); + const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); t.deepEqual(finalState, { user: 'pparker' }); } ); -test.serial.only( +test.serial( 'execute a workflow from the checked out project with credentials and collections', async (t) => { const server = await createLightningServer({ port: 1234 }); @@ -249,7 +249,7 @@ test.serial.only( // Important: the collection value MUST be as string server.collections.upsert('stuff', 'x', JSON.stringify({ id: 'x' })); - await run(`openfn checkout main --log debug -w ${projectsPath}`); + await run(`openfn checkout main --log debug -w ${TMP_DIR}`); // Modify the checked out workflow code await writeFile( @@ -275,8 +275,8 @@ steps: condition: true - id: transform-data name: Transform data - configuration: '1234' - adaptor: "@openfn/language-dhis2@8.0.4" + configuration: 'abcd' + adaptor: "@openfn/language-common@3.2.1" expression: ./transform-data.js ` ); @@ -286,7 +286,7 @@ steps: `${TMP_DIR}/openfn.yaml`, ` project: - endpoint: http://localhost:1234 + endpoint: http://localhost:abcd workspace: credentials: creds.yaml` ); @@ -294,13 +294,14 @@ workspace: // write the credential map await writeFile( `${TMP_DIR}/creds.yaml`, - `1234: + `abcd: username: pparker` ); const { stdout } = await run( - `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${projectsPath}` + `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${TMP_DIR}` ); + console.log(stdout); const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); @@ -318,7 +319,7 @@ workspace: test.serial('merge a project', async (t) => { const readStep = () => readFile( - path.resolve(projectsPath, 'workflows/hello-workflow/transform-data.js'), + path.resolve(TMP_DIR, 'workflows/hello-workflow/transform-data.js'), 'utf8' ).then((str) => str.trim()); @@ -327,9 +328,7 @@ test.serial('merge a project', async (t) => { t.is(initial, '// TODO'); // Run the merge - const { stdout } = await run( - `openfn merge staging -w ${projectsPath} --force` - ); + const { stdout } = await run(`openfn merge staging -w ${TMP_DIR} --force`); // Check the step is updated const merged = await readStep(); From 61edf1e4947fb3b06b0234a729696c817fb0f642 Mon Sep 17 00:00:00 2001 From: Joe Clark Date: Tue, 13 Jan 2026 12:09:32 +0000 Subject: [PATCH 30/30] yet another test fix --- integration-tests/cli/test/project-v2.test.ts | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/integration-tests/cli/test/project-v2.test.ts b/integration-tests/cli/test/project-v2.test.ts index f2644db94..f3b7cab7b 100644 --- a/integration-tests/cli/test/project-v2.test.ts +++ b/integration-tests/cli/test/project-v2.test.ts @@ -187,7 +187,7 @@ test.serial('execute a workflow from the checked out project', async (t) => { t.deepEqual(finalState, { x: 1 }); }); -test.serial.only( +test.serial( 'execute a workflow from the checked out project with a credential map', async (t) => { await run(`openfn checkout main --log debug -w ${TMP_DIR}`); @@ -286,7 +286,7 @@ steps: `${TMP_DIR}/openfn.yaml`, ` project: - endpoint: http://localhost:abcd + endpoint: http://localhost:1234 workspace: credentials: creds.yaml` ); @@ -301,7 +301,6 @@ workspace: const { stdout } = await run( `openfn hello-workflow -o ${TMP_DIR}/output.json --log debug --workspace ${TMP_DIR}` ); - console.log(stdout); const output = await readFile(`${TMP_DIR}/output.json`, 'utf8'); const finalState = JSON.parse(output); @@ -309,14 +308,14 @@ workspace: t.deepEqual(finalState, { data: { id: 'x' }, user: 'pparker', - references: [], }); server.destroy(); } ); -// requires the prior test to run test.serial('merge a project', async (t) => { + await run(`openfn checkout main -w ${TMP_DIR}`); + const readStep = () => readFile( path.resolve(TMP_DIR, 'workflows/hello-workflow/transform-data.js'), @@ -325,7 +324,7 @@ test.serial('merge a project', async (t) => { // assert the initial step code const initial = await readStep(); - t.is(initial, '// TODO'); + t.is(initial, 'fn(() => ({ x: 1}))'); // Run the merge const { stdout } = await run(`openfn merge staging -w ${TMP_DIR} --force`);