From 3094b811d6b534bd103726fb6a30862bed175085 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Fri, 4 Apr 2025 17:42:06 +0200 Subject: [PATCH 01/19] [eas-build] Move processes.ts to `steps` Functionality will be needed in `steps` and can be imported from there See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/build-tools/src/android/gradle.ts | 2 +- packages/build-tools/src/builders/ios.ts | 2 +- packages/build-tools/src/common/setup.ts | 2 +- packages/{build-tools => steps}/src/utils/processes.ts | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/{build-tools => steps}/src/utils/processes.ts (100%) diff --git a/packages/build-tools/src/android/gradle.ts b/packages/build-tools/src/android/gradle.ts index 161894b99..f2a44fc11 100644 --- a/packages/build-tools/src/android/gradle.ts +++ b/packages/build-tools/src/android/gradle.ts @@ -5,9 +5,9 @@ import spawn, { SpawnPromise, SpawnResult } from '@expo/turtle-spawn'; import { Android, Env, Job, Platform } from '@expo/eas-build-job'; import fs from 'fs-extra'; import { bunyan } from '@expo/logger'; +import { getParentAndDescendantProcessPidsAsync } from '@expo/steps'; import { BuildContext } from '../context'; -import { getParentAndDescendantProcessPidsAsync } from '../utils/processes'; export async function ensureLFLineEndingsInGradlewScript( ctx: BuildContext diff --git a/packages/build-tools/src/builders/ios.ts b/packages/build-tools/src/builders/ios.ts index ac4a8dfdb..c0a0fd200 100644 --- a/packages/build-tools/src/builders/ios.ts +++ b/packages/build-tools/src/builders/ios.ts @@ -3,6 +3,7 @@ import { IOSConfig } from '@expo/config-plugins'; import { ManagedArtifactType, BuildMode, BuildPhase, Ios, Workflow } from '@expo/eas-build-job'; import fs from 'fs-extra'; import nullthrows from 'nullthrows'; +import { getParentAndDescendantProcessPidsAsync } from '@expo/steps'; import { Artifacts, BuildContext } from '../context'; import { @@ -20,7 +21,6 @@ import { resolveArtifactPath, resolveBuildConfiguration, resolveScheme } from '. import { setupAsync } from '../common/setup'; import { prebuildAsync } from '../common/prebuild'; import { prepareExecutableAsync } from '../utils/prepareBuildExecutable'; -import { getParentAndDescendantProcessPidsAsync } from '../utils/processes'; import { eagerBundleAsync, shouldUseEagerBundle } from '../common/eagerBundle'; import { runBuilderWithHooksAsync } from './common'; diff --git a/packages/build-tools/src/common/setup.ts b/packages/build-tools/src/common/setup.ts index c04aefe92..d70cd8d80 100644 --- a/packages/build-tools/src/common/setup.ts +++ b/packages/build-tools/src/common/setup.ts @@ -7,6 +7,7 @@ import { BuildTrigger } from '@expo/eas-build-job/dist/common'; import nullthrows from 'nullthrows'; import { ExpoConfig } from '@expo/config'; import { UserFacingError } from '@expo/eas-build-job/dist/errors'; +import { getParentAndDescendantProcessPidsAsync } from '@expo/steps'; import { BuildContext } from '../context'; import { deleteXcodeEnvLocalIfExistsAsync } from '../ios/xcodeEnv'; @@ -14,7 +15,6 @@ import { Hook, runHookIfPresent } from '../utils/hooks'; import { setUpNpmrcAsync } from '../utils/npmrc'; import { isAtLeastNpm7Async } from '../utils/packageManager'; import { readPackageJson } from '../utils/project'; -import { getParentAndDescendantProcessPidsAsync } from '../utils/processes'; import { prepareProjectSourcesAsync } from './projectSources'; import { installDependenciesAsync, resolvePackagerDir } from './installDependencies'; diff --git a/packages/build-tools/src/utils/processes.ts b/packages/steps/src/utils/processes.ts similarity index 100% rename from packages/build-tools/src/utils/processes.ts rename to packages/steps/src/utils/processes.ts From f2d95264e77ddd70ff83a63fef7f16b4b113f118 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Fri, 4 Apr 2025 17:43:54 +0200 Subject: [PATCH 02/19] [eas-build] Add callback to spawn options Added `infoCallbackFn` to `spawnAsync.SpawnOptions` to allow for tracking when the process writes something at info level See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/utils/shell/spawn.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index 47f57e2c1..a562f60a7 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -1,6 +1,6 @@ import { IOType } from 'child_process'; -import { pipeSpawnOutput, bunyan, PipeMode } from '@expo/logger'; +import { pipeSpawnOutput, bunyan, PipeOptions } from '@expo/logger'; import spawnAsyncOriginal, { SpawnResult, SpawnPromise, @@ -9,10 +9,9 @@ import spawnAsyncOriginal, { // We omit 'ignoreStdio' to simplify logic -- only 'stdio' governs stdio. // We omit 'stdio' here to add further down in a logger-based union. -type SpawnOptions = Omit & { - lineTransformer?: (line: string) => string | null; - mode?: PipeMode; -} & ( +type SpawnOptions = Omit & + PipeOptions & + ( | { // If logger is passed, we require stdio to be pipe. logger: bunyan; From 138b2bbf7877d2865103b73e095dc402166040fb Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Fri, 4 Apr 2025 17:44:06 +0200 Subject: [PATCH 03/19] [eas-build] Move processes.ts to `steps` Functionality will be needed in `steps` and can be imported from there See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/index.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/steps/src/index.ts b/packages/steps/src/index.ts index 89bef3c0e..ffbe9b376 100644 --- a/packages/steps/src/index.ts +++ b/packages/steps/src/index.ts @@ -15,3 +15,4 @@ export * as errors from './errors.js'; export * from './interpolation.js'; export * from './utils/shell/spawn.js'; export * from './utils/jsepEval.js'; +export * from './utils/processes.js'; From ea8dfffdf580e616800c19138b042963c505cf6d Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Fri, 4 Apr 2025 17:48:24 +0200 Subject: [PATCH 04/19] [eas-build] Add timeouts Added warn and kill timeouts to build step command execution See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/BuildStep.ts | 67 +++++++++++++++++++++++++++++---- 1 file changed, 60 insertions(+), 7 deletions(-) diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index 7d88462e4..793c826a7 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -29,6 +29,8 @@ import { BuildStepEnv } from './BuildStepEnv.js'; import { BuildRuntimePlatform } from './BuildRuntimePlatform.js'; import { jsepEval } from './utils/jsepEval.js'; import { interpolateJobContext } from './interpolation.js'; +import { nullthrows } from './utils/nullthrows.js'; +import { getParentAndDescendantProcessPidsAsync } from './utils/processes.js'; export enum BuildStepStatus { NEW = 'new', @@ -57,6 +59,10 @@ export type BuildStepFunction = ( const UUID_REGEX = /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/; +const BUILD_STEP_WARN_TIMEOUT_MS = 15 * 60 * 1000; +const BUILD_STEP_KILL_TIMEOUT_MS = 30 * 60 * 1000; +class BuildStepTimeoutError extends Error {} + export interface SerializedBuildStepOutputAccessor { id: string; executed: boolean; @@ -381,13 +387,60 @@ export class BuildStep extends BuildStepOutputAccessor { this.ctx.logger.debug( `Executing script: ${shellCommand}${args !== undefined ? ` ${args.join(' ')}` : ''}` ); - await spawnAsync(shellCommand, args ?? [], { - cwd: this.ctx.workingDirectory, - logger: this.ctx.logger, - env: this.getScriptEnv(), - // stdin is /dev/null, std{out,err} are piped into logger. - stdio: ['ignore', 'pipe', 'pipe'], - }); + let warnTimeout: NodeJS.Timeout | undefined; + let killTimeout: NodeJS.Timeout | undefined; + let killTimedOut: boolean = false; + try { + const spawnPromise = spawnAsync(shellCommand, args ?? [], { + cwd: this.ctx.workingDirectory, + logger: this.ctx.logger, + env: this.getScriptEnv(), + // stdin is /dev/null, std{out,err} are piped into logger. + stdio: ['ignore', 'pipe', 'pipe'], + infoCallbackFn: () => { + if (warnTimeout) { + warnTimeout.refresh(); + } + if (killTimeout) { + killTimeout.refresh(); + } + }, + }); + + warnTimeout = setTimeout(() => { + this.ctx.logger.warn( + 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues' + ); + }, BUILD_STEP_WARN_TIMEOUT_MS); + + killTimeout = setTimeout(async () => { + killTimedOut = true; + this.ctx.logger.error( + 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated' + ); + const ppid = nullthrows(spawnPromise.child.pid); + const pids = await getParentAndDescendantProcessPidsAsync(ppid); + pids.forEach((pid) => { + process.kill(pid); + }); + }, BUILD_STEP_KILL_TIMEOUT_MS); + + await spawnPromise; + } catch (err: any) { + if (killTimedOut) { + throw new BuildStepTimeoutError( + 'Command was inactive for over 30 minutes. Please evaluate if it is correct' + ); + } + throw err; + } finally { + if (warnTimeout) { + clearTimeout(warnTimeout); + } + if (killTimeout) { + clearTimeout(killTimeout); + } + } this.ctx.logger.debug(`Script completed successfully`); } From 9555ec90f92835f30fc1ed587ef47a225301072d Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Tue, 8 Apr 2025 11:45:06 +0200 Subject: [PATCH 05/19] [eas-build] Reformat Improved readability by extracting timeouts to class fields and setting them up to a private method See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/BuildStep.ts | 66 ++++++++++++++++++--------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index 793c826a7..bb573d394 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -5,6 +5,7 @@ import { Buffer } from 'buffer'; import { v4 as uuidv4 } from 'uuid'; import { JobInterpolationContext } from '@expo/eas-build-job'; +import { SpawnPromise, SpawnResult } from '@expo/spawn-async'; import { BuildStepContext, BuildStepGlobalContext } from './BuildStepContext.js'; import { BuildStepInput, BuildStepInputById, makeBuildStepInputByIdMap } from './BuildStepInput.js'; @@ -147,6 +148,9 @@ export class BuildStep extends BuildStepOutputAccessor { private readonly internalId: string; private readonly inputById: BuildStepInputById; protected executed = false; + private commandWarnTimeout: NodeJS.Timeout | undefined; + private commandKillTimeout: NodeJS.Timeout | undefined; + private commandTimedOut: boolean = false; public static getNewId(userDefinedId?: string): string { return userDefinedId ?? uuidv4(); @@ -366,6 +370,27 @@ export class BuildStep extends BuildStepOutputAccessor { toJSON: (value: unknown) => JSON.stringify(value), }; } + + private setCommandSpawnTimeouts(spawnPromise: SpawnPromise): void { + this.commandWarnTimeout = setTimeout(() => { + this.ctx.logger.warn( + 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues' + ); + }, BUILD_STEP_WARN_TIMEOUT_MS); + + this.commandKillTimeout = setTimeout(async () => { + this.commandTimedOut = true; + this.ctx.logger.error( + 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated' + ); + const ppid = nullthrows(spawnPromise.child.pid); + const pids = await getParentAndDescendantProcessPidsAsync(ppid); + pids.forEach((pid) => { + process.kill(pid); + }); + }, BUILD_STEP_KILL_TIMEOUT_MS); + } + private async executeCommandAsync(): Promise { assert(this.command, 'Command must be defined.'); @@ -387,9 +412,7 @@ export class BuildStep extends BuildStepOutputAccessor { this.ctx.logger.debug( `Executing script: ${shellCommand}${args !== undefined ? ` ${args.join(' ')}` : ''}` ); - let warnTimeout: NodeJS.Timeout | undefined; - let killTimeout: NodeJS.Timeout | undefined; - let killTimedOut: boolean = false; + this.commandTimedOut = false; try { const spawnPromise = spawnAsync(shellCommand, args ?? [], { cwd: this.ctx.workingDirectory, @@ -398,47 +421,30 @@ export class BuildStep extends BuildStepOutputAccessor { // stdin is /dev/null, std{out,err} are piped into logger. stdio: ['ignore', 'pipe', 'pipe'], infoCallbackFn: () => { - if (warnTimeout) { - warnTimeout.refresh(); + if (this.commandWarnTimeout) { + this.commandWarnTimeout.refresh(); } - if (killTimeout) { - killTimeout.refresh(); + if (this.commandKillTimeout) { + this.commandKillTimeout.refresh(); } }, }); - warnTimeout = setTimeout(() => { - this.ctx.logger.warn( - 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues' - ); - }, BUILD_STEP_WARN_TIMEOUT_MS); - - killTimeout = setTimeout(async () => { - killTimedOut = true; - this.ctx.logger.error( - 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated' - ); - const ppid = nullthrows(spawnPromise.child.pid); - const pids = await getParentAndDescendantProcessPidsAsync(ppid); - pids.forEach((pid) => { - process.kill(pid); - }); - }, BUILD_STEP_KILL_TIMEOUT_MS); - + this.setCommandSpawnTimeouts(spawnPromise); await spawnPromise; } catch (err: any) { - if (killTimedOut) { + if (this.commandTimedOut) { throw new BuildStepTimeoutError( 'Command was inactive for over 30 minutes. Please evaluate if it is correct' ); } throw err; } finally { - if (warnTimeout) { - clearTimeout(warnTimeout); + if (this.commandWarnTimeout) { + clearTimeout(this.commandWarnTimeout); } - if (killTimeout) { - clearTimeout(killTimeout); + if (this.commandKillTimeout) { + clearTimeout(this.commandKillTimeout); } } this.ctx.logger.debug(`Script completed successfully`); From 9a76072a39d2bc4f70cd071ea90f0b5d45172194 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Krzemie=C5=84?= Date: Tue, 8 Apr 2025 13:57:15 +0200 Subject: [PATCH 06/19] Update packages/steps/src/BuildStep.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stanisław Chmiela --- packages/steps/src/BuildStep.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index bb573d394..70b8b59b4 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -381,7 +381,7 @@ export class BuildStep extends BuildStepOutputAccessor { this.commandKillTimeout = setTimeout(async () => { this.commandTimedOut = true; this.ctx.logger.error( - 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated' + 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated.' ); const ppid = nullthrows(spawnPromise.child.pid); const pids = await getParentAndDescendantProcessPidsAsync(ppid); From 6206192da74dad09eba8778067966ce67ee3d2b1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Rados=C5=82aw=20Krzemie=C5=84?= Date: Tue, 8 Apr 2025 13:57:24 +0200 Subject: [PATCH 07/19] Update packages/steps/src/BuildStep.ts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Stanisław Chmiela --- packages/steps/src/BuildStep.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index 70b8b59b4..2ee8aa1c1 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -374,7 +374,7 @@ export class BuildStep extends BuildStepOutputAccessor { private setCommandSpawnTimeouts(spawnPromise: SpawnPromise): void { this.commandWarnTimeout = setTimeout(() => { this.ctx.logger.warn( - 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues' + 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues.' ); }, BUILD_STEP_WARN_TIMEOUT_MS); From 1792553e51e7bc8ced768f3bdd1eefe8878b9144 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Wed, 9 Apr 2025 18:01:36 +0200 Subject: [PATCH 08/19] [eas-build] Move timeouts to `spawnAsync` Moved the creation, refreshing, handling and removal of optional timeouts into `spawnAsync` itself, while the caller might, but does not have to, define if the timeouts are needed and if they are what their params are See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/BuildStep.ts | 83 +++-------- .../steps/src/scripts/runCustomFunction.ts | 4 +- packages/steps/src/utils/shell/spawn.ts | 135 ++++++++++++++++-- 3 files changed, 149 insertions(+), 73 deletions(-) diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index 2ee8aa1c1..cbc45ccb3 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -5,7 +5,6 @@ import { Buffer } from 'buffer'; import { v4 as uuidv4 } from 'uuid'; import { JobInterpolationContext } from '@expo/eas-build-job'; -import { SpawnPromise, SpawnResult } from '@expo/spawn-async'; import { BuildStepContext, BuildStepGlobalContext } from './BuildStepContext.js'; import { BuildStepInput, BuildStepInputById, makeBuildStepInputByIdMap } from './BuildStepInput.js'; @@ -30,8 +29,6 @@ import { BuildStepEnv } from './BuildStepEnv.js'; import { BuildRuntimePlatform } from './BuildRuntimePlatform.js'; import { jsepEval } from './utils/jsepEval.js'; import { interpolateJobContext } from './interpolation.js'; -import { nullthrows } from './utils/nullthrows.js'; -import { getParentAndDescendantProcessPidsAsync } from './utils/processes.js'; export enum BuildStepStatus { NEW = 'new', @@ -60,8 +57,6 @@ export type BuildStepFunction = ( const UUID_REGEX = /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/; -const BUILD_STEP_WARN_TIMEOUT_MS = 15 * 60 * 1000; -const BUILD_STEP_KILL_TIMEOUT_MS = 30 * 60 * 1000; class BuildStepTimeoutError extends Error {} export interface SerializedBuildStepOutputAccessor { @@ -148,9 +143,6 @@ export class BuildStep extends BuildStepOutputAccessor { private readonly internalId: string; private readonly inputById: BuildStepInputById; protected executed = false; - private commandWarnTimeout: NodeJS.Timeout | undefined; - private commandKillTimeout: NodeJS.Timeout | undefined; - private commandTimedOut: boolean = false; public static getNewId(userDefinedId?: string): string { return userDefinedId ?? uuidv4(); @@ -371,26 +363,6 @@ export class BuildStep extends BuildStepOutputAccessor { }; } - private setCommandSpawnTimeouts(spawnPromise: SpawnPromise): void { - this.commandWarnTimeout = setTimeout(() => { - this.ctx.logger.warn( - 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues.' - ); - }, BUILD_STEP_WARN_TIMEOUT_MS); - - this.commandKillTimeout = setTimeout(async () => { - this.commandTimedOut = true; - this.ctx.logger.error( - 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated.' - ); - const ppid = nullthrows(spawnPromise.child.pid); - const pids = await getParentAndDescendantProcessPidsAsync(ppid); - pids.forEach((pid) => { - process.kill(pid); - }); - }, BUILD_STEP_KILL_TIMEOUT_MS); - } - private async executeCommandAsync(): Promise { assert(this.command, 'Command must be defined.'); @@ -412,41 +384,28 @@ export class BuildStep extends BuildStepOutputAccessor { this.ctx.logger.debug( `Executing script: ${shellCommand}${args !== undefined ? ` ${args.join(' ')}` : ''}` ); - this.commandTimedOut = false; - try { - const spawnPromise = spawnAsync(shellCommand, args ?? [], { - cwd: this.ctx.workingDirectory, - logger: this.ctx.logger, - env: this.getScriptEnv(), - // stdin is /dev/null, std{out,err} are piped into logger. - stdio: ['ignore', 'pipe', 'pipe'], - infoCallbackFn: () => { - if (this.commandWarnTimeout) { - this.commandWarnTimeout.refresh(); - } - if (this.commandKillTimeout) { - this.commandKillTimeout.refresh(); - } + await spawnAsync(shellCommand, args ?? [], { + cwd: this.ctx.workingDirectory, + logger: this.ctx.logger, + env: this.getScriptEnv(), + // stdin is /dev/null, std{out,err} are piped into logger. + stdio: ['ignore', 'pipe', 'pipe'], + noLogsTimeout: { + warn: { + timeoutMinutes: 15, + message: + 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues.', }, - }); - - this.setCommandSpawnTimeouts(spawnPromise); - await spawnPromise; - } catch (err: any) { - if (this.commandTimedOut) { - throw new BuildStepTimeoutError( - 'Command was inactive for over 30 minutes. Please evaluate if it is correct' - ); - } - throw err; - } finally { - if (this.commandWarnTimeout) { - clearTimeout(this.commandWarnTimeout); - } - if (this.commandKillTimeout) { - clearTimeout(this.commandKillTimeout); - } - } + kill: { + timeoutMinutes: 30, + message: + 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated.', + errorClass: BuildStepTimeoutError, + errorMessage: + 'Command was inactive for over 30 minutes. Please evaluate if it is correct.', + }, + }, + }); this.ctx.logger.debug(`Script completed successfully`); } diff --git a/packages/steps/src/scripts/runCustomFunction.ts b/packages/steps/src/scripts/runCustomFunction.ts index 32552d149..a6f62bdb3 100644 --- a/packages/steps/src/scripts/runCustomFunction.ts +++ b/packages/steps/src/scripts/runCustomFunction.ts @@ -1,7 +1,7 @@ import assert from 'assert'; import { createLogger } from '@expo/logger'; -import { SpawnPromise, SpawnResult } from '@expo/spawn-async'; +import { SpawnResult } from '@expo/spawn-async'; import cloneDeep from 'lodash.clonedeep'; import { BuildStepOutput } from '../BuildStepOutput.js'; @@ -94,7 +94,7 @@ async function runCustomJsFunctionAsync(): Promise { await customJavascriptFunction(ctx, { inputs, outputs, env }); - const promises: SpawnPromise[] = []; + const promises: Promise[] = []; for (const output of Object.values(outputs)) { if (output.rawValue) { assert(output.value, 'output.value is required'); diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index a562f60a7..019344103 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -7,6 +7,26 @@ import spawnAsyncOriginal, { SpawnOptions as SpawnOptionsOriginal, } from '@expo/spawn-async'; +import { nullthrows } from '../nullthrows.js'; +import { getParentAndDescendantProcessPidsAsync } from '../processes.js'; + +interface IErrorClass { + new (message?: string | undefined): Error; +} + +type NoLogsTimeoutOptions = { + warn?: { + timeoutMinutes?: number; + message?: string; + }; + kill?: { + timeoutMinutes?: number; + message?: string; + errorClass?: IErrorClass; + errorMessage?: string; + }; +}; + // We omit 'ignoreStdio' to simplify logic -- only 'stdio' governs stdio. // We omit 'stdio' here to add further down in a logger-based union. type SpawnOptions = Omit & @@ -16,29 +36,126 @@ type SpawnOptions = Omit & // If logger is passed, we require stdio to be pipe. logger: bunyan; stdio: 'pipe' | [IOType, 'pipe', 'pipe', ...IOType[]]; + noLogsTimeout?: NoLogsTimeoutOptions; } | { // If logger is not passed, stdio can be anything. // Defaults to inherit. logger?: never; stdio?: SpawnOptionsOriginal['stdio']; + noLogsTimeout?: undefined; } ); -// If -// eslint-disable-next-line async-protect/async-suffix -export function spawnAsync( +const SPAWN_WARN_TIMEOUT_DEFAULT_MINUTES = 15; +const SPAWN_WARN_TIMEOUT_DEFAULT_MESSAGE = + 'Command takes longer then expected and it did not produce any logs in the past ${minutes} minutes. Consider evaluating your command for possible issues.'; +const SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES = 30; +const SPAWN_KILL_TIMEOUT_DEFAULT_MESSAGE = + 'Command takes a very long time and it did not produce any logs in the past ${minutes} minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated.'; +const SPAWN_KILL_TIMEOUT_DEFAULT_ERROR_MESSAGE = + 'Command was inactive for over ${minutes} minutes. Please evaluate if it is correct.'; + +export async function spawnAsync( command: string, args: string[], allOptions: SpawnOptions = { stdio: 'inherit', cwd: process.cwd(), } -): SpawnPromise { - const { logger, ...options } = allOptions; - const promise = spawnAsyncOriginal(command, args, options); - if (logger && promise.child) { - pipeSpawnOutput(logger, promise.child, options); +): Promise { + const { logger, noLogsTimeout, ...options } = allOptions; + let spawnWarnTimeout: NodeJS.Timeout | undefined; + let spawnKillTimeout: NodeJS.Timeout | undefined; + let spawnTimedOut: boolean = false; + let spawnResult: SpawnResult; + + function setCommandSpawnTimeouts( + noLogsTimeout: NoLogsTimeoutOptions, + logger: bunyan, + spawnPromise: SpawnPromise + ): void { + if (noLogsTimeout.warn) { + const warnTimeoutMinutes = + noLogsTimeout.warn.timeoutMinutes ?? SPAWN_WARN_TIMEOUT_DEFAULT_MINUTES; + spawnWarnTimeout = setTimeout( + () => { + logger.warn( + noLogsTimeout.warn?.message ?? + SPAWN_WARN_TIMEOUT_DEFAULT_MESSAGE.replace( + '${minutes}', + warnTimeoutMinutes.toString() + ) + ); + }, + warnTimeoutMinutes * 60 * 1000 + ); + } + + if (noLogsTimeout.kill) { + const killTimeoutMinutes = + noLogsTimeout.kill.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; + spawnKillTimeout = setTimeout( + async () => { + spawnTimedOut = true; + logger.error( + noLogsTimeout.kill?.message ?? + SPAWN_KILL_TIMEOUT_DEFAULT_MESSAGE.replace( + '${minutes}', + killTimeoutMinutes.toString() + ) + ); + const ppid = nullthrows(spawnPromise.child.pid); + const pids = await getParentAndDescendantProcessPidsAsync(ppid); + pids.forEach((pid) => { + process.kill(pid); + }); + }, + killTimeoutMinutes * 60 * 1000 + ); + } + } + + try { + const spawnPromise = spawnAsyncOriginal(command, args, options); + if (logger && spawnPromise.child) { + if (noLogsTimeout) { + const optionsWithCallback = { + ...options, + infoCallbackFn: () => { + if (spawnWarnTimeout) { + spawnWarnTimeout.refresh(); + } + if (spawnKillTimeout) { + spawnKillTimeout.refresh(); + } + }, + }; + pipeSpawnOutput(logger, spawnPromise.child, optionsWithCallback); + setCommandSpawnTimeouts(noLogsTimeout, logger, spawnPromise); + } else { + pipeSpawnOutput(logger, spawnPromise.child, options); + } + } + spawnResult = await spawnPromise; + } catch (err: any) { + if (spawnTimedOut) { + const spawnKillTimeout = + noLogsTimeout?.kill?.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; + const errorMessage = + noLogsTimeout?.kill?.errorMessage ?? + SPAWN_KILL_TIMEOUT_DEFAULT_ERROR_MESSAGE.replace('${minutes}', spawnKillTimeout.toString()); + const ErrorClass = noLogsTimeout?.kill?.errorClass ?? Error; + throw new ErrorClass(errorMessage); + } + throw err; + } finally { + if (spawnWarnTimeout) { + clearTimeout(spawnWarnTimeout); + } + if (spawnKillTimeout) { + clearTimeout(spawnKillTimeout); + } } - return promise; + return spawnResult; } From 08b8fd5e457935375dc82a862ff045418ce7ad7c Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 11:29:33 +0200 Subject: [PATCH 09/19] [eas-build] Simplify message creation Moved the creation of timeout messages to helper functions to unclutter the main function body See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/utils/shell/spawn.ts | 52 +++++++++++++++---------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index 019344103..2cad370c6 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -56,6 +56,34 @@ const SPAWN_KILL_TIMEOUT_DEFAULT_MESSAGE = const SPAWN_KILL_TIMEOUT_DEFAULT_ERROR_MESSAGE = 'Command was inactive for over ${minutes} minutes. Please evaluate if it is correct.'; +function getWarnTimeoutMessage(noLogsTimeout: NoLogsTimeoutOptions): string { + const warnTimeoutMinutes = + noLogsTimeout.warn?.timeoutMinutes ?? SPAWN_WARN_TIMEOUT_DEFAULT_MINUTES; + return ( + noLogsTimeout.warn?.message ?? + SPAWN_WARN_TIMEOUT_DEFAULT_MESSAGE.replace('${minutes}', warnTimeoutMinutes.toString()) + ); +} + +function getKillTimeoutMessage(noLogsTimeout: NoLogsTimeoutOptions): string { + const killTimeoutMinutes = + noLogsTimeout.kill?.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; + return ( + noLogsTimeout.kill?.message ?? + SPAWN_KILL_TIMEOUT_DEFAULT_MESSAGE.replace('${minutes}', killTimeoutMinutes.toString()) + ); +} + +function getKillTimeoutError(noLogsTimeout: NoLogsTimeoutOptions): Error { + const spawnKillTimeout = + noLogsTimeout?.kill?.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; + const errorMessage = + noLogsTimeout?.kill?.errorMessage ?? + SPAWN_KILL_TIMEOUT_DEFAULT_ERROR_MESSAGE.replace('${minutes}', spawnKillTimeout.toString()); + const ErrorClass = noLogsTimeout?.kill?.errorClass ?? Error; + return new ErrorClass(errorMessage); +} + export async function spawnAsync( command: string, args: string[], @@ -80,13 +108,7 @@ export async function spawnAsync( noLogsTimeout.warn.timeoutMinutes ?? SPAWN_WARN_TIMEOUT_DEFAULT_MINUTES; spawnWarnTimeout = setTimeout( () => { - logger.warn( - noLogsTimeout.warn?.message ?? - SPAWN_WARN_TIMEOUT_DEFAULT_MESSAGE.replace( - '${minutes}', - warnTimeoutMinutes.toString() - ) - ); + logger.warn(getWarnTimeoutMessage(noLogsTimeout)); }, warnTimeoutMinutes * 60 * 1000 ); @@ -98,13 +120,7 @@ export async function spawnAsync( spawnKillTimeout = setTimeout( async () => { spawnTimedOut = true; - logger.error( - noLogsTimeout.kill?.message ?? - SPAWN_KILL_TIMEOUT_DEFAULT_MESSAGE.replace( - '${minutes}', - killTimeoutMinutes.toString() - ) - ); + logger.error(getKillTimeoutMessage(noLogsTimeout)); const ppid = nullthrows(spawnPromise.child.pid); const pids = await getParentAndDescendantProcessPidsAsync(ppid); pids.forEach((pid) => { @@ -140,13 +156,7 @@ export async function spawnAsync( spawnResult = await spawnPromise; } catch (err: any) { if (spawnTimedOut) { - const spawnKillTimeout = - noLogsTimeout?.kill?.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; - const errorMessage = - noLogsTimeout?.kill?.errorMessage ?? - SPAWN_KILL_TIMEOUT_DEFAULT_ERROR_MESSAGE.replace('${minutes}', spawnKillTimeout.toString()); - const ErrorClass = noLogsTimeout?.kill?.errorClass ?? Error; - throw new ErrorClass(errorMessage); + throw getKillTimeoutError(noLogsTimeout); } throw err; } finally { From 2890b87133d8668f9a959b47be9c6d563c15cc9d Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 11:30:18 +0200 Subject: [PATCH 10/19] [eas-build] Kill just child pid Removed the iterative killing of descendant pids See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/utils/shell/spawn.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index 2cad370c6..377b959c7 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -8,7 +8,6 @@ import spawnAsyncOriginal, { } from '@expo/spawn-async'; import { nullthrows } from '../nullthrows.js'; -import { getParentAndDescendantProcessPidsAsync } from '../processes.js'; interface IErrorClass { new (message?: string | undefined): Error; @@ -122,10 +121,7 @@ export async function spawnAsync( spawnTimedOut = true; logger.error(getKillTimeoutMessage(noLogsTimeout)); const ppid = nullthrows(spawnPromise.child.pid); - const pids = await getParentAndDescendantProcessPidsAsync(ppid); - pids.forEach((pid) => { - process.kill(pid); - }); + process.kill(ppid); }, killTimeoutMinutes * 60 * 1000 ); From cca6e8b61ecb7c37e72c0df55ba3ddab26e2fc26 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 11:31:24 +0200 Subject: [PATCH 11/19] Revert "[eas-build] Move processes.ts to `steps`" This reverts commit 138b2bbf7877d2865103b73e095dc402166040fb. --- packages/steps/src/index.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/steps/src/index.ts b/packages/steps/src/index.ts index ffbe9b376..89bef3c0e 100644 --- a/packages/steps/src/index.ts +++ b/packages/steps/src/index.ts @@ -15,4 +15,3 @@ export * as errors from './errors.js'; export * from './interpolation.js'; export * from './utils/shell/spawn.js'; export * from './utils/jsepEval.js'; -export * from './utils/processes.js'; From d7e6e298939723e79cba118347831eca351aa160 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 11:31:32 +0200 Subject: [PATCH 12/19] Revert "[eas-build] Move processes.ts to `steps`" This reverts commit 3094b811d6b534bd103726fb6a30862bed175085. --- packages/build-tools/src/android/gradle.ts | 2 +- packages/build-tools/src/builders/ios.ts | 2 +- packages/build-tools/src/common/setup.ts | 2 +- packages/{steps => build-tools}/src/utils/processes.ts | 0 4 files changed, 3 insertions(+), 3 deletions(-) rename packages/{steps => build-tools}/src/utils/processes.ts (100%) diff --git a/packages/build-tools/src/android/gradle.ts b/packages/build-tools/src/android/gradle.ts index f2a44fc11..161894b99 100644 --- a/packages/build-tools/src/android/gradle.ts +++ b/packages/build-tools/src/android/gradle.ts @@ -5,9 +5,9 @@ import spawn, { SpawnPromise, SpawnResult } from '@expo/turtle-spawn'; import { Android, Env, Job, Platform } from '@expo/eas-build-job'; import fs from 'fs-extra'; import { bunyan } from '@expo/logger'; -import { getParentAndDescendantProcessPidsAsync } from '@expo/steps'; import { BuildContext } from '../context'; +import { getParentAndDescendantProcessPidsAsync } from '../utils/processes'; export async function ensureLFLineEndingsInGradlewScript( ctx: BuildContext diff --git a/packages/build-tools/src/builders/ios.ts b/packages/build-tools/src/builders/ios.ts index c0a0fd200..ac4a8dfdb 100644 --- a/packages/build-tools/src/builders/ios.ts +++ b/packages/build-tools/src/builders/ios.ts @@ -3,7 +3,6 @@ import { IOSConfig } from '@expo/config-plugins'; import { ManagedArtifactType, BuildMode, BuildPhase, Ios, Workflow } from '@expo/eas-build-job'; import fs from 'fs-extra'; import nullthrows from 'nullthrows'; -import { getParentAndDescendantProcessPidsAsync } from '@expo/steps'; import { Artifacts, BuildContext } from '../context'; import { @@ -21,6 +20,7 @@ import { resolveArtifactPath, resolveBuildConfiguration, resolveScheme } from '. import { setupAsync } from '../common/setup'; import { prebuildAsync } from '../common/prebuild'; import { prepareExecutableAsync } from '../utils/prepareBuildExecutable'; +import { getParentAndDescendantProcessPidsAsync } from '../utils/processes'; import { eagerBundleAsync, shouldUseEagerBundle } from '../common/eagerBundle'; import { runBuilderWithHooksAsync } from './common'; diff --git a/packages/build-tools/src/common/setup.ts b/packages/build-tools/src/common/setup.ts index d70cd8d80..c04aefe92 100644 --- a/packages/build-tools/src/common/setup.ts +++ b/packages/build-tools/src/common/setup.ts @@ -7,7 +7,6 @@ import { BuildTrigger } from '@expo/eas-build-job/dist/common'; import nullthrows from 'nullthrows'; import { ExpoConfig } from '@expo/config'; import { UserFacingError } from '@expo/eas-build-job/dist/errors'; -import { getParentAndDescendantProcessPidsAsync } from '@expo/steps'; import { BuildContext } from '../context'; import { deleteXcodeEnvLocalIfExistsAsync } from '../ios/xcodeEnv'; @@ -15,6 +14,7 @@ import { Hook, runHookIfPresent } from '../utils/hooks'; import { setUpNpmrcAsync } from '../utils/npmrc'; import { isAtLeastNpm7Async } from '../utils/packageManager'; import { readPackageJson } from '../utils/project'; +import { getParentAndDescendantProcessPidsAsync } from '../utils/processes'; import { prepareProjectSourcesAsync } from './projectSources'; import { installDependenciesAsync, resolvePackagerDir } from './installDependencies'; diff --git a/packages/steps/src/utils/processes.ts b/packages/build-tools/src/utils/processes.ts similarity index 100% rename from packages/steps/src/utils/processes.ts rename to packages/build-tools/src/utils/processes.ts From 459a0966cfc59cdaa6cbec4eb04345888dad71bc Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 11:33:34 +0200 Subject: [PATCH 13/19] [eas-build] Make param optionally undefined Even though it's always set at this point, TS still thinks it can be undefined See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/utils/shell/spawn.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index 377b959c7..0fb191d7e 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -73,7 +73,7 @@ function getKillTimeoutMessage(noLogsTimeout: NoLogsTimeoutOptions): string { ); } -function getKillTimeoutError(noLogsTimeout: NoLogsTimeoutOptions): Error { +function getKillTimeoutError(noLogsTimeout: NoLogsTimeoutOptions | undefined): Error { const spawnKillTimeout = noLogsTimeout?.kill?.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; const errorMessage = From a40fbe628197da40dbe661febbf91639ddc3014a Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 14:11:04 +0200 Subject: [PATCH 14/19] [eas-build] Don't await SpawnPromise Returning the SpawnPromise unawaited, as before, but with `catch` and `finally` callbacks attached See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- .../steps/src/scripts/runCustomFunction.ts | 4 +- packages/steps/src/utils/shell/spawn.ts | 75 ++++++++++--------- 2 files changed, 40 insertions(+), 39 deletions(-) diff --git a/packages/steps/src/scripts/runCustomFunction.ts b/packages/steps/src/scripts/runCustomFunction.ts index a6f62bdb3..32552d149 100644 --- a/packages/steps/src/scripts/runCustomFunction.ts +++ b/packages/steps/src/scripts/runCustomFunction.ts @@ -1,7 +1,7 @@ import assert from 'assert'; import { createLogger } from '@expo/logger'; -import { SpawnResult } from '@expo/spawn-async'; +import { SpawnPromise, SpawnResult } from '@expo/spawn-async'; import cloneDeep from 'lodash.clonedeep'; import { BuildStepOutput } from '../BuildStepOutput.js'; @@ -94,7 +94,7 @@ async function runCustomJsFunctionAsync(): Promise { await customJavascriptFunction(ctx, { inputs, outputs, env }); - const promises: Promise[] = []; + const promises: SpawnPromise[] = []; for (const output of Object.values(outputs)) { if (output.rawValue) { assert(output.value, 'output.value is required'); diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index 0fb191d7e..612227d48 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -83,19 +83,19 @@ function getKillTimeoutError(noLogsTimeout: NoLogsTimeoutOptions | undefined): E return new ErrorClass(errorMessage); } -export async function spawnAsync( +// eslint-disable-next-line async-protect/async-suffix +export function spawnAsync( command: string, args: string[], allOptions: SpawnOptions = { stdio: 'inherit', cwd: process.cwd(), } -): Promise { +): SpawnPromise { const { logger, noLogsTimeout, ...options } = allOptions; let spawnWarnTimeout: NodeJS.Timeout | undefined; let spawnKillTimeout: NodeJS.Timeout | undefined; let spawnTimedOut: boolean = false; - let spawnResult: SpawnResult; function setCommandSpawnTimeouts( noLogsTimeout: NoLogsTimeoutOptions, @@ -128,40 +128,41 @@ export async function spawnAsync( } } - try { - const spawnPromise = spawnAsyncOriginal(command, args, options); - if (logger && spawnPromise.child) { - if (noLogsTimeout) { - const optionsWithCallback = { - ...options, - infoCallbackFn: () => { - if (spawnWarnTimeout) { - spawnWarnTimeout.refresh(); - } - if (spawnKillTimeout) { - spawnKillTimeout.refresh(); - } - }, - }; - pipeSpawnOutput(logger, spawnPromise.child, optionsWithCallback); - setCommandSpawnTimeouts(noLogsTimeout, logger, spawnPromise); - } else { - pipeSpawnOutput(logger, spawnPromise.child, options); - } - } - spawnResult = await spawnPromise; - } catch (err: any) { - if (spawnTimedOut) { - throw getKillTimeoutError(noLogsTimeout); - } - throw err; - } finally { - if (spawnWarnTimeout) { - clearTimeout(spawnWarnTimeout); - } - if (spawnKillTimeout) { - clearTimeout(spawnKillTimeout); + const spawnPromise = spawnAsyncOriginal(command, args, options); + if (logger && spawnPromise.child) { + if (noLogsTimeout) { + const optionsWithCallback = { + ...options, + infoCallbackFn: () => { + if (spawnWarnTimeout) { + spawnWarnTimeout.refresh(); + } + if (spawnKillTimeout) { + spawnKillTimeout.refresh(); + } + }, + }; + pipeSpawnOutput(logger, spawnPromise.child, optionsWithCallback); + setCommandSpawnTimeouts(noLogsTimeout, logger, spawnPromise); + } else { + pipeSpawnOutput(logger, spawnPromise.child, options); } } - return spawnResult; + spawnPromise + .catch((err: any) => { + if (spawnTimedOut) { + throw getKillTimeoutError(noLogsTimeout); + } + throw err; + }) + .finally(() => { + if (spawnWarnTimeout) { + clearTimeout(spawnWarnTimeout); + } + if (spawnKillTimeout) { + clearTimeout(spawnKillTimeout); + } + }); + + return spawnPromise; } From 6f17ed55dba11c35e8ef3a6acce6e9e3ea8c0fd3 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 14:21:42 +0200 Subject: [PATCH 15/19] [eas-build] Use specific error class Added a new subclass of UserFacingError for spawn timeouts See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/eas-build-job/src/errors.ts | 7 +++++++ packages/steps/src/BuildStep.ts | 4 ++-- packages/steps/src/utils/shell/spawn.ts | 9 ++++++--- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/packages/eas-build-job/src/errors.ts b/packages/eas-build-job/src/errors.ts index 3156bdef5..92015ba6f 100644 --- a/packages/eas-build-job/src/errors.ts +++ b/packages/eas-build-job/src/errors.ts @@ -103,3 +103,10 @@ export class CredentialsDistCertMismatchError extends UserFacingError { super('EAS_BUILD_CREDENTIALS_DIST_CERT_MISMATCH', message); } } + +export class SpawnCommandTimeoutError extends UserFacingError { + constructor(message?: string | undefined) { + const defaultMessage = 'Command timed out.'; + super('EAS_BUILD_SPAWN_COMMAND_TIMEOUT', message ?? defaultMessage); + } +} diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index cbc45ccb3..4eb234d6b 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -4,7 +4,7 @@ import path from 'path'; import { Buffer } from 'buffer'; import { v4 as uuidv4 } from 'uuid'; -import { JobInterpolationContext } from '@expo/eas-build-job'; +import { JobInterpolationContext, errors } from '@expo/eas-build-job'; import { BuildStepContext, BuildStepGlobalContext } from './BuildStepContext.js'; import { BuildStepInput, BuildStepInputById, makeBuildStepInputByIdMap } from './BuildStepInput.js'; @@ -57,7 +57,7 @@ export type BuildStepFunction = ( const UUID_REGEX = /^[0-9a-fA-F]{8}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{4}\b-[0-9a-fA-F]{12}$/; -class BuildStepTimeoutError extends Error {} +class BuildStepTimeoutError extends errors.SpawnCommandTimeoutError {} export interface SerializedBuildStepOutputAccessor { id: string; diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index 612227d48..c42178b7a 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -6,11 +6,12 @@ import spawnAsyncOriginal, { SpawnPromise, SpawnOptions as SpawnOptionsOriginal, } from '@expo/spawn-async'; +import { errors } from '@expo/eas-build-job'; import { nullthrows } from '../nullthrows.js'; interface IErrorClass { - new (message?: string | undefined): Error; + new (message?: string | undefined): errors.SpawnCommandTimeoutError; } type NoLogsTimeoutOptions = { @@ -73,13 +74,15 @@ function getKillTimeoutMessage(noLogsTimeout: NoLogsTimeoutOptions): string { ); } -function getKillTimeoutError(noLogsTimeout: NoLogsTimeoutOptions | undefined): Error { +function getKillTimeoutError( + noLogsTimeout: NoLogsTimeoutOptions | undefined +): errors.SpawnCommandTimeoutError { const spawnKillTimeout = noLogsTimeout?.kill?.timeoutMinutes ?? SPAWN_KILL_TIMEOUT_DEFAULT_MINUTES; const errorMessage = noLogsTimeout?.kill?.errorMessage ?? SPAWN_KILL_TIMEOUT_DEFAULT_ERROR_MESSAGE.replace('${minutes}', spawnKillTimeout.toString()); - const ErrorClass = noLogsTimeout?.kill?.errorClass ?? Error; + const ErrorClass = noLogsTimeout?.kill?.errorClass ?? errors.SpawnCommandTimeoutError; return new ErrorClass(errorMessage); } From c2885de953d79b3e2dc0ee87d049402ae0a5eb50 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 15:44:39 +0200 Subject: [PATCH 16/19] [eas-build] Parameterize the timeouts Allowed for setting the timeout in minutes on the step definition See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/eas-build-job/src/step.ts | 26 ++++++++++++++++++++++++++ packages/steps/src/BuildStep.ts | 14 ++++++++++++-- 2 files changed, 38 insertions(+), 2 deletions(-) diff --git a/packages/eas-build-job/src/step.ts b/packages/eas-build-job/src/step.ts index afdb71fa4..494ada67a 100644 --- a/packages/eas-build-job/src/step.ts +++ b/packages/eas-build-job/src/step.ts @@ -87,6 +87,8 @@ export const FunctionStepZ = CommonStepZ.extend({ run: z.never().optional(), shell: z.never().optional(), outputs: z.never().optional(), + 'no-logs-warn-timeout-minutes': z.never().optional(), + 'no-logs-kill-timeout-minutes': z.never().optional(), }); export type FunctionStep = z.infer; @@ -137,6 +139,30 @@ export const ShellStepZ = CommonStepZ.extend({ ]) ) .optional(), + /** + * Number of minutes since last log produced by the script, after which a warning will be logged. + * + * @example + * no-logs-warn-timeout-minutes: 15 + */ + 'no-logs-warn-timeout-minutes': z + .number() + .optional() + .describe( + 'If the script does not produce any logs in this many minutes, a warning will be logged.' + ), + /** + * Number of minutes since last log produced by the script, after which an error will be thrown and the process running the script will be terminated. + * + * @example + * no-logs-kill-timeout-minutes: 30 + */ + 'no-logs-kill-timeout-minutes': z + .number() + .optional() + .describe( + 'If the script does not produce any logs in this many minutes, an error will be thrown and script will be terminated.' + ), uses: z.never().optional(), with: z.never().optional(), diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index 4eb234d6b..e7afcbd30 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -144,6 +144,9 @@ export class BuildStep extends BuildStepOutputAccessor { private readonly inputById: BuildStepInputById; protected executed = false; + private readonly noLogsWarnTimeoutMinutes: number; + private readonly noLogsKillTimeoutMinutes: number; + public static getNewId(userDefinedId?: string): string { return userDefinedId ?? uuidv4(); } @@ -190,6 +193,8 @@ export class BuildStep extends BuildStepOutputAccessor { supportedRuntimePlatforms: maybeSupportedRuntimePlatforms, env, ifCondition, + noLogsWarnTimeoutMinutes, + noLogsKillTimeoutMinutes, }: { id: string; name?: string; @@ -203,6 +208,8 @@ export class BuildStep extends BuildStepOutputAccessor { supportedRuntimePlatforms?: BuildRuntimePlatform[]; env?: BuildStepEnv; ifCondition?: string; + noLogsWarnTimeoutMinutes?: number; + noLogsKillTimeoutMinutes?: number; } ) { assert(command !== undefined || fn !== undefined, 'Either command or fn must be defined.'); @@ -236,6 +243,9 @@ export class BuildStep extends BuildStepOutputAccessor { this.outputsDir = getTemporaryOutputsDirPath(ctx, this.id); this.envsDir = getTemporaryEnvsDirPath(ctx, this.id); + this.noLogsWarnTimeoutMinutes = noLogsWarnTimeoutMinutes ?? 15; + this.noLogsKillTimeoutMinutes = noLogsKillTimeoutMinutes ?? 30; + ctx.registerStep(this); } @@ -392,12 +402,12 @@ export class BuildStep extends BuildStepOutputAccessor { stdio: ['ignore', 'pipe', 'pipe'], noLogsTimeout: { warn: { - timeoutMinutes: 15, + timeoutMinutes: this.noLogsWarnTimeoutMinutes, message: 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues.', }, kill: { - timeoutMinutes: 30, + timeoutMinutes: this.noLogsKillTimeoutMinutes, message: 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated.', errorClass: BuildStepTimeoutError, From dc9d52332577c9f7ced8702bed0dd5c9f8ed5b8b Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Mon, 14 Apr 2025 15:45:26 +0200 Subject: [PATCH 17/19] [eas-build] Fix inner promise rejection Inner promise from `@expo/spawn-async` rejecting caused the outer process to exit unexpectedly See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/utils/shell/spawn.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/steps/src/utils/shell/spawn.ts b/packages/steps/src/utils/shell/spawn.ts index c42178b7a..2f59666e7 100644 --- a/packages/steps/src/utils/shell/spawn.ts +++ b/packages/steps/src/utils/shell/spawn.ts @@ -152,6 +152,10 @@ export function spawnAsync( } } spawnPromise + .then( + (spawnResult) => spawnResult, + (_) => {} + ) .catch((err: any) => { if (spawnTimedOut) { throw getKillTimeoutError(noLogsTimeout); From 4b9aa5147f47ad2d3ab91a77da00dd73434e70eb Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Tue, 15 Apr 2025 13:40:38 +0200 Subject: [PATCH 18/19] [eas-build] Update test Updated test for Step See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/eas-build-job/src/__tests__/step.test.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/eas-build-job/src/__tests__/step.test.ts b/packages/eas-build-job/src/__tests__/step.test.ts index d64d74828..636cead95 100644 --- a/packages/eas-build-job/src/__tests__/step.test.ts +++ b/packages/eas-build-job/src/__tests__/step.test.ts @@ -66,6 +66,8 @@ describe('StepZ', () => { env: { KEY1: 'value1', }, + 'no-logs-warn-timeout-minutes': 15, + 'no-logs-kill-timeout-minutes': 30, }; expect(StepZ.parse(step)).toEqual(step); }); From d2ba0584ef8327dca817de59ef25cf47a7f30df9 Mon Sep 17 00:00:00 2001 From: Radoslaw Krzemien Date: Tue, 15 Apr 2025 13:56:20 +0200 Subject: [PATCH 19/19] [eas-build] Update tests Updated tests for StepsConfigParser checking for customizable timeouts See: https://linear.app/expo/issue/ENG-15188/replicate-the-cancel-hanging-build-functionality-for-workflows --- packages/steps/src/BuildStep.ts | 12 ++++++------ packages/steps/src/StepsConfigParser.ts | 2 ++ .../steps/src/__tests__/StepsConfigParser-test.ts | 6 ++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/packages/steps/src/BuildStep.ts b/packages/steps/src/BuildStep.ts index e7afcbd30..0dd6d270c 100644 --- a/packages/steps/src/BuildStep.ts +++ b/packages/steps/src/BuildStep.ts @@ -144,8 +144,8 @@ export class BuildStep extends BuildStepOutputAccessor { private readonly inputById: BuildStepInputById; protected executed = false; - private readonly noLogsWarnTimeoutMinutes: number; - private readonly noLogsKillTimeoutMinutes: number; + public readonly noLogsWarnTimeoutMinutes?: number; + public readonly noLogsKillTimeoutMinutes?: number; public static getNewId(userDefinedId?: string): string { return userDefinedId ?? uuidv4(); @@ -243,8 +243,8 @@ export class BuildStep extends BuildStepOutputAccessor { this.outputsDir = getTemporaryOutputsDirPath(ctx, this.id); this.envsDir = getTemporaryEnvsDirPath(ctx, this.id); - this.noLogsWarnTimeoutMinutes = noLogsWarnTimeoutMinutes ?? 15; - this.noLogsKillTimeoutMinutes = noLogsKillTimeoutMinutes ?? 30; + this.noLogsWarnTimeoutMinutes = noLogsWarnTimeoutMinutes; + this.noLogsKillTimeoutMinutes = noLogsKillTimeoutMinutes; ctx.registerStep(this); } @@ -402,12 +402,12 @@ export class BuildStep extends BuildStepOutputAccessor { stdio: ['ignore', 'pipe', 'pipe'], noLogsTimeout: { warn: { - timeoutMinutes: this.noLogsWarnTimeoutMinutes, + timeoutMinutes: this.noLogsWarnTimeoutMinutes ?? 15, message: 'Command takes longer then expected and it did not produce any logs in the past 15 minutes. Consider evaluating your command for possible issues.', }, kill: { - timeoutMinutes: this.noLogsKillTimeoutMinutes, + timeoutMinutes: this.noLogsKillTimeoutMinutes ?? 30, message: 'Command takes a very long time and it did not produce any logs in the past 30 minutes. Most likely an unexpected error happened which caused the process to hang and it will be terminated.', errorClass: BuildStepTimeoutError, diff --git a/packages/steps/src/StepsConfigParser.ts b/packages/steps/src/StepsConfigParser.ts index db3996c4d..4240a92c3 100644 --- a/packages/steps/src/StepsConfigParser.ts +++ b/packages/steps/src/StepsConfigParser.ts @@ -128,6 +128,8 @@ export class StepsConfigParser extends AbstractConfigParser { command: step.run, env: step.env, ifCondition: step.if, + noLogsWarnTimeoutMinutes: step['no-logs-warn-timeout-minutes'], + noLogsKillTimeoutMinutes: step['no-logs-kill-timeout-minutes'], }); } diff --git a/packages/steps/src/__tests__/StepsConfigParser-test.ts b/packages/steps/src/__tests__/StepsConfigParser-test.ts index 2410f52d1..3c847fd22 100644 --- a/packages/steps/src/__tests__/StepsConfigParser-test.ts +++ b/packages/steps/src/__tests__/StepsConfigParser-test.ts @@ -195,6 +195,8 @@ describe(StepsConfigParser, () => { }, ], if: '${ always() }', + 'no-logs-warn-timeout-minutes': 5, + 'no-logs-kill-timeout-minutes': 10, }, { id: 'step4', @@ -322,6 +324,8 @@ describe(StepsConfigParser, () => { expect(output3.id).toBe('my_optional_output_without_required'); expect(output3.required).toBe(true); expect(step3.ifCondition).toBe('${ always() }'); + expect(step3.noLogsWarnTimeoutMinutes).toBe(5); + expect(step3.noLogsKillTimeoutMinutes).toBe(10); const step4 = result.buildSteps[3]; expect(step4.id).toEqual('step4'); @@ -370,6 +374,8 @@ describe(StepsConfigParser, () => { expect(input4.required).toBe(true); expect(step4.outputById).toStrictEqual({}); expect(step4.ifCondition).toBe('${ ctx.job.platform } == "android"'); + expect(step4.noLogsWarnTimeoutMinutes).toBeUndefined(); + expect(step4.noLogsKillTimeoutMinutes).toBeUndefined(); }); }); });