From e645f9eeb7a8d49570a8000bdda8d3bbcaa6f0c3 Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Sat, 29 Nov 2025 18:05:24 +0100 Subject: [PATCH 01/12] proof of concept for sparse checkout --- src/parser-includes.ts | 101 +++++++++++++++++++++++------------------ 1 file changed, 57 insertions(+), 44 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 738d95717..e43731b5a 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -76,6 +76,14 @@ export class ParserIncludes { promises.push(this.downloadIncludeRemote(cwd, stateDir, url, fetchIncludes)); } else if (value["remote"]) { promises.push(this.downloadIncludeRemote(cwd, stateDir, value["remote"], fetchIncludes)); + } else if (value["component"]) + { + // TODO: I'm unhappy about calling parseIncludeComponent twice, as it invokes git ls-remote + const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); + if(!isLocalComponent) + { + promises.push(this.downloadIncludeComponent(cwd, stateDir,domain, port, projectPath, ref, componentName, gitData, fetchIncludes)); + } } } @@ -127,53 +135,27 @@ export class ParserIncludes { includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } } else if (value["component"]) { + // TODO: I'm unhappy about calling parseIncludeComponent twice, as it invokes git ls-remote const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); // converts component to project. gitlab allows two different file path ways to include a component - let files = [`${componentName}.yml`, `${componentName}/template.yml`, null]; - - // If a file is present locally, keep only that one in the files array to avoid downloading the other one that never exists - if (!argv.fetchIncludes) { - for (const f of files) { - const localFileName = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; - if (fs.existsSync(localFileName)) { - files = [f]; - break; - } - } - } + const files = [`${componentName}.yml`, `${componentName}/template.yml`]; + let file = null for (const f of files) { - assert(f !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${domain}` + - (port ? `:${port}` : "") + `/${projectPath}\``); - - if (isLocalComponent) { - const localComponentInclude = `${cwd}/${f}`; - if (!(await fs.pathExists(localComponentInclude))) { - continue; - } - - const content = await Parser.loadYaml(localComponentInclude, {inputs: value.inputs || {}}, expandVariables); - includeDatas = includeDatas.concat(await this.init(content, opts)); - break; - } else { - const localFileName = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; - // Check remotely only if the file does not exist locally - if (!fs.existsSync(localFileName) && !(await Utils.remoteFileExist(cwd, f, ref, domain, projectPath, gitData.remote.schema, gitData.remote.port))) { - continue; - } - - const fileDoc = { - include: { - project: projectPath, - file: f, - ref: ref, - inputs: value.inputs || {}, - }, - }; - includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); - break; + let searchPath = `${cwd}/${f}`; + if(!isLocalComponent) { + searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; + } + if (fs.existsSync(searchPath)) { + file = searchPath; } } + assert(file !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${domain}` + + (port ? `:${port}` : "") + `/${projectPath}\``); + + const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables); + // TODO: think about why the project case expands the inner includes? + includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["template"]) { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); const fsUrl = Utils.fsUrl(`https://${domain}/${project}/-/raw/${ref}/${file}`); @@ -254,10 +236,10 @@ export class ParserIncludes { if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) { // https://docs.gitlab.com/ci/components/#semantic-version-ranges let stdout; - try { + if(gitData.remote.schema == "git" || gitData.remote.schema == "ssh") { stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout; - } catch { - stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `https://${domain}:${port ?? 443}/${projectPath}.git`]).stdout; + } else { + stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout; } assert(stdout); const tags = stdout @@ -328,6 +310,37 @@ export class ParserIncludes { } } + static async downloadIncludeComponent (cwd: string, stateDir: string, domain: string, port: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise { + const remote = gitData.remote; + let files = [`${componentName}.yml`, `${componentName}/template.yml`]; + try { + const target = `${stateDir}/includes/${domain}/${project}/${ref}`; + + if(!fetchIncludes && (await fs.pathExists(`${cwd}/${target}/${files[0]}`) || await fs.pathExists(`${cwd}/${target}/${files[1]}`))) return; + + if (remote.schema.startsWith("http")) { + const ext = "tmp-" + Math.random(); + await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`)); + + const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; + await Utils.bashMulti([ + `cd ${cwd}/${stateDir}`, + `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${domain}:${port}/${project}.git ${cwd}/${target}.${ext}`, + `cd ${cwd}/${target}.${ext}`, + `git sparse-checkout set --no-cone ${files[0]} ${files[1]}`, + "git checkout", + `cd ${cwd}/${stateDir}`, + `cp -r ${cwd}/${target}.${ext}/templates ${cwd}/${target}`, + ], cwd); + } else { + await fs.mkdirp(`${cwd}/${target}`); + await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${domain}:${port}/${project}.git ${ref} ${files[0]} ${files[1]} | tar -f - -xC ${target}/`, cwd); + } + } catch (e) { + throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); + } + } + static readonly memoLocalRepoFiles = (() => { const cache = new Map(); return async (path: string) => { From 2782cedd6322ae9d6cc0d25a9b21b72e460006c1 Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Sat, 29 Nov 2025 18:47:28 +0100 Subject: [PATCH 02/12] fix linter errors --- src/parser-includes.ts | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index e43731b5a..f59ba67f2 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -80,9 +80,9 @@ export class ParserIncludes { { // TODO: I'm unhappy about calling parseIncludeComponent twice, as it invokes git ls-remote const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); - if(!isLocalComponent) + if (!isLocalComponent) { - promises.push(this.downloadIncludeComponent(cwd, stateDir,domain, port, projectPath, ref, componentName, gitData, fetchIncludes)); + promises.push(this.downloadIncludeComponent(cwd, stateDir, domain, port, projectPath, ref, componentName, gitData, fetchIncludes)); } } @@ -140,11 +140,11 @@ export class ParserIncludes { // converts component to project. gitlab allows two different file path ways to include a component const files = [`${componentName}.yml`, `${componentName}/template.yml`]; - let file = null + let file = null; for (const f of files) { let searchPath = `${cwd}/${f}`; - if(!isLocalComponent) { - searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; + if (!isLocalComponent) { + searchPath = `${cwd}/${stateDir}/includes/${domain}/${projectPath}/${ref}/${f}`; } if (fs.existsSync(searchPath)) { file = searchPath; @@ -155,7 +155,7 @@ export class ParserIncludes { const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables); // TODO: think about why the project case expands the inner includes? - includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); + includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["template"]) { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); const fsUrl = Utils.fsUrl(`https://${domain}/${project}/-/raw/${ref}/${file}`); @@ -236,7 +236,7 @@ export class ParserIncludes { if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) { // https://docs.gitlab.com/ci/components/#semantic-version-ranges let stdout; - if(gitData.remote.schema == "git" || gitData.remote.schema == "ssh") { + if (gitData.remote.schema == "git" || gitData.remote.schema == "ssh") { stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout; } else { stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout; @@ -312,11 +312,11 @@ export class ParserIncludes { static async downloadIncludeComponent (cwd: string, stateDir: string, domain: string, port: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise { const remote = gitData.remote; - let files = [`${componentName}.yml`, `${componentName}/template.yml`]; + const files = [`${componentName}.yml`, `${componentName}/template.yml`]; try { const target = `${stateDir}/includes/${domain}/${project}/${ref}`; - if(!fetchIncludes && (await fs.pathExists(`${cwd}/${target}/${files[0]}`) || await fs.pathExists(`${cwd}/${target}/${files[1]}`))) return; + if (!fetchIncludes && (await fs.pathExists(`${cwd}/${target}/${files[0]}`) || await fs.pathExists(`${cwd}/${target}/${files[1]}`))) return; if (remote.schema.startsWith("http")) { const ext = "tmp-" + Math.random(); From 1e7d9d2b8120715f14bc309acb888bad452292cb Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Mon, 1 Dec 2025 14:31:04 +0100 Subject: [PATCH 03/12] fix inner include and test --- src/parser-includes.ts | 24 ++++++++++++++++--- .../include-component/integration.test.ts | 2 +- 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index f59ba67f2..98d681d1a 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -154,7 +154,24 @@ export class ParserIncludes { (port ? `:${port}` : "") + `/${projectPath}\``); const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables); - // TODO: think about why the project case expands the inner includes? + // Expand local includes inside to a "project"-like include + fileDoc["include"] = this.expandInclude(fileDoc["include"], opts.variables); + fileDoc["include"].forEach((inner: any, i: number) => { + if (!inner["local"]) return; + if (inner["rules"]) { + const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, gitData); + if (rulesResult.when === "never") { + return; + } + } + fileDoc["include"][i] = { + project: projectPath, + file: inner["local"].replace(/^\//, ""), + ref: ref, + inputs: inner.inputs || {}, + }; + }); + includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["template"]) { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); @@ -330,14 +347,15 @@ export class ParserIncludes { `git sparse-checkout set --no-cone ${files[0]} ${files[1]}`, "git checkout", `cd ${cwd}/${stateDir}`, - `cp -r ${cwd}/${target}.${ext}/templates ${cwd}/${target}`, + `rsync -a --ignore-missing-args ${cwd}/${target}.${ext}/templates ${cwd}/${target}`, // use rsnyc to ignore missing templates/, the check for existence is in a later step ], cwd); } else { await fs.mkdirp(`${cwd}/${target}`); await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${domain}:${port}/${project}.git ${ref} ${files[0]} ${files[1]} | tar -f - -xC ${target}/`, cwd); } } catch (e) { - throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); + console.error(e) + throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); } } diff --git a/tests/test-cases/include-component/integration.test.ts b/tests/test-cases/include-component/integration.test.ts index fc3d43926..076adeac9 100644 --- a/tests/test-cases/include-component/integration.test.ts +++ b/tests/test-cases/include-component/integration.test.ts @@ -20,7 +20,7 @@ test("include-component no component template file (protocol: https)", async () expect(true).toBe(false); } catch (e: any) { assert(e instanceof AssertionError, `Unexpected error thrown:\n ${e}`); - expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml,] must exist in `gitlab.com/components/go`"); + expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml] must exist in `gitlab.com/components/go`"); } }); From 63345821de6d776e9f624d2aabed8f1d2ced0e3c Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Mon, 1 Dec 2025 14:33:32 +0100 Subject: [PATCH 04/12] remove debug log --- src/parser-includes.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 98d681d1a..71e6e0131 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -354,7 +354,6 @@ export class ParserIncludes { await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${domain}:${port}/${project}.git ${ref} ${files[0]} ${files[1]} | tar -f - -xC ${target}/`, cwd); } } catch (e) { - console.error(e) throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); } } From 6f05b3b51c0d7727b14f4845f1dd88e58b0f8a7a Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Mon, 1 Dec 2025 15:55:53 +0100 Subject: [PATCH 05/12] use remote.host and remote.port because components should be on the same instance --- src/parser-includes.ts | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 71e6e0131..399074309 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -82,7 +82,7 @@ export class ParserIncludes { const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); if (!isLocalComponent) { - promises.push(this.downloadIncludeComponent(cwd, stateDir, domain, port, projectPath, ref, componentName, gitData, fetchIncludes)); + promises.push(this.downloadIncludeComponent(cwd, stateDir, projectPath, ref, componentName, gitData, fetchIncludes)); } } @@ -144,7 +144,7 @@ export class ParserIncludes { for (const f of files) { let searchPath = `${cwd}/${f}`; if (!isLocalComponent) { - searchPath = `${cwd}/${stateDir}/includes/${domain}/${projectPath}/${ref}/${f}`; + searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; } if (fs.existsSync(searchPath)) { file = searchPath; @@ -327,11 +327,11 @@ export class ParserIncludes { } } - static async downloadIncludeComponent (cwd: string, stateDir: string, domain: string, port: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise { + static async downloadIncludeComponent (cwd: string, stateDir: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise { const remote = gitData.remote; const files = [`${componentName}.yml`, `${componentName}/template.yml`]; try { - const target = `${stateDir}/includes/${domain}/${project}/${ref}`; + const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`; if (!fetchIncludes && (await fs.pathExists(`${cwd}/${target}/${files[0]}`) || await fs.pathExists(`${cwd}/${target}/${files[1]}`))) return; @@ -342,7 +342,7 @@ export class ParserIncludes { const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; await Utils.bashMulti([ `cd ${cwd}/${stateDir}`, - `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${domain}:${port}/${project}.git ${cwd}/${target}.${ext}`, + `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${cwd}/${target}.${ext}`, `cd ${cwd}/${target}.${ext}`, `git sparse-checkout set --no-cone ${files[0]} ${files[1]}`, "git checkout", @@ -351,7 +351,7 @@ export class ParserIncludes { ], cwd); } else { await fs.mkdirp(`${cwd}/${target}`); - await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${domain}:${port}/${project}.git ${ref} ${files[0]} ${files[1]} | tar -f - -xC ${target}/`, cwd); + await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${files[0]} ${files[1]} | tar -f - -xC ${target}/`, cwd); } } catch (e) { throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); From 28db9b5dd61a005c13ae795073f7672eaf90d837 Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Mon, 1 Dec 2025 16:38:25 +0100 Subject: [PATCH 06/12] cache the parsed components --- src/parser-includes.ts | 52 +++++++++++++++++++++++++----------------- 1 file changed, 31 insertions(+), 21 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 399074309..81cc45231 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -23,6 +23,15 @@ type ParserIncludesInitOptions = { maximumIncludes: number; }; +type ParsedComponent = { + domain: string; + port: string; + projectPath: string; + name: string; + ref: string; + isLocal: boolean; +}; + export class ParserIncludes { private static count: number = 0; @@ -53,12 +62,14 @@ export class ParserIncludes { let includeDatas: any[] = []; const promises = []; const {stateDir, cwd, fetchIncludes, gitData, expandVariables} = opts; + // cache the parsed component, because parseIncludeComponent is expensive and we would call it twice otherwise + const componentParseCache = new Map(); const include = this.expandInclude(gitlabData?.include, opts.variables); this.normalizeTriggerInclude(gitlabData, opts); // Find files to fetch from remote and place in .gitlab-ci-local/includes - for (const value of include) { + for (const [index, value] of include.entries()) { if (value["rules"]) { const include_rules = value["rules"]; const rulesResult = Utils.getRulesResult({argv, cwd, rules: include_rules, variables: opts.variables}, gitData); @@ -76,13 +87,12 @@ export class ParserIncludes { promises.push(this.downloadIncludeRemote(cwd, stateDir, url, fetchIncludes)); } else if (value["remote"]) { promises.push(this.downloadIncludeRemote(cwd, stateDir, value["remote"], fetchIncludes)); - } else if (value["component"]) - { - // TODO: I'm unhappy about calling parseIncludeComponent twice, as it invokes git ls-remote - const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); - if (!isLocalComponent) + } else if (value["component"]) { + const component = this.parseIncludeComponent(value["component"], gitData); + componentParseCache.set(index, component); + if (!component.isLocal) { - promises.push(this.downloadIncludeComponent(cwd, stateDir, projectPath, ref, componentName, gitData, fetchIncludes)); + promises.push(this.downloadIncludeComponent(cwd, stateDir, component.projectPath, component.ref, component.name, gitData, fetchIncludes)); } } @@ -90,7 +100,7 @@ export class ParserIncludes { await Promise.all(promises); - for (const value of include) { + for (const [index, value] of include.entries()) { if (value["rules"]) { const include_rules = value["rules"]; const rulesResult = Utils.getRulesResult({argv, cwd, rules: include_rules, variables: opts.variables}, gitData); @@ -135,23 +145,23 @@ export class ParserIncludes { includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } } else if (value["component"]) { - // TODO: I'm unhappy about calling parseIncludeComponent twice, as it invokes git ls-remote - const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); - // converts component to project. gitlab allows two different file path ways to include a component - const files = [`${componentName}.yml`, `${componentName}/template.yml`]; + const component = componentParseCache.get(index); + assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`); + // gitlab allows two different file path ways to include a component + const files = [`${component.name}.yml`, `${component.name}/template.yml`]; let file = null; for (const f of files) { let searchPath = `${cwd}/${f}`; - if (!isLocalComponent) { - searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; + if (!component.isLocal) { + searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.ref}/${f}`; } if (fs.existsSync(searchPath)) { file = searchPath; } } - assert(file !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${domain}` + - (port ? `:${port}` : "") + `/${projectPath}\``); + assert(file !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${component.domain}` + + (component.port ? `:${component.port}` : "") + `/${component.projectPath}\``); const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables); // Expand local includes inside to a "project"-like include @@ -165,9 +175,9 @@ export class ParserIncludes { } } fileDoc["include"][i] = { - project: projectPath, + project: component.projectPath, file: inner["local"].replace(/^\//, ""), - ref: ref, + ref: component.ref, inputs: inner.inputs || {}, }; }); @@ -237,7 +247,7 @@ export class ParserIncludes { }; } - static parseIncludeComponent (component: string, gitData: GitData): {domain: string; port: string; projectPath: string; componentName: string; ref: string; isLocalComponent: boolean} { + static parseIncludeComponent (component: string, gitData: GitData): ParsedComponent { assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`); const pattern = /(?[^/:\s]+)(:(?\d+))?\/(?.+)\/(?[^@]+)@(?.+)/; // https://regexr.com/7v7hm const gitRemoteMatch = pattern.exec(component); @@ -275,9 +285,9 @@ export class ParserIncludes { domain: domain, port: port, projectPath: projectPath, - componentName: `templates/${gitRemoteMatch.groups["componentName"]}`, + name: `templates/${gitRemoteMatch.groups["componentName"]}`, ref: ref, - isLocalComponent: isLocalComponent, + isLocal: isLocalComponent, }; } From 61ac0ec87fe29439e9f1a23c64696b75ff497aeb Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Wed, 3 Dec 2025 10:37:50 +0100 Subject: [PATCH 07/12] refactor local include expansion --- src/parser-includes.ts | 60 ++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 81cc45231..8ab25824c 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -125,29 +125,13 @@ export class ParserIncludes { , {inputs: value.inputs || {}} , expandVariables); // Expand local includes inside a "project"-like include - fileDoc["include"] = this.expandInclude(fileDoc["include"], opts.variables); - fileDoc["include"].forEach((inner: any, i: number) => { - if (!inner["local"]) return; - if (inner["rules"]) { - const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, gitData); - if (rulesResult.when === "never") { - return; - } - } - fileDoc["include"][i] = { - project: value["project"], - file: inner["local"].replace(/^\//, ""), - ref: value["ref"], - inputs: inner.inputs || {}, - }; - }); - + fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], value["project"], value["ref"], opts); includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } } else if (value["component"]) { const component = componentParseCache.get(index); assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`); - // gitlab allows two different file path ways to include a component + // Gitlab allows two different file paths to include a component const files = [`${component.name}.yml`, `${component.name}/template.yml`]; let file = null; @@ -165,23 +149,7 @@ export class ParserIncludes { const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables); // Expand local includes inside to a "project"-like include - fileDoc["include"] = this.expandInclude(fileDoc["include"], opts.variables); - fileDoc["include"].forEach((inner: any, i: number) => { - if (!inner["local"]) return; - if (inner["rules"]) { - const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, gitData); - if (rulesResult.when === "never") { - return; - } - } - fileDoc["include"][i] = { - project: component.projectPath, - file: inner["local"].replace(/^\//, ""), - ref: component.ref, - inputs: inner.inputs || {}, - }; - }); - + fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.ref, opts); includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["template"]) { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); @@ -291,6 +259,28 @@ export class ParserIncludes { }; } + // Expand local includes inside to a "project"-like include + static expandInnerLocalIncludes (fileIncludes: any, projectPath: string, ref: string, opts: ParserIncludesInitOptions) { + const {argv} = opts; + const updatedIncludes = this.expandInclude(fileIncludes, opts.variables); + updatedIncludes.forEach((inner: any, i: number) => { + if (!inner["local"]) return; + if (inner["rules"]) { + const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, opts.gitData); + if (rulesResult.when === "never") { + return; + } + } + updatedIncludes[i] = { + project: projectPath, + file: inner["local"].replace(/^\//, ""), + ref: ref, + inputs: inner.inputs || {}, + }; + }); + return updatedIncludes; + } + static async downloadIncludeRemote (cwd: string, stateDir: string, url: string, fetchIncludes: boolean): Promise { const fsUrl = Utils.fsUrl(url); try { From 1ddda34e82a691d180dad6c1465180593c0108ff Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Wed, 3 Dec 2025 10:42:25 +0100 Subject: [PATCH 08/12] remove utils.remoteFileExist --- src/utils.ts | 40 ++---------------------------------- tests/utils.test.ts | 49 --------------------------------------------- 2 files changed, 2 insertions(+), 87 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 76bcefcf2..ca3a7c9f6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -8,13 +8,12 @@ import base64url from "base64url"; import execa from "execa"; import assert from "assert"; import {CICDVariable} from "./variables-from-files.js"; -import {GitData, GitSchema} from "./git-data.js"; +import {GitData} from "./git-data.js"; import globby from "globby"; import micromatch from "micromatch"; -import axios, {AxiosRequestConfig} from "axios"; +import {AxiosRequestConfig} from "axios"; import path from "path"; import {Argv} from "./argv.js"; -import {MIMEType} from "node:util"; type RuleResultOpt = { argv: Argv; @@ -403,41 +402,6 @@ export class Utils { return Object.getPrototypeOf(v) === Object.prototype; } - static async remoteFileExist (cwd: string, file: string, ref: string, domain: string, projectPath: string, protocol: GitSchema, port: string) { - switch (protocol) { - case "ssh": - case "git": - try { - await Utils.spawn(`git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), cwd); - return true; - } catch (e: any) { - if (!e.stderr.includes(`remote: fatal: pathspec '${file}' did not match any files`)) throw new Error(e); - return false; - } - - case "http": - case "https": { - try { - const axiosConfig: AxiosRequestConfig = Utils.getAxiosProxyConfig(); - const {status, headers} = await axios.get( - `${protocol}://${domain}:${port}/${projectPath}/-/raw/${ref}/${file}`, - axiosConfig, - ); - const mimeType = new MIMEType(headers["content-type"]); - return ( - status === 200 && - (mimeType.type === "text" && mimeType.subtype === "plain") // handles scenario where self-hosted gitlab returns statuscode 200 when file does not exist - ); - } catch { - return false; - } - } - default: { - Utils.switchStatementExhaustiveCheck(protocol); - } - } - } - static switchStatementExhaustiveCheck (param: never): never { // https://dev.to/babak/exhaustive-type-checking-with-typescript-4l3f throw new Error(`Unhandled case ${param}`); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 17d3b1e22..2334d470f 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -1,54 +1,5 @@ import {GitData} from "../src/git-data.js"; import {Utils} from "../src/utils.js"; -import {initSpawnSpyReject, initSpawnSpy} from "./mocks/utils.mock.js"; - -import {isSshDirFound} from "./utils.js"; - -describe("remoteFileExist", () => { - describe("protocol: git", () => { - const ref = "0.3.1"; - const domain = "gitlab.com"; - const projectPath = "components/go"; - const port = "22"; - const cwd = "."; - - test("exists", async () => { - const file = "templates/test.yml"; - - // NOTE: Only mocks git archive command if `~/.ssh` ssh dir is not found because we have no setup ssh keys in ci env - if (!isSshDirFound()) { - const spyGitArchive = { - cmdArgs: `git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), - returnValue: {output: ""}, - }; - initSpawnSpy([spyGitArchive]); - } - - const fileExist = await Utils.remoteFileExist(cwd, file, ref, domain, projectPath, "git", port); - expect(fileExist).toBe(true); - }); - - test("don't exists", async () => { - const file = "templates/potato.yml"; - - // NOTE: Only mocks git archive command if `~/.ssh` ssh dir is not found because we have no setup ssh keys in ci env - if (!isSshDirFound()) { - const spyGitArchive = { - cmdArgs: `git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), - rejection: { - stderr: `fatal: sent error to the client: git upload-archive: archiver died with error -remote: fatal: pathspec 'templates/potato.yml' did not match any files -remote: git upload-archive: archiver died with error`, - }, - }; - initSpawnSpyReject([spyGitArchive]); - } - - const fileExist = await Utils.remoteFileExist(cwd, file, ref, domain, projectPath, "git", port); - expect(fileExist).toBe(false); - }); - }); -}); describe("evaluateRuleChanges", () => { const tests = [ From 38f46ac88dae64566047a6f43e04be9b7962d8c6 Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Wed, 3 Dec 2025 16:09:06 +0100 Subject: [PATCH 09/12] workaround for git archive --- src/parser-includes.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 8ab25824c..9346ca10b 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -350,8 +350,13 @@ export class ParserIncludes { `rsync -a --ignore-missing-args ${cwd}/${target}.${ext}/templates ${cwd}/${target}`, // use rsnyc to ignore missing templates/, the check for existence is in a later step ], cwd); } else { + // git archive fails if the paths do not exist, to work around this we use a wildcard "templates/component*.yml" + // this resolves to either "templates/component.yml" or "templates/component/template.yml" + // if both exist "templates/component.yml" should be pulled + // Drawback: also pulls all .yml file from templates/component/ + const componentWildcard = files[0].replace(".yml", "*.yml"); await fs.mkdirp(`${cwd}/${target}`); - await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${files[0]} ${files[1]} | tar -f - -xC ${target}/`, cwd); + await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${componentWildcard} | tar -f - -xC ${target}/`, cwd); } } catch (e) { throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); From 1c1e0210cd4487c14d3bca8916783bd31c47d16f Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Thu, 4 Dec 2025 09:26:42 +0100 Subject: [PATCH 10/12] delete include temporary directories --- src/parser-includes.ts | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 9346ca10b..7ee99d791 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -300,6 +300,7 @@ export class ParserIncludes { static async downloadIncludeProjectFile (cwd: string, stateDir: string, project: string, ref: string, file: string, gitData: GitData, fetchIncludes: boolean): Promise { const remote = gitData.remote; const normalizedFile = file.replace(/^\/+/, ""); + let tmpDir = null; try { const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`; if (await fs.pathExists(`${cwd}/${target}/${normalizedFile}`) && !fetchIncludes) return; @@ -307,16 +308,17 @@ export class ParserIncludes { if (remote.schema.startsWith("http")) { const ext = "tmp-" + Math.random(); await fs.mkdirp(path.dirname(`${cwd}/${target}/${normalizedFile}`)); + tmpDir = `${cwd}/${target}.${ext}`; const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; await Utils.bashMulti([ `cd ${cwd}/${stateDir}`, - `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${cwd}/${target}.${ext}`, - `cd ${cwd}/${target}.${ext}`, + `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${tmpDir}`, + `cd ${tmpDir}`, `git sparse-checkout set --no-cone ${normalizedFile}`, "git checkout", `cd ${cwd}/${stateDir}`, - `cp ${cwd}/${target}.${ext}/${normalizedFile} ${cwd}/${target}/${normalizedFile}`, + `cp ${tmpDir}/${normalizedFile} ${cwd}/${target}/${normalizedFile}`, ], cwd); } else { await fs.mkdirp(`${cwd}/${target}`); @@ -324,12 +326,18 @@ export class ParserIncludes { } } catch (e) { throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${normalizedFile} }\n${e}`}); + } finally { + if (tmpDir !== null) { + // always cleanup temporary directory (if created) + await fs.rm(tmpDir, {recursive: true, force: true} ); + } } } static async downloadIncludeComponent (cwd: string, stateDir: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise { const remote = gitData.remote; const files = [`${componentName}.yml`, `${componentName}/template.yml`]; + let tmpDir = null; try { const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`; @@ -337,29 +345,36 @@ export class ParserIncludes { if (remote.schema.startsWith("http")) { const ext = "tmp-" + Math.random(); - await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`)); + await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`)); + tmpDir = `${cwd}/${target}.${ext}`; const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; await Utils.bashMulti([ `cd ${cwd}/${stateDir}`, - `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${cwd}/${target}.${ext}`, - `cd ${cwd}/${target}.${ext}`, + `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${tmpDir}`, + `cd ${tmpDir}`, `git sparse-checkout set --no-cone ${files[0]} ${files[1]}`, "git checkout", `cd ${cwd}/${stateDir}`, - `rsync -a --ignore-missing-args ${cwd}/${target}.${ext}/templates ${cwd}/${target}`, // use rsnyc to ignore missing templates/, the check for existence is in a later step + `mkdir -p ${tmpDir}/templates`, // create templates subdir (if it doesn't exist), as the check out may not create it + `cp -r ${tmpDir}/templates ${cwd}/${target}/templates`, ], cwd); } else { // git archive fails if the paths do not exist, to work around this we use a wildcard "templates/component*.yml" // this resolves to either "templates/component.yml" or "templates/component/template.yml" - // if both exist "templates/component.yml" should be pulled - // Drawback: also pulls all .yml file from templates/component/ - const componentWildcard = files[0].replace(".yml", "*.yml"); + // if both exist "templates/component.yml" will be pulled + // Drawback: also pulls all other .yml files from templates/component/ directory + const componentWildcard = `${componentName}*.yml`; await fs.mkdirp(`${cwd}/${target}`); await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${componentWildcard} | tar -f - -xC ${target}/`, cwd); } } catch (e) { throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); + } finally { + if (tmpDir !== null) { + // always cleanup temporary directory (if created) + await fs.rm(tmpDir, {recursive: true, force: true} ); + } } } From 3ae368da79586287e852d8737ffd82d1a768ccf6 Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Thu, 4 Dec 2025 10:00:51 +0100 Subject: [PATCH 11/12] fix linter errors --- src/parser-includes.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 7ee99d791..d950b36f0 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -329,7 +329,7 @@ export class ParserIncludes { } finally { if (tmpDir !== null) { // always cleanup temporary directory (if created) - await fs.rm(tmpDir, {recursive: true, force: true} ); + await fs.rm(tmpDir, {recursive: true, force: true}); } } } @@ -345,7 +345,7 @@ export class ParserIncludes { if (remote.schema.startsWith("http")) { const ext = "tmp-" + Math.random(); - await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`)); + await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`)); tmpDir = `${cwd}/${target}.${ext}`; const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; @@ -373,7 +373,7 @@ export class ParserIncludes { } finally { if (tmpDir !== null) { // always cleanup temporary directory (if created) - await fs.rm(tmpDir, {recursive: true, force: true} ); + await fs.rm(tmpDir, {recursive: true, force: true}); } } } From 344feafca849d653aed7844e281083330a338ed8 Mon Sep 17 00:00:00 2001 From: Clemens Hesse-Edenfeld Date: Thu, 4 Dec 2025 12:04:17 +0100 Subject: [PATCH 12/12] fix cp --- src/parser-includes.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser-includes.ts b/src/parser-includes.ts index d950b36f0..7b87d56ca 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -357,7 +357,7 @@ export class ParserIncludes { "git checkout", `cd ${cwd}/${stateDir}`, `mkdir -p ${tmpDir}/templates`, // create templates subdir (if it doesn't exist), as the check out may not create it - `cp -r ${tmpDir}/templates ${cwd}/${target}/templates`, + `cp -r ${tmpDir}/templates ${cwd}/${target}`, ], cwd); } else { // git archive fails if the paths do not exist, to work around this we use a wildcard "templates/component*.yml"