diff --git a/src/parser-includes.ts b/src/parser-includes.ts index 738d95717..7b87d56ca 100644 --- a/src/parser-includes.ts +++ b/src/parser-includes.ts @@ -23,6 +23,15 @@ type ParserIncludesInitOptions = { maximumIncludes: number; }; +type ParsedComponent = { + domain: string; + port: string; + projectPath: string; + name: string; + ref: string; + isLocal: boolean; +}; + export class ParserIncludes { private static count: number = 0; @@ -53,12 +62,14 @@ export class ParserIncludes { let includeDatas: any[] = []; const promises = []; const {stateDir, cwd, fetchIncludes, gitData, expandVariables} = opts; + // cache the parsed component, because parseIncludeComponent is expensive and we would call it twice otherwise + const componentParseCache = new Map(); const include = this.expandInclude(gitlabData?.include, opts.variables); this.normalizeTriggerInclude(gitlabData, opts); // Find files to fetch from remote and place in .gitlab-ci-local/includes - for (const value of include) { + for (const [index, value] of include.entries()) { if (value["rules"]) { const include_rules = value["rules"]; const rulesResult = Utils.getRulesResult({argv, cwd, rules: include_rules, variables: opts.variables}, gitData); @@ -76,13 +87,20 @@ export class ParserIncludes { promises.push(this.downloadIncludeRemote(cwd, stateDir, url, fetchIncludes)); } else if (value["remote"]) { promises.push(this.downloadIncludeRemote(cwd, stateDir, value["remote"], fetchIncludes)); + } else if (value["component"]) { + const component = this.parseIncludeComponent(value["component"], gitData); + componentParseCache.set(index, component); + if (!component.isLocal) + { + promises.push(this.downloadIncludeComponent(cwd, stateDir, component.projectPath, component.ref, component.name, gitData, fetchIncludes)); + } } } await Promise.all(promises); - for (const value of include) { + for (const [index, value] of include.entries()) { if (value["rules"]) { const include_rules = value["rules"]; const rulesResult = Utils.getRulesResult({argv, cwd, rules: include_rules, variables: opts.variables}, gitData); @@ -107,73 +125,32 @@ export class ParserIncludes { , {inputs: value.inputs || {}} , expandVariables); // Expand local includes inside a "project"-like include - fileDoc["include"] = this.expandInclude(fileDoc["include"], opts.variables); - fileDoc["include"].forEach((inner: any, i: number) => { - if (!inner["local"]) return; - if (inner["rules"]) { - const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, gitData); - if (rulesResult.when === "never") { - return; - } - } - fileDoc["include"][i] = { - project: value["project"], - file: inner["local"].replace(/^\//, ""), - ref: value["ref"], - inputs: inner.inputs || {}, - }; - }); - + fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], value["project"], value["ref"], opts); includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } } else if (value["component"]) { - const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData); - // converts component to project. gitlab allows two different file path ways to include a component - let files = [`${componentName}.yml`, `${componentName}/template.yml`, null]; - - // If a file is present locally, keep only that one in the files array to avoid downloading the other one that never exists - if (!argv.fetchIncludes) { - for (const f of files) { - const localFileName = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; - if (fs.existsSync(localFileName)) { - files = [f]; - break; - } - } - } + const component = componentParseCache.get(index); + assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`); + // Gitlab allows two different file paths to include a component + const files = [`${component.name}.yml`, `${component.name}/template.yml`]; + let file = null; for (const f of files) { - assert(f !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${domain}` + - (port ? `:${port}` : "") + `/${projectPath}\``); - - if (isLocalComponent) { - const localComponentInclude = `${cwd}/${f}`; - if (!(await fs.pathExists(localComponentInclude))) { - continue; - } - - const content = await Parser.loadYaml(localComponentInclude, {inputs: value.inputs || {}}, expandVariables); - includeDatas = includeDatas.concat(await this.init(content, opts)); - break; - } else { - const localFileName = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`; - // Check remotely only if the file does not exist locally - if (!fs.existsSync(localFileName) && !(await Utils.remoteFileExist(cwd, f, ref, domain, projectPath, gitData.remote.schema, gitData.remote.port))) { - continue; - } - - const fileDoc = { - include: { - project: projectPath, - file: f, - ref: ref, - inputs: value.inputs || {}, - }, - }; - includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); - break; + let searchPath = `${cwd}/${f}`; + if (!component.isLocal) { + searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.ref}/${f}`; + } + if (fs.existsSync(searchPath)) { + file = searchPath; } } + assert(file !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${component.domain}` + + (component.port ? `:${component.port}` : "") + `/${component.projectPath}\``); + + const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables); + // Expand local includes inside to a "project"-like include + fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.ref, opts); + includeDatas = includeDatas.concat(await this.init(fileDoc, opts)); } else if (value["template"]) { const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]); const fsUrl = Utils.fsUrl(`https://${domain}/${project}/-/raw/${ref}/${file}`); @@ -238,7 +215,7 @@ export class ParserIncludes { }; } - static parseIncludeComponent (component: string, gitData: GitData): {domain: string; port: string; projectPath: string; componentName: string; ref: string; isLocalComponent: boolean} { + static parseIncludeComponent (component: string, gitData: GitData): ParsedComponent { assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`); const pattern = /(?[^/:\s]+)(:(?\d+))?\/(?.+)\/(?[^@]+)@(?.+)/; // https://regexr.com/7v7hm const gitRemoteMatch = pattern.exec(component); @@ -254,10 +231,10 @@ export class ParserIncludes { if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) { // https://docs.gitlab.com/ci/components/#semantic-version-ranges let stdout; - try { + if (gitData.remote.schema == "git" || gitData.remote.schema == "ssh") { stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout; - } catch { - stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `https://${domain}:${port ?? 443}/${projectPath}.git`]).stdout; + } else { + stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout; } assert(stdout); const tags = stdout @@ -276,12 +253,34 @@ export class ParserIncludes { domain: domain, port: port, projectPath: projectPath, - componentName: `templates/${gitRemoteMatch.groups["componentName"]}`, + name: `templates/${gitRemoteMatch.groups["componentName"]}`, ref: ref, - isLocalComponent: isLocalComponent, + isLocal: isLocalComponent, }; } + // Expand local includes inside to a "project"-like include + static expandInnerLocalIncludes (fileIncludes: any, projectPath: string, ref: string, opts: ParserIncludesInitOptions) { + const {argv} = opts; + const updatedIncludes = this.expandInclude(fileIncludes, opts.variables); + updatedIncludes.forEach((inner: any, i: number) => { + if (!inner["local"]) return; + if (inner["rules"]) { + const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, opts.gitData); + if (rulesResult.when === "never") { + return; + } + } + updatedIncludes[i] = { + project: projectPath, + file: inner["local"].replace(/^\//, ""), + ref: ref, + inputs: inner.inputs || {}, + }; + }); + return updatedIncludes; + } + static async downloadIncludeRemote (cwd: string, stateDir: string, url: string, fetchIncludes: boolean): Promise { const fsUrl = Utils.fsUrl(url); try { @@ -301,6 +300,7 @@ export class ParserIncludes { static async downloadIncludeProjectFile (cwd: string, stateDir: string, project: string, ref: string, file: string, gitData: GitData, fetchIncludes: boolean): Promise { const remote = gitData.remote; const normalizedFile = file.replace(/^\/+/, ""); + let tmpDir = null; try { const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`; if (await fs.pathExists(`${cwd}/${target}/${normalizedFile}`) && !fetchIncludes) return; @@ -308,16 +308,17 @@ export class ParserIncludes { if (remote.schema.startsWith("http")) { const ext = "tmp-" + Math.random(); await fs.mkdirp(path.dirname(`${cwd}/${target}/${normalizedFile}`)); + tmpDir = `${cwd}/${target}.${ext}`; const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; await Utils.bashMulti([ `cd ${cwd}/${stateDir}`, - `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${cwd}/${target}.${ext}`, - `cd ${cwd}/${target}.${ext}`, + `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${tmpDir}`, + `cd ${tmpDir}`, `git sparse-checkout set --no-cone ${normalizedFile}`, "git checkout", `cd ${cwd}/${stateDir}`, - `cp ${cwd}/${target}.${ext}/${normalizedFile} ${cwd}/${target}/${normalizedFile}`, + `cp ${tmpDir}/${normalizedFile} ${cwd}/${target}/${normalizedFile}`, ], cwd); } else { await fs.mkdirp(`${cwd}/${target}`); @@ -325,6 +326,55 @@ export class ParserIncludes { } } catch (e) { throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${normalizedFile} }\n${e}`}); + } finally { + if (tmpDir !== null) { + // always cleanup temporary directory (if created) + await fs.rm(tmpDir, {recursive: true, force: true}); + } + } + } + + static async downloadIncludeComponent (cwd: string, stateDir: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise { + const remote = gitData.remote; + const files = [`${componentName}.yml`, `${componentName}/template.yml`]; + let tmpDir = null; + try { + const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`; + + if (!fetchIncludes && (await fs.pathExists(`${cwd}/${target}/${files[0]}`) || await fs.pathExists(`${cwd}/${target}/${files[1]}`))) return; + + if (remote.schema.startsWith("http")) { + const ext = "tmp-" + Math.random(); + await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`)); + tmpDir = `${cwd}/${target}.${ext}`; + + const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`; + await Utils.bashMulti([ + `cd ${cwd}/${stateDir}`, + `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${tmpDir}`, + `cd ${tmpDir}`, + `git sparse-checkout set --no-cone ${files[0]} ${files[1]}`, + "git checkout", + `cd ${cwd}/${stateDir}`, + `mkdir -p ${tmpDir}/templates`, // create templates subdir (if it doesn't exist), as the check out may not create it + `cp -r ${tmpDir}/templates ${cwd}/${target}`, + ], cwd); + } else { + // git archive fails if the paths do not exist, to work around this we use a wildcard "templates/component*.yml" + // this resolves to either "templates/component.yml" or "templates/component/template.yml" + // if both exist "templates/component.yml" will be pulled + // Drawback: also pulls all other .yml files from templates/component/ directory + const componentWildcard = `${componentName}*.yml`; + await fs.mkdirp(`${cwd}/${target}`); + await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${componentWildcard} | tar -f - -xC ${target}/`, cwd); + } + } catch (e) { + throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`}); + } finally { + if (tmpDir !== null) { + // always cleanup temporary directory (if created) + await fs.rm(tmpDir, {recursive: true, force: true}); + } } } diff --git a/src/utils.ts b/src/utils.ts index 76bcefcf2..ca3a7c9f6 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -8,13 +8,12 @@ import base64url from "base64url"; import execa from "execa"; import assert from "assert"; import {CICDVariable} from "./variables-from-files.js"; -import {GitData, GitSchema} from "./git-data.js"; +import {GitData} from "./git-data.js"; import globby from "globby"; import micromatch from "micromatch"; -import axios, {AxiosRequestConfig} from "axios"; +import {AxiosRequestConfig} from "axios"; import path from "path"; import {Argv} from "./argv.js"; -import {MIMEType} from "node:util"; type RuleResultOpt = { argv: Argv; @@ -403,41 +402,6 @@ export class Utils { return Object.getPrototypeOf(v) === Object.prototype; } - static async remoteFileExist (cwd: string, file: string, ref: string, domain: string, projectPath: string, protocol: GitSchema, port: string) { - switch (protocol) { - case "ssh": - case "git": - try { - await Utils.spawn(`git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), cwd); - return true; - } catch (e: any) { - if (!e.stderr.includes(`remote: fatal: pathspec '${file}' did not match any files`)) throw new Error(e); - return false; - } - - case "http": - case "https": { - try { - const axiosConfig: AxiosRequestConfig = Utils.getAxiosProxyConfig(); - const {status, headers} = await axios.get( - `${protocol}://${domain}:${port}/${projectPath}/-/raw/${ref}/${file}`, - axiosConfig, - ); - const mimeType = new MIMEType(headers["content-type"]); - return ( - status === 200 && - (mimeType.type === "text" && mimeType.subtype === "plain") // handles scenario where self-hosted gitlab returns statuscode 200 when file does not exist - ); - } catch { - return false; - } - } - default: { - Utils.switchStatementExhaustiveCheck(protocol); - } - } - } - static switchStatementExhaustiveCheck (param: never): never { // https://dev.to/babak/exhaustive-type-checking-with-typescript-4l3f throw new Error(`Unhandled case ${param}`); diff --git a/tests/test-cases/include-component/integration.test.ts b/tests/test-cases/include-component/integration.test.ts index fc3d43926..076adeac9 100644 --- a/tests/test-cases/include-component/integration.test.ts +++ b/tests/test-cases/include-component/integration.test.ts @@ -20,7 +20,7 @@ test("include-component no component template file (protocol: https)", async () expect(true).toBe(false); } catch (e: any) { assert(e instanceof AssertionError, `Unexpected error thrown:\n ${e}`); - expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml,] must exist in `gitlab.com/components/go`"); + expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml] must exist in `gitlab.com/components/go`"); } }); diff --git a/tests/utils.test.ts b/tests/utils.test.ts index 17d3b1e22..2334d470f 100644 --- a/tests/utils.test.ts +++ b/tests/utils.test.ts @@ -1,54 +1,5 @@ import {GitData} from "../src/git-data.js"; import {Utils} from "../src/utils.js"; -import {initSpawnSpyReject, initSpawnSpy} from "./mocks/utils.mock.js"; - -import {isSshDirFound} from "./utils.js"; - -describe("remoteFileExist", () => { - describe("protocol: git", () => { - const ref = "0.3.1"; - const domain = "gitlab.com"; - const projectPath = "components/go"; - const port = "22"; - const cwd = "."; - - test("exists", async () => { - const file = "templates/test.yml"; - - // NOTE: Only mocks git archive command if `~/.ssh` ssh dir is not found because we have no setup ssh keys in ci env - if (!isSshDirFound()) { - const spyGitArchive = { - cmdArgs: `git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), - returnValue: {output: ""}, - }; - initSpawnSpy([spyGitArchive]); - } - - const fileExist = await Utils.remoteFileExist(cwd, file, ref, domain, projectPath, "git", port); - expect(fileExist).toBe(true); - }); - - test("don't exists", async () => { - const file = "templates/potato.yml"; - - // NOTE: Only mocks git archive command if `~/.ssh` ssh dir is not found because we have no setup ssh keys in ci env - if (!isSshDirFound()) { - const spyGitArchive = { - cmdArgs: `git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), - rejection: { - stderr: `fatal: sent error to the client: git upload-archive: archiver died with error -remote: fatal: pathspec 'templates/potato.yml' did not match any files -remote: git upload-archive: archiver died with error`, - }, - }; - initSpawnSpyReject([spyGitArchive]); - } - - const fileExist = await Utils.remoteFileExist(cwd, file, ref, domain, projectPath, "git", port); - expect(fileExist).toBe(false); - }); - }); -}); describe("evaluateRuleChanges", () => { const tests = [