Use sparse checkout for components

src/parser-includes.ts

@@ -23,6 +23,15 @@ type ParserIncludesInitOptions = {
     maximumIncludes: number;
 };
 
+type ParsedComponent = {
+    domain: string;
+    port: string;
+    projectPath: string;
+    name: string;
+    ref: string;
+    isLocal: boolean;
+};
+
 export class ParserIncludes {
     private static count: number = 0;
 
@@ -53,12 +62,14 @@ export class ParserIncludes {
         let includeDatas: any[] = [];
         const promises = [];
         const {stateDir, cwd, fetchIncludes, gitData, expandVariables} = opts;
+        // cache the parsed component, because parseIncludeComponent is expensive and we would call it twice otherwise
+        const componentParseCache = new Map<number, ParsedComponent>();
 
         const include = this.expandInclude(gitlabData?.include, opts.variables);
 
         this.normalizeTriggerInclude(gitlabData, opts);
         // Find files to fetch from remote and place in .gitlab-ci-local/includes
-        for (const value of include) {
+        for (const [index, value] of include.entries()) {
             if (value["rules"]) {
                 const include_rules = value["rules"];
                 const rulesResult = Utils.getRulesResult({argv, cwd, rules: include_rules, variables: opts.variables}, gitData);
@@ -76,13 +87,20 @@ export class ParserIncludes {
                 promises.push(this.downloadIncludeRemote(cwd, stateDir, url, fetchIncludes));
             } else if (value["remote"]) {
                 promises.push(this.downloadIncludeRemote(cwd, stateDir, value["remote"], fetchIncludes));
+            } else if (value["component"]) {
+                const component = this.parseIncludeComponent(value["component"], gitData);
+                componentParseCache.set(index, component);
+                if (!component.isLocal)
+                {
+                    promises.push(this.downloadIncludeComponent(cwd, stateDir, component.projectPath, component.ref, component.name, gitData, fetchIncludes));
+                }
             }
 
         }
 
         await Promise.all(promises);
 
-        for (const value of include) {
+        for (const [index, value] of include.entries()) {
             if (value["rules"]) {
                 const include_rules = value["rules"];
                 const rulesResult = Utils.getRulesResult({argv, cwd, rules: include_rules, variables: opts.variables}, gitData);
@@ -107,73 +125,32 @@ export class ParserIncludes {
                         , {inputs: value.inputs || {}}
                         , expandVariables);
                     // Expand local includes inside a "project"-like include
-                    fileDoc["include"] = this.expandInclude(fileDoc["include"], opts.variables);
-                    fileDoc["include"].forEach((inner: any, i: number) => {
-                        if (!inner["local"]) return;
-                        if (inner["rules"]) {
-                            const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, gitData);
-                            if (rulesResult.when === "never") {
-                                return;
-                            }
-                        }
-                        fileDoc["include"][i] = {
-                            project: value["project"],
-                            file: inner["local"].replace(/^\//, ""),
-                            ref: value["ref"],
-                            inputs: inner.inputs || {},
-                        };
-                    });
-
+                    fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], value["project"], value["ref"], opts);
                     includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
                 }
             } else if (value["component"]) {
-                const {domain, port, projectPath, componentName, ref, isLocalComponent} = this.parseIncludeComponent(value["component"], gitData);
-                // converts component to project. gitlab allows two different file path ways to include a component
-                let files = [`${componentName}.yml`, `${componentName}/template.yml`, null];
-
-                // If a file is present locally, keep only that one in the files array to avoid downloading the other one that never exists
-                if (!argv.fetchIncludes) {
-                    for (const f of files) {
-                        const localFileName = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`;
-                        if (fs.existsSync(localFileName)) {
-                            files = [f];
-                            break;
-                        }
-                    }
-                }
+                const component = componentParseCache.get(index);
+                assert(component !== undefined, `Internal error, component parse cache missing entry [${index}]`);
+                // Gitlab allows two different file paths to include a component
+                const files = [`${component.name}.yml`, `${component.name}/template.yml`];
 
+                let file = null;
                 for (const f of files) {
-                    assert(f !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${domain}` +
-                                        (port ? `:${port}` : "") + `/${projectPath}\``);
-
-                    if (isLocalComponent) {
-                        const localComponentInclude = `${cwd}/${f}`;
-                        if (!(await fs.pathExists(localComponentInclude))) {
-                            continue;
-                        }
-
-                        const content = await Parser.loadYaml(localComponentInclude, {inputs: value.inputs || {}}, expandVariables);
-                        includeDatas = includeDatas.concat(await this.init(content, opts));
-                        break;
-                    } else {
-                        const localFileName = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${projectPath}/${ref}/${f}`;
-                        // Check remotely only if the file does not exist locally
-                        if (!fs.existsSync(localFileName) && !(await Utils.remoteFileExist(cwd, f, ref, domain, projectPath, gitData.remote.schema, gitData.remote.port))) {
-                            continue;
-                        }
-
-                        const fileDoc = {
-                            include: {
-                                project: projectPath,
-                                file: f,
-                                ref: ref,
-                                inputs: value.inputs || {},
-                            },
-                        };
-                        includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
-                        break;
+                    let searchPath = `${cwd}/${f}`;
+                    if (!component.isLocal) {
+                        searchPath = `${cwd}/${stateDir}/includes/${gitData.remote.host}/${component.projectPath}/${component.ref}/${f}`;
+                    }
+                    if (fs.existsSync(searchPath)) {
+                        file = searchPath;
                     }
                 }
+                assert(file !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${component.domain}` +
+                                    (component.port ? `:${component.port}` : "") + `/${component.projectPath}\``);
+
+                const fileDoc = await Parser.loadYaml(file, {inputs: value.inputs || {}}, expandVariables);
+                // Expand local includes inside to a "project"-like include
+                fileDoc["include"] = this.expandInnerLocalIncludes(fileDoc["include"], component.projectPath, component.ref, opts);
+                includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
             } else if (value["template"]) {
                 const {project, ref, file, domain} = this.covertTemplateToProjectFile(value["template"]);
                 const fsUrl = Utils.fsUrl(`https://${domain}/${project}/-/raw/${ref}/${file}`);
@@ -238,7 +215,7 @@ export class ParserIncludes {
         };
     }
 
-    static parseIncludeComponent (component: string, gitData: GitData): {domain: string; port: string; projectPath: string; componentName: string; ref: string; isLocalComponent: boolean} {
+    static parseIncludeComponent (component: string, gitData: GitData): ParsedComponent {
         assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`);
         const pattern = /(?<domain>[^/:\s]+)(:(?<port>\d+))?\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // https://regexr.com/7v7hm
         const gitRemoteMatch = pattern.exec(component);
@@ -254,10 +231,10 @@ export class ParserIncludes {
             if (ref == "~latest" || semanticVersionRangesPattern.test(ref)) {
                 // https://docs.gitlab.com/ci/components/#semantic-version-ranges
                 let stdout;
-                try {
+                if (gitData.remote.schema == "git" || gitData.remote.schema == "ssh") {
                     stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `git@${domain}:${projectPath}`]).stdout;
-                } catch {
-                    stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `https://${domain}:${port ?? 443}/${projectPath}.git`]).stdout;
+                } else {
+                    stdout = Utils.syncSpawn(["git", "ls-remote", "--tags", `${gitData.remote.schema}://${domain}:${port ?? 443}/${projectPath}.git`]).stdout;
                 }
                 assert(stdout);
                 const tags = stdout
@@ -276,12 +253,34 @@ export class ParserIncludes {
             domain: domain,
             port: port,
             projectPath: projectPath,
-            componentName: `templates/${gitRemoteMatch.groups["componentName"]}`,
+            name: `templates/${gitRemoteMatch.groups["componentName"]}`,
             ref: ref,
-            isLocalComponent: isLocalComponent,
+            isLocal: isLocalComponent,
         };
     }
 
+    // Expand local includes inside to a "project"-like include
+    static expandInnerLocalIncludes (fileIncludes: any, projectPath: string, ref: string, opts: ParserIncludesInitOptions) {
+        const {argv} = opts;
+        const updatedIncludes = this.expandInclude(fileIncludes, opts.variables);
+        updatedIncludes.forEach((inner: any, i: number) => {
+            if (!inner["local"]) return;
+            if (inner["rules"]) {
+                const rulesResult = Utils.getRulesResult({argv, cwd: opts.cwd, variables: opts.variables, rules: inner["rules"]}, opts.gitData);
+                if (rulesResult.when === "never") {
+                    return;
+                }
+            }
+            updatedIncludes[i] = {
+                project: projectPath,
+                file: inner["local"].replace(/^\//, ""),
+                ref: ref,
+                inputs: inner.inputs || {},
+            };
+        });
+        return updatedIncludes;
+    }
+
     static async downloadIncludeRemote (cwd: string, stateDir: string, url: string, fetchIncludes: boolean): Promise<void> {
         const fsUrl = Utils.fsUrl(url);
         try {
@@ -301,30 +300,81 @@ export class ParserIncludes {
     static async downloadIncludeProjectFile (cwd: string, stateDir: string, project: string, ref: string, file: string, gitData: GitData, fetchIncludes: boolean): Promise<void> {
         const remote = gitData.remote;
         const normalizedFile = file.replace(/^\/+/, "");
+        let tmpDir = null;
         try {
             const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`;
             if (await fs.pathExists(`${cwd}/${target}/${normalizedFile}`) && !fetchIncludes) return;
 
             if (remote.schema.startsWith("http")) {
                 const ext = "tmp-" + Math.random();
                 await fs.mkdirp(path.dirname(`${cwd}/${target}/${normalizedFile}`));
+                tmpDir = `${cwd}/${target}.${ext}`;
 
                 const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`;
                 await Utils.bashMulti([
                     `cd ${cwd}/${stateDir}`,
-                    `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${cwd}/${target}.${ext}`,
-                    `cd ${cwd}/${target}.${ext}`,
+                    `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${tmpDir}`,
+                    `cd ${tmpDir}`,
                     `git sparse-checkout set --no-cone ${normalizedFile}`,
                     "git checkout",
                     `cd ${cwd}/${stateDir}`,
-                    `cp ${cwd}/${target}.${ext}/${normalizedFile} ${cwd}/${target}/${normalizedFile}`,
+                    `cp ${tmpDir}/${normalizedFile} ${cwd}/${target}/${normalizedFile}`,
                 ], cwd);
             } else {
                 await fs.mkdirp(`${cwd}/${target}`);
                 await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${normalizedFile} | tar -f - -xC ${target}/`, cwd);
             }
         } catch (e) {
             throw new AssertionError({message: `Project include could not be fetched { project: ${project}, ref: ${ref}, file: ${normalizedFile} }\n${e}`});
+        } finally {
+            if (tmpDir !== null) {
+                // always cleanup temporary directory (if created)
+                await fs.rm(tmpDir, {recursive: true, force: true});
+            }
+        }
+    }
+
+    static async downloadIncludeComponent (cwd: string, stateDir: string, project: string, ref: string, componentName: string, gitData: GitData, fetchIncludes: boolean): Promise<void> {
+        const remote = gitData.remote;
+        const files = [`${componentName}.yml`, `${componentName}/template.yml`];
+        let tmpDir = null;
+        try {
+            const target = `${stateDir}/includes/${remote.host}/${project}/${ref}`;
+
+            if (!fetchIncludes && (await fs.pathExists(`${cwd}/${target}/${files[0]}`) || await fs.pathExists(`${cwd}/${target}/${files[1]}`))) return;
+
+            if (remote.schema.startsWith("http")) {
+                const ext = "tmp-" + Math.random();
+                await fs.mkdirp(path.dirname(`${cwd}/${target}/templates`));
+                tmpDir = `${cwd}/${target}.${ext}`;
+
+                const gitCloneBranch = (ref === "HEAD") ? "" : `--branch ${ref}`;
+                await Utils.bashMulti([
+                    `cd ${cwd}/${stateDir}`,
+                    `git clone ${gitCloneBranch} -n --depth=1 --filter=tree:0 ${remote.schema}://${remote.host}:${remote.port}/${project}.git ${tmpDir}`,
+                    `cd ${tmpDir}`,
+                    `git sparse-checkout set --no-cone ${files[0]} ${files[1]}`,
+                    "git checkout",
+                    `cd ${cwd}/${stateDir}`,
+                    `mkdir -p ${tmpDir}/templates`, // create templates subdir (if it doesn't exist), as the check out may not create it
+                    `cp -r ${tmpDir}/templates ${cwd}/${target}`,
+                ], cwd);
+            } else {
+                // git archive fails if the paths do not exist, to work around this we use a wildcard "templates/component*.yml"
+                // this resolves to either "templates/component.yml" or "templates/component/template.yml"
+                // if both exist "templates/component.yml" will be pulled
+                // Drawback: also pulls all other .yml files from templates/component/ directory
+                const componentWildcard = `${componentName}*.yml`;
+                await fs.mkdirp(`${cwd}/${target}`);
+                await Utils.bash(`set -eou pipefail; git archive --remote=ssh://git@${remote.host}:${remote.port}/${project}.git ${ref} ${componentWildcard} | tar -f - -xC ${target}/`, cwd);
+            }
+        } catch (e) {
+            throw new AssertionError({message: `Component include could not be fetched { project: ${project}, ref: ${ref}, file: ${files} }\n${e}`});
+        } finally {
+            if (tmpDir !== null) {
+                // always cleanup temporary directory (if created)
+                await fs.rm(tmpDir, {recursive: true, force: true});
+            }
         }
     }
 

src/utils.ts

@@ -8,13 +8,12 @@ import base64url from "base64url";
 import execa from "execa";
 import assert from "assert";
 import {CICDVariable} from "./variables-from-files.js";
-import {GitData, GitSchema} from "./git-data.js";
+import {GitData} from "./git-data.js";
 import globby from "globby";
 import micromatch from "micromatch";
-import axios, {AxiosRequestConfig} from "axios";
+import {AxiosRequestConfig} from "axios";
 import path from "path";
 import {Argv} from "./argv.js";
-import {MIMEType} from "node:util";
 
 type RuleResultOpt = {
     argv: Argv;
@@ -403,41 +402,6 @@ export class Utils {
         return Object.getPrototypeOf(v) === Object.prototype;
     }
 
-    static async remoteFileExist (cwd: string, file: string, ref: string, domain: string, projectPath: string, protocol: GitSchema, port: string) {
-        switch (protocol) {
-            case "ssh":
-            case "git":
-                try {
-                    await Utils.spawn(`git archive --remote=ssh://git@${domain}:${port}/${projectPath}.git ${ref} ${file}`.split(" "), cwd);
-                    return true;
-                } catch (e: any) {
-                    if (!e.stderr.includes(`remote: fatal: pathspec '${file}' did not match any files`)) throw new Error(e);
-                    return false;
-                }
-
-            case "http":
-            case "https": {
-                try {
-                    const axiosConfig: AxiosRequestConfig = Utils.getAxiosProxyConfig();
-                    const {status, headers} = await axios.get(
-                        `${protocol}://${domain}:${port}/${projectPath}/-/raw/${ref}/${file}`,
-                        axiosConfig,
-                    );
-                    const mimeType = new MIMEType(headers["content-type"]);
-                    return (
-                        status === 200 &&
-                        (mimeType.type === "text" && mimeType.subtype === "plain") // handles scenario where self-hosted gitlab returns statuscode 200 when file does not exist
-                    );
-                } catch {
-                    return false;
-                }
-            }
-            default: {
-                Utils.switchStatementExhaustiveCheck(protocol);
-            }
-        }
-    }
-
     static switchStatementExhaustiveCheck (param: never): never {
         // https://dev.to/babak/exhaustive-type-checking-with-typescript-4l3f
         throw new Error(`Unhandled case ${param}`);

tests/test-cases/include-component/integration.test.ts

@@ -20,7 +20,7 @@ test("include-component no component template file (protocol: https)", async ()
         expect(true).toBe(false);
     } catch (e: any) {
         assert(e instanceof AssertionError, `Unexpected error thrown:\n ${e}`);
-        expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml,] must exist in `gitlab.com/components/go`");
+        expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml] must exist in `gitlab.com/components/go`");
     }
 });