From 250cb3c18e0c2d66003c0638d64fb63133c9ce26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Wed, 7 May 2025 20:07:10 +0200 Subject: [PATCH 1/7] thing --- action.yml | 4 ++ .../plugins/languages/analyzer/analyzer.mjs | 67 ++++++++++++++++++- source/plugins/languages/analyzer/indepth.mjs | 7 ++ source/plugins/languages/analyzer/recent.mjs | 10 +++ source/plugins/languages/analyzers.mjs | 8 +-- source/plugins/languages/index.mjs | 2 +- source/plugins/languages/metadata.yml | 17 +++++ 7 files changed, 107 insertions(+), 8 deletions(-) diff --git a/action.yml b/action.yml index ba2ab546ff5..81956619a42 100644 --- a/action.yml +++ b/action.yml @@ -281,6 +281,10 @@ inputs: description: Skipped repositories default: + plugin_languages_paths_ignored: + description: Ignored folders in repositories (format repo/path or owner/repo/path) + default: + plugin_languages_limit: description: Display limit default: diff --git a/source/plugins/languages/analyzer/analyzer.mjs b/source/plugins/languages/analyzer/analyzer.mjs index 529a367e5cf..db91a7345d8 100644 --- a/source/plugins/languages/analyzer/analyzer.mjs +++ b/source/plugins/languages/analyzer/analyzer.mjs @@ -8,7 +8,7 @@ import { filters } from "../../../app/metrics/utils.mjs" /**Analyzer */ export class Analyzer { /**Constructor */ - constructor(login, {account = "bypass", authoring = [], uid = Math.random(), shell, rest = null, context = {mode: "user"}, skipped = [], categories = ["programming", "markup"], timeout = {global: NaN, repositories: NaN}}) { + constructor(login, {account = "bypass", authoring = [], uid = Math.random(), shell, rest = null, context = {mode: "user"}, skipped = [], pathsIgnored = [], categories = ["programming", "markup"], timeout = {global: NaN, repositories: NaN}}) { //User informations this.login = login this.account = account @@ -26,12 +26,13 @@ export class Analyzer { line: /^(?[-+])\s*(?[\s\S]+)$/, } this.parser = /^(?[\s\S]+?)\/(?[\s\S]+?)(?:@(?[\s\S]+?)(?::(?[\s\S]+))?)?$/ - this.consumed = false //Options this.skipped = skipped + this.pathsIgnored = pathsIgnored this.categories = categories this.timeout = timeout + this.consumed = false //Results this.results = {partial: {global: false, repositories: false}, total: 0, lines: {}, stats: {}, colors: {}, commits: 0, files: 0, missed: {lines: 0, bytes: 0, commits: 0}, elapsed: 0} @@ -106,11 +107,71 @@ export class Analyzer { } } + /**Check if path should be ignored */ + shouldIgnorePath(repo, filePath) { + for (const ignoredPath of this.pathsIgnored) { + // Check for repo:path pattern (using colon as separator) + if (ignoredPath.includes(':')) { + const [repoSpec, pathToIgnore] = ignoredPath.split(':', 2); + + // Handle owner/repo:path format + if (repoSpec.includes('/') && repo === repoSpec) { + if (filePath.startsWith(pathToIgnore)) { + this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore} (colon format)`) + return true; + } + } + // Handle repo:path format (current repo) + else if (repo.endsWith(`/${repoSpec}`)) { + if (filePath.startsWith(pathToIgnore)) { + this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore} (colon format)`) + return true; + } + } + continue; + } + + // Check for owner/repo/path pattern (legacy slash format) + if (ignoredPath.includes('/')) { + const parts = ignoredPath.split('/'); + + // Owner/repo/path format (at least 3 parts) + if (parts.length >= 3) { + const ownerRepo = `${parts[0]}/${parts[1]}`; + const pathToIgnore = parts.slice(2).join('/'); + + if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { + this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore}`) + return true; + } + } + // Handle repo/path format (2 parts) + else if (parts.length === 2) { + const pathToIgnore = parts[1]; + if (repo.endsWith(`/${parts[0]}`) && filePath.startsWith(pathToIgnore)) { + this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore}`) + return true; + } + } + } + // Simple path ignoring for all repos + else { + if (filePath.startsWith(ignoredPath)) { + this.debug(`ignoring file ${filePath} as it matches ignored path ${ignoredPath}`) + return true; + } + } + } + return false; + } + /**Analyze a repository */ async analyze(path, {commits = []} = {}) { const cache = {files: {}, languages: {}} const start = Date.now() let elapsed = 0, processed = 0 + const {repo} = this.parse(path) + if (this.timeout.repositories) this.debug(`timeout for repository analysis set to ${this.timeout.repositories}m`) for (const commit of commits) { @@ -121,7 +182,7 @@ export class Analyzer { break } try { - const {total, files, missed, lines, stats} = await this.linguist(path, {commit, cache}) + const {total, files, missed, lines, stats} = await this.linguist(path, {commit, cache, repo}) this.results.commits++ this.results.total += total this.results.files += files diff --git a/source/plugins/languages/analyzer/indepth.mjs b/source/plugins/languages/analyzer/indepth.mjs index 4179ff82286..cac18705007 100644 --- a/source/plugins/languages/analyzer/indepth.mjs +++ b/source/plugins/languages/analyzer/indepth.mjs @@ -197,7 +197,14 @@ export class IndepthAnalyzer extends Analyzer { const result = {total: 0, files: 0, missed: {lines: 0, bytes: 0}, lines: {}, stats: {}} const edited = new Set() const seen = new Set() + const {repo} = this.parse(path) + for (const edition of commit.editions) { + // Skip this file if it's in an ignored path + if (this.shouldIgnorePath(repo, edition.path.replace(`${path}/`, ''))) { + continue + } + edited.add(edition.path) //Guess file language with linguist (only run it once per sha) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 927c91a7189..1176ae0fca3 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -108,7 +108,17 @@ export class RecentAnalyzer extends Analyzer { const cache = {files: {}, languages} const result = {total: 0, files: 0, missed: {lines: 0, bytes: 0}, lines: {}, stats: {}, languages: {}} const edited = new Set() + for (const edition of commit.editions) { + // Extract repo from edition path + const repoMatch = edition.path.match(/^(?[^\/]+)\/(?[^\/]+)\//) + const repo = repoMatch ? `${repoMatch.groups.owner}/${repoMatch.groups.repo}` : "" + + // Skip this file if it's in an ignored path + if (repo && this.shouldIgnorePath(repo, edition.path.replace(`${repo}/`, ''))) { + continue + } + edited.add(edition.path) //Guess file language with linguist diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index 9e237065c3b..5f771851c7a 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -4,13 +4,13 @@ import { IndepthAnalyzer } from "./analyzer/indepth.mjs" import { RecentAnalyzer } from "./analyzer/recent.mjs" /**Indepth analyzer */ -export async function indepth({login, data, imports, rest, context, repositories}, {skipped, categories, timeout}) { - return new IndepthAnalyzer(login, {shell: imports, uid: data.user.databaseId, skipped, authoring: data.shared["commits.authoring"], timeout, rest, context, categories}).run({repositories}) +export async function indepth({login, data, imports, rest, context, repositories}, {skipped, pathsIgnored = [], categories, timeout}) { + return new IndepthAnalyzer(login, {shell: imports, uid: data.user.databaseId, skipped, pathsIgnored, authoring: data.shared["commits.authoring"], timeout, rest, context, categories}).run({repositories}) } /**Recent languages activity */ -export async function recent({login, data, imports, rest, context, account}, {skipped = [], categories, days = 0, load = 0, timeout}) { - return new RecentAnalyzer(login, {shell: imports, uid: data.user.databaseId, skipped, authoring: data.shared["commits.authoring"], timeout, account, rest, context, days, categories, load}).run() +export async function recent({login, data, imports, rest, context, account}, {skipped = [], pathsIgnored = [], categories, days = 0, load = 0, timeout}) { + return new RecentAnalyzer(login, {shell: imports, uid: data.user.databaseId, skipped, pathsIgnored, authoring: data.shared["commits.authoring"], timeout, account, rest, context, days, categories, load}).run() } //import.meta.main diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index d73e6027818..a6da8576b4d 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -18,7 +18,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled } //Load inputs - let {ignored, skipped, other, colors, aliases, details, threshold, limit, indepth, "indepth.custom": _indepth_custom, "analysis.timeout": _timeout_global, "analysis.timeout.repositories": _timeout_repositories, sections, categories, "recent.categories": _recent_categories, "recent.load": _recent_load, "recent.days": _recent_days} = imports.metadata + let {ignored, skipped, pathsIgnored, other, colors, aliases, details, threshold, limit, indepth, "indepth.custom": _indepth_custom, "analysis.timeout": _timeout_global, "analysis.timeout.repositories": _timeout_repositories, sections, categories, "recent.categories": _recent_categories, "recent.load": _recent_load, "recent.days": _recent_days} = imports.metadata .plugins.languages .inputs({ data, diff --git a/source/plugins/languages/metadata.yml b/source/plugins/languages/metadata.yml index 60e085e076d..94a69c7ea51 100644 --- a/source/plugins/languages/metadata.yml +++ b/source/plugins/languages/metadata.yml @@ -197,3 +197,20 @@ inputs: min: 0 max: 365 zero: disable + + plugin_languages_paths_ignored: + description: | + Ignored folders in repositories + + Use the following formats: + - `folder_name` to ignore a folder in all repositories + - `repo:path/to/folder` to ignore a specific path in a specific repository + - `owner/repo:path/to/folder` to ignore a specific path in a repository with owner + - `repo/path` to ignore a path in a specific repository + - `owner/repo/path` to ignore a path in a specific repository with owner + + The colon format (`:`) is recommended for clarity with longer paths. + type: array + format: comma-separated + default: "" + example: node_modules, vendor, myrepo:external/libs, owner/repo:generated-code From 0b6de46bc80c7823f80bd672aa57b288671100fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Wed, 7 May 2025 20:50:35 +0200 Subject: [PATCH 2/7] fixed eslint shit --- .../plugins/languages/analyzer/analyzer.mjs | 87 +++++++++---------- source/plugins/languages/analyzer/cli.mjs | 46 +++++----- source/plugins/languages/analyzer/indepth.mjs | 54 ++++++------ source/plugins/languages/analyzer/recent.mjs | 58 ++++++------- source/plugins/languages/analyzers.mjs | 10 +-- source/plugins/languages/index.mjs | 28 +++--- 6 files changed, 140 insertions(+), 143 deletions(-) diff --git a/source/plugins/languages/analyzer/analyzer.mjs b/source/plugins/languages/analyzer/analyzer.mjs index db91a7345d8..1aca80e2bbe 100644 --- a/source/plugins/languages/analyzer/analyzer.mjs +++ b/source/plugins/languages/analyzer/analyzer.mjs @@ -3,12 +3,12 @@ import fs from "fs/promises" import os from "os" import paths from "path" import git from "simple-git" -import { filters } from "../../../app/metrics/utils.mjs" +import {filters} from "../../../app/metrics/utils.mjs" /**Analyzer */ export class Analyzer { /**Constructor */ - constructor(login, {account = "bypass", authoring = [], uid = Math.random(), shell, rest = null, context = {mode: "user"}, skipped = [], pathsIgnored = [], categories = ["programming", "markup"], timeout = {global: NaN, repositories: NaN}}) { + constructor(login, {account = "bypass", authoring = [], uid = Math.random(), shell, rest = null, context = {mode:"user"}, skipped = [], pathsIgnored = [], categories = ["programming", "markup"], timeout = {global:NaN, repositories:NaN}}) { //User informations this.login = login this.account = account @@ -21,9 +21,9 @@ export class Analyzer { this.rest = rest this.context = context this.markers = { - hash: /\b[0-9a-f]{40}\b/, - file: /^[+]{3}\sb[/](?[\s\S]+)$/, - line: /^(?[-+])\s*(?[\s\S]+)$/, + hash:/\b[0-9a-f]{40}\b/, + file:/^[+]{3}\sb[/](?[\s\S]+)$/, + line:/^(?[-+])\s*(?[\s\S]+)$/, } this.parser = /^(?[\s\S]+?)\/(?[\s\S]+?)(?:@(?[\s\S]+?)(?::(?[\s\S]+))?)?$/ @@ -35,7 +35,7 @@ export class Analyzer { this.consumed = false //Results - this.results = {partial: {global: false, repositories: false}, total: 0, lines: {}, stats: {}, colors: {}, commits: 0, files: 0, missed: {lines: 0, bytes: 0, commits: 0}, elapsed: 0} + this.results = {partial:{global:false, repositories:false}, total:0, lines:{}, stats:{}, colors:{}, commits:0, files:0, missed:{lines:0, bytes:0, commits:0}, elapsed:0} this.debug(`instantiated a new ${this.constructor.name}`) } @@ -75,7 +75,7 @@ export class Analyzer { if (!this.parser.test(repository)) throw new TypeError(`"${repository}" pattern is not supported`) const {login, name, ...groups} = repository.match(this.parser)?.groups ?? {} - repository = {owner: {login}, name} + repository = {owner:{login}, name} branch = groups.branch ?? null ref = groups.ref ?? null } @@ -89,9 +89,9 @@ export class Analyzer { const {repo, branch, path} = this.parse(repository) let url = /^https?:\/\//.test(repo) ? repo : `https://github.com/${repo}` try { - this.debug(`cloning ${url} to ${path}`) - await fs.rm(path, {recursive: true, force: true}) - await fs.mkdir(path, {recursive: true}) + this.debug(`cloning https://github.com/${repo} to ${path}`) + await fs.rm(path, {recursive:true, force:true}) + await fs.mkdir(path, {recursive:true}) await git(path).clone(url, ".", ["--single-branch"]).status() this.debug(`cloned ${url} to ${path}`) if (branch) { @@ -110,68 +110,65 @@ export class Analyzer { /**Check if path should be ignored */ shouldIgnorePath(repo, filePath) { for (const ignoredPath of this.pathsIgnored) { - // Check for repo:path pattern (using colon as separator) - if (ignoredPath.includes(':')) { - const [repoSpec, pathToIgnore] = ignoredPath.split(':', 2); - - // Handle owner/repo:path format - if (repoSpec.includes('/') && repo === repoSpec) { + //Check for repo:path pattern (using colon as separator) + if (ignoredPath.includes(":")) { + const [repoSpec, pathToIgnore] = ignoredPath.split(":", 2) + + //Handle owner/repo:path format + if (repoSpec.includes("/") && repo === repoSpec) { if (filePath.startsWith(pathToIgnore)) { this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore} (colon format)`) - return true; + return true } } - // Handle repo:path format (current repo) + //Handle repo:path format (current repo) else if (repo.endsWith(`/${repoSpec}`)) { if (filePath.startsWith(pathToIgnore)) { this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore} (colon format)`) - return true; + return true } } - continue; + continue } - - // Check for owner/repo/path pattern (legacy slash format) - if (ignoredPath.includes('/')) { - const parts = ignoredPath.split('/'); - - // Owner/repo/path format (at least 3 parts) + + //Check for owner/repo/path pattern (legacy slash format) + if (ignoredPath.includes("/")) { + const parts = ignoredPath.split("/") + + //Owner/repo/path format (at least 3 parts) if (parts.length >= 3) { - const ownerRepo = `${parts[0]}/${parts[1]}`; - const pathToIgnore = parts.slice(2).join('/'); - - if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { + const ownerRepo = `${parts[0]}/${parts[1]}` + const pathToIgnore = parts.slice(2).join("/") +if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore}`) - return true; + return true } - } - // Handle repo/path format (2 parts) + } + //Handle repo/path format (2 parts) else if (parts.length === 2) { - const pathToIgnore = parts[1]; + const [, pathToIgnore] = parts if (repo.endsWith(`/${parts[0]}`) && filePath.startsWith(pathToIgnore)) { this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore}`) - return true; + return true } } - } - // Simple path ignoring for all repos - else { - if (filePath.startsWith(ignoredPath)) { + } + //Simple path ignoring for all repos + else if (filePath.startsWith(ignoredPath)) { this.debug(`ignoring file ${filePath} as it matches ignored path ${ignoredPath}`) - return true; + return true } - } } - return false; + return false } /**Analyze a repository */ async analyze(path, {commits = []} = {}) { - const cache = {files: {}, languages: {}} + const cache = {files:{}, languages:{}} const start = Date.now() let elapsed = 0, processed = 0 const {repo} = this.parse(path) - + if (this.timeout.repositories) this.debug(`timeout for repository analysis set to ${this.timeout.repositories}m`) for (const commit of commits) { @@ -215,7 +212,7 @@ export class Analyzer { async clean(path) { try { this.debug(`cleaning ${path}`) - await fs.rm(path, {recursive: true, force: true}) + await fs.rm(path, {recursive:true, force:true}) this.debug(`cleaned ${path}`) return true } diff --git a/source/plugins/languages/analyzer/cli.mjs b/source/plugins/languages/analyzer/cli.mjs index 83679997894..a95a3d24517 100644 --- a/source/plugins/languages/analyzer/cli.mjs +++ b/source/plugins/languages/analyzer/cli.mjs @@ -1,8 +1,8 @@ //Imports import OctokitRest from "@octokit/rest" import yargsparser from "yargs-parser" -import { IndepthAnalyzer } from "./indepth.mjs" -import { RecentAnalyzer } from "./recent.mjs" +import {IndepthAnalyzer} from "./indepth.mjs" +import {RecentAnalyzer} from "./recent.mjs" const help = ` @@ -17,37 +17,37 @@ export async function cli() { console.log(help) return null } - const {default: setup} = await import("../../../app/metrics/setup.mjs") - const {conf: {metadata}} = await setup({log: false}) - const {login, _: repositories, mode = "indepth"} = argv + const {default:setup} = await import("../../../app/metrics/setup.mjs") + const {conf:{metadata}} = await setup({log:false}) + const {login, _:repositories, mode = "indepth"} = argv const { - "commits.authoring": authoring, + "commits.authoring":authoring, } = await metadata.plugins.base.inputs({ - q: { - "commits.authoring": argv["commits-authoring"] || login, + q:{ + "commits.authoring":argv["commits-authoring"] || login, }, - account: "bypass", + account:"bypass", }) const { categories, - "analysis.timeout": _timeout_global, - "analysis.timeout.repositories": _timeout_repositories, - "recent.load": _recent_load, - "recent.days": _recent_days, + "analysis.timeout":_timeout_global, + "analysis.timeout.repositories":_timeout_repositories, + "recent.load":_recent_load, + "recent.days":_recent_days, } = await metadata.plugins.languages.inputs({ - q: { - categories: argv.categories || "", - "analysis.timeout": argv["timeout-global"] || "", - "analysis.timeout.repositories": argv["timeout-repositories"] || "", - "recent.load": argv["recent-load"] || "", - "recent.days": argv["recent-days"] || "", + q:{ + categories:argv.categories || "", + "analysis.timeout":argv["timeout-global"] || "", + "analysis.timeout.repositories":argv["timeout-repositories"] || "", + "recent.load":argv["recent-load"] || "", + "recent.days":argv["recent-days"] || "", }, - account: "bypass", + account:"bypass", }) //Prepare call const imports = await import("../../../app/metrics/utils.mjs") - const rest = argv.token ? new OctokitRest.Octokit({auth: argv.token, baseUrl: argv["api-url"]}) : null + const rest = argv.token ? new OctokitRest.Octokit({auth:argv.token, baseUrl:argv["api-url"]}) : null //Language analysis console.log(`analysis mode | ${mode}`) @@ -59,11 +59,11 @@ export async function cli() { case "recent": { console.log(`events to load | ${_recent_load}`) console.log(`events maximum age | ${_recent_days}`) - return new RecentAnalyzer(login, {rest, shell: imports, authoring, categories, timeout: {global: _timeout_global, repositories: _timeout_repositories}, load: _recent_load, days: _recent_days}).run({}) + return new RecentAnalyzer(login, {rest, shell:imports, authoring, categories, timeout:{global:_timeout_global, repositories:_timeout_repositories}, load:_recent_load, days:_recent_days}).run({}) } case "indepth": { console.log(`repositories | ${repositories}`) - return new IndepthAnalyzer(login, {rest, shell: imports, authoring, categories, timeout: {global: _timeout_global, repositories: _timeout_repositories}}).run({repositories}) + return new IndepthAnalyzer(login, {rest, shell:imports, authoring, categories, timeout:{global:_timeout_global, repositories:_timeout_repositories}}).run({repositories}) } } } diff --git a/source/plugins/languages/analyzer/indepth.mjs b/source/plugins/languages/analyzer/indepth.mjs index cac18705007..3ff5a933661 100644 --- a/source/plugins/languages/analyzer/indepth.mjs +++ b/source/plugins/languages/analyzer/indepth.mjs @@ -3,15 +3,15 @@ import fs from "fs/promises" import linguist from "linguist-js" import os from "os" import paths from "path" -import { Analyzer } from "./analyzer.mjs" +import {Analyzer} from "./analyzer.mjs" /**Indepth analyzer */ export class IndepthAnalyzer extends Analyzer { /**Constructor */ constructor() { super(...arguments) - this.manual = {repositories: []} - Object.assign(this.results, {verified: {signature: 0}}) + this.manual = {repositories:[]} + Object.assign(this.results, {verified:{signature:0}}) } /**Run analyzer */ @@ -48,8 +48,8 @@ export class IndepthAnalyzer extends Analyzer { try { this.debug("fetching gpg keys") for (const username of [this.login, "web-flow"]) { - const {data: keys} = await this.rest.users.listGpgKeysForUser({username}) - this.gpg.push(...keys.map(({key_id: id, raw_key: pub, emails}) => ({id, pub, emails}))) + const {data:keys} = await this.rest.users.listGpgKeysForUser({username}) + this.gpg.push(...keys.map(({key_id:id, raw_key:pub, emails}) => ({id, pub, emails}))) if (username === this.login) { for (const {email} of this.gpg.flatMap(({emails}) => emails)) { this.debug(`auto-adding ${email} to commits_authoring (fetched from gpg)`) @@ -83,7 +83,7 @@ export class IndepthAnalyzer extends Analyzer { } finally { this.debug(`cleaning ${path}`) - await fs.rm(path, {recursive: true, force: true}).catch(error => this.debug(`failed to clean ${path} (${error})`)) + await fs.rm(path, {recursive:true, force:true}).catch(error => this.debug(`failed to clean ${path} (${error})`)) } } } @@ -96,14 +96,14 @@ export class IndepthAnalyzer extends Analyzer { for (const author of this.authoring) { //Search by --author { - const output = await this.shell.run(`git log --author='${author}' --pretty=format:"%H" --regexp-ignore-case --no-merges`, {cwd: path, env: {LANG: "en_GB"}}, {log: false, debug: false, prefixed: false}) + const output = await this.shell.run(`git log --author='${author}' --pretty=format:"%H" --regexp-ignore-case --no-merges`, {cwd:path, env:{LANG:"en_GB"}}, {log:false, debug:false, prefixed:false}) const hashes = output.split("\n").map(line => line.trim()).filter(line => this.markers.hash.test(line)) hashes.forEach(hash => commits.add(hash)) this.debug(`found ${hashes.length} for ${author} (using --author)`) } //Search by --grep { - const output = await this.shell.run(`git log --grep='${author}' --pretty=format:"%H" --regexp-ignore-case --no-merges`, {cwd: path, env: {LANG: "en_GB"}}, {log: false, debug: false, prefixed: false}) + const output = await this.shell.run(`git log --grep='${author}' --pretty=format:"%H" --regexp-ignore-case --no-merges`, {cwd:path, env:{LANG:"en_GB"}}, {log:false, debug:false, prefixed:false}) const hashes = output.split("\n").map(line => line.trim()).filter(line => this.markers.hash.test(line)) hashes.forEach(hash => commits.add(hash)) this.debug(`found ${hashes.length} for ${author} (using --grep)`) @@ -112,7 +112,7 @@ export class IndepthAnalyzer extends Analyzer { //Apply ref range if specified if (ref) { this.debug(`filtering commits referenced by ${ref} in ${path}`) - const output = await this.shell.run(`git rev-list --boundary ${ref}`, {cwd: path, env: {LANG: "en_GB"}}, {log: false, debug: false, prefixed: false}) + const output = await this.shell.run(`git rev-list --boundary ${ref}`, {cwd:path, env:{LANG:"en_GB"}}, {log:false, debug:false, prefixed:false}) const hashes = output.split("\n").map(line => line.trim()).filter(line => this.markers.hash.test(line)) commits.forEach(commit => !hashes.includes(commit) ? commits.delete(commit) : null) } @@ -132,9 +132,9 @@ export class IndepthAnalyzer extends Analyzer { try { commits.push({ sha, - name: await this.shell.run(`git log ${sha} --format="%s (authored by %an on %cI)" --max-count=1`, {cwd: path, env: {LANG: "en_GB"}}, {log: false, debug: false, prefixed: false}), - verified: ("verified" in this.results) ? await this.shell.run(`git verify-commit ${sha}`, {cwd: path, env: {LANG: "en_GB"}}, {log: false, debug: false, prefixed: false}).then(() => true).catch(() => null) : null, - editions: await this.editions(path, {sha}), + name:await this.shell.run(`git log ${sha} --format="%s (authored by %an on %cI)" --max-count=1`, {cwd:path, env:{LANG:"en_GB"}}, {log:false, debug:false, prefixed:false}), + verified:("verified" in this.results) ? await this.shell.run(`git verify-commit ${sha}`, {cwd:path, env:{LANG:"en_GB"}}, {log:false, debug:false, prefixed:false}).then(() => true).catch(() => null) : null, + editions:await this.editions(path, {sha}), }) } catch (error) { @@ -149,9 +149,9 @@ export class IndepthAnalyzer extends Analyzer { const editions = [] let edition = null let cursor = 0 - await this.shell.spawn("git", ["log", sha, "--format=''", "--max-count=1", "--patch"], {cwd: path, env: {LANG: "en_GB"}}, { - debug: false, - stdout: line => { + await this.shell.spawn("git", ["log", sha, "--format=''", "--max-count=1", "--patch"], {cwd:path, env:{LANG:"en_GB"}}, { + debug:false, + stdout:line => { try { //Ignore empty lines or unneeded lines cursor++ @@ -161,9 +161,9 @@ export class IndepthAnalyzer extends Analyzer { //File marker if (this.markers.file.test(line)) { edition = { - path: `${path}/${line.match(this.markers.file)?.groups?.file}`.replace(/\\/g, "/"), - added: {lines: 0, bytes: 0}, - deleted: {lines: 0, bytes: 0}, + path:`${path}/${line.match(this.markers.file)?.groups?.file}`.replace(/\\/g, "/"), + added:{lines:0, bytes:0}, + deleted:{lines:0, bytes:0}, } editions.push(edition) return @@ -173,8 +173,8 @@ export class IndepthAnalyzer extends Analyzer { if ((edition) && (this.markers.line.test(line))) { const {op = "+", content = ""} = line.match(this.markers.line)?.groups ?? {} const size = Buffer.byteLength(content, "utf-8") - edition[{"+": "added", "-": "deleted"}[op]].bytes += size - edition[{"+": "added", "-": "deleted"}[op]].lines++ + edition[{"+":"added", "-":"deleted"}[op]].bytes += size + edition[{"+":"added", "-":"deleted"}[op]].lines++ return } } @@ -194,24 +194,24 @@ export class IndepthAnalyzer extends Analyzer { /**Run linguist against a commit and compute edited lines and bytes*/ async linguist(path, {commit, cache}) { - const result = {total: 0, files: 0, missed: {lines: 0, bytes: 0}, lines: {}, stats: {}} + const result = {total:0, files:0, missed:{lines:0, bytes:0}, lines:{}, stats:{}} const edited = new Set() const seen = new Set() const {repo} = this.parse(path) - + for (const edition of commit.editions) { - // Skip this file if it's in an ignored path - if (this.shouldIgnorePath(repo, edition.path.replace(`${path}/`, ''))) { + //Skip this file if it's in an ignored path + if (this.shouldIgnorePath(repo, edition.path.replace(`${path}/`, ""))) { continue } - + edited.add(edition.path) //Guess file language with linguist (only run it once per sha) if ((!(edition.path in cache.files)) && (!seen.has(commit.sha))) { this.debug(`language for file ${edition.path} is not in cache, running linguist at ${commit.sha}`) - await this.shell.run(`git checkout ${commit.sha}`, {cwd: path, env: {LANG: "en_GB"}}, {log: false, debug: false, prefixed: false}) - const {files: {results: files}, languages: {results: languages}} = await linguist(path) + await this.shell.run(`git checkout ${commit.sha}`, {cwd:path, env:{LANG:"en_GB"}}, {log:false, debug:false, prefixed:false}) + const {files:{results:files}, languages:{results:languages}} = await linguist(path) Object.assign(cache.files, files) Object.assign(cache.languages, languages) seen.add(commit.sha) diff --git a/source/plugins/languages/analyzer/recent.mjs b/source/plugins/languages/analyzer/recent.mjs index 1176ae0fca3..a0ba7c6d46a 100644 --- a/source/plugins/languages/analyzer/recent.mjs +++ b/source/plugins/languages/analyzer/recent.mjs @@ -1,7 +1,7 @@ //Imports import linguist from "linguist-js" -import { filters } from "../../../app/metrics/utils.mjs" -import { Analyzer } from "./analyzer.mjs" +import {filters} from "../../../app/metrics/utils.mjs" +import {Analyzer} from "./analyzer.mjs" /**Recent analyzer */ export class RecentAnalyzer extends Analyzer { @@ -10,7 +10,7 @@ export class RecentAnalyzer extends Analyzer { super(...arguments) this.days = arguments[1]?.days ?? 0 this.load = arguments[1]?.load ?? 0 - Object.assign(this.results, {days: this.days}) + Object.assign(this.results, {days:this.days}) } /**Run analyzer */ @@ -23,7 +23,7 @@ export class RecentAnalyzer extends Analyzer { /**Analyze a repository */ async analyze(path) { const patches = await this.patches() - return super.analyze(path, {commits: patches}) + return super.analyze(path, {commits:patches}) } /**Fetch patches */ @@ -33,7 +33,7 @@ export class RecentAnalyzer extends Analyzer { const commits = [], pages = Math.ceil((this.load || Infinity) / 100) if (this.context.mode === "repository") { try { - const {data: {default_branch: branch}} = await this.rest.repos.get(this.context) + const {data:{default_branch:branch}} = await this.rest.repos.get(this.context) this.context.branch = branch this.results.branch = branch this.debug(`default branch for ${this.context.owner}/${this.context.repo} is ${branch}`) @@ -46,10 +46,10 @@ export class RecentAnalyzer extends Analyzer { for (let page = 1; page <= pages; page++) { this.debug(`fetching events page ${page}`) commits.push( - ...(await (this.context.mode === "repository" ? this.rest.activity.listRepoEvents(this.context) : this.rest.activity.listEventsForAuthenticatedUser({username: this.login, per_page: 100, page}))).data + ...(await (this.context.mode === "repository" ? this.rest.activity.listRepoEvents(this.context) : this.rest.activity.listEventsForAuthenticatedUser({username:this.login, per_page:100, page}))).data .filter(({type, payload}) => (type === "PushEvent") && ((this.context.mode !== "repository") || ((this.context.mode === "repository") && (payload?.ref?.includes?.(`refs/heads/${this.context.branch}`))))) - .filter(({actor}) => (this.account === "organization") || (this.context.mode === "repository") ? true : !filters.text(actor.login, [this.login], {debug: false})) - .filter(({repo: {name: repo}}) => !this.ignore(repo)) + .filter(({actor}) => (this.account === "organization") || (this.context.mode === "repository") ? true : !filters.text(actor.login, [this.login], {debug:false})) + .filter(({repo:{name:repo}}) => !this.ignore(repo)) .filter(({created_at}) => ((!this.days) || (new Date(created_at) > new Date(Date.now() - this.days * 24 * 60 * 60 * 1000)))), ) } @@ -67,7 +67,7 @@ export class RecentAnalyzer extends Analyzer { ...await Promise.allSettled( commits .flatMap(({payload}) => payload.commits) - .filter(({committer}) => filters.text(committer?.email, this.authoring, {debug: false})) + .filter(({committer}) => filters.text(committer?.email, this.authoring, {debug:false})) .map(commit => commit.url) .map(async commit => (await this.rest.request(commit)).data), ), @@ -75,15 +75,15 @@ export class RecentAnalyzer extends Analyzer { .filter(({status}) => status === "fulfilled") .map(({value}) => value) .filter(({parents}) => parents.length <= 1) - .map(({sha, commit: {message, committer}, verification, files}) => ({ + .map(({sha, commit:{message, committer}, verification, files}) => ({ sha, - name: `${message} (authored by ${committer.name} on ${committer.date})`, - verified: verification?.verified ?? null, - editions: files.map(({filename, patch = ""}) => { + name:`${message} (authored by ${committer.name} on ${committer.date})`, + verified:verification?.verified ?? null, + editions:files.map(({filename, patch = ""}) => { const edition = { - path: filename, - added: {lines: 0, bytes: 0}, - deleted: {lines: 0, bytes: 0}, + path:filename, + added:{lines:0, bytes:0}, + deleted:{lines:0, bytes:0}, patch, } for (const line of patch.split("\n")) { @@ -92,8 +92,8 @@ export class RecentAnalyzer extends Analyzer { if (this.markers.line.test(line)) { const {op = "+", content = ""} = line.match(this.markers.line)?.groups ?? {} const size = Buffer.byteLength(content, "utf-8") - edition[{"+": "added", "-": "deleted"}[op]].bytes += size - edition[{"+": "added", "-": "deleted"}[op]].lines++ + edition[{"+":"added", "-":"deleted"}[op]].bytes += size + edition[{"+":"added", "-":"deleted"}[op]].lines++ continue } } @@ -104,25 +104,25 @@ export class RecentAnalyzer extends Analyzer { } /**Run linguist against a commit and compute edited lines and bytes*/ - async linguist(_, {commit, cache: {languages}}) { - const cache = {files: {}, languages} - const result = {total: 0, files: 0, missed: {lines: 0, bytes: 0}, lines: {}, stats: {}, languages: {}} + async linguist(_, {commit, cache:{languages}}) { + const cache = {files:{}, languages} + const result = {total:0, files:0, missed:{lines:0, bytes:0}, lines:{}, stats:{}, languages:{}} const edited = new Set() - + for (const edition of commit.editions) { - // Extract repo from edition path - const repoMatch = edition.path.match(/^(?[^\/]+)\/(?[^\/]+)\//) + //Extract repo from edition path + const repoMatch = edition.path.match(/^(?[^/]+)\/(?[^/]+)\//) const repo = repoMatch ? `${repoMatch.groups.owner}/${repoMatch.groups.repo}` : "" - - // Skip this file if it's in an ignored path - if (repo && this.shouldIgnorePath(repo, edition.path.replace(`${repo}/`, ''))) { + + //Skip this file if it's in an ignored path + if (repo && this.shouldIgnorePath(repo, edition.path.replace(`${repo}/`, ""))) { continue } - + edited.add(edition.path) //Guess file language with linguist - const {files: {results: files}, languages: {results: languages}, unknown} = await linguist(edition.path, {fileContent: edition.patch}) + const {files:{results:files}, languages:{results:languages}, unknown} = await linguist(edition.path, {fileContent:edition.patch}) Object.assign(cache.files, files) Object.assign(cache.languages, languages) if (!(edition.path in cache.files)) diff --git a/source/plugins/languages/analyzers.mjs b/source/plugins/languages/analyzers.mjs index 5f771851c7a..32a6455d2a2 100644 --- a/source/plugins/languages/analyzers.mjs +++ b/source/plugins/languages/analyzers.mjs @@ -1,16 +1,16 @@ //Imports -import { cli } from "./analyzer/cli.mjs" -import { IndepthAnalyzer } from "./analyzer/indepth.mjs" -import { RecentAnalyzer } from "./analyzer/recent.mjs" +import {cli} from "./analyzer/cli.mjs" +import {IndepthAnalyzer} from "./analyzer/indepth.mjs" +import {RecentAnalyzer} from "./analyzer/recent.mjs" /**Indepth analyzer */ export async function indepth({login, data, imports, rest, context, repositories}, {skipped, pathsIgnored = [], categories, timeout}) { - return new IndepthAnalyzer(login, {shell: imports, uid: data.user.databaseId, skipped, pathsIgnored, authoring: data.shared["commits.authoring"], timeout, rest, context, categories}).run({repositories}) + return new IndepthAnalyzer(login, {shell:imports, uid:data.user.databaseId, skipped, pathsIgnored, authoring:data.shared["commits.authoring"], timeout, rest, context, categories}).run({repositories}) } /**Recent languages activity */ export async function recent({login, data, imports, rest, context, account}, {skipped = [], pathsIgnored = [], categories, days = 0, load = 0, timeout}) { - return new RecentAnalyzer(login, {shell: imports, uid: data.user.databaseId, skipped, pathsIgnored, authoring: data.shared["commits.authoring"], timeout, account, rest, context, days, categories, load}).run() + return new RecentAnalyzer(login, {shell:imports, uid:data.user.databaseId, skipped, pathsIgnored, authoring:data.shared["commits.authoring"], timeout, account, rest, context, days, categories, load}).run() } //import.meta.main diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index a6da8576b4d..b046c89f330 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -1,5 +1,5 @@ //Imports -import { indepth as indepth_analyzer, recent as recent_analyzer } from "./analyzers.mjs" +import {indepth as indepth_analyzer, recent as recent_analyzer} from "./analyzers.mjs" //Setup export default async function({login, data, imports, q, rest, account}, {enabled = false, extras = false} = {}) { @@ -10,22 +10,22 @@ export default async function({login, data, imports, q, rest, account}, {enabled return null //Context - let context = {mode: "user"} + let context = {mode:"user"} if (q.repo) { console.debug(`metrics/compute/${login}/plugins > languages > switched to repository mode`) - const {owner, repo} = data.user.repositories.nodes.map(({name: repo, owner: {login: owner}}) => ({repo, owner})).shift() - context = {...context, mode: "repository", owner, repo} + const {owner, repo} = data.user.repositories.nodes.map(({name:repo, owner:{login:owner}}) => ({repo, owner})).shift() + context = {...context, mode:"repository", owner, repo} } //Load inputs - let {ignored, skipped, pathsIgnored, other, colors, aliases, details, threshold, limit, indepth, "indepth.custom": _indepth_custom, "analysis.timeout": _timeout_global, "analysis.timeout.repositories": _timeout_repositories, sections, categories, "recent.categories": _recent_categories, "recent.load": _recent_load, "recent.days": _recent_days} = imports.metadata + let {ignored, skipped, /*pathsIgnored,*/ other, colors, aliases, details, threshold, limit, indepth, "indepth.custom":_indepth_custom, "analysis.timeout":_timeout_global, "analysis.timeout.repositories":_timeout_repositories, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata .plugins.languages .inputs({ data, account, q, }) - const timeout = {global: _timeout_global, repositories: _timeout_repositories} + const timeout = {global:_timeout_global, repositories:_timeout_repositories} threshold = (Number(threshold.replace(/%$/, "")) || 0) / 100 skipped.push(...data.shared["repositories.skipped"]) if (!limit) @@ -43,18 +43,18 @@ export default async function({login, data, imports, q, rest, account}, {enabled //Unique languages const repositories = context.mode === "repository" ? data.user.repositories.nodes : [...data.user.repositories.nodes, ...data.user.repositoriesContributedTo.nodes] - const unique = new Set(repositories.flatMap(repository => repository.languages.edges.map(({node: {name}}) => name))).size + const unique = new Set(repositories.flatMap(repository => repository.languages.edges.map(({node:{name}}) => name))).size //Iterate through user's repositories and retrieve languages data console.debug(`metrics/compute/${login}/plugins > languages > processing ${data.user.repositories.nodes.length} repositories`) - const languages = {unique, sections, details, indepth, colors: {}, total: 0, stats: {}, "stats.recent": {}} + const languages = {unique, sections, details, indepth, colors:{}, total:0, stats:{}, "stats.recent":{}} const customColors = {} for (const repository of data.user.repositories.nodes) { //Skip repository if asked if (!imports.filters.repo(repository, skipped)) continue //Process repository languages - for (const {size, node: {color, name}} of Object.values(repository.languages.edges)) { + for (const {size, node:{color, name}} of Object.values(repository.languages.edges)) { languages.stats[name] = (languages.stats[name] ?? 0) + size if (colors[name.toLocaleLowerCase()]) customColors[name] = colors[name.toLocaleLowerCase()] @@ -68,7 +68,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled if ((sections.includes("recently-used")) && (imports.metadata.plugins.languages.extras("indepth", {extras}))) { try { console.debug(`metrics/compute/${login}/plugins > languages > using recent analyzer`) - languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, context, account}, {skipped, categories: _recent_categories ?? categories, days: _recent_days, load: _recent_load, timeout}) + languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, context, account}, {skipped, categories:_recent_categories ?? categories, days:_recent_days, load:_recent_load, timeout}) Object.assign(languages.colors, languages["stats.recent"].colors) } catch (error) { @@ -81,7 +81,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled try { console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`) const existingColors = languages.colors - Object.assign(languages, await indepth_analyzer({login, data, imports, rest, context, repositories: repositories.concat(_indepth_custom)}, {skipped, categories, timeout})) + Object.assign(languages, await indepth_analyzer({login, data, imports, rest, context, repositories:repositories.concat(_indepth_custom)}, {skipped, categories, timeout})) Object.assign(languages.colors, existingColors) console.debug(`metrics/compute/${login}/plugins > languages > indepth analysis processed successfully ${languages.commits} and missed ${languages.missed.commits} commits in ${languages.elapsed.toFixed(2)}m`) } @@ -105,9 +105,9 @@ export default async function({login, data, imports, q, rest, account}, {enabled } //Compute languages stats - for (const {section, stats = {}, lines = {}, missed = {bytes: 0}, total = 0} of [{section: "favorites", stats: languages.stats, lines: languages.lines, total: languages.total, missed: languages.missed}, {section: "recent", ...languages["stats.recent"]}]) { + for (const {section, stats = {}, lines = {}, missed = {bytes:0}, total = 0} of [{section:"favorites", stats:languages.stats, lines:languages.lines, total:languages.total, missed:languages.missed}, {section:"recent", ...languages["stats.recent"]}]) { console.debug(`metrics/compute/${login}/plugins > languages > formatting stats ${section}`) - languages[section] = Object.entries(stats).filter(([name]) => imports.filters.text(name, ignored)).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size: value, color: languages.colors[name], x: 0})).filter(({value}) => value / total > threshold) + languages[section] = Object.entries(stats).filter(([name]) => imports.filters.text(name, ignored)).sort(([_an, a], [_bn, b]) => b - a).slice(0, limit).map(([name, value]) => ({name, value, size:value, color:languages.colors[name], x:0})).filter(({value}) => value / total > threshold) if (other) { let value = indepth ? missed.bytes : Object.entries(stats).filter(([name]) => !Object.values(languages[section]).map(({name}) => name).includes(name)).reduce((a, [_, b]) => a + b, 0) if (value) { @@ -119,7 +119,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled languages[section].push({name:"Other", value, size:value, get lines() { return missed.lines }, set lines(_) { }, x:0}) //eslint-disable-line brace-style, no-empty-function, max-statements-per-line } } - const visible = {total: Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)} + const visible = {total:Object.values(languages[section]).map(({size}) => size).reduce((a, b) => a + b, 0)} for (let i = 0; i < languages[section].length; i++) { const {name} = languages[section][i] languages[section][i].value /= visible.total From 1f65773ddca5312e9efadb1804e5931b9dc3d72a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Wed, 7 May 2025 21:21:53 +0200 Subject: [PATCH 3/7] Update Dockerfile --- Dockerfile | 67 +++++++++++++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3ff5209a48a..bd0f9fd8138 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,29 +5,50 @@ FROM node:20-bookworm-slim COPY . /metrics WORKDIR /metrics -# Setup -RUN chmod +x /metrics/source/app/action/index.mjs \ - # Install latest chrome dev package, fonts to support major charsets and skip chromium download on puppeteer install - # Based on https://github.com/GoogleChrome/puppeteer/blob/master/docs/troubleshooting.md#running-puppeteer-in-docker - && apt-get update \ - && apt-get install -y wget gnupg ca-certificates libgconf-2-4 \ - && wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \ - && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \ - && apt-get update \ - && apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 libx11-xcb1 libxtst6 lsb-release --no-install-recommends \ - # Install deno for miscellaneous scripts - && apt-get install -y curl unzip \ - && curl -fsSL https://deno.land/x/install/install.sh | DENO_INSTALL=/usr/local sh \ - # Install ruby to support github licensed gem - && apt-get install -y ruby-full git g++ cmake pkg-config libssl-dev \ - && gem install licensed \ - # Install python for node-gyp - && apt-get install -y python3 \ - # Clean apt/lists - && rm -rf /var/lib/apt/lists/* \ - # Install node modules and rebuild indexes - && npm ci \ - && npm run build +# Make action executable +RUN chmod +x /metrics/source/app/action/index.mjs + +# Update package lists +RUN apt-get update + +# Install wget, gnupg, and certificates needed for adding Google Chrome repository +RUN apt-get install -y wget gnupg ca-certificates libgconf-2-4 + +# Add Google Chrome repository key +RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - + +# Add Google Chrome repository +RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' + +# Update package lists again after adding new repository +RUN apt-get update + +# Install Chrome and required fonts +RUN apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 libx11-xcb1 libxtst6 lsb-release libxslt-dev libxml2-dev build-essential jq --no-install-recommends + +# Install dependencies for deno +RUN apt-get install -y curl unzip + +# Install deno +RUN curl -fsSL https://deno.land/x/install/install.sh | DENO_INSTALL=/usr/local sh + +# Install ruby and dependencies for licensed gem +RUN apt-get install -y ruby-full git g++ cmake pkg-config libssl-dev + +# Install licensed gem +RUN gem install licensed + +# Install python for node-gyp +RUN apt-get install -y python3 + +# Clean apt cache to reduce image size +RUN rm -rf /var/lib/apt/lists/* + +# Install node modules +RUN npm ci + +# Build the project +RUN npm run build # Environment variables ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD true From f9728aba2a174d11b24fc79c66091605a81760af Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Thu, 8 May 2025 08:26:06 +0200 Subject: [PATCH 4/7] Update analyzer.mjs --- .../plugins/languages/analyzer/analyzer.mjs | 78 +++++++++++++++++-- 1 file changed, 73 insertions(+), 5 deletions(-) diff --git a/source/plugins/languages/analyzer/analyzer.mjs b/source/plugins/languages/analyzer/analyzer.mjs index 1aca80e2bbe..30ad596123c 100644 --- a/source/plugins/languages/analyzer/analyzer.mjs +++ b/source/plugins/languages/analyzer/analyzer.mjs @@ -109,6 +109,7 @@ export class Analyzer { /**Check if path should be ignored */ shouldIgnorePath(repo, filePath) { + this.debug(repo, filePath) for (const ignoredPath of this.pathsIgnored) { //Check for repo:path pattern (using colon as separator) if (ignoredPath.includes(":")) { @@ -139,7 +140,7 @@ export class Analyzer { if (parts.length >= 3) { const ownerRepo = `${parts[0]}/${parts[1]}` const pathToIgnore = parts.slice(2).join("/") -if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { + if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore}`) return true } @@ -155,13 +156,80 @@ if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { } //Simple path ignoring for all repos else if (filePath.startsWith(ignoredPath)) { - this.debug(`ignoring file ${filePath} as it matches ignored path ${ignoredPath}`) - return true - } + this.debug(`ignoring file ${filePath} as it matches ignored path ${ignoredPath}`) + return true + } } return false } + /**Wrapper for linguist to handle path ignoring */ + async filteredLinguist(path, options) { + const {repo} = options + + //First call the original linguist method to get results + const results = await this.linguist(path, options) + + //Filter out results for ignored paths + const filteredLines = {} + const filteredStats = {} + let filteredTotal = 0 + let filteredFiles = 0 + let ignoredFiles = 0 + + //Process each language entry + for (const [language,] of Object.entries(results.lines)) { + filteredLines[language] = 0 + } + + for (const [language,] of Object.entries(results.stats)) { + filteredStats[language] = 0 + } + + this.debug(results) + + //Process file paths and filter out ignored ones + if (results.files_details) { + for (const fileDetail of results.files_details) { + const filePath = fileDetail.path + + if (this.shouldIgnorePath(repo, filePath)) { + ignoredFiles++ + //Skip this file's stats + continue + } + + //Include this file's stats + filteredFiles++ + filteredTotal += fileDetail.total || 0 + + //Add language-specific counts and stats + if (fileDetail.language) { + filteredLines[fileDetail.language] = (filteredLines[fileDetail.language] || 0) + (fileDetail.lines || 0) + filteredStats[fileDetail.language] = (filteredStats[fileDetail.language] || 0) + (fileDetail.bytes || 0) + } + } + + //Return filtered results + if (ignoredFiles > 0) { + this.debug(`Filtered out ${ignoredFiles} files due to path ignore rules`) + } + + return { + ...results, + lines:filteredLines, + stats:filteredStats, + total:filteredTotal, + files:filteredFiles, + files_details:results.files_details.filter(f => !this.shouldIgnorePath(repo, f.path)) + } + } + + //If linguist doesn't provide file details, we can't filter effectively + this.debug("Warning: Unable to filter paths effectively as linguist didn't return file details") + return results + } + /**Analyze a repository */ async analyze(path, {commits = []} = {}) { const cache = {files:{}, languages:{}} @@ -179,7 +247,7 @@ if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { break } try { - const {total, files, missed, lines, stats} = await this.linguist(path, {commit, cache, repo}) + const {total, files, missed, lines, stats} = await this.filteredLinguist(path, {commit, cache, repo}) this.results.commits++ this.results.total += total this.results.files += files From c06bc8dc374a354268173cd77ddfb3078b173a6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Thu, 8 May 2025 09:42:07 +0200 Subject: [PATCH 5/7] Update indepth.mjs --- source/plugins/languages/analyzer/indepth.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/source/plugins/languages/analyzer/indepth.mjs b/source/plugins/languages/analyzer/indepth.mjs index 3ff5a933661..1d0461a67a7 100644 --- a/source/plugins/languages/analyzer/indepth.mjs +++ b/source/plugins/languages/analyzer/indepth.mjs @@ -194,6 +194,7 @@ export class IndepthAnalyzer extends Analyzer { /**Run linguist against a commit and compute edited lines and bytes*/ async linguist(path, {commit, cache}) { + this.debug(path, commit, cache) const result = {total:0, files:0, missed:{lines:0, bytes:0}, lines:{}, stats:{}} const edited = new Set() const seen = new Set() From 98d1d4fa591ee20e74488386c50a9fe1d2678222 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Thu, 8 May 2025 10:09:50 +0200 Subject: [PATCH 6/7] oh my god i think i am stupid --- source/app/action/index.mjs | 170 ++++++++++++++--------------- source/app/metrics/index.mjs | 148 ++++++++++++------------- source/app/metrics/utils.mjs | 136 +++++++++++------------ source/plugins/languages/index.mjs | 6 +- 4 files changed, 230 insertions(+), 230 deletions(-) diff --git a/source/app/action/index.mjs b/source/app/action/index.mjs index 3b42c638b3d..0768037afb0 100644 --- a/source/app/action/index.mjs +++ b/source/app/action/index.mjs @@ -39,7 +39,7 @@ info.section = (left = "", right = " ") => info(`\x1b[36m${left}\x1b[0m`, right) info.group = ({metadata, name, inputs}) => { info.section(metadata.plugins[name]?.name?.match(/(?
[\w\s]+)/i)?.groups?.section?.trim(), " ") for (const [input, value] of Object.entries(inputs)) - info(metadata.plugins[name]?.inputs[input]?.description?.split("\n")[0] ?? metadata.plugins[name]?.inputs[input]?.description ?? input, `${input in preset ? "*" : ""}${value}`, {token: metadata.plugins[name]?.inputs[input]?.type === "token"}) + info(metadata.plugins[name]?.inputs[input]?.description?.split("\n")[0] ?? metadata.plugins[name]?.inputs[input]?.description ?? input, `${input in preset ? "*" : ""}${value}`, {token:metadata.plugins[name]?.inputs[input]?.type === "token"}) } info.break = () => console.log("─".repeat(88)) @@ -72,7 +72,7 @@ async function retry(func, {retries = 1, delay = 0} = {}) { //Process exit function quit(reason) { - const code = {success: 0, skipped: 0, failed: 1}[reason] ?? 0 + const code = {success:0, skipped:0, failed:1}[reason] ?? 0 process.exit(code) } //===================================================================================================== //Runner @@ -96,9 +96,9 @@ function quit(reason) { } //Load configuration - const {conf, Plugins, Templates} = await setup({log: false, community: {templates: core.getInput("setup_community_templates")}, extras: true}) + const {conf, Plugins, Templates} = await setup({log:false, community:{templates:core.getInput("setup_community_templates")}, extras:true}) const {metadata} = conf - conf.settings.extras = {default: true} + conf.settings.extras = {default:true} info("Setup", "complete") info("Version", conf.package.version) @@ -111,48 +111,48 @@ function quit(reason) { } //Core inputs - Object.assign(preset, await presets(core.getInput("config_presets"), {log: false, core})) + Object.assign(preset, await presets(core.getInput("config_presets"), {log:false, core})) const { - user: _user, - repo: _repo, + user:_user, + repo:_repo, token, template, query, - "setup.community.templates": _templates, - filename: _filename, + "setup.community.templates":_templates, + filename:_filename, optimize, verify, - "markdown.cache": _markdown_cache, + "markdown.cache":_markdown_cache, debug, - "debug.flags": dflags, - "debug.print": dprint, - "use.mocked.data": mocked, + "debug.flags":dflags, + "debug.print":dprint, + "use.mocked.data":mocked, dryrun, - "plugins.errors.fatal": die, - "committer.token": _token, - "committer.branch": _branch, - "committer.message": _message, - "committer.gist": _gist, - "use.prebuilt.image": _image, + "plugins.errors.fatal":die, + "committer.token":_token, + "committer.branch":_branch, + "committer.message":_message, + "committer.gist":_gist, + "use.prebuilt.image":_image, retries, - "retries.delay": retries_delay, - "retries.output.action": retries_output_action, - "retries.delay.output.action": retries_delay_output_action, - "output.action": _action, - "output.condition": _output_condition, + "retries.delay":retries_delay, + "retries.output.action":retries_output_action, + "retries.delay.output.action":retries_delay_output_action, + "output.action":_action, + "output.condition":_output_condition, delay, - "quota.required.rest": _quota_required_rest, - "quota.required.graphql": _quota_required_graphql, - "quota.required.search": _quota_required_search, - "notice.release": _notice_releases, - "clean.workflows": _clean_workflows, - "github.api.rest": _github_api_rest, - "github.api.graphql": _github_api_graphql, + "quota.required.rest":_quota_required_rest, + "quota.required.graphql":_quota_required_graphql, + "quota.required.search":_quota_required_search, + "notice.release":_notice_releases, + "clean.workflows":_clean_workflows, + "github.api.rest":_github_api_rest, + "github.api.graphql":_github_api_graphql, ...config } = metadata.plugins.core.inputs.action({core, preset}) - const q = {...query, ...(_repo ? {repo: _repo} : null), template} + const q = {...query, ...(_repo ? {repo:_repo} : null), template} const _output = ["svg", "jpeg", "png", "json", "markdown", "markdown-pdf", "insights"].includes(config["config.output"]) ? config["config.output"] : metadata.templates[template]?.formats?.[0] ?? null - const filename = _filename.replace(/[*]/g, {jpeg: "jpg", markdown: "md", "markdown-pdf": "pdf", insights: "html"}[_output] ?? _output ?? "*") + const filename = _filename.replace(/[*]/g, {jpeg:"jpg", markdown:"md", "markdown-pdf":"pdf", insights:"html"}[_output] ?? _output ?? "*") //Docker image if (_image) @@ -168,7 +168,7 @@ function quit(reason) { q["debug.flags"] = dflags.join(" ") //Token for data gathering - info("GitHub token", token, {token: true}) + info("GitHub token", token, {token:true}) //A GitHub token should start with "gh" along an additional letter for type //See https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats info("GitHub token format", /^github_pat_/.test(token) ? "fine-grained" : /^gh[pousr]_/.test(token) ? "classic" : "legacy or invalid") @@ -179,10 +179,10 @@ function quit(reason) { conf.settings.token = token const api = {} const resources = {} - api.graphql = octokit.graphql.defaults({headers: {authorization: `token ${token}`}, baseUrl: _github_api_graphql || undefined}) + api.graphql = octokit.graphql.defaults({headers:{authorization:`token ${token}`}, baseUrl:_github_api_graphql || undefined}) info("GitHub GraphQL API", "ok") info("GitHub GraphQL API endpoint", api.graphql.baseUrl) - const octoraw = github.getOctokit(token, {baseUrl: _github_api_rest || undefined}) + const octoraw = github.getOctokit(token, {baseUrl:_github_api_rest || undefined}) api.rest = octoraw.rest api.rest.request = octoraw.request info("GitHub REST API", "ok") @@ -196,11 +196,11 @@ function quit(reason) { else if (!/^NOT_NEEDED$/.test(token)) { //Check rate limit let ratelimit = false - const {data} = await api.rest.rateLimit.get().catch(() => ({data: {resources: {}}})) + const {data} = await api.rest.rateLimit.get().catch(() => ({data:{resources:{}}})) Object.assign(resources, data.resources) for (const type of ["core", "graphql", "search"]) { - const name = {core: "REST", graphql: "GraphQL", search: "Search"}[type] - const quota = {core: _quota_required_rest, graphql: _quota_required_graphql, search: _quota_required_search}[type] ?? 1 + const name = {core:"REST", graphql:"GraphQL", search:"Search"}[type] + const quota = {core:_quota_required_rest, graphql:_quota_required_graphql, search:_quota_required_search}[type] ?? 1 info(`API requests (${name})`, resources[type] ? `${resources[type].remaining}/${resources[type].limit}${quota ? ` (${quota}+ required)` : ""}` : "(unknown)") if ((resources[type]) && (resources[type].remaining < quota)) ratelimit = true @@ -230,7 +230,7 @@ function quit(reason) { //Check for new versions if (_notice_releases) { - const {data: [{tag_name: tag}]} = await rest.repos.listReleases({owner: "lowlighter", repo: "metrics"}) + const {data:[{tag_name:tag}]} = await rest.repos.listReleases({owner:"lowlighter", repo:"metrics"}) const current = Number(conf.package.version.match(/(\d+\.\d+)/)?.[1] ?? 0) const latest = Number(tag.match(/(\d+\.\d+)/)?.[1] ?? 0) if (latest > current) @@ -267,7 +267,7 @@ function quit(reason) { committer.merge = _action.match(/^pull-request-(?merge|squash|rebase)$/)?.groups?.method ?? null committer.branch = _branch || github.context.ref.replace(/^refs[/]heads[/]/, "") committer.head = committer.pr ? `metrics-run-${github.context.runId}` : committer.branch - info("Committer token", committer.token, {token: true}) + info("Committer token", committer.token, {token:true}) if (!committer.token) throw new Error("You must provide a valid GitHub token to commit your metrics") info("Committer branch", committer.branch) @@ -286,15 +286,15 @@ function quit(reason) { } //Create head branch if needed try { - await committer.rest.git.getRef({...github.context.repo, ref: `heads/${committer.head}`}) + await committer.rest.git.getRef({...github.context.repo, ref:`heads/${committer.head}`}) info("Committer head branch status", "ok") } catch (error) { console.debug(error) if (/not found/i.test(`${error}`)) { - const {data: {object: {sha}}} = await committer.rest.git.getRef({...github.context.repo, ref: `heads/${committer.branch}`}) + const {data:{object:{sha}}} = await committer.rest.git.getRef({...github.context.repo, ref:`heads/${committer.branch}`}) info("Committer branch current sha", sha) - await committer.rest.git.createRef({...github.context.repo, ref: `refs/heads/${committer.head}`, sha}) + await committer.rest.git.createRef({...github.context.repo, ref:`refs/heads/${committer.head}`, sha}) info("Committer head branch status", "(created)") } else { @@ -304,7 +304,7 @@ function quit(reason) { //Retrieve previous render SHA to be able to update file content through API committer.sha = null try { - const {repository: {object: {oid}}} = await graphql( + const {repository:{object:{oid}}} = await graphql( ` query Sha { repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") { @@ -312,7 +312,7 @@ function quit(reason) { } } `, - {headers: {authorization: `token ${committer.token}`}}, + {headers:{authorization:`token ${committer.token}`}}, ) committer.sha = oid } @@ -343,7 +343,7 @@ function quit(reason) { //Core config info.break() - info.group({metadata, name: "core", inputs: config}) + info.group({metadata, name:"core", inputs:config}) info("Plugin errors", die ? "(exit with error)" : "(displayed in generated image)") const convert = _output || null Object.assign(q, config) @@ -354,7 +354,7 @@ function quit(reason) { await new Promise(async (solve, reject) => { let stdout = "" setTimeout(() => reject("Timeout while waiting for Insights webserver"), 5 * 60 * 1000) - const web = await processes.spawn("node", ["/metrics/source/app/web/index.mjs"], {env: {...process.env}}) + const web = await processes.spawn("node", ["/metrics/source/app/web/index.mjs"], {env:{...process.env}}) web.stdout.on("data", data => (console.debug(`web > ${data}`), stdout += data, /Server ready !/.test(stdout) ? solve() : null)) web.stderr.on("data", data => console.debug(`web > ${data}`)) }) @@ -368,9 +368,9 @@ function quit(reason) { //Base content info.break() - const {base: parts, repositories: _repositories, indepth: _base_indepth, ...base} = metadata.plugins.base.inputs.action({core, preset}) + const {base:parts, repositories:_repositories, indepth:_base_indepth, ...base} = metadata.plugins.base.inputs.action({core, preset}) conf.settings.repositories = _repositories - info.group({metadata, name: "base", inputs: {repositories: conf.settings.repositories, indepth: _base_indepth, ...base}}) + info.group({metadata, name:"base", inputs:{repositories:conf.settings.repositories, indepth:_base_indepth, ...base}}) info("Base sections", parts) base.base = false for (const part of conf.settings.plugins.base.parts) @@ -382,7 +382,7 @@ function quit(reason) { const plugins = {} for (const name of Object.keys(Plugins).filter(key => !["base", "core"].includes(key))) { //Parse inputs - const {[name]: enabled, ...inputs} = metadata.plugins[name].inputs.action({core, preset}) + const {[name]:enabled, ...inputs} = metadata.plugins[name].inputs.action({core, preset}) plugins[name] = {enabled} //Register user inputs if (enabled) { @@ -404,20 +404,20 @@ function quit(reason) { info.break() info.section("Rendering") let {rendered, mime} = await retry(async () => { - const {rendered, mime, errors} = await metrics({login: user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}) + const {rendered, mime, errors} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}) if (errors.length) { console.warn(`::group::${errors.length} error(s) occurred`) - console.warn(util.inspect(errors, {depth: Infinity, maxStringLength: 256})) + console.warn(util.inspect(errors, {depth:Infinity, maxStringLength:256})) console.warn("::endgroup::") } return {rendered, mime} - }, {retries, delay: retries_delay}) + }, {retries, delay:retries_delay}) if (!rendered) throw new Error("Could not render metrics") info("Status", "complete") info("MIME type", mime) const buffer = { - _content: null, + _content:null, get content() { return this._content }, @@ -444,13 +444,13 @@ function quit(reason) { let data = "" await retry(async () => { try { - data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref: `heads/${committer.head}`, path: filename})).data.content, "base64")}` + data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}` } catch (error) { if (error.response.status !== 404) throw error } - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) const previous = await svg.hash(data) info("Previous hash", previous) const current = await svg.hash(rendered) @@ -469,7 +469,7 @@ function quit(reason) { if (dryrun) info("Actions to perform", "(none)") else { - await fs.mkdir(paths.dirname(paths.join("/renders", filename)), {recursive: true}) + await fs.mkdir(paths.dirname(paths.join("/renders", filename)), {recursive:true}) await fs.writeFile(paths.join("/renders", filename), buffer.content) info(`Save to /metrics_renders/${filename}`, "ok") info("Output action", _action) @@ -493,7 +493,7 @@ function quit(reason) { console.debug(`Processing ${path}`) let sha = null try { - const {repository: {object: {oid}}} = await graphql( + const {repository:{object:{oid}}} = await graphql( ` query Sha { repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") { @@ -501,7 +501,7 @@ function quit(reason) { } } `, - {headers: {authorization: `token ${committer.token}`}}, + {headers:{authorization:`token ${committer.token}`}}, ) sha = oid } @@ -513,14 +513,14 @@ function quit(reason) { ...github.context.repo, path, content, - message: `${committer.message} (cache)`, + message:`${committer.message} (cache)`, ...(sha ? {sha} : {}), - branch: committer.pr ? committer.head : committer.branch, + branch:committer.pr ? committer.head : committer.branch, }) rendered = rendered.replace(match, ``) info(`Saving ${path}`, "ok") } - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) } buffer.content = rendered await fs.writeFile(paths.join("/renders", filename), buffer.content) @@ -541,10 +541,10 @@ function quit(reason) { //Upload to gist (this is done as user since committer_token may not have gist rights) if (committer.gist) { await retry(async () => { - await rest.gists.update({gist_id: committer.gist, files: {[filename]: {content: buffer.content.toString()}}}) + await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:buffer.content.toString()}}}) info(`Upload to gist ${committer.gist}`, "ok") committer.commit = false - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) } //Commit metrics @@ -552,14 +552,14 @@ function quit(reason) { await retry(async () => { await committer.rest.repos.createOrUpdateFileContents({ ...github.context.repo, - path: filename, - message: committer.message, - content: buffer.content.toString("base64"), - branch: committer.pr ? committer.head : committer.branch, - ...(committer.sha ? {sha: committer.sha} : {}), + path:filename, + message:committer.message, + content:buffer.content.toString("base64"), + branch:committer.pr ? committer.head : committer.branch, + ...(committer.sha ? {sha:committer.sha} : {}), }) info(`Commit to branch ${committer.branch}`, "ok") - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) } //Pull request @@ -568,7 +568,7 @@ function quit(reason) { let number = null await retry(async () => { try { - ;({data: {number}} = await committer.rest.pulls.create({...github.context.repo, head: committer.head, base: committer.branch, title: `Auto-generated metrics for run #${github.context.runId}`, body: " ", maintainer_can_modify: true})) + ;({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true})) info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)") } catch (error) { @@ -577,7 +577,7 @@ function quit(reason) { if (/A pull request already exists/.test(error)) { info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)") const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title` - const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user: {login}}) => login === "github-actions[bot]") + const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]") if (prs.length < 1) throw new Error("0 matching prs. Cannot proceed.") if (prs.length > 1) @@ -595,7 +595,7 @@ function quit(reason) { } } info("Pull request number", number) - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) //Merge pull request if (committer.merge) { info("Merge method", committer.merge) @@ -603,7 +603,7 @@ function quit(reason) { do { const success = await retry(async () => { //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get) - const {data: {mergeable, mergeable_state: state}} = await committer.rest.pulls.get({...github.context.repo, pull_number: number}) + const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number}) console.debug(`Pull request #${number} mergeable state is "${state}"`) if (mergeable === null) { await wait(15) @@ -612,17 +612,17 @@ function quit(reason) { if (!mergeable) throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`) //Merge pull request - await committer.rest.pulls.merge({...github.context.repo, pull_number: number, merge_method: committer.merge}) + await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge}) info(`Merge #${number} to ${committer.branch}`, "ok") return true - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) if (!success) continue //Delete head branch await retry(async () => { try { await wait(15) - await committer.rest.git.deleteRef({...github.context.repo, ref: `heads/${committer.head}`}) + await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`}) } catch (error) { console.debug(error) @@ -630,7 +630,7 @@ function quit(reason) { throw error } info(`Branch ${committer.head}`, "(deleted)") - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) break } while (--attempts) @@ -642,8 +642,8 @@ function quit(reason) { try { //Get workflow metadata const run_id = github.context.runId - const {data: {workflow_id}} = await rest.actions.getWorkflowRun({...github.context.repo, run_id}) - const {data: {path}} = await rest.actions.getWorkflow({...github.context.repo, workflow_id}) + const {data:{workflow_id}} = await rest.actions.getWorkflowRun({...github.context.repo, run_id}) + const {data:{path}} = await rest.actions.getWorkflow({...github.context.repo, workflow_id}) const workflow = paths.basename(path) info.break() info.section("Cleaning workflows") @@ -657,7 +657,7 @@ function quit(reason) { for (let page = 1; page <= pages; page++) { try { console.debug(`Fetching page ${page}/${pages} of workflow ${workflow}`) - const {data: {workflow_runs, total_count}} = await rest.actions.listWorkflowRuns({...github.context.repo, workflow_id: workflow, branch: committer.branch, status: "completed", page}) + const {data:{workflow_runs, total_count}} = await rest.actions.listWorkflowRuns({...github.context.repo, workflow_id:workflow, branch:committer.branch, status:"completed", page}) pages = total_count / 100 runs.push(...workflow_runs.filter(({conclusion}) => (_clean_workflows.includes("all")) || (_clean_workflows.includes(conclusion))).map(({id}) => ({id}))) } @@ -672,7 +672,7 @@ function quit(reason) { let cleaned = 0 for (const {id} of runs) { try { - await rest.actions.deleteWorkflowRun({...github.context.repo, run_id: id}) + await rest.actions.deleteWorkflowRun({...github.context.repo, run_id:id}) cleaned++ } catch (error) { @@ -695,10 +695,10 @@ function quit(reason) { info.break() info.section("Consumed API requests") info(" * provided that no other app used your quota during execution", "") - const {data: current} = await rest.rateLimit.get().catch(() => ({data: {resources: {}}})) + const {data:current} = await rest.rateLimit.get().catch(() => ({data:{resources:{}}})) for (const type of ["core", "graphql", "search"]) { const used = resources[type].remaining - current.resources[type].remaining - info({core: "REST API", graphql: "GraphQL API", search: "Search API"}[type], (Number.isFinite(used) && (used >= 0)) ? used : "(unknown)") + info({core:"REST API", graphql:"GraphQL API", search:"Search API"}[type], (Number.isFinite(used) && (used >= 0)) ? used : "(unknown)") } } diff --git a/source/app/metrics/index.mjs b/source/app/metrics/index.mjs index a1dd6c19f3e..83c6bd4b93c 100644 --- a/source/app/metrics/index.mjs +++ b/source/app/metrics/index.mjs @@ -10,7 +10,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Debug login = login.replace(/[\n\r]/g, "") console.debug(`metrics/compute/${login} > start`) - console.debug(util.inspect(q, {depth: Infinity, maxStringLength: 256})) + console.debug(util.inspect(q, {depth:Infinity, maxStringLength:256})) //Load template const template = q.template || conf.settings.templates.default @@ -25,11 +25,11 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, const pending = [] const {queries} = conf const imports = { - plugins: Plugins, - templates: Templates, - metadata: conf.metadata, + plugins:Plugins, + templates:Templates, + metadata:conf.metadata, ...utils, - ...utils.formatters({timeZone: q["config.timezone"]}), + ...utils.formatters({timeZone:q["config.timezone"]}), ...(/markdown/.test(convert) ? { imgb64(url, options) { @@ -38,9 +38,9 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } : null), } - const {"debug.flags": dflags, "experimental.features": _experimental, "config.order": _partials} = imports.metadata.plugins.core.inputs({account: "bypass", q}) - const extras = {css: imports.metadata.plugins.core.extras("extras_css", {...conf.settings, error: false}) ? q["extras.css"] ?? "" : "", js: imports.metadata.plugins.core.extras("extras_js", {...conf.settings, error: false}) ? q["extras.js"] ?? "" : ""} - const data = {q, animated: true, large: false, base: {}, config: {}, errors: [], warnings, plugins: {}, computed: {}, extras, postscripts: []} + const {"debug.flags":dflags, "experimental.features":_experimental, "config.order":_partials} = imports.metadata.plugins.core.inputs({account:"bypass", q}) + const extras = {css:imports.metadata.plugins.core.extras("extras_css", {...conf.settings, error:false}) ? q["extras.css"] ?? "" : "", js:imports.metadata.plugins.core.extras("extras_js", {...conf.settings, error:false}) ? q["extras.js"] ?? "" : ""} + const data = {q, animated:true, large:false, base:{}, config:{}, errors:[], warnings, plugins:{}, computed:{}, extras, postscripts:[]} const experimental = new Set(_experimental) if (conf.settings["debug.headless"]) { imports.puppeteer.headless = false @@ -77,7 +77,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Executing base plugin and compute metrics console.debug(`metrics/compute/${login} > compute`) await Plugins.base({login, q, data, rest, graphql, plugins, queries, pending, imports, callbacks}, conf) - await computer({login, q}, {conf, data, rest, graphql, plugins, queries, account: data.account, convert, template, callbacks}, {pending, imports}) + await computer({login, q}, {conf, data, rest, graphql, plugins, queries, account:data.account, convert, template, callbacks}, {pending, imports}) const promised = await Promise.all(pending) //Check plugins errors @@ -87,7 +87,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, if (die) throw new Error("An error occurred during rendering, dying") else - console.debug(util.inspect(errors, {depth: Infinity, maxStringLength: 256})) + console.debug(util.inspect(errors, {depth:Infinity, maxStringLength:256})) } //JSON output @@ -106,7 +106,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } return value })) - return {rendered, mime: "application/json", errors} + return {rendered, mime:"application/json", errors} } //Markdown output @@ -117,12 +117,12 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, try { let template = `${q.markdown}`.replace(/\n/g, "") if (!/^https:/.test(template)) { - const {data: {default_branch: branch, full_name: repo}} = await rest.repos.get({owner: login, repo: q.repo || login}) + const {data:{default_branch:branch, full_name:repo}} = await rest.repos.get({owner:login, repo:q.repo || login}) console.debug(`metrics/compute/${login} > on ${repo} with default branch ${branch}`) template = `https://raw.githubusercontent.com/${repo}/${branch}/${template}` } console.debug(`metrics/compute/${login} > fetching ${template}`) - ;({data: source} = await imports.axios.get(template, {headers: {Accept: "text/plain"}})) + ;({data:source} = await imports.axios.get(template, {headers:{Accept:"text/plain"}})) } catch (error) { console.debug(error) @@ -140,7 +140,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, console.debug(`metrics/compute/${login} > embed called with`) console.debug(q) let {base} = q - q = {..._q, ...Object.fromEntries(Object.keys(Plugins).map(key => [key, false])), ...Object.fromEntries(conf.settings.plugins.base.parts.map(part => [`base.${part}`, false])), template: q.repo ? "repository" : "classic", ...q} + q = {..._q, ...Object.fromEntries(Object.keys(Plugins).map(key => [key, false])), ...Object.fromEntries(conf.settings.plugins.base.parts.map(part => [`base.${part}`, false])), template:q.repo ? "repository" : "classic", ...q} //Translate action syntax to web syntax let parts = [] if (base === true) @@ -159,38 +159,38 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Check users errors const warnings = [] if ((!Object.keys(Plugins).filter(key => q[key]).length) && (!parts.length)) - warnings.push({warning: {message: "No plugin were selected"}}) + warnings.push({warning:{message:"No plugin were selected"}}) const ineffective = Object.keys(q).filter(key => (key.includes(".")) && (key.split(".").at(0) !== "base") && (!(key in imports.metadata.plugins.base.inputs)) && (key.split(".").at(0) in Plugins)).filter(key => !q[key.split(".").at(0)]) - warnings.push(...ineffective.map(key => ({warning: {message: `"${key}" has no effect because "${key.split(".").at(0)}: true" is not set`}}))) + warnings.push(...ineffective.map(key => ({warning:{message:`"${key}" has no effect because "${key.split(".").at(0)}: true" is not set`}}))) //Compute rendering - const {rendered} = await metrics({login, q}, {...arguments[1], convert: ["svg", "png", "jpeg"].includes(q["config.output"]) ? q["config.output"] : null, warnings}, arguments[2]) + const {rendered} = await metrics({login, q}, {...arguments[1], convert:["svg", "png", "jpeg"].includes(q["config.output"]) ? q["config.output"] : null, warnings}, arguments[2]) console.debug(`metrics/compute/${login}/embed > ${name} > success >>>>>>>>>>>>>>>>>>>>>>`) - return `` + return `` } //Rendering template source let rendered = source.replace(/\{\{ (?[\s\S]*?) \}\}/g, "{%= $ %}") console.debug(rendered) - for (const delimiters of [{openDelimiter: "<", closeDelimiter: ">"}, {openDelimiter: "{", closeDelimiter: "}"}]) - rendered = await ejs.render(rendered, {...data, s: imports.s, f: imports.format, embed}, {views, async: true, ...delimiters}) + for (const delimiters of [{openDelimiter:"<", closeDelimiter:">"}, {openDelimiter:"{", closeDelimiter:"}"}]) + rendered = await ejs.render(rendered, {...data, s:imports.s, f:imports.format, embed}, {views, async:true, ...delimiters}) console.debug(`metrics/compute/${login} > success`) //Output if (convert === "markdown-pdf") { return imports.svg.pdf(rendered, { - paddings: q["config.padding"] || conf.settings.padding, - style: extras.css, - twemojis: q["config.twemoji"], - gemojis: q["config.gemoji"], - octicons: q["config.octicon"], + paddings:q["config.padding"] || conf.settings.padding, + style:extras.css, + twemojis:q["config.twemoji"], + gemojis:q["config.gemoji"], + octicons:q["config.octicon"], rest, errors, }) } - return {rendered, mime: "text/html", errors} + return {rendered, mime:"text/html", errors} } //Rendering console.debug(`metrics/compute/${login} > render`) - let rendered = await ejs.render(image, {...data, s: imports.s, f: imports.format, style, fonts}, {views, async: true}) + let rendered = await ejs.render(image, {...data, s:imports.s, f:imports.format, style, fonts}, {views, async:true}) //Additional transformations if (q["config.twemoji"]) @@ -207,7 +207,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, if ((conf.settings?.optimize === true) || (conf.settings?.optimize?.includes?.("svg"))) rendered = await imports.svg.optimize.svg(rendered, q, experimental) //Verify svg - if ((verify) && (imports.metadata.plugins.core.extras("verify", {...conf.settings, error: false}))) { + if ((verify) && (imports.metadata.plugins.core.extras("verify", {...conf.settings, error:false}))) { console.debug(`metrics/compute/${login} > verify SVG`) let libxmljs = null try { @@ -224,7 +224,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } } //Resizing - const {resized, mime} = await imports.svg.resize(rendered, {paddings: q["config.padding"] || conf.settings.padding, convert: convert === "svg" ? null : convert, scripts: [...data.postscripts, extras.js || null].filter(x => x)}) + const {resized, mime} = await imports.svg.resize(rendered, {paddings:q["config.padding"] || conf.settings.padding, convert:convert === "svg" ? null : convert, scripts:[...data.postscripts, extras.js || null].filter(x => x)}) rendered = resized //Result @@ -243,52 +243,52 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Metrics insights metrics.insights = async function({login}, {graphql, rest, conf, callbacks}, {Plugins, Templates}) { - return metrics({login, q: metrics.insights.q}, {graphql, rest, plugins: metrics.insights.plugins, conf, callbacks, convert: "json"}, {Plugins, Templates}) + return metrics({login, q:metrics.insights.q}, {graphql, rest, plugins:metrics.insights.plugins, conf, callbacks, convert:"json"}, {Plugins, Templates}) } metrics.insights.q = { - template: "classic", - achievements: true, - "achievements.threshold": "X", - isocalendar: true, - "isocalendar.duration": "full-year", - languages: true, - "languages.limit": 0, - activity: true, - "activity.limit": 100, - "activity.days": 0, - "activity.timestamps": true, - notable: true, - "notable.repositories": true, - followup: true, - "followup.sections": "repositories, user", - introduction: true, - topics: true, - "topics.mode": "icons", - "topics.limit": 0, - stars: true, - "stars.limit": 6, - reactions: true, - "reactions.details": "percentage", - repositories: true, - "repositories.pinned": 6, - sponsors: true, - calendar: true, - "calendar.limit": 0, + template:"classic", + achievements:true, + "achievements.threshold":"X", + isocalendar:true, + "isocalendar.duration":"full-year", + languages:true, + "languages.limit":0, + activity:true, + "activity.limit":100, + "activity.days":0, + "activity.timestamps":true, + notable:true, + "notable.repositories":true, + followup:true, + "followup.sections":"repositories, user", + introduction:true, + topics:true, + "topics.mode":"icons", + "topics.limit":0, + stars:true, + "stars.limit":6, + reactions:true, + "reactions.details":"percentage", + repositories:true, + "repositories.pinned":6, + sponsors:true, + calendar:true, + "calendar.limit":0, } metrics.insights.plugins = { - achievements: {enabled: true}, - isocalendar: {enabled: true}, - languages: {enabled: true, extras: false}, - activity: {enabled: true, markdown: "extended"}, - notable: {enabled: true}, - followup: {enabled: true}, - introduction: {enabled: true}, - topics: {enabled: true}, - stars: {enabled: true}, - reactions: {enabled: true}, - repositories: {enabled: true}, - sponsors: {enabled: true}, - calendar: {enabled: true}, + achievements:{enabled:true}, + isocalendar:{enabled:true}, + languages:{enabled:true, extras:false}, + activity:{enabled:true, markdown:"extended"}, + notable:{enabled:true}, + followup:{enabled:true}, + introduction:{enabled:true}, + topics:{enabled:true}, + stars:{enabled:true}, + reactions:{enabled:true}, + repositories:{enabled:true}, + sponsors:{enabled:true}, + calendar:{enabled:true}, } //Metrics insights static render @@ -307,7 +307,7 @@ metrics.insights.output = async function({login, imports, conf}, {graphql, rest, await page.goto(`${server}/insights/${login}?embed=1&localstorage=1`) await page.evaluate(async json => localStorage.setItem("local.metrics", json), json) //eslint-disable-line no-undef await page.goto(`${server}/insights/${login}?embed=1&localstorage=1`) - await page.waitForSelector(".container .user", {timeout: 10 * 60 * 1000}) + await page.waitForSelector(".container .user", {timeout:10 * 60 * 1000}) //Rendering console.debug(`metrics/compute/${login} > insights > rendering data`) @@ -320,9 +320,9 @@ metrics.insights.output = async function({login, imports, conf}, {graphql, rest, ${await page.evaluate(() => document.querySelector("main").outerHTML)} - ${(await Promise.all([".css/style.vars.css", ".css/style.css", "insights/.statics/style.css"].map(path => utils.axios.get(`${server}/${path}`)))).map(({data: style}) => ``).join("\n")} + ${(await Promise.all([".css/style.vars.css", ".css/style.css", "insights/.statics/style.css"].map(path => utils.axios.get(`${server}/${path}`)))).map(({data:style}) => ``).join("\n")} ` await browser.close() - return {mime: "text/html", rendered, errors: result.errors} + return {mime:"text/html", rendered, errors:result.errors} } diff --git a/source/app/metrics/utils.mjs b/source/app/metrics/utils.mjs index f210b9e8f10..4cd69a8c4c5 100644 --- a/source/app/metrics/utils.mjs +++ b/source/app/metrics/utils.mjs @@ -4,16 +4,16 @@ import twemojis from "@twemoji/parser" import axios from "axios" import processes from "child_process" import crypto from "crypto" -import { minify as csso } from "csso" +import {minify as csso} from "csso" import * as d3 from "d3" import emoji from "emoji-name-map" -import { fileTypeFromBuffer } from "file-type" +import {fileTypeFromBuffer} from "file-type" import fss from "fs" import fs from "fs/promises" -import { JSDOM } from "jsdom" +import {JSDOM} from "jsdom" import linguist from "linguist-js" -import { marked } from "marked" -import { minimatch } from "minimatch" +import {marked} from "marked" +import {minimatch} from "minimatch" import opengraph from "open-graph-scraper" import os from "os" import paths from "path" @@ -33,7 +33,7 @@ import xmlformat from "xml-formatter" prism_lang() //Exports -export { axios, d3, emoji, fs, git, minimatch, opengraph, os, paths, processes, sharp, url, util } +export {axios, d3, emoji, fs, git, minimatch, opengraph, os, paths, processes, sharp, url, util} /**Returns module __dirname */ export function __module(module) { @@ -44,26 +44,26 @@ export function __module(module) { export const puppeteer = { async launch() { return _puppeteer.launch({ - headless: this.headless, - executablePath: process.env.PUPPETEER_BROWSER_PATH, - args: this.headless ? ["--no-sandbox", "--disable-extensions", "--disable-setuid-sandbox", "--disable-dev-shm-usage"] : [], - ignoreDefaultArgs: ["--disable-extensions"], + headless:this.headless, + executablePath:process.env.PUPPETEER_BROWSER_PATH, + args:this.headless ? ["--no-sandbox", "--disable-extensions", "--disable-setuid-sandbox", "--disable-dev-shm-usage"] : [], + ignoreDefaultArgs:["--disable-extensions"], }) }, - headless: "new", - events: ["load", "domcontentloaded", "networkidle2"], + headless:"new", + events:["load", "domcontentloaded", "networkidle2"], } /**Plural formatter */ export function s(value, end = "") { - return value !== 1 ? {y: "ies", "": "s"}[end] : end + return value !== 1 ? {y:"ies", "":"s"}[end] : end } /**Formatters */ export function formatters({timeZone} = {}) { //Check options try { - new Date().toLocaleString("fr", {timeZoneName: "short", timeZone}) + new Date().toLocaleString("fr", {timeZoneName:"short", timeZone}) } catch { timeZone = undefined @@ -72,7 +72,7 @@ export function formatters({timeZone} = {}) { /**Formatter */ const format = function(n, {sign = false, unit = true, fixed} = {}) { if (unit) { - for (const {u, v} of [{u: "b", v: 10 ** 9}, {u: "m", v: 10 ** 6}, {u: "k", v: 10 ** 3}]) { + for (const {u, v} of [{u:"b", v:10 ** 9}, {u:"m", v:10 ** 6}, {u:"k", v:10 ** 3}]) { if (n / v >= 1) return `${(sign) && (n > 0) ? "+" : ""}${(n / v).toFixed(fixed ?? 2).substr(0, 4).replace(/[.]0*$/, "")}${u}` } @@ -82,7 +82,7 @@ export function formatters({timeZone} = {}) { /**Bytes formatter */ format.bytes = function(n) { - for (const {u, v} of [{u: "E", v: 10 ** 18}, {u: "P", v: 10 ** 15}, {u: "T", v: 10 ** 12}, {u: "G", v: 10 ** 9}, {u: "M", v: 10 ** 6}, {u: "k", v: 10 ** 3}]) { + for (const {u, v} of [{u:"E", v:10 ** 18}, {u:"P", v:10 ** 15}, {u:"T", v:10 ** 12}, {u:"G", v:10 ** 9}, {u:"M", v:10 ** 6}, {u:"k", v:10 ** 3}]) { if (n / v >= 1) return `${(n / v).toFixed(2).substr(0, 4).replace(/[.]0*$/, "")} ${u}B` } @@ -110,11 +110,11 @@ export function formatters({timeZone} = {}) { format.date = function(string, options) { if (options.date) { delete options.date - Object.assign(options, {day: "numeric", month: "short", year: "numeric"}) + Object.assign(options, {day:"numeric", month:"short", year:"numeric"}) } if (options.time) { delete options.time - Object.assign(options, {hour: "2-digit", minute: "2-digit", second: "2-digit"}) + Object.assign(options, {hour:"2-digit", minute:"2-digit", second:"2-digit"}) } return new Intl.DateTimeFormat("en-GB", {timeZone, ...options}).format(new Date(string)) } @@ -131,7 +131,7 @@ export function formatters({timeZone} = {}) { try { //Extras features or enable state error if ((error.extras) || (error.enabled)) - throw {error: {message: error.message, instance: error}} + throw {error:{message:error.message, instance:error}} //Already formatted error if (error.error?.message) throw error @@ -160,9 +160,9 @@ export function formatters({timeZone} = {}) { //Error data console.debug(error.response.data) error = error.response?.data ?? null - throw {error: {message, instance: error}} + throw {error:{message, instance:error}} } - throw {error: {message, instance: error}} + throw {error:{message, instance:error}} } catch (error) { return Object.assign(error, attributes) @@ -182,7 +182,7 @@ export function shuffle(array) { } /**Escape html */ -export function htmlescape(string, u = {"&": true, "<": true, ">": true, '"': true, "'": true}) { +export function htmlescape(string, u = {"&":true, "<":true, ">":true, '"':true, "'":true}) { return string .replace(/&(?!(?:amp|lt|gt|quot|apos);)/g, u["&"] ? "&" : "&") .replace(/": true, '"': tr } /**Unescape html */ -export function htmlunescape(string, u = {"&": true, "<": true, ">": true, '"': true, "'": true}) { +export function htmlunescape(string, u = {"&":true, "<":true, ">":true, '"':true, "'":true}) { return string .replace(/</g, u["<"] ? "<" : "<") .replace(/>/g, u[">"] ? ">" : ">") @@ -209,7 +209,7 @@ export function stripemojis(string) { /**Language analyzer (single file) */ export async function language({filename, patch}) { console.debug(`metrics/language > ${filename}`) - const {files: {results}} = await linguist(filename, {fileContent: patch}) + const {files:{results}} = await linguist(filename, {fileContent:patch}) const result = (results[filename] ?? "unknown").toLocaleLowerCase() console.debug(`metrics/language > ${filename} > result: ${result}`) return result @@ -217,7 +217,7 @@ export async function language({filename, patch}) { /**Run command (use this to execute commands and process whole output at once, may not be suitable for large outputs) */ export async function run(command, options, {prefixed = true, log = true, debug = true} = {}) { - const prefix = {win32: "wsl"}[process.platform] ?? "" + const prefix = {win32:"wsl"}[process.platform] ?? "" command = `${prefixed ? prefix : ""} ${command}`.trim() return new Promise((solve, reject) => { if (debug) @@ -240,7 +240,7 @@ export async function run(command, options, {prefixed = true, log = true, debug /**Spawn command (use this to execute commands and process output on the fly) */ export async function spawn(command, args = [], options = {}, {prefixed = true, timeout = 300 * 1000, stdout, debug = true} = {}) { //eslint-disable-line max-params - const prefix = {win32: "wsl"}[process.platform] ?? "" + const prefix = {win32:"wsl"}[process.platform] ?? "" if ((prefixed) && (prefix)) { args.unshift(command) command = prefix @@ -250,8 +250,8 @@ export async function spawn(command, args = [], options = {}, {prefixed = true, return new Promise((solve, reject) => { if (debug) console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")}`) - const child = processes.spawn(command, args, {...options, shell: true, timeout}) - const reader = readline.createInterface({input: child.stdout}) + const child = processes.spawn(command, args, {...options, shell:true, timeout}) + const reader = readline.createInterface({input:child.stdout}) reader.on("line", stdout) const closed = new Promise(close => reader.on("close", close)) child.on("close", async code => { @@ -286,18 +286,18 @@ export function highlight(code, lang) { /**Markdown-html sanitizer-interpreter */ export async function markdown(text, {mode = "inline", codelines = Infinity} = {}) { //Sanitize user input once to prevent injections and parse into markdown - let rendered = await marked.parse(htmlunescape(htmlsanitize(text)), {highlight, silent: true, xhtml: true}) + let rendered = await marked.parse(htmlunescape(htmlsanitize(text)), {highlight, silent:true, xhtml:true}) //Markdown mode switch (mode) { case "inline": { rendered = htmlsanitize( htmlsanitize(rendered, { - allowedTags: ["h1", "h2", "h3", "h4", "h5", "h6", "br", "blockquote", "code", "span"], - allowedAttributes: {code: ["class"], span: ["class"]}, + allowedTags:["h1", "h2", "h3", "h4", "h5", "h6", "br", "blockquote", "code", "span"], + allowedAttributes:{code:["class"], span:["class"]}, }), { - allowedAttributes: {code: ["class"], span: ["class"]}, - transformTags: {h1: "b", h2: "b", h3: "b", h4: "b", h5: "b", h6: "b", blockquote: "i"}, + allowedAttributes:{code:["class"], span:["class"]}, + transformTags:{h1:"b", h2:"b", h3:"b", h4:"b", h5:"b", h6:"b", blockquote:"i"}, }, ) break @@ -390,7 +390,7 @@ export const filters = { if (patterns[0] === "@use.patterns") { if (debug) console.debug(`metrics/filters/repo > ${repo} > using advanced pattern matching`) - const options = {nocase: true} + const options = {nocase:true} for (let pattern of patterns) { if (pattern.startsWith("#")) continue @@ -463,7 +463,7 @@ export async function imgb64(image, {width, height, fallback = true} = {}) { } //Resize image if ((width) && (height)) - image = image.resize({width: width > 0 ? width : null, height: height > 0 ? height : null}) + image = image.resize({width:width > 0 ? width : null, height:height > 0 ? height : null}) return `data:image/${ext};base64,${(await image.toBuffer()).toString("base64")}` } @@ -478,7 +478,7 @@ export const svg = { } //Additional transformations if (twemojis) - rendered = await svg.twemojis(rendered, {custom: false}) + rendered = await svg.twemojis(rendered, {custom:false}) if ((gemojis) && (rest)) rendered = await svg.gemojis(rendered, {rest}) if (octicons) @@ -487,13 +487,13 @@ export const svg = { //Render through browser and print pdf console.debug("metrics/svg/pdf > loading svg") const page = await svg.resize.browser.newPage() - page.on("console", ({_text: text}) => console.debug(`metrics/svg/pdf > puppeteer > ${text}`)) - await page.setContent(`
${rendered}
`, {waitUntil: puppeteer.events}) + page.on("console", ({_text:text}) => console.debug(`metrics/svg/pdf > puppeteer > ${text}`)) + await page.setContent(`
${rendered}
`, {waitUntil:puppeteer.events}) console.debug("metrics/svg/pdf > loaded svg successfully") const margins = (Array.isArray(paddings) ? paddings : paddings.split(",")).join(" ") console.debug(`metrics/svg/pdf > margins set to ${margins}`) await page.addStyleTag({ - content: ` + content:` main { margin: ${margins}; } main svg { height: 1em; width: 1em; } ${await fs.readFile(paths.join(__module(import.meta.url), "../../../node_modules", "@primer/css/dist/markdown.css")).catch(_ => "")}${style} @@ -503,7 +503,7 @@ export const svg = { //Result await page.close() console.debug("metrics/svg/pdf > rendering complete") - return {rendered, mime: "application/pdf", errors} + return {rendered, mime:"application/pdf", errors} }, /**Render and resize svg */ async resize(rendered, {paddings, convert, scripts = []}) { @@ -513,7 +513,7 @@ export const svg = { console.debug(`metrics/svg/resize > started ${await svg.resize.browser.version()}`) } //Format padding - const padding = {width: 1, height: 1, absolute: {width: 0, height: 0}} + const padding = {width:1, height:1, absolute:{width:0, height:0}} paddings = Array.isArray(paddings) ? paddings : `${paddings}`.split(",").map(x => x.trim()) for (const [i, dimension] of [[0, "width"], [1, "height"]]) { let operands = paddings?.[i] ?? paddings[0] @@ -529,13 +529,13 @@ export const svg = { //Render through browser and resize height console.debug("metrics/svg/resize > loading svg") const page = await svg.resize.browser.newPage() - page.setViewport({width: 980, height: 980}) + page.setViewport({width:980, height:980}) page .on("console", message => console.debug(`metrics/svg/resize > puppeteer > ${message.text()}`)) .on("pageerror", error => console.debug(`metrics/svg/resize > puppeteer > ${error.message}`)) - await page.setContent(rendered, {waitUntil: puppeteer.events}) + await page.setContent(rendered, {waitUntil:puppeteer.events}) console.debug("metrics/svg/resize > loaded svg successfully") - await page.addStyleTag({content: "body { margin: 0; padding: 0; }"}) + await page.addStyleTag({content:"body { margin: 0; padding: 0; }"}) let mime = "image/svg+xml" console.debug("metrics/svg/resize > resizing svg") let height, resized, width @@ -560,7 +560,7 @@ export const svg = { console.debug(`animations are ${animated ? "enabled" : "disabled"}`) await new Promise(solve => setTimeout(solve, 2400)) //Get bounds and resize - let {y: height, width} = document.querySelector("svg #metrics-end").getBoundingClientRect() + let {y:height, width} = document.querySelector("svg #metrics-end").getBoundingClientRect() console.debug(`bounds width=${width}, height=${height}`) height = Math.max(1, Math.ceil(height * padding.height + padding.absolute.height)) width = Math.max(1, Math.ceil(width * padding.width + padding.absolute.width)) @@ -574,7 +574,7 @@ export const svg = { if (animated) document.querySelector("svg").classList.remove("no-animations") //Result - return {resized: new XMLSerializer().serializeToString(document.querySelector("svg")), height, width} + return {resized:new XMLSerializer().serializeToString(document.querySelector("svg")), height, width} }, padding, scripts, @@ -587,7 +587,7 @@ export const svg = { //Convert if required if (convert) { console.debug(`metrics/svg/resize > convert to ${convert}`) - resized = await page.screenshot({type: convert, clip: {x: 0, y: 0, width, height}, omitBackground: true}) + resized = await page.screenshot({type:convert, clip:{x:0, y:0, width, height}, omitBackground:true}) mime = `image/${convert}` } //Result @@ -607,7 +607,7 @@ export const svg = { } //Compute hash const page = await svg.resize.browser.newPage() - await page.setContent(rendered, {waitUntil: puppeteer.events}) + await page.setContent(rendered, {waitUntil:puppeteer.events}) const data = await page.evaluate(async () => { document.querySelector("footer")?.remove() return document.querySelector("svg").outerHTML @@ -623,7 +623,7 @@ export const svg = { //Load emojis console.debug("metrics/svg/twemojis > rendering twemojis") const emojis = new Map() - for (const {text: emoji, url} of twemojis.parse(rendered)) { + for (const {text:emoji, url} of twemojis.parse(rendered)) { if (!emojis.has(emoji)) emojis.set(emoji, (await axios.get(url)).data.replace(/^ rendering gemojis") const emojis = new Map() try { - for (const [emoji, url] of Object.entries((await rest.emojis.get().catch(() => ({data: {}}))).data).map(([key, value]) => [`:${key}:`, value])) { + for (const [emoji, url] of Object.entries((await rest.emojis.get().catch(() => ({data:{}}))).data).map(([key, value]) => [`:${key}:`, value])) { if ((!emojis.has(emoji)) && (new RegExp(emoji, "g").test(rendered))) emojis.set(emoji, ``) } @@ -664,9 +664,9 @@ export const svg = { for (const size of Object.keys(heights)) { const octicon = `:octicon-${name}-${size}:` if (new RegExp(`:octicon-${name}(?:-[0-9]+)?:`, "g").test(rendered)) { - icons.set(octicon, toSVG({height: size, width: size})) + icons.set(octicon, toSVG({height:size, width:size})) if (Number(size) === 16) - icons.set(`:octicon-${name}:`, toSVG({height: size, width: size})) + icons.set(`:octicon-${name}:`, toSVG({height:size, width:size})) } } } @@ -676,7 +676,7 @@ export const svg = { return rendered }, /**Optimizers */ - optimize: { + optimize:{ /**CSS optimizer */ async css(rendered) { //Extract styles @@ -687,9 +687,9 @@ export const svg = { while (regex.test(rendered)) { const style = htmlunescape(rendered.match(regex)?.groups?.style ?? "") rendered = rendered.replace(regex, cleaned) - css.push({raw: style}) + css.push({raw:style}) } - const content = [{raw: rendered, extension: "html"}] + const content = [{raw:rendered, extension:"html"}] //Purge CSS const purged = await new purgecss.PurgeCSS().purge({content, css}) @@ -703,7 +703,7 @@ export const svg = { console.debug("metrics/svg/optimize/xml > skipped as raw option is enabled") return rendered } - return xmlformat(rendered, {lineSeparator: "\n", collapseContent: true}) + return xmlformat(rendered, {lineSeparator:"\n", collapseContent:true}) }, /**SVG optimizer */ async svg(rendered, {raw = false} = {}, experimental = new Set()) { @@ -716,16 +716,16 @@ export const svg = { console.debug("metrics/svg/optimize/svg > this feature require experimental feature flag --optimize-svg") return rendered } - const {error, data: optimized} = await SVGO.optimize(rendered, { - multipass: true, - plugins: SVGO.extendDefaultPlugins([ + const {error, data:optimized} = await SVGO.optimize(rendered, { + multipass:true, + plugins:SVGO.extendDefaultPlugins([ //Additional cleanup - {name: "cleanupListOfValues"}, - {name: "removeRasterImages"}, - {name: "removeScriptElement"}, + {name:"cleanupListOfValues"}, + {name:"removeRasterImages"}, + {name:"removeScriptElement"}, //Force CSS style consistency - {name: "inlineStyles", active: false}, - {name: "removeViewBox", active: false}, + {name:"inlineStyles", active:false}, + {name:"removeViewBox", active:false}, ]), }) if (error) @@ -745,7 +745,7 @@ export async function record({page, width, height, frames, scale = 1, quality = //Register images frames const images = [] for (let i = 0; i < frames; i++) { - images.push(await page.screenshot({type: "png", clip: {width, height, x, y}, omitBackground: background})) + images.push(await page.screenshot({type:"png", clip:{width, height, x, y}, omitBackground:background})) await wait(delay / 1000) if (i % 10 === 0) console.debug(`metrics/record > processed ${i}/${frames} frames`) @@ -753,7 +753,7 @@ export async function record({page, width, height, frames, scale = 1, quality = console.debug(`metrics/record > processed ${frames}/${frames} frames`) //Post-processing console.debug("metrics/record > applying post-processing") - return Promise.all(images.map(async buffer => `data:image/png;base64,${(await (sharp(buffer).resize({width: Math.round(width * scale), height: Math.round(height * scale)}).png({quality}).toBuffer())).toString("base64")}`)) + return Promise.all(images.map(async buffer => `data:image/png;base64,${(await (sharp(buffer).resize({width:Math.round(width * scale), height:Math.round(height * scale)}).png({quality}).toBuffer())).toString("base64")}`)) } /**Create gif from puppeteer browser*/ @@ -774,7 +774,7 @@ export async function gif({page, width, height, frames, x = 0, y = 0, repeat = t encoder.setQuality(quality) //Register frames for (let i = 0; i < frames; i++) { - const buffer = new PNG(await page.screenshot({clip: {width, height, x, y}})) + const buffer = new PNG(await page.screenshot({clip:{width, height, x, y}})) encoder.addFrame(await new Promise(solve => buffer.decode(pixels => solve(pixels)))) if (frames % 10 === 0) console.debug(`metrics/puppeteergif > processed ${i}/${frames} frames`) @@ -828,7 +828,7 @@ export const Graph = { /**Basic Graph */ graph(type, data, {area = true, points = true, text = true, low = NaN, high = NaN, match = null, labels = null, width = 480, height = 315, ticks = 0} = {}) { //Generate SVG - const margin = {top: 10, left: 10, right: 10, bottom: 45} + const margin = {top:10, left:10, right:10, bottom:45} const d3n = new D3node() const svg = d3n.createSVG(width, height) diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index b046c89f330..855b08dbe8a 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -18,7 +18,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled } //Load inputs - let {ignored, skipped, /*pathsIgnored,*/ other, colors, aliases, details, threshold, limit, indepth, "indepth.custom":_indepth_custom, "analysis.timeout":_timeout_global, "analysis.timeout.repositories":_timeout_repositories, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata + let {ignored, skipped, paths_ignored, other, colors, aliases, details, threshold, limit, indepth, "indepth.custom":_indepth_custom, "analysis.timeout":_timeout_global, "analysis.timeout.repositories":_timeout_repositories, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata .plugins.languages .inputs({ data, @@ -68,7 +68,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled if ((sections.includes("recently-used")) && (imports.metadata.plugins.languages.extras("indepth", {extras}))) { try { console.debug(`metrics/compute/${login}/plugins > languages > using recent analyzer`) - languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, context, account}, {skipped, categories:_recent_categories ?? categories, days:_recent_days, load:_recent_load, timeout}) + languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, context, account}, {skipped, pathsIgnored:paths_ignored, categories:_recent_categories ?? categories, days:_recent_days, load:_recent_load, timeout}) Object.assign(languages.colors, languages["stats.recent"].colors) } catch (error) { @@ -81,7 +81,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled try { console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`) const existingColors = languages.colors - Object.assign(languages, await indepth_analyzer({login, data, imports, rest, context, repositories:repositories.concat(_indepth_custom)}, {skipped, categories, timeout})) + Object.assign(languages, await indepth_analyzer({login, data, imports, rest, context, repositories:repositories.concat(_indepth_custom)}, {skipped, pathsIgnored:paths_ignored, categories, timeout})) Object.assign(languages.colors, existingColors) console.debug(`metrics/compute/${login}/plugins > languages > indepth analysis processed successfully ${languages.commits} and missed ${languages.missed.commits} commits in ${languages.elapsed.toFixed(2)}m`) } From 2c05c3e4b8620b6613b65baf6b4a811c5b7241eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=87=BA=E5=8F=A3=E3=83=AC=E3=82=A4?= Date: Sun, 25 May 2025 06:56:45 +0200 Subject: [PATCH 7/7] Some changes Please I beg you work I do not want 6 hour docker builds on each run --- .github/workflows/publish.yml | 30 ++++++++++++++++++ action.yml | 60 +++++++++++++++++------------------ 2 files changed, 59 insertions(+), 31 deletions(-) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000000..b96051f2233 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,30 @@ +name: Build & Push Docker Image to GHCR + +on: + push: + branches: [main] + tags: ["v*"] + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # Needed for GHCR push + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + run: echo "${{ secrets.GHCR_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Build Docker image + run: | + IMAGE_ID=ghcr.io/${{ github.repository }} + IMAGE_ID_LOWER=$(echo "$IMAGE_ID" | tr '[A-Z]' '[a-z]') + docker build -t $IMAGE_ID_LOWER:latest . + + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository }} + IMAGE_ID_LOWER=$(echo "$IMAGE_ID" | tr '[A-Z]' '[a-z]') + docker push $IMAGE_ID_LOWER:latest \ No newline at end of file diff --git a/action.yml b/action.yml index 81956619a42..dee7cdc0675 100644 --- a/action.yml +++ b/action.yml @@ -3,6 +3,11 @@ inputs: + use_prebuilt_image: + description: 'Use pre-built Docker image from registry' + required: false + default: 'true' + # ==================================================================================== # 🗃️ Base content @@ -1489,10 +1494,11 @@ runs: using: composite steps: - run: | - # Check runner compatibility echo "::group::Metrics docker image setup" echo "GitHub action: $METRICS_ACTION ($METRICS_ACTION_PATH)" cd $METRICS_ACTION_PATH + + # Check dependencies for DEPENDENCY in docker jq; do if ! which $DEPENDENCY > /dev/null 2>&1; then echo "::error::\"$DEPENDENCY\" is not installed on current runner but is needed to run metrics" @@ -1500,83 +1506,75 @@ runs: fi done if [[ $MISSING_DEPENDENCIES == "1" ]]; then - echo "Runner compatibility: missing dependencies" exit 1 - else - echo "Runner compatibility: compatible" fi - # Create environment file from inputs and GitHub variables + # Create environment file touch .env for INPUT in $(echo $INPUTS | jq -r 'to_entries|map("INPUT_\(.key|ascii_upcase)=\(.value|@uri)")|.[]'); do echo $INPUT >> .env done env | grep -E '^(GITHUB|ACTIONS|CI|TZ)' >> .env - echo "Environment variables: loaded" - # Renders output folder + # Output folder METRICS_RENDERS="/metrics_renders" sudo mkdir -p $METRICS_RENDERS - echo "Renders output folder: $METRICS_RENDERS" - # Source repository (picked from action name) + # Extract source METRICS_SOURCE=$(echo $METRICS_ACTION | sed -E 's/metrics.*?$//g' | sed -E 's/_//g') echo "Source: $METRICS_SOURCE" - # Version (picked from package.json) + # Extract version METRICS_VERSION=$(grep -Po '(?<="version": ").*(?=")' package.json) echo "Version: $METRICS_VERSION" - # Image tag (extracted from version or from env) + # Tag METRICS_TAG=v$(echo $METRICS_VERSION | sed -r 's/^([0-9]+[.][0-9]+).*/\1/') echo "Image tag: $METRICS_TAG" - # Image name - # Official action + # Determine image source if [[ $METRICS_SOURCE == "lowlighter" ]]; then - # Use registry with pre-built images if [[ ! $METRICS_USE_PREBUILT_IMAGE =~ ^([Ff]alse|[Oo]ff|[Nn]o|0)$ ]]; then - # Is released version set +e METRICS_IS_RELEASED=$(expr $(expr match $METRICS_VERSION .*-beta) == 0) set -e - echo "Is released version: $METRICS_IS_RELEASED" if [[ "$METRICS_IS_RELEASED" -eq "0" ]]; then METRICS_TAG="$METRICS_TAG-beta" - echo "Image tag (updated): $METRICS_TAG" fi METRICS_IMAGE=ghcr.io/lowlighter/metrics:$METRICS_TAG - echo "Using pre-built version $METRICS_TAG, will pull docker image from GitHub registry" if ! docker image pull $METRICS_IMAGE; then - echo "Failed to fetch docker image from GitHub registry, will rebuild it locally" METRICS_IMAGE=metrics:$METRICS_VERSION fi - # Rebuild image else - echo "Using an unreleased version ($METRICS_VERSION)" METRICS_IMAGE=metrics:$METRICS_VERSION fi - # Forked action else - echo "Using a forked version" - METRICS_IMAGE=metrics:forked-$METRICS_VERSION + # Fork (e.g. DeadCodeGames) + if [[ ! $METRICS_USE_PREBUILT_IMAGE =~ ^([Ff]alse|[Oo]ff|[Nn]o|0)$ ]]; then + METRICS_IMAGE=ghcr.io/deadcodegames/metrics:$METRICS_TAG + echo "Trying to pull prebuilt image: $METRICS_IMAGE" + if ! docker image pull $METRICS_IMAGE; then + echo "Failed to pull image, will rebuild locally" + METRICS_IMAGE=metrics:forked-$METRICS_VERSION + fi + else + METRICS_IMAGE=metrics:forked-$METRICS_VERSION + fi fi - echo "Image name: $METRICS_IMAGE" - # Build image if necessary + echo "Using Docker image: $METRICS_IMAGE" + + # Build if missing set +e - docker image inspect $METRICS_IMAGE + docker image inspect $METRICS_IMAGE > /dev/null 2>&1 METRICS_IMAGE_NEEDS_BUILD="$?" set -e if [[ "$METRICS_IMAGE_NEEDS_BUILD" -gt "0" ]]; then - echo "Image $METRICS_IMAGE is not present locally, rebuilding it from Dockerfile" docker build -t $METRICS_IMAGE . - else - echo "Image $METRICS_IMAGE is present locally" fi echo "::endgroup::" - # Run docker image with current environment + # Run the container docker run --init --rm --volume $GITHUB_EVENT_PATH:$GITHUB_EVENT_PATH --volume $METRICS_RENDERS:/renders --env-file .env $METRICS_IMAGE rm .env shell: bash