diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000000..b96051f2233 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,30 @@ +name: Build & Push Docker Image to GHCR + +on: + push: + branches: [main] + tags: ["v*"] + +jobs: + build-and-push: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write # Needed for GHCR push + steps: + - uses: actions/checkout@v4 + + - name: Log in to GHCR + run: echo "${{ secrets.GHCR_TOKEN }}" | docker login ghcr.io -u ${{ github.actor }} --password-stdin + + - name: Build Docker image + run: | + IMAGE_ID=ghcr.io/${{ github.repository }} + IMAGE_ID_LOWER=$(echo "$IMAGE_ID" | tr '[A-Z]' '[a-z]') + docker build -t $IMAGE_ID_LOWER:latest . + + - name: Push image + run: | + IMAGE_ID=ghcr.io/${{ github.repository }} + IMAGE_ID_LOWER=$(echo "$IMAGE_ID" | tr '[A-Z]' '[a-z]') + docker push $IMAGE_ID_LOWER:latest \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index bbb1f2b8918..bd0f9fd8138 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,7 +24,7 @@ RUN sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable RUN apt-get update # Install Chrome and required fonts -RUN apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 libx11-xcb1 libxtst6 lsb-release --no-install-recommends +RUN apt-get install -y google-chrome-stable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 libx11-xcb1 libxtst6 lsb-release libxslt-dev libxml2-dev build-essential jq --no-install-recommends # Install dependencies for deno RUN apt-get install -y curl unzip diff --git a/action.yml b/action.yml index 21b6a00c0a0..68bb66c4042 100644 --- a/action.yml +++ b/action.yml @@ -3,6 +3,11 @@ inputs: + use_prebuilt_image: + description: 'Use pre-built Docker image from registry' + required: false + default: 'true' + # ==================================================================================== # 🗃️ Base content @@ -1512,10 +1517,11 @@ runs: using: composite steps: - run: | - # Check runner compatibility echo "::group::Metrics docker image setup" echo "GitHub action: $METRICS_ACTION ($METRICS_ACTION_PATH)" cd $METRICS_ACTION_PATH + + # Check dependencies for DEPENDENCY in docker jq; do if ! which $DEPENDENCY > /dev/null 2>&1; then echo "::error::\"$DEPENDENCY\" is not installed on current runner but is needed to run metrics" @@ -1523,83 +1529,75 @@ runs: fi done if [[ $MISSING_DEPENDENCIES == "1" ]]; then - echo "Runner compatibility: missing dependencies" exit 1 - else - echo "Runner compatibility: compatible" fi - # Create environment file from inputs and GitHub variables + # Create environment file touch .env for INPUT in $(echo $INPUTS | jq -r 'to_entries|map("INPUT_\(.key|ascii_upcase)=\(.value|@uri)")|.[]'); do echo $INPUT >> .env done env | grep -E '^(GITHUB|ACTIONS|CI|TZ)' >> .env - echo "Environment variables: loaded" - # Renders output folder + # Output folder METRICS_RENDERS="/metrics_renders" sudo mkdir -p $METRICS_RENDERS - echo "Renders output folder: $METRICS_RENDERS" - # Source repository (picked from action name) + # Extract source METRICS_SOURCE=$(echo $METRICS_ACTION | sed -E 's/metrics.*?$//g' | sed -E 's/_//g') echo "Source: $METRICS_SOURCE" - # Version (picked from package.json) + # Extract version METRICS_VERSION=$(grep -Po '(?<="version": ").*(?=")' package.json) echo "Version: $METRICS_VERSION" - # Image tag (extracted from version or from env) + # Tag METRICS_TAG=v$(echo $METRICS_VERSION | sed -r 's/^([0-9]+[.][0-9]+).*/\1/') echo "Image tag: $METRICS_TAG" - # Image name - # Official action + # Determine image source if [[ $METRICS_SOURCE == "lowlighter" ]]; then - # Use registry with pre-built images if [[ ! $METRICS_USE_PREBUILT_IMAGE =~ ^([Ff]alse|[Oo]ff|[Nn]o|0)$ ]]; then - # Is released version set +e METRICS_IS_RELEASED=$(expr $(expr match $METRICS_VERSION .*-beta) == 0) set -e - echo "Is released version: $METRICS_IS_RELEASED" if [[ "$METRICS_IS_RELEASED" -eq "0" ]]; then METRICS_TAG="$METRICS_TAG-beta" - echo "Image tag (updated): $METRICS_TAG" fi METRICS_IMAGE=ghcr.io/lowlighter/metrics:$METRICS_TAG - echo "Using pre-built version $METRICS_TAG, will pull docker image from GitHub registry" if ! docker image pull $METRICS_IMAGE; then - echo "Failed to fetch docker image from GitHub registry, will rebuild it locally" METRICS_IMAGE=metrics:$METRICS_VERSION fi - # Rebuild image else - echo "Using an unreleased version ($METRICS_VERSION)" METRICS_IMAGE=metrics:$METRICS_VERSION fi - # Forked action else - echo "Using a forked version" - METRICS_IMAGE=metrics:forked-$METRICS_VERSION + # Fork (e.g. DeadCodeGames) + if [[ ! $METRICS_USE_PREBUILT_IMAGE =~ ^([Ff]alse|[Oo]ff|[Nn]o|0)$ ]]; then + METRICS_IMAGE=ghcr.io/deadcodegames/metrics:$METRICS_TAG + echo "Trying to pull prebuilt image: $METRICS_IMAGE" + if ! docker image pull $METRICS_IMAGE; then + echo "Failed to pull image, will rebuild locally" + METRICS_IMAGE=metrics:forked-$METRICS_VERSION + fi + else + METRICS_IMAGE=metrics:forked-$METRICS_VERSION + fi fi - echo "Image name: $METRICS_IMAGE" - # Build image if necessary + echo "Using Docker image: $METRICS_IMAGE" + + # Build if missing set +e - docker image inspect $METRICS_IMAGE + docker image inspect $METRICS_IMAGE > /dev/null 2>&1 METRICS_IMAGE_NEEDS_BUILD="$?" set -e if [[ "$METRICS_IMAGE_NEEDS_BUILD" -gt "0" ]]; then - echo "Image $METRICS_IMAGE is not present locally, rebuilding it from Dockerfile" docker build -t $METRICS_IMAGE . - else - echo "Image $METRICS_IMAGE is present locally" fi echo "::endgroup::" - # Run docker image with current environment + # Run the container docker run --init --rm --volume $GITHUB_EVENT_PATH:$GITHUB_EVENT_PATH --volume $METRICS_RENDERS:/renders --env-file .env $METRICS_IMAGE rm .env shell: bash diff --git a/source/app/action/index.mjs b/source/app/action/index.mjs index 3b42c638b3d..0768037afb0 100644 --- a/source/app/action/index.mjs +++ b/source/app/action/index.mjs @@ -39,7 +39,7 @@ info.section = (left = "", right = " ") => info(`\x1b[36m${left}\x1b[0m`, right) info.group = ({metadata, name, inputs}) => { info.section(metadata.plugins[name]?.name?.match(/(?
[\w\s]+)/i)?.groups?.section?.trim(), " ") for (const [input, value] of Object.entries(inputs)) - info(metadata.plugins[name]?.inputs[input]?.description?.split("\n")[0] ?? metadata.plugins[name]?.inputs[input]?.description ?? input, `${input in preset ? "*" : ""}${value}`, {token: metadata.plugins[name]?.inputs[input]?.type === "token"}) + info(metadata.plugins[name]?.inputs[input]?.description?.split("\n")[0] ?? metadata.plugins[name]?.inputs[input]?.description ?? input, `${input in preset ? "*" : ""}${value}`, {token:metadata.plugins[name]?.inputs[input]?.type === "token"}) } info.break = () => console.log("─".repeat(88)) @@ -72,7 +72,7 @@ async function retry(func, {retries = 1, delay = 0} = {}) { //Process exit function quit(reason) { - const code = {success: 0, skipped: 0, failed: 1}[reason] ?? 0 + const code = {success:0, skipped:0, failed:1}[reason] ?? 0 process.exit(code) } //===================================================================================================== //Runner @@ -96,9 +96,9 @@ function quit(reason) { } //Load configuration - const {conf, Plugins, Templates} = await setup({log: false, community: {templates: core.getInput("setup_community_templates")}, extras: true}) + const {conf, Plugins, Templates} = await setup({log:false, community:{templates:core.getInput("setup_community_templates")}, extras:true}) const {metadata} = conf - conf.settings.extras = {default: true} + conf.settings.extras = {default:true} info("Setup", "complete") info("Version", conf.package.version) @@ -111,48 +111,48 @@ function quit(reason) { } //Core inputs - Object.assign(preset, await presets(core.getInput("config_presets"), {log: false, core})) + Object.assign(preset, await presets(core.getInput("config_presets"), {log:false, core})) const { - user: _user, - repo: _repo, + user:_user, + repo:_repo, token, template, query, - "setup.community.templates": _templates, - filename: _filename, + "setup.community.templates":_templates, + filename:_filename, optimize, verify, - "markdown.cache": _markdown_cache, + "markdown.cache":_markdown_cache, debug, - "debug.flags": dflags, - "debug.print": dprint, - "use.mocked.data": mocked, + "debug.flags":dflags, + "debug.print":dprint, + "use.mocked.data":mocked, dryrun, - "plugins.errors.fatal": die, - "committer.token": _token, - "committer.branch": _branch, - "committer.message": _message, - "committer.gist": _gist, - "use.prebuilt.image": _image, + "plugins.errors.fatal":die, + "committer.token":_token, + "committer.branch":_branch, + "committer.message":_message, + "committer.gist":_gist, + "use.prebuilt.image":_image, retries, - "retries.delay": retries_delay, - "retries.output.action": retries_output_action, - "retries.delay.output.action": retries_delay_output_action, - "output.action": _action, - "output.condition": _output_condition, + "retries.delay":retries_delay, + "retries.output.action":retries_output_action, + "retries.delay.output.action":retries_delay_output_action, + "output.action":_action, + "output.condition":_output_condition, delay, - "quota.required.rest": _quota_required_rest, - "quota.required.graphql": _quota_required_graphql, - "quota.required.search": _quota_required_search, - "notice.release": _notice_releases, - "clean.workflows": _clean_workflows, - "github.api.rest": _github_api_rest, - "github.api.graphql": _github_api_graphql, + "quota.required.rest":_quota_required_rest, + "quota.required.graphql":_quota_required_graphql, + "quota.required.search":_quota_required_search, + "notice.release":_notice_releases, + "clean.workflows":_clean_workflows, + "github.api.rest":_github_api_rest, + "github.api.graphql":_github_api_graphql, ...config } = metadata.plugins.core.inputs.action({core, preset}) - const q = {...query, ...(_repo ? {repo: _repo} : null), template} + const q = {...query, ...(_repo ? {repo:_repo} : null), template} const _output = ["svg", "jpeg", "png", "json", "markdown", "markdown-pdf", "insights"].includes(config["config.output"]) ? config["config.output"] : metadata.templates[template]?.formats?.[0] ?? null - const filename = _filename.replace(/[*]/g, {jpeg: "jpg", markdown: "md", "markdown-pdf": "pdf", insights: "html"}[_output] ?? _output ?? "*") + const filename = _filename.replace(/[*]/g, {jpeg:"jpg", markdown:"md", "markdown-pdf":"pdf", insights:"html"}[_output] ?? _output ?? "*") //Docker image if (_image) @@ -168,7 +168,7 @@ function quit(reason) { q["debug.flags"] = dflags.join(" ") //Token for data gathering - info("GitHub token", token, {token: true}) + info("GitHub token", token, {token:true}) //A GitHub token should start with "gh" along an additional letter for type //See https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats info("GitHub token format", /^github_pat_/.test(token) ? "fine-grained" : /^gh[pousr]_/.test(token) ? "classic" : "legacy or invalid") @@ -179,10 +179,10 @@ function quit(reason) { conf.settings.token = token const api = {} const resources = {} - api.graphql = octokit.graphql.defaults({headers: {authorization: `token ${token}`}, baseUrl: _github_api_graphql || undefined}) + api.graphql = octokit.graphql.defaults({headers:{authorization:`token ${token}`}, baseUrl:_github_api_graphql || undefined}) info("GitHub GraphQL API", "ok") info("GitHub GraphQL API endpoint", api.graphql.baseUrl) - const octoraw = github.getOctokit(token, {baseUrl: _github_api_rest || undefined}) + const octoraw = github.getOctokit(token, {baseUrl:_github_api_rest || undefined}) api.rest = octoraw.rest api.rest.request = octoraw.request info("GitHub REST API", "ok") @@ -196,11 +196,11 @@ function quit(reason) { else if (!/^NOT_NEEDED$/.test(token)) { //Check rate limit let ratelimit = false - const {data} = await api.rest.rateLimit.get().catch(() => ({data: {resources: {}}})) + const {data} = await api.rest.rateLimit.get().catch(() => ({data:{resources:{}}})) Object.assign(resources, data.resources) for (const type of ["core", "graphql", "search"]) { - const name = {core: "REST", graphql: "GraphQL", search: "Search"}[type] - const quota = {core: _quota_required_rest, graphql: _quota_required_graphql, search: _quota_required_search}[type] ?? 1 + const name = {core:"REST", graphql:"GraphQL", search:"Search"}[type] + const quota = {core:_quota_required_rest, graphql:_quota_required_graphql, search:_quota_required_search}[type] ?? 1 info(`API requests (${name})`, resources[type] ? `${resources[type].remaining}/${resources[type].limit}${quota ? ` (${quota}+ required)` : ""}` : "(unknown)") if ((resources[type]) && (resources[type].remaining < quota)) ratelimit = true @@ -230,7 +230,7 @@ function quit(reason) { //Check for new versions if (_notice_releases) { - const {data: [{tag_name: tag}]} = await rest.repos.listReleases({owner: "lowlighter", repo: "metrics"}) + const {data:[{tag_name:tag}]} = await rest.repos.listReleases({owner:"lowlighter", repo:"metrics"}) const current = Number(conf.package.version.match(/(\d+\.\d+)/)?.[1] ?? 0) const latest = Number(tag.match(/(\d+\.\d+)/)?.[1] ?? 0) if (latest > current) @@ -267,7 +267,7 @@ function quit(reason) { committer.merge = _action.match(/^pull-request-(?merge|squash|rebase)$/)?.groups?.method ?? null committer.branch = _branch || github.context.ref.replace(/^refs[/]heads[/]/, "") committer.head = committer.pr ? `metrics-run-${github.context.runId}` : committer.branch - info("Committer token", committer.token, {token: true}) + info("Committer token", committer.token, {token:true}) if (!committer.token) throw new Error("You must provide a valid GitHub token to commit your metrics") info("Committer branch", committer.branch) @@ -286,15 +286,15 @@ function quit(reason) { } //Create head branch if needed try { - await committer.rest.git.getRef({...github.context.repo, ref: `heads/${committer.head}`}) + await committer.rest.git.getRef({...github.context.repo, ref:`heads/${committer.head}`}) info("Committer head branch status", "ok") } catch (error) { console.debug(error) if (/not found/i.test(`${error}`)) { - const {data: {object: {sha}}} = await committer.rest.git.getRef({...github.context.repo, ref: `heads/${committer.branch}`}) + const {data:{object:{sha}}} = await committer.rest.git.getRef({...github.context.repo, ref:`heads/${committer.branch}`}) info("Committer branch current sha", sha) - await committer.rest.git.createRef({...github.context.repo, ref: `refs/heads/${committer.head}`, sha}) + await committer.rest.git.createRef({...github.context.repo, ref:`refs/heads/${committer.head}`, sha}) info("Committer head branch status", "(created)") } else { @@ -304,7 +304,7 @@ function quit(reason) { //Retrieve previous render SHA to be able to update file content through API committer.sha = null try { - const {repository: {object: {oid}}} = await graphql( + const {repository:{object:{oid}}} = await graphql( ` query Sha { repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") { @@ -312,7 +312,7 @@ function quit(reason) { } } `, - {headers: {authorization: `token ${committer.token}`}}, + {headers:{authorization:`token ${committer.token}`}}, ) committer.sha = oid } @@ -343,7 +343,7 @@ function quit(reason) { //Core config info.break() - info.group({metadata, name: "core", inputs: config}) + info.group({metadata, name:"core", inputs:config}) info("Plugin errors", die ? "(exit with error)" : "(displayed in generated image)") const convert = _output || null Object.assign(q, config) @@ -354,7 +354,7 @@ function quit(reason) { await new Promise(async (solve, reject) => { let stdout = "" setTimeout(() => reject("Timeout while waiting for Insights webserver"), 5 * 60 * 1000) - const web = await processes.spawn("node", ["/metrics/source/app/web/index.mjs"], {env: {...process.env}}) + const web = await processes.spawn("node", ["/metrics/source/app/web/index.mjs"], {env:{...process.env}}) web.stdout.on("data", data => (console.debug(`web > ${data}`), stdout += data, /Server ready !/.test(stdout) ? solve() : null)) web.stderr.on("data", data => console.debug(`web > ${data}`)) }) @@ -368,9 +368,9 @@ function quit(reason) { //Base content info.break() - const {base: parts, repositories: _repositories, indepth: _base_indepth, ...base} = metadata.plugins.base.inputs.action({core, preset}) + const {base:parts, repositories:_repositories, indepth:_base_indepth, ...base} = metadata.plugins.base.inputs.action({core, preset}) conf.settings.repositories = _repositories - info.group({metadata, name: "base", inputs: {repositories: conf.settings.repositories, indepth: _base_indepth, ...base}}) + info.group({metadata, name:"base", inputs:{repositories:conf.settings.repositories, indepth:_base_indepth, ...base}}) info("Base sections", parts) base.base = false for (const part of conf.settings.plugins.base.parts) @@ -382,7 +382,7 @@ function quit(reason) { const plugins = {} for (const name of Object.keys(Plugins).filter(key => !["base", "core"].includes(key))) { //Parse inputs - const {[name]: enabled, ...inputs} = metadata.plugins[name].inputs.action({core, preset}) + const {[name]:enabled, ...inputs} = metadata.plugins[name].inputs.action({core, preset}) plugins[name] = {enabled} //Register user inputs if (enabled) { @@ -404,20 +404,20 @@ function quit(reason) { info.break() info.section("Rendering") let {rendered, mime} = await retry(async () => { - const {rendered, mime, errors} = await metrics({login: user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}) + const {rendered, mime, errors} = await metrics({login:user, q}, {graphql, rest, plugins, conf, die, verify, convert}, {Plugins, Templates}) if (errors.length) { console.warn(`::group::${errors.length} error(s) occurred`) - console.warn(util.inspect(errors, {depth: Infinity, maxStringLength: 256})) + console.warn(util.inspect(errors, {depth:Infinity, maxStringLength:256})) console.warn("::endgroup::") } return {rendered, mime} - }, {retries, delay: retries_delay}) + }, {retries, delay:retries_delay}) if (!rendered) throw new Error("Could not render metrics") info("Status", "complete") info("MIME type", mime) const buffer = { - _content: null, + _content:null, get content() { return this._content }, @@ -444,13 +444,13 @@ function quit(reason) { let data = "" await retry(async () => { try { - data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref: `heads/${committer.head}`, path: filename})).data.content, "base64")}` + data = `${Buffer.from((await committer.rest.repos.getContent({...github.context.repo, ref:`heads/${committer.head}`, path:filename})).data.content, "base64")}` } catch (error) { if (error.response.status !== 404) throw error } - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) const previous = await svg.hash(data) info("Previous hash", previous) const current = await svg.hash(rendered) @@ -469,7 +469,7 @@ function quit(reason) { if (dryrun) info("Actions to perform", "(none)") else { - await fs.mkdir(paths.dirname(paths.join("/renders", filename)), {recursive: true}) + await fs.mkdir(paths.dirname(paths.join("/renders", filename)), {recursive:true}) await fs.writeFile(paths.join("/renders", filename), buffer.content) info(`Save to /metrics_renders/${filename}`, "ok") info("Output action", _action) @@ -493,7 +493,7 @@ function quit(reason) { console.debug(`Processing ${path}`) let sha = null try { - const {repository: {object: {oid}}} = await graphql( + const {repository:{object:{oid}}} = await graphql( ` query Sha { repository(owner: "${github.context.repo.owner}", name: "${github.context.repo.repo}") { @@ -501,7 +501,7 @@ function quit(reason) { } } `, - {headers: {authorization: `token ${committer.token}`}}, + {headers:{authorization:`token ${committer.token}`}}, ) sha = oid } @@ -513,14 +513,14 @@ function quit(reason) { ...github.context.repo, path, content, - message: `${committer.message} (cache)`, + message:`${committer.message} (cache)`, ...(sha ? {sha} : {}), - branch: committer.pr ? committer.head : committer.branch, + branch:committer.pr ? committer.head : committer.branch, }) rendered = rendered.replace(match, ``) info(`Saving ${path}`, "ok") } - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) } buffer.content = rendered await fs.writeFile(paths.join("/renders", filename), buffer.content) @@ -541,10 +541,10 @@ function quit(reason) { //Upload to gist (this is done as user since committer_token may not have gist rights) if (committer.gist) { await retry(async () => { - await rest.gists.update({gist_id: committer.gist, files: {[filename]: {content: buffer.content.toString()}}}) + await rest.gists.update({gist_id:committer.gist, files:{[filename]:{content:buffer.content.toString()}}}) info(`Upload to gist ${committer.gist}`, "ok") committer.commit = false - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) } //Commit metrics @@ -552,14 +552,14 @@ function quit(reason) { await retry(async () => { await committer.rest.repos.createOrUpdateFileContents({ ...github.context.repo, - path: filename, - message: committer.message, - content: buffer.content.toString("base64"), - branch: committer.pr ? committer.head : committer.branch, - ...(committer.sha ? {sha: committer.sha} : {}), + path:filename, + message:committer.message, + content:buffer.content.toString("base64"), + branch:committer.pr ? committer.head : committer.branch, + ...(committer.sha ? {sha:committer.sha} : {}), }) info(`Commit to branch ${committer.branch}`, "ok") - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) } //Pull request @@ -568,7 +568,7 @@ function quit(reason) { let number = null await retry(async () => { try { - ;({data: {number}} = await committer.rest.pulls.create({...github.context.repo, head: committer.head, base: committer.branch, title: `Auto-generated metrics for run #${github.context.runId}`, body: " ", maintainer_can_modify: true})) + ;({data:{number}} = await committer.rest.pulls.create({...github.context.repo, head:committer.head, base:committer.branch, title:`Auto-generated metrics for run #${github.context.runId}`, body:" ", maintainer_can_modify:true})) info(`Pull request from ${committer.head} to ${committer.branch}`, "(created)") } catch (error) { @@ -577,7 +577,7 @@ function quit(reason) { if (/A pull request already exists/.test(error)) { info(`Pull request from ${committer.head} to ${committer.branch}`, "(already existing)") const q = `repo:${github.context.repo.owner}/${github.context.repo.repo}+type:pr+state:open+Auto-generated metrics for run #${github.context.runId}+in:title` - const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user: {login}}) => login === "github-actions[bot]") + const prs = (await committer.rest.search.issuesAndPullRequests({q})).data.items.filter(({user:{login}}) => login === "github-actions[bot]") if (prs.length < 1) throw new Error("0 matching prs. Cannot proceed.") if (prs.length > 1) @@ -595,7 +595,7 @@ function quit(reason) { } } info("Pull request number", number) - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) //Merge pull request if (committer.merge) { info("Merge method", committer.merge) @@ -603,7 +603,7 @@ function quit(reason) { do { const success = await retry(async () => { //Check pull request mergeability (https://octokit.github.io/rest.js/v18#pulls-get) - const {data: {mergeable, mergeable_state: state}} = await committer.rest.pulls.get({...github.context.repo, pull_number: number}) + const {data:{mergeable, mergeable_state:state}} = await committer.rest.pulls.get({...github.context.repo, pull_number:number}) console.debug(`Pull request #${number} mergeable state is "${state}"`) if (mergeable === null) { await wait(15) @@ -612,17 +612,17 @@ function quit(reason) { if (!mergeable) throw new Error(`Pull request #${number} is not mergeable (state is "${state}")`) //Merge pull request - await committer.rest.pulls.merge({...github.context.repo, pull_number: number, merge_method: committer.merge}) + await committer.rest.pulls.merge({...github.context.repo, pull_number:number, merge_method:committer.merge}) info(`Merge #${number} to ${committer.branch}`, "ok") return true - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) if (!success) continue //Delete head branch await retry(async () => { try { await wait(15) - await committer.rest.git.deleteRef({...github.context.repo, ref: `heads/${committer.head}`}) + await committer.rest.git.deleteRef({...github.context.repo, ref:`heads/${committer.head}`}) } catch (error) { console.debug(error) @@ -630,7 +630,7 @@ function quit(reason) { throw error } info(`Branch ${committer.head}`, "(deleted)") - }, {retries: retries_output_action, delay: retries_delay_output_action}) + }, {retries:retries_output_action, delay:retries_delay_output_action}) break } while (--attempts) @@ -642,8 +642,8 @@ function quit(reason) { try { //Get workflow metadata const run_id = github.context.runId - const {data: {workflow_id}} = await rest.actions.getWorkflowRun({...github.context.repo, run_id}) - const {data: {path}} = await rest.actions.getWorkflow({...github.context.repo, workflow_id}) + const {data:{workflow_id}} = await rest.actions.getWorkflowRun({...github.context.repo, run_id}) + const {data:{path}} = await rest.actions.getWorkflow({...github.context.repo, workflow_id}) const workflow = paths.basename(path) info.break() info.section("Cleaning workflows") @@ -657,7 +657,7 @@ function quit(reason) { for (let page = 1; page <= pages; page++) { try { console.debug(`Fetching page ${page}/${pages} of workflow ${workflow}`) - const {data: {workflow_runs, total_count}} = await rest.actions.listWorkflowRuns({...github.context.repo, workflow_id: workflow, branch: committer.branch, status: "completed", page}) + const {data:{workflow_runs, total_count}} = await rest.actions.listWorkflowRuns({...github.context.repo, workflow_id:workflow, branch:committer.branch, status:"completed", page}) pages = total_count / 100 runs.push(...workflow_runs.filter(({conclusion}) => (_clean_workflows.includes("all")) || (_clean_workflows.includes(conclusion))).map(({id}) => ({id}))) } @@ -672,7 +672,7 @@ function quit(reason) { let cleaned = 0 for (const {id} of runs) { try { - await rest.actions.deleteWorkflowRun({...github.context.repo, run_id: id}) + await rest.actions.deleteWorkflowRun({...github.context.repo, run_id:id}) cleaned++ } catch (error) { @@ -695,10 +695,10 @@ function quit(reason) { info.break() info.section("Consumed API requests") info(" * provided that no other app used your quota during execution", "") - const {data: current} = await rest.rateLimit.get().catch(() => ({data: {resources: {}}})) + const {data:current} = await rest.rateLimit.get().catch(() => ({data:{resources:{}}})) for (const type of ["core", "graphql", "search"]) { const used = resources[type].remaining - current.resources[type].remaining - info({core: "REST API", graphql: "GraphQL API", search: "Search API"}[type], (Number.isFinite(used) && (used >= 0)) ? used : "(unknown)") + info({core:"REST API", graphql:"GraphQL API", search:"Search API"}[type], (Number.isFinite(used) && (used >= 0)) ? used : "(unknown)") } } diff --git a/source/app/metrics/index.mjs b/source/app/metrics/index.mjs index a1dd6c19f3e..83c6bd4b93c 100644 --- a/source/app/metrics/index.mjs +++ b/source/app/metrics/index.mjs @@ -10,7 +10,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Debug login = login.replace(/[\n\r]/g, "") console.debug(`metrics/compute/${login} > start`) - console.debug(util.inspect(q, {depth: Infinity, maxStringLength: 256})) + console.debug(util.inspect(q, {depth:Infinity, maxStringLength:256})) //Load template const template = q.template || conf.settings.templates.default @@ -25,11 +25,11 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, const pending = [] const {queries} = conf const imports = { - plugins: Plugins, - templates: Templates, - metadata: conf.metadata, + plugins:Plugins, + templates:Templates, + metadata:conf.metadata, ...utils, - ...utils.formatters({timeZone: q["config.timezone"]}), + ...utils.formatters({timeZone:q["config.timezone"]}), ...(/markdown/.test(convert) ? { imgb64(url, options) { @@ -38,9 +38,9 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } : null), } - const {"debug.flags": dflags, "experimental.features": _experimental, "config.order": _partials} = imports.metadata.plugins.core.inputs({account: "bypass", q}) - const extras = {css: imports.metadata.plugins.core.extras("extras_css", {...conf.settings, error: false}) ? q["extras.css"] ?? "" : "", js: imports.metadata.plugins.core.extras("extras_js", {...conf.settings, error: false}) ? q["extras.js"] ?? "" : ""} - const data = {q, animated: true, large: false, base: {}, config: {}, errors: [], warnings, plugins: {}, computed: {}, extras, postscripts: []} + const {"debug.flags":dflags, "experimental.features":_experimental, "config.order":_partials} = imports.metadata.plugins.core.inputs({account:"bypass", q}) + const extras = {css:imports.metadata.plugins.core.extras("extras_css", {...conf.settings, error:false}) ? q["extras.css"] ?? "" : "", js:imports.metadata.plugins.core.extras("extras_js", {...conf.settings, error:false}) ? q["extras.js"] ?? "" : ""} + const data = {q, animated:true, large:false, base:{}, config:{}, errors:[], warnings, plugins:{}, computed:{}, extras, postscripts:[]} const experimental = new Set(_experimental) if (conf.settings["debug.headless"]) { imports.puppeteer.headless = false @@ -77,7 +77,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Executing base plugin and compute metrics console.debug(`metrics/compute/${login} > compute`) await Plugins.base({login, q, data, rest, graphql, plugins, queries, pending, imports, callbacks}, conf) - await computer({login, q}, {conf, data, rest, graphql, plugins, queries, account: data.account, convert, template, callbacks}, {pending, imports}) + await computer({login, q}, {conf, data, rest, graphql, plugins, queries, account:data.account, convert, template, callbacks}, {pending, imports}) const promised = await Promise.all(pending) //Check plugins errors @@ -87,7 +87,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, if (die) throw new Error("An error occurred during rendering, dying") else - console.debug(util.inspect(errors, {depth: Infinity, maxStringLength: 256})) + console.debug(util.inspect(errors, {depth:Infinity, maxStringLength:256})) } //JSON output @@ -106,7 +106,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } return value })) - return {rendered, mime: "application/json", errors} + return {rendered, mime:"application/json", errors} } //Markdown output @@ -117,12 +117,12 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, try { let template = `${q.markdown}`.replace(/\n/g, "") if (!/^https:/.test(template)) { - const {data: {default_branch: branch, full_name: repo}} = await rest.repos.get({owner: login, repo: q.repo || login}) + const {data:{default_branch:branch, full_name:repo}} = await rest.repos.get({owner:login, repo:q.repo || login}) console.debug(`metrics/compute/${login} > on ${repo} with default branch ${branch}`) template = `https://raw.githubusercontent.com/${repo}/${branch}/${template}` } console.debug(`metrics/compute/${login} > fetching ${template}`) - ;({data: source} = await imports.axios.get(template, {headers: {Accept: "text/plain"}})) + ;({data:source} = await imports.axios.get(template, {headers:{Accept:"text/plain"}})) } catch (error) { console.debug(error) @@ -140,7 +140,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, console.debug(`metrics/compute/${login} > embed called with`) console.debug(q) let {base} = q - q = {..._q, ...Object.fromEntries(Object.keys(Plugins).map(key => [key, false])), ...Object.fromEntries(conf.settings.plugins.base.parts.map(part => [`base.${part}`, false])), template: q.repo ? "repository" : "classic", ...q} + q = {..._q, ...Object.fromEntries(Object.keys(Plugins).map(key => [key, false])), ...Object.fromEntries(conf.settings.plugins.base.parts.map(part => [`base.${part}`, false])), template:q.repo ? "repository" : "classic", ...q} //Translate action syntax to web syntax let parts = [] if (base === true) @@ -159,38 +159,38 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Check users errors const warnings = [] if ((!Object.keys(Plugins).filter(key => q[key]).length) && (!parts.length)) - warnings.push({warning: {message: "No plugin were selected"}}) + warnings.push({warning:{message:"No plugin were selected"}}) const ineffective = Object.keys(q).filter(key => (key.includes(".")) && (key.split(".").at(0) !== "base") && (!(key in imports.metadata.plugins.base.inputs)) && (key.split(".").at(0) in Plugins)).filter(key => !q[key.split(".").at(0)]) - warnings.push(...ineffective.map(key => ({warning: {message: `"${key}" has no effect because "${key.split(".").at(0)}: true" is not set`}}))) + warnings.push(...ineffective.map(key => ({warning:{message:`"${key}" has no effect because "${key.split(".").at(0)}: true" is not set`}}))) //Compute rendering - const {rendered} = await metrics({login, q}, {...arguments[1], convert: ["svg", "png", "jpeg"].includes(q["config.output"]) ? q["config.output"] : null, warnings}, arguments[2]) + const {rendered} = await metrics({login, q}, {...arguments[1], convert:["svg", "png", "jpeg"].includes(q["config.output"]) ? q["config.output"] : null, warnings}, arguments[2]) console.debug(`metrics/compute/${login}/embed > ${name} > success >>>>>>>>>>>>>>>>>>>>>>`) - return `` + return `` } //Rendering template source let rendered = source.replace(/\{\{ (?[\s\S]*?) \}\}/g, "{%= $ %}") console.debug(rendered) - for (const delimiters of [{openDelimiter: "<", closeDelimiter: ">"}, {openDelimiter: "{", closeDelimiter: "}"}]) - rendered = await ejs.render(rendered, {...data, s: imports.s, f: imports.format, embed}, {views, async: true, ...delimiters}) + for (const delimiters of [{openDelimiter:"<", closeDelimiter:">"}, {openDelimiter:"{", closeDelimiter:"}"}]) + rendered = await ejs.render(rendered, {...data, s:imports.s, f:imports.format, embed}, {views, async:true, ...delimiters}) console.debug(`metrics/compute/${login} > success`) //Output if (convert === "markdown-pdf") { return imports.svg.pdf(rendered, { - paddings: q["config.padding"] || conf.settings.padding, - style: extras.css, - twemojis: q["config.twemoji"], - gemojis: q["config.gemoji"], - octicons: q["config.octicon"], + paddings:q["config.padding"] || conf.settings.padding, + style:extras.css, + twemojis:q["config.twemoji"], + gemojis:q["config.gemoji"], + octicons:q["config.octicon"], rest, errors, }) } - return {rendered, mime: "text/html", errors} + return {rendered, mime:"text/html", errors} } //Rendering console.debug(`metrics/compute/${login} > render`) - let rendered = await ejs.render(image, {...data, s: imports.s, f: imports.format, style, fonts}, {views, async: true}) + let rendered = await ejs.render(image, {...data, s:imports.s, f:imports.format, style, fonts}, {views, async:true}) //Additional transformations if (q["config.twemoji"]) @@ -207,7 +207,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, if ((conf.settings?.optimize === true) || (conf.settings?.optimize?.includes?.("svg"))) rendered = await imports.svg.optimize.svg(rendered, q, experimental) //Verify svg - if ((verify) && (imports.metadata.plugins.core.extras("verify", {...conf.settings, error: false}))) { + if ((verify) && (imports.metadata.plugins.core.extras("verify", {...conf.settings, error:false}))) { console.debug(`metrics/compute/${login} > verify SVG`) let libxmljs = null try { @@ -224,7 +224,7 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, } } //Resizing - const {resized, mime} = await imports.svg.resize(rendered, {paddings: q["config.padding"] || conf.settings.padding, convert: convert === "svg" ? null : convert, scripts: [...data.postscripts, extras.js || null].filter(x => x)}) + const {resized, mime} = await imports.svg.resize(rendered, {paddings:q["config.padding"] || conf.settings.padding, convert:convert === "svg" ? null : convert, scripts:[...data.postscripts, extras.js || null].filter(x => x)}) rendered = resized //Result @@ -243,52 +243,52 @@ export default async function metrics({login, q}, {graphql, rest, plugins, conf, //Metrics insights metrics.insights = async function({login}, {graphql, rest, conf, callbacks}, {Plugins, Templates}) { - return metrics({login, q: metrics.insights.q}, {graphql, rest, plugins: metrics.insights.plugins, conf, callbacks, convert: "json"}, {Plugins, Templates}) + return metrics({login, q:metrics.insights.q}, {graphql, rest, plugins:metrics.insights.plugins, conf, callbacks, convert:"json"}, {Plugins, Templates}) } metrics.insights.q = { - template: "classic", - achievements: true, - "achievements.threshold": "X", - isocalendar: true, - "isocalendar.duration": "full-year", - languages: true, - "languages.limit": 0, - activity: true, - "activity.limit": 100, - "activity.days": 0, - "activity.timestamps": true, - notable: true, - "notable.repositories": true, - followup: true, - "followup.sections": "repositories, user", - introduction: true, - topics: true, - "topics.mode": "icons", - "topics.limit": 0, - stars: true, - "stars.limit": 6, - reactions: true, - "reactions.details": "percentage", - repositories: true, - "repositories.pinned": 6, - sponsors: true, - calendar: true, - "calendar.limit": 0, + template:"classic", + achievements:true, + "achievements.threshold":"X", + isocalendar:true, + "isocalendar.duration":"full-year", + languages:true, + "languages.limit":0, + activity:true, + "activity.limit":100, + "activity.days":0, + "activity.timestamps":true, + notable:true, + "notable.repositories":true, + followup:true, + "followup.sections":"repositories, user", + introduction:true, + topics:true, + "topics.mode":"icons", + "topics.limit":0, + stars:true, + "stars.limit":6, + reactions:true, + "reactions.details":"percentage", + repositories:true, + "repositories.pinned":6, + sponsors:true, + calendar:true, + "calendar.limit":0, } metrics.insights.plugins = { - achievements: {enabled: true}, - isocalendar: {enabled: true}, - languages: {enabled: true, extras: false}, - activity: {enabled: true, markdown: "extended"}, - notable: {enabled: true}, - followup: {enabled: true}, - introduction: {enabled: true}, - topics: {enabled: true}, - stars: {enabled: true}, - reactions: {enabled: true}, - repositories: {enabled: true}, - sponsors: {enabled: true}, - calendar: {enabled: true}, + achievements:{enabled:true}, + isocalendar:{enabled:true}, + languages:{enabled:true, extras:false}, + activity:{enabled:true, markdown:"extended"}, + notable:{enabled:true}, + followup:{enabled:true}, + introduction:{enabled:true}, + topics:{enabled:true}, + stars:{enabled:true}, + reactions:{enabled:true}, + repositories:{enabled:true}, + sponsors:{enabled:true}, + calendar:{enabled:true}, } //Metrics insights static render @@ -307,7 +307,7 @@ metrics.insights.output = async function({login, imports, conf}, {graphql, rest, await page.goto(`${server}/insights/${login}?embed=1&localstorage=1`) await page.evaluate(async json => localStorage.setItem("local.metrics", json), json) //eslint-disable-line no-undef await page.goto(`${server}/insights/${login}?embed=1&localstorage=1`) - await page.waitForSelector(".container .user", {timeout: 10 * 60 * 1000}) + await page.waitForSelector(".container .user", {timeout:10 * 60 * 1000}) //Rendering console.debug(`metrics/compute/${login} > insights > rendering data`) @@ -320,9 +320,9 @@ metrics.insights.output = async function({login, imports, conf}, {graphql, rest, ${await page.evaluate(() => document.querySelector("main").outerHTML)} - ${(await Promise.all([".css/style.vars.css", ".css/style.css", "insights/.statics/style.css"].map(path => utils.axios.get(`${server}/${path}`)))).map(({data: style}) => ``).join("\n")} + ${(await Promise.all([".css/style.vars.css", ".css/style.css", "insights/.statics/style.css"].map(path => utils.axios.get(`${server}/${path}`)))).map(({data:style}) => ``).join("\n")} ` await browser.close() - return {mime: "text/html", rendered, errors: result.errors} + return {mime:"text/html", rendered, errors:result.errors} } diff --git a/source/app/metrics/utils.mjs b/source/app/metrics/utils.mjs index f210b9e8f10..4cd69a8c4c5 100644 --- a/source/app/metrics/utils.mjs +++ b/source/app/metrics/utils.mjs @@ -4,16 +4,16 @@ import twemojis from "@twemoji/parser" import axios from "axios" import processes from "child_process" import crypto from "crypto" -import { minify as csso } from "csso" +import {minify as csso} from "csso" import * as d3 from "d3" import emoji from "emoji-name-map" -import { fileTypeFromBuffer } from "file-type" +import {fileTypeFromBuffer} from "file-type" import fss from "fs" import fs from "fs/promises" -import { JSDOM } from "jsdom" +import {JSDOM} from "jsdom" import linguist from "linguist-js" -import { marked } from "marked" -import { minimatch } from "minimatch" +import {marked} from "marked" +import {minimatch} from "minimatch" import opengraph from "open-graph-scraper" import os from "os" import paths from "path" @@ -33,7 +33,7 @@ import xmlformat from "xml-formatter" prism_lang() //Exports -export { axios, d3, emoji, fs, git, minimatch, opengraph, os, paths, processes, sharp, url, util } +export {axios, d3, emoji, fs, git, minimatch, opengraph, os, paths, processes, sharp, url, util} /**Returns module __dirname */ export function __module(module) { @@ -44,26 +44,26 @@ export function __module(module) { export const puppeteer = { async launch() { return _puppeteer.launch({ - headless: this.headless, - executablePath: process.env.PUPPETEER_BROWSER_PATH, - args: this.headless ? ["--no-sandbox", "--disable-extensions", "--disable-setuid-sandbox", "--disable-dev-shm-usage"] : [], - ignoreDefaultArgs: ["--disable-extensions"], + headless:this.headless, + executablePath:process.env.PUPPETEER_BROWSER_PATH, + args:this.headless ? ["--no-sandbox", "--disable-extensions", "--disable-setuid-sandbox", "--disable-dev-shm-usage"] : [], + ignoreDefaultArgs:["--disable-extensions"], }) }, - headless: "new", - events: ["load", "domcontentloaded", "networkidle2"], + headless:"new", + events:["load", "domcontentloaded", "networkidle2"], } /**Plural formatter */ export function s(value, end = "") { - return value !== 1 ? {y: "ies", "": "s"}[end] : end + return value !== 1 ? {y:"ies", "":"s"}[end] : end } /**Formatters */ export function formatters({timeZone} = {}) { //Check options try { - new Date().toLocaleString("fr", {timeZoneName: "short", timeZone}) + new Date().toLocaleString("fr", {timeZoneName:"short", timeZone}) } catch { timeZone = undefined @@ -72,7 +72,7 @@ export function formatters({timeZone} = {}) { /**Formatter */ const format = function(n, {sign = false, unit = true, fixed} = {}) { if (unit) { - for (const {u, v} of [{u: "b", v: 10 ** 9}, {u: "m", v: 10 ** 6}, {u: "k", v: 10 ** 3}]) { + for (const {u, v} of [{u:"b", v:10 ** 9}, {u:"m", v:10 ** 6}, {u:"k", v:10 ** 3}]) { if (n / v >= 1) return `${(sign) && (n > 0) ? "+" : ""}${(n / v).toFixed(fixed ?? 2).substr(0, 4).replace(/[.]0*$/, "")}${u}` } @@ -82,7 +82,7 @@ export function formatters({timeZone} = {}) { /**Bytes formatter */ format.bytes = function(n) { - for (const {u, v} of [{u: "E", v: 10 ** 18}, {u: "P", v: 10 ** 15}, {u: "T", v: 10 ** 12}, {u: "G", v: 10 ** 9}, {u: "M", v: 10 ** 6}, {u: "k", v: 10 ** 3}]) { + for (const {u, v} of [{u:"E", v:10 ** 18}, {u:"P", v:10 ** 15}, {u:"T", v:10 ** 12}, {u:"G", v:10 ** 9}, {u:"M", v:10 ** 6}, {u:"k", v:10 ** 3}]) { if (n / v >= 1) return `${(n / v).toFixed(2).substr(0, 4).replace(/[.]0*$/, "")} ${u}B` } @@ -110,11 +110,11 @@ export function formatters({timeZone} = {}) { format.date = function(string, options) { if (options.date) { delete options.date - Object.assign(options, {day: "numeric", month: "short", year: "numeric"}) + Object.assign(options, {day:"numeric", month:"short", year:"numeric"}) } if (options.time) { delete options.time - Object.assign(options, {hour: "2-digit", minute: "2-digit", second: "2-digit"}) + Object.assign(options, {hour:"2-digit", minute:"2-digit", second:"2-digit"}) } return new Intl.DateTimeFormat("en-GB", {timeZone, ...options}).format(new Date(string)) } @@ -131,7 +131,7 @@ export function formatters({timeZone} = {}) { try { //Extras features or enable state error if ((error.extras) || (error.enabled)) - throw {error: {message: error.message, instance: error}} + throw {error:{message:error.message, instance:error}} //Already formatted error if (error.error?.message) throw error @@ -160,9 +160,9 @@ export function formatters({timeZone} = {}) { //Error data console.debug(error.response.data) error = error.response?.data ?? null - throw {error: {message, instance: error}} + throw {error:{message, instance:error}} } - throw {error: {message, instance: error}} + throw {error:{message, instance:error}} } catch (error) { return Object.assign(error, attributes) @@ -182,7 +182,7 @@ export function shuffle(array) { } /**Escape html */ -export function htmlescape(string, u = {"&": true, "<": true, ">": true, '"': true, "'": true}) { +export function htmlescape(string, u = {"&":true, "<":true, ">":true, '"':true, "'":true}) { return string .replace(/&(?!(?:amp|lt|gt|quot|apos);)/g, u["&"] ? "&" : "&") .replace(/": true, '"': tr } /**Unescape html */ -export function htmlunescape(string, u = {"&": true, "<": true, ">": true, '"': true, "'": true}) { +export function htmlunescape(string, u = {"&":true, "<":true, ">":true, '"':true, "'":true}) { return string .replace(/</g, u["<"] ? "<" : "<") .replace(/>/g, u[">"] ? ">" : ">") @@ -209,7 +209,7 @@ export function stripemojis(string) { /**Language analyzer (single file) */ export async function language({filename, patch}) { console.debug(`metrics/language > ${filename}`) - const {files: {results}} = await linguist(filename, {fileContent: patch}) + const {files:{results}} = await linguist(filename, {fileContent:patch}) const result = (results[filename] ?? "unknown").toLocaleLowerCase() console.debug(`metrics/language > ${filename} > result: ${result}`) return result @@ -217,7 +217,7 @@ export async function language({filename, patch}) { /**Run command (use this to execute commands and process whole output at once, may not be suitable for large outputs) */ export async function run(command, options, {prefixed = true, log = true, debug = true} = {}) { - const prefix = {win32: "wsl"}[process.platform] ?? "" + const prefix = {win32:"wsl"}[process.platform] ?? "" command = `${prefixed ? prefix : ""} ${command}`.trim() return new Promise((solve, reject) => { if (debug) @@ -240,7 +240,7 @@ export async function run(command, options, {prefixed = true, log = true, debug /**Spawn command (use this to execute commands and process output on the fly) */ export async function spawn(command, args = [], options = {}, {prefixed = true, timeout = 300 * 1000, stdout, debug = true} = {}) { //eslint-disable-line max-params - const prefix = {win32: "wsl"}[process.platform] ?? "" + const prefix = {win32:"wsl"}[process.platform] ?? "" if ((prefixed) && (prefix)) { args.unshift(command) command = prefix @@ -250,8 +250,8 @@ export async function spawn(command, args = [], options = {}, {prefixed = true, return new Promise((solve, reject) => { if (debug) console.debug(`metrics/command/spawn > ${command} with ${args.join(" ")}`) - const child = processes.spawn(command, args, {...options, shell: true, timeout}) - const reader = readline.createInterface({input: child.stdout}) + const child = processes.spawn(command, args, {...options, shell:true, timeout}) + const reader = readline.createInterface({input:child.stdout}) reader.on("line", stdout) const closed = new Promise(close => reader.on("close", close)) child.on("close", async code => { @@ -286,18 +286,18 @@ export function highlight(code, lang) { /**Markdown-html sanitizer-interpreter */ export async function markdown(text, {mode = "inline", codelines = Infinity} = {}) { //Sanitize user input once to prevent injections and parse into markdown - let rendered = await marked.parse(htmlunescape(htmlsanitize(text)), {highlight, silent: true, xhtml: true}) + let rendered = await marked.parse(htmlunescape(htmlsanitize(text)), {highlight, silent:true, xhtml:true}) //Markdown mode switch (mode) { case "inline": { rendered = htmlsanitize( htmlsanitize(rendered, { - allowedTags: ["h1", "h2", "h3", "h4", "h5", "h6", "br", "blockquote", "code", "span"], - allowedAttributes: {code: ["class"], span: ["class"]}, + allowedTags:["h1", "h2", "h3", "h4", "h5", "h6", "br", "blockquote", "code", "span"], + allowedAttributes:{code:["class"], span:["class"]}, }), { - allowedAttributes: {code: ["class"], span: ["class"]}, - transformTags: {h1: "b", h2: "b", h3: "b", h4: "b", h5: "b", h6: "b", blockquote: "i"}, + allowedAttributes:{code:["class"], span:["class"]}, + transformTags:{h1:"b", h2:"b", h3:"b", h4:"b", h5:"b", h6:"b", blockquote:"i"}, }, ) break @@ -390,7 +390,7 @@ export const filters = { if (patterns[0] === "@use.patterns") { if (debug) console.debug(`metrics/filters/repo > ${repo} > using advanced pattern matching`) - const options = {nocase: true} + const options = {nocase:true} for (let pattern of patterns) { if (pattern.startsWith("#")) continue @@ -463,7 +463,7 @@ export async function imgb64(image, {width, height, fallback = true} = {}) { } //Resize image if ((width) && (height)) - image = image.resize({width: width > 0 ? width : null, height: height > 0 ? height : null}) + image = image.resize({width:width > 0 ? width : null, height:height > 0 ? height : null}) return `data:image/${ext};base64,${(await image.toBuffer()).toString("base64")}` } @@ -478,7 +478,7 @@ export const svg = { } //Additional transformations if (twemojis) - rendered = await svg.twemojis(rendered, {custom: false}) + rendered = await svg.twemojis(rendered, {custom:false}) if ((gemojis) && (rest)) rendered = await svg.gemojis(rendered, {rest}) if (octicons) @@ -487,13 +487,13 @@ export const svg = { //Render through browser and print pdf console.debug("metrics/svg/pdf > loading svg") const page = await svg.resize.browser.newPage() - page.on("console", ({_text: text}) => console.debug(`metrics/svg/pdf > puppeteer > ${text}`)) - await page.setContent(`
${rendered}
`, {waitUntil: puppeteer.events}) + page.on("console", ({_text:text}) => console.debug(`metrics/svg/pdf > puppeteer > ${text}`)) + await page.setContent(`
${rendered}
`, {waitUntil:puppeteer.events}) console.debug("metrics/svg/pdf > loaded svg successfully") const margins = (Array.isArray(paddings) ? paddings : paddings.split(",")).join(" ") console.debug(`metrics/svg/pdf > margins set to ${margins}`) await page.addStyleTag({ - content: ` + content:` main { margin: ${margins}; } main svg { height: 1em; width: 1em; } ${await fs.readFile(paths.join(__module(import.meta.url), "../../../node_modules", "@primer/css/dist/markdown.css")).catch(_ => "")}${style} @@ -503,7 +503,7 @@ export const svg = { //Result await page.close() console.debug("metrics/svg/pdf > rendering complete") - return {rendered, mime: "application/pdf", errors} + return {rendered, mime:"application/pdf", errors} }, /**Render and resize svg */ async resize(rendered, {paddings, convert, scripts = []}) { @@ -513,7 +513,7 @@ export const svg = { console.debug(`metrics/svg/resize > started ${await svg.resize.browser.version()}`) } //Format padding - const padding = {width: 1, height: 1, absolute: {width: 0, height: 0}} + const padding = {width:1, height:1, absolute:{width:0, height:0}} paddings = Array.isArray(paddings) ? paddings : `${paddings}`.split(",").map(x => x.trim()) for (const [i, dimension] of [[0, "width"], [1, "height"]]) { let operands = paddings?.[i] ?? paddings[0] @@ -529,13 +529,13 @@ export const svg = { //Render through browser and resize height console.debug("metrics/svg/resize > loading svg") const page = await svg.resize.browser.newPage() - page.setViewport({width: 980, height: 980}) + page.setViewport({width:980, height:980}) page .on("console", message => console.debug(`metrics/svg/resize > puppeteer > ${message.text()}`)) .on("pageerror", error => console.debug(`metrics/svg/resize > puppeteer > ${error.message}`)) - await page.setContent(rendered, {waitUntil: puppeteer.events}) + await page.setContent(rendered, {waitUntil:puppeteer.events}) console.debug("metrics/svg/resize > loaded svg successfully") - await page.addStyleTag({content: "body { margin: 0; padding: 0; }"}) + await page.addStyleTag({content:"body { margin: 0; padding: 0; }"}) let mime = "image/svg+xml" console.debug("metrics/svg/resize > resizing svg") let height, resized, width @@ -560,7 +560,7 @@ export const svg = { console.debug(`animations are ${animated ? "enabled" : "disabled"}`) await new Promise(solve => setTimeout(solve, 2400)) //Get bounds and resize - let {y: height, width} = document.querySelector("svg #metrics-end").getBoundingClientRect() + let {y:height, width} = document.querySelector("svg #metrics-end").getBoundingClientRect() console.debug(`bounds width=${width}, height=${height}`) height = Math.max(1, Math.ceil(height * padding.height + padding.absolute.height)) width = Math.max(1, Math.ceil(width * padding.width + padding.absolute.width)) @@ -574,7 +574,7 @@ export const svg = { if (animated) document.querySelector("svg").classList.remove("no-animations") //Result - return {resized: new XMLSerializer().serializeToString(document.querySelector("svg")), height, width} + return {resized:new XMLSerializer().serializeToString(document.querySelector("svg")), height, width} }, padding, scripts, @@ -587,7 +587,7 @@ export const svg = { //Convert if required if (convert) { console.debug(`metrics/svg/resize > convert to ${convert}`) - resized = await page.screenshot({type: convert, clip: {x: 0, y: 0, width, height}, omitBackground: true}) + resized = await page.screenshot({type:convert, clip:{x:0, y:0, width, height}, omitBackground:true}) mime = `image/${convert}` } //Result @@ -607,7 +607,7 @@ export const svg = { } //Compute hash const page = await svg.resize.browser.newPage() - await page.setContent(rendered, {waitUntil: puppeteer.events}) + await page.setContent(rendered, {waitUntil:puppeteer.events}) const data = await page.evaluate(async () => { document.querySelector("footer")?.remove() return document.querySelector("svg").outerHTML @@ -623,7 +623,7 @@ export const svg = { //Load emojis console.debug("metrics/svg/twemojis > rendering twemojis") const emojis = new Map() - for (const {text: emoji, url} of twemojis.parse(rendered)) { + for (const {text:emoji, url} of twemojis.parse(rendered)) { if (!emojis.has(emoji)) emojis.set(emoji, (await axios.get(url)).data.replace(/^ rendering gemojis") const emojis = new Map() try { - for (const [emoji, url] of Object.entries((await rest.emojis.get().catch(() => ({data: {}}))).data).map(([key, value]) => [`:${key}:`, value])) { + for (const [emoji, url] of Object.entries((await rest.emojis.get().catch(() => ({data:{}}))).data).map(([key, value]) => [`:${key}:`, value])) { if ((!emojis.has(emoji)) && (new RegExp(emoji, "g").test(rendered))) emojis.set(emoji, ``) } @@ -664,9 +664,9 @@ export const svg = { for (const size of Object.keys(heights)) { const octicon = `:octicon-${name}-${size}:` if (new RegExp(`:octicon-${name}(?:-[0-9]+)?:`, "g").test(rendered)) { - icons.set(octicon, toSVG({height: size, width: size})) + icons.set(octicon, toSVG({height:size, width:size})) if (Number(size) === 16) - icons.set(`:octicon-${name}:`, toSVG({height: size, width: size})) + icons.set(`:octicon-${name}:`, toSVG({height:size, width:size})) } } } @@ -676,7 +676,7 @@ export const svg = { return rendered }, /**Optimizers */ - optimize: { + optimize:{ /**CSS optimizer */ async css(rendered) { //Extract styles @@ -687,9 +687,9 @@ export const svg = { while (regex.test(rendered)) { const style = htmlunescape(rendered.match(regex)?.groups?.style ?? "") rendered = rendered.replace(regex, cleaned) - css.push({raw: style}) + css.push({raw:style}) } - const content = [{raw: rendered, extension: "html"}] + const content = [{raw:rendered, extension:"html"}] //Purge CSS const purged = await new purgecss.PurgeCSS().purge({content, css}) @@ -703,7 +703,7 @@ export const svg = { console.debug("metrics/svg/optimize/xml > skipped as raw option is enabled") return rendered } - return xmlformat(rendered, {lineSeparator: "\n", collapseContent: true}) + return xmlformat(rendered, {lineSeparator:"\n", collapseContent:true}) }, /**SVG optimizer */ async svg(rendered, {raw = false} = {}, experimental = new Set()) { @@ -716,16 +716,16 @@ export const svg = { console.debug("metrics/svg/optimize/svg > this feature require experimental feature flag --optimize-svg") return rendered } - const {error, data: optimized} = await SVGO.optimize(rendered, { - multipass: true, - plugins: SVGO.extendDefaultPlugins([ + const {error, data:optimized} = await SVGO.optimize(rendered, { + multipass:true, + plugins:SVGO.extendDefaultPlugins([ //Additional cleanup - {name: "cleanupListOfValues"}, - {name: "removeRasterImages"}, - {name: "removeScriptElement"}, + {name:"cleanupListOfValues"}, + {name:"removeRasterImages"}, + {name:"removeScriptElement"}, //Force CSS style consistency - {name: "inlineStyles", active: false}, - {name: "removeViewBox", active: false}, + {name:"inlineStyles", active:false}, + {name:"removeViewBox", active:false}, ]), }) if (error) @@ -745,7 +745,7 @@ export async function record({page, width, height, frames, scale = 1, quality = //Register images frames const images = [] for (let i = 0; i < frames; i++) { - images.push(await page.screenshot({type: "png", clip: {width, height, x, y}, omitBackground: background})) + images.push(await page.screenshot({type:"png", clip:{width, height, x, y}, omitBackground:background})) await wait(delay / 1000) if (i % 10 === 0) console.debug(`metrics/record > processed ${i}/${frames} frames`) @@ -753,7 +753,7 @@ export async function record({page, width, height, frames, scale = 1, quality = console.debug(`metrics/record > processed ${frames}/${frames} frames`) //Post-processing console.debug("metrics/record > applying post-processing") - return Promise.all(images.map(async buffer => `data:image/png;base64,${(await (sharp(buffer).resize({width: Math.round(width * scale), height: Math.round(height * scale)}).png({quality}).toBuffer())).toString("base64")}`)) + return Promise.all(images.map(async buffer => `data:image/png;base64,${(await (sharp(buffer).resize({width:Math.round(width * scale), height:Math.round(height * scale)}).png({quality}).toBuffer())).toString("base64")}`)) } /**Create gif from puppeteer browser*/ @@ -774,7 +774,7 @@ export async function gif({page, width, height, frames, x = 0, y = 0, repeat = t encoder.setQuality(quality) //Register frames for (let i = 0; i < frames; i++) { - const buffer = new PNG(await page.screenshot({clip: {width, height, x, y}})) + const buffer = new PNG(await page.screenshot({clip:{width, height, x, y}})) encoder.addFrame(await new Promise(solve => buffer.decode(pixels => solve(pixels)))) if (frames % 10 === 0) console.debug(`metrics/puppeteergif > processed ${i}/${frames} frames`) @@ -828,7 +828,7 @@ export const Graph = { /**Basic Graph */ graph(type, data, {area = true, points = true, text = true, low = NaN, high = NaN, match = null, labels = null, width = 480, height = 315, ticks = 0} = {}) { //Generate SVG - const margin = {top: 10, left: 10, right: 10, bottom: 45} + const margin = {top:10, left:10, right:10, bottom:45} const d3n = new D3node() const svg = d3n.createSVG(width, height) diff --git a/source/plugins/languages/analyzer/analyzer.mjs b/source/plugins/languages/analyzer/analyzer.mjs index af2d2c4ba90..30ad596123c 100644 --- a/source/plugins/languages/analyzer/analyzer.mjs +++ b/source/plugins/languages/analyzer/analyzer.mjs @@ -4,7 +4,6 @@ import os from "os" import paths from "path" import git from "simple-git" import {filters} from "../../../app/metrics/utils.mjs" -import core from "@actions/core" /**Analyzer */ export class Analyzer { @@ -88,19 +87,13 @@ export class Analyzer { /**Clone a repository */ async clone(repository) { const {repo, branch, path} = this.parse(repository) - let token - - if (process.env.GITHUB_ACTIONS) { - token = core.getInput("token") - } - - let url = /^https?:\/\//.test(repo) ? repo : `https://${token}@github.com/${repo}` + let url = /^https?:\/\//.test(repo) ? repo : `https://github.com/${repo}` try { this.debug(`cloning https://github.com/${repo} to ${path}`) await fs.rm(path, {recursive:true, force:true}) await fs.mkdir(path, {recursive:true}) await git(path).clone(url, ".", ["--single-branch"]).status() - this.debug(`cloned https://github.com/${repo} to ${path}`) + this.debug(`cloned ${url} to ${path}`) if (branch) { this.debug(`switching to branch ${branch} for ${repo}`) await git(path).branch(branch) @@ -108,7 +101,7 @@ export class Analyzer { return true } catch (error) { - this.debug(`failed to clone https://github.com/${repo} (${error})`) + this.debug(`failed to clone ${url} (${error})`) this.clean(path) return false } @@ -116,6 +109,7 @@ export class Analyzer { /**Check if path should be ignored */ shouldIgnorePath(repo, filePath) { + this.debug(repo, filePath) for (const ignoredPath of this.pathsIgnored) { //Check for repo:path pattern (using colon as separator) if (ignoredPath.includes(":")) { @@ -146,7 +140,7 @@ export class Analyzer { if (parts.length >= 3) { const ownerRepo = `${parts[0]}/${parts[1]}` const pathToIgnore = parts.slice(2).join("/") -if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { + if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { this.debug(`ignoring file ${filePath} in ${repo} as it matches ignored path ${pathToIgnore}`) return true } @@ -162,13 +156,80 @@ if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { } //Simple path ignoring for all repos else if (filePath.startsWith(ignoredPath)) { - this.debug(`ignoring file ${filePath} as it matches ignored path ${ignoredPath}`) - return true - } + this.debug(`ignoring file ${filePath} as it matches ignored path ${ignoredPath}`) + return true + } } return false } + /**Wrapper for linguist to handle path ignoring */ + async filteredLinguist(path, options) { + const {repo} = options + + //First call the original linguist method to get results + const results = await this.linguist(path, options) + + //Filter out results for ignored paths + const filteredLines = {} + const filteredStats = {} + let filteredTotal = 0 + let filteredFiles = 0 + let ignoredFiles = 0 + + //Process each language entry + for (const [language,] of Object.entries(results.lines)) { + filteredLines[language] = 0 + } + + for (const [language,] of Object.entries(results.stats)) { + filteredStats[language] = 0 + } + + this.debug(results) + + //Process file paths and filter out ignored ones + if (results.files_details) { + for (const fileDetail of results.files_details) { + const filePath = fileDetail.path + + if (this.shouldIgnorePath(repo, filePath)) { + ignoredFiles++ + //Skip this file's stats + continue + } + + //Include this file's stats + filteredFiles++ + filteredTotal += fileDetail.total || 0 + + //Add language-specific counts and stats + if (fileDetail.language) { + filteredLines[fileDetail.language] = (filteredLines[fileDetail.language] || 0) + (fileDetail.lines || 0) + filteredStats[fileDetail.language] = (filteredStats[fileDetail.language] || 0) + (fileDetail.bytes || 0) + } + } + + //Return filtered results + if (ignoredFiles > 0) { + this.debug(`Filtered out ${ignoredFiles} files due to path ignore rules`) + } + + return { + ...results, + lines:filteredLines, + stats:filteredStats, + total:filteredTotal, + files:filteredFiles, + files_details:results.files_details.filter(f => !this.shouldIgnorePath(repo, f.path)) + } + } + + //If linguist doesn't provide file details, we can't filter effectively + this.debug("Warning: Unable to filter paths effectively as linguist didn't return file details") + return results + } + /**Analyze a repository */ async analyze(path, {commits = []} = {}) { const cache = {files:{}, languages:{}} @@ -186,7 +247,7 @@ if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { break } try { - const {total, files, missed, lines, stats} = await this.linguist(path, {commit, cache, repo}) + const {total, files, missed, lines, stats} = await this.filteredLinguist(path, {commit, cache, repo}) this.results.commits++ this.results.total += total this.results.files += files @@ -241,4 +302,4 @@ if (repo === ownerRepo && filePath.startsWith(pathToIgnore)) { debug(message) { return console.debug(`metrics/compute/${this.login}/plugins > languages > ${this.constructor.name.replace(/([a-z])([A-Z])/, (_, a, b) => `${a} ${b.toLocaleLowerCase()}`).toLocaleLowerCase()} > ${message}`) } -} \ No newline at end of file +} diff --git a/source/plugins/languages/analyzer/indepth.mjs b/source/plugins/languages/analyzer/indepth.mjs index 3ff5a933661..1d0461a67a7 100644 --- a/source/plugins/languages/analyzer/indepth.mjs +++ b/source/plugins/languages/analyzer/indepth.mjs @@ -194,6 +194,7 @@ export class IndepthAnalyzer extends Analyzer { /**Run linguist against a commit and compute edited lines and bytes*/ async linguist(path, {commit, cache}) { + this.debug(path, commit, cache) const result = {total:0, files:0, missed:{lines:0, bytes:0}, lines:{}, stats:{}} const edited = new Set() const seen = new Set() diff --git a/source/plugins/languages/index.mjs b/source/plugins/languages/index.mjs index b046c89f330..855b08dbe8a 100644 --- a/source/plugins/languages/index.mjs +++ b/source/plugins/languages/index.mjs @@ -18,7 +18,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled } //Load inputs - let {ignored, skipped, /*pathsIgnored,*/ other, colors, aliases, details, threshold, limit, indepth, "indepth.custom":_indepth_custom, "analysis.timeout":_timeout_global, "analysis.timeout.repositories":_timeout_repositories, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata + let {ignored, skipped, paths_ignored, other, colors, aliases, details, threshold, limit, indepth, "indepth.custom":_indepth_custom, "analysis.timeout":_timeout_global, "analysis.timeout.repositories":_timeout_repositories, sections, categories, "recent.categories":_recent_categories, "recent.load":_recent_load, "recent.days":_recent_days} = imports.metadata .plugins.languages .inputs({ data, @@ -68,7 +68,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled if ((sections.includes("recently-used")) && (imports.metadata.plugins.languages.extras("indepth", {extras}))) { try { console.debug(`metrics/compute/${login}/plugins > languages > using recent analyzer`) - languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, context, account}, {skipped, categories:_recent_categories ?? categories, days:_recent_days, load:_recent_load, timeout}) + languages["stats.recent"] = await recent_analyzer({login, data, imports, rest, context, account}, {skipped, pathsIgnored:paths_ignored, categories:_recent_categories ?? categories, days:_recent_days, load:_recent_load, timeout}) Object.assign(languages.colors, languages["stats.recent"].colors) } catch (error) { @@ -81,7 +81,7 @@ export default async function({login, data, imports, q, rest, account}, {enabled try { console.debug(`metrics/compute/${login}/plugins > languages > switching to indepth mode (this may take some time)`) const existingColors = languages.colors - Object.assign(languages, await indepth_analyzer({login, data, imports, rest, context, repositories:repositories.concat(_indepth_custom)}, {skipped, categories, timeout})) + Object.assign(languages, await indepth_analyzer({login, data, imports, rest, context, repositories:repositories.concat(_indepth_custom)}, {skipped, pathsIgnored:paths_ignored, categories, timeout})) Object.assign(languages.colors, existingColors) console.debug(`metrics/compute/${login}/plugins > languages > indepth analysis processed successfully ${languages.commits} and missed ${languages.missed.commits} commits in ${languages.elapsed.toFixed(2)}m`) }