From 2e5bbfde5d630af0ec1a6272a8d1fd1c1b4f2dd1 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Thu, 12 Jun 2025 15:42:18 +0200 Subject: [PATCH 1/8] Split commands.yml into running and saving TODO - add channel for variables about outcome between jobs --- .github/workflows/commands.yml | 74 ++++++++++++++++++++++++++++++++-- 1 file changed, 71 insertions(+), 3 deletions(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index 328e7c1c889..dba81dd62e3 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -7,8 +7,8 @@ on: - cron: '0 13 * * *' permissions: - contents: write - issues: write + contents: read + issues: read pull-requests: read jobs: @@ -80,7 +80,75 @@ jobs: if: steps.command-extractor.outputs.result == 'xlf' id: xlf run: dotnet build src/Compiler /t:UpdateXlf - + - name: Process ilverify command (Update ILVerify baselines) + if: steps.command-extractor.outputs.result == 'ilverify' + id: ilverify + env: + TEST_UPDATE_BSL: 1 + run: | + # Run the ilverify script with TEST_UPDATE_BSL=1 + pwsh tests/ILVerify/ilverify.ps1 + + # Calculate the changes per file + echo "Checking for changes in baseline files..." + FILES_CHANGED=0 + CHANGES_OUTPUT="" + + for file in tests/ILVerify/*.bsl; do + if git diff --quiet "$file"; then + continue + else + FILES_CHANGED=$((FILES_CHANGED + 1)) + LINES_CHANGED=$(git diff --numstat "$file" | awk '{print $1 + $2}') + CHANGES_OUTPUT="${CHANGES_OUTPUT}${file}: ${LINES_CHANGED} lines changed\n" + fi + done + + if [ "$FILES_CHANGED" -eq 0 ]; then + echo "result=The ilverify command ran and did not modify any baseline." >> $GITHUB_OUTPUT + else + echo -e "result=The ilverify command ran and triggered the following number of changes per file:\n${CHANGES_OUTPUT}" >> $GITHUB_OUTPUT + fi + - name: Generate patch + id: generate_patch + run: git diff > workflow.patch + - name: Upload patch + uses: actions/upload-artifact@v4 + with: + name: workflowpatch + path: workflow.patch + save_command: + needs: run_command + permissions: + contents: write + issues: write + pull-requests: write + runs-on: ubuntu-latest + steps: + - name: Get github ref + uses: actions/github-script@v3 + id: get-pr + with: + script: | + const result = await github.pulls.get({ + pull_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + }); + return { "ref": result.data.head.ref, "repository": result.data.head.repo.full_name}; + - name: Checkout repo + uses: actions/checkout@v2 + with: + repository: ${{ fromJson(steps.get-pr.outputs.result).repository }} + ref: ${{ fromJson(steps.get-pr.outputs.result).ref }} + fetch-depth: 0 + - name: Download patch from run step + uses: actions/download-artifact@v4 + with: + name: workflowpatch + - name: Apply patch + id: apply_patch + run: git apply workflow.patch - name: Commit and push changes if: steps.fantomas.outcome == 'success' || steps.xlf.outcome == 'success' || steps.ilverify.outcome == 'success' run: | From d6e98500a00214afd9996d41c380a7f3d8e68b3e Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 13 Jun 2025 14:51:31 +0200 Subject: [PATCH 2/8] Update commands.yml --- .github/workflows/commands.yml | 343 +++++++++++++++------------------ 1 file changed, 160 insertions(+), 183 deletions(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index dba81dd62e3..eddc6622a81 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -1,199 +1,176 @@ -name: Commands on PR +name: Run CLI Commands via PR Comment + on: issue_comment: types: [created] - schedule: - # once a day at 13:00 UTC - - cron: '0 13 * * *' - -permissions: - contents: read - issues: read - pull-requests: read jobs: - cleanup_old_runs: - if: github.event.schedule == '0 13 * * *' + # This first job by definiton runs user-supplied code - you must NOT elevate its permissions to `write` + # Malicious code could change nuget source URL, build targets or even compiler itself to pass a GH token + # And use it to create branches, spam issues etc. Any write-actions happen in the second job, which does not allow + # user extension points (i.e. plain scripts, must NOT run scripts from within checked-out code) + detect-and-run: runs-on: ubuntu-latest - permissions: - actions: write - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + outputs: + command: ${{ steps.parse.outputs.command }} + arg: ${{ steps.parse.outputs.arguments }} + if: github.event.issue.pull_request steps: - - name: Delete old workflow runs - run: | - _UrlPath="/repos/$GITHUB_REPOSITORY/actions/workflows" - _CurrentWorkflowID="$(gh api -X GET "$_UrlPath" | jq '.workflows[] | select(.name == '\""$GITHUB_WORKFLOW"\"') | .id')" + - name: Parse comment + id: parse + uses: drewwyatt/comment-pipeline@v1 + with: + comment: ${{ toJSON(github.event.comment) }} + commands: | + /run fantomas + /run ilverify + /run xlf + /run test-baseline + github-token: ${{ secrets.GITHUB_TOKEN }} - # delete workitems which are 'completed'. (other candidate values of status field are: 'queued' and 'in_progress') + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Checkout PR branch + if: ${{ steps.parse.outputs.command }} + run: gh auth setup-git && gh pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - gh api -X GET "$_UrlPath/$_CurrentWorkflowID/runs" --paginate \ - | jq '.workflow_runs[] | select(.status == "completed") | .id' \ - | xargs -I{} gh api -X DELETE "/repos/$GITHUB_REPOSITORY/actions/runs"/{} + - name: Install dotnet + uses: actions/setup-dotnet@v3 + with: + global-json-file: global.json - run_command: - if: github.event.issue.pull_request != '' && contains(github.event.comment.body, '/run') - runs-on: ubuntu-latest - steps: - - name: Extract command to run - uses: actions/github-script@v3 - id: command-extractor - with: - result-encoding: string - script: | - if (context.eventName !== "issue_comment") throw "Error: This action only works on issue_comment events."; - - // extract the command to run, allowed characters: a-z, A-Z, digits, hyphen, underscore - const regex = /^\/run ([a-zA-Z\d\-\_]+)/; - command = regex.exec(context.payload.comment.body); - if (command == null) throw "Error: No command found in the trigger phrase."; - - return command[1]; - - name: Get github ref - uses: actions/github-script@v3 - id: get-pr - with: - script: | - const result = await github.pulls.get({ - pull_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - return { "ref": result.data.head.ref, "repository": result.data.head.repo.full_name}; - - name: Checkout repo - uses: actions/checkout@v2 - with: - repository: ${{ fromJson(steps.get-pr.outputs.result).repository }} - ref: ${{ fromJson(steps.get-pr.outputs.result).ref }} - fetch-depth: 0 - - name: Install dotnet - uses: actions/setup-dotnet@v3 - with: - global-json-file: global.json - - name: Install dotnet tools - run: dotnet tool restore - - name: Process fantomas command - if: steps.command-extractor.outputs.result == 'fantomas' - id: fantomas - run: dotnet fantomas . - - name: Process xlf command - if: steps.command-extractor.outputs.result == 'xlf' - id: xlf - run: dotnet build src/Compiler /t:UpdateXlf - - name: Process ilverify command (Update ILVerify baselines) - if: steps.command-extractor.outputs.result == 'ilverify' - id: ilverify - env: - TEST_UPDATE_BSL: 1 - run: | - # Run the ilverify script with TEST_UPDATE_BSL=1 - pwsh tests/ILVerify/ilverify.ps1 - - # Calculate the changes per file - echo "Checking for changes in baseline files..." - FILES_CHANGED=0 - CHANGES_OUTPUT="" - - for file in tests/ILVerify/*.bsl; do - if git diff --quiet "$file"; then - continue + - name: Install dotnet tools + run: dotnet tool restore + + - name: Run command + id: run-cmd + env: + TEST_UPDATE_BSL: 1 + continue-on-error: true + run: | + case "${{ steps.parse.outputs.command }}" in + "/run fantomas") dotnet fantomas . ;; + "/run xlf") dotnet build src/Compiler /t:UpdateXlf ;; + "/run ilverify") pwsh tests/ILVerify/ilverify.ps1 ;; + "/run test-baseline") dotnet test ./FSharp.Compiler.Service.sln --filter "${{ steps.parse.outputs.arguments }}" -c Release ;; + *) echo "Unknown command" && exit 1 ;; + esac + + - name: Create patch & metadata + id: meta + if: steps.parse.outputs.command + run: | + echo "run_step_outcome=${{ steps.run-cmd.outcome }}" > result + if [[ "${{ steps.run-cmd.outcome }}" == "success" ]]; then + git diff > repo.patch || true + if [ -s repo.patch ]; then echo "hasPatch=true" >> result; else echo "hasPatch=false" >> result; fi else - FILES_CHANGED=$((FILES_CHANGED + 1)) - LINES_CHANGED=$(git diff --numstat "$file" | awk '{print $1 + $2}') - CHANGES_OUTPUT="${CHANGES_OUTPUT}${file}: ${LINES_CHANGED} lines changed\n" + echo "hasPatch=false" >> result fi - done - - if [ "$FILES_CHANGED" -eq 0 ]; then - echo "result=The ilverify command ran and did not modify any baseline." >> $GITHUB_OUTPUT - else - echo -e "result=The ilverify command ran and triggered the following number of changes per file:\n${CHANGES_OUTPUT}" >> $GITHUB_OUTPUT - fi - - name: Generate patch - id: generate_patch - run: git diff > workflow.patch - - name: Upload patch - uses: actions/upload-artifact@v4 - with: - name: workflowpatch - path: workflow.patch - save_command: - needs: run_command + cat result + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: cli-results + path: | + repo.patch + result + + apply-and-report: + needs: detect-and-run + runs-on: ubuntu-latest permissions: contents: write - issues: write pull-requests: write - runs-on: ubuntu-latest + if: needs.detect-and-run.outputs.command != '' steps: - - name: Get github ref - uses: actions/github-script@v3 - id: get-pr - with: - script: | - const result = await github.pulls.get({ - pull_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - }); - return { "ref": result.data.head.ref, "repository": result.data.head.repo.full_name}; - - name: Checkout repo - uses: actions/checkout@v2 - with: - repository: ${{ fromJson(steps.get-pr.outputs.result).repository }} - ref: ${{ fromJson(steps.get-pr.outputs.result).ref }} - fetch-depth: 0 - - name: Download patch from run step - uses: actions/download-artifact@v4 - with: - name: workflowpatch - - name: Apply patch - id: apply_patch - run: git apply workflow.patch - - name: Commit and push changes - if: steps.fantomas.outcome == 'success' || steps.xlf.outcome == 'success' || steps.ilverify.outcome == 'success' - run: | - # Only commit if there are actual changes - if git diff --quiet; then - echo "No changes to commit, skipping." - exit 0 - fi - - git config --local user.name "github-actions[bot]" - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git commit -a -m 'Automated command ran: ${{ steps.command-extractor.outputs.result }} - - Co-authored-by: ${{ github.event.comment.user.login }} <${{ github.event.comment.user.id }}+${{ github.event.comment.user.login }}@users.noreply.github.com>' - git push origin HEAD:"refs/heads/$PR_HEAD_REF"\ - - name: Post command comment - if: steps.fantomas.outcome == 'success' || steps.xlf.outcome == 'success' || steps.ilverify.outcome == 'success' - uses: actions/github-script@v3 - with: - script: | - // Probably, there's more universal way of getting outputs, but my gh-actions-fu is not that good. - var output = "" - if ("${{steps.command-extractor.outputs.result}}" == 'fantomas') { - output = "${{steps.fantomas.outputs.result}}" - } else if ("${{steps.command-extractor.outputs.result}}" == 'xlf') { - output = "${{steps.xlf.outputs.result}}" - } else if ("${{steps.command-extractor.outputs.result}}" == 'ilverify') { - output = "${{steps.ilverify.outputs.result}}" - } - const body = `Ran ${{ steps.command-extractor.outputs.result }}: https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}\n${output}`; - await github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: body - }); - - name: Post command failed comment - if: failure() - uses: actions/github-script@v3 - with: - script: | - const body = `Failed to run ${{ steps.command-extractor.outputs.result }}: https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${process.env.GITHUB_RUN_ID}`; - await github.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: body - }); + - name: Checkout the repository + uses: actions/checkout@v4 + + - name: Checkout PR branch + run: gh auth setup-git && gh pr checkout ${{ github.event.issue.number }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + + - name: Download artifacts + uses: actions/download-artifact@v4 + with: + name: cli-results + + - name: Read metadata + id: read-meta + run: | + source result + echo "run_step_outcome=$run_step_outcome" >> $GITHUB_OUTPUT + echo "hasPatch=$hasPatch" >> $GITHUB_OUTPUT + + - name: Apply and push patch + if: ${{ steps.read-meta.outputs.run_step_outcome == 'success' && steps.read-meta.outputs.hasPatch == 'true' }} + run: | + patch -p1 -s --force < repo.patch || true + git config user.name "GH Actions" + git config user.email "actions@github.com" + git add -u + git commit -m "Apply patch from ${{ needs.detect-and-run.outputs.command }}" + upstream=$(git rev-parse --abbrev-ref --symbolic-full-name @{u}) + remote=${upstream%%/*} + branch=${upstream#*/} + + echo "Pushing to $remote $branch" + git push "$remote" HEAD:"$branch" + + - name: Count stats + id: stats + if: ${{ steps.read-meta.outputs.run_step_outcome == 'success' && steps.read-meta.outputs.hasPatch == 'true' }} + run: | + files=$(git diff --name-only HEAD~1 HEAD | wc -l) + lines=$(git diff HEAD~1 HEAD | wc -l) + echo "files=$files" >> $GITHUB_OUTPUT + echo "lines=$lines" >> $GITHUB_OUTPUT + - name: Generate and publish report + if: always() + env: + COMMAND: ${{ needs.detect-and-run.outputs.command }} + OUTCOME: ${{ steps.read-meta.outputs.run_step_outcome }} + PATCH: ${{ steps.read-meta.outputs.hasPatch }} + run: | + # Build the markdown report + report=" + # 🔧 CLI Command Report + + - **Command:** \`${COMMAND}\` + - **Outcome:** ${OUTCOME} + + " + + if [[ "$OUTCOME" == "success" ]]; then + if [[ "$PATCH" == "true" ]]; then + report+="✅ Patch applied: + - Files changed: ${{ steps.stats.outputs.files }} + - Lines changed: ${{ steps.stats.outputs.lines }}" + else + report+="✅ Command succeeded, no changes needed." + fi + else + report+="❌ Command **failed** — no patch applied." + fi + + # Output to GitHub Actions UI + echo "$report" >> "$GITHUB_STEP_SUMMARY" + + # Store for use in next step + echo "$report" > pr_report.md + + - name: Comment on PR + if: always() + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ env.PR_NUMBER }} + run: | + # Use gh CLI to comment with multi-line markdown + gh pr comment ${{ github.event.issue.number }} \ + --body-file pr_report.md From 0ac4d2557f73bc25dead504f24b23df34cd3a1fe Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 16 Jun 2025 10:28:52 +0200 Subject: [PATCH 3/8] Also download .NET9 for test execution --- .github/workflows/commands.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index eddc6622a81..b98b44d58ef 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -44,6 +44,12 @@ jobs: - name: Install dotnet tools run: dotnet tool restore + + - name: Setup .NET 9.0.0 Runtime for test execution + if: ${{ steps.parse.outputs.command == '/run test-baseline' }} + uses: actions/setup-dotnet@v4 + with: + dotnet-version: '9.0.x' - name: Run command id: run-cmd @@ -55,7 +61,7 @@ jobs: "/run fantomas") dotnet fantomas . ;; "/run xlf") dotnet build src/Compiler /t:UpdateXlf ;; "/run ilverify") pwsh tests/ILVerify/ilverify.ps1 ;; - "/run test-baseline") dotnet test ./FSharp.Compiler.Service.sln --filter "${{ steps.parse.outputs.arguments }}" -c Release ;; + "/run test-baseline") dotnet test ./FSharp.Compiler.Service.sln --filter "${{ steps.parse.outputs.arguments }}" -c Release || true ;; *) echo "Unknown command" && exit 1 ;; esac From b42cb7000c136ff8bad10ca4a1147dc0fd637da1 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 16 Jun 2025 10:39:25 +0200 Subject: [PATCH 4/8] Update CONTRIBUTING.md --- CONTRIBUTING.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b8c7c63108a..3510ffd7305 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -89,6 +89,26 @@ If you don't know what a pull request is read this article: Date: Mon, 16 Jun 2025 10:39:46 +0200 Subject: [PATCH 5/8] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3510ffd7305..4f83fa70e98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,7 +105,7 @@ For example, the following errors can appear during CI runs: After identifying a failing test which relies on a baseline, the command can then for example be: - `/run test-baseline ParseFile` to update parsing tests related to syntactical tree -- - `/run test-baseline SurfaceAreaTest` to update the API surface area of FSharp.Compiler.Service +- `/run test-baseline SurfaceAreaTest` to update the API surface area of FSharp.Compiler.Service - `/run test-baseline FullyQualifiedName~EmittedIL&FullyQualifiedName~Nullness` to update IL baseline (namespace `EmittedIL`) for tests that touch the `Nullness` feature From 1a962fde1690b82d1099b5aaab880d5d348f2434 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Mon, 16 Jun 2025 10:40:31 +0200 Subject: [PATCH 6/8] Update CONTRIBUTING.md --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4f83fa70e98..84d3f482ed8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -95,7 +95,7 @@ The following comments in a PR can be used as commands to execute scripts which - `/run fantomas` runs `dotnet fantomas .` - `/run ilverify` updates IL verification baseline - `/run xlf` refreshes localisation files for translatable strings -- `/run test-baseline ...` runs tests with the `TEST_UPDATE_BSL: 1` environment variable and an argument supplied filter. Its goal is to refresh baselines. +- `/run test-baseline ...` runs tests with the `TEST_UPDATE_BSL: 1` environment variable and an argument supplied filter (passed to `dotnet test --filter ..`). Its goal is to refresh baselines. This code repository uses a lot of baselines - captures for important output - to spot regressions and willingfully accept changes via PR review. For example, the following errors can appear during CI runs: From f450e8d41ef32ab3d4abd935494f0f8a149dd150 Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Tue, 17 Jun 2025 10:35:23 +0200 Subject: [PATCH 7/8] Update .github/workflows/commands.yml --- .github/workflows/commands.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index b98b44d58ef..476e99492cd 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Parse comment id: parse - uses: drewwyatt/comment-pipeline@v1 + uses: drewwyatt/comment-pipeline@b3034a72ab12ff59a4c3f9a16b32515e36ae4be7 with: comment: ${{ toJSON(github.event.comment) }} commands: | From f4df0c65a9a25104d396868e729e1b86d35f34df Mon Sep 17 00:00:00 2001 From: Tomas Grosup Date: Fri, 20 Jun 2025 08:47:13 +0200 Subject: [PATCH 8/8] Update .github/workflows/commands.yml --- .github/workflows/commands.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/commands.yml b/.github/workflows/commands.yml index 476e99492cd..1c48ab2fdb3 100644 --- a/.github/workflows/commands.yml +++ b/.github/workflows/commands.yml @@ -18,7 +18,7 @@ jobs: steps: - name: Parse comment id: parse - uses: drewwyatt/comment-pipeline@b3034a72ab12ff59a4c3f9a16b32515e36ae4be7 + uses: dotnet/comment-pipeline@1 with: comment: ${{ toJSON(github.event.comment) }} commands: |