From c0df0e445860b2131f742e6997deb388dc5e2293 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Wed, 8 Jan 2025 14:06:19 +0100 Subject: [PATCH 01/11] Disable blueprint tests on headless. --- luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_line.lua | 2 +- luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_single.lua | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_line.lua b/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_line.lua index aa81b408f3a..ec5811014f8 100644 --- a/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_line.lua +++ b/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_line.lua @@ -1,7 +1,7 @@ local widgetName = "Blueprint" function skip() - return Spring.GetGameFrame() <= 0 + return not Platform.gl end function setup() diff --git a/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_single.lua b/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_single.lua index 2ccf36190b6..5b83ace3dbb 100644 --- a/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_single.lua +++ b/luaui/Widgets/Tests/cmd_blueprint/test_cmd_blueprint_single.lua @@ -1,7 +1,7 @@ local widgetName = "Blueprint" function skip() - return Spring.GetGameFrame() <= 0 + return not Platform.gl end function setup() From 46dde62f7339c9aa3820ffcf34a4e2042f060a3e Mon Sep 17 00:00:00 2001 From: Citrine Date: Sun, 21 Jan 2024 12:43:25 -0800 Subject: [PATCH 02/11] feat(test_framework): Add workflow scripts to show test results in PRs The trigger for running tests is disabled until we want to enable them. In order to run, these scripts (like other workflow scripts) must be in the main branch (master). --- .github/workflows/process_test_results.yml | 41 ++++++++++++++++++++++ .github/workflows/run_tests.yml | 34 ++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 .github/workflows/process_test_results.yml create mode 100644 .github/workflows/run_tests.yml diff --git a/.github/workflows/process_test_results.yml b/.github/workflows/process_test_results.yml new file mode 100644 index 00000000000..55a7604eff5 --- /dev/null +++ b/.github/workflows/process_test_results.yml @@ -0,0 +1,41 @@ +name: Process Test Results + +on: + workflow_run: + workflows: ["Run Tests"] + types: + - completed + +permissions: {} + +jobs: + process-test-results: + name: Process Test Results + runs-on: ubuntu-latest + if: github.event.workflow_run.conclusion != 'skipped' + + permissions: + checks: write + + # needed unless run with comment_mode: off + pull-requests: write + + # required by download step to access artifacts API + actions: read + + steps: + - name: Download Artifacts + uses: actions/download-artifact@v4 + with: + path: artifacts + github-token: ${{ github.token }} + run-id: ${{ github.event.workflow_run.id }} + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + time_unit: milliseconds + commit: ${{ github.event.workflow_run.head_sha }} + event_file: artifacts/Event File/event.json + event_name: ${{ github.event.workflow_run.event }} + files: "artifacts/Test Results/*.json" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml new file mode 100644 index 00000000000..19a7d0a603b --- /dev/null +++ b/.github/workflows/run_tests.yml @@ -0,0 +1,34 @@ +name: Run Tests + +on: +# pull_request + workflow_dispatch: + +jobs: + upload-event_file: + name: Upload Event File + runs-on: ubuntu-latest + steps: + - name: Upload Event File + uses: actions/upload-artifact@v4 + with: + name: Event File + path: ${{ github.event_path }} + + run-tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Run Tests + run: docker-compose -f tools/headless_testing/docker-compose.yml up + + - name: Upload Test Results + if: always() + uses: actions/upload-artifact@v4 + with: + name: Test Results + path: | + tools/headless_testing/testlog/results.json From 8d132b32aa294b09e7a9ea5dde1aa42ca5edbe4f Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:27:04 +0100 Subject: [PATCH 03/11] Remove globallos, set Supreme Isthmus to 1.8. --- tools/headless_testing/download-maps.sh | 2 +- tools/headless_testing/startscript.txt | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tools/headless_testing/download-maps.sh b/tools/headless_testing/download-maps.sh index 8901c70e6f9..131eb233f66 100644 --- a/tools/headless_testing/download-maps.sh +++ b/tools/headless_testing/download-maps.sh @@ -1,3 +1,3 @@ #!/bin/bash -engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.6.4" +engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.8" diff --git a/tools/headless_testing/startscript.txt b/tools/headless_testing/startscript.txt index e41b9cf9f84..de123d076d9 100644 --- a/tools/headless_testing/startscript.txt +++ b/tools/headless_testing/startscript.txt @@ -1,6 +1,6 @@ [GAME] { - MapName=Supreme Isthmus v1.6.4; + MapName=Supreme Isthmus v1.8; GameType=Beyond All Reason $VERSION; GameStartDelay=0; StartPosType=0; @@ -10,7 +10,7 @@ FixedRNGSeed = 1; [MODOPTIONS] { - debugcommands=1:cheat|2:godmode|3:globallos|30:runtestsheadless; + debugcommands=1:cheat|2:godmode|30:runtestsheadless; deathmode=neverend; } [ALLYTEAM0] From 22fe7d62d318f25d858d3eea0ae194732b3e26ec Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:32:20 +0100 Subject: [PATCH 04/11] Add skipped tests, cleanup error message. --- common/testing/mocha_json_reporter.lua | 33 ++++++++++++++++++++++---- luaui/Widgets/dbg_test_runner.lua | 1 + 2 files changed, 29 insertions(+), 5 deletions(-) diff --git a/common/testing/mocha_json_reporter.lua b/common/testing/mocha_json_reporter.lua index 56b4570fd4e..a4d9ad40a47 100644 --- a/common/testing/mocha_json_reporter.lua +++ b/common/testing/mocha_json_reporter.lua @@ -8,11 +8,13 @@ function MochaJSONReporter:new() local obj = { totalTests = 0, totalPasses = 0, + totalSkipped = 0, totalFailures = 0, startTime = nil, endTime = nil, duration = nil, - tests = {} + tests = {}, + skipped = {} } setmetatable(obj, self) self.__index = self @@ -28,21 +30,37 @@ function MochaJSONReporter:endTests(duration) self.duration = duration end -function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage) +function MochaJSONReporter:extractError(text) + local errorIndex = text:match'^%[string "[%p%a%s]*%"]:[%d]+:().*' + if errorIndex and errorIndex > 0 then + text = text:sub(errorIndex + 1) + return text + end + errorIndex = text:match'^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*' + if errorIndex and errorIndex > 0 then + text = text:sub(errorIndex) + end + return text +end + +function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage) local result = { title = label, fullTitle = label, file = filePath, duration = duration, } - if success then + if skipped then + self.totalSkipped = self.totalSkipped + 1 + result.err = {} + elseif success then self.totalPasses = self.totalPasses + 1 result.err = {} else self.totalFailures = self.totalFailures + 1 if errorMessage ~= nil then result.err = { - message = errorMessage, + message = self:extractError(errorMessage), stack = errorMessage } else @@ -54,6 +72,9 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM self.totalTests = self.totalTests + 1 self.tests[#(self.tests) + 1] = result + if skipped then + self.skipped[#(self.skipped) + 1] = {fullTitle = label} + end end function MochaJSONReporter:report(filePath) @@ -63,12 +84,14 @@ function MochaJSONReporter:report(filePath) ["tests"] = self.totalTests, ["passes"] = self.totalPasses, ["pending"] = 0, + ["skipped"] = self.totalSkipped, ["failures"] = self.totalFailures, ["start"] = formatTimestamp(self.startTime), ["end"] = formatTimestamp(self.endTime), ["duration"] = self.duration }, - ["tests"] = self.tests + ["tests"] = self.tests, + ["pending"] = self.skipped } local encoded = Json.encode(output) diff --git a/luaui/Widgets/dbg_test_runner.lua b/luaui/Widgets/dbg_test_runner.lua index 8f743a0611a..c026d3221a2 100644 --- a/luaui/Widgets/dbg_test_runner.lua +++ b/luaui/Widgets/dbg_test_runner.lua @@ -90,6 +90,7 @@ local function logTestResult(testResult) testResult.label, testResult.filename, (testResult.result == TestResults.TEST_RESULT.PASS), + (testResult.result == TestResults.TEST_RESULT.SKIP), testResult.milliseconds, testResult.error ) From 2461125510bd8d7b527131c9720dbb1f59c951d6 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:33:55 +0100 Subject: [PATCH 05/11] Run an extra test on headless checking for errors at infolog.txt. --- common/testing/infologtest.lua | 34 +++++++++++++++++++++++++++++++ luaui/Widgets/dbg_test_runner.lua | 6 ++++++ 2 files changed, 40 insertions(+) create mode 100644 common/testing/infologtest.lua diff --git a/common/testing/infologtest.lua b/common/testing/infologtest.lua new file mode 100644 index 00000000000..f9ccae903eb --- /dev/null +++ b/common/testing/infologtest.lua @@ -0,0 +1,34 @@ +-- 'hidden' test checking infolog.txt for errors, used by headless runs. + +local maxErrors = 10 + +local function skipErrors(line) + if string.find(line, 'Could not finalize projectile-texture atlas', nil, true) then + return true + end +end + +local function infologTest() + local errors = {} + local infolog = VFS.LoadFile("infolog.txt") + if infolog then + local fileLines = string.lines(infolog) + for i, line in ipairs(fileLines) do + if string.find(line, 'Error:', nil, true) and not skipErrors(line) then + errors[#errors+1] = line + if #errors > maxErrors then + return errors + end + end + end + end + return errors +end + + +function test() + local errors = infologTest() + if #errors > 0 then + error(table.concat(errors, "\n"), 0) + end +end diff --git a/luaui/Widgets/dbg_test_runner.lua b/luaui/Widgets/dbg_test_runner.lua index c026d3221a2..af97aac7fae 100644 --- a/luaui/Widgets/dbg_test_runner.lua +++ b/luaui/Widgets/dbg_test_runner.lua @@ -49,6 +49,7 @@ local config = { } local testReporter = nil +local headless = false -- utils -- ===== @@ -79,6 +80,7 @@ local function logEndTests(duration) testReporter:endTests(duration) testReporter:report(config.testResultsFilePath) + headless = false end local function logTestResult(testResult) @@ -174,6 +176,9 @@ local function findAllTestFiles(patterns) result[#result + 1] = testFileInfo end end + if headless then + result[#result+1] = {label="infolog", filename="common/testing/infologtest.lua"} + end return result end @@ -1325,6 +1330,7 @@ function widget:Initialize() self, "runtestsheadless", function(cmd, optLine, optWords, data, isRepeat, release, actions) + headless = true config.noColorOutput = true config.quitWhenDone = true config.gameStartTestPatterns = Util.splitPhrases(optLine) From bed382a384b190fb37c830faa545dcb9d5c4af0b Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 18:38:37 +0100 Subject: [PATCH 06/11] Polish and merge github workflows. - Deploy now depends on successful tests. - Action processing tests now fails on inconclusive or failure. --- .github/workflows/process_test_results.yml | 41 ------------------ .github/workflows/run_tests.yml | 34 --------------- .../{quick_deploy.yml => test_and_deploy.yml} | 43 ++++++++++++++++--- 3 files changed, 38 insertions(+), 80 deletions(-) delete mode 100644 .github/workflows/process_test_results.yml delete mode 100644 .github/workflows/run_tests.yml rename .github/workflows/{quick_deploy.yml => test_and_deploy.yml} (58%) diff --git a/.github/workflows/process_test_results.yml b/.github/workflows/process_test_results.yml deleted file mode 100644 index 55a7604eff5..00000000000 --- a/.github/workflows/process_test_results.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: Process Test Results - -on: - workflow_run: - workflows: ["Run Tests"] - types: - - completed - -permissions: {} - -jobs: - process-test-results: - name: Process Test Results - runs-on: ubuntu-latest - if: github.event.workflow_run.conclusion != 'skipped' - - permissions: - checks: write - - # needed unless run with comment_mode: off - pull-requests: write - - # required by download step to access artifacts API - actions: read - - steps: - - name: Download Artifacts - uses: actions/download-artifact@v4 - with: - path: artifacts - github-token: ${{ github.token }} - run-id: ${{ github.event.workflow_run.id }} - - - name: Publish Test Results - uses: EnricoMi/publish-unit-test-result-action@v2 - with: - time_unit: milliseconds - commit: ${{ github.event.workflow_run.head_sha }} - event_file: artifacts/Event File/event.json - event_name: ${{ github.event.workflow_run.event }} - files: "artifacts/Test Results/*.json" diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml deleted file mode 100644 index 19a7d0a603b..00000000000 --- a/.github/workflows/run_tests.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Run Tests - -on: -# pull_request - workflow_dispatch: - -jobs: - upload-event_file: - name: Upload Event File - runs-on: ubuntu-latest - steps: - - name: Upload Event File - uses: actions/upload-artifact@v4 - with: - name: Event File - path: ${{ github.event_path }} - - run-tests: - name: Run Tests - runs-on: ubuntu-latest - steps: - - name: Checkout Repository - uses: actions/checkout@v4 - - - name: Run Tests - run: docker-compose -f tools/headless_testing/docker-compose.yml up - - - name: Upload Test Results - if: always() - uses: actions/upload-artifact@v4 - with: - name: Test Results - path: | - tools/headless_testing/testlog/results.json diff --git a/.github/workflows/quick_deploy.yml b/.github/workflows/test_and_deploy.yml similarity index 58% rename from .github/workflows/quick_deploy.yml rename to .github/workflows/test_and_deploy.yml index d6de550837e..16d00fa8e28 100644 --- a/.github/workflows/quick_deploy.yml +++ b/.github/workflows/test_and_deploy.yml @@ -1,14 +1,44 @@ -# Workflow to push the change to deploy the change to players -# on push to master much quicker without waiting for cron jobs. -name: Deploy +name: Run Tests + on: + workflow_dispatch: + pull_request: push: branches: - - master + - 'master' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + jobs: + run-tests: + name: Run Tests + runs-on: ubuntu-latest + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Run Tests + run: docker compose -f tools/headless_testing/docker-compose.yml up + timeout-minutes: 30 + + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + action_fail: true + action_fail_on_inconclusive: true + check_name: "Test Results" + time_unit: milliseconds + files: "tools/headless_testing/testlog/results.json" + deploy: + name: Deploy runs-on: ubuntu-latest - if: github.repository == 'beyond-all-reason/Beyond-All-Reason' + needs: run-tests + if: | + github.repository == 'beyond-all-reason/Beyond-All-Reason' && + github.event_name == 'push' && + github.ref == 'refs/heads/master' permissions: id-token: write steps: @@ -19,6 +49,7 @@ jobs: ssh -i id.key -o StrictHostKeyChecking=no debian@repos.beyondallreason.dev byar env: SSH_KEY: ${{ secrets.SSH_REPOS_DEPLOY_KEY }} + - name: Authenticate to Google Cloud id: auth uses: google-github-actions/auth@v2 @@ -28,10 +59,12 @@ jobs: token_format: id_token id_token_audience: cdnupdater id_token_include_email: true + - name: Sync files to CDN run: | curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \ -X POST -d '["byar"]' https://rapidsyncer-ssd-7xiouooxaa-ey.a.run.app/sync + - name: Update CDN pointer run: | curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \ From 0e2d576d8e727dda600d58df54d0dc40c284da45 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 19:01:34 +0100 Subject: [PATCH 07/11] Set LogFlushLevel to 0. --- tools/headless_testing/springsettings.cfg | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/headless_testing/springsettings.cfg b/tools/headless_testing/springsettings.cfg index 4cfae692517..5f8ac198062 100644 --- a/tools/headless_testing/springsettings.cfg +++ b/tools/headless_testing/springsettings.cfg @@ -1 +1,2 @@ RapidTagResolutionOrder = repos-cdn.beyondallreason.dev;repos.beyondallreason.dev +LogFlushLevel = 0 From c81efbf3116c1a3b550111de33f8aa2de93cacfa Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 19:08:26 +0100 Subject: [PATCH 08/11] Add missing parenthesis. --- common/testing/mocha_json_reporter.lua | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/testing/mocha_json_reporter.lua b/common/testing/mocha_json_reporter.lua index a4d9ad40a47..f10414e3646 100644 --- a/common/testing/mocha_json_reporter.lua +++ b/common/testing/mocha_json_reporter.lua @@ -31,12 +31,12 @@ function MochaJSONReporter:endTests(duration) end function MochaJSONReporter:extractError(text) - local errorIndex = text:match'^%[string "[%p%a%s]*%"]:[%d]+:().*' + local errorIndex = text:match('^%[string "[%p%a%s]*%"]:[%d]+:().*') if errorIndex and errorIndex > 0 then text = text:sub(errorIndex + 1) return text end - errorIndex = text:match'^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*' + errorIndex = text:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*') if errorIndex and errorIndex > 0 then text = text:sub(errorIndex) end From 716d4dd630fb61960f70b9435e08758b3f7ccc6b Mon Sep 17 00:00:00 2001 From: Saurtron Date: Sat, 21 Dec 2024 19:10:53 +0100 Subject: [PATCH 09/11] Match all infolog.txt errors starting with Error after the timestamp part. --- common/testing/infologtest.lua | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/testing/infologtest.lua b/common/testing/infologtest.lua index f9ccae903eb..8722f6b2114 100644 --- a/common/testing/infologtest.lua +++ b/common/testing/infologtest.lua @@ -14,7 +14,8 @@ local function infologTest() if infolog then local fileLines = string.lines(infolog) for i, line in ipairs(fileLines) do - if string.find(line, 'Error:', nil, true) and not skipErrors(line) then + local errorIndex = line:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] Error().*') + if errorIndex and errorIndex > 0 and not skipErrors(line) then errors[#errors+1] = line if #errors > maxErrors then return errors From ed532ffe2554f3530d109721e4d26916ceaaac97 Mon Sep 17 00:00:00 2001 From: Saurtron Date: Wed, 8 Jan 2025 14:13:26 +0100 Subject: [PATCH 10/11] Enable testsautoheightmap for running tests. --- tools/headless_testing/startscript.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/headless_testing/startscript.txt b/tools/headless_testing/startscript.txt index de123d076d9..69b73a27480 100644 --- a/tools/headless_testing/startscript.txt +++ b/tools/headless_testing/startscript.txt @@ -10,7 +10,7 @@ FixedRNGSeed = 1; [MODOPTIONS] { - debugcommands=1:cheat|2:godmode|30:runtestsheadless; + debugcommands=1:cheat|2:godmode|10:testsautoheightmap 1|30:runtestsheadless; deathmode=neverend; } [ALLYTEAM0] From 4456e4e3115201eaeccb967490522ffd6b4ede9d Mon Sep 17 00:00:00 2001 From: Saurtron Date: Wed, 8 Jan 2025 18:45:10 +0100 Subject: [PATCH 11/11] Don't try to clear outlineInstanceVBO if it doesn't exist. --- luaui/Widgets/api_blueprint.lua | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/luaui/Widgets/api_blueprint.lua b/luaui/Widgets/api_blueprint.lua index 0b6dd3969ba..ba9366dcc53 100644 --- a/luaui/Widgets/api_blueprint.lua +++ b/luaui/Widgets/api_blueprint.lua @@ -527,7 +527,9 @@ local BUILD_MODES_HANDLERS = { local instanceIDs = {} local function clearInstances() - clearInstanceTable(outlineInstanceVBO) + if outlineInstanceVBO then + clearInstanceTable(outlineInstanceVBO) + end if WG.StopDrawUnitShapeGL4 then WG.StopDrawAll(widget:GetInfo().name)