Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
109 commits
Select commit Hold shift + click to select a range
fa56e3d
feat(test_framework): add an in-game lua testing framework
salinecitrine Dec 24, 2023
712484f
style(test_framework): remove author field
salinecitrine Jan 11, 2024
162c953
feat(test_framework): run test code during Update() and GameFrame()
salinecitrine Jan 11, 2024
e35d010
feat(test_framework): add timers and waitTime
salinecitrine Jan 11, 2024
8dec91b
feat(test_framework): expose WG to tests
salinecitrine Jan 11, 2024
830691b
test(cmd_stop_selfd): add test for basic functionality
salinecitrine Jan 11, 2024
edb148d
fix(test_framework): fix waitTime timeout math
salinecitrine Jan 11, 2024
72acf65
fix(test_framework): temporarily disable Update step
salinecitrine Jan 11, 2024
9d5d99f
fix(test_framework): restore step during update, and update wait test
salinecitrine Jan 11, 2024
fc3ee0a
Add summary of serpent.lua
salinecitrine Jan 12, 2024
fa2abf4
style(barwidgets): remove extra whitespace
salinecitrine Jan 12, 2024
eb133c0
feat(barwidgets): add option to expose locals when loading widgets
salinecitrine Jan 12, 2024
7f72589
test(gui_selfd_icons): load widget with locals
salinecitrine Jan 12, 2024
af7b544
test(gui_battle_resource_tracker): load widget with locals
salinecitrine Jan 14, 2024
76d45fc
test(balance): restore game speed to 1 in cleanup
salinecitrine Jan 12, 2024
daa4d76
refactor(test_framework): store locals as an ordered list
salinecitrine Jan 13, 2024
f85ce0b
fix(test_framework): access correct field in callin buffer
salinecitrine Jan 14, 2024
a48fe73
feat(test_framework): don't start tests if tests are already running
salinecitrine Jan 14, 2024
74d1ff3
fix(test_framework): clear callin buffer between tests
salinecitrine Jan 14, 2024
85d5ec8
fix(test_framework): adjust error stack distance to match usage
salinecitrine Jan 14, 2024
ab667b1
test(cmd_stop_selfd): wait a bit longer in between checks
salinecitrine Jan 14, 2024
448caf1
test(test_framework): remove clutter log messages
salinecitrine Jan 14, 2024
46e5a33
test(test_framework): add some more debug log messages
salinecitrine Jan 14, 2024
3accbcf
feat(test_framework): always show all results
salinecitrine Jan 14, 2024
9b28df1
test(gui_battle_resource_tracker): fail earlier if widget is not present
salinecitrine Jan 16, 2024
3755348
docs(test_framework): use better widget/gadget names/descriptions
salinecitrine Jan 16, 2024
6291954
feat(test_framework): only step during update if we are pre-game
salinecitrine Jan 16, 2024
1643bbd
fix(test_framework): clear callin buffer after each waitUntilCallin
salinecitrine Jan 16, 2024
419cef4
test(test_framework): shorten some very long debug log messages
salinecitrine Jan 16, 2024
6351103
feat(test_framework): remove widget if not in dev mode
salinecitrine Jan 17, 2024
0c0aa7e
refactor(test_framework): rearrange and use timer functions
salinecitrine Jan 17, 2024
c01279f
fix(test_framework): add missing RemoveAction
salinecitrine Jan 17, 2024
bfc4b30
fix(test_framework): add nil checks on timer functions
salinecitrine Jan 17, 2024
50364b8
feat(test_framework): add Test.clearCallinBuffer()
salinecitrine Jan 17, 2024
d2682ac
feat(test_framework): add Test.waitUntilCallinArgs()
salinecitrine Jan 17, 2024
71f9698
feat(test_framework): add support for running in headless mode
salinecitrine Jan 16, 2024
8aa4a8c
feat(test_framework): add watchdog to handle runner crash in headless
salinecitrine Jan 17, 2024
c6fa4b8
fix(test_framework): use relativePath, not just filename for matching
salinecitrine Jan 21, 2024
793913d
refactor(test_framework): file -> filename
salinecitrine Jan 21, 2024
7422013
feat(test_framework): add index and filename to test result data
salinecitrine Jan 21, 2024
092676a
feat(test_framework): replace log file with real test results file
salinecitrine Jan 21, 2024
f787fa9
feat(test_framework): add docker setup for headless testing
salinecitrine Jan 21, 2024
9db911b
feat(test_framework): set env vars for pr-downloader and engine config
salinecitrine Jan 21, 2024
dbdeb0e
refactor(test_framework): refine docker scripts
salinecitrine Jan 21, 2024
761438a
feat(test_framework): Add workflow scripts to show test results in PRs
salinecitrine Jan 21, 2024
81a8b20
fix(test_framework): Only enable watchdog if running in headless mode
salinecitrine Jan 22, 2024
2f50a47
feat(test_framework): add support for skip() function in tests
salinecitrine Feb 6, 2024
23f207f
fix(test_framework): disable watchdog on load
salinecitrine Feb 6, 2024
653a6fd
fix(test_framework): fix return values in localsAccess metatable
salinecitrine Feb 6, 2024
d19d526
fix(test_framework): use proper method call syntax in watchdog widget
salinecitrine Feb 6, 2024
df09631
feat(test_framework): add assertTablesEqual
salinecitrine Jan 11, 2024
b017461
test(mex-building-overhaul): add a test for some pregame actions
salinecitrine Feb 6, 2024
a518063
feat(test_framework): add Test.mock()
salinecitrine Feb 10, 2024
be50aeb
fix(test_framework): misplaced return statement
salinecitrine Feb 10, 2024
911ddcb
refactor(test_framework): make SKIP color a little clearer
salinecitrine Feb 10, 2024
9ec24fa
test(mex-building-overhaul): skip pregame test if not pregame
salinecitrine Feb 10, 2024
c3c9d5a
fix(test_framework): remove extra Spring.Echo
salinecitrine Feb 10, 2024
c799609
test(test_framework): add an example test for mocks
salinecitrine Feb 10, 2024
3781c36
Test that building queue can be edited with a mex selected (not overr…
thehobojoe Feb 10, 2024
b0e6a99
Merge branch 'master' into testing_framework
saurtron Dec 18, 2024
fd076be
Try change the docker compose command.
saurtron Dec 18, 2024
632e796
Fix conflict.
saurtron Dec 18, 2024
1db677f
Disable cus_gl4.
saurtron Dec 18, 2024
5a3ebce
Add missing RemoveWidget at gfx_unit_stencil_gl4:goodbye.
saurtron Dec 18, 2024
7e47229
Return from Initialize when unitStencilShader could not be created.
saurtron Dec 18, 2024
a10e5f3
Add debug print.
saurtron Dec 18, 2024
c823ea1
Remove dbg_test_framework widget.
saurtron Dec 18, 2024
5bd737e
Remove if gl.CreateShader doesn't exist.
saurtron Dec 18, 2024
2a8a02e
Remove if no gl.CreateShader.
saurtron Dec 18, 2024
3fd9deb
Protect a few more widgets from no gl.CreateShader.
saurtron Dec 18, 2024
248acad
Protect cmd_extractor_snap Shutdown from no gl4.
saurtron Dec 18, 2024
e3168bc
Allow api_blueprint to work even with no gl4.
saurtron Dec 18, 2024
fb894ad
Add expectCallin to test_wait.
saurtron Dec 18, 2024
067ac23
Give orders with synced run at test_arm_vs_cor_fighters.
saurtron Dec 18, 2024
360bf54
Allow declaring capability dependencies at ghInfo and whInfo.
saurtron Dec 18, 2024
11f6127
Merge branch 'gadget-widget-depends' into testing_framework_with_depends
saurtron Dec 18, 2024
c4b70c9
Print some platform info.
saurtron Dec 18, 2024
70d7846
Fixes.
saurtron Dec 18, 2024
4f5393c
Allow declaring capability dependencies at ghInfo and whInfo.
saurtron Dec 18, 2024
2a876c7
Add depends stance to a few widgets and gadgets.
saurtron Dec 18, 2024
7795b32
Add parameter to gl.GetString
saurtron Dec 18, 2024
11e9991
Move widget capabilities check to InsertWidgetRaw and RemoveWidgetRaw.
saurtron Dec 18, 2024
10df978
Merge remote-tracking branch 'origin/gadget-widget-cap-depends' into …
saurtron Dec 18, 2024
78f65ce
Protect api_unit_tracker_gl4 from error in shutdown.
saurtron Dec 18, 2024
6c5ba98
Merge remote-tracking branch 'origin/gadget-widget-cap-depends' into …
saurtron Dec 18, 2024
03dc98c
Add something random.
saurtron Dec 18, 2024
f78c793
Add stance for pull request.
saurtron Dec 18, 2024
28f6b92
Different empty message.
saurtron Dec 18, 2024
4c8281a
Fix test_wait.lua.
saurtron Dec 18, 2024
5b0f111
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
8968b7c
Count skip as pass at least for now.
saurtron Dec 18, 2024
afea9ea
Count skipped independently.
saurtron Dec 18, 2024
35016e3
Fix typo.
saurtron Dec 18, 2024
d4a2e0a
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
a3779b8
Remove specific types since defaults are fine.
saurtron Dec 18, 2024
40c25f5
Don't remove if no gl.
saurtron Dec 18, 2024
ad57cc2
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
fa6e9f2
SyncedRun for the test_arm_vs_cor_fighters orders.
saurtron Dec 18, 2024
f99feab
Remove battle resource tracker test.
saurtron Dec 18, 2024
03a8e86
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
e06d226
Also run on push to master.
saurtron Dec 18, 2024
2531761
Add missing locals.
saurtron Dec 18, 2024
60a16d2
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
dd38ac7
Update to use Supreme Isthmus v1.8.
saurtron Dec 18, 2024
08dc089
Update to use Supreme Isthmus v1.8.
saurtron Dec 18, 2024
d206969
Also set Supreme 1.8 at startscript.txt.
saurtron Dec 18, 2024
3527150
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
baa7b50
Add skipped as pending.
saurtron Dec 18, 2024
eff0721
Merge branch 'testing_framework_with_depends' into test-running-test
saurtron Dec 18, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 41 additions & 0 deletions .github/workflows/process_test_results.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: Process Test Results

on:
workflow_run:
workflows: ["Run Tests"]
types:
- completed

permissions: {}

jobs:
process-test-results:
name: Process Test Results
runs-on: ubuntu-latest
if: github.event.workflow_run.conclusion != 'skipped'

permissions:
checks: write

# needed unless run with comment_mode: off
pull-requests: write

# required by download step to access artifacts API
actions: read

steps:
- name: Download Artifacts
uses: actions/download-artifact@v4
with:
path: artifacts
github-token: ${{ github.token }}
run-id: ${{ github.event.workflow_run.id }}

- name: Publish Test Results
uses: EnricoMi/publish-unit-test-result-action@v2
with:
time_unit: milliseconds
commit: ${{ github.event.workflow_run.head_sha }}
event_file: artifacts/Event File/event.json
event_name: ${{ github.event.workflow_run.event }}
files: "artifacts/Test Results/*.json"
38 changes: 38 additions & 0 deletions .github/workflows/run_tests.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
name: Run Tests

on:
# pull_request
workflow_dispatch:
pull_request:
push:
branches:
- 'master'

jobs:
upload-event_file:
name: Upload Event File
runs-on: ubuntu-latest
steps:
- name: Upload Event File
uses: actions/upload-artifact@v4
with:
name: Event File
path: ${{ github.event_path }}

run-tests:
name: Run Tests
runs-on: ubuntu-latest
steps:
- name: Checkout Repository
uses: actions/checkout@v4

- name: Run Tests
run: docker compose -f tools/headless_testing/docker-compose.yml up

- name: Upload Test Results
if: always()
uses: actions/upload-artifact@v4
with:
name: Test Results
path: |
tools/headless_testing/testlog/results.json
57 changes: 57 additions & 0 deletions common/platformFunctions.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
if not Platform then return end
local hasGL4 = false
local hasGL = false
local hasShaders = false
local hasFBO = false
local isSyncedCode = (SendToUnsynced ~= nil)

local function determineCapabilities()
if not gl then
return
end
if gl.GetString(0x1F00) ~= "" then
hasGL = true
end
if gl.CreateShader and Platform.glHaveGLSL then
hasShaders = true
end
if gl.CreateFBO then
hasFBO = true
end

if hasFBO and hasShaders and Platform.glHaveGL4 then
hasGL4 = true
end
end

local function checkRequires(allRequires)
if not allRequires or isSyncedCode then
return true
end
for _, req in pairs(allRequires) do
if req == 'gl' and not hasGL then
return false
elseif req == 'gl4' and not hasGL4 then
return false
elseif req == 'shaders' and not hasShaders then
return false
elseif req == 'fbo' and not hasVBO then
return false
end
end
return true
end

local function extendPlatform()
Platform.gl = Platform.gl or hasGL
Platform.gl4 = Platform.gl4 or hasGL4
Platform.glHaveFBO = Platform.glHaveFBO or hasFBO
Platform.check = checkRequires
end

determineCapabilities()
extendPlatform()

Spring.Echo("PLATFORM", isSyncedCode)
Spring.Echo("PLATFORM GL", hasGL, hasGL4, hasShaders, hasFBO)
Spring.Echo("PLATFORM ATTRS", Platform.glHaveGLSL, Platform.glHaveGL4, gl and gl.GetString(0x1F00), Platform.glVendor, Platform.glVersion, Platform.glRenderer)
12 changes: 8 additions & 4 deletions common/testing/mocha_json_reporter.lua
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ function MochaJSONReporter:new()
local obj = {
totalTests = 0,
totalPasses = 0,
totalSkipped = 0,
totalFailures = 0,
startTime = nil,
endTime = nil,
Expand All @@ -28,14 +29,17 @@ function MochaJSONReporter:endTests(duration)
self.duration = duration
end

function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage)
function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage)
local result = {
title = label,
fullTitle = label,
file = filePath,
duration = duration,
}
if success then
if skipped then
self.totalSkipped = self.totalSkipped + 1
result.err = {}
elseif success then
self.totalPasses = self.totalPasses + 1
result.err = {}
else
Expand All @@ -47,7 +51,7 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM
}
else
result.err = {
message = "<unknown error>",
message = "<empty>",
}
end
end
Expand All @@ -62,7 +66,7 @@ function MochaJSONReporter:report(filePath)
["suites"] = 1,
["tests"] = self.totalTests,
["passes"] = self.totalPasses,
["pending"] = 0,
["pending"] = self.totalSkipped,
["failures"] = self.totalFailures,
["start"] = formatTimestamp(self.startTime),
["end"] = formatTimestamp(self.endTime),
Expand Down
2 changes: 2 additions & 0 deletions init.lua
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ if commonFunctions.spring[environment] then
local springFunctions = VFS.Include('common/springFunctions.lua')
Spring.Utilities = Spring.Utilities or springFunctions.Utilities
Spring.Debug = Spring.Debug or springFunctions.Debug
-- extend platform
VFS.Include('common/platformFunctions.lua')
end

if commonFunctions.i18n[environment] then
Expand Down
5 changes: 5 additions & 0 deletions luarules/gadgets.lua
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,11 @@ function gadgetHandler:LoadGadget(filename, overridevfsmode)
return nil -- gadget asked for a quiet death
end

if gadget.GetInfo and (Platform and not Platform.check(gadget.GetInfo().depends)) then
Spring.Echo('Disabling ' .. gadget:GetInfo().name .. ' for missing capabilities')
return nil
end

-- raw access to gadgetHandler
if gadget.GetInfo and gadget:GetInfo().handler then
gadget.gadgetHandler = self
Expand Down
1 change: 1 addition & 0 deletions luarules/gadgets/cus_gl4.lua
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ function gadget:GetInfo()
license = "GNU GPL, v2 or later",
layer = 0,
enabled = true,
depends = {'gl4'},
}
end

Expand Down
109 changes: 109 additions & 0 deletions luarules/gadgets/dbg_test_framework_proxy.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
function gadget:GetInfo()
return {
name = "Test Framework Synced Proxy",
desc = "Proxy for synced commands and code",
date = "2023",
license = "GNU GPL, v2 or later",
version = 0,
layer = 9999,
enabled = true,
}
end

if not gadgetHandler:IsSyncedCode() then
return
end

if not Spring.Utilities.IsDevMode() then
return
end

local LOG_LEVEL = LOG.INFO

VFS.Include('luarules/testing/util.lua')

local function log(level, str, ...)
if level < LOG_LEVEL then
return
end

Spring.Log(
gadget:GetInfo().name,
LOG.NOTICE,
str
)
end

local Extra = {
clearMap = function()
for _, unitID in ipairs(Spring.GetAllUnits()) do
Spring.DestroyUnit(unitID, false, true, nil, true)
end
for _, featureID in ipairs(Spring.GetAllFeatures()) do
Spring.DestroyFeature(featureID)
end
end,
}

local function processCall(msg, env)
log(LOG.DEBUG, "[processCall] " .. msg)
local data, returnId = deserializeFunctionCall(msg)

local fn = evaluateStringPath(data.path, env)

if not fn then
error("[processCall] could not find function: " .. data.path)
end

local pcallOk, pcallResult = splitFirstElement(pack(pcall(fn, unpack(data.args))))

log(LOG.DEBUG, "[processCall.pcall] " .. table.toString({
pcall = { pcallOk, pcallResult },
returnId = returnId,
}))

local serializedReturn = serializeFunctionReturn(pcallOk, pcallResult, returnId)

log(LOG.DEBUG, "[processCall.SendLuaUIMsg] " .. PROXY_RETURN_PREFIX .. serializedReturn)
Spring.SendLuaUIMsg(PROXY_RETURN_PREFIX .. serializedReturn)

end

local function processCode(msg)
log(LOG.DEBUG, "[processCode]")

local fn, returnId = deserializeFunctionRun(msg)

local pcallOk, pcallResult = fn()

log(LOG.DEBUG, "[processCode.pcall] " .. table.toString({
pcall = { pcallOk, pcallResult },
returnId = returnId,
}))

local serializedReturn = serializeFunctionReturn(pcallOk, pcallResult, returnId)

log(LOG.DEBUG, "[processCode.SendLuaUIMsg] " .. PROXY_RETURN_PREFIX .. serializedReturn)
Spring.SendLuaUIMsg(PROXY_RETURN_PREFIX .. serializedReturn)
end

local RECEIVE_MODES = {
[PROXY_PREFIX] = function(msg)
processCall(msg, _G)
end,
[PROXY_EXTRA_PREFIX] = function(msg)
processCall(msg, Extra)
end,
[PROXY_RUN_PREFIX] = function(msg)
processCode(msg)
end,
}

function gadget:RecvLuaMsg(msg, playerID)
for prefix, fn in pairs(RECEIVE_MODES) do
if msg:sub(1, #prefix) == prefix then
fn(msg:sub(#prefix + 1))
return
end
end
end
51 changes: 51 additions & 0 deletions luarules/testing/assertions.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,51 @@
function assertTablesEqual(table1, table2, margin, visited, path)
visited = visited or {}
path = path or {}
margin = margin or 0

local function buildPathString()
local pathString = ""
for _, key in ipairs(path) do
pathString = pathString .. "[" .. tostring(key) .. "]"
end
return pathString
end

if type(table1) ~= "table" or type(table2) ~= "table" then
if type(table1) == "number" and type(table2) == "number" then
assert(math.abs(table1 - table2) <= margin, "Numbers are not close enough at path: " .. buildPathString())
else
assert(table1 == table2, "Tables are not equal at path: " .. buildPathString())
end
return
end

if visited[table1] or visited[table2] then
-- Prevent infinite recursion on circular references
assert(table1 == table2, "Tables are not equal (circular reference) at path: " .. buildPathString())
return
end

visited[table1] = true
visited[table2] = true

for key, value1 in pairs(table1) do
local value2 = table2[key]
table.insert(path, key)
assertTablesEqual(value1, value2, margin, visited, path)
table.remove(path)
end

for key, value2 in pairs(table2) do
local value1 = table1[key]
if value1 == nil then
assert(false, "Tables are not equal, extra key '" .. tostring(key) .. "' in second table at path: " .. buildPathString())
end
end
end



return {
assertTablesEqual = assertTablesEqual,
}
Loading