PR test runner

.github/workflows/run_tests.yml

@@ -0,0 +1,35 @@
+name: Run Tests
+
+on:
+#  pull_request
+  workflow_dispatch:
+  pull_request:
+  push:
+    branches:
+      - 'master'
+
+jobs:
+  run-tests:
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload Event File
+        uses: actions/upload-artifact@v4
+        with:
+          name: Event File
+          path: ${{ github.event_path }}
+
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Run Tests
+        run: docker compose -f tools/headless_testing/docker-compose.yml up
+        timeout-minutes: 30
+
+      - name: Upload Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Test Results
+          path: |
+            tools/headless_testing/testlog/results.json

.github/workflows/test_and_deploy.yml

@@ -0,0 +1,77 @@
+name: Process Tests
+
+on:
+  workflow_run:
+    workflows: ["Run Tests"]
+    types:
+      - completed
+
+permissions: {}
+
+run-name: "Process Tests - ${{ github.event.workflow_run.display_title }}"
+jobs:
+  process-test-results:
+    name: Process Test Results
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+    permissions:
+      checks: write
+      # needed unless run with comment_mode: off
+      pull-requests: write
+      # required by download step to access artifacts API
+      actions: read
+
+    steps:
+      - name: Download Artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: artifacts
+          github-token: ${{ github.token }}
+          run-id: ${{ github.event.workflow_run.id }}
+      - name: Publish Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          action_fail_on_inconclusive: true
+          time_unit: milliseconds
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/Test Results/*.json"
+  deploy:
+    name: Deploy
+    runs-on: ubuntu-latest
+    needs: process-test-results
+    if: |
+      github.repository == 'beyond-all-reason/Beyond-All-Reason' &&
+      github.event.workflow_run.event == 'push' &&
+      github.event.workflow_run.head_branch == 'master'
+    permissions:
+      id-token: write
+    steps:
+      - name: Trigger rebuild
+        run: |
+          echo "$SSH_KEY" > id.key
+          chmod og-rwx id.key
+          ssh -i id.key -o StrictHostKeyChecking=no [email protected] byar
+        env:
+          SSH_KEY: ${{ secrets.SSH_REPOS_DEPLOY_KEY }}
+
+      - name: Authenticate to Google Cloud
+        id: auth
+        uses: google-github-actions/auth@v2
+        with:
+          workload_identity_provider: projects/640511349987/locations/global/workloadIdentityPools/github-actions/providers/github
+          service_account: [email protected]
+          token_format: id_token
+          id_token_audience: cdnupdater
+          id_token_include_email: true
+
+      - name: Sync files to CDN
+        run: |
+          curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \
+            -X POST -d '["byar"]' https://rapidsyncer-ssd-7xiouooxaa-ey.a.run.app/sync
+
+      - name: Update CDN pointer
+        run: |
+          curl --fail -H "Authorization: Bearer ${{ steps.auth.outputs.id_token }}" \
+            -X GET https://bunny-update-edge-rule-7xiouooxaa-ew.a.run.app/update-edge-rule.sh

common/testing/infologtest.lua

@@ -0,0 +1,35 @@
+-- 'hidden' test checking infolog.txt for errors, used by headless runs.
+
+local maxErrors = 10
+
+local function skipErrors(line)
+	if string.find(line, 'Could not finalize projectile-texture atlas', nil, true) then
+		return true
+	end
+end
+
+local function infologTest()
+	local errors = {}
+	local infolog = VFS.LoadFile("infolog.txt")
+	if infolog then
+		local fileLines = string.lines(infolog)
+		for i, line in ipairs(fileLines) do
+			local errorIndex = line:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] Error().*')
+			if errorIndex and errorIndex > 0 and not skipErrors(line) then
+				errors[#errors+1] = line
+				if #errors > maxErrors then
+					return errors
+				end
+			end
+		end
+	end
+	return errors
+end
+
+
+function test()
+	local errors = infologTest()
+	if #errors > 0 then
+		error(table.concat(errors, "\n"), 0)
+	end
+end

common/testing/mocha_json_reporter.lua

@@ -8,11 +8,13 @@ function MochaJSONReporter:new()
 	local obj = {
 		totalTests = 0,
 		totalPasses = 0,
+		totalSkipped = 0,
 		totalFailures = 0,
 		startTime = nil,
 		endTime = nil,
 		duration = nil,
-		tests = {}
+		tests = {},
+		skipped = {}
 	}
 	setmetatable(obj, self)
 	self.__index = self
@@ -28,21 +30,37 @@ function MochaJSONReporter:endTests(duration)
 	self.duration = duration
 end
 
-function MochaJSONReporter:testResult(label, filePath, success, duration, errorMessage)
+function MochaJSONReporter:extractError(text)
+	local errorIndex = text:match('^%[string "[%p%a%s]*%"]:[%d]+:().*')
+	if errorIndex and errorIndex > 0 then
+		text = text:sub(errorIndex + 1)
+		return text
+	end
+	errorIndex = text:match('^%[t=[%d%.:]*%]%[f=[%-%d]*%] ().*')
+	if errorIndex and errorIndex > 0 then
+		text = text:sub(errorIndex)
+	end
+	return text
+end
+
+function MochaJSONReporter:testResult(label, filePath, success, skipped, duration, errorMessage)
 	local result = {
 		title = label,
 		fullTitle = label,
 		file = filePath,
 		duration = duration,
 	}
-	if success then
+	if skipped then
+		self.totalSkipped = self.totalSkipped + 1
+		result.err = {}
+	elseif success then
 		self.totalPasses = self.totalPasses + 1
 		result.err = {}
 	else
 		self.totalFailures = self.totalFailures + 1
 		if errorMessage ~= nil then
 			result.err = {
-				message = errorMessage,
+				message = self:extractError(errorMessage),
 				stack = errorMessage
 			}
 		else
@@ -54,6 +72,9 @@ function MochaJSONReporter:testResult(label, filePath, success, duration, errorM
 
 	self.totalTests = self.totalTests + 1
 	self.tests[#(self.tests) + 1] = result
+	if skipped then
+		self.skipped[#(self.skipped) + 1] = {fullTitle = label}
+	end
 end
 
 function MochaJSONReporter:report(filePath)
@@ -63,12 +84,14 @@ function MochaJSONReporter:report(filePath)
 			["tests"] = self.totalTests,
 			["passes"] = self.totalPasses,
 			["pending"] = 0,
+			["skipped"] = self.totalSkipped,
 			["failures"] = self.totalFailures,
 			["start"] = formatTimestamp(self.startTime),
 			["end"] = formatTimestamp(self.endTime),
 			["duration"] = self.duration
 		},
-		["tests"] = self.tests
+		["tests"] = self.tests,
+		["pending"] = self.skipped
 	}
 
 	local encoded = Json.encode(output)

luaui/Widgets/dbg_test_runner.lua

@@ -49,6 +49,7 @@ local config = {
 }
 
 local testReporter = nil
+local headless = false
 
 -- utils
 -- =====
@@ -79,6 +80,7 @@ local function logEndTests(duration)
 	testReporter:endTests(duration)
 
 	testReporter:report(config.testResultsFilePath)
+	headless = false
 end
 
 local function logTestResult(testResult)
@@ -90,6 +92,7 @@ local function logTestResult(testResult)
 		testResult.label,
 		testResult.filename,
 		(testResult.result == TestResults.TEST_RESULT.PASS),
+		(testResult.result == TestResults.TEST_RESULT.SKIP),
 		testResult.milliseconds,
 		testResult.error
 	)
@@ -173,6 +176,9 @@ local function findAllTestFiles(patterns)
 			result[#result + 1] = testFileInfo
 		end
 	end
+	if headless then
+		result[#result+1] = {label="infolog", filename="common/testing/infologtest.lua"}
+	end
 	return result
 end
 
@@ -1324,6 +1330,7 @@ function widget:Initialize()
 		self,
 		"runtestsheadless",
 		function(cmd, optLine, optWords, data, isRepeat, release, actions)
+			headless = true
 			config.noColorOutput = true
 			config.quitWhenDone = true
 			config.gameStartTestPatterns = Util.splitPhrases(optLine)

tools/headless_testing/download-maps.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.6.4"
+engine/*/pr-downloader --filesystem-writepath "$BAR_ROOT" --download-map "Supreme Isthmus v1.8"

tools/headless_testing/springsettings.cfg

@@ -1 +1,2 @@
 RapidTagResolutionOrder = repos-cdn.beyondallreason.dev;repos.beyondallreason.dev
+LogFlushLevel = 0

tools/headless_testing/startscript.txt

@@ -1,6 +1,6 @@
 [GAME]
 {
-	MapName=Supreme Isthmus v1.6.4;
+	MapName=Supreme Isthmus v1.8;
 	GameType=Beyond All Reason $VERSION;
 	GameStartDelay=0;
 	StartPosType=0;
@@ -10,7 +10,7 @@
 	FixedRNGSeed = 1;
   [MODOPTIONS]
   {
-    debugcommands=1:cheat|2:godmode|3:globallos|30:runtestsheadless;
+    debugcommands=1:cheat|2:godmode|10:testsautoheightmap 1|30:runtestsheadless;
 		deathmode=neverend;
   }
   [ALLYTEAM0]