Merge pull request #3 from AutoModality/AM-337/gtest

Added error reporting

Author: Aaron Roller

.github/workflows/release.yml

@@ -14,6 +14,7 @@ jobs:
         run: |
             [ ${{ steps.xunit-viewer.outputs.report-file }} == "test-reports/index.html" ]
       - name: Attach the report
+        if: always()
         uses: actions/upload-artifact@v1
         with:
           name: test-reports
@@ -36,6 +37,7 @@ jobs:
         run: |
             [ ${{ steps.xunit-viewer.outputs.report-file }} == "${{ env.RESULTS_PATH }}/index.html" ]
       - name: Attach the report
+        if: always()
         uses: actions/upload-artifact@v1
         with:
           name: alternate-results-path-reports
@@ -56,17 +58,60 @@ jobs:
         run: |
             [ ${{ steps.xunit-viewer.outputs.report-file }} == "somewhere/custom-report.html" ]
       - name: Attach the report
+        if: always()
         uses: actions/upload-artifact@v1
         with:
           name: alternate-report-path-reports
           path: somewhere/custom-report.html
+  test-show-failure:
+    runs-on: ubuntu-18.04
+    continue-on-error: true
+    name: Demonstrates failure case
+    steps:
+      - uses: actions/checkout@v1
+      - id: xunit-viewer
+        uses: ./  #use the current project relative to root
+        with:
+          results: test/failure
+      - name: Attach the report
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ steps.xunit-viewer.outputs.report-name }}
+          path: ${{ steps.xunit-viewer.outputs.report-dir }}
+  test-ignore-failure:
+    runs-on: ubuntu-18.04
+    name: Demonstrates failure is not reported
+    steps:
+      - uses: actions/checkout@v1
+      - id: xunit-viewer
+        uses: ./  #use the current project relative to root
+        with:
+          results: test/failure
+          fail: false
+      - name: Attach the report
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: ${{ steps.xunit-viewer.outputs.report-name }}
+          path: ${{ steps.xunit-viewer.outputs.report-dir }}
+  test-detection:
+    runs-on: ubuntu-18.04
+    name: Run tests to ensure detection is working
+    steps:
+      - uses: actions/checkout@v1
+      - name: Run tests
+        run: test/all.sh   
   release:
     runs-on: ubuntu-18.04
     name: Release if all tests pass on master branch
     needs:
       - test-defaults
       - test-alternate-results-path
       - test-alternate-report-path
+      - test-detection
+      - test-show-failure
+      - test-ignore-failure
     steps:
       - uses: actions/checkout@v1
       - name: Semantic Release

.gitignore

@@ -102,3 +102,4 @@ dist
 
 # TernJS port file
 .tern-port
+.history

Dockerfile

@@ -3,6 +3,7 @@ FROM node:lts-alpine
 
 # Copies your code file from your action repository to the filesystem path `/` of the container
 COPY entrypoint.sh /entrypoint.sh
+COPY detection.sh /detection.sh
 
 RUN npm i -g xunit-viewer
 

README.md

@@ -22,6 +22,8 @@ the corresponding Action Run.
 Assumes your test reports exist in a folder named `test-reports` and generates the 
 report at `test-reports/index.html`. 
 
+This test will report a failure if any of the tests failed or errored.  `fail=false` will disable.
+
 ```
 name: Test
 on: push
@@ -36,7 +38,7 @@ jobs:
       - name: Attach the report
         uses: actions/upload-artifact@v1
         with:
-          name: test-reports
+          name: ${{ steps.xunit-viewer.outputs.report-name }}
           path: ${{ steps.xunit-viewer.outputs.report-dir }}
 
 ```

action.yml

@@ -9,19 +9,31 @@ inputs:
     description: File/Folder of test results
     required: true
     default: test-reports
+  title:  
+    description: HTML title e.g. "My Tests". Default is Repository Name.
+    required: false
+    default: ${{ github.repository }}
   output:  
     description: Output filename including optional path
     required: false
+  fail:  
+    description: Return failing exit code when failure or error detected
+    required: false
+    default: "true"
 outputs:
   report-file:
     description: "The file path where the report can be found. Equal to 'output' input."
   report-dir:
     description: "The directory where the report can be found."
+  report-name:
+    description: "Use this name with Artifact Upload for unique attachments"
 runs:
   using: 'docker'
   image: 'Dockerfile'
   args: # these map in order with entrypoint.sh
     - ${{ inputs.results }}
     - ${{ inputs.output }}
+    - ${{ inputs.title }}
+    - ${{ inputs.fail }}
 
 

detection.sh

@@ -0,0 +1,44 @@
+#!/bin/sh
+#NOTICE: Shell, not bash, so it runs on Alpine
+
+set -e
+
+file_or_folder="$1"
+
+# detects failures or errors because xunit-viewer unfortunately does not report
+# exits with non zero code if any error or failure is found
+# uses simple grep to look for failures="0" and errors="0"
+
+
+suite_success(){
+    file="$1"
+    # look for 0 failures and errors
+    # grep reports non zero code if anything but zero failures/errors
+    if cat "$file" | grep "<testsuites";then
+        suite_line=$(cat "$file" | grep "<testsuites")
+        if echo "$suite_line" | grep "failures=\"0\"";then
+            if echo "$suite_line" | grep "errors=\"0\"";then
+                return 0
+            else
+                echo "Error found in $file: $suite_line"
+                return 1
+            fi
+        else
+            echo "Failure found in $file: $suite_line"
+            return 1
+        fi
+    else
+        echo "Skipping file without suite: $file"
+    fi
+}
+
+echo "Inspecting '$file_or_folder'"
+
+if [ -d "$file_or_folder" ];then
+    for file in $file_or_folder/*.xml ; do suite_success "$file"; done
+elif [ -f "$file_or_folder" ]; then
+    suite_success "$file_or_folder"
+else
+    echo "Skipping: No file or folder found at $file_or_folder"
+fi
+

entrypoint.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
+#NOTICE: Shell, not bash, so it runs on Alpine
 
 set -e #exit on error
 
@@ -10,6 +11,9 @@ DEFAULT="none" # a way to handle ordered empty arguments of bash
 # output may be provided, otherwise will default to results dir
 results="${1}"
 output="${2}"
+title="${3:-Tests}"
+fail="${4:-true}"
+
 
 if [[ -z "$results" || "$results" == "$DEFAULT" ]]; then
     echo "result file/folder is required"
@@ -33,7 +37,17 @@ fi
 #ensure path exists
 mkdir -p "$(dirname "$output")"
 
-xunit-viewer --results="$results" --output="$output"
+xunit-viewer --results="$results" --output="$output" --console=true --title="$title"
+
+architecture=$(uname -m)
+report_name="test-results-$GITHUB_REPOSITORY-$GITHUB_WORKFLOW-$architecture-$GITHUB_RUN_ID"
+report_name_escaped=$(echo "$report_name" | tr /\\:*\<\>\|? -)
 
 echo ::set-output name=report-file::"$output"  #reference available to other actions
 echo ::set-output name=report-dir::"$report_dir"  #for easy attachment of a folder
+echo ::set-output name=report-name::"$report_name_escaped"  #to provide a globally unique name for downloading results
+
+# report non zero exit code if any failure or error detected
+if "$fail" == "true";then
+    /detection.sh "$results"
+fi

test/all.sh

@@ -0,0 +1,15 @@
+
+#!/bin/sh
+#NOTICE: Shell, not bash, so it runs on Alpine
+
+set -e
+
+test/one.sh test/success_fixture.xml 0
+test/one.sh test/failure_fixture.xml 1
+test/one.sh test/error_fixture.xml 1
+test/one.sh test 1
+test/one.sh test/success 0
+test/one.sh test/failure 1
+test/one.sh test/error 1
+test/one.sh missing.xml 0
+test/one.sh test/missing.xml 0

test/error/error_fixture.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="5" failures="0" disabled="0" errors="2" timestamp="2020-06-05T02:02:29" time="0.002" name="AllTests">
+  <testsuite name="StrCompare" tests="5" failures="2" disabled="0" errors="0" time="0.002">
+    <testcase name="CStrEqual" status="run" time="0" classname="StrCompare" />
+    <testcase name="CStrEqual2" status="run" time="0" classname="StrCompare" />
+    <testcase name="CStrEqual3" status="run" time="0" classname="StrCompare" />
+    <testcase name="CStrNotEqual" status="run" time="0" classname="StrCompare">
+      <failure message="/github/workspace/catkin_ws/src/am_super/test/sample_tests.cpp:27&#x0A;      Expected: expectVal&#x0A;      Which is: &quot;hello gtest&quot;&#x0A;To be equal to: actualValFalse&#x0A;      Which is: &quot;hello world&quot;" type=""><![CDATA[/github/workspace/catkin_ws/src/am_super/test/sample_tests.cpp:27
+      Expected: expectVal
+      Which is: "hello gtest"
+To be equal to: actualValFalse
+      Which is: "hello world"]]></failure>
+    </testcase>
+    <testcase name="CStrNotEqual4" status="run" time="0.002" classname="StrCompare">
+      <failure message="/github/workspace/catkin_ws/src/am_super/test/sample_tests.cpp:32&#x0A;      Expected: expectVal&#x0A;      Which is: &quot;hello gtest&quot;&#x0A;To be equal to: actualValFalse&#x0A;      Which is: &quot;hello world&quot;" type=""><![CDATA[/github/workspace/catkin_ws/src/am_super/test/sample_tests.cpp:32
+      Expected: expectVal
+      Which is: "hello gtest"
+To be equal to: actualValFalse
+      Which is: "hello world"]]></failure>
+    </testcase>
+  </testsuite>
+</testsuites>

test/error/success_fixture.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites tests="3" failures="0" disabled="0" errors="0" timestamp="2020-06-05T02:02:29" time="0.002" name="AllTests">
+  <testsuite name="StrCompare" tests="5" failures="2" disabled="0" errors="0" time="0.002">
+    <testcase name="CStrEqual" status="run" time="0" classname="StrCompare" />
+    <testcase name="CStrEqual2" status="run" time="0" classname="StrCompare" />
+    <testcase name="CStrEqual3" status="run" time="0" classname="StrCompare" />
+  </testsuite>
+</testsuites>