Show details when a test in regression.sh fails

.github/workflows/test.yml

@@ -46,6 +46,6 @@ jobs:
           options: --rm -v ${{ github.workspace }}:/workspace -w /workspace
           run: |
             set -o errexit
-            apk add bash git less make
+            apk add bash git less make diffutils
             pip install -r requirements-dev.txt
             make test

tests/regression.sh

@@ -2,6 +2,7 @@
 
 TOP_DIR=$(cd $(dirname $0)/.. && pwd) || exit 1
 cd $TOP_DIR || exit 1
+TEST_TMP=$(mktemp -d /tmp/ydiff-regression.XXXXXXXX)
 
 YDIFF=./ydiff.py
 
@@ -17,13 +18,40 @@ function pass()
     fi
 }
 
+function normalize_out()
+{
+    # quote ANSII ESC used to initiate color sequence, and strip any
+    # CR's at the end of the line (windows line endings are CR+LF)
+    local esc=$'\x1b'
+    local cr=$'\r'
+    sed "s/$esc/\x1b/g" | sed "s/$cr$//"
+}
+
 function fail()
 {
+    local expected_out=${1:?}
     if [[ -t 1 ]]; then
-        echo -e "\x1b[01;31mFAIL\x1b[0m" "$*"
+        echo -e "\x1b[01;31mFAIL\x1b[0m" "!= $expected_out"
     else
-        echo "FAIL" "$*"
+        echo "FAIL" "!= $expected_out"
     fi
+    normalize_out <"$TEST_TMP/cmd.out" >"$TEST_TMP/cmd.out.normalized"
+    normalize_out <"$expected_out" >"$TEST_TMP/expected-cmd.out.normalized"
+    {
+        echo "diff of expected vs. actual output (ANSI ESC quoted for readability):"
+        diff -u "$TEST_TMP/expected-cmd.out.normalized" "$TEST_TMP/cmd.out.normalized" | sed "s/^/    /"
+        echo "stderr:"
+        sed "s/^/    /" "$TEST_TMP/cmd.err"
+    } | sed "s/^/    /"
+}
+
+function show_file()
+{
+    local label=${1:?}
+    local file=${2:?}
+    echo "$label:"
+    sed 's/^/    /' "$file"
+    echo
 }
 
 function cmp_output()
@@ -36,17 +64,12 @@ function cmp_output()
     cmd=$(printf "%-7s $YDIFF %-24s < %-30s " $PYTHON "$ydiff_opt" "$input")
     printf "$cmd"
 
-    if [[ $TRAVIS_OS_NAME == windows ]]; then
-        cmp_tool="diff --strip-trailing-cr -q"
-    else
-        cmp_tool="cmp -s"  # --silence does not work on Alpine
-    fi
-
-    if eval $cmd 2>/dev/null | eval $cmp_tool $expected_out - > /dev/null; then
+    eval $cmd 1>"$TEST_TMP/cmd.out" 2>"$TEST_TMP/cmd.err"
+    if diff --strip-trailing-cr "$expected_out" "$TEST_TMP/cmd.out" >/dev/null; then
         pass
         return 0
     else
-        fail "!= $expected_out"
+        fail "$expected_out"
         return 1
     fi
 }