Expanded test suite

* Lang now verifies errors and parse output

* Some new miscellaneous tests

* Easy way to update the tests

* Document workflow in manual

* Use `!` not `~` as separater char for sed

  It is confusing to use `~` when we are talking about paths and home
  directories!

* Test test suite itself (`test/lang-test/infra.sh`)

Additionally, run shellcheck on `tests/lang.sh` to help ensure it is
correct, now that is is more complex.

Co-authored-by: Robert Hensing <roberth@users.noreply.github.com>
Co-authored-by: Valentin Gagarin <valentin.gagarin@tweag.io>

tests/lang.sh

@@ -1,5 +1,17 @@
 source common.sh
 
+set -o pipefail
+
+source lang/framework.sh
+
+# specialize function a bit
+function diffAndAccept() {
+    local -r testName="$1"
+    local -r got="lang/$testName.$2"
+    local -r expected="lang/$testName.$3"
+    diffAndAcceptInner "$testName" "$got" "$expected"
+}
+
 export TEST_VAR=foo # for eval-okay-getenv.nix
 export NIX_REMOTE=dummy://
 export NIX_STORE_DIR=/nix/store
@@ -20,63 +32,114 @@ nix-instantiate --eval -E 'let x = { repeating = x; tracing = builtins.trace x t
 
 set +x
 
-fail=0
+badDiff=0
+badExitCode=0
 
 for i in lang/parse-fail-*.nix; do
     echo "parsing $i (should fail)";
-    i=$(basename $i .nix)
-    if ! expect 1 nix-instantiate --parse - < lang/$i.nix; then
+    i=$(basename "$i" .nix)
+    if expectStderr 1 nix-instantiate --parse - < "lang/$i.nix" > "lang/$i.err"
+    then
+        diffAndAccept "$i" err err.exp
+    else
         echo "FAIL: $i shouldn't parse"
-        fail=1
+        badExitCode=1
     fi
 done
 
 for i in lang/parse-okay-*.nix; do
     echo "parsing $i (should succeed)";
-    i=$(basename $i .nix)
-    if ! expect 0 nix-instantiate --parse - < lang/$i.nix > lang/$i.out; then
+    i=$(basename "$i" .nix)
+    if
+        expect 0 nix-instantiate --parse - < "lang/$i.nix" \
+            1> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.out") \
+            2> >(sed "s!$(pwd)!/pwd!g" > "lang/$i.err")
+    then
+        diffAndAccept "$i" out exp
+        diffAndAccept "$i" err err.exp
+    else
         echo "FAIL: $i should parse"
-        fail=1
+        badExitCode=1
     fi
 done
 
 for i in lang/eval-fail-*.nix; do
     echo "evaluating $i (should fail)";
-    i=$(basename $i .nix)
-    if ! expect 1 nix-instantiate --eval lang/$i.nix; then
+    i=$(basename "$i" .nix)
+    if
+        expectStderr 1 nix-instantiate --show-trace "lang/$i.nix" \
+            | sed "s!$(pwd)!/pwd!g" > "lang/$i.err"
+    then
+        diffAndAccept "$i" err err.exp
+    else
         echo "FAIL: $i shouldn't evaluate"
-        fail=1
+        badExitCode=1
     fi
 done
 
 for i in lang/eval-okay-*.nix; do
     echo "evaluating $i (should succeed)";
-    i=$(basename $i .nix)
+    i=$(basename "$i" .nix)
 
-    if test -e lang/$i.exp; then
-        flags=
-        if test -e lang/$i.flags; then
-            flags=$(cat lang/$i.flags)
-        fi
-        if ! expect 0 env NIX_PATH=lang/dir3:lang/dir4 HOME=/fake-home nix-instantiate $flags --eval --strict lang/$i.nix > lang/$i.out; then
+    if test -e "lang/$i.exp.xml"; then
+        if expect 0 nix-instantiate --eval --xml --no-location --strict \
+                "lang/$i.nix" > "lang/$i.out.xml"
+        then
+            diffAndAccept "$i" out.xml exp.xml
+        else
             echo "FAIL: $i should evaluate"
-            fail=1
-        elif ! diff <(< lang/$i.out sed -e "s|$(pwd)|/pwd|g") lang/$i.exp; then
-            echo "FAIL: evaluation result of $i not as expected"
-            fail=1
+            badExitCode=1
+        fi
+    elif test ! -e "lang/$i.exp-disabled"; then
+        declare -a flags=()
+        if test -e "lang/$i.flags"; then
+            read -r -a flags < "lang/$i.flags"
         fi
-    fi
 
-    if test -e lang/$i.exp.xml; then
-        if ! expect 0 nix-instantiate --eval --xml --no-location --strict \
-                lang/$i.nix > lang/$i.out.xml; then
+        if
+            expect 0 env \
+                NIX_PATH=lang/dir3:lang/dir4 \
+                HOME=/fake-home \
+                nix-instantiate "${flags[@]}" --eval --strict "lang/$i.nix" \
+                1> "lang/$i.out" \
+                2> "lang/$i.err"
+        then
+            sed -i "s!$(pwd)!/pwd!g" "lang/$i.out" "lang/$i.err"
+            diffAndAccept "$i" out exp
+            diffAndAccept "$i" err err.exp
+        else
             echo "FAIL: $i should evaluate"
-            fail=1
-        elif ! cmp -s lang/$i.out.xml lang/$i.exp.xml; then
-            echo "FAIL: XML evaluation result of $i not as expected"
-            fail=1
+            badExitCode=1
         fi
     fi
 done
 
-exit $fail
+if test -n "${_NIX_TEST_ACCEPT-}"; then
+    if (( "$badDiff" )); then
+        echo 'Output did mot match, but accepted output as the persisted expected output.'
+        echo 'That means the next time the tests are run, they should pass.'
+    else
+        echo 'NOTE: Environment variable _NIX_TEST_ACCEPT is defined,'
+        echo 'indicating the unexpected output should be accepted as the expected output going forward,'
+        echo 'but no tests had unexpected output so there was no expected output to update.'
+    fi
+    if (( "$badExitCode" )); then
+        exit "$badExitCode"
+    else
+        skipTest "regenerating golden masters"
+    fi
+else
+    if (( "$badDiff" )); then
+        echo ''
+        echo 'You can rerun this test with:'
+        echo ''
+        echo '    _NIX_TEST_ACCEPT=1 make tests/lang.sh.test'
+        echo ''
+        echo 'to regenerate the files containing the expected output,'
+        echo 'and then view the git diff to decide whether a change is'
+        echo 'good/intentional or bad/unintentional.'
+        echo 'If the diff contains arbitrary or impure information,'
+        echo 'please improve the normalization that the test applies to the output.'
+    fi
+    exit $(( "$badExitCode" + "$badDiff" ))
+fi