tests: Retry matching expected output to improve test robustness

This also reduces test time dramatically on faster hardware.

app/test/Makefile

@@ -42,7 +42,7 @@ endif
 THIS_LIB_BASE=$(shell cd ../.. && pwd)
 CCBN=$(shell basename ${CC})
 BUILD_DIR=${THIS_LIB_BASE}/build/${BUILD_SUBDIR}/${CCBN}
-TMP_DIR=${THIS_LIB_BASE}/tmp
+export TMP_DIR=${THIS_LIB_BASE}/tmp
 TEST_DATA_DIR=${THIS_LIB_BASE}/data
 
 SOURCES=echo count count-pull select select-pull sql 2json serialize flatten pretty desc stack 2db 2tsv jq compare overwrite
@@ -76,15 +76,17 @@ ifneq ($(LEAKS),)
   REDIRECT=>${TMP_DIR}/leaks.txt; grep leak ${TMP_DIR}/leaks.txt | grep bytes \# # stop processing at this step
   REDIRECT1=>${TMP_DIR}/leaks.txt; grep leak ${TMP_DIR}/leaks.txt | grep bytes ) \# # stop processing at this step
   REDIRECT2=>${TMP_DIR}/leaks.txt; grep leak ${TMP_DIR}/leaks.txt | grep bytes ) \# # stop processing at this step
-  CMP=\# # don't run this step
+  export CMP=\# # don't run this step
 else
   PREFIX=
   REDIRECT=>
   REDIRECT1=>
   REDIRECT2=-o
-  CMP=cmp
+  export CMP=cmp
 endif
 
+EXPECT=../../scripts/test-retry-capture-cmp.sh
+
 MAKE_BIN=$(notdir ${MAKE})
 
 help:
@@ -628,11 +630,9 @@ test-sheet-3: ${BUILD_DIR}/bin/zsv_sheet${EXE}
 test-sheet-4: ${BUILD_DIR}/bin/zsv_sheet${EXE}
 	@${TEST_INIT}
 	@(tmux new-session -x 80 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
-	sleep 0.5 && \
+	${EXPECT} $@ indexed && \
 	tmux send-keys -t $@ "C-d" "C-d" "C-u" && \
-	tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
-	tmux send-keys -t $@ "q" && \
-	${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL})
+	${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})
 
 test-sheet-5: ${BUILD_DIR}/bin/zsv_sheet${EXE}
 	@${TEST_INIT}
@@ -682,13 +682,10 @@ test-sheet-9: ${BUILD_DIR}/bin/zsv_sheet${EXE}
 	@${TEST_INIT}
 	@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
 	@(tmux new-session -x 80 -y 6 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \
-	sleep 1 && \
+	${EXPECT} $@ indexed && \
 	tmux send-keys -t $@ "G" && \
 	tmux send-keys -t $@ -N 4096 "k" && \
-	sleep 2 && \
-	tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
-	tmux send-keys -t $@ "q" && \
-	${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
+	${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})
 
 test-sheet-10: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.tsv
 	@${TEST_INIT}

app/test/expected/test-sheet-4-indexed.out

@@ -0,0 +1,5 @@
+Row #     Country   City      AccentCit Region    Populatio Latitude  Longitude
+1         ir        sarmaj-e  Sarmaj-e  13                  34.3578   47.5207
+2         ad        aixirival Aixirival 06                  42.466666 1.5
+3         mm        mokho-atw Mokho-atw 09                  18.033333 96.75
+? for help 1

app/test/expected/test-sheet-9-indexed.out

@@ -0,0 +1,6 @@
+Row #               HA1 HA2 HA3          HB1  HB2  HB3       HC1  HC2  HC3
+1                   A1                   B1                  C1
+2                   A2                   B2                  C2
+3                   A3                   B3                  C3
+4                   A4                   B4                  C4
+? for help 1

scripts/test-retry-capture-cmp.sh

@@ -0,0 +1,56 @@
+#!/bin/sh -eu
+
+target=$1
+if [ -z ${2:-} ]; then
+  stage=""
+else
+  stage=-$2
+fi
+capture=${TMP_DIR}/${target}${stage}.out
+expected=$(pwd)/expected/${target}${stage}.out
+matched=0
+
+cleanup() {
+  if (( matched )); then
+    if [ -z $stage ]; then
+      tmux send-keys -t $target "q"
+    fi
+    exit 0
+  fi
+
+  tmux send-keys -t $target "q"
+  echo 'Incorrect output:'
+  cat $capture
+  ${CMP} -s $capture $expected
+  exit 1
+}
+
+trap cleanup INT TERM QUIT
+
+start_time=$(date +%s)
+warned=0
+while true; do
+  tmux capture-pane -t $target -p > $capture
+  current_time=$(date +%s)
+  span=$(( current_time - start_time ))
+
+  if ${CMP} -s $capture $expected; then
+    echo "$target$stage matched in ${span}s"
+    matched=1
+    break
+  fi
+
+  sleep 0.1
+
+  if (( span > 5 )); then
+    echo "Timeout exceeded"
+    break
+  fi
+
+  if (( span > 2 && !warned )); then
+    warned=1
+    echo "Slow test"
+  fi
+done
+
+cleanup