Skip to content

Commit

Permalink
tests: Retry matching expected output to improve test robustness
Browse files Browse the repository at this point in the history
This also reduces test time dramatically on faster hardware.
  • Loading branch information
richiejp committed Dec 6, 2024
1 parent aa54294 commit 5e4994a
Show file tree
Hide file tree
Showing 17 changed files with 294 additions and 81 deletions.
132 changes: 51 additions & 81 deletions app/test/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ endif
THIS_LIB_BASE=$(shell cd ../.. && pwd)
CCBN=$(shell basename ${CC})
BUILD_DIR=${THIS_LIB_BASE}/build/${BUILD_SUBDIR}/${CCBN}
TMP_DIR=${THIS_LIB_BASE}/tmp
export TMP_DIR=${THIS_LIB_BASE}/tmp
TEST_DATA_DIR=${THIS_LIB_BASE}/data

SOURCES=echo count count-pull select select-pull sql 2json serialize flatten pretty desc stack 2db 2tsv jq compare overwrite
Expand Down Expand Up @@ -76,17 +76,22 @@ ifneq ($(LEAKS),)
REDIRECT=>${TMP_DIR}/leaks.txt; grep leak ${TMP_DIR}/leaks.txt | grep bytes \# # stop processing at this step
REDIRECT1=>${TMP_DIR}/leaks.txt; grep leak ${TMP_DIR}/leaks.txt | grep bytes ) \# # stop processing at this step
REDIRECT2=>${TMP_DIR}/leaks.txt; grep leak ${TMP_DIR}/leaks.txt | grep bytes ) \# # stop processing at this step
CMP=\# # don't run this step
export CMP=\# # don't run this step
else
PREFIX=
REDIRECT=>
REDIRECT1=>
REDIRECT2=-o
CMP=cmp
export CMP=cmp
endif

EXPECT=../../scripts/test-expect.sh

MAKE_BIN=$(notdir ${MAKE})

DATE_TIME:=$(shell date +%F-%H-%M-%S)
export TIMINGS_CSV:=${TMP_DIR}/timings-${DATE_TIME}.csv

help:
@echo "To run all tests: ${MAKE_BIN} test [LEAKS=1]"
@echo "To run individual test: ${MAKE_BIN} test-xxx"
Expand All @@ -105,6 +110,9 @@ test: ${TESTS}
test-paste:
@echo "TO DO: test paste"

${TIMINGS_CSV}:
@echo -n "Test, Stage, Time" > ${TIMINGS_CSV}

.SECONDARY: worldcitiespop_mil.csv

.PHONY: help test test-% test-stack clean
Expand Down Expand Up @@ -606,149 +614,111 @@ test-sheet-1: ${BUILD_DIR}/bin/zsv_sheet${EXE}
tmux -v send-keys -t $@ "q" "exit" && \
${CMP} ${TMP_DIR}/[email protected] expected/[email protected] && ${TEST_PASS} || (for x in `ls tmux-*.log` ; do echo "Log $$x:" && cat $$x ; done && ${TEST_FAIL}))

test-sheet-2: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-2: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@(tmux new-session -x 80 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "C-d" && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL})
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-3: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-3: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@(tmux new-session -x 80 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "f" "sarmaj" Enter && \
sleep 2 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-4: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-4: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@(tmux new-session -x 80 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "C-d" "C-d" "C-u" && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL})
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-5: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-5: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 160 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || ${TEST_FAIL})
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-6: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-6: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 50 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "G" "C-u" "C-u" "C-u" && \
sleep 0.5 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-7: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-7: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 5 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \
sleep 1 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "G" "g" "g" "C-u" "/" "1234" "Enter" && \
sleep 0.5 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-8: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-8: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 160 -y 5 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "f" "e" "Enter" && \
sleep 1 && \
${EXPECT} $@ filtered && \
tmux send-keys -t $@ "G" "C-u" "k" && \
sleep 0.5 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-9: ${BUILD_DIR}/bin/zsv_sheet${EXE}
test-sheet-9: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 6 -d -s $@ "${PREFIX} $< -d 3 ${TEST_DATA_DIR}/test/mixed-line-endings.csv" && \
sleep 1 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "G" && \
tmux send-keys -t $@ -N 4096 "k" && \
sleep 2 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-10: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.tsv
test-sheet-10: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.tsv ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 6 -d -s $@ "${PREFIX} $< -t worldcitiespop_mil.tsv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "f" "e" "Enter" && \
sleep 1 && \
${EXPECT} $@ filtered && \
tmux send-keys -t $@ "G" && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-11: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv
test-sheet-11: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 50 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 1 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ -N 11 "C-d" && \
sleep 1 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-12: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv
test-sheet-12: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 6 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
${EXPECT} $@ indexed && \
tmux send-keys -t $@ "f" "el" "Enter" && \
sleep 0.5 && \
${EXPECT} $@ filtered-el && \
tmux send-keys -t $@ "f" "al" "Enter" && \
sleep 1 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-13: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv
test-sheet-13: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 25 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
tmux send-keys -t $@ "?" "/" "f" && \
sleep 0.5 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-14: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv
test-sheet-14: ${BUILD_DIR}/bin/zsv_sheet${EXE} worldcitiespop_mil.csv ${TIMINGS_CSV}
@${TEST_INIT}
@echo 'set-option default-terminal "${TMUX_TERM}"' > ~/.tmux.conf
@(tmux new-session -x 80 -y 25 -d -s $@ "${PREFIX} $< worldcitiespop_mil.csv" && \
sleep 0.5 && \
tmux send-keys -t $@ "?" && \
sleep 0.5 && \
${EXPECT} $@ help && \
tmux send-keys -t $@ "Escape" && \
sleep 1 && \
tmux capture-pane -t $@ -p ${REDIRECT1} ${TMP_DIR}/$@.out && \
tmux send-keys -t $@ "q" && \
${CMP} ${TMP_DIR}/$@.out expected/$@.out && ${TEST_PASS} || (echo 'Incorrect output:' && cat ${TMP_DIR}/$@.out && ${TEST_FAIL}))
${EXPECT} $@ && ${TEST_PASS} || ${TEST_FAIL})

test-sheet-prop-cmd-opt: ${BUILD_DIR}/bin/zsv_sheet${EXE} ${BUILD_DIR}/bin/zsv_prop${EXE}
@echo "TO DO: test-sheet-prop-cmd-opt"
Expand Down
6 changes: 6 additions & 0 deletions app/test/expected/test-sheet-10-filtered.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
4 id selingon Selingon 17 -8.8374 116.4914
5 ir berimvand Berimvand 13 34.2953 47.1096
8 ad andorra-v Andorra-V 07 42.5 1.5166667
(493039 filtered rows) 1
6 changes: 6 additions & 0 deletions app/test/expected/test-sheet-10-indexed.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
2 ad aixirival Aixirival 06 42.466666 1.5
3 mm mokho-atw Mokho-atw 09 18.033333 96.75
4 id selingon Selingon 17 -8.8374 116.4914
? for help 1
50 changes: 50 additions & 0 deletions app/test/expected/test-sheet-11-indexed.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
2 ad aixirival Aixirival 06 42.466666 1.5
3 mm mokho-atw Mokho-atw 09 18.033333 96.75
4 id selingon Selingon 17 -8.8374 116.4914
5 ir berimvand Berimvand 13 34.2953 47.1096
6 pl chomiaza Chomiaza 73 52.7508 17.841793
7 mg itona Itona 02 -23.86666 47.216666
8 ad andorra-v Andorra-V 07 42.5 1.5166667
9 mx la tortug La Tortug 25 25.75 -108.3333
10 us asaph Asaph PA 41.770833 -77.40527
11 ad andorre-v Andorre-V 07 42.5 1.5166667
12 ru dolmatova Dolmatova 71 57.436791 63.279522
13 ro escu Escu 13 47.133333 23.533333
14 us la presa La Presa CA 35119 32.708055 -116.9963
15 pk makam khu Makam Khu 04 33.650863 72.551536
16 ad aubinya Aubinyà 06 42.45 1.5
17 mm kiosong Kiosöng 11 22.583333 97.05
18 tr donencay Dönençay 28 40.266667 38.583333
19 it roncaglia Roncaglia 05 45.05 9.8
20 ml kourmouss Kourmouss 04 14.75 -5.033333
21 id lamogo Lamogo 38 -4.3945 119.9028
22 ad casas vil Casas Vil 03 42.533333 1.5666667
23 ru otdeleniy Otdeleniy 86 51.726473 39.714345
24 ad certes Certés 06 42.466666 1.5
25 us mound Mound LA 32.339166 -91.02388
26 ad el pui El Pui 04 42.55 1.5166667
27 ad els bons Els Bons 03 42.533333 1.5833333
28 ru roshina Roshina 46 54.641096 43.547234
29 ad els plans Els Plans 02 42.583333 1.6333333
30 ru zelenaya Zelënaya 55 52.616667 54.35
31 mx tejalpa Tejalpa 21 18.35 -98.36666
32 ad el vilar El Vilar 02 42.566666 1.6
33 td narweyt Narweyt 02 15.133333 21.95
34 id babakanme Babakanme 30 -7.5209 108.0738
35 mx rancho sa Rancho Sa 10 25.4 -105.1166
36 ad ercz Ercz 04 42.566666 1.5
37 ad erez Erez 04 42.566666 1.5
38 id cigaleuh Cigaleuh 30 -6.415833 107.61027
39 jp ushitaki Ushitaki 03 41.283333 140.8
40 ml nkalamedo Nkalaméd 07 12.136944 -7.744722
41 la ban huai Ban Huai 13 17.875556 101.21944
42 ad fontaneda Fontaneda 06 42.45 1.4666667
43 kr chinbol Chinbol 13 37.7376 127.2266
44 ad juverri Juverri 06 42.433333 1.5
45 ua arginchik Arginchik 11 45.237894 34.648083
46 il ezuz Ezuz 01 30.792445 34.472671
47 ad l'aldosa L'Aldosa 02 42.583333 1.6333333
48 kh phumi tra Phumi Tra 06 10.866666 104.68333
? for help 1
6 changes: 6 additions & 0 deletions app/test/expected/test-sheet-12-filtered-el.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
4 id selingon Selingon 17 -8.8374 116.4914
23 ru otdeleniy Otdeleniy 86 51.726473 39.714345
26 ad el pui El Pui 04 42.55 1.5166667
27 ad els bons Els Bons 03 42.533333 1.5833333
(59456 filtered rows) 4
6 changes: 6 additions & 0 deletions app/test/expected/test-sheet-12-indexed.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
2 ad aixirival Aixirival 06 42.466666 1.5
3 mm mokho-atw Mokho-atw 09 18.033333 96.75
4 id selingon Selingon 17 -8.8374 116.4914
? for help 1
25 changes: 25 additions & 0 deletions app/test/expected/test-sheet-14-help.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
Key(s) Action Description
q quit Exit the application
<esc> escape Leave the current view or
^ first Jump to the first column
$ last Jump to the last column
<shift><left> first Jump to the first column
<shift><right> last Jump to the last column
k up Move up one row
j down Move down one row
h left Move left one column
l right Move right one column
<up> up Move up one row
<down> down Move down one row
<left> left Move left one column
<right> right Move right one column
<ctrl>d pagedown Move down one page
<ctrl>u pageup Move up one page
<page up> pagedown Move down one page
<page down> pageup Move up one page
g g top Jump to the first row
G bottom Jump to the last row
/ find Set a search term and jum
n next Jump to the next search r
e open Open a another CSV file
<esc> to exit help Key(s)
5 changes: 5 additions & 0 deletions app/test/expected/test-sheet-2-indexed.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
2 ad aixirival Aixirival 06 42.466666 1.5
3 mm mokho-atw Mokho-atw 09 18.033333 96.75
? for help 1
5 changes: 5 additions & 0 deletions app/test/expected/test-sheet-3-indexed.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
2 ad aixirival Aixirival 06 42.466666 1.5
3 mm mokho-atw Mokho-atw 09 18.033333 96.75
? for help 1
5 changes: 5 additions & 0 deletions app/test/expected/test-sheet-4-indexed.out
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
Row # Country City AccentCit Region Populatio Latitude Longitude
1 ir sarmaj-e Sarmaj-e 13 34.3578 47.5207
2 ad aixirival Aixirival 06 42.466666 1.5
3 mm mokho-atw Mokho-atw 09 18.033333 96.75
? for help 1
Loading

0 comments on commit 5e4994a

Please sign in to comment.