Skip to content

Commit

Permalink
Merge pull request #7140 from ipfs/chore/harden_impexp_test
Browse files Browse the repository at this point in the history
Beef up and harden import/export tests
  • Loading branch information
Stebalien authored Apr 10, 2020
2 parents 0765c40 + 768c287 commit 0c619ef
Showing 1 changed file with 56 additions and 27 deletions.
83 changes: 56 additions & 27 deletions test/sharness/t0054-dag-car-import-export.sh
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,52 @@ export -f ipfsi
set -o pipefail

tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz
tab=$'\t'

reset_blockstore() {
node=$1
ipfsi $1 pin ls --quiet --type=recursive | ipfsi $1 pin rm &>/dev/null
ipfsi $1 repo gc &>/dev/null

ipfsi $node pin ls --quiet --type=recursive | ipfsi $node pin rm &>/dev/null
ipfsi $node repo gc &>/dev/null

test_expect_success "pinlist empty" '
[ "$( ipfsi $1 pin ls )" = "" ]
[[ -z "$( ipfsi $node pin ls )" ]]
'
test_expect_success "nothing left to gc" '
[ "$( ipfsi $1 repo gc )" = "" ]
[[ -z "$( ipfsi $node repo gc )" ]]
'
}

# hammer with concurrent gc to ensure nothing clashes
do_import() {
node=$1; shift

bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc1_pid=$!
bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc2_pid=$!
touch spin.gc
timeout -s QUIT 15 bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc &>>gc_out; done" & gc1_pid=$!
timeout -s QUIT 15 bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc &>>gc_out; done" & gc2_pid=$!

ipfsi $node dag import "$@"
timeout -s QUIT 10 bash -c "ipfsi $node dag import $* 2>&1"

rm -f spin.gc || true
sleep 3
kill $gc1_pid $gc2_pid || true
}

run_online_imp_exp_tests() {

reset_blockstore 0
reset_blockstore 1

echo -e "Pinned root\tbafkqaaa\tsuccess" > basic_import_expected
echo -e "Pinned root\tbafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u\tsuccess" >> basic_import_expected
echo -e "Pinned root\tbafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy\tsuccess" >> basic_import_expected
cat > basic_import_expected <<EOE
Pinned root${tab}bafkqaaa${tab}success
Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
EOE

cat >naked_root_import_json_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
EOE


touch spin.gc
test_expect_success "basic import" '
do_import 0 \
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
Expand All @@ -57,12 +65,39 @@ run_online_imp_exp_tests() {

# FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed
test_expect_failure "concurrent GC did not manage to grab anything and remained silent" '
! [[ -s gc_out ]]
test_cmp /dev/null gc_out
'
test_expect_success "basic import output as expected" '
test_cmp basic_import_expected basic_import_actual
'

test_expect_success "basic fetch+export 1" '
ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
'
test_expect_success "export of shuffled testnet export identical to canonical original" '
test_cmp reexported_testnet_128.car ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car
'

test_expect_success "basic fetch+export 2" '
ipfsi 1 dag export bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u > reexported_devnet_genesis.car
'
test_expect_success "export of shuffled devnet export identical to canonical original" '
test_cmp reexported_devnet_genesis.car ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car
'

test_expect_success "pinlist on node1 still empty" '
[[ -z "$( ipfsi 1 pin ls )" ]]
'

test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
| sort > naked_import_result_json_actual
'

test_expect_success "naked import output as expected" '
test_cmp naked_root_import_json_expected naked_import_result_json_actual
'

reset_blockstore 0
reset_blockstore 1

Expand All @@ -81,7 +116,6 @@ run_online_imp_exp_tests() {
bash -c "sleep 60; kill $cat1_pid $cat2_pid 2>/dev/null" &
' &

touch spin.gc
test_expect_success "fifo import" '
do_import 0 \
pipe_testnet \
Expand All @@ -91,7 +125,7 @@ run_online_imp_exp_tests() {
'
# FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed
test_expect_failure "concurrent GC did not manage to grab anything and remained silent" '
! [[ -s gc_out ]]
test_cmp /dev/null gc_out
'

test_expect_success "fifo-import output as expected" '
Expand Down Expand Up @@ -134,16 +168,16 @@ test_expect_success "correct error" '
'


cat >multiroot_import_expected <<EOE
cat >multiroot_import_json_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
EOE
test_expect_success "multiroot import works" '
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_actual
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_json_actual
'
test_expect_success "multiroot import expected output" '
test_cmp multiroot_import_expected multiroot_import_actual
test_cmp multiroot_import_json_expected multiroot_import_json_actual
'


Expand All @@ -154,21 +188,16 @@ test_expect_success "pin-less import works" '
> no-pin_import_actual
'
test_expect_success "expected silence on --pin-roots=false" '
! [[ -s no-pin_import_actual ]]
test_cmp /dev/null no-pin_import_actual
'


cat >naked_root_import_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
EOE
test_expect_success "naked root import works" '
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
| sort > naked_root_import_actual
| sort > naked_root_import_json_actual
'
test_expect_success "naked root import expected output" '
test_cmp naked_root_import_expected naked_root_import_actual
test_cmp naked_root_import_json_expected naked_root_import_json_actual
'


test_done

0 comments on commit 0c619ef

Please sign in to comment.