Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Beef up and harden import/export tests #7140

Merged
merged 3 commits into from
Apr 10, 2020
Merged

Beef up and harden import/export tests #7140

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
4319c12
Fixup all original test issues
ribasushi Apr 10, 2020
edc2779
Extra cross-iptb-network testcases
ribasushi Apr 10, 2020
768c287
Extra potential deadlock debugging
ribasushi Apr 10, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 56 additions & 27 deletions test/sharness/t0054-dag-car-import-export.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,52 @@ export -f ipfsi
set -o pipefail

tar -C ../t0054-dag-car-import-export-data/ --strip-components=1 -Jxf ../t0054-dag-car-import-export-data/test_dataset_car_v0.tar.xz
tab=$'\t'

reset_blockstore() {
node=$1
ipfsi $1 pin ls --quiet --type=recursive | ipfsi $1 pin rm &>/dev/null
ipfsi $1 repo gc &>/dev/null

ipfsi $node pin ls --quiet --type=recursive | ipfsi $node pin rm &>/dev/null
ipfsi $node repo gc &>/dev/null

test_expect_success "pinlist empty" '
[ "$( ipfsi $1 pin ls )" = "" ]
[[ -z "$( ipfsi $node pin ls )" ]]
'
test_expect_success "nothing left to gc" '
[ "$( ipfsi $1 repo gc )" = "" ]
[[ -z "$( ipfsi $node repo gc )" ]]
'
}

# hammer with concurrent gc to ensure nothing clashes
do_import() {
node=$1; shift

bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc1_pid=$!
bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc >>gc_out 2>&1; done" & gc2_pid=$!
touch spin.gc
timeout -s QUIT 15 bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc &>>gc_out; done" & gc1_pid=$!
timeout -s QUIT 15 bash -c "while [[ -e spin.gc ]]; do ipfsi $node repo gc &>>gc_out; done" & gc2_pid=$!

ipfsi $node dag import "$@"
timeout -s QUIT 10 bash -c "ipfsi $node dag import $* 2>&1"

rm -f spin.gc || true
sleep 3
kill $gc1_pid $gc2_pid || true
}

run_online_imp_exp_tests() {

reset_blockstore 0
reset_blockstore 1

echo -e "Pinned root\tbafkqaaa\tsuccess" > basic_import_expected
echo -e "Pinned root\tbafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u\tsuccess" >> basic_import_expected
echo -e "Pinned root\tbafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy\tsuccess" >> basic_import_expected
cat > basic_import_expected <<EOE
Pinned root${tab}bafkqaaa${tab}success
Pinned root${tab}bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u${tab}success
Pinned root${tab}bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy${tab}success
EOE

cat >naked_root_import_json_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
EOE


touch spin.gc
test_expect_success "basic import" '
do_import 0 \
../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
Expand All @@ -57,12 +65,39 @@ run_online_imp_exp_tests() {

# FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed
test_expect_failure "concurrent GC did not manage to grab anything and remained silent" '
! [[ -s gc_out ]]
test_cmp /dev/null gc_out
'
test_expect_success "basic import output as expected" '
test_cmp basic_import_expected basic_import_actual
'

test_expect_success "basic fetch+export 1" '
ipfsi 1 dag export bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy > reexported_testnet_128.car
'
test_expect_success "export of shuffled testnet export identical to canonical original" '
test_cmp reexported_testnet_128.car ../t0054-dag-car-import-export-data/lotus_testnet_export_128.car
'

test_expect_success "basic fetch+export 2" '
ipfsi 1 dag export bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u > reexported_devnet_genesis.car
'
test_expect_success "export of shuffled devnet export identical to canonical original" '
test_cmp reexported_devnet_genesis.car ../t0054-dag-car-import-export-data/lotus_devnet_genesis.car
'

test_expect_success "pinlist on node1 still empty" '
[[ -z "$( ipfsi 1 pin ls )" ]]
'

test_expect_success "import/pin naked roots only, relying on local blockstore having all the data" '
ipfsi 1 dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
| sort > naked_import_result_json_actual
'

test_expect_success "naked import output as expected" '
test_cmp naked_root_import_json_expected naked_import_result_json_actual
'

reset_blockstore 0
reset_blockstore 1

Expand All @@ -81,7 +116,6 @@ run_online_imp_exp_tests() {
bash -c "sleep 60; kill $cat1_pid $cat2_pid 2>/dev/null" &
' &

touch spin.gc
test_expect_success "fifo import" '
do_import 0 \
pipe_testnet \
Expand All @@ -91,7 +125,7 @@ run_online_imp_exp_tests() {
'
# FIXME - positive-test the lack of output when https://github.com/ipfs/go-ipfs/issues/7121 is addressed
test_expect_failure "concurrent GC did not manage to grab anything and remained silent" '
! [[ -s gc_out ]]
test_cmp /dev/null gc_out
'

test_expect_success "fifo-import output as expected" '
Expand Down Expand Up @@ -134,16 +168,16 @@ test_expect_success "correct error" '
'


cat >multiroot_import_expected <<EOE
cat >multiroot_import_json_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceb55n7uxyfaelplulk3ev2xz7gnq6crncf3ahnvu46hqqmpucizcw"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzacebedrc4n2ac6cqdkhs7lmj5e4xiif3gu7nmoborihajxn3fav3vdq"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzacede2hsme6hparlbr4g2x6pylj43olp4uihwjq3plqdjyrdhrv7cp4"},"PinErrorMsg":""}}
EOE
test_expect_success "multiroot import works" '
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_actual
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/lotus_testnet_export_256_multiroot.car | sort > multiroot_import_json_actual
'
test_expect_success "multiroot import expected output" '
test_cmp multiroot_import_expected multiroot_import_actual
test_cmp multiroot_import_json_expected multiroot_import_json_actual
'


Expand All @@ -154,21 +188,16 @@ test_expect_success "pin-less import works" '
> no-pin_import_actual
'
test_expect_success "expected silence on --pin-roots=false" '
! [[ -s no-pin_import_actual ]]
test_cmp /dev/null no-pin_import_actual
'


cat >naked_root_import_expected <<EOE
{"Root":{"Cid":{"/":"bafy2bzaceaxm23epjsmh75yvzcecsrbavlmkcxnva66bkdebdcnyw3bjrc74u"},"PinErrorMsg":""}}
{"Root":{"Cid":{"/":"bafy2bzaced4ueelaegfs5fqu4tzsh6ywbbpfk3cxppupmxfdhbpbhzawfw5oy"},"PinErrorMsg":""}}
EOE
test_expect_success "naked root import works" '
ipfs dag import --enc=json ../t0054-dag-car-import-export-data/combined_naked_roots_genesis_and_128.car \
| sort > naked_root_import_actual
| sort > naked_root_import_json_actual
'
test_expect_success "naked root import expected output" '
test_cmp naked_root_import_expected naked_root_import_actual
test_cmp naked_root_import_json_expected naked_root_import_json_actual
'


test_done