diff --git a/wnfs-bench/hamt.rs b/wnfs-bench/hamt.rs index af82a6b6..27f35e58 100644 --- a/wnfs-bench/hamt.rs +++ b/wnfs-bench/hamt.rs @@ -19,7 +19,7 @@ fn node_set(c: &mut Criterion) { let mut store = MemoryBlockStore::default(); let operations = operations(any::<[u8; 32]>(), any::(), 1_000_000).sample(&mut runner); let node = - &async_std::task::block_on(async { node_from_operations(operations, &mut store).await }) + &async_std::task::block_on(async { node_from_operations(&operations, &mut store).await }) .expect("Couldn't setup HAMT node from operations"); let store = Arc::new(store); @@ -54,7 +54,7 @@ fn node_set_consecutive(c: &mut Criterion) { let operations = operations(any::<[u8; 32]>(), any::(), 1000).sample(&mut runner); let node = async_std::task::block_on(async { - node_from_operations(operations, &mut store).await + node_from_operations(&operations, &mut store).await }) .expect("Couldn't setup HAMT node from operations"); @@ -76,16 +76,14 @@ fn node_load_get(c: &mut Criterion) { let cid = async_std::task::block_on(async { let mut node = Rc::new(>::default()); for i in 0..50 { - node = node.set(i.to_string(), i, &mut store).await.unwrap(); + node = node.set(i.to_string(), i, &store).await.unwrap(); } let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store) .await .unwrap(); - let cid = store.put_serializable(&encoded_hamt).await.unwrap(); - - cid + store.put_serializable(&encoded_hamt).await.unwrap() }); c.bench_function("node load and get", |b| { @@ -110,16 +108,14 @@ fn node_load_remove(c: &mut Criterion) { let cid = async_std::task::block_on(async { let mut node = Rc::new(>::default()); for i in 0..50 { - node = node.set(i.to_string(), i, &mut store).await.unwrap(); + node = node.set(i.to_string(), i, &store).await.unwrap(); } let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store) .await .unwrap(); - let cid = store.put_serializable(&encoded_hamt).await.unwrap(); - - cid + store.put_serializable(&encoded_hamt).await.unwrap() }); c.bench_function("node load and remove", |b| { @@ -142,7 +138,7 @@ fn hamt_load_decode(c: &mut Criterion) { let (cid, bytes) = async_std::task::block_on(async { let mut node = Rc::new(>::default()); for i in 0..50 { - node = node.set(i.to_string(), i, &mut store).await.unwrap(); + node = node.set(i.to_string(), i, &store).await.unwrap(); } let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store) @@ -176,7 +172,7 @@ fn hamt_set_encode(c: &mut Criterion) { }, |(mut store, mut node)| async move { for i in 0..50 { - node = node.set(i.to_string(), i, &mut store).await.unwrap(); + node = node.set(i.to_string(), i, &store).await.unwrap(); } let hamt = Hamt::with_root(node); diff --git a/wnfs-bench/namefilter.rs b/wnfs-bench/namefilter.rs index 7f97de25..df80e4b8 100644 --- a/wnfs-bench/namefilter.rs +++ b/wnfs-bench/namefilter.rs @@ -16,8 +16,9 @@ fn add(c: &mut Criterion) { }, |(mut namefilter, elements)| { for element in elements { - black_box(namefilter.add(&element)); + namefilter.add(&element); } + black_box(&namefilter); }, BatchSize::SmallInput, ) @@ -54,7 +55,8 @@ fn saturate(c: &mut Criterion) { namefilter }, |mut namefilter| { - black_box(namefilter.saturate()); + namefilter.saturate(); + black_box(&namefilter); }, BatchSize::SmallInput, ) diff --git a/wnfs/Cargo.toml b/wnfs/Cargo.toml index 8d9e8732..6f0b29d4 100644 --- a/wnfs/Cargo.toml +++ b/wnfs/Cargo.toml @@ -26,13 +26,14 @@ async-stream = "0.3" async-trait = "0.1" bitvec = { version = "1.0", features = ["serde"] } chrono = { version = "0.4", default-features = false, features = ["clock", "std"] } +either = "1.8" futures = "0.3" futures-util = "0.3" -hashbrown = "0.13" -lazy_static = "1.4" libipld = { version = "0.15", features = ["dag-cbor", "derive", "serde-codec"] } log = "0.4" +once_cell = "1.16" proptest = { version = "1.0", optional = true } +rand = { version = "0.8", optional = true } rand_core = "0.6" semver = { version = "1.0", features = ["serde"] } serde = { version = "1.0", features = ["rc"] } @@ -55,4 +56,4 @@ path = "src/lib.rs" [features] default = [] wasm = [] -test_strategies = ["proptest"] +test_strategies = ["proptest", "rand"] diff --git a/wnfs/proptest-regressions/private/hamt/diff/key_value.txt b/wnfs/proptest-regressions/private/hamt/diff/key_value.txt new file mode 100644 index 00000000..f9910708 --- /dev/null +++ b/wnfs/proptest-regressions/private/hamt/diff/key_value.txt @@ -0,0 +1,10 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 5e555d7196fc6adeea0338b071168600760e42f646ab23a1718d3e45d167b65c # shrinks to input = _DiffCorrespondenceArgs { ops: Operations([Remove("r"), Remove("0u"), Insert("aqbj0y", 8759736365109863483), Insert("4s", 10044956238670467519), Insert("dnd", 6060135432809277420), Remove("pb8b"), Remove("y"), Insert("baihhn45", 11463665909768295174), Insert("sq7", 4811876933643406092), Insert("7p2dpe8q", 12533010396669215881), Remove("to"), Remove("5"), Remove("348d"), Insert("4usi", 6747423100670732089), Remove("1xz"), Remove("39"), Remove("jiz917"), Remove("67b0"), Remove("u"), Insert("44a", 7870292772246520017), Remove("i5"), Remove("cd56zlx"), Insert("i74", 6534526574668539170), Insert("punb", 8473891721762386415), Insert("0", 10780825455149569896), Remove("3w6mz"), Insert("s0eguz", 11046279096949854934), Insert("3kq", 2784467905212969289), Insert("wb9x", 4784810419246373260), Insert("6e53", 8282203486355092676), Insert("hywri", 5137386236814319476), Remove("c46"), Insert("j20vtq1z", 1112325979113224409), Remove("69218i6"), Remove("9bc"), Remove("8u4zy6"), Insert("m", 13451127568675789321), Insert("4zaujs2", 4047325072326692822), Insert("gtv89d7", 7244565589581245304), Insert("a3d", 6591904120187804917), Remove("630mh7c"), Insert("j7", 13930972651537777602), Insert("unvs", 14396077056944091654), Insert("7v9", 10830219039014455780), Insert("wo4", 17465392163036022156), Remove("6"), Insert("p435", 4491338255758162835), Remove("10b923n1"), Insert("1i", 10572089488915295653), Remove("f2ok"), Remove("reohep"), Remove("6dyfad"), Insert("u275bp", 14719964432689961856), Insert("84uywc5", 3034869085899589692), Remove("4lqyyn"), Remove("n1x"), Insert("2", 4103134472614339993), Insert("n8t", 9755734175390624725), Insert("ux6000j", 9700209894676242316), Remove("1127dc6t"), Insert("22t0r8", 6796782628145486429), Insert("hy04", 2403701566790270018), Remove("nq7zr74r"), Insert("ylpbd83f", 15368764264547351299), Insert("87rv", 12252309813607948940), Remove("5"), Insert("4", 12982583174298221718), Insert("0puw", 5013753213686266093), Remove("l588t"), Remove("i"), Insert("7chy6ni", 972220594862718031), Insert("33", 15229741910201849945), Insert("r1o2", 5544133324611432971), Insert("hf6p", 9651098830073773988), Remove("m0q"), Insert("py4", 5643875776673896126), Remove("ys75z"), Remove("y0p"), Insert("o82", 5637522023550207602), Remove("l"), Remove("ck75w6p"), Insert("4e3", 9658913443678631064), Insert("9v", 9423557612194703216), Remove("128fr69o"), Insert("9nq03cjc", 5195524287983201984), Remove("s4l881p8"), Insert("u", 7931681966693414839)]) } +cc 783c9bd6e2cac79e26900fa17c9e9f8bb9f2318643070afff6392ad0f2cc59cd # shrinks to input = _AddRemoveFlipArgs { ops: Operations([Insert("l", 8190415290856170167), Insert("rq", 7802272870926789370), Insert("3ax", 11452151341966869560), Remove("90spto"), Remove("50"), Remove("jy"), Insert("93c9x7n", 17533421366072020711), Insert("a12ng10v", 7160346824398479785), Remove("10apn"), Remove("nz2712t"), Insert("d650a", 15596522953897214152), Insert("i3f7fhti", 293876842522201024), Insert("f1k6", 10772800145220521215), Insert("l64e", 808564228799298117), Remove("hng"), Remove("vu862d"), Remove("b2ms7qt"), Remove("2vjy3"), Remove("g3l2c19"), Remove("46"), Remove("jtra3754"), Remove("214fv"), Remove("8047mr5"), Insert("j1xt", 17455986792880115757), Insert("x", 10840329018809920586), Remove("1n9"), Remove("1lf18a"), Remove("t3zhz"), Remove("r"), Remove("qnz"), Remove("rml"), Remove("277o21o1"), Remove("l5tmj1"), Insert("8nof47cp", 12803114712855644898), Insert("03wm8", 10674708628263752206), Insert("2n", 12647826351949495019), Insert("09uci", 911344669669063258), Insert("ch9r", 4122732132649509988), Remove("2560"), Remove("wpe3p"), Remove("kr6p"), Insert("tnx5", 15780716971213936273), Remove("995i6ho3"), Remove("6z9m"), Insert("j", 9310781360903656187), Insert("903", 17962523713258320062), Insert("20bu", 39644780053866288), Remove("3gww3l"), Insert("1dwu", 4906399812115179669), Remove("s"), Insert("3ab", 2430071676156752155), Insert("8htw6v4o", 15798029957745473163), Remove("762u85b9"), Insert("w305", 10940860920108359882), Remove("f"), Insert("p2k4193p", 7417383211714364929), Insert("5m828", 10189603807998671317), Insert("14", 2273392941122468858), Insert("9", 15883828856151696571), Insert("yd6z803", 4218895444353587596), Remove("d"), Insert("e78g1", 9803365149477112329), Insert("y7nmm", 8704575196200429261), Insert("0h", 11264660707081105088), Remove("n667u0tp"), Remove("1f925"), Insert("0jj", 3415302823512910611), Remove("21h"), Insert("868345", 7057953949462148973), Insert("56sx", 13319564524444190980), Insert("19y29137", 6237959749701385176), Remove("3n"), Insert("e", 1844240028980683962), Remove("8"), Insert("47", 775572366098639473), Remove("m3cx42"), Remove("730m5n")]) } +cc e23b5493b19d1a9639faafc952b0eb0607a7c7577fdd1d92588be7ce4a85dd95 # shrinks to input = _AddRemoveFlipArgs { ops: Operations([Insert("21", 7057345610017908876), Insert("2m", 14042773570795727584), Remove("3p1"), Remove("76l04l"), Insert("knbdy", 18146268831894905032), Remove("60snt2"), Remove("lu9hi"), Remove("0k4b"), Insert("45h", 2579283501447206608), Remove("bw"), Insert("2x07p43", 8493324281333499417), Insert("q1t40", 7601167973431252114), Remove("h91"), Remove("rbk5"), Remove("0g"), Remove("cj1qy"), Insert("nvfyfsh", 4423885449729888548), Insert("8f1bs", 14160566796080055282), Remove("35e65"), Remove("c4"), Insert("46t", 15274819395966855748), Remove("djw79l"), Remove("j6i94k"), Remove("52"), Remove("4"), Remove("4741v9zw"), Insert("0v6rv0n6", 8007587074198761281), Remove("rm1u"), Insert("996l67", 13690975871034554449), Insert("2ws", 17420284283461900707), Remove("7tk7"), Remove("s"), Insert("xt4660m", 16933411867363950079), Remove("v"), Insert("4", 231938933949663025), Remove("db3f7lu1"), Remove("9"), Remove("s70f"), Remove("8x9d"), Remove("1"), Remove("qs78x4"), Remove("74yrh1v"), Remove("6"), Insert("7i", 3708293114264745095), Remove("85ki3"), Insert("sh83va33", 14851363518059892503), Remove("fx"), Remove("m834"), Insert("eq4m", 5758164870050785584), Remove("1"), Remove("3hrrqqo"), Insert("cvogb08a", 8522917429326002565), Insert("r0", 12544564931559276845), Remove("ju02zc54"), Remove("18ja"), Remove("4wa4h3e7"), Insert("m", 2466305068150690676), Insert("jrjdj5", 12612076267141588934), Insert("75vvn56k", 1821145576455514124), Insert("o", 7704537420839772406), Remove("w5d"), Remove("9p6gz870"), Remove("929636d6"), Insert("7", 8361347580713022873), Remove("5"), Insert("vu5nn4u", 11981653745046670523), Insert("5oab", 8201077123413368720), Remove("ec85295"), Remove("lp"), Insert("7gdq2xr7", 12164408117300005328), Insert("q", 3516141357951345603), Remove("a2ad4"), Insert("f", 2086973970166535168)]) } +cc 9db676834d52f0de6c41187beea9df6feead91cc534fee2f1d46fed728379bd3 # shrinks to input = _DiffCorrespondenceArgs { ops_changes: (Operations([Insert("13g", 0), Insert("5s", 0)]), [Add("13g", 1000)]) } diff --git a/wnfs/proptest-regressions/private/hamt/diff/node.txt b/wnfs/proptest-regressions/private/hamt/diff/node.txt new file mode 100644 index 00000000..d89b1b3b --- /dev/null +++ b/wnfs/proptest-regressions/private/hamt/diff/node.txt @@ -0,0 +1,11 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 72dc4e2a4374846a71044bee2d230bfa131730e3520e72ef85d379b8478f40be # shrinks to input = _RemoveHashmapCorrespondenceArgs { ops: Operations([Insert("c1s1", 2861968474096821110), Insert("1", 7543637671228506879), Insert("96", 2252768383817693563), Remove("4b7fw77"), Insert("g", 11439732489252793716), Remove("ww53m"), Remove("6v4xcoa6")]) } +cc 1726d1bd34f08be2b81a527eabc4012325c57055b171bb09405ac8bb945a520b # shrinks to input = _DiffHashmapCorrespondenceArgs { ops: Operations([Remove("0")]) } +cc e4b58a42567fcd5ffb91c9da25c6b4f713565fe27ef294eba6b8998f15b36149 # shrinks to input = _DiffHashmapCorrespondenceArgs { ops: Operations([Remove("3nvc20f"), Insert("u", 9956079658192318434), Remove("0a9r7g9"), Insert("87v", 14695044301822388386), Remove("2r51d6"), Remove("jl454"), Remove("j20j20pm"), Insert("g", 5152991923846126473), Remove("03y40"), Remove("fg9765m"), Insert("8t", 4474459820086240620), Remove("7"), Remove("85j2l46"), Insert("wky9u", 11445290414692833415), Insert("8lc", 9546591671211870912), Insert("np0hl19", 16482876861009537883), Remove("44cfut"), Remove("k7orxzg"), Remove("t3w"), Insert("3j", 8037164116380012272), Remove("m1q24kf8"), Insert("5yk", 2981284836867171899), Insert("v7w9kbfr", 16526934658811960738), Remove("l9k59"), Insert("941u", 8583149886141079763), Insert("81p42wa", 14443267362832524868), Insert("lw8", 12108614233900920745), Remove("tdub47k"), Insert("9wx157sz", 8251470049628788226), Insert("zwjb", 12946990402202987744), Insert("10fn", 1761380674163905782), Remove("ojw"), Insert("1v200w", 5604804115321922268), Remove("b"), Remove("i"), Insert("d637", 15852238920165317932), Insert("q64", 1730417650685085634), Insert("tj1", 14819700385153246002), Remove("src7b"), Remove("5nw0jkkr"), Remove("l81k"), Remove("wnbb1"), Remove("tgv7wd9"), Remove("1"), Insert("75mv5", 17583178423204481260), Remove("7ny1o80"), Remove("7"), Insert("z0cix6", 331083845447588779), Insert("16b", 4585931953995488213), Remove("6j5k3"), Insert("bwlx", 17352226830058118319), Remove("5xm4l2e9"), Remove("awshh4"), Insert("1dd8e53", 14849683218925113909), Insert("c57a", 532223834441501308), Remove("1"), Insert("7krc", 12089486759419077079), Insert("1", 8958205764608539889), Remove("ga94"), Remove("oq2p1du0"), Insert("s87n3um", 5818462765028773932), Remove("89ps"), Remove("t"), Remove("a7tcl4"), Insert("24", 11260399054011738518), Insert("m7r1r", 1164430910219864659), Remove("9h78"), Insert("001", 9872829549486100367), Remove("0117k31"), Remove("9k"), Insert("ztt", 5115805548357263331), Insert("5", 3579437175790913731), Remove("jbn9nr8y"), Insert("cba1hv41", 3167024047889073856), Insert("1ey63u2", 12275380225424398350), Insert("5o39", 6072018527218905335), Remove("928k6"), Remove("df1"), Remove("g"), Remove("x2s"), Remove("5k04kp94"), Remove("h973r36w"), Remove("p983xo3"), Insert("662", 12503934591749486865), Insert("158yl0", 3391647112296275715), Remove("p"), Insert("i2vg2no", 14100769604499849852), Remove("39udv"), Remove("m41p5r"), Insert("91", 12441709321095061151), Remove("a"), Remove("6"), Insert("8g", 11695773606635918662), Insert("000p", 14719758053479480581), Remove("8ei")]) } +cc 195a0e55fe7ca790273f4006594ed16b4a07969f3048eff2d0301c2c8c00da02 # shrinks to input = _DiffHashmapCorrespondenceArgs { ops: Operations([Remove("ahi95"), Insert("q92hcdz6", 12013588257816538438), Insert("g3", 10591691140536598674), Remove("e4jo60p9"), Insert("vs8qcp", 4011281999148158115), Remove("bi6s3d8f"), Remove("rq88vj"), Remove("znc39i"), Insert("14lt0", 10950353491372813321), Insert("02o6", 17427054465394299211), Insert("0", 7557074686536222252), Remove("1gq"), Insert("832te8", 18212087089492431808), Insert("sb", 14276893202905350281), Insert("w", 15998670375294947940), Insert("7", 16867301993208336769), Insert("sx7xb", 155999688409670117), Insert("14", 14609764202045242970), Insert("4", 6925978030280114190), Insert("irpw1", 7585210218031281310), Remove("p"), Remove("ze0s"), Remove("11fi75"), Remove("13av9"), Insert("l3hsy1s2", 7492885477565579271), Insert("ub1ym38", 7552406135646256719), Insert("7s8mwl", 18135249620231959636), Remove("96t"), Remove("rq6pxfg4"), Remove("reomie1"), Remove("inel"), Remove("89f99"), Insert("pl2", 2269060021916044316)]) } +cc 974898ad0bafcd27f2d71fb09d4b96f04680f2e626a18ad2eca582dd2a8bbf56 # shrinks to input = _DiffCorrespondenceArgs { ops: Operations([Insert("nze37", 5607293976437903960), Insert("w83ri", 13974003653085375353), Remove("3pj3f"), Insert("0v", 14449994618968824478), Insert("2zx23", 490777705537419654), Insert("8", 4901286928634009637), Insert("gu390i", 7302034693882484860), Insert("w1xjnk", 15275939612861193177), Insert("gp6nt3k", 16818862727418097197), Insert("0ef", 14774889956068947404), Insert("4f659", 13213841622869572750), Insert("2dv", 1996392139953980495), Insert("78if6", 11581101175803594395), Remove("j21h5"), Remove("bj06f"), Remove("t5se"), Insert("05", 9684822998161819070), Insert("94290vt", 12578288451391820397), Remove("2p1p7"), Remove("h855517"), Remove("l46m4o9x"), Insert("6", 5235519396526454081), Remove("on0k9"), Remove("a"), Insert("773", 6805938963484506619), Remove("40h"), Insert("9", 6366563695199188883), Insert("y2", 6653609466314396468), Insert("wegqcp", 10364286821529963026), Remove("fv0"), Insert("uigibv9", 5477988444587535454), Insert("c775zga2", 12731603435233519486), Insert("a6", 6544273158821719696), Insert("f2xh2", 14507375301219512205), Remove("5ly"), Remove("01"), Remove("o1cx7r1r"), Insert("4r6m9", 9438169518613924728), Insert("36g", 4813958754157346055), Insert("716", 7658772099189076995), Remove("4z7758ph"), Insert("x", 7512642240091346227), Remove("64k6xlz6"), Remove("f1a7a"), Remove("5m4f1m"), Remove("43bh2bw"), Remove("o2"), Insert("c5e8lmi", 3850797232426528274), Insert("b", 15686353040049894950), Insert("ybjrsi5", 10115613010033310831), Insert("mpyyu", 10047631643897079445), Remove("gms"), Insert("0238", 12225635164454194279), Remove("6secht"), Insert("q0lh", 1625660415707077002), Insert("09sg08vk", 5529754594624688099), Insert("11g8", 18147176070903365111), Insert("7uq15g1", 5045832724365655421), Remove("i7ljxl"), Insert("2", 4056524782711557091), Insert("888d", 7200406577966948479), Insert("2s3z6sb9", 11277545678417675335), Remove("9cva"), Insert("p8a0h0l1", 7796497246403854634), Insert("v4", 2163990801486059232), Remove("ta4"), Remove("085h7u"), Insert("w82o28", 8929686504171384478), Remove("726u"), Insert("27c8jrv", 12363820087431725551), Remove("eog"), Insert("4w9n5o7f", 5852413551123062477), Remove("8c4c"), Remove("mi2"), Insert("r18", 7881014950756268623), Insert("6ow4", 3847558428371643988), Insert("3j", 8116029199181678606), Insert("k8", 14197183690972410544), Insert("0e57ego2", 16092422570903853179), Remove("n6er"), Insert("5f0hnc6s", 13820376120759977329)]) } diff --git a/wnfs/proptest-regressions/private/hamt/merge.txt b/wnfs/proptest-regressions/private/hamt/merge.txt new file mode 100644 index 00000000..b0fcfa5f --- /dev/null +++ b/wnfs/proptest-regressions/private/hamt/merge.txt @@ -0,0 +1,8 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc a534906dfc176f8c4cd538fdf1bf0f0899462ac9444d4b966f02471f697b5fef # shrinks to input = _MergeCommutativityArgs { ops: Operations([Insert("yf0h9w", 5841962655239405927), Insert("2iu58c9", 3370888731860934047), Insert("m9j9t", 7265696903202593462), Remove("j3vs"), Remove("f4"), Insert("i5qb", 10523669173817519006), Insert("f67r9vuw", 12317150070157799956), Insert("4yibko9", 1749416059759925490), Insert("1", 11104016840992901739), Remove("k56vtf"), Insert("57ul0s", 10760630981796699974), Remove("9b2vc84"), Remove("m1e5"), Remove("d4kn"), Insert("gyl", 5768830030276443147), Insert("36", 1062845700696758957), Remove("dqz0q7nh"), Insert("h2a2r3h", 1448791360660457216), Remove("ych543"), Remove("7xk"), Insert("r", 12267725769721984134), Insert("6x", 7521533771167090565), Insert("2u5duw", 142243144271675893), Remove("k0v8ju"), Remove("51u8zwr"), Remove("gi1h2"), Remove("511i4k0"), Remove("478"), Remove("32"), Remove("3qvaw0xa"), Insert("e3", 8846047171313551908), Remove("csyme0o0")]) } +cc 4e6b67ab3a19fc8a07dfdfc61970354373d689e8b320b619aab92cfe5506af7c # shrinks to input = _MergeAssociativityArgs { ops_changes: (Operations([Remove("6"), Remove("8r"), Insert("9ne", 896), Insert("3", 198), Insert("g", 152), Remove("u4"), Insert("l5k", 800), Remove("ux"), Insert("0cs", 395), Remove("736"), Insert("5gk", 395), Insert("1v5", 971)]), [Add("5gk", 1331), Modify("9ne", 1310), Add("3", 1907)], [Remove("l5k")]) } diff --git a/wnfs/src/common/blockstore.rs b/wnfs/src/common/blockstore.rs index 34502c94..9f71348e 100644 --- a/wnfs/src/common/blockstore.rs +++ b/wnfs/src/common/blockstore.rs @@ -4,7 +4,6 @@ use std::{borrow::Cow, io::Cursor}; use anyhow::Result; use async_trait::async_trait; -use hashbrown::HashMap; use libipld::{ cbor::DagCborCodec, cid::Version, @@ -14,6 +13,7 @@ use libipld::{ }; use rand_core::RngCore; use serde::{de::DeserializeOwned, Serialize}; +use std::collections::HashMap; use crate::{ private::{Key, NONCE_SIZE}, diff --git a/wnfs/src/common/error.rs b/wnfs/src/common/error.rs index b45154bf..5bd3dc9b 100644 --- a/wnfs/src/common/error.rs +++ b/wnfs/src/common/error.rs @@ -68,6 +68,12 @@ pub enum FsError { #[error("Cannot find shard for file content")] FileShardNotFound, + + #[error("The hashprefix index is out of bounds")] + InvalidHashPrefixIndex, + + #[error("Key does not exist in HAMT")] + KeyNotFoundInHamt, } pub fn error(err: impl std::error::Error + Send + Sync + 'static) -> Result { diff --git a/wnfs/src/common/link.rs b/wnfs/src/common/link.rs index b64f8b9a..7f0a03b1 100644 --- a/wnfs/src/common/link.rs +++ b/wnfs/src/common/link.rs @@ -1,10 +1,10 @@ +use crate::{AsyncSerialize, BlockStore, IpldEq}; use anyhow::Result; use async_once_cell::OnceCell; use async_trait::async_trait; use libipld::Cid; use serde::de::DeserializeOwned; - -use crate::{AsyncSerialize, BlockStore, IpldEq}; +use std::fmt::{self, Debug, Formatter}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -15,7 +15,6 @@ use crate::{AsyncSerialize, BlockStore, IpldEq}; /// It supports representing the "link" with a Cid or the deserialized value itself. /// /// Link needs a `BlockStore` to be able to resolve Cids to corresponding values of `T` and vice versa. -#[derive(Debug)] pub enum Link { /// A variant of `Link` that started out as a `Cid`. /// If the decoded value is resolved using `resolve_value`, then the `value_cache` gets populated and @@ -91,7 +90,7 @@ impl Link { } /// Gets an owned value from type. It attempts to it get from the store if it is not present in type. - pub async fn get_owned_value(self, store: &B) -> Result + pub async fn resolve_owned_value(self, store: &B) -> Result where T: DeserializeOwned, { @@ -204,6 +203,18 @@ where } } +impl Debug for Link +where + T: Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + match self { + Self::Encoded { cid, .. } => f.debug_tuple("Link::Encoded").field(cid).finish(), + Self::Decoded { value, .. } => f.debug_tuple("Link::Decoded").field(value).finish(), + } + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- diff --git a/wnfs/src/common/mod.rs b/wnfs/src/common/mod.rs index c4cd362d..e1d2c305 100644 --- a/wnfs/src/common/mod.rs +++ b/wnfs/src/common/mod.rs @@ -26,4 +26,5 @@ pub const MAX_BLOCK_SIZE: usize = usize::pow(2, 18); // Type Definitions //-------------------------------------------------------------------------------------------------- +/// The general size of digests in WNFS. pub type HashOutput = [u8; HASH_BYTE_SIZE]; diff --git a/wnfs/src/common/utils.rs b/wnfs/src/common/utils.rs index c125a135..e32843e2 100644 --- a/wnfs/src/common/utils.rs +++ b/wnfs/src/common/utils.rs @@ -1,4 +1,4 @@ -use crate::{error, FsError}; +use crate::{error, FsError, HashOutput}; use anyhow::Result; #[cfg(any(test, feature = "test_strategies"))] use proptest::{ @@ -7,7 +7,8 @@ use proptest::{ }; use rand_core::RngCore; use serde::de::Visitor; -use std::fmt; +use std::{fmt, rc::Rc}; + //-------------------------------------------------------------------------------------------------- // Type Definitions //-------------------------------------------------------------------------------------------------- @@ -20,6 +21,11 @@ pub trait Sampleable { fn sample(&self, runner: &mut TestRunner) -> Self::Value; } +pub(crate) trait UnwrapOrClone { + type Output; + fn unwrap_or_clone(self) -> Self::Output; +} + //-------------------------------------------------------------------------------------------------- // Implementations //-------------------------------------------------------------------------------------------------- @@ -54,6 +60,20 @@ where } } +impl UnwrapOrClone for Rc +where + T: Clone, +{ + type Output = Result; + + fn unwrap_or_clone(self) -> Self::Output { + match Rc::try_unwrap(self) { + Ok(value) => Ok(value), + Err(rc) => Ok(rc.as_ref().clone()), + } + } +} + //-------------------------------------------------------------------------------------------------- // Functions //-------------------------------------------------------------------------------------------------- @@ -71,10 +91,10 @@ pub(crate) fn split_last(path_segments: &[String]) -> Result<(&[String], &String /// /// ``` /// use rand::thread_rng; -/// use wnfs::utils::get_random_bytes; +/// use wnfs::utils; /// /// let rng = &mut thread_rng(); -/// let bytes = get_random_bytes::<32>(rng); +/// let bytes = utils::get_random_bytes::<32>(rng); /// /// assert_eq!(bytes.len(), 32); /// ``` @@ -84,6 +104,27 @@ pub fn get_random_bytes(rng: &mut impl RngCore) -> [u8; N] { bytes } +/// Creates a [`HashOutput`][HashOutput] ([u8; 32]) from a possibly incomplete slice. +/// +/// If the slice is smaller than `HashOutput`, the remaining bytes are filled with zeros. +/// +/// # Examples +/// +/// ``` +/// use wnfs::utils; +/// +/// let digest = utils::make_digest(&[0xff, 0x22]); +/// +/// assert_eq!(digest.len(), 32); +/// ``` +/// +/// [HashOutput]: crate::HashOutput +pub fn make_digest(bytes: &[u8]) -> HashOutput { + let mut nibbles = [0u8; 32]; + nibbles[..bytes.len()].copy_from_slice(bytes); + nibbles +} + //-------------------------------------------------------------------------------------------------- // Macros //-------------------------------------------------------------------------------------------------- @@ -114,6 +155,11 @@ pub(crate) mod test_setup { proptest::test_runner::RngAlgorithm::ChaCha ) }; + [ runner ] => { + proptest::test_runner::TestRunner::new( + proptest::test_runner::Config::default() + ) + }; [ store ] => { $crate::MemoryBlockStore::new() }; diff --git a/wnfs/src/lib.rs b/wnfs/src/lib.rs index 357fc4cf..d048a2e0 100644 --- a/wnfs/src/lib.rs +++ b/wnfs/src/lib.rs @@ -28,6 +28,7 @@ pub use traits::*; pub mod ipld { pub use libipld::{ cbor::DagCborCodec, + cid::Version, codec::{Codec, Decode, Encode}, Cid, IpldCodec, }; diff --git a/wnfs/src/private/file.rs b/wnfs/src/private/file.rs index c66baccc..311a1bb7 100644 --- a/wnfs/src/private/file.rs +++ b/wnfs/src/private/file.rs @@ -475,7 +475,6 @@ mod tests { use super::*; use crate::utils::test_setup; use rand::Rng; - use test_strategy::proptest; #[async_std::test] async fn can_create_empty_file() { @@ -510,6 +509,14 @@ mod tests { content[2 * MAX_BLOCK_CONTENT_SIZE..4 * MAX_BLOCK_CONTENT_SIZE] ); } +} + +#[cfg(test)] +mod proptests { + use super::MAX_BLOCK_CONTENT_SIZE; + use crate::utils::test_setup; + use futures::StreamExt; + use test_strategy::proptest; #[proptest(cases = 100)] fn can_include_and_get_content_from_file( diff --git a/wnfs/src/private/forest.rs b/wnfs/src/private/forest.rs index 75f7de57..d37be0b5 100644 --- a/wnfs/src/private/forest.rs +++ b/wnfs/src/private/forest.rs @@ -1,13 +1,14 @@ -use std::{collections::BTreeSet, rc::Rc}; - +use super::{ + hamt::{self, Hamt}, + namefilter::Namefilter, + Key, PrivateNode, PrivateRef, +}; +use crate::{utils::UnwrapOrClone, BlockStore, HashOutput, Hasher, Link}; use anyhow::Result; use libipld::Cid; use log::debug; use rand_core::RngCore; - -use crate::{BlockStore, HashOutput}; - -use super::{hamt::Hamt, namefilter::Namefilter, Key, PrivateNode, PrivateRef}; +use std::{collections::BTreeSet, fmt, rc::Rc}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -19,7 +20,7 @@ use super::{hamt::Hamt, namefilter::Namefilter, Key, PrivateNode, PrivateRef}; /// an accompanying block store. And on lookup, the nodes are decrypted and deserialized with the same private /// refs. /// -/// It is called a forest because it is a collection of file trees. +/// It is called a forest because it can store a collection of file trees. /// /// # Examples /// @@ -30,7 +31,6 @@ use super::{hamt::Hamt, namefilter::Namefilter, Key, PrivateNode, PrivateRef}; /// /// println!("{:?}", forest); /// ``` -// TODO(appcypher): Change Cid to PrivateLink. pub type PrivateForest = Hamt>; //-------------------------------------------------------------------------------------------------- @@ -49,10 +49,8 @@ impl PrivateForest { /// /// ``` /// use std::rc::Rc; - /// /// use chrono::Utc; /// use rand::thread_rng; - /// /// use wnfs::{ /// private::{PrivateForest, PrivateRef}, PrivateNode, /// BlockStore, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult, @@ -117,10 +115,8 @@ impl PrivateForest { /// /// ``` /// use std::rc::Rc; - /// /// use chrono::Utc; /// use rand::thread_rng; - /// /// use wnfs::{ /// private::{PrivateForest, PrivateRef}, PrivateNode, /// BlockStore, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult, @@ -186,11 +182,9 @@ impl PrivateForest { /// /// ``` /// use std::rc::Rc; - /// /// use chrono::Utc; /// use rand::thread_rng; /// use sha3::Sha3_256; - /// /// use wnfs::{ /// private::{PrivateForest, PrivateRef}, PrivateNode, /// BlockStore, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult, Hasher @@ -246,7 +240,7 @@ impl PrivateForest { .unwrap_or_default(); values.insert(value); - let mut forest = Rc::try_unwrap(self).unwrap_or_else(|rc| (*rc).clone()); + let mut forest = self.unwrap_or_clone()?; forest.root = forest.root.set(name, values, store).await?; Ok(Rc::new(forest)) } @@ -298,22 +292,185 @@ impl PrivateForest { } } -// //-------------------------------------------------------------------------------------------------- -// // Tests -// //-------------------------------------------------------------------------------------------------- +impl Hamt, H> +where + H: Hasher + fmt::Debug + Clone + 'static, +{ + /// Merges a private forest with another. If there is a conflict with the values,they are union + /// combined into a single value in the final merge node + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// use chrono::{Utc, Days}; + /// use rand::{thread_rng, Rng}; + /// use wnfs::{ + /// private::{PrivateForest, PrivateRef}, PrivateNode, + /// BlockStore, MemoryBlockStore, Namefilter, PrivateDirectory, PrivateOpResult, + /// }; + /// + /// #[async_std::main] + /// async fn main() { + /// let store = &mut MemoryBlockStore::default(); + /// let rng = &mut thread_rng(); + /// + /// let ratchet_seed = rng.gen::<[u8; 32]>(); + /// let inumber = rng.gen::<[u8; 32]>(); + /// + /// let main_forest = Rc::new(PrivateForest::new()); + /// let root_dir = Rc::new(PrivateDirectory::with_seed( + /// Namefilter::default(), + /// Utc::now(), + /// ratchet_seed, + /// inumber + /// )); + /// let main_forest = main_forest + /// .put( + /// root_dir.header.get_saturated_name(), + /// &root_dir.header.get_private_ref(), + /// &PrivateNode::Dir(Rc::clone(&root_dir)), + /// store, + /// rng + /// ) + /// .await + /// .unwrap(); + /// + /// let other_forest = Rc::new(PrivateForest::new()); + /// let root_dir = Rc::new(PrivateDirectory::with_seed( + /// Namefilter::default(), + /// Utc::now().checked_add_days(Days::new(1)).unwrap(), + /// ratchet_seed, + /// inumber + /// )); + /// let other_forest = other_forest + /// .put( + /// root_dir.header.get_saturated_name(), + /// &root_dir.header.get_private_ref(), + /// &PrivateNode::Dir(Rc::clone(&root_dir)), + /// store, + /// rng + /// ) + /// .await + /// .unwrap(); + /// + /// let merge_forest = main_forest.merge(&other_forest, store).await.unwrap(); + /// + /// assert_eq!( + /// 2, + /// merge_forest + /// .root + /// .get(&root_dir.header.get_saturated_name(), store) + /// .await + /// .unwrap() + /// .unwrap() + /// .len() + /// ); + /// } + /// ``` + pub async fn merge(&self, other: &Self, store: &mut B) -> Result { + let merge_node = hamt::merge( + Link::from(Rc::clone(&self.root)), + Link::from(Rc::clone(&other.root)), + |a, b| Ok(a.union(b).cloned().collect()), + store, + ) + .await?; + + Ok(Self { + version: self.version.clone(), + root: merge_node, + }) + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- #[cfg(test)] mod tests { + use super::*; + use crate::{ + private::{HashNibbles, Node, PrivateDirectory}, + utils::test_setup, + MemoryBlockStore, + }; + use chrono::Utc; + use helper::*; use proptest::test_runner::{RngAlgorithm, TestRng}; use std::rc::Rc; - use test_log::test; - use chrono::Utc; + mod helper { + use crate::{utils, HashOutput, Hasher, Namefilter}; + use libipld::{Cid, Multihash}; + use once_cell::sync::Lazy; + use rand::{thread_rng, RngCore}; + + pub(super) static HASH_KV_PAIRS: Lazy> = + Lazy::new(|| { + vec![ + ( + utils::make_digest(&[0xA0]), + generate_saturated_name_hash(&mut thread_rng()), + generate_cid(&mut thread_rng()), + ), + ( + utils::make_digest(&[0xA3]), + generate_saturated_name_hash(&mut thread_rng()), + generate_cid(&mut thread_rng()), + ), + ( + utils::make_digest(&[0xA7]), + generate_saturated_name_hash(&mut thread_rng()), + generate_cid(&mut thread_rng()), + ), + ( + utils::make_digest(&[0xAC]), + generate_saturated_name_hash(&mut thread_rng()), + generate_cid(&mut thread_rng()), + ), + ( + utils::make_digest(&[0xAE]), + generate_saturated_name_hash(&mut thread_rng()), + generate_cid(&mut thread_rng()), + ), + ] + }); + + #[derive(Debug, Clone)] + pub(super) struct MockHasher; + impl Hasher for MockHasher { + fn hash>(key: &K) -> HashOutput { + HASH_KV_PAIRS + .iter() + .find(|(_, v, _)| key.as_ref() == v.as_ref()) + .unwrap() + .0 + } + } - use super::*; - use crate::{private::PrivateDirectory, MemoryBlockStore}; + pub(super) fn generate_saturated_name_hash(rng: &mut impl RngCore) -> Namefilter { + let mut namefilter = Namefilter::default(); + namefilter.add(&utils::get_random_bytes::<32>(rng)); + namefilter.saturate(); + namefilter + } + + pub(super) fn generate_cid(rng: &mut impl RngCore) -> Cid { + let bytes = { + let mut tmp = [0u8; 10]; + let (a, b) = tmp.split_at_mut(2); + a.copy_from_slice(&[0x55, 0x08]); + b.copy_from_slice(&utils::get_random_bytes::<8>(rng)); + tmp + }; + + Cid::new_v1(0x55, Multihash::from_bytes(&bytes).unwrap()) + } + } - #[test(async_std::test)] + #[async_std::test] async fn inserted_items_can_be_fetched() { let store = &mut MemoryBlockStore::new(); let forest = Rc::new(PrivateForest::new()); @@ -343,7 +500,7 @@ mod tests { assert_eq!(retrieved, private_node); } - #[test(async_std::test)] + #[async_std::test] async fn inserted_multivalue_items_can_be_fetched_with_bias() { let store = &mut MemoryBlockStore::new(); let forest = Rc::new(PrivateForest::new()); @@ -413,4 +570,74 @@ mod tests { assert_eq!(retrieved, private_node_conflict); } + + #[async_std::test] + async fn can_merge_nodes_with_different_structure_and_modified_changes() { + let (store, rng) = test_setup::init!(mut store, mut rng); + + // A node that adds the first 3 pairs of HASH_KV_PAIRS. + let mut other_node = Rc::new(Node::<_, _, MockHasher>::default()); + for (digest, k, v) in HASH_KV_PAIRS.iter().take(3) { + other_node = other_node + .set_value( + &mut HashNibbles::new(digest), + k.clone(), + BTreeSet::from([*v]), + store, + ) + .await + .unwrap(); + } + + // Another node that keeps the first pair, modify the second pair, removes the third pair, and adds the fourth and fifth pair. + let mut main_node = Rc::new(Node::<_, _, MockHasher>::default()); + main_node = main_node + .set_value( + &mut HashNibbles::new(&HASH_KV_PAIRS[0].0), + HASH_KV_PAIRS[0].1.clone(), + BTreeSet::from([HASH_KV_PAIRS[0].2]), + store, + ) + .await + .unwrap(); + + let new_cid = generate_cid(rng); + main_node = main_node + .set_value( + &mut HashNibbles::new(&HASH_KV_PAIRS[1].0), + HASH_KV_PAIRS[1].1.clone(), + BTreeSet::from([new_cid]), + store, + ) + .await + .unwrap(); + + for (digest, k, v) in HASH_KV_PAIRS.iter().skip(3).take(2) { + main_node = main_node + .set_value( + &mut HashNibbles::new(digest), + k.clone(), + BTreeSet::from([*v]), + store, + ) + .await + .unwrap(); + } + + let main_forest = Hamt::, _>::with_root(main_node); + let other_forest = Hamt::, _>::with_root(other_node); + + let merge_forest = main_forest.merge(&other_forest, store).await.unwrap(); + + for (i, (digest, _, v)) in HASH_KV_PAIRS.iter().take(5).enumerate() { + let retrieved = merge_forest.root.get_by_hash(digest, store).await.unwrap(); + if i != 1 { + assert_eq!(retrieved.unwrap(), &BTreeSet::from([*v])); + } else { + // The second pair should contain two merged Cids. + assert!(retrieved.unwrap().contains(&new_cid)); + assert!(retrieved.unwrap().contains(&HASH_KV_PAIRS[1].2)); + } + } + } } diff --git a/wnfs/src/private/hamt/diff/key_value.rs b/wnfs/src/private/hamt/diff/key_value.rs new file mode 100644 index 00000000..6584bafc --- /dev/null +++ b/wnfs/src/private/hamt/diff/key_value.rs @@ -0,0 +1,551 @@ +use super::ChangeType; +use crate::{private::Node, BlockStore, Hasher, Link, Pair}; +use anyhow::{Ok, Result}; +use either::Either::{self, *}; +use serde::de::DeserializeOwned; +use std::{hash::Hash, rc::Rc}; + +//-------------------------------------------------------------------------------------------------- +// Type Definitions +//-------------------------------------------------------------------------------------------------- + +/// Represents a change to some key-value pair of a HAMT node. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct KeyValueChange { + pub r#type: ChangeType, + pub key: K, + pub main_value: Option, + pub other_value: Option, +} + +type EitherPairOrNode<'a, K, V, H> = Option, &'a Rc>>>; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Compare two nodes and get the key-value changes made to the main node. +/// +/// This implementation gets all the changes to main node at the leaf node level. +/// +/// This is a more expensive operation because it gathers the key value pairs under a node has +/// been added or removed even though we can simply return a reference to the node itself. +/// +/// # Examples +/// +/// ``` +/// use std::rc::Rc; +/// use wnfs::{private::{Node, diff}, Link, Pair, MemoryBlockStore}; +/// +/// #[async_std::main] +/// async fn main() { +/// let store = &mut MemoryBlockStore::new(); +/// let mut main_node = Rc::new(Node::<[u8; 4], String>::default()); +/// for i in 0u32..3 { +/// main_node = main_node +/// .set(i.to_le_bytes(), i.to_string(), store) +/// .await +/// .unwrap(); +/// } +/// +/// let mut other_node = Rc::new(Node::<[u8; 4], String>::default()); +/// other_node = other_node +/// .set(0_u32.to_le_bytes(), 0_u32.to_string(), store) +/// .await +/// .unwrap(); +/// +/// let changes = diff::kv_diff( +/// Link::from(Rc::clone(&main_node)), +/// Link::from(Rc::clone(&other_node)), +/// store, +/// ) +/// .await +/// .unwrap(); +/// +/// +/// println!("Changes {:#?}", changes); +/// } +/// ``` +pub async fn kv_diff( + main_link: Link>>, + other_link: Link>>, + store: &mut B, +) -> Result>> +where + K: DeserializeOwned + Clone + Eq + Hash + AsRef<[u8]>, + V: DeserializeOwned + Clone + Eq, + H: Hasher + Clone + 'static, + B: BlockStore, +{ + let node_changes = super::node_diff(main_link.clone(), other_link.clone(), store).await?; + + let main_node = main_link.resolve_value(store).await?; + let other_node = other_link.resolve_value(store).await?; + + let mut kv_changes = Vec::new(); + for change in node_changes { + match change.r#type { + ChangeType::Add => { + let result = main_node.get_node_at(&change.hashprefix, store).await?; + kv_changes + .extend(generate_add_or_remove_changes(result, ChangeType::Add, store).await?); + } + ChangeType::Remove => { + let result = other_node.get_node_at(&change.hashprefix, store).await?; + kv_changes.extend( + generate_add_or_remove_changes(result, ChangeType::Remove, store).await?, + ); + } + ChangeType::Modify => match ( + main_node.get_node_at(&change.hashprefix, store).await?, + other_node.get_node_at(&change.hashprefix, store).await?, + ) { + (Some(Left(main_pair)), Some(Left(other_pair))) => { + kv_changes.push(KeyValueChange { + r#type: ChangeType::Modify, + key: main_pair.key.clone(), + main_value: Some(main_pair.value.clone()), + other_value: Some(other_pair.value.clone()), + }); + } + _ => unreachable!("Node change type is Modify but nodes not found or not pairs."), + }, + } + } + + Ok(kv_changes) +} + +async fn generate_add_or_remove_changes<'a, K, V, H, B>( + node: EitherPairOrNode<'a, K, V, H>, + r#type: ChangeType, + store: &B, +) -> Result>> +where + B: BlockStore, + K: DeserializeOwned + Clone, + V: DeserializeOwned + Clone, + H: Hasher + Clone + 'static, +{ + match node { + Some(Left(Pair { key, value })) => Ok(vec![KeyValueChange { + r#type, + key: key.clone(), + main_value: if r#type == ChangeType::Add { + Some(value.clone()) + } else { + None + }, + other_value: if r#type == ChangeType::Remove { + Some(value.clone()) + } else { + None + }, + }]), + Some(Right(node)) => { + node.flat_map( + &|Pair { key, value }| { + Ok(KeyValueChange { + r#type, + key: key.clone(), + main_value: if r#type == ChangeType::Add { + Some(value.clone()) + } else { + None + }, + other_value: if r#type == ChangeType::Remove { + Some(value.clone()) + } else { + None + }, + }) + }, + store, + ) + .await + } + _ => unreachable!("Node change type is Remove but node is not found."), + } +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::{ChangeType::*, *}; + use crate::{ + private::{HashNibbles, Node}, + utils::test_setup, + }; + use helper::*; + use std::rc::Rc; + + mod helper { + use once_cell::sync::Lazy; + + use crate::{utils, HashOutput, Hasher}; + + pub(super) static HASH_KV_PAIRS: Lazy> = Lazy::new(|| { + vec![ + (utils::make_digest(&[0xA0]), "first"), + (utils::make_digest(&[0xA3]), "second"), + (utils::make_digest(&[0xA7]), "third"), + (utils::make_digest(&[0xAC]), "fourth"), + (utils::make_digest(&[0xAE]), "fifth"), + ] + }); + + #[derive(Debug, Clone)] + pub(crate) struct MockHasher; + impl Hasher for MockHasher { + fn hash>(key: &K) -> HashOutput { + HASH_KV_PAIRS + .iter() + .find(|(_, v)| key.as_ref() == >::as_ref(v)) + .unwrap() + .0 + } + } + } + + #[async_std::test] + async fn can_diff_main_node_with_added_removed_pairs() { + let store = test_setup::init!(mut store); + + let mut main_node = Rc::new(Node::<[u8; 4], String>::default()); + for i in 0u32..3 { + main_node = main_node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let mut other_node = Rc::new(Node::<[u8; 4], String>::default()); + other_node = other_node + .set(0_u32.to_le_bytes(), 0_u32.to_string(), store) + .await + .unwrap(); + + let changes = kv_diff( + Link::from(Rc::clone(&main_node)), + Link::from(Rc::clone(&other_node)), + store, + ) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + KeyValueChange { + r#type: Add, + key: [2, 0, 0, 0,], + main_value: Some(String::from("2")), + other_value: None, + }, + KeyValueChange { + r#type: Add, + key: [1, 0, 0, 0,], + main_value: Some(String::from("1")), + other_value: None, + }, + ] + ); + + let changes = kv_diff(Link::from(other_node), Link::from(main_node), store) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + KeyValueChange { + r#type: Remove, + key: [2, 0, 0, 0,], + main_value: None, + other_value: Some(String::from("2")), + }, + KeyValueChange { + r#type: Remove, + key: [1, 0, 0, 0,], + main_value: None, + other_value: Some(String::from("1")), + }, + ] + ); + } + + #[async_std::test] + async fn can_diff_main_node_with_no_changes() { + let store = test_setup::init!(mut store); + + let mut main_node = Rc::new(Node::<_, _>::default()); + for i in 0_u32..3 { + main_node = main_node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let mut other_node = Rc::new(Node::<_, _>::default()); + for i in 0_u32..3 { + other_node = other_node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let changes = kv_diff(Link::from(main_node), Link::from(other_node), store) + .await + .unwrap(); + + assert!(changes.is_empty()); + } + + #[async_std::test] + async fn can_diff_nodes_with_different_structure_and_modified_changes() { + let store = test_setup::init!(mut store); + + // A node that adds the first 3 pairs of HASH_KV_PAIRS. + let mut other_node = Rc::new(Node::<_, _, MockHasher>::default()); + for (digest, kv) in HASH_KV_PAIRS.iter().take(3) { + other_node = other_node + .set_value( + &mut HashNibbles::new(digest), + kv.to_string(), + kv.to_string(), + store, + ) + .await + .unwrap(); + } + + // Another node that keeps the first pair, modify the second pair, removes the third pair, and adds the fourth and fifth pair. + let mut main_node = Rc::new(Node::<_, _, MockHasher>::default()); + main_node = main_node + .set_value( + &mut HashNibbles::new(&HASH_KV_PAIRS[0].0), + HASH_KV_PAIRS[0].1.to_string(), + HASH_KV_PAIRS[0].1.to_string(), + store, + ) + .await + .unwrap(); + + main_node = main_node + .set_value( + &mut HashNibbles::new(&HASH_KV_PAIRS[1].0), + HASH_KV_PAIRS[1].1.to_string(), + String::from("second_modified"), + store, + ) + .await + .unwrap(); + + for (digest, kv) in HASH_KV_PAIRS.iter().skip(3).take(2) { + main_node = main_node + .set_value( + &mut HashNibbles::new(digest), + kv.to_string(), + kv.to_string(), + store, + ) + .await + .unwrap(); + } + + let changes = kv_diff( + Link::from(Rc::clone(&main_node)), + Link::from(Rc::clone(&other_node)), + store, + ) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + KeyValueChange { + r#type: Modify, + key: "second".to_string(), + main_value: Some("second_modified".to_string()), + other_value: Some("second".to_string()), + }, + KeyValueChange { + r#type: Remove, + key: "third".to_string(), + main_value: None, + other_value: Some("third".to_string()), + }, + KeyValueChange { + r#type: Add, + key: "fourth".to_string(), + main_value: Some("fourth".to_string()), + other_value: None, + }, + KeyValueChange { + r#type: Add, + key: "fifth".to_string(), + main_value: Some("fifth".to_string()), + other_value: None, + }, + ] + ); + + let changes = kv_diff(Link::from(other_node), Link::from(main_node), store) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + KeyValueChange { + r#type: Modify, + key: "second".to_string(), + main_value: Some("second".to_string()), + other_value: Some("second_modified".to_string()), + }, + KeyValueChange { + r#type: Add, + key: "third".to_string(), + main_value: Some("third".to_string()), + other_value: None, + }, + KeyValueChange { + r#type: Remove, + key: "fourth".to_string(), + main_value: None, + other_value: Some("fourth".to_string()), + }, + KeyValueChange { + r#type: Remove, + key: "fifth".to_string(), + main_value: None, + other_value: Some("fifth".to_string()), + }, + ] + ); + } +} + +#[cfg(test)] +mod proptests { + use crate::{ + private::{ + strategies::{self, generate_kvs, generate_ops_and_changes, Change, Operations}, + ChangeType, + }, + utils::test_setup, + Link, + }; + use async_std::task; + use std::{collections::HashSet, rc::Rc}; + use test_strategy::proptest; + + #[proptest(cases = 100, max_shrink_iters = 4000)] + fn diff_correspondence( + #[strategy(generate_ops_and_changes())] ops_changes: ( + Operations, + Vec>, + ), + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + let (ops, strategy_changes) = ops_changes; + + let other_node = strategies::prepare_node( + strategies::node_from_operations(&ops, store).await.unwrap(), + &strategy_changes, + store, + ) + .await + .unwrap(); + + let main_node = + strategies::apply_changes(Rc::clone(&other_node), &strategy_changes, store) + .await + .unwrap(); + + let changes = super::kv_diff( + Link::from(Rc::clone(&main_node)), + Link::from(Rc::clone(&other_node)), + store, + ) + .await + .unwrap(); + + assert_eq!(strategy_changes.len(), changes.len()); + for strategy_change in strategy_changes { + assert!(changes.iter().any(|c| match &strategy_change { + Change::Add(k, _) => c.r#type == ChangeType::Add && &c.key == k, + Change::Modify(k, _) => { + c.r#type == ChangeType::Modify && &c.key == k + } + Change::Remove(k) => { + c.r#type == ChangeType::Remove && &c.key == k + } + })); + } + }); + } + + #[proptest(cases = 1000, max_shrink_iters = 40000)] + fn diff_unique_keys( + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs1: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs2: Vec<(String, u64)>, + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + + let node1 = strategies::node_from_kvs(kvs1, store).await.unwrap(); + let node2 = strategies::node_from_kvs(kvs2, store).await.unwrap(); + + let changes = super::kv_diff(Link::from(node1), Link::from(node2), store) + .await + .unwrap(); + + let change_set = changes + .iter() + .map(|c| c.key.clone()) + .collect::>(); + + assert_eq!(change_set.len(), changes.len()); + }); + } + + #[proptest(cases = 100)] + fn add_remove_flip( + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs1: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs2: Vec<(String, u64)>, + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + + let node1 = strategies::node_from_kvs(kvs1, store).await.unwrap(); + let node2 = strategies::node_from_kvs(kvs2, store).await.unwrap(); + + let changes = super::kv_diff( + Link::from(Rc::clone(&node1)), + Link::from(Rc::clone(&node2)), + store, + ) + .await + .unwrap(); + + let flipped_changes = super::kv_diff(Link::from(node2), Link::from(node1), store) + .await + .unwrap(); + + assert_eq!(changes.len(), flipped_changes.len()); + for change in changes { + assert!(flipped_changes.iter().any(|c| match change.r#type { + ChangeType::Add => c.r#type == ChangeType::Remove && c.key == change.key, + ChangeType::Remove => c.r#type == ChangeType::Add && c.key == change.key, + ChangeType::Modify => c.r#type == ChangeType::Modify && c.key == change.key, + })); + } + }); + } +} diff --git a/wnfs/src/private/hamt/diff/mod.rs b/wnfs/src/private/hamt/diff/mod.rs new file mode 100644 index 00000000..21205ad1 --- /dev/null +++ b/wnfs/src/private/hamt/diff/mod.rs @@ -0,0 +1,17 @@ +mod key_value; +mod node; + +pub use key_value::*; +pub use node::*; + +//-------------------------------------------------------------------------------------------------- +// Type Definitions +//-------------------------------------------------------------------------------------------------- + +/// This type represents the different kinds of changes to a node. +#[derive(Debug, Copy, Clone, PartialEq, Eq)] +pub enum ChangeType { + Add, + Remove, + Modify, +} diff --git a/wnfs/src/private/hamt/diff/node.rs b/wnfs/src/private/hamt/diff/node.rs new file mode 100644 index 00000000..a9f29f90 --- /dev/null +++ b/wnfs/src/private/hamt/diff/node.rs @@ -0,0 +1,600 @@ +use super::ChangeType; +use crate::{ + private::{HashNibbles, HashPrefix, Node, Pointer, HAMT_BITMASK_BIT_SIZE}, + utils::UnwrapOrClone, + BlockStore, Hasher, Link, Pair, +}; +use anyhow::Result; +use async_recursion::async_recursion; +use serde::de::DeserializeOwned; +use std::{collections::HashMap, hash::Hash, mem, rc::Rc}; + +//-------------------------------------------------------------------------------------------------- +// Type Definitions +//-------------------------------------------------------------------------------------------------- + +/// Represents a change to some node or key-value pair of a HAMT. +#[derive(Debug, Clone, PartialEq)] +pub struct NodeChange { + pub r#type: ChangeType, + pub hashprefix: HashPrefix, +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Compare two nodes and get differences in keys relative to the main node. +/// +/// This implementation returns early once it detects differences between intermediate nodes of the +/// HAMT. In such cases, it returns the hash prefix for the set of keys that were added, removed +/// or modified. +/// +/// When a node has been added or removed, this implementation does not visit the children, instead +/// it returns the hashprefix representing the node. This leads a more efficient implementation that does +/// not contain keys and values and stops at node level if the node itself has been added or removed. +/// +/// # Examples +/// +/// ``` +/// use std::rc::Rc; +/// use wnfs::{private::{Node, diff}, Link, Pair, MemoryBlockStore}; +/// +/// #[async_std::main] +/// async fn main() { +/// let store = &mut MemoryBlockStore::new(); +/// let mut main_node = Rc::new(Node::<[u8; 4], String>::default()); +/// for i in 0u32..3 { +/// main_node = main_node +/// .set(i.to_le_bytes(), i.to_string(), store) +/// .await +/// .unwrap(); +/// } +/// +/// let mut other_node = Rc::new(Node::<[u8; 4], String>::default()); +/// other_node = other_node +/// .set(0_u32.to_le_bytes(), 0_u32.to_string(), store) +/// .await +/// .unwrap(); +/// +/// let changes = diff::node_diff( +/// Link::from(Rc::clone(&main_node)), +/// Link::from(Rc::clone(&other_node)), +/// store, +/// ) +/// .await +/// .unwrap(); +/// +/// +/// println!("Changes {:#?}", changes); +/// } +/// ``` +#[async_recursion(?Send)] +pub async fn node_diff( + main_link: Link>>, + other_link: Link>>, + store: &mut B, +) -> Result> +where + K: DeserializeOwned + Clone + Eq + Hash + AsRef<[u8]>, + V: DeserializeOwned + Clone + Eq, + H: Hasher + Clone + 'static, + B: BlockStore, +{ + node_diff_helper(main_link, other_link, HashPrefix::default(), store).await +} + +#[async_recursion(?Send)] +pub async fn node_diff_helper( + main_link: Link>>, + other_link: Link>>, + hashprefix: HashPrefix, + store: &mut B, +) -> Result> +where + K: DeserializeOwned + Clone + Eq + Hash + AsRef<[u8]>, + V: DeserializeOwned + Clone + Eq, + H: Hasher + Clone + 'static, + B: BlockStore, +{ + // If Cids are available, check to see if they are equal so we can skip further comparisons. + if let (Some(cid), Some(cid2)) = (main_link.get_cid(), other_link.get_cid()) { + if cid == cid2 { + return Ok(vec![]); + } + } + + // Otherwise, get nodes from store. + let mut main_node = main_link + .resolve_owned_value(store) + .await? + .unwrap_or_clone()?; + + let mut other_node = other_link + .resolve_owned_value(store) + .await? + .unwrap_or_clone()?; + + let mut changes = vec![]; + for index in 0..HAMT_BITMASK_BIT_SIZE { + // Create hashprefix for child. + let mut hashprefix = hashprefix.clone(); + hashprefix.push(index as u8); + + match (main_node.bitmask[index], other_node.bitmask[index]) { + (true, false) => { + // Main has a value, other doesn't. + changes.extend(generate_add_or_remove_changes( + &main_node.pointers[main_node.get_value_index(index)], + ChangeType::Add, + hashprefix, + )); + } + (false, true) => { + // Main doesn't have a value, other does. + changes.extend(generate_add_or_remove_changes( + &other_node.pointers[other_node.get_value_index(index)], + ChangeType::Remove, + hashprefix, + )); + } + (true, true) => { + // Main and other have a value. They may be the same or different so we check. + let main_index = main_node.get_value_index(index); + let main_pointer = mem::take(main_node.pointers.get_mut(main_index).unwrap()); + + let other_index = other_node.get_value_index(index); + let other_pointer = mem::take(other_node.pointers.get_mut(other_index).unwrap()); + + changes.extend( + generate_modify_changes(main_pointer, other_pointer, hashprefix, store).await?, + ); + } + (false, false) => { /*No change */ } + } + } + + Ok(changes) +} + +fn generate_add_or_remove_changes( + node_pointer: &Pointer, + r#type: ChangeType, + hashprefix: HashPrefix, +) -> Vec +where + K: AsRef<[u8]>, + H: Hasher + Clone, +{ + match node_pointer { + Pointer::Values(values) => values + .iter() + .map(|Pair { key, .. }| { + let digest = H::hash(&key); + let hashprefix = HashPrefix::with_length(digest, digest.len() as u8 * 2); + NodeChange { r#type, hashprefix } + }) + .collect(), + Pointer::Link(_) => { + vec![NodeChange { r#type, hashprefix }] + } + } +} + +async fn generate_modify_changes( + main_pointer: Pointer, + other_pointer: Pointer, + hashprefix: HashPrefix, + store: &mut B, +) -> Result> +where + K: DeserializeOwned + Clone + Eq + Hash + AsRef<[u8]>, + V: DeserializeOwned + Clone + Eq, + H: Hasher + Clone + 'static, + B: BlockStore, +{ + match (main_pointer, other_pointer) { + (Pointer::Link(main_link), Pointer::Link(other_link)) => { + node_diff_helper(main_link, other_link, hashprefix, store).await + } + (Pointer::Values(main_values), Pointer::Values(other_values)) => { + let mut changes = vec![]; + let mut main_map = HashMap::<&K, &V>::default(); + let other_map = HashMap::<&K, &V>::from_iter( + other_values.iter().map(|Pair { key, value }| (key, value)), + ); + + for Pair { key, value } in &main_values { + match other_map.get(&key) { + Some(v) => { + if *v != value { + let digest = H::hash(&key); + let hashprefix = + HashPrefix::with_length(digest, digest.len() as u8 * 2); + changes.push(NodeChange { + r#type: ChangeType::Modify, + hashprefix, + }); + } + } + None => { + let digest = H::hash(&key); + let hashprefix = HashPrefix::with_length(digest, digest.len() as u8 * 2); + changes.push(NodeChange { + r#type: ChangeType::Add, + hashprefix, + }) + } + } + + main_map.insert(key, value); + } + + for Pair { key, .. } in &other_values { + if matches!(main_map.get(key), None) { + let digest = H::hash(&key); + let hashprefix = HashPrefix::with_length(digest, digest.len() as u8 * 2); + changes.push(NodeChange { + r#type: ChangeType::Remove, + hashprefix, + }) + } + } + + Ok(changes) + } + (Pointer::Values(main_values), Pointer::Link(other_link)) => { + let main_link = Link::from( + create_node_from_pairs::<_, _, H, _>(main_values, hashprefix.len(), store).await?, + ); + + node_diff_helper(main_link, other_link, hashprefix, store).await + } + (Pointer::Link(main_link), Pointer::Values(other_values)) => { + let other_link = Link::from( + create_node_from_pairs::<_, _, H, _>(other_values, hashprefix.len(), store).await?, + ); + + node_diff_helper(main_link, other_link, hashprefix, store).await + } + } +} + +async fn create_node_from_pairs( + values: Vec>, + hashprefix_length: usize, + store: &B, +) -> Result>> +where + K: DeserializeOwned + Clone + AsRef<[u8]>, + V: DeserializeOwned + Clone, + H: Hasher + Clone + 'static, +{ + let mut node = Rc::new(Node::<_, _, H>::default()); + for Pair { key, value } in values { + let digest = &H::hash(&key); + let hashnibbles = &mut HashNibbles::with_cursor(digest, hashprefix_length); + node = node.set_value(hashnibbles, key, value, store).await?; + } + Ok(node) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::{ChangeType::*, *}; + use crate::{ + private::{Node, MAX_HASH_NIBBLE_LENGTH}, + utils::{self, test_setup}, + }; + use helper::*; + use sha3::Sha3_256; + use std::rc::Rc; + + mod helper { + use crate::{utils, HashOutput, Hasher}; + use once_cell::sync::Lazy; + + pub(super) static HASH_KV_PAIRS: Lazy> = Lazy::new(|| { + vec![ + (utils::make_digest(&[0xA0]), "first"), + (utils::make_digest(&[0xA3]), "second"), + (utils::make_digest(&[0xA7]), "third"), + (utils::make_digest(&[0xAC]), "fourth"), + (utils::make_digest(&[0xAE]), "fifth"), + ] + }); + + #[derive(Debug, Clone)] + pub(super) struct MockHasher; + impl Hasher for MockHasher { + fn hash>(key: &K) -> HashOutput { + HASH_KV_PAIRS + .iter() + .find(|(_, v)| key.as_ref() == >::as_ref(v)) + .unwrap() + .0 + } + } + } + + #[async_std::test] + async fn can_diff_main_node_with_added_removed_pairs() { + let store = test_setup::init!(mut store); + + let mut main_node = Rc::new(Node::<[u8; 4], String>::default()); + for i in 0u32..3 { + main_node = main_node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let mut other_node = Rc::new(Node::<[u8; 4], String>::default()); + other_node = other_node + .set(0_u32.to_le_bytes(), 0_u32.to_string(), store) + .await + .unwrap(); + + let changes = node_diff( + Link::from(Rc::clone(&main_node)), + Link::from(Rc::clone(&other_node)), + store, + ) + .await + .unwrap(); + + let hashprefix_of_1 = HashPrefix::with_length( + Sha3_256::hash(&1_u32.to_le_bytes()), + MAX_HASH_NIBBLE_LENGTH as u8, + ); + + let hashprefix_of_2 = HashPrefix::with_length( + Sha3_256::hash(&2_u32.to_le_bytes()), + MAX_HASH_NIBBLE_LENGTH as u8, + ); + + assert_eq!( + changes, + vec![ + NodeChange { + r#type: Add, + hashprefix: hashprefix_of_2.clone() + }, + NodeChange { + r#type: Add, + hashprefix: hashprefix_of_1.clone(), + }, + ] + ); + + let changes = node_diff(Link::from(other_node), Link::from(main_node), store) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + NodeChange { + r#type: Remove, + hashprefix: hashprefix_of_2 + }, + NodeChange { + r#type: Remove, + hashprefix: hashprefix_of_1, + }, + ] + ); + } + + #[async_std::test] + async fn can_diff_main_node_with_no_changes() { + let store = test_setup::init!(mut store); + + let mut main_node = Rc::new(Node::<_, _>::default()); + for i in 0_u32..3 { + main_node = main_node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let mut other_node = Rc::new(Node::<_, _>::default()); + for i in 0_u32..3 { + other_node = other_node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let changes = node_diff(Link::from(main_node), Link::from(other_node), store) + .await + .unwrap(); + + assert!(changes.is_empty()); + } + + #[async_std::test] + async fn can_diff_nodes_with_different_structure_and_modified_changes() { + let store = test_setup::init!(mut store); + + // A node that adds the first 3 pairs of HASH_KV_PAIRS. + let mut other_node = Rc::new(Node::<_, _, MockHasher>::default()); + for (digest, kv) in HASH_KV_PAIRS.iter().take(3) { + other_node = other_node + .set_value( + &mut HashNibbles::new(digest), + kv.to_string(), + kv.to_string(), + store, + ) + .await + .unwrap(); + } + + // Another node that keeps the first pair, modify the second pair, removes the third pair, and adds the fourth and fifth pair. + let mut main_node = Rc::new(Node::<_, _, MockHasher>::default()); + main_node = main_node + .set_value( + &mut HashNibbles::new(&HASH_KV_PAIRS[0].0), + HASH_KV_PAIRS[0].1.to_string(), + HASH_KV_PAIRS[0].1.to_string(), + store, + ) + .await + .unwrap(); + + main_node = main_node + .set_value( + &mut HashNibbles::new(&HASH_KV_PAIRS[1].0), + HASH_KV_PAIRS[1].1.to_string(), + String::from("second_modified"), + store, + ) + .await + .unwrap(); + + for (digest, kv) in HASH_KV_PAIRS.iter().skip(3).take(2) { + main_node = main_node + .set_value( + &mut HashNibbles::new(digest), + kv.to_string(), + kv.to_string(), + store, + ) + .await + .unwrap(); + } + + let changes = node_diff( + Link::from(Rc::clone(&main_node)), + Link::from(Rc::clone(&other_node)), + store, + ) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + NodeChange { + r#type: Modify, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xA3, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + NodeChange { + r#type: Remove, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xA7, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + NodeChange { + r#type: Add, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xAC, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + NodeChange { + r#type: Add, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xAE, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + ] + ); + + let changes = node_diff(Link::from(other_node), Link::from(main_node), store) + .await + .unwrap(); + + assert_eq!( + changes, + vec![ + NodeChange { + r#type: Modify, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xA3, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + NodeChange { + r#type: Add, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xA7, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + NodeChange { + r#type: Remove, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xAC, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + NodeChange { + r#type: Remove, + hashprefix: HashPrefix::with_length( + utils::make_digest(&[0xAE, 0x00]), + MAX_HASH_NIBBLE_LENGTH as u8 + ), + }, + ] + ); + } +} + +#[cfg(test)] +mod proptests { + use super::*; + use crate::{ + private::strategies::{self, generate_kvs}, + utils::test_setup, + }; + use async_std::task; + use test_strategy::proptest; + + #[proptest] + fn add_remove_flip( + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs1: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs2: Vec<(String, u64)>, + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + + let node1 = strategies::node_from_kvs(kvs1, store).await.unwrap(); + let node2 = strategies::node_from_kvs(kvs2, store).await.unwrap(); + + let changes = node_diff( + Link::from(Rc::clone(&node1)), + Link::from(Rc::clone(&node2)), + store, + ) + .await + .unwrap(); + + let flipped_changes = node_diff(Link::from(node2), Link::from(node1), store) + .await + .unwrap(); + + assert_eq!(changes.len(), flipped_changes.len()); + for change in changes { + assert!(flipped_changes.iter().any(|c| match change.r#type { + ChangeType::Add => + c.r#type == ChangeType::Remove && c.hashprefix == change.hashprefix, + ChangeType::Remove => + c.r#type == ChangeType::Add && c.hashprefix == change.hashprefix, + ChangeType::Modify => + c.r#type == ChangeType::Modify && c.hashprefix == change.hashprefix, + })); + } + }); + } +} diff --git a/wnfs/src/private/hamt/hamt.rs b/wnfs/src/private/hamt/hamt.rs index d78f9114..26f45b20 100644 --- a/wnfs/src/private/hamt/hamt.rs +++ b/wnfs/src/private/hamt/hamt.rs @@ -1,5 +1,5 @@ -use std::{collections::BTreeMap, rc::Rc, str::FromStr}; - +use super::{diff, KeyValueChange, Node, NodeChange, HAMT_VERSION}; +use crate::{AsyncSerialize, BlockStore, Hasher, Link}; use anyhow::Result; use async_trait::async_trait; use libipld::{serde as ipld_serde, Ipld}; @@ -9,10 +9,8 @@ use serde::{ ser::Error as SerError, Deserialize, Deserializer, Serialize, Serializer, }; - -use crate::{AsyncSerialize, BlockStore}; - -use super::{Node, HAMT_VERSION}; +use sha3::Sha3_256; +use std::{collections::BTreeMap, fmt, hash::Hash, rc::Rc, str::FromStr}; //-------------------------------------------------------------------------------------------------- // Type Definitions @@ -31,9 +29,12 @@ use super::{Node, HAMT_VERSION}; /// let hamt = Hamt::::new(); /// println!("HAMT: {:?}", hamt); /// ``` -#[derive(Debug, Clone, PartialEq)] -pub struct Hamt { - pub root: Rc>, +#[derive(Debug, Clone)] +pub struct Hamt +where + H: Hasher, +{ + pub root: Rc>, pub version: Version, } @@ -41,7 +42,7 @@ pub struct Hamt { // Implementations //-------------------------------------------------------------------------------------------------- -impl Hamt { +impl Hamt { /// Creates a new empty HAMT. /// /// # Examples @@ -68,15 +69,119 @@ impl Hamt { /// use wnfs::private::{Hamt, Node}; /// /// let hamt = Hamt::::with_root(Rc::new(Node::default())); + /// /// println!("HAMT: {:?}", hamt); /// ``` - pub fn with_root(root: Rc>) -> Self { + pub fn with_root(root: Rc>) -> Self { Self { root, version: HAMT_VERSION, } } + /// Gets the difference between two HAMTs at the node level. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// use wnfs::{ + /// private::{Hamt, Node}, + /// MemoryBlockStore + /// }; + /// + /// #[async_std::main] + /// async fn main() { + /// let store = &mut MemoryBlockStore::default(); + /// + /// let main_hamt = Hamt::::with_root({ + /// let node = Rc::new(Node::default()); + /// let node = node.set("foo".into(), 400, store).await.unwrap(); + /// let node = node.set("bar".into(), 500, store).await.unwrap(); + /// node + /// }); + /// + /// let other_hamt = Hamt::::with_root({ + /// let node = Rc::new(Node::default()); + /// let node = node.set("foo".into(), 200, store).await.unwrap(); + /// let node = node.set("qux".into(), 600, store).await.unwrap(); + /// node + /// }); + /// + /// let node_diff = main_hamt.node_diff(&other_hamt, store).await.unwrap(); + /// + /// println!("node_diff: {:#?}", node_diff); + /// } + /// ``` + pub async fn node_diff( + &self, + other: &Self, + store: &mut B, + ) -> Result> + where + K: DeserializeOwned + Clone + fmt::Debug + Eq + Hash + AsRef<[u8]>, + V: DeserializeOwned + Clone + fmt::Debug + Eq, + H: Clone + fmt::Debug + 'static, + { + diff::node_diff( + Link::from(Rc::clone(&self.root)), + Link::from(Rc::clone(&other.root)), + store, + ) + .await + } + + /// Gets the difference between two HAMTs at the key-value level. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// use wnfs::{ + /// private::{Hamt, Node}, + /// MemoryBlockStore + /// }; + /// + /// #[async_std::main] + /// async fn main() { + /// let store = &mut MemoryBlockStore::default(); + /// + /// let main_hamt = Hamt::::with_root({ + /// let node = Rc::new(Node::default()); + /// let node = node.set("foo".into(), 400, store).await.unwrap(); + /// let node = node.set("bar".into(), 500, store).await.unwrap(); + /// node + /// }); + /// + /// let other_hamt = Hamt::::with_root({ + /// let node = Rc::new(Node::default()); + /// let node = node.set("foo".into(), 200, store).await.unwrap(); + /// let node = node.set("qux".into(), 600, store).await.unwrap(); + /// node + /// }); + /// + /// let kv_diff = main_hamt.kv_diff(&other_hamt, store).await.unwrap(); + /// + /// println!("kv_diff: {:#?}", kv_diff); + /// } + pub async fn kv_diff( + &self, + other: &Self, + store: &mut B, + ) -> Result>> + where + K: DeserializeOwned + Clone + fmt::Debug + Eq + Hash + AsRef<[u8]>, + V: DeserializeOwned + Clone + fmt::Debug + Eq, + H: Clone + fmt::Debug + 'static, + { + diff::kv_diff( + Link::from(Rc::clone(&self.root)), + Link::from(Rc::clone(&other.root)), + store, + ) + .await + } + async fn to_ipld(&self, store: &mut B) -> Result where K: Serialize, @@ -91,7 +196,7 @@ impl Hamt { } #[async_trait(?Send)] -impl AsyncSerialize for Hamt +impl AsyncSerialize for Hamt where K: Serialize, V: Serialize, @@ -148,12 +253,23 @@ where } } -impl Default for Hamt { +impl Default for Hamt { fn default() -> Self { Self::new() } } +impl PartialEq for Hamt +where + K: PartialEq, + V: PartialEq, + H: Hasher, +{ + fn eq(&self, other: &Self) -> bool { + self.root == other.root && self.version == other.version + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- diff --git a/wnfs/src/private/hamt/hash.rs b/wnfs/src/private/hamt/hash.rs index c4b19d4e..090b28bb 100644 --- a/wnfs/src/private/hamt/hash.rs +++ b/wnfs/src/private/hamt/hash.rs @@ -1,15 +1,17 @@ +use super::error::HamtError; +use crate::{utils, HashOutput, HASH_BYTE_SIZE}; use anyhow::{bail, Result}; use sha3::{Digest, Sha3_256}; - -use crate::{HashOutput, HASH_BYTE_SIZE}; - -use super::error::HamtError; +use std::fmt::Debug; //-------------------------------------------------------------------------------------------------- // Constants //-------------------------------------------------------------------------------------------------- -const MAX_CURSOR_DEPTH: usize = HASH_BYTE_SIZE * 2; +/// The number of nibbles in a [`HashOutput`][HashOutput]. +/// +/// [HashOutput]: crate::HashOutput +pub const MAX_HASH_NIBBLE_LENGTH: usize = HASH_BYTE_SIZE * 2; //-------------------------------------------------------------------------------------------------- // Type Definition @@ -39,29 +41,67 @@ pub trait Hasher { } /// HashNibbles is a wrapper around a byte slice that provides a cursor for traversing the nibbles. -#[derive(Debug, Clone)] -pub(super) struct HashNibbles<'a> { +#[derive(Clone)] +pub(crate) struct HashNibbles<'a> { pub digest: &'a HashOutput, cursor: usize, } +/// This represents the location of a intermediate or leaf node in the HAMT. +/// +/// It is based on the hash of the key with a length info for knowing how deep +/// to traverse the tree to find the intermediate or leaf node. +/// +/// # Examples +/// +/// ``` +/// use wnfs::{private::HashPrefix, utils}; +/// +/// let hashprefix = HashPrefix::with_length(utils::make_digest(&[0xff, 0x22]), 4); +/// +/// println!("{:?}", hashprefix); +/// ``` +#[derive(Clone, Default)] +pub struct HashPrefix { + pub digest: HashOutput, + length: u8, +} + +/// An iterator over the nibbles of a HashPrefix. +/// +/// # Examples +/// +/// ``` +/// use wnfs::{private::HashPrefix, utils}; +/// +/// let hashprefix = HashPrefix::with_length(utils::make_digest(&[0xff, 0x22]), 4); +/// for i in hashprefix.iter() { +/// println!("{}", i); +/// } +/// ``` +#[derive(Clone)] +pub struct HashPrefixIterator<'a> { + pub hashprefix: &'a HashPrefix, + cursor: u8, +} + //-------------------------------------------------------------------------------------------------- // Implementation //-------------------------------------------------------------------------------------------------- impl<'a> HashNibbles<'a> { /// Creates a new `HashNibbles` instance from a `[u8; 32]` hash. - pub fn new(digest: &'a HashOutput) -> HashNibbles<'a> { + pub(crate) fn new(digest: &'a HashOutput) -> HashNibbles<'a> { Self::with_cursor(digest, 0) } - /// Constructs hash nibbles with custom cursor index. - pub fn with_cursor(digest: &'a HashOutput, cursor: usize) -> HashNibbles<'a> { + /// Constructs a `HashNibbles` with custom cursor index. + pub(crate) fn with_cursor(digest: &'a HashOutput, cursor: usize) -> HashNibbles<'a> { Self { digest, cursor } } /// Gets the next nibble from the hash. - pub fn try_next(&mut self) -> Result { + pub(crate) fn try_next(&mut self) -> Result { if let Some(nibble) = self.next() { return Ok(nibble as usize); } @@ -70,7 +110,7 @@ impl<'a> HashNibbles<'a> { /// Gets the current cursor position. #[inline] - pub fn get_cursor(&self) -> usize { + pub(crate) fn get_cursor(&self) -> usize { self.cursor } } @@ -79,7 +119,7 @@ impl Iterator for HashNibbles<'_> { type Item = u8; fn next(&mut self) -> Option { - if self.cursor >= MAX_CURSOR_DEPTH { + if self.cursor >= MAX_HASH_NIBBLE_LENGTH { return None; } @@ -95,6 +135,20 @@ impl Iterator for HashNibbles<'_> { } } +impl Debug for HashNibbles<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut nibbles_str = String::new(); + for nibble in HashNibbles::with_cursor(self.digest, 0) { + nibbles_str.push_str(&format!("{nibble:1X}")); + } + + f.debug_struct("HashNibbles") + .field("hash", &nibbles_str) + .field("cursor", &self.cursor) + .finish() + } +} + impl Hasher for Sha3_256 { fn hash>(data: &D) -> HashOutput { let mut hasher = Self::default(); @@ -103,6 +157,179 @@ impl Hasher for Sha3_256 { } } +impl HashPrefix { + /// Creates a new `HashPrefix` instance from a `[u8; 32]` hash. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let hashprefix = HashPrefix::with_length(utils::make_digest(&[0xff, 0x22]), 4); + /// + /// println!("{:?}", hashprefix); + /// ``` + pub fn with_length(digest: HashOutput, length: u8) -> HashPrefix { + Self { digest, length } + } + + /// Pushes a nibble to the end of the hash. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let mut hashprefix = HashPrefix::default(); + /// for i in 0..16_u8 { + /// hashprefix.push(i); + /// } + /// + /// assert_eq!(hashprefix.len(), 16); + /// ``` + pub fn push(&mut self, nibble: u8) { + if self.length >= MAX_HASH_NIBBLE_LENGTH as u8 { + panic!("HashPrefix is full"); + } + + let offset = self.length as usize / 2; + let byte = self.digest[offset]; + let byte = if self.length as usize % 2 == 0 { + nibble << 4 + } else { + byte | (nibble & 0x0F) + }; + + self.digest[offset] = byte; + self.length += 1; + } + + /// Gets the length of the hash. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let mut hashprefix = HashPrefix::default(); + /// for i in 0..16_u8 { + /// hashprefix.push(i); + /// } + /// + /// assert_eq!(hashprefix.len(), 16); + /// ``` + #[inline(always)] + pub fn len(&self) -> usize { + self.length as usize + } + + /// Checks if the hash is empty. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let hashprefix = HashPrefix::default(); + /// assert!(hashprefix.is_empty()); + /// ``` + pub fn is_empty(&self) -> bool { + self.length == 0 + } + + /// Get the nibble at specified offset. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let mut hashprefix = HashPrefix::default(); + /// for i in 0..16_u8 { + /// hashprefix.push(i); + /// } + /// + /// assert_eq!(hashprefix.get(15), Some(0x0f)); + /// ``` + pub fn get(&self, index: u8) -> Option { + if index >= self.length { + return None; + } + + let byte = self.digest.get(index as usize / 2)?; + Some(if index % 2 == 0 { + byte >> 4 + } else { + byte & 0x0F + }) + } + + /// Creates an iterator over the nibbles of the hash. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let hashprefix = HashPrefix::with_length(utils::make_digest(&[0xff, 0x22]), 4); + /// for i in hashprefix.iter() { + /// println!("{}", i); + /// } + /// ``` + pub fn iter(&self) -> HashPrefixIterator { + HashPrefixIterator { + hashprefix: self, + cursor: 0, + } + } + + /// Checks if the HashPrefix is a prefix of some arbitrary byte slice. + /// + /// # Examples + /// + /// ``` + /// use wnfs::{private::HashPrefix, utils}; + /// + /// let hashprefix = HashPrefix::with_length(utils::make_digest(&[0xff, 0x22]), 4); + /// + /// assert!(hashprefix.is_prefix_of(&[0xff, 0x22, 0x33])); + /// ``` + pub fn is_prefix_of(&self, bytes: &[u8]) -> bool { + self == &HashPrefix::with_length(utils::make_digest(bytes), self.length) + } +} + +impl Debug for HashPrefix { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "0x")?; + for nibble in self.iter() { + write!(f, "{nibble:1X}")?; + } + + Ok(()) + } +} + +impl PartialEq for HashPrefix { + fn eq(&self, other: &Self) -> bool { + self.iter().eq(other.iter()) + } +} + +impl Iterator for HashPrefixIterator<'_> { + type Item = u8; + + fn next(&mut self) -> Option { + if self.cursor >= self.hashprefix.length { + return None; + } + + let byte = self.hashprefix.get(self.cursor)?; + self.cursor += 1; + Some(byte) + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- @@ -133,9 +360,28 @@ mod tests { // Exhaust the iterator. let _ = hashnibbles - .take(MAX_CURSOR_DEPTH - expected_nibbles.len()) + .take(MAX_HASH_NIBBLE_LENGTH - expected_nibbles.len()) .collect::>(); assert_eq!(hashnibbles.next(), None); } + + #[test] + fn can_push_and_get_nibbles_from_hashprefix() { + let mut hashprefix = HashPrefix::default(); + for i in 0..HASH_BYTE_SIZE { + hashprefix.push((i % 16) as u8); + hashprefix.push((15 - i % 16) as u8); + } + + assert!(!hashprefix.is_empty()); + + for i in 0..HASH_BYTE_SIZE { + assert_eq!(hashprefix.get(i as u8 * 2).unwrap(), (i % 16) as u8); + assert_eq!( + hashprefix.get(i as u8 * 2 + 1).unwrap(), + (15 - i % 16) as u8 + ); + } + } } diff --git a/wnfs/src/private/hamt/merge.rs b/wnfs/src/private/hamt/merge.rs new file mode 100644 index 00000000..bff81e09 --- /dev/null +++ b/wnfs/src/private/hamt/merge.rs @@ -0,0 +1,196 @@ +use super::{diff, ChangeType, Node}; +use crate::{BlockStore, FsError, Hasher, Link}; +use anyhow::Result; +use serde::de::DeserializeOwned; +use std::{hash::Hash, rc::Rc}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +/// Merges a node with another with the help of a resolver function. +pub async fn merge( + main_link: Link>>, + other_link: Link>>, + f: F, + store: &mut B, +) -> Result>> +where + F: Fn(&V, &V) -> Result, + K: DeserializeOwned + Eq + Clone + Hash + AsRef<[u8]>, + V: DeserializeOwned + Eq + Clone, + H: Hasher + Clone + 'static, +{ + let kv_changes = diff::kv_diff(main_link.clone(), other_link.clone(), store).await?; + + let main_node = main_link.resolve_owned_value(store).await?; + let other_node = other_link.resolve_owned_value(store).await?; + + let mut merge_node = Rc::clone(&main_node); + for change in kv_changes { + match change.r#type { + ChangeType::Remove => { + merge_node = merge_node + .set(change.key, change.other_value.unwrap(), store) + .await?; + } + ChangeType::Modify => { + let main_value = main_node + .get(&change.key, store) + .await? + .ok_or(FsError::KeyNotFoundInHamt)?; + + let other_value = other_node + .get(&change.key, store) + .await? + .ok_or(FsError::KeyNotFoundInHamt)?; + + merge_node = merge_node + .set(change.key, f(main_value, other_value)?, store) + .await?; + } + _ => (), + } + } + + Ok(merge_node) +} + +//-------------------------------------------------------------------------------------------------- +// Tests +//-------------------------------------------------------------------------------------------------- + +#[cfg(test)] +mod proptests { + use crate::{ + private::strategies::{self, generate_kvs}, + utils::test_setup, + Link, + }; + use async_std::task; + use std::{cmp, rc::Rc}; + use test_strategy::proptest; + + #[proptest(cases = 100)] + fn merge_associativity( + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs1: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs2: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs3: Vec<(String, u64)>, + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + + let node1 = strategies::node_from_kvs(kvs1, store).await.unwrap(); + let node2 = strategies::node_from_kvs(kvs2, store).await.unwrap(); + let node3 = strategies::node_from_kvs(kvs3, store).await.unwrap(); + + let merge_node_left_assoc = { + let tmp = super::merge( + Link::from(Rc::clone(&node1)), + Link::from(Rc::clone(&node2)), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap(); + + super::merge( + Link::from(tmp), + Link::from(Rc::clone(&node3)), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap() + }; + + let merge_node_right_assoc = { + let tmp = super::merge( + Link::from(node2), + Link::from(node3), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap(); + + super::merge( + Link::from(node1), + Link::from(tmp), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap() + }; + + assert_eq!(merge_node_left_assoc, merge_node_right_assoc); + }); + } + + #[proptest(cases = 100)] + fn merge_commutativity( + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs1: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs2: Vec<(String, u64)>, + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + + let node1 = strategies::node_from_kvs(kvs1, store).await.unwrap(); + let node2 = strategies::node_from_kvs(kvs2, store).await.unwrap(); + + let merge_node_1 = super::merge( + Link::from(Rc::clone(&node1)), + Link::from(Rc::clone(&node2)), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap(); + + let merge_node_2 = super::merge( + Link::from(node2), + Link::from(node1), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap(); + + assert_eq!(merge_node_1, merge_node_2); + }) + } + + #[proptest(cases = 100)] + fn merge_idempotency( + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs1: Vec<(String, u64)>, + #[strategy(generate_kvs("[a-z0-9]{1,3}", 0u64..1000, 0..100))] kvs2: Vec<(String, u64)>, + ) { + task::block_on(async { + let store = test_setup::init!(mut store); + + let node1 = strategies::node_from_kvs(kvs1, store).await.unwrap(); + let node2 = strategies::node_from_kvs(kvs2, store).await.unwrap(); + + let merge_node_1 = super::merge( + Link::from(Rc::clone(&node1)), + Link::from(Rc::clone(&node2)), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap(); + + let merge_node_2 = super::merge( + Link::from(Rc::clone(&merge_node_1)), + Link::from(node2), + |a, b| Ok(cmp::min(*a, *b)), + store, + ) + .await + .unwrap(); + + assert_eq!(merge_node_1, merge_node_2); + }) + } +} diff --git a/wnfs/src/private/hamt/mod.rs b/wnfs/src/private/hamt/mod.rs index 51a40ed5..4ba650c4 100644 --- a/wnfs/src/private/hamt/mod.rs +++ b/wnfs/src/private/hamt/mod.rs @@ -1,17 +1,21 @@ //! This implementation is based on [ipld_hamt](https://github.com/filecoin-project/ref-fvm/tree/master/ipld/hamt). mod constants; +pub mod diff; mod error; #[allow(clippy::module_inception)] mod hamt; mod hash; +mod merge; mod node; mod pointer; pub(crate) use constants::*; +pub use diff::*; pub use hamt::*; pub use hash::*; +pub use merge::*; pub use node::*; pub use pointer::*; diff --git a/wnfs/src/private/hamt/node.rs b/wnfs/src/private/hamt/node.rs index 4d125641..50f42db5 100644 --- a/wnfs/src/private/hamt/node.rs +++ b/wnfs/src/private/hamt/node.rs @@ -1,11 +1,17 @@ -use std::{fmt::Debug, marker::PhantomData, rc::Rc}; - -use crate::{private::HAMT_VALUES_BUCKET_SIZE, AsyncSerialize, BlockStore, HashOutput, Link}; +use super::{ + error::HamtError, + hash::{HashNibbles, Hasher}, + HashPrefix, Pair, Pointer, HAMT_BITMASK_BIT_SIZE, HAMT_BITMASK_BYTE_SIZE, +}; +use crate::{ + private::HAMT_VALUES_BUCKET_SIZE, utils::UnwrapOrClone, AsyncSerialize, BlockStore, FsError, + HashOutput, Link, +}; use anyhow::{bail, Result}; use async_recursion::async_recursion; use async_trait::async_trait; use bitvec::array::BitArray; - +use either::{Either, Either::*}; use futures::future::LocalBoxFuture; use libipld::{serde as ipld_serde, Ipld}; use log::debug; @@ -15,11 +21,12 @@ use serde::{ Deserializer, Serialize, Serializer, }; use sha3::Sha3_256; - -use super::{ - error::HamtError, - hash::{HashNibbles, Hasher}, - Pair, Pointer, HAMT_BITMASK_BIT_SIZE, HAMT_BITMASK_BYTE_SIZE, +use std::{ + collections::HashMap, + fmt::{self, Debug, Formatter}, + hash::Hash, + marker::PhantomData, + rc::Rc, }; //-------------------------------------------------------------------------------------------------- @@ -42,7 +49,7 @@ pub type BitMaskType = [u8; HAMT_BITMASK_BYTE_SIZE]; /// /// assert!(node.is_empty()); /// ``` -#[derive(Debug, Clone)] +#[derive(Clone)] pub struct Node where H: Hasher, @@ -249,7 +256,7 @@ where } /// Calculates the value index from the bitmask index. - fn get_value_index(&self, bit_index: usize) -> usize { + pub(crate) fn get_value_index(&self, bit_index: usize) -> usize { let shift_amount = HAMT_BITMASK_BIT_SIZE - bit_index; let mask = if shift_amount < HAMT_BITMASK_BIT_SIZE { let mut tmp = BitArray::::new([0xff, 0xff]); @@ -262,7 +269,7 @@ where (mask & self.bitmask).count_ones() } - fn set_value<'a, B: BlockStore>( + pub(crate) fn set_value<'a, B: BlockStore>( self: Rc, hashnibbles: &'a mut HashNibbles, key: K, @@ -283,7 +290,7 @@ where bit_index, value_index ); - let mut node = Rc::try_unwrap(self).unwrap_or_else(|rc| (*rc).clone()); + let mut node = self.unwrap_or_clone()?; // If the bit is not set yet, insert a new pointer. if !node.bitmask[bit_index] { @@ -343,7 +350,7 @@ where } #[async_recursion(?Send)] - async fn get_value<'a, B: BlockStore>( + pub(crate) async fn get_value<'a, B: BlockStore>( &'a self, hashnibbles: &mut HashNibbles, store: &B, @@ -375,7 +382,7 @@ where // It's internal and is only more complex because async_recursion doesn't work here #[allow(clippy::type_complexity)] - fn remove_value<'k, 'v, 'a, B: BlockStore>( + pub(crate) fn remove_value<'k, 'v, 'a, B: BlockStore>( self: Rc, hashnibbles: &'a mut HashNibbles, store: &'a B, @@ -396,7 +403,7 @@ where let value_index = self.get_value_index(bit_index); - let mut node = Rc::try_unwrap(self).unwrap_or_else(|rc| (*rc).clone()); + let mut node = self.unwrap_or_clone()?; let removed = match &mut node.pointers[value_index] { // If there is only one value, we can remove the entire pointer. @@ -432,7 +439,7 @@ where let child = Rc::clone(link.resolve_value(store).await?); let (child, removed) = child.remove_value(hashnibbles, store).await?; if removed.is_some() { - // If something has been deleted, we attempt toc canonicalize the pointer. + // If something has been deleted, we attempt to canonicalize the pointer. if let Some(pointer) = Pointer::Link(Link::from(child)).canonicalize(store).await? { @@ -452,6 +459,189 @@ where Ok((Rc::new(node), removed)) }) } + + /// Visits all the leaf nodes in the trie and calls the given function on each of them. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// use wnfs::{private::{Node, Pair}, utils, Hasher, MemoryBlockStore}; + /// + /// #[async_std::main] + /// async fn main() { + /// let store = &mut MemoryBlockStore::new(); + /// let mut node = Rc::new(Node::<[u8; 4], String>::default()); + /// for i in 0..99_u32 { + /// node = node + /// .set(i.to_le_bytes(), i.to_string(), store) + /// .await + /// .unwrap(); + /// } + /// + /// let keys = node + /// .flat_map(&|Pair { key, .. }| Ok(*key), store) + /// .await + /// .unwrap(); + /// + /// assert_eq!(keys.len(), 99); + /// } + /// ``` + #[async_recursion(?Send)] + pub async fn flat_map(&self, f: &F, store: &B) -> Result> + where + B: BlockStore, + F: Fn(&Pair) -> Result, + K: DeserializeOwned, + V: DeserializeOwned, + { + let mut items = >::new(); + for p in self.pointers.iter() { + match p { + Pointer::Values(values) => { + for pair in values { + items.push(f(pair)?); + } + } + Pointer::Link(link) => { + let child = link.resolve_value(store).await?; + items.extend(child.flat_map(f, store).await?); + } + } + } + + Ok(items) + } + + /// Given a hashprefix representing the path to a node in the trie. This function will + /// return the key-value pair or the intermediate node that the hashprefix points to. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// use sha3::Sha3_256; + /// use wnfs::{ + /// private::{Node, HashPrefix}, + /// utils, Hasher, MemoryBlockStore + /// }; + /// + /// #[async_std::main] + /// async fn main() { + /// let store = &mut MemoryBlockStore::new(); + /// + /// let mut node = Rc::new(Node::<[u8; 4], String>::default()); + /// for i in 0..100_u32 { + /// node = node + /// .set(i.to_le_bytes(), i.to_string(), store) + /// .await + /// .unwrap(); + /// } + /// + /// let hashprefix = HashPrefix::with_length(utils::make_digest(&[0x8C]), 2); + /// let result = node.get_node_at(&hashprefix, store).await.unwrap(); + /// + /// println!("Result: {:#?}", result); + /// } + /// ``` + #[async_recursion(?Send)] + pub async fn get_node_at<'a, B>( + &'a self, + hashprefix: &HashPrefix, + store: &B, + ) -> Result, &'a Rc>>> + where + K: DeserializeOwned + AsRef<[u8]>, + V: DeserializeOwned, + B: BlockStore, + { + self.get_node_at_helper(hashprefix, 0, store).await + } + + #[async_recursion(?Send)] + async fn get_node_at_helper<'a, B>( + &'a self, + hashprefix: &HashPrefix, + index: u8, + store: &B, + ) -> Result, &'a Rc>>> + where + K: DeserializeOwned + AsRef<[u8]>, + V: DeserializeOwned, + B: BlockStore, + { + let bit_index = hashprefix + .get(index) + .ok_or(FsError::InvalidHashPrefixIndex)? as usize; + + if !self.bitmask[bit_index] { + return Ok(None); + } + + let value_index = self.get_value_index(bit_index); + match &self.pointers[value_index] { + Pointer::Values(values) => Ok({ + values + .iter() + .find(|p| hashprefix.is_prefix_of(&H::hash(&p.key))) + .map(Left) + }), + Pointer::Link(link) => { + let child = link.resolve_value(store).await?; + if index == hashprefix.len() as u8 - 1 { + return Ok(Some(Right(child))); + } + + child.get_node_at_helper(hashprefix, index + 1, store).await + } + } + } + + /// Generates a hashmap from the node. + /// + /// # Examples + /// + /// ``` + /// use std::rc::Rc; + /// use sha3::Sha3_256; + /// use wnfs::{private::Node, Hasher, MemoryBlockStore}; + /// + /// #[async_std::main] + /// async fn main() { + /// let store = &mut MemoryBlockStore::new(); + /// + /// let mut node = Rc::new(Node::<[u8; 4], String>::default()); + /// for i in 0..100_u32 { + /// node = node + /// .set(i.to_le_bytes(), i.to_string(), store) + /// .await + /// .unwrap(); + /// } + /// + /// let map = node.to_hashmap(store).await.unwrap(); + /// + /// assert_eq!(map.len(), 100); + /// } + /// ``` + pub async fn to_hashmap(&self, store: &B) -> Result> + where + K: DeserializeOwned + Clone + Eq + Hash, + V: DeserializeOwned + Clone, + { + let mut map = HashMap::new(); + let key_values = self + .flat_map( + &|Pair { key, value }| Ok((key.clone(), value.clone())), + store, + ) + .await?; + + for (key, value) in key_values { + map.insert(key, value); + } + + Ok(map) + } } impl Node { @@ -564,6 +754,25 @@ where } } +impl Debug for Node +where + K: Debug, + V: Debug, + H: Hasher + Debug, +{ + fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result { + let mut bitmask_str = String::new(); + for i in self.bitmask.as_raw_slice().iter().rev() { + bitmask_str.push_str(&format!("{i:08b}")); + } + + f.debug_struct("Node") + .field("bitmask", &bitmask_str) + .field("pointers", &self.pointers) + .finish() + } +} + //-------------------------------------------------------------------------------------------------- // Tests //-------------------------------------------------------------------------------------------------- @@ -571,41 +780,45 @@ where #[cfg(test)] mod tests { use super::*; - use crate::{HashOutput, MemoryBlockStore}; - use lazy_static::lazy_static; - use test_log::test; - - fn digest(bytes: &[u8]) -> HashOutput { - let mut nibbles = [0u8; 32]; - nibbles[..bytes.len()].copy_from_slice(bytes); - nibbles - } - - lazy_static! { - static ref HASH_KV_PAIRS: Vec<(HashOutput, &'static str)> = vec![ - (digest(&[0xE0]), "first"), - (digest(&[0xE1]), "second"), - (digest(&[0xE2]), "third"), - (digest(&[0xE3]), "fourth"), - ]; - } - - #[derive(Debug, Clone)] - struct MockHasher; - impl Hasher for MockHasher { - fn hash>(key: &K) -> HashOutput { - let s = std::str::from_utf8(key.as_ref()).unwrap(); - HASH_KV_PAIRS.iter().find(|(_, v)| s == *v).unwrap().0 + use crate::{ + utils::{self, test_setup}, + MemoryBlockStore, + }; + use helper::*; + + mod helper { + use crate::{utils, HashOutput, Hasher}; + use once_cell::sync::Lazy; + + pub(super) static HASH_KV_PAIRS: Lazy> = Lazy::new(|| { + vec![ + (utils::make_digest(&[0xE0]), "first"), + (utils::make_digest(&[0xE1]), "second"), + (utils::make_digest(&[0xE2]), "third"), + (utils::make_digest(&[0xE3]), "fourth"), + ] + }); + + #[derive(Debug, Clone)] + pub(super) struct MockHasher; + impl Hasher for MockHasher { + fn hash>(key: &K) -> HashOutput { + HASH_KV_PAIRS + .iter() + .find(|(_, v)| key.as_ref() == >::as_ref(v)) + .unwrap() + .0 + } } } - #[test(async_std::test)] + #[async_std::test] async fn get_value_fetches_deeply_linked_value() { let store = &mut MemoryBlockStore::default(); // Insert 4 values to trigger the creation of a linked node. let mut working_node = Rc::new(Node::::default()); - for (digest, kv) in HASH_KV_PAIRS.iter() { + for (digest, kv) in HASH_KV_PAIRS.iter().take(4) { let hashnibbles = &mut HashNibbles::new(digest); working_node = working_node .set_value(hashnibbles, kv.to_string(), kv.to_string(), store) @@ -614,7 +827,7 @@ mod tests { } // Get the values. - for (digest, kv) in HASH_KV_PAIRS.iter() { + for (digest, kv) in HASH_KV_PAIRS.iter().take(4) { let hashnibbles = &mut HashNibbles::new(digest); let value = working_node.get_value(hashnibbles, store).await.unwrap(); @@ -622,13 +835,13 @@ mod tests { } } - #[test(async_std::test)] + #[async_std::test] async fn remove_value_canonicalizes_linked_node() { let store = &mut MemoryBlockStore::default(); // Insert 4 values to trigger the creation of a linked node. let mut working_node = Rc::new(Node::::default()); - for (digest, kv) in HASH_KV_PAIRS.iter() { + for (digest, kv) in HASH_KV_PAIRS.iter().take(4) { let hashnibbles = &mut HashNibbles::new(digest); working_node = working_node .set_value(hashnibbles, kv.to_string(), kv.to_string(), store) @@ -662,7 +875,7 @@ mod tests { assert!(value.is_none()); } - #[test(async_std::test)] + #[async_std::test] async fn set_value_splits_when_bucket_threshold_reached() { let store = &mut MemoryBlockStore::default(); @@ -707,7 +920,7 @@ mod tests { } } - #[test(async_std::test)] + #[async_std::test] async fn get_value_index_gets_correct_index() { let store = &mut MemoryBlockStore::default(); let hash_expected_idx_samples = [ @@ -731,7 +944,7 @@ mod tests { let mut working_node = Rc::new(Node::::default()); for (hash, expected_idx) in hash_expected_idx_samples.into_iter() { - let bytes = digest(&hash[..]); + let bytes = utils::make_digest(&hash[..]); let hashnibbles = &mut HashNibbles::new(&bytes); working_node = working_node @@ -754,7 +967,7 @@ mod tests { } } - #[test(async_std::test)] + #[async_std::test] async fn node_can_insert_pair_and_retrieve() { let store = MemoryBlockStore::default(); let node = Rc::new(Node::::default()); @@ -766,7 +979,7 @@ mod tests { assert_eq!(value, &(10, 0.315)); } - #[test(async_std::test)] + #[async_std::test] async fn node_is_same_with_irrelevant_remove() { // These two keys' hashes have the same first nibble (7) let insert_key: String = "GL59 Tg4phDb bv".into(); @@ -781,7 +994,7 @@ mod tests { assert_eq!(node0.count_values().unwrap(), 1); } - #[test(async_std::test)] + #[async_std::test] async fn node_history_independence_regression() { let store = &mut MemoryBlockStore::default(); @@ -807,19 +1020,81 @@ mod tests { assert_eq!(cid1, cid2); } + + #[async_std::test] + async fn can_map_over_leaf_nodes() { + let store = test_setup::init!(mut store); + + let mut node = Rc::new(Node::<[u8; 4], String>::default()); + for i in 0..99_u32 { + node = node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let keys = node + .flat_map(&|Pair { key, .. }| Ok(*key), store) + .await + .unwrap(); + + assert_eq!(keys.len(), 99); + } + + #[async_std::test] + async fn can_fetch_node_at_hashprefix() { + let store = test_setup::init!(mut store); + + let mut node = Rc::new(Node::::default()); + for (digest, kv) in HASH_KV_PAIRS.iter() { + let hashnibbles = &mut HashNibbles::new(digest); + node = node + .set_value(hashnibbles, kv.to_string(), kv.to_string(), store) + .await + .unwrap(); + } + + for (digest, kv) in HASH_KV_PAIRS.iter().take(4) { + let hashprefix = HashPrefix::with_length(*digest, 2); + let result = node.get_node_at(&hashprefix, store).await.unwrap(); + let (key, value) = (kv.to_string(), kv.to_string()); + assert_eq!(result, Some(Either::Left(&Pair { key, value }))); + } + + let hashprefix = HashPrefix::with_length(utils::make_digest(&[0xE0]), 1); + let result = node.get_node_at(&hashprefix, store).await.unwrap(); + + assert!(matches!(result, Some(Either::Right(_)))); + } + + #[async_std::test] + async fn can_generate_hashmap_from_node() { + let store = test_setup::init!(mut store); + + let mut node = Rc::new(Node::<[u8; 4], String>::default()); + const NUM_VALUES: u32 = 1000; + for i in (u32::MAX - NUM_VALUES..u32::MAX).rev() { + node = node + .set(i.to_le_bytes(), i.to_string(), store) + .await + .unwrap(); + } + + let map = node.to_hashmap(store).await.unwrap(); + assert_eq!(map.len(), NUM_VALUES as usize); + for i in (u32::MAX - NUM_VALUES..u32::MAX).rev() { + assert_eq!(map.get(&i.to_le_bytes()).unwrap(), &i.to_string()); + } + } } #[cfg(test)] mod proptests { - - use crate::private::hamt::strategies::*; + use super::*; + use crate::{dagcbor, private::hamt::strategies::*, MemoryBlockStore}; use proptest::prelude::*; use test_strategy::proptest; - use crate::{dagcbor, MemoryBlockStore}; - - use super::*; - fn small_key() -> impl Strategy { (0..1000).prop_map(|i| format!("key {i}")) } @@ -835,7 +1110,7 @@ mod proptests { ) { async_std::task::block_on(async move { let store = &mut MemoryBlockStore::default(); - let node = node_from_operations(operations, store).await.unwrap(); + let node = node_from_operations(&operations, store).await.unwrap(); let node = node.set(key.clone(), value, store).await.unwrap(); let cid1 = store.put_async_serializable(&node).await.unwrap(); @@ -857,7 +1132,7 @@ mod proptests { ) { async_std::task::block_on(async move { let store = &mut MemoryBlockStore::default(); - let node = node_from_operations(operations, store).await.unwrap(); + let node = node_from_operations(&operations, store).await.unwrap(); let (node, _) = node.remove(&key, store).await.unwrap(); let cid1 = store.put_async_serializable(&node).await.unwrap(); @@ -878,7 +1153,7 @@ mod proptests { ) { async_std::task::block_on(async move { let store = &mut MemoryBlockStore::default(); - let node = node_from_operations(operations, store).await.unwrap(); + let node = node_from_operations(&operations, store).await.unwrap(); let encoded_node = dagcbor::async_encode(&node, store).await.unwrap(); let decoded_node = dagcbor::decode::>(encoded_node.as_ref()).unwrap(); @@ -899,8 +1174,8 @@ mod proptests { let store = &mut MemoryBlockStore::default(); - let node1 = node_from_operations(original, store).await.unwrap(); - let node2 = node_from_operations(shuffled, store).await.unwrap(); + let node1 = node_from_operations(&original, store).await.unwrap(); + let node2 = node_from_operations(&shuffled, store).await.unwrap(); let cid1 = store.put_async_serializable(&node1).await.unwrap(); let cid2 = store.put_async_serializable(&node2).await.unwrap(); @@ -919,9 +1194,9 @@ mod proptests { ) { let (original, shuffled) = pair; - let map1 = hash_map_from_operations(original); - let map2 = hash_map_from_operations(shuffled); + let map1 = HashMap::from(&original); + let map2 = HashMap::from(&shuffled); - assert_eq!(map1, map2); + prop_assert_eq!(map1, map2); } } diff --git a/wnfs/src/private/hamt/pointer.rs b/wnfs/src/private/hamt/pointer.rs index 13ceabf1..089a47ef 100644 --- a/wnfs/src/private/hamt/pointer.rs +++ b/wnfs/src/private/hamt/pointer.rs @@ -79,7 +79,7 @@ impl Pointer { { match self { Pointer::Link(link) => { - let node = link.get_owned_value(store).await?; + let node = link.resolve_owned_value(store).await?; match node.pointers.len() { 0 => Ok(None), 1 if matches!(node.pointers[0], Pointer::Values(_)) => { diff --git a/wnfs/src/private/hamt/strategies/changes.rs b/wnfs/src/private/hamt/strategies/changes.rs new file mode 100644 index 00000000..9686d1fa --- /dev/null +++ b/wnfs/src/private/hamt/strategies/changes.rs @@ -0,0 +1,99 @@ +#![cfg(test)] + +use super::{operations, Operations}; +use crate::{private::Node, BlockStore}; +use anyhow::Result; +use proptest::{collection::vec, strategy::Strategy}; +use serde::de::DeserializeOwned; +use std::{collections::HashMap, fmt::Debug, rc::Rc}; + +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + +#[derive(Debug, Clone)] +pub(crate) enum Change { + Add(K, V), + Remove(K), + Modify(K, V), +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(crate) fn generate_changes( + value_gen: impl Strategy, + pairs: Vec<(K, V)>, +) -> impl Strategy>> { + let rngs = vec((0..=3, value_gen), pairs.len()); + rngs.prop_map(move |randoms| { + pairs + .clone() + .into_iter() + .zip(randoms.into_iter()) + .filter(|(_, (num, _))| *num != 0) + .map(|((k, _), (num, val))| match num { + 1 => Change::Add(k, val), + 2 => Change::Remove(k), + 3 => Change::Modify(k, val), + _ => unreachable!(), + }) + .collect::>() + }) +} + +pub(crate) fn generate_ops_and_changes( +) -> impl Strategy, Vec>)> { + operations("[a-z0-9]{1,3}", 0..1000u64, 1..20).prop_flat_map(|ops| { + let map = HashMap::from(&ops); + let pairs = map.into_iter().collect::>(); + generate_changes(1000..2000_u64, pairs).prop_map(move |changes| (ops.clone(), changes)) + }) +} + +pub(crate) async fn apply_changes( + mut node: Rc>, + changes: &Vec>, + store: &B, +) -> Result>> +where + K: Debug + Clone + AsRef<[u8]> + DeserializeOwned, + V: Debug + Clone + DeserializeOwned, + B: BlockStore, +{ + for change in changes { + match change { + Change::Add(k, v) => { + node = node.set(k.clone(), v.clone(), store).await?; + } + Change::Remove(k) => { + (node, _) = node.remove(k, store).await?; + } + Change::Modify(k, v) => { + node = node.set(k.clone(), v.clone(), store).await?; + } + } + } + + Ok(node) +} + +pub(crate) async fn prepare_node( + mut node: Rc>, + changes: &Vec>, + store: &B, +) -> Result>> +where + K: Debug + Clone + AsRef<[u8]> + DeserializeOwned, + V: Debug + Clone + DeserializeOwned, + B: BlockStore, +{ + for change in changes { + if let Change::Add(k, _) = change { + (node, _) = node.remove(k, store).await?; + } + } + + Ok(node) +} diff --git a/wnfs/src/private/hamt/strategies/kv.rs b/wnfs/src/private/hamt/strategies/kv.rs new file mode 100644 index 00000000..2d46e769 --- /dev/null +++ b/wnfs/src/private/hamt/strategies/kv.rs @@ -0,0 +1,43 @@ +#![cfg(test)] + +use crate::{private::Node, BlockStore}; +use anyhow::Result; +use proptest::{collection::vec, sample::SizeRange, strategy::Strategy}; +use serde::{de::DeserializeOwned, Serialize}; +use std::{collections::HashMap, fmt::Debug, hash::Hash, rc::Rc}; + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + +pub(crate) fn generate_kvs( + key: impl Strategy, + value: impl Strategy, + size: impl Into, +) -> impl Strategy> +where + K: Eq + Hash, +{ + vec((key, value), size).prop_map(|vec| { + vec.into_iter() + .collect::>() + .into_iter() + .collect() + }) +} + +pub(crate) async fn node_from_kvs( + pairs: Vec<(K, V)>, + store: &mut impl BlockStore, +) -> Result>> +where + K: DeserializeOwned + Serialize + Clone + Debug + AsRef<[u8]>, + V: DeserializeOwned + Serialize + Clone + Debug, +{ + let mut node: Rc> = Rc::new(Node::default()); + for (k, v) in pairs { + node = node.set(k, v, store).await?; + } + + Ok(node) +} diff --git a/wnfs/src/private/hamt/strategies/mod.rs b/wnfs/src/private/hamt/strategies/mod.rs new file mode 100644 index 00000000..ef1c6a99 --- /dev/null +++ b/wnfs/src/private/hamt/strategies/mod.rs @@ -0,0 +1,9 @@ +mod changes; +mod kv; +mod operations; + +#[cfg(test)] +pub(crate) use changes::*; +#[cfg(test)] +pub(crate) use kv::*; +pub use operations::*; diff --git a/wnfs/src/private/hamt/strategies.rs b/wnfs/src/private/hamt/strategies/operations.rs similarity index 84% rename from wnfs/src/private/hamt/strategies.rs rename to wnfs/src/private/hamt/strategies/operations.rs index a93e1c68..7936fa87 100644 --- a/wnfs/src/private/hamt/strategies.rs +++ b/wnfs/src/private/hamt/strategies/operations.rs @@ -4,6 +4,10 @@ use proptest::{collection::*, prelude::*, strategy::Shuffleable}; use serde::{de::DeserializeOwned, Serialize}; use std::{collections::HashMap, fmt::Debug, hash::Hash, rc::Rc}; +//-------------------------------------------------------------------------------------------------- +// Types +//-------------------------------------------------------------------------------------------------- + /// Represents an operation that can be performed on a map-like data structure. /// /// # Examples @@ -24,6 +28,27 @@ pub enum Operation { Remove(K), } +/// A list of operations that can be applied to a map-like data structure. +/// +/// # Examples +/// +/// ``` +/// use wnfs::private::hamt::strategies::{self, Operation, Operations}; +/// use wnfs::utils::Sampleable; +/// use proptest::{arbitrary::any, test_runner::TestRunner}; +/// +/// let mut runner = &mut TestRunner::deterministic(); +/// let ops = strategies::operations(any::<[u8; 32]>(), any::(), 2).sample(runner); +/// +/// assert_eq!(ops.0.len(), 2); +/// ``` +#[derive(Debug, Clone)] +pub struct Operations(pub Vec>); + +//-------------------------------------------------------------------------------------------------- +// Implementations +//-------------------------------------------------------------------------------------------------- + impl Operation { fn can_be_swapped_with(&self, other: &Operation) -> bool where @@ -57,23 +82,6 @@ impl Operation { } } -/// A list of operations that can be applied to a map-like data structure. -/// -/// # Examples -/// -/// ``` -/// use wnfs::private::hamt::strategies::{self, Operation, Operations}; -/// use wnfs::utils::Sampleable; -/// use proptest::{arbitrary::any, test_runner::TestRunner}; -/// -/// let mut runner = &mut TestRunner::deterministic(); -/// let ops = strategies::operations(any::<[u8; 32]>(), any::(), 2).sample(runner); -/// -/// assert_eq!(ops.0.len(), 2); -/// ``` -#[derive(Debug, Clone)] -pub struct Operations(pub Vec>); - impl Shuffleable for Operations { fn shuffle_len(&self) -> usize { self.0.len() @@ -122,6 +130,31 @@ impl Shuffleable for Operations { } } +impl From<&Operations> for HashMap +where + K: Hash + Eq + Clone, + V: Clone, +{ + fn from(ops: &Operations) -> Self { + let mut map = HashMap::default(); + for op in &ops.0 { + match op { + Operation::Insert(key, value) => { + map.insert(key.clone(), value.clone()); + } + Operation::Remove(key) => { + map.remove(key); + } + } + } + map + } +} + +//-------------------------------------------------------------------------------------------------- +// Functions +//-------------------------------------------------------------------------------------------------- + /// This creates a node from a list of operations. /// /// # Examples @@ -138,13 +171,13 @@ impl Shuffleable for Operations { /// let ops = strategies::operations(any::<[u8; 32]>(), any::(), 10).sample(runner); /// /// let store = &mut MemoryBlockStore::new(); -/// let node = strategies::node_from_operations(ops, store).await.unwrap(); +/// let node = strategies::node_from_operations(&ops, store).await.unwrap(); /// /// println!("{:?}", node); /// } /// ``` pub async fn node_from_operations( - operations: Operations, + operations: &Operations, store: &mut B, ) -> Result>> where @@ -152,13 +185,13 @@ where V: DeserializeOwned + Serialize + Clone + Debug, { let mut node: Rc> = Rc::new(Node::default()); - for op in operations.0 { + for op in &operations.0 { match op { Operation::Insert(key, value) => { - node = node.set(key.clone(), value, store).await?; + node = node.set(key.clone(), value.clone(), store).await?; } Operation::Remove(key) => { - (node, _) = node.remove(&key, store).await?; + (node, _) = node.remove(key, store).await?; } }; } @@ -166,38 +199,6 @@ where Ok(node) } -/// Create a hashmap based on provided operations. -/// -/// # Examples -/// -/// ``` -/// use wnfs::private::hamt::strategies::{self, Operation, Operations}; -/// use wnfs::utils::Sampleable; -/// use proptest::{arbitrary::any, test_runner::TestRunner}; -/// -/// let mut runner = &mut TestRunner::deterministic(); -/// let ops = strategies::operations(any::<[u8; 32]>(), any::(), 10).sample(runner); -/// let hash_map = strategies::hash_map_from_operations(ops); -/// -/// println!("{:?}", hash_map); -/// ``` -pub fn hash_map_from_operations( - operations: Operations, -) -> HashMap { - let mut map = HashMap::default(); - for op in operations.0 { - match op { - Operation::Insert(key, value) => { - map.insert(key, value); - } - Operation::Remove(key) => { - map.remove(&key); - } - } - } - map -} - /// Creates an insert or remove operation strategy based on the key and value provided. /// /// # Examples diff --git a/wnfs/src/private/key.rs b/wnfs/src/private/key.rs index f00fe191..55413bc1 100644 --- a/wnfs/src/private/key.rs +++ b/wnfs/src/private/key.rs @@ -167,6 +167,7 @@ mod proptests { use super::*; use proptest::{ prelude::any, + prop_assert_eq, test_runner::{RngAlgorithm, TestRng}, }; use test_strategy::proptest; @@ -183,6 +184,6 @@ mod proptests { let encrypted = key.encrypt(&Key::generate_nonce(rng), &data).unwrap(); let decrypted = key.decrypt(&encrypted).unwrap(); - assert_eq!(decrypted, data); + prop_assert_eq!(decrypted, data); } } diff --git a/wnfs/src/private/namefilter/bloomfilter.rs b/wnfs/src/private/namefilter/bloomfilter.rs index b387a537..b30c1b1c 100644 --- a/wnfs/src/private/namefilter/bloomfilter.rs +++ b/wnfs/src/private/namefilter/bloomfilter.rs @@ -27,7 +27,7 @@ use crate::utils::ByteArrayVisitor; /// /// assert!(filter.contains(&[0xF5u8; 32])); /// ``` -#[derive(Clone, PartialEq, Eq, PartialOrd)] +#[derive(Clone, PartialEq, Eq, PartialOrd, Hash)] pub struct BloomFilter { pub(super) bits: BitArray<[u8; N]>, } @@ -314,6 +314,7 @@ mod tests { #[cfg(test)] mod proptests { + use proptest::prop_assert_eq; use test_strategy::proptest; use super::HashIndexIterator; @@ -327,7 +328,7 @@ mod proptests { .collect::>(); for (indices, count) in indices { - assert_eq!(indices.len(), count); + prop_assert_eq!(indices.len(), count); } } } diff --git a/wnfs/src/public/directory.rs b/wnfs/src/public/directory.rs index 65ee58fe..1aae1eae 100644 --- a/wnfs/src/public/directory.rs +++ b/wnfs/src/public/directory.rs @@ -641,7 +641,7 @@ impl PublicDirectory { // Remove the entry from its parent directory let removed_node = match directory.userland.remove(node_name) { - Some(link) => link.get_owned_value(store).await?, + Some(link) => link.resolve_owned_value(store).await?, None => bail!(FsError::NotFound), };