Skip to content

Commit

Permalink
feat(hamt): diff and merge implementation (#94)
Browse files Browse the repository at this point in the history
* Implement node diff and merge

- lean diff method that focuses on the tree
- exhaustive diff that holds a copy of changed key value pairs

* Remove unnecessary spacing

* Add unittests and merge impl

* Fix HAMT PartialEq issue

* Basic proptests

- Add some docs
- Not satisfied with the proptests yet

* Add more proptests

* Fix `get_node_at`, ...

- Implement merge for `Node<k, V, H>`
- Add more proptests, unittests and docs

* Fix proptests

- Remove hashbrown crate
Co-authored-by: Philipp Krüger <[email protected]>

* Fix benches

* Fix tests and so on

- Simplify tests
- UnwrapOrClone trait
- Prefer once_cell
- Remove depth param from diff

* Remove version checks and so on

- Remove version checks in hamt
- CHange HashKey to HashPrefix
  • Loading branch information
appcypher authored Jan 6, 2023
1 parent 60d4c24 commit 883b3ab
Show file tree
Hide file tree
Showing 30 changed files with 2,690 additions and 204 deletions.
20 changes: 8 additions & 12 deletions wnfs-bench/hamt.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ fn node_set(c: &mut Criterion) {
let mut store = MemoryBlockStore::default();
let operations = operations(any::<[u8; 32]>(), any::<u64>(), 1_000_000).sample(&mut runner);
let node =
&async_std::task::block_on(async { node_from_operations(operations, &mut store).await })
&async_std::task::block_on(async { node_from_operations(&operations, &mut store).await })
.expect("Couldn't setup HAMT node from operations");

let store = Arc::new(store);
Expand Down Expand Up @@ -54,7 +54,7 @@ fn node_set_consecutive(c: &mut Criterion) {
let operations =
operations(any::<[u8; 32]>(), any::<u64>(), 1000).sample(&mut runner);
let node = async_std::task::block_on(async {
node_from_operations(operations, &mut store).await
node_from_operations(&operations, &mut store).await
})
.expect("Couldn't setup HAMT node from operations");

Expand All @@ -76,16 +76,14 @@ fn node_load_get(c: &mut Criterion) {
let cid = async_std::task::block_on(async {
let mut node = Rc::new(<Node<_, _>>::default());
for i in 0..50 {
node = node.set(i.to_string(), i, &mut store).await.unwrap();
node = node.set(i.to_string(), i, &store).await.unwrap();
}

let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store)
.await
.unwrap();

let cid = store.put_serializable(&encoded_hamt).await.unwrap();

cid
store.put_serializable(&encoded_hamt).await.unwrap()
});

c.bench_function("node load and get", |b| {
Expand All @@ -110,16 +108,14 @@ fn node_load_remove(c: &mut Criterion) {
let cid = async_std::task::block_on(async {
let mut node = Rc::new(<Node<_, _>>::default());
for i in 0..50 {
node = node.set(i.to_string(), i, &mut store).await.unwrap();
node = node.set(i.to_string(), i, &store).await.unwrap();
}

let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store)
.await
.unwrap();

let cid = store.put_serializable(&encoded_hamt).await.unwrap();

cid
store.put_serializable(&encoded_hamt).await.unwrap()
});

c.bench_function("node load and remove", |b| {
Expand All @@ -142,7 +138,7 @@ fn hamt_load_decode(c: &mut Criterion) {
let (cid, bytes) = async_std::task::block_on(async {
let mut node = Rc::new(<Node<_, _>>::default());
for i in 0..50 {
node = node.set(i.to_string(), i, &mut store).await.unwrap();
node = node.set(i.to_string(), i, &store).await.unwrap();
}

let encoded_hamt = dagcbor::async_encode(&Hamt::with_root(node), &mut store)
Expand Down Expand Up @@ -176,7 +172,7 @@ fn hamt_set_encode(c: &mut Criterion) {
},
|(mut store, mut node)| async move {
for i in 0..50 {
node = node.set(i.to_string(), i, &mut store).await.unwrap();
node = node.set(i.to_string(), i, &store).await.unwrap();
}

let hamt = Hamt::with_root(node);
Expand Down
6 changes: 4 additions & 2 deletions wnfs-bench/namefilter.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,9 @@ fn add(c: &mut Criterion) {
},
|(mut namefilter, elements)| {
for element in elements {
black_box(namefilter.add(&element));
namefilter.add(&element);
}
black_box(&namefilter);
},
BatchSize::SmallInput,
)
Expand Down Expand Up @@ -54,7 +55,8 @@ fn saturate(c: &mut Criterion) {
namefilter
},
|mut namefilter| {
black_box(namefilter.saturate());
namefilter.saturate();
black_box(&namefilter);
},
BatchSize::SmallInput,
)
Expand Down
7 changes: 4 additions & 3 deletions wnfs/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,14 @@ async-stream = "0.3"
async-trait = "0.1"
bitvec = { version = "1.0", features = ["serde"] }
chrono = { version = "0.4", default-features = false, features = ["clock", "std"] }
either = "1.8"
futures = "0.3"
futures-util = "0.3"
hashbrown = "0.13"
lazy_static = "1.4"
libipld = { version = "0.15", features = ["dag-cbor", "derive", "serde-codec"] }
log = "0.4"
once_cell = "1.16"
proptest = { version = "1.0", optional = true }
rand = { version = "0.8", optional = true }
rand_core = "0.6"
semver = { version = "1.0", features = ["serde"] }
serde = { version = "1.0", features = ["rc"] }
Expand All @@ -55,4 +56,4 @@ path = "src/lib.rs"
[features]
default = []
wasm = []
test_strategies = ["proptest"]
test_strategies = ["proptest", "rand"]
10 changes: 10 additions & 0 deletions wnfs/proptest-regressions/private/hamt/diff/key_value.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 5e555d7196fc6adeea0338b071168600760e42f646ab23a1718d3e45d167b65c # shrinks to input = _DiffCorrespondenceArgs { ops: Operations([Remove("r"), Remove("0u"), Insert("aqbj0y", 8759736365109863483), Insert("4s", 10044956238670467519), Insert("dnd", 6060135432809277420), Remove("pb8b"), Remove("y"), Insert("baihhn45", 11463665909768295174), Insert("sq7", 4811876933643406092), Insert("7p2dpe8q", 12533010396669215881), Remove("to"), Remove("5"), Remove("348d"), Insert("4usi", 6747423100670732089), Remove("1xz"), Remove("39"), Remove("jiz917"), Remove("67b0"), Remove("u"), Insert("44a", 7870292772246520017), Remove("i5"), Remove("cd56zlx"), Insert("i74", 6534526574668539170), Insert("punb", 8473891721762386415), Insert("0", 10780825455149569896), Remove("3w6mz"), Insert("s0eguz", 11046279096949854934), Insert("3kq", 2784467905212969289), Insert("wb9x", 4784810419246373260), Insert("6e53", 8282203486355092676), Insert("hywri", 5137386236814319476), Remove("c46"), Insert("j20vtq1z", 1112325979113224409), Remove("69218i6"), Remove("9bc"), Remove("8u4zy6"), Insert("m", 13451127568675789321), Insert("4zaujs2", 4047325072326692822), Insert("gtv89d7", 7244565589581245304), Insert("a3d", 6591904120187804917), Remove("630mh7c"), Insert("j7", 13930972651537777602), Insert("unvs", 14396077056944091654), Insert("7v9", 10830219039014455780), Insert("wo4", 17465392163036022156), Remove("6"), Insert("p435", 4491338255758162835), Remove("10b923n1"), Insert("1i", 10572089488915295653), Remove("f2ok"), Remove("reohep"), Remove("6dyfad"), Insert("u275bp", 14719964432689961856), Insert("84uywc5", 3034869085899589692), Remove("4lqyyn"), Remove("n1x"), Insert("2", 4103134472614339993), Insert("n8t", 9755734175390624725), Insert("ux6000j", 9700209894676242316), Remove("1127dc6t"), Insert("22t0r8", 6796782628145486429), Insert("hy04", 2403701566790270018), Remove("nq7zr74r"), Insert("ylpbd83f", 15368764264547351299), Insert("87rv", 12252309813607948940), Remove("5"), Insert("4", 12982583174298221718), Insert("0puw", 5013753213686266093), Remove("l588t"), Remove("i"), Insert("7chy6ni", 972220594862718031), Insert("33", 15229741910201849945), Insert("r1o2", 5544133324611432971), Insert("hf6p", 9651098830073773988), Remove("m0q"), Insert("py4", 5643875776673896126), Remove("ys75z"), Remove("y0p"), Insert("o82", 5637522023550207602), Remove("l"), Remove("ck75w6p"), Insert("4e3", 9658913443678631064), Insert("9v", 9423557612194703216), Remove("128fr69o"), Insert("9nq03cjc", 5195524287983201984), Remove("s4l881p8"), Insert("u", 7931681966693414839)]) }
cc 783c9bd6e2cac79e26900fa17c9e9f8bb9f2318643070afff6392ad0f2cc59cd # shrinks to input = _AddRemoveFlipArgs { ops: Operations([Insert("l", 8190415290856170167), Insert("rq", 7802272870926789370), Insert("3ax", 11452151341966869560), Remove("90spto"), Remove("50"), Remove("jy"), Insert("93c9x7n", 17533421366072020711), Insert("a12ng10v", 7160346824398479785), Remove("10apn"), Remove("nz2712t"), Insert("d650a", 15596522953897214152), Insert("i3f7fhti", 293876842522201024), Insert("f1k6", 10772800145220521215), Insert("l64e", 808564228799298117), Remove("hng"), Remove("vu862d"), Remove("b2ms7qt"), Remove("2vjy3"), Remove("g3l2c19"), Remove("46"), Remove("jtra3754"), Remove("214fv"), Remove("8047mr5"), Insert("j1xt", 17455986792880115757), Insert("x", 10840329018809920586), Remove("1n9"), Remove("1lf18a"), Remove("t3zhz"), Remove("r"), Remove("qnz"), Remove("rml"), Remove("277o21o1"), Remove("l5tmj1"), Insert("8nof47cp", 12803114712855644898), Insert("03wm8", 10674708628263752206), Insert("2n", 12647826351949495019), Insert("09uci", 911344669669063258), Insert("ch9r", 4122732132649509988), Remove("2560"), Remove("wpe3p"), Remove("kr6p"), Insert("tnx5", 15780716971213936273), Remove("995i6ho3"), Remove("6z9m"), Insert("j", 9310781360903656187), Insert("903", 17962523713258320062), Insert("20bu", 39644780053866288), Remove("3gww3l"), Insert("1dwu", 4906399812115179669), Remove("s"), Insert("3ab", 2430071676156752155), Insert("8htw6v4o", 15798029957745473163), Remove("762u85b9"), Insert("w305", 10940860920108359882), Remove("f"), Insert("p2k4193p", 7417383211714364929), Insert("5m828", 10189603807998671317), Insert("14", 2273392941122468858), Insert("9", 15883828856151696571), Insert("yd6z803", 4218895444353587596), Remove("d"), Insert("e78g1", 9803365149477112329), Insert("y7nmm", 8704575196200429261), Insert("0h", 11264660707081105088), Remove("n667u0tp"), Remove("1f925"), Insert("0jj", 3415302823512910611), Remove("21h"), Insert("868345", 7057953949462148973), Insert("56sx", 13319564524444190980), Insert("19y29137", 6237959749701385176), Remove("3n"), Insert("e", 1844240028980683962), Remove("8"), Insert("47", 775572366098639473), Remove("m3cx42"), Remove("730m5n")]) }
cc e23b5493b19d1a9639faafc952b0eb0607a7c7577fdd1d92588be7ce4a85dd95 # shrinks to input = _AddRemoveFlipArgs { ops: Operations([Insert("21", 7057345610017908876), Insert("2m", 14042773570795727584), Remove("3p1"), Remove("76l04l"), Insert("knbdy", 18146268831894905032), Remove("60snt2"), Remove("lu9hi"), Remove("0k4b"), Insert("45h", 2579283501447206608), Remove("bw"), Insert("2x07p43", 8493324281333499417), Insert("q1t40", 7601167973431252114), Remove("h91"), Remove("rbk5"), Remove("0g"), Remove("cj1qy"), Insert("nvfyfsh", 4423885449729888548), Insert("8f1bs", 14160566796080055282), Remove("35e65"), Remove("c4"), Insert("46t", 15274819395966855748), Remove("djw79l"), Remove("j6i94k"), Remove("52"), Remove("4"), Remove("4741v9zw"), Insert("0v6rv0n6", 8007587074198761281), Remove("rm1u"), Insert("996l67", 13690975871034554449), Insert("2ws", 17420284283461900707), Remove("7tk7"), Remove("s"), Insert("xt4660m", 16933411867363950079), Remove("v"), Insert("4", 231938933949663025), Remove("db3f7lu1"), Remove("9"), Remove("s70f"), Remove("8x9d"), Remove("1"), Remove("qs78x4"), Remove("74yrh1v"), Remove("6"), Insert("7i", 3708293114264745095), Remove("85ki3"), Insert("sh83va33", 14851363518059892503), Remove("fx"), Remove("m834"), Insert("eq4m", 5758164870050785584), Remove("1"), Remove("3hrrqqo"), Insert("cvogb08a", 8522917429326002565), Insert("r0", 12544564931559276845), Remove("ju02zc54"), Remove("18ja"), Remove("4wa4h3e7"), Insert("m", 2466305068150690676), Insert("jrjdj5", 12612076267141588934), Insert("75vvn56k", 1821145576455514124), Insert("o", 7704537420839772406), Remove("w5d"), Remove("9p6gz870"), Remove("929636d6"), Insert("7", 8361347580713022873), Remove("5"), Insert("vu5nn4u", 11981653745046670523), Insert("5oab", 8201077123413368720), Remove("ec85295"), Remove("lp"), Insert("7gdq2xr7", 12164408117300005328), Insert("q", 3516141357951345603), Remove("a2ad4"), Insert("f", 2086973970166535168)]) }
cc 9db676834d52f0de6c41187beea9df6feead91cc534fee2f1d46fed728379bd3 # shrinks to input = _DiffCorrespondenceArgs { ops_changes: (Operations([Insert("13g", 0), Insert("5s", 0)]), [Add("13g", 1000)]) }
11 changes: 11 additions & 0 deletions wnfs/proptest-regressions/private/hamt/diff/node.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
# Seeds for failure cases proptest has generated in the past. It is
# automatically read and these particular cases re-run before any
# novel cases are generated.
#
# It is recommended to check this file in to source control so that
# everyone who runs the test benefits from these saved cases.
cc 72dc4e2a4374846a71044bee2d230bfa131730e3520e72ef85d379b8478f40be # shrinks to input = _RemoveHashmapCorrespondenceArgs { ops: Operations([Insert("c1s1", 2861968474096821110), Insert("1", 7543637671228506879), Insert("96", 2252768383817693563), Remove("4b7fw77"), Insert("g", 11439732489252793716), Remove("ww53m"), Remove("6v4xcoa6")]) }
cc 1726d1bd34f08be2b81a527eabc4012325c57055b171bb09405ac8bb945a520b # shrinks to input = _DiffHashmapCorrespondenceArgs { ops: Operations([Remove("0")]) }
cc e4b58a42567fcd5ffb91c9da25c6b4f713565fe27ef294eba6b8998f15b36149 # shrinks to input = _DiffHashmapCorrespondenceArgs { ops: Operations([Remove("3nvc20f"), Insert("u", 9956079658192318434), Remove("0a9r7g9"), Insert("87v", 14695044301822388386), Remove("2r51d6"), Remove("jl454"), Remove("j20j20pm"), Insert("g", 5152991923846126473), Remove("03y40"), Remove("fg9765m"), Insert("8t", 4474459820086240620), Remove("7"), Remove("85j2l46"), Insert("wky9u", 11445290414692833415), Insert("8lc", 9546591671211870912), Insert("np0hl19", 16482876861009537883), Remove("44cfut"), Remove("k7orxzg"), Remove("t3w"), Insert("3j", 8037164116380012272), Remove("m1q24kf8"), Insert("5yk", 2981284836867171899), Insert("v7w9kbfr", 16526934658811960738), Remove("l9k59"), Insert("941u", 8583149886141079763), Insert("81p42wa", 14443267362832524868), Insert("lw8", 12108614233900920745), Remove("tdub47k"), Insert("9wx157sz", 8251470049628788226), Insert("zwjb", 12946990402202987744), Insert("10fn", 1761380674163905782), Remove("ojw"), Insert("1v200w", 5604804115321922268), Remove("b"), Remove("i"), Insert("d637", 15852238920165317932), Insert("q64", 1730417650685085634), Insert("tj1", 14819700385153246002), Remove("src7b"), Remove("5nw0jkkr"), Remove("l81k"), Remove("wnbb1"), Remove("tgv7wd9"), Remove("1"), Insert("75mv5", 17583178423204481260), Remove("7ny1o80"), Remove("7"), Insert("z0cix6", 331083845447588779), Insert("16b", 4585931953995488213), Remove("6j5k3"), Insert("bwlx", 17352226830058118319), Remove("5xm4l2e9"), Remove("awshh4"), Insert("1dd8e53", 14849683218925113909), Insert("c57a", 532223834441501308), Remove("1"), Insert("7krc", 12089486759419077079), Insert("1", 8958205764608539889), Remove("ga94"), Remove("oq2p1du0"), Insert("s87n3um", 5818462765028773932), Remove("89ps"), Remove("t"), Remove("a7tcl4"), Insert("24", 11260399054011738518), Insert("m7r1r", 1164430910219864659), Remove("9h78"), Insert("001", 9872829549486100367), Remove("0117k31"), Remove("9k"), Insert("ztt", 5115805548357263331), Insert("5", 3579437175790913731), Remove("jbn9nr8y"), Insert("cba1hv41", 3167024047889073856), Insert("1ey63u2", 12275380225424398350), Insert("5o39", 6072018527218905335), Remove("928k6"), Remove("df1"), Remove("g"), Remove("x2s"), Remove("5k04kp94"), Remove("h973r36w"), Remove("p983xo3"), Insert("662", 12503934591749486865), Insert("158yl0", 3391647112296275715), Remove("p"), Insert("i2vg2no", 14100769604499849852), Remove("39udv"), Remove("m41p5r"), Insert("91", 12441709321095061151), Remove("a"), Remove("6"), Insert("8g", 11695773606635918662), Insert("000p", 14719758053479480581), Remove("8ei")]) }
cc 195a0e55fe7ca790273f4006594ed16b4a07969f3048eff2d0301c2c8c00da02 # shrinks to input = _DiffHashmapCorrespondenceArgs { ops: Operations([Remove("ahi95"), Insert("q92hcdz6", 12013588257816538438), Insert("g3", 10591691140536598674), Remove("e4jo60p9"), Insert("vs8qcp", 4011281999148158115), Remove("bi6s3d8f"), Remove("rq88vj"), Remove("znc39i"), Insert("14lt0", 10950353491372813321), Insert("02o6", 17427054465394299211), Insert("0", 7557074686536222252), Remove("1gq"), Insert("832te8", 18212087089492431808), Insert("sb", 14276893202905350281), Insert("w", 15998670375294947940), Insert("7", 16867301993208336769), Insert("sx7xb", 155999688409670117), Insert("14", 14609764202045242970), Insert("4", 6925978030280114190), Insert("irpw1", 7585210218031281310), Remove("p"), Remove("ze0s"), Remove("11fi75"), Remove("13av9"), Insert("l3hsy1s2", 7492885477565579271), Insert("ub1ym38", 7552406135646256719), Insert("7s8mwl", 18135249620231959636), Remove("96t"), Remove("rq6pxfg4"), Remove("reomie1"), Remove("inel"), Remove("89f99"), Insert("pl2", 2269060021916044316)]) }
cc 974898ad0bafcd27f2d71fb09d4b96f04680f2e626a18ad2eca582dd2a8bbf56 # shrinks to input = _DiffCorrespondenceArgs { ops: Operations([Insert("nze37", 5607293976437903960), Insert("w83ri", 13974003653085375353), Remove("3pj3f"), Insert("0v", 14449994618968824478), Insert("2zx23", 490777705537419654), Insert("8", 4901286928634009637), Insert("gu390i", 7302034693882484860), Insert("w1xjnk", 15275939612861193177), Insert("gp6nt3k", 16818862727418097197), Insert("0ef", 14774889956068947404), Insert("4f659", 13213841622869572750), Insert("2dv", 1996392139953980495), Insert("78if6", 11581101175803594395), Remove("j21h5"), Remove("bj06f"), Remove("t5se"), Insert("05", 9684822998161819070), Insert("94290vt", 12578288451391820397), Remove("2p1p7"), Remove("h855517"), Remove("l46m4o9x"), Insert("6", 5235519396526454081), Remove("on0k9"), Remove("a"), Insert("773", 6805938963484506619), Remove("40h"), Insert("9", 6366563695199188883), Insert("y2", 6653609466314396468), Insert("wegqcp", 10364286821529963026), Remove("fv0"), Insert("uigibv9", 5477988444587535454), Insert("c775zga2", 12731603435233519486), Insert("a6", 6544273158821719696), Insert("f2xh2", 14507375301219512205), Remove("5ly"), Remove("01"), Remove("o1cx7r1r"), Insert("4r6m9", 9438169518613924728), Insert("36g", 4813958754157346055), Insert("716", 7658772099189076995), Remove("4z7758ph"), Insert("x", 7512642240091346227), Remove("64k6xlz6"), Remove("f1a7a"), Remove("5m4f1m"), Remove("43bh2bw"), Remove("o2"), Insert("c5e8lmi", 3850797232426528274), Insert("b", 15686353040049894950), Insert("ybjrsi5", 10115613010033310831), Insert("mpyyu", 10047631643897079445), Remove("gms"), Insert("0238", 12225635164454194279), Remove("6secht"), Insert("q0lh", 1625660415707077002), Insert("09sg08vk", 5529754594624688099), Insert("11g8", 18147176070903365111), Insert("7uq15g1", 5045832724365655421), Remove("i7ljxl"), Insert("2", 4056524782711557091), Insert("888d", 7200406577966948479), Insert("2s3z6sb9", 11277545678417675335), Remove("9cva"), Insert("p8a0h0l1", 7796497246403854634), Insert("v4", 2163990801486059232), Remove("ta4"), Remove("085h7u"), Insert("w82o28", 8929686504171384478), Remove("726u"), Insert("27c8jrv", 12363820087431725551), Remove("eog"), Insert("4w9n5o7f", 5852413551123062477), Remove("8c4c"), Remove("mi2"), Insert("r18", 7881014950756268623), Insert("6ow4", 3847558428371643988), Insert("3j", 8116029199181678606), Insert("k8", 14197183690972410544), Insert("0e57ego2", 16092422570903853179), Remove("n6er"), Insert("5f0hnc6s", 13820376120759977329)]) }
Loading

0 comments on commit 883b3ab

Please sign in to comment.