diff --git a/makefile b/makefile index 0fc2ef8..46d369b 100644 --- a/makefile +++ b/makefile @@ -1,9 +1,7 @@ setup: uv venv --python 3.12 uv pip install -r requirements.txt - uv pip install git+https://github.com/ilan-gold/zarrs-python.git@ld/chunk_codec_concurrency - # uv pip install --upgrade --force-reinstall ../zarrs-python - cargo binstall zarrs_tools@0.6.0-beta.1 + cargo binstall zarrs_tools@0.7.1 binstall: curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash diff --git a/measurements/benchmark_read_all.csv b/measurements/benchmark_read_all.csv index c565eee..e10be4a 100644 --- a/measurements/benchmark_read_all.csv +++ b/measurements/benchmark_read_all.csv @@ -1,6 +1,6 @@ Metric,Time (s),Time (s),Time (s),Time (s),Time (s),Time (s),Memory (GB),Memory (GB),Memory (GB),Memory (GB),Memory (GB),Memory (GB) Implementation,zarrs_rust,tensorstore_python,zarr_python,zarrs_python,zarr_dask_python,zarrs_dask_python,zarrs_rust,tensorstore_python,zarr_python,zarrs_python,zarr_dask_python,zarrs_dask_python Image,,,,,,,,,,,, -data/benchmark.zarr,2.37,2.98,3.42,3.09,5.03,4.54,9.078368,11.447836,8.76694,9.22934,16.839548,16.847508 -data/benchmark_compress.zarr,2.15,2.94,3.06,2.51,5.19,4.0,9.3663,9.688068,8.988624,9.42066,17.131092,17.061112 -data/benchmark_compress_shard.zarr,1.65,2.02,29.08,2.28,40.73,4.24,8.458552,8.63794,9.01874,8.508508,17.01946,16.937372 +data/benchmark.zarr,2.39,2.88,3.1,2.99,5.21,4.92,9.1216,11.357908,8.751452,9.225376,16.881816,16.886612 +data/benchmark_compress.zarr,2.13,2.51,3.12,2.57,5.25,4.55,9.366084,9.5691,8.987,9.405832,17.160644,16.998852 +data/benchmark_compress_shard.zarr,1.61,2.05,29.54,2.26,44.98,4.33,8.457372,8.621676,9.01544,8.510408,17.066332,16.984744 diff --git a/measurements/benchmark_read_all.md b/measurements/benchmark_read_all.md index ea2117b..356def1 100644 --- a/measurements/benchmark_read_all.md +++ b/measurements/benchmark_read_all.md @@ -1,5 +1,5 @@ | Image | Time (s)
zarrs
rust |
tensorstore
python |
zarr
python |
zarrs
python |
zarr
dask
python |
zarrs
dask
python | Memory (GB)
zarrs
rust |
tensorstore
python |
zarr
python |
zarrs
python |
zarr
dask
python |
zarrs
dask
python | |:-----------------------------------|----------------------------:|----------------------------:|---------------------:|----------------------:|-----------------------------:|------------------------------:|-------------------------------:|----------------------------:|---------------------:|----------------------:|-----------------------------:|------------------------------:| -| data/benchmark.zarr | 2.37 | 2.98 | 3.42 | 3.09 | 5.03 | 4.54 | 9.08 | 11.45 | 8.77 | 9.23 | 16.84 | 16.85 | -| data/benchmark_compress.zarr | 2.15 | 2.94 | 3.06 | 2.51 | 5.19 | 4.00 | 9.37 | 9.69 | 8.99 | 9.42 | 17.13 | 17.06 | -| data/benchmark_compress_shard.zarr | 1.65 | 2.02 | 29.08 | 2.28 | 40.73 | 4.24 | 8.46 | 8.64 | 9.02 | 8.51 | 17.02 | 16.94 | \ No newline at end of file +| data/benchmark.zarr | 2.39 | 2.88 | 3.10 | 2.99 | 5.21 | 4.92 | 9.12 | 11.36 | 8.75 | 9.23 | 16.88 | 16.89 | +| data/benchmark_compress.zarr | 2.13 | 2.51 | 3.12 | 2.57 | 5.25 | 4.55 | 9.37 | 9.57 | 8.99 | 9.41 | 17.16 | 17.00 | +| data/benchmark_compress_shard.zarr | 1.61 | 2.05 | 29.54 | 2.26 | 44.98 | 4.33 | 8.46 | 8.62 | 9.02 | 8.51 | 17.07 | 16.98 | \ No newline at end of file diff --git a/measurements/benchmark_read_chunks.csv b/measurements/benchmark_read_chunks.csv index eaa0d9e..10bafde 100644 --- a/measurements/benchmark_read_chunks.csv +++ b/measurements/benchmark_read_chunks.csv @@ -1,21 +1,21 @@ Metric,,Time (s),Time (s),Time (s),Time (s),Time (s),Time (s),Memory (GB),Memory (GB),Memory (GB),Memory (GB),Memory (GB),Memory (GB) Implementation,,zarrs_rust,tensorstore_python,zarr_python,zarrs_python,zarr_dask_python,zarrs_dask_python,zarrs_rust,tensorstore_python,zarr_python,zarrs_python,zarr_dask_python,zarrs_dask_python Image,Concurrency,,,,,,,,,,,, -data/benchmark.zarr,1,4.1,5.46,5.0,6.45,5.52,6.74,0.038192,0.123992,0.115932,0.120476,0.124684,0.131216 -data/benchmark.zarr,2,2.43,3.26,3.25,3.89,3.42,3.76,0.073104,0.190044,0.183576,0.186528,0.19014,0.193088 -data/benchmark.zarr,4,1.62,2.46,2.71,2.55,2.82,2.71,0.136564,0.312456,0.312724,0.313584,0.321028,0.302072 -data/benchmark.zarr,8,1.54,2.2,2.68,2.61,3.18,2.63,0.258544,0.414088,0.574772,0.475676,0.57898,0.410292 -data/benchmark.zarr,16,1.5,2.18,2.85,2.48,3.17,2.65,0.522484,0.713756,1.05534,0.726668,1.036324,0.668352 -data/benchmark.zarr,32,1.54,2.36,2.92,2.5,3.2,2.68,0.771308,1.564,2.06194,1.253872,1.93922,1.20242 -data/benchmark_compress.zarr,1,4.6,6.49,6.06,5.98,6.16,6.33,0.048276,0.15028,0.131444,0.374032,0.135604,0.348036 -data/benchmark_compress.zarr,2,2.2,3.76,3.65,3.46,3.78,3.57,0.09458,0.219012,0.213024,0.423576,0.259176,0.412632 -data/benchmark_compress.zarr,4,1.13,2.4,2.56,2.21,2.5,2.27,0.177688,0.364212,0.46168,0.530944,0.407076,0.507436 -data/benchmark_compress.zarr,8,0.69,1.95,2.22,1.93,2.32,2.09,0.327952,0.582932,0.741816,0.696572,0.644508,0.62458 -data/benchmark_compress.zarr,16,0.71,1.94,2.22,1.92,2.42,2.06,0.512548,0.887996,1.176644,0.93344,1.207852,0.879792 -data/benchmark_compress.zarr,32,0.76,1.93,2.42,1.97,2.73,2.23,0.723472,1.533288,2.17394,1.475916,2.337784,1.352212 -data/benchmark_compress_shard.zarr,1,2.28,2.81,35.19,3.64,26.36,3.59,0.082008,0.136236,0.33542,0.137716,0.314544,0.14142 -data/benchmark_compress_shard.zarr,2,1.53,2.17,33.19,2.2,24.7,2.39,0.12454,0.178248,0.441248,0.209328,0.428516,0.19052 -data/benchmark_compress_shard.zarr,4,1.32,1.84,32.03,1.85,27.12,2.05,0.203408,0.265768,0.586076,0.305272,0.638644,0.305424 -data/benchmark_compress_shard.zarr,8,1.16,1.72,31.47,1.92,29.55,2.01,0.360468,0.41328,0.834048,0.466712,0.958224,0.42636 -data/benchmark_compress_shard.zarr,16,1.06,1.67,31.72,1.88,33.62,2.05,0.619252,0.522008,1.372836,0.70762,1.784544,0.674872 -data/benchmark_compress_shard.zarr,32,1.13,1.73,33.18,1.97,36.8,2.21,0.884464,0.912768,2.448768,1.228768,3.489496,1.234684 +data/benchmark.zarr,1,4.76,5.74,4.93,6.62,5.64,6.9,0.038416,0.119924,0.113,0.1173,0.16642,0.168356 +data/benchmark.zarr,2,2.28,3.49,3.01,3.68,3.62,4.01,0.071064,0.186696,0.178636,0.183236,0.231764,0.235456 +data/benchmark.zarr,4,1.57,2.43,2.75,2.54,3.14,3.27,0.130256,0.318244,0.313612,0.31204,0.364584,0.349972 +data/benchmark.zarr,8,1.53,2.15,2.74,2.63,3.11,2.76,0.249144,0.407156,0.5737,0.454324,0.598492,0.483656 +data/benchmark.zarr,16,1.56,2.03,2.8,2.34,3.35,2.94,0.514944,0.739828,1.029564,0.7405,1.030476,0.721012 +data/benchmark.zarr,32,1.6,2.43,2.88,2.39,3.28,3.09,0.78834,1.627972,2.048828,1.280544,2.043356,1.241544 +data/benchmark_compress.zarr,1,4.11,5.64,6.25,5.66,6.77,6.35,0.047556,0.145592,0.128384,0.14066,0.17968,0.195464 +data/benchmark_compress.zarr,2,2.05,3.27,3.63,3.14,4.03,3.69,0.095644,0.229376,0.211856,0.220604,0.288052,0.26788 +data/benchmark_compress.zarr,4,1.1,1.98,2.52,2.14,2.8,2.55,0.164932,0.367956,0.430372,0.379696,0.44482,0.380744 +data/benchmark_compress.zarr,8,0.69,1.51,2.21,1.93,2.64,2.37,0.301296,0.516724,0.732408,0.591456,0.792676,0.565356 +data/benchmark_compress.zarr,16,0.74,1.59,2.19,1.92,2.64,2.36,0.5247,0.827748,1.260192,0.867852,1.250276,0.818392 +data/benchmark_compress.zarr,32,0.79,1.63,2.33,1.94,2.78,2.37,0.688444,1.424012,2.18272,1.33024,2.164008,1.313892 +data/benchmark_compress_shard.zarr,1,2.42,2.81,36.58,3.57,28.63,3.86,0.081292,0.137776,0.309596,0.136036,0.354168,0.18636 +data/benchmark_compress_shard.zarr,2,1.65,2.23,31.37,2.17,27.13,2.76,0.120428,0.17838,0.427832,0.208684,0.470932,0.224104 +data/benchmark_compress_shard.zarr,4,1.31,1.92,30.28,1.91,29.61,2.32,0.204636,0.26516,0.583808,0.313416,0.648056,0.312048 +data/benchmark_compress_shard.zarr,8,1.16,1.79,29.38,1.78,33.13,2.21,0.362204,0.387804,0.840772,0.456744,0.995296,0.460824 +data/benchmark_compress_shard.zarr,16,1.12,1.73,30.11,1.82,36.78,2.22,0.609532,0.545428,1.395968,0.737816,1.817132,0.720144 +data/benchmark_compress_shard.zarr,32,1.17,1.72,31.79,1.76,39.55,2.27,0.883988,0.903796,2.491672,1.246516,3.559228,1.233076 diff --git a/measurements/benchmark_read_chunks.md b/measurements/benchmark_read_chunks.md index a72c0bd..62a556c 100644 --- a/measurements/benchmark_read_chunks.md +++ b/measurements/benchmark_read_chunks.md @@ -1,20 +1,20 @@ | | Image | Concurrency | Time (s)
zarrs
rust |
tensorstore
python |
zarr
python |
zarrs
python |
zarr
dask
python |
zarrs
dask
python | Memory (GB)
zarrs
rust |
tensorstore
python |
zarr
python |
zarrs
python |
zarr
dask
python |
zarrs
dask
python | |---:|:-----------------------------------|--------------:|----------------------------:|----------------------------:|---------------------:|----------------------:|-----------------------------:|------------------------------:|-------------------------------:|----------------------------:|---------------------:|----------------------:|-----------------------------:|------------------------------:| -| 0 | data/benchmark.zarr | 1 | 4.10 | 5.46 | 5.00 | 6.45 | 5.52 | 6.74 | 0.04 | 0.12 | 0.12 | 0.12 | 0.12 | 0.13 | -| 1 | data/benchmark.zarr | 2 | 2.43 | 3.26 | 3.25 | 3.89 | 3.42 | 3.76 | 0.07 | 0.19 | 0.18 | 0.19 | 0.19 | 0.19 | -| 2 | data/benchmark.zarr | 4 | 1.62 | 2.46 | 2.71 | 2.55 | 2.82 | 2.71 | 0.14 | 0.31 | 0.31 | 0.31 | 0.32 | 0.30 | -| 3 | data/benchmark.zarr | 8 | 1.54 | 2.20 | 2.68 | 2.61 | 3.18 | 2.63 | 0.26 | 0.41 | 0.57 | 0.48 | 0.58 | 0.41 | -| 4 | data/benchmark.zarr | 16 | 1.50 | 2.18 | 2.85 | 2.48 | 3.17 | 2.65 | 0.52 | 0.71 | 1.06 | 0.73 | 1.04 | 0.67 | -| 5 | data/benchmark.zarr | 32 | 1.54 | 2.36 | 2.92 | 2.50 | 3.20 | 2.68 | 0.77 | 1.56 | 2.06 | 1.25 | 1.94 | 1.20 | -| 6 | data/benchmark_compress.zarr | 1 | 4.60 | 6.49 | 6.06 | 5.98 | 6.16 | 6.33 | 0.05 | 0.15 | 0.13 | 0.37 | 0.14 | 0.35 | -| 7 | data/benchmark_compress.zarr | 2 | 2.20 | 3.76 | 3.65 | 3.46 | 3.78 | 3.57 | 0.09 | 0.22 | 0.21 | 0.42 | 0.26 | 0.41 | -| 8 | data/benchmark_compress.zarr | 4 | 1.13 | 2.40 | 2.56 | 2.21 | 2.50 | 2.27 | 0.18 | 0.36 | 0.46 | 0.53 | 0.41 | 0.51 | -| 9 | data/benchmark_compress.zarr | 8 | 0.69 | 1.95 | 2.22 | 1.93 | 2.32 | 2.09 | 0.33 | 0.58 | 0.74 | 0.70 | 0.64 | 0.62 | -| 10 | data/benchmark_compress.zarr | 16 | 0.71 | 1.94 | 2.22 | 1.92 | 2.42 | 2.06 | 0.51 | 0.89 | 1.18 | 0.93 | 1.21 | 0.88 | -| 11 | data/benchmark_compress.zarr | 32 | 0.76 | 1.93 | 2.42 | 1.97 | 2.73 | 2.23 | 0.72 | 1.53 | 2.17 | 1.48 | 2.34 | 1.35 | -| 12 | data/benchmark_compress_shard.zarr | 1 | 2.28 | 2.81 | 35.19 | 3.64 | 26.36 | 3.59 | 0.08 | 0.14 | 0.34 | 0.14 | 0.31 | 0.14 | -| 13 | data/benchmark_compress_shard.zarr | 2 | 1.53 | 2.17 | 33.19 | 2.20 | 24.70 | 2.39 | 0.12 | 0.18 | 0.44 | 0.21 | 0.43 | 0.19 | -| 14 | data/benchmark_compress_shard.zarr | 4 | 1.32 | 1.84 | 32.03 | 1.85 | 27.12 | 2.05 | 0.20 | 0.27 | 0.59 | 0.31 | 0.64 | 0.31 | -| 15 | data/benchmark_compress_shard.zarr | 8 | 1.16 | 1.72 | 31.47 | 1.92 | 29.55 | 2.01 | 0.36 | 0.41 | 0.83 | 0.47 | 0.96 | 0.43 | -| 16 | data/benchmark_compress_shard.zarr | 16 | 1.06 | 1.67 | 31.72 | 1.88 | 33.62 | 2.05 | 0.62 | 0.52 | 1.37 | 0.71 | 1.78 | 0.67 | -| 17 | data/benchmark_compress_shard.zarr | 32 | 1.13 | 1.73 | 33.18 | 1.97 | 36.80 | 2.21 | 0.88 | 0.91 | 2.45 | 1.23 | 3.49 | 1.23 | \ No newline at end of file +| 0 | data/benchmark.zarr | 1 | 4.76 | 5.74 | 4.93 | 6.62 | 5.64 | 6.90 | 0.04 | 0.12 | 0.11 | 0.12 | 0.17 | 0.17 | +| 1 | data/benchmark.zarr | 2 | 2.28 | 3.49 | 3.01 | 3.68 | 3.62 | 4.01 | 0.07 | 0.19 | 0.18 | 0.18 | 0.23 | 0.24 | +| 2 | data/benchmark.zarr | 4 | 1.57 | 2.43 | 2.75 | 2.54 | 3.14 | 3.27 | 0.13 | 0.32 | 0.31 | 0.31 | 0.36 | 0.35 | +| 3 | data/benchmark.zarr | 8 | 1.53 | 2.15 | 2.74 | 2.63 | 3.11 | 2.76 | 0.25 | 0.41 | 0.57 | 0.45 | 0.60 | 0.48 | +| 4 | data/benchmark.zarr | 16 | 1.56 | 2.03 | 2.80 | 2.34 | 3.35 | 2.94 | 0.51 | 0.74 | 1.03 | 0.74 | 1.03 | 0.72 | +| 5 | data/benchmark.zarr | 32 | 1.60 | 2.43 | 2.88 | 2.39 | 3.28 | 3.09 | 0.79 | 1.63 | 2.05 | 1.28 | 2.04 | 1.24 | +| 6 | data/benchmark_compress.zarr | 1 | 4.11 | 5.64 | 6.25 | 5.66 | 6.77 | 6.35 | 0.05 | 0.15 | 0.13 | 0.14 | 0.18 | 0.20 | +| 7 | data/benchmark_compress.zarr | 2 | 2.05 | 3.27 | 3.63 | 3.14 | 4.03 | 3.69 | 0.10 | 0.23 | 0.21 | 0.22 | 0.29 | 0.27 | +| 8 | data/benchmark_compress.zarr | 4 | 1.10 | 1.98 | 2.52 | 2.14 | 2.80 | 2.55 | 0.16 | 0.37 | 0.43 | 0.38 | 0.44 | 0.38 | +| 9 | data/benchmark_compress.zarr | 8 | 0.69 | 1.51 | 2.21 | 1.93 | 2.64 | 2.37 | 0.30 | 0.52 | 0.73 | 0.59 | 0.79 | 0.57 | +| 10 | data/benchmark_compress.zarr | 16 | 0.74 | 1.59 | 2.19 | 1.92 | 2.64 | 2.36 | 0.52 | 0.83 | 1.26 | 0.87 | 1.25 | 0.82 | +| 11 | data/benchmark_compress.zarr | 32 | 0.79 | 1.63 | 2.33 | 1.94 | 2.78 | 2.37 | 0.69 | 1.42 | 2.18 | 1.33 | 2.16 | 1.31 | +| 12 | data/benchmark_compress_shard.zarr | 1 | 2.42 | 2.81 | 36.58 | 3.57 | 28.63 | 3.86 | 0.08 | 0.14 | 0.31 | 0.14 | 0.35 | 0.19 | +| 13 | data/benchmark_compress_shard.zarr | 2 | 1.65 | 2.23 | 31.37 | 2.17 | 27.13 | 2.76 | 0.12 | 0.18 | 0.43 | 0.21 | 0.47 | 0.22 | +| 14 | data/benchmark_compress_shard.zarr | 4 | 1.31 | 1.92 | 30.28 | 1.91 | 29.61 | 2.32 | 0.20 | 0.27 | 0.58 | 0.31 | 0.65 | 0.31 | +| 15 | data/benchmark_compress_shard.zarr | 8 | 1.16 | 1.79 | 29.38 | 1.78 | 33.13 | 2.21 | 0.36 | 0.39 | 0.84 | 0.46 | 1.00 | 0.46 | +| 16 | data/benchmark_compress_shard.zarr | 16 | 1.12 | 1.73 | 30.11 | 1.82 | 36.78 | 2.22 | 0.61 | 0.55 | 1.40 | 0.74 | 1.82 | 0.72 | +| 17 | data/benchmark_compress_shard.zarr | 32 | 1.17 | 1.72 | 31.79 | 1.76 | 39.55 | 2.27 | 0.88 | 0.90 | 2.49 | 1.25 | 3.56 | 1.23 | \ No newline at end of file diff --git a/measurements/benchmark_roundtrip.csv b/measurements/benchmark_roundtrip.csv index 4e03b0f..3148371 100644 --- a/measurements/benchmark_roundtrip.csv +++ b/measurements/benchmark_roundtrip.csv @@ -1,6 +1,6 @@ Metric,Time (s),Time (s),Time (s),Time (s),Time (s),Time (s),Memory (GB),Memory (GB),Memory (GB),Memory (GB),Memory (GB),Memory (GB) Implementation,zarrs_rust,tensorstore_python,zarr_python,zarrs_python,zarr_dask_python,zarrs_dask_python,zarrs_rust,tensorstore_python,zarr_python,zarrs_python,zarr_dask_python,zarrs_dask_python Image,,,,,,,,,,,, -data/benchmark.zarr,2.3,8.28,17.78,4.94,15.38,4.17,0.792712,0.883052,9.031084,9.266244,1.621484,1.561132 -data/benchmark_compress.zarr,1.3,7.71,14.65,4.1,15.24,3.59,1.052528,1.612392,9.13668,9.548896,1.613448,1.580084 -data/benchmark_compress_shard.zarr,2.97,4.87,142.37,4.27,46.56,3.65,0.22328,2.32528,9.0285,8.661988,2.644088,1.53796 +data/benchmark.zarr,2.27,9.7,18.47,5.01,11.36,3.88,0.79338,0.879184,9.0254,9.259884,1.534688,1.78304 +data/benchmark_compress.zarr,1.3,6.76,15.34,4.09,10.63,3.54,1.060276,1.157956,9.009768,9.5209,1.762008,1.586404 +data/benchmark_compress_shard.zarr,2.84,3.75,180.9,6.62,45.65,3.33,0.22384,1.53856,9.017404,8.722708,2.675728,1.574932 diff --git a/measurements/benchmark_roundtrip.md b/measurements/benchmark_roundtrip.md index 2b191a9..f869259 100644 --- a/measurements/benchmark_roundtrip.md +++ b/measurements/benchmark_roundtrip.md @@ -1,5 +1,5 @@ | Image | Time (s)
zarrs
rust |
tensorstore
python |
zarr
python |
zarrs
python |
zarr
dask
python |
zarrs
dask
python | Memory (GB)
zarrs
rust |
tensorstore
python |
zarr
python |
zarrs
python |
zarr
dask
python |
zarrs
dask
python | |:-----------------------------------|----------------------------:|----------------------------:|---------------------:|----------------------:|-----------------------------:|------------------------------:|-------------------------------:|----------------------------:|---------------------:|----------------------:|-----------------------------:|------------------------------:| -| data/benchmark.zarr | 2.30 | 8.28 | 17.78 | 4.94 | 15.38 | 4.17 | 0.79 | 0.88 | 9.03 | 9.27 | 1.62 | 1.56 | -| data/benchmark_compress.zarr | 1.30 | 7.71 | 14.65 | 4.10 | 15.24 | 3.59 | 1.05 | 1.61 | 9.14 | 9.55 | 1.61 | 1.58 | -| data/benchmark_compress_shard.zarr | 2.97 | 4.87 | 142.37 | 4.27 | 46.56 | 3.65 | 0.22 | 2.33 | 9.03 | 8.66 | 2.64 | 1.54 | \ No newline at end of file +| data/benchmark.zarr | 2.27 | 9.70 | 18.47 | 5.01 | 11.36 | 3.88 | 0.79 | 0.88 | 9.03 | 9.26 | 1.53 | 1.78 | +| data/benchmark_compress.zarr | 1.30 | 6.76 | 15.34 | 4.09 | 10.63 | 3.54 | 1.06 | 1.16 | 9.01 | 9.52 | 1.76 | 1.59 | +| data/benchmark_compress_shard.zarr | 2.84 | 3.75 | 180.90 | 6.62 | 45.65 | 3.33 | 0.22 | 1.54 | 9.02 | 8.72 | 2.68 | 1.57 | \ No newline at end of file diff --git a/plots/benchmark_read_all.pdf b/plots/benchmark_read_all.pdf index 6acea16..baec47f 100644 Binary files a/plots/benchmark_read_all.pdf and b/plots/benchmark_read_all.pdf differ diff --git a/plots/benchmark_read_all.svg b/plots/benchmark_read_all.svg index ac198fb..4bd6b29 100644 --- a/plots/benchmark_read_all.svg +++ b/plots/benchmark_read_all.svg @@ -34,109 +34,109 @@ z +" clip-path="url(#p09428c31a6)" style="fill: #1f77b4"/> +" clip-path="url(#p09428c31a6)" style="fill: #1f77b4"/> +" clip-path="url(#p09428c31a6)" style="fill: #1f77b4"/> +" clip-path="url(#p09428c31a6)" style="fill: #ff7f0e"/> +" clip-path="url(#p09428c31a6)" style="fill: #ff7f0e"/> +" clip-path="url(#p09428c31a6)" style="fill: #ff7f0e"/> +" clip-path="url(#p09428c31a6)" style="fill: #2ca02c"/> +" clip-path="url(#p09428c31a6)" style="fill: #2ca02c"/> +" clip-path="url(#p09428c31a6)" style="fill: #2ca02c"/> +" clip-path="url(#p09428c31a6)" style="fill: #d62728"/> +" clip-path="url(#p09428c31a6)" style="fill: #d62728"/> +" clip-path="url(#p09428c31a6)" style="fill: #d62728"/> - - + @@ -326,7 +326,7 @@ z - + @@ -363,7 +363,7 @@ z - + @@ -478,16 +478,16 @@ z +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -533,11 +533,11 @@ z +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -583,11 +583,11 @@ z +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -623,11 +623,11 @@ z +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -643,11 +643,11 @@ L 322.143704 195.54674 +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -687,11 +687,11 @@ z +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -707,11 +707,11 @@ L 322.143704 153.308095 +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -757,11 +757,11 @@ z +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -777,11 +777,11 @@ L 322.143704 111.06945 +" clip-path="url(#p09428c31a6)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -929,8 +929,8 @@ L 322.143704 258.904707 " style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/> - - + + - - + - - + + - - + - - + + - + - - + + - - + - - + + - - - + + - - + + - + - - + + - - + - - + + - - + + - + - + - - - + + + - + - - + + + + + - + - - + + - + @@ -1259,26 +1241,26 @@ z +" clip-path="url(#p39035273c5)" style="fill: #1f77b4"/> +" clip-path="url(#p39035273c5)" style="fill: #1f77b4"/> +" clip-path="url(#p39035273c5)" style="fill: #1f77b4"/> +" clip-path="url(#p39035273c5)" style="fill: #ff7f0e"/> +" clip-path="url(#p39035273c5)" style="fill: #ff7f0e"/> +" clip-path="url(#p39035273c5)" style="fill: #ff7f0e"/> +" clip-path="url(#p39035273c5)" style="fill: #2ca02c"/> +" clip-path="url(#p39035273c5)" style="fill: #2ca02c"/> +" clip-path="url(#p39035273c5)" style="fill: #2ca02c"/> +" clip-path="url(#p39035273c5)" style="fill: #d62728"/> +" clip-path="url(#p39035273c5)" style="fill: #d62728"/> +" clip-path="url(#p39035273c5)" style="fill: #d62728"/> - + @@ -1380,7 +1362,7 @@ z - + @@ -1402,7 +1384,7 @@ z - + @@ -1438,11 +1420,11 @@ z +" clip-path="url(#p39035273c5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -1454,54 +1436,54 @@ L 644.99976 258.904707 - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - - + @@ -1795,34 +1758,55 @@ L 359.549015 89.950127 L 644.99976 258.904707 " style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/> - - - + + + - - + + - + - + - + - + + + + - + @@ -1831,72 +1815,72 @@ L 644.99976 258.904707 - - - + + + - - + + - - - + + + - + - - - + + + - + - + - + - + - + - + - + - - - + + + - + - + - + @@ -1906,19 +1890,19 @@ L 644.99976 258.904707 - - + @@ -1984,16 +1968,16 @@ z - - - - + + + - - @@ -2094,30 +2051,23 @@ z - + - - - - - - - - - + + - - - - + + + @@ -2141,23 +2091,30 @@ z - - + + - - - - + + + + - - - - + + - - - - + + + @@ -2254,13 +2209,11 @@ z - - - - + + - - + + @@ -2289,7 +2242,7 @@ z - + @@ -2314,10 +2267,10 @@ z - + - + diff --git a/plots/benchmark_read_all_dask.pdf b/plots/benchmark_read_all_dask.pdf index d8a1722..00d42dd 100644 Binary files a/plots/benchmark_read_all_dask.pdf and b/plots/benchmark_read_all_dask.pdf differ diff --git a/plots/benchmark_read_all_dask.svg b/plots/benchmark_read_all_dask.svg index 19e7cc5..05e5a65 100644 --- a/plots/benchmark_read_all_dask.svg +++ b/plots/benchmark_read_all_dask.svg @@ -25,75 +25,75 @@ z - +" clip-path="url(#pb8e3d0cd29)" style="fill: #1f77b4"/> - +" clip-path="url(#pb8e3d0cd29)" style="fill: #1f77b4"/> - +" clip-path="url(#pb8e3d0cd29)" style="fill: #1f77b4"/> - +" clip-path="url(#pb8e3d0cd29)" style="fill: #ff7f0e"/> - +" clip-path="url(#pb8e3d0cd29)" style="fill: #ff7f0e"/> - +" clip-path="url(#pb8e3d0cd29)" style="fill: #ff7f0e"/> - - + - + - + - + - + - + @@ -333,7 +333,7 @@ z - + +L 316.885628 258.904707 +" clip-path="url(#pb8e3d0cd29)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -474,18 +474,18 @@ z - + - + - + - + - + - + - + - + - + @@ -580,18 +580,18 @@ L 320.76 186.054472 - + - + - + - + - + - + @@ -642,18 +642,18 @@ L 320.76 137.487649 - + - + - + - + - + - + @@ -710,18 +710,18 @@ L 320.76 88.920826 - + - + - + - + - - + + - - - - - - - - - - - - - - - - + + + + + + + + + + + - - + + - - - - + - - + + + + + - - + + - - + + + + + - - + + - - - + + - - +" clip-path="url(#p8dc42703ab)" style="fill: #1f77b4"/> - +" clip-path="url(#p8dc42703ab)" style="fill: #1f77b4"/> - +" clip-path="url(#p8dc42703ab)" style="fill: #1f77b4"/> - +" clip-path="url(#p8dc42703ab)" style="fill: #ff7f0e"/> - +" clip-path="url(#p8dc42703ab)" style="fill: #ff7f0e"/> - +" clip-path="url(#p8dc42703ab)" style="fill: #ff7f0e"/> - + - + @@ -1160,12 +1129,12 @@ z - + - + @@ -1182,12 +1151,12 @@ z - + - + @@ -1200,7 +1169,7 @@ z - + @@ -1216,228 +1185,201 @@ z - +" clip-path="url(#p8dc42703ab)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + - - + + + + + + + - + - + - - + + + + - + - + - - - + + + + + - + - + - - + + - - + + + - + - + - - - - - - + + + + + + - + - + - - + + - + + + - + - + - - + + - + + + - + - + - - + + - + + + - - - - - - - - - - - - - - - - - - + - + - - - - - + + + - + + + + + + + + + + - + - + - + @@ -1679,37 +1621,26 @@ z - - - - - - - - - + + - + - - - + + + - - - - - + + + - - - + @@ -1727,9 +1658,9 @@ Q 108.006381 32.92528 110.006381 32.92528 z " style="fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter"/> - - - + + + + - @@ -1814,50 +1745,47 @@ z - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1868,7 +1796,7 @@ L 112.006381 19.43462 z " style="fill: #1f77b4"/> - + @@ -1944,8 +1872,8 @@ L 286.642777 19.43462 z " style="fill: #ff7f0e"/> - - + + - + - - + + - - + + diff --git a/plots/benchmark_read_chunks.pdf b/plots/benchmark_read_chunks.pdf index 073fa0c..dcba448 100644 Binary files a/plots/benchmark_read_chunks.pdf and b/plots/benchmark_read_chunks.pdf differ diff --git a/plots/benchmark_read_chunks.svg b/plots/benchmark_read_chunks.svg index bb6db56..b44869c 100644 --- a/plots/benchmark_read_chunks.svg +++ b/plots/benchmark_read_chunks.svg @@ -35,12 +35,12 @@ z - - + @@ -69,7 +69,7 @@ z - + @@ -104,7 +104,7 @@ z - + @@ -138,7 +138,7 @@ z - + @@ -176,7 +176,7 @@ z - + @@ -214,7 +214,7 @@ z - + @@ -442,16 +442,16 @@ z +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -497,11 +497,11 @@ z +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -547,11 +547,11 @@ z +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -587,11 +587,11 @@ z +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -607,11 +607,11 @@ L 312.063418 186.69199 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -651,11 +651,11 @@ z +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -671,11 +671,11 @@ L 312.063418 140.294379 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -721,11 +721,11 @@ z +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -741,11 +741,11 @@ L 312.063418 93.896768 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -948,106 +948,106 @@ z - + - + - + - + - +L 256.989326 221.954174 +L 312.063418 219.634294 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #1f77b4; stroke-width: 1.5"/> - + - + - +L 312.063418 166.277041 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke-dasharray: 5.55,2.4; stroke-dashoffset: 0; stroke: #d62728; stroke-width: 1.5"/> - +L 256.989326 204.323082 +L 312.063418 202.003202 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/> +L 91.767051 152.821734 +L 146.841142 167.204993 +L 201.915234 173.236683 +L 256.989326 176.020539 +L 312.063418 176.484516 +" clip-path="url(#pbe575d23a5)" style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/> - + - + - + @@ -1086,7 +1086,7 @@ z - + @@ -1099,7 +1099,7 @@ z - + @@ -1112,7 +1112,7 @@ z - + @@ -1125,7 +1125,7 @@ z - + @@ -1139,7 +1139,7 @@ z - + @@ -1175,18 +1175,18 @@ z - + - + - + - + - + - + @@ -1227,198 +1227,198 @@ L 640.018425 115.450214 - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1562,111 +1562,111 @@ z +L 419.722058 222.978531 +L 474.79615 198.47605 +L 529.870241 172.250552 +L 584.944333 142.891165 +L 640.018425 125.669604 +" clip-path="url(#p26124a4ba1)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #1f77b4; stroke-width: 1.5"/> - + - + - + - + - + - + - + - + - + - +" clip-path="url(#p26124a4ba1)" style="fill: none; stroke: #2ca02c; stroke-width: 1.5; stroke-linecap: square"/> - + - + - + - - @@ -1894,17 +1890,10 @@ z - + - - - - - - - - - + + @@ -1914,7 +1903,7 @@ L 24 36.790297 " style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/> - + - - + + - - - + + + - - - - + + - - - + + @@ -2062,13 +2056,11 @@ L 196.491804 43.487404 - - - - + + - - + + - + @@ -2284,10 +2276,10 @@ z - + - + diff --git a/plots/benchmark_read_chunks_dask.pdf b/plots/benchmark_read_chunks_dask.pdf index 176199b..09c57e3 100644 Binary files a/plots/benchmark_read_chunks_dask.pdf and b/plots/benchmark_read_chunks_dask.pdf differ diff --git a/plots/benchmark_read_chunks_dask.svg b/plots/benchmark_read_chunks_dask.svg index 4b44a72..57dced7 100644 --- a/plots/benchmark_read_chunks_dask.svg +++ b/plots/benchmark_read_chunks_dask.svg @@ -35,12 +35,12 @@ z - - + @@ -69,7 +69,7 @@ z - + @@ -104,7 +104,7 @@ z - + @@ -138,7 +138,7 @@ z - + @@ -176,7 +176,7 @@ z - + @@ -214,7 +214,7 @@ z - + @@ -442,16 +442,16 @@ z +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -497,11 +497,11 @@ z +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -547,11 +547,11 @@ z +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -587,11 +587,11 @@ z +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -607,11 +607,11 @@ L 315.17675 183.879046 +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -651,11 +651,11 @@ z +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -671,11 +671,11 @@ L 315.17675 135.606139 +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -721,11 +721,11 @@ z +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -741,11 +741,11 @@ L 315.17675 87.333232 +" clip-path="url(#p3b528ea966)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -948,52 +948,52 @@ z - + - + - + - + - + - + - + @@ -1032,7 +1032,7 @@ z - + @@ -1045,7 +1045,7 @@ z - + @@ -1058,7 +1058,7 @@ z - + @@ -1071,7 +1071,7 @@ z - + @@ -1085,7 +1085,7 @@ z - + @@ -1121,18 +1121,18 @@ z - + - + - + @@ -1141,138 +1141,138 @@ L 640.018425 137.819252 - + - - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -1416,57 +1416,57 @@ z +L 417.231392 228.529115 +L 472.92815 202.564112 +L 528.624908 174.15649 +L 584.321667 143.014642 +L 640.018425 103.779615 +" clip-path="url(#p5565aa6c1c)" style="fill: none; stroke-dasharray: 1.5,2.475; stroke-dashoffset: 0; stroke: #1f77b4; stroke-width: 1.5"/> - + - + - + - +" clip-path="url(#p5565aa6c1c)" style="fill: none; stroke: #1f77b4; stroke-width: 1.5; stroke-linecap: square"/> - + - - + + + - @@ -1626,50 +1624,47 @@ z - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1753,7 +1748,7 @@ L 198.636396 22.93462 " style="fill: none; stroke: #ff7f0e; stroke-width: 1.5; stroke-linecap: square"/> - + - + @@ -1970,10 +1965,10 @@ z - + - + diff --git a/plots/benchmark_roundtrip.pdf b/plots/benchmark_roundtrip.pdf index ab8d13d..16f12c9 100644 Binary files a/plots/benchmark_roundtrip.pdf and b/plots/benchmark_roundtrip.pdf differ diff --git a/plots/benchmark_roundtrip.svg b/plots/benchmark_roundtrip.svg index 3952639..d9ddfc6 100644 --- a/plots/benchmark_roundtrip.svg +++ b/plots/benchmark_roundtrip.svg @@ -25,123 +25,123 @@ z - +" clip-path="url(#p9ef87c5af2)" style="fill: #1f77b4"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #1f77b4"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #1f77b4"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #ff7f0e"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #ff7f0e"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #ff7f0e"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #2ca02c"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #2ca02c"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #2ca02c"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #d62728"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #d62728"/> - +" clip-path="url(#p9ef87c5af2)" style="fill: #d62728"/> - - + - + - + - + - + - + @@ -381,7 +381,7 @@ z - + +L 327.125039 258.904707 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -532,12 +532,12 @@ z +L 327.125039 237.439458 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -604,12 +604,12 @@ z +L 327.125039 215.97421 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -624,12 +624,12 @@ L 324.634372 215.97421 +L 327.125039 194.508961 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -665,12 +665,12 @@ z +L 327.125039 173.043713 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -706,12 +706,12 @@ z +L 327.125039 151.578464 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -727,12 +727,12 @@ L 324.634372 151.578464 +L 327.125039 130.113216 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -748,12 +748,12 @@ L 324.634372 130.113216 +L 327.125039 108.647967 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -769,12 +769,12 @@ L 324.634372 108.647967 +L 327.125039 87.182719 +" clip-path="url(#p9ef87c5af2)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -891,12 +891,12 @@ L 41.674294 87.182719 - - + + - - + + - + + @@ -976,54 +989,8 @@ z - - - - - - - - - - - - - - - + + - - - - - - - - - - - - - - - - - - - - - + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - + + - + - + - - + + - + - + - - + + - - + + - - - + + + + + + - - + + - - + + - + + - - - + + + - - + + - - +" clip-path="url(#p069d844f33)" style="fill: #1f77b4"/> - +" clip-path="url(#p069d844f33)" style="fill: #1f77b4"/> - +" clip-path="url(#p069d844f33)" style="fill: #1f77b4"/> - +" clip-path="url(#p069d844f33)" style="fill: #ff7f0e"/> - +" clip-path="url(#p069d844f33)" style="fill: #ff7f0e"/> - +" clip-path="url(#p069d844f33)" style="fill: #ff7f0e"/> - +" clip-path="url(#p069d844f33)" style="fill: #2ca02c"/> - +" clip-path="url(#p069d844f33)" style="fill: #2ca02c"/> - +" clip-path="url(#p069d844f33)" style="fill: #2ca02c"/> - +" clip-path="url(#p069d844f33)" style="fill: #d62728"/> - +" clip-path="url(#p069d844f33)" style="fill: #d62728"/> - +" clip-path="url(#p069d844f33)" style="fill: #d62728"/> - + - + @@ -1280,12 +1345,12 @@ z - + - + @@ -1302,12 +1367,12 @@ z - + - + @@ -1320,7 +1385,7 @@ z - + @@ -1336,54 +1401,54 @@ z - +" clip-path="url(#p069d844f33)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - - - - - - - - - - - - - - + - + - - - + - - - - + @@ -1736,167 +1760,123 @@ z - - - - - - + + + - + - - - + + + - + - - - + + + - - + + - - - - - - + + + - - + + - - - - + + + + - - + + - + - + - - - + + + - - + + - - - + + + - + - - - + + + - + - - - + + + - + - - - + + + - - + + - - + @@ -1962,16 +1942,16 @@ z - - - - + + + - - @@ -2072,30 +2025,23 @@ z - + - - - - - - - - - + + - - - - + + + @@ -2119,23 +2065,30 @@ z - - + + - - - - + + + + - - - - + + - - - - + + + @@ -2232,13 +2183,11 @@ z - - - - + + - - + + @@ -2267,7 +2216,7 @@ z - + @@ -2292,11 +2241,11 @@ z - - + + - - + + diff --git a/plots/benchmark_roundtrip_dask.pdf b/plots/benchmark_roundtrip_dask.pdf index f7f2381..aff3b9d 100644 Binary files a/plots/benchmark_roundtrip_dask.pdf and b/plots/benchmark_roundtrip_dask.pdf differ diff --git a/plots/benchmark_roundtrip_dask.svg b/plots/benchmark_roundtrip_dask.svg index a8faf66..bba3746 100644 --- a/plots/benchmark_roundtrip_dask.svg +++ b/plots/benchmark_roundtrip_dask.svg @@ -34,61 +34,61 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: #1f77b4"/> +" clip-path="url(#pd8f35cf91b)" style="fill: #1f77b4"/> +" clip-path="url(#pd8f35cf91b)" style="fill: #1f77b4"/> +" clip-path="url(#pd8f35cf91b)" style="fill: #ff7f0e"/> +" clip-path="url(#pd8f35cf91b)" style="fill: #ff7f0e"/> +" clip-path="url(#pd8f35cf91b)" style="fill: #ff7f0e"/> - - + @@ -278,7 +278,7 @@ z - + @@ -315,7 +315,7 @@ z - + @@ -430,16 +430,16 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - - + @@ -476,11 +476,11 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -515,11 +515,11 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -558,11 +558,11 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -607,11 +607,11 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -654,11 +654,11 @@ z +" clip-path="url(#pd8f35cf91b)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -807,8 +807,8 @@ L 319.376296 258.904707 " style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/> - - + + - - + - - + + - - - - - - - - - - - - - - + + - - - + + + - - - - + + + - - - + + + - + + + + + + + + + - + - - + + - - + + @@ -1038,18 +1024,18 @@ z +" clip-path="url(#p45fd2aa662)" style="fill: #1f77b4"/> +" clip-path="url(#p45fd2aa662)" style="fill: #1f77b4"/> +" clip-path="url(#p45fd2aa662)" style="fill: #1f77b4"/> +" clip-path="url(#p45fd2aa662)" style="fill: #ff7f0e"/> +" clip-path="url(#p45fd2aa662)" style="fill: #ff7f0e"/> +" clip-path="url(#p45fd2aa662)" style="fill: #ff7f0e"/> - + @@ -1111,7 +1097,7 @@ z - + @@ -1133,7 +1119,7 @@ z - + @@ -1169,11 +1155,11 @@ z +" clip-path="url(#p45fd2aa662)" style="fill: none; stroke: #b0b0b0; stroke-width: 0.8; stroke-linecap: square"/> - + @@ -1196,18 +1182,18 @@ z - + - + - + @@ -1216,18 +1202,18 @@ L 644.99976 222.161284 - + - + - + @@ -1236,18 +1222,18 @@ L 644.99976 185.417861 - + - + - + @@ -1256,18 +1242,18 @@ L 644.99976 148.674437 - + - + - + @@ -1276,18 +1262,18 @@ L 644.99976 111.931014 - + - + - + @@ -1473,82 +1459,118 @@ L 644.99976 258.904707 " style="fill: none; stroke: #000000; stroke-width: 0.8; stroke-linejoin: miter; stroke-linecap: square"/> - - + + - - + + - - + + + + + - - + + - + + + + - + - - + + - - + + - - + + - - + - - + + - + @@ -1567,8 +1589,8 @@ z " style="fill: #ffffff; opacity: 0.8; stroke: #cccccc; stroke-linejoin: miter"/> - - + + - - @@ -1673,50 +1655,47 @@ z - + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1804,7 +1783,7 @@ z " style="fill: #ff7f0e"/> - + - + - + - + diff --git a/requirements.txt b/requirements.txt index 440fcbb..be5424e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ -click==8.1.7 -numpy==1.26.4 -dask==2024.11.2 -xarray==2024.10.0 +click==8.1.8 +numpy==2.2.4 +dask==2025.3.0 +xarray==2025.3.1 # xarray-tensorstore==0.1.5 -zarr==3.0.0b2 -zarrs==0.1.0 +zarr==3.0.6 +zarrs==0.1.3 pandas==2.2.3 tabulate==0.9.0 -tensorstore==0.1.67 -matplotlib==3.9.2 +tensorstore==0.1.73 +matplotlib==3.10.1 diff --git a/scripts/zarr_dask_python_benchmark_read.py b/scripts/zarr_dask_python_benchmark_read.py index 2966b13..e66532a 100755 --- a/scripts/zarr_dask_python_benchmark_read.py +++ b/scripts/zarr_dask_python_benchmark_read.py @@ -19,8 +19,8 @@ def main(path, concurrent_chunks, read_all): # if "benchmark_compress_shard.zarr" in path: # sys.exit(1) - - arr = da.from_zarr(path) + z = zarr.open_array(path) + arr = da.from_zarr(path, chunks=z.shards) start_time = timeit.default_timer() if read_all: diff --git a/scripts/zarr_dask_python_benchmark_roundtrip.py b/scripts/zarr_dask_python_benchmark_roundtrip.py index f1e75f5..039934e 100755 --- a/scripts/zarr_dask_python_benchmark_roundtrip.py +++ b/scripts/zarr_dask_python_benchmark_roundtrip.py @@ -17,8 +17,9 @@ def main(path, output): # if "benchmark_compress_shard.zarr" in path: # sys.exit(1) + z = zarr.open_array(path) - arr = da.from_zarr(path) + arr = da.from_zarr(path, chunks=z.shards) start_time = timeit.default_timer() da.to_zarr(arr, output) elapsed = timeit.default_timer() - start_time diff --git a/scripts/zarr_python_benchmark_read.py b/scripts/zarr_python_benchmark_read.py index ad9884a..367b9b0 100755 --- a/scripts/zarr_python_benchmark_read.py +++ b/scripts/zarr_python_benchmark_read.py @@ -8,7 +8,7 @@ import sys import zarr -from zarr.storage import LocalStore, RemoteStore +from zarr.storage import LocalStore, FsspecStore from zarr.core.indexing import BlockIndexer from zarr.core.buffer import default_buffer_prototype @@ -33,14 +33,14 @@ async def main(path, concurrent_chunks, read_all): # sys.exit(1) if path.startswith("http"): - store = RemoteStore(url=path) # broken with zarr-python 3.0.0a0 + store = FsspecStore.from_url(url=path) # broken with zarr-python 3.0.0a0 else: - store = LocalStore(path) + store = LocalStore(path, read_only=True) dataset = zarr.open(store=store, mode='r') domain_shape = dataset.shape - chunk_shape = dataset.chunks + chunk_shape = dataset.shards or dataset.chunks print("Domain shape", domain_shape) print("Chunk shape", chunk_shape) @@ -62,7 +62,7 @@ async def chunk_read(chunk_index): tg.create_task(chunk_read(chunk_index)) elif concurrent_chunks == 1: for chunk_index in np.ndindex(*num_chunks): - dataset.get_block_selection(chunk_index) + dataset[tuple(slice(i * s, (1 + i) * s) for i, s in zip(chunk_index, chunk_shape))] else: semaphore = asyncio.Semaphore(concurrent_chunks) async def chunk_read_concurrent_limit(chunk_index): diff --git a/scripts/zarr_python_benchmark_roundtrip.py b/scripts/zarr_python_benchmark_roundtrip.py index cafbc5f..62d9dff 100755 --- a/scripts/zarr_python_benchmark_roundtrip.py +++ b/scripts/zarr_python_benchmark_roundtrip.py @@ -8,7 +8,7 @@ import sys import zarr -from zarr.storage import LocalStore, RemoteStore +from zarr.storage import LocalStore, FsspecStore from zarr.core.indexing import BlockIndexer from zarr.core.buffer import default_buffer_prototype @@ -32,9 +32,9 @@ async def main(path, output): # sys.exit(1) if path.startswith("http"): - store = RemoteStore(url=path) # broken with zarr-python 3.0.0a0 + store = FsspecStore.from_url(url=path) # broken with zarr-python 3.0.0a0 else: - store = LocalStore(path) + store = LocalStore(path, read_only=True) dataset = zarr.open(store=store, mode='r') dataset_out = zarr.create(store=LocalStore(output), mode='w', shape=dataset.shape, chunks=dataset.chunks, dtype=dataset.dtype, codecs=dataset.metadata.codecs) diff --git a/scripts/zarrs_dask_python_benchmark_read.py b/scripts/zarrs_dask_python_benchmark_read.py index 9c5197e..fbf030c 100755 --- a/scripts/zarrs_dask_python_benchmark_read.py +++ b/scripts/zarrs_dask_python_benchmark_read.py @@ -14,6 +14,7 @@ "threading.num_workers": None, "array.write_empty_chunks": False, "codec_pipeline": { + 'batch_size': 1, "path": "zarrs.ZarrsCodecPipeline", "validate_checksums": True, "store_empty_chunks": False, @@ -29,8 +30,8 @@ def main(path, concurrent_chunks, read_all): # if "benchmark_compress_shard.zarr" in path: # sys.exit(1) - - arr = da.from_zarr(path) + z = zarr.open_array(path) + arr = da.from_zarr(path, chunks=z.shards) start_time = timeit.default_timer() if read_all: diff --git a/scripts/zarrs_dask_python_benchmark_roundtrip.py b/scripts/zarrs_dask_python_benchmark_roundtrip.py index 1878bfe..16045b3 100755 --- a/scripts/zarrs_dask_python_benchmark_roundtrip.py +++ b/scripts/zarrs_dask_python_benchmark_roundtrip.py @@ -13,6 +13,7 @@ "threading.num_workers": None, "array.write_empty_chunks": False, "codec_pipeline": { + 'batch_size': 1, "path": "zarrs.ZarrsCodecPipeline", "validate_checksums": True, "store_empty_chunks": False, @@ -27,8 +28,8 @@ def main(path, output): # if "benchmark_compress_shard.zarr" in path: # sys.exit(1) - - arr = da.from_zarr(path) + z = zarr.open_array(path) + arr = da.from_zarr(path, chunks=z.shards) start_time = timeit.default_timer() da.to_zarr(arr, output) elapsed = timeit.default_timer() - start_time diff --git a/scripts/zarrs_python_benchmark_read.py b/scripts/zarrs_python_benchmark_read.py index e5c9b13..9fdef17 100755 --- a/scripts/zarrs_python_benchmark_read.py +++ b/scripts/zarrs_python_benchmark_read.py @@ -8,7 +8,7 @@ import sys import zarr -from zarr.storage import LocalStore, RemoteStore +from zarr.storage import LocalStore, FsspecStore from zarr.core.indexing import BlockIndexer from zarr.core.buffer import default_buffer_prototype @@ -17,6 +17,7 @@ "threading.num_workers": None, "array.write_empty_chunks": False, "codec_pipeline": { + 'batch_size': 1, "path": "zarrs.ZarrsCodecPipeline", "validate_checksums": True, "store_empty_chunks": False, @@ -43,14 +44,14 @@ async def main(path, concurrent_chunks, read_all): # sys.exit(1) if path.startswith("http"): - store = RemoteStore(url=path) # broken with zarr-python 3.0.0a0 + store = FsspecStore.from_url(url=path) # broken with zarr-python 3.0.0a0 else: - store = LocalStore(path) + store = LocalStore(path, read_only=True) dataset = zarr.open(store=store, mode='r') domain_shape = dataset.shape - chunk_shape = dataset.chunks + chunk_shape = dataset.shards or dataset.chunks print("Domain shape", domain_shape) print("Chunk shape", chunk_shape) @@ -72,7 +73,7 @@ async def chunk_read(chunk_index): tg.create_task(chunk_read(chunk_index)) elif concurrent_chunks == 1: for chunk_index in np.ndindex(*num_chunks): - dataset.get_block_selection(chunk_index) + dataset[tuple(slice(i * s, (1 + i) * s) for i, s in zip(chunk_index, chunk_shape))] else: semaphore = asyncio.Semaphore(concurrent_chunks) async def chunk_read_concurrent_limit(chunk_index): diff --git a/scripts/zarrs_python_benchmark_roundtrip.py b/scripts/zarrs_python_benchmark_roundtrip.py index 9bbc4f5..bc99c41 100755 --- a/scripts/zarrs_python_benchmark_roundtrip.py +++ b/scripts/zarrs_python_benchmark_roundtrip.py @@ -4,13 +4,14 @@ import click import zarr -from zarr.storage import LocalStore, RemoteStore +from zarr.storage import LocalStore, FsspecStore import zarrs zarr.config.set({ "threading.num_workers": None, "array.write_empty_chunks": False, "codec_pipeline": { + 'batch_size': 1, "path": "zarrs.ZarrsCodecPipeline", "validate_checksums": True, "store_empty_chunks": False, @@ -24,9 +25,9 @@ @click.argument('output', type=str) def main(path, output): if path.startswith("http"): - store = RemoteStore(url=path) # broken with zarr-python 3.0.0a0 + store = FsspecStore.from_url(url=path) # broken with zarr-python 3.0.0a0 else: - store = LocalStore(path) + store = LocalStore(path, read_only=True) dataset = zarr.open(store=store, mode='r') dataset_out = zarr.create(store=LocalStore(output), mode='w', shape=dataset.shape, chunks=dataset.chunks, dtype=dataset.dtype, codecs=dataset.metadata.codecs)