File tree Expand file tree Collapse file tree 1 file changed +6
-2
lines changed Expand file tree Collapse file tree 1 file changed +6
-2
lines changed Original file line number Diff line number Diff line change 12
12
This benchmark creates N numpy arrays with sizes following a normal distribution
13
13
and measures the time and memory consumption for storing them in TreeStore, h5py, and zarr.
14
14
15
+ The arrays in h5py/zarr are compressed with the same defaults as in TreeStore.
16
+ Moreover, the chunks for storing arrays in h5py/zarr are set to Blosc2's blocks
17
+ (first partition) which should lead to same compression ratio as in TreeStore.
18
+
15
19
Note: This adapts to zarr v3+ API if available.
16
20
"""
17
21
@@ -154,7 +158,7 @@ def store_arrays_in_h5py(arrays, output_file):
154
158
else :
155
159
grp = f [group_name ]
156
160
157
- # Store array with compression
161
+ # Store array with compression; use arr.blocks (first partition in Blosc2) as chunks
158
162
grp .create_dataset (dataset_name , data = arr [:],
159
163
# compression="gzip", shuffle=True,
160
164
# To compare apples with apples, use Blosc2 compression with Zstd compression
@@ -213,7 +217,7 @@ def store_arrays_in_zarr(arrays, output_dir):
213
217
else :
214
218
grp = root [group_name ]
215
219
216
- # Store array with blosc2 compression
220
+ # Store array with blosc2 compression; use arr.blocks (first partition in Blosc2) as chunks
217
221
if zarr .__version__ >= "3" :
218
222
grp .create_array (
219
223
name = dataset_name ,
You can’t perform that action at this time.
0 commit comments