Skip to content

Commit e7be543

Browse files
joshmoorejoshmoore
authored
Merge pull request #15 from sbesson/numthreads_serialization_fix
Blosc compressor: numThreads serialization fix
2 parents d4f85aa + f7f35f9 commit e7be543

File tree

5 files changed

+279
-4
lines changed

5 files changed

+279
-4
lines changed

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66

77
<groupId>dev.zarr</groupId>
88
<artifactId>jzarr</artifactId>
9-
<version>0.4.0</version>
9+
<version>0.4.1-SNAPSHOT</version>
1010

1111
<name>JZarr</name>
1212

‎src/main/java/com/bc/zarr/CompressorFactory.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
package com.bc.zarr;
2828

29+
import com.fasterxml.jackson.annotation.JsonIgnore;
2930
import com.sun.jna.ptr.NativeLongByReference;
3031
import org.blosc.BufferSizes;
3132
import org.blosc.IBloscDll;
@@ -321,6 +322,7 @@ public String getCname() {
321322
return cname;
322323
}
323324

325+
@JsonIgnore
324326
public int getNumThreads() {
325327
return nthreads;
326328
}

‎src/test/java/com/bc/zarr/CompressorFactoryTest.java‎

Lines changed: 93 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,4 +119,96 @@ public void create_compressor_not_supported() {
119119
assertEquals("Compressor id:'kkkkkkk' not supported.", expected.getMessage());
120120
}
121121
}
122-
}
122+
123+
@Test
124+
public void createBloscValidCnames() {
125+
String[] cnames = { "zstd", "blosclz", "lz4", "lz4hc", "zlib" };
126+
for (int i = 0; i < cnames.length; i += 1) {
127+
final Compressor compressor = CompressorFactory.create("blosc", "cname", cnames[i]);
128+
assertNotNull(compressor);
129+
assertEquals("blosc", compressor.getId());
130+
assertEquals(
131+
"compressor=blosc/cname=" + cnames[i] +
132+
"/clevel=5/blocksize=0/shuffle=1", compressor.toString());
133+
}
134+
}
135+
136+
@Test
137+
public void createBloscInvalidCname() {
138+
try {
139+
CompressorFactory.create("blosc", "cname", "unsupported");
140+
fail("IllegalArgumentException expected");
141+
} catch (IllegalArgumentException expected) {
142+
assertEquals("blosc: compressor not supported: 'unsupported'; expected one of [zstd, blosclz, lz4, lz4hc, zlib]", expected.getMessage());
143+
}
144+
}
145+
146+
@Test
147+
public void createBloscValidClevel() {
148+
final Compressor compressor = CompressorFactory.create("blosc", "clevel", 1);
149+
assertNotNull(compressor);
150+
assertEquals("blosc", compressor.getId());
151+
assertEquals(
152+
"compressor=blosc/cname=lz4" +
153+
"/clevel=1/blocksize=0/shuffle=1", compressor.toString());
154+
}
155+
156+
@Test
157+
public void createBloscInvalidClevel() {
158+
try {
159+
CompressorFactory.create("blosc", "clevel", -1);
160+
fail("IllegalArgumentException expected");
161+
} catch (IllegalArgumentException expected) {
162+
assertEquals("blosc: clevel parameter must be between 0 and 9 but was: -1", expected.getMessage());
163+
}
164+
165+
try {
166+
CompressorFactory.create("blosc", "clevel", 10);
167+
fail("IllegalArgumentException expected");
168+
} catch (IllegalArgumentException expected) {
169+
assertEquals(
170+
"blosc: clevel parameter must be between 0 and 9 but was: 10",
171+
expected.getMessage());
172+
}
173+
}
174+
175+
@Test
176+
public void createBloscValidShuffles() {
177+
int[] shuffles = { 0, 1, 2 };
178+
for (int i = 0; i < shuffles.length; i += 1) {
179+
final Compressor compressor = CompressorFactory.create("blosc", "shuffle", shuffles[i]);
180+
assertNotNull(compressor);
181+
assertEquals("blosc", compressor.getId());
182+
assertEquals(
183+
"compressor=blosc/cname=lz4" +
184+
"/clevel=5/blocksize=0/shuffle=" +
185+
shuffles[i], compressor.toString());
186+
}
187+
}
188+
189+
@Test
190+
public void createBloscInvalidShuffle() {
191+
try {
192+
CompressorFactory.create("blosc", "shuffle", -1);
193+
fail("IllegalArgumentException expected");
194+
} catch (IllegalArgumentException expected) {
195+
assertEquals(
196+
"blosc: shuffle type not supported: '-1'; expected one of [0 (NOSHUFFLE), 1 (BYTESHUFFLE), 2 (BITSHUFFLE)]",
197+
expected.getMessage());
198+
}
199+
}
200+
201+
@Test
202+
public void createBloscValidBlockSizes() {
203+
int[] blockSizes = { 0, 1, 20 };
204+
for (int i = 0; i < blockSizes.length; i += 1) {
205+
final Compressor compressor = CompressorFactory.create("blosc", "blocksize", blockSizes[i]);
206+
assertNotNull(compressor);
207+
assertEquals("blosc", compressor.getId());
208+
assertEquals(
209+
"compressor=blosc/cname=lz4" +
210+
"/clevel=5/blocksize=" + blockSizes[i] +
211+
"/shuffle=1", compressor.toString());
212+
}
213+
}
214+
}

‎src/test/java/com/bc/zarr/ZarrArrayDataReaderWriterTest_2D_writeAndReadData.java‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,11 +57,11 @@ public void setUp() throws Exception {
5757
}
5858

5959
@Test
60-
public void getCompressor() throws IOException {
60+
public void getNullCompressor() throws IOException {
6161
final int[] shape = {1, 1};
6262
final int[] chunkShape = {1, 1};
6363
final DataType dataType = DataType.i1; // Byte
64-
final Compressor compressor = CompressorFactory.nullCompressor;
64+
final Compressor compressor = CompressorFactory.create("null");
6565
final ArrayParams parameters = new ArrayParams()
6666
.shape(shape).chunks(chunkShape)
6767
.dataType(dataType)
@@ -72,6 +72,36 @@ public void getCompressor() throws IOException {
7272
assertEquals(compressor, array.getCompressor());
7373
}
7474

75+
@Test
76+
public void getBloscCompressor() throws IOException {
77+
final int[] shape = {1, 1};
78+
final int[] chunkShape = {1, 1};
79+
final DataType dataType = DataType.i1; // Byte
80+
final Compressor compressor = CompressorFactory.create("blosc");
81+
final ArrayParams parameters = new ArrayParams()
82+
.shape(shape).chunks(chunkShape)
83+
.dataType(dataType)
84+
.compressor(compressor);
85+
final ZarrArray array = ZarrArray.create(
86+
new ZarrPath(arrayName), store, parameters, null);
87+
assertEquals(compressor, array.getCompressor());
88+
}
89+
90+
@Test
91+
public void getZlibCompressor() throws IOException {
92+
final int[] shape = {1, 1};
93+
final int[] chunkShape = {1, 1};
94+
final DataType dataType = DataType.i1; // Byte
95+
final Compressor compressor = CompressorFactory.create("zlib");
96+
final ArrayParams parameters = new ArrayParams()
97+
.shape(shape).chunks(chunkShape)
98+
.dataType(dataType)
99+
.compressor(compressor);
100+
final ZarrArray array = ZarrArray.create(
101+
new ZarrPath(arrayName), store, parameters, null);
102+
assertEquals(compressor, array.getCompressor());
103+
}
104+
75105
@Test
76106
public void writeAndRead_Byte_Full() throws IOException, InvalidRangeException {
77107
//preparation

‎src/test/java/com/bc/zarr/ZarrUtilsTestCompression.java‎

Lines changed: 151 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,151 @@
1+
/*
2+
*
3+
* MIT License
4+
*
5+
* Copyright (c) 2020. Brockmann Consult GmbH ([email protected])
6+
*
7+
* Permission is hereby granted, free of charge, to any person obtaining a copy
8+
* of this software and associated documentation files (the "Software"), to deal
9+
* in the Software without restriction, including without limitation the rights
10+
* to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11+
* copies of the Software, and to permit persons to whom the Software is
12+
* furnished to do so, subject to the following conditions:
13+
*
14+
* The above copyright notice and this permission notice shall be included in all
15+
* copies or substantial portions of the Software.
16+
*
17+
* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18+
* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19+
* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20+
* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21+
* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22+
* OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23+
* SOFTWARE.
24+
*
25+
*/
26+
27+
package com.bc.zarr;
28+
29+
import static org.hamcrest.Matchers.*;
30+
import static org.hamcrest.MatcherAssert.*;
31+
import static org.junit.Assert.assertNull;
32+
import static org.junit.Assert.assertNotNull;
33+
import static org.junit.Assert.assertEquals;
34+
import static org.junit.Assert.assertArrayEquals;
35+
36+
import org.junit.*;
37+
import org.junit.runner.RunWith;
38+
import org.junit.runners.Parameterized;
39+
40+
import java.io.IOException;
41+
import java.io.PrintWriter;
42+
import java.io.StringReader;
43+
import java.io.StringWriter;
44+
import java.nio.ByteOrder;
45+
import java.util.Arrays;
46+
import java.util.Collection;
47+
48+
@RunWith(Parameterized.class)
49+
public class ZarrUtilsTestCompression {
50+
51+
private ZarrHeader _zarrHeader;
52+
private final String compression;
53+
54+
@Parameterized.Parameters
55+
public static Collection<String[]> getCompressions() {
56+
return Arrays.asList(new String[][] {
57+
{"blosc"},
58+
{"zlib"},
59+
{"null"}
60+
});
61+
}
62+
63+
public ZarrUtilsTestCompression(String compression) {
64+
this.compression = compression;
65+
}
66+
67+
@Before
68+
public void setUp() {
69+
final int[] chunks = {5, 6};
70+
final Compressor compressor = CompressorFactory.create(compression);
71+
final String dtype = "i4";
72+
final int[] shape = {10, 15};
73+
_zarrHeader = new ZarrHeader(shape, chunks, dtype, ByteOrder.BIG_ENDIAN, 3.6d, compressor, DimensionSeparator.DOT.getSeparatorChar());
74+
}
75+
76+
@Test
77+
public void toJson() throws IOException {
78+
final StringWriter writer = new StringWriter();
79+
80+
ZarrUtils.toJson(_zarrHeader, writer);
81+
assertThat(strip(writer.toString()), is(equalToIgnoringWhiteSpace(expectedJson(compression))));
82+
}
83+
84+
85+
@Test
86+
public void fromJson() throws IOException {
87+
//execution
88+
final ZarrHeader zarrHeader = ZarrUtils.fromJson(new StringReader(expectedJson(compression)), ZarrHeader.class);
89+
90+
//verification
91+
assertNotNull(zarrHeader);
92+
assertThat(zarrHeader.getChunks(), is(equalTo(_zarrHeader.getChunks())));
93+
assertThat(zarrHeader.getDtype(), is(equalTo(_zarrHeader.getDtype())));
94+
if (compression == "null") {
95+
assertNull(zarrHeader.getCompressor());
96+
} else {
97+
assertNotNull(zarrHeader.getCompressor());
98+
assertThat(zarrHeader.getCompressor().toString(), is(equalTo(_zarrHeader.getCompressor().toString())));
99+
}
100+
assertThat(zarrHeader.getFill_value().doubleValue(), is(equalTo(_zarrHeader.getFill_value().doubleValue())));
101+
assertThat(zarrHeader.getShape(), is(equalTo(_zarrHeader.getShape())));
102+
}
103+
104+
105+
private String expectedJson(String compression) {
106+
final StringWriter sw = new StringWriter();
107+
final PrintWriter pw = new PrintWriter(sw);
108+
pw.println("{");
109+
pw.println(" \"chunks\": [");
110+
pw.println(" 5,");
111+
pw.println(" 6");
112+
pw.println(" ],");
113+
if (compression == "null") {
114+
pw.println(" \"compressor\": null,");
115+
} else if (compression == "zlib") {
116+
pw.println(" \"compressor\": {");
117+
pw.println(" \"level\": 1,");
118+
pw.println(" \"id\": \"zlib\"");
119+
pw.println(" },");
120+
} else if (compression == "blosc") {
121+
pw.println(" \"compressor\": {");
122+
pw.println(" \"clevel\": 5,");
123+
pw.println(" \"blocksize\": 0,");
124+
pw.println(" \"shuffle\": 1,");
125+
pw.println(" \"cname\": \"lz4\",");
126+
pw.println(" \"id\": \"blosc\"");
127+
pw.println(" },");
128+
}
129+
pw.println(" \"dtype\": \">i4\",");
130+
pw.println(" \"fill_value\": 3.6,");
131+
pw.println(" \"filters\": null,");
132+
pw.println(" \"order\": \"C\",");
133+
pw.println(" \"shape\": [");
134+
pw.println(" 10,");
135+
pw.println(" 15");
136+
pw.println(" ],");
137+
pw.println(" \"dimension_separator\": \".\",");
138+
pw.println(" \"zarr_format\": 2");
139+
pw.println("}");
140+
141+
return strip(sw.toString());
142+
}
143+
144+
private String strip(String s) {
145+
s = s.replace("\r", "").replace("\n", "");
146+
s = s.replace(" ", "");
147+
// while (s.contains(" ")) s = s.replace(" ", " ");
148+
return s;
149+
}
150+
151+
}

0 commit comments

Comments
 (0)