From d6aa61150e03957c1ed3c4356a92b2e8171c2321 Mon Sep 17 00:00:00 2001 From: AntonIXO Date: Mon, 31 Oct 2022 19:23:57 +0300 Subject: [PATCH 1/4] Add zstd with mixin --- build.gradle | 2 ++ .../c2me/opts/chunkio/common/Config.java | 1 + .../MixinChunkStreamVersion.java | 4 ++++ .../mixin/compression/zstd/AddZstd.java | 19 +++++++++++++++++++ .../resources/c2me-opts-chunkio.mixins.json | 1 + gradle.properties | 1 + 6 files changed, 28 insertions(+) create mode 100644 c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java diff --git a/build.gradle b/build.gradle index 58f896252..eca4b354b 100644 --- a/build.gradle +++ b/build.gradle @@ -124,6 +124,7 @@ configure (allprojects - project(":tests")) { implementation "com.github.LlamaLad7:MixinExtras:${mixinextras_version}" annotationProcessor "com.github.LlamaLad7:MixinExtras:${mixinextras_version}" + implementation "com.github.luben:zstd-jni:${zstd_jni_version}" } } @@ -153,6 +154,7 @@ dependencies { include implementation("org.threadly:threadly:${threadly_version}") include implementation("net.objecthunter:exp4j:${exp4j_version}") include implementation("com.github.LlamaLad7:MixinExtras:${mixinextras_version}") + include implementation("com.github.luben:zstd-jni:${zstd_jni_version}") // PSA: Some older mods, compiled on Loom 0.2.1, might have outdated Maven POMs. // You may need to force-disable transitiveness on them. diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java index 0c9ecd043..c5d7cbcda 100644 --- a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java @@ -26,6 +26,7 @@ public class Config { 1 for GZip (RFC1952) (Vanilla compatible)\s 2 for Zlib (RFC1950) (Vanilla default) (Vanilla compatible)\s 3 for Uncompressed (Fastest, but higher disk usage) (Vanilla compatible)\s + 4 for zstd Experimental, purposed for tests! (Vanilla Incompatible, won't loads worlds saved with this option without it)\\s \s Original chunk data will still readable after modifying this option \s as this option only affects newly stored chunks\s diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java index 54fb85105..f836b0660 100644 --- a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java @@ -1,5 +1,7 @@ package com.ishland.c2me.opts.chunkio.mixin.compression.increase_buffer_size; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; import net.minecraft.world.storage.ChunkStreamVersion; import org.spongepowered.asm.mixin.Dynamic; import org.spongepowered.asm.mixin.Mixin; @@ -31,6 +33,8 @@ private static ChunkStreamVersion redirectChunkStreamVersionConstructor(int id, return new ChunkStreamVersion(id, in -> new InflaterInputStream(in, new Inflater(), 16 * 1024), out -> new DeflaterOutputStream(out, new Deflater(), 16 * 1024)); } else if (id == 3) { // UNCOMPRESSED return new ChunkStreamVersion(id, BufferedInputStream::new, BufferedOutputStream::new); + } else if (id == 4) { // zstd + return new ChunkStreamVersion(id, in -> new ZstdInputStream(in), out -> new ZstdOutputStream(out)); } else { return new ChunkStreamVersion(id, inputStreamWrapper, outputStreamWrapper); } diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java new file mode 100644 index 000000000..825b73229 --- /dev/null +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java @@ -0,0 +1,19 @@ +package com.ishland.c2me.opts.chunkio.mixin.compression.zstd; + +import net.minecraft.world.storage.ChunkStreamVersion; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.gen.Invoker; + +@Mixin(ChunkStreamVersion.class) +public class AddZstd { + + @Invoker("add") + public static ChunkStreamVersion add(ChunkStreamVersion version) { + return null; + } + + private static final ChunkStreamVersion zstd = add(new ChunkStreamVersion(4, inputStream -> new ZstdInputStream(inputStream), outputStream ->new ZstdOutputStream(outputStream))); + +} diff --git a/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json b/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json index a9e72a64a..78afb7fa0 100644 --- a/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json +++ b/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json @@ -6,6 +6,7 @@ "mixins": [ "compression.increase_buffer_size.MixinChunkStreamVersion", "compression.modify_default_chunk_compression.MixinRegionFile", + "compression.zstd.AddZstd", "hide_sync_disk_writes_behind_flag.MixinRegionBasedStorage", "limit_nbt_cache.MixinStorageIoWorker" ] diff --git a/gradle.properties b/gradle.properties index dca01c7bb..b6980e882 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,3 +17,4 @@ night_config_version=3.6.5 threadly_version=7.0 exp4j_version=0.4.8 mixinextras_version=0.1.1 +zstd_jni_version=1.5.5-4 From 8567a79f29677f4f4ff76efe44e083b312e1cda0 Mon Sep 17 00:00:00 2001 From: AntonIXO Date: Mon, 31 Oct 2022 21:24:27 +0300 Subject: [PATCH 2/4] Comments --- .../c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java index 825b73229..7a8eb0c5b 100644 --- a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java @@ -9,11 +9,13 @@ @Mixin(ChunkStreamVersion.class) public class AddZstd { + // Use invoker to use private "add" function @Invoker("add") public static ChunkStreamVersion add(ChunkStreamVersion version) { return null; } + // Add new compression option private static final ChunkStreamVersion zstd = add(new ChunkStreamVersion(4, inputStream -> new ZstdInputStream(inputStream), outputStream ->new ZstdOutputStream(outputStream))); } From 5de24599b86b06ae05d79309ce8f45427c8c1a02 Mon Sep 17 00:00:00 2001 From: AntonIXO Date: Sat, 1 Jul 2023 14:56:12 +0300 Subject: [PATCH 3/4] Format config --- .../main/java/com/ishland/c2me/opts/chunkio/common/Config.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java index c5d7cbcda..44b2f36ee 100644 --- a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java @@ -26,7 +26,7 @@ public class Config { 1 for GZip (RFC1952) (Vanilla compatible)\s 2 for Zlib (RFC1950) (Vanilla default) (Vanilla compatible)\s 3 for Uncompressed (Fastest, but higher disk usage) (Vanilla compatible)\s - 4 for zstd Experimental, purposed for tests! (Vanilla Incompatible, won't loads worlds saved with this option without it)\\s + 4 for zstd (Experimental, purposed for tests) (Vanilla Incompatible, won't loads worlds saved with this option, without this setting)\\s \s Original chunk data will still readable after modifying this option \s as this option only affects newly stored chunks\s From 6656e2d2636274e698be04ef22302ce49c9a0dde Mon Sep 17 00:00:00 2001 From: AntonIXO Date: Sat, 15 Jul 2023 00:21:04 +0300 Subject: [PATCH 4/4] Add benchmark --- src/main/java/com/ishland/c2me/C2MEMod.java | 37 ++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/src/main/java/com/ishland/c2me/C2MEMod.java b/src/main/java/com/ishland/c2me/C2MEMod.java index 154961ce5..07e395652 100644 --- a/src/main/java/com/ishland/c2me/C2MEMod.java +++ b/src/main/java/com/ishland/c2me/C2MEMod.java @@ -7,6 +7,8 @@ import net.minecraft.world.storage.ChunkStreamVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -28,14 +30,16 @@ public void onInitialize() { if (Boolean.getBoolean("com.ishland.c2me.runCompressionBenchmark")) { LOGGER.info("Benchmarking chunk stream speed"); LOGGER.info("Warming up"); - for (int i = 0; i < 3; i++) { + for (int i = 0; i < 4; i++) { runBenchmark("GZIP", ChunkStreamVersion.GZIP, true); runBenchmark("DEFLATE", ChunkStreamVersion.DEFLATE, true); runBenchmark("UNCOMPRESSED", ChunkStreamVersion.UNCOMPRESSED, true); + runZstdBenchmark("zstd", true); } runBenchmark("GZIP", ChunkStreamVersion.GZIP, false); runBenchmark("DEFLATE", ChunkStreamVersion.DEFLATE, false); runBenchmark("UNCOMPRESSED", ChunkStreamVersion.UNCOMPRESSED, false); + runZstdBenchmark("zstd", false); } if (Boolean.getBoolean("com.ishland.c2me.runConsistencyTest")) { consistencyTest(); @@ -73,6 +77,37 @@ private void runBenchmark(String name, ChunkStreamVersion version, boolean suppr } } + private void runZstdBenchmark(String name, boolean suppressLog) { + try { + final DecimalFormat decimalFormat = new DecimalFormat("0.###"); + if (!suppressLog) LOGGER.info("Generating 128MB random data"); + final byte[] bytes = new byte[128 * 1024 * 1024]; + new Random().nextBytes(bytes); + if (!suppressLog) LOGGER.info("Starting benchmark for zstd"); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + { + final OutputStream wrappedOutputStream = new ZstdOutputStream(outputStream); + long startTime = System.nanoTime(); + wrappedOutputStream.write(bytes); + wrappedOutputStream.close(); + long endTime = System.nanoTime(); + if (!suppressLog) LOGGER.info("{} write speed: {} MB/s ({} MB/s compressed)", name, decimalFormat.format((bytes.length / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0)), decimalFormat.format((outputStream.size() / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0))); + if (!suppressLog) LOGGER.info("{} compression ratio: {} %", name, decimalFormat.format(outputStream.size() / (double) bytes.length * 100.0)); + } + { + final ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); + final InputStream wrappedInputStream = new ZstdInputStream(inputStream); + long startTime = System.nanoTime(); + final byte[] readAllBytes = wrappedInputStream.readAllBytes(); + wrappedInputStream.close(); + long endTime = System.nanoTime(); + if (!suppressLog) LOGGER.info("{} read speed: {} MB/s ({} MB/s compressed)", name, decimalFormat.format((readAllBytes.length / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0)), decimalFormat.format((outputStream.size() / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0))); + } + } catch (Throwable t) { + t.printStackTrace(); + } + } + private void consistencyTest() { int taskSize = 512; AtomicIntegerArray array = new AtomicIntegerArray(taskSize);