diff --git a/build.gradle b/build.gradle index 58f896252..eca4b354b 100644 --- a/build.gradle +++ b/build.gradle @@ -124,6 +124,7 @@ configure (allprojects - project(":tests")) { implementation "com.github.LlamaLad7:MixinExtras:${mixinextras_version}" annotationProcessor "com.github.LlamaLad7:MixinExtras:${mixinextras_version}" + implementation "com.github.luben:zstd-jni:${zstd_jni_version}" } } @@ -153,6 +154,7 @@ dependencies { include implementation("org.threadly:threadly:${threadly_version}") include implementation("net.objecthunter:exp4j:${exp4j_version}") include implementation("com.github.LlamaLad7:MixinExtras:${mixinextras_version}") + include implementation("com.github.luben:zstd-jni:${zstd_jni_version}") // PSA: Some older mods, compiled on Loom 0.2.1, might have outdated Maven POMs. // You may need to force-disable transitiveness on them. diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java index 0c9ecd043..44b2f36ee 100644 --- a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/common/Config.java @@ -26,6 +26,7 @@ public class Config { 1 for GZip (RFC1952) (Vanilla compatible)\s 2 for Zlib (RFC1950) (Vanilla default) (Vanilla compatible)\s 3 for Uncompressed (Fastest, but higher disk usage) (Vanilla compatible)\s + 4 for zstd (Experimental, purposed for tests) (Vanilla Incompatible, won't loads worlds saved with this option, without this setting)\\s \s Original chunk data will still readable after modifying this option \s as this option only affects newly stored chunks\s diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java index 54fb85105..f836b0660 100644 --- a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/increase_buffer_size/MixinChunkStreamVersion.java @@ -1,5 +1,7 @@ package com.ishland.c2me.opts.chunkio.mixin.compression.increase_buffer_size; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; import net.minecraft.world.storage.ChunkStreamVersion; import org.spongepowered.asm.mixin.Dynamic; import org.spongepowered.asm.mixin.Mixin; @@ -31,6 +33,8 @@ private static ChunkStreamVersion redirectChunkStreamVersionConstructor(int id, return new ChunkStreamVersion(id, in -> new InflaterInputStream(in, new Inflater(), 16 * 1024), out -> new DeflaterOutputStream(out, new Deflater(), 16 * 1024)); } else if (id == 3) { // UNCOMPRESSED return new ChunkStreamVersion(id, BufferedInputStream::new, BufferedOutputStream::new); + } else if (id == 4) { // zstd + return new ChunkStreamVersion(id, in -> new ZstdInputStream(in), out -> new ZstdOutputStream(out)); } else { return new ChunkStreamVersion(id, inputStreamWrapper, outputStreamWrapper); } diff --git a/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java new file mode 100644 index 000000000..7a8eb0c5b --- /dev/null +++ b/c2me-opts-chunkio/src/main/java/com/ishland/c2me/opts/chunkio/mixin/compression/zstd/AddZstd.java @@ -0,0 +1,21 @@ +package com.ishland.c2me.opts.chunkio.mixin.compression.zstd; + +import net.minecraft.world.storage.ChunkStreamVersion; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; +import org.spongepowered.asm.mixin.Mixin; +import org.spongepowered.asm.mixin.gen.Invoker; + +@Mixin(ChunkStreamVersion.class) +public class AddZstd { + + // Use invoker to use private "add" function + @Invoker("add") + public static ChunkStreamVersion add(ChunkStreamVersion version) { + return null; + } + + // Add new compression option + private static final ChunkStreamVersion zstd = add(new ChunkStreamVersion(4, inputStream -> new ZstdInputStream(inputStream), outputStream ->new ZstdOutputStream(outputStream))); + +} diff --git a/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json b/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json index a9e72a64a..78afb7fa0 100644 --- a/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json +++ b/c2me-opts-chunkio/src/main/resources/c2me-opts-chunkio.mixins.json @@ -6,6 +6,7 @@ "mixins": [ "compression.increase_buffer_size.MixinChunkStreamVersion", "compression.modify_default_chunk_compression.MixinRegionFile", + "compression.zstd.AddZstd", "hide_sync_disk_writes_behind_flag.MixinRegionBasedStorage", "limit_nbt_cache.MixinStorageIoWorker" ] diff --git a/gradle.properties b/gradle.properties index dca01c7bb..b6980e882 100644 --- a/gradle.properties +++ b/gradle.properties @@ -17,3 +17,4 @@ night_config_version=3.6.5 threadly_version=7.0 exp4j_version=0.4.8 mixinextras_version=0.1.1 +zstd_jni_version=1.5.5-4 diff --git a/src/main/java/com/ishland/c2me/C2MEMod.java b/src/main/java/com/ishland/c2me/C2MEMod.java index 154961ce5..07e395652 100644 --- a/src/main/java/com/ishland/c2me/C2MEMod.java +++ b/src/main/java/com/ishland/c2me/C2MEMod.java @@ -7,6 +7,8 @@ import net.minecraft.world.storage.ChunkStreamVersion; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.github.luben.zstd.ZstdInputStream; +import com.github.luben.zstd.ZstdOutputStream; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; @@ -28,14 +30,16 @@ public void onInitialize() { if (Boolean.getBoolean("com.ishland.c2me.runCompressionBenchmark")) { LOGGER.info("Benchmarking chunk stream speed"); LOGGER.info("Warming up"); - for (int i = 0; i < 3; i++) { + for (int i = 0; i < 4; i++) { runBenchmark("GZIP", ChunkStreamVersion.GZIP, true); runBenchmark("DEFLATE", ChunkStreamVersion.DEFLATE, true); runBenchmark("UNCOMPRESSED", ChunkStreamVersion.UNCOMPRESSED, true); + runZstdBenchmark("zstd", true); } runBenchmark("GZIP", ChunkStreamVersion.GZIP, false); runBenchmark("DEFLATE", ChunkStreamVersion.DEFLATE, false); runBenchmark("UNCOMPRESSED", ChunkStreamVersion.UNCOMPRESSED, false); + runZstdBenchmark("zstd", false); } if (Boolean.getBoolean("com.ishland.c2me.runConsistencyTest")) { consistencyTest(); @@ -73,6 +77,37 @@ private void runBenchmark(String name, ChunkStreamVersion version, boolean suppr } } + private void runZstdBenchmark(String name, boolean suppressLog) { + try { + final DecimalFormat decimalFormat = new DecimalFormat("0.###"); + if (!suppressLog) LOGGER.info("Generating 128MB random data"); + final byte[] bytes = new byte[128 * 1024 * 1024]; + new Random().nextBytes(bytes); + if (!suppressLog) LOGGER.info("Starting benchmark for zstd"); + final ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + { + final OutputStream wrappedOutputStream = new ZstdOutputStream(outputStream); + long startTime = System.nanoTime(); + wrappedOutputStream.write(bytes); + wrappedOutputStream.close(); + long endTime = System.nanoTime(); + if (!suppressLog) LOGGER.info("{} write speed: {} MB/s ({} MB/s compressed)", name, decimalFormat.format((bytes.length / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0)), decimalFormat.format((outputStream.size() / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0))); + if (!suppressLog) LOGGER.info("{} compression ratio: {} %", name, decimalFormat.format(outputStream.size() / (double) bytes.length * 100.0)); + } + { + final ByteArrayInputStream inputStream = new ByteArrayInputStream(outputStream.toByteArray()); + final InputStream wrappedInputStream = new ZstdInputStream(inputStream); + long startTime = System.nanoTime(); + final byte[] readAllBytes = wrappedInputStream.readAllBytes(); + wrappedInputStream.close(); + long endTime = System.nanoTime(); + if (!suppressLog) LOGGER.info("{} read speed: {} MB/s ({} MB/s compressed)", name, decimalFormat.format((readAllBytes.length / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0)), decimalFormat.format((outputStream.size() / 1024.0 / 1024.0) / ((endTime - startTime) / 1_000_000_000.0))); + } + } catch (Throwable t) { + t.printStackTrace(); + } + } + private void consistencyTest() { int taskSize = 512; AtomicIntegerArray array = new AtomicIntegerArray(taskSize);