From a2db3f89e3901412d8822f5fd67c6002ca6e79f0 Mon Sep 17 00:00:00 2001 From: ikpil Date: Sat, 9 Dec 2023 01:21:33 +0900 Subject: [PATCH] add zstd --- README.md | 69 +++++++++++-------- .../BenchmarkFastLZ.cs | 10 +-- .../BenchmarkLZ4.cs | 12 ++-- .../BenchmarkLZMA.cs | 18 ++--- .../BenchmarkMemCopy.cs | 10 +-- .../BenchmarkZstd.cs | 40 +++++++++++ .../DotCompressorBenchmark.Tools.csproj | 1 + 7 files changed, 106 insertions(+), 54 deletions(-) create mode 100644 src/DotCompressorBenchmark.Tools/BenchmarkZstd.cs diff --git a/README.md b/README.md index 5231a3c..123ca09 100644 --- a/README.md +++ b/README.md @@ -33,39 +33,50 @@ If you have any compression libraries you'd like to add, please let me know. - zip - [system.io.compression.ziparchive](https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.ziparchive) - snappy - [Snappier](https://github.com/brantburnett/Snappier) - bzip2 - [SharpZipLib](https://github.com/icsharpcode/SharpZipLib) +- zstd - [ZstdSharp](https://github.com/oleg-st/ZstdSharp) ### Benchmark - CPU : Ryzen 3600 single core - RAM : 64GB +- File : silesia.tar +- Size : 211948544 -| Name | Comp. MB/s | Decomp. MB/s | Compr.Size | Ratio | Filename | File size | -|-------------------|------------|--------------|------------|----------|-------------|-----------| -| memcpy | 7268.78 | 15795.43 | 211948544 | 100.00 | silesia.tar | 211948544 | -| lz4fast -0 | 165.21 | 2423.67 | 100881461 | 47.60 | silesia.tar | 211948544 | -| lz4hc -6 | 15.71 | 2406.49 | 78386370 | 36.98 | silesia.tar | 211948544 | -| lz4hc -12 | 3.45 | 2366.64 | 77263302 | 36.45 | silesia.tar | 211948544 | -| lz4hc -9 | 10.00 | 2356.09 | 77885122 | 36.75 | silesia.tar | 211948544 | -| lz4hc -3 | 25.75 | 2222.53 | 81343053 | 38.38 | silesia.tar | 211948544 | -| snappy | 218.09 | 843.74 | 102380218 | 48.30 | silesia.tar | 211948544 | -| fastlz -2 | 67.31 | 365.93 | 100147467 | 47.25 | silesia.tar | 211948544 | -| fastlz -1 | 66.13 | 346.14 | 103856237 | 49.00 | silesia.tar | 211948544 | -| zip -Fastest | 34.97 | 378.53 | 75887013 | 35.80 | silesia.tar | 211948544 | -| zip -Optimal | 13.39 | 398.10 | 68352124 | 32.25 | silesia.tar | 211948544 | -| brotli -Fastest | 87.12 | 261.09 | 73444548 | 34.65 | silesia.tar | 211948544 | -| deflate -Fastest | 35.66 | 210.66 | 75886907 | 35.80 | silesia.tar | 211948544 | -| gzip -Fastest | 35.48 | 206.51 | 75886925 | 35.80 | silesia.tar | 211948544 | -| zlib -Fastest | 34.31 | 163.11 | 75886913 | 35.80 | silesia.tar | 211948544 | -| brotli -Optimal | 20.37 | 176.39 | 64211140 | 30.30 | silesia.tar | 211948544 | -| deflate -Optimal | 13.63 | 86.24 | 68352018 | 32.25 | silesia.tar | 211948544 | -| gzip -Optimal | 13.70 | 81.59 | 68352036 | 32.25 | silesia.tar | 211948544 | -| zlib -Optimal | 13.52 | 74.30 | 68352024 | 32.25 | silesia.tar | 211948544 | -| lzma 22.1.1 -5 | 0.24 | 24.07 | 49743984 | 23.47 | silesia.tar | 211948544 | -| lzma 22.1.1 -9 | 0.23 | 24.06 | 49564567 | 23.39 | silesia.tar | 211948544 | -| lzma 22.1.1 -4 | 0.23 | 23.94 | 50444814 | 23.80 | silesia.tar | 211948544 | -| lzma 22.1.1 -2 | 0.31 | 21.38 | 53730001 | 25.35 | silesia.tar | 211948544 | -| lzma 22.1.1 -0 | 0.40 | 18.04 | 59953750 | 28.29 | silesia.tar | 211948544 | -| bzip2 -1 | 1.99 | 31.60 | 60533303 | 28.56 | silesia.tar | 211948544 | -| bzip2 -5 | 1.78 | 30.28 | 55723253 | 26.29 | silesia.tar | 211948544 | -| bzip2 -9 | 1.63 | 29.83 | 54535438 | 25.73 | silesia.tar | 211948544 | +| Name | Comp. MB/s | Decomp. MB/s | Compr.Size | Ratio | Filename | File size | +|------------------|------------|--------------|------------|----------|-------------|-----------| +| memcpy | 7268.78 | 15795.43 | 211948544 | 100.00 | silesia.tar | 211948544 | +| lz4fast -0 | 165.21 | 2423.67 | 100881461 | 47.60 | silesia.tar | 211948544 | +| lz4hc -6 | 15.71 | 2406.49 | 78386370 | 36.98 | silesia.tar | 211948544 | +| lz4hc -12 | 3.45 | 2366.64 | 77263302 | 36.45 | silesia.tar | 211948544 | +| lz4hc -9 | 10.00 | 2356.09 | 77885122 | 36.75 | silesia.tar | 211948544 | +| lz4hc -3 | 25.75 | 2222.53 | 81343053 | 38.38 | silesia.tar | 211948544 | +| snappy | 218.09 | 843.74 | 102380218 | 48.30 | silesia.tar | 211948544 | +| fastlz -2 | 67.31 | 365.93 | 100147467 | 47.25 | silesia.tar | 211948544 | +| fastlz -1 | 66.13 | 346.14 | 103856237 | 49.00 | silesia.tar | 211948544 | +| zip -Fastest | 34.97 | 378.53 | 75887013 | 35.80 | silesia.tar | 211948544 | +| zip -Optimal | 13.39 | 398.10 | 68352124 | 32.25 | silesia.tar | 211948544 | +| brotli -Fastest | 87.12 | 261.09 | 73444548 | 34.65 | silesia.tar | 211948544 | +| deflate -Fastest | 35.66 | 210.66 | 75886907 | 35.80 | silesia.tar | 211948544 | +| gzip -Fastest | 35.48 | 206.51 | 75886925 | 35.80 | silesia.tar | 211948544 | +| zlib -Fastest | 34.31 | 163.11 | 75886913 | 35.80 | silesia.tar | 211948544 | +| brotli -Optimal | 20.37 | 176.39 | 64211140 | 30.30 | silesia.tar | 211948544 | +| deflate -Optimal | 13.63 | 86.24 | 68352018 | 32.25 | silesia.tar | 211948544 | +| gzip -Optimal | 13.70 | 81.59 | 68352036 | 32.25 | silesia.tar | 211948544 | +| zlib -Optimal | 13.52 | 74.30 | 68352024 | 32.25 | silesia.tar | 211948544 | +| lzma 22.1.1 -5 | 0.24 | 24.07 | 49743984 | 23.47 | silesia.tar | 211948544 | +| lzma 22.1.1 -9 | 0.23 | 24.06 | 49564567 | 23.39 | silesia.tar | 211948544 | +| lzma 22.1.1 -4 | 0.23 | 23.94 | 50444814 | 23.80 | silesia.tar | 211948544 | +| lzma 22.1.1 -2 | 0.31 | 21.38 | 53730001 | 25.35 | silesia.tar | 211948544 | +| lzma 22.1.1 -0 | 0.40 | 18.04 | 59953750 | 28.29 | silesia.tar | 211948544 | +| bzip2 -1 | 1.99 | 31.60 | 60533303 | 28.56 | silesia.tar | 211948544 | +| bzip2 -5 | 1.78 | 30.28 | 55723253 | 26.29 | silesia.tar | 211948544 | +| bzip2 -9 | 1.63 | 29.83 | 54535438 | 25.73 | silesia.tar | 211948544 | +| zstd -11 | 5.22 | 801.16 | 58266350 | 27.49 | silesia.tar | 211948544 | +| zstd -15 | 1.59 | 802.37 | 57174130 | 26.98 | silesia.tar | 211948544 | +| zstd -2 | 77.17 | 716.97 | 69484490 | 32.78 | silesia.tar | 211948544 | +| zstd -8 | 11.09 | 777.71 | 60020696 | 28.32 | silesia.tar | 211948544 | +| zstd -5 | 22.69 | 704.05 | 63041984 | 29.74 | silesia.tar | 211948544 | +| zstd -1 | 87.04 | 633.52 | 73418039 | 34.64 | silesia.tar | 211948544 | +| zstd -18 | 0.69 | 694.44 | 53423198 | 25.21 | silesia.tar | 211948544 | +| zstd -22 | 0.42 | 603.32 | 52441255 | 24.74 | silesia.tar | 211948544 | diff --git a/src/DotCompressorBenchmark.Tools/BenchmarkFastLZ.cs b/src/DotCompressorBenchmark.Tools/BenchmarkFastLZ.cs index 052b105..a83b260 100644 --- a/src/DotCompressorBenchmark.Tools/BenchmarkFastLZ.cs +++ b/src/DotCompressorBenchmark.Tools/BenchmarkFastLZ.cs @@ -43,19 +43,19 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt return Benchmarks.Roundtrip(Name, filename, srcBytes, dstBytes, (s, d) => Compress(s, d, _level), Decompress); } - private static long Compress(byte[] srcBytes, byte[] dstBytes, int level) + private static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, int level) { if (2 == level) { - return FastLZ.CompressLevel2(srcBytes, 0, srcBytes.Length, dstBytes); + return FastLZ.CompressLevel2(uncompressedBytes, 0, uncompressedBytes.Length, compressedBytes); } - return FastLZ.CompressLevel1(srcBytes, 0, srcBytes.Length, dstBytes); + return FastLZ.CompressLevel1(uncompressedBytes, 0, uncompressedBytes.Length, compressedBytes); } - private static long Decompress(byte[] srcBytes, long size, byte[] dstBytes) + private static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes) { - return FastLZ.Decompress(srcBytes, size, dstBytes, dstBytes.Length); + return FastLZ.Decompress(compressedBytes, size, uncompressedBytes, uncompressedBytes.Length); } } \ No newline at end of file diff --git a/src/DotCompressorBenchmark.Tools/BenchmarkLZ4.cs b/src/DotCompressorBenchmark.Tools/BenchmarkLZ4.cs index 5fb16fa..0723663 100644 --- a/src/DotCompressorBenchmark.Tools/BenchmarkLZ4.cs +++ b/src/DotCompressorBenchmark.Tools/BenchmarkLZ4.cs @@ -53,17 +53,17 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt (s, d) => Compress(s, d, _level), Decompress); } - public static long Compress(byte[] srcBytes, byte[] dstBytes, LZ4Level level) + public static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, LZ4Level level) { - var writer = new FixedArrayBufferWriter(dstBytes); - LZ4Pickler.Pickle(srcBytes, writer, level); + var writer = new FixedArrayBufferWriter(compressedBytes); + LZ4Pickler.Pickle(uncompressedBytes, writer, level); return writer.WrittenCount; } - public static long Decompress(byte[] srcBytes, long size, byte[] dstBytes) + public static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes) { - var writer = new FixedArrayBufferWriter(dstBytes); - LZ4Pickler.Unpickle(srcBytes.AsSpan(0, (int)size), writer); + var writer = new FixedArrayBufferWriter(uncompressedBytes); + LZ4Pickler.Unpickle(compressedBytes.AsSpan(0, (int)size), writer); return writer.WrittenCount; } } \ No newline at end of file diff --git a/src/DotCompressorBenchmark.Tools/BenchmarkLZMA.cs b/src/DotCompressorBenchmark.Tools/BenchmarkLZMA.cs index 8e67b0e..5146660 100644 --- a/src/DotCompressorBenchmark.Tools/BenchmarkLZMA.cs +++ b/src/DotCompressorBenchmark.Tools/BenchmarkLZMA.cs @@ -56,7 +56,7 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt (s, d) => Compress(s, d, _level), Decompress); } - public static long Compress(byte[] srcBytes, byte[] dstBytes, int level) + public static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, int level) { var props = new CLzmaEncProps(); props.level = level; @@ -92,8 +92,8 @@ public static long Compress(byte[] srcBytes, byte[] dstBytes, int level) if (props.mc == 0) props.mc = ((16 + (props.fb >> 1)) >> (0 != props.btMode ? 0 : 1)); if (props.numThreads < 0) props.numThreads = ((0 != props.btMode && 0 != props.algo) ? 2 : 1); - using MemoryStream inStream = new MemoryStream(srcBytes); - using MemoryStream outStream = new MemoryStream(dstBytes); + using MemoryStream inStream = new MemoryStream(uncompressedBytes); + using MemoryStream outStream = new MemoryStream(compressedBytes); var encoder = new Encoder(); encoder.SetCoderProperties(new CoderPropID[] { @@ -125,16 +125,16 @@ public static long Compress(byte[] srcBytes, byte[] dstBytes, int level) props.writeEndMark, }); encoder.WriteCoderProperties(outStream); - outStream.Write(BitConverter.GetBytes((long)srcBytes.Length), 0, 8); - encoder.Code(inStream, outStream, srcBytes.Length, dstBytes.Length, null); + outStream.Write(BitConverter.GetBytes((long)uncompressedBytes.Length), 0, 8); + encoder.Code(inStream, outStream, uncompressedBytes.Length, compressedBytes.Length, null); return outStream.Position; } - public static long Decompress(byte[] srcBytes, long size, byte[] dstBytes) + public static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes) { - using var inStream = new MemoryStream(srcBytes, 0, (int)size); - using var outStream = new MemoryStream(dstBytes); + using var inStream = new MemoryStream(compressedBytes, 0, (int)size); + using var outStream = new MemoryStream(uncompressedBytes); byte[] properties = new byte[5]; inStream.Read(properties, 0, 5); // header @@ -145,7 +145,7 @@ public static long Decompress(byte[] srcBytes, long size, byte[] dstBytes) var decoder = new Decoder(); decoder.SetDecoderProperties(properties); - decoder.Code(inStream, outStream, size, dstBytes.Length, null); + decoder.Code(inStream, outStream, size, uncompressedBytes.Length, null); return outStream.Position; } diff --git a/src/DotCompressorBenchmark.Tools/BenchmarkMemCopy.cs b/src/DotCompressorBenchmark.Tools/BenchmarkMemCopy.cs index 71704b0..c7501bd 100644 --- a/src/DotCompressorBenchmark.Tools/BenchmarkMemCopy.cs +++ b/src/DotCompressorBenchmark.Tools/BenchmarkMemCopy.cs @@ -42,15 +42,15 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt } - public static long CompressMemcpy(byte[] srcBytes, byte[] dstBytes) + public static long CompressMemcpy(byte[] uncompressedBytes, byte[] compressedBytes) { - srcBytes.CopyTo(new Span(dstBytes)); - return srcBytes.Length; + uncompressedBytes.CopyTo(new Span(compressedBytes)); + return uncompressedBytes.Length; } - public static long DecompressMemcpy(byte[] srcBytes, long size, byte[] dstBytes) + public static long DecompressMemcpy(byte[] compressedBytes, long size, byte[] uncompressedBytes) { - srcBytes.AsSpan(0, (int)size).CopyTo(new Span(dstBytes)); + compressedBytes.AsSpan(0, (int)size).CopyTo(new Span(uncompressedBytes)); return size; } } \ No newline at end of file diff --git a/src/DotCompressorBenchmark.Tools/BenchmarkZstd.cs b/src/DotCompressorBenchmark.Tools/BenchmarkZstd.cs new file mode 100644 index 0000000..009f927 --- /dev/null +++ b/src/DotCompressorBenchmark.Tools/BenchmarkZstd.cs @@ -0,0 +1,40 @@ +using System.IO; +using System.IO.Compression; +using ZstdSharp; + +namespace DotCompressorBenchmark.Tools; + +public class BenchmarkZstd : IBenchmark +{ + public string Name { get; } + private readonly int _level; + + public BenchmarkZstd(int level) + { + _level = level; + Name = $"zstd -{_level.ToString()}"; + } + + public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstBytes) + { + return Benchmarks.Roundtrip(Name, filename, srcBytes, dstBytes, (s, d) => Compress(s, d, _level), Decompress); + } + + public static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, int level) + { + using var compressedStream = new MemoryStream(compressedBytes); + using (var cs = new CompressionStream(compressedStream, level)) + { + cs.Write(uncompressedBytes, 0, uncompressedBytes.Length); + } + + return compressedStream.Position; + } + + public static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes) + { + using var ms = new MemoryStream(compressedBytes, 0, (int)size); + using var ds = new DecompressionStream(ms); + return ds.Read(uncompressedBytes); + } +} \ No newline at end of file diff --git a/src/DotCompressorBenchmark.Tools/DotCompressorBenchmark.Tools.csproj b/src/DotCompressorBenchmark.Tools/DotCompressorBenchmark.Tools.csproj index 39a9039..c7abc55 100644 --- a/src/DotCompressorBenchmark.Tools/DotCompressorBenchmark.Tools.csproj +++ b/src/DotCompressorBenchmark.Tools/DotCompressorBenchmark.Tools.csproj @@ -12,6 +12,7 @@ +