Skip to content

Commit

Permalink
add zstd
Browse files Browse the repository at this point in the history
  • Loading branch information
@ikpil
ikpil committed Dec 8, 2023
1 parent 68dab9d commit a2db3f8
Show file tree
Hide file tree
Showing 7 changed files with 106 additions and 54 deletions.
69 changes: 40 additions & 29 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,39 +33,50 @@ If you have any compression libraries you'd like to add, please let me know.
- zip - [system.io.compression.ziparchive](https://learn.microsoft.com/en-us/dotnet/api/system.io.compression.ziparchive)
- snappy - [Snappier](https://github.com/brantburnett/Snappier)
- bzip2 - [SharpZipLib](https://github.com/icsharpcode/SharpZipLib)
- zstd - [ZstdSharp](https://github.com/oleg-st/ZstdSharp)

### Benchmark
- CPU : Ryzen 3600 single core
- RAM : 64GB
- File : silesia.tar
- Size : 211948544

| Name | Comp. MB/s | Decomp. MB/s | Compr.Size | Ratio | Filename | File size |
|-------------------|------------|--------------|------------|----------|-------------|-----------|
| memcpy | 7268.78 | 15795.43 | 211948544 | 100.00 | silesia.tar | 211948544 |
| lz4fast -0 | 165.21 | 2423.67 | 100881461 | 47.60 | silesia.tar | 211948544 |
| lz4hc -6 | 15.71 | 2406.49 | 78386370 | 36.98 | silesia.tar | 211948544 |
| lz4hc -12 | 3.45 | 2366.64 | 77263302 | 36.45 | silesia.tar | 211948544 |
| lz4hc -9 | 10.00 | 2356.09 | 77885122 | 36.75 | silesia.tar | 211948544 |
| lz4hc -3 | 25.75 | 2222.53 | 81343053 | 38.38 | silesia.tar | 211948544 |
| snappy | 218.09 | 843.74 | 102380218 | 48.30 | silesia.tar | 211948544 |
| fastlz -2 | 67.31 | 365.93 | 100147467 | 47.25 | silesia.tar | 211948544 |
| fastlz -1 | 66.13 | 346.14 | 103856237 | 49.00 | silesia.tar | 211948544 |
| zip -Fastest | 34.97 | 378.53 | 75887013 | 35.80 | silesia.tar | 211948544 |
| zip -Optimal | 13.39 | 398.10 | 68352124 | 32.25 | silesia.tar | 211948544 |
| brotli -Fastest | 87.12 | 261.09 | 73444548 | 34.65 | silesia.tar | 211948544 |
| deflate -Fastest | 35.66 | 210.66 | 75886907 | 35.80 | silesia.tar | 211948544 |
| gzip -Fastest | 35.48 | 206.51 | 75886925 | 35.80 | silesia.tar | 211948544 |
| zlib -Fastest | 34.31 | 163.11 | 75886913 | 35.80 | silesia.tar | 211948544 |
| brotli -Optimal | 20.37 | 176.39 | 64211140 | 30.30 | silesia.tar | 211948544 |
| deflate -Optimal | 13.63 | 86.24 | 68352018 | 32.25 | silesia.tar | 211948544 |
| gzip -Optimal | 13.70 | 81.59 | 68352036 | 32.25 | silesia.tar | 211948544 |
| zlib -Optimal | 13.52 | 74.30 | 68352024 | 32.25 | silesia.tar | 211948544 |
| lzma 22.1.1 -5 | 0.24 | 24.07 | 49743984 | 23.47 | silesia.tar | 211948544 |
| lzma 22.1.1 -9 | 0.23 | 24.06 | 49564567 | 23.39 | silesia.tar | 211948544 |
| lzma 22.1.1 -4 | 0.23 | 23.94 | 50444814 | 23.80 | silesia.tar | 211948544 |
| lzma 22.1.1 -2 | 0.31 | 21.38 | 53730001 | 25.35 | silesia.tar | 211948544 |
| lzma 22.1.1 -0 | 0.40 | 18.04 | 59953750 | 28.29 | silesia.tar | 211948544 |
| bzip2 -1 | 1.99 | 31.60 | 60533303 | 28.56 | silesia.tar | 211948544 |
| bzip2 -5 | 1.78 | 30.28 | 55723253 | 26.29 | silesia.tar | 211948544 |
| bzip2 -9 | 1.63 | 29.83 | 54535438 | 25.73 | silesia.tar | 211948544 |
| Name | Comp. MB/s | Decomp. MB/s | Compr.Size | Ratio | Filename | File size |
|------------------|------------|--------------|------------|----------|-------------|-----------|
| memcpy | 7268.78 | 15795.43 | 211948544 | 100.00 | silesia.tar | 211948544 |
| lz4fast -0 | 165.21 | 2423.67 | 100881461 | 47.60 | silesia.tar | 211948544 |
| lz4hc -6 | 15.71 | 2406.49 | 78386370 | 36.98 | silesia.tar | 211948544 |
| lz4hc -12 | 3.45 | 2366.64 | 77263302 | 36.45 | silesia.tar | 211948544 |
| lz4hc -9 | 10.00 | 2356.09 | 77885122 | 36.75 | silesia.tar | 211948544 |
| lz4hc -3 | 25.75 | 2222.53 | 81343053 | 38.38 | silesia.tar | 211948544 |
| snappy | 218.09 | 843.74 | 102380218 | 48.30 | silesia.tar | 211948544 |
| fastlz -2 | 67.31 | 365.93 | 100147467 | 47.25 | silesia.tar | 211948544 |
| fastlz -1 | 66.13 | 346.14 | 103856237 | 49.00 | silesia.tar | 211948544 |
| zip -Fastest | 34.97 | 378.53 | 75887013 | 35.80 | silesia.tar | 211948544 |
| zip -Optimal | 13.39 | 398.10 | 68352124 | 32.25 | silesia.tar | 211948544 |
| brotli -Fastest | 87.12 | 261.09 | 73444548 | 34.65 | silesia.tar | 211948544 |
| deflate -Fastest | 35.66 | 210.66 | 75886907 | 35.80 | silesia.tar | 211948544 |
| gzip -Fastest | 35.48 | 206.51 | 75886925 | 35.80 | silesia.tar | 211948544 |
| zlib -Fastest | 34.31 | 163.11 | 75886913 | 35.80 | silesia.tar | 211948544 |
| brotli -Optimal | 20.37 | 176.39 | 64211140 | 30.30 | silesia.tar | 211948544 |
| deflate -Optimal | 13.63 | 86.24 | 68352018 | 32.25 | silesia.tar | 211948544 |
| gzip -Optimal | 13.70 | 81.59 | 68352036 | 32.25 | silesia.tar | 211948544 |
| zlib -Optimal | 13.52 | 74.30 | 68352024 | 32.25 | silesia.tar | 211948544 |
| lzma 22.1.1 -5 | 0.24 | 24.07 | 49743984 | 23.47 | silesia.tar | 211948544 |
| lzma 22.1.1 -9 | 0.23 | 24.06 | 49564567 | 23.39 | silesia.tar | 211948544 |
| lzma 22.1.1 -4 | 0.23 | 23.94 | 50444814 | 23.80 | silesia.tar | 211948544 |
| lzma 22.1.1 -2 | 0.31 | 21.38 | 53730001 | 25.35 | silesia.tar | 211948544 |
| lzma 22.1.1 -0 | 0.40 | 18.04 | 59953750 | 28.29 | silesia.tar | 211948544 |
| bzip2 -1 | 1.99 | 31.60 | 60533303 | 28.56 | silesia.tar | 211948544 |
| bzip2 -5 | 1.78 | 30.28 | 55723253 | 26.29 | silesia.tar | 211948544 |
| bzip2 -9 | 1.63 | 29.83 | 54535438 | 25.73 | silesia.tar | 211948544 |
| zstd -11 | 5.22 | 801.16 | 58266350 | 27.49 | silesia.tar | 211948544 |
| zstd -15 | 1.59 | 802.37 | 57174130 | 26.98 | silesia.tar | 211948544 |
| zstd -2 | 77.17 | 716.97 | 69484490 | 32.78 | silesia.tar | 211948544 |
| zstd -8 | 11.09 | 777.71 | 60020696 | 28.32 | silesia.tar | 211948544 |
| zstd -5 | 22.69 | 704.05 | 63041984 | 29.74 | silesia.tar | 211948544 |
| zstd -1 | 87.04 | 633.52 | 73418039 | 34.64 | silesia.tar | 211948544 |
| zstd -18 | 0.69 | 694.44 | 53423198 | 25.21 | silesia.tar | 211948544 |
| zstd -22 | 0.42 | 603.32 | 52441255 | 24.74 | silesia.tar | 211948544 |


10 changes: 5 additions & 5 deletions src/DotCompressorBenchmark.Tools/BenchmarkFastLZ.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,19 +43,19 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt
return Benchmarks.Roundtrip(Name, filename, srcBytes, dstBytes, (s, d) => Compress(s, d, _level), Decompress);
}

private static long Compress(byte[] srcBytes, byte[] dstBytes, int level)
private static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, int level)
{
if (2 == level)
{
return FastLZ.CompressLevel2(srcBytes, 0, srcBytes.Length, dstBytes);
return FastLZ.CompressLevel2(uncompressedBytes, 0, uncompressedBytes.Length, compressedBytes);
}

return FastLZ.CompressLevel1(srcBytes, 0, srcBytes.Length, dstBytes);
return FastLZ.CompressLevel1(uncompressedBytes, 0, uncompressedBytes.Length, compressedBytes);
}


private static long Decompress(byte[] srcBytes, long size, byte[] dstBytes)
private static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes)
{
return FastLZ.Decompress(srcBytes, size, dstBytes, dstBytes.Length);
return FastLZ.Decompress(compressedBytes, size, uncompressedBytes, uncompressedBytes.Length);
}
}
12 changes: 6 additions & 6 deletions src/DotCompressorBenchmark.Tools/BenchmarkLZ4.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,17 +53,17 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt
(s, d) => Compress(s, d, _level), Decompress);
}

public static long Compress(byte[] srcBytes, byte[] dstBytes, LZ4Level level)
public static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, LZ4Level level)
{
var writer = new FixedArrayBufferWriter<byte>(dstBytes);
LZ4Pickler.Pickle(srcBytes, writer, level);
var writer = new FixedArrayBufferWriter<byte>(compressedBytes);
LZ4Pickler.Pickle(uncompressedBytes, writer, level);
return writer.WrittenCount;
}

public static long Decompress(byte[] srcBytes, long size, byte[] dstBytes)
public static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes)
{
var writer = new FixedArrayBufferWriter<byte>(dstBytes);
LZ4Pickler.Unpickle(srcBytes.AsSpan(0, (int)size), writer);
var writer = new FixedArrayBufferWriter<byte>(uncompressedBytes);
LZ4Pickler.Unpickle(compressedBytes.AsSpan(0, (int)size), writer);
return writer.WrittenCount;
}
}
18 changes: 9 additions & 9 deletions src/DotCompressorBenchmark.Tools/BenchmarkLZMA.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt
(s, d) => Compress(s, d, _level), Decompress);
}

public static long Compress(byte[] srcBytes, byte[] dstBytes, int level)
public static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, int level)
{
var props = new CLzmaEncProps();
props.level = level;
Expand Down Expand Up @@ -92,8 +92,8 @@ public static long Compress(byte[] srcBytes, byte[] dstBytes, int level)
if (props.mc == 0) props.mc = ((16 + (props.fb >> 1)) >> (0 != props.btMode ? 0 : 1));
if (props.numThreads < 0) props.numThreads = ((0 != props.btMode && 0 != props.algo) ? 2 : 1);

using MemoryStream inStream = new MemoryStream(srcBytes);
using MemoryStream outStream = new MemoryStream(dstBytes);
using MemoryStream inStream = new MemoryStream(uncompressedBytes);
using MemoryStream outStream = new MemoryStream(compressedBytes);
var encoder = new Encoder();
encoder.SetCoderProperties(new CoderPropID[]
{
Expand Down Expand Up @@ -125,16 +125,16 @@ public static long Compress(byte[] srcBytes, byte[] dstBytes, int level)
props.writeEndMark,
});
encoder.WriteCoderProperties(outStream);
outStream.Write(BitConverter.GetBytes((long)srcBytes.Length), 0, 8);
encoder.Code(inStream, outStream, srcBytes.Length, dstBytes.Length, null);
outStream.Write(BitConverter.GetBytes((long)uncompressedBytes.Length), 0, 8);
encoder.Code(inStream, outStream, uncompressedBytes.Length, compressedBytes.Length, null);

return outStream.Position;
}

public static long Decompress(byte[] srcBytes, long size, byte[] dstBytes)
public static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes)
{
using var inStream = new MemoryStream(srcBytes, 0, (int)size);
using var outStream = new MemoryStream(dstBytes);
using var inStream = new MemoryStream(compressedBytes, 0, (int)size);
using var outStream = new MemoryStream(uncompressedBytes);

byte[] properties = new byte[5];
inStream.Read(properties, 0, 5); // header
Expand All @@ -145,7 +145,7 @@ public static long Decompress(byte[] srcBytes, long size, byte[] dstBytes)

var decoder = new Decoder();
decoder.SetDecoderProperties(properties);
decoder.Code(inStream, outStream, size, dstBytes.Length, null);
decoder.Code(inStream, outStream, size, uncompressedBytes.Length, null);

return outStream.Position;
}
Expand Down
10 changes: 5 additions & 5 deletions src/DotCompressorBenchmark.Tools/BenchmarkMemCopy.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,15 +42,15 @@ public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstByt
}


public static long CompressMemcpy(byte[] srcBytes, byte[] dstBytes)
public static long CompressMemcpy(byte[] uncompressedBytes, byte[] compressedBytes)
{
srcBytes.CopyTo(new Span<byte>(dstBytes));
return srcBytes.Length;
uncompressedBytes.CopyTo(new Span<byte>(compressedBytes));
return uncompressedBytes.Length;
}

public static long DecompressMemcpy(byte[] srcBytes, long size, byte[] dstBytes)
public static long DecompressMemcpy(byte[] compressedBytes, long size, byte[] uncompressedBytes)
{
srcBytes.AsSpan(0, (int)size).CopyTo(new Span<byte>(dstBytes));
compressedBytes.AsSpan(0, (int)size).CopyTo(new Span<byte>(uncompressedBytes));
return size;
}
}
40 changes: 40 additions & 0 deletions src/DotCompressorBenchmark.Tools/BenchmarkZstd.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
using System.IO;
using System.IO.Compression;
using ZstdSharp;

namespace DotCompressorBenchmark.Tools;

public class BenchmarkZstd : IBenchmark
{
public string Name { get; }
private readonly int _level;

public BenchmarkZstd(int level)
{
_level = level;
Name = $"zstd -{_level.ToString()}";
}

public BenchmarkResult Roundtrip(string filename, byte[] srcBytes, byte[] dstBytes)
{
return Benchmarks.Roundtrip(Name, filename, srcBytes, dstBytes, (s, d) => Compress(s, d, _level), Decompress);
}

public static long Compress(byte[] uncompressedBytes, byte[] compressedBytes, int level)
{
using var compressedStream = new MemoryStream(compressedBytes);
using (var cs = new CompressionStream(compressedStream, level))
{
cs.Write(uncompressedBytes, 0, uncompressedBytes.Length);
}

return compressedStream.Position;
}

public static long Decompress(byte[] compressedBytes, long size, byte[] uncompressedBytes)
{
using var ms = new MemoryStream(compressedBytes, 0, (int)size);
using var ds = new DecompressionStream(ms);
return ds.Read(uncompressedBytes);
}
}
1 change: 1 addition & 0 deletions src/DotCompressorBenchmark.Tools/DotCompressorBenchmark.Tools.csproj
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
<PackageReference Include="Snappier" Version="1.1.3" />
<PackageReference Include="LZMA-SDK" Version="22.1.1" />
<PackageReference Include="SharpZipLib" Version="1.4.2" />
<PackageReference Include="ZstdSharp.Port" Version="0.7.4" />
</ItemGroup>

<ItemGroup>
Expand Down

0 comments on commit a2db3f8

Please sign in to comment.