Use isal for faster compression

m417z · Sep 14, 2022 · 9ff7e84 · 9ff7e84
1 parent 9a56e95
commit 9ff7e84
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml
@@ -37,7 +37,7 @@ jobs:
       - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install 'git+https://github.com/m417z/pymultitor.git' mitmproxy orjson requests
+          pip install 'git+https://github.com/m417z/pymultitor.git' isal mitmproxy orjson requests
       - name: Run periodic update
         run: |
           cd data

diff --git a/data/config.py b/data/config.py
@@ -6,4 +6,4 @@
 extract_in_a_new_thread = False
 exit_on_first_error = True
 high_mem_usage_for_performance = False
-compression_level = 9
+compression_level = 3
diff --git a/data/upd05_group_by_filename.py b/data/upd05_group_by_filename.py
@@ -1,9 +1,9 @@
+from isal import igzip as gzip
 from datetime import datetime
 from pathlib import Path
 import itertools
 import bisect
 import orjson
-import gzip
 import json
 
 import config