Improve dependency installation flow

.github/workflows/deploy.yml

@@ -15,10 +15,22 @@ jobs:
         uses: actions/setup-python@v2
         with:
           python-version: '3.x'
-      - name: Install dependencies
+      - name: Install Windows dependencies
+        if: runner.os == 'Windows'
+        run: |
+          choco install -y aria2
+          choco install -y tor
+          7z x data\tools.zip -odata\tools
+      - name: Install Linux dependencies
+        if: runner.os == 'Linux'
+        run: |
+          sudo apt-get install -y aria2
+          sudo apt-get install -y cabextract
+          sudo apt-get install -y tor
+      - name: Install Python dependencies
         run: |
           python -m pip install --upgrade pip
-          pip install orjson requests
+          pip install 'git+git://github.com/m417z/pymultitor.git' 'mitmproxy==6.*' orjson requests
       - name: Run Periodic Update
         run: |
           cd data

data/deploy.py

@@ -86,27 +86,11 @@ def run_virustotal_updates():
     if datetime.now() >= time_to_stop:
         return None
 
-    # Install pymultitor.
-    commands = [
-        ['pip', 'install', 'mitmproxy==6.*'],
-        #['pip', 'install', 'pymultitor'],
-        ['pip', 'install', 'git+git://github.com/m417z/pymultitor.git'],  # use a fork until PRs are merged
-    ]
+    if not check_pymultitor():
+        subprocess.Popen(['pymultitor', '--on-error-code', '429', '--tor-timeout', '0'])
 
-    for args in commands:
-        subprocess.run(args, check=True)
-
-    # Install and set up Tor.
-    tor_params_for_pymultitor = ['--tor-timeout', '0']
-    if platform.system() == 'Windows':
-        tor_params_for_pymultitor += ['--tor-cmd', r'tools\tor-win32-0.4.5.10\Tor\tor.exe']
-    else:
-        subprocess.run(['sudo', 'apt-get', 'install', '-y', 'tor'], check=True)
-
-    subprocess.Popen(['pymultitor', '--on-error-code', '429'] + tor_params_for_pymultitor)
-
-    while not check_pymultitor():
-        time.sleep(1)
+        while not check_pymultitor():
+            time.sleep(1)
 
     virustotal_path = config.out_path.joinpath('virustotal')
     files_count_before = sum(1 for x in virustotal_path.glob('*.json') if not x.name.startswith('_'))
@@ -134,10 +118,6 @@ def run_deploy():
     if datetime.now() >= time_to_stop:
         return None
 
-    if not Path('tools').exists():
-        with zipfile.ZipFile('tools.zip', 'r') as zip_ref:
-            zip_ref.extractall('tools')
-
     progress_file = config.out_path.joinpath('_progress.json')
     if progress_file.is_file():
         with open(progress_file, 'r') as f:

data/upd02_get_manifests_from_updates.py

@@ -77,12 +77,7 @@ def download_update(windows_version, update_kb):
     #    with open(local_path, 'wb') as f:
     #        shutil.copyfileobj(r.raw, f)
 
-    if platform.system() == 'Windows':
-        aria2c_app = 'tools/aria2c.exe'
-    else:
-        aria2c_app = 'aria2c'
-
-    args = [aria2c_app, '-x4', '-o', local_path, '--allow-overwrite=true', download_url]
+    args = ['aria2c', '-x4', '-o', local_path, '--allow-overwrite=true', download_url]
     subprocess.run(args, check=True, stdout=None if config.verbose_run else subprocess.DEVNULL)
 
     return download_url, local_dir, local_path