diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index a42b85c3c840..32595c713c84 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,10 +15,22 @@ jobs: uses: actions/setup-python@v2 with: python-version: '3.x' - - name: Install dependencies + - name: Install Windows dependencies + if: runner.os == 'Windows' + run: | + choco install -y aria2 + choco install -y tor + 7z x data\tools.zip -odata\tools + - name: Install Linux dependencies + if: runner.os == 'Linux' + run: | + sudo apt-get install -y aria2 + sudo apt-get install -y cabextract + sudo apt-get install -y tor + - name: Install Python dependencies run: | python -m pip install --upgrade pip - pip install orjson requests + pip install 'git+git://github.com/m417z/pymultitor.git' 'mitmproxy==6.*' orjson requests - name: Run Periodic Update run: | cd data diff --git a/data/deploy.py b/data/deploy.py index 1da7eb04fe56..9497c5b00ea4 100644 --- a/data/deploy.py +++ b/data/deploy.py @@ -86,27 +86,11 @@ def run_virustotal_updates(): if datetime.now() >= time_to_stop: return None - # Install pymultitor. - commands = [ - ['pip', 'install', 'mitmproxy==6.*'], - #['pip', 'install', 'pymultitor'], - ['pip', 'install', 'git+git://github.com/m417z/pymultitor.git'], # use a fork until PRs are merged - ] + if not check_pymultitor(): + subprocess.Popen(['pymultitor', '--on-error-code', '429', '--tor-timeout', '0']) - for args in commands: - subprocess.run(args, check=True) - - # Install and set up Tor. - tor_params_for_pymultitor = ['--tor-timeout', '0'] - if platform.system() == 'Windows': - tor_params_for_pymultitor += ['--tor-cmd', r'tools\tor-win32-0.4.5.10\Tor\tor.exe'] - else: - subprocess.run(['sudo', 'apt-get', 'install', '-y', 'tor'], check=True) - - subprocess.Popen(['pymultitor', '--on-error-code', '429'] + tor_params_for_pymultitor) - - while not check_pymultitor(): - time.sleep(1) + while not check_pymultitor(): + time.sleep(1) virustotal_path = config.out_path.joinpath('virustotal') files_count_before = sum(1 for x in virustotal_path.glob('*.json') if not x.name.startswith('_')) @@ -134,10 +118,6 @@ def run_deploy(): if datetime.now() >= time_to_stop: return None - if not Path('tools').exists(): - with zipfile.ZipFile('tools.zip', 'r') as zip_ref: - zip_ref.extractall('tools') - progress_file = config.out_path.joinpath('_progress.json') if progress_file.is_file(): with open(progress_file, 'r') as f: diff --git a/data/tools.zip b/data/tools.zip index ada9bc8abfa5..40f2437bd85d 100644 Binary files a/data/tools.zip and b/data/tools.zip differ diff --git a/data/upd02_get_manifests_from_updates.py b/data/upd02_get_manifests_from_updates.py index fb796dac1053..c6d8bbe877f5 100644 --- a/data/upd02_get_manifests_from_updates.py +++ b/data/upd02_get_manifests_from_updates.py @@ -77,12 +77,7 @@ def download_update(windows_version, update_kb): # with open(local_path, 'wb') as f: # shutil.copyfileobj(r.raw, f) - if platform.system() == 'Windows': - aria2c_app = 'tools/aria2c.exe' - else: - aria2c_app = 'aria2c' - - args = [aria2c_app, '-x4', '-o', local_path, '--allow-overwrite=true', download_url] + args = ['aria2c', '-x4', '-o', local_path, '--allow-overwrite=true', download_url] subprocess.run(args, check=True, stdout=None if config.verbose_run else subprocess.DEVNULL) return download_url, local_dir, local_path