diff --git a/.git_archival.txt b/.git_archival.txt new file mode 100644 index 00000000..b1a286bb --- /dev/null +++ b/.git_archival.txt @@ -0,0 +1,4 @@ +node: $Format:%H$ +node-date: $Format:%cI$ +describe-name: $Format:%(describe:tags=true,match=*[0-9]*)$ +ref-names: $Format:%D$ \ No newline at end of file diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..00a7b00c --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +.git_archival.txt export-subst diff --git a/.github/workflows/check-combinations.yml b/.github/workflows/check-combinations.yml index 09112ee0..938887cf 100644 --- a/.github/workflows/check-combinations.yml +++ b/.github/workflows/check-combinations.yml @@ -14,7 +14,7 @@ jobs: check_combinations: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 with: fetch-depth: 20 diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 71c1d2f0..bd21e1f2 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -16,7 +16,7 @@ jobs: name: Check packaging 📦 runs-on: ubuntu-20.04 steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 with: fetch-depth: 20 - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 9f502519..3a1585da 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -28,7 +28,7 @@ jobs: steps: - name: Checkout repository - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Initialize CodeQL uses: github/codeql-action/init@v2 diff --git a/.github/workflows/publish-to-pypi.yml b/.github/workflows/publish-to-pypi.yml index 5d8da89a..eb452629 100644 --- a/.github/workflows/publish-to-pypi.yml +++ b/.github/workflows/publish-to-pypi.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Checkout 🛎️ - uses: actions/checkout@v2 + uses: actions/checkout@v3 with: fetch-depth: 50 - name: Fetch release tag diff --git a/.github/workflows/release-note.yml b/.github/workflows/release-note.yml index b7c039ad..9dfa31bf 100644 --- a/.github/workflows/release-note.yml +++ b/.github/workflows/release-note.yml @@ -7,7 +7,7 @@ jobs: create-release-notes: runs-on: ubuntu-latest steps: - - uses: actions/checkout@master + - uses: actions/checkout@v3 - name: Create Release Notes uses: docker://decathlon/release-notes-generator-action:2.0.0 env: diff --git a/.github/workflows/test-install-qt.yml b/.github/workflows/test-install-qt.yml index e882d744..0f070cf8 100644 --- a/.github/workflows/test-install-qt.yml +++ b/.github/workflows/test-install-qt.yml @@ -1,6 +1,6 @@ name: Test on GH actions environment -on: push +on: [push, pull_request] jobs: test: @@ -8,7 +8,7 @@ jobs: strategy: matrix: os: [windows-latest, macOS-latest, ubuntu-latest] - py: ["3.10"] + py: ["3.8", "3.10", "3.11"] qtver: [5.9.9, 5.12.8, 6.1.0] artifact: [standard] include: @@ -25,7 +25,7 @@ jobs: py: "3.10" qtver: 6.1.0 steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: fetch-depth: 20 - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* @@ -112,15 +112,16 @@ jobs: if qtver.startswith('6'): command_line6 = [] command_line6.extend(prefix) - if platform == 'ubuntu-20.04': + if platform.startswith("ubuntu"): command_line6.extend([qtver, "linux", "android", "android_armv7"]) timeout = 360 - elif platform == "macOS-latest": + elif platform.startswith("macOS"): command_line6.extend([qtver, "mac", "ios", "ios"]) timeout = 360 else: command_line6.extend([qtver, "windows", "android", "android_armv7"]) timeout = 360 + print("Execute: {}".format(command_line6)) try: res = subprocess.run(command_line6, timeout=timeout, check=True) except subprocess.CalledProcessError as cpe: diff --git a/.github/workflows/upload-release-artifacts.yml b/.github/workflows/upload-release-artifacts.yml index 83ef4a8b..026380ff 100644 --- a/.github/workflows/upload-release-artifacts.yml +++ b/.github/workflows/upload-release-artifacts.yml @@ -12,14 +12,20 @@ on: jobs: build-standalone: - runs-on: ${{ matrix.os }} + runs-on: ${{ matrix.system.os }} strategy: matrix: - os: [windows-latest] + system: [ + {os: windows-latest, arch: x86, output_file: 'aqt_x86.exe', primary_artifact: '', secondary_artifact: 'aqt_x86.exe'}, + {os: windows-latest, arch: x64, output_file: 'aqt_x64.exe', primary_artifact: 'aqt.exe', secondary_artifact: 'aqt_x64.exe'}, + {os: macOS-latest, arch: x64, output_file: 'aqt_x64', primary_artifact: 'aqt-macos', secondary_artifact: 'aqt-macos_x64'} + ] py: [3.9] - arch: [x86, x64] + defaults: + run: + shell: ${{ matrix.system.os == 'windows-latest' && 'pwsh' || 'bash' }} {0} steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v3 with: fetch-depth: 20 - run: git fetch --depth=1 origin +refs/tags/*:refs/tags/* @@ -27,35 +33,35 @@ jobs: uses: actions/setup-python@v4 with: python-version: ${{ matrix.py }} - architecture: ${{ matrix.arch }} + architecture: ${{ matrix.system.arch }} - name: Build standalone binary run: | python -m venv venv - venv/Scripts/activate.ps1 + ${{ matrix.system.os == 'windows-latest' && 'venv/Scripts/activate.ps1' || 'chmod +x venv/bin/activate && venv/bin/activate' }} python -m pip install -U pip wheel setuptools setuptools_scm pyinstaller python -m pip install . - python tools/build_standalone.py ${{ matrix.arch }} - deactivate - Remove-Item venv -Recurse -Force - shell: pwsh + python tools/build_standalone.py ${{ matrix.system.arch }} + ${{ matrix.system.os == 'windows-latest' && 'deactivate' || 'chmod +x venv/bin/deactivate && venv/bin/deactivate' }} + ${{ matrix.system.os == 'windows-latest' && 'Remove-Item venv -Recurse -Force' || 'rm -rf venv' }} - name: Upload to Release uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: dist\aqt_x64.exe - asset_name: aqt.exe + file: dist/${{ matrix.system.output_file }} + asset_name: ${{ matrix.system.primary_artifact }} tag: ${{ github.ref }} overwrite: true - if: matrix.arch=='x64' && startsWith(github.ref, 'refs/tags/v') + if: matrix.system.primary_artifact!='' && startsWith(github.ref, 'refs/tags/v') - name: Upload to Release for all architectures uses: svenstaro/upload-release-action@v2 with: repo_token: ${{ secrets.GITHUB_TOKEN }} - file: dist\aqt_${{ matrix.arch }}.exe + file: dist/${{ matrix.system.output_file }} + asset_name: ${{ matrix.system.secondary_artifact }} tag: ${{ github.ref }} overwrite: true - if: startsWith(github.ref, 'refs/tags/v') + if: matrix.system.secondary_artifact!='' && startsWith(github.ref, 'refs/tags/v') - name: Update continuous build uses: svenstaro/upload-release-action@v2 with: @@ -63,9 +69,9 @@ jobs: overwrite: true prerelease: true tag: Continuous - file: dist\aqt_x64.exe - asset_name: aqt.exe - if: matrix.arch=='x64' && startsWith(github.ref, 'refs/tags/v') + file: dist/${{ matrix.system.output_file }} + asset_name: ${{ matrix.system.primary_artifact }} + if: matrix.system.primary_artifact!='' && startsWith(github.ref, 'refs/tags/v') - name: Update continuous build for all architectures uses: svenstaro/upload-release-action@v2 with: @@ -73,5 +79,6 @@ jobs: overwrite: true prerelease: true tag: Continuous - file: dist\aqt_${{ matrix.arch }}.exe - if: startsWith(github.ref, 'refs/tags/v') + file: dist/${{ matrix.system.output_file }} + asset_name: ${{ matrix.system.secondary_artifact }} + if: matrix.system.secondary_artifact!='' && startsWith(github.ref, 'refs/tags/v') diff --git a/.readthedocs.yml b/.readthedocs.yml index 0b53257e..c8eed333 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -13,7 +13,6 @@ python: path: . extra_requirements: - docs - system_packages: false build: os: ubuntu-22.04 diff --git a/MANIFEST.in b/MANIFEST.in index 46922963..fa5e0cab 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,4 +1,5 @@ include *.rst +include *.txt include LICENSE include pyproject.toml include .flake8 diff --git a/aqt/archives.py b/aqt/archives.py index 214ca0e7..65598447 100644 --- a/aqt/archives.py +++ b/aqt/archives.py @@ -27,7 +27,7 @@ from defusedxml import ElementTree -from aqt.exceptions import ArchiveDownloadError, ArchiveListError, NoPackageFound +from aqt.exceptions import ArchiveDownloadError, ArchiveListError, ChecksumDownloadFailure, NoPackageFound from aqt.helper import Settings, get_hash, getUrl, ssplit from aqt.metadata import QtRepoProperty, Version @@ -390,7 +390,16 @@ def _get_archives_base(self, name, target_packages): def _download_update_xml(self, update_xml_path): """Hook for unit test.""" - xml_hash = get_hash(update_xml_path, "sha256", self.timeout) + if not Settings.ignore_hash: + try: + xml_hash = get_hash(update_xml_path, Settings.hash_algorithm, self.timeout) + except ChecksumDownloadFailure: + self.logger.warning( + "Failed to download checksum for the file 'Updates.xml'. This may happen on unofficial mirrors." + ) + xml_hash = None + else: + xml_hash = None return getUrl(posixpath.join(self.base, update_xml_path), self.timeout, xml_hash) def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages: Optional[ModuleToPackage]): diff --git a/aqt/combinations.json b/aqt/combinations.json index d81f655b..5767f991 100644 --- a/aqt/combinations.json +++ b/aqt/combinations.json @@ -234,6 +234,83 @@ "qtwebview" ], "qt_version": "6.4" + }, + { + "modules": [ + "debug_info", + "qt3d", + "qt5compat", + "qtcharts", + "qtconnectivity", + "qtdatavis3d", + "qtgrpc", + "qthttpserver", + "qtimageformats", + "qtlanguageserver", + "qtlocation", + "qtlottie", + "qtmultimedia", + "qtnetworkauth", + "qtpdf", + "qtpositioning", + "qtquick3d", + "qtquick3dphysics", + "qtquickeffectmaker", + "qtquicktimeline", + "qtremoteobjects", + "qtscxml", + "qtsensors", + "qtserialbus", + "qtserialport", + "qtshadertools", + "qtspeech", + "qtvirtualkeyboard", + "qtwaylandcompositor", + "qtwebchannel", + "qtwebengine", + "qtwebsockets", + "qtwebview" + ], + "qt_version": "6.5" + }, + { + "modules": [ + "debug_info", + "qt3d", + "qt5compat", + "qtcharts", + "qtconnectivity", + "qtdatavis3d", + "qtgraphs", + "qtgrpc", + "qthttpserver", + "qtimageformats", + "qtlanguageserver", + "qtlocation", + "qtlottie", + "qtmultimedia", + "qtnetworkauth", + "qtpdf", + "qtpositioning", + "qtquick3d", + "qtquick3dphysics", + "qtquickeffectmaker", + "qtquicktimeline", + "qtremoteobjects", + "qtscxml", + "qtsensors", + "qtserialbus", + "qtserialport", + "qtshadertools", + "qtspeech", + "qtvirtualkeyboard", + "qtwaylandcompositor", + "qtwebchannel", + "qtwebengine", + "qtwebsockets", + "qtwebview" + ], + "qt_version": "6.6" } ], "new_archive": [ @@ -291,6 +368,16 @@ "os_name": "linux", "target": "desktop" }, + { + "arch": "wasm_multithread", + "os_name": "linux", + "target": "desktop" + }, + { + "arch": "wasm_singlethread", + "os_name": "linux", + "target": "desktop" + }, { "arch": "android", "os_name": "mac", @@ -326,6 +413,16 @@ "os_name": "mac", "target": "desktop" }, + { + "arch": "wasm_multithread", + "os_name": "mac", + "target": "desktop" + }, + { + "arch": "wasm_singlethread", + "os_name": "mac", + "target": "desktop" + }, { "arch": "ios", "os_name": "mac", @@ -361,6 +458,16 @@ "os_name": "windows", "target": "desktop" }, + { + "arch": "wasm_multithread", + "os_name": "windows", + "target": "desktop" + }, + { + "arch": "wasm_singlethread", + "os_name": "windows", + "target": "desktop" + }, { "arch": "win32_mingw53", "os_name": "windows", @@ -478,6 +585,12 @@ } ], "tools": [ + { + "arch": "qt.tools.qtcreator", + "os_name": "linux", + "target": "desktop", + "tool_name": "sdktool" + }, { "arch": "qt.tools.cmake", "os_name": "linux", @@ -503,7 +616,7 @@ "tool_name": "tools_generic" }, { - "arch": "qt.tools.ifw.45", + "arch": "qt.tools.ifw.46", "os_name": "linux", "target": "desktop", "tool_name": "tools_ifw" @@ -521,16 +634,10 @@ "tool_name": "tools_ninja" }, { - "arch": "qt.tools.openssl.src", - "os_name": "linux", - "target": "desktop", - "tool_name": "tools_openssl_src" - }, - { - "arch": "qt.tools.openssl.gcc_64", + "arch": "qt.tools.opensslv3.src", "os_name": "linux", "target": "desktop", - "tool_name": "tools_openssl_x64" + "tool_name": "tools_opensslv3_src" }, { "arch": "qt.tools.qt3dstudio", @@ -562,6 +669,30 @@ "target": "desktop", "tool_name": "tools_qtcreator" }, + { + "arch": "qt.tools.qtcreator_gui", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreator_gui.src", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatordbg", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatordev", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, { "arch": "qt.tools.qtdesignstudio", "os_name": "linux", @@ -574,12 +705,36 @@ "target": "desktop", "tool_name": "tools_qtdesignstudio_generation2" }, + { + "arch": "qtdesignstudio.lts", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_qtdesignstudio_generation2_lts" + }, { "arch": "qt.tools.qtcreator.telemetry", "os_name": "linux", "target": "desktop", "tool_name": "tools_telemetry" }, + { + "arch": "qt.tools.qtcreator_gui.telemetry_evaluator", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_telemetry_eval_gui" + }, + { + "arch": "qt.tools.qtcreator_gui.telemetry", + "os_name": "linux", + "target": "desktop", + "tool_name": "tools_telemetry_gui" + }, + { + "arch": "qt.tools.qtcreator", + "os_name": "mac", + "target": "desktop", + "tool_name": "sdktool" + }, { "arch": "qt.tools.cmake", "os_name": "mac", @@ -605,7 +760,7 @@ "tool_name": "tools_generic" }, { - "arch": "qt.tools.ifw.45", + "arch": "qt.tools.ifw.46", "os_name": "mac", "target": "desktop", "tool_name": "tools_ifw" @@ -646,6 +801,24 @@ "target": "desktop", "tool_name": "tools_qtcreator" }, + { + "arch": "qt.tools.qtcreator_gui", + "os_name": "mac", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatordbg", + "os_name": "mac", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatordev", + "os_name": "mac", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, { "arch": "qt.tools.qtdesignstudio", "os_name": "mac", @@ -658,12 +831,36 @@ "target": "desktop", "tool_name": "tools_qtdesignstudio_generation2" }, + { + "arch": "qtdesignstudio.lts", + "os_name": "mac", + "target": "desktop", + "tool_name": "tools_qtdesignstudio_generation2_lts" + }, { "arch": "qt.tools.qtcreator.telemetry", "os_name": "mac", "target": "desktop", "tool_name": "tools_telemetry" }, + { + "arch": "qt.tools.qtcreator_gui.telemetry_evaluator", + "os_name": "mac", + "target": "desktop", + "tool_name": "tools_telemetry_eval_gui" + }, + { + "arch": "qt.tools.qtcreator_gui.telemetry", + "os_name": "mac", + "target": "desktop", + "tool_name": "tools_telemetry_gui" + }, + { + "arch": "qt.tools.qtcreator", + "os_name": "windows", + "target": "desktop", + "tool_name": "sdktool" + }, { "arch": "qt.tools.cmake", "os_name": "windows", @@ -689,7 +886,7 @@ "tool_name": "tools_generic" }, { - "arch": "qt.tools.ifw.45", + "arch": "qt.tools.ifw.46", "os_name": "windows", "target": "desktop", "tool_name": "tools_ifw" @@ -773,22 +970,16 @@ "tool_name": "tools_ninja" }, { - "arch": "qt.tools.openssl.src", - "os_name": "windows", - "target": "desktop", - "tool_name": "tools_openssl_src" - }, - { - "arch": "qt.tools.openssl.win_x64", + "arch": "qt.tools.opensslv3.src", "os_name": "windows", "target": "desktop", - "tool_name": "tools_openssl_x64" + "tool_name": "tools_opensslv3_src" }, { - "arch": "qt.tools.openssl.win_x86", + "arch": "qt.tools.opensslv3.win_x64", "os_name": "windows", "target": "desktop", - "tool_name": "tools_openssl_x86" + "tool_name": "tools_opensslv3_x64" }, { "arch": "qt.tools.qt3dstudio", @@ -820,6 +1011,30 @@ "target": "desktop", "tool_name": "tools_qtcreator" }, + { + "arch": "qt.tools.qtcreator_gui", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatorcdbext", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatordbg", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, + { + "arch": "qt.tools.qtcreatordev", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_qtcreator_gui" + }, { "arch": "qt.tools.windows_kits_debuggers", "os_name": "windows", @@ -838,12 +1053,30 @@ "target": "desktop", "tool_name": "tools_qtdesignstudio_generation2" }, + { + "arch": "qtdesignstudio.lts", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_qtdesignstudio_generation2_lts" + }, { "arch": "qt.tools.qtcreator.telemetry", "os_name": "windows", "target": "desktop", "tool_name": "tools_telemetry" }, + { + "arch": "qt.tools.qtcreator_gui.telemetry_evaluator", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_telemetry_eval_gui" + }, + { + "arch": "qt.tools.qtcreator_gui.telemetry", + "os_name": "windows", + "target": "desktop", + "tool_name": "tools_telemetry_gui" + }, { "arch": "qt.tools.vcredist", "os_name": "windows", @@ -962,7 +1195,14 @@ "6.3.1", "6.3.2", "6.4.0", - "6.4.1" + "6.4.1", + "6.4.2", + "6.4.3", + "6.5.0", + "6.5.1", + "6.5.2", + "6.5.3", + "6.6.0" ] } ] \ No newline at end of file diff --git a/aqt/exceptions.py b/aqt/exceptions.py index 012a8c8c..972488c0 100644 --- a/aqt/exceptions.py +++ b/aqt/exceptions.py @@ -102,3 +102,11 @@ class UpdaterError(AqtException): class OutOfMemory(AqtException): pass + + +class OutOfDiskSpace(AqtException): + pass + + +class DiskAccessNotPermitted(AqtException): + pass diff --git a/aqt/helper.py b/aqt/helper.py index 2e6dee7b..4529acb5 100644 --- a/aqt/helper.py +++ b/aqt/helper.py @@ -94,7 +94,15 @@ def getUrl(url: str, timeout: Tuple[float, float], expected_hash: Optional[bytes raise ArchiveDownloadError(msg) result: str = r.text filename = url.split("/")[-1] - actual_hash = hashlib.sha256(bytes(result, "utf-8")).digest() + _kwargs = {"usedforsecurity": False} if sys.version_info >= (3, 9) else {} + if Settings.hash_algorithm == "sha256": + actual_hash = hashlib.sha256(bytes(result, "utf-8"), **_kwargs).digest() + elif Settings.hash_algorithm == "sha1": + actual_hash = hashlib.sha1(bytes(result, "utf-8"), **_kwargs).digest() + elif Settings.hash_algorithm == "md5": + actual_hash = hashlib.md5(bytes(result, "utf-8"), **_kwargs).digest() + else: + raise ArchiveChecksumError(f"Unknown hash algorithm: {Settings.hash_algorithm}.\nPlease check settings.ini") if expected_hash is not None and expected_hash != actual_hash: raise ArchiveChecksumError( f"Downloaded file {filename} is corrupted! Detect checksum error.\n" @@ -104,7 +112,7 @@ def getUrl(url: str, timeout: Tuple[float, float], expected_hash: Optional[bytes return result -def downloadBinaryFile(url: str, out: Path, hash_algo: str, exp: bytes, timeout: Tuple[float, float]) -> None: +def downloadBinaryFile(url: str, out: Path, hash_algo: str, exp: Optional[bytes], timeout: Tuple[float, float]) -> None: logger = getLogger("aqt.helper") filename = Path(url).name with requests.sessions.Session() as session: @@ -126,7 +134,10 @@ def downloadBinaryFile(url: str, out: Path, hash_algo: str, exp: bytes, timeout: except requests.exceptions.Timeout as e: raise ArchiveConnectionError(f"Connection timeout: {e.args}") from e else: - hash = hashlib.new(hash_algo) + if sys.version_info >= (3, 9): + hash = hashlib.new(hash_algo, usedforsecurity=False) + else: + hash = hashlib.new(hash_algo) try: with open(out, "wb") as fd: for chunk in r.iter_content(chunk_size=8196): @@ -456,6 +467,14 @@ def max_retries_on_checksum_error(self): def max_retries_to_retrieve_hash(self): return self.config.getint("requests", "max_retries_to_retrieve_hash", fallback=int(self.max_retries)) + @property + def hash_algorithm(self): + return self.config.get("requests", "hash_algorithm", fallback="sha256") + + @property + def ignore_hash(self): + return self.config.getboolean("requests", "INSECURE_NOT_FOR_PRODUCTION_ignore_hash", fallback=False) + @property def backoff_factor(self): return self.config.getfloat("requests", "retry_backoff", fallback=0.1) diff --git a/aqt/installer.py b/aqt/installer.py index 1a16cad3..458e2178 100644 --- a/aqt/installer.py +++ b/aqt/installer.py @@ -22,6 +22,7 @@ # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import argparse +import errno import gc import multiprocessing import os @@ -30,7 +31,9 @@ import signal import subprocess import sys +import tarfile import time +import zipfile from logging import getLogger from logging.handlers import QueueHandler from pathlib import Path @@ -47,6 +50,8 @@ ArchiveListError, CliInputError, CliKeyboardInterrupt, + DiskAccessNotPermitted, + OutOfDiskSpace, OutOfMemory, ) from aqt.helper import ( @@ -213,21 +218,42 @@ def _check_qt_arg_versions(self, version): def _check_qt_arg_version_offline(self, version): return version in Settings.available_offline_installer_version - def _set_sevenzip(self, external): + def _warning_unknown_qt_version(self, qt_version: str) -> str: + return self._warning_on_bad_combination(f'Qt version "{qt_version}"') + + def _warning_unknown_target_arch_combo(self, args: List[str]) -> str: + return self._warning_on_bad_combination(f"target combination \"{' '.join(args)}\"") + + def _warning_unexpected_modules(self, unexpected_modules: List[str]) -> str: + return self._warning_on_bad_combination(f"modules {unexpected_modules}") + + def _warning_on_bad_combination(self, combo_message: str) -> str: + return ( + f"Specified {combo_message} did not exist when this version of aqtinstall was released. " + "This may not install properly, but we will try our best." + ) + + def _set_sevenzip(self, external: Optional[str]) -> Optional[str]: sevenzip = external + fallback = Settings.zipcmd if sevenzip is None: - return None - + if EXT7Z: + self.logger.warning(f"The py7zr module failed to load. Falling back to '{fallback}' for .7z extraction.") + self.logger.warning("You can use the '--external | -E' flags to select your own extraction tool.") + sevenzip = fallback + else: + # Just use py7zr + return None try: subprocess.run( [sevenzip, "--help"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, ) + return sevenzip except FileNotFoundError as e: - raise CliInputError("Specified 7zip command executable does not exist: {!r}".format(sevenzip)) from e - - return sevenzip + qualifier = "Specified" if sevenzip == external else "Fallback" + raise CliInputError(f"{qualifier} 7zip command executable does not exist: '{sevenzip}'") from e @staticmethod def _set_arch(arch: Optional[str], os_name: str, target: str, qt_version_or_spec: str) -> str: @@ -257,13 +283,10 @@ def _check_mirror(self, mirror): return False return True - def _check_modules_arg(self, qt_version, modules): - if modules is None: - return True + def _select_unexpected_modules(self, qt_version: str, modules: Optional[List[str]]) -> List[str]: + """Returns a sorted list of all the requested modules that do not exist in the combinations.json file.""" available = Settings.available_modules(qt_version) - if available is None: - return False - return all([m in available for m in modules]) + return sorted(set(modules or []) - set(available or [])) @staticmethod def _determine_qt_version( @@ -341,9 +364,6 @@ def run_install_qt(self, args: InstallArgParser): timeout = (Settings.connection_timeout, Settings.response_timeout) modules = args.modules sevenzip = self._set_sevenzip(args.external) - if EXT7Z and sevenzip is None: - # override when py7zr is not exist - sevenzip = self._set_sevenzip("7z") if args.base is not None: if not self._check_mirror(args.base): raise CliInputError( @@ -373,7 +393,7 @@ def run_install_qt(self, args: InstallArgParser): base_path = Path(base_dir) expect_desktop_archdir, autodesk_arch = self._get_autodesktop_dir_and_arch( - should_autoinstall, os_name, target, base_path, _version, is_wasm=(arch.startswith("wasm")) + should_autoinstall, os_name, target, base_path, _version, arch ) def get_auto_desktop_archives() -> List[QtPackage]: @@ -388,14 +408,14 @@ def to_archives(baseurl: str) -> QtArchives: auto_desktop_archives: List[QtPackage] = get_auto_desktop_archives() if not self._check_qt_arg_versions(qt_version): - self.logger.warning("Specified Qt version is unknown: {}.".format(qt_version)) + self.logger.warning(self._warning_unknown_qt_version(qt_version)) if not self._check_qt_arg_combination(qt_version, os_name, target, arch): - self.logger.warning( - "Specified target combination is not valid or unknown: {} {} {}".format(os_name, target, arch) - ) + self.logger.warning(self._warning_unknown_target_arch_combo([os_name, target, arch])) all_extra = True if modules is not None and "all" in modules else False - if not all_extra and not self._check_modules_arg(qt_version, modules): - self.logger.warning("Some of specified modules are unknown.") + if not all_extra: + unexpected_modules = self._select_unexpected_modules(qt_version, modules) + if unexpected_modules: + self.logger.warning(self._warning_unexpected_modules(unexpected_modules)) qt_archives: QtArchives = retry_on_bad_connection( lambda base_url: QtArchives( @@ -458,14 +478,11 @@ def _run_src_doc_examples(self, flavor, args, cmd_name: Optional[str] = None): else: timeout = (Settings.connection_timeout, Settings.response_timeout) sevenzip = self._set_sevenzip(args.external) - if EXT7Z and sevenzip is None: - # override when py7zr is not exist - sevenzip = self._set_sevenzip(Settings.zipcmd) modules = getattr(args, "modules", None) # `--modules` is invalid for `install-src` archives = args.archives all_extra = True if modules is not None and "all" in modules else False if not self._check_qt_arg_versions(qt_version): - self.logger.warning("Specified Qt version is unknown: {}.".format(qt_version)) + self.logger.warning(self._warning_unknown_qt_version(qt_version)) srcdocexamples_archives: SrcDocExamplesArchives = retry_on_bad_connection( lambda base_url: SrcDocExamplesArchives( @@ -532,9 +549,6 @@ def run_install_tool(self, args: InstallToolArgParser): else: base_dir = output_dir sevenzip = self._set_sevenzip(args.external) - if EXT7Z and sevenzip is None: - # override when py7zr is not exist - sevenzip = self._set_sevenzip(Settings.zipcmd) version = getattr(args, "version", None) if version is not None: Cli._validate_version_str(version, allow_minus=True) @@ -562,7 +576,7 @@ def run_install_tool(self, args: InstallToolArgParser): for arch in archs: if not self._check_tools_arg_combination(os_name, tool_name, arch): - self.logger.warning("Specified target combination is not valid: {} {} {}".format(os_name, tool_name, arch)) + self.logger.warning(self._warning_unknown_target_arch_combo([os_name, tool_name, arch])) tool_archives: ToolArchives = retry_on_bad_connection( lambda base_url: ToolArchives( @@ -709,8 +723,9 @@ def _set_install_qt_parser(self, install_qt_parser, *, is_legacy: bool): install_qt_parser.add_argument( "--autodesktop", action="store_true", - help="For android, ios, and Qt6/wasm installations, a standard desktop Qt installation is required. " - "When enabled, this option installs the required desktop version automatically.", + help="For Qt6 android, ios, wasm, and msvc_arm64 installations, an additional desktop Qt installation is " + "required. When enabled, this option installs the required desktop version automatically. " + "It has no effect when the desktop installation is not required.", ) def _set_install_tool_parser(self, install_tool_parser, *, is_legacy: bool): @@ -898,11 +913,11 @@ def _make_list_tool_parser(self, subparsers: argparse._SubParsersAction): "list-tool", formatter_class=argparse.RawDescriptionHelpFormatter, epilog="Examples:\n" - "$ aqt list-tool mac desktop # print all tools for mac desktop\n" - "$ aqt list-tool mac desktop tools_ifw # print all tool variant names for QtIFW\n" - "$ aqt list-tool mac desktop ifw # print all tool variant names for QtIFW\n" - "$ aqt list-tool mac desktop -l tools_ifw # print tool variant names with metadata for QtIFW\n" - "$ aqt list-tool mac desktop -l ifw # print tool variant names with metadata for QtIFW\n", + "$ aqt list-tool mac desktop # print all tools for mac desktop\n" + "$ aqt list-tool mac desktop tools_ifw # print all tool variant names for QtIFW\n" + "$ aqt list-tool mac desktop ifw # print all tool variant names for QtIFW\n" + "$ aqt list-tool mac desktop tools_ifw --long # print tool variant names with metadata for QtIFW\n" + "$ aqt list-tool mac desktop ifw --long # print tool variant names with metadata for QtIFW\n", ) list_parser.add_argument("host", choices=["linux", "mac", "windows"], help="host os name") list_parser.add_argument( @@ -1052,25 +1067,34 @@ def _validate_version_str( raise CliInputError(f"Invalid version: '{version_str}'! Please use the form '5.X.Y'.") from e def _get_autodesktop_dir_and_arch( - self, should_autoinstall: bool, host: str, target: str, base_path: Path, version: Version, is_wasm: bool = False + self, should_autoinstall: bool, host: str, target: str, base_path: Path, version: Version, arch: str ) -> Tuple[Optional[str], Optional[str]]: """Returns expected_desktop_arch_dir, desktop_arch_to_install""" - is_wasm_qt6 = target == "desktop" and is_wasm and version >= Version("6.0.0") - if target not in ["ios", "android"] and not is_wasm_qt6: - # We do not need to worry about the desktop directory if target is not mobile, or it's not Qt6 wasm. + is_wasm = arch.startswith("wasm") + is_msvc = "msvc" in arch + is_win_desktop_msvc_arm64 = host == "windows" and target == "desktop" and is_msvc and arch.endswith("arm64") + if version < Version("6.0.0") or ( + target not in ["ios", "android"] and not is_wasm and not is_win_desktop_msvc_arm64 + ): + # We only need to worry about the desktop directory for Qt6 mobile or wasm installs. return None, None - installed_desktop_arch_dir = QtRepoProperty.find_installed_desktop_qt_dir(host, base_path, version) + installed_desktop_arch_dir = QtRepoProperty.find_installed_desktop_qt_dir(host, base_path, version, is_msvc=is_msvc) if installed_desktop_arch_dir: # An acceptable desktop Qt is already installed, so don't do anything. self.logger.info(f"Found installed {host}-desktop Qt at {installed_desktop_arch_dir}") return installed_desktop_arch_dir.name, None - default_desktop_arch = MetadataFactory(ArchiveId("qt", host, "desktop")).fetch_default_desktop_arch(version) + default_desktop_arch = MetadataFactory(ArchiveId("qt", host, "desktop")).fetch_default_desktop_arch(version, is_msvc) desktop_arch_dir = QtRepoProperty.get_arch_dir_name(host, default_desktop_arch, version) expected_desktop_arch_path = base_path / dir_for_version(version) / desktop_arch_dir - qt_type = "Qt6-WASM" if is_wasm_qt6 else target + if is_win_desktop_msvc_arm64: + qt_type = "MSVC Arm64" + elif is_wasm: + qt_type = "Qt6-WASM" + else: + qt_type = target if should_autoinstall: # No desktop Qt is installed, but the user has requested installation. Find out what to install. self.logger.info( @@ -1116,6 +1140,23 @@ def close_worker_pool_on_exception(exception: BaseException): pool.starmap(installer, tasks) pool.close() pool.join() + except PermissionError as e: # subclass of OSError + close_worker_pool_on_exception(e) + raise DiskAccessNotPermitted( + f"Failed to write to base directory at {base_dir}", + suggested_action=[ + "Check that the destination is writable and does not already contain files owned by another user." + ], + ) from e + except OSError as e: + close_worker_pool_on_exception(e) + if e.errno == errno.ENOSPC: + raise OutOfDiskSpace( + "Insufficient disk space to complete installation.", + suggested_action=["Check available disk space.", "Check size requirements for installation."], + ) from e + else: + raise except KeyboardInterrupt as e: close_worker_pool_on_exception(e) raise CliKeyboardInterrupt("Installer halted by keyboard interrupt.") from e @@ -1176,12 +1217,12 @@ def installer( logger.addHandler(qh) # timeout = (Settings.connection_timeout, Settings.response_timeout) - hash = get_hash(qt_package.archive_path, algorithm="sha256", timeout=timeout) + hash = get_hash(qt_package.archive_path, Settings.hash_algorithm, timeout) if not Settings.ignore_hash else None def download_bin(_base_url): url = posixpath.join(_base_url, qt_package.archive_path) logger.debug("Download URL: {}".format(url)) - return downloadBinaryFile(url, archive, "sha256", hash, timeout) + return downloadBinaryFile(url, archive, Settings.hash_algorithm, hash, timeout) retry_on_errors( action=lambda: retry_on_bad_connection(download_bin, base_url), @@ -1190,7 +1231,19 @@ def download_bin(_base_url): name=f"Downloading {name}", ) gc.collect() - if command is None: + + if tarfile.is_tarfile(archive): + with tarfile.open(archive) as tar_archive: + if hasattr(tarfile, "data_filter"): + tar_archive.extractall(filter="tar", path=base_dir) + else: + # remove this when the minimum Python version is 3.12 + logger.warning("Extracting may be unsafe; consider updating Python to 3.11.4 or greater") + tar_archive.extractall(path=base_dir) + elif zipfile.is_zipfile(archive): + with zipfile.ZipFile(archive) as zip_archive: + zip_archive.extractall(path=base_dir) + elif command is None: with py7zr.SevenZipFile(archive, "r") as szf: szf.extractall(path=base_dir) else: diff --git a/aqt/metadata.py b/aqt/metadata.py index 11c9ec0e..9f992a76 100644 --- a/aqt/metadata.py +++ b/aqt/metadata.py @@ -246,6 +246,8 @@ def to_folder(self, qt_version_no_dots: str, extension: Optional[str] = None) -> def all_extensions(self, version: Version) -> List[str]: if self.target == "desktop" and QtRepoProperty.is_in_wasm_range(self.host, version): return ["", "wasm"] + elif self.target == "desktop" and QtRepoProperty.is_in_wasm_threaded_range(version): + return ["", "wasm_singlethread", "wasm_multithread"] elif self.target == "android" and version >= Version("6.0.0"): return list(ArchiveId.EXTENSIONS_REQUIRED_ANDROID_QT6) else: @@ -392,6 +394,11 @@ def get_arch_dir_name(host: str, arch: str, version: Version) -> str: def default_linux_desktop_arch_dir() -> str: return "gcc_64" + @staticmethod + def default_win_msvc_desktop_arch_dir(_version: Version) -> str: + """_version is unused, but we expect it to matter for future releases""" + return "msvc2019_64" + @staticmethod def default_mac_desktop_arch_dir(version: Version) -> str: return "macos" if version in SimpleSpec(">=6.1.2") else "clang_64" @@ -400,6 +407,10 @@ def default_mac_desktop_arch_dir(version: Version) -> str: def extension_for_arch(architecture: str, is_version_ge_6: bool) -> str: if architecture == "wasm_32": return "wasm" + elif architecture == "wasm_singlethread": + return "wasm_singlethread" + elif architecture == "wasm_multithread": + return "wasm_multithread" elif architecture.startswith("android_") and is_version_ge_6: ext = architecture[len("android_") :] if ext in ArchiveId.EXTENSIONS_REQUIRED_ANDROID_QT6: @@ -459,7 +470,7 @@ def select_superior_arch(lhs: ArchBitsVer, rhs: ArchBitsVer) -> ArchBitsVer: return default_arch @staticmethod - def find_installed_desktop_qt_dir(host: str, base_path: Path, version: Version) -> Optional[Path]: + def find_installed_desktop_qt_dir(host: str, base_path: Path, version: Version, is_msvc: bool = False) -> Optional[Path]: """ Locates the default installed desktop qt directory, somewhere in base_path. """ @@ -470,6 +481,9 @@ def find_installed_desktop_qt_dir(host: str, base_path: Path, version: Version) elif host == "linux": arch_path = installed_qt_version_dir / QtRepoProperty.default_linux_desktop_arch_dir() return arch_path if (arch_path / "bin/qmake").is_file() else None + elif host == "windows" and is_msvc: + arch_path = installed_qt_version_dir / QtRepoProperty.default_win_msvc_desktop_arch_dir(version) + return arch_path if (arch_path / "bin/qmake.exe").is_file() else None def contains_qmake_exe(arch_path: Path) -> bool: return (arch_path / "bin/qmake.exe").is_file() @@ -483,11 +497,15 @@ def contains_qmake_exe(arch_path: Path) -> bool: @staticmethod def is_in_wasm_range(host: str, version: Version) -> bool: return ( - version in SimpleSpec(">=6.2.0") + version in SimpleSpec(">=6.2.0,<6.5.0") or (host == "linux" and version in SimpleSpec(">=5.13,<6")) or version in SimpleSpec(">=5.13.1,<6") ) + @staticmethod + def is_in_wasm_threaded_range(version: Version) -> bool: + return version in SimpleSpec(">=6.5.0") + class MetadataFactory: """Retrieve metadata of Qt variations, versions, and descriptions from Qt site.""" @@ -503,7 +521,7 @@ def __init__( self, archive_id: ArchiveId, *, - base_url: str = Settings.baseurl, + base_url: Optional[str] = None, spec: Optional[SimpleSpec] = None, is_latest_version: bool = False, modules_query: Optional[ModulesQuery] = None, @@ -529,11 +547,14 @@ def __init__( self.logger = getLogger("aqt.metadata") self.archive_id = archive_id self.spec = spec - self.base_url = base_url + self.base_url = base_url or Settings.baseurl if archive_id.is_tools(): if tool_name is not None: - _tool_name: str = "tools_" + tool_name if not tool_name.startswith("tools_") else tool_name + if not tool_name.startswith("tools_") and tool_name != "sdktool": + _tool_name = f"tools_{tool_name}" + else: + _tool_name = tool_name if is_long_listing: self.request_type = "tool long listing" self._action: MetadataFactory.Action = lambda: self.fetch_tool_long_listing(_tool_name) @@ -581,9 +602,23 @@ def getList(self) -> Metadata: def fetch_arches(self, version: Version) -> List[str]: arches = [] + qt_ver_str = self._get_qt_version_str(version) for extension in self.archive_id.all_extensions(version): - qt_ver_str = self._get_qt_version_str(version) - modules = self._fetch_module_metadata(self.archive_id.to_folder(qt_ver_str, extension)) + modules: Dict[str, Dict[str, str]] = {} + try: + modules = self._fetch_module_metadata(self.archive_id.to_folder(qt_ver_str, extension)) + except ArchiveDownloadError as e: + if extension == "": + raise + else: + self.logger.debug(e) + self.logger.debug( + f"Failed to retrieve arches list with extension `{extension}`. " + f"Please check that this extension exists for this version of Qt: " + f"if not, code changes will be necessary." + ) + # It's ok to swallow this error: we will still print the other available architectures that aqt can + # install successfully. This is to prevent future errors such as those reported in #643 for name in modules.keys(): ver, arch = name.split(".")[-2:] @@ -672,7 +707,7 @@ def _to_version(self, qt_ver: str, arch: Optional[str]) -> Version: def fetch_http(self, rest_of_url: str, is_check_hash: bool = True) -> str: timeout = (Settings.connection_timeout, Settings.response_timeout) - expected_hash = get_hash(rest_of_url, "sha256", timeout) if is_check_hash else None + expected_hash = get_hash(rest_of_url, Settings.hash_algorithm, timeout) if is_check_hash else None base_urls = self.base_url, random.choice(Settings.fallbacks) err: BaseException = AssertionError("unraisable") @@ -709,6 +744,8 @@ def link_to_folder(link: bs4.element.Tag) -> str: continue if folder.startswith(filter_category): yield folder + if filter_category == "tools" and folder == "sdktool": + yield folder except Exception as e: raise ArchiveConnectionError( f"Failed to retrieve the expected HTML page at {html_url}", @@ -755,7 +792,7 @@ def _get_qt_version_str(self, version: Version) -> str: def _fetch_module_metadata(self, folder: str, predicate: Optional[Callable[[Element], bool]] = None): rest_of_url = posixpath.join(self.archive_id.to_url(), folder, "Updates.xml") - xml = self.fetch_http(rest_of_url) + xml = self.fetch_http(rest_of_url) if not Settings.ignore_hash else self.fetch_http(rest_of_url, False) return xml_to_modules( xml, predicate=predicate if predicate else MetadataFactory._has_nonempty_downloads, @@ -891,12 +928,14 @@ def describe_filters(self) -> str: return str(self.archive_id) return "{} with spec {}".format(self.archive_id, self.spec) - def fetch_default_desktop_arch(self, version: Version) -> str: + def fetch_default_desktop_arch(self, version: Version, is_msvc: bool = False) -> str: assert self.archive_id.target == "desktop", "This function is meant to fetch desktop architectures" if self.archive_id.host == "linux": return "gcc_64" elif self.archive_id.host == "mac": return "clang_64" + elif self.archive_id.host == "windows" and is_msvc: + return "win64_msvc2019_64" arches = [arch for arch in self.fetch_arches(version) if QtRepoProperty.MINGW_ARCH_PATTERN.match(arch)] selected_arch = QtRepoProperty.select_default_mingw(arches, is_dir=False) if not selected_arch: diff --git a/aqt/settings.ini b/aqt/settings.ini index 9b379686..3641e00e 100644 --- a/aqt/settings.ini +++ b/aqt/settings.ini @@ -16,6 +16,8 @@ max_retries_on_connection_error: 5 retry_backoff: 0.1 max_retries_on_checksum_error: 5 max_retries_to_retrieve_hash: 5 +hash_algorithm: sha256 +INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False [mirrors] trusted_mirrors: @@ -27,7 +29,6 @@ blacklist: fallbacks: https://qtproject.mirror.liquidtelecom.com/ https://mirrors.aliyun.com/qt/ - https://mirrors.sjtug.sjtu.edu.cn/qt/ https://mirrors.ustc.edu.cn/qtproject/ https://ftp.jaist.ac.jp/pub/qtproject/ https://ftp.yz.yamagata-u.ac.jp/pub/qtproject/ diff --git a/aqt/updater.py b/aqt/updater.py index 58d8f95d..e891bd9c 100644 --- a/aqt/updater.py +++ b/aqt/updater.py @@ -38,7 +38,12 @@ def unpatched_paths() -> List[str]: - return ["/home/qt/work/install", "/Users/qt/work/install"] + return [ + "/home/qt/work/install/", + "/Users/qt/work/install/", + "\\home\\qt\\work\\install\\", + "\\Users\\qt\\work\\install\\", + ] class Updater: @@ -185,7 +190,7 @@ def patch_qmake_script(self, base_dir, qt_version: str, os_name: str, desktop_ar qmake_path = self.prefix / "bin" / ("qmake.bat" if os_name == "windows" else "qmake") self.logger.info(f"Patching {qmake_path}") for unpatched in unpatched_paths(): - self._patch_textfile(qmake_path, f"{unpatched}/bin", patched, is_executable=True) + self._patch_textfile(qmake_path, f"{unpatched}bin", patched, is_executable=True) def patch_qtcore(self, target): """patch to QtCore""" @@ -245,7 +250,7 @@ def patch_target_qt_conf(self, base_dir: str, qt_version: str, arch_dir: str, os self._patch_textfile(target_qt_conf, old_host_lib_execs, f"HostLibraryExecutables={new_host_lib_execs}") for unpatched in unpatched_paths(): - self._patch_textfile(target_qt_conf, f"Prefix={unpatched}/target", new_targetprefix) + self._patch_textfile(target_qt_conf, f"Prefix={unpatched}target", new_targetprefix) self._patch_textfile(target_qt_conf, "HostPrefix=../../", new_hostprefix) self._patch_textfile(target_qt_conf, "HostData=target", new_hostdata) @@ -287,10 +292,13 @@ def update(cls, target: TargetConfig, base_path: Path, installed_desktop_arch_di "ios", "android", "wasm_32", + "wasm_singlethread", + "wasm_multithread", "android_x86_64", "android_arm64_v8a", "android_x86", "android_armv7", + "win64_msvc2019_arm64", ]: # desktop version updater.make_qtconf(base_dir, version_dir, arch_dir) updater.patch_qmake() @@ -310,13 +318,13 @@ def update(cls, target: TargetConfig, base_path: Path, installed_desktop_arch_di updater.patch_qtcore(target) elif version in SimpleSpec(">=5.0,<6.0"): updater.patch_qmake() - else: # qt6 mobile or wasm + else: # qt6 mobile, wasm, or msvc-arm64 if installed_desktop_arch_dir is not None: desktop_arch_dir = installed_desktop_arch_dir else: # Use MetadataFactory to check what the default architecture should be meta = MetadataFactory(ArchiveId("qt", os_name, "desktop")) - desktop_arch_dir = meta.fetch_default_desktop_arch(version) + desktop_arch_dir = meta.fetch_default_desktop_arch(version, is_msvc="msvc" in target.arch) updater.patch_qmake_script(base_dir, version_dir, target.os_name, desktop_arch_dir) updater.patch_target_qt_conf(base_dir, version_dir, arch_dir, target.os_name, desktop_arch_dir) diff --git a/ci/generate_azure_pipelines_matrices.py b/ci/generate_azure_pipelines_matrices.py index be0d5933..a418c352 100644 --- a/ci/generate_azure_pipelines_matrices.py +++ b/ci/generate_azure_pipelines_matrices.py @@ -189,6 +189,21 @@ def __init__(self, platform, build_jobs): spec=">1,<5.15", # Don't redirect output! Must be wrapped in quotes! mirror=random.choice(MIRRORS), ), + BuildJob( + "install-qt", + "6.5.2", + "windows", + "desktop", + "win64_msvc2019_arm64", + "msvc2019_arm64", + is_autodesktop=True, # Should install win64_msvc2019_arm64 in parallel + ), + BuildJob( + # Archives stored as .zip + "install-src", "6.4.3", "windows", "desktop", "gcc_64", "gcc_64", subarchives="qtlottie", + # Fail the job if this path does not exist: + check_output_cmd="ls -lh ./6.4.3/Src/qtlottie/", + ), ] ) @@ -217,10 +232,17 @@ def __init__(self, platform, build_jobs): subarchives="qtbase qttools qt icu", ), BuildJob( + # Archives stored as .7z "install-src", "6.1.0", "linux", "desktop", "gcc_64", "gcc_64", subarchives="qtlottie", # Fail the job if this path does not exist: check_output_cmd="ls -lh ./6.1.0/Src/qtlottie/", ), + BuildJob( + # Archives stored as .tar.gz + "install-src", "6.4.3", "linux", "desktop", "gcc_64", "gcc_64", subarchives="qtlottie", + # Fail the job if this path does not exist: + check_output_cmd="ls -lh ./6.4.3/Src/qtlottie/", + ), # Should install the `qtlottie` module, even though the archive `qtlottieanimation` is not specified: BuildJob( "install-doc", "6.1.0", "linux", "desktop", "gcc_64", "gcc_64", @@ -275,6 +297,16 @@ def __init__(self, platform, build_jobs): BuildJob("install-qt", "6.4.0", "linux", "desktop", "wasm_32", "wasm_32", is_autodesktop=True, emsdk_version="sdk-3.1.14-64bit", autodesk_arch_folder="gcc_64") ) +for job_queue, host, desk_arch in ( + (linux_build_jobs, "linux", "gcc_64"), + (mac_build_jobs, "mac", "clang_64"), + (windows_build_jobs, "windows", "mingw_64"), +): + for wasm_arch in ("wasm_singlethread", "wasm_multithread"): + job_queue.append( + BuildJob("install-qt", "6.5.0", host, "desktop", wasm_arch, wasm_arch, + is_autodesktop=True, emsdk_version="sdk-3.1.25-64bit", autodesk_arch_folder=desk_arch) + ) mac_build_jobs.append( BuildJob("install-qt", "5.14.2", "mac", "desktop", "wasm_32", "wasm_32") ) diff --git a/ci/steps.yml b/ci/steps.yml index 6b04aa55..d09a7642 100644 --- a/ci/steps.yml +++ b/ci/steps.yml @@ -103,6 +103,13 @@ steps: else # fail if 'wasm_32' is in the list ! aqt list-qt $(HOST) $(TARGET) --arch $(QT_VERSION) | grep -w -q "wasm_32" fi + + # Extra check for Qt 6.5.0 WASM: + for host in mac linux windows; do + export WASM_ARCHES=$(aqt list-qt $host desktop --arch 6.5.0) + echo $WASM_ARCHES | grep -w -q "wasm_singlethread" # fail if 'wasm_singlethread' is not in the list + echo $WASM_ARCHES | grep -w -q "wasm_multithread" # fail if 'wasm_multithread' is not in the list + done fi aqt list-qt $(HOST) $(TARGET) --spec "$(SPEC)" --arch latest @@ -165,7 +172,16 @@ steps: export PATH=$(QT_BINDIR):$PATH qmake $(Build.BinariesDirectory)/tests/accelbubble make - condition: and(eq(variables['TARGET'], 'android'), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')), ne(variables['SUBCOMMAND'], 'list'), ne(variables['SUBCOMMAND'], 'install-tool')) + condition: | + and( + eq(variables['TARGET'], 'android'), + or( + eq(variables['Agent.OS'], 'Linux'), + eq(variables['Agent.OS'], 'Darwin') + ), + ne(variables['SUBCOMMAND'], 'list'), + ne(variables['SUBCOMMAND'], 'install-tool') + ) displayName: Build accelbubble example application to test for android ##---------------------------------------------------- @@ -243,7 +259,17 @@ steps: export PATH=$(QT_BINDIR):$PATH qmake $(Build.BinariesDirectory)/tests/helloworld make - condition: and(eq( variables['TARGET'], 'desktop' ), ne( variables['ARCH'], 'wasm_32' ), or(eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin')), eq(variables['MODULE'], ''), eq(variables['SUBCOMMAND'], 'install-qt')) + condition: | + and( + eq( variables['TARGET'], 'desktop' ), + not( startsWith( variables['ARCH'], 'wasm_' ) ), + or( + eq(variables['Agent.OS'], 'Linux'), + eq(variables['Agent.OS'], 'Darwin') + ), + eq(variables['MODULE'], ''), + eq(variables['SUBCOMMAND'], 'install-qt') + ) displayName: Build test with qmake for Linux and macOS w/o extra module - powershell: | if ( $env:TOOLCHAIN -eq 'MSVC' ) { @@ -277,7 +303,12 @@ steps: qmake $(Build.BinariesDirectory)\tests\helloworld mingw32-make } - condition: and(eq( variables['Agent.OS'], 'Windows_NT'), eq(variables['MODULE'], ''), eq(variables['SUBCOMMAND'], 'install-qt')) + condition: | + and( + eq(variables['Agent.OS'], 'Windows_NT'), + eq(variables['MODULE'], ''), + eq(variables['SUBCOMMAND'], 'install-qt') + ) displayName: build test with qmake w/o extra module - powershell: | # Load modules from cache @@ -295,7 +326,13 @@ steps: cd .. qmake $(Build.BinariesDirectory)\tests\redditclient nmake - condition: and(eq( variables['Agent.OS'], 'Windows_NT'), eq(variables['TOOLCHAIN'], 'MSVC'), ne(variables['MODULE'], ''), ne(variables['VSVER'], '2019')) + condition: | + and( + eq(variables['Agent.OS'], 'Windows_NT'), + eq(variables['TOOLCHAIN'], 'MSVC'), + ne(variables['MODULE'], ''), + ne(variables['VSVER'], '2019') + ) displayName: build test with qmake with MSVC with extra module - bash: | set -ex @@ -308,15 +345,19 @@ steps: qmake $(Build.BinariesDirectory)/tests/redditclient make condition: | - and(eq( variables['TARGET'], 'desktop'), - or(eq(variables['Agent.OS'], 'Linux'), - eq(variables['Agent.OS'], 'Darwin')), - ne(variables['MODULE'], ''), - eq(variables['SUBCOMMAND'], 'install-qt')) + and( + eq( variables['TARGET'], 'desktop'), + or( + eq(variables['Agent.OS'], 'Linux'), + eq(variables['Agent.OS'], 'Darwin') + ), + ne(variables['MODULE'], ''), + eq(variables['SUBCOMMAND'], 'install-qt') + ) displayName: Build test with qmake for Linux and macOS with extra module ##---------------------------------------------------- - # wasm_32 on linux and mac + # wasm_32/single/multithread on linux and mac - script: | set -uex git clone --depth=1 --branch=$(EMSDK_TAG) https://github.com/emscripten-core/emsdk.git @@ -337,7 +378,7 @@ steps: workingDirectory: $(Build.BinariesDirectory) condition: | and( - eq( variables['ARCH'], 'wasm_32' ), + startsWith( variables['ARCH'], 'wasm_' ), or( eq(variables['Agent.OS'], 'Linux'), eq(variables['Agent.OS'], 'Darwin') @@ -345,7 +386,7 @@ steps: ) displayName: 'Build WebAssembler sample project on mac/linux' - # wasm_32 on Windows cmd.exe + # wasm_32/single/multithread on Windows cmd.exe - powershell: | git clone --depth=1 --branch=$(EMSDK_TAG) https://github.com/emscripten-core/emsdk.git cd emsdk @@ -380,7 +421,7 @@ steps: workingDirectory: $(Build.BinariesDirectory) condition: | and( - eq( variables['ARCH'], 'wasm_32' ), + startsWith( variables['ARCH'], 'wasm_' ), eq( variables['Agent.OS'], 'Windows_NT' ) ) displayName: 'Build WebAssembler sample project on windows' diff --git a/docs/CHANGELOG.rst b/docs/CHANGELOG.rst index 5304746e..a456d97d 100644 --- a/docs/CHANGELOG.rst +++ b/docs/CHANGELOG.rst @@ -13,6 +13,135 @@ All notable changes to this project will be documented in this file. `Unreleased`_ ============= +`v3.1.10`_ (14, Nov. 2023) +========================== + +Fixed +----- +- list_* commands ignore base url setting (#731,#732) + +Changed +------- +- chore: support build on git export (#730) + +`v3.1.9`_ (6, Nov. 2023) +======================== + +Security +-------- +* CVE-2023-32681: Bump requests@2.31.0 (#724) + +Changed +------- +* Remove a specific mirror from fallback (#688) +* add ``debug`` extras for test and check (#725) +* Bump pytest-remotedata@0.4.1 +* Bump flake8,flake8-isort@6.0.0 (#726) +* docs: change interpreted text to inline literals (#728) + +Added +----- +* macOS binary build (#722) +* ``ignore_hash`` and ``hash_algorithm`` options (#684) + +`v3.1.8`_ (1, Nov. 2023) +======================== + +Changed +------- +- Add 6.5.3 and openssl as known versions (#718) +- Docs: remove deprecated configuration description (#714) +- Test: test on python 3.8, 3.9 and 3.11 (#715) +- Docs: Update documentation for ``--autodesktop`` flag (#713) +- Use 'tar' filter when extracting tarfiles (#707) +- Log a warning when aqtinstall falls back to an external 7z extraction tool (#705) +- Bump py7zr@0.20.6(#702) + +Fixed +----- +- Fix failed CI (#716) +- Fix installation of win64_msvc2019_arm64 arch (#711) +- Fix ``test_install`` that fails on Python<3.11.4 (#708) +- Fix failing documentation builds (#706) +- Fix: exception when target path is relative (#702) + +`v3.1.7`_ (1, Aug. 2023) +======================== + +Added +----- +Add support for standalone sdktool installation(#677) + +Fixed +----- +- Fixed command to check tools_mingw90 (#680) +- Fixed help text for list-tool + +Changed +------- +* Add Qt 6.6.0, 6.5.2 and 6.5.1 as known version(#685,#698) +* Default blacklist setting(#689) +* Add test for sdktool(#678) + + +`v3.1.6`_ (4, May, 2023) +======================== + +Added +----- +* Add opensslv3 as known module (#674) +* Add code signature for standalone binary + +`v3.1.5`_ (30, Mar. 2023) +========================= + +Fixed +----- +* Fix failure to install Qt 6.4.3 source and docs on Windows(#665) +* Fix failed .tar.gz extraction in ``install-src`` and ``install-doc`` (#663) + +`v3.1.4`_ (25, Mar. 2023) +========================= + +Changed +------- +* Add Qt 6.4.3 as known version(#661) +* Catch OSError(errno.ENOSPC) and PermissionError (#657) +* Update security policy + + +`v3.1.3`_ (2, Mar. 2023) +======================== + +Changed +------- +* make the message about "unknown" Qt versions and modules + more friendly and easy to understand (#646,#654) + + +`v3.1.2`_ (17, Feb. 2023) +========================= + +Fixed +----- +* CI: Pin checkout at v3 in all workflows(#649) +* Fix list-qt and install-qt handling of WASM for Qt 6.5.0 (#648) + +Changed +------- +* Update combinations.xml (#650) +* Update documentation for ``--autodesktop`` flag (#638) + +`v3.1.1`_ (10, Feb. 2023) +========================= + +Fixed +----- +* CI: Pin EMSDK version (#641) +* Test: update tox.ini config (#634) +* Fix errors in install-* caused by duplicate modules (#633) + + `v3.1.0`_ (5, Dec. 2022) ======================== @@ -169,7 +298,17 @@ Security * Check Update.xml file with SHA256 hash (#493) -.. _Unreleased: https://github.com/miurahr/aqtinstall/compare/v3.1.0...HEAD +.. _Unreleased: https://github.com/miurahr/aqtinstall/compare/v3.1.10...HEAD +.. _v3.1.10: https://github.com/miurahr/aqtinstall/compare/v3.1.9...v3.1.10 +.. _v3.1.9: https://github.com/miurahr/aqtinstall/compare/v3.1.8...v3.1.9 +.. _v3.1.8: https://github.com/miurahr/aqtinstall/compare/v3.1.7...v3.1.8 +.. _v3.1.7: https://github.com/miurahr/aqtinstall/compare/v3.1.6...v3.1.7 +.. _v3.1.6: https://github.com/miurahr/aqtinstall/compare/v3.1.5...v3.1.6 +.. _v3.1.5: https://github.com/miurahr/aqtinstall/compare/v3.1.4...v3.1.5 +.. _v3.1.4: https://github.com/miurahr/aqtinstall/compare/v3.1.3...v3.1.4 +.. _v3.1.3: https://github.com/miurahr/aqtinstall/compare/v3.1.2...v3.1.3 +.. _v3.1.2: https://github.com/miurahr/aqtinstall/compare/v3.1.1...v3.1.2 +.. _v3.1.1: https://github.com/miurahr/aqtinstall/compare/v3.1.0...v3.1.1 .. _v3.1.0: https://github.com/miurahr/aqtinstall/compare/v3.0.2...v3.1.0 .. _v3.0.2: https://github.com/miurahr/aqtinstall/compare/v3.0.1...v3.0.2 .. _v3.0.1: https://github.com/miurahr/aqtinstall/compare/v3.0.0...v3.0.1 diff --git a/docs/CODE_OF_CONDUCT.rst b/docs/CODE_OF_CONDUCT.rst index 28e829ee..ddc11553 100644 --- a/docs/CODE_OF_CONDUCT.rst +++ b/docs/CODE_OF_CONDUCT.rst @@ -140,8 +140,8 @@ Community Impact Guidelines were inspired by `Mozilla's code of conduct enforcem For answers to common questions about this code of conduct, see the `FAQ`_ or its translations_. -.. _`Contributor Covenant`: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html +.. _`Contributor Covenant`: https://www.contributor-covenant.org/version/2/0/code_of_conduct/ .. _`homepage`: https://www.contributor-covenant.org -.. _`Mozilla's code of conduct enforcement ladder`: https://github.com/mozilla/diversity +.. _`Mozilla's code of conduct enforcement ladder`: https://github.com/mozilla/inclusion .. _`FAQ`: https://www.contributor-covenant.org/faq .. _`translations`: https://www.contributor-covenant.org/translations diff --git a/docs/SECURITY.rst b/docs/SECURITY.rst index d9b7ce57..ed02bae6 100644 --- a/docs/SECURITY.rst +++ b/docs/SECURITY.rst @@ -7,11 +7,11 @@ Supported Versions +---------+---------------------+ | Version | Status | +=========+=====================+ -| 3.0.x | Stable version | +| 3.1.x | Stable version | +---------+---------------------+ -| 2.2.x | Security fixes only | +| 3.0.x | Security fixes only | +---------+---------------------+ -| < 2.2 | not supported | +| < 3.0 | not supported | +---------+---------------------+ Reporting a Vulnerability diff --git a/docs/authors.rst b/docs/authors.rst index efc9aa98..70dec185 100644 --- a/docs/authors.rst +++ b/docs/authors.rst @@ -14,6 +14,7 @@ David also leads many developments and reviews effort after 2.0. All contributors, listed alphabetically, are: +* Alberto Mardegan(ignore_hash option) * Andrei Yankovich (tools ifw installation) * Aurélien Gâteau (patching to qmake) * Benjamin O (Github Actions and more) @@ -25,15 +26,17 @@ All contributors, listed alphabetically, are: * Gamso (improve parsing of update.xml) * Julien Marrec (mypy, type hints) * Kyle Altendorf (7z binary path search) -* lightmare (Documents) -* Mike Tzou (Update fallback url) +* @lebarsfa (ignore_hash/hash_algorithm options) +* @lightmare (Documents) * Martin Delille (Documents) +* Mike Tzou (Update fallback url) * mite-user (folder index handling of download web sites) * Mizux Seihax (Qt versions) * Mozi (CI/workflow improvement, log format) * Nelson Chen (CI tests) * @nikitalita (Binary distribution) * @pylipp (Documents) +* @Steveice10 (MacOS binary build) * Sztergbaum Roman (Version database) * Thomas Grainger (CLI entry point) * @tsteven4 (fix patching to qmake, pkgconfig and libtool) diff --git a/docs/cli.rst b/docs/cli.rst index 6552353a..756e61cc 100644 --- a/docs/cli.rst +++ b/docs/cli.rst @@ -387,7 +387,7 @@ are described here: .. _py7zr: https://pypi.org/project/py7zr/ .. _7-zip: https://www.7-zip.org/ -.. _Choco: https://community.chocolatey.org/packages/7zip/19.0 +.. _Choco: https://community.chocolatey.org/packages/7zip/ .. _brew: https://formulae.brew.sh/formula/p7zip .. option:: --internal @@ -545,9 +545,10 @@ There are various combinations to accept according to Qt version. .. option:: --autodesktop - If you are installing an ios or android version of Qt, or the WASM version of Qt6, + If you are installing an ios, android, WASM, or msvc_arm64 version of Qt6, the corresponding desktop version of Qt must be installed alongside of it. Turn this option on to install it automatically. + This option will have no effect if the desktop version of Qt is not required. .. option:: --noarchives @@ -808,7 +809,14 @@ Example: Installing Android (armv7) Qt 5.13.2: .. code-block:: console - aqt install-qt linux android 5.13.2 android_armv7 --autodesktop + aqt install-qt linux android 5.13.2 android_armv7 + + +Example: Installing Android (armv7) Qt 6.4.2: + +.. code-block:: console + + aqt install-qt linux android 6.4.2 android_armv7 --autodesktop Example: Install examples, doc and source: @@ -911,18 +919,34 @@ Example: Install vcredist on Windows: .. code-block:: doscon - aqt install-tool windows tools_vcredist + aqt install-tool windows desktop tools_vcredist .\Qt\Tools\vcredist\vcredist_msvc2019_x64.exe /norestart /q -Example: Install MinGW on Windows +Example: Install MinGW 8.1.0 on Windows: .. code-block:: doscon - aqt install-tool -O c:\Qt windows tools_mingw qt.tools.win64_mingw810 + aqt install-tool -O c:\Qt windows desktop tools_mingw qt.tools.win64_mingw810 set PATH=C:\Qt\Tools\mingw810_64\bin +Example: Install MinGW 11.2.0 on Windows: + +.. code-block:: doscon + + aqt install-tool -O c:\Qt windows desktop tools_mingw90 + set PATH=C:\Qt\Tools\mingw1120_64\bin + +.. note:: + + This is not a typo; it is a mislabelled tool name! + ``tools_mingw90`` and the tool variant ``qt.tools.win64_mingw900`` + do not contain MinGW 9.0.0; they actually contain MinGW 11.2.0! + Verify with ``aqt list-tool --long windows desktop tools_mingw90`` + in a wide terminal. + + Example: Show help message .. code-block:: console diff --git a/docs/conf.py b/docs/conf.py index 1acd7a88..f2bea7be 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -55,7 +55,7 @@ # General information about the project. project = u'aqtinstall' -copyright = u'2019-2021, Hiroshi Miura' +copyright = u'2019-2023, Hiroshi Miura' author = u'Hiroshi Miura' # The version info for the project you're documenting, acts as replacement for diff --git a/docs/configuration.rst b/docs/configuration.rst index 9d78b00d..525b0d40 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -31,6 +31,8 @@ A file is like as follows: retry_backoff: 0.1 max_retries_on_checksum_error: 5 max_retries_to_retrieve_hash: 5 + hash_algorithm: sha256 + INSECURE_NOT_FOR_PRODUCTION_ignore_hash: False [mirrors] trusted_mirrors: @@ -97,7 +99,7 @@ min_module_size: and they are exactly 40 bytes when uncompressed. The authors feel that it is not useful for ``aqt list-*`` to list these empty modules. If you want to print these modules with ``aqt list-*``, please feel free to change - the `min_module_size` value to something less than 40. + the ``min_module_size`` value to something less than 40. This setting has no effect on your ability to install these modules. ``aqt install-*`` can will still install them without any warnings. @@ -130,6 +132,18 @@ max_retries_on_checksum_error: This setting controls how many times ``aqt`` will attempt to download a file, in the case of a checksum error. +hash_algorithm: + This is either ``sha256``, ``sha1`` or ``md5``. ``sha256`` is the only safe + value to use here. Default is ``sha256`` if not set. + See also ``trusted_mirrors`` setting. + +INSECURE_NOT_FOR_PRODUCTION_ignore_hash: + This is either ``True`` or ``False``. + The ``True`` setting disables hash checking when downloading files. Although + this is not recommended, this may help when hashes are not available. + The ``False`` setting will enforce hash checking. This is highly recommended + to avoid corrupted files. + The ``[mirrors]`` section is a configuration for mirror handling. @@ -156,8 +170,8 @@ blacklist: Some mirror sites ignore a connection from IP addresses out of their preffered one. It will cause connection error or connection timeout. There are some known mirror sites in default. - When you are happy with the default sites, - you can override with your custom settings. + If you are not happy with the default sites, + you can override them with custom settings. fallbacks: It is a list of URL where is a good for access. diff --git a/docs/getting_started.rst b/docs/getting_started.rst index 4e5b6c2d..18c709f5 100644 --- a/docs/getting_started.rst +++ b/docs/getting_started.rst @@ -83,7 +83,7 @@ is not on this list, we can use the `Available Qt versions`_ wiki page to get a rough idea of what versions support the architecture we want, and then use :ref:`aqt list-qt ` to confirm that the architecture is available. -Let's say that we want to install Qt 6.2.0 with architecture `win64_mingw81`. +Let's say that we want to install Qt 6.2.0 with architecture ``win64_mingw81``. The installation command we need is: .. code-block:: console @@ -120,7 +120,7 @@ installed by default, using this command: $ aqt install-qt linux desktop 6.2.0 gcc_64 --external 7z .. _py7zr: https://pypi.org/project/py7zr/ -.. _p7zip: http://p7zip.sourceforge.net/ +.. _p7zip: https://p7zip.sourceforge.net/ .. _7-zip: https://www.7-zip.org/ Changing the output directory @@ -128,7 +128,7 @@ Changing the output directory By default, ``aqt`` will install all of the Qt packages into the current working directory, in the subdirectory ``.///``. -For example, if we install Qt 6.2.0 for Windows desktop with arch `win64_mingw81`, +For example, if we install Qt 6.2.0 for Windows desktop with arch ``win64_mingw81``, it would end up in ``./6.2.0/win64_mingw81``. If you would prefer to install it to another location, you @@ -160,6 +160,8 @@ combination, so we will need to supply :ref:`aqt list-qt ` with qtcharts qtdatavis3d qtlottie qtnetworkauth qtpurchasing qtquick3d qtquicktimeline qtscript qtvirtualkeyboard qtwebengine qtwebglplugin +.. _long_modules explanation: + Let's say that we want to know more about these modules before we install them. We can use the ``--long-modules`` flag for that: @@ -182,10 +184,30 @@ We can use the ``--long-modules`` flag for that: Note that if your terminal is wider than 95 characters, this command will show release dates and sizes in extra columns to the right. -If you try this, you will notice that `debug_info` is 5.9 gigabytes installed. +If you try this, you will notice that ``debug_info`` is 5.9 gigabytes installed. + +Also, notice that the 'Display Name' indicates which compiler the module is +intended to be used with. In this case, for the architecture ``win64_mingw81``, +you will most likely want to use the "MinGW 8.1.0 64-bit" compiler. +Here's what the command prints when you use it with the ambiguously-named +``win64_mingw`` architecture: + +.. code-block:: console + + $ python -m aqt list-qt windows desktop --long-modules 6.2.4 win64_mingw + Module Name Display Name + ======================================================================= + debug_info Desktop MinGW 11.2.0 64-bit debug information files + qt3d Qt 3D for MinGW 11.2.0 64-bit + qt5compat Qt 5 Compatibility Module for MinGW 11.2.0 64-bit + qtactiveqt Qt 3D for MinGW 11.2.0 64-bit + qtcharts Qt Charts for MinGW 11.2.0 64-bit + ... + +You can find out how to install MinGW 8.1.0 and 11.2.0 in the `Installing Tools`_ section. -Let's say that we want to install `qtcharts` and `qtnetworkauth`. -We can do that by using the `-m` flag with the :ref:`aqt install-qt ` command. +Let's say that we want to install ``qtcharts`` and ``qtnetworkauth``. +We can do that by using the ``-m`` flag with the :ref:`aqt install-qt ` command. This flag receives the name of at least one module as an argument: .. code-block:: console @@ -213,8 +235,8 @@ The ``xargs`` equivalent to this command is an exercise left to the reader. If you want to install all available modules, you are probably better off using the ``all`` keyword, as discussed above. This scripting example is presented to give you a sense of how to accomplish something more complicated. -Perhaps you want to install all modules except `qtnetworkauth`; you could write a script -that removes `qtnetworkauth` from the output of :ref:`aqt list-qt `, +Perhaps you want to install all modules except ``qtnetworkauth``; you could write a script +that removes ``qtnetworkauth`` from the output of :ref:`aqt list-qt `, and pipe that into :ref:`aqt install-qt `. This exercise is left to the reader. @@ -245,7 +267,7 @@ Let's install Qt for Android. This will be similar to installing Qt for Desktop $ aqt install-qt windows android 6.2.4 android_armv7 -m qtcharts qtnetworkauth # Install -Please note that when you install Qt for android or ios, the installation will not +Please note that when you install Qt6 for android or ios, the installation will not be functional unless you install the corresponding desktop version of Qt alongside it. You can do this automatically with the ``--autodesktop`` flag: @@ -269,6 +291,9 @@ We can find out whether or not that architecture is available for our version of Not every version of Qt supports WASM. This command shows us that we cannot use WASM with Qt 6.1.3. +Please note that the WASM architecture for Qt 6.5.0+ changed from ``wasm_32`` to ``wasm_singlethread`` and +``wasm_multithread``. Always use ``aqt list-qt`` to check what architectures are available for the desired version of Qt. + We can check the modules available as before: .. code-block:: console @@ -310,11 +335,12 @@ Let's find out what tools are available for Windows Desktop by using the tools_openssl_src tools_ninja tools_mingw + tools_mingw90 tools_ifw tools_conan tools_cmake -Let's see what tool variants are available in `tools_mingw`: +Let's see what tool variants are available in ``tools_mingw``: .. code-block:: console @@ -352,31 +378,55 @@ Let's see some more details, using the ``-l`` or ``--long`` flag: qt.tools.win64_mingw810 8.1.0-1-202004170606 2020-04-17 The ``-l`` flag causes :ref:`aqt list-tool ` to print a table -that shows plenty of data pertinent to each tool variant available in `tools_mingw`. +that shows plenty of data pertinent to each tool variant available in ``tools_mingw``. :ref:`aqt list-tool ` additionally prints the 'Display Name' and 'Description' for each tool if your terminal is wider than 95 characters; terminals that are narrower than this cannot display this table in a readable way. -Now let's install `mingw`, using the :ref:`aqt install-tool ` command. +Please be aware that the tool ``tools_mingw90`` appears to be mislabelled: + +.. code-block:: console + + $ aqt list-tool windows desktop tools_mingw90 -l + + Tool Variant Name Version Release Date + ============================================================= + qt.tools.win64_mingw900 9.0.0-1-202203221220 2022-03-22 + + $ aqt list-tool windows desktop tools_mingw90 -l + + Tool Variant Name Version Release Date Display Name Description + ============================================================================================================ + qt.tools.win64_mingw900 9.0.0-1-202203221220 2022-03-22 MinGW 11.2.0 64-bit MinGW-builds 11.2.0 + 64-bit toolchain with + gcc 11.2.0 + +The 'narrow display' for ``tools_mingw90`` cuts off the two columns of the table that +show you what's really in that package: ``MinGW 11.2.0 64-bit``. +If you are using the ``win64_mingw`` architecture for Qt 6.2.2+, then this is +probably the compiler you want to install (see `long_modules explanation`_). + + +Now let's install ``mingw``, using the :ref:`aqt install-tool ` command. This command receives four parameters: 1. The host operating system (windows, mac, or linux) 2. The target SDK (desktop, android, ios, or winrt) -3. The name of the tool (this is `tools_mingw` in our case) +3. The name of the tool (this is ``tools_mingw`` in our case) 4. (Optional) The tool variant name. We saw a list of these when we ran - :ref:`aqt list-tool ` with the `tool name` argument filled in. + :ref:`aqt list-tool ` with the ``tool name`` argument filled in. -To install `mingw`, you could use this command (please don't): +To install ``mingw``, you could use this command (please don't): .. code-block:: console $ aqt install-tool windows desktop tools_mingw # please don't run this! -Using this command will install every tool variant available in `tools_mingw`; +Using this command will install every tool variant available in ``tools_mingw``; in this case, you would install 10 different versions of the same tool. -For some tools, like `qtcreator` or `ifw`, this is an appropriate thing to do, +For some tools, like ``qtcreator`` or ``ifw``, this is an appropriate thing to do, since each tool variant is a different program. -However, for tools like `mingw` and `vcredist`, it would make more sense to use +However, for tools like ``mingw`` and ``vcredist``, it would make more sense to use :ref:`aqt list-tool ` to see what tool variants are available, and then install just the tool variant you are interested in, like this: @@ -406,7 +456,7 @@ reduce the footprint of your Qt installation. .. note:: - Be careful about using the ``--archives`` flag; it is marked `Advanced` for a reason! + Be careful about using the ``--archives`` flag; it is marked ``Advanced`` for a reason! It is very easy to misuse this command and end up with a Qt installation that is missing the components that you need. Don't use it unless you know what you are doing! diff --git a/docs/previous_changes.rst b/docs/previous_changes.rst index f2328d99..6cf5fd46 100644 --- a/docs/previous_changes.rst +++ b/docs/previous_changes.rst @@ -620,7 +620,7 @@ Changed Fixed ----- -* Work around for http://download.qt.io/ returns wrong metalink xml data.(#105, #106) +* Work around for https://download.qt.io/ returns wrong metalink xml data.(#105, #106) `v0.8a1`_ (28, Feb., 2020) diff --git a/pyproject.toml b/pyproject.toml index c3af7711..8cef78cc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,8 +26,8 @@ dependencies = [ "defusedxml", "humanize", "patch>=1.16", - "py7zr>=0.20.2", - "requests>2.20.0", + "py7zr>=0.20.6", + "requests>=2.31.0", "semantic-version", "texttable", ] @@ -49,19 +49,18 @@ aqt = "aqt.__main__:main" test = [ "pytest>=6.0", "pytest-pep8", - "pytest-remotedata<0.4.0", "pytest-cov", + "pytest-remotedata>=0.4.1", "pytest-socket", - "pytest-leaks", "pytest-timeout", "pympler", ] check = [ "mypy>=0.990", - "flake8<6", + "flake8>=6.0.0,<7.0.0", "flake8-black", "flake8-colors", - "flake8-isort>=5.0.3,<6.0.0", + "flake8-isort>=6.0.0,<7.0.0", "flake8-pyi", "flake8-typing-imports", "docutils", @@ -71,9 +70,12 @@ check = [ "packaging", ] docs = [ - "sphinx>=5.0", - "sphinx_rtd_theme", - "sphinx-py3doc-enhanced-theme", + "sphinx>=7.0", + "sphinx_rtd_theme>=1.3", + "sphinx-py3doc-enhanced-theme>=2.4", +] +debug = [ + "pytest-leaks", ] [project.urls] @@ -211,18 +213,13 @@ commands = [testenv:py39d] basepython = python3.9d -extras = test +extras = test, debug commands = python3.9-dbg -m pytest -v --no-cov -R : -k "test_install" -deps = - pytest - pytest-leaks - pytest-remotedata<0.4.0 - pytest-socket - pytest-cov [testenv:mprof] basepython = python3.9 +extras = debug commands = mprof run --multiprocess python -m aqt install-qt -O /tmp -d /tmp linux desktop 6.2.1 mprof plot --output memory-profile.png @@ -232,6 +229,7 @@ deps = [testenv:fil] basepython = python3.9 +extras = debug commands = fil-profile run -m aqt install-qt -O /tmp -d /tmp linux desktop 6.2.1 deps = diff --git a/tests/data/linux-desktop-expect.json b/tests/data/linux-desktop-expect.json index 4bfde464..0841d20e 100644 --- a/tests/data/linux-desktop-expect.json +++ b/tests/data/linux-desktop-expect.json @@ -9,7 +9,9 @@ "5.14.0 5.14.1 5.14.2", "5.15.0 5.15.1 5.15.2", "6.0.0 6.0.1 6.0.2 6.0.3", - "6.1.0" + "6.1.0", + "6.2.0", + "6.5.0" ], "preview": [ "5.9-preview", @@ -25,8 +27,11 @@ "wasm": [ "5.13.0 5.13.1 5.13.2", "5.14.0 5.14.1 5.14.2", - "5.15.0 5.15.1 5.15.2" + "5.15.0 5.15.1 5.15.2", + "6.2.0" ], + "wasm_singlethread": ["6.5.0"], + "wasm_multithread": ["6.5.0"], "wasm_preview": [ "5.13-preview", "5.14-preview", @@ -74,6 +79,7 @@ "tools_ifw", "tools_generic", "tools_conan", - "tools_cmake" + "tools_cmake", + "sdktool" ] } \ No newline at end of file diff --git a/tests/data/linux-desktop.html b/tests/data/linux-desktop.html index dbee03dd..2e149703 100644 --- a/tests/data/linux-desktop.html +++ b/tests/data/linux-desktop.html @@ -38,6 +38,14 @@

Index of /online/qtsdkrepository/linux_x64/desktop

 tools_generic/13-Apr-2021 14:39 -    tools_conan/15-Feb-2021 12:14 -    tools_cmake/07-Jan-2021 14:22 -   + sdktool/05-May-2023 10:53 -   + qt6_650_wasm_singlethread/01-Jan-2023 00:00 -   + qt6_650_wasm_multithread/01-Jan-2023 00:00 -   + qt6_650_src_doc_examples/01-Jan-2023 00:00 -   + qt6_650/01-Jan-2023 00:00 -   + qt6_620_wasm/29-Sep-2021 12:46 -   + qt6_620_src_doc_examples/29-Sep-2021 12:43 -   + qt6_620/29-Sep-2021 12:34 -    qt6_610_src_doc_examples/30-Apr-2021 08:09 -    qt6_610/30-Apr-2021 08:07 -    qt6_603_src_doc_examples/31-Mar-2021 07:31 -   diff --git a/tests/data/mac-desktop-expect.json b/tests/data/mac-desktop-expect.json index b0d4addb..3f1a7fcf 100644 --- a/tests/data/mac-desktop-expect.json +++ b/tests/data/mac-desktop-expect.json @@ -9,7 +9,9 @@ "5.14.0 5.14.1 5.14.2", "5.15.0 5.15.1 5.15.2", "6.0.0 6.0.1 6.0.2 6.0.3", - "6.1.0" + "6.1.0", + "6.2.0", + "6.5.0" ], "preview": [ "5.6-preview", @@ -26,8 +28,11 @@ "wasm": [ "5.13.1 5.13.2", "5.14.0 5.14.1 5.14.2", - "5.15.0 5.15.1 5.15.2" + "5.15.0 5.15.1 5.15.2", + "6.2.0" ], + "wasm_singlethread": ["6.5.0"], + "wasm_multithread": ["6.5.0"], "wasm_preview": [ "5.13-preview", "5.14-preview", @@ -75,6 +80,7 @@ "tools_ifw", "tools_generic", "tools_conan", - "tools_cmake" + "tools_cmake", + "sdktool" ] } \ No newline at end of file diff --git a/tests/data/mac-desktop-sdktool-expect.json b/tests/data/mac-desktop-sdktool-expect.json new file mode 100644 index 00000000..30a51a27 --- /dev/null +++ b/tests/data/mac-desktop-sdktool-expect.json @@ -0,0 +1,14 @@ +{ + "modules": [ + "qt.tools.qtcreator" + ], + "long_listing": [ + [ + "qt.tools.qtcreator", + "10.0.1-0-202305050734", + "2023-05-05", + "SDKTool", + "SDKTool" + ] + ] +} \ No newline at end of file diff --git a/tests/data/mac-desktop-sdktool-update.xml b/tests/data/mac-desktop-sdktool-update.xml new file mode 100644 index 00000000..636a22f3 --- /dev/null +++ b/tests/data/mac-desktop-sdktool-update.xml @@ -0,0 +1,20 @@ + + {AnyApplication} + 1.0.0 + true + + qt.tools.qtcreator + SDKTool + SDKTool + 10.0.1-0-202305050734 + 2023-05-05 + + true + true + qtcreator_sdktool.7z + + 450d20347716f794124272a38d7cf46df9c50169 + + c6d6c9ff8b3c0a75f1bb1804eb4955a6409ac867 + 2023-05-05-0734_meta.7z + diff --git a/tests/data/mac-desktop.html b/tests/data/mac-desktop.html index 5f7497c9..b80f45a3 100644 --- a/tests/data/mac-desktop.html +++ b/tests/data/mac-desktop.html @@ -38,6 +38,14 @@

Index of /online/qtsdkrepository/mac_x64/desktop

 tools_generic/13-Apr-2021 14:39 -    tools_conan/15-Feb-2021 12:15 -    tools_cmake/07-Jan-2021 14:22 -   + sdktool/05-May-2023 10:53 -   + qt6_650_wasm_singlethread/01-Jan-2023 00:00 -   + qt6_650_wasm_multithread/01-Jan-2023 00:00 -   + qt6_650_src_doc_examples/01-Jan-2023 00:00 -   + qt6_650/01-Jan-2023 00:00 -   + qt6_620_wasm/29-Sep-2021 12:46 -   + qt6_620_src_doc_examples/29-Sep-2021 12:43 -   + qt6_620/29-Sep-2021 12:34 -    qt6_610_src_doc_examples/30-Apr-2021 08:13 -    qt6_610/30-Apr-2021 08:13 -    qt6_603_src_doc_examples/31-Mar-2021 07:34 -   diff --git a/tests/data/windows-650-wasm-multi-expect.json b/tests/data/windows-650-wasm-multi-expect.json new file mode 100644 index 00000000..3a4fab54 --- /dev/null +++ b/tests/data/windows-650-wasm-multi-expect.json @@ -0,0 +1,26 @@ +{ + "architectures": [ + "wasm_multithread" + ], + "modules_by_arch": { + "wasm_multithread": [ + "qtcharts", + "qtdatavis3d", + "qtgrpc", + "qthttpserver", + "qtimageformats", + "qtlottie", + "qtmultimedia", + "qtquick3dphysics", + "qtscxml", + "qtspeech", + "qtvirtualkeyboard", + "qtwebchannel", + "qtwebsockets", + "qt5compat", + "qtquick3d", + "qtquicktimeline", + "qtshadertools" + ] + } +} \ No newline at end of file diff --git a/tests/data/windows-650-wasm-multi-update.xml b/tests/data/windows-650-wasm-multi-update.xml new file mode 100644 index 00000000..85ff2d89 --- /dev/null +++ b/tests/data/windows-650-wasm-multi-update.xml @@ -0,0 +1,290 @@ + + {AnyApplication} + 1.0.0 + true + + qt.qt6.650.addons.qtcharts.wasm_multithread + Qt Charts for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtcharts, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtcharts-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + a84793455e6f234204956d2a535ef7487e12f746 + + + qt.qt6.650.addons.qtdatavis3d.wasm_multithread + Qt Data Visualization for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtdatavis3d, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtdatavis3d-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 942fb21a8ba4338a2c94c15bc3564d9ed016aa91 + + + qt.qt6.650.addons.qtgrpc.wasm_multithread + Qt Protobuf and Qt GRPC for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtgrpc, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtgrpc-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + eab733c1b40b2ba93eac301090ed2e40d1bfb309 + + + qt.qt6.650.addons.qthttpserver.wasm_multithread + Qt HTTP Server for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qthttpserver, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qthttpserver-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 14b847981005d193938f7c3e8f6608d8518a6366 + + + qt.qt6.650.addons.qtimageformats.wasm_multithread + Qt ImageFormats for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtimageformats, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtimageformats-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + ff3b43b3e1b9fe078761b75dada9d725f47d9af3 + + + qt.qt6.650.addons.qtlottie.wasm_multithread + Qt Lottie Animation for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtlottie, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtlottie-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 3869563cef20f85ecc55e66fd0875a37a820175c + + + qt.qt6.650.addons.qtmultimedia.wasm_multithread + Qt Multimedia for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtmultimedia, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtmultimedia-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 8f7053696b1258f4e0511fd233e24be98b96bddf + + + qt.qt6.650.addons.qtquick3dphysics + Quick: 3D Physics + Qt Quick 3D Physics provides a high-level QML module adding physical simulation capabilities to Qt Quick 3D. + 6.5.0-0-202301241223 + 2023-01-24 + false + qt.qt6.650.doc.qtquick3dphysics, qt.qt6.650.examples.qtquick3dphysics, qt.qt6.650.qtquick3d, qt.qt6.650.qtshadertools + + + + 05989ccb769ed6b46a60e453d401dd4415609363 + + + qt.qt6.650.addons.qtquick3dphysics.wasm_multithread + Quick: 3D Physics for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtquick3dphysics, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtquick3dphysics-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 7cfea014ddc41082f6ac52a72930ec174d2cd484 + + + qt.qt6.650.addons.qtscxml.wasm_multithread + Qt State Machines for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtscxml, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtscxml-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + a6fb82aa712eae15f582835a3e858ad4b94961ac + + + qt.qt6.650.addons.qtspeech.wasm_multithread + Qt Speech for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtspeech, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtspeech-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 2e444441796b80824a742107514797a2d8e97234 + + + qt.qt6.650.addons.qtvirtualkeyboard.wasm_multithread + Qt Virtual Keyboard for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtvirtualkeyboard, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtvirtualkeyboard-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + fd072ead9d8d55fac3a9efb0e2c7a92af8fb047d + + + qt.qt6.650.addons.qtwebchannel.wasm_multithread + Qt WebChannel for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtwebchannel, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtwebchannel-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 802b47541de55f8528e436d0354421138e598f90 + + + qt.qt6.650.addons.qtwebsockets.wasm_multithread + Qt WebSockets for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtwebsockets, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtwebsockets-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + f5ea2a5ab7011919f9efd72c482bd41d4b9125b6 + + + qt.qt6.650.qt5compat.wasm_multithread + Qt 5 Compatibility Module for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qt5compat, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qt5compat-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 8573219387c9d6448da92f9c0eb2cd77fc5e4d6a + + + qt.qt6.650.qtquick3d.wasm_multithread + Qt Quick 3D for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qtquick3d, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtquick3d-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 5a92d04743dc47cd410922b312018b2ca13707a9 + + + qt.qt6.650.qtquicktimeline.wasm_multithread + Qt Quick Timeline for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qtquicktimeline, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtquicktimeline-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + 7bce5d03611d856cf1675844c736a9f67dea055d + + + qt.qt6.650.qtshadertools.wasm_multithread + Qt Shader Tools for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qtshadertools, qt.qt6.650.wasm_multithread + qt.qt6.650.wasm_multithread + true + + + qtshadertools-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + c39e4c3ccda1d23253bf8049b417284380cc2715 + + + qt.qt6.650.wasm_multithread + WebAssembly (multi-threaded) + Qt 6.5.0 Prebuilt Components for WebAssembly (multi-threaded). + 6.5.0-0-202301241223 + 2023-01-24 + + + false + + 700 + qtbase-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z, qtdeclarative-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z, qtsvg-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z, qttools-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z, qttranslations-MacOS-MacOS_12-Clang-MacOS-WebAssembly-X86_64.7z + + c49a2335d6bb9ecd42a4d04c0693b928eeb03f19 + + e6fc59b20903417c2ab109b8aeb90f73e5cbabc7 + 2023-01-24-1447_meta.7z + diff --git a/tests/data/windows-650-wasm-single-expect.json b/tests/data/windows-650-wasm-single-expect.json new file mode 100644 index 00000000..29ede737 --- /dev/null +++ b/tests/data/windows-650-wasm-single-expect.json @@ -0,0 +1,26 @@ +{ + "architectures": [ + "wasm_singlethread" + ], + "modules_by_arch": { + "wasm_singlethread": [ + "qtcharts", + "qtdatavis3d", + "qtgrpc", + "qthttpserver", + "qtimageformats", + "qtlottie", + "qtmultimedia", + "qtquick3dphysics", + "qtscxml", + "qtspeech", + "qtvirtualkeyboard", + "qtwebchannel", + "qtwebsockets", + "qt5compat", + "qtquick3d", + "qtquicktimeline", + "qtshadertools" + ] + } +} \ No newline at end of file diff --git a/tests/data/windows-650-wasm-single-update.xml b/tests/data/windows-650-wasm-single-update.xml new file mode 100644 index 00000000..3c997f06 --- /dev/null +++ b/tests/data/windows-650-wasm-single-update.xml @@ -0,0 +1,290 @@ + + {AnyApplication} + 1.0.0 + true + + qt.qt6.650.addons.qtcharts.wasm_singlethread + Qt Charts for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtcharts, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtcharts-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 8f5ed7a6950398e9c0009a1e71514c885d4950ae + + + qt.qt6.650.addons.qtdatavis3d.wasm_singlethread + Qt Data Visualization for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtdatavis3d, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtdatavis3d-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + b721d4ccc387c2bce2064acda62e838bf3b0faf5 + + + qt.qt6.650.addons.qtgrpc.wasm_singlethread + Qt Protobuf and Qt GRPC for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtgrpc, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtgrpc-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 8ee03c76df8602753c17e09471969050ff9ad143 + + + qt.qt6.650.addons.qthttpserver.wasm_singlethread + Qt HTTP Server for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qthttpserver, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qthttpserver-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + a517c99db4438b786775d896dce33f26776484ce + + + qt.qt6.650.addons.qtimageformats.wasm_singlethread + Qt ImageFormats for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtimageformats, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtimageformats-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 21721377df4a85a447883183d4c43490537bc5f7 + + + qt.qt6.650.addons.qtlottie.wasm_singlethread + Qt Lottie Animation for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtlottie, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtlottie-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + e84a9a593ac3ce61fbf01f28839adca16ba72662 + + + qt.qt6.650.addons.qtmultimedia.wasm_singlethread + Qt Multimedia for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtmultimedia, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtmultimedia-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + b3cec4c569f81833fe8950eac8ef8ad18efa9c8e + + + qt.qt6.650.addons.qtquick3dphysics + Quick: 3D Physics + Qt Quick 3D Physics provides a high-level QML module adding physical simulation capabilities to Qt Quick 3D. + 6.5.0-0-202301241223 + 2023-01-24 + false + qt.qt6.650.doc.qtquick3dphysics, qt.qt6.650.examples.qtquick3dphysics, qt.qt6.650.qtquick3d, qt.qt6.650.qtshadertools + + + + d7ae546041aa8afc9d75615538a7de5c74df2c76 + + + qt.qt6.650.addons.qtquick3dphysics.wasm_singlethread + Quick: 3D Physics for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtquick3dphysics, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtquick3dphysics-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + ffae40c5ea3a64fad5d3c891b3321fb84f5dbb10 + + + qt.qt6.650.addons.qtscxml.wasm_singlethread + Qt State Machines for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtscxml, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtscxml-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 196ec95bf0b8b94c71101b5b100e76a8219c7aa6 + + + qt.qt6.650.addons.qtspeech.wasm_singlethread + Qt Speech for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtspeech, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtspeech-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 22af0b136ee5af85d5231e909644f05a2610c333 + + + qt.qt6.650.addons.qtvirtualkeyboard.wasm_singlethread + Qt Virtual Keyboard for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtvirtualkeyboard, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtvirtualkeyboard-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 0bf68731445a07c3d0343a7b2be575792e1d74ec + + + qt.qt6.650.addons.qtwebchannel.wasm_singlethread + Qt WebChannel for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtwebchannel, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtwebchannel-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 3b77f28422d469dd639f53cfa3c90f17581599f2 + + + qt.qt6.650.addons.qtwebsockets.wasm_singlethread + Qt WebSockets for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.addons.qtwebsockets, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtwebsockets-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + c966978a11ef2baee84e26bc8cee4ca636b87130 + + + qt.qt6.650.qt5compat.wasm_singlethread + Qt 5 Compatibility Module for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qt5compat, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qt5compat-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 4362670c3f297d5f298602e5820843150a394585 + + + qt.qt6.650.qtquick3d.wasm_singlethread + Qt Quick 3D for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qtquick3d, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtquick3d-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + a8cb6b3e15ed4357b60895a5b8f06938f381cbe4 + + + qt.qt6.650.qtquicktimeline.wasm_singlethread + Qt Quick Timeline for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qtquicktimeline, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtquicktimeline-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + e3b17d993959c4f9bf0bf1b1eef3eaa32dfa1135 + + + qt.qt6.650.qtshadertools.wasm_singlethread + Qt Shader Tools for WebAssembly + + 6.5.0-0-202301241223 + 2023-01-24 + qt.qt6.650.qtshadertools, qt.qt6.650.wasm_singlethread + qt.qt6.650.wasm_singlethread + true + + + qtshadertools-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + 4a159d7a4d73ea9e02a20600425228ea5fc72368 + + + qt.qt6.650.wasm_singlethread + WebAssembly (single-threaded) + Qt 6.5.0 Prebuilt Components for WebAssembly (single-threaded). + 6.5.0-0-202301241223 + 2023-01-24 + + + false + + 700 + qtbase-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z, qtdeclarative-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z, qtsvg-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z, qttools-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z, qttranslations-Windows-Windows_10_22H2-Clang-Windows-WebAssembly-X86_64.7z + + b3131ac0be2b917e3450eed517fa52aaef7031b3 + + c2e369a310d73abefd8f1f1c038762aad5349c31 + 2023-01-24-1445_meta.7z + diff --git a/tests/data/windows-desktop-expect.json b/tests/data/windows-desktop-expect.json index 75913497..f0c18732 100644 --- a/tests/data/windows-desktop-expect.json +++ b/tests/data/windows-desktop-expect.json @@ -10,7 +10,8 @@ "5.15.0 5.15.1 5.15.2", "6.0.0 6.0.1 6.0.2 6.0.3", "6.1.0", - "6.2.0" + "6.2.0", + "6.5.0" ], "preview": [ "5.6-preview", @@ -27,8 +28,11 @@ "wasm": [ "5.13.1 5.13.2", "5.14.0 5.14.1 5.14.2", - "5.15.0 5.15.1 5.15.2" + "5.15.0 5.15.1 5.15.2", + "6.2.0" ], + "wasm_singlethread": ["6.5.0"], + "wasm_multithread": ["6.5.0"], "wasm_preview": [ "5.13-preview", "5.14-preview", @@ -87,6 +91,7 @@ "tools_ifw", "tools_generic", "tools_conan", - "tools_cmake" + "tools_cmake", + "sdktool" ] } \ No newline at end of file diff --git a/tests/data/windows-desktop.html b/tests/data/windows-desktop.html index fef3b4b8..d7ca25bf 100644 --- a/tests/data/windows-desktop.html +++ b/tests/data/windows-desktop.html @@ -43,6 +43,11 @@

Index of /online/qtsdkrepository/windows_x86/desktop

 tools_generic/13-Apr-2021 14:39 -    tools_conan/15-Feb-2021 12:15 -    tools_cmake/07-Jan-2021 14:22 -   + sdktool/05-May-2023 10:53 -   + qt6_650_wasm_singlethread/01-Jan-2023 00:00 -   + qt6_650_wasm_multithread/01-Jan-2023 00:00 -   + qt6_650_src_doc_examples/01-Jan-2023 00:00 -   + qt6_650/01-Jan-2023 00:00 -    qt6_620_wasm/29-Sep-2021 12:46 -    qt6_620_src_doc_examples/29-Sep-2021 12:43 -    qt6_620/29-Sep-2021 12:34 -   diff --git a/tests/test_cli.py b/tests/test_cli.py index 09c58689..35265aed 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -7,6 +7,7 @@ import pytest from aqt.exceptions import CliInputError +from aqt.helper import Settings from aqt.installer import Cli from aqt.metadata import MetadataFactory, SimpleSpec, Version @@ -53,13 +54,23 @@ def test_cli_help(capsys): assert expected_help(out) -def test_cli_check_module(): +@pytest.mark.parametrize( + "qt_version, modules, unexpected_modules", + ( + ("5.11.3", ["qtcharts", "qtwebengine"], []), + ("5.11.3", ["not_exist"], ["not_exist"]), + ("5.11.3", ["qtcharts", "qtwebengine", "not_exist"], ["not_exist"]), + ("5.11.3", None, []), + ("5.15.0", ["Unknown"], ["Unknown"]), + ), +) +def test_cli_select_unexpected_modules(qt_version: str, modules: Optional[List[str]], unexpected_modules: List[str]): cli = Cli() cli._setup_settings() - assert cli._check_modules_arg("5.11.3", ["qtcharts", "qtwebengine"]) - assert not cli._check_modules_arg("5.7", ["not_exist"]) - assert cli._check_modules_arg("5.14.0", None) - assert not cli._check_modules_arg("5.15.0", ["Unknown"]) + assert cli._select_unexpected_modules(qt_version, modules) == unexpected_modules + + nonexistent_qt = "5.16.0" + assert cli._select_unexpected_modules(nonexistent_qt, modules) == sorted(modules or []) def test_cli_check_combination(): @@ -89,8 +100,8 @@ def test_cli_check_version(): ("windows", "winrt", "mingw32", "6", None, False), ("windows", "winrt", "mingw32", "bad spec", None, True), ("windows", "android", "android_x86", "6", Version("6.1.0"), False), - ("windows", "desktop", "android_x86", "6", Version("6.2.0"), False), # does not validate arch - ("windows", "desktop", "android_fake", "6", Version("6.2.0"), False), # does not validate arch + ("windows", "desktop", "android_x86", "6", Version("6.5.0"), False), # does not validate arch + ("windows", "desktop", "android_fake", "6", Version("6.5.0"), False), # does not validate arch ), ) def test_cli_determine_qt_version( @@ -372,13 +383,63 @@ def _mocked_run(*args): ) -def test_cli_set_7zip(monkeypatch): +@pytest.mark.parametrize("external_tool_exists", (True, False)) +def test_set_7zip_checks_external_tool_when_specified(monkeypatch, capsys, external_tool_exists: bool): cli = Cli() cli._setup_settings() - with pytest.raises(CliInputError) as err: - cli._set_sevenzip("some_nonexistent_binary") - assert err.type == CliInputError - assert format(err.value) == "Specified 7zip command executable does not exist: 'some_nonexistent_binary'" + external = "my_7z_extractor" + + def mock_subprocess_run(args, **kwargs): + assert args[0] == external + if not external_tool_exists: + raise FileNotFoundError() + + monkeypatch.setattr("aqt.installer.subprocess.run", mock_subprocess_run) + monkeypatch.setattr("aqt.installer.EXT7Z", False) + if external_tool_exists: + assert external == cli._set_sevenzip(external) + else: + with pytest.raises(CliInputError) as err: + cli._set_sevenzip(external) + assert format(err.value) == format(f"Specified 7zip command executable does not exist: '{external}'") + assert capsys.readouterr()[1] == "" + + +@pytest.mark.parametrize("fallback_exists", (True, False)) +def test_set_7zip_uses_fallback_when_py7zr_missing(monkeypatch, capsys, fallback_exists: bool): + cli = Cli() + cli._setup_settings() + external, fallback = None, Settings.zipcmd + + def mock_subprocess_run(args, **kwargs): + assert args[0] == fallback + if not fallback_exists: + raise FileNotFoundError() + + monkeypatch.setattr("aqt.installer.subprocess.run", mock_subprocess_run) + monkeypatch.setattr("aqt.installer.EXT7Z", True) + if fallback_exists: + assert fallback == cli._set_sevenzip(external) + else: + with pytest.raises(CliInputError) as err: + cli._set_sevenzip(external) + assert format(err.value) == format(f"Fallback 7zip command executable does not exist: '{fallback}'") + assert f"Falling back to '{fallback}'" in capsys.readouterr()[1] + + +@pytest.mark.parametrize("fallback_exists", (True, False)) +def test_set_7zip_chooses_p7zr_when_ext_missing(monkeypatch, capsys, fallback_exists: bool): + cli = Cli() + cli._setup_settings() + external = None + + def mock_subprocess_run(args, **kwargs): + assert False, "Should not try to run anything" + + monkeypatch.setattr("aqt.installer.subprocess.run", mock_subprocess_run) + monkeypatch.setattr("aqt.installer.EXT7Z", False) + assert cli._set_sevenzip(external) is None + assert capsys.readouterr()[1] == "" @pytest.mark.parametrize( @@ -405,58 +466,130 @@ def mock_mkdir(*args, **kwargs): @pytest.mark.parametrize( - "host, is_auto, mocked_mingw, existing_arch_dirs, expect", + "host, target, arch, is_auto, mocked_arches, existing_arch_dirs, expect", ( ( # not installed "windows", + "android", + "android_armv7", False, - "win64_mingw99", + ["win64_mingw99"], ["not_mingw"], {"install": None, "instruct": "win64_mingw99", "use_dir": "mingw99_64"}, ), ( # Alt Desktop Qt already installed "windows", + "android", + "android_armv7", False, - "win64_mingw99", + ["win64_mingw99"], ["mingw128_32"], {"install": None, "instruct": None, "use_dir": "mingw128_32"}, ), # not installed - ("linux", False, None, ["gcc_32"], {"install": None, "instruct": "gcc_64", "use_dir": "gcc_64"}), + ( + "linux", + "android", + "android_armv7", + False, + [], + ["gcc_32"], + {"install": None, "instruct": "gcc_64", "use_dir": "gcc_64"}, + ), ( # Desktop Qt already installed "linux", + "android", + "android_armv7", False, - None, + [], ["gcc_64"], {"install": None, "instruct": None, "use_dir": "gcc_64"}, ), ( # not installed "windows", + "android", + "android_armv7", True, - "win64_mingw99", + ["win64_mingw99"], ["not_mingw"], {"install": "win64_mingw99", "instruct": None, "use_dir": "mingw99_64"}, ), ( # Alt Desktop Qt already installed "windows", + "android", + "android_armv7", True, - "win64_mingw99", + ["win64_mingw99"], ["mingw128_32"], {"install": None, "instruct": None, "use_dir": "mingw128_32"}, ), # not installed - ("linux", True, None, ["gcc_32"], {"install": "gcc_64", "instruct": None, "use_dir": "gcc_64"}), + ( + "linux", + "android", + "android_armv7", + True, + [], + ["gcc_32"], + {"install": "gcc_64", "instruct": None, "use_dir": "gcc_64"}, + ), ( # Desktop Qt already installed "linux", + "android", + "android_armv7", True, - None, + [], ["gcc_64"], {"install": None, "instruct": None, "use_dir": "gcc_64"}, ), + ( # MSVC arm64 with --autodesktop: should install min64_msvc2019_64 + "windows", + "desktop", + "win64_msvc2019_arm64", + True, + ["win64_mingw", "win64_msvc2019_64", "win64_msvc2019_arm64", "wasm_singlethread", "wasm_multithread"], + ["mingw128_32"], + {"install": "win64_msvc2019_64", "instruct": None, "use_dir": "msvc2019_64"}, + ), + ( # MSVC arm64 without --autodesktop: should point to min64_msvc2019_64 + "windows", + "desktop", + "win64_msvc2019_arm64", + False, + ["win64_mingw", "win64_msvc2019_64", "win64_msvc2019_arm64", "wasm_singlethread", "wasm_multithread"], + ["mingw128_32"], + {"install": None, "instruct": "win64_msvc2019_64", "use_dir": "msvc2019_64"}, + ), + ( # MSVC arm64 without --autodesktop, with correct target already installed + "windows", + "desktop", + "win64_msvc2019_arm64", + False, + ["win64_mingw", "win64_msvc2019_64", "win64_msvc2019_arm64", "wasm_singlethread", "wasm_multithread"], + ["msvc2019_64"], + {"install": None, "instruct": None, "use_dir": "msvc2019_64"}, + ), + ( # MSVC arm64 without --autodesktop, with wrong target already installed + "windows", + "desktop", + "win64_msvc2019_arm64", + False, + ["win64_mingw", "win64_msvc2019_64", "win64_msvc2019_arm64", "wasm_singlethread", "wasm_multithread"], + ["mingw128_32"], + {"install": None, "instruct": "win64_msvc2019_64", "use_dir": "msvc2019_64"}, + ), ), ) -def test_get_autodesktop_dir_and_arch( - monkeypatch, capsys, host: str, is_auto: bool, mocked_mingw: str, existing_arch_dirs: List[str], expect: Dict[str, str] +def test_get_autodesktop_dir_and_arch_non_android( + monkeypatch, + capsys, + host: str, + target: str, + arch: str, + is_auto: bool, + mocked_arches: List[str], + existing_arch_dirs: List[str], + expect: Dict[str, str], ): """ :is_auto: Simulates passing `--autodesktop` to aqt @@ -467,16 +600,16 @@ def test_get_autodesktop_dir_and_arch( :expect[use_dir]: The directory that includes `bin/qmake`; we will patch files in the mobile installation with this value """ - monkeypatch.setattr(MetadataFactory, "fetch_arches", lambda *args: [mocked_mingw]) + monkeypatch.setattr(MetadataFactory, "fetch_arches", lambda *args: mocked_arches) monkeypatch.setattr(Cli, "run", lambda *args: 0) - target = "android" version = "6.2.3" cli = Cli() cli._setup_settings() + flavor = "MSVC Arm64" if arch == "win64_msvc2019_arm64" else target expect_msg_prefix = ( - f"You are installing the {target} version of Qt, " + f"You are installing the {flavor} version of Qt, " f"which requires that the desktop version of Qt is also installed." ) @@ -487,7 +620,7 @@ def test_get_autodesktop_dir_and_arch( qmake.parent.mkdir(parents=True) qmake.write_text("exe file") autodesktop_arch_dir, autodesktop_arch_to_install = cli._get_autodesktop_dir_and_arch( - is_auto, host, target, base_dir, Version(version) + is_auto, host, target, base_dir, Version(version), arch ) # It should choose the correct desktop arch directory for updates assert autodesktop_arch_dir == expect["use_dir"] diff --git a/tests/test_install.py b/tests/test_install.py index 9cae23d4..0d386f67 100644 --- a/tests/test_install.py +++ b/tests/test_install.py @@ -1,3 +1,4 @@ +import errno import hashlib import logging import os @@ -5,6 +6,7 @@ import re import subprocess import sys +import tarfile import textwrap from dataclasses import dataclass from datetime import datetime @@ -107,7 +109,25 @@ def xml_package_update(self) -> str: ) def write_compressed_archive(self, dest: Path) -> None: - with TemporaryDirectory() as temp_dir, py7zr.SevenZipFile(dest / self.filename_7z, "w") as archive: + def open_writable_archive(): + if self.filename_7z.endswith(".7z"): + return py7zr.SevenZipFile(dest / self.filename_7z, "w") + elif self.filename_7z.endswith(".tar.xz"): + return tarfile.open(dest / self.filename_7z, "w:xz") + # elif self.filename_7z.endswith(".zip"): + # return tarfile.open(dest / "DUMMY-NOT-USED", "w") + else: + assert False, "Archive type not supported" + + def write_to_archive(arc, src, arcname): + if self.filename_7z.endswith(".7z"): + arc.writeall(path=src, arcname=arcname) + elif self.filename_7z.endswith(".tar.xz"): + arc.add(name=src, arcname=arcname) + # elif self.filename_7z.endswith(".zip"): + # shutil.make_archive(str(dest / self.filename_7z), "zip", src) + + with TemporaryDirectory() as temp_dir, open_writable_archive() as archive: temp_path = Path(temp_dir) for folder in ("bin", "lib", "mkspecs"): @@ -121,7 +141,7 @@ def write_compressed_archive(self, dest: Path) -> None: full_path.write_text(patched_file.unpatched_content, "utf_8") archive_name = "5.9" if self.version == "5.9.0" else self.version - archive.writeall(path=temp_path, arcname=archive_name) + write_to_archive(archive, temp_path, arcname=archive_name) def make_mock_geturl_download_archive( @@ -137,7 +157,7 @@ def make_mock_geturl_download_archive( if desktop_archives is None: desktop_archives = [] for _archive in [*standard_archives, *desktop_archives]: - assert _archive.filename_7z.endswith(".7z") + assert re.match(r".*\.(7z|tar\.xz)$", _archive.filename_7z), "Unsupported file type" standard_xml = "\n{}\n".format( "\n".join([archive.xml_package_update() for archive in standard_archives]) @@ -145,8 +165,15 @@ def make_mock_geturl_download_archive( desktop_xml = "\n{}\n".format( "\n".join([archive.xml_package_update() for archive in desktop_archives]) ) + merged_xml = "\n{}{}\n".format( + "\n".join([archive.xml_package_update() for archive in standard_archives]), + "\n".join([archive.xml_package_update() for archive in desktop_archives]), + ) def mock_getUrl(url: str, *args, **kwargs) -> str: + if standard_updates_url == desktop_updates_url and url.endswith(standard_updates_url): + # Edge case where both standard and desktop come from the same Updates.xml: ie msvc2019_arm64 and msvc2019_64 + return merged_xml for xml, updates_url in ( (standard_xml, standard_updates_url), (desktop_xml, desktop_updates_url), @@ -310,6 +337,23 @@ def tool_archive(host: str, tool_name: str, variant: str, date: datetime = datet r"INFO : Time elapsed: .* second" ), ), + ( + "install-tool linux desktop sdktool qt.tools.qtcreator".split(), + "linux", + "desktop", + "10.0.1-0-202305050734", + {"std": ""}, + {"std": ""}, + {"std": "linux_x64/desktop/sdktool/Updates.xml"}, + {"std": [tool_archive("linux", "sdktool", "qt.tools.qtcreator")]}, + re.compile( + r"^INFO : aqtinstall\(aqt\) v.* on Python 3.*\n" + r"INFO : Downloading qt.tools.qtcreator...\n" + r"Finished installation of sdktool-linux-qt.tools.qtcreator.7z in .*\n" + r"INFO : Finished installation\n" + r"INFO : Time elapsed: .* second" + ), + ), ( "tool linux tools_qtcreator 1.2.3-0-197001020304 qt.tools.qtcreator".split(), "linux", @@ -419,6 +463,42 @@ def tool_archive(host: str, tool_name: str, variant: str, date: datetime = datet r"INFO : Time elapsed: .* second" ), ), + ( + "install-src linux desktop 6.5.0".split(), + "linux", + "desktop", + "6.5.0", + {"std": ""}, + {"std": ""}, + {"std": "linux_x64/desktop/qt6_650_src_doc_examples/Updates.xml"}, + { + "std": [ + MockArchive( + filename_7z="qtbase-everywhere-src-6.5.0.tar.xz", + update_xml_name="qt.qt6.650.src", + version="6.5.0", + contents=( + PatchedFile( + filename="Src/qtbase/QtBaseSource.cpp", + unpatched_content="int main(){ return 0; }", + patched_content=None, # not patched + ), + ), + ), + ] + }, + re.compile( + r"^INFO : aqtinstall\(aqt\) v.* on Python 3.*\n" + r"WARNING : The parameter 'target' with value 'desktop' is deprecated " + r"and marked for removal in a future version of aqt\.\n" + r"In the future, please omit this parameter\.\n" + r"INFO : Downloading qtbase\.\.\.\n" + r"([^\n]*Extracting may be unsafe; consider updating Python to 3.11.4 or greater\n)?" + r"Finished installation of qtbase-everywhere-src-6\.5\.0\.tar\.xz in .*\n" + r"INFO : Finished installation\n" + r"INFO : Time elapsed: .* second" + ), + ), ( "install-src windows 5.14.2".split(), "windows", @@ -659,6 +739,68 @@ def tool_archive(host: str, tool_name: str, variant: str, date: datetime = datet r"INFO : Time elapsed: .* second" ), ), + ( + "install-qt windows desktop 6.5.2 win64_msvc2019_arm64 --autodesktop".split(), + "windows", + "desktop", + "6.5.2", + {"std": "win64_msvc2019_arm64", "desk": "win64_msvc2019_64"}, + {"std": "msvc2019_arm64", "desk": "msvc2019_64"}, + {"std": "windows_x86/desktop/qt6_652/Updates.xml", "desk": "windows_x86/desktop/qt6_652/Updates.xml"}, + { + "std": [ + MockArchive( + filename_7z="qtbase-windows-win64_msvc2019_arm64.7z", + update_xml_name="qt.qt6.652.win64_msvc2019_arm64", + contents=( + # Qt 6 msvc-arm64 should patch qconfig.pri, qmake script and target_qt.conf + PatchedFile( + filename="mkspecs/qconfig.pri", + unpatched_content="... blah blah blah ...\n" + "QT_EDITION = Not OpenSource\n" + "QT_LICHECK = Not Empty\n" + "... blah blah blah ...\n", + patched_content="... blah blah blah ...\n" + "QT_EDITION = OpenSource\n" + "QT_LICHECK =\n" + "... blah blah blah ...\n", + ), + PatchedFile( + filename="bin/target_qt.conf", + unpatched_content="Prefix=/Users/qt/work/install/target\n" + "HostPrefix=../../\n" + "HostData=target\n", + patched_content="Prefix={base_dir}{sep}6.5.2{sep}msvc2019_arm64{sep}target\n" + "HostPrefix=../../msvc2019_64\n" + "HostData=../msvc2019_arm64\n", + ), + PatchedFile( + filename="bin/qmake.bat", + unpatched_content="... blah blah blah ...\n" + "/Users/qt/work/install/bin\n" + "... blah blah blah ...\n", + patched_content="... blah blah blah ...\n" + "{base_dir}\\6.5.2\\msvc2019_64\\bin\n" + "... blah blah blah ...\n", + ), + ), + ), + ], + "desk": [plain_qtbase_archive("qt.qt6.652.win64_msvc2019_64", "win64_msvc2019_64", host="windows")], + }, + re.compile( + r"^INFO : aqtinstall\(aqt\) v.* on Python 3.*\n" + r"INFO : You are installing the MSVC Arm64 version of Qt, which requires that the desktop version of " + r"Qt is also installed. Now installing Qt: desktop 6.5.2 win64_msvc2019_64\n" + r"INFO : Downloading qtbase...\n" + r"Finished installation of qtbase-windows-win64_msvc2019_arm64.7z in .*\n" + r"INFO : Downloading qtbase...\n" + r"Finished installation of qtbase-windows-win64_msvc2019_64.7z in .*\n" + r"INFO : Patching .*6\.5\.2[/\\]msvc2019_arm64[/\\]bin[/\\]qmake.bat\n" + r"INFO : Finished installation\n" + r"INFO : Time elapsed: .* second" + ), + ), ( "install-qt linux android 6.4.1 android_arm64_v8a".split(), "linux", @@ -1011,8 +1153,8 @@ def test_install( monkeypatch.setattr("aqt.helper.getUrl", mock_get_url) monkeypatch.setattr("aqt.installer.downloadBinaryFile", mock_download_archive) monkeypatch.setattr( - "aqt.metadata.MetadataFactory.fetch_default_desktop_arch", - lambda *args: {"windows": "win64_mingw1234", "linux": "gcc_64", "mac": "clang_64"}[host], + "aqt.metadata.MetadataFactory.fetch_arches", + lambda *args: [{"windows": "win64_mingw1234", "linux": "gcc_64", "mac": "clang_64"}[host]], ) with TemporaryDirectory() as output_dir: @@ -1052,7 +1194,8 @@ def test_install( ( "install-qt windows desktop 5.16.0 win32_mingw73", None, - "WARNING : Specified Qt version is unknown: 5.16.0.\n" + 'WARNING : Specified Qt version "5.16.0" did not exist when this version of aqtinstall was released. ' + "This may not install properly, but we will try our best.\n" "ERROR : Failed to locate XML data for Qt version '5.16.0'.\n" "==============================Suggested follow-up:==============================\n" "* Please use 'aqt list-qt windows desktop' to show versions available.\n", @@ -1060,7 +1203,8 @@ def test_install( ( "install-qt windows desktop 5.15.0 bad_arch", "windows-5150-update.xml", - "WARNING : Specified target combination is not valid or unknown: windows desktop bad_arch\n" + 'WARNING : Specified target combination "windows desktop bad_arch" did not exist when this version of ' + "aqtinstall was released. This may not install properly, but we will try our best.\n" "ERROR : The packages ['qt_base'] were not found while parsing XML of package information!\n" "==============================Suggested follow-up:==============================\n" "* Please use 'aqt list-qt windows desktop --arch 5.15.0' to show architectures available.\n", @@ -1068,7 +1212,8 @@ def test_install( ( "install-qt windows desktop 5.15.0 win32_mingw73 -m nonexistent foo", "windows-5150-update.xml", - "WARNING : Some of specified modules are unknown.\n" + "WARNING : Specified modules ['foo', 'nonexistent'] did not exist when this version of aqtinstall " + "was released. This may not install properly, but we will try our best.\n" "ERROR : The packages ['foo', 'nonexistent', 'qt_base'] were not found" " while parsing XML of package information!\n" "==============================Suggested follow-up:==============================\n" @@ -1106,7 +1251,8 @@ def test_install( ( "install-tool windows desktop tools_vcredist nonexistent", "windows-desktop-tools_vcredist-update.xml", - "WARNING : Specified target combination is not valid: windows tools_vcredist nonexistent\n" + 'WARNING : Specified target combination "windows tools_vcredist nonexistent" did not exist when this version of ' + "aqtinstall was released. This may not install properly, but we will try our best.\n" "ERROR : The package 'nonexistent' was not found while parsing XML of package information!\n" "==============================Suggested follow-up:==============================\n" "* Please use 'aqt list-tool windows desktop tools_vcredist' to show tool variants available.\n", @@ -1114,7 +1260,8 @@ def test_install( ( "install-tool windows desktop tools_nonexistent nonexistent", None, - "WARNING : Specified target combination is not valid: windows tools_nonexistent nonexistent\n" + 'WARNING : Specified target combination "windows tools_nonexistent nonexistent" did not exist when this ' + "version of aqtinstall was released. This may not install properly, but we will try our best.\n" "ERROR : Failed to locate XML data for the tool 'tools_nonexistent'.\n" "==============================Suggested follow-up:==============================\n" "* Please use 'aqt list-tool windows desktop' to show tools available.\n", @@ -1151,10 +1298,10 @@ def mock_get_url(url, *args, **kwargs): @pytest.mark.parametrize( - "exception_class, settings_file, expect_end_msg, expect_return", + "exception, settings_file, expect_end_msg, expect_return", ( ( - RuntimeError, + RuntimeError(), "../aqt/settings.ini", "===========================PLEASE FILE A BUG REPORT===========================\n" "You have discovered a bug in aqt.\n" @@ -1163,14 +1310,14 @@ def mock_get_url(url, *args, **kwargs): Cli.UNHANDLED_EXCEPTION_CODE, ), ( - KeyboardInterrupt, + KeyboardInterrupt(), "../aqt/settings.ini", "WARNING : Caught KeyboardInterrupt, terminating installer workers\n" "ERROR : Installer halted by keyboard interrupt.", 1, ), ( - MemoryError, + MemoryError(), "../aqt/settings.ini", "WARNING : Caught MemoryError, terminating installer workers\n" "ERROR : Out of memory when downloading and extracting archives in parallel.\n" @@ -1182,7 +1329,7 @@ def mock_get_url(url, *args, **kwargs): 1, ), ( - MemoryError, + MemoryError(), "data/settings_no_concurrency.ini", "WARNING : Caught MemoryError, terminating installer workers\n" "ERROR : Out of memory when downloading and extracting archives.\n" @@ -1192,11 +1339,39 @@ def mock_get_url(url, *args, **kwargs): "(see https://aqtinstall.readthedocs.io/en/latest/cli.html#cmdoption-list-tool-external)", 1, ), + ( + OSError(errno.ENOSPC, "No space left on device"), + "../aqt/settings.ini", + "WARNING : Caught OSError, terminating installer workers\n" + "ERROR : Insufficient disk space to complete installation.\n" + "==============================Suggested follow-up:==============================\n" + "* Check available disk space.\n" + "* Check size requirements for installation.", + 1, + ), + ( + OSError(), + "../aqt/settings.ini", + "===========================PLEASE FILE A BUG REPORT===========================\n" + "You have discovered a bug in aqt.\n" + "Please file a bug report at https://github.com/miurahr/aqtinstall/issues\n" + "Please remember to include a copy of this program's output in your report.", + Cli.UNHANDLED_EXCEPTION_CODE, + ), + ( + PermissionError(), + "../aqt/settings.ini", + "WARNING : Caught PermissionError, terminating installer workers\n" + f"ERROR : Failed to write to base directory at {os.getcwd()}\n" + "==============================Suggested follow-up:==============================\n" + "* Check that the destination is writable and does not already contain files owned by another user.", + 1, + ), ), ) -def test_install_pool_exception(monkeypatch, capsys, exception_class, settings_file, expect_end_msg, expect_return): +def test_install_pool_exception(monkeypatch, capsys, exception, settings_file, expect_end_msg, expect_return): def mock_installer_func(*args): - raise exception_class() + raise exception host, target, ver, arch = "windows", "desktop", "6.1.0", "win64_mingw81" updates_url = "windows_x86/desktop/qt6_610/Updates.xml" @@ -1226,6 +1401,8 @@ def mock_extractor_that_fails(*args, **kwargs): monkeypatch.setattr("aqt.installer.subprocess.run", mock_extractor_that_fails) with pytest.raises(ArchiveExtractionError) as err, TemporaryDirectory() as temp_dir: + with open(Path(temp_dir) / "archive", "w"): + pass installer( qt_package=QtPackage( "name", diff --git a/tests/test_list.py b/tests/test_list.py index 9877abd3..fc0c5cda 100644 --- a/tests/test_list.py +++ b/tests/test_list.py @@ -406,43 +406,50 @@ def test_long_qt_modules(monkeypatch, host: str, target: str, version: str, arch assert table._rows(table.long_heading_keys) == expect["modules_long_by_arch"][arch] -@pytest.fixture -def expected_windows_desktop_plus_wasm_5140() -> Dict: - input_filenames = "windows-5140-expect.json", "windows-5140-wasm-expect.json" +def expected_windows_desktop_plus_wasm_5140(is_wasm_threaded: bool) -> Dict: + if is_wasm_threaded: + input_filenames = ( + "windows-5140-expect.json", + "windows-650-wasm-single-expect.json", + "windows-650-wasm-multi-expect.json", + ) + else: + input_filenames = "windows-5140-expect.json", "windows-5140-wasm-expect.json" to_join = [json.loads((Path(__file__).parent / "data" / f).read_text("utf-8")) for f in input_filenames] return { "architectures": [arch for _dict in to_join for arch in _dict["architectures"]], - "modules_by_arch": {**to_join[0]["modules_by_arch"], **to_join[1]["modules_by_arch"]}, + "modules_by_arch": {k: v for _dict in to_join for k, v in _dict["modules_by_arch"].items()}, } @pytest.mark.parametrize( - "args, expect", + "args, is_wasm_threaded, expect", ( - ("--modules latest win64_msvc2017_64", ["modules_by_arch", "win64_msvc2017_64"]), - ("--spec 5.14 --modules latest win64_msvc2017_64", ["modules_by_arch", "win64_msvc2017_64"]), - ("--modules 5.14.0 win32_mingw73", ["modules_by_arch", "win32_mingw73"]), - ("--modules 5.14.0 win32_msvc2017", ["modules_by_arch", "win32_msvc2017"]), - ("--modules 5.14.0 win64_mingw73", ["modules_by_arch", "win64_mingw73"]), - ("--modules 5.14.0 win64_msvc2015_64", ["modules_by_arch", "win64_msvc2015_64"]), - ("--modules 5.14.0 win64_msvc2017_64", ["modules_by_arch", "win64_msvc2017_64"]), - ("--arch latest", ["architectures"]), - ("--spec 5.14 --arch latest", ["architectures"]), - ("--arch 5.14.0", ["architectures"]), + ("--modules latest win64_msvc2017_64", False, ["modules_by_arch", "win64_msvc2017_64"]), + ("--spec 5.14 --modules latest win64_msvc2017_64", False, ["modules_by_arch", "win64_msvc2017_64"]), + ("--modules 5.14.0 win32_mingw73", False, ["modules_by_arch", "win32_mingw73"]), + ("--modules 5.14.0 win32_msvc2017", False, ["modules_by_arch", "win32_msvc2017"]), + ("--modules 5.14.0 win64_mingw73", False, ["modules_by_arch", "win64_mingw73"]), + ("--modules 5.14.0 win64_msvc2015_64", False, ["modules_by_arch", "win64_msvc2015_64"]), + ("--modules 5.14.0 win64_msvc2017_64", False, ["modules_by_arch", "win64_msvc2017_64"]), + ("--modules 6.5.0 wasm_singlethread", True, ["modules_by_arch", "wasm_singlethread"]), + ("--modules 6.5.0 wasm_multithread", True, ["modules_by_arch", "wasm_multithread"]), + ("--arch latest", True, ["architectures"]), + ("--spec 5.14 --arch latest", False, ["architectures"]), + ("--arch 5.14.0", False, ["architectures"]), ), ) def test_list_qt_cli( monkeypatch, capsys, - expected_windows_desktop_plus_wasm_5140: Dict[str, Set[str]], args: str, + is_wasm_threaded: bool, expect: Union[Set[str], List[str]], ): - htmlfile, xmlfile, wasm_xmlfile = "windows-desktop.html", "windows-5140-update.xml", "windows-5140-wasm-update.xml" version_string_to_replace = "qt5.5140" if isinstance(expect, list): # In this case, 'expect' is a list of keys to follow to the expected values. - expected_dict = expected_windows_desktop_plus_wasm_5140 + expected_dict = expected_windows_desktop_plus_wasm_5140(is_wasm_threaded) for key in expect: # Follow the chain of keys to the list of values. expected_dict = expected_dict[key] assert isinstance(expected_dict, list) @@ -452,13 +459,21 @@ def test_list_qt_cli( assert isinstance(expect_set, set) def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str: - htmltext = (Path(__file__).parent / "data" / htmlfile).read_text("utf-8") + htmltext = (Path(__file__).parent / "data" / "windows-desktop.html").read_text("utf-8") if not rest_of_url.endswith("Updates.xml"): return htmltext - norm_xmltext = (Path(__file__).parent / "data" / xmlfile).read_text("utf-8") - wasm_xmltext = (Path(__file__).parent / "data" / wasm_xmlfile).read_text("utf-8") - xmltext = wasm_xmltext if rest_of_url.endswith("_wasm/Updates.xml") else norm_xmltext + def get_xml_filename() -> str: + if rest_of_url.endswith("_wasm/Updates.xml"): + return "windows-5140-wasm-update.xml" + elif rest_of_url.endswith("_wasm_singlethread/Updates.xml"): + return "windows-650-wasm-single-update.xml" + elif rest_of_url.endswith("_wasm_multithread/Updates.xml"): + return "windows-650-wasm-multi-update.xml" + else: + return "windows-5140-update.xml" + + xmltext = (Path(__file__).parent / "data" / get_xml_filename()).read_text("utf-8") # If we are serving an Updates.xml, `aqt list` will look for a Qt version number. # We will replace the version numbers in the file with the requested version. match = re.search(r"qt(\d)_(\d+)", rest_of_url) @@ -476,6 +491,29 @@ def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str: assert output_set == expect_set +def test_list_missing_wasm_updates(monkeypatch, capsys): + """Require that MetadataFactory is resilient to missing wasm updates.xml files""" + data_dir = Path(__file__).parent / "data" + expect = set(json.loads((data_dir / "windows-620-expect.json").read_text("utf-8"))["architectures"]) + + def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str: + htmltext = (Path(__file__).parent / "data" / "windows-desktop.html").read_text("utf-8") + if rest_of_url.endswith("windows_x86/desktop/"): + return htmltext + elif rest_of_url.endswith("windows_x86/desktop/qt6_620/Updates.xml"): + return (data_dir / "windows-620-update.xml").read_text("utf-8") + else: + raise ArchiveDownloadError(f"No such file at {rest_of_url}") + + monkeypatch.setattr(MetadataFactory, "fetch_http", _mock_fetch_http) + + cli = Cli() + rv = cli.run("list-qt windows desktop --arch 6.2.0".split()) + assert rv == 0 + out, err = capsys.readouterr() + assert set(out.strip().split()) == expect + + @pytest.mark.parametrize( "qt_ver_str, expect_set", (("6.2.0", {"android_x86", "android_x86_64", "android_armv7", "android_arm64_v8a"}),) ) @@ -1028,7 +1066,13 @@ def fetch_expected_tooldata(json_filename: str) -> ToolData: return ToolData(tools) -@pytest.mark.parametrize("host, target, tool_name", (("mac", "desktop", "tools_cmake"),)) +@pytest.mark.parametrize( + "host, target, tool_name", + ( + ("mac", "desktop", "tools_cmake"), + ("mac", "desktop", "sdktool"), + ), +) def test_list_tool_cli(monkeypatch, capsys, host: str, target: str, tool_name: str): html_file = f"{host}-{target}.html" xml_file = f"{host}-{target}-{tool_name}-update.xml" @@ -1046,7 +1090,7 @@ def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str: if not rest_of_url.endswith("Updates.xml"): return htmltext folder = urlparse(rest_of_url).path.split("/")[-2] - assert folder.startswith("tools_") + assert folder.startswith("tools_") or folder in ["sdktool"] return xmltext monkeypatch.setattr(MetadataFactory, "fetch_http", _mock_fetch_http) @@ -1054,27 +1098,31 @@ def _mock_fetch_http(_, rest_of_url, *args, **kwargs: str) -> str: cli = Cli() cli.run(["list-tool", host, target]) out, err = capsys.readouterr() + assert not err output_set = set(out.strip().split()) assert output_set == expected_tools cli.run(["list-tool", host, target, tool_name]) out, err = capsys.readouterr() + assert not err output_set = set(out.strip().split()) assert output_set == expected_tool_modules # Test abbreviated tool name: "aqt list-tool mac desktop ifw" - assert tool_name.startswith("tools_") - short_tool_name = tool_name[6:] + assert tool_name.startswith("tools_") or tool_name in ["sdktool"] + short_tool_name = tool_name[6:] if tool_name.startswith("tools_") else tool_name cli.run(["list-tool", host, target, short_tool_name]) out, err = capsys.readouterr() + assert not err output_set = set(out.strip().split()) assert output_set == expected_tool_modules cli.run(["list-tool", host, target, tool_name, "-l"]) out, err = capsys.readouterr() + assert not err expected_tooldata = format(fetch_expected_tooldata(xml_expect)) - assert out.strip() == expected_tooldata + assert out.strip() == expected_tooldata.strip() def test_fetch_http_ok(monkeypatch):