Merge branch 'master' into topic/miurahr/ci/use-review-dog-for-pr

.gitattributes

@@ -1 +1,26 @@
 .git_archival.txt  export-subst
+
+ # Archives
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.br filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+
+# Documents
+*.pdf filter=lfs diff=lfs merge=lfs -text
+
+# Images
+*.gif filter=lfs diff=lfs merge=lfs -text
+*.ico filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.png filter=lfs diff=lfs merge=lfs -text
+*.psd filter=lfs diff=lfs merge=lfs -text
+*.webp filter=lfs diff=lfs merge=lfs -text
+
+# Fonts
+*.woff2 filter=lfs diff=lfs merge=lfs -text
+
+# Other
+*.exe filter=lfs diff=lfs merge=lfs -text
+*.jar filter=lfs diff=lfs merge=lfs -text

.github/workflows/check.yml

@@ -20,6 +20,7 @@ jobs:
       with:
         fetch-depth: 20
         fetch-tags: true
+        lfs: true
     - name: Set up Python 3.12
       uses: actions/setup-python@v5
       with:
@@ -29,10 +30,21 @@ jobs:
         pip install -U pip
         pip install tox tox-gh-actions coveralls coverage[toml]
     - name: Test
-      run: tox
+      run: |
+        tox
+        coverage xml -o cobertura.xml
       env:
         PYTEST_ADDOPTS: --cov-config=pyproject.toml --cov --cov-append --verbose
     - name: Upload Coverage
       run: coveralls --service=github
       env:
         GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    - uses: actions/setup-java@v4
+      with:
+        distribution: 'temurin'
+        java-version: '21'
+    - name: Send coverage to codacy
+      run: |
+        java -jar ci/codacy-coverage-reporter-assembly.jar report -l Python -t ${PROJECT_TOKEN} -r cobertura.xml
+      env:
+        PROJECT_TOKEN: ${{ secrets.CODACY_PROJECT_TOKEN }}

.github/workflows/codeql-analysis.yml

@@ -31,9 +31,9 @@ jobs:
       uses: actions/checkout@v4
 
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v2
+      uses: github/codeql-action/init@v3
       with:
         languages: ${{ matrix.language }}
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v2
+      uses: github/codeql-action/analyze@v3

.github/workflows/test-install-qt.yml

@@ -31,6 +31,11 @@ jobs:
             py: "3.12"
             qtver: 6.8.0
             artifact: standard
+          - os: ubuntu-latest
+            py: "3.12"
+            # 6.8.1 introduces Operations elements in Updates.xml
+            qtver: 6.8.1
+            artifact: standard
     steps:
       - uses: actions/checkout@v4
         with:

README.rst

@@ -4,6 +4,7 @@ Another Qt installer(aqt)
 - Release: |pypi|
 - Documentation: |docs|
 - Test status: |gha| and Coverage: |coveralls|
+- Code Quality: |codacy|
 - Project maturity |Package health|
 
 .. |pypi| image:: https://badge.fury.io/py/aqtinstall.svg
@@ -17,7 +18,8 @@ Another Qt installer(aqt)
 .. |Package health| image:: https://snyk.io/advisor/python/aqtinstall/badge.svg
   :target: https://snyk.io/advisor/python/aqtinstall
   :alt: aqtinstall
-
+.. |codacy| image:: https://app.codacy.com/project/badge/Grade/188accbe7f8f406abf61b888773bf5e3
+  :target: https://app.codacy.com/gh/miurahr/aqtinstall/dashboard?utm_source=gh&utm_medium=referral&utm_content=&utm_campaign=Badge_grade
 
 
 This is a utility alternative to the official graphical Qt installer, for using in CI environment

aqt/archives.py

@@ -21,6 +21,7 @@
 # CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 import posixpath
 from dataclasses import dataclass, field
+from itertools import islice, zip_longest
 from logging import getLogger
 from typing import Dict, Iterable, List, Optional, Set, Tuple
 from xml.etree.ElementTree import Element  # noqa
@@ -32,6 +33,12 @@
 from aqt.metadata import QtRepoProperty, Version
 
 
+@dataclass
+class UpdateXmls:
+    target_folder: str
+    xml_text: str
+
+
 @dataclass
 class TargetConfig:
     version: str
@@ -46,6 +53,7 @@ class QtPackage:
     base_url: str
     archive_path: str
     archive: str
+    archive_install_path: str
     package_desc: str
     pkg_update_name: str
     version: Optional[Version] = field(default=None)
@@ -128,12 +136,13 @@ class PackageUpdate:
     dependencies: Iterable[str]
     auto_dependon: Iterable[str]
     downloadable_archives: Iterable[str]
+    archive_install_paths: Iterable[str]
     default: bool
     virtual: bool
     base: str
 
     def __post_init__(self):
-        for iter_of_str in self.dependencies, self.auto_dependon, self.downloadable_archives:
+        for iter_of_str in self.dependencies, self.auto_dependon, self.downloadable_archives, self.archive_install_paths:
             assert isinstance(iter_of_str, Iterable) and not isinstance(iter_of_str, str)
         for _str in self.name, self.display_name, self.description, self.release_date, self.full_version, self.base:
             assert isinstance(_str, str)
@@ -177,6 +186,7 @@ def fromstring(base, update_xml_text: str):
         except ElementTree.ParseError as perror:
             raise ArchiveListError(f"Downloaded metadata is corrupted. {perror}") from perror
         updates = Updates()
+        extract_xpath = "Operations/Operation[@name='Extract']/Argument"
         for packageupdate in update_xml.iter("PackageUpdate"):
             pkg_name = updates._get_text(packageupdate.find("Name"))
             display_name = updates._get_text(packageupdate.find("DisplayName"))
@@ -186,8 +196,20 @@ def fromstring(base, update_xml_text: str):
             dependencies = updates._get_list(packageupdate.find("Dependencies"))
             auto_dependon = updates._get_list(packageupdate.find("AutoDependOn"))
             archives = updates._get_list(packageupdate.find("DownloadableArchives"))
+            archive_install_paths = updates._get_list(None)
             default = updates._get_boolean(packageupdate.find("Default"))
             virtual = updates._get_boolean(packageupdate.find("Virtual"))
+            if packageupdate.find(extract_xpath) is not None:
+                arc_args = map(
+                    lambda x: x.text,
+                    islice(packageupdate.iterfind(extract_xpath), 1, None, 2),
+                )
+                archives = ssplit(", ".join(arc_args))
+                path_args = map(
+                    lambda x: x.text.replace("@TargetDir@/", "", 1),
+                    islice(packageupdate.iterfind(extract_xpath), 0, None, 2),
+                )
+                archive_install_paths = ssplit(", ".join(path_args))
             updates.package_updates.append(
                 PackageUpdate(
                     pkg_name,
@@ -198,6 +220,7 @@ def fromstring(base, update_xml_text: str):
                     dependencies,
                     auto_dependon,
                     archives,
+                    archive_install_paths,
                     default,
                     virtual,
                     base,
@@ -354,6 +377,7 @@ def _target_packages(self) -> ModuleToPackage:
             package_names = [
                 f"qt.qt{self.version.major}.{self._version_str()}.{suffix}",
                 f"qt.{self._version_str()}.{suffix}",
+                f"extensions.{module}.{self._version_str()}.{self.arch}",
             ]
             if not module.startswith("addons."):
                 package_names.append(f"qt.qt{self.version.major}.{self._version_str()}.addons.{suffix}")
@@ -386,7 +410,7 @@ def _get_archives_base(self, name, target_packages):
         os_name = self.os_name
         if self.target == "android" and self.version >= Version("6.7.0"):
             os_name = "all_os"
-        if self.os_name == "windows":
+        elif self.os_name == "windows":
             os_name += "_x86"
         elif os_name != "linux_arm64" and os_name != "all_os" and os_name != "windows_arm64":
             os_name += "_x64"
@@ -399,18 +423,48 @@ def _get_archives_base(self, name, target_packages):
         )
         update_xml_url = posixpath.join(os_target_folder, "Updates.xml")
         update_xml_text = self._download_update_xml(update_xml_url)
-        self._parse_update_xml(os_target_folder, update_xml_text, target_packages)
+        update_xmls = [UpdateXmls(os_target_folder, update_xml_text)]
 
-    def _download_update_xml(self, update_xml_path):
+        if self.version >= Version("6.8.0"):
+            arch = self.arch
+            if self.os_name == "windows":
+                arch = self.arch.replace("win64_", "", 1)
+            elif self.os_name == "linux":
+                arch = "x86_64"
+            elif self.os_name == "linux_arm64":
+                arch = "arm64"
+            for ext in ["qtwebengine", "qtpdf"]:
+                extensions_target_folder = posixpath.join(
+                    "online/qtsdkrepository", os_name, "extensions", ext, self._version_str(), arch
+                )
+                extensions_xml_url = posixpath.join(extensions_target_folder, "Updates.xml")
+                # The extension may or may not exist for this version and arch.
+                try:
+                    extensions_xml_text = self._download_update_xml(extensions_xml_url, True)
+                except ArchiveDownloadError:
+                    # In case _download_update_xml ignores the hash and tries to get the url.
+                    pass
+                if extensions_xml_text:
+                    self.logger.info("Found extension {}".format(ext))
+                    update_xmls.append(UpdateXmls(extensions_target_folder, extensions_xml_text))
+
+        self._parse_update_xmls(update_xmls, target_packages)
+
+    def _download_update_xml(self, update_xml_path, silent=False):
         """Hook for unit test."""
         if not Settings.ignore_hash:
             try:
                 xml_hash = get_hash(update_xml_path, Settings.hash_algorithm, self.timeout)
             except ChecksumDownloadFailure:
-                self.logger.warning(
-                    "Failed to download checksum for the file 'Updates.xml'. This may happen on unofficial mirrors."
-                )
-                xml_hash = None
+                if silent:
+                    return None
+                else:
+                    self.logger.warning(
+                        "Failed to download checksum for the file '{}'. This may happen on unofficial mirrors.".format(
+                            update_xml_path
+                        )
+                    )
+                    xml_hash = None
         else:
             xml_hash = None
         return getUrl(posixpath.join(self.base, update_xml_path), self.timeout, xml_hash)
@@ -429,7 +483,9 @@ def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages:
                 target_packages.remove_module_for_package(packageupdate.name)
             should_filter_archives: bool = bool(self.subarchives) and self.should_filter_archives(packageupdate.name)
 
-            for archive in packageupdate.downloadable_archives:
+            for archive, archive_install_path in zip_longest(
+                packageupdate.downloadable_archives, packageupdate.archive_install_paths, fillvalue=""
+            ):
                 archive_name = archive.split("-", maxsplit=1)[0]
                 if should_filter_archives and self.subarchives is not None and archive_name not in self.subarchives:
                     continue
@@ -447,10 +503,17 @@ def _parse_update_xml(self, os_target_folder, update_xml_text, target_packages:
                         base_url=base_url,
                         archive_path=archive_path,
                         archive=archive,
+                        archive_install_path=archive_install_path,
                         package_desc=packageupdate.description,
                         pkg_update_name=packageupdate.name,  # For testing purposes
                     )
                 )
+
+    def _parse_update_xmls(self, update_xmls, target_packages: Optional[ModuleToPackage]):
+        if not target_packages:
+            target_packages = ModuleToPackage({})
+        for update_xml in update_xmls:
+            self._parse_update_xml(update_xml.target_folder, update_xml.xml_text, target_packages)
         # if we have located every requested package, then target_packages will be empty
         if not self.all_extra and len(target_packages) > 0:
             message = f"The packages {target_packages} were not found while parsing XML of package information!"
@@ -468,10 +531,11 @@ def _append_tool_update(self, os_target_folder, update_xml, target, tool_version
             raise NoPackageFound(message, suggested_action=self.help_msg())
         package_desc = packageupdate.description
         downloadable_archives = packageupdate.downloadable_archives
+        archive_install_paths = packageupdate.archive_install_paths
         if not downloadable_archives:
             message = f"The package '{self.arch}' contains no downloadable archives!"
             raise NoPackageFound(message)
-        for archive in downloadable_archives:
+        for archive, archive_install_path in zip_longest(downloadable_archives, archive_install_paths, fillvalue=""):
             archive_path = posixpath.join(
                 # online/qtsdkrepository/linux_x64/desktop/tools_ifw/
                 os_target_folder,
@@ -486,6 +550,7 @@ def _append_tool_update(self, os_target_folder, update_xml, target, tool_version
                     base_url=self.base,
                     archive_path=archive_path,
                     archive=archive,
+                    archive_install_path=archive_install_path,
                     package_desc=package_desc,
                     pkg_update_name=name,  # Redundant
                 )

aqt/installer.py

@@ -1157,6 +1157,7 @@ def installer(
     name = qt_package.name
     base_url = qt_package.base_url
     archive: Path = archive_dest / qt_package.archive
+    base_dir = posixpath.join(base_dir, qt_package.archive_install_path)
     start_time = time.perf_counter()
     Settings.load_settings(file=settings_ini)
     # setup queue logger