Use py-rattler to fetch repodata in proxy mode

.github/workflows/ci.yml

@@ -15,7 +15,7 @@ env:
 jobs:
   test_quetz:
     # timeout for the whole job
-    timeout-minutes: 10
+    timeout-minutes: 12
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
@@ -79,7 +79,7 @@ jobs:
       - name: Testing server
         shell: bash -l -eo pipefail {0}
         # timeout for the step
-        timeout-minutes: 5
+        timeout-minutes: 8
         env:
           TEST_DB_BACKEND: ${{ matrix.test_database }}
           QUETZ_TEST_DBINIT: ${{ matrix.db_init }}

.pre-commit-config.yaml

@@ -1,4 +1,4 @@
-exclude: (quetz/migrations)
+exclude: ^(quetz/migrations|quetz/tests/data/test-server/)
 repos:
   - repo: https://github.com/astral-sh/ruff-pre-commit
     # Ruff version.
@@ -27,4 +27,4 @@ repos:
           - types-toml
           - types-ujson
           - types-aiofiles
-        args: [--show-error-codes, --implicit-optional]
+        args: [--show-error-codes, --implicit-optional]

environment.yml

@@ -52,5 +52,6 @@ dependencies:
   - pytest-asyncio
   - pytest-timeout
   - pydantic >=2
+  - py-rattler
   - pip:
       - git+https://github.com/jupyter-server/jupyter_releaser.git@v2

quetz/config.py

@@ -62,6 +62,7 @@ class Config:
                 ConfigEntry("package_unpack_threads", int, 1),
                 ConfigEntry("frontend_dir", str, default=""),
                 ConfigEntry("redirect_http_to_https", bool, False),
+                ConfigEntry("rattler_cache_dir", str, default="rattler_cache"),
             ],
         ),
         ConfigSection(

quetz/tasks/mirror.py

@@ -1,15 +1,20 @@
+import asyncio
 import contextlib
 import json
 import logging
 import os
 import shutil
 from concurrent.futures import ThreadPoolExecutor
 from http.client import IncompleteRead
+from pathlib import Path, PurePath
 from tempfile import SpooledTemporaryFile
 from typing import List
 
+import aiofiles
+import aiofiles.os
 import requests
 from fastapi import HTTPException, status
+from rattler import Channel, ChannelConfig, Platform, fetch_repo_data
 from tenacity import TryAgain, retry
 from tenacity.after import after_log
 from tenacity.stop import stop_after_attempt
@@ -59,6 +64,11 @@ def __init__(self, host, session):
     def open(self, path):
         return RemoteFile(self.host, path, self.session)
 
+    @property
+    def rattler_channel(self):
+        host_path = PurePath(self.host)
+        return Channel(host_path.name, ChannelConfig(f"{host_path.parent}/"))
+
 
 class RemoteServerError(Exception):
     pass
@@ -106,6 +116,36 @@ def json(self):
         return json.load(self.file)
 
 
+async def download_repodata(repository: RemoteRepository, channel: str, platform: str):
+    cache_path = Path(Config().general_rattler_cache_dir) / channel / platform
+    logger.debug(f"Fetching {platform} repodata from {repository.rattler_channel}")
+    try:
+        await fetch_repo_data(
+            channels=[repository.rattler_channel],
+            platforms=[Platform(platform)],
+            cache_path=cache_path,
+            callback=None,
+        )
+    except Exception as e:
+        logger.error(f"Failed to fetch repodata: {e}")
+        raise
+    files = await aiofiles.os.listdir(cache_path)
+    try:
+        json_file = [
+            filename
+            for filename in files
+            if filename.endswith(".json") and not filename.endswith(".info.json")
+        ][0]
+    except IndexError:
+        logger.error(f"No json file found in rattler cache: {cache_path}")
+        raise RemoteFileNotFound
+    else:
+        async with aiofiles.open(cache_path / json_file, "rb") as f:
+            contents = await f.read()
+        logger.debug(f"Retrieved repodata from rattler cache: {cache_path / json_file}")
+        return contents
+
+
 def download_remote_file(
     repository: RemoteRepository, pkgstore: PackageStore, channel: str, path: str
 ):
@@ -122,13 +162,18 @@ def download_remote_file(
     # Acquire a lock to prevent multiple concurrent downloads of the same file
     with pkgstore.create_download_lock(channel, path):
         logger.debug(f"Downloading {path} from {channel} to pkgstore")
-        remote_file = repository.open(path)
-        data_stream = remote_file.file
-
-        if path.endswith(".json"):
-            add_static_file(data_stream.read(), channel, None, path, pkgstore)
+        if path.endswith("/repodata.json"):
+            platform = str(PurePath(path).parent)
+            repodata = asyncio.run(download_repodata(repository, channel, platform))
+            add_static_file(repodata, channel, None, path, pkgstore)
         else:
-            pkgstore.add_package(data_stream, channel, path)
+            remote_file = repository.open(path)
+            data_stream = remote_file.file
+
+            if path.endswith(".json"):
+                add_static_file(data_stream.read(), channel, None, path, pkgstore)
+            else:
+                pkgstore.add_package(data_stream, channel, path)
 
     pkgstore.delete_download_lock(channel, path)
 

quetz/tests/data/test-server/README.md

@@ -0,0 +1,7 @@
+# test-server
+
+A simple server to serve repodata for test purposes.
+Originally implemented by [mamba](https://github.com/mamba-org/mamba/tree/a8d595b6ff8ac182e60741c3e8cbd142e7d19905/mamba/tests)
+under the BSD-3-Clause license.
+
+Copied from [rattler](https://github.com/mamba-org/rattler/tree/9a3f2cc92a50fec4f6f7c13488441ca4ac15269b/test-data/test-server).

quetz/tests/data/test-server/repo/channeldata.json

@@ -0,0 +1,38 @@
+{
+  "channeldata_version": 1,
+  "packages": {
+    "test-package": {
+      "activate.d": false,
+      "binary_prefix": false,
+      "deactivate.d": false,
+      "description": null,
+      "dev_url": null,
+      "doc_source_url": null,
+      "doc_url": null,
+      "home": "https://github.com/mamba-org/mamba",
+      "icon_hash": null,
+      "icon_url": null,
+      "identifiers": null,
+      "keywords": null,
+      "license": "BSD",
+      "post_link": false,
+      "pre_link": false,
+      "pre_unlink": false,
+      "recipe_origin": null,
+      "run_exports": {},
+      "source_git_url": null,
+      "source_url": null,
+      "subdirs": [
+        "noarch"
+      ],
+      "summary": "I am just a test package!",
+      "tags": null,
+      "text_prefix": false,
+      "timestamp": 1613117294,
+      "version": "0.1"
+    }
+  },
+  "subdirs": [
+    "noarch"
+  ]
+}

quetz/tests/data/test-server/repo/index.html

@@ -0,0 +1,90 @@
+<html>
+<head>
+  <title>repo</title>
+  <style type="text/css">
+    a, a:active {
+      text-decoration: none; color: blue;
+    }
+    a:visited {
+      color: #48468F;
+    }
+    a:hover, a:focus {
+      text-decoration: underline; color: red;
+    }
+    body {
+      background-color: #F5F5F5;
+    }
+    h2 {
+      margin-bottom: 12px;
+    }
+    th, td {
+      font: 100% monospace; text-align: left;
+    }
+    th {
+      font-weight: bold; padding-right: 14px; padding-bottom: 3px;
+    }
+    th.tight {
+        padding-right: 6px;
+    }
+    td {
+      padding-right: 14px;
+    }
+    td.tight {
+        padding-right: 8px;
+    }
+    td.s, th.s {
+      text-align: right;
+    }
+    td.summary {
+      white-space: nowrap;
+      overflow: hidden;
+    }
+    td.packagename {
+      white-space: nowrap;
+      text-overflow: ellipsis;
+      overflow: hidden;
+      max-width: 180px;
+      padding-right: 8px;
+    }
+    td.version {
+      //white-space: nowrap;
+      overflow: hidden;
+      max-width: 90px;
+      padding-right: 8px;
+    }
+    table {
+      background-color: white;
+      border-top: 1px solid #646464;
+      border-bottom: 1px solid #646464;
+      padding-top: 10px;
+      padding-bottom: 14px;
+    }
+    address {
+      color: #787878;
+      padding-top: 10px;
+    }
+  </style>
+</head>
+<body>
+  <h2>repo</h2>
+  <h3><a href="rss.xml">RSS Feed</a>&nbsp;&nbsp;&nbsp;<a href="channeldata.json">channeldata.json</a></h3>
+<a href="noarch">noarch</a>&nbsp;&nbsp;&nbsp;  <table>
+    <tr>
+      <th style="padding-right:18px;">Package</th>
+      <th>Latest Version</th>
+      <th>Doc</th>
+      <th>Dev</th>
+      <th>License</th>
+<th class="tight">noarch</th>      <th>Summary</th>
+    </tr>
+    <tr>
+      <td class="packagename"><a href="https://github.com/mamba-org/mamba" alt="test-package">test-package</a></td>
+      <td class="version">0.1</td>
+      <td></td>
+      <td></td>
+      <td class="tight">BSD</td>
+<td>X</td>      <td class="summary">I am just a test package!</td>
+    </tr>  </table>
+  <address>Updated: 2021-02-12 09:02:37 +0000 - Files: 1</address>
+</body>
+</html>

quetz/tests/data/test-server/repo/noarch/current_repodata.json

@@ -0,0 +1,25 @@
+{
+  "info": {
+    "subdir": "noarch"
+  },
+  "packages": {
+    "test-package-0.1-0.tar.bz2": {
+      "build": "0",
+      "build_number": 0,
+      "depends": [],
+      "license": "BSD",
+      "license_family": "BSD",
+      "md5": "2a8595f37faa2950e1b433acbe91d481",
+      "name": "test-package",
+      "noarch": "generic",
+      "sha256": "b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988",
+      "size": 5719,
+      "subdir": "noarch",
+      "timestamp": 1613117294885,
+      "version": "0.1"
+    }
+  },
+  "packages.conda": {},
+  "removed": [],
+  "repodata_version": 1
+}

quetz/tests/data/test-server/repo/noarch/index.html

@@ -0,0 +1,88 @@
+<html>
+<head>
+  <title>repo/noarch</title>
+  <style type="text/css">
+    a, a:active {
+      text-decoration: none; color: blue;
+    }
+    a:visited {
+      color: #48468F;
+    }
+    a:hover, a:focus {
+      text-decoration: underline; color: red;
+    }
+    body {
+      background-color: #F5F5F5;
+    }
+    h2 {
+      margin-bottom: 12px;
+    }
+    th, td {
+      font: 100% monospace; text-align: left;
+    }
+    th {
+      font-weight: bold; padding-right: 14px; padding-bottom: 3px;
+    }
+    td {
+      padding-right: 20px;
+    }
+    td.s, th.s {
+      text-align: right;
+    }
+    table {
+      background-color: white;
+      border-top: 1px solid #646464;
+      border-bottom: 1px solid #646464;
+      padding-top: 10px;
+      padding-bottom: 14px;
+    }
+    address {
+      color: #787878;
+      padding-top: 10px;
+    }
+  </style>
+</head>
+<body>
+  <h2>repo/noarch</h2>
+  <table>
+    <tr>
+      <th>Filename</th>
+      <th>Size</th>
+      <th>Last Modified</th>
+      <th>SHA256</th>
+      <th>MD5</th>
+    </tr>
+    <tr>
+      <td><a href="repodata.json" alt="repodata.json">repodata.json</a></td>
+      <td class="s">586 B</td>
+      <td>2021-02-12 09:01:48 +0000</td>
+      <td>cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b04</td>
+      <td>7501ec77771889b42a39c615158cb9c4</td>
+    </tr>    <tr>
+      <td><a href="repodata.json.bz2" alt="repodata.json.bz2">repodata.json.bz2</a></td>
+      <td class="s">351 B</td>
+      <td>2021-02-12 09:01:48 +0000</td>
+      <td>9a0288ca48c6b8caa348d7cafefd0981c2d25dcb4a5837a5187ab200b8b9fb45</td>
+      <td>0c926155642f0e894d97dc8a5af7007b</td>
+    </tr>    <tr>
+      <td><a href="repodata_from_packages.json" alt="repodata_from_packages.json">repodata_from_packages.json</a></td>
+      <td class="s">586 B</td>
+      <td>2021-02-12 09:01:48 +0000</td>
+      <td>cc5f72aaa8d3f508c8adca196fe05cf4b19e1ca1006cfcbb3892d73160bd3b04</td>
+      <td>7501ec77771889b42a39c615158cb9c4</td>
+    </tr>    <tr>
+      <td><a href="repodata_from_packages.json.bz2" alt="repodata_from_packages.json.bz2">repodata_from_packages.json.bz2</a></td>
+      <td class="s">351 B</td>
+      <td>2021-02-12 09:01:48 +0000</td>
+      <td>9a0288ca48c6b8caa348d7cafefd0981c2d25dcb4a5837a5187ab200b8b9fb45</td>
+      <td>0c926155642f0e894d97dc8a5af7007b</td>
+    </tr>    <tr>
+      <td><a href="test-package-0.1-0.tar.bz2" alt="test-package-0.1-0.tar.bz2">test-package-0.1-0.tar.bz2</a></td>
+      <td class="s">6 KB</td>
+      <td>2021-02-12 08:08:14 +0000</td>
+      <td>b908ffce2d26d94c58c968abf286568d4bcf87d1cfe6c994958351724a6f6988</td>
+      <td>2a8595f37faa2950e1b433acbe91d481</td>
+    </tr>  </table>
+  <address>Updated: 2021-02-12 09:02:37 +0000 - Files: 1</address>
+</body>
+</html>