feat: multipart upload/download directories recursively (#848)

- When uploading/downloading a directory, glob all directories and files
- Fix AverageTransferSpeedColumn so it renders per task, not for all
tasks

Resolve BE-2133

sdk/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "beta9"
-version = "0.1.150"
+version = "0.1.151"
 description = ""
 authors = ["beam.cloud <[email protected]>"]
 packages = [

sdk/src/beta9/multipart.py

@@ -610,15 +610,18 @@ def __init__(
         self.service = service
         self.progress = progress
 
-    def list_dir(self, remote_path: RemotePath) -> List[RemotePath]:
-        res = self.service.list_path(ListPathRequest(path=remote_path.path))
+    def list_dir(self, remote_path: RemotePath, recursive: bool = False) -> List[RemotePath]:
+        path = remote_path.path
+        if recursive:
+            path = f"{path}/**".replace("//", "/")
+
+        res = self.service.list_path(ListPathRequest(path=path))
         if not res.ok:
             raise RuntimeError(f"{remote_path} ({res.err_msg})")
 
         return [
             RemotePath(remote_path.scheme, remote_path.volume_name, p.path, is_dir=p.is_dir)
             for p in res.path_infos
-            if not p.is_dir
         ]
 
     def is_dir(self, remote_path: RemotePath) -> bool:
@@ -674,7 +677,7 @@ def download(self, remote_path: RemotePath, local_path: Path) -> None:
         if self.is_dir(remote_path):
             local_path.mkdir(parents=True, exist_ok=True)
 
-            for rpath in self.list_dir(remote_path):
+            for rpath in self.list_dir(remote_path, recursive=True):
                 lpath = local_path / rpath.volume_path
                 if not remote_path.volume_path.endswith("/"):
                     lpath = local_path / rpath.name

sdk/src/beta9/terminal.py

@@ -199,26 +199,24 @@ class AverageTransferSpeedColumn(ProgressColumn):
     Renders the average data transfer speed over the entire lifetime of the transfer.
     """
 
-    def __init__(self) -> None:
-        super().__init__()
-        self.average_mib_s = 0.0
-
     def render(self, task: Task) -> Text:
+        task.fields.setdefault("average_mib_s", 0.0)
+
         # If the task hasn't started or there's no elapsed time yet, we can't compute an average
         if not task.started or task.elapsed == 0:
             return Text("?", style="progress.data.speed")
 
         if task.completed == task.total:
-            return Text(f"{self.average_mib_s:.2f} MiB/s", style="progress.data.speed")
+            return Text(f"{task.fields['average_mib_s']:.2f} MiB/s", style="progress.data.speed")
 
         # Calculate average speed in bytes per second
         average_bps = task.completed / (task.elapsed or 1)
 
         # Convert bytes per second to MiB/s (1 MiB = 1024 * 1024 bytes)
-        self.average_mib_s = average_bps / (1024**2)
+        task.fields["average_mib_s"] = average_bps / (1024**2)
 
         # Format to a reasonable precision (e.g., 2 decimal places)
-        return Text(f"{self.average_mib_s:.2f} MiB/s", style="progress.data.speed")
+        return Text(f"{task.fields['average_mib_s']:.2f} MiB/s", style="progress.data.speed")
 
 
 def StyledProgress() -> CustomProgress: