From c71a250ebb75e64f92d3b0c0f4ea63f1abf03161 Mon Sep 17 00:00:00 2001 From: Ean Garvey <87458719+monorimet@users.noreply.github.com> Date: Tue, 7 Jan 2025 11:25:43 -0600 Subject: [PATCH] (shortfin-sd) Interleave workers and their fibers by device. (#587) This enables proper "filling" of multi-device topologies that populate each device evenly, rather than block allocating work to one device at a time. Co-authored-by: Ean Garvey --- shortfin/python/shortfin_apps/sd/components/service.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/shortfin/python/shortfin_apps/sd/components/service.py b/shortfin/python/shortfin_apps/sd/components/service.py index 9b09632a6..814916bee 100644 --- a/shortfin/python/shortfin_apps/sd/components/service.py +++ b/shortfin/python/shortfin_apps/sd/components/service.py @@ -76,8 +76,10 @@ def __init__( self.workers = [] self.fibers = [] self.idle_fibers = set() - for idx, device in enumerate(self.sysman.ls.devices): - for i in range(self.workers_per_device): + # For each worker index we create one on each device, and add their fibers to the idle set. + # This roughly ensures that the first picked fibers are distributed across available devices. + for i in range(self.workers_per_device): + for idx, device in enumerate(self.sysman.ls.devices): worker = sysman.ls.create_worker(f"{name}-inference-{device.name}-{i}") self.workers.append(worker) for idx, device in enumerate(self.sysman.ls.devices):