Fix L0_multi_gpu

triton-inference-server · Sep 20, 2024 · 8fbc234 · 8fbc234
1 parent 46dcfad
commit 8fbc234
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/src/model.py b/src/model.py
@@ -161,6 +161,7 @@ def init_engine(self):
         self.llm_engine = AsyncLLMEngine.from_engine_args(aync_engine_args)
 
         # Create vLLM custom metrics
+        self.vllm_metrics = None
         if (
             "REPORT_CUSTOM_METRICS" in self.model_config["parameters"]
             and self.model_config["parameters"]["REPORT_CUSTOM_METRICS"]["string_value"]
@@ -574,7 +575,8 @@ def finalize(self):
             self._response_thread = None
 
         # Shutdown the logger thread.
-        self.vllm_metrics.finalize()
+        if self.vllm_metrics is not None:
+            self.vllm_metrics.finalize()
 
         # When using parallel tensors, the stub process may not shutdown due to
         # unreleased references, so manually run the garbage collector once.