Save device name stat prior to running stages (#138)

examples/cli/plugins/example_combined/turnkeyml_plugin_example_combined/runtime.py

@@ -28,7 +28,7 @@ def __init__(
         inputs=None,
         delay_before_benchmarking: str = "0",
     ):
-        # Custom runtime args always arive as strings, so we need to convert them
+        # Custom runtime args always arrive as strings, so we need to convert them
         # to the appropriate data type here
         self.delay_before_benchmarking = int(delay_before_benchmarking)
 
@@ -86,7 +86,7 @@ def benchmark(self):
         return MeasuredPerformance(
             mean_latency=self.mean_latency,
             throughput=self.throughput,
-            device=self.device_name,
+            device=self.device_name(),
             device_type=self.device_type,
             runtime=self.runtime,
             runtime_version=self.runtime_version,
@@ -111,9 +111,6 @@ def throughput(self) -> float:
                 "Queried throughput before self.benchmark() was called"
             )
 
-    @property
-    def device_name(self) -> str:
-        return (
-            f"Device Family {self.device_type.family}, Device Part {self.device_type.part}, "
-            f"Device Configuration {self.device_type.config}"
-        )
+    @staticmethod
+    def device_name() -> str:
+        return "Example Device"

examples/cli/plugins/example_rt/turnkeyml_plugin_example_rt/runtime.py

@@ -57,7 +57,7 @@ def benchmark(self) -> MeasuredPerformance:
         return MeasuredPerformance(
             mean_latency=self.mean_latency,
             throughput=self.throughput,
-            device=self.device_name,
+            device=self.device_name(),
             device_type=self.device_type,
             runtime=self.runtime,
             runtime_version=self.runtime_version,
@@ -82,6 +82,6 @@ def throughput(self) -> float:
                 "Queried throughput before self.benchmark() was called"
             )
 
-    @property
-    def device_name(self) -> str:
+    @staticmethod
+    def device_name() -> str:
         return "the x86 cpu of your dreams"

src/turnkeyml/analyze/script.py

@@ -371,6 +371,11 @@ def explore_invocation(
             fs.Keys.BENCHMARK_STATUS, build.FunctionStatus.NOT_STARTED.value
         )
 
+        # Save the device name that will be used for the benchmark
+        stats.save_model_eval_stat(
+            fs.Keys.DEVICE, runtime_info["RuntimeClass"].device_name()
+        )
+
     build_state = None
     perf = None
     benchmark_logfile_path = ""
@@ -667,7 +672,7 @@ def explore_frame(
 
             # Starting in version 2.2.0, torch dynamo added wrappers to callbacks
             # while tracing frames, which conflicts with TurnkeML's analysis. Here,
-            # we supress errors caused by those callback wrappers and only raise an
+            # we suppress errors caused by those callback wrappers and only raise an
             # error if the compiled model actually tries to execute within TurnkeyML.
             td = torch._dynamo  # pylint: disable=protected-access
             td.config.suppress_errors = True
@@ -800,7 +805,7 @@ def forward_spy(*args, **kwargs):
                 and invocation_info.is_target
                 and (model_info.build_model)
             ):
-                # Disable all modifications while we evalute the model
+                # Disable all modifications while we evaluate the model
                 # This is needed in case a tool called during evaluation wants to
                 # trace the model. There are some scenarios (e.g., ipex.quantization.prepare),
                 # that raise an exception when they encounter forward_spy()

src/turnkeyml/common/filesystem.py

@@ -332,8 +332,10 @@ class Keys:
     PERFORMANCE = "performance"
     # Runtime used for the benchmark
     RUNTIME = "runtime"
-    # Device used for the benchmark
+    # Type of device used for the benchmark (e.g., "x86")
     DEVICE_TYPE = "device_type"
+    # Specific device used for the benchmark
+    DEVICE = "device"
     # Name of the model
     MODEL_NAME = "model_name"
     # References the per-evaluation stats section

src/turnkeyml/run/basert.py

@@ -228,7 +228,7 @@ def benchmark(self) -> MeasuredPerformance:
         return MeasuredPerformance(
             mean_latency=self.mean_latency,
             throughput=self.throughput,
-            device=self.device_name,
+            device=self.device_name(),
             device_type=self.device_type,
             runtime=self.runtime,
             runtime_version=self.runtime_version,
@@ -260,9 +260,9 @@ def throughput(self) -> float:
         Returns the throughput, in IPS, for the benchmarking run.
         """
 
-    @property
+    @staticmethod
     @abstractmethod
-    def device_name(self) -> str:
+    def device_name() -> str:
         """
         Returns the full device name for the device used in benchmarking.
         For example, a benchmark on a `x86` device might have a device name like

src/turnkeyml/run/onnxrt/runtime.py

@@ -5,7 +5,11 @@
 import turnkeyml.common.exceptions as exp
 from turnkeyml.run.onnxrt.execute import ORT_VERSION
 from turnkeyml.common.filesystem import Stats
-from turnkeyml.run.onnxrt.execute import create_conda_env, execute_benchmark
+from turnkeyml.run.onnxrt.execute import (
+    create_conda_env,
+    execute_benchmark,
+    get_cpu_specs,
+)
 import turnkeyml.run.plugin_helpers as plugin_helpers
 
 
@@ -95,6 +99,6 @@ def mean_latency(self):
     def throughput(self):
         return float(self._get_stat("Throughput"))
 
-    @property
-    def device_name(self):
-        return self._get_stat("CPU Name")
+    @staticmethod
+    def device_name() -> str:
+        return get_cpu_specs()["CPU Name"]

src/turnkeyml/run/tensorrt/runtime.py

@@ -59,18 +59,24 @@ def __init__(
             requires_docker=True,
         )
 
+        self.device_name = self._dynamic_device_name
+
     def _setup(self) -> None:
         # Check if at least one NVIDIA GPU is available locally
-        result = subprocess.run(
-            ["nvidia-smi"],
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            encoding="utf-8",
-            check=False,
-        )
-
-        if "NVIDIA" not in result.stdout or result.returncode == 1:
-            msg = "No NVIDIA GPUs available on the local machine"
+        msg = "No NVIDIA GPUs available on the local machine"
+        try:
+            result = subprocess.run(
+                ["nvidia-smi"],
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                encoding="utf-8",
+                check=False,
+            )
+
+            if "NVIDIA" not in result.stdout or result.returncode == 1:
+
+                raise exp.BenchmarkException(msg)
+        except FileNotFoundError:
             raise exp.BenchmarkException(msg)
 
     def _execute(
@@ -143,6 +149,14 @@ def mean_latency(self):
     def throughput(self):
         return float(self._get_stat("Throughput").split(" ")[0])
 
-    @property
-    def device_name(self):
+    def _dynamic_device_name(self):
+        # Return the specific Nvidia GPU, which is only known
+        # after we invoke the TensorRT docker
         return self._get_stat("Selected Device")
+
+    @staticmethod
+    def device_name():
+        # Return a generic response, since we haven't invoked
+        # the docker yet. At instantiation time, this method
+        # is replaced by the dynamic version.
+        return "Nvidia GPU"

src/turnkeyml/run/torchrt/runtime.py

@@ -124,7 +124,7 @@ def _calculate_performance(
         return MeasuredPerformance(
             mean_latency=self.mean_latency,
             throughput=self.throughput,
-            device=self.device_name,
+            device=self.device_name(),
             device_type=self.device_type,
             runtime=self.runtime,
             runtime_version=self.runtime_version,
@@ -218,6 +218,6 @@ def throughput(self) -> float:
                 "Queried throughput before self.benchmark() was called"
             )
 
-    @property
-    def device_name(self) -> str:
+    @staticmethod
+    def device_name() -> str:
         return get_cpu_specs()["CPU Name"]

src/turnkeyml/version.py

@@ -1 +1 @@
-__version__ = "1.1.3"
+__version__ = "2.0.0"