From ab7436e533a2a479314986c2a8208e6db9161a30 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Sat, 12 Oct 2024 16:44:29 +0200 Subject: [PATCH 01/15] Initial commit for osx vm setup --- .gitignore | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.gitignore b/.gitignore index f7d251f6..ea8aa17d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,8 @@ +.vagrant +macos-sonoma.conf +macos-sonoma/ +Vagrantfile +Session.vim .idea .venv */__pycache__/* From 5d19a207e635c46a8d58e2bdb2d27ba276a0286a Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Fri, 25 Oct 2024 17:24:00 +0200 Subject: [PATCH 02/15] Skeleton implementation of powermetrics --- examples/powermetrics-profiling/README.md | 0 .../powermetrics-profiling/RunnerConfig.py | 101 ++++++++++++++++++ .../Plugins/Profilers/PowerMetrics.py | 89 +++++++++++++++ 3 files changed, 190 insertions(+) create mode 100644 examples/powermetrics-profiling/README.md create mode 100644 examples/powermetrics-profiling/RunnerConfig.py create mode 100644 experiment-runner/Plugins/Profilers/PowerMetrics.py diff --git a/examples/powermetrics-profiling/README.md b/examples/powermetrics-profiling/README.md new file mode 100644 index 00000000..e69de29b diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py new file mode 100644 index 00000000..638049f8 --- /dev/null +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -0,0 +1,101 @@ +from EventManager.Models.RunnerEvents import RunnerEvents +from EventManager.EventSubscriptionController import EventSubscriptionController +from ConfigValidator.Config.Models.RunTableModel import RunTableModel +from ConfigValidator.Config.Models.FactorModel import FactorModel +from ConfigValidator.Config.Models.RunnerContext import RunnerContext +from ConfigValidator.Config.Models.OperationType import OperationType +from ProgressManager.Output.OutputProcedure import OutputProcedure as output + +from typing import Dict, List, Any, Optional +from pathlib import Path +from os.path import dirname, realpath + +class RunnerConfig: + ROOT_DIR = Path(dirname(realpath(__file__))) + + # ================================ USER SPECIFIC CONFIG ================================ + """The name of the experiment.""" + name: str = "new_runner_experiment" + + """The path in which Experiment Runner will create a folder with the name `self.name`, in order to store the + results from this experiment. (Path does not need to exist - it will be created if necessary.) + Output path defaults to the config file's path, inside the folder 'experiments'""" + results_output_path: Path = ROOT_DIR / 'experiments' + + """Experiment operation type. Unless you manually want to initiate each run, use `OperationType.AUTO`.""" + operation_type: OperationType = OperationType.AUTO + + """The time Experiment Runner will wait after a run completes. + This can be essential to accommodate for cooldown periods on some systems.""" + time_between_runs_in_ms: int = 1000 + + # Dynamic configurations can be one-time satisfied here before the program takes the config as-is + # e.g. Setting some variable based on some criteria + def __init__(self): + """Executes immediately after program start, on config load""" + + EventSubscriptionController.subscribe_to_multiple_events([ + (RunnerEvents.BEFORE_EXPERIMENT, self.before_experiment), + (RunnerEvents.BEFORE_RUN , self.before_run ), + (RunnerEvents.START_RUN , self.start_run ), + (RunnerEvents.START_MEASUREMENT, self.start_measurement), + (RunnerEvents.INTERACT , self.interact ), + (RunnerEvents.STOP_MEASUREMENT , self.stop_measurement ), + (RunnerEvents.STOP_RUN , self.stop_run ), + (RunnerEvents.POPULATE_RUN_DATA, self.populate_run_data), + (RunnerEvents.AFTER_EXPERIMENT , self.after_experiment ) + ]) + self.run_table_model = None # Initialized later + output.console_log("Custom config loaded") + + def create_run_table_model(self) -> RunTableModel: + """Create and return the run_table model here. A run_table is a List (rows) of tuples (columns), + representing each run performed""" + pass + + def before_experiment(self) -> None: + """Perform any activity required before starting the experiment here + Invoked only once during the lifetime of the program.""" + pass + + def before_run(self) -> None: + """Perform any activity required before starting a run. + No context is available here as the run is not yet active (BEFORE RUN)""" + pass + + def start_run(self, context: RunnerContext) -> None: + """Perform any activity required for starting the run here. + For example, starting the target system to measure. + Activities after starting the run should also be performed here.""" + pass + + def start_measurement(self, context: RunnerContext) -> None: + """Perform any activity required for starting measurements.""" + pass + + def interact(self, context: RunnerContext) -> None: + """Perform any interaction with the running target system here, or block here until the target finishes.""" + pass + + def stop_measurement(self, context: RunnerContext) -> None: + """Perform any activity here required for stopping measurements.""" + pass + + def stop_run(self, context: RunnerContext) -> None: + """Perform any activity here required for stopping the run. + Activities after stopping the run should also be performed here.""" + pass + + def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: + """Parse and process any measurement data here. + You can also store the raw measurement data under `context.run_dir` + Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" + pass + + def after_experiment(self) -> None: + """Perform any activity required after stopping the experiment here + Invoked only once during the lifetime of the program.""" + pass + + # ================================ DO NOT ALTER BELOW THIS LINE ================================ + experiment_path: Path = None diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py new file mode 100644 index 00000000..5ae83d5c --- /dev/null +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -0,0 +1,89 @@ +from __future__ import annotations +import enum +from collections.abc import Callable +from pathlib import Path + +# How to format the output +class PMFormatTypes(enum.Enum): + PM_FMT_TEXT = "text" + PM_FMT_PLIST = "plist" + +# How to order results +class PMOrderTypes(enum.Enum): + PM_ORDER_PID = "pid" + PM_ORDER_WAKEUP = "wakeups" + PM_ORDER_CPU = "cputime" + PM_ORDER_HYBRID = "composite" + +# Different categories of stats available +class PMStatTypes(enum.Enum): + PM_STAT_BATTERY = 0 # Battery statistics + PM_STAT_INTERRUPT = 1 # Interrupt distribution + PM_STAT_POWER = 2 # CPU energy usage + PM_STAT_IO = 3 # Disc / Network + PM_STAT_BACKLIGHT = 4 # Backlight usage (not portable) + +class PowerMetrics(object): + """An integration of OSX powermetrics into experiment-runner as a data source plugin""" + def __init__(self, sample_frequency: int = 5000, + out_file: Path = None, sample_count: int = None, + order: PMOrderTypes = None, format: PMFormatTypes = None, + poweravg: int = None, wakeup_cost: int = None, + buffer_size: int = None, hide_power: bool = False, + hide_cpu: bool = False, hide_gpu: bool = False, + show_initial: bool = False, show_summary: bool = False): + + self.pm_process = None + self.logfile = "test" + + # All paramters we can pass to powermetrics + self.parameters = { + "--output-file": self.logfile, + "--sample-interval": sample_frequency, # default + "--sample-count": 0, # 0 for inifinite + "--order": PMOrderTypes.PM_ORDER_CPU, + "--format": PMFormatTypes.PM_FMT_PLIST, + "--poweravg": 10, # default + #"--wakeup-cost": 10, + #"--buffer-size": 0, + "--hide-platform-power": None, + "--hide-cpu-duty-cycle": None, + "--hide-gpu-duty-cycle": None, + "--show-initial-usage": None, + "--show-usage-summary": None + } + + # Ensure that powermetrics is not currently running when we delete this object + def __del__(self): + if self.pm_process: + pass + + # Check that we are running on OSX, and that the powermetrics command exists + def validate_platform(self): + pass + + # Apply channel and mains settings to the picolog + def update_parameters(self, **paramters): + pass + + # Log data from powermetrics to a logfile + def log(self, logfile = None, dev = None, timeout: int = 60, finished_fn: Callable[[], bool] = None): + log_data = {} + if logfile: + self.logfile = logfile + + print('Logging...') + + # Write all of the data to a log file (if requested) + if self.logfile: + with open(self.logfile,'w') as f: + pass + + return log_data + + def print_config(self, handle): + pass + + @staticmethod + def parse_log(logfile): + pass From 4b55d540919868efa15d214679eef4a9b8d8f6c6 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Sat, 26 Oct 2024 18:05:55 +0200 Subject: [PATCH 03/15] updated parameters list, start of process handling and updated parameters --- .../powermetrics-profiling/RunnerConfig.py | 38 ++++++++-- .../Plugins/Profilers/PowerMetrics.py | 73 ++++++++++++------- 2 files changed, 78 insertions(+), 33 deletions(-) diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py index 638049f8..474bb5c1 100644 --- a/examples/powermetrics-profiling/RunnerConfig.py +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -5,10 +5,12 @@ from ConfigValidator.Config.Models.RunnerContext import RunnerContext from ConfigValidator.Config.Models.OperationType import OperationType from ProgressManager.Output.OutputProcedure import OutputProcedure as output +from Plugins.Profilers.PowerMetrics import PowerMetrics from typing import Dict, List, Any, Optional from pathlib import Path from os.path import dirname, realpath +import time class RunnerConfig: ROOT_DIR = Path(dirname(realpath(__file__))) @@ -51,12 +53,21 @@ def __init__(self): def create_run_table_model(self) -> RunTableModel: """Create and return the run_table model here. A run_table is a List (rows) of tuples (columns), representing each run performed""" - pass + + # Create the experiment run table with factors, and desired data columns + factor1 = FactorModel("test_factor", [1, 2]) + self.run_table_model = RunTableModel( + factors = [factor1], + data_columns=["poweravg", "wakeups", "cpu-usage"]) + + return self.run_table_model def before_experiment(self) -> None: """Perform any activity required before starting the experiment here Invoked only once during the lifetime of the program.""" - pass + + # Create the powermetrics object we will use to collect data + self.meter = PowerMetrics() def before_run(self) -> None: """Perform any activity required before starting a run. @@ -67,19 +78,27 @@ def start_run(self, context: RunnerContext) -> None: """Perform any activity required for starting the run here. For example, starting the target system to measure. Activities after starting the run should also be performed here.""" - pass + + # Optionally change powermetrics parameters between runs (if needed) + #self.meter.update_parameters() def start_measurement(self, context: RunnerContext) -> None: """Perform any activity required for starting measurements.""" - pass + + # Start measuring useing powermetrics (write to log file) + self.meter.start_pm() def interact(self, context: RunnerContext) -> None: """Perform any interaction with the running target system here, or block here until the target finishes.""" - pass + + # Wait (block) for a bit to collect some data + time.sleep(20) def stop_measurement(self, context: RunnerContext) -> None: """Perform any activity here required for stopping measurements.""" - pass + + # Stop measuring at the end of a run + self.meter.stop_pm() def stop_run(self, context: RunnerContext) -> None: """Perform any activity here required for stopping the run. @@ -90,7 +109,12 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: """Parse and process any measurement data here. You can also store the raw measurement data under `context.run_dir` Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" - pass + + # Retrieve data from run + run_results = self.meter.parse_logs() + + # Parse it as required for your experiment and add it to the run table + def after_experiment(self) -> None: """Perform any activity required after stopping the experiment here diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 5ae83d5c..5d87f378 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -2,6 +2,8 @@ import enum from collections.abc import Callable from pathlib import Path +import subprocess +import shlex # How to format the output class PMFormatTypes(enum.Enum): @@ -15,6 +17,23 @@ class PMOrderTypes(enum.Enum): PM_ORDER_CPU = "cputime" PM_ORDER_HYBRID = "composite" +# Which sources to sample from +class PMSampleTypes(enum.Enum): + PM_SAMPLE_TASKS = "tasks" # per task cpu usage and wakeup stats + PM_SAMPLE_BAT = "battery" # battery and backlight info + PM_SAMPLE_NET = "network" # network usage info + PM_SAMPLE_DISK = "disk" # disk usage info + PM_SAMPLE_INT = "interrupts" # interrupt distribution + PM_SAMPLE_CPU_POWER = "cpu_power" # c-state residency, power and frequency info + PM_SAMPLE_TEMP = "thermal" # thermal pressure notifications + PM_SAMPLE_SFI = "sfi" # selective forced idle information + PM_SAMPLE_GPU_POWER = "gpu_power" # gpu c-state residency, p-state residency and frequency info + PM_SAMPLE_AGPM = "gpu_agpm_stats" # Statistics reported by AGPM + PM_SAMPLE_SMC = "smc" # SMC sensors + PM_SAMPLE_DCC = "gpu_dcc_stats" # gpu duty cycle info + PM_SAMPLE_NVME = "nvme_ssd" # NVMe power state information + LM_SAMPLE_THROTTLE = "io_throttle_ssd" # IO Throttling information + # Different categories of stats available class PMStatTypes(enum.Enum): PM_STAT_BATTERY = 0 # Battery statistics @@ -25,12 +44,9 @@ class PMStatTypes(enum.Enum): class PowerMetrics(object): """An integration of OSX powermetrics into experiment-runner as a data source plugin""" - def __init__(self, sample_frequency: int = 5000, + def __init__(self, sample_frequency: int = 5000, out_file: Path = None, sample_count: int = None, - order: PMOrderTypes = None, format: PMFormatTypes = None, - poweravg: int = None, wakeup_cost: int = None, - buffer_size: int = None, hide_power: bool = False, - hide_cpu: bool = False, hide_gpu: bool = False, + order: PMOrderTypes = None, poweravg: int = None, wakeup_cost: int = None, show_initial: bool = False, show_summary: bool = False): self.pm_process = None @@ -41,16 +57,16 @@ def __init__(self, sample_frequency: int = 5000, "--output-file": self.logfile, "--sample-interval": sample_frequency, # default "--sample-count": 0, # 0 for inifinite - "--order": PMOrderTypes.PM_ORDER_CPU, - "--format": PMFormatTypes.PM_FMT_PLIST, + "--order": PMOrderTypes.PM_ORDER_CPU.value, + "--format": PMFormatTypes.PM_FMT_PLIST.value, "--poweravg": 10, # default #"--wakeup-cost": 10, #"--buffer-size": 0, - "--hide-platform-power": None, - "--hide-cpu-duty-cycle": None, - "--hide-gpu-duty-cycle": None, - "--show-initial-usage": None, - "--show-usage-summary": None + "--hide-platform-power": "", + "--hide-cpu-duty-cycle": "", + "--hide-gpu-duty-cycle": "", + "--show-initial-usage": "", + "--show-usage-summary": "" } # Ensure that powermetrics is not currently running when we delete this object @@ -66,24 +82,29 @@ def validate_platform(self): def update_parameters(self, **paramters): pass - # Log data from powermetrics to a logfile - def log(self, logfile = None, dev = None, timeout: int = 60, finished_fn: Callable[[], bool] = None): - log_data = {} - if logfile: - self.logfile = logfile + def start_pm(self): + cmd = " ".join([f"{key} {value}" for key, value in self.parameters.items()]) - print('Logging...') - - # Write all of the data to a log file (if requested) - if self.logfile: - with open(self.logfile,'w') as f: - pass + try: + self.pm_process = subprocess.Popen(["powermetrics", *shlex.split(cmd)], stdout=subprocess.PIPE) + except Exception as e: + print(e) + + def stop_pm(self): + if not self.pm_process: + return + + try: + self.pm_process.terminate() + stdout, stderr = self.pm_process.communicate() + + print(stdout) + except Exception as e: + print(e) - return log_data - def print_config(self, handle): pass @staticmethod - def parse_log(logfile): + def parse_logs(logfile, stats: list): pass From eafb7ade4bc572babefd626c18ba17f2c0ac44cb Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Wed, 6 Nov 2024 18:25:18 +0100 Subject: [PATCH 04/15] Some output parsing, and parameter usage redesign --- .gitignore | 2 + .../powermetrics-profiling/RunnerConfig.py | 2 +- .../Plugins/Profilers/PowerMetrics.py | 71 ++++++++++--------- 3 files changed, 42 insertions(+), 33 deletions(-) diff --git a/.gitignore b/.gitignore index ea8aa17d..23f8e9e2 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,5 @@ tmp/ _trial_temp/ pycharm-interpreter.sh python + +scratch.py diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py index 474bb5c1..0fc5b8e3 100644 --- a/examples/powermetrics-profiling/RunnerConfig.py +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -111,7 +111,7 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" # Retrieve data from run - run_results = self.meter.parse_logs() + run_results = self.meter.parse_logs("../../../powermetrics_outs/plist_power.txt") # Parse it as required for your experiment and add it to the run table diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 5d87f378..1420adc5 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -4,6 +4,7 @@ from pathlib import Path import subprocess import shlex +import plistlib # How to format the output class PMFormatTypes(enum.Enum): @@ -32,54 +33,44 @@ class PMSampleTypes(enum.Enum): PM_SAMPLE_SMC = "smc" # SMC sensors PM_SAMPLE_DCC = "gpu_dcc_stats" # gpu duty cycle info PM_SAMPLE_NVME = "nvme_ssd" # NVMe power state information - LM_SAMPLE_THROTTLE = "io_throttle_ssd" # IO Throttling information - -# Different categories of stats available -class PMStatTypes(enum.Enum): - PM_STAT_BATTERY = 0 # Battery statistics - PM_STAT_INTERRUPT = 1 # Interrupt distribution - PM_STAT_POWER = 2 # CPU energy usage - PM_STAT_IO = 3 # Disc / Network - PM_STAT_BACKLIGHT = 4 # Backlight usage (not portable) + PM_SAMPLE_THROTTLE = "io_throttle_ssd" # IO Throttling information class PowerMetrics(object): """An integration of OSX powermetrics into experiment-runner as a data source plugin""" - def __init__(self, sample_frequency: int = 5000, - out_file: Path = None, sample_count: int = None, - order: PMOrderTypes = None, poweravg: int = None, wakeup_cost: int = None, - show_initial: bool = False, show_summary: bool = False): + def __init__(self, sample_frequency: int = 5000, + out_file: Path = None, poweravg: int = None, + show_summary: bool = False, samplers: list[PMSampleTypes] = None, + additional_args: list[str] = None): self.pm_process = None self.logfile = "test" - - # All paramters we can pass to powermetrics - self.parameters = { + + # Grab all available power stats by default + self.default_parameters = { "--output-file": self.logfile, - "--sample-interval": sample_frequency, # default - "--sample-count": 0, # 0 for inifinite - "--order": PMOrderTypes.PM_ORDER_CPU.value, + "--sample-interval": sample_frequency, "--format": PMFormatTypes.PM_FMT_PLIST.value, - "--poweravg": 10, # default - #"--wakeup-cost": 10, - #"--buffer-size": 0, - "--hide-platform-power": "", - "--hide-cpu-duty-cycle": "", - "--hide-gpu-duty-cycle": "", - "--show-initial-usage": "", - "--show-usage-summary": "" + "--samplers": [PMSampleTypes.PM_SAMPLE_CPU_POWER.value, + PMSampleTypes.PM_SAMPLE_GPU_POWER.value, + PMSampleTypes.PM_SAMPLE_AGPM.value], + "--hide-cpu-duty-cycle": "" } # Ensure that powermetrics is not currently running when we delete this object def __del__(self): if self.pm_process: - pass + self.pm_process.terminate() # Check that we are running on OSX, and that the powermetrics command exists def validate_platform(self): pass - # Apply channel and mains settings to the picolog - def update_parameters(self, **paramters): + # Set the parameters used for power metrics to a new set + def update_parameters(self, new_params: dict): + for p, v in new_params.items(): + self.default_parameters[p] = v + + def format_cmd(self): pass def start_pm(self): @@ -101,10 +92,26 @@ def stop_pm(self): print(stdout) except Exception as e: print(e) + + @staticmethod + def get_stat_types(logfile): + pass - def print_config(self, handle): + @staticmethod + def get_power(logfile): pass @staticmethod def parse_logs(logfile, stats: list): - pass + fp = open(logfile, "rb") + + plists = [] + cur_plist = bytearray() + for l in fp.readlines(): + if l[0] == 0: + plists.append(plistlib.loads(cur_plist)) + + cur_plist = bytearray() + cur_plist.extend(l[1:]) + else: + cur_plist.extend(l) From e25dd9b8843d966796b9c17e945a8549b4913173 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Thu, 7 Nov 2024 18:05:46 +0100 Subject: [PATCH 05/15] More parsing and some documentation --- .../Plugins/Profilers/PowerMetrics.py | 111 ++++++++++++++---- 1 file changed, 89 insertions(+), 22 deletions(-) diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 1420adc5..590b0b5e 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -37,14 +37,14 @@ class PMSampleTypes(enum.Enum): class PowerMetrics(object): """An integration of OSX powermetrics into experiment-runner as a data source plugin""" - def __init__(self, sample_frequency: int = 5000, - out_file: Path = None, poweravg: int = None, - show_summary: bool = False, samplers: list[PMSampleTypes] = None, - additional_args: list[str] = None): + def __init__(self, sample_frequency: int = 5000, + out_file: Path = "pm_out.plist", samplers: list[PMSampleTypes] = [], + additional_args: list[str] = None, hide_cpu_duty_cycle: bool = True): self.pm_process = None self.logfile = "test" + self.additional_args = additional_args # Grab all available power stats by default self.default_parameters = { "--output-file": self.logfile, @@ -52,8 +52,8 @@ def __init__(self, sample_frequency: int = "--format": PMFormatTypes.PM_FMT_PLIST.value, "--samplers": [PMSampleTypes.PM_SAMPLE_CPU_POWER.value, PMSampleTypes.PM_SAMPLE_GPU_POWER.value, - PMSampleTypes.PM_SAMPLE_AGPM.value], - "--hide-cpu-duty-cycle": "" + PMSampleTypes.PM_SAMPLE_AGPM.value] + samplers, + "--hide-cpu-duty-cycle": hide_cpu_duty_cycle } # Ensure that powermetrics is not currently running when we delete this object @@ -62,26 +62,32 @@ def __del__(self): self.pm_process.terminate() # Check that we are running on OSX, and that the powermetrics command exists - def validate_platform(self): + def __validate_platform(self): pass - # Set the parameters used for power metrics to a new set - def update_parameters(self, new_params: dict): - for p, v in new_params.items(): - self.default_parameters[p] = v - - def format_cmd(self): - pass + def __format_cmd(self): + cmd = "" + + return cmd def start_pm(self): - cmd = " ".join([f"{key} {value}" for key, value in self.parameters.items()]) + """ + Starts the powermetrics process, with the parameters in default_parameters + additional_args. + """ + cmd = self.__format_cmd() try: self.pm_process = subprocess.Popen(["powermetrics", *shlex.split(cmd)], stdout=subprocess.PIPE) except Exception as e: - print(e) + print(e) def stop_pm(self): + """ + Terminates the powermetrics process, as it was running indefinetly. This method collects the stdout and stderr + + Returns: + stdout, stderr of the powermetrics process. + """ if not self.pm_process: return @@ -90,19 +96,78 @@ def stop_pm(self): stdout, stderr = self.pm_process.communicate() print(stdout) + return stdout, stderr except Exception as e: print(e) - @staticmethod - def get_stat_types(logfile): - pass + # Set the parameters used for power metrics to a new set + def update_parameters(self, new_params: dict): + """ + Updates the list of parameters, to be in line with new_params. + Note that samplers will be set to the new list if present, make sure + to include the previous set if you still want to use them. + + Parameters: + new_params (dict): A dictionary containing the new list of parameters. For + parameters with no value (like --hide-cpu-duty-cycle) use a boolean to indicate + if they can be used. + Returns: + The new list of parameters, can also be valided with self.default_parameters + """ + for p, v in new_params.items(): + self.default_parameters[p] = v + @staticmethod - def get_power(logfile): - pass + def get_plist_power(pm_plists: list[dict]): + """ + Extracts from a list of plists, the relavent power statistics if present. If no + power stats are present, this returns an empty list. This is mainly a helper method, + to make the plists easier to work with. + + Parameters: + pm_plists (list[dict]): The list of plists created by parse_pm_plist + + Returns: + A list of dicts, each containing a subset of the available stats related to power. + """ + power_plists = [] + + for plist in pm_plists: + stats = {} + if "GPU" in plist.keys(): + stats["GPU"] = plist["GPU"].copy() + del stats["GPU"]["misc_counters"] + del stats["GPU"]["pstates"] + + if "processor" in plist.keys(): + stats["processor"] = plist["processor"] + del stats["processor"]["packages"] + if "agpm_stats" in plist.keys(): + stats["agpm_stats"] = plist["agpm_stats"] + + if "timestamp" in plist.keys(): + stats["timestamp"] = plist["timestamp"] + + power_plists.apend(stats) + + return power_plists + @staticmethod - def parse_logs(logfile, stats: list): + def parse_pm_plist(logfile: Path): + """ + Parses a provided logfile from powermetrics in plist format. Powermetrics outputs a plist + for every sample taken, it included a newline after the closing <\plist>, we account for that here + to make things easier to parse. + + Parameters: + logfile (Path): The path to the plist logfile created by powermetrics + + Returns: + A list of dicts, each representing the plist for a given sample + """ + fp = open(logfile, "rb") plists = [] @@ -115,3 +180,5 @@ def parse_logs(logfile, stats: list): cur_plist.extend(l[1:]) else: cur_plist.extend(l) + + return plists From 231195ec756a5703b9d79e88bd0174fc55ab45d1 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Sun, 10 Nov 2024 18:24:28 +0100 Subject: [PATCH 06/15] Parameter update/varification, and error handling --- .../Plugins/Profilers/PowerMetrics.py | 96 +++++++++++++++---- 1 file changed, 75 insertions(+), 21 deletions(-) diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 590b0b5e..8b4a5a0f 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -3,8 +3,9 @@ from collections.abc import Callable from pathlib import Path import subprocess -import shlex import plistlib +import platform +import shutil # How to format the output class PMFormatTypes(enum.Enum): @@ -37,12 +38,19 @@ class PMSampleTypes(enum.Enum): class PowerMetrics(object): """An integration of OSX powermetrics into experiment-runner as a data source plugin""" - def __init__(self, sample_frequency: int = 5000, - out_file: Path = "pm_out.plist", samplers: list[PMSampleTypes] = [], - additional_args: list[str] = None, hide_cpu_duty_cycle: bool = True): + def __init__(self, + sample_frequency: int = 5000, + out_file: Path = "pm_out.plist", + additional_samplers: list[PMSampleTypes] = [], + additional_args: list[str] = [], + hide_cpu_duty_cycle: bool = True, + order: PMOrderTypes = PMOrderTypes.PM_ORDER_CPU): + # Double check we have the required software for this plugin + self.__validate_platform() + self.pm_process = None - self.logfile = "test" + self.logfile = out_file self.additional_args = additional_args # Grab all available power stats by default @@ -50,10 +58,11 @@ def __init__(self, sample_frequency: int "--output-file": self.logfile, "--sample-interval": sample_frequency, "--format": PMFormatTypes.PM_FMT_PLIST.value, - "--samplers": [PMSampleTypes.PM_SAMPLE_CPU_POWER.value, - PMSampleTypes.PM_SAMPLE_GPU_POWER.value, - PMSampleTypes.PM_SAMPLE_AGPM.value] + samplers, - "--hide-cpu-duty-cycle": hide_cpu_duty_cycle + "--samplers": [PMSampleTypes.PM_SAMPLE_CPU_POWER, + PMSampleTypes.PM_SAMPLE_GPU_POWER, + PMSampleTypes.PM_SAMPLE_AGPM] + additional_samplers, + "--hide-cpu-duty-cycle": hide_cpu_duty_cycle, + "--order": order.value } # Ensure that powermetrics is not currently running when we delete this object @@ -63,23 +72,50 @@ def __del__(self): # Check that we are running on OSX, and that the powermetrics command exists def __validate_platform(self): - pass - + if "OSX" not in platform.system(): + raise RuntimeError("The OSX platform is required for this plugin") + + if shutil.which("powermetrics") is None: + raise RuntimeError("The powermetrics tool is required for this plugin") + def __format_cmd(self): - cmd = "" - - return cmd + cmd = ["powermetrics"] + + # Add in the default parameters + for p, v in self.default_parameters.items(): + if v is False: + continue + + # Add the parameter + cmd.append(p) + + # Add the value + if "samplers" in p and isinstance(v, list): + cmd.append(",".join([x.value for x in v])) + elif not isinstance(v, bool): + cmd.append(str(v)) + + return cmd + self.additional_args def start_pm(self): """ Starts the powermetrics process, with the parameters in default_parameters + additional_args. """ - cmd = self.__format_cmd() - try: - self.pm_process = subprocess.Popen(["powermetrics", *shlex.split(cmd)], stdout=subprocess.PIPE) + self.pm_process = subprocess.Popen(self.__format_cmd(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = subprocess.communicate() + + if stderr: + self.pm_process.terminate() + self.pm_process = None + raise RuntimeError(f"Powermetrics encountered an error while starting: {stderr}") + except Exception as e: - print(e) + print(f"Could not start powermetrics: {e}") + raise RuntimeError("Powermetrics plugin could not start") def stop_pm(self): """ @@ -94,11 +130,14 @@ def stop_pm(self): try: self.pm_process.terminate() stdout, stderr = self.pm_process.communicate() + + if stderr: + raise RuntimeError("powermetrics encountered an error during measurement") - print(stdout) return stdout, stderr except Exception as e: - print(e) + print(f"Could not stop powermetrics: {e}") + raise RuntimeError("Powermetrics plugin could not stop") # Set the parameters used for power metrics to a new set def update_parameters(self, new_params: dict): @@ -116,6 +155,22 @@ def update_parameters(self, new_params: dict): The new list of parameters, can also be valided with self.default_parameters """ for p, v in new_params.items(): + # Double check new_params where possible + if "samplers" in p: + assert(isinstance(v, list)) + for e in v: + assert(isinstance(e, PMSampleTypes)) + if "format" in p: + assert(isinstance(v, PMFormatTypes)) + v = v.value + if "order" in p: + assert(isinstance(v, PMOrderTypes)) + v = v.value + if "output-file" in p: + assert(isinstance(v, str)) + if "hide-cpu-duty-cycle" in p: + assert(isinstance(v, bool)) + self.default_parameters[p] = v @staticmethod @@ -167,7 +222,6 @@ def parse_pm_plist(logfile: Path): Returns: A list of dicts, each representing the plist for a given sample """ - fp = open(logfile, "rb") plists = [] From f63b90c67dbbddf12ef9aff6e49108f923622201 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Mon, 18 Nov 2024 18:27:31 +0100 Subject: [PATCH 07/15] Framework for new plugins + example for PM --- .../powermetrics-profiling/RunnerConfig.py | 13 ++- experiment-runner/Plugins/Profilers/PS.py | 96 +++++++++++++++++++ .../Plugins/Profilers/PowerJoular.py | 87 +++++++++++++++++ .../Plugins/Profilers/PowerMetrics.py | 6 +- 4 files changed, 194 insertions(+), 8 deletions(-) create mode 100644 experiment-runner/Plugins/Profilers/PS.py create mode 100644 experiment-runner/Plugins/Profilers/PowerJoular.py diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py index 0fc5b8e3..7b1ac132 100644 --- a/examples/powermetrics-profiling/RunnerConfig.py +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -11,6 +11,7 @@ from pathlib import Path from os.path import dirname, realpath import time +import numpy as np class RunnerConfig: ROOT_DIR = Path(dirname(realpath(__file__))) @@ -58,7 +59,7 @@ def create_run_table_model(self) -> RunTableModel: factor1 = FactorModel("test_factor", [1, 2]) self.run_table_model = RunTableModel( factors = [factor1], - data_columns=["poweravg", "wakeups", "cpu-usage"]) + data_columns=["joules", "avg_cpu", "avg_gpu"]) return self.run_table_model @@ -111,10 +112,14 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" # Retrieve data from run - run_results = self.meter.parse_logs("../../../powermetrics_outs/plist_power.txt") - - # Parse it as required for your experiment and add it to the run table + run_results = self.meter.parse_pm_plist("../../../powermetrics_outs/plist_power.txt") + # Parse it as required for your experiment and add it to the run table + return { + "joules": sum(map(lambda x: x["processor"]["package_joules"], run_results)), + "avg_cpu": np.mean(map(lambda x: x["processor"]["packages"]["cores_active_ratio"], run_results)), + "avg_gpu": np.mean(map(lambda x: x["processor"]["packages"]["gpu_active_ratio"], run_results)), + } def after_experiment(self) -> None: """Perform any activity required after stopping the experiment here diff --git a/experiment-runner/Plugins/Profilers/PS.py b/experiment-runner/Plugins/Profilers/PS.py new file mode 100644 index 00000000..c2baa1fa --- /dev/null +++ b/experiment-runner/Plugins/Profilers/PS.py @@ -0,0 +1,96 @@ +from __future__ import annotations +import enum +from collections.abc import Callable +from pathlib import Path +import subprocess +import plistlib +import platform +import shutil +import time +import pandas as pd + +class PowerJoular(object): + """An integration of OSX powermetrics into experiment-runner as a data source plugin""" + def __init__(self): + self.ps_process = None + self.additional_args = None + pass + + # Ensure that powermetrics is not currently running when we delete this object + def __del__(self): + if self.pj_process: + self.pj_process.terminate() + + # Check that we are running on OSX, and that the powermetrics command exists + def __validate_platform(self): + pass + + def __format_cmd(self): + cmd = [] + return cmd + self.additional_args + + def start_ps(self): + """ + Starts the powermetrics process, with the parameters in default_parameters + additional_args. + """ + # man 1 ps + # %cpu: + # cpu utilization of the process in "##.#" format. Currently, it is the CPU time used + # divided by the time the process has been running (cputime/realtime ratio), expressed + # as a percentage. It will not add up to 100% unless you are lucky. (alias pcpu). + profiler_cmd = f'ps -p {self.target.pid} --noheader -o %cpu' + wrapper_script = f''' + while true; do {profiler_cmd}; sleep 1; done + ''' + + time.sleep(1) # allow the process to run a little before measuring + self.profiler = subprocess.Popen(['sh', '-c', wrapper_script], + stdout=subprocess.PIPE, stderr=subprocess.PIPE + ) + + def stop_pm(self): + """ + Terminates the powermetrics process, as it was running indefinetly. This method collects the stdout and stderr + + Returns: + stdout, stderr of the powermetrics process. + """ + self.ps_profiler.kill() + self.ps_profiler.wait() + + # Set the parameters used for power metrics to a new set + def update_parameters(self, new_params: dict): + """ + Updates the list of parameters, to be in line with new_params. + Note that samplers will be set to the new list if present, make sure + to include the previous set if you still want to use them. + + Parameters: + new_params (dict): A dictionary containing the new list of parameters. For + parameters with no value (like --hide-cpu-duty-cycle) use a boolean to indicate + if they can be used. + + Returns: + The new list of parameters, can also be valided with self.default_parameters + """ + pass + + def parse_pj_logs(self, logfile: Path): + """ + Parses a provided logfile from powermetrics in plist format. Powermetrics outputs a plist + for every sample taken, it included a newline after the closing <\plist>, we account for that here + to make things easier to parse. + + Parameters: + logfile (Path): The path to the plist logfile created by powermetrics + + Returns: + A list of dicts, each representing the plist for a given sample + """ + df = pd.DataFrame(columns=['cpu_usage']) + for i, l in enumerate(self.profiler.stdout.readlines()): + cpu_usage=float(l.decode('ascii').strip()) + df.loc[i] = [cpu_usage] + + df.to_csv(self.run_dir / 'raw_data.csv', index=False) + diff --git a/experiment-runner/Plugins/Profilers/PowerJoular.py b/experiment-runner/Plugins/Profilers/PowerJoular.py new file mode 100644 index 00000000..f6953ac6 --- /dev/null +++ b/experiment-runner/Plugins/Profilers/PowerJoular.py @@ -0,0 +1,87 @@ +from __future__ import annotations +import enum +from collections.abc import Callable +from pathlib import Path +import subprocess +import plistlib +import platform +from shutil import shlex +import time +import signal +import os +import pandas as pd + +class PowerJoular(object): + """An integration of OSX powermetrics into experiment-runner as a data source plugin""" + def __init__(self): + self.ps_process = None + self.additional_args = None + pass + + # Ensure that powermetrics is not currently running when we delete this object + def __del__(self): + if self.pj_process: + self.pj_process.terminate() + + # Check that we are running on OSX, and that the powermetrics command exists + def __validate_platform(self): + pass + + def __format_cmd(self): + cmd = [] + return cmd + self.additional_args + + def start_pj(self): + """ + Starts the powermetrics process, with the parameters in default_parameters + additional_args. + """ + + profiler_cmd = f'powerjoular -l -p {self.target.pid} -f {self.run_dir / "powerjoular.csv"}' + + time.sleep(1) # allow the process to run a little before measuring + self.profiler = subprocess.Popen(shlex.split(profiler_cmd)) + + def stop_pj(self): + """ + Terminates the powermetrics process, as it was running indefinetly. This method collects the stdout and stderr + + Returns: + stdout, stderr of the powermetrics process. + """ + os.kill(self.profiler.pid, signal.SIGINT) # graceful shutdown of powerjoular + self.profiler.wait() + + # Set the parameters used for power metrics to a new set + def update_parameters(self, new_params: dict): + """ + Updates the list of parameters, to be in line with new_params. + Note that samplers will be set to the new list if present, make sure + to include the previous set if you still want to use them. + + Parameters: + new_params (dict): A dictionary containing the new list of parameters. For + parameters with no value (like --hide-cpu-duty-cycle) use a boolean to indicate + if they can be used. + + Returns: + The new list of parameters, can also be valided with self.default_parameters + """ + pass + + + @staticmethod + def parse_pj_logs(logfile: Path): + """ + Parses a provided logfile from powermetrics in plist format. Powermetrics outputs a plist + for every sample taken, it included a newline after the closing <\plist>, we account for that here + to make things easier to parse. + + Parameters: + logfile (Path): The path to the plist logfile created by powermetrics + + Returns: + A list of dicts, each representing the plist for a given sample + """ + # powerjoular.csv - Power consumption of the whole system + # powerjoular.csv-PID.csv - Power consumption of that specific process + df = pd.read_csv(context.run_dir / f"powerjoular.csv-{self.target.pid}.csv") diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 8b4a5a0f..2c218fc4 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -114,8 +114,7 @@ def start_pm(self): raise RuntimeError(f"Powermetrics encountered an error while starting: {stderr}") except Exception as e: - print(f"Could not start powermetrics: {e}") - raise RuntimeError("Powermetrics plugin could not start") + raise RuntimeError(f"Powermetrics plugin could not start: {e}") def stop_pm(self): """ @@ -136,8 +135,7 @@ def stop_pm(self): return stdout, stderr except Exception as e: - print(f"Could not stop powermetrics: {e}") - raise RuntimeError("Powermetrics plugin could not stop") + raise RuntimeError(f"Powermetrics plugin could not stop {e}") # Set the parameters used for power metrics to a new set def update_parameters(self, new_params: dict): From 304a4d115a22a4a5507f530e14fdead57012caa9 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Tue, 26 Nov 2024 19:23:27 +0100 Subject: [PATCH 08/15] Updated Framework for plugins + defined a abstract base class for creating cli or device based plugins --- .../Plugins/Profilers/DataSource.py | 131 ++++++++++++++++++ .../Plugins/Profilers/PowerJoular.py | 18 ++- .../Plugins/Profilers/PowerMetrics.py | 35 +++++ .../Plugins/Profilers/{PS.py => Ps.py} | 72 +++++++++- .../Plugins/Profilers/WattsUpPro.py | 2 +- 5 files changed, 251 insertions(+), 7 deletions(-) create mode 100644 experiment-runner/Plugins/Profilers/DataSource.py rename experiment-runner/Plugins/Profilers/{PS.py => Ps.py} (68%) diff --git a/experiment-runner/Plugins/Profilers/DataSource.py b/experiment-runner/Plugins/Profilers/DataSource.py new file mode 100644 index 00000000..63c6f9d4 --- /dev/null +++ b/experiment-runner/Plugins/Profilers/DataSource.py @@ -0,0 +1,131 @@ +from abc import ABC +from collections import UserDict +import platform +import shutil + +class DataSource(ABC): + def __init__(self): + self.__validate_platform() + + def __validate_platform(self): + for platform in self.supported_platforms: + if "OSX" in platform.system(): + return + + raise RuntimeError(f"One of: {self.supported_platforms} is required for this plugin") + + @property + @abstractmethod + def supported_platforms(self): -> list[str] + pass + + @property + @abstractmethod + def source_name(self): -> str + pass + + @abstractmethod + def __del__(self): + pass + + @abstractmethod + @staticmethod + def parse_log(): + pass + +class DeviceSource(DataSource): + def __init__(self): + self.super().__init__() + self.device_handle = None + + def __del__(): + if self.device_handle: + self.close_device() + + @abstractmethod + def list_devices(self): + pass + + @abstractmethod + def open_device(self): + pass + + @abstractmethod + def close_device(self): + pass + + @abstractmethod + def set_mode(self): + pass + + @abstractmethod + def log(self, timeout: int = 60, logfile: Path = None): + pass + +class ParameterDict(UserDict): + def __setitem__(self, key, item): + pass + + def __getitem__(self, key): + pass + + def __delitem__(self, key): + pass + +class CLISource(DataSource): + def __init__(self): + self.super().__init__() + self.process = None + self.default_args = None + self.additional_args = None + + def __validate_platform(self): + self.super().__validate_platform() + + if shutil.which("powermetrics") is None: + raise RuntimeError(f"The {self.source_name} cli tool is required for this plugin") + + def __validate_start(self, stdout, stderr): + if stderr: + raise RuntimeError(f"{self.source_name} did not start correctly") + + def __validate_stop(self, stdout, stderr): + if stderr: + raise RuntimeWarning(f"{self.source_name} did not stop correctly") + + @abstractmethod + def __format_cmd(self): + pass + + @abstractmethod + def update_parameters(self, new_parameters: dict): + pass + + def start(self): + try: + self.pm_process = subprocess.Popen(self.__format_cmd(), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + + stdout, stderr = subprocess.communicate() + + except Exception as e: + raise RuntimeError(f"{self.source_name} process could not start: {e}") + + self.__validate_start(stdout, stderr) + + def stop(self): + if not self.process: + return + + try: + self.process.terminate() + stdout, stderr = self.process.communicate() + + except Exception as e: + raise RuntimeError(f"{self.source_name} process could not stop {e}") + + self.__validate_stop(stdout, stderr) + + + diff --git a/experiment-runner/Plugins/Profilers/PowerJoular.py b/experiment-runner/Plugins/Profilers/PowerJoular.py index f6953ac6..a851d8de 100644 --- a/experiment-runner/Plugins/Profilers/PowerJoular.py +++ b/experiment-runner/Plugins/Profilers/PowerJoular.py @@ -10,9 +10,22 @@ import signal import os import pandas as pd +from Plugins.Profilers.DataSource import CLISource -class PowerJoular(object): - """An integration of OSX powermetrics into experiment-runner as a data source plugin""" +# Supported Paramters for the PowerJoular metrics plugin +POWERJOULAR_PARAMETERS = { + "-p": int, + "-a": str, + "-f": Path, + "-o": Path, + "-t": None, + "-l": None, + "-m": str, + "-s": str +} + +class PowerJoular(CLISource): + """An integration of PowerJoular into experiment-runner as a data source plugin""" def __init__(self): self.ps_process = None self.additional_args = None @@ -82,6 +95,7 @@ def parse_pj_logs(logfile: Path): Returns: A list of dicts, each representing the plist for a given sample """ + # powerjoular.csv - Power consumption of the whole system # powerjoular.csv-PID.csv - Power consumption of that specific process df = pd.read_csv(context.run_dir / f"powerjoular.csv-{self.target.pid}.csv") diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 2c218fc4..ec97670c 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -36,6 +36,41 @@ class PMSampleTypes(enum.Enum): PM_SAMPLE_NVME = "nvme_ssd" # NVMe power state information PM_SAMPLE_THROTTLE = "io_throttle_ssd" # IO Throttling information +# Supported Paramters for the power metrics plugin +POWERMETRICS_PARAMETERS = { + "--poweravg": int, + "--buffer-size": int, + "--format": PMFormatTypes, + "--sample-rate": int, + "--sample-count": int, + "--output-file": Path, + "--order": PMOrderTypes, + "--samplers": list[PMSampleTypes], + "--wakeup-cost": int, + "--unhide-info": list[PMSampleTypes], + "--show-all": None, + "--show-initial-usage": None, + "--show-usage-summary": None, + "--show-extra-power-info": None, + "--show-pstates": None, + "--show-plimits": None, + "--show-cpu-qos": None, + "--show-cpu-scalability": None, + "--show-hwp-capability": None, + "--show-process-coalition": None, + "--show-responsible-pid": None, + "--show-process-wait-times": None, + "--show-process-qos-tiers": None, + "--show-process-io": None, + "--show-process-gpu": None, + "--show-process-netstats": None, + "--show-process-qos": None, + "--show-process-energy": None, + "--show-process-samp-norm": None, + "--handle-invalid-values": None, + "--hide-cpu-duty-cycle": None, +} + class PowerMetrics(object): """An integration of OSX powermetrics into experiment-runner as a data source plugin""" def __init__(self, diff --git a/experiment-runner/Plugins/Profilers/PS.py b/experiment-runner/Plugins/Profilers/Ps.py similarity index 68% rename from experiment-runner/Plugins/Profilers/PS.py rename to experiment-runner/Plugins/Profilers/Ps.py index c2baa1fa..df66f29d 100644 --- a/experiment-runner/Plugins/Profilers/PS.py +++ b/experiment-runner/Plugins/Profilers/Ps.py @@ -9,8 +9,72 @@ import time import pandas as pd -class PowerJoular(object): - """An integration of OSX powermetrics into experiment-runner as a data source plugin""" +PS_PARAMTERS = { + "-A": None, + "-a": None, + "a": None, + "-d": None, + "-N": None, + "r": None, + "T": None, + "x": None, + + "-C": list[str], + "-G": list[int], + "-g": list[str], + "-p": list[int], + "--ppid": list[int], + "-q": list[int], + "-s": list[int], + "-t": list[int], + "-u": list[int], + "-U": list[int], + + "-D": str, + "-F": None, + "-f": None, + "f": None, + "-H": None, + "-j": None, + "j": None, + "-l": None, + "l": None, + "-M": None, + "-O": str, + "O": str, + "-o": str, + "-P": None, + "s": None, + "u": None, + "v": None, + "X": None, + "--context": None, + "--headers": None, + "--no-headers": None, + "--cols": int, + "--rows": int, + "--signames": None, + + "H": None, + "-L": None, + "-m": None, + "-T": None, + + "-c": None, + "c": None, + "e": None, + "k": str, + "L": None, + "n": None, + "S": None, + "-y": None, + "-w": None, +# "-V": None, # We dont support version or help +# "--help": None # We dont support the help option +} + +class Ps(object): + """An integration of the Linux ps utility into experiment-runner as a data source plugin""" def __init__(self): self.ps_process = None self.additional_args = None @@ -18,8 +82,8 @@ def __init__(self): # Ensure that powermetrics is not currently running when we delete this object def __del__(self): - if self.pj_process: - self.pj_process.terminate() + if self.ps_process: + self.ps_process.terminate() # Check that we are running on OSX, and that the powermetrics command exists def __validate_platform(self): diff --git a/experiment-runner/Plugins/Profilers/WattsUpPro.py b/experiment-runner/Plugins/Profilers/WattsUpPro.py index 43dd6ee7..6cac77f7 100644 --- a/experiment-runner/Plugins/Profilers/WattsUpPro.py +++ b/experiment-runner/Plugins/Profilers/WattsUpPro.py @@ -27,7 +27,7 @@ def __init__(self, port: str = None, interval=1.0): print( 'Default port is /dev/ttyUSB0 for Linux') raise RuntimeError("Invalid port") - self.s = serial.Serial(port, 115200 ) + self.s = serial.Serial(port, 115200) self.logfile = None self.interval = interval # initialize lists for keeping data From c33e852fe592d96d69dd5e1636ed522288d32cba Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Wed, 27 Nov 2024 19:25:32 +0100 Subject: [PATCH 09/15] Refactor of plugins to match the new DataSource generic --- .../powermetrics-profiling/RunnerConfig.py | 6 +- .../Plugins/Profilers/DataSource.py | 143 ++++++++---- .../Plugins/Profilers/PowerJoular.py | 101 ++------- .../Plugins/Profilers/PowerMetrics.py | 188 ++++------------ experiment-runner/Plugins/Profilers/Ps.py | 203 +++++++----------- 5 files changed, 238 insertions(+), 403 deletions(-) diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py index 7b1ac132..287a5464 100644 --- a/examples/powermetrics-profiling/RunnerConfig.py +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -87,7 +87,7 @@ def start_measurement(self, context: RunnerContext) -> None: """Perform any activity required for starting measurements.""" # Start measuring useing powermetrics (write to log file) - self.meter.start_pm() + self.meter.start() def interact(self, context: RunnerContext) -> None: """Perform any interaction with the running target system here, or block here until the target finishes.""" @@ -99,7 +99,7 @@ def stop_measurement(self, context: RunnerContext) -> None: """Perform any activity here required for stopping measurements.""" # Stop measuring at the end of a run - self.meter.stop_pm() + self.meter.stop() def stop_run(self, context: RunnerContext) -> None: """Perform any activity here required for stopping the run. @@ -112,7 +112,7 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" # Retrieve data from run - run_results = self.meter.parse_pm_plist("../../../powermetrics_outs/plist_power.txt") + run_results = self.meter.parse_log("../../../powermetrics_outs/plist_power.txt") # Parse it as required for your experiment and add it to the run table return { diff --git a/experiment-runner/Plugins/Profilers/DataSource.py b/experiment-runner/Plugins/Profilers/DataSource.py index 63c6f9d4..a4070f4c 100644 --- a/experiment-runner/Plugins/Profilers/DataSource.py +++ b/experiment-runner/Plugins/Profilers/DataSource.py @@ -3,9 +3,53 @@ import platform import shutil +class ParameterDict(UserDict): + def valid_key(self, key): + return isinstance(key, str) \ + or isinstance(key, tuple) \ + or isinstance(key, list[str]) + + def str_to_tuple(self, key): + return tuple([key]) + + def __setitem__(self, key, item): + if not self.valid_key(key): + raise RuntimeError("Unexpected key value") + + for params in self.data.keys(): + if set(key).issubset(params): + raise RuntimeError("Keys cannot have duplicate elements") + + if isinstance(key, str): + key = self.str_to_tuple(key) + + super().__setitem__(tuple(key), item) + + def __getitem__(self, key): + if not self.valid_key(key): + raise RuntimeError("Unexpected key, expected `str` or `list[str]`") + + if isinstance(key, str): + key = self.str_to_tuple(key) + + for params, val in self.data.items(): + if set(key).issubset(params): + return val + + # Pass to default handler if we cant find it + super().__getitem__(tuple(key)) + + # Must pass entire valid key to delete element + def __delitem__(self, key): + if isinstance(key, str): + key = self.str_to_tuple(key) + + super().__delitem__(tuple(key)) + class DataSource(ABC): def __init__(self): self.__validate_platform() + self.logfile = None def __validate_platform(self): for platform in self.supported_platforms: @@ -28,61 +72,28 @@ def source_name(self): -> str def __del__(self): pass - @abstractmethod @staticmethod + @abstractmethod def parse_log(): pass -class DeviceSource(DataSource): + +class CLISource(DataSource): def __init__(self): self.super().__init__() - self.device_handle = None - def __del__(): - if self.device_handle: - self.close_device() - - @abstractmethod - def list_devices(self): - pass - - @abstractmethod - def open_device(self): - pass - - @abstractmethod - def close_device(self): - pass - - @abstractmethod - def set_mode(self): - pass + self.process = None + self.args = None + @property @abstractmethod - def log(self, timeout: int = 60, logfile: Path = None): + def parameters(self): -> ParameterDict pass -class ParameterDict(UserDict): - def __setitem__(self, key, item): - pass - - def __getitem__(self, key): - pass - - def __delitem__(self, key): - pass - -class CLISource(DataSource): - def __init__(self): - self.super().__init__() - self.process = None - self.default_args = None - self.additional_args = None - def __validate_platform(self): self.super().__validate_platform() - if shutil.which("powermetrics") is None: + if shutil.which(self.source_name) is None: raise RuntimeError(f"The {self.source_name} cli tool is required for this plugin") def __validate_start(self, stdout, stderr): @@ -92,14 +103,25 @@ def __validate_start(self, stdout, stderr): def __validate_stop(self, stdout, stderr): if stderr: raise RuntimeWarning(f"{self.source_name} did not stop correctly") + + def __validate_parameters(self, parameters): + # TODO: Ensure types match here + pass - @abstractmethod def __format_cmd(self): - pass - - @abstractmethod + cmd = [self.source_name] + + # Add in the default parameters + for p, v in self.default_parameters.items(): + #TODO: Parse a parameter dict into a string format + pass + + return cmd + def update_parameters(self, new_parameters: dict): - pass + for p, v in new_params.items(): + # TODO: parse parameters, add to parameter dict + pass def start(self): try: @@ -127,5 +149,32 @@ def stop(self): self.__validate_stop(stdout, stderr) - +class DeviceSource(DataSource): + def __init__(self): + self.super().__init__() + self.device_handle = None + + def __del__(): + if self.device_handle: + self.close_device() + + @abstractmethod + def list_devices(self): + pass + + @abstractmethod + def open_device(self): + pass + + @abstractmethod + def close_device(self): + pass + + @abstractmethod + def set_mode(self): + pass + + @abstractmethod + def log(self, timeout: int = 60, logfile: Path = None): + pass diff --git a/experiment-runner/Plugins/Profilers/PowerJoular.py b/experiment-runner/Plugins/Profilers/PowerJoular.py index a851d8de..ab893e4e 100644 --- a/experiment-runner/Plugins/Profilers/PowerJoular.py +++ b/experiment-runner/Plugins/Profilers/PowerJoular.py @@ -10,92 +10,37 @@ import signal import os import pandas as pd -from Plugins.Profilers.DataSource import CLISource +from Plugins.Profilers.DataSource import CLISource, ParameterDict # Supported Paramters for the PowerJoular metrics plugin POWERJOULAR_PARAMETERS = { - "-p": int, - "-a": str, - "-f": Path, - "-o": Path, - "-t": None, - "-l": None, - "-m": str, - "-s": str + ("-p",): int, + ("-a",): str, + ("-f",): Path, + ("-o",): Path, + ("-t",): None, + ("-l",): None, + ("-m",): str, + ("-s",): str } class PowerJoular(CLISource): - """An integration of PowerJoular into experiment-runner as a data source plugin""" - def __init__(self): - self.ps_process = None - self.additional_args = None - pass - - # Ensure that powermetrics is not currently running when we delete this object - def __del__(self): - if self.pj_process: - self.pj_process.terminate() - - # Check that we are running on OSX, and that the powermetrics command exists - def __validate_platform(self): - pass - - def __format_cmd(self): - cmd = [] - return cmd + self.additional_args - - def start_pj(self): - """ - Starts the powermetrics process, with the parameters in default_parameters + additional_args. - """ - - profiler_cmd = f'powerjoular -l -p {self.target.pid} -f {self.run_dir / "powerjoular.csv"}' - - time.sleep(1) # allow the process to run a little before measuring - self.profiler = subprocess.Popen(shlex.split(profiler_cmd)) - - def stop_pj(self): - """ - Terminates the powermetrics process, as it was running indefinetly. This method collects the stdout and stderr + parameters = ParameterDict(POWERJOULAR_PARAMETERS) + source_name = "powerjoular" + supported_platforms = ["Linux"] - Returns: - stdout, stderr of the powermetrics process. - """ - os.kill(self.profiler.pid, signal.SIGINT) # graceful shutdown of powerjoular - self.profiler.wait() - - # Set the parameters used for power metrics to a new set - def update_parameters(self, new_params: dict): - """ - Updates the list of parameters, to be in line with new_params. - Note that samplers will be set to the new list if present, make sure - to include the previous set if you still want to use them. - - Parameters: - new_params (dict): A dictionary containing the new list of parameters. For - parameters with no value (like --hide-cpu-duty-cycle) use a boolean to indicate - if they can be used. - - Returns: - The new list of parameters, can also be valided with self.default_parameters - """ - pass - - + """An integration of PowerJoular into experiment-runner as a data source plugin""" + def __init__(self, + sample_frequency: int = 5000, + out_file: Path = "powerjoular.csv", + additional_args: list[str] = []): + + self.logfile = outfile + # TODO: Convert to appropriate dict + self.args = f'powerjoular -l -p {self.target.pid} -f {self.run_dir / "powerjoular.csv"}' + @staticmethod - def parse_pj_logs(logfile: Path): - """ - Parses a provided logfile from powermetrics in plist format. Powermetrics outputs a plist - for every sample taken, it included a newline after the closing <\plist>, we account for that here - to make things easier to parse. - - Parameters: - logfile (Path): The path to the plist logfile created by powermetrics - - Returns: - A list of dicts, each representing the plist for a given sample - """ - + def parse_log(logfile: Path): # powerjoular.csv - Power consumption of the whole system # powerjoular.csv-PID.csv - Power consumption of that specific process df = pd.read_csv(context.run_dir / f"powerjoular.csv-{self.target.pid}.csv") diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index ec97670c..19248359 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -7,6 +7,8 @@ import platform import shutil +from Plugins.Profilers.DataSource import ParameterDict, CLISource + # How to format the output class PMFormatTypes(enum.Enum): PM_FMT_TEXT = "text" @@ -38,58 +40,56 @@ class PMSampleTypes(enum.Enum): # Supported Paramters for the power metrics plugin POWERMETRICS_PARAMETERS = { - "--poweravg": int, - "--buffer-size": int, - "--format": PMFormatTypes, - "--sample-rate": int, - "--sample-count": int, - "--output-file": Path, - "--order": PMOrderTypes, - "--samplers": list[PMSampleTypes], - "--wakeup-cost": int, - "--unhide-info": list[PMSampleTypes], - "--show-all": None, - "--show-initial-usage": None, - "--show-usage-summary": None, - "--show-extra-power-info": None, - "--show-pstates": None, - "--show-plimits": None, - "--show-cpu-qos": None, - "--show-cpu-scalability": None, - "--show-hwp-capability": None, - "--show-process-coalition": None, - "--show-responsible-pid": None, - "--show-process-wait-times": None, - "--show-process-qos-tiers": None, - "--show-process-io": None, - "--show-process-gpu": None, - "--show-process-netstats": None, - "--show-process-qos": None, - "--show-process-energy": None, - "--show-process-samp-norm": None, - "--handle-invalid-values": None, - "--hide-cpu-duty-cycle": None, + ("--poweravg", "-a"): int, + ("--buffer-size", "-b"): int, + ("--format", "-f"): PMFormatTypes, + ("--sample-rate", "-i"): int, + ("--sample-count", "-n"): int, + ("--output-file", "-o"): Path, + ("--order", "-r"): PMOrderTypes, + ("--samplers", "-s"): list[PMSampleTypes], + ("--wakeup-cost", "-t"): int, + ("--unhide-info",): list[PMSampleTypes], + ("--show-all", "-A"): None, + ("--show-initial-usage",): None, + ("--show-usage-summary",): None, + ("--show-extra-power-info",): None, + ("--show-pstates",): None, + ("--show-plimits",): None, + ("--show-cpu-qos",): None, + ("--show-cpu-scalability",): None, + ("--show-hwp-capability",): None, + ("--show-process-coalition",): None, + ("--show-responsible-pid",): None, + ("--show-process-wait-times",): None, + ("--show-process-qos-tiers",): None, + ("--show-process-io",): None, + ("--show-process-gpu",): None, + ("--show-process-netstats",): None, + ("--show-process-qos"): None, + ("--show-process-energy",): None, + ("--show-process-samp-norm",): None, + ("--handle-invalid-values",): None, + ("--hide-cpu-duty-cycle",): None, } -class PowerMetrics(object): +class PowerMetrics(CLISource): + parameters = ParameterDict(POWERMETRICS_PARAMETERS) + source_name = "powermetrics" + supported_platforms = ["OSX"] + """An integration of OSX powermetrics into experiment-runner as a data source plugin""" def __init__(self, sample_frequency: int = 5000, out_file: Path = "pm_out.plist", + additional_args: list[str] = [], additional_samplers: list[PMSampleTypes] = [], - additional_args: list[str] = [], hide_cpu_duty_cycle: bool = True, order: PMOrderTypes = PMOrderTypes.PM_ORDER_CPU): - - # Double check we have the required software for this plugin - self.__validate_platform() - self.pm_process = None self.logfile = out_file - - self.additional_args = additional_args # Grab all available power stats by default - self.default_parameters = { + self.args = { "--output-file": self.logfile, "--sample-interval": sample_frequency, "--format": PMFormatTypes.PM_FMT_PLIST.value, @@ -100,112 +100,6 @@ def __init__(self, "--order": order.value } - # Ensure that powermetrics is not currently running when we delete this object - def __del__(self): - if self.pm_process: - self.pm_process.terminate() - - # Check that we are running on OSX, and that the powermetrics command exists - def __validate_platform(self): - if "OSX" not in platform.system(): - raise RuntimeError("The OSX platform is required for this plugin") - - if shutil.which("powermetrics") is None: - raise RuntimeError("The powermetrics tool is required for this plugin") - - def __format_cmd(self): - cmd = ["powermetrics"] - - # Add in the default parameters - for p, v in self.default_parameters.items(): - if v is False: - continue - - # Add the parameter - cmd.append(p) - - # Add the value - if "samplers" in p and isinstance(v, list): - cmd.append(",".join([x.value for x in v])) - elif not isinstance(v, bool): - cmd.append(str(v)) - - return cmd + self.additional_args - - def start_pm(self): - """ - Starts the powermetrics process, with the parameters in default_parameters + additional_args. - """ - try: - self.pm_process = subprocess.Popen(self.__format_cmd(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) - - stdout, stderr = subprocess.communicate() - - if stderr: - self.pm_process.terminate() - self.pm_process = None - raise RuntimeError(f"Powermetrics encountered an error while starting: {stderr}") - - except Exception as e: - raise RuntimeError(f"Powermetrics plugin could not start: {e}") - - def stop_pm(self): - """ - Terminates the powermetrics process, as it was running indefinetly. This method collects the stdout and stderr - - Returns: - stdout, stderr of the powermetrics process. - """ - if not self.pm_process: - return - - try: - self.pm_process.terminate() - stdout, stderr = self.pm_process.communicate() - - if stderr: - raise RuntimeError("powermetrics encountered an error during measurement") - - return stdout, stderr - except Exception as e: - raise RuntimeError(f"Powermetrics plugin could not stop {e}") - - # Set the parameters used for power metrics to a new set - def update_parameters(self, new_params: dict): - """ - Updates the list of parameters, to be in line with new_params. - Note that samplers will be set to the new list if present, make sure - to include the previous set if you still want to use them. - - Parameters: - new_params (dict): A dictionary containing the new list of parameters. For - parameters with no value (like --hide-cpu-duty-cycle) use a boolean to indicate - if they can be used. - - Returns: - The new list of parameters, can also be valided with self.default_parameters - """ - for p, v in new_params.items(): - # Double check new_params where possible - if "samplers" in p: - assert(isinstance(v, list)) - for e in v: - assert(isinstance(e, PMSampleTypes)) - if "format" in p: - assert(isinstance(v, PMFormatTypes)) - v = v.value - if "order" in p: - assert(isinstance(v, PMOrderTypes)) - v = v.value - if "output-file" in p: - assert(isinstance(v, str)) - if "hide-cpu-duty-cycle" in p: - assert(isinstance(v, bool)) - - self.default_parameters[p] = v - @staticmethod def get_plist_power(pm_plists: list[dict]): """ @@ -243,7 +137,7 @@ def get_plist_power(pm_plists: list[dict]): return power_plists @staticmethod - def parse_pm_plist(logfile: Path): + def parse_log(logfile: Path): """ Parses a provided logfile from powermetrics in plist format. Powermetrics outputs a plist for every sample taken, it included a newline after the closing <\plist>, we account for that here diff --git a/experiment-runner/Plugins/Profilers/Ps.py b/experiment-runner/Plugins/Profilers/Ps.py index df66f29d..c6bf91dc 100644 --- a/experiment-runner/Plugins/Profilers/Ps.py +++ b/experiment-runner/Plugins/Profilers/Ps.py @@ -8,149 +8,96 @@ import shutil import time import pandas as pd +from Plugins.Profilers.DataSource import CLISource, ParameterDict PS_PARAMTERS = { - "-A": None, - "-a": None, - "a": None, - "-d": None, - "-N": None, - "r": None, - "T": None, - "x": None, - - "-C": list[str], - "-G": list[int], - "-g": list[str], - "-p": list[int], - "--ppid": list[int], - "-q": list[int], - "-s": list[int], - "-t": list[int], - "-u": list[int], - "-U": list[int], - - "-D": str, - "-F": None, - "-f": None, - "f": None, - "-H": None, - "-j": None, - "j": None, - "-l": None, - "l": None, - "-M": None, - "-O": str, - "O": str, - "-o": str, - "-P": None, - "s": None, - "u": None, - "v": None, - "X": None, - "--context": None, - "--headers": None, - "--no-headers": None, - "--cols": int, - "--rows": int, - "--signames": None, + ("-A", "-e"): None, + ("-a"): None, + ("a"): None, + ("-d"): None, + ("-N", "--deselect"): None, + ("r"): None, + ("T"): None, + ("x"): None, + + ("-C"): list[str], + ("-G", "--Group"): list[int], + ("-g", "--group"): list[str], + ("-p", "p", "--pid"): list[int], + ("--ppid"): list[int], + ("-q", "q", "--quick-pid"): list[int], + ("-s","--sid"): list[int], + ("-t", "t", "--tty"): list[int], + ("-u", "U", "--user"): list[int], + ("-U", "--User"): list[int], + + ("-D"): str, + ("-F"): None, + ("-f"): None, + ("f", "--forest"): None, + ("-H"): None, + ("-j"): None, + ("j"): None, + ("-l"): None, + ("l"): None, + ("-M", "Z"): None, + ("-O"): str, + ("O"): str, + ("-o", "o", "--format"): str, + ("-P"): None, + ("s"): None, + ("u"): None, + ("v"): None, + ("X"): None, + ("--context"): None, + ("--headers"): None, + ("--no-headers"): None, + ("--cols", "--columns", "--width"): int, + ("--rows", "--lines"): int, + ("--signames"): None, - "H": None, - "-L": None, - "-m": None, - "-T": None, - - "-c": None, - "c": None, - "e": None, - "k": str, - "L": None, - "n": None, - "S": None, - "-y": None, - "-w": None, -# "-V": None, # We dont support version or help -# "--help": None # We dont support the help option + ("H"): None, + ("-L"): None, + ("-m", "m"): None, + ("-T"): None, + + ("-c"): None, + ("c"): None, + ("e"): None, + ("k", "--sort"): str, # There is a format type here, maybe regex this eventually + ("L"): None, + ("n"): None, + ("S", "--cumulative"): None, + ("-y"): None, + ("-w", "w"): None, } -class Ps(object): - """An integration of the Linux ps utility into experiment-runner as a data source plugin""" - def __init__(self): - self.ps_process = None - self.additional_args = None - pass - - # Ensure that powermetrics is not currently running when we delete this object - def __del__(self): - if self.ps_process: - self.ps_process.terminate() - - # Check that we are running on OSX, and that the powermetrics command exists - def __validate_platform(self): - pass +class Ps(CliSource): + parameters = ParameterDict(PS_PARAMTERS) + source_name = "ps" + supported_platforms = ["Linux"] - def __format_cmd(self): - cmd = [] - return cmd + self.additional_args + """An integration of the Linux ps utility into experiment-runner as a data source plugin""" + def __init__(self, + sample_frequency: int = 5000, + out_file: Path = "powerjoular.csv", + additional_args: list[str] = []): + + self.logfile = out_file + + # TODO: Convert this into a params dict + self.args = f'ps -p {self.target.pid} --noheader -o %cpu' - def start_ps(self): - """ - Starts the powermetrics process, with the parameters in default_parameters + additional_args. - """ - # man 1 ps + # man 1 ps # %cpu: # cpu utilization of the process in "##.#" format. Currently, it is the CPU time used # divided by the time the process has been running (cputime/realtime ratio), expressed # as a percentage. It will not add up to 100% unless you are lucky. (alias pcpu). - profiler_cmd = f'ps -p {self.target.pid} --noheader -o %cpu' wrapper_script = f''' while true; do {profiler_cmd}; sleep 1; done ''' - time.sleep(1) # allow the process to run a little before measuring - self.profiler = subprocess.Popen(['sh', '-c', wrapper_script], - stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) - - def stop_pm(self): - """ - Terminates the powermetrics process, as it was running indefinetly. This method collects the stdout and stderr - - Returns: - stdout, stderr of the powermetrics process. - """ - self.ps_profiler.kill() - self.ps_profiler.wait() - - # Set the parameters used for power metrics to a new set - def update_parameters(self, new_params: dict): - """ - Updates the list of parameters, to be in line with new_params. - Note that samplers will be set to the new list if present, make sure - to include the previous set if you still want to use them. - - Parameters: - new_params (dict): A dictionary containing the new list of parameters. For - parameters with no value (like --hide-cpu-duty-cycle) use a boolean to indicate - if they can be used. - - Returns: - The new list of parameters, can also be valided with self.default_parameters - """ - pass - - def parse_pj_logs(self, logfile: Path): - """ - Parses a provided logfile from powermetrics in plist format. Powermetrics outputs a plist - for every sample taken, it included a newline after the closing <\plist>, we account for that here - to make things easier to parse. - - Parameters: - logfile (Path): The path to the plist logfile created by powermetrics - - Returns: - A list of dicts, each representing the plist for a given sample - """ + def parse_log(self, logfile: Path): df = pd.DataFrame(columns=['cpu_usage']) for i, l in enumerate(self.profiler.stdout.readlines()): cpu_usage=float(l.decode('ascii').strip()) From 8a0dc3f2fb8f4f640e47efff06162caa40d4939c Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Wed, 4 Dec 2024 17:03:27 +0100 Subject: [PATCH 10/15] Updated runner configs and completed plugin implementations --- .../RunnerConfig.py | 35 ++++---- examples/linux-ps-profiling/RunnerConfig.py | 53 ++++++------ .../powermetrics-profiling/RunnerConfig.py | 18 ++--- .../Plugins/Profilers/DataSource.py | 80 +++++++++++-------- .../Plugins/Profilers/PowerJoular.py | 32 ++++---- .../Plugins/Profilers/PowerMetrics.py | 11 ++- experiment-runner/Plugins/Profilers/Ps.py | 67 ++++++++++------ 7 files changed, 160 insertions(+), 136 deletions(-) diff --git a/examples/linux-powerjoular-profiling/RunnerConfig.py b/examples/linux-powerjoular-profiling/RunnerConfig.py index 244b343d..3aa464bc 100644 --- a/examples/linux-powerjoular-profiling/RunnerConfig.py +++ b/examples/linux-powerjoular-profiling/RunnerConfig.py @@ -5,6 +5,7 @@ from ConfigValidator.Config.Models.RunnerContext import RunnerContext from ConfigValidator.Config.Models.OperationType import OperationType from ProgressManager.Output.OutputProcedure import OutputProcedure as output +from Plugins.Profilers.PowerJoular import PowerJoular from typing import Dict, List, Any, Optional from pathlib import Path @@ -90,14 +91,16 @@ def start_run(self, context: RunnerContext) -> None: # Configure the environment based on the current variation subprocess.check_call(shlex.split(f'cpulimit -b -p {self.target.pid} --limit {cpu_limit}')) + time.sleep(1) # allow the process to run a little before measuring def start_measurement(self, context: RunnerContext) -> None: """Perform any activity required for starting measurements.""" - - profiler_cmd = f'powerjoular -l -p {self.target.pid} -f {context.run_dir / "powerjoular.csv"}' - - time.sleep(1) # allow the process to run a little before measuring - self.profiler = subprocess.Popen(shlex.split(profiler_cmd)) + + # Set up the powerjoular object, provide an (optional) target and output file name + self.meter = PowerJoular(target_pid=self.target.pid, + out_file=context.run_dir / "powerjoular.csv") + # Start measuring with powerjoular + stdout = self.meter.start() def interact(self, context: RunnerContext) -> None: """Perform any interaction with the running target system here, or block here until the target finishes.""" @@ -109,14 +112,14 @@ def interact(self, context: RunnerContext) -> None: def stop_measurement(self, context: RunnerContext) -> None: """Perform any activity here required for stopping measurements.""" - - os.kill(self.profiler.pid, signal.SIGINT) # graceful shutdown of powerjoular - self.profiler.wait() + + # Stop the measurements + stdout = self.meter.stop() def stop_run(self, context: RunnerContext) -> None: """Perform any activity here required for stopping the run. Activities after stopping the run should also be performed here.""" - + self.target.kill() self.target.wait() @@ -124,13 +127,17 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: """Parse and process any measurement data here. You can also store the raw measurement data under `context.run_dir` Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" + + out_file = context.run_dir / "powerjoular.csv" + + results_global = self.meter.parse_log(out_file) + # If you specified a target_pid or used the -p paramter + # a second csv for that target will be generated + # results_process = self.meter.parse_log(f"{out_file}-{self.target.pid}.csv" ) - # powerjoular.csv - Power consumption of the whole system - # powerjoular.csv-PID.csv - Power consumption of that specific process - df = pd.read_csv(context.run_dir / f"powerjoular.csv-{self.target.pid}.csv") run_data = { - 'avg_cpu': round(df['CPU Utilization'].sum(), 3), - 'total_energy': round(df['CPU Power'].sum(), 3), + 'avg_cpu': round(results_global['CPU Utilization'].sum(), 3), + 'total_energy': round(results_global['CPU Power'].sum(), 3), } return run_data diff --git a/examples/linux-ps-profiling/RunnerConfig.py b/examples/linux-ps-profiling/RunnerConfig.py index 83060cec..eaa88c39 100644 --- a/examples/linux-ps-profiling/RunnerConfig.py +++ b/examples/linux-ps-profiling/RunnerConfig.py @@ -5,12 +5,13 @@ from ConfigValidator.Config.Models.RunnerContext import RunnerContext from ConfigValidator.Config.Models.OperationType import OperationType from ProgressManager.Output.OutputProcedure import OutputProcedure as output +from Plugins.Profilers.PowerJoular import PS from typing import Dict, List, Any, Optional from pathlib import Path from os.path import dirname, realpath -import pandas as pd +import numpy as np import time import subprocess import shlex @@ -64,14 +65,16 @@ def create_run_table_model(self) -> RunTableModel: exclude_variations = [ {cpu_limit_factor: [70], pin_core_factor: [False]} # all runs having the combination <'70', 'False'> will be excluded ], - data_columns=['avg_cpu'] + data_columns=["avg_cpu", "avg_mem"] ) return self.run_table_model def before_experiment(self) -> None: """Perform any activity required before starting the experiment here Invoked only once during the lifetime of the program.""" - subprocess.check_call(['make'], cwd=self.ROOT_DIR) # compile + + # compile the target program + subprocess.check_call(['make'], cwd=self.ROOT_DIR) def before_run(self) -> None: """Perform any activity required before starting a run. @@ -93,27 +96,21 @@ def start_run(self, context: RunnerContext) -> None: # Configure the environment based on the current variation if pin_core: - subprocess.check_call(shlex.split(f'taskset -cp 0 {self.target.pid}')) + subprocess.check_call(shlex.split(f'taskset -cp 0 {self.target.pid}')) + + # Limit the targets cputime subprocess.check_call(shlex.split(f'cpulimit -b -p {self.target.pid} --limit {cpu_limit}')) + time.sleep(1) # allow the process to run a little before measuring def start_measurement(self, context: RunnerContext) -> None: """Perform any activity required for starting measurements.""" - - # man 1 ps - # %cpu: - # cpu utilization of the process in "##.#" format. Currently, it is the CPU time used - # divided by the time the process has been running (cputime/realtime ratio), expressed - # as a percentage. It will not add up to 100% unless you are lucky. (alias pcpu). - profiler_cmd = f'ps -p {self.target.pid} --noheader -o %cpu' - wrapper_script = f''' - while true; do {profiler_cmd}; sleep 1; done - ''' - - time.sleep(1) # allow the process to run a little before measuring - self.profiler = subprocess.Popen(['sh', '-c', wrapper_script], - stdout=subprocess.PIPE, stderr=subprocess.PIPE - ) + + # Set up the ps object, provide an (optional) target and output file name + self.meter = PS(out_file=context.run_dir / "ps.csv", + target_pid=self.target.pid) + # Start measuring with ps + self.meter.start() def interact(self, context: RunnerContext) -> None: """Perform any interaction with the running target system here, or block here until the target finishes.""" @@ -126,8 +123,8 @@ def interact(self, context: RunnerContext) -> None: def stop_measurement(self, context: RunnerContext) -> None: """Perform any activity here required for stopping measurements.""" - self.profiler.kill() - self.profiler.wait() + # Stop the measurements + stdout = self.meter.stop() def stop_run(self, context: RunnerContext) -> None: """Perform any activity here required for stopping the run. @@ -141,17 +138,13 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: You can also store the raw measurement data under `context.run_dir` Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" - df = pd.DataFrame(columns=['cpu_usage']) - for i, l in enumerate(self.profiler.stdout.readlines()): - cpu_usage=float(l.decode('ascii').strip()) - df.loc[i] = [cpu_usage] - - df.to_csv(context.run_dir / 'raw_data.csv', index=False) + results = self.meter.parse_log(context.run_dir / "ps.csv", + column_names=["cpu_usage", "memory_usage"]) - run_data = { - 'avg_cpu': round(df['cpu_usage'].mean(), 3) + return { + "avg_cpu": round(np.mean(results['cpu_usage']), 3), + "avg_mem": round(np.mean(results['memory_usage']), 3) } - return run_data def after_experiment(self) -> None: """Perform any activity required after stopping the experiment here diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py index 287a5464..1657cbfe 100644 --- a/examples/powermetrics-profiling/RunnerConfig.py +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -66,9 +66,7 @@ def create_run_table_model(self) -> RunTableModel: def before_experiment(self) -> None: """Perform any activity required before starting the experiment here Invoked only once during the lifetime of the program.""" - - # Create the powermetrics object we will use to collect data - self.meter = PowerMetrics() + pass def before_run(self) -> None: """Perform any activity required before starting a run. @@ -79,14 +77,14 @@ def start_run(self, context: RunnerContext) -> None: """Perform any activity required for starting the run here. For example, starting the target system to measure. Activities after starting the run should also be performed here.""" - - # Optionally change powermetrics parameters between runs (if needed) - #self.meter.update_parameters() + pass def start_measurement(self, context: RunnerContext) -> None: """Perform any activity required for starting measurements.""" - - # Start measuring useing powermetrics (write to log file) + + # Create the powermetrics object we will use to collect data + self.meter = PowerMetrics(out_file=context.run_dir / "powermetrics.plist") + # Start measuring useing powermetrics self.meter.start() def interact(self, context: RunnerContext) -> None: @@ -99,7 +97,7 @@ def stop_measurement(self, context: RunnerContext) -> None: """Perform any activity here required for stopping measurements.""" # Stop measuring at the end of a run - self.meter.stop() + stdout = self.meter.stop() def stop_run(self, context: RunnerContext) -> None: """Perform any activity here required for stopping the run. @@ -112,7 +110,7 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: Returns a dictionary with keys `self.run_table_model.data_columns` and their values populated""" # Retrieve data from run - run_results = self.meter.parse_log("../../../powermetrics_outs/plist_power.txt") + run_results = self.meter.parse_log(context.run_dir / "powermetrics.plist") # Parse it as required for your experiment and add it to the run table return { diff --git a/experiment-runner/Plugins/Profilers/DataSource.py b/experiment-runner/Plugins/Profilers/DataSource.py index a4070f4c..0607ee83 100644 --- a/experiment-runner/Plugins/Profilers/DataSource.py +++ b/experiment-runner/Plugins/Profilers/DataSource.py @@ -1,12 +1,14 @@ -from abc import ABC -from collections import UserDict +from abc import ABC, abstractmethod +from collections import UserDict, Iterable +from pathlib import Path import platform import shutil +import subprocess class ParameterDict(UserDict): def valid_key(self, key): return isinstance(key, str) \ - or isinstance(key, tuple) \ + or isinstance(key, tuple) \ or isinstance(key, list[str]) def str_to_tuple(self, key): @@ -52,20 +54,19 @@ def __init__(self): self.logfile = None def __validate_platform(self): - for platform in self.supported_platforms: - if "OSX" in platform.system(): - return + if platform.system() in self.supported_platforms: + return raise RuntimeError(f"One of: {self.supported_platforms} is required for this plugin") @property @abstractmethod - def supported_platforms(self): -> list[str] + def supported_platforms(self) -> list[str]: pass @property @abstractmethod - def source_name(self): -> str + def source_name(self) -> str: pass @abstractmethod @@ -87,7 +88,7 @@ def __init__(self): @property @abstractmethod - def parameters(self): -> ParameterDict + def parameters(self) -> ParameterDict: pass def __validate_platform(self): @@ -96,65 +97,78 @@ def __validate_platform(self): if shutil.which(self.source_name) is None: raise RuntimeError(f"The {self.source_name} cli tool is required for this plugin") - def __validate_start(self, stdout, stderr): - if stderr: + def __validate_start(self): + if self.process.poll() != None: raise RuntimeError(f"{self.source_name} did not start correctly") def __validate_stop(self, stdout, stderr): if stderr: - raise RuntimeWarning(f"{self.source_name} did not stop correctly") + raise RuntimeWarning(f"{self.source_name} did not stop correctly, or encountered an error") - def __validate_parameters(self, parameters): - # TODO: Ensure types match here - pass + def __validate_parameters(self, parameters: dict): + for p, v in parameters: + if not isinstance(v, self.parameters[p]): + raise RuntimeError(f"Unexpected type: {type(v)} for parameter {p}") def __format_cmd(self): - cmd = [self.source_name] + self.__validate_parameters(self.args) + + cmd = self.source_name - # Add in the default parameters - for p, v in self.default_parameters.items(): - #TODO: Parse a parameter dict into a string format - pass - + # Transform the parameter dict into string format to be parsed by shlex + for p, v in self.args.items(): + if isinstance(v, Iterable): + cmd += f" {p} {",".join(map(str, v))}" + else: + cmd += f" {p} {v}" + return cmd - def update_parameters(self, new_parameters: dict): - for p, v in new_params.items(): - # TODO: parse parameters, add to parameter dict - pass + def update_parameters(self, add: dict={}, remove: list[str]=[]): + # Check if the new sets of parameters are sane + self.__validate_parameters(add) + + for p, v in add.items(): + self.args[p] = v + + for p in remove: + if p in self.args.keys(): + del self.args[p] + + # Double check that our typeing is still valid + self.__validate_parameters(self.args) def start(self): try: - self.pm_process = subprocess.Popen(self.__format_cmd(), + self.process = subprocess.Popen(shutil.shlex.split(self.__format_cmd()), stdout=subprocess.PIPE, stderr=subprocess.PIPE) - - stdout, stderr = subprocess.communicate() - except Exception as e: raise RuntimeError(f"{self.source_name} process could not start: {e}") - self.__validate_start(stdout, stderr) + self.__validate_start() def stop(self): if not self.process: return try: - self.process.terminate() - stdout, stderr = self.process.communicate() + # most cli utilities should stop with ctrl-c + self.process.send_signal(subprocess.signal.SIGINT) + stdout, stderr = self.process.communicate(timeout=5) except Exception as e: raise RuntimeError(f"{self.source_name} process could not stop {e}") self.__validate_stop(stdout, stderr) + return stdout.decode("utf-8") class DeviceSource(DataSource): def __init__(self): self.super().__init__() self.device_handle = None - def __del__(): + def __del__(self): if self.device_handle: self.close_device() diff --git a/experiment-runner/Plugins/Profilers/PowerJoular.py b/experiment-runner/Plugins/Profilers/PowerJoular.py index ab893e4e..271e8ed3 100644 --- a/experiment-runner/Plugins/Profilers/PowerJoular.py +++ b/experiment-runner/Plugins/Profilers/PowerJoular.py @@ -1,14 +1,5 @@ from __future__ import annotations -import enum -from collections.abc import Callable from pathlib import Path -import subprocess -import plistlib -import platform -from shutil import shlex -import time -import signal -import os import pandas as pd from Plugins.Profilers.DataSource import CLISource, ParameterDict @@ -33,14 +24,21 @@ class PowerJoular(CLISource): def __init__(self, sample_frequency: int = 5000, out_file: Path = "powerjoular.csv", - additional_args: list[str] = []): + additional_args: dict = {}, + target_pid: int = None): + + self.logfile = out_file + self.args = { + "-l": None, + "-f": self.logfile, + } + + if target_pid: + self.update_parameters(add={"-p": target_pid}) + + self.update_parameters(add=additional_args) - self.logfile = outfile - # TODO: Convert to appropriate dict - self.args = f'powerjoular -l -p {self.target.pid} -f {self.run_dir / "powerjoular.csv"}' - @staticmethod def parse_log(logfile: Path): - # powerjoular.csv - Power consumption of the whole system - # powerjoular.csv-PID.csv - Power consumption of that specific process - df = pd.read_csv(context.run_dir / f"powerjoular.csv-{self.target.pid}.csv") + # Things are already in csv format here, no checks needed + return pd.read_csv(logfile).to_dict() diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index 19248359..b7911aea 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -1,11 +1,7 @@ from __future__ import annotations import enum -from collections.abc import Callable from pathlib import Path -import subprocess import plistlib -import platform -import shutil from Plugins.Profilers.DataSource import ParameterDict, CLISource @@ -76,13 +72,13 @@ class PMSampleTypes(enum.Enum): class PowerMetrics(CLISource): parameters = ParameterDict(POWERMETRICS_PARAMETERS) source_name = "powermetrics" - supported_platforms = ["OSX"] + supported_platforms = ["OS X"] """An integration of OSX powermetrics into experiment-runner as a data source plugin""" def __init__(self, sample_frequency: int = 5000, out_file: Path = "pm_out.plist", - additional_args: list[str] = [], + additional_args: dict = {}, additional_samplers: list[PMSampleTypes] = [], hide_cpu_duty_cycle: bool = True, order: PMOrderTypes = PMOrderTypes.PM_ORDER_CPU): @@ -100,6 +96,8 @@ def __init__(self, "--order": order.value } + self.update_parameters(add=additional_args) + @staticmethod def get_plist_power(pm_plists: list[dict]): """ @@ -154,6 +152,7 @@ def parse_log(logfile: Path): plists = [] cur_plist = bytearray() for l in fp.readlines(): + # Powermetrics outputs plists with null bytes inbetween. We account for this if l[0] == 0: plists.append(plistlib.loads(cur_plist)) diff --git a/experiment-runner/Plugins/Profilers/Ps.py b/experiment-runner/Plugins/Profilers/Ps.py index c6bf91dc..07d415c2 100644 --- a/experiment-runner/Plugins/Profilers/Ps.py +++ b/experiment-runner/Plugins/Profilers/Ps.py @@ -1,12 +1,5 @@ from __future__ import annotations -import enum -from collections.abc import Callable from pathlib import Path -import subprocess -import plistlib -import platform -import shutil -import time import pandas as pd from Plugins.Profilers.DataSource import CLISource, ParameterDict @@ -43,7 +36,7 @@ ("-M", "Z"): None, ("-O"): str, ("O"): str, - ("-o", "o", "--format"): str, + ("-o", "o", "--format"): list[str], ("-P"): None, ("s"): None, ("u"): None, @@ -72,36 +65,58 @@ ("-w", "w"): None, } -class Ps(CliSource): +class Ps(CLISource): parameters = ParameterDict(PS_PARAMTERS) source_name = "ps" supported_platforms = ["Linux"] """An integration of the Linux ps utility into experiment-runner as a data source plugin""" def __init__(self, - sample_frequency: int = 5000, - out_file: Path = "powerjoular.csv", - additional_args: list[str] = []): - - self.logfile = out_file - - # TODO: Convert this into a params dict - self.args = f'ps -p {self.target.pid} --noheader -o %cpu' + sleep_interval: int = 1, + out_file: Path = "ps.csv", + additional_args: dict = [], + target_pid: list[int] = None, + out_format: list[str] = ["%cpu", "%mem"]): # man 1 ps # %cpu: # cpu utilization of the process in "##.#" format. Currently, it is the CPU time used # divided by the time the process has been running (cputime/realtime ratio), expressed # as a percentage. It will not add up to 100% unless you are lucky. (alias pcpu). - wrapper_script = f''' - while true; do {profiler_cmd}; sleep 1; done - ''' + # %mem: + # How much memory the process is currently using + self.logfile = out_file + self.sleep_interval = sleep_interval + self.args = { + "--noheader": None, + "-o": out_format + } + + if target_pid: + self.update_parameters(add={"-p": target_pid}) - def parse_log(self, logfile: Path): - df = pd.DataFrame(columns=['cpu_usage']) - for i, l in enumerate(self.profiler.stdout.readlines()): - cpu_usage=float(l.decode('ascii').strip()) - df.loc[i] = [cpu_usage] + self.update_parameters(add=additional_args) + + def __format_cmd(self): + cmd = self.super().__format_cmd(); - df.to_csv(self.run_dir / 'raw_data.csv', index=False) + output_cmd = "" + if self.logfile is not None: + output_cmd = f" > {self.logfile}" + + # This wraps the ps utility so that it runs continously and also outputs into a csv like format + return f''' + sh -c "while true; do {cmd} | awk '{{$1=$1}};1' | tr ' ' ','{output_cmd}; sleep {self.sleep_interval}; done" + ''' + + # The csv saved by default has no header, this must be provided by the user + @staticmethod + def parse_log(logfile: Path, column_names: list[str]): + # Ps has many options, we dont check them all. converting to csv might fail in some cases + try: + df = pd.read_csv(logfile, names=column_names).to_dict() + except Exception as e: + print(f"Could not parse ps ouput csv: {e}") + + return df.to_dict() From 6ca2574915c4c382b4ed510914ad681dee3ee9c2 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Sun, 8 Dec 2024 17:46:15 +0100 Subject: [PATCH 11/15] Added testing files for the plugins + tested ps module --- .../Plugins/Profilers/DataSource.py | 102 ++++++++++++------ experiment-runner/Plugins/Profilers/Ps.py | 19 ++-- test/Plugins/Profilers/test_PowerJoular.py | 19 ++++ test/Plugins/Profilers/test_PowerMetrics.py | 21 ++++ test/Plugins/Profilers/test_Ps.py | 67 ++++++++++++ 5 files changed, 187 insertions(+), 41 deletions(-) create mode 100644 test/Plugins/Profilers/test_PowerJoular.py create mode 100644 test/Plugins/Profilers/test_PowerMetrics.py create mode 100755 test/Plugins/Profilers/test_Ps.py diff --git a/experiment-runner/Plugins/Profilers/DataSource.py b/experiment-runner/Plugins/Profilers/DataSource.py index 0607ee83..98a8e805 100644 --- a/experiment-runner/Plugins/Profilers/DataSource.py +++ b/experiment-runner/Plugins/Profilers/DataSource.py @@ -1,9 +1,13 @@ from abc import ABC, abstractmethod -from collections import UserDict, Iterable +from collections import UserDict +from collections.abc import Iterable # This is only valid >= python 3.10 I think from pathlib import Path +from typing import get_origin, get_args import platform +import shlex import shutil import subprocess +import signal class ParameterDict(UserDict): def valid_key(self, key): @@ -18,19 +22,19 @@ def __setitem__(self, key, item): if not self.valid_key(key): raise RuntimeError("Unexpected key value") + if isinstance(key, str): + key = self.str_to_tuple(key) + for params in self.data.keys(): if set(key).issubset(params): raise RuntimeError("Keys cannot have duplicate elements") - - if isinstance(key, str): - key = self.str_to_tuple(key) super().__setitem__(tuple(key), item) def __getitem__(self, key): if not self.valid_key(key): - raise RuntimeError("Unexpected key, expected `str` or `list[str]`") - + raise RuntimeError("Unexpected key type, expected `str` or `list[str]`") + if isinstance(key, str): key = self.str_to_tuple(key) @@ -48,12 +52,22 @@ def __delitem__(self, key): super().__delitem__(tuple(key)) + def __contains__(self, key): + if isinstance(key, str): + key = self.str_to_tuple(key) + + for params, val in self.data.items(): + if set(key).issubset(params): + return True + + return False + class DataSource(ABC): def __init__(self): - self.__validate_platform() + self._validate_platform() self.logfile = None - def __validate_platform(self): + def _validate_platform(self): if platform.system() in self.supported_platforms: return @@ -81,43 +95,68 @@ def parse_log(): class CLISource(DataSource): def __init__(self): - self.super().__init__() + super().__init__() self.process = None self.args = None + + def __del__(self): + if self.process: + self.process.kill() @property @abstractmethod def parameters(self) -> ParameterDict: pass - def __validate_platform(self): - self.super().__validate_platform() + def _validate_platform(self): + super()._validate_platform() if shutil.which(self.source_name) is None: raise RuntimeError(f"The {self.source_name} cli tool is required for this plugin") - def __validate_start(self): + def _validate_start(self): if self.process.poll() != None: raise RuntimeError(f"{self.source_name} did not start correctly") - def __validate_stop(self, stdout, stderr): + def _validate_stop(self, stdout, stderr): if stderr: - raise RuntimeWarning(f"{self.source_name} did not stop correctly, or encountered an error") + raise RuntimeWarning(f"{self.source_name} did not stop correctly, or encountered an error: {stderr}") - def __validate_parameters(self, parameters: dict): - for p, v in parameters: - if not isinstance(v, self.parameters[p]): - raise RuntimeError(f"Unexpected type: {type(v)} for parameter {p}") + # Should work well with single level type generics e.g. list[int] + # TODO: Expand this to be more robust with other types + def _validate_type(self, param, p_type): + if isinstance(param, Iterable): + if type(param) != get_origin(p_type): + return False + + if type(param[0]) != get_args(p_type)[0]: + return False + + return True + + return isinstance(param, p_type) - def __format_cmd(self): - self.__validate_parameters(self.args) + def _validate_parameters(self, parameters: dict): + for p, v in parameters.items(): + if p not in self.parameters: + raise RuntimeError(f"Unexpected parameter: {p}") + + if self.parameters[p] == None: + continue + + if not self._validate_type(v, self.parameters[p]): + raise RuntimeError(f"Unexpected type: {type(v)} for parameter {p}") + def _format_cmd(self): + self._validate_parameters(self.args) cmd = self.source_name # Transform the parameter dict into string format to be parsed by shlex for p, v in self.args.items(): - if isinstance(v, Iterable): + if v == None: + cmd += f" {p}" + elif isinstance(v, Iterable): cmd += f" {p} {",".join(map(str, v))}" else: cmd += f" {p} {v}" @@ -126,7 +165,7 @@ def __format_cmd(self): def update_parameters(self, add: dict={}, remove: list[str]=[]): # Check if the new sets of parameters are sane - self.__validate_parameters(add) + self._validate_parameters(add) for p, v in add.items(): self.args[p] = v @@ -136,36 +175,37 @@ def update_parameters(self, add: dict={}, remove: list[str]=[]): del self.args[p] # Double check that our typeing is still valid - self.__validate_parameters(self.args) + self._validate_parameters(self.args) def start(self): try: - self.process = subprocess.Popen(shutil.shlex.split(self.__format_cmd()), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + self.process = subprocess.Popen(shlex.split(self._format_cmd()), + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) except Exception as e: + self.process.kill() raise RuntimeError(f"{self.source_name} process could not start: {e}") - self.__validate_start() + self._validate_start() def stop(self): if not self.process: return try: - # most cli utilities should stop with ctrl-c - self.process.send_signal(subprocess.signal.SIGINT) + self.process.terminate() stdout, stderr = self.process.communicate(timeout=5) except Exception as e: + self.process.kill() raise RuntimeError(f"{self.source_name} process could not stop {e}") - self.__validate_stop(stdout, stderr) + self._validate_stop(stdout.decode("utf-8"), stderr.decode("utf-8")) return stdout.decode("utf-8") class DeviceSource(DataSource): def __init__(self): - self.super().__init__() + super().__init__() self.device_handle = None def __del__(self): diff --git a/experiment-runner/Plugins/Profilers/Ps.py b/experiment-runner/Plugins/Profilers/Ps.py index 07d415c2..d5bcf461 100644 --- a/experiment-runner/Plugins/Profilers/Ps.py +++ b/experiment-runner/Plugins/Profilers/Ps.py @@ -44,7 +44,7 @@ ("X"): None, ("--context"): None, ("--headers"): None, - ("--no-headers"): None, + ("--no-headers", "--noheader"): None, ("--cols", "--columns", "--width"): int, ("--rows", "--lines"): int, ("--signames"): None, @@ -74,10 +74,11 @@ class Ps(CLISource): def __init__(self, sleep_interval: int = 1, out_file: Path = "ps.csv", - additional_args: dict = [], + additional_args: dict = {}, target_pid: list[int] = None, out_format: list[str] = ["%cpu", "%mem"]): - + + super().__init__() # man 1 ps # %cpu: # cpu utilization of the process in "##.#" format. Currently, it is the CPU time used @@ -97,24 +98,22 @@ def __init__(self, self.update_parameters(add=additional_args) - def __format_cmd(self): - cmd = self.super().__format_cmd(); + def _format_cmd(self): + cmd = super()._format_cmd() output_cmd = "" if self.logfile is not None: - output_cmd = f" > {self.logfile}" + output_cmd = f" >> {self.logfile}" # This wraps the ps utility so that it runs continously and also outputs into a csv like format - return f''' - sh -c "while true; do {cmd} | awk '{{$1=$1}};1' | tr ' ' ','{output_cmd}; sleep {self.sleep_interval}; done" - ''' + return f'''sh -c "while true; do {cmd} | awk '{{$1=$1}};1' | tr ' ' ','{output_cmd}; sleep {self.sleep_interval}; done"''' # The csv saved by default has no header, this must be provided by the user @staticmethod def parse_log(logfile: Path, column_names: list[str]): # Ps has many options, we dont check them all. converting to csv might fail in some cases try: - df = pd.read_csv(logfile, names=column_names).to_dict() + df = pd.read_csv(logfile, names=column_names) except Exception as e: print(f"Could not parse ps ouput csv: {e}") diff --git a/test/Plugins/Profilers/test_PowerJoular.py b/test/Plugins/Profilers/test_PowerJoular.py new file mode 100644 index 00000000..e1806ddd --- /dev/null +++ b/test/Plugins/Profilers/test_PowerJoular.py @@ -0,0 +1,19 @@ +import os +import unittest +import shutil +import tempfile + +from Plugins.Profilers.PowerJoular import PowerJoular + +class TestPowerJoular(unittest.TestCase): + def test_update(self): + pass + + def test_invalid_update(self): + pass + + def test_run(self): + pass + +if __name__ == '__main__': + unittest.main() diff --git a/test/Plugins/Profilers/test_PowerMetrics.py b/test/Plugins/Profilers/test_PowerMetrics.py new file mode 100644 index 00000000..9062beab --- /dev/null +++ b/test/Plugins/Profilers/test_PowerMetrics.py @@ -0,0 +1,21 @@ +import os +import unittest +import shutil +import tempfile +import sys + +sys.path.append("experiment-runner") +from Plugins.Profilers.PowerMetrics import PowerMetrics + +class TestPowerMetrics(unittest.TestCase): + def test_update(self): + pass + + def test_invalid_update(self): + pass + + def test_run(self): + pass + +if __name__ == '__main__': + unittest.main() diff --git a/test/Plugins/Profilers/test_Ps.py b/test/Plugins/Profilers/test_Ps.py new file mode 100755 index 00000000..d38513ca --- /dev/null +++ b/test/Plugins/Profilers/test_Ps.py @@ -0,0 +1,67 @@ +import os +import time +import unittest +import sys +import psutil + +sys.path.append("experiment-runner") +from Plugins.Profilers.Ps import Ps + +class TestPs(unittest.TestCase): + def test_update(self): + ps = Ps() + original_args = ps.args.copy() + + ps.update_parameters(add={"-e": None}) + self.assertIn(("-e", None), ps.args.items()) + + ps.update_parameters(remove=["-e"]) + self.assertDictEqual(original_args, ps.args) + + ps.update_parameters(add={"--cols": 2}) + self.assertIn(("--cols", 2), ps.args.items()) + + ps.update_parameters(add={"-p": [1,2,3]}) + self.assertIn(("-p", [1,2,3]), ps.args.items()) + + def test_invalid_update(self): + ps = Ps() + + with self.assertRaises(RuntimeError): + ps.update_parameters(add={"--not-a-valid-parameter": None}) + + original_args = ps.args.copy() + + # This should be a null op + ps.update_parameters(remove=["--not-a-valid-parameter"]) + self.assertDictEqual(original_args, ps.args) + + with self.assertRaises(RuntimeError): + ps.update_parameters(add={"--cols": "not the correct type"}) + + with self.assertRaises(RuntimeError): + ps.update_parameters(add={"-p": ["not", "correct", "type"]}) + + + def test_run(self): + valid_pid = psutil.pids()[1] # Could possibly fail if the process finishes before we test + test_outfile = "/tmp/ps_test_out.csv" + ps = Ps(out_file=test_outfile, target_pid=[valid_pid]) + + sleep_len = 2 + headers = ps.args["-o"] + + ps.start() + time.sleep(sleep_len) + ps.stop() + + # We should see 5 entries in the log + log = ps.parse_log(test_outfile, headers) + + for hdr in headers: + self.assertEqual(len(log[hdr]), sleep_len) + + os.remove(test_outfile) + +if __name__ == '__main__': + unittest.main() From 39d97760534641752d9efd11217c2063fd85162f Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Mon, 9 Dec 2024 17:48:28 +0100 Subject: [PATCH 12/15] More testing --- .../Plugins/Profilers/DataSource.py | 9 +-- .../Plugins/Profilers/PowerJoular.py | 12 ++- .../Plugins/Profilers/PowerMetrics.py | 4 +- test/Plugins/Profilers/test_PowerJoular.py | 73 ++++++++++++++++-- test/Plugins/Profilers/test_PowerMetrics.py | 74 +++++++++++++++++-- test/Plugins/Profilers/test_Ps.py | 59 ++++++++------- 6 files changed, 184 insertions(+), 47 deletions(-) diff --git a/experiment-runner/Plugins/Profilers/DataSource.py b/experiment-runner/Plugins/Profilers/DataSource.py index 98a8e805..c73adcd2 100644 --- a/experiment-runner/Plugins/Profilers/DataSource.py +++ b/experiment-runner/Plugins/Profilers/DataSource.py @@ -1,13 +1,12 @@ from abc import ABC, abstractmethod from collections import UserDict -from collections.abc import Iterable # This is only valid >= python 3.10 I think +from collections.abc import Iterable # This import is only valid >= python 3.10 I think from pathlib import Path from typing import get_origin, get_args import platform import shlex import shutil import subprocess -import signal class ParameterDict(UserDict): def valid_key(self, key): @@ -126,7 +125,7 @@ def _validate_stop(self, stdout, stderr): # Should work well with single level type generics e.g. list[int] # TODO: Expand this to be more robust with other types def _validate_type(self, param, p_type): - if isinstance(param, Iterable): + if p_type != str and isinstance(param, Iterable): if type(param) != get_origin(p_type): return False @@ -134,7 +133,7 @@ def _validate_type(self, param, p_type): return False return True - + return isinstance(param, p_type) def _validate_parameters(self, parameters: dict): @@ -146,7 +145,7 @@ def _validate_parameters(self, parameters: dict): continue if not self._validate_type(v, self.parameters[p]): - raise RuntimeError(f"Unexpected type: {type(v)} for parameter {p}") + raise RuntimeError(f"Unexpected type: {type(v)} for parameter {p}, expected {self.parameters[p]}") def _format_cmd(self): self._validate_parameters(self.args) diff --git a/experiment-runner/Plugins/Profilers/PowerJoular.py b/experiment-runner/Plugins/Profilers/PowerJoular.py index 271e8ed3..a3957fd5 100644 --- a/experiment-runner/Plugins/Profilers/PowerJoular.py +++ b/experiment-runner/Plugins/Profilers/PowerJoular.py @@ -26,17 +26,25 @@ def __init__(self, out_file: Path = "powerjoular.csv", additional_args: dict = {}, target_pid: int = None): - + + super().__init__() self.logfile = out_file self.args = { "-l": None, - "-f": self.logfile, + "-f": Path(self.logfile), } if target_pid: self.update_parameters(add={"-p": target_pid}) self.update_parameters(add=additional_args) + + @property + def target_logfile(self): + if "-p" in self.args.keys(): + return f"{self.logfile}-{self.args["-p"]}.csv" + + return None @staticmethod def parse_log(logfile: Path): diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index b7911aea..fa20217c 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -82,7 +82,7 @@ def __init__(self, additional_samplers: list[PMSampleTypes] = [], hide_cpu_duty_cycle: bool = True, order: PMOrderTypes = PMOrderTypes.PM_ORDER_CPU): - + super().__init__() self.logfile = out_file # Grab all available power stats by default self.args = { @@ -99,7 +99,7 @@ def __init__(self, self.update_parameters(add=additional_args) @staticmethod - def get_plist_power(pm_plists: list[dict]): + def parse_plist_power(pm_plists: list[dict]): """ Extracts from a list of plists, the relavent power statistics if present. If no power stats are present, this returns an empty list. This is mainly a helper method, diff --git a/test/Plugins/Profilers/test_PowerJoular.py b/test/Plugins/Profilers/test_PowerJoular.py index e1806ddd..fafa627d 100644 --- a/test/Plugins/Profilers/test_PowerJoular.py +++ b/test/Plugins/Profilers/test_PowerJoular.py @@ -1,19 +1,78 @@ import os import unittest -import shutil -import tempfile +import time +import psutil +import sys +sys.path.append("experiment-runner") from Plugins.Profilers.PowerJoular import PowerJoular class TestPowerJoular(unittest.TestCase): + def tearDown(self): + if self.plugin is None: + return + + if self.plugin.target_logfile \ + and os.path.exists(self.plugin.target_logfile): + + os.remove(self.plugin.target_logfile) + + if os.path.exists(self.plugin.logfile): + os.remove(self.plugin.logfile) + + self.plugin = None + def test_update(self): - pass - + self.plugin = PowerJoular() + original_args = self.plugin.args.copy() + + self.plugin.update_parameters(add={"-t": None}) + self.assertIn(("-t", None), self.plugin.args.items()) + + self.plugin.update_parameters(remove=["-t"]) + self.assertDictEqual(original_args, self.plugin.args) + + self.plugin.update_parameters(add={"-a": "program"}) + self.assertIn(("-a", "program"), self.plugin.args.items()) + def test_invalid_update(self): - pass - + self.plugin = PowerJoular() + + with self.assertRaises(RuntimeError): + self.plugin.update_parameters(add={"--not-a-valid-parameter": None}) + + original_args = self.plugin.args.copy() + + # This should be a null op + self.plugin.update_parameters(remove=["--not-a-valid-parameter"]) + self.assertDictEqual(original_args, self.plugin.args) + + with self.assertRaises(RuntimeError): + self.plugin.update_parameters(add={"-p": "not the correct type"}) + def test_run(self): - pass + valid_pid = psutil.pids()[0] # Could possibly fail if the process finishes before we test + test_outfile = "/tmp/pj_test_out.csv" + self.plugin = PowerJoular(out_file=test_outfile, target_pid=valid_pid) + + sleep_len = 2 + + self.plugin.start() + time.sleep(sleep_len) + self.plugin.stop() + + self.assertTrue(os.path.exists(test_outfile)) + self.assertTrue(os.path.exists(self.plugin.target_logfile)) + + # We should see sleep_len - 1 entries in the log + log = self.plugin.parse_log(test_outfile) + target_log = self.plugin.parse_log(self.plugin.target_logfile) + + for k, v in log.items(): + self.assertEqual(len(v), sleep_len - 1) + + for k, v in target_log.items(): + self.assertEqual(len(v), sleep_len - 1) if __name__ == '__main__': unittest.main() diff --git a/test/Plugins/Profilers/test_PowerMetrics.py b/test/Plugins/Profilers/test_PowerMetrics.py index 9062beab..e71a924b 100644 --- a/test/Plugins/Profilers/test_PowerMetrics.py +++ b/test/Plugins/Profilers/test_PowerMetrics.py @@ -1,21 +1,83 @@ import os import unittest -import shutil -import tempfile +import time import sys sys.path.append("experiment-runner") from Plugins.Profilers.PowerMetrics import PowerMetrics +from Plugins.Profilers.PowerMetrics import PMSampleTypes class TestPowerMetrics(unittest.TestCase): + def tearDown(self): + if self.plugin is None: + return + + if os.path.exists(self.plugin.logfile): + os.remove(self.plugin.logfile) + + self.plugin = None + def test_update(self): - pass - + self.plugin = PowerMetrics() + original_args = self.plugin.args.copy() + + self.plugin.update_parameters(add={"--show-process-qos": None}) + self.assertIn(("-t", None), self.plugin.args.items()) + + self.plugin.update_parameters(remove=["--show-process-qos"]) + self.assertDictEqual(original_args, self.plugin.args) + + self.plugin.update_parameters(add={"--unhide-info": + [PMSampleTypes.PM_SAMPLE_TASKS, PMSampleTypes.PM_SAMPLE_SFI]}) + self.assertIn(("--unhide-info", + [PMSampleTypes.PM_SAMPLE_TASKS, PMSampleTypes.PM_SAMPLE_SFI]), self.plugin.args.items()) + def test_invalid_update(self): - pass + self.plugin = PowerMetrics() + + with self.assertRaises(RuntimeError): + self.plugin.update_parameters(add={"--not-a-valid-parameter": None}) + + original_args = self.plugin.args.copy() + + # This should be a null op + self.plugin.update_parameters(remove=["--not-a-valid-parameter"]) + self.assertDictEqual(original_args, self.plugin.args) + + with self.assertRaises(RuntimeError): + self.plugin.update_parameters(add={"--unhide-info": ["not", "correct", "type"]}) def test_run(self): - pass + test_outfile = "/tmp/pm_test_out.csv" + self.plugin = PowerMetrics(out_file=test_outfile, sample_frequency=1000) + + sleep_len = 2 + + self.plugin.start() + time.sleep(sleep_len) + self.plugin.stop() + + self.assertTrue(os.path.exists(test_outfile)) + + log = self.plugin.parse_log(test_outfile) + power_data = self.plugin.parse_plist_power(log) + + # powermetrics returns a seperate plist for each measurement + self.assertEqual(len(log), sleep_len/(self.plugin.sample_frequency/1000)) + + # Make sure we have results from each sampler + for sampler in map(lambda x: x.value, self.plugin.args["--samplers"]): + # As names of samplers can differ from names of the data headers, we approximate this a bit. + for l in log: + self.assertTrue(filter(lambda x: sampler.lower() in x.lower(), l.keys()) > 0) + + # Check that our power data filter is also working + for l in power_data: + self.assertIn("GPU", l.keys()) + self.assertIn("agpm_stats", l.keys()) + self.assertIn("processor", l.keys()) + self.assertIn("timestamp", l.keys()) + if __name__ == '__main__': unittest.main() diff --git a/test/Plugins/Profilers/test_Ps.py b/test/Plugins/Profilers/test_Ps.py index d38513ca..e03b309c 100755 --- a/test/Plugins/Profilers/test_Ps.py +++ b/test/Plugins/Profilers/test_Ps.py @@ -8,60 +8,69 @@ from Plugins.Profilers.Ps import Ps class TestPs(unittest.TestCase): + def tearDown(self): + if self.plugin is None: + return + + if os.path.exists(self.plugin.logfile): + os.remove(self.plugin.logfile) + + self.plugin = None + def test_update(self): - ps = Ps() - original_args = ps.args.copy() + self.plugin = Ps() + original_args = self.plugin.args.copy() - ps.update_parameters(add={"-e": None}) - self.assertIn(("-e", None), ps.args.items()) + self.plugin.update_parameters(add={"-e": None}) + self.assertIn(("-e", None), self.plugin.args.items()) - ps.update_parameters(remove=["-e"]) - self.assertDictEqual(original_args, ps.args) + self.plugin.update_parameters(remove=["-e"]) + self.assertDictEqual(original_args, self.plugin.args) - ps.update_parameters(add={"--cols": 2}) - self.assertIn(("--cols", 2), ps.args.items()) + self.plugin.update_parameters(add={"--cols": 2}) + self.assertIn(("--cols", 2), self.plugin.args.items()) - ps.update_parameters(add={"-p": [1,2,3]}) - self.assertIn(("-p", [1,2,3]), ps.args.items()) + self.plugin.update_parameters(add={"-p": [1,2,3]}) + self.assertIn(("-p", [1,2,3]), self.plugin.args.items()) def test_invalid_update(self): - ps = Ps() + self.plugin = Ps() with self.assertRaises(RuntimeError): - ps.update_parameters(add={"--not-a-valid-parameter": None}) + self.plugin.update_parameters(add={"--not-a-valid-parameter": None}) - original_args = ps.args.copy() + original_args = self.plugin.args.copy() # This should be a null op - ps.update_parameters(remove=["--not-a-valid-parameter"]) - self.assertDictEqual(original_args, ps.args) + self.plugin.update_parameters(remove=["--not-a-valid-parameter"]) + self.assertDictEqual(original_args, self.plugin.args) with self.assertRaises(RuntimeError): - ps.update_parameters(add={"--cols": "not the correct type"}) + self.plugin.update_parameters(add={"--cols": "not the correct type"}) with self.assertRaises(RuntimeError): - ps.update_parameters(add={"-p": ["not", "correct", "type"]}) + self.plugin.update_parameters(add={"-p": ["not", "correct", "type"]}) def test_run(self): valid_pid = psutil.pids()[1] # Could possibly fail if the process finishes before we test test_outfile = "/tmp/ps_test_out.csv" - ps = Ps(out_file=test_outfile, target_pid=[valid_pid]) + self.plugin = Ps(out_file=test_outfile, target_pid=[valid_pid]) sleep_len = 2 - headers = ps.args["-o"] + headers = self.plugin.args["-o"] - ps.start() + self.plugin.start() time.sleep(sleep_len) - ps.stop() + self.plugin.stop() - # We should see 5 entries in the log - log = ps.parse_log(test_outfile, headers) + self.assertTrue(os.path.exists(test_outfile)) + + # We should see 2 entries in the log + log = self.plugin.parse_log(test_outfile, headers) for hdr in headers: self.assertEqual(len(log[hdr]), sleep_len) - os.remove(test_outfile) - if __name__ == '__main__': unittest.main() From b5ebb3aef164077ec35066e65c1193bd3b4fbaea Mon Sep 17 00:00:00 2001 From: b383229 Date: Mon, 9 Dec 2024 20:01:22 +0100 Subject: [PATCH 13/15] new stuff --- .../powermetrics-profiling/RunnerConfig.py | 4 +- .../Plugins/Profilers/DataSource.py | 5 +- .../Plugins/Profilers/PowerMetrics.py | 51 ++++++++++--------- test/Plugins/Profilers/test_PowerMetrics.py | 26 ++++++---- 4 files changed, 47 insertions(+), 39 deletions(-) diff --git a/examples/powermetrics-profiling/RunnerConfig.py b/examples/powermetrics-profiling/RunnerConfig.py index 1657cbfe..417aeccb 100644 --- a/examples/powermetrics-profiling/RunnerConfig.py +++ b/examples/powermetrics-profiling/RunnerConfig.py @@ -115,8 +115,8 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: # Parse it as required for your experiment and add it to the run table return { "joules": sum(map(lambda x: x["processor"]["package_joules"], run_results)), - "avg_cpu": np.mean(map(lambda x: x["processor"]["packages"]["cores_active_ratio"], run_results)), - "avg_gpu": np.mean(map(lambda x: x["processor"]["packages"]["gpu_active_ratio"], run_results)), + "avg_cpu": np.mean(list(map(lambda x: x["processor"]["packages"][0]["cores_active_ratio"], run_results))), + "avg_gpu": np.mean(list(map(lambda x: x["processor"]["packages"][0]["gpu_active_ratio"], run_results))), } def after_experiment(self) -> None: diff --git a/experiment-runner/Plugins/Profilers/DataSource.py b/experiment-runner/Plugins/Profilers/DataSource.py index c73adcd2..e4c5891c 100644 --- a/experiment-runner/Plugins/Profilers/DataSource.py +++ b/experiment-runner/Plugins/Profilers/DataSource.py @@ -5,6 +5,7 @@ from typing import get_origin, get_args import platform import shlex +from enum import StrEnum import shutil import subprocess @@ -125,7 +126,7 @@ def _validate_stop(self, stdout, stderr): # Should work well with single level type generics e.g. list[int] # TODO: Expand this to be more robust with other types def _validate_type(self, param, p_type): - if p_type != str and isinstance(param, Iterable): + if p_type != str and not isinstance(param, StrEnum) and isinstance(param, Iterable): if type(param) != get_origin(p_type): return False @@ -155,7 +156,7 @@ def _format_cmd(self): for p, v in self.args.items(): if v == None: cmd += f" {p}" - elif isinstance(v, Iterable): + elif isinstance(v, Iterable) and not (isinstance(v, StrEnum) or isinstance(v, str)): cmd += f" {p} {",".join(map(str, v))}" else: cmd += f" {p} {v}" diff --git a/experiment-runner/Plugins/Profilers/PowerMetrics.py b/experiment-runner/Plugins/Profilers/PowerMetrics.py index fa20217c..e94a448a 100644 --- a/experiment-runner/Plugins/Profilers/PowerMetrics.py +++ b/experiment-runner/Plugins/Profilers/PowerMetrics.py @@ -1,24 +1,24 @@ from __future__ import annotations -import enum +from enum import StrEnum from pathlib import Path import plistlib from Plugins.Profilers.DataSource import ParameterDict, CLISource # How to format the output -class PMFormatTypes(enum.Enum): +class PMFormatTypes(StrEnum): PM_FMT_TEXT = "text" PM_FMT_PLIST = "plist" # How to order results -class PMOrderTypes(enum.Enum): +class PMOrderTypes(StrEnum): PM_ORDER_PID = "pid" PM_ORDER_WAKEUP = "wakeups" PM_ORDER_CPU = "cputime" PM_ORDER_HYBRID = "composite" # Which sources to sample from -class PMSampleTypes(enum.Enum): +class PMSampleTypes(StrEnum): PM_SAMPLE_TASKS = "tasks" # per task cpu usage and wakeup stats PM_SAMPLE_BAT = "battery" # battery and backlight info PM_SAMPLE_NET = "network" # network usage info @@ -72,7 +72,7 @@ class PMSampleTypes(enum.Enum): class PowerMetrics(CLISource): parameters = ParameterDict(POWERMETRICS_PARAMETERS) source_name = "powermetrics" - supported_platforms = ["OS X"] + supported_platforms = ["Darwin"] """An integration of OSX powermetrics into experiment-runner as a data source plugin""" def __init__(self, @@ -86,14 +86,14 @@ def __init__(self, self.logfile = out_file # Grab all available power stats by default self.args = { - "--output-file": self.logfile, - "--sample-interval": sample_frequency, - "--format": PMFormatTypes.PM_FMT_PLIST.value, + "--output-file": Path(self.logfile), + "--sample-rate": sample_frequency, + "--format": PMFormatTypes.PM_FMT_PLIST, "--samplers": [PMSampleTypes.PM_SAMPLE_CPU_POWER, PMSampleTypes.PM_SAMPLE_GPU_POWER, PMSampleTypes.PM_SAMPLE_AGPM] + additional_samplers, "--hide-cpu-duty-cycle": hide_cpu_duty_cycle, - "--order": order.value + "--order": order } self.update_parameters(add=additional_args) @@ -117,8 +117,9 @@ def parse_plist_power(pm_plists: list[dict]): stats = {} if "GPU" in plist.keys(): stats["GPU"] = plist["GPU"].copy() - del stats["GPU"]["misc_counters"] - del stats["GPU"]["pstates"] + for gpu in range(len(stats["GPU"])): + del stats["GPU"][gpu]["misc_counters"] + del stats["GPU"][gpu]["p_states"] if "processor" in plist.keys(): stats["processor"] = plist["processor"] @@ -130,7 +131,7 @@ def parse_plist_power(pm_plists: list[dict]): if "timestamp" in plist.keys(): stats["timestamp"] = plist["timestamp"] - power_plists.apend(stats) + power_plists.append(stats) return power_plists @@ -145,20 +146,20 @@ def parse_log(logfile: Path): logfile (Path): The path to the plist logfile created by powermetrics Returns: - A list of dicts, each representing the plist for a given sample + A list of dicts, each representing the plist for a given sample """ - fp = open(logfile, "rb") - plists = [] cur_plist = bytearray() - for l in fp.readlines(): - # Powermetrics outputs plists with null bytes inbetween. We account for this - if l[0] == 0: - plists.append(plistlib.loads(cur_plist)) - - cur_plist = bytearray() - cur_plist.extend(l[1:]) - else: - cur_plist.extend(l) - + with open(logfile, "rb") as fp: + for l in fp.readlines(): + # Powermetrics outputs plists with null bytes inbetween. We account for this + if l[0] == 0: + cur_plist.extend(l[1:]) + else: + cur_plist.extend(l) + + if b"\n" in l: + plists.append(plistlib.loads(cur_plist)) + cur_plist = bytearray() + return plists diff --git a/test/Plugins/Profilers/test_PowerMetrics.py b/test/Plugins/Profilers/test_PowerMetrics.py index e71a924b..4ca8241d 100644 --- a/test/Plugins/Profilers/test_PowerMetrics.py +++ b/test/Plugins/Profilers/test_PowerMetrics.py @@ -2,27 +2,28 @@ import unittest import time import sys +from pprint import pprint sys.path.append("experiment-runner") from Plugins.Profilers.PowerMetrics import PowerMetrics from Plugins.Profilers.PowerMetrics import PMSampleTypes class TestPowerMetrics(unittest.TestCase): - def tearDown(self): - if self.plugin is None: - return + # def tearDown(self): + # if self.plugin is None: + # return - if os.path.exists(self.plugin.logfile): - os.remove(self.plugin.logfile) + # if os.path.exists(self.plugin.logfile): + # os.remove(self.plugin.logfile) - self.plugin = None + # self.plugin = None def test_update(self): self.plugin = PowerMetrics() original_args = self.plugin.args.copy() self.plugin.update_parameters(add={"--show-process-qos": None}) - self.assertIn(("-t", None), self.plugin.args.items()) + self.assertIn(("--show-process-qos", None), self.plugin.args.items()) self.plugin.update_parameters(remove=["--show-process-qos"]) self.assertDictEqual(original_args, self.plugin.args) @@ -63,13 +64,18 @@ def test_run(self): power_data = self.plugin.parse_plist_power(log) # powermetrics returns a seperate plist for each measurement - self.assertEqual(len(log), sleep_len/(self.plugin.sample_frequency/1000)) - + self.assertEqual(len(log), (sleep_len/(self.plugin.args["--sample-rate"]/1000)) - 1) + # Make sure we have results from each sampler for sampler in map(lambda x: x.value, self.plugin.args["--samplers"]): + # TODO: This doesnt properly check all results, only for the parameters used in this test # As names of samplers can differ from names of the data headers, we approximate this a bit. for l in log: - self.assertTrue(filter(lambda x: sampler.lower() in x.lower(), l.keys()) > 0) + if "cpu_power" in sampler: + self.assertIn("package_joules", l["processor"].keys()) + self.assertIn("package_watts", l["processor"].keys()) + else: + self.assertTrue(len(list(filter(lambda x: sampler.lower() in x.lower() or x.lower() in sampler.lower(), l.keys()))) > 0) # Check that our power data filter is also working for l in power_data: From 6b6e5c11d8586edb1103328323b8f767c70a2747 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Tue, 10 Dec 2024 14:37:04 +0100 Subject: [PATCH 14/15] Tested runner cofigs + adjustment so runtable is readable even when running as root --- .../RunnerConfig.py | 20 ++++++++----------- examples/linux-ps-profiling/RunnerConfig.py | 12 +++++------ .../Output/CSVOutputManager.py | 9 ++++++++- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/examples/linux-powerjoular-profiling/RunnerConfig.py b/examples/linux-powerjoular-profiling/RunnerConfig.py index 3aa464bc..aeb613dd 100644 --- a/examples/linux-powerjoular-profiling/RunnerConfig.py +++ b/examples/linux-powerjoular-profiling/RunnerConfig.py @@ -5,18 +5,16 @@ from ConfigValidator.Config.Models.RunnerContext import RunnerContext from ConfigValidator.Config.Models.OperationType import OperationType from ProgressManager.Output.OutputProcedure import OutputProcedure as output + from Plugins.Profilers.PowerJoular import PowerJoular from typing import Dict, List, Any, Optional from pathlib import Path from os.path import dirname, realpath -import os -import signal -import pandas as pd import time import subprocess -import shlex +import numpy as np class RunnerConfig: ROOT_DIR = Path(dirname(realpath(__file__))) @@ -89,7 +87,7 @@ def start_run(self, context: RunnerContext) -> None: ) # Configure the environment based on the current variation - subprocess.check_call(shlex.split(f'cpulimit -b -p {self.target.pid} --limit {cpu_limit}')) + subprocess.check_call(f'cpulimit -p {self.target.pid} --limit {cpu_limit} &', shell=True) time.sleep(1) # allow the process to run a little before measuring @@ -100,7 +98,7 @@ def start_measurement(self, context: RunnerContext) -> None: self.meter = PowerJoular(target_pid=self.target.pid, out_file=context.run_dir / "powerjoular.csv") # Start measuring with powerjoular - stdout = self.meter.start() + self.meter.start() def interact(self, context: RunnerContext) -> None: """Perform any interaction with the running target system here, or block here until the target finishes.""" @@ -133,13 +131,11 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: results_global = self.meter.parse_log(out_file) # If you specified a target_pid or used the -p paramter # a second csv for that target will be generated - # results_process = self.meter.parse_log(f"{out_file}-{self.target.pid}.csv" ) - - run_data = { - 'avg_cpu': round(results_global['CPU Utilization'].sum(), 3), - 'total_energy': round(results_global['CPU Power'].sum(), 3), + # results_process = self.meter.parse_log(self.meter.target_logfile) + return { + 'avg_cpu': round(np.mean(list(results_global['CPU Utilization'].values())), 3), + 'total_energy': round(sum(list(results_global['CPU Power'].values())), 3), } - return run_data def after_experiment(self) -> None: """Perform any activity required after stopping the experiment here diff --git a/examples/linux-ps-profiling/RunnerConfig.py b/examples/linux-ps-profiling/RunnerConfig.py index eaa88c39..b686308d 100644 --- a/examples/linux-ps-profiling/RunnerConfig.py +++ b/examples/linux-ps-profiling/RunnerConfig.py @@ -5,7 +5,7 @@ from ConfigValidator.Config.Models.RunnerContext import RunnerContext from ConfigValidator.Config.Models.OperationType import OperationType from ProgressManager.Output.OutputProcedure import OutputProcedure as output -from Plugins.Profilers.PowerJoular import PS +from Plugins.Profilers.Ps import Ps from typing import Dict, List, Any, Optional from pathlib import Path @@ -99,7 +99,7 @@ def start_run(self, context: RunnerContext) -> None: subprocess.check_call(shlex.split(f'taskset -cp 0 {self.target.pid}')) # Limit the targets cputime - subprocess.check_call(shlex.split(f'cpulimit -b -p {self.target.pid} --limit {cpu_limit}')) + subprocess.check_call(f'cpulimit --limit={cpu_limit} -p {self.target.pid} &', shell=True) time.sleep(1) # allow the process to run a little before measuring @@ -107,8 +107,8 @@ def start_measurement(self, context: RunnerContext) -> None: """Perform any activity required for starting measurements.""" # Set up the ps object, provide an (optional) target and output file name - self.meter = PS(out_file=context.run_dir / "ps.csv", - target_pid=self.target.pid) + self.meter = Ps(out_file=context.run_dir / "ps.csv", + target_pid=[self.target.pid]) # Start measuring with ps self.meter.start() @@ -142,8 +142,8 @@ def populate_run_data(self, context: RunnerContext) -> Optional[Dict[str, Any]]: column_names=["cpu_usage", "memory_usage"]) return { - "avg_cpu": round(np.mean(results['cpu_usage']), 3), - "avg_mem": round(np.mean(results['memory_usage']), 3) + "avg_cpu": round(np.mean(list(results['cpu_usage'].values())), 3), + "avg_mem": round(np.mean(list(results['memory_usage'].values())), 3) } def after_experiment(self) -> None: diff --git a/experiment-runner/ProgressManager/Output/CSVOutputManager.py b/experiment-runner/ProgressManager/Output/CSVOutputManager.py index c42c9fde..fd7d1b50 100644 --- a/experiment-runner/ProgressManager/Output/CSVOutputManager.py +++ b/experiment-runner/ProgressManager/Output/CSVOutputManager.py @@ -6,6 +6,8 @@ from tempfile import NamedTemporaryFile import shutil import csv +import os +import pwd from typing import Dict, List @@ -62,6 +64,11 @@ def update_row_data(self, updated_row: dict): writer.writerow(row) shutil.move(tempfile.name, self._experiment_path / 'run_table.csv') + + # Change permissions so the files can be accessed if run as root (needed for some plugins) + user = pwd.getpwnam(os.getlogin()) + os.chown(self._experiment_path / "run_table.csv", user.pw_uid, user.pw_gid) + output.console_log_WARNING(f"CSVManager: Updated row {updated_row['__run_id']}") # with open(self.experiment_path + '/run_table.csv', 'w', newline='') as myfile: @@ -72,4 +79,4 @@ def update_row_data(self, updated_row: dict): # if name == row['name']: # row['name'] = input("enter new name for {}".format(name)) # # write the row either way - # writer.writerow({'name': row['name'], 'number': row['number'], 'address': row['address']}) \ No newline at end of file + # writer.writerow({'name': row['name'], 'number': row['number'], 'address': row['address']}) From 8cbc6597148d67cee71ac650a3210dcbc91a0053 Mon Sep 17 00:00:00 2001 From: Max Karsten Date: Tue, 10 Dec 2024 14:49:14 +0100 Subject: [PATCH 15/15] Update to READMEs --- .../linux-powerjoular-profiling/README.md | 3 ++- examples/powermetrics-profiling/README.md | 25 +++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/examples/linux-powerjoular-profiling/README.md b/examples/linux-powerjoular-profiling/README.md index 974dd0f2..5c85d61d 100644 --- a/examples/linux-powerjoular-profiling/README.md +++ b/examples/linux-powerjoular-profiling/README.md @@ -20,9 +20,10 @@ pip install -r requirements.txt ## Running From the root directory of the repo, run the following command: +NOTE: This program must be run as root, as powerjoular requires this for its use of Intel RAPL. ```bash -python experiment-runner/ examples/linux-powerjoular-profiling/RunnerConfig.py +sudo python experiment-runner/ examples/linux-powerjoular-profiling/RunnerConfig.py ``` ## Results diff --git a/examples/powermetrics-profiling/README.md b/examples/powermetrics-profiling/README.md index e69de29b..c63c0ab5 100644 --- a/examples/powermetrics-profiling/README.md +++ b/examples/powermetrics-profiling/README.md @@ -0,0 +1,25 @@ + +# `powermetrics` profiler + +A simple example using the OS X [powermetrics](https://developer.apple.com/library/archive/documentation/Performance/Conceptual/power_efficiency_guidelines_osx/PrioritizeWorkAtTheTaskLevel.html#//apple_ref/doc/uid/TP40013929-CH35-SW10) cli tool to measure the ambient power consumtption of the system. + +## Requirements + +Install the requirements to run: + +```bash +pip install -r requirements.txt +``` + +## Running + +From the root directory of the repo, run the following command: +NOTE: This program must be run as root, as powermetrics requires this + +```bash +sudo python experiment-runner/ examples/powermetrics-profiling/RunnerConfig.py +``` + +## Results + +The results are generated in the `examples/powermetrics-profiling/experiments` folder.