diff --git a/client/src/components/History/CurrentHistory/HistoryOperations/SelectionOperations.vue b/client/src/components/History/CurrentHistory/HistoryOperations/SelectionOperations.vue
index 0569f4b06cf4..9b70de885019 100644
--- a/client/src/components/History/CurrentHistory/HistoryOperations/SelectionOperations.vue
+++ b/client/src/components/History/CurrentHistory/HistoryOperations/SelectionOperations.vue
@@ -91,6 +91,7 @@
id="change-dbkey-of-selected-content"
title="Change Database/Build?"
title-tag="h2"
+ body-class="modal-with-selector"
@ok="changeDbkeyOfSelected">
-
+
View Report
> {metrics}; cat "$f" >> {metrics} 2>/dev/null;
fi;
done;
-fi
-""".replace(
- "\n", " "
-).strip()
-MEMORY_USAGE_TEMPLATE = """
-if [ -e "/proc/$$/cgroup" -a -d "{cgroup_mount}" ]; then
cgroup_path=$(cat "/proc/$$/cgroup" | awk -F':' '$2=="memory"{{print $3}}');
if [ ! -e "{cgroup_mount}/memory$cgroup_path/memory.max_usage_in_bytes" ]; then
cgroup_path="";
@@ -61,6 +97,16 @@
""".replace(
"\n", " "
).strip()
+CGROUPSV2_TEMPLATE = r"""
+if [ -e "/proc/$$/cgroup" -a -f "{cgroup_mount}/cgroup.controllers" ]; then
+ cgroup_path=$(cat "/proc/$$/cgroup" | awk -F':' '($1=="0") {{print $3}}');
+ for f in {cgroup_mount}/${{cgroup_path}}/{{cpu,memory}}.*; do
+ echo "__$(basename $f)__" >> {metrics}; cat "$f" >> {metrics} 2>/dev/null;
+ done;
+fi
+""".replace(
+ "\n", " "
+).strip()
Metric = namedtuple("Metric", ("key", "subkey", "value"))
@@ -76,7 +122,7 @@ def format(self, key, value):
return title, nice_size(value)
except ValueError:
pass
- elif isinstance(value, (numbers.Integral, numbers.Real)) and value == int(value):
+ elif isinstance(value, (decimal.Decimal, numbers.Integral, numbers.Real)) and value == int(value):
value = int(value)
return title, value
@@ -90,33 +136,36 @@ class CgroupPlugin(InstrumentPlugin):
def __init__(self, **kwargs):
self.verbose = asbool(kwargs.get("verbose", False))
self.cgroup_mount = kwargs.get("cgroup_mount", "/sys/fs/cgroup")
+ self.version = str(kwargs.get("version", "auto"))
+ assert self.version in VALID_VERSIONS, f"cgroup metric version option must be one of {VALID_VERSIONS}"
params_str = kwargs.get("params", None)
if isinstance(params_str, list):
params = params_str
elif params_str:
params = [v.strip() for v in params_str.split(",")]
else:
- params = list(TITLES.keys())
+ params = list(DEFAULT_PARAMS)
self.params = params
def post_execute_instrument(self, job_directory: str) -> List[str]:
commands: List[str] = []
- commands.append(self.__record_cgroup_cpu_usage(job_directory))
- commands.append(self.__record_cgroup_memory_usage(job_directory))
+ if self.version in ("auto", "1"):
+ commands.append(self.__record_cgroup_v1_usage(job_directory))
+ if self.version in ("auto", "2"):
+ commands.append(self.__record_cgroup_v2_usage(job_directory))
return commands
def job_properties(self, job_id, job_directory: str) -> Dict[str, Any]:
metrics = self.__read_metrics(self.__cgroup_metrics_file(job_directory))
return metrics
- def __record_cgroup_cpu_usage(self, job_directory: str) -> str:
- # comounted cgroups (which cpu and cpuacct are on the supported Linux distros) can appear in any order (cpu,cpuacct or cpuacct,cpu)
- return CPU_USAGE_TEMPLATE.format(
+ def __record_cgroup_v1_usage(self, job_directory: str) -> str:
+ return CGROUPSV1_TEMPLATE.format(
metrics=self.__cgroup_metrics_file(job_directory), cgroup_mount=self.cgroup_mount
)
- def __record_cgroup_memory_usage(self, job_directory: str) -> str:
- return MEMORY_USAGE_TEMPLATE.format(
+ def __record_cgroup_v2_usage(self, job_directory: str) -> str:
+ return CGROUPSV2_TEMPLATE.format(
metrics=self.__cgroup_metrics_file(job_directory), cgroup_mount=self.cgroup_mount
)
diff --git a/test/unit/job_metrics/test_cgroups.py b/test/unit/job_metrics/test_cgroups.py
index 863ea89c558b..7c6b41ae8ed9 100644
--- a/test/unit/job_metrics/test_cgroups.py
+++ b/test/unit/job_metrics/test_cgroups.py
@@ -1,6 +1,6 @@
from galaxy.job_metrics.instrumenters.cgroup import CgroupPlugin
-CGROUP_PRODUCTION_EXAMPLE_2201 = """__cpu.cfs_period_us__
+CGROUPV1_PRODUCTION_EXAMPLE_2201 = """__cpu.cfs_period_us__
100000
__cpu.cfs_quota_us__
-1
@@ -105,11 +105,156 @@
1
"""
+CGROUPV2_PRODUCTION_EXAMPLE_232 = """__cpu.idle__
+0
+__cpu.max__
+max 100000
+__cpu.max.burst__
+0
+__cpu.stat__
+usage_usec 8992210
+user_usec 6139150
+system_usec 2853059
+core_sched.force_idle_usec 0
+nr_periods 0
+nr_throttled 0
+throttled_usec 0
+nr_bursts 0
+burst_usec 0
+__cpu.weight__
+100
+__cpu.weight.nice__
+0
+__memory.current__
+139350016
+__memory.events__
+low 0
+high 0
+max 0
+oom 0
+oom_kill 0
+oom_group_kill 0
+__memory.events.local__
+low 0
+high 0
+max 0
+oom 0
+oom_kill 0
+oom_group_kill 0
+__memory.high__
+max
+__memory.low__
+0
+__memory.max__
+max
+__memory.min__
+0
+__memory.numa_stat__
+anon N0=864256
+file N0=129146880
+kernel_stack N0=32768
+pagetables N0=131072
+sec_pagetables N0=0
+shmem N0=0
+file_mapped N0=0
+file_dirty N0=0
+file_writeback N0=0
+swapcached N0=0
+anon_thp N0=0
+file_thp N0=0
+shmem_thp N0=0
+inactive_anon N0=819200
+active_anon N0=20480
+inactive_file N0=51507200
+active_file N0=77639680
+unevictable N0=0
+slab_reclaimable N0=8638552
+slab_unreclaimable N0=340136
+workingset_refault_anon N0=0
+workingset_refault_file N0=77
+workingset_activate_anon N0=0
+workingset_activate_file N0=0
+workingset_restore_anon N0=0
+workingset_restore_file N0=0
+workingset_nodereclaim N0=0
+__memory.oom.group__
+0
+__memory.peak__
+339906560
+__memory.reclaim__
+__memory.stat__
+anon 860160
+file 129146880
+kernel 9211904
+kernel_stack 32768
+pagetables 126976
+sec_pagetables 0
+percpu 0
+sock 0
+vmalloc 0
+shmem 0
+zswap 0
+zswapped 0
+file_mapped 0
+file_dirty 0
+file_writeback 0
+swapcached 0
+anon_thp 0
+file_thp 0
+shmem_thp 0
+inactive_anon 815104
+active_anon 20480
+inactive_file 51507200
+active_file 77639680
+unevictable 0
+slab_reclaimable 8642480
+slab_unreclaimable 340904
+slab 8983384
+workingset_refault_anon 0
+workingset_refault_file 77
+workingset_activate_anon 0
+workingset_activate_file 0
+workingset_restore_anon 0
+workingset_restore_file 0
+workingset_nodereclaim 0
+pgscan 0
+pgsteal 0
+pgscan_kswapd 0
+pgscan_direct 0
+pgsteal_kswapd 0
+pgsteal_direct 0
+pgfault 132306
+pgmajfault 524
+pgrefill 0
+pgactivate 18958
+pgdeactivate 0
+pglazyfree 0
+pglazyfreed 0
+zswpin 0
+zswpout 0
+thp_fault_alloc 19
+thp_collapse_alloc 0
+__memory.swap.current__
+0
+__memory.swap.events__
+high 0
+max 0
+fail 0
+__memory.swap.high__
+max
+__memory.swap.max__
+max
+__memory.zswap.current__
+0
+__memory.zswap.max__
+max
+"""
+
-def test_cgroup_collection(tmpdir):
+def test_cgroupv1_collection(tmpdir):
plugin = CgroupPlugin()
job_dir = tmpdir.mkdir("job")
- job_dir.join("__instrument_cgroup__metrics").write(CGROUP_PRODUCTION_EXAMPLE_2201)
+ job_dir.join("__instrument_cgroup__metrics").write(CGROUPV1_PRODUCTION_EXAMPLE_2201)
properties = plugin.job_properties(1, job_dir)
assert "cpuacct.usage" in properties
assert properties["cpuacct.usage"] == 7265342042
@@ -117,6 +262,17 @@ def test_cgroup_collection(tmpdir):
assert properties["memory.limit_in_bytes"] == 9223372036854771712
+def test_cgroupv2_collection(tmpdir):
+ plugin = CgroupPlugin()
+ job_dir = tmpdir.mkdir("job")
+ job_dir.join("__instrument_cgroup__metrics").write(CGROUPV2_PRODUCTION_EXAMPLE_232)
+ properties = plugin.job_properties(1, job_dir)
+ assert "cpu.stat.usage_usec" in properties
+ assert properties["cpu.stat.usage_usec"] == 8992210
+ assert "memory.peak" in properties
+ assert properties["memory.peak"] == 339906560
+
+
def test_instrumentation(tmpdir):
# don't actually run the instrumentation but at least exercise the code the and make
# sure templating includes cgroup_mount override.
diff --git a/test/unit/job_metrics/test_job_metrics.py b/test/unit/job_metrics/test_job_metrics.py
index 07930c0684ec..a80df5e9b413 100644
--- a/test/unit/job_metrics/test_job_metrics.py
+++ b/test/unit/job_metrics/test_job_metrics.py
@@ -49,6 +49,20 @@ def test_job_metrics_format_cgroup():
assert_title="Memory limit on cgroup (MEM)",
assert_value="8.0 EB",
)
+ _assert_format(
+ "cgroup",
+ "cpu.stat.usage_usec",
+ 7982357892.000000,
+ assert_title="CPU usage time",
+ assert_value="2.0 hours and 13.0 minutes",
+ )
+ _assert_format(
+ "cgroup",
+ "memory.peak",
+ 45097156608,
+ assert_title="Max memory usage recorded",
+ assert_value="42.0 GB",
+ )
def test_job_metrics_uname():