veperf

#!/usr/bin/python3
#
# Copyright (C) 2018-2023 Erich Focht
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License along
# with this program; if not, write to the Free Software Foundation, Inc.,
# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.


import argparse
from veosinfo import *
import os, time
import signal
import sys


sysfs_path = dict()
pwr_sum = dict()
pwr_nmeas = 0
t_start = 0

def sigint_handler(signum, frame):
    sys.exit()

def get_pid_comm(pid):
    #COMM=`cat /proc/$pid/cmdline | sed -e 's,^.*--,,'`
    try:
        with open("/proc/%d/cmdline" % pid, "r") as f:
            for line in f:
                line = line.rstrip(os.linesep)
                idx = line.find("--")
                if idx > 0:
                    return line[idx + 2:]
    except:
        pass
    return "-"

def get_sensor(nodeid, name):
    with open(sysfs_path[nodeid] + "/" + name) as f:
        return float(f.read())

def power(nodeid):
    family = int(ve_info[nodeid]["family"])
    if family <= 2:
        volt = get_sensor(nodeid, "sensor_8") / 1000000.0
        voltEdge = get_sensor(nodeid, "sensor_9") / 1000000.0
        current = get_sensor(nodeid, "sensor_12") / 1000.0
        currentEdge = get_sensor(nodeid, "sensor_13") / 1000.0
    else:
        volt = get_sensor(nodeid, "sensor_36") / 1000000.0
        voltEdge = get_sensor(nodeid, "sensor_37") / 1000000.0
        current = get_sensor(nodeid, "sensor_41") / 1000.0
        currentEdge = get_sensor(nodeid, "sensor_42") / 1000.0
    pwr = 5 + volt * current + voltEdge * currentEdge
    pwr_sum[nodeid] += pwr
    return pwr

# energy consumed since start of measurement on specified VE
def energy(nodeid):
    pwr_avg = pwr_sum[nodeid] / pwr_nmeas
    e = pwr_avg * (time.time() - t_start) / 1000
    return e

def print_metrics():
    for nodeid in sorted(METR.keys()):
        s_mops = 0
        s_mflops = 0
        s_loadbw = 0
        s_storebw = 0
        print("-> VE%d" % nodeid)
        for pid in sorted(METR[nodeid].keys()):
            m = METR[nodeid][pid]
            comm = get_pid_comm(pid)
            print("%-8d  %8.2fs  %7.3f  %7.0f  %8.0f  %5.0f  %7.1f%%  %9.1f%%  %6.0f%%  %7.0f%%  %7.0f%%  %8.3f  %8.3f  %-10s" % \
                (pid, m["USRSEC"], m.get("EFFTIME", 0), m.get("MOPS", 0), m.get("MFLOPS", 0),
                 m.get("AVGVL", 0), m.get("VOPRAT", 0), m.get("VTIMERATIO", 0),
                 m.get("L1CACHEMISS", 0), m.get("CPUPORTCONF", 0), m.get("VLDLLCHIT", 0),
                 m.get("LOADBW", 0), m.get("STOREBW", 0),
                 comm))
            s_mops += m.get("MOPS", 0)
            s_mflops += m.get("MFLOPS", 0)
            s_loadbw += m.get("LOADBW", 0)
            s_storebw += m.get("STOREBW", 0)
        if (s_loadbw > 0):
            print("SUM VE%d: MOPS=%-8.0f MFLOPS=%-8.0f LOADBW=%.3fGB/s STOREBW=%.3fGB/s POWER=%.1fW ENERGY=%.2fkJ" %
                  (nodeid, s_mops, s_mflops, s_loadbw, s_storebw, power(nodeid), energy(nodeid)))
        else:
            print("SUM VE%d: MOPS=%-8.0f MFLOPS=%-8.0f POWER=%.1fW ENERGY=%.2fkJ" %
                  (nodeid, s_mops, s_mflops, power(nodeid), energy(nodeid)))


def print_label():
    print("%-8s  %9s  %7s  %7s  %8s  %5s  %8s  %10s  %7s  %8s  %8s  %8s  %8s  %-10s" % \
        ("pid", "USRSEC", "EFFTIME", "MOPS", "MFLOPS", "AVGVL", "VOPRAT",
         "VTIMERATIO", "L1CMISS", "PORTCONF", "VLLCHIT", "LOADBW", "STOREBW", "comm"))

#
################################################################################
#

DESCR = """Show performance metrics of VE tasks.

VEOS allows to read performance counters of own VE processes from the VH
with no or extremely low overhead for the VE processes. Root can access
all VE processes' metrics and will see all VE tasks.

The MOPS and MFLOPS metrics as well as bandwidths (if available), POWER and
ENERGY are aggregated per node at the end of each node output block.

Displayed metrics:
------------------

USRSEC:      Task's user time on VE
EFFTIME:     Effective time: ratio between user and elapsed time.
             A value lower than 1.0 is a sign for syscalls.
MOPS:        Millions of Operations Per Second
MFLOPS:      Millions of Floating Point Ops Per Second
AVGVL:       Average vector length
VOPRAT:      Vector operation ratio [percent]
VTIMERATIO:  Vector time ratio (vector time / user time) [percent]
L1CMISS:     L1 cache miss time [percent]
PORTCONF:    CPU port conflict time [percent]
VLLCHIT:     Vector load LLC hits [percent] (counter not active with MPI)
LOADBW:      Load bandwidth (LLC to Core) [GB/s] (see below)
STOREBW:     Store bandwidth (Core to LLC) [GB/s] (see below)

POWER:       Power consumption of each VE
ENERGY:      Energy used by each VE since start of veperf measurement

The PMMR register of each VE core is controlling Which performance metrics
are actually active, MPI and non-MPI programs might measure slightly different
metrics. In order to measure load and store bandwidths, you must run with the
environment variable VE_PERF_MODE=VECTOR-MEM set.

"""

signal.signal(signal.SIGINT, sigint_handler)
parser = argparse.ArgumentParser( description=DESCR,
                                  formatter_class=argparse.RawTextHelpFormatter )
parser.add_argument("-d", "--delay", type=int, default=[2], nargs=1,
                    help="Measurement interval [seconds]. Default: 2s.")
parser.add_argument("-n", "--node", action="append",
                    help="VE Node ID to be included into measurement. Default: all nodes.")
parser.add_argument("interv", type=int, nargs='?', metavar='INTERVALS',
                    help="number of measurement intervals before exiting")
args, rest = parser.parse_known_args()

DELAY = args.delay[0]
INTERV = -1
if args.interv is not None:
    try:
        INTERV = args.interv + 1
    except:
        pass

ni = node_info()
ve_info = dict()
nodeids = []
for i in xrange(ni['total_node_count']):
    nodeid = ni['nodeid'][i]
    if ni['status'][i] != 0:
        continue
    nodeids.append(nodeid)
    ve_info[nodeid] = cpu_info(nodeid)

if args.node is not None:
    nodeids = []
    for n in args.node:
        nodeids.append(int(n))

for nodeid in nodeids:
    sysfs_path[nodeid] = ve_sysfs_path(nodeid)
    pwr_sum[nodeid] = 0
t_start = time.time()
        
DATA = dict()
PREV = dict()
METR = dict()
for nodeid in nodeids:
    DATA[nodeid] = dict()
    PREV[nodeid] = dict()
    METR[nodeid] = dict()

cnt = 0
while 1:
    TSTART = time.time()
    
    # loop over VE node IDs
    for nodeid in nodeids:
        pids = sorted(ve_pids(nodeid))

        for prevpid in PREV[nodeid].keys():
            if prevpid not in pids:
                del PREV[nodeid][prevpid]
                if prevpid in METR[nodeid].keys():
                    del METR[nodeid][prevpid]

        for pid in pids:
            try:
                DATA[nodeid][pid] = ve_pid_perf(nodeid, pid)
            except:
                continue
            if pid in PREV[nodeid].keys():
                METR[nodeid][pid] = calc_metrics(nodeid, PREV[nodeid][pid], DATA[nodeid][pid])
            PREV[nodeid][pid] = DATA[nodeid][pid]
            del DATA[nodeid][pid]

    if cnt > 0:
        pwr_nmeas += 1
        print_label()
        print_metrics()

    TEND = time.time()
    cnt += 1
    if cnt == INTERV:
        break
    time.sleep(max(DELAY - (TEND - TSTART), 0.5))