From 37f9fe00666d455f63230e23bcf808cf540177b2 Mon Sep 17 00:00:00 2001 From: "Vallee, Geoffroy R" Date: Tue, 28 Aug 2018 17:27:28 -0400 Subject: [PATCH] First complete implementation of the SICM support --- opal/mca/mpool/sicm/Makefile.am | 39 +++ opal/mca/mpool/sicm/configure.m4 | 52 ++++ opal/mca/mpool/sicm/mpool_sicm.h | 68 +++++ opal/mca/mpool/sicm/mpool_sicm_component.c | 274 +++++++++++++++++++++ opal/mca/mpool/sicm/mpool_sicm_module.c | 125 ++++++++++ 5 files changed, 558 insertions(+) create mode 100644 opal/mca/mpool/sicm/Makefile.am create mode 100644 opal/mca/mpool/sicm/configure.m4 create mode 100644 opal/mca/mpool/sicm/mpool_sicm.h create mode 100644 opal/mca/mpool/sicm/mpool_sicm_component.c create mode 100644 opal/mca/mpool/sicm/mpool_sicm_module.c diff --git a/opal/mca/mpool/sicm/Makefile.am b/opal/mca/mpool/sicm/Makefile.am new file mode 100644 index 00000000000..69c6371df02 --- /dev/null +++ b/opal/mca/mpool/sicm/Makefile.am @@ -0,0 +1,39 @@ +# -*- indent-tabs-mode:nil -*- +# +# Copyright (c) 2018 UT-Battelle, LLC +# All rights reserved. +# +# Additional copyrights may follow +# +# $HEADERS$ +# + + +AM_CPPFLAGS = $(mpool_sicm_CPPFLAGS) +AM_LDFLAGS = $(mpool_sicm_LDFLAGS) + +if MCA_BUILD_opal_mpool_sicm_DSO +component_noinst = +component_install = mca_mpool_sicm.la +else +component_noinst = libmca_mpool_sicm.la +component_install = +endif + +sicm_SOURCES = mpool_sicm.h \ + mpool_sicm_component.c \ + mpool_sicm_module.c + +mcacomponentdir = $(opallibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_mpool_sicm_la_SOURCES = $(sicm_SOURCES) +nodist_mca_mpool_sicm_la_SOURCES = $(sicm_nodist_SOURCES) +mca_mpool_sicm_la_LIBASS = $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la \ + $(mpool_sicm_LIBS) +mca_mpool_sicm_la_LDFLAGS = -module -avoid-version $(mpool_sicm_LDFLAGS) + +noinst_LTLIBRARIES = $(component_noinst) +libmca_mpool_sicm_la_SOURCES = $(sicm_SOURCES) +nodist_libmca_mpool_sicm_la_SOURCES = $(sicm_nodist_SOURCES) +libmca_mpool_sicm_la_LIBADD = $(mpool_sicm_LIBS) +libmca_mpool_sicm_la_LDFLAGS = -module -avoid-version $(mpool_sicm_LDFLAGS) diff --git a/opal/mca/mpool/sicm/configure.m4 b/opal/mca/mpool/sicm/configure.m4 new file mode 100644 index 00000000000..83691e5a9a6 --- /dev/null +++ b/opal/mca/mpool/sicm/configure.m4 @@ -0,0 +1,52 @@ +# -*- shell-script -*- +# +# Copyright (c) 2018 UT-Battelle, LLC +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# check if SICM support can be found. +AC_DEFUN([MCA_opal_mpool_sicm_CONFIG], + [OPAL_VAR_SCOPE_PUSH([opal_mpool_sicm_happy]) + AC_CONFIG_FILES([opal/mca/mpool/sicm/Makefile]) + + _sicm_cppflags="" + _sicm_ldflags="" + _sicm_libs="" + + AC_ARG_WITH([sicm], + [AC_HELP_STRING([--with-sicm(=DIR)], + [Build with Simple Interface Complex Memory library support]) + ] + ) + OPAL_CHECK_WITHDIR([sicm], [$with_sicm], [include/sicm_low.h]) + + AS_IF([test "$with_sicm" != "no"], + [# At the moment, we always assume that users will use their own installation of SICM + + AS_IF([test ! -d "$with_sicm"], + [AC_MSG_RESULT([not found]) + AC_MSG_WARN([Directory $with_sicm not found]) + AC_MSG_ERROR([Cannot continue]) + ], + [AC_MSG_RESULT([found]) + _sicm_cppflags="-I$with_sicm/include" + _sicm_ldflags="-L$with_sicm/lib" + _sicm_libs="-lsicm" + opal_mpool_sicm_happy="yes" + ] + ) + + ],[opal_mpool_sicm_happy=no] + ) + + AC_SUBST([mpool_sicm_CPPFLAGS],"$_sicm_cppflags") + AC_SUBST([mpool_sicm_LDFLAGS],"$_sicm_ldflags -lsicm") + AC_SUBST([mpool_sicm_LIBS],"$_sicm_libs") + OPAL_VAR_SCOPE_POP + ] +) diff --git a/opal/mca/mpool/sicm/mpool_sicm.h b/opal/mca/mpool/sicm/mpool_sicm.h new file mode 100644 index 00000000000..fcc80f5fa62 --- /dev/null +++ b/opal/mca/mpool/sicm/mpool_sicm.h @@ -0,0 +1,68 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 UT-Battelle, LLC + * All rights reserved + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADERS$ + */ + +#ifndef MCA_MPOOL_SICM_H +#define MCA_MPOOL_SICM_H + +#include "opal_config.h" +#include "opal/threads/mutex_unix.h" + +#include "opal/mca/event/event.h" +#include "opal/mca/mpool/mpool.h" + +#include "sicm_low.h" + +BEGIN_C_DECLS + +struct mca_mpool_sicm_module_t { + mca_mpool_base_module_t super; + bool sicm_is_initialized; + sicm_device_tag target_device_type; + opal_mutex_t lock; +}; +typedef struct mca_mpool_sicm_module_t mca_mpool_sicm_module_t; + +#if 0 +struct mca_mpool_sicm_module_le_t { + opal_list_item_t super; + mca_mpool_sicm_module_t module; +}; +typedef struct mca_mpool_sicm_module_le_t mca_mpool_sicm_module_le_t; +OBJ_CLASS_DECLARATION(mca_mpool_sicm_module_le_t); +#endif + +struct mca_mpool_sicm_component_t { + mca_mpool_base_component_t super; + int module_count; + mca_mpool_sicm_module_t **modules; + int priority; + int output; + sicm_device_list devices; +}; +typedef struct mca_mpool_sicm_component_t mca_mpool_sicm_component_t; +OPAL_MODULE_DECLSPEC extern mca_mpool_sicm_component_t mca_mpool_sicm_component; + +int mpool_sicm_module_init (mca_mpool_sicm_module_t *module); + +void mpool_sicm_finalize (mca_mpool_base_module_t *module); + +#if 0 +static void* mpool_sicm_alloc (mca_mpool_base_module_t *module, size_t size, size_t align, uint32_t flags); + +static void* mpool_sicm_realloc (mca_mpool_base_module_t *module, void *addr, size_t size); + +static void mpool_sicm_free (mca_mpool_base_module_t *module, void *addr); +#endif + +END_C_DECLS + +#endif /* MCA_MPOOL_SICM_H */ diff --git a/opal/mca/mpool/sicm/mpool_sicm_component.c b/opal/mca/mpool/sicm/mpool_sicm_component.c new file mode 100644 index 00000000000..db136f34178 --- /dev/null +++ b/opal/mca/mpool/sicm/mpool_sicm_component.c @@ -0,0 +1,274 @@ +/* + * Copyright (c) 2018 UT-Battelle, LLC + * All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADERS$ + */ + +#include "opal/mca/mpool/mpool.h" +#include "opal/mca/mpool/sicm/mpool_sicm.h" + +static int mpool_sicm_priority; +static int mpool_sicm_verbose; + +extern mca_mpool_sicm_module_t mca_mpool_sicm_module; + +static int mpool_sicm_register (void); +static int mpool_sicm_open (void); +static int mpool_sicm_close (void); +static int mpool_sicm_query (const char *hints, int *priority, mca_mpool_base_module_t **module); + +mca_mpool_sicm_component_t mca_mpool_sicm_component = { + { + .mpool_version = { + MCA_MPOOL_BASE_VERSION_3_0_0, + "sicm", + MCA_BASE_MAKE_VERSION ( + component, + OPAL_MAJOR_VERSION, + OPAL_MINOR_VERSION, + OPAL_RELEASE_VERSION), + .mca_open_component = mpool_sicm_open, + .mca_close_component = mpool_sicm_close, + .mca_register_component_params = mpool_sicm_register + }, + .mpool_data = { + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + .mpool_query = mpool_sicm_query, + }, + .modules = NULL, + .module_count = 0, +}; + + +static int +mpool_sicm_register (void) +{ + mpool_sicm_priority = 0; + mpool_sicm_verbose = 0; + +#if 0 + mca_base_component_var_register (&mpool_sicm_component.super.mpool_version, + "default_type", + "Default sicm type to use", + MCA_BASE_VAR_TYPE_INT, + mpool_sicm_type_enum, + 0, + 0, + OPAL_INFO_LVL_5, + MCA_BASE_VAR_SCOPE_LOCAL + &mpool_sicm_component.default_type); +#endif + + mca_base_component_var_register (&mca_mpool_sicm_component.super.mpool_version, + "priority", + "Default priority of the SICM mpool component (default: 0)", + MCA_BASE_VAR_TYPE_INT, + NULL, + 0, + 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_LOCAL, + &mpool_sicm_priority); + + mca_base_component_var_register (&mca_mpool_sicm_component.super.mpool_version, + "verbose", + "Default level of verbosity for the SICM mpool component (default: 0)", + MCA_BASE_VAR_TYPE_INT, + NULL, + 0, + 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_LOCAL, + &mpool_sicm_verbose); + + return OPAL_SUCCESS; +} + +static int +_mpool_sicm_extend_module_list (void) +{ + int n_mods = mca_mpool_sicm_component.module_count + 1; + mca_mpool_sicm_module_t *_new_mod = NULL; + + if (n_mods == 1) + { + mca_mpool_sicm_component.modules = (mca_mpool_sicm_module_t**) malloc (sizeof (mca_mpool_sicm_module_t*)); + } + else + { + mca_mpool_sicm_component.modules = (mca_mpool_sicm_module_t**) realloc (mca_mpool_sicm_component.modules, n_mods * sizeof (mca_mpool_sicm_module_t*)); + } + + if (mca_mpool_sicm_component.modules == NULL) + return OPAL_ERR_OUT_OF_RESOURCE; + + _new_mod = (mca_mpool_sicm_module_t*) malloc (sizeof (mca_mpool_sicm_module_t)); + if (_new_mod == NULL) + return OPAL_ERR_OUT_OF_RESOURCE; + + mpool_sicm_module_init (_new_mod); + mca_mpool_sicm_component.modules[mca_mpool_sicm_component.module_count] = _new_mod; + + mca_mpool_sicm_component.module_count = n_mods; + + return OPAL_SUCCESS; +} + +static int +mpool_sicm_open (void) +{ + if (mpool_sicm_verbose != 0) + { + mca_mpool_sicm_component.output = opal_output_open (NULL); + } + else + { + mca_mpool_sicm_component.output = -1; + } + + // In case we can allocate memory based on various parameters, + // we will instantiate a SICM module for each configuration + // For now, we create one default module. + _mpool_sicm_extend_module_list (); + + return OPAL_SUCCESS; +} + +static int +mpool_sicm_close (void) +{ + for (int _i = 0; _i < mca_mpool_sicm_component.module_count; _i++) + { + mca_mpool_sicm_module_t *_m = mca_mpool_sicm_component.modules[_i]; + _m->super.mpool_finalize (&_m->super); + } + + free (mca_mpool_sicm_component.modules); + mca_mpool_sicm_component.modules = NULL; + + return OPAL_SUCCESS; +} + +static void +_parse_hints (const char *hints, int *priority, mca_mpool_base_module_t **module) +{ + int _priority = 0; + sicm_device_tag device_type = SICM_DRAM; + + if (hints) + { + char **_hints; + int _i = 0; + + // _hints is NULL terminated + _hints = opal_argv_split (hints, ','); + if (_hints == NULL) + return; + + while (_hints[_i] != NULL) + { + char *_key = _hints[_i]; + char *_val = NULL; + char *_str = NULL; + + // The hints we are looking for are in the form of a key/value pair. + // Separate the key from the value without copy. + _str = strchr (_key, '='); + if (_str != NULL) + { + _val = _str + 1; + *_str = '\0'; + } + + if (strcasecmp("mpool", _key) == 0) + { + if (_val != NULL && strcasecmp ("sicm", _val) == 0) + { + // This module is the target of the request + _priority = 100; + } + } + + if (strcasecmp("sicm_device_type", _key) == 0) + { + // Because SICM does not define enums in a precise way, we need to explicitely + // figure out what the tag is + if (strcmp (_val, "SICM_KNL_HBM") == 0) + device_type = SICM_KNL_HBM; + + if (strcmp (_val, "SICM_POWERPC_HBM") == 0) + device_type = SICM_POWERPC_HBM; + + } + + // TODO: make sure we get all the hints to drive the memory operation + + _i++; + } + + opal_argv_free (_hints); + } + + if (_priority > 0) + { + int i = 0; + mca_mpool_sicm_module_t *_m = NULL; + + // We find the SICM module instance that handles the target type of device + do + { + if (mca_mpool_sicm_component.modules[i]->target_device_type == device_type) + { + _m = mca_mpool_sicm_component.modules[i]; + break; + } + + if (mca_mpool_sicm_component.modules[i]->target_device_type == INVALID_TAG) + { + mca_mpool_sicm_component.modules[i]->target_device_type = device_type; + _m = mca_mpool_sicm_component.modules[i]; + break; + } + + i++; + } while (i < mca_mpool_sicm_component.module_count); + + if (_m == NULL) + { + // We did not find a suitable module so we create a new one + _mpool_sicm_extend_module_list (); + assert (mca_mpool_sicm_component.modules[mca_mpool_sicm_component.module_count]->target_device_type == INVALID_TAG); + _m = mca_mpool_sicm_component.modules[mca_mpool_sicm_component.module_count]; + _m->target_device_type = device_type; + } + + *module = (mca_mpool_base_module_t*)_m; + } + + *priority = _priority; +} + +static int +mpool_sicm_query (const char *hints, int *priority, mca_mpool_base_module_t **module) +{ + + // hints is a string of comma-sperated keys that are used to pass parameters in + if (hints) + { + // Parse the list of hints + _parse_hints (hints, priority, module); + } + else + { + *priority = 0; + } + + return OPAL_SUCCESS; +} + diff --git a/opal/mca/mpool/sicm/mpool_sicm_module.c b/opal/mca/mpool/sicm/mpool_sicm_module.c new file mode 100644 index 00000000000..aa6d8a028e3 --- /dev/null +++ b/opal/mca/mpool/sicm/mpool_sicm_module.c @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2018 UT-Battelle, LLC + * All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADERS$ + */ + +#include "opal_config.h" + +#include "opal/mca/mpool/base/base.h" +#include "opal/mca/mpool/sicm/mpool_sicm.h" + +static void* mpool_sicm_alloc (mca_mpool_base_module_t *module, size_t size, size_t align, uint32_t flags); +static void* mpool_sicm_realloc (mca_mpool_base_module_t *module, void *addr, size_t size); +static void mpool_sicm_free (mca_mpool_base_module_t *module, void *addr); + +mca_mpool_sicm_module_t mca_mpool_sicm_module = { + .super = { + .mpool_component = &mca_mpool_sicm_component.super, + .mpool_alloc = mpool_sicm_alloc, + .mpool_realloc = mpool_sicm_realloc, + .mpool_free = mpool_sicm_free, + }, + .sicm_is_initialized = false, +}; + +int +mpool_sicm_module_init (mca_mpool_sicm_module_t *module) +{ + module->super.mpool_component = &mca_mpool_sicm_component.super; + module->super.mpool_base = NULL; + module->super.mpool_alloc = mpool_sicm_alloc; + module->super.mpool_realloc = mpool_sicm_realloc; + module->super.mpool_free = mpool_sicm_free; + module->super.mpool_finalize = mpool_sicm_finalize; + module->super.mpool_ft_event = NULL; + module->super.flags = MCA_MPOOL_FLAGS_MPI_ALLOC_MEM; + + OBJ_CONSTRUCT (&module->lock, opal_mutex_t); + + mca_mpool_sicm_component.devices = sicm_init (); + module->target_device_type = INVALID_TAG; + module->sicm_is_initialized = true; + + return OPAL_SUCCESS; +} + +static void* +mpool_sicm_alloc (mca_mpool_base_module_t *module, size_t size, size_t align, uint32_t flags) +{ + sicm_arena arena; + void *mem = NULL; + sicm_device *dev; + sicm_device_tag device_tag = SICM_DRAM; + mca_mpool_sicm_module_t *_m; + + _m = (mca_mpool_sicm_module_t*)module; + + if (_m->sicm_is_initialized == false) + mpool_sicm_module_init (_m); + + if (flags == SICM_DRAM) + device_tag = SICM_DRAM; + + if (flags == SICM_KNL_HBM) + device_tag = SICM_KNL_HBM; + + if (flags == SICM_POWERPC_HBM) + device_tag = SICM_POWERPC_HBM; + + dev = sicm_find_device (&mca_mpool_sicm_component.devices, device_tag, 0, NULL); + if (dev == NULL) + return NULL; + + arena = sicm_arena_create (0, dev); + if (arena == NULL) + return NULL; + + if (align > 0) + mem = sicm_arena_alloc_aligned (arena, size, align); + else + mem = sicm_arena_alloc (arena, size); + + return mem; +} + +static void* +mpool_sicm_realloc (mca_mpool_base_module_t *module, void *addr, size_t size) +{ + sicm_arena arena; + + arena = sicm_arena_lookup (addr); + if (arena == NULL) + return NULL; + + return sicm_arena_realloc (arena, addr, size); +} + +static void +mpool_sicm_free (mca_mpool_base_module_t *module, void *addr) +{ + sicm_arena arena; + + arena = sicm_arena_lookup (addr); + if (arena == NULL) + return; + + sicm_free (addr); + + // No API to destroy an arena - Memory leak +} + +void +mpool_sicm_finalize (mca_mpool_base_module_t *module) +{ + mca_mpool_sicm_module_t *_m = (mca_mpool_sicm_module_t*) module; + + // No function to finalize sicm and/or free devices - Memory leaks + + OBJ_DESTRUCT (&_m->lock); +}