From 43dd40cf7d9c6c9ec967c86e2ed5032b070b4f66 Mon Sep 17 00:00:00 2001 From: George Stelle Date: Wed, 22 Jan 2020 10:19:50 -0700 Subject: [PATCH 1/9] Updated hwloc to work with hwloc 2.1, needs scrutiny --- src/affinity/hwloc.c | 50 ++++++++++++++++++++++---------------------- 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/src/affinity/hwloc.c b/src/affinity/hwloc.c index 17718cec1..fe910f5cf 100644 --- a/src/affinity/hwloc.c +++ b/src/affinity/hwloc.c @@ -59,7 +59,7 @@ static unsigned int num_usable_by_depth(unsigned int depth) qthread_debug(AFFINITY_DETAILS, "obj %i of this type is NULL!!!\n", i); continue; } - if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, HWLOC_OBJ_PU) == 0) { + if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU) == 0) { qthread_debug(AFFINITY_DETAILS, "obj %i of this type has no PUs!!!\n", i); continue; } @@ -80,7 +80,7 @@ static unsigned int num_usable_by_type(hwloc_obj_type_t tp) qthread_debug(AFFINITY_DETAILS, "obj %i of this type is NULL!!!\n", i); continue; } - if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, HWLOC_OBJ_PU) == 0) { + if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU) == 0) { qthread_debug(AFFINITY_DETAILS, "obj %i of this type has no PUs!!!\n", i); continue; } @@ -123,9 +123,9 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, * below is used to compare to the cache level */ const size_t shepindexofL1cache = 5; hwloc_obj_type_t shep_type_options[] = { - HWLOC_OBJ_NODE, HWLOC_OBJ_CACHE, HWLOC_OBJ_SOCKET, HWLOC_OBJ_CORE, + HWLOC_OBJ_NODE, HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_SOCKET, HWLOC_OBJ_CORE, HWLOC_OBJ_PU, - HWLOC_OBJ_CACHE, HWLOC_OBJ_CACHE, HWLOC_OBJ_CACHE, HWLOC_OBJ_CACHE + HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_CACHE_UNIFIED }; { const char *qsh = qt_internal_get_env_str("SHEPHERD_BOUNDARY", "node"); @@ -200,7 +200,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, if ((shep_depth == HWLOC_TYPE_DEPTH_UNKNOWN) || (shep_depth == HWLOC_TYPE_DEPTH_MULTIPLE)) { if ((shep_type_idx > 0) && - (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE)) { + (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE_UNIFIED)) { /* caches are almost always weird; so if the user asked for them, just give best effort */ unsigned int maxdepth = hwloc_topology_get_depth(topology); unsigned int curdepth; @@ -211,7 +211,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, for (curdepth = maxdepth; curdepth > 0; --curdepth) { unsigned int realdepth = curdepth - 1; hwloc_obj_type_t t = hwloc_get_depth_type(topology, realdepth); - if (t == HWLOC_OBJ_CACHE) { + if (t == HWLOC_OBJ_CACHE_UNIFIED) { level++; qthread_debug(AFFINITY_DETAILS, "L%u at depth %u (nbobjs is %u)\n", @@ -261,7 +261,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, shep_depth = hwloc_get_type_depth(topology, shep_type_options[shep_type_idx]); if (shep_depth <= 0) { qthread_debug(AFFINITY_DETAILS, "invalid shepherd type (%s), finding another one...\n", typenames[shep_type_idx]); - if (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE) { + if (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE_UNIFIED) { shep_type_idx = 1; } shep_type_idx++; @@ -405,7 +405,7 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns hwloc_obj_t obj = hwloc_get_obj_inside_cpuset_by_depth(topology, allowed_cpuset, shep_depth, 0); int workerobjs_per_shep; - if (hwloc_compare_types(wkr_type, HWLOC_OBJ_CACHE) == 0) { + if (hwloc_compare_types(wkr_type, HWLOC_OBJ_CACHE_UNIFIED) == 0) { qthread_debug(AFFINITY_DETAILS, "worker type is a cache; must handle specially...\n"); wkr_depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PU); qthread_debug(AFFINITY_DETAILS, "PU wkr_depth = %u\n", wkr_depth); @@ -414,18 +414,18 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns qthread_debug(AFFINITY_DETAILS, "wkridx=%i wkr_index=%i\n", wkridx, wkr_index); do { wkr_depth--; - tmp = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth, 0); - qthread_debug(AFFINITY_DETAILS, "wkr_depth %i, type = %s %i\n", wkr_depth, hwloc_obj_type_string(tmp->type), hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE)); - } while (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE) > 0); - if (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE) != 0) { + tmp = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth, 0); + qthread_debug(AFFINITY_DETAILS, "wkr_depth %i, type = %s %i\n", wkr_depth, hwloc_obj_type_string(tmp->type), hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED)); + } while (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED) > 0); + if (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED) != 0) { fprintf(stderr, "QTHREADS: worker unit (%s) does not exist on this machine.\n", typenames[wkr_index]); - assert(hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE) == 0); + assert(hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED) == 0); goto guess_my_weight; } } qthread_debug(AFFINITY_DETAILS, "found the desired level of cache!!!\n"); assert(wkr_depth >= shep_depth); - workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth); + workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth); qthread_debug(AFFINITY_CALLS, "workerobjs type = %s, per_shep = %u\n", hwloc_obj_type_string(wkr_type), workerobjs_per_shep); switch (workerobjs_per_shep) { case 0: @@ -439,7 +439,7 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns } } else { qthread_debug(AFFINITY_DETAILS, "worker type is NOT a cache...\n"); - workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, wkr_type); + workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, wkr_type); qthread_debug(AFFINITY_CALLS, "workerobjs = %s, per_shep = %u\n", hwloc_obj_type_string(wkr_type), workerobjs_per_shep); switch (workerobjs_per_shep) { case 0: @@ -474,7 +474,7 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns for (size_t idx = 0; idx < nshepherds && idx < max_idx; ++idx) { hwloc_obj_t obj = hwloc_get_obj_inside_cpuset_by_depth(topology, allowed_cpuset, shep_depth, idx); - unsigned int weight = WEIGHT(obj->allowed_cpuset); + unsigned int weight = WEIGHT(obj->cpuset); qthread_debug(AFFINITY_DETAILS, "%s %u has %u weight\n", hwloc_obj_type_string(hwloc_get_depth_type(topology, shep_depth)), (unsigned int)idx, weight); total += weight; if ((idx == 0) || (ret < weight)) { @@ -508,9 +508,9 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, assert(wkr_depth >= 0); qthread_debug(AFFINITY_DETAILS, "wkr_depth = %u\n", wkr_depth); qthread_debug(AFFINITY_DETAILS, "num_wkrs = %u\n", hwloc_get_nbobjs_inside_cpuset_by_depth(topology, allowed_cpuset, wkr_depth)); - qthread_debug(AFFINITY_DETAILS, "num_wkrs = %u\n", hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth)); - hwloc_obj_t worker0 = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth, 0); - unsigned int workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth); + qthread_debug(AFFINITY_DETAILS, "num_wkrs = %u\n", hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth)); + hwloc_obj_t worker0 = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth, 0); + unsigned int workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth); assert(workerobjs_per_shep > 0); assert(worker0); @@ -524,13 +524,13 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, maxshepobjs, hwloc_obj_type_string(hwloc_get_depth_type(topology, shep_depth)), (int)hwloc_get_nbobjs_inside_cpuset_by_type(topology, allowed_cpuset, HWLOC_OBJ_PU)); - int shep_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, HWLOC_OBJ_PU); + int shep_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU); assert(shep_pus > 0); - unsigned int worker_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, worker0->allowed_cpuset, HWLOC_OBJ_PU); + unsigned int worker_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, worker0->cpuset, HWLOC_OBJ_PU); unsigned int wraparounds = me->packed_worker_id / (maxshepobjs * nworkerspershep); unsigned int worker_wraparounds = me->worker_id / workerobjs_per_shep; hwloc_obj_t sub_obj = - hwloc_get_obj_inside_cpuset_by_type(topology, obj->allowed_cpuset, + hwloc_get_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU, ((me->worker_id * worker_pus) + (wraparounds * nworkerspershep) + @@ -548,7 +548,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, qthread_debug(AFFINITY_DETAILS, "%u: (%i*%i) + (((%i * %i) + (%i * %i) + %i) % %i)\n", (unsigned)me->packed_worker_id, (int)shep_pus, (int)myshep->node, (int)me->worker_id, (int)worker_pus, (int)wraparounds, (int)nworkerspershep, (int)worker_wraparounds, (int)shep_pus); - ASPRINTF(&str, sub_obj->allowed_cpuset); + ASPRINTF(&str, sub_obj->cpuset); qthread_debug(AFFINITY_BEHAVIOR, "binding shep %i worker %i (%i) to PU %i, newPU %i, mask %s\n", (int)myshep->shepherd_id, (int)me->worker_id, @@ -559,7 +559,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, FREE(str, strlen(str)); } #endif /* ifdef QTHREAD_DEBUG_AFFINITY */ - if (hwloc_set_cpubind(topology, sub_obj->allowed_cpuset, HWLOC_CPUBIND_THREAD)) { + if (hwloc_set_cpubind(topology, sub_obj->cpuset, HWLOC_CPUBIND_THREAD)) { char *str; int i = errno; #ifdef __APPLE__ @@ -567,7 +567,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, return; } #endif - ASPRINTF(&str, sub_obj->allowed_cpuset); + ASPRINTF(&str, sub_obj->cpuset); fprintf(stderr, "Couldn't bind to cpuset %s because %s (%i)\n", str, strerror(i), i); FREE(str, strlen(str)); From 41a32a7b0630f754f437292e9963cc8f94b313fe Mon Sep 17 00:00:00 2001 From: Jan Ciesko Date: Fri, 18 Dec 2020 19:44:57 -0600 Subject: [PATCH 2/9] Replaced deprecated HWLOC API calls. --- src/affinity/hwloc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/affinity/hwloc.c b/src/affinity/hwloc.c index fe910f5cf..8a2168e96 100644 --- a/src/affinity/hwloc.c +++ b/src/affinity/hwloc.c @@ -333,7 +333,7 @@ void INTERNAL qt_affinity_mem_tonode(void *addr, DEBUG_ONLY(hwloc_topology_check(topology)); hwloc_bitmap_set(nodeset, node); - hwloc_set_area_membind_nodeset(topology, addr, bytes, nodeset, + hwloc_set_area_membind(topology, addr, bytes, nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_NOCPUBIND); hwloc_bitmap_free(nodeset); @@ -354,7 +354,7 @@ void INTERNAL *qt_affinity_alloc_onnode(size_t bytes, DEBUG_ONLY(hwloc_topology_check(topology)); nodeset = hwloc_bitmap_alloc(); hwloc_bitmap_set(nodeset, node); - ret = hwloc_alloc_membind_nodeset(topology, bytes, nodeset, + ret = hwloc_alloc_membind(topology, bytes, nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_NOCPUBIND); hwloc_bitmap_free(nodeset); From 502c2de82f66d987438eddce3b96e413f85a7295 Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Tue, 22 Dec 2020 08:55:25 -0800 Subject: [PATCH 3/9] Update calls to hwloc2.x for hwloc_v2 to fix compilation error --- src/affinity/hwloc_v2.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/affinity/hwloc_v2.c b/src/affinity/hwloc_v2.c index 7cae25067..73945586a 100644 --- a/src/affinity/hwloc_v2.c +++ b/src/affinity/hwloc_v2.c @@ -89,7 +89,7 @@ static void print_system_view(hwloc_topology_t sys_topo) int const num_cores = hwloc_get_nbobjs_inside_cpuset_by_type( - sys_topo, allowed_cpuset, HWLOC_OBJ_CORE); + sys_topo, allowed_cpuset, HWLOC_OBJ_CACHE_UNIFIED); printf("TOPO: number of available COREs: %d\n", num_cores); } @@ -179,7 +179,7 @@ static void init_type_options(void) while (NULL != obj) { topo_types[type_id] = obj->type; - if (0 == hwloc_compare_types(HWLOC_OBJ_CACHE, obj->type)) { + if (0 == hwloc_compare_types(HWLOC_OBJ_CACHE_UNIFIED, obj->type)) { snprintf(topo_type_names[type_id], 8, "L%dcache", cache_level); cache_level += 1; } else { @@ -559,7 +559,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, sys_topo, allowed_cpuset, qt_topo.shep_level, i); hwloc_obj_t logical_core_obj = hwloc_get_obj_inside_cpuset_by_type( - sys_topo, shep_obj->allowed_cpuset, HWLOC_OBJ_CORE, j); + sys_topo, shep_obj->cpuset, HWLOC_OBJ_CORE, j); qt_topo.worker_map[uid].bind_obj = hwloc_get_ancestor_obj_by_depth( sys_topo, qt_topo.worker_obj->depth, logical_core_obj); @@ -587,7 +587,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, ASSERT_ONLY(hwloc_topology_check(sys_topo)); hwloc_obj_t target_obj = qt_topo.worker_map[me->unique_id - 1].bind_obj; - if (hwloc_set_cpubind(sys_topo, target_obj->allowed_cpuset, + if (hwloc_set_cpubind(sys_topo, target_obj->cpuset, HWLOC_CPUBIND_THREAD)) { char *str; int i = errno; @@ -596,7 +596,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, return; } #endif - hwloc_bitmap_asprintf(&str, target_obj->allowed_cpuset); + hwloc_bitmap_asprintf(&str, target_obj->cpuset); fprintf(stderr, "Couldn't bind to cpuset %s because %s (%i)\n", str, strerror(i), i); FREE(str, strlen(str)); From 2190cf568545f9862fd3642ee0dd596965786006 Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Tue, 22 Dec 2020 18:32:45 -0800 Subject: [PATCH 4/9] Update calls to hwloc2.x for hwloc_via_chapel to fix compilation error --- src/affinity/hwloc_via_chapel.c | 51 ++++++++++++++++----------------- 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/src/affinity/hwloc_via_chapel.c b/src/affinity/hwloc_via_chapel.c index 5fe6eaf21..7ea25f109 100644 --- a/src/affinity/hwloc_via_chapel.c +++ b/src/affinity/hwloc_via_chapel.c @@ -61,7 +61,7 @@ static unsigned int num_usable_by_depth(unsigned int depth) qthread_debug(AFFINITY_DETAILS, "obj %i of this type is NULL!!!\n", i); continue; } - if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, HWLOC_OBJ_PU) == 0) { + if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU) == 0) { qthread_debug(AFFINITY_DETAILS, "obj %i of this type has no PUs!!!\n", i); continue; } @@ -82,7 +82,7 @@ static unsigned int num_usable_by_type(hwloc_obj_type_t tp) qthread_debug(AFFINITY_DETAILS, "obj %i of this type is NULL!!!\n", i); continue; } - if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, HWLOC_OBJ_PU) == 0) { + if (hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU) == 0) { qthread_debug(AFFINITY_DETAILS, "obj %i of this type has no PUs!!!\n", i); continue; } @@ -129,11 +129,10 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, * below is used to compare to the cache level */ const size_t shepindexofL1cache = 5; hwloc_obj_type_t shep_type_options[] = { - HWLOC_OBJ_NODE, HWLOC_OBJ_CACHE, HWLOC_OBJ_SOCKET, HWLOC_OBJ_CORE, + HWLOC_OBJ_NODE, HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_SOCKET, HWLOC_OBJ_CORE, HWLOC_OBJ_PU, - HWLOC_OBJ_CACHE, HWLOC_OBJ_CACHE, HWLOC_OBJ_CACHE, HWLOC_OBJ_CACHE - }; - { + HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_CACHE_UNIFIED, HWLOC_OBJ_CACHE_UNIFIED}; +{ const char *qsh = qt_internal_get_env_str("SHEPHERD_BOUNDARY", "node"); if (qsh) { @@ -206,7 +205,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, if ((shep_depth == HWLOC_TYPE_DEPTH_UNKNOWN) || (shep_depth == HWLOC_TYPE_DEPTH_MULTIPLE)) { if ((shep_type_idx > 0) && - (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE)) { + (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE_UNIFIED)) { /* caches are almost always weird; so if the user asked for them, just give best effort */ unsigned int maxdepth = hwloc_topology_get_depth(topology); unsigned int curdepth; @@ -217,7 +216,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, for (curdepth = maxdepth; curdepth > 0; --curdepth) { unsigned int realdepth = curdepth - 1; hwloc_obj_type_t t = hwloc_get_depth_type(topology, realdepth); - if (t == HWLOC_OBJ_CACHE) { + if (t == HWLOC_OBJ_CACHE_UNIFIED) { level++; qthread_debug(AFFINITY_DETAILS, "L%u at depth %u (nbobjs is %u)\n", @@ -267,7 +266,7 @@ void INTERNAL qt_affinity_init(qthread_shepherd_id_t *nbshepherds, shep_depth = hwloc_get_type_depth(topology, shep_type_options[shep_type_idx]); if (shep_depth <= 0) { qthread_debug(AFFINITY_DETAILS, "invalid shepherd type (%s), finding another one...\n", typenames[shep_type_idx]); - if (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE) { + if (shep_type_options[shep_type_idx] == HWLOC_OBJ_CACHE_UNIFIED) { shep_type_idx = 1; } shep_type_idx++; @@ -411,7 +410,7 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns hwloc_obj_t obj = hwloc_get_obj_inside_cpuset_by_depth(topology, allowed_cpuset, shep_depth, 0); int workerobjs_per_shep; - if (hwloc_compare_types(wkr_type, HWLOC_OBJ_CACHE) == 0) { + if (hwloc_compare_types(wkr_type, HWLOC_OBJ_CACHE_UNIFIED) == 0) { qthread_debug(AFFINITY_DETAILS, "worker type is a cache; must handle specially...\n"); wkr_depth = hwloc_get_type_depth(topology, HWLOC_OBJ_PU); qthread_debug(AFFINITY_DETAILS, "PU wkr_depth = %u\n", wkr_depth); @@ -420,18 +419,18 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns qthread_debug(AFFINITY_DETAILS, "wkridx=%i wkr_index=%i\n", wkridx, wkr_index); do { wkr_depth--; - tmp = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth, 0); - qthread_debug(AFFINITY_DETAILS, "wkr_depth %i, type = %s %i\n", wkr_depth, hwloc_obj_type_string(tmp->type), hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE)); - } while (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE) > 0); - if (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE) != 0) { + tmp = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth, 0); + qthread_debug(AFFINITY_DETAILS, "wkr_depth %i, type = %s %i\n", wkr_depth, hwloc_obj_type_string(tmp->type), hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED)); + } while (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED) > 0); + if (hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED) != 0) { fprintf(stderr, "QTHREADS: worker unit (%s) does not exist on this machine.\n", typenames[wkr_index]); - assert(hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE) == 0); + assert(hwloc_compare_types(tmp->type, HWLOC_OBJ_CACHE_UNIFIED) == 0); goto guess_my_weight; } } qthread_debug(AFFINITY_DETAILS, "found the desired level of cache!!!\n"); assert(wkr_depth >= shep_depth); - workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth); + workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth); qthread_debug(AFFINITY_CALLS, "workerobjs type = %s, per_shep = %u\n", hwloc_obj_type_string(wkr_type), workerobjs_per_shep); switch (workerobjs_per_shep) { case 0: @@ -445,7 +444,7 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns } } else { qthread_debug(AFFINITY_DETAILS, "worker type is NOT a cache...\n"); - workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, wkr_type); + workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, wkr_type); qthread_debug(AFFINITY_CALLS, "workerobjs = %s, per_shep = %u\n", hwloc_obj_type_string(wkr_type), workerobjs_per_shep); switch (workerobjs_per_shep) { case 0: @@ -480,7 +479,7 @@ qthread_worker_id_t INTERNAL guess_num_workers_per_shep(qthread_shepherd_id_t ns for (size_t idx = 0; idx < nshepherds && idx < max_idx; ++idx) { hwloc_obj_t obj = hwloc_get_obj_inside_cpuset_by_depth(topology, allowed_cpuset, shep_depth, idx); - unsigned int weight = WEIGHT(obj->allowed_cpuset); + unsigned int weight = WEIGHT(obj->cpuset); qthread_debug(AFFINITY_DETAILS, "%s %u has %u weight\n", hwloc_obj_type_string(hwloc_get_depth_type(topology, shep_depth)), (unsigned int)idx, weight); total += weight; if ((idx == 0) || (ret < weight)) { @@ -514,9 +513,9 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, assert(wkr_depth >= 0); qthread_debug(AFFINITY_DETAILS, "wkr_depth = %u\n", wkr_depth); qthread_debug(AFFINITY_DETAILS, "num_wkrs = %u\n", hwloc_get_nbobjs_inside_cpuset_by_depth(topology, allowed_cpuset, wkr_depth)); - qthread_debug(AFFINITY_DETAILS, "num_wkrs = %u\n", hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth)); - hwloc_obj_t worker0 = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth, 0); - unsigned int workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->allowed_cpuset, wkr_depth); + qthread_debug(AFFINITY_DETAILS, "num_wkrs = %u\n", hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth)); + hwloc_obj_t worker0 = hwloc_get_obj_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth, 0); + unsigned int workerobjs_per_shep = hwloc_get_nbobjs_inside_cpuset_by_depth(topology, obj->cpuset, wkr_depth); assert(workerobjs_per_shep > 0); assert(worker0); @@ -530,13 +529,13 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, maxshepobjs, hwloc_obj_type_string(hwloc_get_depth_type(topology, shep_depth)), (int)hwloc_get_nbobjs_inside_cpuset_by_type(topology, allowed_cpuset, HWLOC_OBJ_PU)); - int shep_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->allowed_cpuset, HWLOC_OBJ_PU); + int shep_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU); assert(shep_pus > 0); unsigned int worker_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, worker0->allowed_cpuset, HWLOC_OBJ_PU); unsigned int wraparounds = me->packed_worker_id / (maxshepobjs * nworkerspershep); unsigned int worker_wraparounds = me->worker_id / workerobjs_per_shep; hwloc_obj_t sub_obj = - hwloc_get_obj_inside_cpuset_by_type(topology, obj->allowed_cpuset, + hwloc_get_obj_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU, ((me->worker_id * worker_pus) + (wraparounds * nworkerspershep) + @@ -554,7 +553,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, qthread_debug(AFFINITY_DETAILS, "%u: (%i*%i) + (((%i * %i) + (%i * %i) + %i) % %i)\n", (unsigned)me->packed_worker_id, (int)shep_pus, (int)myshep->node, (int)me->worker_id, (int)worker_pus, (int)wraparounds, (int)nworkerspershep, (int)worker_wraparounds, (int)shep_pus); - ASPRINTF(&str, sub_obj->allowed_cpuset); + ASPRINTF(&str, sub_obj->cpuset); qthread_debug(AFFINITY_BEHAVIOR, "binding shep %i worker %i (%i) to PU %i, newPU %i, mask %s\n", (int)myshep->shepherd_id, (int)me->worker_id, @@ -565,7 +564,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, FREE(str, strlen(str)); } #endif /* ifdef QTHREAD_DEBUG_AFFINITY */ - if (hwloc_set_cpubind(topology, sub_obj->allowed_cpuset, HWLOC_CPUBIND_THREAD)) { + if (hwloc_set_cpubind(topology, sub_obj->cpuset, HWLOC_CPUBIND_THREAD)) { char *str; int i = errno; #ifdef __APPLE__ @@ -573,7 +572,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, return; } #endif - ASPRINTF(&str, sub_obj->allowed_cpuset); + ASPRINTF(&str, sub_obj->cpuset); fprintf(stderr, "Couldn't bind to cpuset %s because %s (%i)\n", str, strerror(i), i); FREE(str, strlen(str)); From 896879de8b1827e03b772703c0eb3e5bba06ee4d Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Tue, 22 Dec 2020 18:36:36 -0800 Subject: [PATCH 5/9] Update calls to hwloc2.x for binders to fix compilation error --- src/affinity/binders.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/affinity/binders.c b/src/affinity/binders.c index 1525e9b4a..40a84970b 100644 --- a/src/affinity/binders.c +++ b/src/affinity/binders.c @@ -47,12 +47,12 @@ void INTERNAL qt_affinity_balanced(int num_workers, } else { // No children workers.binds[start] = hwloc_bitmap_alloc(); - hwloc_bitmap_copy(workers.binds[start], obj->allowed_cpuset); + hwloc_bitmap_copy(workers.binds[start], obj->cpuset); if (num_workers > 1) { printf("warning: PU oversubscribed\n"); } - if (hwloc_bitmap_weight(obj->allowed_cpuset) != 1) { - printf("error: expected pu, got weight %d\n", hwloc_bitmap_weight(obj->allowed_cpuset)); + if (hwloc_bitmap_weight(obj->cpuset) != 1) { + printf("error: expected pu, got weight %d\n", hwloc_bitmap_weight(obj->cpuset)); exit(-1); } } @@ -73,9 +73,9 @@ int INTERNAL qt_affinity_compact(int num_workers, } else { // No children, should be PU workers.binds[workers.num - n] = hwloc_bitmap_alloc(); - hwloc_bitmap_copy(workers.binds[workers.num - n], obj->allowed_cpuset); - if (hwloc_bitmap_weight(obj->allowed_cpuset) != 1){ - printf("error: expected pu, got weight %d\n", hwloc_bitmap_weight(obj->allowed_cpuset)); + hwloc_bitmap_copy(workers.binds[workers.num - n], obj->cpuset); + if (hwloc_bitmap_weight(obj->cpuset) != 1){ + printf("error: expected pu, got weight %d\n", hwloc_bitmap_weight(obj->cpuset)); exit(-1); } return n - 1; From b8d64e1ce83fb0161b1b5beb7ce663306e4b9f8e Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Tue, 22 Dec 2020 18:42:39 -0800 Subject: [PATCH 6/9] Replace deprecated HWLOC API call (hwloc_set_area_membind_nodeset) --- src/affinity/hwloc_v2.c | 2 +- src/affinity/hwloc_via_chapel.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/affinity/hwloc_v2.c b/src/affinity/hwloc_v2.c index 73945586a..d5b3f776a 100644 --- a/src/affinity/hwloc_v2.c +++ b/src/affinity/hwloc_v2.c @@ -683,7 +683,7 @@ void INTERNAL qt_affinity_mem_tonode(void *addr, DEBUG_ONLY(hwloc_topology_check(sys_topo)); hwloc_bitmap_set(nodeset, node); - hwloc_set_area_membind_nodeset(sys_topo, addr, bytes, nodeset, + hwloc_set_area_membind(sys_topo, addr, bytes, nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_NOCPUBIND); hwloc_bitmap_free(nodeset); diff --git a/src/affinity/hwloc_via_chapel.c b/src/affinity/hwloc_via_chapel.c index 7ea25f109..5a5ade6cf 100644 --- a/src/affinity/hwloc_via_chapel.c +++ b/src/affinity/hwloc_via_chapel.c @@ -338,7 +338,7 @@ void INTERNAL qt_affinity_mem_tonode(void *addr, DEBUG_ONLY(hwloc_topology_check(topology)); hwloc_bitmap_set(nodeset, node); - hwloc_set_area_membind_nodeset(topology, addr, bytes, nodeset, + hwloc_set_area_membind(topology, addr, bytes, nodeset, HWLOC_MEMBIND_BIND, HWLOC_MEMBIND_NOCPUBIND); hwloc_bitmap_free(nodeset); From 92f3d83df4da297dfe45ac0672398770c8d23596 Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Tue, 22 Dec 2020 19:42:59 -0800 Subject: [PATCH 7/9] Update change log --- NEWS | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/NEWS b/NEWS index 21e0268b7..d91f2b472 100644 --- a/NEWS +++ b/NEWS @@ -1,9 +1,16 @@ ---- 1.16 (anticipated 2020) --- +--- 1.16 (current) --- + +Improvements: + - Add support for Spack testing (spack test run qthreads) + +Bugfixes: + - Fix compilation error when using HWLOC2.x + - Remove use of deprecated hwloc APIs (minimal required version is hwloc1.15) Documentation: - User guide for basic Qthreads usage (see userguide subdirectory for LaTex and included code source files) ---- 1.15 (current) --- +--- 1.15 --- New Features: - Add experimental support for Thread Local Storage and new Condition variable initializers From 1cbd0c8b9426b1ff94668cea2913ab1528358096 Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Wed, 23 Dec 2020 10:37:44 -0800 Subject: [PATCH 8/9] Minimal supported hwloc version is 1.5 --- NEWS | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/NEWS b/NEWS index d91f2b472..40a7949ad 100644 --- a/NEWS +++ b/NEWS @@ -2,10 +2,7 @@ Improvements: - Add support for Spack testing (spack test run qthreads) - -Bugfixes: - - Fix compilation error when using HWLOC2.x - - Remove use of deprecated hwloc APIs (minimal required version is hwloc1.15) + - Add support for hwloc2.x (minimal required version is hwloc1.5) Documentation: - User guide for basic Qthreads usage (see userguide subdirectory for LaTex and included code source files) From 0c0937314e4024494d5362ce4c8a28ba89a2a834 Mon Sep 17 00:00:00 2001 From: "Jan Ciesko (-EXP)" Date: Wed, 6 Jan 2021 10:36:42 -0800 Subject: [PATCH 9/9] Update use of 'allowed_cpuset' to 'cpuset' for hwloc_via_chapel --- src/affinity/hwloc_via_chapel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/affinity/hwloc_via_chapel.c b/src/affinity/hwloc_via_chapel.c index 5a5ade6cf..9131e5d56 100644 --- a/src/affinity/hwloc_via_chapel.c +++ b/src/affinity/hwloc_via_chapel.c @@ -531,7 +531,7 @@ void INTERNAL qt_affinity_set(qthread_worker_t *me, (int)hwloc_get_nbobjs_inside_cpuset_by_type(topology, allowed_cpuset, HWLOC_OBJ_PU)); int shep_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, obj->cpuset, HWLOC_OBJ_PU); assert(shep_pus > 0); - unsigned int worker_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, worker0->allowed_cpuset, HWLOC_OBJ_PU); + unsigned int worker_pus = hwloc_get_nbobjs_inside_cpuset_by_type(topology, worker0->cpuset, HWLOC_OBJ_PU); unsigned int wraparounds = me->packed_worker_id / (maxshepobjs * nworkerspershep); unsigned int worker_wraparounds = me->worker_id / workerobjs_per_shep; hwloc_obj_t sub_obj =