From 82801ac3bb6b194b563dd6cc8bfd5a48182253c0 Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Mon, 9 Dec 2024 16:23:24 -0500 Subject: [PATCH 1/4] Get system memory size from procfs --- .../core/driver/xdna/amd_xdna_driver.cpp | 30 ++++++++++++++++++- .../hsa-runtime/core/inc/amd_xdna_driver.h | 3 ++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp index edee88b73..15a283e64 100644 --- a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp +++ b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp @@ -45,6 +45,8 @@ #include #include +#include +#include #include #include @@ -64,7 +66,7 @@ XdnaDriver::~XdnaDriver() { FreeDeviceHeap(); } hsa_status_t XdnaDriver::DiscoverDriver() { const int max_minor_num(64); - const std::string devnode_prefix("/dev/accel/accel"); + static const std::string devnode_prefix("/dev/accel/accel"); for (int i = 0; i < max_minor_num; ++i) { std::unique_ptr xdna_drv( @@ -84,6 +86,32 @@ hsa_status_t XdnaDriver::DiscoverDriver() { return HSA_STATUS_ERROR; } +uint64_t XdnaDriver::GetSystemMemoryByteSize() { + static const std::string meminfo_node("/proc/meminfo"); + + FILE *file = std::fopen(meminfo_node.c_str(), "r"); + if (file == nullptr) + throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_FILE, + "Invalid memory info file."); + + const int expected_args = 2; + int result = 0; + char property_name[256]; + unsigned long long property_val = 0; + while ((result = std::fscanf(file, "%255s %llu\n", property_name, + &property_val)) == expected_args) { + if (std::strcmp(property_name, "MemTotal:") == 0) + break; + } + std::fclose(file); + + if ((result == EOF) || (result != expected_args)) + throw AMD::hsa_exception(HSA_STATUS_ERROR, "Could not parse file."); + + // memory reported is in Kbytes + return property_val * 1024ull; +} + uint64_t XdnaDriver::GetDevHeapByteSize() { return dev_heap_size; } diff --git a/runtime/hsa-runtime/core/inc/amd_xdna_driver.h b/runtime/hsa-runtime/core/inc/amd_xdna_driver.h index 4c0cad47f..156a9b0b4 100644 --- a/runtime/hsa-runtime/core/inc/amd_xdna_driver.h +++ b/runtime/hsa-runtime/core/inc/amd_xdna_driver.h @@ -133,6 +133,9 @@ class XdnaDriver final : public core::Driver { static hsa_status_t DiscoverDriver(); + /// @brief Returns the size of the system memory heap in bytes. + static uint64_t GetSystemMemoryByteSize(); + /// @brief Returns the size of the dev heap in bytes. static uint64_t GetDevHeapByteSize(); From 413f83b64f490f7ce7c826d01bd69a854754f606 Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Mon, 9 Dec 2024 16:35:55 -0500 Subject: [PATCH 2/4] Adding system memory size and extra non-kernarg pool to AIE agents --- runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp index 3c823b464..e9f7a3a59 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp @@ -282,11 +282,18 @@ void AieAgent::InitRegionList() { /// This should be easier once the ROCt source is incorporated into the /// ROCr source. Since the AIE itself currently has no memory regions of /// its own all memory is just the system DRAM. + const uint64_t total_system_memory = XdnaDriver::GetSystemMemoryByteSize(); /// For allocating kernel arguments or other objects that only need /// system memory. HsaMemoryProperties sys_mem_props = {}; sys_mem_props.HeapType = HSA_HEAPTYPE_SYSTEM; + sys_mem_props.SizeInBytes = total_system_memory; + + /// For any other allocation, e.g., buffers. + HsaMemoryProperties other_mem_props = {}; + other_mem_props.HeapType = HSA_HEAPTYPE_SYSTEM; + other_mem_props.SizeInBytes = total_system_memory; /// For allocating memory for programmable device image (PDI) files. These /// need to be mapped to the device so the hardware can access the PDIs. @@ -296,11 +303,13 @@ void AieAgent::InitRegionList() { /// As of now the AIE devices support coarse-grain memory regions that require /// explicit sync operations. - regions_.reserve(2); + regions_.reserve(3); regions_.push_back( new MemoryRegion(false, true, false, false, true, this, sys_mem_props)); regions_.push_back( new MemoryRegion(false, false, false, false, true, this, dev_mem_props)); + regions_.push_back(new MemoryRegion(false, false, false, false, true, this, + other_mem_props)); } void AieAgent::GetAgentProperties() { From 91e24d27d75e49b9e8a275d43cadff89b975fe7e Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Mon, 9 Dec 2024 16:43:33 -0500 Subject: [PATCH 3/4] Making expression into a statement --- runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp b/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp index e9f7a3a59..d655d0fa2 100644 --- a/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp +++ b/runtime/hsa-runtime/core/runtime/amd_aie_agent.cpp @@ -298,7 +298,7 @@ void AieAgent::InitRegionList() { /// For allocating memory for programmable device image (PDI) files. These /// need to be mapped to the device so the hardware can access the PDIs. HsaMemoryProperties dev_mem_props = {}; - dev_mem_props.HeapType = HSA_HEAPTYPE_DEVICE_SVM, + dev_mem_props.HeapType = HSA_HEAPTYPE_DEVICE_SVM; dev_mem_props.SizeInBytes = XdnaDriver::GetDevHeapByteSize(); /// As of now the AIE devices support coarse-grain memory regions that require From c3470e16f4b4996b5b8682fca4bec978b4db430f Mon Sep 17 00:00:00 2001 From: Yiannis Papadopoulos Date: Mon, 9 Dec 2024 17:15:34 -0500 Subject: [PATCH 4/4] Using sysconf to calculate system memory --- .../core/driver/xdna/amd_xdna_driver.cpp | 29 +++---------------- 1 file changed, 4 insertions(+), 25 deletions(-) diff --git a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp index 15a283e64..997fabf00 100644 --- a/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp +++ b/runtime/hsa-runtime/core/driver/xdna/amd_xdna_driver.cpp @@ -44,9 +44,8 @@ #include #include +#include -#include -#include #include #include @@ -87,29 +86,9 @@ hsa_status_t XdnaDriver::DiscoverDriver() { } uint64_t XdnaDriver::GetSystemMemoryByteSize() { - static const std::string meminfo_node("/proc/meminfo"); - - FILE *file = std::fopen(meminfo_node.c_str(), "r"); - if (file == nullptr) - throw AMD::hsa_exception(HSA_STATUS_ERROR_INVALID_FILE, - "Invalid memory info file."); - - const int expected_args = 2; - int result = 0; - char property_name[256]; - unsigned long long property_val = 0; - while ((result = std::fscanf(file, "%255s %llu\n", property_name, - &property_val)) == expected_args) { - if (std::strcmp(property_name, "MemTotal:") == 0) - break; - } - std::fclose(file); - - if ((result == EOF) || (result != expected_args)) - throw AMD::hsa_exception(HSA_STATUS_ERROR, "Could not parse file."); - - // memory reported is in Kbytes - return property_val * 1024ull; + const long pagesize = sysconf(_SC_PAGESIZE); + const long page_count = sysconf(_SC_PHYS_PAGES); + return pagesize * page_count; } uint64_t XdnaDriver::GetDevHeapByteSize() {