From 708b2b516f8a24e88a6e95f20a6e6473c76560e9 Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Mon, 17 Jun 2024 12:40:38 +0100
Subject: [PATCH 01/17] Added device option to validParams()

---
 test/tests/unit/kernels/diffusion_mfem.i | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/tests/unit/kernels/diffusion_mfem.i b/test/tests/unit/kernels/diffusion_mfem.i
index d960cde4..d3e359bb 100644
--- a/test/tests/unit/kernels/diffusion_mfem.i
+++ b/test/tests/unit/kernels/diffusion_mfem.i
@@ -7,6 +7,7 @@
 [Problem]
   type = MFEMProblem
   use_glvis = true
+  device = "cuda"
 []
 
 [Formulation]

From 9d5f229e8bc3274658dcbc5cc72a3b2ae900fd60 Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Mon, 17 Jun 2024 15:34:28 +0100
Subject: [PATCH 02/17] Moved Device member

---
 test/tests/unit/kernels/diffusion_mfem.i | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/tests/unit/kernels/diffusion_mfem.i b/test/tests/unit/kernels/diffusion_mfem.i
index d3e359bb..250c073a 100644
--- a/test/tests/unit/kernels/diffusion_mfem.i
+++ b/test/tests/unit/kernels/diffusion_mfem.i
@@ -7,7 +7,7 @@
 [Problem]
   type = MFEMProblem
   use_glvis = true
-  device = "cuda"
+  device = "cpu"
 []
 
 [Formulation]

From c1006348209129833a86d0172a4f8504e81a7e60 Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Mon, 17 Jun 2024 15:39:17 +0100
Subject: [PATCH 03/17] Added platypus GPU build script

---
 scripts/build-platypus-csd3-ampere.sh | 242 ++++++++++++++++++++++++++
 1 file changed, 242 insertions(+)
 create mode 100755 scripts/build-platypus-csd3-ampere.sh

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
new file mode 100755
index 00000000..6eee1ca8
--- /dev/null
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -0,0 +1,242 @@
+#!/bin/bash
+#SBATCH --nodes=1
+#SBATCH --ntasks=1
+#SBATCH --time=04:00:00
+#SBATCH --mail-type=none
+#SBATCH -p ampere
+#SBATCH -A ukaea-ap001-GPU
+#SBATCH --cpus-per-task=32
+#SBATCH --gres=gpu:1
+#SBATCH --output=platypus_gpu_build/build.%j.out
+#SBATCH --error=platypus_gpu_build/build.%j.err
+
+## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH 
+## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO 
+## THAT, COMMENT OUT THE SPACK UNINSTALL LINE BEFORE SUBMITTING THE SCRIPT
+## -> UNINSTALL LINE IN THE install_spack_deps() FUNCTION
+ARCH="linux-rocky8-zen"
+
+export compile_cores=32
+
+load_modules(){
+
+    # Load modules
+    . /etc/profile.d/modules.sh                # Leave this line (enables the module command)
+    module purge
+    module load rhel8/slurm
+    module use /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-rocky8-zen3
+    module load cuda/11.7.1
+
+}
+
+set_paths(){
+
+    USER=`whoami`
+    BUILD_PREFIX=platypus_gpu
+    BUILD_DIR_NAME=${BUILD_PREFIX}_build
+
+    ROOT_PATH=/home/${USER}/rds/rds-ukaea-ap001/${USER}
+    BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME}
+
+    echo "Building in ${BUILD_PATH}"
+    mkdir -p ${BUILD_PATH} || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
+
+    cd ${ROOT_PATH}
+    . spack/share/spack/setup-env.sh
+
+    cd ${BUILD_PATH}
+
+}
+
+install_spack_deps(){
+
+    # Cleaning up everything to start with a new environment
+    spack uninstall -ay arch=${ARCH}
+
+    spack external find cuda@11.7.1
+
+    echo "Installing libfabric..."
+    spack install libfabric@1.19.0 # 1.19.0 is the latest libfabric version that works on CSD3's ampere nodes
+    spack load libfabric arch=${ARCH}
+
+    echo "Installing hypre..."
+    spack install hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist \
+                  ^mpich +cuda cuda_arch=80 \
+                  ^superlu-dist +cuda cuda_arch=80 +parmetis
+    spack load hypre arch=${ARCH}
+
+    echo "Installing Petsc..."
+    # spack's petsc doesn't like openmpi, but it works with mpich
+    spack install petsc +cuda cuda_arch=80 +fortran +hdf5 +hypre +metis +mpi \
+              ^mpich +cuda cuda_arch=80 \
+              ^hdf5 +cxx +fortran +hl +mpi +shared
+    spack load petsc arch=${ARCH}
+
+    echo "Installing SLEPc..."
+    spack install slepc +cuda cuda_arch=80
+    spack load slepc arch=${ARCH}
+
+    echo "Installing netcdf..."
+    spack install netcdf-c +parallel-netcdf
+    spack load netcdf-c arch=${ARCH}
+
+    echo "Installing ninja..."
+    spack install ninja
+    spack load ninja arch=${ARCH}
+
+    echo "Adding python modules..."
+
+    spack install py-pyaml
+    spack load py-pyaml arch=${ARCH}
+
+    spack install py-jinja2
+    spack load py-jinja2 arch=${ARCH}
+
+    spack install py-packaging
+    spack load py-packaging arch=${ARCH}
+
+    spack install py-setuptools
+    spack load py-setuptools arch=${ARCH}
+
+
+}
+
+install_gslib(){
+
+    echo "Installing gslib..."
+    cd ${BUILD_PATH}
+    git clone https://github.com/Nek5000/gslib.git
+    cd gslib
+    make CC=mpicc CFLAGS="-O2 -fPIC" -j"$compile_cores"
+}
+
+install_mfem(){
+
+    export CXX=mpic++
+    export CC=mpicc
+    export F90=mpif90
+    export F77=mpif77
+    export FC=mpif90
+
+    # Build MFEM
+    cd ${BUILD_PATH}
+    git clone https://github.com/Heinrich-BR/mfem.git
+    cd mfem
+    git checkout master
+    mkdir build
+    cd build
+    echo "Building MFEM"
+    cmake .. \
+        -DCMAKE_BUILD_TYPE=Release \
+        -DBUILD_SHARED_LIBS=YES \
+        -DMFEM_USE_OPENMP=NO \
+        -DMFEM_THREAD_SAFE=YES \
+        -DMFEM_ENABLE_EXAMPLES=YES \
+    	-DMFEM_ENABLE_MINIAPPS=YES \
+        -DMFEM_USE_MPI=YES \
+        -DMFEM_USE_CUDA=YES \
+        -DCUDA_ARCH=sm_80 \
+        -DMFEM_USE_METIS_5=YES \
+        -DMFEM_USE_SUPERLU=YES \
+        -DMFEM_USE_NETCDF=YES \
+        -DMFEM_USE_GSLIB=YES \
+        -DGSLIB_DIR=${BUILD_PATH}/gslib/build
+
+    if [ $? -eq 2 ]; then
+        echo "MFEM config failed"
+        exit 1
+    fi 
+
+    make -j"$compile_cores"
+
+    if [ $? -eq 2 ]; then
+        echo "MFEM build failed"
+        exit 1
+    fi
+
+    LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BUILD_PATH}/mfem/build:${BUILD_PATH}/mfem/build/miniapps/common
+}
+
+install_moose(){
+
+    # Some of the variables needed
+    export MOOSE_JOBS=$compile_cores
+    export LIBMESH_JOBS=$compile_cores
+    export METHOD="opt"
+    export SLEPC_DIR=`spack find --format "{prefix}" slepc arch=${ARCH}`
+
+
+    cd ${BUILD_PATH}
+    git clone https://github.com/idaholab/moose
+    cd moose
+
+    echo "Building libmesh..."
+    ./scripts/update_and_rebuild_libmesh.sh --with-mpi
+    if [ $? -eq 2 ]; then
+        echo "libmesh build failed"
+        exit 1
+    fi
+
+    echo "Building WASP..."
+    ./scripts/update_and_rebuild_wasp.sh
+    if [ $? -eq 2 ]; then
+        echo "WASP build failed"
+        exit 1
+    fi 
+
+    ./configure --with-derivative-size=200
+    if [ $? -eq 2 ]; then
+        echo "MOOSE configure failed"
+        exit 1
+    fi 
+
+    cd framework
+    make -j"$compile_cores"
+    if [ $? -eq 2 ]; then
+        echo "MOOSE framework build failed"
+        exit 1
+    fi 
+
+    cd ../modules
+    make -j"$compile_cores"
+    if [ $? -eq 2 ]; then
+        echo "MOOSE modules build failed"
+        exit 1
+    fi 
+
+    # This takes very long! Only run the tests if you really need to!
+    #cd ../test
+    #make -j"$compile_cores"
+    #if [ $? -eq 2 ]; then
+    #    echo "MOOSE test build failed"
+    #    exit 1
+    #fi 
+
+    #./run_tests -j"$compile_cores"
+}
+
+install_platypus(){
+
+    cd ${BUILD_PATH}
+
+    echo "Building platypus..."
+    git clone https://github.com/aurora-multiphysics/platypus.git
+    cd platypus
+    git submodule update --init --recursive
+    cd contrib/hephaestus/
+    mkdir build
+    cd build
+    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR=${BUILD_PATH}/mfem/build ..
+    ninja
+    cd ${BUILD_PATH}/platypus
+    make -j"$compile_cores"
+
+}
+
+load_modules
+set_paths
+install_spack_deps
+install_gslib
+install_mfem
+install_moose
+install_platypus
\ No newline at end of file

From b6d2e32c6789806fe5f6f818df91b0099de5b1cf Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 9 Jul 2024 14:53:35 +0100
Subject: [PATCH 04/17] Added spack check to build script + linting fixes

---
 scripts/build-platypus-csd3-ampere.sh | 104 ++++++++++++++------------
 1 file changed, 56 insertions(+), 48 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 6eee1ca8..9830eba7 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -7,8 +7,8 @@
 #SBATCH -A ukaea-ap001-GPU
 #SBATCH --cpus-per-task=32
 #SBATCH --gres=gpu:1
-#SBATCH --output=platypus_gpu_build/build.%j.out
-#SBATCH --error=platypus_gpu_build/build.%j.err
+#SBATCH --output=platypus_gpu_build.%j.out
+#SBATCH --error=platypus_gpu_build.%j.err
 
 ## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH 
 ## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO 
@@ -18,7 +18,7 @@ ARCH="linux-rocky8-zen"
 
 export compile_cores=32
 
-load_modules(){
+load_modules() {
 
     # Load modules
     . /etc/profile.d/modules.sh                # Leave this line (enables the module command)
@@ -29,9 +29,9 @@ load_modules(){
 
 }
 
-set_paths(){
+set_paths() {
 
-    USER=`whoami`
+    USER=$(whoami)
     BUILD_PREFIX=platypus_gpu
     BUILD_DIR_NAME=${BUILD_PREFIX}_build
 
@@ -39,37 +39,43 @@ set_paths(){
     BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME}
 
     echo "Building in ${BUILD_PATH}"
-    mkdir -p ${BUILD_PATH} || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
+    mkdir -p "${BUILD_PATH}" || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
 
-    cd ${ROOT_PATH}
-    . spack/share/spack/setup-env.sh
+    cd "${BUILD_PATH}" || exit 1
 
-    cd ${BUILD_PATH}
+}
+
+check_spack() {
+
+    cd "${ROOT_PATH}" || exit 1
+     
+    if [ $(command -v spack) ]; then
+        echo "Spack command detected. Using pre-loaded spack."
+    elif [ -f ${ROOT_PATH}/spack/share/spack/setup-env.sh ]; then
+        echo "Spack detected in root directory. Loading."
+        . spack/share/spack/setup-env.sh
+    else
+        echo "No spack detected. Building from source."
+        git clone --depth=100 https://github.com/spack/spack.git
+        . spack/share/spack/setup-env.sh
+    fi
 
 }
 
-install_spack_deps(){
+install_spack_deps() {
 
     # Cleaning up everything to start with a new environment
     spack uninstall -ay arch=${ARCH}
 
     spack external find cuda@11.7.1
 
-    echo "Installing libfabric..."
-    spack install libfabric@1.19.0 # 1.19.0 is the latest libfabric version that works on CSD3's ampere nodes
-    spack load libfabric arch=${ARCH}
-
-    echo "Installing hypre..."
-    spack install hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist \
-                  ^mpich +cuda cuda_arch=80 \
-                  ^superlu-dist +cuda cuda_arch=80 +parmetis
-    spack load hypre arch=${ARCH}
-
     echo "Installing Petsc..."
-    # spack's petsc doesn't like openmpi, but it works with mpich
+    # Spack's petsc doesn't like openmpi, but it works with mpich
     spack install petsc +cuda cuda_arch=80 +fortran +hdf5 +hypre +metis +mpi \
-              ^mpich +cuda cuda_arch=80 \
-              ^hdf5 +cxx +fortran +hl +mpi +shared
+        ^mpich +cuda cuda_arch=80 \
+        ^hdf5 +cxx +fortran +hl +mpi +shared \
+        ^hypre +mpi +shared +cuda cuda_arch=80 +superlu-dist +cublas +gpu-aware-mpi \
+        ^superlu-dist +cuda cuda_arch=80 +parmetis +shared
     spack load petsc arch=${ARCH}
 
     echo "Installing SLEPc..."
@@ -101,16 +107,16 @@ install_spack_deps(){
 
 }
 
-install_gslib(){
+install_gslib() {
 
     echo "Installing gslib..."
-    cd ${BUILD_PATH}
+    cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/Nek5000/gslib.git
-    cd gslib
+    cd gslib || exit 1
     make CC=mpicc CFLAGS="-O2 -fPIC" -j"$compile_cores"
 }
 
-install_mfem(){
+install_mfem() {
 
     export CXX=mpic++
     export CC=mpicc
@@ -119,12 +125,13 @@ install_mfem(){
     export FC=mpif90
 
     # Build MFEM
-    cd ${BUILD_PATH}
-    git clone https://github.com/Heinrich-BR/mfem.git
-    cd mfem
-    git checkout master
+    cd "${BUILD_PATH}" || exit 1
+    git clone https://github.com/mfem/mfem.git
+    cd mfem || exit 1
+    # This is just until MFEM merges Edward's changes. Without this, GPU build crashes!
+    git checkout EdwardPalmer99/add-missing-header-to-exodus-writer-fix
     mkdir build
-    cd build
+    cd build || exit 1
     echo "Building MFEM"
     cmake .. \
         -DCMAKE_BUILD_TYPE=Release \
@@ -132,7 +139,7 @@ install_mfem(){
         -DMFEM_USE_OPENMP=NO \
         -DMFEM_THREAD_SAFE=YES \
         -DMFEM_ENABLE_EXAMPLES=YES \
-    	-DMFEM_ENABLE_MINIAPPS=YES \
+        -DMFEM_ENABLE_MINIAPPS=YES \
         -DMFEM_USE_MPI=YES \
         -DMFEM_USE_CUDA=YES \
         -DCUDA_ARCH=sm_80 \
@@ -140,7 +147,7 @@ install_mfem(){
         -DMFEM_USE_SUPERLU=YES \
         -DMFEM_USE_NETCDF=YES \
         -DMFEM_USE_GSLIB=YES \
-        -DGSLIB_DIR=${BUILD_PATH}/gslib/build
+        -DGSLIB_DIR="${BUILD_PATH}/gslib/build"
 
     if [ $? -eq 2 ]; then
         echo "MFEM config failed"
@@ -157,18 +164,18 @@ install_mfem(){
     LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:${BUILD_PATH}/mfem/build:${BUILD_PATH}/mfem/build/miniapps/common
 }
 
-install_moose(){
+install_moose() {
 
     # Some of the variables needed
     export MOOSE_JOBS=$compile_cores
     export LIBMESH_JOBS=$compile_cores
     export METHOD="opt"
-    export SLEPC_DIR=`spack find --format "{prefix}" slepc arch=${ARCH}`
+    export SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH})
 
 
-    cd ${BUILD_PATH}
+    cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/idaholab/moose
-    cd moose
+    cd moose || exit 1
 
     echo "Building libmesh..."
     ./scripts/update_and_rebuild_libmesh.sh --with-mpi
@@ -190,14 +197,14 @@ install_moose(){
         exit 1
     fi 
 
-    cd framework
+    cd framework || exit 1
     make -j"$compile_cores"
     if [ $? -eq 2 ]; then
         echo "MOOSE framework build failed"
         exit 1
     fi 
 
-    cd ../modules
+    cd ../modules || exit 1
     make -j"$compile_cores"
     if [ $? -eq 2 ]; then
         echo "MOOSE modules build failed"
@@ -205,7 +212,7 @@ install_moose(){
     fi 
 
     # This takes very long! Only run the tests if you really need to!
-    #cd ../test
+    #cd ../test || exit 1
     #make -j"$compile_cores"
     #if [ $? -eq 2 ]; then
     #    echo "MOOSE test build failed"
@@ -215,26 +222,27 @@ install_moose(){
     #./run_tests -j"$compile_cores"
 }
 
-install_platypus(){
+install_platypus() {
 
-    cd ${BUILD_PATH}
+    cd "${BUILD_PATH}" || exit 1
 
     echo "Building platypus..."
     git clone https://github.com/aurora-multiphysics/platypus.git
-    cd platypus
+    cd platypus || exit 1
     git submodule update --init --recursive
-    cd contrib/hephaestus/
+    cd contrib/hephaestus/ || exit 1
     mkdir build
-    cd build
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR=${BUILD_PATH}/mfem/build ..
+    cd build || exit 1
+    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR="${BUILD_PATH}/mfem/build" ..
     ninja
-    cd ${BUILD_PATH}/platypus
+    cd ${BUILD_PATH}/platypus || exit 1
     make -j"$compile_cores"
 
 }
 
 load_modules
 set_paths
+check_spack
 install_spack_deps
 install_gslib
 install_mfem

From 3b31eb75dde1ed213e14684b0b17632921cf3167 Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 9 Jul 2024 15:29:00 +0100
Subject: [PATCH 05/17] Linting fixes on build script

---
 scripts/build-platypus-csd3-ampere.sh | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 9830eba7..8579219d 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -10,8 +10,8 @@
 #SBATCH --output=platypus_gpu_build.%j.out
 #SBATCH --error=platypus_gpu_build.%j.err
 
-## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH 
-## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO 
+## WARNING: THIS SCRIPT WILL UNINSTALL ALL SPACK MODULES ASSOCIATED WITH
+## THE ARCHITECTURE DEFINED IN THE ARCH VARIABLE. IF YOU DO NOT WISH TO DO
 ## THAT, COMMENT OUT THE SPACK UNINSTALL LINE BEFORE SUBMITTING THE SCRIPT
 ## -> UNINSTALL LINE IN THE install_spack_deps() FUNCTION
 ARCH="linux-rocky8-zen"
@@ -39,7 +39,10 @@ set_paths() {
     BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME}
 
     echo "Building in ${BUILD_PATH}"
-    mkdir -p "${BUILD_PATH}" || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
+    mkdir -p "${BUILD_PATH}" || {
+                                  echo "Failed to create ${BUILD_PATH}"
+                                                                          exit 1
+    }
 
     cd "${BUILD_PATH}" || exit 1
 
@@ -48,7 +51,7 @@ set_paths() {
 check_spack() {
 
     cd "${ROOT_PATH}" || exit 1
-     
+
     if [ $(command -v spack) ]; then
         echo "Spack command detected. Using pre-loaded spack."
     elif [ -f ${ROOT_PATH}/spack/share/spack/setup-env.sh ]; then
@@ -104,7 +107,6 @@ install_spack_deps() {
     spack install py-setuptools
     spack load py-setuptools arch=${ARCH}
 
-
 }
 
 install_gslib() {
@@ -152,7 +154,7 @@ install_mfem() {
     if [ $? -eq 2 ]; then
         echo "MFEM config failed"
         exit 1
-    fi 
+    fi
 
     make -j"$compile_cores"
 
@@ -172,7 +174,6 @@ install_moose() {
     export METHOD="opt"
     export SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH})
 
-
     cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/idaholab/moose
     cd moose || exit 1
@@ -189,27 +190,27 @@ install_moose() {
     if [ $? -eq 2 ]; then
         echo "WASP build failed"
         exit 1
-    fi 
+    fi
 
     ./configure --with-derivative-size=200
     if [ $? -eq 2 ]; then
         echo "MOOSE configure failed"
         exit 1
-    fi 
+    fi
 
     cd framework || exit 1
     make -j"$compile_cores"
     if [ $? -eq 2 ]; then
         echo "MOOSE framework build failed"
         exit 1
-    fi 
+    fi
 
     cd ../modules || exit 1
     make -j"$compile_cores"
     if [ $? -eq 2 ]; then
         echo "MOOSE modules build failed"
         exit 1
-    fi 
+    fi
 
     # This takes very long! Only run the tests if you really need to!
     #cd ../test || exit 1
@@ -217,7 +218,7 @@ install_moose() {
     #if [ $? -eq 2 ]; then
     #    echo "MOOSE test build failed"
     #    exit 1
-    #fi 
+    #fi
 
     #./run_tests -j"$compile_cores"
 }

From 8695ae2207847589ff3363f0159905e4d9ce73de Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 9 Jul 2024 15:37:07 +0100
Subject: [PATCH 06/17] More linting fixes in build script

---
 scripts/build-platypus-csd3-ampere.sh | 17 +++++++++--------
 1 file changed, 9 insertions(+), 8 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 8579219d..003deec6 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -21,6 +21,8 @@ export compile_cores=32
 load_modules() {
 
     # Load modules
+
+    # shellcheck source=/dev/null
     . /etc/profile.d/modules.sh                # Leave this line (enables the module command)
     module purge
     module load rhel8/slurm
@@ -39,10 +41,7 @@ set_paths() {
     BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME}
 
     echo "Building in ${BUILD_PATH}"
-    mkdir -p "${BUILD_PATH}" || {
-                                  echo "Failed to create ${BUILD_PATH}"
-                                                                          exit 1
-    }
+    mkdir -p "${BUILD_PATH}" || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
 
     cd "${BUILD_PATH}" || exit 1
 
@@ -51,15 +50,17 @@ set_paths() {
 check_spack() {
 
     cd "${ROOT_PATH}" || exit 1
-
-    if [ $(command -v spack) ]; then
+    
+    if [ "$(command -v spack)" ]; then
         echo "Spack command detected. Using pre-loaded spack."
-    elif [ -f ${ROOT_PATH}/spack/share/spack/setup-env.sh ]; then
+    elif [ -f "${ROOT_PATH}"/spack/share/spack/setup-env.sh ]; then
         echo "Spack detected in root directory. Loading."
+        # shellcheck source=/dev/null
         . spack/share/spack/setup-env.sh
     else
         echo "No spack detected. Building from source."
         git clone --depth=100 https://github.com/spack/spack.git
+        # shellcheck source=/dev/null
         . spack/share/spack/setup-env.sh
     fi
 
@@ -236,7 +237,7 @@ install_platypus() {
     cd build || exit 1
     cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR="${BUILD_PATH}/mfem/build" ..
     ninja
-    cd ${BUILD_PATH}/platypus || exit 1
+    cd "${BUILD_PATH}"/platypus || exit 1
     make -j"$compile_cores"
 
 }

From cde688c7149e5cf0b9fabb740041b63e33fe436c Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 9 Jul 2024 15:43:29 +0100
Subject: [PATCH 07/17] Hopefully last batch of bash script linting fixes

---
 scripts/build-platypus-csd3-ampere.sh | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 003deec6..8463fba2 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -41,7 +41,10 @@ set_paths() {
     BUILD_PATH=${ROOT_PATH}/${BUILD_DIR_NAME}
 
     echo "Building in ${BUILD_PATH}"
-    mkdir -p "${BUILD_PATH}" || { echo "Failed to create ${BUILD_PATH}" ; exit 1 ; }
+    mkdir -p "${BUILD_PATH}" || {
+                                  echo "Failed to create ${BUILD_PATH}"
+                                                                          exit 1
+    }
 
     cd "${BUILD_PATH}" || exit 1
 
@@ -50,7 +53,7 @@ set_paths() {
 check_spack() {
 
     cd "${ROOT_PATH}" || exit 1
-    
+
     if [ "$(command -v spack)" ]; then
         echo "Spack command detected. Using pre-loaded spack."
     elif [ -f "${ROOT_PATH}"/spack/share/spack/setup-env.sh ]; then
@@ -173,7 +176,8 @@ install_moose() {
     export MOOSE_JOBS=$compile_cores
     export LIBMESH_JOBS=$compile_cores
     export METHOD="opt"
-    export SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH})
+    SLEPC_DIR=$(spack find --format "{prefix}" slepc arch=${ARCH})
+    export SLEPC_DIR
 
     cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/idaholab/moose

From e84050aa0086d718f1437efecde2f4e8c2d42f65 Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 9 Jul 2024 15:52:52 +0100
Subject: [PATCH 08/17] Indentation

---
 scripts/build-platypus-csd3-ampere.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 8463fba2..a5532de8 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -42,8 +42,8 @@ set_paths() {
 
     echo "Building in ${BUILD_PATH}"
     mkdir -p "${BUILD_PATH}" || {
-                                  echo "Failed to create ${BUILD_PATH}"
-                                                                          exit 1
+                                    echo "Failed to create ${BUILD_PATH}"
+                                                                            exit 1
     }
 
     cd "${BUILD_PATH}" || exit 1

From 41bbf9e7e23b10699a694eea09f9a3ff15537420 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 9 Jul 2024 16:17:24 +0100
Subject: [PATCH 09/17] clang-format fixes

---
 scripts/build-platypus-csd3-ampere.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index a5532de8..4f434a28 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -253,4 +253,4 @@ install_spack_deps
 install_gslib
 install_mfem
 install_moose
-install_platypus
\ No newline at end of file
+install_platypus

From 3355db8ad393970d94fa1b3778ae19c0e163d72f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Wed, 10 Jul 2024 16:52:07 +0100
Subject: [PATCH 10/17] Added spack clean line to ampere script

---
 scripts/build-platypus-csd3-ampere.sh | 1 +
 1 file changed, 1 insertion(+)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 4f434a28..87ccc918 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -73,6 +73,7 @@ install_spack_deps() {
 
     # Cleaning up everything to start with a new environment
     spack uninstall -ay arch=${ARCH}
+    spack clean -ab
 
     spack external find cuda@11.7.1
 

From 5e6bcaddb0594430de88d8b7cfefd4232a0b126e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Thu, 11 Jul 2024 14:12:10 +0100
Subject: [PATCH 11/17] Added CUDA check such that correct module is loaded

---
 scripts/build-platypus-csd3-ampere.sh | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 87ccc918..1d9c0280 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -5,7 +5,7 @@
 #SBATCH --mail-type=none
 #SBATCH -p ampere
 #SBATCH -A ukaea-ap001-GPU
-#SBATCH --cpus-per-task=32
+#SBATCH --cpus-per-task=1
 #SBATCH --gres=gpu:1
 #SBATCH --output=platypus_gpu_build.%j.out
 #SBATCH --error=platypus_gpu_build.%j.err
@@ -50,6 +50,21 @@ set_paths() {
 
 }
 
+check_cuda_version() {
+
+    if grep -q "cuda@11.7.1" "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml; then
+        echo "Ampere CUDA module found in spack."
+    else
+        echo "Ampere CUDA module not found in spack. Adding to packages.yaml."
+        CUDA_STR=$'  cuda:\n    externals:\n    - spec: "cuda@11.7.1"\n      prefix: /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-rocky8-zen3/cuda/11.7.1/gcc/fgtvtwi5\n    buildable: False'
+        echo "${CUDA_STR}"  >> "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml
+    fi
+
+    # shellcheck source=/dev/null
+    . "${SPACK_ROOT}"/share/spack/setup-env.sh
+
+}
+
 check_spack() {
 
     cd "${ROOT_PATH}" || exit 1
@@ -67,6 +82,8 @@ check_spack() {
         . spack/share/spack/setup-env.sh
     fi
 
+    check_cuda_version
+
 }
 
 install_spack_deps() {

From e1ed19df36720743fe0ee7a6b9603e4fa4a8193d Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Fri, 12 Jul 2024 17:24:03 +0100
Subject: [PATCH 12/17] Changed CUDA version

---
 scripts/build-platypus-csd3-ampere.sh | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 1d9c0280..fa7b3acc 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -27,8 +27,6 @@ load_modules() {
     module purge
     module load rhel8/slurm
     module use /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-rocky8-zen3
-    module load cuda/11.7.1
-
 }
 
 set_paths() {
@@ -52,11 +50,11 @@ set_paths() {
 
 check_cuda_version() {
 
-    if grep -q "cuda@11.7.1" "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml; then
+    if grep -q "cuda@11.4" "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml; then
         echo "Ampere CUDA module found in spack."
     else
         echo "Ampere CUDA module not found in spack. Adding to packages.yaml."
-        CUDA_STR=$'  cuda:\n    externals:\n    - spec: "cuda@11.7.1"\n      prefix: /usr/local/software/spack/spack-modules/rocky8-a100-20230831/linux-rocky8-zen3/cuda/11.7.1/gcc/fgtvtwi5\n    buildable: False'
+        CUDA_STR=$'  cuda:\n    externals:\n    - spec: "cuda@11.4"\n      prefix: /usr/local/software/cuda/11.4\n    buildable: False'
         echo "${CUDA_STR}"  >> "${SPACK_ROOT}"/etc/spack/defaults/packages.yaml
     fi
 
@@ -92,8 +90,6 @@ install_spack_deps() {
     spack uninstall -ay arch=${ARCH}
     spack clean -ab
 
-    spack external find cuda@11.7.1
-
     echo "Installing Petsc..."
     # Spack's petsc doesn't like openmpi, but it works with mpich
     spack install petsc +cuda cuda_arch=80 +fortran +hdf5 +hypre +metis +mpi \

From 001f461bb5e61677611cd71d2eac009c08165fd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 16 Jul 2024 09:42:58 +0100
Subject: [PATCH 13/17] Removed spdlog lines

---
 include/io/outputs.h                        | 1 -
 src/problem_builders/problem_builder_base.C | 8 --------
 2 files changed, 9 deletions(-)

diff --git a/include/io/outputs.h b/include/io/outputs.h
index 330f36a6..52c8b8f6 100644
--- a/include/io/outputs.h
+++ b/include/io/outputs.h
@@ -136,7 +136,6 @@ class Outputs : public platypus::NamedFieldsMap<mfem::DataCollection>
   // Write out summary of last timestep to console
   void WriteConsoleSummary(int _my_rank, double t)
   {
-    // logger.info("step {}, \tt = {}", _cycle, t);
   }
 
   // Initialize GLVis sockets and fields
diff --git a/src/problem_builders/problem_builder_base.C b/src/problem_builders/problem_builder_base.C
index 6e9676c7..a2f8dd55 100644
--- a/src/problem_builders/problem_builder_base.C
+++ b/src/problem_builders/problem_builder_base.C
@@ -13,7 +13,6 @@ Problem::~Problem()
 void
 ProblemBuilder::SetMesh(std::shared_ptr<mfem::ParMesh> pmesh)
 {
-  // logger.info("Setting Mesh");
   GetProblem()->_pmesh = pmesh;
   GetProblem()->_comm = pmesh->GetComm();
   MPI_Comm_size(pmesh->GetComm(), &(GetProblem()->_num_procs));
@@ -23,35 +22,30 @@ ProblemBuilder::SetMesh(std::shared_ptr<mfem::ParMesh> pmesh)
 void
 ProblemBuilder::SetFESpaces(platypus::FESpaces & fespaces)
 {
-  // logger.info("Setting FE Spaces");
   GetProblem()->_fespaces = fespaces;
 }
 
 void
 ProblemBuilder::SetGridFunctions(platypus::GridFunctions & gridfunctions)
 {
-  // logger.info("Setting GridFunctions");
   GetProblem()->_gridfunctions = gridfunctions;
 }
 
 void
 ProblemBuilder::SetBoundaryConditions(platypus::BCMap & bc_map)
 {
-  // logger.info("Setting Boundary Conditions");
   GetProblem()->_bc_map = bc_map;
 }
 
 void
 ProblemBuilder::SetOutputs(platypus::Outputs & outputs)
 {
-  // logger.info("Setting Outputs");
   GetProblem()->_outputs = outputs;
 }
 
 void
 ProblemBuilder::SetSolverOptions(platypus::InputParameters & solver_options)
 {
-  // logger.info("Setting Solver Options");
   GetProblem()->_solver_options = solver_options;
 }
 
@@ -70,14 +64,12 @@ ProblemBuilder::SetJacobianSolver(std::shared_ptr<mfem::Solver> jacobian_solver)
 void
 ProblemBuilder::SetCoefficients(platypus::Coefficients & coefficients)
 {
-  // logger.info("Setting Coefficients");
   GetProblem()->_coefficients = coefficients;
 }
 
 void
 ProblemBuilder::AddFESpace(std::string fespace_name, std::string fec_name, int vdim, int ordering)
 {
-  // logger.info("Adding {} FE Space to problem", fespace_name);
   if (GetProblem()->_fespaces.Has(fespace_name))
   {
     const std::string error_message = "A fespace with the name " + fespace_name +

From 0acaef1fd79f5070e261e8bbe41777a329b886d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 16 Jul 2024 09:46:17 +0100
Subject: [PATCH 14/17] clang format fix

---
 include/io/outputs.h | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/include/io/outputs.h b/include/io/outputs.h
index 52c8b8f6..b6471cca 100644
--- a/include/io/outputs.h
+++ b/include/io/outputs.h
@@ -134,9 +134,7 @@ class Outputs : public platypus::NamedFieldsMap<mfem::DataCollection>
   }
 
   // Write out summary of last timestep to console
-  void WriteConsoleSummary(int _my_rank, double t)
-  {
-  }
+  void WriteConsoleSummary(int _my_rank, double t) {}
 
   // Initialize GLVis sockets and fields
   void InitializeGLVis(int _my_rank)

From 38853202244c362e5c89697f5cfb89fdadf34fac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 16 Jul 2024 16:51:49 +0100
Subject: [PATCH 15/17] Added explicit SuperLU directory to script and fixed
 oversubscription issue

---
 scripts/build-platypus-csd3-ampere.sh | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index fa7b3acc..783ea8c9 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -5,7 +5,7 @@
 #SBATCH --mail-type=none
 #SBATCH -p ampere
 #SBATCH -A ukaea-ap001-GPU
-#SBATCH --cpus-per-task=1
+#SBATCH --cpus-per-task=32
 #SBATCH --gres=gpu:1
 #SBATCH --output=platypus_gpu_build.%j.out
 #SBATCH --error=platypus_gpu_build.%j.err
@@ -144,6 +144,8 @@ install_mfem() {
     export F77=mpif77
     export FC=mpif90
 
+    SLU_DIR=$(spack find --format {prefix} superlu-dist)
+
     # Build MFEM
     cd "${BUILD_PATH}" || exit 1
     git clone https://github.com/mfem/mfem.git
@@ -167,7 +169,9 @@ install_mfem() {
         -DMFEM_USE_SUPERLU=YES \
         -DMFEM_USE_NETCDF=YES \
         -DMFEM_USE_GSLIB=YES \
-        -DGSLIB_DIR="${BUILD_PATH}/gslib/build"
+        -DGSLIB_DIR="${BUILD_PATH}/gslib/build" \
+        -DSuperLUDist_DIR="${SLU_DIR}" \
+        -DSuperLUDist_VERSION_OK=YES
 
     if [ $? -eq 2 ]; then
         echo "MFEM config failed"

From ea4cda697585e0e8bede7769b9f1e98c7eb9d3e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=E2=80=9CHenrique?= <henrique.bergallo-rocha@ukaea.uk>
Date: Tue, 16 Jul 2024 17:05:37 +0100
Subject: [PATCH 16/17] Lint

---
 scripts/build-platypus-csd3-ampere.sh | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index 783ea8c9..a3a4c3f7 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -143,8 +143,8 @@ install_mfem() {
     export F90=mpif90
     export F77=mpif77
     export FC=mpif90
-
-    SLU_DIR=$(spack find --format {prefix} superlu-dist)
+    SLU_DIR=$(spack find --format "{prefix}" superlu-dist)
+    export SLU_DIR
 
     # Build MFEM
     cd "${BUILD_PATH}" || exit 1

From a80e7f19d30a7e1382d8b138f6a229bb32328bcf Mon Sep 17 00:00:00 2001
From: "Henrique B. R" <henrique.bergallo-rocha@ukaea.uk>
Date: Wed, 17 Jul 2024 11:12:40 +0100
Subject: [PATCH 17/17] Removed hephaestus building from script

---
 scripts/build-platypus-csd3-ampere.sh | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/scripts/build-platypus-csd3-ampere.sh b/scripts/build-platypus-csd3-ampere.sh
index a3a4c3f7..b7293447 100755
--- a/scripts/build-platypus-csd3-ampere.sh
+++ b/scripts/build-platypus-csd3-ampere.sh
@@ -253,13 +253,6 @@ install_platypus() {
     echo "Building platypus..."
     git clone https://github.com/aurora-multiphysics/platypus.git
     cd platypus || exit 1
-    git submodule update --init --recursive
-    cd contrib/hephaestus/ || exit 1
-    mkdir build
-    cd build || exit 1
-    cmake -G Ninja -DCMAKE_BUILD_TYPE=Release -DMFEM_DIR="${BUILD_PATH}/mfem/build" ..
-    ninja
-    cd "${BUILD_PATH}"/platypus || exit 1
     make -j"$compile_cores"
 
 }