initial perlmutter

.github/workflows/cmake.yml

@@ -142,9 +142,9 @@ jobs:
     # - name: Install Valgrind
     #   run: sudo apt-get install -yq valgrind
 
-    - name: Run CTest
-      run: ctest --test-dir ${{ runner.temp }}/build-pcms
+    # - name: Run CTest
+    #   run: ctest --test-dir ${{ runner.temp }}/build-pcms
 
-    - name: Print Test
-      if: always()
-      run: cat ${{ runner.temp }}/build-pcms/Testing/Temporary/LastTest.log
+    # - name: Print Test
+    #   if: always()
+    #   run: cat ${{ runner.temp }}/build-pcms/Testing/Temporary/LastTest.log

.github/workflows/gpu-systems.yml

@@ -0,0 +1,59 @@
+name: Systems
+on:
+  push:
+
+concurrency:
+  group: systems-${{ github.event.repository.name }}
+  cancel-in-progress: true
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+    strategy:
+      matrix:
+        machine: ["Perlmutter"]
+
+    steps:
+
+    - name: checkout repo
+      uses: actions/checkout@v4
+      with:
+        path: ${{ github.event.repository.name }}
+
+    - name: setup python
+      uses: actions/setup-python@v5
+      with:
+        python-version: '3.10'
+
+    - name: install packing
+      run: sudo apt install python3-packaging
+
+    - name: install globus
+      run: |
+          python -m ensurepip --upgrade --user
+          python -m pip install globus-compute-endpoint
+
+    - name: use globus
+      working-directory: ${{ github.event.repository.name }}/.github/workflows
+      env:
+        GLOBUS_ID: ${{ secrets.GLOBUS_COMPUTE_ID }}
+        GLOBUS_SECRET: ${{ secrets.GLOBUS_COMPUTE_SECRET }}
+      run: |
+        export GLOBUS_COMPUTE_CLIENT_ID="$GLOBUS_ID"
+        export GLOBUS_COMPUTE_CLIENT_SECRET="$GLOBUS_SECRET"
+        if [ ${{matrix.machine}} == Perlmutter ]; then TARGET_ENDPOINT=0dd4499a-8d76-4977-bae9-841e4bb2f616; fi
+        if [ ${{matrix.machine}} == Frontier ]; then TARGET_ENDPOINT=d625c6cf-de7a-4228-ac44-56247a642fe0; fi
+        python run-globus.py ${{ github.event.repository.name }} ${{ github.sha }} $TARGET_ENDPOINT
+
+    - name: print build log
+      working-directory: ${{ github.event.repository.name }}/.github/workflows
+      run: cat ${{ github.event.repository.name }}-result/Build.log
+
+    - name: print test log
+      working-directory: ${{ github.event.repository.name }}/.github/workflows
+      run: cat ${{ github.event.repository.name }}-result/Test.log
+
+    - name: check failed test
+      working-directory: ${{ github.event.repository.name }}/.github/workflows
+      run: if grep "Failed" ${{ github.event.repository.name }}-result/Test.log; then return 1; fi

.github/workflows/perlmutter/env.sh

@@ -0,0 +1 @@
+module load cray-fftw

.github/workflows/perlmutter/install.sh

@@ -0,0 +1,108 @@
+#!/bin/bash -e
+
+name=$1
+branch=$2
+
+cd $SCRATCH/globus-compute/$name-test
+
+# # kokkos
+# rm build-kokkos -rf
+# # rm kokkos -rf
+# git clone -b 4.2.00 https://github.com/kokkos/kokkos.git
+# cmake -S kokkos -B build-kokkos \
+#   -DCMAKE_INSTALL_PREFIX=build-kokkos/install \
+#   -DCMAKE_BUILD_TYPE="Release" \
+#   -DCMAKE_CXX_COMPILER=$PWD/kokkos/bin/nvcc_wrapper \
+#   -DKokkos_ARCH_AMPERE80=ON \
+#   -DKokkos_ENABLE_SERIAL=ON \
+#   -DKokkos_ENABLE_OPENMP=off \
+#   -DKokkos_ENABLE_CUDA=on \
+#   -DKokkos_ENABLE_CUDA_LAMBDA=on \
+#   -DKokkos_ENABLE_CUDA_CONSTEXPR=on \
+#   -DKokkos_ENABLE_DEBUG=off
+# cmake --build build-kokkos -j 24 --target install
+
+# # ADIOS2
+# rm build-ADIOS2 -rf
+# # rm ADIOS2 -rf
+# git clone [email protected]:ornladios/ADIOS2.git
+# cmake -S ADIOS2/ -B build-ADIOS2 \
+#   -DCMAKE_INSTALL_PREFIX=build-ADIOS2/install \
+#   -DADIOS2_USE_CUDA=on \
+#   -DCMAKE_BUILD_TYPE=Release \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DCMAKE_C_COMPILER=cc
+# cmake --build build-ADIOS2 --target install -j8
+
+# # perfstubs
+# rm build-perfstubs -rf
+# # rm perfstubs -rf
+# git clone [email protected]:UO-OACISS/perfstubs.git
+# cmake -S perfstubs -B build-perfstubs \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DCMAKE_C_COMPILER=cc \
+#   -DCMAKE_INSTALL_PREFIX=build-perfstubs/install
+# cmake --build build-perfstubs -j2 --target install
+
+# # redev
+# rm build-redev -rf
+# # rm redev -rf
+# git clone [email protected]:SCOREC/redev.git
+# cmake -S redev -B build-redev \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DCMAKE_C_COMPILER=cc \
+#   -DMPIEXEC_EXECUTABLE=`which srun` \
+#   -DCMAKE_BUILD_TYPE=Release \
+#   -DCMAKE_INSTALL_PREFIX=build-redev/install \
+#   -Dperfstubs_DIR=$PWD/build-perfstubs \
+#   -DADIOS2_ROOT=build-ADIOS2/install
+# cmake --build build-redev -j2 --target install
+
+# # omega_h
+# rm build-omega_h -rf
+# # rm omega_h -rf
+# git clone [email protected]:SCOREC/omega_h.git
+# cmake -S omega_h -B build-omega_h \
+#   -DCMAKE_INSTALL_PREFIX=build-omega_h/install \
+#   -DCMAKE_BUILD_TYPE=Release \
+#   -DBUILD_SHARED_LIBS=off \
+#   -DOmega_h_USE_Kokkos=ON \
+#   -DOmega_h_USE_CUDA=on \
+#   -DOmega_h_CUDA_ARCH=80 \
+#   -DOmega_h_USE_MPI=on  \
+#   -DMPIEXEC_EXECUTABLE=srun \
+#   -DBUILD_TESTING=off  \
+#   -DCMAKE_C_COMPILER=cc \
+#   -DCMAKE_CXX_COMPILER=CC \
+#   -DKokkos_PREFIX=build-kokkos/install/lib64/cmake
+# cmake --build build-omega_h -j24 --target install
+
+# # Catch2
+# rm build-Catch2 -rf
+# # rm Catch2 -rf
+# git clone https://github.com/catchorg/Catch2
+# cmake -S Catch2 -B build-Catch2 \
+#   -DCMAKE_INSTALL_PREFIX=$PWD/build-Catch2/install \
+#   -DCMAKE_CXX_COMPILER=CC
+# cmake --build build-Catch2 -j2 --target install
+
+# pcms
+rm pcms -rf
+rm build-pcms -rf
+rm pcms_testcases -rf
+
+git clone https://github.com/SCOREC/pcms.git
+cd pcms && git checkout $branch && cd -
+git clone [email protected]:jacobmerson/pcms_testcases.git
+
+cmake -S pcms -B build-pcms \
+  -DCMAKE_C_COMPILER=cc \
+  -DCMAKE_CXX_COMPILER=CC \
+  -DCMAKE_BUILD_TYPE=Release \
+  -Dredev_DIR=$PWD/build-redev/install/lib64/cmake/redev \
+  -Dperfstubs_DIR=$PWD/build-perfstubs \
+  -DOmega_h_DIR=$PWD/build-omega_h/install/lib64/cmake/Omega_h/ \
+  -DKokkos_DIR=$PWD/build-kokkos/install/lib64/cmake/Kokkos/ \
+  -DCatch2_DIR=$PWD/build-Catch2/install/lib64/cmake/Catch2/ \
+  -DPCMS_TEST_DATA_DIR=$PWD/pcms_testcases
+cmake --build build-pcms -j8

.github/workflows/perlmutter/run.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+name=$1
+test=$2
+
+cd $SCRATCH/globus-compute/$name-test
+source env.sh
+
+mkdir -p results
+cd build-$name
+salloc --time 00:20:00 --constraint gpu --qos=interactive --nodes=1 --ntasks-per-node=40 --cpus-per-task=1 --gpus=1 --account=m4564 ctest $test 2>&1 | tee ../results/summary.log
+cat $PWD/Testing/Temporary/LastTest.log 2>&1 | tee ../results/test.log

.github/workflows/run-globus.py

@@ -0,0 +1,40 @@
+# How to use
+# 1. Login to https://app.globus.org/settings/developers and copy a project app id and secret
+# 2. Use the id and secret to create and endpoint https://funcx.readthedocs.io/en/latest/sdk.html#client-credentials-with-clients
+#     $ export FUNCX_SDK_CLIENT_ID="b0500dab-ebd4-430f-b962-0c85bd43bdbb"
+#     $ export FUNCX_SDK_CLIENT_SECRET="ABCDEFGHIJKLMNOP0123456789="
+# 3. Set up an endpoint on the computer that will run the tests, using these instructions: https://funcx.readthedocs.io/en/latest/endpoints.html
+# 4. Create install-test.sh and run-test.sh on target computer
+
+from globus_compute_sdk import Executor
+import sys
+import os
+
+name = sys.argv[1]
+branch = sys.argv[2]
+endpoint = sys.argv[3]
+
+with open('perlmutter/env.sh', 'rb') as file:
+    env_file = file.read()
+
+with open('perlmutter/install.sh', 'rb') as file:
+    install_file = file.read()
+
+def run_on_endpoint(name, branch):
+    import subprocess
+
+    install = subprocess.run([install_file], shell=True, encoding="utf_8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    return (install, None)
+
+gce = Executor(endpoint_id = endpoint)
+future = gce.submit(run_on_endpoint, name, branch)
+result = future.result()
+
+os.popen("mkdir -p "+name+"-result").read()
+with open(name+"-result/Build.log", "w") as text_file:
+    text_file.write("%s" % result[0].stdout)
+    text_file.close()
+if result[0].returncode == 0:
+    with open(name+"-result/Test.log", "w") as text_file:
+        text_file.write("%s" % result[1].stdout)
+        text_file.close()

.github/workflows/test_on_system.py

@@ -0,0 +1,38 @@
+# How to use
+# 1. Login to https://app.globus.org/settings/developers and copy a project app id and secret
+# 2. Use the id and secret to create and endpoint https://funcx.readthedocs.io/en/latest/sdk.html#client-credentials-with-clients
+#     $ export FUNCX_SDK_CLIENT_ID="b0500dab-ebd4-430f-b962-0c85bd43bdbb"
+#     $ export FUNCX_SDK_CLIENT_SECRET="ABCDEFGHIJKLMNOP0123456789="
+# 3. Set up an endpoint on the computer that will run the tests, using these instructions: https://funcx.readthedocs.io/en/latest/endpoints.html
+# 4. Create install-test.sh and run-test.sh on target computer
+
+from globus_compute_sdk import Executor
+import sys
+import os
+
+name = sys.argv[1]
+branch = sys.argv[2]
+endpoint = sys.argv[3]
+
+def run_on_endpoint(name, branch):
+    import subprocess
+
+    install = subprocess.run(["./install-test.sh "+name+" "+branch], shell=True, encoding="utf_8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    if install.returncode == 1:
+        return (install, None)
+
+    result = subprocess.run(["./run-test.sh "+name], shell=True, encoding="utf_8", stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+    return (install, result)
+
+gce = Executor(endpoint_id = endpoint)
+future = gce.submit(run_on_endpoint, name, branch)
+result = future.result()
+
+os.popen("mkdir -p "+name+"-result").read()
+with open(name+"-result/Build.log", "w") as text_file:
+    text_file.write("%s" % result[0].stdout)
+    text_file.close()
+if result[0].returncode == 0:
+    with open(name+"-result/Test.log", "w") as text_file:
+        text_file.write("%s" % result[1].stdout)
+        text_file.close()