Merge remote-tracking branch 'upstream/develop' into develop

.github/CODEOWNERS

@@ -211,3 +211,4 @@ ush/python/pygfs/utils/marine_da_utils.py @guillaumevernieres @AndrewEichmann-NO
 
 # Specific workflow scripts
 workflow/generate_workflows.sh @DavidHuber-NOAA
+workflow/build_compute.py @DavidHuber-NOAA @aerorahul

.gitignore

@@ -85,6 +85,9 @@ parm/wafs
 
 # Ignore sorc and logs folders from externals
 #--------------------------------------------
+sorc/build.xml
+sorc/build.db
+sorc/build_lock.db
 sorc/*log
 sorc/logs
 sorc/calc_analysis.fd

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -18,4 +18,4 @@ arguments:
   yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_defaults_ci.yaml
 
 skip_ci_on_hosts:
-  - wcoss2
+  - None

ci/cases/pr/C96_S2SWA_gefs_replay_ics.yaml

@@ -19,4 +19,4 @@ arguments:
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96mx100/20240610
 
 skip_ci_on_hosts:
-  - wcoss2
+  - None

ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

@@ -3,4 +3,4 @@ defaults:
 base:
   DO_JEDISNOWDA: "YES"
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
-  DO_TEST_MODE: "YES"
+  DO_TEST_MODE: "NO"

ci/cases/yamls/gfs_defaults_ci.yaml

@@ -2,4 +2,4 @@ defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
-  DO_TEST_MODE: "YES"
+  DO_TEST_MODE: "NO"

ci/cases/yamls/gfs_extended_ci.yaml

@@ -12,4 +12,4 @@ base:
   FCST_BREAKPOINTS: 192
   FHMAX_GFS: 384
   FHMAX_HF_GFS: 120
-  DO_TEST_MODE: "YES"
+  DO_TEST_MODE: "NO"

ci/cases/yamls/soca_gfs_defaults_ci.yaml

@@ -6,4 +6,6 @@ base:
 
 marineanl:
   SOCA_NINNER: 1
-  DO_TEST_MODE: "YES"
+  DO_TEST_MODE: "NO"
+marineanl:
+  JCB_ALGO_YAML_VAR: "{{ HOMEgfs }}/sorc/gdas.cd/test/gw-ci/soca/jcb-prototype_3dfgat_3DVarAOWCDA.yaml.j2"

ci/cases/yamls/ufs_hybatmDA_defaults.ci.yaml

@@ -5,10 +5,17 @@ base:
   DO_JEDIATMVAR: "YES"
   DO_JEDIATMENS: "YES"
   ACCOUNT: {{ 'HPC_ACCOUNT' | getenv }}
+  DO_TEST_MODE: "NO"
 atmanl:
+  JCB_ALGO_YAML_VAR: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_3dvar_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_FV3INC: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_3dvar-fv3inc_ufs_hybatmDA.yaml.j2"
   LAYOUT_X_ATMANL: 4
   LAYOUT_Y_ATMANL: 4
 atmensanl:
+  JCB_ALGO_YAML_LETKF: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_OBS: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_observer_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_SOL: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf_solver_ufs_hybatmDA.yaml.j2"
+  JCB_ALGO_YAML_FV3INC: "${HOMEgfs}/sorc/gdas.cd/test/gw-ci/atm/jcb-prototype_lgetkf-fv3inc_ufs_hybatmDA.yaml.j2"
   LAYOUT_X_ATMENSANL: 4
   LAYOUT_Y_ATMENSANL: 4
 esfc:

ci/platforms/config.gaea

@@ -3,6 +3,6 @@
 export GFS_CI_ROOT=/gpfs/f5/epic/proj-shared/global/GFS_CI_ROOT
 export ICSDIR_ROOT=/gpfs/f5/epic/proj-shared/global/glopara/data/ICSDIR
 export STMP="/gpfs/f5/epic/scratch/${USER}"
-export SLURM_ACCOUNT=ufs-ard
+export HPC_ACCOUNT=ufs-ard
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ci/platforms/config.hercules

@@ -2,7 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/HERCULES
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export HPC_ACCOUNT=nems
+export HPC_ACCOUNT=fv3-cpu
 export max_concurrent_cases=5
 export max_concurrent_pr=4
 

ci/scripts/utils/ci_utils_wrapper.sh

@@ -3,7 +3,7 @@
 HOMEgfs="$(cd "$(dirname  "${BASH_SOURCE[0]}")/../../.." >/dev/null 2>&1 && pwd )"
 source "${HOMEgfs}/ush/detect_machine.sh"
 
-utitilty_function="${1}"
+utility_function="${1}"
 
 source "${HOMEgfs}/ci/scripts/utils/ci_utils.sh"
-${utitilty_function} "${@:2}"
+${utility_function} "${@:2}"

docs/source/clone.rst

@@ -18,35 +18,39 @@ Clone the `global-workflow` and `cd` into the `sorc` directory:
    git clone --recursive https://github.com/NOAA-EMC/global-workflow
    cd global-workflow/sorc
 
-For forecast-only (coupled or uncoupled) build of the components:
+.. _build_examples:
+
+The build_all.sh script can be used to build all required components of the global workflow.  The accepted arguments is a list of systems to be built.  This includes builds for GFS and GEFS forecast-only experiments, GSI and GDASApp-based DA for cycled GFS experiments.  See `feature availability <hpc.html#feature-availability-by-hpc>`__ to see which system(s) are available on each supported system.
 
 ::
 
-   ./build_all.sh
+   ./build_all.sh [gfs] [gefs] [gs] [gdas] [all]
 
-For cycled (w/ data assimilation) use the `-g` option during build:
+For example, to run GFS experiments with GSI DA, execute:
 
 ::
 
-   ./build_all.sh -g
+   ./build_all.sh gfs gsi
 
-For coupled cycling (include new UFSDA) use the `-gu` options during build:
+This builds the GFS, UFS-utils, GFS-utils, WW3 with PDLIB (structured wave grids), UPP, GSI, GSI-monitor, and GSI-utils executables.
 
-[Currently only available on Hera, Orion, and Hercules]
+For coupled cycling (include new UFSDA) execute:
 
 ::
 
-   ./build_all.sh -gu
+   ./build_all.sh gfs gdas
 
+This builds all of the same executables, except it builds the GDASApp instead of the GSI.
 
-For building without PDLIB (unstructured grid) for the wave model, use the `-w` options during build:
+To run GEFS (forecast-only) execute:
 
 ::
 
-   ./build_all.sh -w
+   ./build_all.sh gefs
 
+This builds the GEFS, UFS-utils, GFS-utils, WW3 *without* PDLIB (unstructure wave grids), and UPP executables.
 
-Build workflow components and link workflow artifacts such as executables, etc.
+Once the building is complete, link workflow artifacts such as executables, configuration files, and scripts via
 
 ::
 
@@ -107,40 +111,19 @@ Under the ``/sorc`` folder is a script to build all components called ``build_al
 
 ::
 
-   ./build_all.sh [-a UFS_app][-g][-h][-u][-v]
+   ./build_all.sh [-a UFS_app][-k][-h][-v] [list of system(s) to build]
   -a UFS_app:
     Build a specific UFS app instead of the default
-  -g:
-    Build GSI
+  -k:
+    Kill all builds immediately if one fails
   -h:
     Print this help message and exit
-  -j:
-    Specify maximum number of build jobs (n)
-  -u:
-    Build UFS-DA
   -v:
     Execute all build scripts with -v option to turn on verbose where supported
 
-For forecast-only (coupled or uncoupled) build of the components:
-
-::
-
-   ./build_all.sh
-
-For cycled (w/ data assimilation) use the `-g` option during build:
-
-::
-
-   ./build_all.sh -g
-
-For coupled cycling (include new UFSDA) use the `-gu` options during build:
-
-[Currently only available on Hera, Orion, and Hercules]
-
-::
-
-   ./build_all.sh -gu
+  Lastly, pass to build_all.sh a list of systems to build.  This includes `gfs`, `gefs`, `sfs` (not fully supported), `gsi`, `gdas`, and `all`.
 
+For examples of how to use this script, see :ref:`build examples <build_examples>`.
 
 ^^^^^^^^^^^^^^^
 Link components
@@ -156,4 +139,3 @@ After running the checkout and build scripts run the link script:
 
 Where:
    ``-o``: Run in operations (NCO) mode. This creates copies instead of using symlinks and is generally only used by NCO during installation into production.
-

env/AWSPW.env

@@ -33,7 +33,29 @@ else
     exit 2
 fi
 
-if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+if [[ "${step}" = "prep" ]] || [[ "${step}" = "prepbufr" ]]; then
+
+    export POE="NO"
+    export BACK="NO"
+    export sys_tp="AWSPW"
+    export launcher_PREP="srun"
+
+elif [[ "${step}" = "prepsnowobs" ]]; then
+
+    export APRUN_CALCFIMS="${APRUN_default}"
+
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${APRUN_default}"
+
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
+elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
     export launcher="srun --mpi=pmi2 -l"
 
@@ -52,52 +74,16 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
 
 elif [[ "${step}" = "post" ]]; then
 
-    export NTHREADS_NP=${NTHREADS1}
-    export APRUN_NP="${APRUN_default}"
-
-    export NTHREADS_DWN=${threads_per_task_dwn:-1}
-    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
-    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
-
-elif [[ "${step}" = "atmos_products" ]]; then
-
-    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+    export NTHREADS_UPP=${NTHREADS1}
+    export APRUN_UPP="${APRUN_default} --cpus-per-task=${NTHREADS_UPP}"
 
 elif [[ "${step}" = "oceanice_products" ]]; then
 
     export NTHREADS_OCNICEPOST=${NTHREADS1}
     export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
 
-elif [[ "${step}" = "ecen" ]]; then
-
-    export NTHREADS_ECEN=${NTHREADSmax}
-    export APRUN_ECEN="${APRUN_default}"
-
-    export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
-    [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
-    export APRUN_CHGRES="time"
-
-    export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
-    [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
-    export APRUN_CALCINC="${APRUN_default}"
-
-elif [[ "${step}" = "esfc" ]]; then
-
-    export NTHREADS_ESFC=${NTHREADSmax}
-    export APRUN_ESFC="${APRUN_default}"
-
-    export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
-    [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
-    export APRUN_CYCLE="${APRUN_default}"
-
-elif [[ "${step}" = "epos" ]]; then
-
-    export NTHREADS_EPOS=${NTHREADSmax}
-    export APRUN_EPOS="${APRUN_default}"
-
-elif [[ "${step}" = "fit2obs" ]]; then
+elif [[ "${step}" = "atmos_products" ]]; then
 
-    export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN_default}"
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on AWS
 
 fi

env/AZUREPW.env

@@ -15,6 +15,7 @@ export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
 # Configure MPI environment
 export OMP_STACKSIZE=2048000
 export NTHSTACK=1024000000
+export UCX_TLS=ud,sm,self
 
 ulimit -s unlimited
 ulimit -a
@@ -50,6 +51,10 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
     export wavempexec=${launcher}
     export wave_mpmd=${mpmd_opt}
 
+elif [[ "${step}" = "prep_emissions" ]]; then
+
+    export APRUN="${APRUN_default}"
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -71,33 +76,33 @@ elif [[ "${step}" = "oceanice_products" ]]; then
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}
-    export APRUN_ECEN="${APRUN}"
+    export APRUN_ECEN="${APRUN_default}"
 
     export NTHREADS_CHGRES=${threads_per_task_chgres:-12}
     [[ ${NTHREADS_CHGRES} -gt ${max_tasks_per_node} ]] && export NTHREADS_CHGRES=${max_tasks_per_node}
     export APRUN_CHGRES="time"
 
     export NTHREADS_CALCINC=${threads_per_task_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${max_threads_per_task} ]] && export NTHREADS_CALCINC=${max_threads_per_task}
-    export APRUN_CALCINC="${APRUN}"
+    export APRUN_CALCINC="${APRUN_default}"
 
 elif [[ "${step}" = "esfc" ]]; then
 
     export NTHREADS_ESFC=${NTHREADSmax}
-    export APRUN_ESFC="${APRUN}"
+    export APRUN_ESFC="${APRUN_default}"
 
     export NTHREADS_CYCLE=${threads_per_task_cycle:-14}
     [[ ${NTHREADS_CYCLE} -gt ${max_tasks_per_node} ]] && export NTHREADS_CYCLE=${max_tasks_per_node}
-    export APRUN_CYCLE="${APRUN}"
+    export APRUN_CYCLE="${APRUN_default}"
 
 elif [[ "${step}" = "epos" ]]; then
 
     export NTHREADS_EPOS=${NTHREADSmax}
-    export APRUN_EPOS="${APRUN}"
+    export APRUN_EPOS="${APRUN_default}"
 
 elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN}"
+    export MPIRUN="${APRUN_default}"
 
 fi

env/GOOGLEPW.env

@@ -45,7 +45,7 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
 elif [[ "${step}" = "prep_emissions" ]]; then
 
-    export APRUN
+    export APRUN="${APRUN_default}"
 
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
@@ -102,6 +102,6 @@ elif [[ "${step}" = "epos" ]]; then
 elif [[ "${step}" = "fit2obs" ]]; then
 
     export NTHREADS_FIT2OBS=${NTHREADS1}
-    export MPIRUN="${APRUN}"
+    export MPIRUN="${APRUN_default}"
 
 fi

env/HERA.env

@@ -115,14 +115,16 @@ elif [[ "${step}" = "snowanl" ]]; then
     export APRUN_CALCFIMS="${launcher} -n 1"
 
     export NTHREADS_SNOWANL=${NTHREADSmax}
-    export APRUN_SNOWANL="${APRUN_default} --cpus-per-task=${NTHREADS_SNOWANL}"
+    export APRUN_SNOWANL="${APRUN_default} --mem=0 --cpus-per-task=${NTHREADS_SNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
-elif [[ "${step}" = "esnowrecen" ]]; then
+elif [[ "${step}" = "esnowanl" ]]; then
 
-    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
-    export APRUN_ESNOWRECEN="${APRUN_default} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+    export APRUN_CALCFIMS="${launcher} -n 1"
+
+    export NTHREADS_ESNOWANL=${NTHREADSmax}
+    export APRUN_ESNOWANL="${APRUN_default} --mem=0 --cpus-per-task=${NTHREADS_ESNOWANL}"
 
     export APRUN_APPLY_INCR="${launcher} -n 6"