From 5485db8648c1a19a233d2ed614e7ed852bbb140a Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Thu, 5 Dec 2024 07:11:22 -0600 Subject: [PATCH 1/3] Add support for traditional threading. 'ESMF threading' is still default --- tests/default_vars.sh | 5 +- tests/fv3_conf/fv3_qsub.IN_acorn | 3 +- tests/fv3_conf/fv3_qsub.IN_wcoss2 | 3 +- tests/parm/params_grib2_tbl_new | 0 tests/parm/ufs.configure.atm.IN | 3 +- tests/parm/ufs.configure.atm_esmf.IN | 27 --- tests/parm/ufs.configure.atm_lm4.IN | 2 +- tests/parm/ufs.configure.atm_lnd.IN | 2 +- tests/parm/ufs.configure.atmaero.IN | 4 +- tests/parm/ufs.configure.atmaero_esmf.IN | 40 ----- tests/parm/ufs.configure.atmaq.IN | 2 +- tests/parm/ufs.configure.atmw.IN | 2 +- tests/parm/ufs.configure.blocked_atm_wav.IN | 2 +- .../ufs.configure.blocked_atm_wav_2way.IN | 2 +- tests/parm/ufs.configure.cpld_atm_fbh.IN | 2 +- tests/parm/ufs.configure.datm_cdeps.IN | 2 +- tests/parm/ufs.configure.hafs_atm.IN | 2 +- tests/parm/ufs.configure.hafs_atm_docn.IN | 2 +- tests/parm/ufs.configure.hafs_atm_ocn.IN | 2 +- tests/parm/ufs.configure.hafs_atm_ocn_wav.IN | 2 +- .../ufs.configure.hafs_atm_ocn_wav_inline.IN | 2 +- .../ufs.configure.hafs_atm_ocn_wav_mom6.IN | 2 +- tests/parm/ufs.configure.hafs_atm_wav.IN | 2 +- tests/parm/ufs.configure.leapfrog_atm_wav.IN | 4 +- .../ufs.configure.leapfrog_atm_wav_esmf.IN | 41 ----- tests/parm/ufs.configure.s2s.IN | 20 ++- ...ux_esmf.IN => ufs.configure.s2s_aoflux.IN} | 2 +- tests/parm/ufs.configure.s2s_esmf.IN | 139 --------------- tests/parm/ufs.configure.s2sa.IN | 7 +- tests/parm/ufs.configure.s2sa_esmf.IN | 137 --------------- tests/parm/ufs.configure.s2sw.IN | 9 +- tests/parm/ufs.configure.s2sw_esmf.IN | 149 ---------------- ...ast_esmf.IN => ufs.configure.s2sw_fast.IN} | 2 +- tests/parm/ufs.configure.s2swa.IN | 8 +- tests/parm/ufs.configure.s2swa_esmf.IN | 161 ------------------ ...st_esmf.IN => ufs.configure.s2swa_fast.IN} | 3 +- tests/rt.sh | 6 +- tests/rt_utils.sh | 126 +++++++++++++- tests/run_test.sh | 35 ++-- tests/tests/atm_ds2s_docn_dice | 2 +- tests/tests/atm_ds2s_docn_pcice | 2 +- tests/tests/atmaero_control_p8 | 2 +- tests/tests/atmaero_control_p8_rad | 2 +- tests/tests/atmaero_control_p8_rad_micro | 2 +- tests/tests/cpld_2threads_p8 | 3 + tests/tests/cpld_control_c48 | 2 +- tests/tests/cpld_control_noaero_p8 | 4 +- tests/tests/cpld_control_noaero_p8_agrid | 2 +- tests/tests/cpld_control_nowave_noaero_p8 | 2 +- tests/tests/cpld_control_pdlib_p8 | 2 +- tests/tests/cpld_debug_noaero_p8 | 2 +- tests/tests/cpld_debug_pdlib_p8 | 2 +- tests/tests/cpld_mpi_pdlib_p8 | 2 +- tests/tests/cpld_restart_c48 | 2 +- tests/tests/cpld_restart_pdlib_p8 | 2 +- tests/tests/cpld_s2sa_p8 | 2 +- tests/tests/cpld_warmstart_c48 | 2 +- 57 files changed, 243 insertions(+), 760 deletions(-) mode change 100755 => 100644 tests/parm/params_grib2_tbl_new delete mode 100644 tests/parm/ufs.configure.atm_esmf.IN delete mode 100644 tests/parm/ufs.configure.atmaero_esmf.IN delete mode 100644 tests/parm/ufs.configure.leapfrog_atm_wav_esmf.IN rename tests/parm/{ufs.configure.s2s_aoflux_esmf.IN => ufs.configure.s2s_aoflux.IN} (98%) delete mode 100644 tests/parm/ufs.configure.s2s_esmf.IN delete mode 100644 tests/parm/ufs.configure.s2sa_esmf.IN delete mode 100644 tests/parm/ufs.configure.s2sw_esmf.IN rename tests/parm/{ufs.configure.s2sw_fast_esmf.IN => ufs.configure.s2sw_fast.IN} (98%) delete mode 100644 tests/parm/ufs.configure.s2swa_esmf.IN rename tests/parm/{ufs.configure.s2swa_fast_esmf.IN => ufs.configure.s2swa_fast.IN} (98%) diff --git a/tests/default_vars.sh b/tests/default_vars.sh index fa43453bd2..919ab4ac4b 100644 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -419,6 +419,7 @@ export CMP_DATAONLY=false # Defaults for ufs.configure export esmf_logkind="ESMF_LOGKIND_MULTI" +export ESMF_THREADING=true export DumpFields="false" export MED_history_n=1000000 @@ -504,7 +505,7 @@ else fi # ufs.configure defaults -export UFS_CONFIGURE=ufs.configure.atm_esmf.IN +export UFS_CONFIGURE=ufs.configure.atm.IN export MODEL_CONFIGURE=model_configure.IN export atm_model=fv3 @@ -1342,7 +1343,7 @@ export_fire_behavior() { # Defaults for the coupled 5-component export_cmeps() { - export UFS_CONFIGURE=ufs.configure.s2swa_fast_esmf.IN + export UFS_CONFIGURE=ufs.configure.s2swa_fast.IN export med_model=cmeps export atm_model=fv3 export chm_model=gocart diff --git a/tests/fv3_conf/fv3_qsub.IN_acorn b/tests/fv3_conf/fv3_qsub.IN_acorn index b3d0102328..5cb4781c23 100644 --- a/tests/fv3_conf/fv3_qsub.IN_acorn +++ b/tests/fv3_conf/fv3_qsub.IN_acorn @@ -10,10 +10,11 @@ #PBS -l walltime=00:@[WLCLK]:00 set -eux -echo -n " $( date +%s )," > job_timestamp.txt cd $PBS_O_WORKDIR +echo -n " $( date +%s )," > job_timestamp.txt + set +x module use $PWD/modulefiles module load modules.fv3 diff --git a/tests/fv3_conf/fv3_qsub.IN_wcoss2 b/tests/fv3_conf/fv3_qsub.IN_wcoss2 index b3d0102328..5cb4781c23 100644 --- a/tests/fv3_conf/fv3_qsub.IN_wcoss2 +++ b/tests/fv3_conf/fv3_qsub.IN_wcoss2 @@ -10,10 +10,11 @@ #PBS -l walltime=00:@[WLCLK]:00 set -eux -echo -n " $( date +%s )," > job_timestamp.txt cd $PBS_O_WORKDIR +echo -n " $( date +%s )," > job_timestamp.txt + set +x module use $PWD/modulefiles module load modules.fv3 diff --git a/tests/parm/params_grib2_tbl_new b/tests/parm/params_grib2_tbl_new old mode 100755 new mode 100644 diff --git a/tests/parm/ufs.configure.atm.IN b/tests/parm/ufs.configure.atm.IN index 3b6a305247..44dd5667b8 100644 --- a/tests/parm/ufs.configure.atm.IN +++ b/tests/parm/ufs.configure.atm.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM @@ -15,6 +15,7 @@ EARTH_attributes:: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 Diagnostic = 0 diff --git a/tests/parm/ufs.configure.atm_esmf.IN b/tests/parm/ufs.configure.atm_esmf.IN deleted file mode 100644 index 8eb7b94dc7..0000000000 --- a/tests/parm/ufs.configure.atm_esmf.IN +++ /dev/null @@ -1,27 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: ATM -EARTH_attributes:: - Verbosity = 0 -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 - Diagnostic = 0 -:: - -# Run Sequence # -runSeq:: - ATM -:: diff --git a/tests/parm/ufs.configure.atm_lm4.IN b/tests/parm/ufs.configure.atm_lm4.IN index 16bc68285d..2eaa5111c7 100644 --- a/tests/parm/ufs.configure.atm_lm4.IN +++ b/tests/parm/ufs.configure.atm_lm4.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM LND diff --git a/tests/parm/ufs.configure.atm_lnd.IN b/tests/parm/ufs.configure.atm_lnd.IN index 4210c3bf0f..6a01acf8cf 100644 --- a/tests/parm/ufs.configure.atm_lnd.IN +++ b/tests/parm/ufs.configure.atm_lnd.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM LND diff --git a/tests/parm/ufs.configure.atmaero.IN b/tests/parm/ufs.configure.atmaero.IN index 6c60333e07..a23b750941 100644 --- a/tests/parm/ufs.configure.atmaero.IN +++ b/tests/parm/ufs.configure.atmaero.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM CHM @@ -15,6 +15,7 @@ EARTH_attributes:: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 :: @@ -22,6 +23,7 @@ ATM_attributes:: # CHM # CHM_model: @[chm_model] CHM_petlist_bounds: @[chm_petlist_bounds] +CHM_omp_num_threads: @[chm_omp_num_threads] CHM_attributes:: Verbosity = 0 :: diff --git a/tests/parm/ufs.configure.atmaero_esmf.IN b/tests/parm/ufs.configure.atmaero_esmf.IN deleted file mode 100644 index 0c20f20905..0000000000 --- a/tests/parm/ufs.configure.atmaero_esmf.IN +++ /dev/null @@ -1,40 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: ATM CHM -EARTH_attributes:: - Verbosity = 0 -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 -:: - -# CHM # -CHM_model: @[chm_model] -CHM_petlist_bounds: @[chm_petlist_bounds] -CHM_omp_num_threads: @[chm_omp_num_threads] -CHM_attributes:: - Verbosity = 0 -:: - -# Run Sequence # -runSeq:: - @@[coupling_interval_sec] - ATM phase1 - ATM -> CHM - CHM - CHM -> ATM - ATM phase2 - @ -:: diff --git a/tests/parm/ufs.configure.atmaq.IN b/tests/parm/ufs.configure.atmaq.IN index 7c725ec158..bd7d712e98 100644 --- a/tests/parm/ufs.configure.atmaq.IN +++ b/tests/parm/ufs.configure.atmaq.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM AQM diff --git a/tests/parm/ufs.configure.atmw.IN b/tests/parm/ufs.configure.atmw.IN index 3d9caa18e7..7a5b142ee3 100644 --- a/tests/parm/ufs.configure.atmw.IN +++ b/tests/parm/ufs.configure.atmw.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM WAV diff --git a/tests/parm/ufs.configure.blocked_atm_wav.IN b/tests/parm/ufs.configure.blocked_atm_wav.IN index b0354b287c..8ee55464bd 100644 --- a/tests/parm/ufs.configure.blocked_atm_wav.IN +++ b/tests/parm/ufs.configure.blocked_atm_wav.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM WAV diff --git a/tests/parm/ufs.configure.blocked_atm_wav_2way.IN b/tests/parm/ufs.configure.blocked_atm_wav_2way.IN index d66d77eede..7b502f567e 100644 --- a/tests/parm/ufs.configure.blocked_atm_wav_2way.IN +++ b/tests/parm/ufs.configure.blocked_atm_wav_2way.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM WAV diff --git a/tests/parm/ufs.configure.cpld_atm_fbh.IN b/tests/parm/ufs.configure.cpld_atm_fbh.IN index 6b586d6585..60d464c9e0 100644 --- a/tests/parm/ufs.configure.cpld_atm_fbh.IN +++ b/tests/parm/ufs.configure.cpld_atm_fbh.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM FBH diff --git a/tests/parm/ufs.configure.datm_cdeps.IN b/tests/parm/ufs.configure.datm_cdeps.IN index afaa2f98cf..01e016fce4 100644 --- a/tests/parm/ufs.configure.datm_cdeps.IN +++ b/tests/parm/ufs.configure.datm_cdeps.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN ICE diff --git a/tests/parm/ufs.configure.hafs_atm.IN b/tests/parm/ufs.configure.hafs_atm.IN index 2d4ab6996b..3f98d6d693 100644 --- a/tests/parm/ufs.configure.hafs_atm.IN +++ b/tests/parm/ufs.configure.hafs_atm.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM diff --git a/tests/parm/ufs.configure.hafs_atm_docn.IN b/tests/parm/ufs.configure.hafs_atm_docn.IN index 09e469b5b9..2a65426232 100644 --- a/tests/parm/ufs.configure.hafs_atm_docn.IN +++ b/tests/parm/ufs.configure.hafs_atm_docn.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN diff --git a/tests/parm/ufs.configure.hafs_atm_ocn.IN b/tests/parm/ufs.configure.hafs_atm_ocn.IN index 91dd787e69..89f67b57a2 100644 --- a/tests/parm/ufs.configure.hafs_atm_ocn.IN +++ b/tests/parm/ufs.configure.hafs_atm_ocn.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN diff --git a/tests/parm/ufs.configure.hafs_atm_ocn_wav.IN b/tests/parm/ufs.configure.hafs_atm_ocn_wav.IN index 9b089d556a..62d28b351f 100644 --- a/tests/parm/ufs.configure.hafs_atm_ocn_wav.IN +++ b/tests/parm/ufs.configure.hafs_atm_ocn_wav.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI_ON -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN WAV diff --git a/tests/parm/ufs.configure.hafs_atm_ocn_wav_inline.IN b/tests/parm/ufs.configure.hafs_atm_ocn_wav_inline.IN index 47f36bad1f..b289a36b4d 100644 --- a/tests/parm/ufs.configure.hafs_atm_ocn_wav_inline.IN +++ b/tests/parm/ufs.configure.hafs_atm_ocn_wav_inline.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI_ON -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN WAV diff --git a/tests/parm/ufs.configure.hafs_atm_ocn_wav_mom6.IN b/tests/parm/ufs.configure.hafs_atm_ocn_wav_mom6.IN index 81930e3209..eb023c2570 100644 --- a/tests/parm/ufs.configure.hafs_atm_ocn_wav_mom6.IN +++ b/tests/parm/ufs.configure.hafs_atm_ocn_wav_mom6.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI_ON -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN WAV diff --git a/tests/parm/ufs.configure.hafs_atm_wav.IN b/tests/parm/ufs.configure.hafs_atm_wav.IN index bd0192db36..4d0a38511f 100644 --- a/tests/parm/ufs.configure.hafs_atm_wav.IN +++ b/tests/parm/ufs.configure.hafs_atm_wav.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI_ON -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM WAV diff --git a/tests/parm/ufs.configure.leapfrog_atm_wav.IN b/tests/parm/ufs.configure.leapfrog_atm_wav.IN index fe33f5da9f..89079607af 100644 --- a/tests/parm/ufs.configure.leapfrog_atm_wav.IN +++ b/tests/parm/ufs.configure.leapfrog_atm_wav.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: ATM WAV @@ -15,6 +15,7 @@ EARTH_attributes:: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -23,6 +24,7 @@ ATM_attributes:: # WAV # WAV_model: @[wav_model] WAV_petlist_bounds: @[wav_petlist_bounds] +WAV_omp_num_threads: @[wav_omp_num_threads] WAV_attributes:: Verbosity = 0 :: diff --git a/tests/parm/ufs.configure.leapfrog_atm_wav_esmf.IN b/tests/parm/ufs.configure.leapfrog_atm_wav_esmf.IN deleted file mode 100644 index c4b6055fd1..0000000000 --- a/tests/parm/ufs.configure.leapfrog_atm_wav_esmf.IN +++ /dev/null @@ -1,41 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: ATM WAV -EARTH_attributes:: - Verbosity = 0 -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] -:: - -# WAV # -WAV_model: @[wav_model] -WAV_petlist_bounds: @[wav_petlist_bounds] -WAV_omp_num_threads: @[wav_omp_num_threads] -WAV_attributes:: - Verbosity = 0 -:: - - - -# Run Sequence # -runSeq:: - @@[coupling_interval_sec] - ATM - ATM -> WAV - WAV - @ -:: diff --git a/tests/parm/ufs.configure.s2s.IN b/tests/parm/ufs.configure.s2s.IN index 9ff939a6c7..17ccc678c9 100644 --- a/tests/parm/ufs.configure.s2s.IN +++ b/tests/parm/ufs.configure.s2s.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN ICE @@ -15,11 +15,13 @@ EARTH_attributes:: # MED # MED_model: @[med_model] MED_petlist_bounds: @[med_petlist_bounds] +MED_omp_num_threads: @[med_omp_num_threads] :: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -30,6 +32,7 @@ ATM_attributes:: # OCN # OCN_model: @[ocn_model] OCN_petlist_bounds: @[ocn_petlist_bounds] +OCN_omp_num_threads: @[ocn_omp_num_threads] OCN_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -43,6 +46,7 @@ OCN_attributes:: # ICE # ICE_model: @[ice_model] ICE_petlist_bounds: @[ice_petlist_bounds] +ICE_omp_num_threads: @[ice_omp_num_threads] ICE_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -93,6 +97,20 @@ MED_attributes:: coupling_mode = @[CPLMODE] pio_rearranger = @[pio_rearranger] ocean_albedo_limit = @[ocean_albedo_limit] + histaux_ice2med_file1_auxname = ice.1h.aux + histaux_ice2med_file1_doavg = .false. + histaux_ice2med_file1_enabled = .true. + histaux_ice2med_file1_flds = all + histaux_ice2med_file1_history_n = @[MED_history_n] + histaux_ice2med_file1_history_option = nhours + histaux_ice2med_file1_ntperfile = 9999 + histaux_ocn2med_file1_auxname = ocn.1h.aux + histaux_ocn2med_file1_doavg = .false. + histaux_ocn2med_file1_enabled = .true. + histaux_ocn2med_file1_flds = all + histaux_ocn2med_file1_history_n = @[MED_history_n] + histaux_ocn2med_file1_history_option = nhours + histaux_ocn2med_file1_ntperfile = 9999 :: ALLCOMP_attributes:: diff --git a/tests/parm/ufs.configure.s2s_aoflux_esmf.IN b/tests/parm/ufs.configure.s2s_aoflux.IN similarity index 98% rename from tests/parm/ufs.configure.s2s_aoflux_esmf.IN rename to tests/parm/ufs.configure.s2s_aoflux.IN index c5c2705e03..f2e416f931 100644 --- a/tests/parm/ufs.configure.s2s_aoflux_esmf.IN +++ b/tests/parm/ufs.configure.s2s_aoflux.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN ICE diff --git a/tests/parm/ufs.configure.s2s_esmf.IN b/tests/parm/ufs.configure.s2s_esmf.IN deleted file mode 100644 index 999731d927..0000000000 --- a/tests/parm/ufs.configure.s2s_esmf.IN +++ /dev/null @@ -1,139 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: MED ATM OCN ICE -EARTH_attributes:: - Verbosity = 0 -:: - -# MED # -MED_model: @[med_model] -MED_petlist_bounds: @[med_petlist_bounds] -MED_omp_num_threads: @[med_omp_num_threads] -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true -:: - -# OCN # -OCN_model: @[ocn_model] -OCN_petlist_bounds: @[ocn_petlist_bounds] -OCN_omp_num_threads: @[ocn_omp_num_threads] -OCN_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ocn = @[MESH_OCN] - use_coldstart = @[use_coldstart] - use_mommesh = @[use_mommesh] -:: - -# ICE # -ICE_model: @[ice_model] -ICE_petlist_bounds: @[ice_petlist_bounds] -ICE_omp_num_threads: @[ice_omp_num_threads] -ICE_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ice = @[MESH_ICE] - eps_imesh = @[eps_imesh] - stop_n = @[RESTART_N] - stop_option = nhours - stop_ymd = -999 -:: - -# CMEPS warm run sequence -runSeq:: -@@[coupling_interval_slow_sec] - MED med_phases_prep_ocn_avg - MED -> OCN :remapMethod=redist - OCN - @@[coupling_interval_fast_sec] - MED med_phases_prep_atm - MED med_phases_prep_ice - MED -> ATM :remapMethod=redist - MED -> ICE :remapMethod=redist - ATM - ICE - ATM -> MED :remapMethod=redist - MED med_phases_post_atm - ICE -> MED :remapMethod=redist - MED med_phases_post_ice - MED med_phases_ocnalb_run - MED med_phases_prep_ocn_accum - @ - OCN -> MED :remapMethod=redist - MED med_phases_post_ocn - MED med_phases_restart_write -@ -:: - -# CMEPS variables - -DRIVER_attributes:: -:: - -MED_attributes:: - ATM_model = @[atm_model] - ICE_model = @[ice_model] - OCN_model = @[ocn_model] - coupling_mode = @[CPLMODE] - pio_rearranger = @[pio_rearranger] - ocean_albedo_limit = @[ocean_albedo_limit] - histaux_ice2med_file1_auxname = ice.1h.aux - histaux_ice2med_file1_doavg = .false. - histaux_ice2med_file1_enabled = .true. - histaux_ice2med_file1_flds = all - histaux_ice2med_file1_history_n = @[MED_history_n] - histaux_ice2med_file1_history_option = nhours - histaux_ice2med_file1_ntperfile = 9999 - histaux_ocn2med_file1_auxname = ocn.1h.aux - histaux_ocn2med_file1_doavg = .false. - histaux_ocn2med_file1_enabled = .true. - histaux_ocn2med_file1_flds = all - histaux_ocn2med_file1_history_n = @[MED_history_n] - histaux_ocn2med_file1_history_option = nhours - histaux_ocn2med_file1_ntperfile = 9999 -:: - -ALLCOMP_attributes:: - ScalarFieldCount = 3 - ScalarFieldIdxGridNX = 1 - ScalarFieldIdxGridNY = 2 - ScalarFieldIdxGridNTile = 3 - ScalarFieldName = cpl_scalars - start_type = @[RUNTYPE] - restart_dir = @[CMEPS_RESTART_DIR] - case_name = ufs.cpld - restart_n = @[RESTART_N] - restart_option = nhours - restart_ymd = -999 - write_restart_at_endofrun = @[WRITE_ENDOFRUN_RESTART] - dbug_flag = @[cap_dbug_flag] - stop_n = @[FHMAX] - stop_option = nhours - stop_ymd = -999 - orb_eccen = 1.e36 - orb_iyear = 2000 - orb_iyear_align = 2000 - orb_mode = fixed_year - orb_mvelp = 1.e36 - orb_obliq = 1.e36 -:: diff --git a/tests/parm/ufs.configure.s2sa.IN b/tests/parm/ufs.configure.s2sa.IN index f49831094c..db685a49bc 100644 --- a/tests/parm/ufs.configure.s2sa.IN +++ b/tests/parm/ufs.configure.s2sa.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM CHM OCN ICE @@ -15,11 +15,13 @@ EARTH_attributes:: # MED # MED_model: @[med_model] MED_petlist_bounds: @[med_petlist_bounds] +MED_omp_num_threads: @[med_omp_num_threads] :: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -30,6 +32,7 @@ ATM_attributes:: # CHM # CHM_model: @[chm_model] CHM_petlist_bounds: @[chm_petlist_bounds] +CHM_omp_num_threads: @[chm_omp_num_threads] CHM_attributes:: Verbosity = 0 :: @@ -37,6 +40,7 @@ CHM_attributes:: # OCN # OCN_model: @[ocn_model] OCN_petlist_bounds: @[ocn_petlist_bounds] +OCN_omp_num_threads: @[ocn_omp_num_threads] OCN_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -50,6 +54,7 @@ OCN_attributes:: # ICE # ICE_model: @[ice_model] ICE_petlist_bounds: @[ice_petlist_bounds] +ICE_omp_num_threads: @[ice_omp_num_threads] ICE_attributes:: Verbosity = 0 DumpFields = @[DumpFields] diff --git a/tests/parm/ufs.configure.s2sa_esmf.IN b/tests/parm/ufs.configure.s2sa_esmf.IN deleted file mode 100644 index 3921f3881b..0000000000 --- a/tests/parm/ufs.configure.s2sa_esmf.IN +++ /dev/null @@ -1,137 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: MED ATM CHM OCN ICE -EARTH_attributes:: - Verbosity = 0 -:: - -# MED # -MED_model: @[med_model] -MED_petlist_bounds: @[med_petlist_bounds] -MED_omp_num_threads: @[med_omp_num_threads] -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true -:: - -# CHM # -CHM_model: @[chm_model] -CHM_petlist_bounds: @[chm_petlist_bounds] -CHM_omp_num_threads: @[chm_omp_num_threads] -CHM_attributes:: - Verbosity = 0 -:: - -# OCN # -OCN_model: @[ocn_model] -OCN_petlist_bounds: @[ocn_petlist_bounds] -OCN_omp_num_threads: @[ocn_omp_num_threads] -OCN_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ocn = @[MESH_OCN] - use_coldstart = @[use_coldstart] - use_mommesh = @[use_mommesh] -:: - -# ICE # -ICE_model: @[ice_model] -ICE_petlist_bounds: @[ice_petlist_bounds] -ICE_omp_num_threads: @[ice_omp_num_threads] -ICE_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ice = @[MESH_ICE] - eps_imesh = @[eps_imesh] - stop_n = @[RESTART_N] - stop_option = nhours - stop_ymd = -999 -:: - -# CMEPS warm run sequence -runSeq:: -@@[coupling_interval_slow_sec] - MED med_phases_prep_ocn_avg - MED -> OCN :remapMethod=redist - OCN - @@[coupling_interval_fast_sec] - MED med_phases_prep_atm - MED med_phases_prep_ice - MED -> ATM :remapMethod=redist - MED -> ICE :remapMethod=redist - ATM phase1 - ATM -> CHM - CHM - CHM -> ATM - ATM phase2 - ICE - ATM -> MED :remapMethod=redist - MED med_phases_post_atm - ICE -> MED :remapMethod=redist - MED med_phases_post_ice - MED med_phases_ocnalb_run - MED med_phases_prep_ocn_accum - @ - OCN -> MED :remapMethod=redist - MED med_phases_post_ocn - MED med_phases_restart_write -@ -:: - -# CMEPS variables - -DRIVER_attributes:: -:: - -MED_attributes:: - ATM_model = @[atm_model] - ICE_model = @[ice_model] - OCN_model = @[ocn_model] - coupling_mode = @[CPLMODE] - pio_rearranger = @[pio_rearranger] - ocean_albedo_limit = @[ocean_albedo_limit] -:: - -ALLCOMP_attributes:: - ScalarFieldCount = 3 - ScalarFieldIdxGridNX = 1 - ScalarFieldIdxGridNY = 2 - ScalarFieldIdxGridNTile = 3 - ScalarFieldName = cpl_scalars - start_type = @[RUNTYPE] - restart_dir = @[CMEPS_RESTART_DIR] - case_name = ufs.cpld - restart_n = @[RESTART_N] - restart_option = nhours - restart_ymd = -999 - write_restart_at_endofrun = @[WRITE_ENDOFRUN_RESTART] - dbug_flag = @[cap_dbug_flag] - stop_n = @[FHMAX] - stop_option = nhours - stop_ymd = -999 - orb_eccen = 1.e36 - orb_iyear = 2000 - orb_iyear_align = 2000 - orb_mode = fixed_year - orb_mvelp = 1.e36 - orb_obliq = 1.e36 -:: diff --git a/tests/parm/ufs.configure.s2sw.IN b/tests/parm/ufs.configure.s2sw.IN index 9ba2a9c2e1..38b6ab9a76 100644 --- a/tests/parm/ufs.configure.s2sw.IN +++ b/tests/parm/ufs.configure.s2sw.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN ICE WAV @@ -15,11 +15,13 @@ EARTH_attributes:: # MED # MED_model: @[med_model] MED_petlist_bounds: @[med_petlist_bounds] +MED_omp_num_threads: @[med_omp_num_threads] :: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -30,6 +32,7 @@ ATM_attributes:: # OCN # OCN_model: @[ocn_model] OCN_petlist_bounds: @[ocn_petlist_bounds] +OCN_omp_num_threads: @[ocn_omp_num_threads] OCN_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -43,6 +46,7 @@ OCN_attributes:: # ICE # ICE_model: @[ice_model] ICE_petlist_bounds: @[ice_petlist_bounds] +ICE_omp_num_threads: @[ice_omp_num_threads] ICE_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -58,6 +62,7 @@ ICE_attributes:: # WAV # WAV_model: @[wav_model] WAV_petlist_bounds: @[wav_petlist_bounds] +WAV_omp_num_threads: @[wav_omp_num_threads] WAV_attributes:: Verbosity = 0 OverwriteSlice = false @@ -109,7 +114,6 @@ runSeq:: DRIVER_attributes:: :: - MED_attributes:: ATM_model = @[atm_model] ICE_model = @[ice_model] @@ -119,7 +123,6 @@ MED_attributes:: pio_rearranger = @[pio_rearranger] ocean_albedo_limit = @[ocean_albedo_limit] :: - ALLCOMP_attributes:: ScalarFieldCount = 3 ScalarFieldIdxGridNX = 1 diff --git a/tests/parm/ufs.configure.s2sw_esmf.IN b/tests/parm/ufs.configure.s2sw_esmf.IN deleted file mode 100644 index 0bfd8dc3d7..0000000000 --- a/tests/parm/ufs.configure.s2sw_esmf.IN +++ /dev/null @@ -1,149 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: MED ATM OCN ICE WAV -EARTH_attributes:: - Verbosity = 0 -:: - -# MED # -MED_model: @[med_model] -MED_petlist_bounds: @[med_petlist_bounds] -MED_omp_num_threads: @[med_omp_num_threads] -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true -:: - -# OCN # -OCN_model: @[ocn_model] -OCN_petlist_bounds: @[ocn_petlist_bounds] -OCN_omp_num_threads: @[ocn_omp_num_threads] -OCN_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ocn = @[MESH_OCN] - use_coldstart = @[use_coldstart] - use_mommesh = @[use_mommesh] -:: - -# ICE # -ICE_model: @[ice_model] -ICE_petlist_bounds: @[ice_petlist_bounds] -ICE_omp_num_threads: @[ice_omp_num_threads] -ICE_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ice = @[MESH_ICE] - eps_imesh = @[eps_imesh] - stop_n = @[RESTART_N] - stop_option = nhours - stop_ymd = -999 -:: - -# WAV # -WAV_model: @[wav_model] -WAV_petlist_bounds: @[wav_petlist_bounds] -WAV_omp_num_threads: @[wav_omp_num_threads] -WAV_attributes:: - Verbosity = 0 - OverwriteSlice = false - mesh_wav = @[MESH_WAV] - user_histname = @[WW3_user_histname] - use_historync = @[WW3_historync] - use_restartnc = @[WW3_restartnc] - restart_from_binary = @[WW3_restart_from_binary] - pio_typename = @[WW3_PIO_FORMAT] - pio_numiotasks = @[WW3_PIO_IOTASKS] - pio_stride = @[WW3_PIO_STRIDE] - pio_rearranger = @[WW3_PIO_REARR] - pio_root = @[WW3_PIO_ROOT] -:: - -# CMEPS warm run sequence -runSeq:: -@@[coupling_interval_slow_sec] - MED med_phases_prep_wav_avg - MED med_phases_prep_ocn_avg - MED -> WAV :remapMethod=redist - MED -> OCN :remapMethod=redist - WAV - OCN - @@[coupling_interval_fast_sec] - MED med_phases_prep_atm - MED med_phases_prep_ice - MED -> ATM :remapMethod=redist - MED -> ICE :remapMethod=redist - ATM - ICE - ATM -> MED :remapMethod=redist - MED med_phases_post_atm - ICE -> MED :remapMethod=redist - MED med_phases_post_ice - MED med_phases_ocnalb_run - MED med_phases_prep_ocn_accum - MED med_phases_prep_wav_accum - @ - OCN -> MED :remapMethod=redist - WAV -> MED :remapMethod=redist - MED med_phases_post_ocn - MED med_phases_post_wav - MED med_phases_restart_write -@ -:: - -# CMEPS variables - -DRIVER_attributes:: -:: -MED_attributes:: - ATM_model = @[atm_model] - ICE_model = @[ice_model] - OCN_model = @[ocn_model] - WAV_model = @[wav_model] - coupling_mode = @[CPLMODE] - pio_rearranger = @[pio_rearranger] - ocean_albedo_limit = @[ocean_albedo_limit] -:: -ALLCOMP_attributes:: - ScalarFieldCount = 3 - ScalarFieldIdxGridNX = 1 - ScalarFieldIdxGridNY = 2 - ScalarFieldIdxGridNTile = 3 - ScalarFieldName = cpl_scalars - start_type = @[RUNTYPE] - restart_dir = @[CMEPS_RESTART_DIR] - case_name = ufs.cpld - restart_n = @[RESTART_N] - restart_option = nhours - restart_ymd = -999 - write_restart_at_endofrun = @[WRITE_ENDOFRUN_RESTART] - dbug_flag = @[cap_dbug_flag] - stop_n = @[FHMAX] - stop_option = nhours - stop_ymd = -999 - orb_eccen = 1.e36 - orb_iyear = 2000 - orb_iyear_align = 2000 - orb_mode = fixed_year - orb_mvelp = 1.e36 - orb_obliq = 1.e36 -:: diff --git a/tests/parm/ufs.configure.s2sw_fast_esmf.IN b/tests/parm/ufs.configure.s2sw_fast.IN similarity index 98% rename from tests/parm/ufs.configure.s2sw_fast_esmf.IN rename to tests/parm/ufs.configure.s2sw_fast.IN index 623fab0124..a4ac78def4 100644 --- a/tests/parm/ufs.configure.s2sw_fast_esmf.IN +++ b/tests/parm/ufs.configure.s2sw_fast.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM OCN ICE WAV diff --git a/tests/parm/ufs.configure.s2swa.IN b/tests/parm/ufs.configure.s2swa.IN index 2fa6ac40c3..cb930fd4b3 100644 --- a/tests/parm/ufs.configure.s2swa.IN +++ b/tests/parm/ufs.configure.s2swa.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: @[esmf_logkind] -globalResourceControl: false +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM CHM OCN ICE WAV @@ -15,11 +15,13 @@ EARTH_attributes:: # MED # MED_model: @[med_model] MED_petlist_bounds: @[med_petlist_bounds] +MED_omp_num_threads: @[med_omp_num_threads] :: # ATM # ATM_model: @[atm_model] ATM_petlist_bounds: @[atm_petlist_bounds] +ATM_omp_num_threads: @[atm_omp_num_threads] ATM_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -30,6 +32,7 @@ ATM_attributes:: # CHM # CHM_model: @[chm_model] CHM_petlist_bounds: @[chm_petlist_bounds] +CHM_omp_num_threads: @[chm_omp_num_threads] CHM_attributes:: Verbosity = 0 :: @@ -37,6 +40,7 @@ CHM_attributes:: # OCN # OCN_model: @[ocn_model] OCN_petlist_bounds: @[ocn_petlist_bounds] +OCN_omp_num_threads: @[ocn_omp_num_threads] OCN_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -50,6 +54,7 @@ OCN_attributes:: # ICE # ICE_model: @[ice_model] ICE_petlist_bounds: @[ice_petlist_bounds] +ICE_omp_num_threads: @[ice_omp_num_threads] ICE_attributes:: Verbosity = 0 DumpFields = @[DumpFields] @@ -65,6 +70,7 @@ ICE_attributes:: # WAV # WAV_model: @[wav_model] WAV_petlist_bounds: @[wav_petlist_bounds] +WAV_omp_num_threads: @[wav_omp_num_threads] WAV_attributes:: Verbosity = 0 OverwriteSlice = false diff --git a/tests/parm/ufs.configure.s2swa_esmf.IN b/tests/parm/ufs.configure.s2swa_esmf.IN deleted file mode 100644 index 91410741de..0000000000 --- a/tests/parm/ufs.configure.s2swa_esmf.IN +++ /dev/null @@ -1,161 +0,0 @@ -############################################# -#### UFS Run-Time Configuration File ###### -############################################# - -# ESMF # -logKindFlag: @[esmf_logkind] -globalResourceControl: true - -# EARTH # -EARTH_component_list: MED ATM CHM OCN ICE WAV -EARTH_attributes:: - Verbosity = 0 -:: - -# MED # -MED_model: @[med_model] -MED_petlist_bounds: @[med_petlist_bounds] -MED_omp_num_threads: @[med_omp_num_threads] -:: - -# ATM # -ATM_model: @[atm_model] -ATM_petlist_bounds: @[atm_petlist_bounds] -ATM_omp_num_threads: @[atm_omp_num_threads] -ATM_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true -:: - -# CHM # -CHM_model: @[chm_model] -CHM_petlist_bounds: @[chm_petlist_bounds] -CHM_omp_num_threads: @[chm_omp_num_threads] -CHM_attributes:: - Verbosity = 0 -:: - -# OCN # -OCN_model: @[ocn_model] -OCN_petlist_bounds: @[ocn_petlist_bounds] -OCN_omp_num_threads: @[ocn_omp_num_threads] -OCN_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ocn = @[MESH_OCN] - use_coldstart = @[use_coldstart] - use_mommesh = @[use_mommesh] -:: - -# ICE # -ICE_model: @[ice_model] -ICE_petlist_bounds: @[ice_petlist_bounds] -ICE_omp_num_threads: @[ice_omp_num_threads] -ICE_attributes:: - Verbosity = 0 - DumpFields = @[DumpFields] - ProfileMemory = false - OverwriteSlice = true - mesh_ice = @[MESH_ICE] - eps_imesh = @[eps_imesh] - stop_n = @[RESTART_N] - stop_option = nhours - stop_ymd = -999 -:: - -# WAV # -WAV_model: @[wav_model] -WAV_petlist_bounds: @[wav_petlist_bounds] -WAV_omp_num_threads: @[wav_omp_num_threads] -WAV_attributes:: - Verbosity = 0 - OverwriteSlice = false - mesh_wav = @[MESH_WAV] - user_histname = @[WW3_user_histname] - use_historync = @[WW3_historync] - use_restartnc = @[WW3_restartnc] - restart_from_binary = @[WW3_restart_from_binary] - pio_typename = @[WW3_PIO_FORMAT] - pio_numiotasks = @[WW3_PIO_IOTASKS] - pio_stride = @[WW3_PIO_STRIDE] - pio_rearranger = @[WW3_PIO_REARR] - pio_root = @[WW3_PIO_ROOT] -:: - -# CMEPS warm run sequence -runSeq:: -@@[coupling_interval_slow_sec] - MED med_phases_prep_wav_avg - MED med_phases_prep_ocn_avg - MED -> WAV :remapMethod=redist - MED -> OCN :remapMethod=redist - WAV - OCN - @@[coupling_interval_fast_sec] - MED med_phases_prep_atm - MED med_phases_prep_ice - MED -> ATM :remapMethod=redist - MED -> ICE :remapMethod=redist - ATM phase1 - ATM -> CHM - CHM - CHM -> ATM - ATM phase2 - ICE - ATM -> MED :remapMethod=redist - MED med_phases_post_atm - ICE -> MED :remapMethod=redist - MED med_phases_post_ice - MED med_phases_ocnalb_run - MED med_phases_prep_ocn_accum - MED med_phases_prep_wav_accum - @ - OCN -> MED :remapMethod=redist - WAV -> MED :remapMethod=redist - MED med_phases_post_ocn - MED med_phases_post_wav - MED med_phases_restart_write -@ -:: - -# CMEPS variables - -DRIVER_attributes:: -:: -MED_attributes:: - ATM_model = @[atm_model] - ICE_model = @[ice_model] - OCN_model = @[ocn_model] - WAV_model = @[wav_model] - coupling_mode = @[CPLMODE] - pio_rearranger = @[pio_rearranger] - ocean_albedo_limit = @[ocean_albedo_limit] -:: -ALLCOMP_attributes:: - ScalarFieldCount = 3 - ScalarFieldIdxGridNX = 1 - ScalarFieldIdxGridNY = 2 - ScalarFieldIdxGridNTile = 3 - ScalarFieldName = cpl_scalars - start_type = @[RUNTYPE] - restart_dir = @[CMEPS_RESTART_DIR] - case_name = ufs.cpld - restart_n = @[RESTART_N] - restart_option = nhours - restart_ymd = -999 - write_restart_at_endofrun = @[WRITE_ENDOFRUN_RESTART] - dbug_flag = @[cap_dbug_flag] - stop_n = @[FHMAX] - stop_option = nhours - stop_ymd = -999 - orb_eccen = 1.e36 - orb_iyear = 2000 - orb_iyear_align = 2000 - orb_mode = fixed_year - orb_mvelp = 1.e36 - orb_obliq = 1.e36 -:: diff --git a/tests/parm/ufs.configure.s2swa_fast_esmf.IN b/tests/parm/ufs.configure.s2swa_fast.IN similarity index 98% rename from tests/parm/ufs.configure.s2swa_fast_esmf.IN rename to tests/parm/ufs.configure.s2swa_fast.IN index 5ded338a95..e5ed589d7f 100644 --- a/tests/parm/ufs.configure.s2swa_fast_esmf.IN +++ b/tests/parm/ufs.configure.s2swa_fast.IN @@ -4,7 +4,7 @@ # ESMF # logKindFlag: ESMF_LOGKIND_MULTI -globalResourceControl: true +globalResourceControl: @[ESMF_THREADING] # EARTH # EARTH_component_list: MED ATM CHM OCN ICE WAV @@ -16,6 +16,7 @@ EARTH_attributes:: MED_model: @[med_model] MED_petlist_bounds: @[med_petlist_bounds] MED_omp_num_threads: @[med_omp_num_threads] +:: # ATM # ATM_model: @[atm_model] diff --git a/tests/rt.sh b/tests/rt.sh index 672c876e53..d887f38cc7 100755 --- a/tests/rt.sh +++ b/tests/rt.sh @@ -1266,7 +1266,11 @@ EOF ( source "${PATHRT}/tests/${TEST_NAME}" - compute_petbounds_and_tasks + if [[ ${ESMF_THREADING} == true ]]; then + compute_petbounds_and_tasks_esmf_threading + else + compute_petbounds_and_tasks_traditional_threading + fi TPN=$(( TPN / THRD )) NODES=$(( TASKS / TPN )) diff --git a/tests/rt_utils.sh b/tests/rt_utils.sh index ecf4259421..8ea92b6ec3 100755 --- a/tests/rt_utils.sh +++ b/tests/rt_utils.sh @@ -21,7 +21,122 @@ redirect_out_err() { # redirect_out_err command will return non-zero if "$@" or tee return non-zero. } -function compute_petbounds_and_tasks() { +function compute_petbounds_and_tasks_traditional_threading() { + + # each test MUST define ${COMPONENT}_tasks variable for all components it is using + # and MUST NOT define those that it's not using or set the value to 0. + + # ATM is a special case since it is running on the sum of compute and io tasks. + # CHM component and mediator are running on ATM compute tasks only. + + if [[ ${DATM_CDEPS} = 'false' ]]; then + if [[ ${ATM_compute_tasks:-0} -eq 0 ]]; then + ATM_compute_tasks=$((INPES * JNPES * NTILES)) + fi + if [[ ${QUILTING} = '.true.' ]]; then + ATM_io_tasks=$((WRITE_GROUP * WRTTASK_PER_GROUP)) + else + ATM_io_tasks=0 + fi + ATM_tasks=$((ATM_compute_tasks + ATM_io_tasks)) + fi + + local n=0 + unset atm_petlist_bounds ocn_petlist_bounds ice_petlist_bounds wav_petlist_bounds chm_petlist_bounds med_petlist_bounds aqm_petlist_bounds fbh_petlist_bounds + + local _tasks + + # ATM + if [[ ${ATM_tasks:-0} -gt 0 ]]; then + atm_petlist_bounds="${n} $((n + ATM_tasks - 1))" + n=$((n + ATM_tasks)) + _tasks=$(( ATM_tasks*atm_omp_num_threads )) + atm_nodes=$(( _tasks / TPN )) + if (( atm_nodes * TPN < _tasks )); then + atm_nodes=$(( atm_nodes + 1 )) + fi + fi + + # OCN + if [[ ${OCN_tasks:-0} -gt 0 ]]; then + ocn_petlist_bounds="${n} $((n + OCN_tasks - 1))" + n=$((n + OCN_tasks)) + _tasks=$(( OCN_tasks*ocn_omp_num_threads )) + ocn_nodes=$(( _tasks / TPN )) + if (( ocn_nodes * TPN < _tasks )); then + ocn_nodes=$(( ocn_nodes + 1 )) + fi + fi + + # ICE + if [[ ${ICE_tasks:-0} -gt 0 ]]; then + ice_petlist_bounds="${n} $((n + ICE_tasks - 1))" + n=$((n + ICE_tasks)) + _tasks=$(( ICE_tasks*ice_omp_num_threads )) + ice_nodes=$(( _tasks / TPN )) + if (( ice_nodes * TPN < _tasks )); then + ice_nodes=$(( ice_nodes + 1 )) + fi + fi + + # WAV + if [[ ${WAV_tasks:-0} -gt 0 ]]; then + wav_petlist_bounds="${n} $((n + WAV_tasks - 1))" + n=$((n + WAV_tasks)) + _tasks=$(( WAV_tasks*wav_omp_num_threads )) + wav_nodes=$(( _tasks / TPN )) + if (( wav_nodes * TPN < _tasks )); then + wav_nodes=$(( wav_nodes + 1 )) + fi + fi + + # CHM + chm_petlist_bounds="0 $((ATM_compute_tasks - 1))" + + # MED - mediator (CMEPS) runs on at most 300 tasks. + MED_compute_tasks=$((ATM_compute_tasks<=300 ? ATM_compute_tasks : 300)) + med_petlist_bounds="0 $((MED_compute_tasks - 1))" + + # AQM + aqm_petlist_bounds="0 $((ATM_compute_tasks - 1))" + + # LND + if [[ ${lnd_model:-} = "lm4" ]]; then + # set lnd_petlist_bounds to be same as ATM_compute_tasks + lnd_petlist_bounds="0 $((ATM_compute_tasks - 1))" + elif [[ ${LND_tasks:-0} -gt 0 ]]; then # noahmp component or other + lnd_petlist_bounds="${n} $((n + LND_tasks - 1))" + n=$((n + LND_tasks)) + fi + + # FBH + if [[ ${FBH_tasks:-0} -gt 0 ]]; then + fbh_petlist_bounds="${n} $((n + FBH_tasks - 1))" + n=$((n + FBH_tasks)) + fi + + unset _tasks + + UFS_tasks=${n} + + if [[ ${RTVERBOSE} == true ]]; then + echo "ATM_petlist_bounds: ${atm_petlist_bounds:-}" + echo "OCN_petlist_bounds: ${ocn_petlist_bounds:-}" + echo "ICE_petlist_bounds: ${ice_petlist_bounds:-}" + echo "WAV_petlist_bounds: ${wav_petlist_bounds:-}" + echo "CHM_petlist_bounds: ${chm_petlist_bounds:-}" + echo "MED_petlist_bounds: ${med_petlist_bounds:-}" + echo "AQM_petlist_bounds: ${aqm_petlist_bounds:-}" + echo "LND_petlist_bounds: ${lnd_petlist_bounds:-}" + echo "FBH_petlist_bounds: ${fbh_petlist_bounds:-}" + echo "UFS_tasks : ${UFS_tasks:-}" + fi + + # TASKS is now set to UFS_TASKS + export TASKS=${UFS_tasks} +} + +function compute_petbounds_and_tasks_esmf_threading() { # each test MUST define ${COMPONENT}_tasks variable for all components it is using # and MUST NOT define those that it's not using or set the value to 0. @@ -72,8 +187,9 @@ function compute_petbounds_and_tasks() { # CHM chm_petlist_bounds="0 $((ATM_compute_tasks * atm_omp_num_threads - 1))" - # MED - med_petlist_bounds="0 $((ATM_compute_tasks * atm_omp_num_threads - 1))" + # MED - mediator (CMEPS) runs on at most 300 tasks. + MED_compute_tasks=$((ATM_compute_tasks<=300 ? ATM_compute_tasks : 300)) + med_petlist_bounds="0 $((MED_compute_tasks * atm_omp_num_threads - 1))" # AQM aqm_petlist_bounds="0 $((ATM_compute_tasks * atm_omp_num_threads - 1))" @@ -82,8 +198,8 @@ function compute_petbounds_and_tasks() { if [[ ${lnd_model:-} = "lm4" ]]; then # set lnd_petlist_bounds to be same as ATM_compute_tasks lnd_petlist_bounds="0 $((ATM_compute_tasks - 1))" - elif [[ ${LND_tasks:-0} -gt 0 ]]; then # noahmp component or other - LND_tasks=$((LND_tasks * lnd_omp_num_threads)) + elif [[ ${LND_tasks:-0} -gt 0 ]]; then # noahmp component or other + LND_tasks=$((LND_tasks * lnd_omp_num_threads)) lnd_petlist_bounds="${n} $((n + LND_tasks - 1))" n=$((n + LND_tasks)) fi diff --git a/tests/run_test.sh b/tests/run_test.sh index 8e8e7c34d3..4702877f69 100755 --- a/tests/run_test.sh +++ b/tests/run_test.sh @@ -163,10 +163,19 @@ else exit 1 fi -compute_petbounds_and_tasks +if [[ ${ESMF_THREADING} == true ]]; then + compute_petbounds_and_tasks_esmf_threading +else + compute_petbounds_and_tasks_traditional_threading +fi if [[ -f ${PATHRT}/parm/${UFS_CONFIGURE} ]]; then - atparse < "${PATHRT}/parm/${UFS_CONFIGURE}" > ufs.configure + ( + atparse < "${PATHRT}/parm/${UFS_CONFIGURE}" > ufs.configure + if [[ ${ESMF_THREADING} != true ]]; then + sed -i -e "/_omp_num_threads:/d" ufs.configure + fi + ) else echo "Cannot find file ${UFS_CONFIGURE} set by variable UFS_CONFIGURE" exit 1 @@ -240,7 +249,7 @@ fi if [[ "Q${FIELD_TABLE:-}" != Q ]]; then cp "${PATHRT}/parm/field_table/${FIELD_TABLE}" field_table fi - + # fix files if [[ ${FV3} == true ]]; then cp "${INPUTDATA_ROOT}"/FV3_fix/*.txt . @@ -367,16 +376,20 @@ if (( NODES * TPN < TASKS )); then fi export NODES -UFS_TASKS=${TASKS} -TASKS=$(( NODES * TPN )) -export TASKS +if [[ ${ESMF_THREADING} == true ]]; then + UFS_TASKS=${TASKS} + TASKS=$(( NODES * TPN )) + export TASKS -PPN=$(( UFS_TASKS / NODES )) -if (( UFS_TASKS - ( PPN * NODES ) > 0 )); then - PPN=$((PPN + 1)) + PPN=$(( UFS_TASKS / NODES )) + if (( UFS_TASKS - ( PPN * NODES ) > 0 )); then + PPN=$((PPN + 1)) + fi + export PPN + export UFS_TASKS +else + PPN=${TPN} fi -export PPN -export UFS_TASKS if [[ ${SCHEDULER} = 'pbs' ]]; then if [[ -e ${PATHRT}/fv3_conf/fv3_qsub.IN_${MACHINE_ID} ]]; then diff --git a/tests/tests/atm_ds2s_docn_dice b/tests/tests/atm_ds2s_docn_dice index cd477d3bc1..dc1e33896f 100644 --- a/tests/tests/atm_ds2s_docn_dice +++ b/tests/tests/atm_ds2s_docn_dice @@ -127,7 +127,7 @@ export DIAG_TABLE=diag_table_gfsv16 export FIELD_TABLE=field_table_thompson_noaero_tke export INPUT_NML=global_control.nml.IN export MODEL_CONFIGURE=model_configure.IN -export UFS_CONFIGURE=ufs.configure.s2s_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s.IN export FV3_RUN="cpld_control_run.IN cpld_docn_dice.IN" if [[ $MACHINE_ID = hera ]] || [[ $MACHINE_ID = hercules ]] || [[ $MACHINE_ID = orion ]] || [[ $MACHINE_ID = gaea ]]; then diff --git a/tests/tests/atm_ds2s_docn_pcice b/tests/tests/atm_ds2s_docn_pcice index fef8e3cdf3..4f6204e6fc 100644 --- a/tests/tests/atm_ds2s_docn_pcice +++ b/tests/tests/atm_ds2s_docn_pcice @@ -124,5 +124,5 @@ export DIAG_TABLE=diag_table_gfsv16 export FIELD_TABLE=field_table_thompson_noaero_tke export INPUT_NML=global_control.nml.IN export MODEL_CONFIGURE=model_configure.IN -export UFS_CONFIGURE=ufs.configure.s2s_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s.IN export FV3_RUN="cpld_control_run.IN cpld_docn_pcice.IN" diff --git a/tests/tests/atmaero_control_p8 b/tests/tests/atmaero_control_p8 index 6d4012d580..8c40560673 100644 --- a/tests/tests/atmaero_control_p8 +++ b/tests/tests/atmaero_control_p8 @@ -67,7 +67,7 @@ export WRITE_DOPOST=.true. export CPL=.true. export CPLCHM=.true. -export UFS_CONFIGURE=ufs.configure.atmaero_esmf.IN +export UFS_CONFIGURE=ufs.configure.atmaero.IN export atm_model='fv3' export chm_model='gocart' diff --git a/tests/tests/atmaero_control_p8_rad b/tests/tests/atmaero_control_p8_rad index 4eac5a3303..b384a7ff4a 100644 --- a/tests/tests/atmaero_control_p8_rad +++ b/tests/tests/atmaero_control_p8_rad @@ -66,7 +66,7 @@ export WRITE_DOPOST=.true. export CPL=.true. export CPLCHM=.true. -export UFS_CONFIGURE=ufs.configure.atmaero_esmf.IN +export UFS_CONFIGURE=ufs.configure.atmaero.IN export atm_model='fv3' export chm_model='gocart' diff --git a/tests/tests/atmaero_control_p8_rad_micro b/tests/tests/atmaero_control_p8_rad_micro index 1c4bbe4478..9a57bbc9ac 100644 --- a/tests/tests/atmaero_control_p8_rad_micro +++ b/tests/tests/atmaero_control_p8_rad_micro @@ -66,7 +66,7 @@ export WRITE_DOPOST=.true. export CPL=.true. export CPLCHM=.true. -export UFS_CONFIGURE=ufs.configure.atmaero_esmf.IN +export UFS_CONFIGURE=ufs.configure.atmaero.IN export atm_model='fv3' export chm_model='gocart' diff --git a/tests/tests/cpld_2threads_p8 b/tests/tests/cpld_2threads_p8 index 21346f4a09..3a30a6f6bf 100644 --- a/tests/tests/cpld_2threads_p8 +++ b/tests/tests/cpld_2threads_p8 @@ -103,3 +103,6 @@ export N_SPLIT=5 # HR4 GFSv17 GWD update export DO_GWD_OPT_PSL=.true. export DO_GSL_DRAG_SS=.false. + +ESMF_THREADING=false +THRD=$THRD_cpl_thrd diff --git a/tests/tests/cpld_control_c48 b/tests/tests/cpld_control_c48 index 0f86ec5d5b..7a9de35986 100644 --- a/tests/tests/cpld_control_c48 +++ b/tests/tests/cpld_control_c48 @@ -137,7 +137,7 @@ export eps_imesh=4.0e-1 export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2s_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_control_noaero_p8 b/tests/tests/cpld_control_noaero_p8 index 58853ae7d0..70c4be0283 100644 --- a/tests/tests/cpld_control_noaero_p8 +++ b/tests/tests/cpld_control_noaero_p8 @@ -65,7 +65,7 @@ export LIST_FILES="sfcf021.tile1.nc \ RESTART/20210323.060000.MOM.res.nc \ RESTART/iced.2021-03-23-21600.nc \ RESTART/ufs.cpld.cpl.r.2021-03-23-21600.nc \ - ufs.cpld.ww3.r.2021-03-23-21600.nc \ + ufs.cpld.ww3.r.2021-03-23-21600.nc \ 20210323.060000.out_pnt.ww3 \ 20210323.060000.out_grd.ww3 " @@ -86,7 +86,7 @@ export eps_imesh=2.5e-1 export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2sw_fast_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sw_fast.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_control_noaero_p8_agrid b/tests/tests/cpld_control_noaero_p8_agrid index 5036092ba4..6c294b3abd 100644 --- a/tests/tests/cpld_control_noaero_p8_agrid +++ b/tests/tests/cpld_control_noaero_p8_agrid @@ -93,7 +93,7 @@ export MOM6_ALLOW_LANDMASK_CHANGES=True export eps_imesh=2.5e-1 export DIAG_TABLE=diag_table_template -export UFS_CONFIGURE=ufs.configure.s2s_aoflux_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s_aoflux.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_control_nowave_noaero_p8 b/tests/tests/cpld_control_nowave_noaero_p8 index 240b03d2d2..031e48c98f 100644 --- a/tests/tests/cpld_control_nowave_noaero_p8 +++ b/tests/tests/cpld_control_nowave_noaero_p8 @@ -104,7 +104,7 @@ export MOM6_TOPOEDITS=ufs.topo_edits_011818.nc export MOM6_ALLOW_LANDMASK_CHANGES=True export DIAG_TABLE=diag_table_template -export UFS_CONFIGURE=ufs.configure.s2s_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_control_pdlib_p8 b/tests/tests/cpld_control_pdlib_p8 index 120504e818..a2ca423a56 100644 --- a/tests/tests/cpld_control_pdlib_p8 +++ b/tests/tests/cpld_control_pdlib_p8 @@ -102,7 +102,7 @@ export WW3_MODDEF=mod_def.exp.${WW3_DOMAIN} export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2sw_fast_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sw_fast.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_debug_noaero_p8 b/tests/tests/cpld_debug_noaero_p8 index f703ad51cd..0e566adeef 100644 --- a/tests/tests/cpld_debug_noaero_p8 +++ b/tests/tests/cpld_debug_noaero_p8 @@ -77,7 +77,7 @@ export CICE_DIAGFREQ=`expr $((FHMAX-$FHROT)) \* 3600 / $DT_CICE` export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2sw_fast_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sw_fast.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_debug_pdlib_p8 b/tests/tests/cpld_debug_pdlib_p8 index e2d0ea77c7..8e115a070a 100644 --- a/tests/tests/cpld_debug_pdlib_p8 +++ b/tests/tests/cpld_debug_pdlib_p8 @@ -96,7 +96,7 @@ export WW3_MODDEF=mod_def.exp.${WW3_DOMAIN} export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2sw_fast_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sw_fast.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_mpi_pdlib_p8 b/tests/tests/cpld_mpi_pdlib_p8 index 5afb46a3de..2c0b91ac34 100644 --- a/tests/tests/cpld_mpi_pdlib_p8 +++ b/tests/tests/cpld_mpi_pdlib_p8 @@ -104,7 +104,7 @@ export WW3_MODDEF=mod_def.exp.${WW3_DOMAIN} export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2sw_fast_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sw_fast.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_restart_c48 b/tests/tests/cpld_restart_c48 index 4803fd5f46..7507d0f492 100644 --- a/tests/tests/cpld_restart_c48 +++ b/tests/tests/cpld_restart_c48 @@ -168,7 +168,7 @@ export eps_imesh=4.0e-1 export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2s_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_restart_pdlib_p8 b/tests/tests/cpld_restart_pdlib_p8 index 8ca61d4e3a..9a3ea199d5 100644 --- a/tests/tests/cpld_restart_pdlib_p8 +++ b/tests/tests/cpld_restart_pdlib_p8 @@ -108,7 +108,7 @@ export WW3_MODDEF=mod_def.exp.${WW3_DOMAIN} export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2sw_fast_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sw_fast.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_s2sa_p8 b/tests/tests/cpld_s2sa_p8 index a9fba26d01..80fc63eb56 100644 --- a/tests/tests/cpld_s2sa_p8 +++ b/tests/tests/cpld_s2sa_p8 @@ -75,7 +75,7 @@ export MOM6_ALLOW_LANDMASK_CHANGES=True export eps_imesh=2.5e-1 export DIAG_TABLE=diag_table_template -export UFS_CONFIGURE=ufs.configure.s2sa_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2sa.IN export FV3_RUN=cpld_control_run.IN diff --git a/tests/tests/cpld_warmstart_c48 b/tests/tests/cpld_warmstart_c48 index 8f65c964de..0ba17be79d 100644 --- a/tests/tests/cpld_warmstart_c48 +++ b/tests/tests/cpld_warmstart_c48 @@ -165,7 +165,7 @@ export eps_imesh=4.0e-1 export DIAG_TABLE=diag_table_template export FIELD_TABLE=field_table_thompson_noaero_tke -export UFS_CONFIGURE=ufs.configure.s2s_esmf.IN +export UFS_CONFIGURE=ufs.configure.s2s.IN export FV3_RUN=cpld_control_run.IN From f089968fe8941e81bd28f88431eef062205906ae Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 10 Dec 2024 09:53:33 -0600 Subject: [PATCH 2/3] Convert all '2threads' tests to traditional threading --- tests/tests/control_2threads_p8 | 3 +++ tests/tests/conus13km_2threads | 3 +++ tests/tests/conus13km_debug_2threads | 3 +++ tests/tests/hrrr_control_2threads | 2 ++ tests/tests/hrrr_control_2threads_dyn32_phy32 | 2 ++ tests/tests/rap_2threads | 3 +++ tests/tests/rap_2threads_dyn32_phy32 | 3 +++ tests/tests/regional_2threads | 3 +++ 8 files changed, 22 insertions(+) diff --git a/tests/tests/control_2threads_p8 b/tests/tests/control_2threads_p8 index d48022f634..22b5a8f3f5 100644 --- a/tests/tests/control_2threads_p8 +++ b/tests/tests/control_2threads_p8 @@ -154,3 +154,6 @@ export DOGP_LWSCAT=.true. export DOGP_SGS_CNV=.true. export SATMEDMF=.true. + +ESMF_THREADING=false +THRD=$atm_omp_num_threads diff --git a/tests/tests/conus13km_2threads b/tests/tests/conus13km_2threads index 9d8620c42a..f37809fdc1 100644 --- a/tests/tests/conus13km_2threads +++ b/tests/tests/conus13km_2threads @@ -25,3 +25,6 @@ export INPES=12 export JNPES=12 export WRTTASK_PER_GROUP=6 export QUILTING_RESTART=.false. + +ESMF_THREADING=false +THRD=$atm_omp_num_threads diff --git a/tests/tests/conus13km_debug_2threads b/tests/tests/conus13km_debug_2threads index 9daadc7dd6..b565c216a5 100644 --- a/tests/tests/conus13km_debug_2threads +++ b/tests/tests/conus13km_debug_2threads @@ -24,3 +24,6 @@ export INPES=12 export JNPES=12 export WRTTASK_PER_GROUP=6 export QUILTING_RESTART=.false. + +ESMF_THREADING=false +THRD=$atm_omp_num_threads diff --git a/tests/tests/hrrr_control_2threads b/tests/tests/hrrr_control_2threads index a74ab76253..73a26a688c 100644 --- a/tests/tests/hrrr_control_2threads +++ b/tests/tests/hrrr_control_2threads @@ -61,3 +61,5 @@ export JNPES=$JNPES_thrd export WRTTASK_PER_GROUP=6 export RESTART_INTERVAL="6 -1" export OUTPUT_FH='0 09 12' + +ESMF_THREADING=false diff --git a/tests/tests/hrrr_control_2threads_dyn32_phy32 b/tests/tests/hrrr_control_2threads_dyn32_phy32 index a92253d796..e5f7b11988 100644 --- a/tests/tests/hrrr_control_2threads_dyn32_phy32 +++ b/tests/tests/hrrr_control_2threads_dyn32_phy32 @@ -62,3 +62,5 @@ export JNPES=$JNPES_thrd export WRTTASK_PER_GROUP=6 export RESTART_INTERVAL="6 -1" export OUTPUT_FH='0 09 12' + +ESMF_THREADING=false diff --git a/tests/tests/rap_2threads b/tests/tests/rap_2threads index ab5fdac9d6..f8b5da8c40 100644 --- a/tests/tests/rap_2threads +++ b/tests/tests/rap_2threads @@ -61,3 +61,6 @@ export atm_omp_num_threads=2 export INPES=$INPES_thrd export JNPES=$JNPES_thrd export WRTTASK_PER_GROUP=6 + +ESMF_THREADING=false +THRD=$atm_omp_num_threads diff --git a/tests/tests/rap_2threads_dyn32_phy32 b/tests/tests/rap_2threads_dyn32_phy32 index 8642231ad1..a943c30deb 100644 --- a/tests/tests/rap_2threads_dyn32_phy32 +++ b/tests/tests/rap_2threads_dyn32_phy32 @@ -62,3 +62,6 @@ export atm_omp_num_threads=2 export INPES=$INPES_thrd export JNPES=$JNPES_thrd export WRTTASK_PER_GROUP=6 + +ESMF_THREADING=false +THRD=$atm_omp_num_threads diff --git a/tests/tests/regional_2threads b/tests/tests/regional_2threads index 9d3e3d5ce3..0dbf5efc75 100644 --- a/tests/tests/regional_2threads +++ b/tests/tests/regional_2threads @@ -43,3 +43,6 @@ export OUTPUT_FH="3 -1" WRITE_GROUP=1 WRTTASK_PER_GROUP=10 NTILES=1 + +ESMF_THREADING=false +THRD=$atm_omp_num_threads From 441a09f3b6f26cd50dbb11e672bdf71ef992a520 Mon Sep 17 00:00:00 2001 From: Dusan Jovic Date: Tue, 10 Dec 2024 12:09:03 -0600 Subject: [PATCH 3/3] Add new config variable 'histaux_enabled' Use this variable to turn on writing histaux files every MED_history_n. By default it is .false. Currently only cpld_control_nowave_noaero_p8 test has it turned on. --- tests/default_vars.sh | 222 +++++++++++----------- tests/parm/ufs.configure.s2s.IN | 4 +- tests/tests/cpld_control_nowave_noaero_p8 | 2 + 3 files changed, 116 insertions(+), 112 deletions(-) diff --git a/tests/default_vars.sh b/tests/default_vars.sh index 919ab4ac4b..d31fb19946 100644 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -7,116 +7,118 @@ # ############################################################################### - THRD=1 - - export INPES_atmaero=4 - export JNPES_atmaero=8 - export WPG_atmaero=6 - - export THRD_cpl_atmw=1 - export INPES_cpl_atmw=3 - export JNPES_cpl_atmw=8 - export WPG_cpl_atmw=6 - export WAV_tasks_cpl_atmw=30 - export WAV_thrds_cpl_atmw=1 - - export THRD_cpl_c48=1 - export INPES_cpl_c48=1 - export JNPES_cpl_c48=1 - export WPG_cpl_c48=6 - export OCN_tasks_cpl_c48=4 - export ICE_tasks_cpl_c48=4 - - export THRD_cpl_dflt=1 - export INPES_cpl_dflt=3 - export JNPES_cpl_dflt=8; - export WPG_cpl_dflt=6 - export OCN_tasks_cpl_dflt=20 - export ICE_tasks_cpl_dflt=10 - export WAV_tasks_cpl_dflt=20 - - export THRD_cpl_thrd=2 - export INPES_cpl_thrd=3 - export JNPES_cpl_thrd=4 - export WPG_cpl_thrd=6 - export OCN_tasks_cpl_thrd=20 - export OCN_thrds_cpl_thrd=1 - export ICE_tasks_cpl_thrd=10 - export ICE_thrds_cpl_thrd=1 - export WAV_tasks_cpl_thrd=12 - export WAV_thrds_cpl_thrd=2 - - export THRD_cpl_dcmp=1 - export INPES_cpl_dcmp=4 - export JNPES_cpl_dcmp=6 - export WPG_cpl_dcmp=6 - export OCN_tasks_cpl_dcmp=20 - export ICE_tasks_cpl_dcmp=10 - export WAV_tasks_cpl_dcmp=20 - - export THRD_cpl_mpi=1 - export INPES_cpl_mpi=4 - export JNPES_cpl_mpi=8 - export WPG_cpl_mpi=6 - export OCN_tasks_cpl_mpi=34 - export ICE_tasks_cpl_mpi=20 - export WAV_tasks_cpl_mpi=28 - - export THRD_cpl_bmrk=2 - export INPES_cpl_bmrk=8 - export JNPES_cpl_bmrk=8 - export WPG_cpl_bmrk=48 - export OCN_tasks_cpl_bmrk=120 - export OCN_thrds_cpl_bmrk=1 - export ICE_tasks_cpl_bmrk=48 - export ICE_thrds_cpl_bmrk=1 - export WAV_tasks_cpl_bmrk=80 - export WAV_thrds_cpl_bmrk=2 - - export THRD_cpl_c192=2 - export INPES_cpl_c192=6 - export JNPES_cpl_c192=8 - export WPG_cpl_c192=12 - export OCN_tasks_cpl_c192=60 - export ICE_tasks_cpl_c192=24 - export WAV_tasks_cpl_c192=80 - - export ATM_compute_tasks_cdeps_100=12 - export OCN_tasks_cdeps_100=16 - export ICE_tasks_cdeps_100=12 - - export ATM_compute_tasks_cdeps_025=40 - export OCN_tasks_cdeps_025=120 - export ICE_tasks_cdeps_025=48 - - export INPES_aqm=33 - export JNPES_aqm=8 - - export THRD_cpl_unstr=1 - export INPES_cpl_unstr=3 - export JNPES_cpl_unstr=8 - export WPG_cpl_unstr=6 - export OCN_tasks_cpl_unstr=20 - export ICE_tasks_cpl_unstr=10 - export WAV_tasks_cpl_unstr=60 - - export THRD_cpl_unstr_mpi=1 - export INPES_cpl_unstr_mpi=4 - export JNPES_cpl_unstr_mpi=8 - export WPG_cpl_unstr_mpi=6 - export OCN_tasks_cpl_unstr_mpi=34 - export ICE_tasks_cpl_unstr_mpi=20 - export WAV_tasks_cpl_unstr_mpi=50 - - export aqm_omp_num_threads=1 - export atm_omp_num_threads=1 - export chm_omp_num_threads=1 - export ice_omp_num_threads=1 - export lnd_omp_num_threads=1 - export med_omp_num_threads=1 - export ocn_omp_num_threads=1 - export wav_omp_num_threads=1 - export fbh_omp_num_threads=1 +THRD=1 + +export INPES_atmaero=4 +export JNPES_atmaero=8 +export WPG_atmaero=6 + +export THRD_cpl_atmw=1 +export INPES_cpl_atmw=3 +export JNPES_cpl_atmw=8 +export WPG_cpl_atmw=6 +export WAV_tasks_cpl_atmw=30 +export WAV_thrds_cpl_atmw=1 + +export THRD_cpl_c48=1 +export INPES_cpl_c48=1 +export JNPES_cpl_c48=1 +export WPG_cpl_c48=6 +export OCN_tasks_cpl_c48=4 +export ICE_tasks_cpl_c48=4 + +export THRD_cpl_dflt=1 +export INPES_cpl_dflt=3 +export JNPES_cpl_dflt=8; +export WPG_cpl_dflt=6 +export OCN_tasks_cpl_dflt=20 +export ICE_tasks_cpl_dflt=10 +export WAV_tasks_cpl_dflt=20 + +export THRD_cpl_thrd=2 +export INPES_cpl_thrd=3 +export JNPES_cpl_thrd=4 +export WPG_cpl_thrd=6 +export OCN_tasks_cpl_thrd=20 +export OCN_thrds_cpl_thrd=1 +export ICE_tasks_cpl_thrd=10 +export ICE_thrds_cpl_thrd=1 +export WAV_tasks_cpl_thrd=12 +export WAV_thrds_cpl_thrd=2 + +export THRD_cpl_dcmp=1 +export INPES_cpl_dcmp=4 +export JNPES_cpl_dcmp=6 +export WPG_cpl_dcmp=6 +export OCN_tasks_cpl_dcmp=20 +export ICE_tasks_cpl_dcmp=10 +export WAV_tasks_cpl_dcmp=20 + +export THRD_cpl_mpi=1 +export INPES_cpl_mpi=4 +export JNPES_cpl_mpi=8 +export WPG_cpl_mpi=6 +export OCN_tasks_cpl_mpi=34 +export ICE_tasks_cpl_mpi=20 +export WAV_tasks_cpl_mpi=28 + +export THRD_cpl_bmrk=2 +export INPES_cpl_bmrk=8 +export JNPES_cpl_bmrk=8 +export WPG_cpl_bmrk=48 +export OCN_tasks_cpl_bmrk=120 +export OCN_thrds_cpl_bmrk=1 +export ICE_tasks_cpl_bmrk=48 +export ICE_thrds_cpl_bmrk=1 +export WAV_tasks_cpl_bmrk=80 +export WAV_thrds_cpl_bmrk=2 + +export THRD_cpl_c192=2 +export INPES_cpl_c192=6 +export JNPES_cpl_c192=8 +export WPG_cpl_c192=12 +export OCN_tasks_cpl_c192=60 +export ICE_tasks_cpl_c192=24 +export WAV_tasks_cpl_c192=80 + +export ATM_compute_tasks_cdeps_100=12 +export OCN_tasks_cdeps_100=16 +export ICE_tasks_cdeps_100=12 + +export ATM_compute_tasks_cdeps_025=40 +export OCN_tasks_cdeps_025=120 +export ICE_tasks_cdeps_025=48 + +export INPES_aqm=33 +export JNPES_aqm=8 + +export THRD_cpl_unstr=1 +export INPES_cpl_unstr=3 +export JNPES_cpl_unstr=8 +export WPG_cpl_unstr=6 +export OCN_tasks_cpl_unstr=20 +export ICE_tasks_cpl_unstr=10 +export WAV_tasks_cpl_unstr=60 + +export THRD_cpl_unstr_mpi=1 +export INPES_cpl_unstr_mpi=4 +export JNPES_cpl_unstr_mpi=8 +export WPG_cpl_unstr_mpi=6 +export OCN_tasks_cpl_unstr_mpi=34 +export ICE_tasks_cpl_unstr_mpi=20 +export WAV_tasks_cpl_unstr_mpi=50 + +export aqm_omp_num_threads=1 +export atm_omp_num_threads=1 +export chm_omp_num_threads=1 +export ice_omp_num_threads=1 +export lnd_omp_num_threads=1 +export med_omp_num_threads=1 +export ocn_omp_num_threads=1 +export wav_omp_num_threads=1 +export fbh_omp_num_threads=1 + +export histaux_enabled=.false. if [[ ${MACHINE_ID} = wcoss2 || ${MACHINE_ID} = acorn ]]; then diff --git a/tests/parm/ufs.configure.s2s.IN b/tests/parm/ufs.configure.s2s.IN index 17ccc678c9..f70b837c7a 100644 --- a/tests/parm/ufs.configure.s2s.IN +++ b/tests/parm/ufs.configure.s2s.IN @@ -99,14 +99,14 @@ MED_attributes:: ocean_albedo_limit = @[ocean_albedo_limit] histaux_ice2med_file1_auxname = ice.1h.aux histaux_ice2med_file1_doavg = .false. - histaux_ice2med_file1_enabled = .true. + histaux_ice2med_file1_enabled = @[histaux_enabled] histaux_ice2med_file1_flds = all histaux_ice2med_file1_history_n = @[MED_history_n] histaux_ice2med_file1_history_option = nhours histaux_ice2med_file1_ntperfile = 9999 histaux_ocn2med_file1_auxname = ocn.1h.aux histaux_ocn2med_file1_doavg = .false. - histaux_ocn2med_file1_enabled = .true. + histaux_ocn2med_file1_enabled = @[histaux_enabled] histaux_ocn2med_file1_flds = all histaux_ocn2med_file1_history_n = @[MED_history_n] histaux_ocn2med_file1_history_option = nhours diff --git a/tests/tests/cpld_control_nowave_noaero_p8 b/tests/tests/cpld_control_nowave_noaero_p8 index 031e48c98f..4599c895b3 100644 --- a/tests/tests/cpld_control_nowave_noaero_p8 +++ b/tests/tests/cpld_control_nowave_noaero_p8 @@ -114,3 +114,5 @@ export N_SPLIT=5 # HR4 GFSv17 GWD update export DO_GWD_OPT_PSL=.true. export DO_GSL_DRAG_SS=.false. + +export histaux_enabled=.true.