From b411c6cde73ade61872c2169a7f9f05525bf1d60 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 18 Jul 2024 20:18:31 +0000 Subject: [PATCH 01/15] Create feature/mpas-dycore branch --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index a3775fc12d..937184f7f2 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "FV3"] path = FV3 - url = https://github.com/NOAA-EMC/fv3atm - branch = develop + url = https://github.com/NCAR/fv3atm + branch = feature/mpas-dycore [submodule "WW3"] path = WW3 url = https://github.com/NOAA-EMC/WW3 From 82dbe8cb063d1ff6e12a599a987a8f08150e98ae Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 19 Jul 2024 17:31:46 +0000 Subject: [PATCH 02/15] Reorganize code for MPAS dycore. Isolate FV3 and MPAS pieces. --- CMakeLists.txt | 24 ++++++++++++++++++++---- FV3 | 2 +- driver/UFSDriver.F90 | 11 +++++++++++ 3 files changed, 32 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e5fdd1e83a..f02f312a3e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -23,7 +23,8 @@ if(NOT (APP IN_LIST VALID_APPS)) endif() set(FMS OFF CACHE BOOL "Enable FMS") -set(FV3 OFF CACHE BOOL "Enable FV3") +set(FV3 OFF CACHE BOOL "Enable FV3 dycore") +set(MPAS OFF CACHE BOOL "Enable MPAS dycore") set(AQM OFF CACHE BOOL "Enable AQM") set(UFS_GOCART OFF CACHE BOOL "Enable GOCART") set(MOM6 OFF CACHE BOOL "Enable MOM6") @@ -41,7 +42,8 @@ include(cmake/configure_apps.cmake) message("") message("FMS .............. ${FMS}") -message("FV3 .............. ${FV3}") +message("FV3_dy ........... ${FV3}") +message("MPAS_dy .......... ${MPAS}") message("AQM .............. ${AQM}") message("GOCART ........... ${UFS_GOCART}") message("MOM6 ............. ${MOM6}") @@ -185,10 +187,18 @@ endif() ############################################################################### ### Atmosphere Components [FV3, MPAS?] +### DJS2024: There will be a generalized atmospheric component with multiple +### dycores, not multiple atmospheric components (one for each dycore). +### So, - rename the atmospheric component FV3 to UFSATM, then +### - if FV3 or MPAS, add_sudirectory(UFSATM) +### ############################################################################### if(FV3) add_subdirectory(FV3) endif() +if (MPAS) + add_subdirectory(FV3) +endif() ############################################################################### ### AQM @@ -296,9 +306,15 @@ if(STOCH_PHYS) endif() if(FV3) - add_dependencies(ufs fv3atm) + add_dependencies(ufs ufsatm) list(APPEND _ufs_defs_private FRONT_FV3=fv3atm_cap_mod) - list(APPEND _ufs_libs_public fv3atm) + list(APPEND _ufs_libs_public ufsatm) +endif() + +if(MPAS) + add_dependencies(ufs ufsatm) + list(APPEND _ufs_defs_private FRONT_MPAS=mpasatm_cap_mod) + list(APPEND _ufs_libs_public ufsatm) endif() if(MOM6) diff --git a/FV3 b/FV3 index 3581423f39..eb63f9fb2b 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 3581423f397e2eb0c1f0cd7f6b728593511278a9 +Subproject commit eb63f9fb2b2c59ae74055a262f447887a047fb99 diff --git a/driver/UFSDriver.F90 b/driver/UFSDriver.F90 index 53c51d818b..9ad721618d 100644 --- a/driver/UFSDriver.F90 +++ b/driver/UFSDriver.F90 @@ -48,6 +48,9 @@ MODULE UFSDriver #ifdef FRONT_FV3 use FRONT_FV3, only: FV3_SS => SetServices #endif +#ifdef FRONT_MPAS + use FRONT_MPAS, only: MPAS_SS => SetServices +#endif #ifdef FRONT_CDEPS_DATM use FRONT_CDEPS_DATM, only: DATM_SS => SetServices #endif @@ -363,6 +366,14 @@ subroutine SetModelServices(driver, rc) found_comp = .true. end if #endif +#ifdef FRONT_MPAS + if (trim(model) == "mpas") then + call NUOPC_DriverAddComp(driver, trim(prefix), MPAS_SS, & + info=info, petList=petList, comp=comp, rc=rc) + if (ChkErr(rc,__LINE__,u_FILE_u)) return + found_comp = .true. + end if +#endif #if defined FRONT_CDEPS_DATM if (trim(model) == "datm" ) then !TODO: Remove bail code and pass info and SetVM to DriverAddComp From 4b3b5e37621b242b64cd12671be54c8cc24344b9 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 19 Jul 2024 17:32:18 +0000 Subject: [PATCH 03/15] Update submodules --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 937184f7f2..f533ca3a2e 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "FV3"] path = FV3 - url = https://github.com/NCAR/fv3atm - branch = feature/mpas-dycore + url = https://github.com/dustinswales/fv3atm + branch = feature/mpas-dycore-reorg [submodule "WW3"] path = WW3 url = https://github.com/NOAA-EMC/WW3 From b01ced78506c3b52370831a14949b2cf67994f81 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 19 Jul 2024 19:43:44 +0000 Subject: [PATCH 04/15] Added more build step pieces, up until the MPAS dycore build --- CMakeLists.txt | 4 ++-- FV3 | 2 +- cmake/configure_apps.cmake | 5 ++++- 3 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f02f312a3e..3898c6fd18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/CMakeModules/Modules) ############################################################################### # Valid applications and choices -list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS) +list(APPEND VALID_APPS ATM ATMAERO ATMAQ ATMW ATMWM ATML ATMMPAS LND S2S S2SA S2SW S2SWA S2SWAL ATM_DS2S ATM_DS2S-PCICE HAFS HAFSW HAFS-MOM6 HAFS-MOM6W HAFS-ALL NG-GODAS) set(APP NONE CACHE BOOL "Application Name") if(NOT (APP IN_LIST VALID_APPS)) message(FATAL_ERROR "${APP} is not a valid application.\nValid Applications are: ${VALID_APPS}") @@ -157,7 +157,7 @@ if(FMS) elseif (APP MATCHES "^(S2S|S2SA|S2SW|S2SWA|S2SWAL|ATM_DS2S|ATM_DS2S-PCICE|NG-GODAS|HAFS-MOM6|HAFS-MOM6W)$") add_library(fms ALIAS FMS::fms_r8) endif() - if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|HAFS|HAFS-ALL)$") + if(APP MATCHES "^(ATM|ATMAERO|ATMAQ|ATMWM|ATMW|ATML|ATMMPAS|HAFS|HAFS-ALL)$") if(32BIT) add_library(fms ALIAS FMS::fms_r4) else() diff --git a/FV3 b/FV3 index eb63f9fb2b..1301e07acd 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit eb63f9fb2b2c59ae74055a262f447887a047fb99 +Subproject commit 1301e07acdbedc306e0c0ca96649c35c785ef44b diff --git a/cmake/configure_apps.cmake b/cmake/configure_apps.cmake index b6db3a5759..a61c3ff250 100644 --- a/cmake/configure_apps.cmake +++ b/cmake/configure_apps.cmake @@ -13,7 +13,7 @@ ############################################################################### ### Configure Application Components ############################################################################### -if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML)$") +if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML|ATMMPAS)$") set(FMS ON CACHE BOOL "Enable FMS" FORCE) set(FV3 ON CACHE BOOL "Enable FV3" FORCE) set(STOCH_PHYS ON CACHE BOOL "Enable Stochastic Physics" FORCE) @@ -31,6 +31,9 @@ if(APP MATCHES "^(ATM|ATMW|ATMWM|ATMAQ|ATML)$") set(CMEPS ON CACHE BOOL "Enable CMEPS" FORCE) set(NOAHMP ON CACHE BOOL "Enable NOAHMP" FORCE) message("Configuring UFS app in Atmosphere with Air Quality mode") + elseif(APP MATCHES "ATMMPAS") + set(MPAS ON CACHE BOOL "Enable MPAS dycore" FORCE) + message("Configuring UFS app in Atmosphere with MPAS dycore") else() message("Configuring UFS app in Atmosphere Only mode") endif() From 9e82655af3646b5b3d5e25e92c9f4a8fde31136d Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 30 Jul 2024 22:13:03 +0000 Subject: [PATCH 05/15] Some more changes. Almost building. --- CMakeLists.txt | 11 ++++------- FV3 | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3898c6fd18..5387962ed1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -196,9 +196,6 @@ endif() if(FV3) add_subdirectory(FV3) endif() -if (MPAS) - add_subdirectory(FV3) -endif() ############################################################################### ### AQM @@ -306,15 +303,15 @@ if(STOCH_PHYS) endif() if(FV3) - add_dependencies(ufs ufsatm) + add_dependencies(ufs ufsatm_fv3) list(APPEND _ufs_defs_private FRONT_FV3=fv3atm_cap_mod) - list(APPEND _ufs_libs_public ufsatm) + list(APPEND _ufs_libs_public ufsatm_fv3) endif() if(MPAS) - add_dependencies(ufs ufsatm) + add_dependencies(ufs ufsatm_mpas) list(APPEND _ufs_defs_private FRONT_MPAS=mpasatm_cap_mod) - list(APPEND _ufs_libs_public ufsatm) + list(APPEND _ufs_libs_public ufsatm_mpas) endif() if(MOM6) diff --git a/FV3 b/FV3 index 1301e07acd..587ea55925 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 1301e07acdbedc306e0c0ca96649c35c785ef44b +Subproject commit 587ea55925d09bef2b4e8aed296b70ad8e97d2c8 From f0c3a780c1a443d04cf37e4459adcb54d9d07cbf Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Thu, 1 Aug 2024 21:00:03 +0000 Subject: [PATCH 06/15] MPAS building in UFSATM! Some more reorganization. Initial infrastructure for CCPP. More CMake modifications --- CMakeLists.txt | 13 +++++++++++-- FV3 | 2 +- driver/UFSDriver.F90 | 12 ++++++------ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5387962ed1..5d0b3be19d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -174,6 +174,12 @@ find_package(bacio 2.4.0 REQUIRED) find_package(sp 2.3.3 REQUIRED) find_package(w3emc 2.9.2 REQUIRED) +if(MPAS) + find_package(PnetCDF REQUIRED COMPONENTS Fortran) + find_package(PIO REQUIRED COMPONENTS Fortran C) + set(FV3 OFF) +endif() + # Configure Python find_package(Python 3.6 REQUIRED COMPONENTS Interpreter) message("Found Python: ${Python_EXECUTABLE}") @@ -189,13 +195,16 @@ endif() ### Atmosphere Components [FV3, MPAS?] ### DJS2024: There will be a generalized atmospheric component with multiple ### dycores, not multiple atmospheric components (one for each dycore). -### So, - rename the atmospheric component FV3 to UFSATM, then -### - if FV3 or MPAS, add_sudirectory(UFSATM) +### So, - rename the atmospheric component FV3 to UFSATM or ATM, then +### - if FV3 or MPAS, add_sudirectory(UFSATM or ATM) ### ############################################################################### if(FV3) add_subdirectory(FV3) endif() +if (MPAS) + add_subdirectory(FV3) +endif() ############################################################################### ### AQM diff --git a/FV3 b/FV3 index 587ea55925..b4f6463fa4 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 587ea55925d09bef2b4e8aed296b70ad8e97d2c8 +Subproject commit b4f6463fa403b39fd80c51378c3aa1e42e466f21 diff --git a/driver/UFSDriver.F90 b/driver/UFSDriver.F90 index 9ad721618d..c9fb8f2ca9 100644 --- a/driver/UFSDriver.F90 +++ b/driver/UFSDriver.F90 @@ -367,12 +367,12 @@ subroutine SetModelServices(driver, rc) end if #endif #ifdef FRONT_MPAS - if (trim(model) == "mpas") then - call NUOPC_DriverAddComp(driver, trim(prefix), MPAS_SS, & - info=info, petList=petList, comp=comp, rc=rc) - if (ChkErr(rc,__LINE__,u_FILE_u)) return - found_comp = .true. - end if +! if (trim(model) == "mpas") then +! call NUOPC_DriverAddComp(driver, trim(prefix), MPAS_SS, & +! info=info, petList=petList, comp=comp, rc=rc) +! if (ChkErr(rc,__LINE__,u_FILE_u)) return +! found_comp = .true. +! end if #endif #if defined FRONT_CDEPS_DATM if (trim(model) == "datm" ) then From fe47b338ba40fed9d8572bf0a26e463e1cb8f185 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 7 Aug 2024 16:49:01 +0000 Subject: [PATCH 07/15] Forgot to link to personal for of MPAS submodule --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index b4f6463fa4..8168a66d9d 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit b4f6463fa403b39fd80c51378c3aa1e42e466f21 +Subproject commit 8168a66d9da8e45a62b1cea452d37ac2da3c928b From f26d5380643ebabf5d4665a5a02e28c467760748 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 13 Aug 2024 15:57:45 +0000 Subject: [PATCH 08/15] Address reviewers comments --- FV3 | 2 +- tests/rt.conf | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/FV3 b/FV3 index 8168a66d9d..b56c4863de 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 8168a66d9da8e45a62b1cea452d37ac2da3c928b +Subproject commit b56c4863de299415d234ed2221632822ceb40bcb diff --git a/tests/rt.conf b/tests/rt.conf index 2a644b651d..dd7f7dd8cd 100644 --- a/tests/rt.conf +++ b/tests/rt.conf @@ -434,3 +434,6 @@ RUN | cpld_debug_pdlib_p8 | + hera hercules ### CDEPS Data Atmosphere test ### COMPILE | datm_cdeps | gnu | -DAPP=NG-GODAS | + hera hercules | fv3 | RUN | datm_cdeps_control_cfsr | + hera hercules | baseline | + +# UFS with MPAS dynamical core +COMPILE | atm_mpas_dyn32 | intel | -DAPP=ATMMPAS -DCCPP_SUITES=FV3_GFS_v16,FV3_GFS_v17_p8_ugwpv1 -D32BIT=ON | | mpas | \ No newline at end of file From 4c2f9005ad86ba61d3f53191c28a745398e61e6c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 13 Aug 2024 17:51:30 +0000 Subject: [PATCH 09/15] Address comments --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index b56c4863de..64afb902eb 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit b56c4863de299415d234ed2221632822ceb40bcb +Subproject commit 64afb902eb2ae364bc6679d60253012898160188 From 4952814d85ae2a9a092c80c60e5f647f15afff24 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Fri, 23 Aug 2024 15:38:07 +0000 Subject: [PATCH 10/15] Add MPAS to default_vars.sh --- tests/default_vars.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/default_vars.sh b/tests/default_vars.sh index ef44dfe4a9..493e847298 100644 --- a/tests/default_vars.sh +++ b/tests/default_vars.sh @@ -494,6 +494,7 @@ export MODEL_CONFIGURE=model_configure.IN export atm_model=fv3 export FV3=true +export MPAS=true export S2S=false export HAFS=false export AQM=false @@ -1065,6 +1066,7 @@ export_cmeps() { export_cpl () { export FV3=true +export MPAS=false export S2S=true export HAFS=false export AQM=false @@ -1251,6 +1253,7 @@ export_35d_run () export_datm_cdeps () { export FV3=false + export MPAS=false export S2S=false export HAFS=false export AQM=false @@ -1328,6 +1331,7 @@ export_datm_cdeps () export_hafs_datm_cdeps () { export FV3=false + export MPAS=false export S2S=false export HAFS=true export AQM=false @@ -1347,6 +1351,7 @@ export_hafs_datm_cdeps () export_hafs_docn_cdeps () { export FV3=true + export MPAS=false export S2S=false export HAFS=true export AQM=false @@ -1366,6 +1371,7 @@ export_hafs_docn_cdeps () export_hafs_regional () { export FV3=true + export MPAS=false export S2S=false export HAFS=true export AQM=false @@ -1439,6 +1445,7 @@ export_hafs () { export_fv3_v16 export FV3=true +export MPAS=false export S2S=false export HAFS=true export AQM=false From d6418b41ad32d634b40961dd368e82013b0d649c Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 3 Sep 2024 16:35:47 +0000 Subject: [PATCH 11/15] Move MPAS to ufs-community reporitory --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index 64afb902eb..5dec7c2c49 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 64afb902eb2ae364bc6679d60253012898160188 +Subproject commit 5dec7c2c49225ec1ec368c39ed4d06dfdfa4c5fc From 7d0e826d74e60874274f5fb7a4e412dc4b3ba032 Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Tue, 3 Sep 2024 17:20:21 +0000 Subject: [PATCH 12/15] Revert unneeded change --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index 5dec7c2c49..1b2c665998 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 5dec7c2c49225ec1ec368c39ed4d06dfdfa4c5fc +Subproject commit 1b2c6659987b04f7b823d17a416fd65fa59889ba From ee3323122b0391acd919512276621a56b6c9868f Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 2 Oct 2024 01:35:58 +0000 Subject: [PATCH 13/15] Synced FV3 --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index 1b2c665998..afb09b5be4 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit 1b2c6659987b04f7b823d17a416fd65fa59889ba +Subproject commit afb09b5be4b64b1f390671350189d76b7182444a From 55f4f66d4d66fa388e91f959aae3d7a44b75c3fa Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Wed, 2 Oct 2024 15:34:55 +0000 Subject: [PATCH 14/15] Updated FV3 submodule hash --- FV3 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/FV3 b/FV3 index afb09b5be4..f5712486a2 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit afb09b5be4b64b1f390671350189d76b7182444a +Subproject commit f5712486a2090beab2c9bb33a80c4ec703db749e From d08975fa4f721b5b4781fe4f200c43faa186935f Mon Sep 17 00:00:00 2001 From: Dustin Swales Date: Mon, 7 Oct 2024 15:01:21 +0000 Subject: [PATCH 15/15] Housekeeping --- .gitmodules | 4 ++-- FV3 | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitmodules b/.gitmodules index 7a97520069..8c87a67a6c 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,7 +1,7 @@ [submodule "FV3"] path = FV3 - url = https://github.com/dustinswales/fv3atm - branch = feature/mpas-dycore-reorg + url = https://github.com/NCAR/fv3atm + branch = feature/mpas-dycore [submodule "WW3"] path = WW3 url = https://github.com/NOAA-EMC/WW3 diff --git a/FV3 b/FV3 index f5712486a2..fe342d0391 160000 --- a/FV3 +++ b/FV3 @@ -1 +1 @@ -Subproject commit f5712486a2090beab2c9bb33a80c4ec703db749e +Subproject commit fe342d039198a92138ac5de157f31c23f26b5863