Mods to allow user control of collective reads from input.nml. Defaul…

…t behavior is unchanged, user has to activate collective reads

fms2_io/fms2_io.F90

@@ -386,8 +386,11 @@ module fms2_io_mod
 integer :: deflate_level = default_deflate_level !< Netcdf deflate level to use in nf90_def_var
                                                  !! (integer between 1 to 9)
 logical :: shuffle = .false. !< Flag indicating whether to use the netcdf shuffle filter
+
+integer :: num_collective=0 !< The number of files targetted for MPIIO collective treatment
+
 namelist / fms2_io_nml / &
-                      ncchksz, netcdf_default_format, header_buffer_val, deflate_level, shuffle
+                      ncchksz, netcdf_default_format, header_buffer_val, deflate_level, shuffle, num_collective
 
 contains
 
@@ -413,7 +416,12 @@ subroutine fms2_io_init ()
     call mpp_error(FATAL, &
       "deflate_level in fms2_io_nml must be a positive number between 1 and 9 as it is required by NetCDF")
   endif
-  call netcdf_io_init (ncchksz,header_buffer_val,netcdf_default_format, deflate_level, shuffle)
+  if (num_collective .lt. 0) then
+    call mpp_error(FATAL, &
+      "num_collective in fms2_io_nml must be a positive number (0+)")
+  endif
+
+  call netcdf_io_init (ncchksz,header_buffer_val,netcdf_default_format, deflate_level, shuffle, num_collective)
   call blackboxio_init (ncchksz)
 !> Mark the fms2_io as initialized
   fms2_io_is_initialized = .true.

fms2_io/include/netcdf_read_data.inc

@@ -355,19 +355,18 @@ subroutine netcdf_read_data_2d(fileobj, variable_name, buf, unlim_dim_level, &
     c(unlim_dim_index) = unlim_dim_level
   endif
   if (fileobj%is_root) then
+    !write(6,'("netcdf_read_data_2d:"3A)') trim(fileobj%path), '   ', trim(variable_name)
     varid = get_variable_id(fileobj%ncid, trim(variable_name), msg=append_error_msg)
     select type(buf)
       type is (integer(kind=i4_kind))
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       type is (integer(kind=i8_kind))
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       type is (real(kind=r4_kind))
+        if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       type is (real(kind=r8_kind))
-         if(string_compare(trim(fileobj%path), "INPUT/C3463_grid.tile7.nc", .true.) .or. &
-           string_compare(trim(fileobj%path), "INPUT/sfc_data.nc"        , .true.) ) then
-          err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
-        endif
+        if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       class default
         call error("Unsupported variable type: "//trim(append_error_msg))
@@ -451,23 +450,18 @@ subroutine netcdf_read_data_3d(fileobj, variable_name, buf, unlim_dim_level, &
     c(unlim_dim_index) = unlim_dim_level
   endif
   if (fileobj%is_root) then
+    !write(6,'("netcdf_read_data_3d:"3A)') trim(fileobj%path),'  ',trim(variable_name)
     varid = get_variable_id(fileobj%ncid, trim(variable_name), msg=append_error_msg)
     select type(buf)
       type is (integer(kind=i4_kind))
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       type is (integer(kind=i8_kind))
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       type is (real(kind=r4_kind))
-        if(string_compare(trim(fileobj%path), "INPUT/fv_tracer.res.tile1.nc"   , .true.) .or. &
-           string_compare(trim(fileobj%path), "INPUT/fv_core.res.tile1.nc"  , .true.) ) then
-          err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
-        endif
+        if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       type is (real(kind=r8_kind))
-        if(string_compare(trim(fileobj%path), "INPUT/phy_data.nc"   , .true.) .or. &
-           string_compare(trim(fileobj%path), "INPUT/sfc_data.nc", .true.) ) then
-          err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
-        endif
+        if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective)
         err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e)
       class default
         call error("Unsupported variable type: "//trim(append_error_msg))

fms2_io/netcdf_io.F90

@@ -61,6 +61,10 @@ module netcdf_io_mod
 logical, private :: fms2_shuffle = .false. !< Flag indicating whether to use the netcdf shuffle filter
 logical, private :: fms2_is_netcdf4 = .false. !< Flag indicating whether the default netcdf file format is netcdf4
 
+integer :: fms2_num_collective=0 !< The number of files targetted for MPIIO collective treatment
+character (len=NF90_MAX_NAME), allocatable :: fn_collective(:)
+namelist / fms2_io_collective_nml / fn_collective
+
 !> @}
 
 !> @brief information needed fr regional restart variables
@@ -150,6 +154,7 @@ module netcdf_io_mod
   character (len=20) :: time_name
   type(dimension_information) :: bc_dimensions !<information about the current dimensions for regional
                                                !! restart variables
+  logical :: use_collective = .false. !< Flag telling if we should open the file for collective input
 
 endtype FmsNetcdfFile_t
 
@@ -338,19 +343,23 @@ module netcdf_io_mod
 contains
 
 !> @brief Accepts the namelist fms2_io_nml variables relevant to netcdf_io_mod
-subroutine netcdf_io_init (chksz, header_buffer_val, netcdf_default_format, deflate_level, shuffle)
+subroutine netcdf_io_init (chksz, header_buffer_val, netcdf_default_format, deflate_level, shuffle, num_collective)
 integer,              intent(in) :: chksz                 !< Chunksize (bytes) used in nc_open and nc_create
 character (len = 10), intent(in) :: netcdf_default_format !< Netcdf format type param used in nc_create
 integer,              intent(in) :: header_buffer_val     !< Value used in NF__ENDDEF
 integer,              intent(in) :: deflate_level         !< Netcdf deflate level to use in nf90_def_var
                                                           !! (integer between 1 to 9)
 logical,              intent(in) :: shuffle               !< Flag indicating whether to use the netcdf shuffle filter
+integer,              intent(in) :: num_collective        !< The number of files targetted for MPIIO collective treatment
+integer :: mystat
 
  fms2_ncchksz = chksz
  fms2_deflate_level = deflate_level
  fms2_shuffle = shuffle
  fms2_is_netcdf4 = .false.
  fms2_header_buffer_val = header_buffer_val
+ fms2_num_collective = num_collective
+
  if (string_compare(netcdf_default_format, "64bit", .true.)) then
      fms2_nc_format_param = nf90_64bit_offset
      call string_copy(fms2_nc_format, "64bit")
@@ -366,6 +375,12 @@ subroutine netcdf_io_init (chksz, header_buffer_val, netcdf_default_format, defl
      '. The acceptable values are "64bit", "classic", "netcdf4". Check fms2_io_nml: netcdf_default_format')
  endif
 
+ if (fms2_num_collective .gt. 0) then
+   allocate(fn_collective(fms2_num_collective))
+   READ (input_nml_file, NML=fms2_io_collective_nml, IOSTAT=mystat)
+   !print*,'netcdf_io_init: ',fn_collective
+ endif
+
 end subroutine netcdf_io_init
 
 !> @brief Check for errors returned by netcdf.
@@ -562,7 +577,7 @@ function netcdf_file_open(fileobj, path, mode, nc_format, pelist, is_restart, do
   logical :: success
 
   integer :: nc_format_param
-  integer :: err
+  integer :: i,err
   character(len=256) :: buf !< Filename with .res in the filename if it is a restart
   character(len=256) :: buf2 !< Filename with the filename appendix if there is one
   logical :: is_res
@@ -645,12 +660,11 @@ function netcdf_file_open(fileobj, path, mode, nc_format, pelist, is_restart, do
     endif
 
     if (string_compare(mode, "read", .true.)) then
-      if(string_compare(trim(fileobj%path), "INPUT/phy_data.nc"           , .true.) .or. &
-         string_compare(trim(fileobj%path), "INPUT/fv_tracer.res.tile1.nc", .true.) .or. &
-         string_compare(trim(fileobj%path), "INPUT/sfc_data.nc"              , .true.) .or. &
-         string_compare(trim(fileobj%path), "INPUT/C3463_grid.tile7.nc"      , .true.) .or. &
-         string_compare(trim(fileobj%path), "INPUT/C3463_grid.tile7.halo3.nc", .true.) .or. &
-         string_compare(trim(fileobj%path), "INPUT/fv_core.res.tile1.nc"  , .true.) ) then
+      !print*,'netcdf_file_open: ',trim(fileobj%path)
+      do i=1,fms2_num_collective
+        if(string_compare(trim(fileobj%path), trim(fn_collective(i)), .true.)) fileobj%use_collective = .true.
+      enddo
+      if(fileobj%use_collective) then
         err = nf90_open(trim(fileobj%path), ior(NF90_NOWRITE, NF90_MPIIO), fileobj%ncid, comm=mpp_comm_private, info=MPI_INFO_NULL)
       else
         err = nf90_open(trim(fileobj%path), nf90_nowrite, fileobj%ncid, chunksize=fms2_ncchksz)

mpp/mpp.F90

@@ -1266,7 +1266,7 @@ module mpp_mod
   logical              :: debug = .false.
   integer              :: npes=1, root_pe=0, pe=0
   integer(i8_kind)     :: tick, ticks_per_sec, max_ticks, start_tick, end_tick, tick0=0
-  integer              :: mpp_comm_private
+  integer,public       :: mpp_comm_private
   logical              :: first_call_system_clock_mpi=.TRUE.
   real(r8_kind)        :: mpi_count0=0  !< use to prevent integer overflow
   real(r8_kind)        :: mpi_tick_rate=0.d0  !< clock rate for mpi_wtick()