diff --git a/fms2_io/fms2_io.F90 b/fms2_io/fms2_io.F90 index 00f6046a67..dd055f526a 100644 --- a/fms2_io/fms2_io.F90 +++ b/fms2_io/fms2_io.F90 @@ -386,8 +386,11 @@ module fms2_io_mod integer :: deflate_level = default_deflate_level !< Netcdf deflate level to use in nf90_def_var !! (integer between 1 to 9) logical :: shuffle = .false. !< Flag indicating whether to use the netcdf shuffle filter + +integer :: num_collective=0 !< The number of files targetted for MPIIO collective treatment + namelist / fms2_io_nml / & - ncchksz, netcdf_default_format, header_buffer_val, deflate_level, shuffle + ncchksz, netcdf_default_format, header_buffer_val, deflate_level, shuffle, num_collective contains @@ -413,7 +416,12 @@ subroutine fms2_io_init () call mpp_error(FATAL, & "deflate_level in fms2_io_nml must be a positive number between 1 and 9 as it is required by NetCDF") endif - call netcdf_io_init (ncchksz,header_buffer_val,netcdf_default_format, deflate_level, shuffle) + if (num_collective .lt. 0) then + call mpp_error(FATAL, & + "num_collective in fms2_io_nml must be a positive number (0+)") + endif + + call netcdf_io_init (ncchksz,header_buffer_val,netcdf_default_format, deflate_level, shuffle, num_collective) call blackboxio_init (ncchksz) !> Mark the fms2_io as initialized fms2_io_is_initialized = .true. diff --git a/fms2_io/include/netcdf_read_data.inc b/fms2_io/include/netcdf_read_data.inc index af91389f57..a365365599 100644 --- a/fms2_io/include/netcdf_read_data.inc +++ b/fms2_io/include/netcdf_read_data.inc @@ -355,6 +355,7 @@ subroutine netcdf_read_data_2d(fileobj, variable_name, buf, unlim_dim_level, & c(unlim_dim_index) = unlim_dim_level endif if (fileobj%is_root) then + !write(6,'("netcdf_read_data_2d:"3A)') trim(fileobj%path), ' ', trim(variable_name) varid = get_variable_id(fileobj%ncid, trim(variable_name), msg=append_error_msg) select type(buf) type is (integer(kind=i4_kind)) @@ -362,12 +363,10 @@ subroutine netcdf_read_data_2d(fileobj, variable_name, buf, unlim_dim_level, & type is (integer(kind=i8_kind)) err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e) type is (real(kind=r4_kind)) + if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e) type is (real(kind=r8_kind)) - if(string_compare(trim(fileobj%path), "INPUT/C3463_grid.tile7.nc", .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/sfc_data.nc" , .true.) ) then - err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) - endif + if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e) class default call error("Unsupported variable type: "//trim(append_error_msg)) @@ -451,6 +450,7 @@ subroutine netcdf_read_data_3d(fileobj, variable_name, buf, unlim_dim_level, & c(unlim_dim_index) = unlim_dim_level endif if (fileobj%is_root) then + !write(6,'("netcdf_read_data_3d:"3A)') trim(fileobj%path),' ',trim(variable_name) varid = get_variable_id(fileobj%ncid, trim(variable_name), msg=append_error_msg) select type(buf) type is (integer(kind=i4_kind)) @@ -458,16 +458,10 @@ subroutine netcdf_read_data_3d(fileobj, variable_name, buf, unlim_dim_level, & type is (integer(kind=i8_kind)) err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e) type is (real(kind=r4_kind)) - if(string_compare(trim(fileobj%path), "INPUT/fv_tracer.res.tile1.nc" , .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/fv_core.res.tile1.nc" , .true.) ) then - err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) - endif + if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e) type is (real(kind=r8_kind)) - if(string_compare(trim(fileobj%path), "INPUT/phy_data.nc" , .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/sfc_data.nc", .true.) ) then - err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) - endif + if(fileobj%use_collective) err = nf90_var_par_access(fileobj%ncid, varid, nf90_collective) err = nf90_get_var(fileobj%ncid, varid, buf, start=c, count=e) class default call error("Unsupported variable type: "//trim(append_error_msg)) diff --git a/fms2_io/netcdf_io.F90 b/fms2_io/netcdf_io.F90 index 25839c9bad..a78a52bd7b 100644 --- a/fms2_io/netcdf_io.F90 +++ b/fms2_io/netcdf_io.F90 @@ -61,6 +61,10 @@ module netcdf_io_mod logical, private :: fms2_shuffle = .false. !< Flag indicating whether to use the netcdf shuffle filter logical, private :: fms2_is_netcdf4 = .false. !< Flag indicating whether the default netcdf file format is netcdf4 +integer :: fms2_num_collective=0 !< The number of files targetted for MPIIO collective treatment +character (len=NF90_MAX_NAME), allocatable :: fn_collective(:) +namelist / fms2_io_collective_nml / fn_collective + !> @} !> @brief information needed fr regional restart variables @@ -150,6 +154,7 @@ module netcdf_io_mod character (len=20) :: time_name type(dimension_information) :: bc_dimensions ! @brief Accepts the namelist fms2_io_nml variables relevant to netcdf_io_mod -subroutine netcdf_io_init (chksz, header_buffer_val, netcdf_default_format, deflate_level, shuffle) +subroutine netcdf_io_init (chksz, header_buffer_val, netcdf_default_format, deflate_level, shuffle, num_collective) integer, intent(in) :: chksz !< Chunksize (bytes) used in nc_open and nc_create character (len = 10), intent(in) :: netcdf_default_format !< Netcdf format type param used in nc_create integer, intent(in) :: header_buffer_val !< Value used in NF__ENDDEF integer, intent(in) :: deflate_level !< Netcdf deflate level to use in nf90_def_var !! (integer between 1 to 9) logical, intent(in) :: shuffle !< Flag indicating whether to use the netcdf shuffle filter +integer, intent(in) :: num_collective !< The number of files targetted for MPIIO collective treatment +integer :: mystat fms2_ncchksz = chksz fms2_deflate_level = deflate_level fms2_shuffle = shuffle fms2_is_netcdf4 = .false. fms2_header_buffer_val = header_buffer_val + fms2_num_collective = num_collective + if (string_compare(netcdf_default_format, "64bit", .true.)) then fms2_nc_format_param = nf90_64bit_offset call string_copy(fms2_nc_format, "64bit") @@ -366,6 +375,12 @@ subroutine netcdf_io_init (chksz, header_buffer_val, netcdf_default_format, defl '. The acceptable values are "64bit", "classic", "netcdf4". Check fms2_io_nml: netcdf_default_format') endif + if (fms2_num_collective .gt. 0) then + allocate(fn_collective(fms2_num_collective)) + READ (input_nml_file, NML=fms2_io_collective_nml, IOSTAT=mystat) + !print*,'netcdf_io_init: ',fn_collective + endif + end subroutine netcdf_io_init !> @brief Check for errors returned by netcdf. @@ -562,7 +577,7 @@ function netcdf_file_open(fileobj, path, mode, nc_format, pelist, is_restart, do logical :: success integer :: nc_format_param - integer :: err + integer :: i,err character(len=256) :: buf !< Filename with .res in the filename if it is a restart character(len=256) :: buf2 !< Filename with the filename appendix if there is one logical :: is_res @@ -645,12 +660,11 @@ function netcdf_file_open(fileobj, path, mode, nc_format, pelist, is_restart, do endif if (string_compare(mode, "read", .true.)) then - if(string_compare(trim(fileobj%path), "INPUT/phy_data.nc" , .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/fv_tracer.res.tile1.nc", .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/sfc_data.nc" , .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/C3463_grid.tile7.nc" , .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/C3463_grid.tile7.halo3.nc", .true.) .or. & - string_compare(trim(fileobj%path), "INPUT/fv_core.res.tile1.nc" , .true.) ) then + !print*,'netcdf_file_open: ',trim(fileobj%path) + do i=1,fms2_num_collective + if(string_compare(trim(fileobj%path), trim(fn_collective(i)), .true.)) fileobj%use_collective = .true. + enddo + if(fileobj%use_collective) then err = nf90_open(trim(fileobj%path), ior(NF90_NOWRITE, NF90_MPIIO), fileobj%ncid, comm=mpp_comm_private, info=MPI_INFO_NULL) else err = nf90_open(trim(fileobj%path), nf90_nowrite, fileobj%ncid, chunksize=fms2_ncchksz) diff --git a/mpp/mpp.F90 b/mpp/mpp.F90 index 7d07e1937c..61c317a16f 100644 --- a/mpp/mpp.F90 +++ b/mpp/mpp.F90 @@ -1266,7 +1266,7 @@ module mpp_mod logical :: debug = .false. integer :: npes=1, root_pe=0, pe=0 integer(i8_kind) :: tick, ticks_per_sec, max_ticks, start_tick, end_tick, tick0=0 - integer :: mpp_comm_private + integer,public :: mpp_comm_private logical :: first_call_system_clock_mpi=.TRUE. real(r8_kind) :: mpi_count0=0 !< use to prevent integer overflow real(r8_kind) :: mpi_tick_rate=0.d0 !< clock rate for mpi_wtick()