FMS  2025.04
Flexible Modeling System
ensemble_manager.F90
1 !***********************************************************************
2 !* Apache License 2.0
3 !*
4 !* This file is part of the GFDL Flexible Modeling System (FMS).
5 !*
6 !* Licensed under the Apache License, Version 2.0 (the "License");
7 !* you may not use this file except in compliance with the License.
8 !* You may obtain a copy of the License at
9 !*
10 !* http://www.apache.org/licenses/LICENSE-2.0
11 !*
12 !* FMS is distributed in the hope that it will be useful, but WITHOUT
13 !* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied;
14 !* without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
15 !* PARTICULAR PURPOSE. See the License for the specific language
16 !* governing permissions and limitations under the License.
17 !***********************************************************************
18 !> @defgroup ensemble_manager_mod ensemble_manager_mod
19 !> @ingroup coupler
20 !> @brief Routines for setting up and managing ensembles and ensemble pe lists.
21 
22 !> @addtogroup ensemble_manager_mod
23 !> @{
24 module ensemble_manager_mod
25 
26 
27  use fms_mod, only : check_nml_error
28  use mpp_mod, only : mpp_npes, stdout, stdlog, mpp_error, fatal
29  use mpp_mod, only : mpp_pe, mpp_declare_pelist
30  use mpp_mod, only : input_nml_file
31  use fms2_io_mod, only : fms2_io_set_filename_appendix=>set_filename_appendix
32 
33  IMPLICIT NONE
34 
35  private
36 
37  integer, parameter :: MAX_ENSEMBLE_SIZE = 100
38 
39 
40  integer, allocatable, dimension(:,:) :: ensemble_pelist
41  integer, allocatable, dimension(:,:) :: ensemble_pelist_ocean
42  integer, allocatable, dimension(:,:) :: ensemble_pelist_atmos
43  integer, allocatable, dimension(:,:) :: ensemble_pelist_land
44  integer, allocatable, dimension(:,:) :: ensemble_pelist_ice
45  integer, allocatable, dimension(:) :: ensemble_pelist_ocean_filter
46  integer, allocatable, dimension(:) :: ensemble_pelist_atmos_filter
47  integer, allocatable, dimension(:) :: ensemble_pelist_land_filter
48  integer, allocatable, dimension(:) :: ensemble_pelist_ice_filter
49 
50  integer :: ensemble_size = 1
51  integer :: ensemble_id = 1
52  integer :: pe, total_npes_pm=0,ocean_npes_pm=0,atmos_npes_pm=0
53  integer :: land_npes_pm=0,ice_npes_pm=0
54 
56  public :: ensemble_pelist_setup
58 contains
59 
60 !> @brief Initializes @ref ensemble_manager_mod
61 !!
62 !> @throw FATAL, "ensemble_manager_mod: ensemble_nml variable ensemble_size must be a positive integer"
63 !! @throw FATAL, "ensemble_manager_mod: ensemble_nml variable ensemble_size should be no larger
64 !! than MAX_ENSEMBLE_SIZE, change ensemble_size or increase MAX_ENSEMBLE_SIZE"
65 !! @throw FATAL, "ensemble_size must be divis by npes"
66 !! @throw FATAL, "get_ensemble_pelist: size of pelist 1st index < ensemble_size"
67 !! @throw FATAL, "get_ensemble_pelist: size of pelist 2nd index < ocean_npes_pm"
68 !! @throw FATAL, "get_ensemble_pelist: size of pelist 2nd index < atmos_npes_pm"
69 !! @throw FATAL, "get_ensemble_pelist: size of pelist 2nd index < land_npes_pm"
70 !! @throw FATAL, "get_ensemble_pelist: size of pelist 2nd index < ice_npes_pm"
71 !! @throw FATAL, "get_ensemble_pelist: unknown argument name=[name]"
72 !! @throw FATAL, "get_ensemble_pelist: size of pelist 2nd index < total_npes_pm"
73  subroutine ensemble_manager_init()
74 
75 
76  integer :: i, io_status, npes, ierr
77 
78  namelist /ensemble_nml/ ensemble_size
79 
80  read (input_nml_file, ensemble_nml, iostat=io_status)
81  ierr = check_nml_error(io_status, 'ensemble_nml')
82 
83  if(ensemble_size < 1) call mpp_error(fatal, &
84  'ensemble_manager_mod: ensemble_nml variable ensemble_size must be a positive integer')
85  if(ensemble_size > max_ensemble_size) call mpp_error(fatal, &
86  'ensemble_manager_mod: ensemble_nml variable ensemble_size should be no larger than MAX_ENSEMBLE_SIZE, '// &
87  'change ensemble_size or increase MAX_ENSEMBLE_SIZE')
88 
89  pe = mpp_pe()
90  npes = mpp_npes()
91  if (npes < ensemble_size) then
92  call mpp_error(fatal,'npes must be >= ensemble_size')
93  endif
94  total_npes_pm = npes/ensemble_size
95  if (mod(npes, total_npes_pm) /= 0) call mpp_error(fatal,'ensemble_size must be divis by npes')
96 
97  call mpp_declare_pelist((/(i,i=0,npes-1)/),'_ens0') ! for ensemble driver
98 
99  end subroutine ensemble_manager_init
100 
101  !> @brief Getter function for ensemble_id
102  !! @return integer of ensemble id
103  function get_ensemble_id()
104  integer :: get_ensemble_id
105  get_ensemble_id = ensemble_id
106  end function get_ensemble_id
107 
108  !> @brief Returns ensemble size integer array
109  !! @return integer array of sizes
110  function get_ensemble_size()
111 
112  integer, dimension(6) :: get_ensemble_size
113 
114  get_ensemble_size(1) = ensemble_size
115  get_ensemble_size(2) = total_npes_pm
116  get_ensemble_size(3) = ocean_npes_pm
117  get_ensemble_size(4) = atmos_npes_pm
118  get_ensemble_size(5) = land_npes_pm
119  get_ensemble_size(6) = ice_npes_pm
120 
121  end function get_ensemble_size
122 
123  !> @brief Gets pe list for current ensemble or a given ensemble component.
124  subroutine get_ensemble_pelist(pelist, name)
125 
126  integer, intent(inout) :: pelist(:,:) !< Ensemble pelist
127  character(len=*), intent(in), optional :: name !< Component name.
128 
129  if (size(pelist,1) < ensemble_size) &
130  call mpp_error(fatal,'get_ensemble_pelist: size of pelist 1st index < ensemble_size')
131 
132  if(present(name)) then
133  select case(name)
134  case('ocean')
135  if (size(pelist,2) < ocean_npes_pm)&
136  call mpp_error(fatal,'get_ensemble_pelist: size of pelist 2nd index < ocean_npes_pm')
137  pelist = 0
138  pelist(1:ensemble_size,1:ocean_npes_pm) = &
139  ensemble_pelist_ocean(1:ensemble_size,1:ocean_npes_pm)
140 
141  case('atmos')
142  if (size(pelist,2) < atmos_npes_pm)&
143  call mpp_error(fatal,'get_ensemble_pelist: size of pelist 2nd index < atmos_npes_pm')
144  pelist = 0
145  pelist(1:ensemble_size,1:atmos_npes_pm) = &
146  ensemble_pelist_atmos(1:ensemble_size,1:atmos_npes_pm)
147 
148  case('land')
149  if (size(pelist,2) < land_npes_pm)&
150  call mpp_error(fatal,'get_ensemble_pelist: size of pelist 2nd index < land_npes_pm')
151  pelist = 0
152  pelist(1:ensemble_size,1:land_npes_pm) = &
153  ensemble_pelist_land(1:ensemble_size,1:land_npes_pm)
154 
155  case('ice')
156  if (size(pelist,2) < ice_npes_pm)&
157  call mpp_error(fatal,'get_ensemble_pelist: size of pelist 2nd index < ice_npes_pm')
158  pelist = 0
159  pelist(1:ensemble_size,1:ice_npes_pm) = &
160  ensemble_pelist_ice(1:ensemble_size,1:ice_npes_pm)
161 
162  case default
163  call mpp_error(fatal,'get_ensemble_pelist: unknown argument name='//name)
164  end select
165  else
166  if (size(pelist,2) < total_npes_pm)&
167  call mpp_error(fatal,'get_ensemble_pelist: size of pelist 2nd index < total_npes_pm')
168  pelist = 0
169  pelist(1:ensemble_size,1:total_npes_pm) = &
170  ensemble_pelist(1:ensemble_size,1:total_npes_pm)
171  endif
172 
173  return
174  end subroutine get_ensemble_pelist
175 
176 !> @brief Gets filter pelist for a given ensemble component.
177 !!
178 !! @throw FATAL, "get_ensemble_filter_pelist: size of pelist argument < ensemble_size * ocean_npes_pm"
179 !! @throw FATAL, "get_ensemble_filter_pelist: size of pelist argument < ensemble_size * atmos_npes_pm"
180 !! @throw FATAL, "get_ensemble_filter_pelist: size of pelist argument < ensemble_size * land_npes_pm"
181 !! @throw FATAL, "get_ensemble_filter_pelist: size of pelist argument < ensemble_size * ice_npes_pm"
182 !! @throw FATAL, "get_ensemble_filter_pelist: unknown argument name=[name]"
183  subroutine get_ensemble_filter_pelist(pelist, name)
184 
185  integer, intent(inout) :: pelist(:) !< Returned filter pe list
186  character(len=*), intent(in) :: name !< Ensemble component name
187 
188  select case(name)
189  case('ocean')
190  if (size(pelist) < ensemble_size * ocean_npes_pm)&
191  call mpp_error(fatal,'get_ensemble_filter_pelist: size of pelist argument < ensemble_size * ocean_npes_pm')
192  pelist = 0
193  pelist(1:ensemble_size*ocean_npes_pm) = &
194  ensemble_pelist_ocean_filter(1:ensemble_size*ocean_npes_pm)
195 
196  case('atmos')
197  if (size(pelist) < ensemble_size * atmos_npes_pm)&
198  call mpp_error(fatal,'get_ensemble_filter_pelist: size of pelist argument < ensemble_size * atmos_npes_pm')
199  pelist = 0
200  pelist(1:ensemble_size*atmos_npes_pm) = &
201  ensemble_pelist_atmos_filter(1:ensemble_size*atmos_npes_pm)
202 
203  case('land')
204  if (size(pelist) < ensemble_size * land_npes_pm)&
205  call mpp_error(fatal,'get_ensemble_filter_pelist: size of pelist argument < ensemble_size * land_npes_pm')
206  pelist = 0
207  pelist(1:ensemble_size*land_npes_pm) = &
208  ensemble_pelist_land_filter(1:ensemble_size*land_npes_pm)
209 
210  case('ice')
211  if (size(pelist) < ensemble_size * ice_npes_pm)&
212  call mpp_error(fatal,'get_ensemble_filter_pelist: size of pelist argument < ensemble_size * ice_npes_pm')
213  pelist = 0
214  pelist(1:ensemble_size*ice_npes_pm) = &
215  ensemble_pelist_ice_filter(1:ensemble_size*ice_npes_pm)
216 
217  case default
218  call mpp_error(fatal,'get_ensemble_filter_pelist: unknown argument name='//name)
219  end select
220 
221 
222  return
223  end subroutine get_ensemble_filter_pelist
224 
225 !nnz: I think the following block of code should be contained in a subroutine
226 ! to consolidate and ensure the consistency of declaring the various pelists.
227 !> @brief Sets up pe list for an ensemble.
228 !!
229 !! @throw FATAL, "ensemble_manager_mod: land_npes > atmos_npes"
230 !! @throw FATAL, "ensemble_manager_mod: ice_npes > atmos_npes"
231  subroutine ensemble_pelist_setup(concurrent, atmos_npes, ocean_npes, land_npes, ice_npes, &
232  Atm_pelist, Ocean_pelist, Land_pelist, Ice_pelist)
233  logical, intent(in) :: concurrent
234  integer, intent(in) :: atmos_npes, ocean_npes
235  integer, intent(in) :: land_npes, ice_npes
236  integer, dimension(:), intent(inout) :: atm_pelist, ocean_pelist
237  integer, dimension(:), intent(inout) :: land_pelist, ice_pelist
238  integer :: atmos_pe_start, atmos_pe_end, ocean_pe_start, ocean_pe_end
239  integer :: land_pe_start, land_pe_end, ice_pe_start, ice_pe_end
240  character(len=10) :: pelist_name, text
241  integer :: npes, n, m ,i
242 
243  npes = total_npes_pm
244 
245  ! make sure land_npes and ice_npes are not greater than atmos_npes
246  if(land_npes > atmos_npes) call mpp_error(fatal, 'ensemble_manager_mod: land_npes > atmos_npes')
247  if(ice_npes > atmos_npes) call mpp_error(fatal, 'ensemble_manager_mod: ice_npes > atmos_npes')
248 
249  allocate(ensemble_pelist(ensemble_size,total_npes_pm))
250  allocate(ensemble_pelist_ocean(1:ensemble_size, 1:ocean_npes) )
251  allocate(ensemble_pelist_atmos(1:ensemble_size, 1:atmos_npes) )
252  allocate(ensemble_pelist_land(1:ensemble_size, 1:land_npes) )
253  allocate(ensemble_pelist_ice(1:ensemble_size, 1:ice_npes) )
254 
255  atmos_pe_start = 0
256  ocean_pe_start = 0
257  land_pe_start = 0
258  ice_pe_start = 0
259  if( concurrent .OR. atmos_npes+ocean_npes == npes )then
260  ocean_pe_start = ensemble_size*atmos_npes
261  endif
262  do n=1,ensemble_size
263  atmos_pe_end = atmos_pe_start + atmos_npes - 1
264  ocean_pe_end = ocean_pe_start + ocean_npes - 1
265  land_pe_end = land_pe_start + land_npes - 1
266  ice_pe_end = ice_pe_start + ice_npes - 1
267  ensemble_pelist_atmos(n, 1:atmos_npes) = (/(i,i=atmos_pe_start,atmos_pe_end)/)
268  ensemble_pelist_ocean(n, 1:ocean_npes) = (/(i,i=ocean_pe_start,ocean_pe_end)/)
269  ensemble_pelist_land(n, 1:land_npes) = (/(i,i=land_pe_start, land_pe_end)/)
270  ensemble_pelist_ice(n, 1:ice_npes) = (/(i,i=ice_pe_start, ice_pe_end)/)
271  ensemble_pelist(n, 1:atmos_npes) = ensemble_pelist_atmos(n, 1:atmos_npes)
272  if( concurrent .OR. atmos_npes+ocean_npes == npes ) &
273  ensemble_pelist(n, atmos_npes+1:npes) = ensemble_pelist_ocean(n, 1:ocean_npes)
274  if(any(ensemble_pelist(n,:) == pe)) ensemble_id = n
275  write(pelist_name,'(a,i2.2)') '_ens',n
276  call mpp_declare_pelist(ensemble_pelist(n,:), trim(pelist_name))
277  atmos_pe_start = atmos_pe_end + 1
278  ocean_pe_start = ocean_pe_end + 1
279  land_pe_start = atmos_pe_start
280  ice_pe_start = atmos_pe_start
281  enddo
282 
283  atm_pelist(:) = ensemble_pelist_atmos(ensemble_id,:)
284  ocean_pelist(:) = ensemble_pelist_ocean(ensemble_id,:)
285  land_pelist(:) = ensemble_pelist_land(ensemble_id,:)
286  ice_pelist(:) = ensemble_pelist_ice(ensemble_id,:)
287 
288  ! write(pelist_name,'(a,i2.2)') '_ocn_ens',ensemble_id
289  ! call mpp_declare_pelist(Ocean%pelist , trim(pelist_name) )
290 
291  ! write(pelist_name,'(a,i2.2)') '_atm_ens',ensemble_id
292  ! call mpp_declare_pelist(Atm%pelist , trim(pelist_name) )
293  !
294  !nnz: The above is sufficient for non-concurrent mode.
295  ! BUT
296  ! For atmosphere_init to work in ensemble, concurrent mode
297  ! the following Atm_pelist should be declared (per ensemble member)
298  ! instead of the above Atm%pelist!
299  !
300  ! allocate( Atm_pelist(1:ensemble_size, 1:atmos_npes) )
301  ! do n=1,ensemble_size
302  ! do i=1, atmos_npes
303  ! Atm_pelist(n, i) = ensemble_pelist(n, i)
304  ! enddo
305  ! write(pelist_name,'(a,i2.2)') '_atm_ens',n
306  ! call mpp_declare_pelist(Atm_pelist(n,:) , trim(pelist_name) )
307  ! enddo
308  !
309  ! The way I understand this with the help of Totalview is:
310  ! With mpp_declare_pelist(Atm%pelist)
311  ! When we are in fv_arrays_init when mp_init(comID) is called
312  ! comID is the same for the atmospheric PE's for both ensemble members
313  ! since peset(5)%id is the same (7) for those PE's, so the PE count is double what it should be inside
314  ! mp_init().
315  ! It is also true that for Ocean PE's, peset(4)%id is the same (6) for Ocean PE's in both ensemble members
316  ! but for Ocean it is not a problem because Ocean is not trying to create new communicators
317  ! from this peset whereas ATM does (vis mp_init).
318  !
319  ! Who sets peset(i)%id ? Can it be modified to assign different %id for the two subsets.
320  ! peset(i)%id = 0 for Ocean PE's on ATM pesets and for ATM PE's on Ocean pesets.
321  !
322  ! With mpp_declare_pelist(Atm_pelist(n,:)) n=1,...,ensemble_size
323  ! we get separate pesets for each ATM ensemble member and each with a different %id and mp_init is cured.
324  !
325  ! There is also a matter of precedence. If we have both calls
326  ! call mpp_declare_pelist(Atm%pelist , trim(pelist_name) )
327  ! and
328  ! call mpp_declare_pelist(Atm_pelist(n,:) , trim(pelist_name) )
329  ! then concurrent run fails because with call mpp_set_current_pelist( Atm%pelist )
330  ! peset(i) is searched for i=1,2,... and the first pelist that matches argument, its peset is set as current.
331  !
332  ! To be consistent with ATM and OCEAN we can do the following
333  ! (eventhough mpp_declare_pelist(Ocean%pelist) is adequate right now.)
334 
335  if( concurrent )then
336  do n=1,ensemble_size
337  write(pelist_name,'(a,i2.2)') 'atm_ens',n
338  call mpp_declare_pelist(ensemble_pelist_atmos(n,:) , trim(pelist_name) )
339  write(pelist_name,'(a,i2.2)') 'ocn_ens',n
340  call mpp_declare_pelist(ensemble_pelist_ocean(n,:) , trim(pelist_name) )
341  write(pelist_name,'(a,i2.2)') 'lnd_ens',n
342  call mpp_declare_pelist(ensemble_pelist_land(n,:) , trim(pelist_name) )
343  write(pelist_name,'(a,i2.2)') 'ice_ens',n
344  call mpp_declare_pelist(ensemble_pelist_ice(n,:) , trim(pelist_name) )
345  enddo
346  else
347  write(pelist_name,'(a,i2.2)') 'atm_ens',ensemble_id
348  call mpp_declare_pelist(atm_pelist , trim(pelist_name) )
349  write(pelist_name,'(a,i2.2)') 'ocn_ens',ensemble_id
350  call mpp_declare_pelist(ocean_pelist , trim(pelist_name) )
351  write(pelist_name,'(a,i2.2)') 'lnd_ens',ensemble_id
352  call mpp_declare_pelist(land_pelist , trim(pelist_name) )
353  write(pelist_name,'(a,i2.2)') 'ice_ens',ensemble_id
354  call mpp_declare_pelist(ice_pelist , trim(pelist_name) )
355  endif
356 
357  ocean_npes_pm = ocean_npes
358  atmos_npes_pm = atmos_npes
359  land_npes_pm = land_npes
360  ice_npes_pm = ice_npes
361 
362  !Declare pelist of all Ocean, Atmos, Land and Ice pes across all ensembles ( filters )
363  allocate(ensemble_pelist_ocean_filter(ensemble_size * ocean_npes_pm))
364  allocate(ensemble_pelist_atmos_filter(ensemble_size * atmos_npes_pm))
365  allocate(ensemble_pelist_land_filter(ensemble_size * land_npes_pm))
366  allocate(ensemble_pelist_ice_filter(ensemble_size * ice_npes_pm))
367  do n=1,ensemble_size
368  do m=1,ocean_npes_pm
369  i=(n-1)*ocean_npes_pm + m
370  ensemble_pelist_ocean_filter(i) = ensemble_pelist_ocean(n,m)
371  enddo
372  do m=1,atmos_npes_pm
373  i=(n-1)*atmos_npes_pm + m
374  ensemble_pelist_atmos_filter(i) = ensemble_pelist_atmos(n,m)
375  enddo
376  do m=1,land_npes_pm
377  i=(n-1)*land_npes_pm + m
378  ensemble_pelist_land_filter(i) = ensemble_pelist_land(n,m)
379  enddo
380  do m=1,ice_npes_pm
381  i=(n-1)*ice_npes_pm + m
382  ensemble_pelist_ice_filter(i) = ensemble_pelist_ice(n,m)
383  enddo
384  enddo
385 
386  write(pelist_name,'(a)') 'ocn_filter'
387  call mpp_declare_pelist(ensemble_pelist_ocean_filter, trim(pelist_name) )
388 
389  write(pelist_name,'(a)') 'atm_filter'
390  call mpp_declare_pelist(ensemble_pelist_atmos_filter, trim(pelist_name) )
391 
392  write(pelist_name,'(a)') 'lnd_filter'
393  call mpp_declare_pelist(ensemble_pelist_land_filter, trim(pelist_name) )
394 
395  write(pelist_name,'(a)') 'ice_filter'
396  call mpp_declare_pelist(ensemble_pelist_ice_filter, trim(pelist_name) )
397 
398  !
399  !Rename output files to identify the ensemble
400  !If ensemble_size=1 do not rename files so that the same coupler
401  !can be used for non-ensemble experiments
402  !
403  if (ensemble_size > 1) then
404  write( text,'(a,i2.2)' ) 'ens_', ensemble_id
405  !Append ensemble_id to the restart filenames
406  call fms2_io_set_filename_appendix(trim(text))
407  endif
408 
409  end subroutine ensemble_pelist_setup
410 
411 
412 end module ensemble_manager_mod
413 !> @}
414 ! close documentation grouping
integer function, public get_ensemble_id()
Getter function for ensemble_id.
integer function, dimension(6), public get_ensemble_size()
Returns ensemble size integer array.
subroutine, public ensemble_pelist_setup(concurrent, atmos_npes, ocean_npes, land_npes, ice_npes, Atm_pelist, Ocean_pelist, Land_pelist, Ice_pelist)
Sets up pe list for an ensemble.
subroutine, public ensemble_manager_init()
Initializes ensemble_manager_mod.
subroutine, public get_ensemble_pelist(pelist, name)
Gets pe list for current ensemble or a given ensemble component.
subroutine, public get_ensemble_filter_pelist(pelist, name)
Gets filter pelist for a given ensemble component.
integer function, public check_nml_error(IOSTAT, NML_NAME)
Checks the iostat argument that is returned after reading a namelist and determines if the error code...
Definition: fms.F90:523
integer function stdout()
This function returns the current standard fortran unit numbers for output.
Definition: mpp_util.inc:42
subroutine mpp_declare_pelist(pelist, name, commID)
Declare a pelist.
Definition: mpp_util.inc:469
integer function stdlog()
This function returns the current standard fortran unit numbers for log messages. Log messages,...
Definition: mpp_util.inc:58
integer function mpp_npes()
Returns processor count for current pelist.
Definition: mpp_util.inc:420
integer function mpp_pe()
Returns processor ID.
Definition: mpp_util.inc:406
Error handler.
Definition: mpp.F90:381