MPI-AMRVAC  3.0
The MPI - Adaptive Mesh Refinement - Versatile Advection Code
m_octree_mg_2d.t
Go to the documentation of this file.
1 
2 
3 ! Single module generated from the octree-mg sources.
4 ! This file can be easier to include in existing projects.
5 !
6 ! Notes:
7 ! 1. The module name is here extended by _1d, _2d or _3d
8 ! 2. The free space Poisson solver is not included here.
9 ! 3. It is best to make changes in the original repository at
10 ! https://github.com/jannisteunissen/octree-mg
11 ! 4. This file can be generated as follows:
12 ! cd octree-mg/single_module
13 ! ./to_single_module.sh
14 !
15 ! The modules can be compiled with:
16 ! mpif90 -c m_octree_mg_1d.f90 [other options]
17 ! mpif90 -c m_octree_mg_2d.f90 [other options]
18 ! mpif90 -c m_octree_mg_3d.f90 [other options]
19 ! mpif90 -c m_octree_mg.f90 -cpp -DNDIM=<1,2,3> [other options]
20 
21 
23  use mpi
24  implicit none
25  private
26 
27  !! File ../src/m_data_structures.f90
28 
29  !> Type of reals
30  integer, parameter, public :: dp = kind(0.0d0)
31 
32  !> Type for 64-bit integers
33  integer, parameter, public :: i8 = selected_int_kind(18)
34 
35  !> Indicates a standard Laplacian
36  integer, parameter, public :: mg_laplacian = 1
37 
38  !> Indicates a variable-coefficient Laplacian
39  integer, parameter, public :: mg_vlaplacian = 2
40 
41  !> Indicates a constant-coefficient Helmholtz equation
42  integer, parameter, public :: mg_helmholtz = 3
43 
44  !> Indicates a variable-coefficient Helmholtz equation
45  integer, parameter, public :: mg_vhelmholtz = 4
46 
47  !> Indicates a anisotropic-coefficient Helmholtz equation
48  integer, parameter, public :: mg_ahelmholtz = 5
49 
50  !> Cartesian coordinate system
51  integer, parameter, public :: mg_cartesian = 1
52  !> Cylindrical coordinate system
53  integer, parameter, public :: mg_cylindrical = 2
54  !> Spherical coordinate system
55  integer, parameter, public :: mg_spherical = 3
56 
57  integer, parameter, public :: mg_smoother_gs = 1
58  integer, parameter, public :: mg_smoother_gsrb = 2
59  integer, parameter, public :: mg_smoother_jacobi = 3
60 
61  !> Problem dimension
62  integer, parameter, public :: mg_ndim = 2
63 
64  !> Number of predefined multigrid variables
65  integer, parameter, public :: mg_num_vars = 4
66  !> Maximum number of variables
67  integer, parameter, public :: mg_max_num_vars = 10
68  !> Index of solution
69  integer, parameter, public :: mg_iphi = 1
70  !> Index of right-hand side
71  integer, parameter, public :: mg_irhs = 2
72  !> Index of previous solution (used for correction)
73  integer, parameter, public :: mg_iold = 3
74  !> Index of residual
75  integer, parameter, public :: mg_ires = 4
76 
77  !> Index of the variable coefficient (at cell centers)
78  integer, parameter, public :: mg_iveps = 5
79 
80  !> Indexes of anisotropic variable coefficients
81  integer, parameter, public :: mg_iveps1 = 5
82  integer, parameter, public :: mg_iveps2 = 6
83  integer, parameter, public :: mg_iveps3 = 7
84 
85  !> Minimum allowed grid level
86  integer, parameter, public :: mg_lvl_lo = -20
87  !> Maximum allowed grid level
88  integer, parameter, public :: mg_lvl_hi = 20
89 
90  !> Value to indicate a Dirichlet boundary condition
91  integer, parameter, public :: mg_bc_dirichlet = -10
92 
93  !> Value to indicate a Neumann boundary condition
94  integer, parameter, public :: mg_bc_neumann = -11
95 
96  !> Value to indicate a continuous boundary condition
97  integer, parameter, public :: mg_bc_continuous = -12
98 
99  !> Special value that indicates there is no box
100  integer, parameter, public :: mg_no_box = 0
101  !> Special value that indicates there is a physical boundary
102  integer, parameter, public :: mg_physical_boundary = -1
103 
104  !> Maximum number of timers to use
105  integer, parameter, public :: mg_max_timers = 20
106 
107  ! Numbering of children (same location as **corners**)
108  integer, parameter, public :: mg_num_children = 4
109 
110  ! Index offset for each child
111  integer, parameter, public :: mg_child_dix(2, 4) = reshape([0,0,1,0,0,1,1,1], [2,4])
112  ! Reverse child index in each direction
113  integer, parameter, public :: mg_child_rev(4, 2) = reshape([2,1,4,3,3,4,1,2], [4,2])
114  ! Children adjacent to a neighbor
115  integer, parameter, public :: mg_child_adj_nb(2, 4) = reshape([1,3,2,4,1,2,3,4], [2,4])
116  ! Which children have a low index per dimension
117  logical, parameter, public :: mg_child_low(2, 4) = reshape([.true., .true., &
118  .false., .true., .true., .false., .false., .false.], [2, 4])
119 
120  ! Neighbor topology information
121  integer, parameter, public :: mg_num_neighbors = 4
122  integer, parameter, public :: mg_neighb_lowx = 1
123  integer, parameter, public :: mg_neighb_highx = 2
124  integer, parameter, public :: mg_neighb_lowy = 3
125  integer, parameter, public :: mg_neighb_highy = 4
126 
127  ! Index offsets of neighbors
128  integer, parameter, public :: mg_neighb_dix(2, 4) = reshape([-1,0,1,0,0,-1,0,1], [2,4])
129  ! Which neighbors have a lower index
130  logical, parameter, public :: mg_neighb_low(4) = [.true., .false., .true., .false.]
131  ! Opposite of nb_low, but now as -1,1 integers
132  integer, parameter, public :: mg_neighb_high_pm(4) = [-1, 1, -1, 1]
133 
134  ! Reverse neighbors
135  integer, parameter, public :: mg_neighb_rev(4) = [2, 1, 4, 3]
136  ! Direction (dimension) for a neighbor
137  integer, parameter, public :: mg_neighb_dim(4) = [1, 1, 2, 2]
138 
139  !> Lists of blocks per refinement level
140  type, public :: mg_lvl_t
141  integer, allocatable :: leaves(:)
142  integer, allocatable :: parents(:)
143  integer, allocatable :: ref_bnds(:)
144  integer, allocatable :: ids(:)
145  integer, allocatable :: my_leaves(:)
146  integer, allocatable :: my_parents(:)
147  integer, allocatable :: my_ref_bnds(:)
148  integer, allocatable :: my_ids(:)
149  end type mg_lvl_t
150 
151  !> Box data structure
152  type, public :: mg_box_t
153  integer :: rank !< Which process owns this box
154  integer :: id !< Box id (index in boxes(:) array)
155  integer :: lvl !< Refinement level
156  integer :: ix(2) !< Spatial index
157  integer :: parent !< Id of parent
158  integer :: children(2**2) !< Ids of children
159  integer :: neighbors(2*2) !< Ids of neighbors
160  real(dp) :: r_min(2) !< Minimum coordinate
161  real(dp) :: dr(2) !< Grid spacing
162  !> Cell-centered data
163  real(dp), allocatable :: cc(:, :, :)
164  end type mg_box_t
165 
166  !> Buffer type (one is used for each pair of communicating processes)
167  type, public :: mg_buf_t
168  integer :: i_send !< Index in send array
169  integer :: i_recv
170  integer :: i_ix
171  integer, allocatable :: ix(:) !< Will be used to sort the data
172  real(dp), allocatable :: send(:)
173  real(dp), allocatable :: recv(:)
174  end type mg_buf_t
175 
176  type, public :: mg_comm_t
177  integer, allocatable :: n_send(:, :)
178  integer, allocatable :: n_recv(:, :)
179  end type mg_comm_t
180 
181  type, public :: mg_bc_t
182  integer :: bc_type = mg_bc_dirichlet !< Type of boundary condition
183  real(dp) :: bc_value = 0.0_dp !< Value (for e.g. Dirichlet or Neumann)
184  !> To set user-defined boundary conditions (overrides bc(:))
185  procedure(mg_subr_bc), pointer, nopass :: boundary_cond => null()
186  !> To set a user-defined refinement boundary method
187  procedure(mg_subr_rb), pointer, nopass :: refinement_bnd => null()
188  end type mg_bc_t
189 
190  type, public :: mg_timer_t
191  character(len=20) :: name
192  real(dp) :: t = 0.0_dp
193  real(dp) :: t0
194  end type mg_timer_t
195 
196  type, public :: mg_t
197  !> Whether the multigrid tree structure has been created
198  logical :: tree_created = .false.
199  !> Whether storage has been allocated for boxes
200  logical :: is_allocated = .false.
201  !> Number of extra cell-centered variable (e.g., for coefficients)
202  integer :: n_extra_vars = 0
203  !> MPI communicator
204  integer :: comm = -1
205  !> Number of MPI tasks
206  integer :: n_cpu = -1
207  !> MPI rank of this task
208  integer :: my_rank = -1
209  !> Size of boxes in cells, be equal for all dimensions
210  integer :: box_size = -1
211  !> Highest grid level in the tree
212  integer :: highest_lvl = -1
213  !> Lowest grid level in the tree
214  integer :: lowest_lvl = -1
215  !> First normal level of the quadtree/octree, at coarser levels parents
216  !> have only one child
217  integer :: first_normal_lvl = -1
218  !> Total number of boxes in the tree (among all processes)
219  integer :: n_boxes = 0
220  !> Size of boxes per level (differs for coarsest levels)
221  integer :: box_size_lvl(mg_lvl_lo:mg_lvl_hi)
222  !> Size of domain per level (if uniformly refined)
223  integer :: domain_size_lvl(2, mg_lvl_lo:mg_lvl_hi)
224  !> Grid spacing per level
225  real(dp) :: dr(2, mg_lvl_lo:mg_lvl_hi)
226  !> Minimum coordinates
227  real(dp) :: r_min(2)
228  !> List of all levels
229  type(mg_lvl_t) :: lvls(mg_lvl_lo:mg_lvl_hi)
230  !> Array with all boxes in the tree. Only boxes owned by this task are
231  !> actually allocated
232  type(mg_box_t), allocatable :: boxes(:)
233  !> Buffer for communication with each other task
234  type(mg_buf_t), allocatable :: buf(:)
235 
236  !> Communication info for restriction
237  type(mg_comm_t) :: comm_restrict
238  !> Communication info for prolongation
239  type(mg_comm_t) :: comm_prolong
240  !> Communication info for ghost cell filling
241  type(mg_comm_t) :: comm_ghostcell
242 
243  !> Whether boundary condition data has been stored for mg solution
244  logical :: phi_bc_data_stored = .false.
245 
246  !> Whether a dimension is periodic
247  logical :: periodic(2) = .false.
248 
249  !> To store pre-defined boundary conditions per direction per variable
251 
252  !> Type of operator
253  integer :: operator_type = mg_laplacian
254  !> Type of grid geometry
255  integer :: geometry_type = mg_cartesian
256 
257  !> Whether the mean has to be subtracted from the multigrid solution
258  logical :: subtract_mean = .false.
259  !> Type of multigrid smoother
260  integer :: smoother_type = mg_smoother_gs
261  !> Number of substeps for the smoother (for GSRB this is 2)
262  integer :: n_smoother_substeps = 1
263  !> Number of cycles when doing downwards relaxation
264  integer :: n_cycle_down = 2
265  !> Number of cycles when doing upwards relaxation
266  integer :: n_cycle_up = 2
267  !> Maximum number of cycles on the coarse grid
268  integer :: max_coarse_cycles = 1000
269  integer :: coarsest_grid(2) = 2
270  !> Stop coarse grid when max. residual is smaller than this
271  real(dp) :: residual_coarse_abs = 1e-8_dp
272  !> Stop coarse grid when residual has been reduced by this factor
273  real(dp) :: residual_coarse_rel = 1e-8_dp
274 
275  !> Multigrid operator (e.g., Laplacian)
276  procedure(mg_box_op), pointer, nopass :: box_op => null()
277 
278  !> Multigrid smoother
279  procedure(mg_box_gsrb), pointer, nopass :: box_smoother => null()
280 
281  !> Multigrid prolongation method
282  procedure(mg_box_prolong), pointer, nopass :: box_prolong => null()
283 
284  !> Number of timers
285  integer :: n_timers = 0
286  !> Values for the timers
287  type(mg_timer_t) :: timers(mg_max_timers)
288  end type mg_t
289 
290  interface
291  !> To fill ghost cells near physical boundaries
292  subroutine mg_subr_bc(box, nc, iv, nb, bc_type, bc)
293  import
294  type(mg_box_t), intent(in) :: box
295  integer, intent(in) :: nc
296  integer, intent(in) :: iv !< Index of variable
297  integer, intent(in) :: nb !< Direction
298  integer, intent(out) :: bc_type !< Type of b.c.
299  !> Boundary values
300  real(dp), intent(out) :: bc(nc)
301  end subroutine mg_subr_bc
302 
303  !> To fill ghost cells near refinement boundaries
304  subroutine mg_subr_rb(box, nc, iv, nb, cgc)
305  import
306  type(mg_box_t), intent(inout) :: box
307  integer, intent(in) :: nc
308  integer, intent(in) :: iv !< Index of variable
309  integer, intent(in) :: nb !< Direction
310  !> Coarse data
311  real(dp), intent(in) :: cgc(nc)
312  end subroutine mg_subr_rb
313 
314  !> Subroutine that performs A * cc(..., i_in) = cc(..., i_out)
315  subroutine mg_box_op(mg, id, nc, i_out)
316  import
317  type(mg_t), intent(inout) :: mg
318  integer, intent(in) :: id
319  integer, intent(in) :: nc
320  integer, intent(in) :: i_out
321  end subroutine mg_box_op
322 
323  !> Subroutine that performs Gauss-Seidel relaxation
324  subroutine mg_box_gsrb(mg, id, nc, redblack_cntr)
325  import
326  type(mg_t), intent(inout) :: mg
327  integer, intent(in) :: id
328  integer, intent(in) :: nc
329  integer, intent(in) :: redblack_cntr
330  end subroutine mg_box_gsrb
331 
332  !> Subroutine that performs prolongation to a single child
333  subroutine mg_box_prolong(mg, p_id, dix, nc, iv, fine)
334  import
335  type(mg_t), intent(inout) :: mg
336  integer, intent(in) :: p_id !< Id of parent
337  integer, intent(in) :: dix(2) !< Offset of child in parent grid
338  integer, intent(in) :: nc !< Child grid size
339  integer, intent(in) :: iv !< Prolong from this variable
340  real(dp), intent(out) :: fine(nc, nc) !< Prolonged values
341  end subroutine mg_box_prolong
342  end interface
343 
344  ! Public methods
345  public :: mg_subr_bc
346  public :: mg_subr_rb
347  public :: mg_box_op
348  public :: mg_box_gsrb
349  public :: mg_box_prolong
350  public :: mg_has_children
351  public :: mg_ix_to_ichild
352  public :: mg_get_child_offset
353  public :: mg_highest_uniform_lvl
354  public :: mg_number_of_unknowns
355  public :: mg_get_face_coords
356  public :: mg_add_timer
357  public :: mg_timer_start
358  public :: mg_timer_end
359  public :: mg_timers_show
360 
361  !! File ../src/m_allocate_storage.f90
362 
363  public :: mg_allocate_storage
364  public :: mg_deallocate_storage
365 
366  !! File ../src/m_diffusion.f90
367 
368 !> Module for implicitly solving diffusion equations
369 
370  public :: diffusion_solve
371  public :: diffusion_solve_vcoeff
372  public :: diffusion_solve_acoeff
373 
374  !! File ../src/m_laplacian.f90
375 
376 !> Module which contains multigrid procedures for a Laplacian operator
377 
378  public :: laplacian_set_methods
379 
380  !! File ../src/m_vhelmholtz.f90
381 
382 !> Module which contains multigrid procedures for a Helmholtz operator of the
383 !> form: div(D grad(phi)) - lambda*phi = f, where D has a smooth spatial
384 !> variation
385 
386  !> The lambda used for the Helmholtz equation (should be >= 0)
387  real(dp), public, protected :: vhelmholtz_lambda = 0.0_dp
388 
389  public :: vhelmholtz_set_methods
390  public :: vhelmholtz_set_lambda
391 
392  !! File ../src/m_build_tree.f90
393 
394  ! Public methods
395  public :: mg_build_rectangle
396  public :: mg_add_children
397  public :: mg_set_leaves_parents
398  public :: mg_set_next_level_ids
400  public :: mg_set_neighbors_lvl
401 
402  !! File ../src/m_load_balance.f90
403 !> Module for load balancing a tree (that already has been constructed). The
404 !> load balancing determines which ranks (MPI processes) allocated physical
405 !> storage for boxes. The tree structure itself is present on all processes.
406 
407  ! Public methods
408  public :: mg_load_balance_simple
409  public :: mg_load_balance
410  public :: mg_load_balance_parents
411 
412  !! File ../src/m_vlaplacian.f90
413 
414 !> Module which contains multigrid procedures for a variable-coefficient
415 !> Laplacian operator, assuming the variation is smooth
416 
417  public :: vlaplacian_set_methods
418 
419  !! File ../src/m_communication.f90
420 
421  ! Public methods
422  public :: mg_comm_init
423  public :: sort_and_transfer_buffers
424 
425  !! File ../src/m_ghost_cells.f90
426 
427  ! Public methods
428  public :: mg_ghost_cell_buffer_size
429  public :: mg_fill_ghost_cells
430  public :: mg_fill_ghost_cells_lvl
431  public :: mg_phi_bc_store
432 
433  !! File ../src/m_prolong.f90
434 
435  ! Public methods
436  public :: mg_prolong
437  public :: mg_prolong_buffer_size
438  public :: mg_prolong_sparse
439 
440  !! File ../src/m_helmholtz.f90
441 
442 !> Module which contains multigrid procedures for a Helmholtz operator of the
443 !> form: laplacian(phi) - lambda*phi = f
444 
445  !> The lambda used for the Helmholtz equation (should be >= 0)
446  real(dp), public, protected :: helmholtz_lambda = 0.0_dp
447 
448  public :: helmholtz_set_methods
449  public :: helmholtz_set_lambda
450 
451  !! File ../src/m_multigrid.f90
452 
453  integer :: timer_total_vcycle = -1
454  integer :: timer_total_fmg = -1
455  integer :: timer_smoother = -1
456  integer :: timer_smoother_gc = -1
457  integer :: timer_coarse = -1
458  integer :: timer_correct = -1
459  integer :: timer_update_coarse = -1
460 
461  ! Public methods
462  public :: mg_fas_vcycle
463  public :: mg_fas_fmg
464  public :: mg_set_methods
465  public :: mg_apply_op
466 
467  !! File ../src/m_restrict.f90
468 
469  ! Public methods
470  public :: mg_restrict
471  public :: mg_restrict_lvl
472  public :: mg_restrict_buffer_size
473 
474  !! File ../src/m_mrgrnk.f90
475 
476  Integer, Parameter :: kdp = selected_real_kind(15)
477  public :: mrgrnk
478  private :: kdp
479  private :: i_mrgrnk
480  interface mrgrnk
481  module procedure i_mrgrnk
482  end interface mrgrnk
483 
484  !! File ../src/m_ahelmholtz.f90
485 
486 !> Module which contains multigrid procedures for a Helmholtz operator of the
487 !> form: div(D grad(phi)) - lambda*phi = f, where D has a smooth spatial
488 !> variation and a component in each spatial direction
489 
490  !> The lambda used for the Helmholtz equation (should be >= 0)
491  real(dp), public, protected :: ahelmholtz_lambda = 0.0_dp
492 
493  public :: ahelmholtz_set_methods
494  public :: ahelmholtz_set_lambda
495 contains
496 
497  !! File ../src/m_data_structures.f90
498 
499  !> Return .true. if a box has children
500  elemental logical function mg_has_children(box)
501  type(mg_box_t), intent(in) :: box
502 
503  ! Boxes are either fully refined or not, so we only need to check one of the
504  ! children
505  mg_has_children = (box%children(1) /= mg_no_box)
506  end function mg_has_children
507 
508  !> Compute the child index for a box with spatial index ix. With child index
509  !> we mean the index in the children(:) array of its parent.
510  integer function mg_ix_to_ichild(ix)
511  integer, intent(in) :: ix(2) !< Spatial index of the box
512  ! The index can range from 1 (all ix odd) and 2**$D (all ix even)
513  mg_ix_to_ichild = 4 - 2 * iand(ix(2), 1) - iand(ix(1), 1)
514  end function mg_ix_to_ichild
515 
516  !> Get the offset of a box with respect to its parent (e.g. in 2d, there can
517  !> be a child at offset 0,0, one at n_cell/2,0, one at 0,n_cell/2 and one at
518  !> n_cell/2, n_cell/2)
519  pure function mg_get_child_offset(mg, id) result(ix_offset)
520  type(mg_t), intent(in) :: mg
521  integer, intent(in) :: id
522  integer :: ix_offset(2)
523 
524  if (mg%boxes(id)%lvl <= mg%first_normal_lvl) then
525  ix_offset(:) = 0
526  else
527  ix_offset = iand(mg%boxes(id)%ix-1, 1) * &
528  ishft(mg%box_size, -1) ! * n_cell / 2
529  end if
530  end function mg_get_child_offset
531 
532  pure function mg_highest_uniform_lvl(mg) result(lvl)
533  type(mg_t), intent(in) :: mg
534  integer :: lvl
535 
536  do lvl = mg%first_normal_lvl, mg%highest_lvl-1
537  ! Exit if a grid is partially refined
538  if (size(mg%lvls(lvl)%leaves) /= 0 .and. &
539  size(mg%lvls(lvl)%parents) /= 0) exit
540  end do
541  ! If the loop did not exit, we get lvl equals mg%highest_lvl
542  end function mg_highest_uniform_lvl
543 
544  !> Determine total number of unknowns (on leaves)
545  function mg_number_of_unknowns(mg) result(n_unknowns)
546  type(mg_t), intent(in) :: mg
547  integer :: lvl
548  integer(i8) :: n_unknowns
549 
550  n_unknowns = 0
551  do lvl = mg%first_normal_lvl, mg%highest_lvl
552  n_unknowns = n_unknowns + size(mg%lvls(lvl)%leaves)
553  end do
554  n_unknowns = n_unknowns * int(mg%box_size**3, i8)
555  end function mg_number_of_unknowns
556 
557  !> Get coordinates at the face of a box
558  subroutine mg_get_face_coords(box, nb, nc, x)
559  type(mg_box_t), intent(in) :: box
560  integer, intent(in) :: nb
561  integer, intent(in) :: nc
562  real(dp), intent(out) :: x(nc, 2)
563  integer :: i, ixs(2-1)
564  integer :: nb_dim
565  real(dp) :: rmin(2)
566 
567  nb_dim = mg_neighb_dim(nb)
568 
569  ! Determine directions perpendicular to neighbor
570  ixs = [(i, i = 1, 2-1)]
571  ixs(nb_dim:) = ixs(nb_dim:) + 1
572 
573  rmin = box%r_min
574  if (.not. mg_neighb_low(nb)) then
575  rmin(nb_dim) = rmin(nb_dim) + box%dr(nb_dim) * nc
576  end if
577 
578  do i = 1, nc
579  x(i, :) = rmin
580  x(i, ixs(1)) = x(i, ixs(1)) + (i-0.5d0) * box%dr(ixs(1))
581  end do
582  end subroutine mg_get_face_coords
583 
584  integer function mg_add_timer(mg, name)
585  type(mg_t), intent(inout) :: mg
586  character(len=*), intent(in) :: name
587 
588  mg%n_timers = mg%n_timers + 1
589  mg_add_timer = mg%n_timers
590  mg%timers(mg_add_timer)%name = name
591  end function mg_add_timer
592 
593  subroutine mg_timer_start(timer)
594  use mpi
595  type(mg_timer_t), intent(inout) :: timer
596  timer%t0 = mpi_wtime()
597  end subroutine mg_timer_start
598 
599  subroutine mg_timer_end(timer)
600  use mpi
601  type(mg_timer_t), intent(inout) :: timer
602  timer%t = timer%t + mpi_wtime() - timer%t0
603  end subroutine mg_timer_end
604 
605  subroutine mg_timers_show(mg)
606  use mpi
607  type(mg_t), intent(in) :: mg
608  integer :: n, ierr
609  real(dp) :: tmin(mg%n_timers)
610  real(dp) :: tmax(mg%n_timers)
611 
612  call mpi_reduce(mg%timers(1:mg%n_timers)%t, tmin, mg%n_timers, &
613  mpi_double, mpi_min, 0, mg%comm, ierr)
614  call mpi_reduce(mg%timers(1:mg%n_timers)%t, tmax, mg%n_timers, &
615  mpi_double, mpi_max, 0, mg%comm, ierr)
616 
617  if (mg%my_rank == 0) then
618  write(*, "(A20,2A16)") "name ", "min(s)", "max(s)"
619  do n = 1, mg%n_timers
620  write(*, "(A20,2F16.6)") mg%timers(n)%name, &
621  tmin(n), tmax(n)
622  end do
623  end if
624  end subroutine mg_timers_show
625 
626  !! File ../src/m_allocate_storage.f90
627 
628  !> Deallocate all allocatable arrays
629  subroutine mg_deallocate_storage(mg)
630  type(mg_t), intent(inout) :: mg
631  integer :: lvl
632 
633  if (.not. mg%is_allocated) &
634  error stop "deallocate_storage: tree is not allocated"
635 
636  deallocate(mg%boxes)
637  deallocate(mg%buf)
638 
639  deallocate(mg%comm_restrict%n_send)
640  deallocate(mg%comm_restrict%n_recv)
641 
642  deallocate(mg%comm_prolong%n_send)
643  deallocate(mg%comm_prolong%n_recv)
644 
645  deallocate(mg%comm_ghostcell%n_send)
646  deallocate(mg%comm_ghostcell%n_recv)
647 
648  do lvl = mg%lowest_lvl, mg%highest_lvl
649  deallocate(mg%lvls(lvl)%ids)
650  deallocate(mg%lvls(lvl)%leaves)
651  deallocate(mg%lvls(lvl)%parents)
652  deallocate(mg%lvls(lvl)%ref_bnds)
653  deallocate(mg%lvls(lvl)%my_ids)
654  deallocate(mg%lvls(lvl)%my_leaves)
655  deallocate(mg%lvls(lvl)%my_parents)
656  deallocate(mg%lvls(lvl)%my_ref_bnds)
657  end do
658 
659  mg%is_allocated = .false.
660  mg%n_boxes = 0
661  mg%phi_bc_data_stored = .false.
662  end subroutine mg_deallocate_storage
663 
664  !> Allocate communication buffers and local boxes for a tree that has already
665  !> been created
666  subroutine mg_allocate_storage(mg)
667 
668  type(mg_t), intent(inout) :: mg
669  integer :: i, id, lvl, nc
670  integer :: n_send(0:mg%n_cpu-1, 3)
671  integer :: n_recv(0:mg%n_cpu-1, 3)
672  integer :: dsize(3)
673  integer :: n_in, n_out, n_id
674 
675  if (.not. mg%tree_created) &
676  error stop "allocate_storage: tree is not yet created"
677 
678  if (mg%is_allocated) &
679  error stop "allocate_storage: tree is already allocated"
680 
681  do lvl = mg%lowest_lvl, mg%highest_lvl
682  nc = mg%box_size_lvl(lvl)
683  do i = 1, size(mg%lvls(lvl)%my_ids)
684  id = mg%lvls(lvl)%my_ids(i)
685  allocate(mg%boxes(id)%cc(0:nc+1, 0:nc+1, &
686  mg_num_vars + mg%n_extra_vars))
687 
688  ! Set all initial values to zero
689  mg%boxes(id)%cc(:, :, :) = 0.0_dp
690  end do
691  end do
692 
693  allocate(mg%buf(0:mg%n_cpu-1))
694 
695  call mg_ghost_cell_buffer_size(mg, n_send(:, 1), &
696  n_recv(:, 1), dsize(1))
697  call mg_restrict_buffer_size(mg, n_send(:, 2), &
698  n_recv(:, 2), dsize(2))
699  call mg_prolong_buffer_size(mg, n_send(:, 3), &
700  n_recv(:, 3), dsize(3))
701 
702  do i = 0, mg%n_cpu-1
703  n_out = maxval(n_send(i, :) * dsize(:))
704  n_in = maxval(n_recv(i, :) * dsize(:))
705  n_id = maxval(n_send(i, :))
706  allocate(mg%buf(i)%send(n_out))
707  allocate(mg%buf(i)%recv(n_in))
708  allocate(mg%buf(i)%ix(n_id))
709  end do
710 
711  mg%is_allocated = .true.
712  end subroutine mg_allocate_storage
713 
714  !! File ../src/m_diffusion.f90
715 
716  !> Solve a diffusion equation implicitly, assuming a constant diffusion
717  !> coefficient. The solution at time t should be stored in mg_iphi, which is
718  !> on output replaced by the solution at time t+dt.
719  subroutine diffusion_solve(mg, dt, diffusion_coeff, order, max_res)
720 
721  type(mg_t), intent(inout) :: mg
722  real(dp), intent(in) :: dt
723  real(dp), intent(in) :: diffusion_coeff
724  integer, intent(in) :: order
725  real(dp), intent(in) :: max_res
726  integer, parameter :: max_its = 10
727  integer :: n
728  real(dp) :: res
729 
730  mg%operator_type = mg_helmholtz
731  call mg_set_methods(mg)
732 
733  select case (order)
734  case (1)
735  call helmholtz_set_lambda(1/(dt * diffusion_coeff))
736  call set_rhs(mg, -1/(dt * diffusion_coeff), 0.0_dp)
737  case (2)
738  call helmholtz_set_lambda(0.0d0)
739  call mg_apply_op(mg, mg_irhs)
740  call helmholtz_set_lambda(2/(dt * diffusion_coeff))
741  call set_rhs(mg, -2/(dt * diffusion_coeff), -1.0_dp)
742  case default
743  error stop "diffusion_solve: order should be 1 or 2"
744  end select
745 
746  ! Start with an FMG cycle
747  call mg_fas_fmg(mg, .true., max_res=res)
748 
749  ! Add V-cycles if necessary
750  do n = 1, max_its
751  if (res <= max_res) exit
752  call mg_fas_vcycle(mg, max_res=res)
753  end do
754 
755  if (n == max_its + 1) then
756  if (mg%my_rank == 0) &
757  print *, "Did you specify boundary conditions correctly?"
758  error stop "diffusion_solve: no convergence"
759  end if
760  end subroutine diffusion_solve
761 
762  !> Solve a diffusion equation implicitly, assuming a variable diffusion
763  !> coefficient which has been stored in mg_iveps (also on coarse grids). The
764  !> solution at time t should be stored in mg_iphi, which is on output replaced
765  !> by the solution at time t+dt.
766  subroutine diffusion_solve_vcoeff(mg, dt, order, max_res)
767 
768  type(mg_t), intent(inout) :: mg
769  real(dp), intent(in) :: dt
770  integer, intent(in) :: order
771  real(dp), intent(in) :: max_res
772  integer, parameter :: max_its = 10
773  integer :: n
774  real(dp) :: res
775 
776  mg%operator_type = mg_vhelmholtz
777  call mg_set_methods(mg)
778 
779  select case (order)
780  case (1)
781  call vhelmholtz_set_lambda(1/dt)
782  call set_rhs(mg, -1/dt, 0.0_dp)
783  case (2)
784  call vhelmholtz_set_lambda(0.0d0)
785  call mg_apply_op(mg, mg_irhs)
786  call vhelmholtz_set_lambda(2/dt)
787  call set_rhs(mg, -2/dt, -1.0_dp)
788  case default
789  error stop "diffusion_solve: order should be 1 or 2"
790  end select
791 
792  ! Start with an FMG cycle
793  call mg_fas_fmg(mg, .true., max_res=res)
794 
795  ! Add V-cycles if necessary
796  do n = 1, max_its
797  if (res <= max_res) exit
798  call mg_fas_vcycle(mg, max_res=res)
799  end do
800 
801  if (n == max_its + 1) then
802  if (mg%my_rank == 0) then
803  print *, "Did you specify boundary conditions correctly?"
804  print *, "Or is the variation in diffusion too large?"
805  end if
806  error stop "diffusion_solve: no convergence"
807  end if
808  end subroutine diffusion_solve_vcoeff
809 
810  !> Solve a diffusion equation implicitly, assuming anisotropic diffusion
811  !> coefficient which has been stored in mg_iveps1, mg_iveps2, mg_iveps3
812  !> (also on coarse grids). The
813  !> solution at time t should be stored in mg_iphi, which is on output replaced
814  !> by the solution at time t+dt.
815  subroutine diffusion_solve_acoeff(mg, dt, order, max_res)
816 
817  type(mg_t), intent(inout) :: mg
818  real(dp), intent(in) :: dt
819  integer, intent(in) :: order
820  real(dp), intent(in) :: max_res
821  integer, parameter :: max_its = 100
822  integer :: n
823  real(dp) :: res
824 
825  mg%operator_type = mg_ahelmholtz
826  call mg_set_methods(mg)
827 
828  select case (order)
829  case (1)
830  call ahelmholtz_set_lambda(1/dt)
831  call set_rhs(mg, -1/dt, 0.0_dp)
832  case (2)
833  call ahelmholtz_set_lambda(0.0d0)
834  call mg_apply_op(mg, mg_irhs)
835  call ahelmholtz_set_lambda(2/dt)
836  call set_rhs(mg, -2/dt, -1.0_dp)
837  case default
838  error stop "diffusion_solve: order should be 1 or 2"
839  end select
840 
841  ! Start with an FMG cycle
842  call mg_fas_fmg(mg, .true., max_res=res)
843 
844  ! Add V-cycles if necessary
845  do n = 1, max_its
846  print*, n, res
847  if (res <= max_res) exit
848  call mg_fas_vcycle(mg, max_res=res)
849  end do
850 
851  if (n == max_its + 1) then
852  if (mg%my_rank == 0) then
853  print *, "Did you specify boundary conditions correctly?"
854  print *, "Or is the variation in diffusion too large?"
855  end if
856  error stop "diffusion_solve: no convergence"
857  end if
858  end subroutine diffusion_solve_acoeff
859 
860  subroutine set_rhs(mg, f1, f2)
861  type(mg_t), intent(inout) :: mg
862  real(dp), intent(in) :: f1, f2
863  integer :: lvl, i, id, nc
864 
865  do lvl = 1, mg%highest_lvl
866  nc = mg%box_size_lvl(lvl)
867  do i = 1, size(mg%lvls(lvl)%my_leaves)
868  id = mg%lvls(lvl)%my_leaves(i)
869  mg%boxes(id)%cc(1:nc, 1:nc, mg_irhs) = &
870  f1 * mg%boxes(id)%cc(1:nc, 1:nc, mg_iphi) + &
871  f2 * mg%boxes(id)%cc(1:nc, 1:nc, mg_irhs)
872  end do
873  end do
874  end subroutine set_rhs
875 
876  !! File ../src/m_laplacian.f90
877 
878  subroutine laplacian_set_methods(mg)
879  type(mg_t), intent(inout) :: mg
880 
881  if (all(mg%periodic)) then
882  ! For a fully periodic Laplacian, remove the mean from the rhs and phi so
883  ! that a unique and periodic solution can be found
884  mg%subtract_mean = .true.
885  end if
886 
887  select case (mg%geometry_type)
888  case (mg_cartesian)
889  mg%box_op => box_lpl
890 
891  select case (mg%smoother_type)
892  ! case (smoother_jacobi)
893  ! mg%box_smoother => box_jacobi_lpl
895  mg%box_smoother => box_gs_lpl
896  case default
897  error stop "laplacian_set_methods: unsupported smoother type"
898  end select
899  case (mg_cylindrical)
900  mg%box_op => box_clpl
901 
902  select case (mg%smoother_type)
904  mg%box_smoother => box_gs_clpl
905  case default
906  error stop "laplacian_set_methods: unsupported smoother type"
907  end select
908  case default
909  error stop "laplacian_set_methods: unsupported geometry"
910  end select
911 
912  end subroutine laplacian_set_methods
913 
914  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator
915  subroutine box_gs_lpl(mg, id, nc, redblack_cntr)
916  type(mg_t), intent(inout) :: mg
917  integer, intent(in) :: id
918  integer, intent(in) :: nc
919  integer, intent(in) :: redblack_cntr !< Iteration counter
920  integer :: i, j, i0, di
921  real(dp) :: idr2(2), fac
922  logical :: redblack
923 
924  idr2 = 1/mg%dr(:, mg%boxes(id)%lvl)**2
925  fac = 0.5_dp / sum(idr2)
926  i0 = 1
927 
928  redblack = (mg%smoother_type == mg_smoother_gsrb)
929  if (redblack) then
930  di = 2
931  else
932  di = 1
933  end if
934 
935  ! The parity of redblack_cntr determines which cells we use. If
936  ! redblack_cntr is even, we use the even cells and vice versa.
937  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
938  do j = 1, nc
939  if (redblack) &
940  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
941 
942  do i = i0, nc, di
943  cc(i, j, n) = fac * ( &
944  idr2(1) * (cc(i+1, j, n) + cc(i-1, j, n)) + &
945  idr2(2) * (cc(i, j+1, n) + cc(i, j-1, n)) - &
946  cc(i, j, mg_irhs))
947  end do
948  end do
949  end associate
950  end subroutine box_gs_lpl
951 
952 ! !> Perform Jacobi relaxation on box for a Laplacian operator
953 ! subroutine box_jacobi_lpl(mg, id, nc, cntr)
954 ! type(mg_t), intent(inout) :: mg
955 ! integer, intent(in) :: id
956 ! integer, intent(in) :: nc
957 ! integer, intent(in) :: cntr !< Not used
958 ! integer :: i, j
959 ! real(dp), parameter :: w = 2.0_dp / 3
960 ! real(dp) :: tmp(0:nc+1, 0:nc+1)
961 ! real(dp) :: dr2
962 ! #if 2 == 3
963 ! real(dp), parameter :: sixth = 1/6.0_dp
964 ! #endif
965 
966 ! dr2 = mg%dr(mg%boxes(id)%lvl)**2
967 
968 ! associate (box => mg%boxes(id))
969 ! tmp = box%cc(:, :, mg_iphi)
970 ! do j=1, nc; do i=1, nc
971 ! #if 2 == 2
972 ! box%cc(i, j, mg_iphi) = (1-w) * box%cc(i, j, mg_iphi) + &
973 ! 0.25_dp * w * ( &
974 ! tmp(i+1, j) + tmp(i-1, j) + &
975 ! tmp(i, j+1) + tmp(i, j-1) - &
976 ! dr2 * box%cc(i, j, mg_irhs))
977 ! #elif 2 == 3
978 ! box%cc(i, j, k, mg_iphi) = (1-w) * &
979 ! box%cc(i, j, k, mg_iphi) + &
980 ! sixth * w * ( &
981 ! tmp(i+1, j, k) + tmp(i-1, j, k) + &
982 ! tmp(i, j+1, k) + tmp(i, j-1, k) + &
983 ! tmp(i, j, k+1) + tmp(i, j, k-1) - &
984 ! dr2 * box%cc(i, j, k, mg_irhs))
985 ! #endif
986 ! end do; end do
987 ! end associate
988 ! end subroutine box_jacobi_lpl
989 
990  !> Perform Laplacian operator on a box
991  subroutine box_lpl(mg, id, nc, i_out)
992  type(mg_t), intent(inout) :: mg
993  integer, intent(in) :: id
994  integer, intent(in) :: nc
995  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
996  integer :: i, j
997  real(dp) :: idr2(2)
998 
999  idr2 = 1 / mg%dr(:, mg%boxes(id)%lvl)**2
1000 
1001  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1002  do j = 1, nc
1003  do i = 1, nc
1004  cc(i, j, i_out) = &
1005  idr2(1) * (cc(i-1, j, n) + cc(i+1, j, n) - 2 * cc(i, j, n)) + &
1006  idr2(2) * (cc(i, j-1, n) + cc(i, j+1, n) - 2 * cc(i, j, n))
1007  end do
1008  end do
1009  end associate
1010  end subroutine box_lpl
1011 
1012  !> Perform Laplacian operator on a box in cylindrical geometry, using (r,z)
1013  !> and (r,phi,z) coordinates in 2D/3D.
1014  subroutine box_clpl(mg, id, nc, i_out)
1015  type(mg_t), intent(inout) :: mg
1016  integer, intent(in) :: id
1017  integer, intent(in) :: nc
1018  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
1019  integer :: i, j
1020  real(dp) :: dr(2), idr2(2)
1021  real(dp) :: r_face(nc+1), r_inv(nc)
1022 
1023  dr = mg%dr(:, mg%boxes(id)%lvl)
1024  idr2 = 1 / dr**2
1025  r_face = mg%boxes(id)%r_min(1) + dr(1) * [(i, i=0,nc)]
1026  r_inv = 1/(mg%boxes(id)%r_min(1) + dr(1) * [(i-0.5_dp, i=1,nc)])
1027 
1028  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1029  do j = 1, nc
1030  do i = 1, nc
1031  cc(i, j, i_out) = idr2(1) * (&
1032  r_face(i) * r_inv(i) * cc(i-1, j, n) + &
1033  r_face(i+1) * r_inv(i) * cc(i+1, j, n) &
1034  - 2 * cc(i, j, n)) &
1035  + idr2(2) * (cc(i, j-1, n) + cc(i, j+1, n) - 2 * cc(i, j, n))
1036  end do
1037  end do
1038  end associate
1039  end subroutine box_clpl
1040 
1041  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator in
1042  !> cylindrical geometry. TODO: in 3D this does not converge well, maybe it
1043  !> will for a stretched grid.
1044  subroutine box_gs_clpl(mg, id, nc, redblack_cntr)
1045  type(mg_t), intent(inout) :: mg
1046  integer, intent(in) :: id
1047  integer, intent(in) :: nc
1048  integer, intent(in) :: redblack_cntr !< Iteration counter
1049  integer :: i, j, i0, di
1050  logical :: redblack
1051  real(dp) :: idr2(2), dr(2), dr2(2), fac
1052  real(dp) :: r_face(nc+1), r_inv(nc)
1053 
1054  dr = mg%dr(:, mg%boxes(id)%lvl)
1055  dr2 = dr**2
1056  idr2 = 1/dr**2
1057  fac = 0.5_dp / sum(idr2)
1058  r_face = mg%boxes(id)%r_min(1) + dr(1) * [(i, i=0,nc)]
1059  r_inv = 1/(mg%boxes(id)%r_min(1) + dr(1) * [(i-0.5_dp, i=1,nc)])
1060 
1061  i0 = 1
1062  redblack = (mg%smoother_type == mg_smoother_gsrb)
1063  if (redblack) then
1064  di = 2
1065  else
1066  di = 1
1067  end if
1068 
1069  ! The parity of redblack_cntr determines which cells we use. If
1070  ! redblack_cntr is even, we use the even cells and vice versa.
1071  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1072  do j = 1, nc
1073  if (redblack) &
1074  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
1075 
1076  do i = i0, nc, di
1077  cc(i, j, n) = fac * (idr2(1) * &
1078  (r_face(i+1) * r_inv(i) * cc(i+1, j, n) + &
1079  r_face(i) * r_inv(i) * cc(i-1, j, n)) + &
1080  idr2(2) * (cc(i, j+1, n) + cc(i, j-1, n)) &
1081  - cc(i, j, mg_irhs))
1082  end do
1083  end do
1084  end associate
1085  end subroutine box_gs_clpl
1086 
1087  !! File ../src/m_vhelmholtz.f90
1088 
1089  subroutine vhelmholtz_set_methods(mg)
1090  type(mg_t), intent(inout) :: mg
1091 
1092  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
1093  error stop "vhelmholtz_set_methods: mg%n_extra_vars == 0"
1094  else
1095  mg%n_extra_vars = max(1, mg%n_extra_vars)
1096  end if
1097 
1098  mg%subtract_mean = .false.
1099 
1100  ! Use Neumann zero boundary conditions for the variable coefficient, since
1101  ! it is needed in ghost cells.
1102  mg%bc(:, mg_iveps)%bc_type = mg_bc_neumann
1103  mg%bc(:, mg_iveps)%bc_value = 0.0_dp
1104 
1105  select case (mg%geometry_type)
1106  case (mg_cartesian)
1107  mg%box_op => box_vhelmh
1108 
1109  select case (mg%smoother_type)
1111  mg%box_smoother => box_gs_vhelmh
1112  case default
1113  error stop "vhelmholtz_set_methods: unsupported smoother type"
1114  end select
1115  case default
1116  error stop "vhelmholtz_set_methods: unsupported geometry"
1117  end select
1118 
1119  end subroutine vhelmholtz_set_methods
1120 
1121  subroutine vhelmholtz_set_lambda(lambda)
1122  real(dp), intent(in) :: lambda
1123 
1124  if (lambda < 0) &
1125  error stop "vhelmholtz_set_lambda: lambda < 0 not allowed"
1126 
1127  vhelmholtz_lambda = lambda
1128  end subroutine vhelmholtz_set_lambda
1129 
1130  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
1131  subroutine box_gs_vhelmh(mg, id, nc, redblack_cntr)
1132  type(mg_t), intent(inout) :: mg
1133  integer, intent(in) :: id
1134  integer, intent(in) :: nc
1135  integer, intent(in) :: redblack_cntr !< Iteration counter
1136  integer :: i, j, i0, di
1137  logical :: redblack
1138  real(dp) :: idr2(2*2), u(2*2)
1139  real(dp) :: a0, a(2*2), c(2*2)
1140 
1141  ! Duplicate 1/dr^2 array to multiply neighbor values
1142  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1143  idr2(2:2*2:2) = idr2(1:2*2:2)
1144  i0 = 1
1145 
1146  redblack = (mg%smoother_type == mg_smoother_gsrb)
1147  if (redblack) then
1148  di = 2
1149  else
1150  di = 1
1151  end if
1152 
1153  ! The parity of redblack_cntr determines which cells we use. If
1154  ! redblack_cntr is even, we use the even cells and vice versa.
1155  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1156  i_eps => mg_iveps)
1157  do j = 1, nc
1158  if (redblack) &
1159  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
1160 
1161  do i = i0, nc, di
1162  a0 = cc(i, j, i_eps)
1163  u(1:2) = cc(i-1:i+1:2, j, n)
1164  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1165  u(3:4) = cc(i, j-1:j+1:2, n)
1166  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1167  c(:) = 2 * a0 * a(:) / (a0 + a(:)) * idr2
1168 
1169  cc(i, j, n) = &
1170  (sum(c(:) * u(:)) - cc(i, j, mg_irhs)) / &
1171  (sum(c(:)) + vhelmholtz_lambda)
1172  end do
1173  end do
1174  end associate
1175  end subroutine box_gs_vhelmh
1176 
1177  !> Perform Helmholtz operator on a box
1178  subroutine box_vhelmh(mg, id, nc, i_out)
1179  type(mg_t), intent(inout) :: mg
1180  integer, intent(in) :: id
1181  integer, intent(in) :: nc
1182  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
1183  integer :: i, j
1184  real(dp) :: idr2(2*2), a0, u0, u(2*2), a(2*2)
1185 
1186  ! Duplicate 1/dr^2 array to multiply neighbor values
1187  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1188  idr2(2:2*2:2) = idr2(1:2*2:2)
1189 
1190  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1191  i_eps => mg_iveps)
1192  do j = 1, nc
1193  do i = 1, nc
1194  a0 = cc(i, j, i_eps)
1195  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1196  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1197  u0 = cc(i, j, n)
1198  u(1:2) = cc(i-1:i+1:2, j, n)
1199  u(3:4) = cc(i, j-1:j+1:2, n)
1200 
1201  cc(i, j, i_out) = sum(2 * idr2 * &
1202  a0*a(:)/(a0 + a(:)) * (u(:) - u0)) - &
1203  vhelmholtz_lambda * u0
1204  end do
1205  end do
1206  end associate
1207  end subroutine box_vhelmh
1208 
1209  !! File ../src/m_build_tree.f90
1210 
1211  subroutine mg_build_rectangle(mg, domain_size, box_size, dx, r_min, &
1212  periodic, n_finer)
1213  type(mg_t), intent(inout) :: mg
1214  integer, intent(in) :: domain_size(2)
1215  integer, intent(in) :: box_size
1216  real(dp), intent(in) :: dx(2)
1217  real(dp), intent(in) :: r_min(2)
1218  logical, intent(in) :: periodic(2)
1219  integer, intent(in) :: n_finer
1220  integer :: i, j, lvl, n, id, nx(2)
1221  integer :: boxes_per_dim(2, mg_lvl_lo:1)
1222  integer :: periodic_offset(2)
1223 
1224  if (modulo(box_size, 2) /= 0) &
1225  error stop "box_size should be even"
1226  if (any(modulo(domain_size, box_size) /= 0)) &
1227  error stop "box_size does not divide domain_size"
1228 
1229  if (all(periodic)) then
1230  mg%subtract_mean = .true.
1231  end if
1232 
1233  nx = domain_size
1234  mg%box_size = box_size
1235  mg%box_size_lvl(1) = box_size
1236  mg%domain_size_lvl(:, 1) = domain_size
1237  mg%first_normal_lvl = 1
1238  mg%dr(:, 1) = dx
1239  mg%r_min(:) = r_min
1240  mg%periodic = periodic
1241  boxes_per_dim(:, :) = 0
1242  boxes_per_dim(:, 1) = domain_size / box_size
1243 
1244  do lvl = 1, mg_lvl_lo+1, -1
1245  ! For a Gauss-Seidel (non red-black) smoother, we should avoid boxes
1246  ! containing a single cell
1247  if (any(modulo(nx, 2) == 1 .or. nx == mg%coarsest_grid .or. &
1248  (mg%box_size_lvl(lvl) == mg%coarsest_grid .and. &
1249  mg%smoother_type == mg_smoother_gs))) exit
1250 
1251  if (all(modulo(nx/mg%box_size_lvl(lvl), 2) == 0)) then
1252  mg%box_size_lvl(lvl-1) = mg%box_size_lvl(lvl)
1253  boxes_per_dim(:, lvl-1) = boxes_per_dim(:, lvl)/2
1254  mg%first_normal_lvl = lvl-1
1255  else
1256  mg%box_size_lvl(lvl-1) = mg%box_size_lvl(lvl)/2
1257  boxes_per_dim(:, lvl-1) = boxes_per_dim(:, lvl)
1258  end if
1259 
1260  mg%dr(:, lvl-1) = mg%dr(:, lvl) * 2
1261  nx = nx / 2
1262  mg%domain_size_lvl(:, lvl-1) = nx
1263  end do
1264 
1265  mg%lowest_lvl = lvl
1266  mg%highest_lvl = 1
1267 
1268  do lvl = 2, mg_lvl_hi
1269  mg%dr(:, lvl) = mg%dr(:, lvl-1) * 0.5_dp
1270  mg%box_size_lvl(lvl) = box_size
1271  mg%domain_size_lvl(:, lvl) = 2 * mg%domain_size_lvl(:, lvl-1)
1272  end do
1273 
1274  n = sum(product(boxes_per_dim, dim=1)) + n_finer
1275  allocate(mg%boxes(n))
1276 
1277  ! Create lowest level
1278  nx = boxes_per_dim(:, mg%lowest_lvl)
1279  periodic_offset = [nx(1)-1, (nx(2)-1)*nx(1)]
1280 
1281  do j=1,nx(2); do i=1,nx(1)
1282  mg%n_boxes = mg%n_boxes + 1
1283  n = mg%n_boxes
1284 
1285  mg%boxes(n)%rank = 0
1286  mg%boxes(n)%id = n
1287  mg%boxes(n)%lvl = mg%lowest_lvl
1288  mg%boxes(n)%ix(:) = [i, j]
1289  mg%boxes(n)%r_min(:) = r_min + (mg%boxes(n)%ix(:) - 1) * &
1290  mg%box_size_lvl(mg%lowest_lvl) * mg%dr(:, mg%lowest_lvl)
1291  mg%boxes(n)%dr(:) = mg%dr(:, mg%lowest_lvl)
1292  mg%boxes(n)%parent = mg_no_box
1293  mg%boxes(n)%children(:) = mg_no_box
1294 
1295  ! Set default neighbors
1296  mg%boxes(n)%neighbors(:) = [n-1, n+1, n-nx(1), n+nx(1)]
1297 
1298  ! Handle boundaries
1299  where ([i, j] == 1 .and. .not. periodic)
1300  mg%boxes(n)%neighbors(1:mg_num_neighbors:2) = &
1302  end where
1303  where ([i, j] == 1 .and. periodic)
1304  mg%boxes(n)%neighbors(1:mg_num_neighbors:2) = &
1305  n + periodic_offset
1306  end where
1307 
1308  where ([i, j] == nx .and. .not. periodic)
1309  mg%boxes(n)%neighbors(2:mg_num_neighbors:2) = &
1311  end where
1312  where ([i, j] == nx .and. periodic)
1313  mg%boxes(n)%neighbors(2:mg_num_neighbors:2) = &
1314  n - periodic_offset
1315  end where
1316  end do; end do
1317 
1318  mg%lvls(mg%lowest_lvl)%ids = [(n, n=1, mg%n_boxes)]
1319 
1320  ! Add higher levels
1321  do lvl = mg%lowest_lvl, 0
1322  if (mg%box_size_lvl(lvl+1) == mg%box_size_lvl(lvl)) then
1323  do i = 1, size(mg%lvls(lvl)%ids)
1324  id = mg%lvls(lvl)%ids(i)
1325  call mg_add_children(mg, id)
1326  end do
1327 
1328  call mg_set_leaves_parents(mg%boxes, mg%lvls(lvl))
1329  call mg_set_next_level_ids(mg, lvl)
1330  call mg_set_neighbors_lvl(mg, lvl+1)
1331  else
1332  do i = 1, size(mg%lvls(lvl)%ids)
1333  id = mg%lvls(lvl)%ids(i)
1334  call add_single_child(mg, id, size(mg%lvls(lvl)%ids))
1335  end do
1336 
1337  call mg_set_leaves_parents(mg%boxes, mg%lvls(lvl))
1338  call mg_set_next_level_ids(mg, lvl)
1339  end if
1340  end do
1341 
1342  call mg_set_leaves_parents(mg%boxes, mg%lvls(1))
1343 
1344  ! No refinement boundaries
1345  do lvl = mg%lowest_lvl, 1
1346  if (allocated(mg%lvls(lvl)%ref_bnds)) &
1347  deallocate(mg%lvls(lvl)%ref_bnds)
1348  allocate(mg%lvls(lvl)%ref_bnds(0))
1349  end do
1350 
1351  mg%tree_created = .true.
1352  end subroutine mg_build_rectangle
1353 
1354  subroutine mg_set_neighbors_lvl(mg, lvl)
1355  type(mg_t), intent(inout) :: mg
1356  integer, intent(in) :: lvl
1357  integer :: i, id
1358 
1359  do i = 1, size(mg%lvls(lvl)%ids)
1360  id = mg%lvls(lvl)%ids(i)
1361  call set_neighbs(mg%boxes, id)
1362  end do
1363  end subroutine mg_set_neighbors_lvl
1364 
1365  subroutine mg_set_next_level_ids(mg, lvl)
1366  type(mg_t), intent(inout) :: mg
1367  integer, intent(in) :: lvl
1368  integer :: n, i, id
1369 
1370  if (allocated(mg%lvls(lvl+1)%ids)) &
1371  deallocate(mg%lvls(lvl+1)%ids)
1372 
1373  ! Set next level ids to children of this level
1374  if (mg%box_size_lvl(lvl+1) == mg%box_size_lvl(lvl)) then
1375  n = mg_num_children * size(mg%lvls(lvl)%parents)
1376  allocate(mg%lvls(lvl+1)%ids(n))
1377 
1378  n = mg_num_children
1379  do i = 1, size(mg%lvls(lvl)%parents)
1380  id = mg%lvls(lvl)%parents(i)
1381  mg%lvls(lvl+1)%ids(n*(i-1)+1:n*i) = mg%boxes(id)%children
1382  end do
1383  else
1384  n = size(mg%lvls(lvl)%parents)
1385  allocate(mg%lvls(lvl+1)%ids(n))
1386 
1387  n = 1
1388  do i = 1, size(mg%lvls(lvl)%parents)
1389  id = mg%lvls(lvl)%parents(i)
1390  mg%lvls(lvl+1)%ids(i) = mg%boxes(id)%children(1)
1391  end do
1392  end if
1393 
1394  end subroutine mg_set_next_level_ids
1395 
1396  ! Set the neighbors of id (using their parent)
1397  subroutine set_neighbs(boxes, id)
1398  type(mg_box_t), intent(inout) :: boxes(:)
1399  integer, intent(in) :: id
1400  integer :: nb, nb_id
1401 
1402  do nb = 1, mg_num_neighbors
1403  if (boxes(id)%neighbors(nb) == mg_no_box) then
1404  nb_id = find_neighb(boxes, id, nb)
1405  if (nb_id > mg_no_box) then
1406  boxes(id)%neighbors(nb) = nb_id
1407  boxes(nb_id)%neighbors(mg_neighb_rev(nb)) = id
1408  end if
1409  end if
1410  end do
1411  end subroutine set_neighbs
1412 
1413  !> Get the id of neighbor nb of boxes(id), through its parent
1414  function find_neighb(boxes, id, nb) result(nb_id)
1415  type(mg_box_t), intent(in) :: boxes(:) !< List with all the boxes
1416  integer, intent(in) :: id !< Box whose neighbor we are looking for
1417  integer, intent(in) :: nb !< Neighbor index
1418  integer :: nb_id, p_id, c_ix, d, old_pid
1419 
1420  p_id = boxes(id)%parent
1421  old_pid = p_id
1422  c_ix = mg_ix_to_ichild(boxes(id)%ix)
1423  d = mg_neighb_dim(nb)
1424 
1425  ! Check if neighbor is in same direction as ix is (low/high). If so,
1426  ! use neighbor of parent
1427  if (mg_child_low(d, c_ix) .eqv. mg_neighb_low(nb)) then
1428  p_id = boxes(p_id)%neighbors(nb)
1429  end if
1430 
1431  ! The child ix of the neighbor is reversed in direction d
1432  nb_id = boxes(p_id)%children(mg_child_rev(c_ix, d))
1433  end function find_neighb
1434 
1435  !> Create a list of leaves and a list of parents for a level
1436  subroutine mg_set_leaves_parents(boxes, level)
1437  type(mg_box_t), intent(in) :: boxes(:) !< List of boxes
1438  type(mg_lvl_t), intent(inout) :: level !< Level type which contains the indices of boxes
1439  integer :: i, id, i_leaf, i_parent
1440  integer :: n_parents, n_leaves
1441 
1442  n_parents = count(mg_has_children(boxes(level%ids)))
1443  n_leaves = size(level%ids) - n_parents
1444 
1445  if (.not. allocated(level%parents)) then
1446  allocate(level%parents(n_parents))
1447  else if (n_parents /= size(level%parents)) then
1448  deallocate(level%parents)
1449  allocate(level%parents(n_parents))
1450  end if
1451 
1452  if (.not. allocated(level%leaves)) then
1453  allocate(level%leaves(n_leaves))
1454  else if (n_leaves /= size(level%leaves)) then
1455  deallocate(level%leaves)
1456  allocate(level%leaves(n_leaves))
1457  end if
1458 
1459  i_leaf = 0
1460  i_parent = 0
1461  do i = 1, size(level%ids)
1462  id = level%ids(i)
1463  if (mg_has_children(boxes(id))) then
1464  i_parent = i_parent + 1
1465  level%parents(i_parent) = id
1466  else
1467  i_leaf = i_leaf + 1
1468  level%leaves(i_leaf) = id
1469  end if
1470  end do
1471  end subroutine mg_set_leaves_parents
1472 
1473  !> Create a list of refinement boundaries (from the coarse side)
1474  subroutine mg_set_refinement_boundaries(boxes, level)
1475  type(mg_box_t), intent(in) :: boxes(:)
1476  type(mg_lvl_t), intent(inout) :: level
1477  integer, allocatable :: tmp(:)
1478  integer :: i, id, nb, nb_id, ix
1479 
1480  if (allocated(level%ref_bnds)) deallocate(level%ref_bnds)
1481 
1482  if (size(level%parents) == 0) then
1483  ! There are no refinement boundaries
1484  allocate(level%ref_bnds(0))
1485  else
1486  allocate(tmp(size(level%leaves)))
1487  ix = 0
1488  do i = 1, size(level%leaves)
1489  id = level%leaves(i)
1490 
1491  do nb = 1, mg_num_neighbors
1492  nb_id = boxes(id)%neighbors(nb)
1493  if (nb_id > mg_no_box) then
1494  if (mg_has_children(boxes(nb_id))) then
1495  ix = ix + 1
1496  tmp(ix) = id
1497  exit
1498  end if
1499  end if
1500  end do
1501  end do
1502 
1503  allocate(level%ref_bnds(ix))
1504  level%ref_bnds(:) = tmp(1:ix)
1505  end if
1506  end subroutine mg_set_refinement_boundaries
1507 
1508  subroutine mg_add_children(mg, id)
1509  type(mg_t), intent(inout) :: mg
1510  integer, intent(in) :: id !< Id of box that gets children
1511  integer :: lvl, i, nb, child_nb(2**(2-1))
1512  integer :: c_ids(mg_num_children)
1513  integer :: c_id, c_ix_base(2)
1514 
1515  if (mg%n_boxes + mg_num_children > size(mg%boxes)) then
1516  error stop "mg_add_children: not enough space"
1517  end if
1518 
1519  c_ids = [(mg%n_boxes+i, i=1,mg_num_children)]
1520  mg%n_boxes = mg%n_boxes + mg_num_children
1521  mg%boxes(id)%children = c_ids
1522  c_ix_base = 2 * mg%boxes(id)%ix - 1
1523  lvl = mg%boxes(id)%lvl+1
1524 
1525  do i = 1, mg_num_children
1526  c_id = c_ids(i)
1527  mg%boxes(c_id)%rank = mg%boxes(id)%rank
1528  mg%boxes(c_id)%ix = c_ix_base + mg_child_dix(:, i)
1529  mg%boxes(c_id)%lvl = lvl
1530  mg%boxes(c_id)%parent = id
1531  mg%boxes(c_id)%children = mg_no_box
1532  mg%boxes(c_id)%neighbors = mg_no_box
1533  mg%boxes(c_id)%r_min = mg%boxes(id)%r_min + &
1534  mg%dr(:, lvl) * mg_child_dix(:, i) * mg%box_size
1535  mg%boxes(c_id)%dr(:) = mg%dr(:, lvl)
1536  end do
1537 
1538  ! Set boundary conditions at children
1539  do nb = 1, mg_num_neighbors
1540  if (mg%boxes(id)%neighbors(nb) < mg_no_box) then
1541  child_nb = c_ids(mg_child_adj_nb(:, nb)) ! Neighboring children
1542  mg%boxes(child_nb)%neighbors(nb) = mg%boxes(id)%neighbors(nb)
1543  end if
1544  end do
1545  end subroutine mg_add_children
1546 
1547  subroutine add_single_child(mg, id, n_boxes_lvl)
1548  type(mg_t), intent(inout) :: mg
1549  integer, intent(in) :: id !< Id of box that gets children
1550  integer, intent(in) :: n_boxes_lvl
1551  integer :: lvl, c_id
1552 
1553  c_id = mg%n_boxes + 1
1554  mg%n_boxes = mg%n_boxes + 1
1555  mg%boxes(id)%children(1) = c_id
1556  lvl = mg%boxes(id)%lvl+1
1557 
1558  mg%boxes(c_id)%rank = mg%boxes(id)%rank
1559  mg%boxes(c_id)%ix = mg%boxes(id)%ix
1560  mg%boxes(c_id)%lvl = lvl
1561  mg%boxes(c_id)%parent = id
1562  mg%boxes(c_id)%children = mg_no_box
1563  where (mg%boxes(id)%neighbors == mg_physical_boundary)
1564  mg%boxes(c_id)%neighbors = mg%boxes(id)%neighbors
1565  elsewhere
1566  mg%boxes(c_id)%neighbors = mg%boxes(id)%neighbors + n_boxes_lvl
1567  end where
1568  mg%boxes(c_id)%r_min = mg%boxes(id)%r_min
1569  mg%boxes(c_id)%dr(:) = mg%dr(:, lvl)
1570 
1571  end subroutine add_single_child
1572 
1573  !! File ../src/m_load_balance.f90
1574 
1575  !> Load balance all boxes in the multigrid tree, by simply distributing the
1576  !> load per grid level. This method will only work well for uniform grids.
1577  !>
1578  !> Note that in a typical application the load balancing of the leaves is
1579  !> already determined, then mg_load_balance_parents can be used.
1580  subroutine mg_load_balance_simple(mg)
1581  type(mg_t), intent(inout) :: mg
1582  integer :: i, id, lvl, single_cpu_lvl
1583  integer :: work_left, my_work, i_cpu
1584 
1585  ! Up to this level, all boxes have to be on a single processor because they
1586  ! have a different size and the communication routines do not support this
1587  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1588 
1589  do lvl = mg%lowest_lvl, single_cpu_lvl
1590  do i = 1, size(mg%lvls(lvl)%ids)
1591  id = mg%lvls(lvl)%ids(i)
1592  mg%boxes(id)%rank = 0
1593  end do
1594  end do
1595 
1596  ! Distribute the boxes equally. Due to the way the mesh is constructed, the
1597  ! mg%lvls(lvl)%ids array already contains a Morton-like ordering.
1598  do lvl = single_cpu_lvl+1, mg%highest_lvl
1599  work_left = size(mg%lvls(lvl)%ids)
1600  my_work = 0
1601  i_cpu = 0
1602 
1603  do i = 1, size(mg%lvls(lvl)%ids)
1604  if ((mg%n_cpu - i_cpu - 1) * my_work >= work_left) then
1605  i_cpu = i_cpu + 1
1606  my_work = 0
1607  end if
1608 
1609  my_work = my_work + 1
1610  work_left = work_left - 1
1611 
1612  id = mg%lvls(lvl)%ids(i)
1613  mg%boxes(id)%rank = i_cpu
1614  end do
1615  end do
1616 
1617  do lvl = mg%lowest_lvl, mg%highest_lvl
1618  call update_lvl_info(mg, mg%lvls(lvl))
1619  end do
1620 
1621  end subroutine mg_load_balance_simple
1622 
1623  !> Load balance all boxes in the multigrid tree. Compared to
1624  !> mg_load_balance_simple, this method does a better job of setting the ranks
1625  !> of parent boxes
1626  !>
1627  !> Note that in a typical application the load balancing of the leaves is
1628  !> already determined, then mg_load_balance_parents can be used.
1629  subroutine mg_load_balance(mg)
1630  type(mg_t), intent(inout) :: mg
1631  integer :: i, id, lvl, single_cpu_lvl
1632  integer :: work_left, my_work(0:mg%n_cpu), i_cpu
1633  integer :: c_ids(mg_num_children)
1634  integer :: c_ranks(mg_num_children)
1635  integer :: coarse_rank
1636 
1637  ! Up to this level, all boxes have to be on a single processor because they
1638  ! have a different size and the communication routines do not support this
1639  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1640 
1641  ! Distribute the boxes equally. Due to the way the mesh is constructed, the
1642  ! mg%lvls(lvl)%ids array already contains a Morton-like ordering.
1643  do lvl = mg%highest_lvl, single_cpu_lvl+1, -1
1644  ! For parents determine the rank based on their child ranks
1645  my_work(:) = 0
1646 
1647  do i = 1, size(mg%lvls(lvl)%parents)
1648  id = mg%lvls(lvl)%parents(i)
1649 
1650  c_ids = mg%boxes(id)%children
1651  c_ranks = mg%boxes(c_ids)%rank
1652  i_cpu = most_popular(c_ranks, my_work, mg%n_cpu)
1653  mg%boxes(id)%rank = i_cpu
1654  my_work(i_cpu) = my_work(i_cpu) + 1
1655  end do
1656 
1657  work_left = size(mg%lvls(lvl)%leaves)
1658  i_cpu = 0
1659 
1660  do i = 1, size(mg%lvls(lvl)%leaves)
1661  ! Skip this CPU if it already has enough work
1662  if ((mg%n_cpu - i_cpu - 1) * my_work(i_cpu) >= &
1663  work_left + sum(my_work(i_cpu+1:))) then
1664  i_cpu = i_cpu + 1
1665  end if
1666 
1667  my_work(i_cpu) = my_work(i_cpu) + 1
1668  work_left = work_left - 1
1669 
1670  id = mg%lvls(lvl)%leaves(i)
1671  mg%boxes(id)%rank = i_cpu
1672  end do
1673  end do
1674 
1675  ! Determine most popular CPU for coarse grids
1676  if (single_cpu_lvl < mg%highest_lvl) then
1677  coarse_rank = most_popular(mg%boxes(&
1678  mg%lvls(single_cpu_lvl+1)%ids)%rank, my_work, mg%n_cpu)
1679  else
1680  coarse_rank = 0
1681  end if
1682 
1683  do lvl = mg%lowest_lvl, single_cpu_lvl
1684  do i = 1, size(mg%lvls(lvl)%ids)
1685  id = mg%lvls(lvl)%ids(i)
1686  mg%boxes(id)%rank = coarse_rank
1687  end do
1688  end do
1689 
1690  do lvl = mg%lowest_lvl, mg%highest_lvl
1691  call update_lvl_info(mg, mg%lvls(lvl))
1692  end do
1693 
1694  end subroutine mg_load_balance
1695 
1696  !> Load balance the parents (non-leafs). Assign them to the rank that has most
1697  !> children.
1699  type(mg_t), intent(inout) :: mg
1700  integer :: i, id, lvl
1701  integer :: c_ids(mg_num_children)
1702  integer :: c_ranks(mg_num_children)
1703  integer :: single_cpu_lvl, coarse_rank
1704  integer :: my_work(0:mg%n_cpu), i_cpu
1705 
1706  ! Up to this level, all boxes have to be on a single processor because they
1707  ! have a different size and the communication routines do not support this
1708  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1709 
1710  do lvl = mg%highest_lvl-1, single_cpu_lvl+1, -1
1711  my_work(:) = 0
1712 
1713  ! Determine amount of work for the leaves
1714  do i = 1, size(mg%lvls(lvl)%leaves)
1715  id = mg%lvls(lvl)%leaves(i)
1716  i_cpu = mg%boxes(id)%rank
1717  my_work(i_cpu) = my_work(i_cpu) + 1
1718  end do
1719 
1720  do i = 1, size(mg%lvls(lvl)%parents)
1721  id = mg%lvls(lvl)%parents(i)
1722 
1723  c_ids = mg%boxes(id)%children
1724  c_ranks = mg%boxes(c_ids)%rank
1725  i_cpu = most_popular(c_ranks, my_work, mg%n_cpu)
1726  mg%boxes(id)%rank = i_cpu
1727  my_work(i_cpu) = my_work(i_cpu) + 1
1728  end do
1729 
1730  end do
1731 
1732  ! Determine most popular CPU for coarse grids
1733  if (single_cpu_lvl < mg%highest_lvl) then
1734  coarse_rank = most_popular(mg%boxes(&
1735  mg%lvls(single_cpu_lvl+1)%ids)%rank, my_work, mg%n_cpu)
1736  else
1737  coarse_rank = 0
1738  end if
1739 
1740  do lvl = mg%lowest_lvl, single_cpu_lvl
1741  do i = 1, size(mg%lvls(lvl)%ids)
1742  id = mg%lvls(lvl)%ids(i)
1743  mg%boxes(id)%rank = coarse_rank
1744  end do
1745  end do
1746 
1747  do lvl = mg%lowest_lvl, mg%highest_lvl
1748  call update_lvl_info(mg, mg%lvls(lvl))
1749  end do
1750 
1751  end subroutine mg_load_balance_parents
1752 
1753  !> Determine most popular rank in the list. In case of ties, assign the rank
1754  !> with the least work.
1755  pure integer function most_popular(list, work, n_cpu)
1756  integer, intent(in) :: list(:) !< List of MPI ranks
1757  integer, intent(in) :: n_cpu
1758  integer, intent(in) :: work(0:n_cpu-1) !< Existing work per rank
1759  integer :: i, best_count, current_count
1760  integer :: my_work, best_work
1761 
1762  best_count = 0
1763  best_work = 0
1764  most_popular = -1
1765 
1766  do i = 1, size(list)
1767  current_count = count(list == list(i))
1768  my_work = work(list(i))
1769 
1770  ! In case of ties, select task with lowest work
1771  if (current_count > best_count .or. &
1772  current_count == best_count .and. my_work < best_work) then
1773  best_count = current_count
1774  best_work = my_work
1775  most_popular = list(i)
1776  end if
1777  end do
1778 
1779  end function most_popular
1780 
1781  subroutine update_lvl_info(mg, lvl)
1782  type(mg_t), intent(inout) :: mg
1783  type(mg_lvl_t), intent(inout) :: lvl
1784 
1785  lvl%my_ids = pack(lvl%ids, &
1786  mg%boxes(lvl%ids)%rank == mg%my_rank)
1787  lvl%my_leaves = pack(lvl%leaves, &
1788  mg%boxes(lvl%leaves)%rank == mg%my_rank)
1789  lvl%my_parents = pack(lvl%parents, &
1790  mg%boxes(lvl%parents)%rank == mg%my_rank)
1791  lvl%my_ref_bnds = pack(lvl%ref_bnds, &
1792  mg%boxes(lvl%ref_bnds)%rank == mg%my_rank)
1793  end subroutine update_lvl_info
1794 
1795  !! File ../src/m_vlaplacian.f90
1796 
1797  subroutine vlaplacian_set_methods(mg)
1798  type(mg_t), intent(inout) :: mg
1799 
1800  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
1801  error stop "vlaplacian_set_methods: mg%n_extra_vars == 0"
1802  else
1803  mg%n_extra_vars = max(1, mg%n_extra_vars)
1804  end if
1805 
1806  mg%subtract_mean = .false.
1807 
1808  ! Use Neumann zero boundary conditions for the variable coefficient, since
1809  ! it is needed in ghost cells.
1810  mg%bc(:, mg_iveps)%bc_type = mg_bc_neumann
1811  mg%bc(:, mg_iveps)%bc_value = 0.0_dp
1812 
1813  select case (mg%geometry_type)
1814  case (mg_cartesian)
1815  mg%box_op => box_vlpl
1816 
1817  ! TODO: test whether a custom prolongation works better
1818  ! mg%box_prolong => vlpl_prolong
1819 
1820  select case (mg%smoother_type)
1822  mg%box_smoother => box_gs_vlpl
1823  case default
1824  error stop "vlaplacian_set_methods: unsupported smoother type"
1825  end select
1826 
1827  case default
1828  error stop "vlaplacian_set_methods: unsupported geometry"
1829  end select
1830 
1831  end subroutine vlaplacian_set_methods
1832 
1833  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator
1834  subroutine box_gs_vlpl(mg, id, nc, redblack_cntr)
1835  type(mg_t), intent(inout) :: mg
1836  integer, intent(in) :: id
1837  integer, intent(in) :: nc
1838  integer, intent(in) :: redblack_cntr !< Iteration counter
1839  integer :: i, j, i0, di
1840  logical :: redblack
1841  real(dp) :: idr2(2*2), u(2*2)
1842  real(dp) :: a0, a(2*2), c(2*2)
1843 
1844  ! Duplicate 1/dr^2 array to multiply neighbor values
1845  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1846  idr2(2:2*2:2) = idr2(1:2*2:2)
1847  i0 = 1
1848 
1849  redblack = (mg%smoother_type == mg_smoother_gsrb)
1850  if (redblack) then
1851  di = 2
1852  else
1853  di = 1
1854  end if
1855 
1856  ! The parity of redblack_cntr determines which cells we use. If
1857  ! redblack_cntr is even, we use the even cells and vice versa.
1858  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1859  i_eps => mg_iveps)
1860  do j = 1, nc
1861  if (redblack) &
1862  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
1863 
1864  do i = i0, nc, di
1865  a0 = cc(i, j, i_eps)
1866  u(1:2) = cc(i-1:i+1:2, j, n)
1867  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1868  u(3:4) = cc(i, j-1:j+1:2, n)
1869  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1870  c(:) = 2 * a0 * a(:) / (a0 + a(:)) * idr2
1871 
1872  cc(i, j, n) = &
1873  (sum(c(:) * u(:)) - cc(i, j, mg_irhs)) / sum(c(:))
1874  end do
1875  end do
1876  end associate
1877  end subroutine box_gs_vlpl
1878 
1879  !> Perform Laplacian operator on a box
1880  subroutine box_vlpl(mg, id, nc, i_out)
1881  type(mg_t), intent(inout) :: mg
1882  integer, intent(in) :: id
1883  integer, intent(in) :: nc
1884  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
1885  integer :: i, j
1886  real(dp) :: idr2(2*2), a0, u0, u(2*2), a(2*2)
1887 
1888  ! Duplicate 1/dr^2 array to multiply neighbor values
1889  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1890  idr2(2:2*2:2) = idr2(1:2*2:2)
1891 
1892  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1893  i_eps => mg_iveps)
1894  do j = 1, nc
1895  do i = 1, nc
1896  a0 = cc(i, j, i_eps)
1897  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1898  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1899  u0 = cc(i, j, n)
1900  u(1:2) = cc(i-1:i+1:2, j, n)
1901  u(3:4) = cc(i, j-1:j+1:2, n)
1902 
1903  cc(i, j, i_out) = sum(2 * idr2 * &
1904  a0*a(:)/(a0 + a(:)) * (u(:) - u0))
1905  end do
1906  end do
1907  end associate
1908  end subroutine box_vlpl
1909 
1910  !> Prolong from a parent to a child with index offset dix. This method could
1911  !> sometimes work better than the default prolongation, which does not take
1912  !> the variation in epsilon into account
1913 ! subroutine vlpl_prolong(mg, p_id, dix, nc, iv, fine)
1914 ! type(mg_t), intent(inout) :: mg
1915 ! integer, intent(in) :: p_id !< Id of parent
1916 ! integer, intent(in) :: dix(2) !< Offset of child in parent grid
1917 ! integer, intent(in) :: nc !< Child grid size
1918 ! integer, intent(in) :: iv !< Prolong from this variable
1919 ! real(dp), intent(out) :: fine(nc, nc) !< Prolonged values
1920 
1921 ! integer :: i, j, hnc
1922 ! #if 2 == 2
1923 ! integer :: ic, jc
1924 ! real(dp) :: f0, flx, fhx, fly, fhy
1925 ! real(dp) :: c0, clx, chx, cly, chy
1926 ! #elif 2 == 3
1927 ! integer :: ic, jc, kc
1928 ! real(dp) :: f0, flx, fhx, fly, fhy, flz, fhz
1929 ! real(dp) :: c0, clx, chx, cly, chy, clz, chz
1930 ! #endif
1931 
1932 ! hnc = nc/2
1933 
1934 ! associate (crs => mg%boxes(p_id)%cc, &
1935 ! i_eps => mg_iveps)
1936 ! #if 2 == 2
1937 ! do j = 1, hnc
1938 ! jc = j + dix(2)
1939 ! do i = 1, hnc
1940 ! ic = i + dix(1)
1941 
1942 ! f0 = crs(ic, jc, iv)
1943 ! flx = crs(ic-1, jc, iv)
1944 ! fhx = crs(ic+1, jc, iv)
1945 ! fly = crs(ic, jc-1, iv)
1946 ! fhy = crs(ic, jc+1, iv)
1947 
1948 ! c0 = 2 * crs(ic, jc, i_eps)
1949 ! clx = crs(ic-1, jc, i_eps)
1950 ! chx = crs(ic+1, jc, i_eps)
1951 ! cly = crs(ic, jc-1, i_eps)
1952 ! chy = crs(ic, jc+1, i_eps)
1953 
1954 ! fine(2*i-1, 2*j-1) = (f0*c0 + flx*clx + fly*cly) / &
1955 ! (c0 + clx + cly)
1956 ! fine(2*i , 2*j-1) = (f0*c0 + fhx*chx + fly*cly) / &
1957 ! (c0 + chx + cly)
1958 ! fine(2*i-1, 2*j) = (f0*c0 + flx*clx + fhy*chy) / &
1959 ! (c0 + clx + chy)
1960 ! fine(2*i , 2*j) = (f0*c0 + fhx*chx + fhy*chy) / &
1961 ! (c0 + chx + chy)
1962 ! end do
1963 ! end do
1964 ! #elif 2 == 3
1965 ! do k = 1, hnc
1966 ! kc = k + dix(3)
1967 ! do j = 1, hnc
1968 ! jc = j + dix(2)
1969 ! do i = 1, hnc
1970 ! ic = i + dix(1)
1971 
1972 ! f0 = crs(ic, jc, kc, iv)
1973 ! flx = crs(ic-1, jc, kc, iv)
1974 ! fhx = crs(ic+1, jc, kc, iv)
1975 ! fly = crs(ic, jc-1, kc, iv)
1976 ! fhy = crs(ic, jc+1, kc, iv)
1977 ! flz = crs(ic, jc, kc-1, iv)
1978 ! fhz = crs(ic, jc, kc+1, iv)
1979 
1980 ! c0 = crs(ic, jc, kc, i_eps)
1981 ! clx = crs(ic-1, jc, kc, i_eps)
1982 ! chx = crs(ic+1, jc, kc, i_eps)
1983 ! cly = crs(ic, jc-1, kc, i_eps)
1984 ! chy = crs(ic, jc+1, kc, i_eps)
1985 ! clz = crs(ic, jc, kc-1, i_eps)
1986 ! chz = crs(ic, jc, kc+1, i_eps)
1987 
1988 ! fine(2*i-1, 2*j-1, 2*k-1) = (f0*c0 + flx*clx + fly*cly + flz*clz) / &
1989 ! (c0 + clx + cly + clz)
1990 ! fine(2*i, 2*j-1, 2*k-1) = (f0*c0 + fhx*chx + fly*cly + flz*clz) / &
1991 ! (c0 + chx + cly + clz)
1992 ! fine(2*i-1, 2*j, 2*k-1) = (f0*c0 + flx*clx + fhy*chy + flz*clz) / &
1993 ! (c0 + clx + chy + clz)
1994 ! fine(2*i, 2*j, 2*k-1) = (f0*c0 + fhx*chx + fhy*chy + flz*clz) / &
1995 ! (c0 + chx + chy + clz)
1996 ! fine(2*i-1, 2*j-1, 2*k) = (f0*c0 + flx*clx + fly*cly + fhz*chz) / &
1997 ! (c0 + clx + cly + chz)
1998 ! fine(2*i, 2*j-1, 2*k) = (f0*c0 + fhx*chx + fly*cly + fhz*chz) / &
1999 ! (c0 + chx + cly + chz)
2000 ! fine(2*i-1, 2*j, 2*k) = (f0*c0 + flx*clx + fhy*chy + fhz*chz) / &
2001 ! (c0 + clx + chy + chz)
2002 ! fine(2*i, 2*j, 2*k) = (f0*c0 + fhx*chx + fhy*chy + fhz*chz) / &
2003 ! (c0 + chx + chy + chz)
2004 ! end do
2005 ! end do
2006 ! end do
2007 ! #endif
2008 ! end associate
2009 ! end subroutine vlpl_prolong
2010 
2011  !! File ../src/m_communication.f90
2012 
2013  !> Initialize MPI if needed, and store MPI information
2014  subroutine mg_comm_init(mg, comm)
2015  use mpi
2016  type(mg_t), intent(inout) :: mg
2017  !> MPI communicator (default: MPI_COMM_WORLD)
2018  integer, intent(in), optional :: comm
2019  integer :: ierr
2020  logical :: initialized
2021 
2022  call mpi_initialized(initialized, ierr)
2023  if (.not. initialized) then
2024  call mpi_init(ierr)
2025  end if
2026 
2027  if (present(comm)) then
2028  mg%comm = comm
2029  else
2030  mg%comm = mpi_comm_world
2031  end if
2032 
2033  call mpi_comm_rank(mg%comm, mg%my_rank, ierr)
2034  call mpi_comm_size(mg%comm, mg%n_cpu, ierr)
2035  end subroutine mg_comm_init
2036 
2037  subroutine sort_and_transfer_buffers(mg, dsize)
2038  use mpi
2039  type(mg_t), intent(inout) :: mg
2040  integer, intent(in) :: dsize
2041  integer :: i, n_send, n_recv
2042  integer :: send_req(mg%n_cpu)
2043  integer :: recv_req(mg%n_cpu)
2044  integer :: ierr
2045 
2046  n_send = 0
2047  n_recv = 0
2048 
2049  do i = 0, mg%n_cpu - 1
2050  if (mg%buf(i)%i_send > 0) then
2051  n_send = n_send + 1
2052  call sort_sendbuf(mg%buf(i), dsize)
2053  call mpi_isend(mg%buf(i)%send, mg%buf(i)%i_send, mpi_double, &
2054  i, 0, mg%comm, send_req(n_send), ierr)
2055  end if
2056  if (mg%buf(i)%i_recv > 0) then
2057  n_recv = n_recv + 1
2058  call mpi_irecv(mg%buf(i)%recv, mg%buf(i)%i_recv, mpi_double, &
2059  i, 0, mg%comm, recv_req(n_recv), ierr)
2060  end if
2061  end do
2062 
2063  call mpi_waitall(n_recv, recv_req(1:n_recv), mpi_statuses_ignore, ierr)
2064  call mpi_waitall(n_send, send_req(1:n_send), mpi_statuses_ignore, ierr)
2065 
2066  end subroutine sort_and_transfer_buffers
2067 
2068  !> Sort send buffers according to the idbuf array
2069  subroutine sort_sendbuf(gc, dsize)
2070 
2071  type(mg_buf_t), intent(inout) :: gc
2072  integer, intent(in) :: dsize !< Size of send buffer elements
2073  integer :: ix_sort(gc%i_ix)
2074  real(dp) :: buf_cpy(gc%i_send)
2075  integer :: i, j, n
2076 
2077  call mrgrnk(gc%ix(1:gc%i_ix), ix_sort)
2078 
2079  buf_cpy = gc%send(1:gc%i_send)
2080 
2081  do n = 1, gc%i_ix
2082  i = (n-1) * dsize
2083  j = (ix_sort(n)-1) * dsize
2084  gc%send(i+1:i+dsize) = buf_cpy(j+1:j+dsize)
2085  end do
2086  gc%ix(1:gc%i_ix) = gc%ix(ix_sort)
2087 
2088  end subroutine sort_sendbuf
2089 
2090  !! File ../src/m_ghost_cells.f90
2091 
2092  !> Specify minimum buffer size (per process) for communication
2093  subroutine mg_ghost_cell_buffer_size(mg, n_send, n_recv, dsize)
2094  type(mg_t), intent(inout) :: mg
2095  integer, intent(out) :: n_send(0:mg%n_cpu-1)
2096  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
2097  integer, intent(out) :: dsize
2098  integer :: i, id, lvl, nc
2099 
2100  allocate(mg%comm_ghostcell%n_send(0:mg%n_cpu-1, &
2101  mg%first_normal_lvl:mg%highest_lvl))
2102  allocate(mg%comm_ghostcell%n_recv(0:mg%n_cpu-1, &
2103  mg%first_normal_lvl:mg%highest_lvl))
2104 
2105  dsize = mg%box_size**(2-1)
2106 
2107  do lvl = mg%first_normal_lvl, mg%highest_lvl
2108  nc = mg%box_size_lvl(lvl)
2109  mg%buf(:)%i_send = 0
2110  mg%buf(:)%i_recv = 0
2111  mg%buf(:)%i_ix = 0
2112 
2113  do i = 1, size(mg%lvls(lvl)%my_ids)
2114  id = mg%lvls(lvl)%my_ids(i)
2115  call buffer_ghost_cells(mg, id, nc, 1, dry_run=.true.)
2116  end do
2117 
2118  if (lvl > 1) then
2119  do i = 1, size(mg%lvls(lvl-1)%my_ref_bnds)
2120  id = mg%lvls(lvl-1)%my_ref_bnds(i)
2121  call buffer_refinement_boundaries(mg, id, nc, 1, dry_run=.true.)
2122  end do
2123  end if
2124 
2125  ! Set ghost cells to received data
2126  mg%buf(:)%i_recv = 0
2127  do i = 1, size(mg%lvls(lvl)%my_ids)
2128  id = mg%lvls(lvl)%my_ids(i)
2129  call set_ghost_cells(mg, id, nc, 1, dry_run=.true.)
2130  end do
2131 
2132  mg%comm_ghostcell%n_send(:, lvl) = mg%buf(:)%i_send/dsize
2133  mg%comm_ghostcell%n_recv(:, lvl) = mg%buf(:)%i_recv/dsize
2134  end do
2135 
2136  n_send = maxval(mg%comm_ghostcell%n_send, dim=2)
2137  n_recv = maxval(mg%comm_ghostcell%n_recv, dim=2)
2138  end subroutine mg_ghost_cell_buffer_size
2139 
2140  !> Store boundary conditions for the solution variable, this can speed up
2141  !> calculations if the same boundary conditions are re-used.
2142  subroutine mg_phi_bc_store(mg)
2143  type(mg_t), intent(inout) :: mg
2144  integer :: lvl, nc
2145 
2146  nc = mg%box_size
2147 
2148  do lvl = mg%lowest_lvl, mg%highest_lvl
2149  nc = mg%box_size_lvl(lvl)
2150  call mg_phi_bc_store_lvl(mg, lvl, nc)
2151  end do
2152 
2153  mg%phi_bc_data_stored = .true.
2154  end subroutine mg_phi_bc_store
2155 
2156  subroutine mg_phi_bc_store_lvl(mg, lvl, nc)
2157  type(mg_t), intent(inout) :: mg
2158  integer, intent(in) :: lvl
2159  integer, intent(in) :: nc
2160  real(dp) :: bc(nc)
2161  integer :: i, id, nb, nb_id, bc_type
2162 
2163  do i = 1, size(mg%lvls(lvl)%my_ids)
2164  id = mg%lvls(lvl)%my_ids(i)
2165  do nb = 1, mg_num_neighbors
2166  nb_id = mg%boxes(id)%neighbors(nb)
2167  if (nb_id < mg_no_box) then
2168  ! Physical boundary
2169  if (associated(mg%bc(nb, mg_iphi)%boundary_cond)) then
2170  call mg%bc(nb, mg_iphi)%boundary_cond(mg%boxes(id), nc, &
2171  mg_iphi, nb, bc_type, bc)
2172  else
2173  bc_type = mg%bc(nb, mg_iphi)%bc_type
2174  bc = mg%bc(nb, mg_iphi)%bc_value
2175  end if
2176 
2177  ! Store the boundary condition type. This is not globally set in
2178  ! the tree, but all negative values are treated the same in
2179  ! other parts of the code
2180  mg%boxes(id)%neighbors(nb) = bc_type
2181 
2182  ! Store ghost cell data in the right-hand side
2183  call box_set_gc(mg%boxes(id), nb, nc, mg_irhs, bc)
2184  end if
2185  end do
2186  end do
2187  end subroutine mg_phi_bc_store_lvl
2188 
2189  !> Fill ghost cells at all grid levels
2190  subroutine mg_fill_ghost_cells(mg, iv)
2191  type(mg_t) :: mg
2192  integer, intent(in) :: iv !< Index of variable
2193  integer :: lvl
2194 
2195  do lvl = mg%lowest_lvl, mg%highest_lvl
2196  call mg_fill_ghost_cells_lvl(mg, lvl, iv)
2197  end do
2198  end subroutine mg_fill_ghost_cells
2199 
2200  !> Fill ghost cells at a grid level
2201  subroutine mg_fill_ghost_cells_lvl(mg, lvl, iv)
2202 
2203  type(mg_t) :: mg
2204  integer, intent(in) :: lvl
2205  integer, intent(in) :: iv !< Index of variable
2206  integer :: i, id, dsize, nc
2207 
2208  if (lvl < mg%lowest_lvl) &
2209  error stop "fill_ghost_cells_lvl: lvl < lowest_lvl"
2210  if (lvl > mg%highest_lvl) &
2211  error stop "fill_ghost_cells_lvl: lvl > highest_lvl"
2212 
2213  nc = mg%box_size_lvl(lvl)
2214 
2215  if (lvl >= mg%first_normal_lvl) then
2216  dsize = nc**(2-1)
2217  mg%buf(:)%i_send = 0
2218  mg%buf(:)%i_recv = 0
2219  mg%buf(:)%i_ix = 0
2220 
2221  do i = 1, size(mg%lvls(lvl)%my_ids)
2222  id = mg%lvls(lvl)%my_ids(i)
2223  call buffer_ghost_cells(mg, id, nc, iv, .false.)
2224  end do
2225 
2226  if (lvl > 1) then
2227  do i = 1, size(mg%lvls(lvl-1)%my_ref_bnds)
2228  id = mg%lvls(lvl-1)%my_ref_bnds(i)
2229  call buffer_refinement_boundaries(mg, id, nc, iv, .false.)
2230  end do
2231  end if
2232 
2233  ! Transfer data between processes
2234  mg%buf(:)%i_recv = mg%comm_ghostcell%n_recv(:, lvl) * dsize
2235  call sort_and_transfer_buffers(mg, dsize)
2236 
2237  ! Set ghost cells to received data
2238  mg%buf(:)%i_recv = 0
2239  end if
2240 
2241  do i = 1, size(mg%lvls(lvl)%my_ids)
2242  id = mg%lvls(lvl)%my_ids(i)
2243  call set_ghost_cells(mg, id, nc, iv, .false.)
2244  end do
2245  end subroutine mg_fill_ghost_cells_lvl
2246 
2247  subroutine buffer_ghost_cells(mg, id, nc, iv, dry_run)
2248  type(mg_t), intent(inout) :: mg
2249  integer, intent(in) :: id
2250  integer, intent(in) :: nc
2251  integer, intent(in) :: iv
2252  logical, intent(in) :: dry_run
2253  integer :: nb, nb_id, nb_rank
2254 
2255  do nb = 1, mg_num_neighbors
2256  nb_id = mg%boxes(id)%neighbors(nb)
2257 
2258  if (nb_id > mg_no_box) then
2259  ! There is a neighbor
2260  nb_rank = mg%boxes(nb_id)%rank
2261 
2262  if (nb_rank /= mg%my_rank) then
2263  call buffer_for_nb(mg, mg%boxes(id), nc, iv, nb_id, nb_rank, &
2264  nb, dry_run)
2265  end if
2266  end if
2267  end do
2268  end subroutine buffer_ghost_cells
2269 
2270  subroutine buffer_refinement_boundaries(mg, id, nc, iv, dry_run)
2271  type(mg_t), intent(inout) :: mg
2272  integer, intent(in) :: id
2273  integer, intent(in) :: nc
2274  integer, intent(in) :: iv
2275  logical, intent(in) :: dry_run
2276  integer :: nb, nb_id, c_ids(2**(2-1))
2277  integer :: n, c_id, c_rank
2278 
2279  do nb = 1, mg_num_neighbors
2280  nb_id = mg%boxes(id)%neighbors(nb)
2281  if (nb_id > mg_no_box) then
2282  if (mg_has_children(mg%boxes(nb_id))) then
2283  c_ids = mg%boxes(nb_id)%children(&
2285 
2286  do n = 1, mg_num_children/2
2287  c_id = c_ids(n)
2288  c_rank = mg%boxes(c_id)%rank
2289 
2290  if (c_rank /= mg%my_rank) then
2291  ! Send all coarse ghost cells
2292  call buffer_for_fine_nb(mg, mg%boxes(id), nc, iv, c_id, &
2293  c_rank, nb, dry_run)
2294  end if
2295  end do
2296  end if
2297  end if
2298  end do
2299  end subroutine buffer_refinement_boundaries
2300 
2301  !> The routine that actually fills the ghost cells
2302  subroutine set_ghost_cells(mg, id, nc, iv, dry_run)
2303  type(mg_t), intent(inout) :: mg
2304  integer, intent(in) :: id
2305  integer, intent(in) :: nc
2306  integer, intent(in) :: iv
2307  logical, intent(in) :: dry_run
2308  real(dp) :: bc(nc)
2309  integer :: nb, nb_id, nb_rank, bc_type
2310 
2311  do nb = 1, mg_num_neighbors
2312  nb_id = mg%boxes(id)%neighbors(nb)
2313 
2314  if (nb_id > mg_no_box) then
2315  ! There is a neighbor
2316  nb_rank = mg%boxes(nb_id)%rank
2317 
2318  if (nb_rank /= mg%my_rank) then
2319  call fill_buffered_nb(mg, mg%boxes(id), nb_rank, &
2320  nb, nc, iv, dry_run)
2321  else if (.not. dry_run) then
2322  call copy_from_nb(mg%boxes(id), mg%boxes(nb_id), &
2323  nb, nc, iv)
2324  end if
2325  else if (nb_id == mg_no_box) then
2326  ! Refinement boundary
2327  call fill_refinement_bnd(mg, id, nb, nc, iv, dry_run)
2328  else if (.not. dry_run) then
2329  ! Physical boundary
2330  if (mg%phi_bc_data_stored .and. iv == mg_iphi) then
2331  ! Copy the boundary conditions stored in the ghost cells of the
2332  ! right-hand side
2333  call box_get_gc(mg%boxes(id), nb, nc, mg_irhs, bc)
2334  bc_type = nb_id
2335  else
2336  if (associated(mg%bc(nb, iv)%boundary_cond)) then
2337  call mg%bc(nb, iv)%boundary_cond(mg%boxes(id), nc, iv, &
2338  nb, bc_type, bc)
2339  else
2340  bc_type = mg%bc(nb, iv)%bc_type
2341  bc = mg%bc(nb, iv)%bc_value
2342  end if
2343  end if
2344 
2345  call box_set_gc(mg%boxes(id), nb, nc, iv, bc)
2346  call bc_to_gc(mg, id, nc, iv, nb, bc_type)
2347  end if
2348  end do
2349  end subroutine set_ghost_cells
2350 
2351  subroutine fill_refinement_bnd(mg, id, nb, nc, iv, dry_run)
2352  type(mg_t), intent(inout) :: mg
2353  integer, intent(in) :: id
2354  integer, intent(in) :: nc
2355  integer, intent(in) :: iv
2356  integer, intent(in) :: nb
2357  logical, intent(in) :: dry_run
2358  real(dp) :: gc(nc)
2359  integer :: p_id, p_nb_id, ix_offset(2)
2360  integer :: i, dsize, p_nb_rank
2361 
2362  dsize = nc**(2-1)
2363  p_id = mg%boxes(id)%parent
2364  p_nb_id = mg%boxes(p_id)%neighbors(nb)
2365  p_nb_rank = mg%boxes(p_nb_id)%rank
2366 
2367  if (p_nb_rank /= mg%my_rank) then
2368  i = mg%buf(p_nb_rank)%i_recv
2369  if (.not. dry_run) then
2370  gc = reshape(mg%buf(p_nb_rank)%recv(i+1:i+dsize), shape(gc))
2371  end if
2372  mg%buf(p_nb_rank)%i_recv = mg%buf(p_nb_rank)%i_recv + dsize
2373  else if (.not. dry_run) then
2374  ix_offset = mg_get_child_offset(mg, id)
2375  call box_gc_for_fine_neighbor(mg%boxes(p_nb_id), mg_neighb_rev(nb), &
2376  ix_offset, nc, iv, gc)
2377  end if
2378 
2379  if (.not. dry_run) then
2380  if (associated(mg%bc(nb, iv)%refinement_bnd)) then
2381  call mg%bc(nb, iv)%refinement_bnd(mg%boxes(id), nc, iv, nb, gc)
2382  else
2383  call sides_rb(mg%boxes(id), nc, iv, nb, gc)
2384  end if
2385  end if
2386  end subroutine fill_refinement_bnd
2387 
2388  subroutine copy_from_nb(box, box_nb, nb, nc, iv)
2389  type(mg_box_t), intent(inout) :: box
2390  type(mg_box_t), intent(in) :: box_nb
2391  integer, intent(in) :: nb
2392  integer, intent(in) :: nc
2393  integer, intent(in) :: iv
2394  real(dp) :: gc(nc)
2395 
2396  call box_gc_for_neighbor(box_nb, mg_neighb_rev(nb), nc, iv, gc)
2397  call box_set_gc(box, nb, nc, iv, gc)
2398  end subroutine copy_from_nb
2399 
2400  subroutine buffer_for_nb(mg, box, nc, iv, nb_id, nb_rank, nb, dry_run)
2401  type(mg_t), intent(inout) :: mg
2402  type(mg_box_t), intent(inout) :: box
2403  integer, intent(in) :: nc
2404  integer, intent(in) :: iv
2405  integer, intent(in) :: nb_id
2406  integer, intent(in) :: nb_rank
2407  integer, intent(in) :: nb
2408  logical, intent(in) :: dry_run
2409  integer :: i, dsize
2410  real(dp) :: gc(nc)
2411 
2412  i = mg%buf(nb_rank)%i_send
2413  dsize = nc**(2-1)
2414 
2415  if (.not. dry_run) then
2416  call box_gc_for_neighbor(box, nb, nc, iv, gc)
2417  mg%buf(nb_rank)%send(i+1:i+dsize) = pack(gc, .true.)
2418  end if
2419 
2420  ! Later the buffer is sorted, using the fact that loops go from low to high
2421  ! box id, and we fill ghost cells according to the neighbor order
2422  i = mg%buf(nb_rank)%i_ix
2423  if (.not. dry_run) then
2424  mg%buf(nb_rank)%ix(i+1) = mg_num_neighbors * nb_id + mg_neighb_rev(nb)
2425  end if
2426 
2427  mg%buf(nb_rank)%i_send = mg%buf(nb_rank)%i_send + dsize
2428  mg%buf(nb_rank)%i_ix = mg%buf(nb_rank)%i_ix + 1
2429  end subroutine buffer_for_nb
2430 
2431  subroutine buffer_for_fine_nb(mg, box, nc, iv, fine_id, fine_rank, nb, dry_run)
2432  type(mg_t), intent(inout) :: mg
2433  type(mg_box_t), intent(inout) :: box
2434  integer, intent(in) :: nc
2435  integer, intent(in) :: iv
2436  integer, intent(in) :: fine_id
2437  integer, intent(in) :: fine_rank
2438  integer, intent(in) :: nb
2439  logical, intent(in) :: dry_run
2440  integer :: i, dsize, ix_offset(2)
2441  real(dp) :: gc(nc)
2442 
2443  i = mg%buf(fine_rank)%i_send
2444  dsize = nc**(2-1)
2445 
2446  if (.not. dry_run) then
2447  ix_offset = mg_get_child_offset(mg, fine_id)
2448  call box_gc_for_fine_neighbor(box, nb, ix_offset, nc, iv, gc)
2449  mg%buf(fine_rank)%send(i+1:i+dsize) = pack(gc, .true.)
2450  end if
2451 
2452  ! Later the buffer is sorted, using the fact that loops go from low to high
2453  ! box id, and we fill ghost cells according to the neighbor order
2454  i = mg%buf(fine_rank)%i_ix
2455  if (.not. dry_run) then
2456  mg%buf(fine_rank)%ix(i+1) = mg_num_neighbors * fine_id + &
2457  mg_neighb_rev(nb)
2458  end if
2459 
2460  mg%buf(fine_rank)%i_send = mg%buf(fine_rank)%i_send + dsize
2461  mg%buf(fine_rank)%i_ix = mg%buf(fine_rank)%i_ix + 1
2462  end subroutine buffer_for_fine_nb
2463 
2464  subroutine fill_buffered_nb(mg, box, nb_rank, nb, nc, iv, dry_run)
2465  type(mg_t), intent(inout) :: mg
2466  type(mg_box_t), intent(inout) :: box
2467  integer, intent(in) :: nb_rank
2468  integer, intent(in) :: nb
2469  integer, intent(in) :: nc
2470  integer, intent(in) :: iv
2471  logical, intent(in) :: dry_run
2472  integer :: i, dsize
2473  real(dp) :: gc(nc)
2474 
2475  i = mg%buf(nb_rank)%i_recv
2476  dsize = nc**(2-1)
2477 
2478  if (.not. dry_run) then
2479  gc = reshape(mg%buf(nb_rank)%recv(i+1:i+dsize), shape(gc))
2480  call box_set_gc(box, nb, nc, iv, gc)
2481  end if
2482  mg%buf(nb_rank)%i_recv = mg%buf(nb_rank)%i_recv + dsize
2483 
2484  end subroutine fill_buffered_nb
2485 
2486  subroutine box_gc_for_neighbor(box, nb, nc, iv, gc)
2487  type(mg_box_t), intent(in) :: box
2488  integer, intent(in) :: nb, nc, iv
2489  real(dp), intent(out) :: gc(nc)
2490 
2491  select case (nb)
2492  case (mg_neighb_lowx)
2493  gc = box%cc(1, 1:nc, iv)
2494  case (mg_neighb_highx)
2495  gc = box%cc(nc, 1:nc, iv)
2496  case (mg_neighb_lowy)
2497  gc = box%cc(1:nc, 1, iv)
2498  case (mg_neighb_highy)
2499  gc = box%cc(1:nc, nc, iv)
2500  end select
2501  end subroutine box_gc_for_neighbor
2502 
2503  !> Get ghost cells for a fine neighbor
2504  subroutine box_gc_for_fine_neighbor(box, nb, di, nc, iv, gc)
2505  type(mg_box_t), intent(in) :: box
2506  integer, intent(in) :: nb !< Direction of fine neighbor
2507  integer, intent(in) :: di(2) !< Index offset of fine neighbor
2508  integer, intent(in) :: nc, iv
2509  real(dp), intent(out) :: gc(nc)
2510  real(dp) :: tmp(0:nc/2+1), grad(2-1)
2511  integer :: i, hnc
2512 
2513  hnc = nc/2
2514 
2515  ! First fill a temporary array with data next to the fine grid
2516  select case (nb)
2517  case (mg_neighb_lowx)
2518  tmp = box%cc(1, di(2):di(2)+hnc+1, iv)
2519  case (mg_neighb_highx)
2520  tmp = box%cc(nc, di(2):di(2)+hnc+1, iv)
2521  case (mg_neighb_lowy)
2522  tmp = box%cc(di(1):di(1)+hnc+1, 1, iv)
2523  case (mg_neighb_highy)
2524  tmp = box%cc(di(1):di(1)+hnc+1, nc, iv)
2525  case default
2526  error stop
2527  end select
2528 
2529  ! Now interpolate the coarse grid data to obtain values 'straight' next to
2530  ! the fine grid points
2531  do i = 1, hnc
2532  grad(1) = 0.125_dp * (tmp(i+1) - tmp(i-1))
2533  gc(2*i-1) = tmp(i) - grad(1)
2534  gc(2*i) = tmp(i) + grad(1)
2535  end do
2536  end subroutine box_gc_for_fine_neighbor
2537 
2538  subroutine box_get_gc(box, nb, nc, iv, gc)
2539  type(mg_box_t), intent(in) :: box
2540  integer, intent(in) :: nb, nc, iv
2541  real(dp), intent(out) :: gc(nc)
2542 
2543  select case (nb)
2544  case (mg_neighb_lowx)
2545  gc = box%cc(0, 1:nc, iv)
2546  case (mg_neighb_highx)
2547  gc = box%cc(nc+1, 1:nc, iv)
2548  case (mg_neighb_lowy)
2549  gc = box%cc(1:nc, 0, iv)
2550  case (mg_neighb_highy)
2551  gc = box%cc(1:nc, nc+1, iv)
2552  end select
2553  end subroutine box_get_gc
2554 
2555  subroutine box_set_gc(box, nb, nc, iv, gc)
2556  type(mg_box_t), intent(inout) :: box
2557  integer, intent(in) :: nb, nc, iv
2558  real(dp), intent(in) :: gc(nc)
2559 
2560  select case (nb)
2561  case (mg_neighb_lowx)
2562  box%cc(0, 1:nc, iv) = gc
2563  case (mg_neighb_highx)
2564  box%cc(nc+1, 1:nc, iv) = gc
2565  case (mg_neighb_lowy)
2566  box%cc(1:nc, 0, iv) = gc
2567  case (mg_neighb_highy)
2568  box%cc(1:nc, nc+1, iv) = gc
2569  end select
2570  end subroutine box_set_gc
2571 
2572  subroutine bc_to_gc(mg, id, nc, iv, nb, bc_type)
2573  type(mg_t), intent(inout) :: mg
2574  integer, intent(in) :: id
2575  integer, intent(in) :: nc
2576  integer, intent(in) :: iv
2577  integer, intent(in) :: nb !< Neighbor direction
2578  integer, intent(in) :: bc_type !< Type of b.c.
2579  real(dp) :: c0, c1, c2, dr
2580 
2581  ! If we call the interior point x1, x2 and the ghost point x0, then a
2582  ! Dirichlet boundary value b can be imposed as:
2583  ! x0 = -x1 + 2*b
2584  ! A Neumann b.c. can be imposed as:
2585  ! x0 = x1 +/- dx * b
2586  ! A continuous boundary (same slope) as:
2587  ! x0 = 2 * x1 - x2
2588  ! Below, we set coefficients to handle these cases
2589  select case (bc_type)
2590  case (mg_bc_dirichlet)
2591  c0 = 2
2592  c1 = -1
2593  c2 = 0
2594  case (mg_bc_neumann)
2595  dr = mg%dr(mg_neighb_dim(nb), mg%boxes(id)%lvl)
2596  c0 = dr * mg_neighb_high_pm(nb) ! This gives a + or - sign
2597  c1 = 1
2598  c2 = 0
2599  case (mg_bc_continuous)
2600  c0 = 0
2601  c1 = 2
2602  c2 = -1
2603  case default
2604  error stop "bc_to_gc: unknown boundary condition"
2605  end select
2606 
2607  select case (nb)
2608  case (mg_neighb_lowx)
2609  mg%boxes(id)%cc(0, 1:nc, iv) = &
2610  c0 * mg%boxes(id)%cc(0, 1:nc, iv) + &
2611  c1 * mg%boxes(id)%cc(1, 1:nc, iv) + &
2612  c2 * mg%boxes(id)%cc(2, 1:nc, iv)
2613  case (mg_neighb_highx)
2614  mg%boxes(id)%cc(nc+1, 1:nc, iv) = &
2615  c0 * mg%boxes(id)%cc(nc+1, 1:nc, iv) + &
2616  c1 * mg%boxes(id)%cc(nc, 1:nc, iv) + &
2617  c2 * mg%boxes(id)%cc(nc-1, 1:nc, iv)
2618  case (mg_neighb_lowy)
2619  mg%boxes(id)%cc(1:nc, 0, iv) = &
2620  c0 * mg%boxes(id)%cc(1:nc, 0, iv) + &
2621  c1 * mg%boxes(id)%cc(1:nc, 1, iv) + &
2622  c2 * mg%boxes(id)%cc(1:nc, 2, iv)
2623  case (mg_neighb_highy)
2624  mg%boxes(id)%cc(1:nc, nc+1, iv) = &
2625  c0 * mg%boxes(id)%cc(1:nc, nc+1, iv) + &
2626  c1 * mg%boxes(id)%cc(1:nc, nc, iv) + &
2627  c2 * mg%boxes(id)%cc(1:nc, nc-1, iv)
2628  end select
2629  end subroutine bc_to_gc
2630 
2631  !> Fill ghost cells near refinement boundaries which preserves diffusive fluxes.
2632  subroutine sides_rb(box, nc, iv, nb, gc)
2633  type(mg_box_t), intent(inout) :: box
2634  integer, intent(in) :: nc
2635  integer, intent(in) :: iv
2636  integer, intent(in) :: nb !< Ghost cell direction
2637  !> Interpolated coarse grid ghost cell data (but not yet in the nb direction)
2638  real(dp), intent(in) :: gc(nc)
2639  integer :: di, dj
2640  integer :: i, j, ix, dix
2641 
2642  if (mg_neighb_low(nb)) then
2643  ix = 1
2644  dix = 1
2645  else
2646  ix = nc
2647  dix = -1
2648  end if
2649 
2650  select case (mg_neighb_dim(nb))
2651  case (1)
2652  i = ix
2653  di = dix
2654 
2655  do j = 1, nc
2656  dj = -1 + 2 * iand(j, 1)
2657  box%cc(i-di, j, iv) = 0.5_dp * gc(j) &
2658  + 0.75_dp * box%cc(i, j, iv) &
2659  - 0.25_dp * box%cc(i+di, j, iv)
2660  end do
2661  case (2)
2662  j = ix
2663  dj = dix
2664  do i = 1, nc
2665  di = -1 + 2 * iand(i, 1)
2666  box%cc(i, j-dj, iv) = 0.5_dp * gc(i) &
2667  + 0.75_dp * box%cc(i, j, iv) &
2668  - 0.25_dp * box%cc(i, j+dj, iv)
2669  end do
2670  end select
2671 
2672  end subroutine sides_rb
2673 
2674  !! File ../src/m_prolong.f90
2675 
2676  !> Specify minimum buffer size (per process) for communication
2677  subroutine mg_prolong_buffer_size(mg, n_send, n_recv, dsize)
2678  type(mg_t), intent(inout) :: mg
2679  integer, intent(out) :: n_send(0:mg%n_cpu-1)
2680  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
2681  integer, intent(out) :: dsize
2682  integer :: lvl, min_lvl
2683 
2684  if (.not. allocated(mg%comm_restrict%n_send)) then
2685  error stop "Call restrict_buffer_size before prolong_buffer_size"
2686  end if
2687 
2688  min_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
2689  allocate(mg%comm_prolong%n_send(0:mg%n_cpu-1, &
2690  min_lvl:mg%highest_lvl))
2691  allocate(mg%comm_prolong%n_recv(0:mg%n_cpu-1, &
2692  min_lvl:mg%highest_lvl))
2693 
2694  mg%comm_prolong%n_recv(:, mg%highest_lvl) = 0
2695  mg%comm_prolong%n_send(:, mg%highest_lvl) = 0
2696 
2697  do lvl = min_lvl, mg%highest_lvl-1
2698  mg%comm_prolong%n_recv(:, lvl) = &
2699  mg%comm_restrict%n_send(:, lvl+1)
2700  mg%comm_prolong%n_send(:, lvl) = &
2701  mg%comm_restrict%n_recv(:, lvl+1)
2702  end do
2703 
2704  ! Send fine grid points, because this is more flexible than sending coarse
2705  ! grid points (e.g., when multiple variables are used for interpolation)
2706  dsize = (mg%box_size)**2
2707  n_send = maxval(mg%comm_prolong%n_send, dim=2)
2708  n_recv = maxval(mg%comm_prolong%n_recv, dim=2)
2709  end subroutine mg_prolong_buffer_size
2710 
2711  !> Prolong variable iv from lvl to variable iv_to at lvl+1
2712  subroutine mg_prolong(mg, lvl, iv, iv_to, method, add)
2713 
2714  type(mg_t), intent(inout) :: mg
2715  integer, intent(in) :: lvl !< Level to prolong from
2716  integer, intent(in) :: iv !< Source variable
2717  integer, intent(in) :: iv_to !< Target variable
2718  procedure(mg_box_prolong) :: method !< Prolongation method
2719  logical, intent(in) :: add !< If true, add to current values
2720  integer :: i, id, dsize, nc
2721 
2722  if (lvl == mg%highest_lvl) error stop "cannot prolong highest level"
2723  if (lvl < mg%lowest_lvl) error stop "cannot prolong below lowest level"
2724 
2725  ! Below the first normal level, all boxes are on the same CPU
2726  if (lvl >= mg%first_normal_lvl-1) then
2727  dsize = mg%box_size**2
2728  mg%buf(:)%i_send = 0
2729  mg%buf(:)%i_ix = 0
2730 
2731  do i = 1, size(mg%lvls(lvl)%my_ids)
2732  id = mg%lvls(lvl)%my_ids(i)
2733  call prolong_set_buffer(mg, id, mg%box_size, iv, method)
2734  end do
2735 
2736  mg%buf(:)%i_recv = mg%comm_prolong%n_recv(:, lvl) * dsize
2737  call sort_and_transfer_buffers(mg, dsize)
2738  mg%buf(:)%i_recv = 0
2739  end if
2740 
2741  nc = mg%box_size_lvl(lvl+1)
2742  do i = 1, size(mg%lvls(lvl+1)%my_ids)
2743  id = mg%lvls(lvl+1)%my_ids(i)
2744  call prolong_onto(mg, id, nc, iv, iv_to, add, method)
2745  end do
2746  end subroutine mg_prolong
2747 
2748  !> In case the fine grid is on a different CPU, perform the prolongation and
2749  !> store the fine-grid values in the send buffer.
2750  !>
2751  subroutine prolong_set_buffer(mg, id, nc, iv, method)
2752  type(mg_t), intent(inout) :: mg
2753  integer, intent(in) :: id
2754  integer, intent(in) :: nc
2755  integer, intent(in) :: iv
2756  procedure(mg_box_prolong) :: method
2757  integer :: i, dix(2)
2758  integer :: i_c, c_id, c_rank, dsize
2759  real(dp) :: tmp(nc, nc)
2760 
2761  dsize = nc**2
2762 
2763  do i_c = 1, mg_num_children
2764  c_id = mg%boxes(id)%children(i_c)
2765  if (c_id > mg_no_box) then
2766  c_rank = mg%boxes(c_id)%rank
2767  if (c_rank /= mg%my_rank) then
2768  dix = mg_get_child_offset(mg, c_id)
2769  call method(mg, id, dix, nc, iv, tmp)
2770 
2771  i = mg%buf(c_rank)%i_send
2772  mg%buf(c_rank)%send(i+1:i+dsize) = pack(tmp, .true.)
2773  mg%buf(c_rank)%i_send = mg%buf(c_rank)%i_send + dsize
2774 
2775  i = mg%buf(c_rank)%i_ix
2776  mg%buf(c_rank)%ix(i+1) = c_id
2777  mg%buf(c_rank)%i_ix = mg%buf(c_rank)%i_ix + 1
2778  end if
2779  end if
2780  end do
2781  end subroutine prolong_set_buffer
2782 
2783  !> Prolong onto a child box
2784  subroutine prolong_onto(mg, id, nc, iv, iv_to, add, method)
2785  type(mg_t), intent(inout) :: mg
2786  integer, intent(in) :: id
2787  integer, intent(in) :: nc
2788  integer, intent(in) :: iv !< Prolong from this variable
2789  integer, intent(in) :: iv_to !< Prolong to this variable
2790  logical, intent(in) :: add !< If true, add to current values
2791  procedure(mg_box_prolong) :: method
2792  integer :: hnc, p_id, p_rank, i, dix(2), dsize
2793  real(dp) :: tmp(nc, nc)
2794 
2795  hnc = nc/2
2796  p_id = mg%boxes(id)%parent
2797  p_rank = mg%boxes(p_id)%rank
2798 
2799  if (p_rank == mg%my_rank) then
2800  dix = mg_get_child_offset(mg, id)
2801  call method(mg, p_id, dix, nc, iv, tmp)
2802  else
2803  dsize = nc**2
2804  i = mg%buf(p_rank)%i_recv
2805  tmp = reshape(mg%buf(p_rank)%recv(i+1:i+dsize), [nc, nc])
2806  mg%buf(p_rank)%i_recv = mg%buf(p_rank)%i_recv + dsize
2807  end if
2808 
2809  if (add) then
2810  mg%boxes(id)%cc(1:nc, 1:nc, iv_to) = &
2811  mg%boxes(id)%cc(1:nc, 1:nc, iv_to) + tmp
2812  else
2813  mg%boxes(id)%cc(1:nc, 1:nc, iv_to) = tmp
2814  end if
2815 
2816  end subroutine prolong_onto
2817 
2818  !> Prolong from a parent to a child with index offset dix
2819  subroutine mg_prolong_sparse(mg, p_id, dix, nc, iv, fine)
2820  type(mg_t), intent(inout) :: mg
2821  integer, intent(in) :: p_id !< Id of parent
2822  integer, intent(in) :: dix(2) !< Offset of child in parent grid
2823  integer, intent(in) :: nc !< Child grid size
2824  integer, intent(in) :: iv !< Prolong from this variable
2825  real(dp), intent(out) :: fine(nc, nc) !< Prolonged values
2826 
2827  integer :: i, j, hnc
2828  integer :: ic, jc
2829  real(dp) :: f0, flx, fhx, fly, fhy
2830 
2831  hnc = nc/2
2832 
2833  associate(crs => mg%boxes(p_id)%cc)
2834  do j = 1, hnc
2835  jc = j + dix(2)
2836  do i = 1, hnc
2837  ic = i + dix(1)
2838 
2839  f0 = 0.5_dp * crs(ic, jc, iv)
2840  flx = 0.25_dp * crs(ic-1, jc, iv)
2841  fhx = 0.25_dp * crs(ic+1, jc, iv)
2842  fly = 0.25_dp * crs(ic, jc-1, iv)
2843  fhy = 0.25_dp * crs(ic, jc+1, iv)
2844 
2845  fine(2*i-1, 2*j-1) = f0 + flx + fly
2846  fine(2*i , 2*j-1) = f0 + fhx + fly
2847  fine(2*i-1, 2*j) = f0 + flx + fhy
2848  fine(2*i , 2*j) = f0 + fhx + fhy
2849  end do
2850  end do
2851  end associate
2852  end subroutine mg_prolong_sparse
2853 
2854  !! File ../src/m_helmholtz.f90
2855 
2856  subroutine helmholtz_set_methods(mg)
2857  type(mg_t), intent(inout) :: mg
2858 
2859  mg%subtract_mean = .false.
2860 
2861  select case (mg%geometry_type)
2862  case (mg_cartesian)
2863  mg%box_op => box_helmh
2864 
2865  select case (mg%smoother_type)
2867  mg%box_smoother => box_gs_helmh
2868  case default
2869  error stop "helmholtz_set_methods: unsupported smoother type"
2870  end select
2871  case default
2872  error stop "helmholtz_set_methods: unsupported geometry"
2873  end select
2874 
2875  end subroutine helmholtz_set_methods
2876 
2877  subroutine helmholtz_set_lambda(lambda)
2878  real(dp), intent(in) :: lambda
2879 
2880  if (lambda < 0) &
2881  error stop "helmholtz_set_lambda: lambda < 0 not allowed"
2882 
2883  helmholtz_lambda = lambda
2884  end subroutine helmholtz_set_lambda
2885 
2886  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
2887  subroutine box_gs_helmh(mg, id, nc, redblack_cntr)
2888  type(mg_t), intent(inout) :: mg
2889  integer, intent(in) :: id
2890  integer, intent(in) :: nc
2891  integer, intent(in) :: redblack_cntr !< Iteration counter
2892  integer :: i, j, i0, di
2893  real(dp) :: idr2(2), fac
2894  logical :: redblack
2895 
2896  idr2 = 1/mg%dr(:, mg%boxes(id)%lvl)**2
2897  fac = 1.0_dp / (2 * sum(idr2) + helmholtz_lambda)
2898  i0 = 1
2899 
2900  redblack = (mg%smoother_type == mg_smoother_gsrb)
2901  if (redblack) then
2902  di = 2
2903  else
2904  di = 1
2905  end if
2906 
2907  ! The parity of redblack_cntr determines which cells we use. If
2908  ! redblack_cntr is even, we use the even cells and vice versa.
2909  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
2910  do j = 1, nc
2911  if (redblack) &
2912  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
2913 
2914  do i = i0, nc, di
2915  cc(i, j, n) = fac * ( &
2916  idr2(1) * (cc(i+1, j, n) + cc(i-1, j, n)) + &
2917  idr2(2) * (cc(i, j+1, n) + cc(i, j-1, n)) - &
2918  cc(i, j, mg_irhs))
2919  end do
2920  end do
2921  end associate
2922  end subroutine box_gs_helmh
2923 
2924  !> Perform Helmholtz operator on a box
2925  subroutine box_helmh(mg, id, nc, i_out)
2926  type(mg_t), intent(inout) :: mg
2927  integer, intent(in) :: id
2928  integer, intent(in) :: nc
2929  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
2930  integer :: i, j
2931  real(dp) :: idr2(2)
2932 
2933  idr2 = 1 / mg%dr(:, mg%boxes(id)%lvl)**2
2934 
2935  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
2936  do j = 1, nc
2937  do i = 1, nc
2938  cc(i, j, i_out) = &
2939  idr2(1) * (cc(i-1, j, n) + cc(i+1, j, n) - 2 * cc(i, j, n)) + &
2940  idr2(2) * (cc(i, j-1, n) + cc(i, j+1, n) - 2 * cc(i, j, n)) - &
2941  helmholtz_lambda * cc(i, j, n)
2942  end do
2943  end do
2944  end associate
2945  end subroutine box_helmh
2946 
2947  !! File ../src/m_multigrid.f90
2948 
2949  subroutine mg_set_methods(mg)
2950 
2951  type(mg_t), intent(inout) :: mg
2952 
2953  ! Set default prolongation method (routines below can override this)
2954  mg%box_prolong => mg_prolong_sparse
2955 
2956  select case (mg%operator_type)
2957  case (mg_laplacian)
2958  call laplacian_set_methods(mg)
2959  case (mg_vlaplacian)
2960  call vlaplacian_set_methods(mg)
2961  case (mg_helmholtz)
2962  call helmholtz_set_methods(mg)
2963  case (mg_vhelmholtz)
2964  call vhelmholtz_set_methods(mg)
2965  case (mg_ahelmholtz)
2966  call ahelmholtz_set_methods(mg)
2967  case default
2968  error stop "mg_set_methods: unknown operator"
2969  end select
2970 
2971  ! For red-black, perform two smoothing sub-steps so that all unknowns are
2972  ! updated per cycle
2973  if (mg%smoother_type == mg_smoother_gsrb) then
2974  mg%n_smoother_substeps = 2
2975  else
2976  mg%n_smoother_substeps = 1
2977  end if
2978  end subroutine mg_set_methods
2979 
2980  subroutine check_methods(mg)
2981  type(mg_t), intent(inout) :: mg
2982 
2983  if (.not. associated(mg%box_op) .or. &
2984  .not. associated(mg%box_smoother)) then
2985  call mg_set_methods(mg)
2986  end if
2987 
2988  end subroutine check_methods
2989 
2990  subroutine mg_add_timers(mg)
2991  type(mg_t), intent(inout) :: mg
2992  timer_total_vcycle = mg_add_timer(mg, "mg total V-cycle")
2993  timer_total_fmg = mg_add_timer(mg, "mg total FMG cycle")
2994  timer_smoother = mg_add_timer(mg, "mg smoother")
2995  timer_smoother_gc = mg_add_timer(mg, "mg smoother g.c.")
2996  timer_coarse = mg_add_timer(mg, "mg coarse")
2997  timer_correct = mg_add_timer(mg, "mg correct")
2998  timer_update_coarse = mg_add_timer(mg, "mg update coarse")
2999  end subroutine mg_add_timers
3000 
3001  !> Perform FAS-FMG cycle (full approximation scheme, full multigrid).
3002  subroutine mg_fas_fmg(mg, have_guess, max_res)
3003  type(mg_t), intent(inout) :: mg
3004  logical, intent(in) :: have_guess !< If false, start from phi = 0
3005  real(dp), intent(out), optional :: max_res !< Store max(abs(residual))
3006  integer :: lvl, i, id
3007 
3008  call check_methods(mg)
3009  if (timer_smoother == -1) call mg_add_timers(mg)
3010 
3011  call mg_timer_start(mg%timers(timer_total_fmg))
3012 
3013  if (.not. have_guess) then
3014  do lvl = mg%highest_lvl, mg%lowest_lvl, -1
3015  do i = 1, size(mg%lvls(lvl)%my_ids)
3016  id = mg%lvls(lvl)%my_ids(i)
3017  mg%boxes(id)%cc(:, :, mg_iphi) = 0.0_dp
3018  end do
3019  end do
3020  end if
3021 
3022  ! Ensure ghost cells are filled correctly
3023  call mg_fill_ghost_cells_lvl(mg, mg%highest_lvl, mg_iphi)
3024 
3025  do lvl = mg%highest_lvl, mg%lowest_lvl+1, -1
3026  ! Set rhs on coarse grid and restrict phi
3027  call mg_timer_start(mg%timers(timer_update_coarse))
3028  call update_coarse(mg, lvl)
3029  call mg_timer_end(mg%timers(timer_update_coarse))
3030  end do
3031 
3032  if (mg%subtract_mean) then
3033  ! For fully periodic solutions, the mean source term has to be zero
3034  call subtract_mean(mg, mg_irhs, .false.)
3035  end if
3036 
3037  do lvl = mg%lowest_lvl, mg%highest_lvl
3038  ! Store phi_old
3039  do i = 1, size(mg%lvls(lvl)%my_ids)
3040  id = mg%lvls(lvl)%my_ids(i)
3041  mg%boxes(id)%cc(:, :, mg_iold) = &
3042  mg%boxes(id)%cc(:, :, mg_iphi)
3043  end do
3044 
3045  if (lvl > mg%lowest_lvl) then
3046  ! Correct solution at this lvl using lvl-1 data
3047  ! phi = phi + prolong(phi_coarse - phi_old_coarse)
3048  call mg_timer_start(mg%timers(timer_correct))
3049  call correct_children(mg, lvl-1)
3050  call mg_timer_end(mg%timers(timer_correct))
3051 
3052  ! Update ghost cells
3053  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3054  end if
3055 
3056  ! Perform V-cycle, possibly set residual on last iteration
3057  if (lvl == mg%highest_lvl) then
3058  call mg_fas_vcycle(mg, lvl, max_res, standalone=.false.)
3059  else
3060  call mg_fas_vcycle(mg, lvl, standalone=.false.)
3061  end if
3062  end do
3063 
3064  call mg_timer_end(mg%timers(timer_total_fmg))
3065  end subroutine mg_fas_fmg
3066 
3067  !> Perform FAS V-cycle (full approximation scheme).
3068  subroutine mg_fas_vcycle(mg, highest_lvl, max_res, standalone)
3069  use mpi
3070  type(mg_t), intent(inout) :: mg
3071  integer, intent(in), optional :: highest_lvl !< Maximum level for V-cycle
3072  real(dp), intent(out), optional :: max_res !< Store max(abs(residual))
3073  !> Whether the V-cycle is called by itself (default: true)
3074  logical, intent(in), optional :: standalone
3075  integer :: lvl, min_lvl, i, max_lvl, ierr
3076  real(dp) :: init_res, res
3077  logical :: is_standalone
3078 
3079  is_standalone = .true.
3080  if (present(standalone)) is_standalone = standalone
3081 
3082  call check_methods(mg)
3083  if (timer_smoother == -1) call mg_add_timers(mg)
3084 
3085  if (is_standalone) &
3086  call mg_timer_start(mg%timers(timer_total_vcycle))
3087 
3088  if (mg%subtract_mean .and. .not. present(highest_lvl)) then
3089  ! Assume that this is a stand-alone call. For fully periodic solutions,
3090  ! ensure the mean source term is zero.
3091  call subtract_mean(mg, mg_irhs, .false.)
3092  end if
3093 
3094  min_lvl = mg%lowest_lvl
3095  max_lvl = mg%highest_lvl
3096  if (present(highest_lvl)) max_lvl = highest_lvl
3097 
3098  ! Ensure ghost cells are filled correctly
3099  if (is_standalone) then
3100  call mg_fill_ghost_cells_lvl(mg, max_lvl, mg_iphi)
3101  end if
3102 
3103  do lvl = max_lvl, min_lvl+1, -1
3104  ! Downwards relaxation
3105  call smooth_boxes(mg, lvl, mg%n_cycle_down)
3106 
3107  ! Set rhs on coarse grid, restrict phi, and copy mg_iphi to mg_iold for the
3108  ! correction later
3109  call mg_timer_start(mg%timers(timer_update_coarse))
3110  call update_coarse(mg, lvl)
3111  call mg_timer_end(mg%timers(timer_update_coarse))
3112  end do
3113 
3114  call mg_timer_start(mg%timers(timer_coarse))
3115  if (.not. all(mg%boxes(mg%lvls(min_lvl)%ids)%rank == &
3116  mg%boxes(mg%lvls(min_lvl)%ids(1))%rank)) then
3117  error stop "Multiple CPUs for coarse grid (not implemented yet)"
3118  end if
3119 
3120  init_res = max_residual_lvl(mg, min_lvl)
3121  do i = 1, mg%max_coarse_cycles
3122  call smooth_boxes(mg, min_lvl, mg%n_cycle_up+mg%n_cycle_down)
3123  res = max_residual_lvl(mg, min_lvl)
3124  if (res < mg%residual_coarse_rel * init_res .or. &
3125  res < mg%residual_coarse_abs) exit
3126  end do
3127  call mg_timer_end(mg%timers(timer_coarse))
3128 
3129  ! Do the upwards part of the v-cycle in the tree
3130  do lvl = min_lvl+1, max_lvl
3131  ! Correct solution at this lvl using lvl-1 data
3132  ! phi = phi + prolong(phi_coarse - phi_old_coarse)
3133  call mg_timer_start(mg%timers(timer_correct))
3134  call correct_children(mg, lvl-1)
3135 
3136  ! Have to fill ghost cells after correction
3137  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3138  call mg_timer_end(mg%timers(timer_correct))
3139 
3140  ! Upwards relaxation
3141  call smooth_boxes(mg, lvl, mg%n_cycle_up)
3142  end do
3143 
3144  if (present(max_res)) then
3145  init_res = 0.0_dp
3146  do lvl = min_lvl, max_lvl
3147  res = max_residual_lvl(mg, lvl)
3148  init_res = max(res, init_res)
3149  end do
3150  call mpi_allreduce(init_res, max_res, 1, &
3151  mpi_double, mpi_max, mg%comm, ierr)
3152  end if
3153 
3154  ! Subtract mean(phi) from phi
3155  if (mg%subtract_mean) then
3156  call subtract_mean(mg, mg_iphi, .true.)
3157  end if
3158 
3159  if (is_standalone) &
3160  call mg_timer_end(mg%timers(timer_total_vcycle))
3161  end subroutine mg_fas_vcycle
3162 
3163  subroutine subtract_mean(mg, iv, include_ghostcells)
3164  use mpi
3165  type(mg_t), intent(inout) :: mg
3166  integer, intent(in) :: iv
3167  logical, intent(in) :: include_ghostcells
3168  integer :: i, id, lvl, nc, ierr
3169  real(dp) :: sum_iv, mean_iv, volume
3170 
3171  nc = mg%box_size
3172  sum_iv = get_sum(mg, iv)
3173  call mpi_allreduce(sum_iv, mean_iv, 1, &
3174  mpi_double, mpi_sum, mg%comm, ierr)
3175 
3176  ! Divide by total grid volume to get mean
3177  volume = nc**2 * product(mg%dr(:, 1)) * size(mg%lvls(1)%ids)
3178  mean_iv = mean_iv / volume
3179 
3180  do lvl = mg%lowest_lvl, mg%highest_lvl
3181  nc = mg%box_size_lvl(lvl)
3182 
3183  do i = 1, size(mg%lvls(lvl)%my_ids)
3184  id = mg%lvls(lvl)%my_ids(i)
3185  if (include_ghostcells) then
3186  mg%boxes(id)%cc(:, :, iv) = &
3187  mg%boxes(id)%cc(:, :, iv) - mean_iv
3188  else
3189  mg%boxes(id)%cc(1:nc, 1:nc, iv) = &
3190  mg%boxes(id)%cc(1:nc, 1:nc, iv) - mean_iv
3191  end if
3192  end do
3193  end do
3194  end subroutine subtract_mean
3195 
3196  real(dp) function get_sum(mg, iv)
3197  type(mg_t), intent(in) :: mg
3198  integer, intent(in) :: iv
3199  integer :: lvl, i, id, nc
3200  real(dp) :: w
3201 
3202  get_sum = 0.0_dp
3203  do lvl = 1, mg%highest_lvl
3204  nc = mg%box_size_lvl(lvl)
3205  w = product(mg%dr(:, lvl)) ! Adjust for non-Cartesian cases
3206  do i = 1, size(mg%lvls(lvl)%my_leaves)
3207  id = mg%lvls(lvl)%my_leaves(i)
3208  get_sum = get_sum + w * &
3209  sum(mg%boxes(id)%cc(1:nc, 1:nc, iv))
3210  end do
3211  end do
3212  end function get_sum
3213 
3214  real(dp) function max_residual_lvl(mg, lvl)
3215  type(mg_t), intent(inout) :: mg
3216  integer, intent(in) :: lvl
3217  integer :: i, id, nc
3218  real(dp) :: res
3219 
3220  nc = mg%box_size_lvl(lvl)
3221  max_residual_lvl = 0.0_dp
3222 
3223  do i = 1, size(mg%lvls(lvl)%my_ids)
3224  id = mg%lvls(lvl)%my_ids(i)
3225  call residual_box(mg, id, nc)
3226  res = maxval(abs(mg%boxes(id)%cc(1:nc, 1:nc, mg_ires)))
3227  max_residual_lvl = max(max_residual_lvl, res)
3228  end do
3229  end function max_residual_lvl
3230 
3231  ! subroutine solve_coarse_grid(mg)
3232  ! use m_fishpack
3233  ! type(mg_t), intent(inout) :: mg
3234 
3235  ! real(dp) :: rhs(mg%box_size, mg%box_size)
3236  ! real(dp) :: rmin(2), rmax(2)
3237  ! integer :: nc, nx(2), my_boxes, total_boxes
3238 
3239  ! my_boxes = size(mg%lvls(1)%my_ids)
3240  ! total_boxes = size(mg%lvls(1)%ids)
3241  ! nc = mg%box_size
3242 
3243  ! if (my_boxes == total_boxes) then
3244  ! nx(:) = nc
3245  ! rmin = [0.0_dp, 0.0_dp]
3246  ! rmax = mg%dr(1) * [nc, nc]
3247  ! rhs = mg%boxes(1)%cc(1:nc, 1:nc, mg_irhs)
3248 
3249  ! #if 2 == 2
3250  ! call fishpack_2d(nx, rhs, mg%bc, rmin, rmax)
3251  ! #elif 2 == 3
3252  ! call fishpack_3d(nx, rhs, mg%bc, rmin, rmax)
3253  ! #endif
3254 
3255  ! mg%boxes(1)%cc(1:nc, 1:nc, mg_iphi) = rhs
3256  ! else if (my_boxes > 0) then
3257  ! error stop "Boxes at level 1 at different processors"
3258  ! end if
3259 
3260  ! call fill_ghost_cells_lvl(mg, 1)
3261  ! end subroutine solve_coarse_grid
3262 
3263  ! Set rhs on coarse grid, restrict phi, and copy mg_iphi to mg_iold for the
3264  ! correction later
3265  subroutine update_coarse(mg, lvl)
3266  type(mg_t), intent(inout) :: mg !< Tree containing full grid
3267  integer, intent(in) :: lvl !< Update coarse values at lvl-1
3268  integer :: i, id, nc, nc_c
3269 
3270  nc = mg%box_size_lvl(lvl)
3271  nc_c = mg%box_size_lvl(lvl-1)
3272 
3273  ! Compute residual
3274  do i = 1, size(mg%lvls(lvl)%my_ids)
3275  id = mg%lvls(lvl)%my_ids(i)
3276  call residual_box(mg, id, nc)
3277  end do
3278 
3279  ! Restrict phi and the residual
3280  call mg_restrict_lvl(mg, mg_iphi, lvl)
3281  call mg_restrict_lvl(mg, mg_ires, lvl)
3282 
3283  call mg_fill_ghost_cells_lvl(mg, lvl-1, mg_iphi)
3284 
3285  ! Set rhs_c = laplacian(phi_c) + restrict(res) where it is refined, and
3286  ! store current coarse phi in old.
3287  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3288  id = mg%lvls(lvl-1)%my_parents(i)
3289 
3290  ! Set rhs = L phi
3291  call mg%box_op(mg, id, nc_c, mg_irhs)
3292 
3293  ! Add the fine grid residual to rhs
3294  mg%boxes(id)%cc(1:nc_c, 1:nc_c, mg_irhs) = &
3295  mg%boxes(id)%cc(1:nc_c, 1:nc_c, mg_irhs) + &
3296  mg%boxes(id)%cc(1:nc_c, 1:nc_c, mg_ires)
3297 
3298  ! Story a copy of phi
3299  mg%boxes(id)%cc(:, :, mg_iold) = &
3300  mg%boxes(id)%cc(:, :, mg_iphi)
3301  end do
3302  end subroutine update_coarse
3303 
3304  ! Sets phi = phi + prolong(phi_coarse - phi_old_coarse)
3305  subroutine correct_children(mg, lvl)
3306  type(mg_t), intent(inout) :: mg
3307  integer, intent(in) :: lvl
3308  integer :: i, id
3309 
3310  do i = 1, size(mg%lvls(lvl)%my_parents)
3311  id = mg%lvls(lvl)%my_parents(i)
3312 
3313  ! Store the correction in mg_ires
3314  mg%boxes(id)%cc(:, :, mg_ires) = &
3315  mg%boxes(id)%cc(:, :, mg_iphi) - &
3316  mg%boxes(id)%cc(:, :, mg_iold)
3317  end do
3318 
3319  call mg_prolong(mg, lvl, mg_ires, mg_iphi, mg%box_prolong, add=.true.)
3320  end subroutine correct_children
3321 
3322  subroutine smooth_boxes(mg, lvl, n_cycle)
3323  type(mg_t), intent(inout) :: mg
3324  integer, intent(in) :: lvl
3325  integer, intent(in) :: n_cycle !< Number of cycles to perform
3326  integer :: n, i, id, nc
3327 
3328  nc = mg%box_size_lvl(lvl)
3329 
3330  do n = 1, n_cycle * mg%n_smoother_substeps
3331  call mg_timer_start(mg%timers(timer_smoother))
3332  do i = 1, size(mg%lvls(lvl)%my_ids)
3333  id = mg%lvls(lvl)%my_ids(i)
3334  call mg%box_smoother(mg, id, nc, n)
3335  end do
3336  call mg_timer_end(mg%timers(timer_smoother))
3337 
3338  call mg_timer_start(mg%timers(timer_smoother_gc))
3339  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3340  call mg_timer_end(mg%timers(timer_smoother_gc))
3341  end do
3342  end subroutine smooth_boxes
3343 
3344  subroutine residual_box(mg, id, nc)
3345  type(mg_t), intent(inout) :: mg
3346  integer, intent(in) :: id
3347  integer, intent(in) :: nc
3348 
3349  call mg%box_op(mg, id, nc, mg_ires)
3350 
3351  mg%boxes(id)%cc(1:nc, 1:nc, mg_ires) = &
3352  mg%boxes(id)%cc(1:nc, 1:nc, mg_irhs) &
3353  - mg%boxes(id)%cc(1:nc, 1:nc, mg_ires)
3354  end subroutine residual_box
3355 
3356  !> Apply operator to the tree and store in variable i_out
3357  subroutine mg_apply_op(mg, i_out, op)
3358  type(mg_t), intent(inout) :: mg
3359  integer, intent(in) :: i_out
3360  procedure(mg_box_op), optional :: op
3361  integer :: lvl, i, id, nc
3362 
3363  do lvl = mg%lowest_lvl, mg%highest_lvl
3364  nc = mg%box_size_lvl(lvl)
3365  do i = 1, size(mg%lvls(lvl)%my_ids)
3366  id = mg%lvls(lvl)%my_ids(i)
3367  if (present(op)) then
3368  call op(mg, id, nc, i_out)
3369  else
3370  call mg%box_op(mg, id, nc, i_out)
3371  end if
3372  end do
3373  end do
3374  end subroutine mg_apply_op
3375 
3376  !! File ../src/m_restrict.f90
3377 
3378  !> Specify minimum buffer size (per process) for communication
3379  subroutine mg_restrict_buffer_size(mg, n_send, n_recv, dsize)
3380  type(mg_t), intent(inout) :: mg
3381  integer, intent(out) :: n_send(0:mg%n_cpu-1)
3382  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
3383  integer, intent(out) :: dsize
3384  integer :: n_out(0:mg%n_cpu-1, mg%first_normal_lvl:mg%highest_lvl)
3385  integer :: n_in(0:mg%n_cpu-1, mg%first_normal_lvl:mg%highest_lvl)
3386  integer :: lvl, i, id, p_id, p_rank
3387  integer :: i_c, c_id, c_rank, min_lvl
3388 
3389  n_out(:, :) = 0
3390  n_in(:, :) = 0
3391  min_lvl = max(mg%lowest_lvl+1, mg%first_normal_lvl)
3392 
3393  do lvl = min_lvl, mg%highest_lvl
3394  ! Number of messages to receive (at lvl-1)
3395  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3396  id = mg%lvls(lvl-1)%my_parents(i)
3397  do i_c = 1, mg_num_children
3398  c_id = mg%boxes(id)%children(i_c)
3399 
3400  if (c_id > mg_no_box) then
3401  c_rank = mg%boxes(c_id)%rank
3402  if (c_rank /= mg%my_rank) then
3403  n_in(c_rank, lvl) = n_in(c_rank, lvl) + 1
3404  end if
3405  end if
3406  end do
3407  end do
3408 
3409  ! Number of messages to send
3410  do i = 1, size(mg%lvls(lvl)%my_ids)
3411  id = mg%lvls(lvl)%my_ids(i)
3412 
3413  p_id = mg%boxes(id)%parent
3414  p_rank = mg%boxes(p_id)%rank
3415  if (p_rank /= mg%my_rank) then
3416  n_out(p_rank, lvl) = n_out(p_rank, lvl) + 1
3417  end if
3418 
3419  end do
3420  end do
3421 
3422  allocate(mg%comm_restrict%n_send(0:mg%n_cpu-1, &
3423  mg%first_normal_lvl:mg%highest_lvl))
3424  allocate(mg%comm_restrict%n_recv(0:mg%n_cpu-1, &
3425  mg%first_normal_lvl:mg%highest_lvl))
3426  mg%comm_restrict%n_send = n_out
3427  mg%comm_restrict%n_recv = n_in
3428 
3429  dsize = (mg%box_size/2)**2
3430  n_send = maxval(n_out, dim=2)
3431  n_recv = maxval(n_in, dim=2)
3432  end subroutine mg_restrict_buffer_size
3433 
3434  !> Restrict all levels
3435  subroutine mg_restrict(mg, iv)
3436  type(mg_t), intent(inout) :: mg
3437  integer, intent(in) :: iv
3438  integer :: lvl
3439 
3440  do lvl = mg%highest_lvl, mg%lowest_lvl+1, -1
3441  call mg_restrict_lvl(mg, iv, lvl)
3442  end do
3443  end subroutine mg_restrict
3444 
3445  !> Restrict from lvl to lvl-1
3446  subroutine mg_restrict_lvl(mg, iv, lvl)
3447 
3448  type(mg_t), intent(inout) :: mg
3449  integer, intent(in) :: iv
3450  integer, intent(in) :: lvl
3451  integer :: i, id, dsize, nc
3452 
3453  if (lvl <= mg%lowest_lvl) error stop "cannot restrict lvl <= lowest_lvl"
3454 
3455  nc = mg%box_size_lvl(lvl)
3456 
3457  if (lvl >= mg%first_normal_lvl) then
3458  dsize = (nc/2)**2
3459 
3460  mg%buf(:)%i_send = 0
3461  mg%buf(:)%i_ix = 0
3462 
3463  do i = 1, size(mg%lvls(lvl)%my_ids)
3464  id = mg%lvls(lvl)%my_ids(i)
3465  call restrict_set_buffer(mg, id, iv)
3466  end do
3467 
3468  mg%buf(:)%i_recv = mg%comm_restrict%n_recv(:, lvl) * dsize
3469  call sort_and_transfer_buffers(mg, dsize)
3470  mg%buf(:)%i_recv = 0
3471  end if
3472 
3473  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3474  id = mg%lvls(lvl-1)%my_parents(i)
3475  call restrict_onto(mg, id, nc, iv)
3476  end do
3477  end subroutine mg_restrict_lvl
3478 
3479  subroutine restrict_set_buffer(mg, id, iv)
3480  type(mg_t), intent(inout) :: mg
3481  integer, intent(in) :: id
3482  integer, intent(in) :: iv
3483  integer :: i, j, n, hnc, p_id, p_rank
3484  real(dp) :: tmp(mg%box_size/2, mg%box_size/2)
3485 
3486  hnc = mg%box_size/2
3487  p_id = mg%boxes(id)%parent
3488  p_rank = mg%boxes(p_id)%rank
3489 
3490  if (p_rank /= mg%my_rank) then
3491  do j = 1, hnc
3492  do i = 1, hnc
3493  tmp(i, j) = 0.25_dp * &
3494  sum(mg%boxes(id)%cc(2*i-1:2*i, 2*j-1:2*j, iv))
3495  end do
3496  end do
3497 
3498  ! Buffer
3499  n = size(tmp)
3500  i = mg%buf(p_rank)%i_send
3501  mg%buf(p_rank)%send(i+1:i+n) = pack(tmp, .true.)
3502  mg%buf(p_rank)%i_send = mg%buf(p_rank)%i_send + n
3503 
3504  ! To later sort the send buffer according to parent order
3505  i = mg%buf(p_rank)%i_ix
3506  n = mg_ix_to_ichild(mg%boxes(id)%ix)
3507  mg%buf(p_rank)%ix(i+1) = mg_num_children * p_id + n
3508  mg%buf(p_rank)%i_ix = mg%buf(p_rank)%i_ix + 1
3509  end if
3510  end subroutine restrict_set_buffer
3511 
3512  subroutine restrict_onto(mg, id, nc, iv)
3513  type(mg_t), intent(inout) :: mg
3514  integer, intent(in) :: id
3515  integer, intent(in) :: nc
3516  integer, intent(in) :: iv
3517  integer :: i, j, hnc, dsize, i_c, c_id
3518  integer :: c_rank, dix(2)
3519 
3520  hnc = nc/2
3521  dsize = hnc**2
3522 
3523  do i_c = 1, mg_num_children
3524  c_id = mg%boxes(id)%children(i_c)
3525  if (c_id == mg_no_box) cycle ! For coarsened grid
3526  c_rank = mg%boxes(c_id)%rank
3527  dix = mg_get_child_offset(mg, c_id)
3528 
3529  if (c_rank == mg%my_rank) then
3530  do j=1, hnc; do i=1, hnc
3531  mg%boxes(id)%cc(dix(1)+i, dix(2)+j, iv) = 0.25_dp * &
3532  sum(mg%boxes(c_id)%cc(2*i-1:2*i, 2*j-1:2*j, iv))
3533  end do; end do
3534  else
3535  i = mg%buf(c_rank)%i_recv
3536  mg%boxes(id)%cc(dix(1)+1:dix(1)+hnc, &
3537  dix(2)+1:dix(2)+hnc, iv) = &
3538  reshape(mg%buf(c_rank)%recv(i+1:i+dsize), [hnc, hnc])
3539  mg%buf(c_rank)%i_recv = mg%buf(c_rank)%i_recv + dsize
3540  end if
3541  end do
3542 
3543  end subroutine restrict_onto
3544 
3545  !! File ../src/m_mrgrnk.f90
3546 
3547  Subroutine i_mrgrnk (XDONT, IRNGT)
3548  ! __________________________________________________________
3549  ! MRGRNK = Merge-sort ranking of an array
3550  ! For performance reasons, the first 2 passes are taken
3551  ! out of the standard loop, and use dedicated coding.
3552  ! __________________________________________________________
3553  ! __________________________________________________________
3554  Integer, Dimension (:), Intent (In) :: xdont
3555  Integer, Dimension (:), Intent (Out) :: irngt
3556  ! __________________________________________________________
3557  Integer :: xvala, xvalb
3558  !
3559  Integer, Dimension (SIZE(IRNGT)) :: jwrkt
3560  Integer :: lmtna, lmtnc, irng1, irng2
3561  Integer :: nval, iind, iwrkd, iwrk, iwrkf, jinda, iinda, iindb
3562  !
3563  nval = min(SIZE(xdont), SIZE(irngt))
3564  Select Case (nval)
3565  Case (:0)
3566  Return
3567  Case (1)
3568  irngt(1) = 1
3569  Return
3570  Case Default
3571  Continue
3572  End Select
3573  !
3574  ! Fill-in the index array, creating ordered couples
3575  !
3576  Do iind = 2, nval, 2
3577  If (xdont(iind-1) <= xdont(iind)) Then
3578  irngt(iind-1) = iind - 1
3579  irngt(iind) = iind
3580  Else
3581  irngt(iind-1) = iind
3582  irngt(iind) = iind - 1
3583  End If
3584  End Do
3585  If (modulo(nval, 2) /= 0) Then
3586  irngt(nval) = nval
3587  End If
3588  !
3589  ! We will now have ordered subsets A - B - A - B - ...
3590  ! and merge A and B couples into C - C - ...
3591  !
3592  lmtna = 2
3593  lmtnc = 4
3594  !
3595  ! First iteration. The length of the ordered subsets goes from 2 to 4
3596  !
3597  Do
3598  If (nval <= 2) Exit
3599  !
3600  ! Loop on merges of A and B into C
3601  !
3602  Do iwrkd = 0, nval - 1, 4
3603  If ((iwrkd+4) > nval) Then
3604  If ((iwrkd+2) >= nval) Exit
3605  !
3606  ! 1 2 3
3607  !
3608  If (xdont(irngt(iwrkd+2)) <= xdont(irngt(iwrkd+3))) Exit
3609  !
3610  ! 1 3 2
3611  !
3612  If (xdont(irngt(iwrkd+1)) <= xdont(irngt(iwrkd+3))) Then
3613  irng2 = irngt(iwrkd+2)
3614  irngt(iwrkd+2) = irngt(iwrkd+3)
3615  irngt(iwrkd+3) = irng2
3616  !
3617  ! 3 1 2
3618  !
3619  Else
3620  irng1 = irngt(iwrkd+1)
3621  irngt(iwrkd+1) = irngt(iwrkd+3)
3622  irngt(iwrkd+3) = irngt(iwrkd+2)
3623  irngt(iwrkd+2) = irng1
3624  End If
3625  Exit
3626  End If
3627  !
3628  ! 1 2 3 4
3629  !
3630  If (xdont(irngt(iwrkd+2)) <= xdont(irngt(iwrkd+3))) cycle
3631  !
3632  ! 1 3 x x
3633  !
3634  If (xdont(irngt(iwrkd+1)) <= xdont(irngt(iwrkd+3))) Then
3635  irng2 = irngt(iwrkd+2)
3636  irngt(iwrkd+2) = irngt(iwrkd+3)
3637  If (xdont(irng2) <= xdont(irngt(iwrkd+4))) Then
3638  ! 1 3 2 4
3639  irngt(iwrkd+3) = irng2
3640  Else
3641  ! 1 3 4 2
3642  irngt(iwrkd+3) = irngt(iwrkd+4)
3643  irngt(iwrkd+4) = irng2
3644  End If
3645  !
3646  ! 3 x x x
3647  !
3648  Else
3649  irng1 = irngt(iwrkd+1)
3650  irng2 = irngt(iwrkd+2)
3651  irngt(iwrkd+1) = irngt(iwrkd+3)
3652  If (xdont(irng1) <= xdont(irngt(iwrkd+4))) Then
3653  irngt(iwrkd+2) = irng1
3654  If (xdont(irng2) <= xdont(irngt(iwrkd+4))) Then
3655  ! 3 1 2 4
3656  irngt(iwrkd+3) = irng2
3657  Else
3658  ! 3 1 4 2
3659  irngt(iwrkd+3) = irngt(iwrkd+4)
3660  irngt(iwrkd+4) = irng2
3661  End If
3662  Else
3663  ! 3 4 1 2
3664  irngt(iwrkd+2) = irngt(iwrkd+4)
3665  irngt(iwrkd+3) = irng1
3666  irngt(iwrkd+4) = irng2
3667  End If
3668  End If
3669  End Do
3670  !
3671  ! The Cs become As and Bs
3672  !
3673  lmtna = 4
3674  Exit
3675  End Do
3676  !
3677  ! Iteration loop. Each time, the length of the ordered subsets
3678  ! is doubled.
3679  !
3680  Do
3681  If (lmtna >= nval) Exit
3682  iwrkf = 0
3683  lmtnc = 2 * lmtnc
3684  !
3685  ! Loop on merges of A and B into C
3686  !
3687  Do
3688  iwrk = iwrkf
3689  iwrkd = iwrkf + 1
3690  jinda = iwrkf + lmtna
3691  iwrkf = iwrkf + lmtnc
3692  If (iwrkf >= nval) Then
3693  If (jinda >= nval) Exit
3694  iwrkf = nval
3695  End If
3696  iinda = 1
3697  iindb = jinda + 1
3698  !
3699  ! Shortcut for the case when the max of A is smaller
3700  ! than the min of B. This line may be activated when the
3701  ! initial set is already close to sorted.
3702  !
3703  ! IF (XDONT(IRNGT(JINDA)) <= XDONT(IRNGT(IINDB))) CYCLE
3704  !
3705  ! One steps in the C subset, that we build in the final rank array
3706  !
3707  ! Make a copy of the rank array for the merge iteration
3708  !
3709  jwrkt(1:lmtna) = irngt(iwrkd:jinda)
3710  !
3711  xvala = xdont(jwrkt(iinda))
3712  xvalb = xdont(irngt(iindb))
3713  !
3714  Do
3715  iwrk = iwrk + 1
3716  !
3717  ! We still have unprocessed values in both A and B
3718  !
3719  If (xvala > xvalb) Then
3720  irngt(iwrk) = irngt(iindb)
3721  iindb = iindb + 1
3722  If (iindb > iwrkf) Then
3723  ! Only A still with unprocessed values
3724  irngt(iwrk+1:iwrkf) = jwrkt(iinda:lmtna)
3725  Exit
3726  End If
3727  xvalb = xdont(irngt(iindb))
3728  Else
3729  irngt(iwrk) = jwrkt(iinda)
3730  iinda = iinda + 1
3731  If (iinda > lmtna) exit! Only B still with unprocessed values
3732  xvala = xdont(jwrkt(iinda))
3733  End If
3734  !
3735  End Do
3736  End Do
3737  !
3738  ! The Cs become As and Bs
3739  !
3740  lmtna = 2 * lmtna
3741  End Do
3742  !
3743  Return
3744  !
3745  End Subroutine i_mrgrnk
3746 
3747  !! File ../src/m_ahelmholtz.f90
3748 
3749  subroutine ahelmholtz_set_methods(mg)
3750  type(mg_t), intent(inout) :: mg
3751 
3752  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
3753  error stop "ahelmholtz_set_methods: mg%n_extra_vars == 0"
3754  else
3755  mg%n_extra_vars = max(2, mg%n_extra_vars)
3756  end if
3757 
3758  ! Use Neumann zero boundary conditions for the variable coefficient, since
3759  ! it is needed in ghost cells.
3760  mg%bc(:, mg_iveps1)%bc_type = mg_bc_neumann
3761  mg%bc(:, mg_iveps1)%bc_value = 0.0_dp
3762 
3763  mg%bc(:, mg_iveps2)%bc_type = mg_bc_neumann
3764  mg%bc(:, mg_iveps2)%bc_value = 0.0_dp
3765 
3766 
3767  select case (mg%geometry_type)
3768  case (mg_cartesian)
3769  mg%box_op => box_ahelmh
3770 
3771  select case (mg%smoother_type)
3773  mg%box_smoother => box_gs_ahelmh
3774  case default
3775  error stop "ahelmholtz_set_methods: unsupported smoother type"
3776  end select
3777  case default
3778  error stop "ahelmholtz_set_methods: unsupported geometry"
3779  end select
3780 
3781  end subroutine ahelmholtz_set_methods
3782 
3783  subroutine ahelmholtz_set_lambda(lambda)
3784  real(dp), intent(in) :: lambda
3785 
3786  if (lambda < 0) &
3787  error stop "ahelmholtz_set_lambda: lambda < 0 not allowed"
3788 
3789  ahelmholtz_lambda = lambda
3790  end subroutine ahelmholtz_set_lambda
3791 
3792  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
3793  subroutine box_gs_ahelmh(mg, id, nc, redblack_cntr)
3794  type(mg_t), intent(inout) :: mg
3795  integer, intent(in) :: id
3796  integer, intent(in) :: nc
3797  integer, intent(in) :: redblack_cntr !< Iteration counter
3798  integer :: i, j, i0, di
3799  logical :: redblack
3800  real(dp) :: idr2(2*2), u(2*2)
3801  real(dp) :: a0(2*2), a(2*2), c(2*2)
3802 
3803  ! Duplicate 1/dr^2 array to multiply neighbor values
3804  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
3805  idr2(2:2*2:2) = idr2(1:2*2:2)
3806  i0 = 1
3807 
3808  redblack = (mg%smoother_type == mg_smoother_gsrb)
3809  if (redblack) then
3810  di = 2
3811  else
3812  di = 1
3813  end if
3814 
3815  ! The parity of redblack_cntr determines which cells we use. If
3816  ! redblack_cntr is even, we use the even cells and vice versa.
3817  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
3818  i_eps1 => mg_iveps1, &
3819  i_eps2 => mg_iveps2)
3820 
3821  do j = 1, nc
3822  if (redblack) &
3823  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
3824 
3825  do i = i0, nc, di
3826  a0(1:2) = cc(i, j, i_eps1)
3827  a0(3:4) = cc(i, j, i_eps2)
3828  u(1:2) = cc(i-1:i+1:2, j, n)
3829  a(1:2) = cc(i-1:i+1:2, j, i_eps1)
3830  u(3:4) = cc(i, j-1:j+1:2, n)
3831  a(3:4) = cc(i, j-1:j+1:2, i_eps2)
3832  c(:) = 2 * a0(:) * a(:) / (a0(:) + a(:)) * idr2
3833 
3834  cc(i, j, n) = &
3835  (sum(c(:) * u(:)) - cc(i, j, mg_irhs)) / &
3836  (sum(c(:)) + ahelmholtz_lambda)
3837  end do
3838  end do
3839  end associate
3840  end subroutine box_gs_ahelmh
3841 
3842  !> Perform Helmholtz operator on a box
3843  subroutine box_ahelmh(mg, id, nc, i_out)
3844  type(mg_t), intent(inout) :: mg
3845  integer, intent(in) :: id
3846  integer, intent(in) :: nc
3847  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
3848  integer :: i, j
3849  real(dp) :: idr2(2*2), a0(2*2), u0, u(2*2), a(2*2)
3850 
3851  ! Duplicate 1/dr^2 array to multiply neighbor values
3852  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
3853  idr2(2:2*2:2) = idr2(1:2*2:2)
3854 
3855  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
3856  i_eps1 => mg_iveps1, &
3857  i_eps2 => mg_iveps2)
3858  do j = 1, nc
3859  do i = 1, nc
3860  a0(1:2) = cc(i, j, i_eps1)
3861  a0(3:4) = cc(i, j, i_eps2)
3862  a(1:2) = cc(i-1:i+1:2, j, i_eps1)
3863  a(3:4) = cc(i, j-1:j+1:2, i_eps2)
3864  u0 = cc(i, j, n)
3865  u(1:2) = cc(i-1:i+1:2, j, n)
3866  u(3:4) = cc(i, j-1:j+1:2, n)
3867 
3868  cc(i, j, i_out) = sum(2 * idr2 * &
3869  a0(:)*a(:)/(a0(:) + a(:)) * (u(:) - u0)) - &
3870  ahelmholtz_lambda * u0
3871  end do
3872  end do
3873  end associate
3874  end subroutine box_ahelmh
3875 
3876 end module m_octree_mg_2d
To fill ghost cells near physical boundaries.
subroutine, public mg_comm_init(mg, comm)
Prolong from a parent to a child with index offset dix. This method could sometimes work better than ...
subroutine, public mg_fas_vcycle(mg, highest_lvl, max_res, standalone)
Perform FAS V-cycle (full approximation scheme).
subroutine, public vhelmholtz_set_methods(mg)
subroutine, public mg_ghost_cell_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
integer, parameter, public mg_num_children
subroutine, public mg_timer_end(timer)
subroutine, public mg_restrict(mg, iv)
Restrict all levels.
integer, parameter, public mg_vhelmholtz
Indicates a variable-coefficient Helmholtz equation.
integer, parameter, public mg_iveps1
Indexes of anisotropic variable coefficients.
integer, parameter, public mg_ahelmholtz
Indicates a anisotropic-coefficient Helmholtz equation.
integer, parameter, public mg_vlaplacian
Indicates a variable-coefficient Laplacian.
integer, dimension(2, 4), parameter, public mg_neighb_dix
integer, parameter, public mg_helmholtz
Indicates a constant-coefficient Helmholtz equation.
integer, parameter, public mg_physical_boundary
Special value that indicates there is a physical boundary.
subroutine, public mg_prolong_sparse(mg, p_id, dix, nc, iv, fine)
Prolong from a parent to a child with index offset dix.
subroutine, public mg_get_face_coords(box, nb, nc, x)
Get coordinates at the face of a box.
subroutine, public sort_and_transfer_buffers(mg, dsize)
integer, parameter, public dp
Type of reals.
pure integer function, public mg_highest_uniform_lvl(mg)
integer function, public mg_ix_to_ichild(ix)
Compute the child index for a box with spatial index ix. With child index we mean the index in the ch...
integer, parameter, public mg_ndim
Problem dimension.
subroutine, public mg_prolong_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
subroutine, public diffusion_solve_acoeff(mg, dt, order, max_res)
Solve a diffusion equation implicitly, assuming anisotropic diffusion coefficient which has been stor...
subroutine, public mg_fas_fmg(mg, have_guess, max_res)
Perform FAS-FMG cycle (full approximation scheme, full multigrid).
integer, parameter, public mg_ires
Index of residual.
integer, parameter, public mg_neighb_highy
subroutine, public mg_set_neighbors_lvl(mg, lvl)
integer, parameter, public mg_iveps3
subroutine, public mg_load_balance_simple(mg)
Load balance all boxes in the multigrid tree, by simply distributing the load per grid level....
subroutine, public helmholtz_set_methods(mg)
integer, parameter, public mg_bc_continuous
Value to indicate a continuous boundary condition.
subroutine, public mg_timers_show(mg)
integer, parameter, public mg_iphi
Index of solution.
subroutine, public mg_add_children(mg, id)
subroutine, public mg_prolong(mg, lvl, iv, iv_to, method, add)
Prolong variable iv from lvl to variable iv_to at lvl+1.
integer, dimension(4), parameter, public mg_neighb_dim
integer, dimension(2, 4), parameter, public mg_child_dix
integer, parameter, public mg_spherical
Spherical coordinate system.
subroutine, public mg_timer_start(timer)
integer(i8) function, public mg_number_of_unknowns(mg)
Determine total number of unknowns (on leaves)
integer, parameter, public mg_neighb_highx
integer, parameter, public mg_bc_dirichlet
Value to indicate a Dirichlet boundary condition.
subroutine, public helmholtz_set_lambda(lambda)
integer, parameter, public mg_laplacian
Indicates a standard Laplacian.
integer, dimension(4), parameter, public mg_neighb_rev
subroutine, public ahelmholtz_set_lambda(lambda)
real(dp), public, protected ahelmholtz_lambda
Module which contains multigrid procedures for a Helmholtz operator of the form: div(D grad(phi)) - l...
integer, parameter, public mg_lvl_hi
Maximum allowed grid level.
integer, dimension(4), parameter, public mg_neighb_high_pm
integer, parameter, public mg_bc_neumann
Value to indicate a Neumann boundary condition.
integer, parameter, public mg_irhs
Index of right-hand side.
logical, dimension(2, 4), parameter, public mg_child_low
integer, parameter, public mg_iveps2
elemental logical function, public mg_has_children(box)
Return .true. if a box has children.
subroutine sort_sendbuf(gc, dsize)
Sort send buffers according to the idbuf array.
subroutine, public diffusion_solve_vcoeff(mg, dt, order, max_res)
Solve a diffusion equation implicitly, assuming a variable diffusion coefficient which has been store...
integer, parameter, public mg_cylindrical
Cylindrical coordinate system.
real(dp), public, protected vhelmholtz_lambda
Module for implicitly solving diffusion equations.
subroutine, public mg_restrict_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
integer, parameter, public mg_smoother_jacobi
subroutine, public laplacian_set_methods(mg)
subroutine, public mg_deallocate_storage(mg)
Deallocate all allocatable arrays.
subroutine, public ahelmholtz_set_methods(mg)
subroutine, public mg_load_balance(mg)
Load balance all boxes in the multigrid tree. Compared to mg_load_balance_simple, this method does a ...
subroutine, public mg_fill_ghost_cells_lvl(mg, lvl, iv)
Fill ghost cells at a grid level.
subroutine, public vhelmholtz_set_lambda(lambda)
subroutine, public mg_set_next_level_ids(mg, lvl)
integer, parameter, public mg_num_neighbors
integer, parameter, public mg_cartesian
Cartesian coordinate system.
integer, dimension(2, 4), parameter, public mg_child_adj_nb
subroutine, public mg_set_refinement_boundaries(boxes, level)
Create a list of refinement boundaries (from the coarse side)
integer, parameter, public mg_neighb_lowx
integer, parameter, public mg_neighb_lowy
integer, parameter, public mg_num_vars
Number of predefined multigrid variables.
subroutine subtract_mean(mg, iv, include_ghostcells)
integer, parameter, public mg_lvl_lo
Minimum allowed grid level.
subroutine, public mg_set_leaves_parents(boxes, level)
Create a list of leaves and a list of parents for a level.
subroutine, public mg_load_balance_parents(mg)
Load balance the parents (non-leafs). Assign them to the rank that has most children.
subroutine, public mg_fill_ghost_cells(mg, iv)
Fill ghost cells at all grid levels.
subroutine, public mg_restrict_lvl(mg, iv, lvl)
Restrict from lvl to lvl-1.
subroutine, public vlaplacian_set_methods(mg)
integer, parameter, public mg_iveps
Index of the variable coefficient (at cell centers)
subroutine, public mg_allocate_storage(mg)
Allocate communication buffers and local boxes for a tree that has already been created.
real(dp) function get_sum(mg, iv)
subroutine, public mg_build_rectangle(mg, domain_size, box_size, dx, r_min, periodic, n_finer)
subroutine, public mg_apply_op(mg, i_out, op)
Apply operator to the tree and store in variable i_out.
integer, parameter, public i8
Type for 64-bit integers.
subroutine, public mg_set_methods(mg)
integer, parameter, public mg_max_num_vars
Maximum number of variables.
pure integer function, dimension(2), public mg_get_child_offset(mg, id)
Get the offset of a box with respect to its parent (e.g. in 2d, there can be a child at offset 0,...
integer, parameter, public mg_iold
Index of previous solution (used for correction)
logical, dimension(4), parameter, public mg_neighb_low
integer, parameter, public mg_max_timers
Maximum number of timers to use.
integer, parameter, public mg_smoother_gs
real(dp), public, protected helmholtz_lambda
Module for load balancing a tree (that already has been constructed). The load balancing determines w...
subroutine, public diffusion_solve(mg, dt, diffusion_coeff, order, max_res)
Solve a diffusion equation implicitly, assuming a constant diffusion coefficient. The solution at tim...
subroutine, public mg_phi_bc_store(mg)
Store boundary conditions for the solution variable, this can speed up calculations if the same bound...
integer, dimension(4, 2), parameter, public mg_child_rev
integer, parameter, public mg_no_box
Special value that indicates there is no box.
integer, parameter, public mg_smoother_gsrb
integer function, public mg_add_timer(mg, name)
Box data structure.
Buffer type (one is used for each pair of communicating processes)
Lists of blocks per refinement level.