MPI-AMRVAC  3.1
The MPI - Adaptive Mesh Refinement - Versatile Advection Code
m_octree_mg_2d.t
Go to the documentation of this file.
1 
2 
3 ! Single module generated from the octree-mg sources.
4 ! This file can be easier to include in existing projects.
5 !
6 ! Notes:
7 ! 1. The module name is here extended by _1d, _2d or _3d
8 ! 2. The free space Poisson solver is not included here.
9 ! 3. It is best to make changes in the original repository at
10 ! https://github.com/jannisteunissen/octree-mg
11 ! 4. This file can be generated as follows:
12 ! cd octree-mg/single_module
13 ! ./to_single_module.sh
14 !
15 ! The modules can be compiled with:
16 ! mpif90 -c m_octree_mg_1d.f90 [other options]
17 ! mpif90 -c m_octree_mg_2d.f90 [other options]
18 ! mpif90 -c m_octree_mg_3d.f90 [other options]
19 ! mpif90 -c m_octree_mg.f90 -cpp -DNDIM=<1,2,3> [other options]
20 
21 
23  use mpi
24  implicit none
25  private
26 
27  !! File ../src/m_data_structures.f90
28 
29  !> Type of reals
30  integer, parameter, public :: dp = kind(0.0d0)
31 
32  !> Type for 64-bit integers
33  integer, parameter, public :: i8 = selected_int_kind(18)
34 
35  !> Indicates a standard Laplacian
36  integer, parameter, public :: mg_laplacian = 1
37 
38  !> Indicates a variable-coefficient Laplacian
39  integer, parameter, public :: mg_vlaplacian = 2
40 
41  !> Indicates a constant-coefficient Helmholtz equation
42  integer, parameter, public :: mg_helmholtz = 3
43 
44  !> Indicates a variable-coefficient Helmholtz equation
45  integer, parameter, public :: mg_vhelmholtz = 4
46 
47  !> Indicates a anisotropic-coefficient Helmholtz equation
48  integer, parameter, public :: mg_ahelmholtz = 5
49 
50  !> Cartesian coordinate system
51  integer, parameter, public :: mg_cartesian = 1
52  !> Cylindrical coordinate system
53  integer, parameter, public :: mg_cylindrical = 2
54  !> Spherical coordinate system
55  integer, parameter, public :: mg_spherical = 3
56 
57  integer, parameter, public :: mg_smoother_gs = 1
58  integer, parameter, public :: mg_smoother_gsrb = 2
59  integer, parameter, public :: mg_smoother_jacobi = 3
60 
61  !> Problem dimension
62  integer, parameter, public :: mg_ndim = 2
63 
64  !> Number of predefined multigrid variables
65  integer, parameter, public :: mg_num_vars = 4
66  !> Maximum number of variables
67  integer, parameter, public :: mg_max_num_vars = 10
68  !> Index of solution
69  integer, parameter, public :: mg_iphi = 1
70  !> Index of right-hand side
71  integer, parameter, public :: mg_irhs = 2
72  !> Index of previous solution (used for correction)
73  integer, parameter, public :: mg_iold = 3
74  !> Index of residual
75  integer, parameter, public :: mg_ires = 4
76 
77  !> Index of the variable coefficient (at cell centers)
78  integer, parameter, public :: mg_iveps = 5
79 
80  !> Indexes of anisotropic variable coefficients
81  integer, parameter, public :: mg_iveps1 = 5
82  integer, parameter, public :: mg_iveps2 = 6
83  integer, parameter, public :: mg_iveps3 = 7
84 
85  !> Minimum allowed grid level
86  integer, parameter, public :: mg_lvl_lo = -20
87  !> Maximum allowed grid level
88  integer, parameter, public :: mg_lvl_hi = 20
89 
90  !> Value to indicate a Dirichlet boundary condition
91  integer, parameter, public :: mg_bc_dirichlet = -10
92 
93  !> Value to indicate a Neumann boundary condition
94  integer, parameter, public :: mg_bc_neumann = -11
95 
96  !> Value to indicate a continuous boundary condition
97  integer, parameter, public :: mg_bc_continuous = -12
98 
99  !> Special value that indicates there is no box
100  integer, parameter, public :: mg_no_box = 0
101  !> Special value that indicates there is a physical boundary
102  integer, parameter, public :: mg_physical_boundary = -1
103 
104  !> Maximum number of timers to use
105  integer, parameter, public :: mg_max_timers = 20
106 
107  ! Numbering of children (same location as **corners**)
108  integer, parameter, public :: mg_num_children = 4
109 
110  ! Index offset for each child
111  integer, parameter, public :: mg_child_dix(2, 4) = reshape([0,0,1,0,0,1,1,1], [2,4])
112  ! Reverse child index in each direction
113  integer, parameter, public :: mg_child_rev(4, 2) = reshape([2,1,4,3,3,4,1,2], [4,2])
114  ! Children adjacent to a neighbor
115  integer, parameter, public :: mg_child_adj_nb(2, 4) = reshape([1,3,2,4,1,2,3,4], [2,4])
116  ! Which children have a low index per dimension
117  logical, parameter, public :: mg_child_low(2, 4) = reshape([.true., .true., &
118  .false., .true., .true., .false., .false., .false.], [2, 4])
119 
120  ! Neighbor topology information
121  integer, parameter, public :: mg_num_neighbors = 4
122  integer, parameter, public :: mg_neighb_lowx = 1
123  integer, parameter, public :: mg_neighb_highx = 2
124  integer, parameter, public :: mg_neighb_lowy = 3
125  integer, parameter, public :: mg_neighb_highy = 4
126 
127  ! Index offsets of neighbors
128  integer, parameter, public :: mg_neighb_dix(2, 4) = reshape([-1,0,1,0,0,-1,0,1], [2,4])
129  ! Which neighbors have a lower index
130  logical, parameter, public :: mg_neighb_low(4) = [.true., .false., .true., .false.]
131  ! Opposite of nb_low, but now as -1,1 integers
132  integer, parameter, public :: mg_neighb_high_pm(4) = [-1, 1, -1, 1]
133 
134  ! Reverse neighbors
135  integer, parameter, public :: mg_neighb_rev(4) = [2, 1, 4, 3]
136  ! Direction (dimension) for a neighbor
137  integer, parameter, public :: mg_neighb_dim(4) = [1, 1, 2, 2]
138 
139  !> Lists of blocks per refinement level
140  type, public :: mg_lvl_t
141  integer, allocatable :: leaves(:)
142  integer, allocatable :: parents(:)
143  integer, allocatable :: ref_bnds(:)
144  integer, allocatable :: ids(:)
145  integer, allocatable :: my_leaves(:)
146  integer, allocatable :: my_parents(:)
147  integer, allocatable :: my_ref_bnds(:)
148  integer, allocatable :: my_ids(:)
149  end type mg_lvl_t
150 
151  !> Box data structure
152  type, public :: mg_box_t
153  integer :: rank !< Which process owns this box
154  integer :: id !< Box id (index in boxes(:) array)
155  integer :: lvl !< Refinement level
156  integer :: ix(2) !< Spatial index
157  integer :: parent !< Id of parent
158  integer :: children(2**2) !< Ids of children
159  integer :: neighbors(2*2) !< Ids of neighbors
160  real(dp) :: r_min(2) !< Minimum coordinate
161  real(dp) :: dr(2) !< Grid spacing
162  !> Cell-centered data
163  real(dp), allocatable :: cc(:, :, :)
164  end type mg_box_t
165 
166  !> Buffer type (one is used for each pair of communicating processes)
167  type, public :: mg_buf_t
168  integer :: i_send !< Index in send array
169  integer :: i_recv
170  integer :: i_ix
171  integer, allocatable :: ix(:) !< Will be used to sort the data
172  real(dp), allocatable :: send(:)
173  real(dp), allocatable :: recv(:)
174  end type mg_buf_t
175 
176  type, public :: mg_comm_t
177  integer, allocatable :: n_send(:, :)
178  integer, allocatable :: n_recv(:, :)
179  end type mg_comm_t
180 
181  type, public :: mg_bc_t
182  integer :: bc_type = mg_bc_dirichlet !< Type of boundary condition
183  real(dp) :: bc_value = 0.0_dp !< Value (for e.g. Dirichlet or Neumann)
184  !> To set user-defined boundary conditions (overrides bc(:))
185  procedure(mg_subr_bc), pointer, nopass :: boundary_cond => null()
186  !> To set a user-defined refinement boundary method
187  procedure(mg_subr_rb), pointer, nopass :: refinement_bnd => null()
188  end type mg_bc_t
189 
190  type, public :: mg_timer_t
191  character(len=20) :: name
192  real(dp) :: t = 0.0_dp
193  real(dp) :: t0
194  end type mg_timer_t
195 
196  type, public :: mg_t
197  !> Whether the multigrid tree structure has been created
198  logical :: tree_created = .false.
199  !> Whether storage has been allocated for boxes
200  logical :: is_allocated = .false.
201  !> Number of extra cell-centered variable (e.g., for coefficients)
202  integer :: n_extra_vars = 0
203  !> MPI communicator
204  integer :: comm = -1
205  !> Number of MPI tasks
206  integer :: n_cpu = -1
207  !> MPI rank of this task
208  integer :: my_rank = -1
209  !> Size of boxes in cells, be equal for all dimensions
210  integer :: box_size = -1
211  !> Highest grid level in the tree
212  integer :: highest_lvl = -1
213  !> Lowest grid level in the tree
214  integer :: lowest_lvl = -1
215  !> First normal level of the quadtree/octree, at coarser levels parents
216  !> have only one child
217  integer :: first_normal_lvl = -1
218  !> Total number of boxes in the tree (among all processes)
219  integer :: n_boxes = 0
220  !> Size of boxes per level (differs for coarsest levels)
221  integer :: box_size_lvl(mg_lvl_lo:mg_lvl_hi)
222  !> Size of domain per level (if uniformly refined)
223  integer :: domain_size_lvl(2, mg_lvl_lo:mg_lvl_hi)
224  !> Grid spacing per level
225  real(dp) :: dr(2, mg_lvl_lo:mg_lvl_hi)
226  !> Minimum coordinates
227  real(dp) :: r_min(2)
228  !> List of all levels
229  type(mg_lvl_t) :: lvls(mg_lvl_lo:mg_lvl_hi)
230  !> Array with all boxes in the tree. Only boxes owned by this task are
231  !> actually allocated
232  type(mg_box_t), allocatable :: boxes(:)
233  !> Buffer for communication with each other task
234  type(mg_buf_t), allocatable :: buf(:)
235 
236  !> Communication info for restriction
237  type(mg_comm_t) :: comm_restrict
238  !> Communication info for prolongation
239  type(mg_comm_t) :: comm_prolong
240  !> Communication info for ghost cell filling
241  type(mg_comm_t) :: comm_ghostcell
242 
243  !> Whether boundary condition data has been stored for mg solution
244  logical :: phi_bc_data_stored = .false.
245 
246  !> Whether a dimension is periodic
247  logical :: periodic(2) = .false.
248 
249  !> To store pre-defined boundary conditions per direction per variable
251 
252  !> Type of operator
253  integer :: operator_type = mg_laplacian
254  !> Type of grid geometry
255  integer :: geometry_type = mg_cartesian
256 
257  !> Whether the mean has to be subtracted from the multigrid solution
258  logical :: subtract_mean = .false.
259  !> Type of multigrid smoother
260  integer :: smoother_type = mg_smoother_gs
261  !> Number of substeps for the smoother (for GSRB this is 2)
262  integer :: n_smoother_substeps = 1
263  !> Number of cycles when doing downwards relaxation
264  integer :: n_cycle_down = 2
265  !> Number of cycles when doing upwards relaxation
266  integer :: n_cycle_up = 2
267  !> Maximum number of cycles on the coarse grid
268  integer :: max_coarse_cycles = 1000
269  integer :: coarsest_grid(2) = 2
270  !> Stop coarse grid when max. residual is smaller than this
271  real(dp) :: residual_coarse_abs = 1e-8_dp
272  !> Stop coarse grid when residual has been reduced by this factor
273  real(dp) :: residual_coarse_rel = 1e-8_dp
274 
275  !> Multigrid operator (e.g., Laplacian)
276  procedure(mg_box_op), pointer, nopass :: box_op => null()
277 
278  !> Multigrid smoother
279  procedure(mg_box_gsrb), pointer, nopass :: box_smoother => null()
280 
281  !> Multigrid prolongation method
282  procedure(mg_box_prolong), pointer, nopass :: box_prolong => null()
283 
284  !> Number of timers
285  integer :: n_timers = 0
286  !> Values for the timers
287  type(mg_timer_t) :: timers(mg_max_timers)
288  end type mg_t
289 
290  interface
291  !> To fill ghost cells near physical boundaries
292  subroutine mg_subr_bc(box, nc, iv, nb, bc_type, bc)
293  import
294  type(mg_box_t), intent(in) :: box
295  integer, intent(in) :: nc
296  integer, intent(in) :: iv !< Index of variable
297  integer, intent(in) :: nb !< Direction
298  integer, intent(out) :: bc_type !< Type of b.c.
299  !> Boundary values
300  real(dp), intent(out) :: bc(nc)
301  end subroutine mg_subr_bc
302 
303  !> To fill ghost cells near refinement boundaries
304  subroutine mg_subr_rb(box, nc, iv, nb, cgc)
305  import
306  type(mg_box_t), intent(inout) :: box
307  integer, intent(in) :: nc
308  integer, intent(in) :: iv !< Index of variable
309  integer, intent(in) :: nb !< Direction
310  !> Coarse data
311  real(dp), intent(in) :: cgc(nc)
312  end subroutine mg_subr_rb
313 
314  !> Subroutine that performs A * cc(..., i_in) = cc(..., i_out)
315  subroutine mg_box_op(mg, id, nc, i_out)
316  import
317  type(mg_t), intent(inout) :: mg
318  integer, intent(in) :: id
319  integer, intent(in) :: nc
320  integer, intent(in) :: i_out
321  end subroutine mg_box_op
322 
323  !> Subroutine that performs Gauss-Seidel relaxation
324  subroutine mg_box_gsrb(mg, id, nc, redblack_cntr)
325  import
326  type(mg_t), intent(inout) :: mg
327  integer, intent(in) :: id
328  integer, intent(in) :: nc
329  integer, intent(in) :: redblack_cntr
330  end subroutine mg_box_gsrb
331 
332  !> Subroutine that performs prolongation to a single child
333  subroutine mg_box_prolong(mg, p_id, dix, nc, iv, fine)
334  import
335  type(mg_t), intent(inout) :: mg
336  integer, intent(in) :: p_id !< Id of parent
337  integer, intent(in) :: dix(2) !< Offset of child in parent grid
338  integer, intent(in) :: nc !< Child grid size
339  integer, intent(in) :: iv !< Prolong from this variable
340  real(dp), intent(out) :: fine(nc, nc) !< Prolonged values
341  end subroutine mg_box_prolong
342  end interface
343 
344  ! Public methods
345  public :: mg_subr_bc
346  public :: mg_subr_rb
347  public :: mg_box_op
348  public :: mg_box_gsrb
349  public :: mg_box_prolong
350  public :: mg_has_children
351  public :: mg_ix_to_ichild
352  public :: mg_get_child_offset
353  public :: mg_highest_uniform_lvl
354  public :: mg_number_of_unknowns
355  public :: mg_get_face_coords
356  public :: mg_add_timer
357  public :: mg_timer_start
358  public :: mg_timer_end
359  public :: mg_timers_show
360 
361  !! File ../src/m_allocate_storage.f90
362 
363  public :: mg_allocate_storage
364  public :: mg_deallocate_storage
365 
366  !! File ../src/m_diffusion.f90
367 
368 !> Module for implicitly solving diffusion equations
369 
370  public :: diffusion_solve
371  public :: diffusion_solve_vcoeff
372  public :: diffusion_solve_acoeff
373 
374  !! File ../src/m_laplacian.f90
375 
376 !> Module which contains multigrid procedures for a Laplacian operator
377 
378  public :: laplacian_set_methods
379 
380  !! File ../src/m_vhelmholtz.f90
381 
382 !> Module which contains multigrid procedures for a Helmholtz operator of the
383 !> form: div(D grad(phi)) - lambda*phi = f, where D has a smooth spatial
384 !> variation
385 
386  !> The lambda used for the Helmholtz equation (should be >= 0)
387  real(dp), public, protected :: vhelmholtz_lambda = 0.0_dp
388 
389  public :: vhelmholtz_set_methods
390  public :: vhelmholtz_set_lambda
391 
392  !! File ../src/m_build_tree.f90
393 
394  ! Public methods
395  public :: mg_build_rectangle
396  public :: mg_add_children
397  public :: mg_set_leaves_parents
398  public :: mg_set_next_level_ids
400  public :: mg_set_neighbors_lvl
401 
402  !! File ../src/m_load_balance.f90
403 !> Module for load balancing a tree (that already has been constructed). The
404 !> load balancing determines which ranks (MPI processes) allocated physical
405 !> storage for boxes. The tree structure itself is present on all processes.
406 
407  ! Public methods
408  public :: mg_load_balance_simple
409  public :: mg_load_balance
410  public :: mg_load_balance_parents
411 
412  !! File ../src/m_vlaplacian.f90
413 
414 !> Module which contains multigrid procedures for a variable-coefficient
415 !> Laplacian operator, assuming the variation is smooth
416 
417  public :: vlaplacian_set_methods
418 
419  !! File ../src/m_communication.f90
420 
421  ! Public methods
422  public :: mg_comm_init
423  public :: sort_and_transfer_buffers
424 
425  !! File ../src/m_ghost_cells.f90
426 
427  ! Public methods
428  public :: mg_ghost_cell_buffer_size
429  public :: mg_fill_ghost_cells
430  public :: mg_fill_ghost_cells_lvl
431  public :: mg_phi_bc_store
432 
433  !! File ../src/m_prolong.f90
434 
435  ! Public methods
436  public :: mg_prolong
437  public :: mg_prolong_buffer_size
438  public :: mg_prolong_sparse
439 
440  !! File ../src/m_helmholtz.f90
441 
442 !> Module which contains multigrid procedures for a Helmholtz operator of the
443 !> form: laplacian(phi) - lambda*phi = f
444 
445  !> The lambda used for the Helmholtz equation (should be >= 0)
446  real(dp), public, protected :: helmholtz_lambda = 0.0_dp
447 
448  public :: helmholtz_set_methods
449  public :: helmholtz_set_lambda
450 
451  !! File ../src/m_multigrid.f90
452 
453  integer :: timer_total_vcycle = -1
454  integer :: timer_total_fmg = -1
455  integer :: timer_smoother = -1
456  integer :: timer_smoother_gc = -1
457  integer :: timer_coarse = -1
458  integer :: timer_correct = -1
459  integer :: timer_update_coarse = -1
460 
461  ! Public methods
462  public :: mg_fas_vcycle
463  public :: mg_fas_fmg
464  public :: mg_set_methods
465  public :: mg_apply_op
466 
467  !! File ../src/m_restrict.f90
468 
469  ! Public methods
470  public :: mg_restrict
471  public :: mg_restrict_lvl
472  public :: mg_restrict_buffer_size
473 
474  !! File ../src/m_mrgrnk.f90
475 
476  Integer, Parameter :: kdp = selected_real_kind(15)
477  public :: mrgrnk
478  private :: kdp
479  private :: i_mrgrnk
480  interface mrgrnk
481  module procedure i_mrgrnk
482  end interface mrgrnk
483 
484  !! File ../src/m_ahelmholtz.f90
485 
486 !> Module which contains multigrid procedures for a Helmholtz operator of the
487 !> form: div(D grad(phi)) - lambda*phi = f, where D has a smooth spatial
488 !> variation and a component in each spatial direction
489 
490  !> The lambda used for the Helmholtz equation (should be >= 0)
491  real(dp), public, protected :: ahelmholtz_lambda = 0.0_dp
492 
493  public :: ahelmholtz_set_methods
494  public :: ahelmholtz_set_lambda
495 contains
496 
497  !! File ../src/m_data_structures.f90
498 
499  !> Return .true. if a box has children
500  elemental logical function mg_has_children(box)
501  type(mg_box_t), intent(in) :: box
502 
503  ! Boxes are either fully refined or not, so we only need to check one of the
504  ! children
505  mg_has_children = (box%children(1) /= mg_no_box)
506  end function mg_has_children
507 
508  !> Compute the child index for a box with spatial index ix. With child index
509  !> we mean the index in the children(:) array of its parent.
510  integer function mg_ix_to_ichild(ix)
511  integer, intent(in) :: ix(2) !< Spatial index of the box
512  ! The index can range from 1 (all ix odd) and 2**$D (all ix even)
513  mg_ix_to_ichild = 4 - 2 * iand(ix(2), 1) - iand(ix(1), 1)
514  end function mg_ix_to_ichild
515 
516  !> Get the offset of a box with respect to its parent (e.g. in 2d, there can
517  !> be a child at offset 0,0, one at n_cell/2,0, one at 0,n_cell/2 and one at
518  !> n_cell/2, n_cell/2)
519  pure function mg_get_child_offset(mg, id) result(ix_offset)
520  type(mg_t), intent(in) :: mg
521  integer, intent(in) :: id
522  integer :: ix_offset(2)
523 
524  if (mg%boxes(id)%lvl <= mg%first_normal_lvl) then
525  ix_offset(:) = 0
526  else
527  ix_offset = iand(mg%boxes(id)%ix-1, 1) * &
528  ishft(mg%box_size, -1) ! * n_cell / 2
529  end if
530  end function mg_get_child_offset
531 
532  pure function mg_highest_uniform_lvl(mg) result(lvl)
533  type(mg_t), intent(in) :: mg
534  integer :: lvl
535 
536  do lvl = mg%first_normal_lvl, mg%highest_lvl-1
537  ! Exit if a grid is partially refined
538  if (size(mg%lvls(lvl)%leaves) /= 0 .and. &
539  size(mg%lvls(lvl)%parents) /= 0) exit
540  end do
541  ! If the loop did not exit, we get lvl equals mg%highest_lvl
542  end function mg_highest_uniform_lvl
543 
544  !> Determine total number of unknowns (on leaves)
545  function mg_number_of_unknowns(mg) result(n_unknowns)
546  type(mg_t), intent(in) :: mg
547  integer :: lvl
548  integer(i8) :: n_unknowns
549 
550  n_unknowns = 0
551  do lvl = mg%first_normal_lvl, mg%highest_lvl
552  n_unknowns = n_unknowns + size(mg%lvls(lvl)%leaves)
553  end do
554  n_unknowns = n_unknowns * int(mg%box_size**3, i8)
555  end function mg_number_of_unknowns
556 
557  !> Get coordinates at the face of a box
558  subroutine mg_get_face_coords(box, nb, nc, x)
559  type(mg_box_t), intent(in) :: box
560  integer, intent(in) :: nb
561  integer, intent(in) :: nc
562  real(dp), intent(out) :: x(nc, 2)
563  integer :: i, ixs(2-1)
564  integer :: nb_dim
565  real(dp) :: rmin(2)
566 
567  nb_dim = mg_neighb_dim(nb)
568 
569  ! Determine directions perpendicular to neighbor
570  ixs = [(i, i = 1, 2-1)]
571  ixs(nb_dim:) = ixs(nb_dim:) + 1
572 
573  rmin = box%r_min
574  if (.not. mg_neighb_low(nb)) then
575  rmin(nb_dim) = rmin(nb_dim) + box%dr(nb_dim) * nc
576  end if
577 
578  do i = 1, nc
579  x(i, :) = rmin
580  x(i, ixs(1)) = x(i, ixs(1)) + (i-0.5d0) * box%dr(ixs(1))
581  end do
582  end subroutine mg_get_face_coords
583 
584  integer function mg_add_timer(mg, name)
585  type(mg_t), intent(inout) :: mg
586  character(len=*), intent(in) :: name
587 
588  mg%n_timers = mg%n_timers + 1
589  mg_add_timer = mg%n_timers
590  mg%timers(mg_add_timer)%name = name
591  end function mg_add_timer
592 
593  subroutine mg_timer_start(timer)
594  use mpi
595  type(mg_timer_t), intent(inout) :: timer
596  timer%t0 = mpi_wtime()
597  end subroutine mg_timer_start
598 
599  subroutine mg_timer_end(timer)
600  use mpi
601  type(mg_timer_t), intent(inout) :: timer
602  timer%t = timer%t + mpi_wtime() - timer%t0
603  end subroutine mg_timer_end
604 
605  subroutine mg_timers_show(mg)
606  use mpi
607  type(mg_t), intent(in) :: mg
608  integer :: n, ierr
609  real(dp) :: tmin(mg%n_timers)
610  real(dp) :: tmax(mg%n_timers)
611 
612  call mpi_reduce(mg%timers(1:mg%n_timers)%t, tmin, mg%n_timers, &
613  mpi_double, mpi_min, 0, mg%comm, ierr)
614  call mpi_reduce(mg%timers(1:mg%n_timers)%t, tmax, mg%n_timers, &
615  mpi_double, mpi_max, 0, mg%comm, ierr)
616 
617  if (mg%my_rank == 0) then
618  write(*, "(A20,2A16)") "name ", "min(s)", "max(s)"
619  do n = 1, mg%n_timers
620  write(*, "(A20,2F16.6)") mg%timers(n)%name, &
621  tmin(n), tmax(n)
622  end do
623  end if
624  end subroutine mg_timers_show
625 
626  !! File ../src/m_allocate_storage.f90
627 
628  !> Deallocate all allocatable arrays
629  subroutine mg_deallocate_storage(mg)
630  type(mg_t), intent(inout) :: mg
631  integer :: lvl
632 
633  if (.not. mg%is_allocated) &
634  error stop "deallocate_storage: tree is not allocated"
635 
636  deallocate(mg%boxes)
637  deallocate(mg%buf)
638 
639  deallocate(mg%comm_restrict%n_send)
640  deallocate(mg%comm_restrict%n_recv)
641 
642  deallocate(mg%comm_prolong%n_send)
643  deallocate(mg%comm_prolong%n_recv)
644 
645  deallocate(mg%comm_ghostcell%n_send)
646  deallocate(mg%comm_ghostcell%n_recv)
647 
648  do lvl = mg%lowest_lvl, mg%highest_lvl
649  deallocate(mg%lvls(lvl)%ids)
650  deallocate(mg%lvls(lvl)%leaves)
651  deallocate(mg%lvls(lvl)%parents)
652  deallocate(mg%lvls(lvl)%ref_bnds)
653  deallocate(mg%lvls(lvl)%my_ids)
654  deallocate(mg%lvls(lvl)%my_leaves)
655  deallocate(mg%lvls(lvl)%my_parents)
656  deallocate(mg%lvls(lvl)%my_ref_bnds)
657  end do
658 
659  mg%is_allocated = .false.
660  mg%n_boxes = 0
661  mg%phi_bc_data_stored = .false.
662  end subroutine mg_deallocate_storage
663 
664  !> Allocate communication buffers and local boxes for a tree that has already
665  !> been created
666  subroutine mg_allocate_storage(mg)
667 
668  type(mg_t), intent(inout) :: mg
669  integer :: i, id, lvl, nc
670  integer :: n_send(0:mg%n_cpu-1, 3)
671  integer :: n_recv(0:mg%n_cpu-1, 3)
672  integer :: dsize(3)
673  integer :: n_in, n_out, n_id
674 
675  if (.not. mg%tree_created) &
676  error stop "allocate_storage: tree is not yet created"
677 
678  if (mg%is_allocated) &
679  error stop "allocate_storage: tree is already allocated"
680 
681  do lvl = mg%lowest_lvl, mg%highest_lvl
682  nc = mg%box_size_lvl(lvl)
683  do i = 1, size(mg%lvls(lvl)%my_ids)
684  id = mg%lvls(lvl)%my_ids(i)
685  allocate(mg%boxes(id)%cc(0:nc+1, 0:nc+1, &
686  mg_num_vars + mg%n_extra_vars))
687 
688  ! Set all initial values to zero
689  mg%boxes(id)%cc(:, :, :) = 0.0_dp
690  end do
691  end do
692 
693  allocate(mg%buf(0:mg%n_cpu-1))
694 
695  call mg_ghost_cell_buffer_size(mg, n_send(:, 1), &
696  n_recv(:, 1), dsize(1))
697  call mg_restrict_buffer_size(mg, n_send(:, 2), &
698  n_recv(:, 2), dsize(2))
699  call mg_prolong_buffer_size(mg, n_send(:, 3), &
700  n_recv(:, 3), dsize(3))
701 
702  do i = 0, mg%n_cpu-1
703  n_out = maxval(n_send(i, :) * dsize(:))
704  n_in = maxval(n_recv(i, :) * dsize(:))
705  n_id = maxval(n_send(i, :))
706  allocate(mg%buf(i)%send(n_out))
707  allocate(mg%buf(i)%recv(n_in))
708  allocate(mg%buf(i)%ix(n_id))
709  end do
710 
711  mg%is_allocated = .true.
712  end subroutine mg_allocate_storage
713 
714  !! File ../src/m_diffusion.f90
715 
716  !> Solve a diffusion equation implicitly, assuming a constant diffusion
717  !> coefficient. The solution at time t should be stored in mg_iphi, which is
718  !> on output replaced by the solution at time t+dt.
719  subroutine diffusion_solve(mg, dt, diffusion_coeff, order, max_res)
720 
721  type(mg_t), intent(inout) :: mg
722  real(dp), intent(in) :: dt
723  real(dp), intent(in) :: diffusion_coeff
724  integer, intent(in) :: order
725  real(dp), intent(in) :: max_res
726  integer, parameter :: max_its = 10
727  integer :: n
728  real(dp) :: res
729 
730  mg%operator_type = mg_helmholtz
731  call mg_set_methods(mg)
732 
733  select case (order)
734  case (1)
735  call helmholtz_set_lambda(1/(dt * diffusion_coeff))
736  call set_rhs(mg, -1/(dt * diffusion_coeff), 0.0_dp)
737  case (2)
738  call helmholtz_set_lambda(0.0d0)
739  call mg_apply_op(mg, mg_irhs)
740  call helmholtz_set_lambda(2/(dt * diffusion_coeff))
741  call set_rhs(mg, -2/(dt * diffusion_coeff), -1.0_dp)
742  case default
743  error stop "diffusion_solve: order should be 1 or 2"
744  end select
745 
746  ! Start with an FMG cycle
747  call mg_fas_fmg(mg, .true., max_res=res)
748 
749  ! Add V-cycles if necessary
750  do n = 1, max_its
751  if (res <= max_res) exit
752  call mg_fas_vcycle(mg, max_res=res)
753  end do
754 
755  if (n == max_its + 1) then
756  if (mg%my_rank == 0) &
757  print *, "Did you specify boundary conditions correctly?"
758  error stop "diffusion_solve: no convergence"
759  end if
760  end subroutine diffusion_solve
761 
762  !> Solve a diffusion equation implicitly, assuming a variable diffusion
763  !> coefficient which has been stored in mg_iveps (also on coarse grids). The
764  !> solution at time t should be stored in mg_iphi, which is on output replaced
765  !> by the solution at time t+dt.
766  subroutine diffusion_solve_vcoeff(mg, dt, order, max_res)
767 
768  type(mg_t), intent(inout) :: mg
769  real(dp), intent(in) :: dt
770  integer, intent(in) :: order
771  real(dp), intent(in) :: max_res
772  integer, parameter :: max_its = 10
773  integer :: n
774  real(dp) :: res
775 
776  mg%operator_type = mg_vhelmholtz
777  call mg_set_methods(mg)
778 
779  select case (order)
780  case (1)
781  call vhelmholtz_set_lambda(1/dt)
782  call set_rhs(mg, -1/dt, 0.0_dp)
783  case (2)
784  call vhelmholtz_set_lambda(0.0d0)
785  call mg_apply_op(mg, mg_irhs)
786  call vhelmholtz_set_lambda(2/dt)
787  call set_rhs(mg, -2/dt, -1.0_dp)
788  case default
789  error stop "diffusion_solve: order should be 1 or 2"
790  end select
791 
792  ! Start with an FMG cycle
793  call mg_fas_fmg(mg, .true., max_res=res)
794 
795  ! Add V-cycles if necessary
796  do n = 1, max_its
797  if (res <= max_res) exit
798  call mg_fas_vcycle(mg, max_res=res)
799  end do
800 
801  if (n == max_its + 1) then
802  if (mg%my_rank == 0) then
803  print *, "Did you specify boundary conditions correctly?"
804  print *, "Or is the variation in diffusion too large?"
805  end if
806  error stop "diffusion_solve: no convergence"
807  end if
808  end subroutine diffusion_solve_vcoeff
809 
810  !> Solve a diffusion equation implicitly, assuming anisotropic diffusion
811  !> coefficient which has been stored in mg_iveps1, mg_iveps2, mg_iveps3
812  !> (also on coarse grids). The
813  !> solution at time t should be stored in mg_iphi, which is on output replaced
814  !> by the solution at time t+dt.
815  subroutine diffusion_solve_acoeff(mg, dt, order, max_res)
816 
817  type(mg_t), intent(inout) :: mg
818  real(dp), intent(in) :: dt
819  integer, intent(in) :: order
820  real(dp), intent(in) :: max_res
821  integer, parameter :: max_its = 100
822  integer :: n
823  real(dp) :: res
824 
825  mg%operator_type = mg_ahelmholtz
826  call mg_set_methods(mg)
827 
828  select case (order)
829  case (1)
830  call ahelmholtz_set_lambda(1/dt)
831  call set_rhs(mg, -1/dt, 0.0_dp)
832  case (2)
833  call ahelmholtz_set_lambda(0.0d0)
834  call mg_apply_op(mg, mg_irhs)
835  call ahelmholtz_set_lambda(2/dt)
836  call set_rhs(mg, -2/dt, -1.0_dp)
837  case default
838  error stop "diffusion_solve: order should be 1 or 2"
839  end select
840 
841  ! Start with an FMG cycle
842  call mg_fas_fmg(mg, .true., max_res=res)
843 
844  ! Add V-cycles if necessary
845  do n = 1, max_its
846  print*, n, res
847  if (res <= max_res) exit
848  call mg_fas_vcycle(mg, max_res=res)
849  end do
850 
851  if (n == max_its + 1) then
852  if (mg%my_rank == 0) then
853  print *, "Did you specify boundary conditions correctly?"
854  print *, "Or is the variation in diffusion too large?"
855  end if
856  error stop "diffusion_solve: no convergence"
857  end if
858  end subroutine diffusion_solve_acoeff
859 
860  subroutine set_rhs(mg, f1, f2)
861  type(mg_t), intent(inout) :: mg
862  real(dp), intent(in) :: f1, f2
863  integer :: lvl, i, id, nc
864 
865  do lvl = 1, mg%highest_lvl
866  nc = mg%box_size_lvl(lvl)
867  do i = 1, size(mg%lvls(lvl)%my_leaves)
868  id = mg%lvls(lvl)%my_leaves(i)
869  mg%boxes(id)%cc(1:nc, 1:nc, mg_irhs) = &
870  f1 * mg%boxes(id)%cc(1:nc, 1:nc, mg_iphi) + &
871  f2 * mg%boxes(id)%cc(1:nc, 1:nc, mg_irhs)
872  end do
873  end do
874  end subroutine set_rhs
875 
876  !! File ../src/m_laplacian.f90
877 
878  subroutine laplacian_set_methods(mg)
879  type(mg_t), intent(inout) :: mg
880 
881  if (all(mg%periodic)) then
882  ! For a fully periodic Laplacian, remove the mean from the rhs and phi so
883  ! that a unique and periodic solution can be found
884  mg%subtract_mean = .true.
885  end if
886 
887  select case (mg%geometry_type)
888  case (mg_cartesian)
889  mg%box_op => box_lpl
890 
891  select case (mg%smoother_type)
892  ! case (smoother_jacobi)
893  ! mg%box_smoother => box_jacobi_lpl
895  mg%box_smoother => box_gs_lpl
896  case default
897  error stop "laplacian_set_methods: unsupported smoother type"
898  end select
899  case (mg_cylindrical)
900  mg%box_op => box_clpl
901 
902  select case (mg%smoother_type)
904  mg%box_smoother => box_gs_clpl
905  case default
906  error stop "laplacian_set_methods: unsupported smoother type"
907  end select
908  case default
909  error stop "laplacian_set_methods: unsupported geometry"
910  end select
911 
912  end subroutine laplacian_set_methods
913 
914  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator
915  subroutine box_gs_lpl(mg, id, nc, redblack_cntr)
916  type(mg_t), intent(inout) :: mg
917  integer, intent(in) :: id
918  integer, intent(in) :: nc
919  integer, intent(in) :: redblack_cntr !< Iteration counter
920  integer :: i, j, i0, di
921  real(dp) :: idr2(2), fac
922  logical :: redblack
923 
924  idr2 = 1/mg%dr(:, mg%boxes(id)%lvl)**2
925  fac = 0.5_dp / sum(idr2)
926  i0 = 1
927 
928  redblack = (mg%smoother_type == mg_smoother_gsrb)
929  if (redblack) then
930  di = 2
931  else
932  di = 1
933  end if
934 
935  ! The parity of redblack_cntr determines which cells we use. If
936  ! redblack_cntr is even, we use the even cells and vice versa.
937  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
938  do j = 1, nc
939  if (redblack) &
940  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
941 
942  do i = i0, nc, di
943  cc(i, j, n) = fac * ( &
944  idr2(1) * (cc(i+1, j, n) + cc(i-1, j, n)) + &
945  idr2(2) * (cc(i, j+1, n) + cc(i, j-1, n)) - &
946  cc(i, j, mg_irhs))
947  end do
948  end do
949  end associate
950  end subroutine box_gs_lpl
951 
952 ! !> Perform Jacobi relaxation on box for a Laplacian operator
953 ! subroutine box_jacobi_lpl(mg, id, nc, cntr)
954 ! type(mg_t), intent(inout) :: mg
955 ! integer, intent(in) :: id
956 ! integer, intent(in) :: nc
957 ! integer, intent(in) :: cntr !< Not used
958 ! integer :: i, j
959 ! real(dp), parameter :: w = 2.0_dp / 3
960 ! real(dp) :: tmp(0:nc+1, 0:nc+1)
961 ! real(dp) :: dr2
962 ! #if 2 == 3
963 ! real(dp), parameter :: sixth = 1/6.0_dp
964 ! #endif
965 
966 ! dr2 = mg%dr(mg%boxes(id)%lvl)**2
967 
968 ! associate (box => mg%boxes(id))
969 ! tmp = box%cc(:, :, mg_iphi)
970 ! do j=1, nc; do i=1, nc
971 ! #if 2 == 2
972 ! box%cc(i, j, mg_iphi) = (1-w) * box%cc(i, j, mg_iphi) + &
973 ! 0.25_dp * w * ( &
974 ! tmp(i+1, j) + tmp(i-1, j) + &
975 ! tmp(i, j+1) + tmp(i, j-1) - &
976 ! dr2 * box%cc(i, j, mg_irhs))
977 ! #elif 2 == 3
978 ! box%cc(i, j, k, mg_iphi) = (1-w) * &
979 ! box%cc(i, j, k, mg_iphi) + &
980 ! sixth * w * ( &
981 ! tmp(i+1, j, k) + tmp(i-1, j, k) + &
982 ! tmp(i, j+1, k) + tmp(i, j-1, k) + &
983 ! tmp(i, j, k+1) + tmp(i, j, k-1) - &
984 ! dr2 * box%cc(i, j, k, mg_irhs))
985 ! #endif
986 ! end do; end do
987 ! end associate
988 ! end subroutine box_jacobi_lpl
989 
990  !> Perform Laplacian operator on a box
991  subroutine box_lpl(mg, id, nc, i_out)
992  type(mg_t), intent(inout) :: mg
993  integer, intent(in) :: id
994  integer, intent(in) :: nc
995  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
996  integer :: i, j
997  real(dp) :: idr2(2)
998 
999  idr2 = 1 / mg%dr(:, mg%boxes(id)%lvl)**2
1000 
1001  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1002  do j = 1, nc
1003  do i = 1, nc
1004  cc(i, j, i_out) = &
1005  idr2(1) * (cc(i-1, j, n) + cc(i+1, j, n) - 2 * cc(i, j, n)) + &
1006  idr2(2) * (cc(i, j-1, n) + cc(i, j+1, n) - 2 * cc(i, j, n))
1007  end do
1008  end do
1009  end associate
1010  end subroutine box_lpl
1011 
1012  !> Perform Laplacian operator on a box in cylindrical geometry, using (r,z)
1013  !> and (r,phi,z) coordinates in 2D/3D.
1014  subroutine box_clpl(mg, id, nc, i_out)
1015  type(mg_t), intent(inout) :: mg
1016  integer, intent(in) :: id
1017  integer, intent(in) :: nc
1018  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
1019  integer :: i, j
1020  real(dp) :: dr(2), idr2(2)
1021  real(dp) :: r_face(nc+1), r_inv(nc)
1022 
1023  dr = mg%dr(:, mg%boxes(id)%lvl)
1024  idr2 = 1 / dr**2
1025  r_face = mg%boxes(id)%r_min(1) + dr(1) * [(i, i=0,nc)]
1026  r_inv = 1/(mg%boxes(id)%r_min(1) + dr(1) * [(i-0.5_dp, i=1,nc)])
1027 
1028  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1029  do j = 1, nc
1030  do i = 1, nc
1031  cc(i, j, i_out) = idr2(1) * (&
1032  r_face(i) * r_inv(i) * cc(i-1, j, n) + &
1033  r_face(i+1) * r_inv(i) * cc(i+1, j, n) &
1034  - 2 * cc(i, j, n)) &
1035  + idr2(2) * (cc(i, j-1, n) + cc(i, j+1, n) - 2 * cc(i, j, n))
1036  end do
1037  end do
1038  end associate
1039  end subroutine box_clpl
1040 
1041  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator in
1042  !> cylindrical geometry. TODO: in 3D this does not converge well, maybe it
1043  !> will for a stretched grid.
1044  subroutine box_gs_clpl(mg, id, nc, redblack_cntr)
1045  type(mg_t), intent(inout) :: mg
1046  integer, intent(in) :: id
1047  integer, intent(in) :: nc
1048  integer, intent(in) :: redblack_cntr !< Iteration counter
1049  integer :: i, j, i0, di
1050  logical :: redblack
1051  real(dp) :: idr2(2), dr(2), dr2(2), fac
1052  real(dp) :: r_face(nc+1), r_inv(nc)
1053 
1054  dr = mg%dr(:, mg%boxes(id)%lvl)
1055  dr2 = dr**2
1056  idr2 = 1/dr**2
1057  fac = 0.5_dp / sum(idr2)
1058  r_face = mg%boxes(id)%r_min(1) + dr(1) * [(i, i=0,nc)]
1059  r_inv = 1/(mg%boxes(id)%r_min(1) + dr(1) * [(i-0.5_dp, i=1,nc)])
1060 
1061  i0 = 1
1062  redblack = (mg%smoother_type == mg_smoother_gsrb)
1063  if (redblack) then
1064  di = 2
1065  else
1066  di = 1
1067  end if
1068 
1069  ! The parity of redblack_cntr determines which cells we use. If
1070  ! redblack_cntr is even, we use the even cells and vice versa.
1071  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
1072  do j = 1, nc
1073  if (redblack) &
1074  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
1075 
1076  do i = i0, nc, di
1077  cc(i, j, n) = fac * (idr2(1) * &
1078  (r_face(i+1) * r_inv(i) * cc(i+1, j, n) + &
1079  r_face(i) * r_inv(i) * cc(i-1, j, n)) + &
1080  idr2(2) * (cc(i, j+1, n) + cc(i, j-1, n)) &
1081  - cc(i, j, mg_irhs))
1082  end do
1083  end do
1084  end associate
1085  end subroutine box_gs_clpl
1086 
1087  !! File ../src/m_vhelmholtz.f90
1088 
1089  subroutine vhelmholtz_set_methods(mg)
1090  type(mg_t), intent(inout) :: mg
1091 
1092  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
1093  error stop "vhelmholtz_set_methods: mg%n_extra_vars == 0"
1094  else
1095  mg%n_extra_vars = max(1, mg%n_extra_vars)
1096  end if
1097 
1098  mg%subtract_mean = .false.
1099 
1100  ! Use Neumann zero boundary conditions for the variable coefficient, since
1101  ! it is needed in ghost cells.
1102  mg%bc(:, mg_iveps)%bc_type = mg_bc_neumann
1103  mg%bc(:, mg_iveps)%bc_value = 0.0_dp
1104 
1105  select case (mg%geometry_type)
1106  case (mg_cartesian)
1107  mg%box_op => box_vhelmh
1108 
1109  select case (mg%smoother_type)
1111  mg%box_smoother => box_gs_vhelmh
1112  case default
1113  error stop "vhelmholtz_set_methods: unsupported smoother type"
1114  end select
1115  case default
1116  error stop "vhelmholtz_set_methods: unsupported geometry"
1117  end select
1118 
1119  end subroutine vhelmholtz_set_methods
1120 
1121  subroutine vhelmholtz_set_lambda(lambda)
1122  real(dp), intent(in) :: lambda
1123 
1124  if (lambda < 0) &
1125  error stop "vhelmholtz_set_lambda: lambda < 0 not allowed"
1126 
1127  vhelmholtz_lambda = lambda
1128  end subroutine vhelmholtz_set_lambda
1129 
1130  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
1131  subroutine box_gs_vhelmh(mg, id, nc, redblack_cntr)
1132  type(mg_t), intent(inout) :: mg
1133  integer, intent(in) :: id
1134  integer, intent(in) :: nc
1135  integer, intent(in) :: redblack_cntr !< Iteration counter
1136  integer :: i, j, i0, di
1137  logical :: redblack
1138  real(dp) :: idr2(2*2), u(2*2)
1139  real(dp) :: a0, a(2*2), c(2*2)
1140 
1141  ! Duplicate 1/dr^2 array to multiply neighbor values
1142  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1143  idr2(2:2*2:2) = idr2(1:2*2:2)
1144  i0 = 1
1145 
1146  redblack = (mg%smoother_type == mg_smoother_gsrb)
1147  if (redblack) then
1148  di = 2
1149  else
1150  di = 1
1151  end if
1152 
1153  ! The parity of redblack_cntr determines which cells we use. If
1154  ! redblack_cntr is even, we use the even cells and vice versa.
1155  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1156  i_eps => mg_iveps)
1157  do j = 1, nc
1158  if (redblack) &
1159  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
1160 
1161  do i = i0, nc, di
1162  a0 = cc(i, j, i_eps)
1163  u(1:2) = cc(i-1:i+1:2, j, n)
1164  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1165  u(3:4) = cc(i, j-1:j+1:2, n)
1166  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1167  c(:) = 2 * a0 * a(:) / (a0 + a(:)) * idr2
1168 
1169  cc(i, j, n) = &
1170  (sum(c(:) * u(:)) - cc(i, j, mg_irhs)) / &
1171  (sum(c(:)) + vhelmholtz_lambda)
1172  end do
1173  end do
1174  end associate
1175  end subroutine box_gs_vhelmh
1176 
1177  !> Perform Helmholtz operator on a box
1178  subroutine box_vhelmh(mg, id, nc, i_out)
1179  type(mg_t), intent(inout) :: mg
1180  integer, intent(in) :: id
1181  integer, intent(in) :: nc
1182  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
1183  integer :: i, j
1184  real(dp) :: idr2(2*2), a0, u0, u(2*2), a(2*2)
1185 
1186  ! Duplicate 1/dr^2 array to multiply neighbor values
1187  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1188  idr2(2:2*2:2) = idr2(1:2*2:2)
1189 
1190  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1191  i_eps => mg_iveps)
1192  do j = 1, nc
1193  do i = 1, nc
1194  a0 = cc(i, j, i_eps)
1195  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1196  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1197  u0 = cc(i, j, n)
1198  u(1:2) = cc(i-1:i+1:2, j, n)
1199  u(3:4) = cc(i, j-1:j+1:2, n)
1200 
1201  cc(i, j, i_out) = sum(2 * idr2 * &
1202  a0*a(:)/(a0 + a(:)) * (u(:) - u0)) - &
1203  vhelmholtz_lambda * u0
1204  end do
1205  end do
1206  end associate
1207  end subroutine box_vhelmh
1208 
1209  !! File ../src/m_build_tree.f90
1210 
1211  subroutine mg_build_rectangle(mg, domain_size, box_size, dx, r_min, &
1212  periodic, n_finer)
1213  type(mg_t), intent(inout) :: mg
1214  integer, intent(in) :: domain_size(2)
1215  integer, intent(in) :: box_size
1216  real(dp), intent(in) :: dx(2)
1217  real(dp), intent(in) :: r_min(2)
1218  logical, intent(in) :: periodic(2)
1219  integer, intent(in) :: n_finer
1220  integer :: i, j, lvl, n, id, nx(2), ib, nb(2)
1221  integer :: boxes_per_dim(2, mg_lvl_lo:1)
1222  integer :: periodic_offset(2)
1223 
1224  if (modulo(box_size, 2) /= 0) &
1225  error stop "box_size should be even"
1226  if (any(modulo(domain_size, box_size) /= 0)) &
1227  error stop "box_size does not divide domain_size"
1228 
1229  if (all(periodic)) then
1230  mg%subtract_mean = .true.
1231  end if
1232 
1233  nx = domain_size
1234  mg%box_size = box_size
1235  mg%box_size_lvl(1) = box_size
1236  mg%domain_size_lvl(:, 1) = domain_size
1237  mg%first_normal_lvl = 1
1238  mg%dr(:, 1) = dx
1239  mg%r_min(:) = r_min
1240  mg%periodic = periodic
1241  boxes_per_dim(:, :) = 0
1242  boxes_per_dim(:, 1) = domain_size / box_size
1243 
1244  do lvl = 1, mg_lvl_lo+1, -1
1245  ! For a Gauss-Seidel (non red-black) smoother, we should avoid boxes
1246  ! containing a single cell
1247  if (any(modulo(nx, 2) == 1 .or. nx == mg%coarsest_grid .or. &
1248  (mg%box_size_lvl(lvl) == mg%coarsest_grid .and. &
1249  mg%smoother_type == mg_smoother_gs))) exit
1250 
1251  if (all(modulo(nx/mg%box_size_lvl(lvl), 2) == 0)) then
1252  mg%box_size_lvl(lvl-1) = mg%box_size_lvl(lvl)
1253  boxes_per_dim(:, lvl-1) = boxes_per_dim(:, lvl)/2
1254  mg%first_normal_lvl = lvl-1
1255  else
1256  mg%box_size_lvl(lvl-1) = mg%box_size_lvl(lvl)/2
1257  boxes_per_dim(:, lvl-1) = boxes_per_dim(:, lvl)
1258  end if
1259 
1260  mg%dr(:, lvl-1) = mg%dr(:, lvl) * 2
1261  nx = nx / 2
1262  mg%domain_size_lvl(:, lvl-1) = nx
1263  end do
1264 
1265  mg%lowest_lvl = lvl
1266  mg%highest_lvl = 1
1267 
1268  do lvl = 2, mg_lvl_hi
1269  mg%dr(:, lvl) = mg%dr(:, lvl-1) * 0.5_dp
1270  mg%box_size_lvl(lvl) = box_size
1271  mg%domain_size_lvl(:, lvl) = 2 * mg%domain_size_lvl(:, lvl-1)
1272  end do
1273 
1274  n = sum(product(boxes_per_dim, dim=1)) + n_finer
1275  allocate(mg%boxes(n))
1276 
1277  ! Create lowest level
1278  nx = boxes_per_dim(:, mg%lowest_lvl)
1279  periodic_offset = [nx(1)-1, (nx(2)-1)*nx(1)]
1280 
1281  do j=1,nx(2); do i=1,nx(1)
1282  mg%n_boxes = mg%n_boxes + 1
1283  n = mg%n_boxes
1284 
1285  mg%boxes(n)%rank = 0
1286  mg%boxes(n)%id = n
1287  mg%boxes(n)%lvl = mg%lowest_lvl
1288  mg%boxes(n)%ix(:) = [i, j]
1289  mg%boxes(n)%r_min(:) = r_min + (mg%boxes(n)%ix(:) - 1) * &
1290  mg%box_size_lvl(mg%lowest_lvl) * mg%dr(:, mg%lowest_lvl)
1291  mg%boxes(n)%dr(:) = mg%dr(:, mg%lowest_lvl)
1292  mg%boxes(n)%parent = mg_no_box
1293  mg%boxes(n)%children(:) = mg_no_box
1294 
1295  ! Set default neighbors
1296  mg%boxes(n)%neighbors(:) = [n-1, n+1, n-nx(1), n+nx(1)]
1297 
1298  ! Handle boundaries
1299  nb=[i, j]
1300  do ib=1,2
1301  if(nb(ib)==1 .and. .not.periodic(ib)) then
1302  mg%boxes(n)%neighbors(ib*2-1) = mg_physical_boundary
1303  end if
1304  if(nb(ib)==1 .and. periodic(ib)) then
1305  mg%boxes(n)%neighbors(ib*2-1) = n + periodic_offset(ib)
1306  end if
1307  if(nb(ib)==nx(ib) .and. .not.periodic(ib)) then
1308  mg%boxes(n)%neighbors(ib*2) = mg_physical_boundary
1309  end if
1310  if(nb(ib)==nx(ib) .and. periodic(ib)) then
1311  mg%boxes(n)%neighbors(ib*2) = n - periodic_offset(ib)
1312  end if
1313  end do
1314  !where ([i, j] == 1 .and. .not. periodic)
1315  ! mg%boxes(n)%neighbors(1:mg_num_neighbors:2) = &
1316  ! mg_physical_boundary
1317  !end where
1318  !where ([i, j] == 1 .and. periodic)
1319  ! mg%boxes(n)%neighbors(1:mg_num_neighbors:2) = &
1320  ! n + periodic_offset
1321  !end where
1322 
1323  !where ([i, j] == nx .and. .not. periodic)
1324  ! mg%boxes(n)%neighbors(2:mg_num_neighbors:2) = &
1325  ! mg_physical_boundary
1326  !end where
1327  !where ([i, j] == nx .and. periodic)
1328  ! mg%boxes(n)%neighbors(2:mg_num_neighbors:2) = &
1329  ! n - periodic_offset
1330  !end where
1331  end do; end do
1332 
1333  mg%lvls(mg%lowest_lvl)%ids = [(n, n=1, mg%n_boxes)]
1334 
1335  ! Add higher levels
1336  do lvl = mg%lowest_lvl, 0
1337  if (mg%box_size_lvl(lvl+1) == mg%box_size_lvl(lvl)) then
1338  do i = 1, size(mg%lvls(lvl)%ids)
1339  id = mg%lvls(lvl)%ids(i)
1340  call mg_add_children(mg, id)
1341  end do
1342 
1343  call mg_set_leaves_parents(mg%boxes, mg%lvls(lvl))
1344  call mg_set_next_level_ids(mg, lvl)
1345  call mg_set_neighbors_lvl(mg, lvl+1)
1346  else
1347  do i = 1, size(mg%lvls(lvl)%ids)
1348  id = mg%lvls(lvl)%ids(i)
1349  call add_single_child(mg, id, size(mg%lvls(lvl)%ids))
1350  end do
1351 
1352  call mg_set_leaves_parents(mg%boxes, mg%lvls(lvl))
1353  call mg_set_next_level_ids(mg, lvl)
1354  end if
1355  end do
1356 
1357  call mg_set_leaves_parents(mg%boxes, mg%lvls(1))
1358 
1359  ! No refinement boundaries
1360  do lvl = mg%lowest_lvl, 1
1361  if (allocated(mg%lvls(lvl)%ref_bnds)) &
1362  deallocate(mg%lvls(lvl)%ref_bnds)
1363  allocate(mg%lvls(lvl)%ref_bnds(0))
1364  end do
1365 
1366  mg%tree_created = .true.
1367  end subroutine mg_build_rectangle
1368 
1369  subroutine mg_set_neighbors_lvl(mg, lvl)
1370  type(mg_t), intent(inout) :: mg
1371  integer, intent(in) :: lvl
1372  integer :: i, id
1373 
1374  do i = 1, size(mg%lvls(lvl)%ids)
1375  id = mg%lvls(lvl)%ids(i)
1376  call set_neighbs(mg%boxes, id)
1377  end do
1378  end subroutine mg_set_neighbors_lvl
1379 
1380  subroutine mg_set_next_level_ids(mg, lvl)
1381  type(mg_t), intent(inout) :: mg
1382  integer, intent(in) :: lvl
1383  integer :: n, i, id
1384 
1385  if (allocated(mg%lvls(lvl+1)%ids)) &
1386  deallocate(mg%lvls(lvl+1)%ids)
1387 
1388  ! Set next level ids to children of this level
1389  if (mg%box_size_lvl(lvl+1) == mg%box_size_lvl(lvl)) then
1390  n = mg_num_children * size(mg%lvls(lvl)%parents)
1391  allocate(mg%lvls(lvl+1)%ids(n))
1392 
1393  n = mg_num_children
1394  do i = 1, size(mg%lvls(lvl)%parents)
1395  id = mg%lvls(lvl)%parents(i)
1396  mg%lvls(lvl+1)%ids(n*(i-1)+1:n*i) = mg%boxes(id)%children
1397  end do
1398  else
1399  n = size(mg%lvls(lvl)%parents)
1400  allocate(mg%lvls(lvl+1)%ids(n))
1401 
1402  n = 1
1403  do i = 1, size(mg%lvls(lvl)%parents)
1404  id = mg%lvls(lvl)%parents(i)
1405  mg%lvls(lvl+1)%ids(i) = mg%boxes(id)%children(1)
1406  end do
1407  end if
1408 
1409  end subroutine mg_set_next_level_ids
1410 
1411  ! Set the neighbors of id (using their parent)
1412  subroutine set_neighbs(boxes, id)
1413  type(mg_box_t), intent(inout) :: boxes(:)
1414  integer, intent(in) :: id
1415  integer :: nb, nb_id
1416 
1417  do nb = 1, mg_num_neighbors
1418  if (boxes(id)%neighbors(nb) == mg_no_box) then
1419  nb_id = find_neighb(boxes, id, nb)
1420  if (nb_id > mg_no_box) then
1421  boxes(id)%neighbors(nb) = nb_id
1422  boxes(nb_id)%neighbors(mg_neighb_rev(nb)) = id
1423  end if
1424  end if
1425  end do
1426  end subroutine set_neighbs
1427 
1428  !> Get the id of neighbor nb of boxes(id), through its parent
1429  function find_neighb(boxes, id, nb) result(nb_id)
1430  type(mg_box_t), intent(in) :: boxes(:) !< List with all the boxes
1431  integer, intent(in) :: id !< Box whose neighbor we are looking for
1432  integer, intent(in) :: nb !< Neighbor index
1433  integer :: nb_id, p_id, c_ix, d, old_pid
1434 
1435  p_id = boxes(id)%parent
1436  old_pid = p_id
1437  c_ix = mg_ix_to_ichild(boxes(id)%ix)
1438  d = mg_neighb_dim(nb)
1439 
1440  ! Check if neighbor is in same direction as ix is (low/high). If so,
1441  ! use neighbor of parent
1442  if (mg_child_low(d, c_ix) .eqv. mg_neighb_low(nb)) then
1443  p_id = boxes(p_id)%neighbors(nb)
1444  end if
1445 
1446  ! The child ix of the neighbor is reversed in direction d
1447  nb_id = boxes(p_id)%children(mg_child_rev(c_ix, d))
1448  end function find_neighb
1449 
1450  !> Create a list of leaves and a list of parents for a level
1451  subroutine mg_set_leaves_parents(boxes, level)
1452  type(mg_box_t), intent(in) :: boxes(:) !< List of boxes
1453  type(mg_lvl_t), intent(inout) :: level !< Level type which contains the indices of boxes
1454  integer :: i, id, i_leaf, i_parent
1455  integer :: n_parents, n_leaves
1456 
1457  n_parents = count(mg_has_children(boxes(level%ids)))
1458  n_leaves = size(level%ids) - n_parents
1459 
1460  if (.not. allocated(level%parents)) then
1461  allocate(level%parents(n_parents))
1462  else if (n_parents /= size(level%parents)) then
1463  deallocate(level%parents)
1464  allocate(level%parents(n_parents))
1465  end if
1466 
1467  if (.not. allocated(level%leaves)) then
1468  allocate(level%leaves(n_leaves))
1469  else if (n_leaves /= size(level%leaves)) then
1470  deallocate(level%leaves)
1471  allocate(level%leaves(n_leaves))
1472  end if
1473 
1474  i_leaf = 0
1475  i_parent = 0
1476  do i = 1, size(level%ids)
1477  id = level%ids(i)
1478  if (mg_has_children(boxes(id))) then
1479  i_parent = i_parent + 1
1480  level%parents(i_parent) = id
1481  else
1482  i_leaf = i_leaf + 1
1483  level%leaves(i_leaf) = id
1484  end if
1485  end do
1486  end subroutine mg_set_leaves_parents
1487 
1488  !> Create a list of refinement boundaries (from the coarse side)
1489  subroutine mg_set_refinement_boundaries(boxes, level)
1490  type(mg_box_t), intent(in) :: boxes(:)
1491  type(mg_lvl_t), intent(inout) :: level
1492  integer, allocatable :: tmp(:)
1493  integer :: i, id, nb, nb_id, ix
1494 
1495  if (allocated(level%ref_bnds)) deallocate(level%ref_bnds)
1496 
1497  if (size(level%parents) == 0) then
1498  ! There are no refinement boundaries
1499  allocate(level%ref_bnds(0))
1500  else
1501  allocate(tmp(size(level%leaves)))
1502  ix = 0
1503  do i = 1, size(level%leaves)
1504  id = level%leaves(i)
1505 
1506  do nb = 1, mg_num_neighbors
1507  nb_id = boxes(id)%neighbors(nb)
1508  if (nb_id > mg_no_box) then
1509  if (mg_has_children(boxes(nb_id))) then
1510  ix = ix + 1
1511  tmp(ix) = id
1512  exit
1513  end if
1514  end if
1515  end do
1516  end do
1517 
1518  allocate(level%ref_bnds(ix))
1519  level%ref_bnds(:) = tmp(1:ix)
1520  end if
1521  end subroutine mg_set_refinement_boundaries
1522 
1523  subroutine mg_add_children(mg, id)
1524  type(mg_t), intent(inout) :: mg
1525  integer, intent(in) :: id !< Id of box that gets children
1526  integer :: lvl, i, nb, child_nb(2**(2-1))
1527  integer :: c_ids(mg_num_children)
1528  integer :: c_id, c_ix_base(2)
1529 
1530  if (mg%n_boxes + mg_num_children > size(mg%boxes)) then
1531  error stop "mg_add_children: not enough space"
1532  end if
1533 
1534  c_ids = [(mg%n_boxes+i, i=1,mg_num_children)]
1535  mg%n_boxes = mg%n_boxes + mg_num_children
1536  mg%boxes(id)%children = c_ids
1537  c_ix_base = 2 * mg%boxes(id)%ix - 1
1538  lvl = mg%boxes(id)%lvl+1
1539 
1540  do i = 1, mg_num_children
1541  c_id = c_ids(i)
1542  mg%boxes(c_id)%rank = mg%boxes(id)%rank
1543  mg%boxes(c_id)%ix = c_ix_base + mg_child_dix(:, i)
1544  mg%boxes(c_id)%lvl = lvl
1545  mg%boxes(c_id)%parent = id
1546  mg%boxes(c_id)%children = mg_no_box
1547  mg%boxes(c_id)%neighbors = mg_no_box
1548  mg%boxes(c_id)%r_min = mg%boxes(id)%r_min + &
1549  mg%dr(:, lvl) * mg_child_dix(:, i) * mg%box_size
1550  mg%boxes(c_id)%dr(:) = mg%dr(:, lvl)
1551  end do
1552 
1553  ! Set boundary conditions at children
1554  do nb = 1, mg_num_neighbors
1555  if (mg%boxes(id)%neighbors(nb) < mg_no_box) then
1556  child_nb = c_ids(mg_child_adj_nb(:, nb)) ! Neighboring children
1557  mg%boxes(child_nb)%neighbors(nb) = mg%boxes(id)%neighbors(nb)
1558  end if
1559  end do
1560  end subroutine mg_add_children
1561 
1562  subroutine add_single_child(mg, id, n_boxes_lvl)
1563  type(mg_t), intent(inout) :: mg
1564  integer, intent(in) :: id !< Id of box that gets children
1565  integer, intent(in) :: n_boxes_lvl
1566  integer :: lvl, c_id
1567 
1568  c_id = mg%n_boxes + 1
1569  mg%n_boxes = mg%n_boxes + 1
1570  mg%boxes(id)%children(1) = c_id
1571  lvl = mg%boxes(id)%lvl+1
1572 
1573  mg%boxes(c_id)%rank = mg%boxes(id)%rank
1574  mg%boxes(c_id)%ix = mg%boxes(id)%ix
1575  mg%boxes(c_id)%lvl = lvl
1576  mg%boxes(c_id)%parent = id
1577  mg%boxes(c_id)%children = mg_no_box
1578  where (mg%boxes(id)%neighbors == mg_physical_boundary)
1579  mg%boxes(c_id)%neighbors = mg%boxes(id)%neighbors
1580  elsewhere
1581  mg%boxes(c_id)%neighbors = mg%boxes(id)%neighbors + n_boxes_lvl
1582  end where
1583  mg%boxes(c_id)%r_min = mg%boxes(id)%r_min
1584  mg%boxes(c_id)%dr(:) = mg%dr(:, lvl)
1585 
1586  end subroutine add_single_child
1587 
1588  !! File ../src/m_load_balance.f90
1589 
1590  !> Load balance all boxes in the multigrid tree, by simply distributing the
1591  !> load per grid level. This method will only work well for uniform grids.
1592  !>
1593  !> Note that in a typical application the load balancing of the leaves is
1594  !> already determined, then mg_load_balance_parents can be used.
1595  subroutine mg_load_balance_simple(mg)
1596  type(mg_t), intent(inout) :: mg
1597  integer :: i, id, lvl, single_cpu_lvl
1598  integer :: work_left, my_work, i_cpu
1599 
1600  ! Up to this level, all boxes have to be on a single processor because they
1601  ! have a different size and the communication routines do not support this
1602  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1603 
1604  do lvl = mg%lowest_lvl, single_cpu_lvl
1605  do i = 1, size(mg%lvls(lvl)%ids)
1606  id = mg%lvls(lvl)%ids(i)
1607  mg%boxes(id)%rank = 0
1608  end do
1609  end do
1610 
1611  ! Distribute the boxes equally. Due to the way the mesh is constructed, the
1612  ! mg%lvls(lvl)%ids array already contains a Morton-like ordering.
1613  do lvl = single_cpu_lvl+1, mg%highest_lvl
1614  work_left = size(mg%lvls(lvl)%ids)
1615  my_work = 0
1616  i_cpu = 0
1617 
1618  do i = 1, size(mg%lvls(lvl)%ids)
1619  if ((mg%n_cpu - i_cpu - 1) * my_work >= work_left) then
1620  i_cpu = i_cpu + 1
1621  my_work = 0
1622  end if
1623 
1624  my_work = my_work + 1
1625  work_left = work_left - 1
1626 
1627  id = mg%lvls(lvl)%ids(i)
1628  mg%boxes(id)%rank = i_cpu
1629  end do
1630  end do
1631 
1632  do lvl = mg%lowest_lvl, mg%highest_lvl
1633  call update_lvl_info(mg, mg%lvls(lvl))
1634  end do
1635 
1636  end subroutine mg_load_balance_simple
1637 
1638  !> Load balance all boxes in the multigrid tree. Compared to
1639  !> mg_load_balance_simple, this method does a better job of setting the ranks
1640  !> of parent boxes
1641  !>
1642  !> Note that in a typical application the load balancing of the leaves is
1643  !> already determined, then mg_load_balance_parents can be used.
1644  subroutine mg_load_balance(mg)
1645  type(mg_t), intent(inout) :: mg
1646  integer :: i, id, lvl, single_cpu_lvl
1647  integer :: work_left, my_work(0:mg%n_cpu), i_cpu
1648  integer :: c_ids(mg_num_children)
1649  integer :: c_ranks(mg_num_children)
1650  integer :: coarse_rank
1651 
1652  ! Up to this level, all boxes have to be on a single processor because they
1653  ! have a different size and the communication routines do not support this
1654  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1655 
1656  ! Distribute the boxes equally. Due to the way the mesh is constructed, the
1657  ! mg%lvls(lvl)%ids array already contains a Morton-like ordering.
1658  do lvl = mg%highest_lvl, single_cpu_lvl+1, -1
1659  ! For parents determine the rank based on their child ranks
1660  my_work(:) = 0
1661 
1662  do i = 1, size(mg%lvls(lvl)%parents)
1663  id = mg%lvls(lvl)%parents(i)
1664 
1665  c_ids = mg%boxes(id)%children
1666  c_ranks = mg%boxes(c_ids)%rank
1667  i_cpu = most_popular(c_ranks, my_work, mg%n_cpu)
1668  mg%boxes(id)%rank = i_cpu
1669  my_work(i_cpu) = my_work(i_cpu) + 1
1670  end do
1671 
1672  work_left = size(mg%lvls(lvl)%leaves)
1673  i_cpu = 0
1674 
1675  do i = 1, size(mg%lvls(lvl)%leaves)
1676  ! Skip this CPU if it already has enough work
1677  if ((mg%n_cpu - i_cpu - 1) * my_work(i_cpu) >= &
1678  work_left + sum(my_work(i_cpu+1:))) then
1679  i_cpu = i_cpu + 1
1680  end if
1681 
1682  my_work(i_cpu) = my_work(i_cpu) + 1
1683  work_left = work_left - 1
1684 
1685  id = mg%lvls(lvl)%leaves(i)
1686  mg%boxes(id)%rank = i_cpu
1687  end do
1688  end do
1689 
1690  ! Determine most popular CPU for coarse grids
1691  if (single_cpu_lvl < mg%highest_lvl) then
1692  coarse_rank = most_popular(mg%boxes(&
1693  mg%lvls(single_cpu_lvl+1)%ids)%rank, my_work, mg%n_cpu)
1694  else
1695  coarse_rank = 0
1696  end if
1697 
1698  do lvl = mg%lowest_lvl, single_cpu_lvl
1699  do i = 1, size(mg%lvls(lvl)%ids)
1700  id = mg%lvls(lvl)%ids(i)
1701  mg%boxes(id)%rank = coarse_rank
1702  end do
1703  end do
1704 
1705  do lvl = mg%lowest_lvl, mg%highest_lvl
1706  call update_lvl_info(mg, mg%lvls(lvl))
1707  end do
1708 
1709  end subroutine mg_load_balance
1710 
1711  !> Load balance the parents (non-leafs). Assign them to the rank that has most
1712  !> children.
1714  type(mg_t), intent(inout) :: mg
1715  integer :: i, id, lvl
1716  integer :: c_ids(mg_num_children)
1717  integer :: c_ranks(mg_num_children)
1718  integer :: single_cpu_lvl, coarse_rank
1719  integer :: my_work(0:mg%n_cpu), i_cpu
1720 
1721  ! Up to this level, all boxes have to be on a single processor because they
1722  ! have a different size and the communication routines do not support this
1723  single_cpu_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
1724 
1725  do lvl = mg%highest_lvl-1, single_cpu_lvl+1, -1
1726  my_work(:) = 0
1727 
1728  ! Determine amount of work for the leaves
1729  do i = 1, size(mg%lvls(lvl)%leaves)
1730  id = mg%lvls(lvl)%leaves(i)
1731  i_cpu = mg%boxes(id)%rank
1732  my_work(i_cpu) = my_work(i_cpu) + 1
1733  end do
1734 
1735  do i = 1, size(mg%lvls(lvl)%parents)
1736  id = mg%lvls(lvl)%parents(i)
1737 
1738  c_ids = mg%boxes(id)%children
1739  c_ranks = mg%boxes(c_ids)%rank
1740  i_cpu = most_popular(c_ranks, my_work, mg%n_cpu)
1741  mg%boxes(id)%rank = i_cpu
1742  my_work(i_cpu) = my_work(i_cpu) + 1
1743  end do
1744 
1745  end do
1746 
1747  ! Determine most popular CPU for coarse grids
1748  if (single_cpu_lvl < mg%highest_lvl) then
1749  coarse_rank = most_popular(mg%boxes(&
1750  mg%lvls(single_cpu_lvl+1)%ids)%rank, my_work, mg%n_cpu)
1751  else
1752  coarse_rank = 0
1753  end if
1754 
1755  do lvl = mg%lowest_lvl, single_cpu_lvl
1756  do i = 1, size(mg%lvls(lvl)%ids)
1757  id = mg%lvls(lvl)%ids(i)
1758  mg%boxes(id)%rank = coarse_rank
1759  end do
1760  end do
1761 
1762  do lvl = mg%lowest_lvl, mg%highest_lvl
1763  call update_lvl_info(mg, mg%lvls(lvl))
1764  end do
1765 
1766  end subroutine mg_load_balance_parents
1767 
1768  !> Determine most popular rank in the list. In case of ties, assign the rank
1769  !> with the least work.
1770  pure integer function most_popular(list, work, n_cpu)
1771  integer, intent(in) :: list(:) !< List of MPI ranks
1772  integer, intent(in) :: n_cpu
1773  integer, intent(in) :: work(0:n_cpu-1) !< Existing work per rank
1774  integer :: i, best_count, current_count
1775  integer :: my_work, best_work
1776 
1777  best_count = 0
1778  best_work = 0
1779  most_popular = -1
1780 
1781  do i = 1, size(list)
1782  current_count = count(list == list(i))
1783  my_work = work(list(i))
1784 
1785  ! In case of ties, select task with lowest work
1786  if (current_count > best_count .or. &
1787  current_count == best_count .and. my_work < best_work) then
1788  best_count = current_count
1789  best_work = my_work
1790  most_popular = list(i)
1791  end if
1792  end do
1793 
1794  end function most_popular
1795 
1796  subroutine update_lvl_info(mg, lvl)
1797  type(mg_t), intent(inout) :: mg
1798  type(mg_lvl_t), intent(inout) :: lvl
1799 
1800  lvl%my_ids = pack(lvl%ids, &
1801  mg%boxes(lvl%ids)%rank == mg%my_rank)
1802  lvl%my_leaves = pack(lvl%leaves, &
1803  mg%boxes(lvl%leaves)%rank == mg%my_rank)
1804  lvl%my_parents = pack(lvl%parents, &
1805  mg%boxes(lvl%parents)%rank == mg%my_rank)
1806  lvl%my_ref_bnds = pack(lvl%ref_bnds, &
1807  mg%boxes(lvl%ref_bnds)%rank == mg%my_rank)
1808  end subroutine update_lvl_info
1809 
1810  !! File ../src/m_vlaplacian.f90
1811 
1812  subroutine vlaplacian_set_methods(mg)
1813  type(mg_t), intent(inout) :: mg
1814 
1815  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
1816  error stop "vlaplacian_set_methods: mg%n_extra_vars == 0"
1817  else
1818  mg%n_extra_vars = max(1, mg%n_extra_vars)
1819  end if
1820 
1821  mg%subtract_mean = .false.
1822 
1823  ! Use Neumann zero boundary conditions for the variable coefficient, since
1824  ! it is needed in ghost cells.
1825  mg%bc(:, mg_iveps)%bc_type = mg_bc_neumann
1826  mg%bc(:, mg_iveps)%bc_value = 0.0_dp
1827 
1828  select case (mg%geometry_type)
1829  case (mg_cartesian)
1830  mg%box_op => box_vlpl
1831 
1832  ! TODO: test whether a custom prolongation works better
1833  ! mg%box_prolong => vlpl_prolong
1834 
1835  select case (mg%smoother_type)
1837  mg%box_smoother => box_gs_vlpl
1838  case default
1839  error stop "vlaplacian_set_methods: unsupported smoother type"
1840  end select
1841 
1842  case default
1843  error stop "vlaplacian_set_methods: unsupported geometry"
1844  end select
1845 
1846  end subroutine vlaplacian_set_methods
1847 
1848  !> Perform Gauss-Seidel relaxation on box for a Laplacian operator
1849  subroutine box_gs_vlpl(mg, id, nc, redblack_cntr)
1850  type(mg_t), intent(inout) :: mg
1851  integer, intent(in) :: id
1852  integer, intent(in) :: nc
1853  integer, intent(in) :: redblack_cntr !< Iteration counter
1854  integer :: i, j, i0, di
1855  logical :: redblack
1856  real(dp) :: idr2(2*2), u(2*2)
1857  real(dp) :: a0, a(2*2), c(2*2)
1858 
1859  ! Duplicate 1/dr^2 array to multiply neighbor values
1860  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1861  idr2(2:2*2:2) = idr2(1:2*2:2)
1862  i0 = 1
1863 
1864  redblack = (mg%smoother_type == mg_smoother_gsrb)
1865  if (redblack) then
1866  di = 2
1867  else
1868  di = 1
1869  end if
1870 
1871  ! The parity of redblack_cntr determines which cells we use. If
1872  ! redblack_cntr is even, we use the even cells and vice versa.
1873  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1874  i_eps => mg_iveps)
1875  do j = 1, nc
1876  if (redblack) &
1877  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
1878 
1879  do i = i0, nc, di
1880  a0 = cc(i, j, i_eps)
1881  u(1:2) = cc(i-1:i+1:2, j, n)
1882  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1883  u(3:4) = cc(i, j-1:j+1:2, n)
1884  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1885  c(:) = 2 * a0 * a(:) / (a0 + a(:)) * idr2
1886 
1887  cc(i, j, n) = &
1888  (sum(c(:) * u(:)) - cc(i, j, mg_irhs)) / sum(c(:))
1889  end do
1890  end do
1891  end associate
1892  end subroutine box_gs_vlpl
1893 
1894  !> Perform Laplacian operator on a box
1895  subroutine box_vlpl(mg, id, nc, i_out)
1896  type(mg_t), intent(inout) :: mg
1897  integer, intent(in) :: id
1898  integer, intent(in) :: nc
1899  integer, intent(in) :: i_out !< Index of variable to store Laplacian in
1900  integer :: i, j
1901  real(dp) :: idr2(2*2), a0, u0, u(2*2), a(2*2)
1902 
1903  ! Duplicate 1/dr^2 array to multiply neighbor values
1904  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
1905  idr2(2:2*2:2) = idr2(1:2*2:2)
1906 
1907  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
1908  i_eps => mg_iveps)
1909  do j = 1, nc
1910  do i = 1, nc
1911  a0 = cc(i, j, i_eps)
1912  a(1:2) = cc(i-1:i+1:2, j, i_eps)
1913  a(3:4) = cc(i, j-1:j+1:2, i_eps)
1914  u0 = cc(i, j, n)
1915  u(1:2) = cc(i-1:i+1:2, j, n)
1916  u(3:4) = cc(i, j-1:j+1:2, n)
1917 
1918  cc(i, j, i_out) = sum(2 * idr2 * &
1919  a0*a(:)/(a0 + a(:)) * (u(:) - u0))
1920  end do
1921  end do
1922  end associate
1923  end subroutine box_vlpl
1924 
1925  !> Prolong from a parent to a child with index offset dix. This method could
1926  !> sometimes work better than the default prolongation, which does not take
1927  !> the variation in epsilon into account
1928 ! subroutine vlpl_prolong(mg, p_id, dix, nc, iv, fine)
1929 ! type(mg_t), intent(inout) :: mg
1930 ! integer, intent(in) :: p_id !< Id of parent
1931 ! integer, intent(in) :: dix(2) !< Offset of child in parent grid
1932 ! integer, intent(in) :: nc !< Child grid size
1933 ! integer, intent(in) :: iv !< Prolong from this variable
1934 ! real(dp), intent(out) :: fine(nc, nc) !< Prolonged values
1935 
1936 ! integer :: i, j, hnc
1937 ! #if 2 == 2
1938 ! integer :: ic, jc
1939 ! real(dp) :: f0, flx, fhx, fly, fhy
1940 ! real(dp) :: c0, clx, chx, cly, chy
1941 ! #elif 2 == 3
1942 ! integer :: ic, jc, kc
1943 ! real(dp) :: f0, flx, fhx, fly, fhy, flz, fhz
1944 ! real(dp) :: c0, clx, chx, cly, chy, clz, chz
1945 ! #endif
1946 
1947 ! hnc = nc/2
1948 
1949 ! associate (crs => mg%boxes(p_id)%cc, &
1950 ! i_eps => mg_iveps)
1951 ! #if 2 == 2
1952 ! do j = 1, hnc
1953 ! jc = j + dix(2)
1954 ! do i = 1, hnc
1955 ! ic = i + dix(1)
1956 
1957 ! f0 = crs(ic, jc, iv)
1958 ! flx = crs(ic-1, jc, iv)
1959 ! fhx = crs(ic+1, jc, iv)
1960 ! fly = crs(ic, jc-1, iv)
1961 ! fhy = crs(ic, jc+1, iv)
1962 
1963 ! c0 = 2 * crs(ic, jc, i_eps)
1964 ! clx = crs(ic-1, jc, i_eps)
1965 ! chx = crs(ic+1, jc, i_eps)
1966 ! cly = crs(ic, jc-1, i_eps)
1967 ! chy = crs(ic, jc+1, i_eps)
1968 
1969 ! fine(2*i-1, 2*j-1) = (f0*c0 + flx*clx + fly*cly) / &
1970 ! (c0 + clx + cly)
1971 ! fine(2*i , 2*j-1) = (f0*c0 + fhx*chx + fly*cly) / &
1972 ! (c0 + chx + cly)
1973 ! fine(2*i-1, 2*j) = (f0*c0 + flx*clx + fhy*chy) / &
1974 ! (c0 + clx + chy)
1975 ! fine(2*i , 2*j) = (f0*c0 + fhx*chx + fhy*chy) / &
1976 ! (c0 + chx + chy)
1977 ! end do
1978 ! end do
1979 ! #elif 2 == 3
1980 ! do k = 1, hnc
1981 ! kc = k + dix(3)
1982 ! do j = 1, hnc
1983 ! jc = j + dix(2)
1984 ! do i = 1, hnc
1985 ! ic = i + dix(1)
1986 
1987 ! f0 = crs(ic, jc, kc, iv)
1988 ! flx = crs(ic-1, jc, kc, iv)
1989 ! fhx = crs(ic+1, jc, kc, iv)
1990 ! fly = crs(ic, jc-1, kc, iv)
1991 ! fhy = crs(ic, jc+1, kc, iv)
1992 ! flz = crs(ic, jc, kc-1, iv)
1993 ! fhz = crs(ic, jc, kc+1, iv)
1994 
1995 ! c0 = crs(ic, jc, kc, i_eps)
1996 ! clx = crs(ic-1, jc, kc, i_eps)
1997 ! chx = crs(ic+1, jc, kc, i_eps)
1998 ! cly = crs(ic, jc-1, kc, i_eps)
1999 ! chy = crs(ic, jc+1, kc, i_eps)
2000 ! clz = crs(ic, jc, kc-1, i_eps)
2001 ! chz = crs(ic, jc, kc+1, i_eps)
2002 
2003 ! fine(2*i-1, 2*j-1, 2*k-1) = (f0*c0 + flx*clx + fly*cly + flz*clz) / &
2004 ! (c0 + clx + cly + clz)
2005 ! fine(2*i, 2*j-1, 2*k-1) = (f0*c0 + fhx*chx + fly*cly + flz*clz) / &
2006 ! (c0 + chx + cly + clz)
2007 ! fine(2*i-1, 2*j, 2*k-1) = (f0*c0 + flx*clx + fhy*chy + flz*clz) / &
2008 ! (c0 + clx + chy + clz)
2009 ! fine(2*i, 2*j, 2*k-1) = (f0*c0 + fhx*chx + fhy*chy + flz*clz) / &
2010 ! (c0 + chx + chy + clz)
2011 ! fine(2*i-1, 2*j-1, 2*k) = (f0*c0 + flx*clx + fly*cly + fhz*chz) / &
2012 ! (c0 + clx + cly + chz)
2013 ! fine(2*i, 2*j-1, 2*k) = (f0*c0 + fhx*chx + fly*cly + fhz*chz) / &
2014 ! (c0 + chx + cly + chz)
2015 ! fine(2*i-1, 2*j, 2*k) = (f0*c0 + flx*clx + fhy*chy + fhz*chz) / &
2016 ! (c0 + clx + chy + chz)
2017 ! fine(2*i, 2*j, 2*k) = (f0*c0 + fhx*chx + fhy*chy + fhz*chz) / &
2018 ! (c0 + chx + chy + chz)
2019 ! end do
2020 ! end do
2021 ! end do
2022 ! #endif
2023 ! end associate
2024 ! end subroutine vlpl_prolong
2025 
2026  !! File ../src/m_communication.f90
2027 
2028  !> Initialize MPI if needed, and store MPI information
2029  subroutine mg_comm_init(mg, comm)
2030  use mpi
2031  type(mg_t), intent(inout) :: mg
2032  !> MPI communicator (default: MPI_COMM_WORLD)
2033  integer, intent(in), optional :: comm
2034  integer :: ierr
2035  logical :: initialized
2036 
2037  call mpi_initialized(initialized, ierr)
2038  if (.not. initialized) then
2039  call mpi_init(ierr)
2040  end if
2041 
2042  if (present(comm)) then
2043  mg%comm = comm
2044  else
2045  mg%comm = mpi_comm_world
2046  end if
2047 
2048  call mpi_comm_rank(mg%comm, mg%my_rank, ierr)
2049  call mpi_comm_size(mg%comm, mg%n_cpu, ierr)
2050  end subroutine mg_comm_init
2051 
2052  subroutine sort_and_transfer_buffers(mg, dsize)
2053  use mpi
2054  type(mg_t), intent(inout) :: mg
2055  integer, intent(in) :: dsize
2056  integer :: i, n_send, n_recv
2057  integer :: send_req(mg%n_cpu)
2058  integer :: recv_req(mg%n_cpu)
2059  integer :: ierr
2060 
2061  n_send = 0
2062  n_recv = 0
2063 
2064  do i = 0, mg%n_cpu - 1
2065  if (mg%buf(i)%i_send > 0) then
2066  n_send = n_send + 1
2067  call sort_sendbuf(mg%buf(i), dsize)
2068  call mpi_isend(mg%buf(i)%send, mg%buf(i)%i_send, mpi_double, &
2069  i, 0, mg%comm, send_req(n_send), ierr)
2070  end if
2071  if (mg%buf(i)%i_recv > 0) then
2072  n_recv = n_recv + 1
2073  call mpi_irecv(mg%buf(i)%recv, mg%buf(i)%i_recv, mpi_double, &
2074  i, 0, mg%comm, recv_req(n_recv), ierr)
2075  end if
2076  end do
2077 
2078  call mpi_waitall(n_recv, recv_req(1:n_recv), mpi_statuses_ignore, ierr)
2079  call mpi_waitall(n_send, send_req(1:n_send), mpi_statuses_ignore, ierr)
2080 
2081  end subroutine sort_and_transfer_buffers
2082 
2083  !> Sort send buffers according to the idbuf array
2084  subroutine sort_sendbuf(gc, dsize)
2085 
2086  type(mg_buf_t), intent(inout) :: gc
2087  integer, intent(in) :: dsize !< Size of send buffer elements
2088  integer :: ix_sort(gc%i_ix)
2089  real(dp) :: buf_cpy(gc%i_send)
2090  integer :: i, j, n
2091 
2092  call mrgrnk(gc%ix(1:gc%i_ix), ix_sort)
2093 
2094  buf_cpy = gc%send(1:gc%i_send)
2095 
2096  do n = 1, gc%i_ix
2097  i = (n-1) * dsize
2098  j = (ix_sort(n)-1) * dsize
2099  gc%send(i+1:i+dsize) = buf_cpy(j+1:j+dsize)
2100  end do
2101  gc%ix(1:gc%i_ix) = gc%ix(ix_sort)
2102 
2103  end subroutine sort_sendbuf
2104 
2105  !! File ../src/m_ghost_cells.f90
2106 
2107  !> Specify minimum buffer size (per process) for communication
2108  subroutine mg_ghost_cell_buffer_size(mg, n_send, n_recv, dsize)
2109  type(mg_t), intent(inout) :: mg
2110  integer, intent(out) :: n_send(0:mg%n_cpu-1)
2111  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
2112  integer, intent(out) :: dsize
2113  integer :: i, id, lvl, nc
2114 
2115  allocate(mg%comm_ghostcell%n_send(0:mg%n_cpu-1, &
2116  mg%first_normal_lvl:mg%highest_lvl))
2117  allocate(mg%comm_ghostcell%n_recv(0:mg%n_cpu-1, &
2118  mg%first_normal_lvl:mg%highest_lvl))
2119 
2120  dsize = mg%box_size**(2-1)
2121 
2122  do lvl = mg%first_normal_lvl, mg%highest_lvl
2123  nc = mg%box_size_lvl(lvl)
2124  mg%buf(:)%i_send = 0
2125  mg%buf(:)%i_recv = 0
2126  mg%buf(:)%i_ix = 0
2127 
2128  do i = 1, size(mg%lvls(lvl)%my_ids)
2129  id = mg%lvls(lvl)%my_ids(i)
2130  call buffer_ghost_cells(mg, id, nc, 1, dry_run=.true.)
2131  end do
2132 
2133  if (lvl > 1) then
2134  do i = 1, size(mg%lvls(lvl-1)%my_ref_bnds)
2135  id = mg%lvls(lvl-1)%my_ref_bnds(i)
2136  call buffer_refinement_boundaries(mg, id, nc, 1, dry_run=.true.)
2137  end do
2138  end if
2139 
2140  ! Set ghost cells to received data
2141  mg%buf(:)%i_recv = 0
2142  do i = 1, size(mg%lvls(lvl)%my_ids)
2143  id = mg%lvls(lvl)%my_ids(i)
2144  call set_ghost_cells(mg, id, nc, 1, dry_run=.true.)
2145  end do
2146 
2147  mg%comm_ghostcell%n_send(:, lvl) = mg%buf(:)%i_send/dsize
2148  mg%comm_ghostcell%n_recv(:, lvl) = mg%buf(:)%i_recv/dsize
2149  end do
2150 
2151  n_send = maxval(mg%comm_ghostcell%n_send, dim=2)
2152  n_recv = maxval(mg%comm_ghostcell%n_recv, dim=2)
2153  end subroutine mg_ghost_cell_buffer_size
2154 
2155  !> Store boundary conditions for the solution variable, this can speed up
2156  !> calculations if the same boundary conditions are re-used.
2157  subroutine mg_phi_bc_store(mg)
2158  type(mg_t), intent(inout) :: mg
2159  integer :: lvl, nc
2160 
2161  nc = mg%box_size
2162 
2163  do lvl = mg%lowest_lvl, mg%highest_lvl
2164  nc = mg%box_size_lvl(lvl)
2165  call mg_phi_bc_store_lvl(mg, lvl, nc)
2166  end do
2167 
2168  mg%phi_bc_data_stored = .true.
2169  end subroutine mg_phi_bc_store
2170 
2171  subroutine mg_phi_bc_store_lvl(mg, lvl, nc)
2172  type(mg_t), intent(inout) :: mg
2173  integer, intent(in) :: lvl
2174  integer, intent(in) :: nc
2175  real(dp) :: bc(nc)
2176  integer :: i, id, nb, nb_id, bc_type
2177 
2178  do i = 1, size(mg%lvls(lvl)%my_ids)
2179  id = mg%lvls(lvl)%my_ids(i)
2180  do nb = 1, mg_num_neighbors
2181  nb_id = mg%boxes(id)%neighbors(nb)
2182  if (nb_id < mg_no_box) then
2183  ! Physical boundary
2184  if (associated(mg%bc(nb, mg_iphi)%boundary_cond)) then
2185  call mg%bc(nb, mg_iphi)%boundary_cond(mg%boxes(id), nc, &
2186  mg_iphi, nb, bc_type, bc)
2187  else
2188  bc_type = mg%bc(nb, mg_iphi)%bc_type
2189  bc = mg%bc(nb, mg_iphi)%bc_value
2190  end if
2191 
2192  ! Store the boundary condition type. This is not globally set in
2193  ! the tree, but all negative values are treated the same in
2194  ! other parts of the code
2195  mg%boxes(id)%neighbors(nb) = bc_type
2196 
2197  ! Store ghost cell data in the right-hand side
2198  call box_set_gc(mg%boxes(id), nb, nc, mg_irhs, bc)
2199  end if
2200  end do
2201  end do
2202  end subroutine mg_phi_bc_store_lvl
2203 
2204  !> Fill ghost cells at all grid levels
2205  subroutine mg_fill_ghost_cells(mg, iv)
2206  type(mg_t) :: mg
2207  integer, intent(in) :: iv !< Index of variable
2208  integer :: lvl
2209 
2210  do lvl = mg%lowest_lvl, mg%highest_lvl
2211  call mg_fill_ghost_cells_lvl(mg, lvl, iv)
2212  end do
2213  end subroutine mg_fill_ghost_cells
2214 
2215  !> Fill ghost cells at a grid level
2216  subroutine mg_fill_ghost_cells_lvl(mg, lvl, iv)
2217 
2218  type(mg_t) :: mg
2219  integer, intent(in) :: lvl
2220  integer, intent(in) :: iv !< Index of variable
2221  integer :: i, id, dsize, nc
2222 
2223  if (lvl < mg%lowest_lvl) &
2224  error stop "fill_ghost_cells_lvl: lvl < lowest_lvl"
2225  if (lvl > mg%highest_lvl) &
2226  error stop "fill_ghost_cells_lvl: lvl > highest_lvl"
2227 
2228  nc = mg%box_size_lvl(lvl)
2229 
2230  if (lvl >= mg%first_normal_lvl) then
2231  dsize = nc**(2-1)
2232  mg%buf(:)%i_send = 0
2233  mg%buf(:)%i_recv = 0
2234  mg%buf(:)%i_ix = 0
2235 
2236  do i = 1, size(mg%lvls(lvl)%my_ids)
2237  id = mg%lvls(lvl)%my_ids(i)
2238  call buffer_ghost_cells(mg, id, nc, iv, .false.)
2239  end do
2240 
2241  if (lvl > 1) then
2242  do i = 1, size(mg%lvls(lvl-1)%my_ref_bnds)
2243  id = mg%lvls(lvl-1)%my_ref_bnds(i)
2244  call buffer_refinement_boundaries(mg, id, nc, iv, .false.)
2245  end do
2246  end if
2247 
2248  ! Transfer data between processes
2249  mg%buf(:)%i_recv = mg%comm_ghostcell%n_recv(:, lvl) * dsize
2250  call sort_and_transfer_buffers(mg, dsize)
2251 
2252  ! Set ghost cells to received data
2253  mg%buf(:)%i_recv = 0
2254  end if
2255 
2256  do i = 1, size(mg%lvls(lvl)%my_ids)
2257  id = mg%lvls(lvl)%my_ids(i)
2258  call set_ghost_cells(mg, id, nc, iv, .false.)
2259  end do
2260  end subroutine mg_fill_ghost_cells_lvl
2261 
2262  subroutine buffer_ghost_cells(mg, id, nc, iv, dry_run)
2263  type(mg_t), intent(inout) :: mg
2264  integer, intent(in) :: id
2265  integer, intent(in) :: nc
2266  integer, intent(in) :: iv
2267  logical, intent(in) :: dry_run
2268  integer :: nb, nb_id, nb_rank
2269 
2270  do nb = 1, mg_num_neighbors
2271  nb_id = mg%boxes(id)%neighbors(nb)
2272 
2273  if (nb_id > mg_no_box) then
2274  ! There is a neighbor
2275  nb_rank = mg%boxes(nb_id)%rank
2276 
2277  if (nb_rank /= mg%my_rank) then
2278  call buffer_for_nb(mg, mg%boxes(id), nc, iv, nb_id, nb_rank, &
2279  nb, dry_run)
2280  end if
2281  end if
2282  end do
2283  end subroutine buffer_ghost_cells
2284 
2285  subroutine buffer_refinement_boundaries(mg, id, nc, iv, dry_run)
2286  type(mg_t), intent(inout) :: mg
2287  integer, intent(in) :: id
2288  integer, intent(in) :: nc
2289  integer, intent(in) :: iv
2290  logical, intent(in) :: dry_run
2291  integer :: nb, nb_id, c_ids(2**(2-1))
2292  integer :: n, c_id, c_rank
2293 
2294  do nb = 1, mg_num_neighbors
2295  nb_id = mg%boxes(id)%neighbors(nb)
2296  if (nb_id > mg_no_box) then
2297  if (mg_has_children(mg%boxes(nb_id))) then
2298  c_ids = mg%boxes(nb_id)%children(&
2300 
2301  do n = 1, mg_num_children/2
2302  c_id = c_ids(n)
2303  c_rank = mg%boxes(c_id)%rank
2304 
2305  if (c_rank /= mg%my_rank) then
2306  ! Send all coarse ghost cells
2307  call buffer_for_fine_nb(mg, mg%boxes(id), nc, iv, c_id, &
2308  c_rank, nb, dry_run)
2309  end if
2310  end do
2311  end if
2312  end if
2313  end do
2314  end subroutine buffer_refinement_boundaries
2315 
2316  !> The routine that actually fills the ghost cells
2317  subroutine set_ghost_cells(mg, id, nc, iv, dry_run)
2318  type(mg_t), intent(inout) :: mg
2319  integer, intent(in) :: id
2320  integer, intent(in) :: nc
2321  integer, intent(in) :: iv
2322  logical, intent(in) :: dry_run
2323  real(dp) :: bc(nc)
2324  integer :: nb, nb_id, nb_rank, bc_type
2325 
2326  do nb = 1, mg_num_neighbors
2327  nb_id = mg%boxes(id)%neighbors(nb)
2328 
2329  if (nb_id > mg_no_box) then
2330  ! There is a neighbor
2331  nb_rank = mg%boxes(nb_id)%rank
2332 
2333  if (nb_rank /= mg%my_rank) then
2334  call fill_buffered_nb(mg, mg%boxes(id), nb_rank, &
2335  nb, nc, iv, dry_run)
2336  else if (.not. dry_run) then
2337  call copy_from_nb(mg%boxes(id), mg%boxes(nb_id), &
2338  nb, nc, iv)
2339  end if
2340  else if (nb_id == mg_no_box) then
2341  ! Refinement boundary
2342  call fill_refinement_bnd(mg, id, nb, nc, iv, dry_run)
2343  else if (.not. dry_run) then
2344  ! Physical boundary
2345  if (mg%phi_bc_data_stored .and. iv == mg_iphi) then
2346  ! Copy the boundary conditions stored in the ghost cells of the
2347  ! right-hand side
2348  call box_get_gc(mg%boxes(id), nb, nc, mg_irhs, bc)
2349  bc_type = nb_id
2350  else
2351  if (associated(mg%bc(nb, iv)%boundary_cond)) then
2352  call mg%bc(nb, iv)%boundary_cond(mg%boxes(id), nc, iv, &
2353  nb, bc_type, bc)
2354  else
2355  bc_type = mg%bc(nb, iv)%bc_type
2356  bc = mg%bc(nb, iv)%bc_value
2357  end if
2358  end if
2359 
2360  call box_set_gc(mg%boxes(id), nb, nc, iv, bc)
2361  call bc_to_gc(mg, id, nc, iv, nb, bc_type)
2362  end if
2363  end do
2364  end subroutine set_ghost_cells
2365 
2366  subroutine fill_refinement_bnd(mg, id, nb, nc, iv, dry_run)
2367  type(mg_t), intent(inout) :: mg
2368  integer, intent(in) :: id
2369  integer, intent(in) :: nc
2370  integer, intent(in) :: iv
2371  integer, intent(in) :: nb
2372  logical, intent(in) :: dry_run
2373  real(dp) :: gc(nc)
2374  integer :: p_id, p_nb_id, ix_offset(2)
2375  integer :: i, dsize, p_nb_rank
2376 
2377  dsize = nc**(2-1)
2378  p_id = mg%boxes(id)%parent
2379  p_nb_id = mg%boxes(p_id)%neighbors(nb)
2380  p_nb_rank = mg%boxes(p_nb_id)%rank
2381 
2382  if (p_nb_rank /= mg%my_rank) then
2383  i = mg%buf(p_nb_rank)%i_recv
2384  if (.not. dry_run) then
2385  gc = reshape(mg%buf(p_nb_rank)%recv(i+1:i+dsize), shape(gc))
2386  end if
2387  mg%buf(p_nb_rank)%i_recv = mg%buf(p_nb_rank)%i_recv + dsize
2388  else if (.not. dry_run) then
2389  ix_offset = mg_get_child_offset(mg, id)
2390  call box_gc_for_fine_neighbor(mg%boxes(p_nb_id), mg_neighb_rev(nb), &
2391  ix_offset, nc, iv, gc)
2392  end if
2393 
2394  if (.not. dry_run) then
2395  if (associated(mg%bc(nb, iv)%refinement_bnd)) then
2396  call mg%bc(nb, iv)%refinement_bnd(mg%boxes(id), nc, iv, nb, gc)
2397  else
2398  call sides_rb(mg%boxes(id), nc, iv, nb, gc)
2399  end if
2400  end if
2401  end subroutine fill_refinement_bnd
2402 
2403  subroutine copy_from_nb(box, box_nb, nb, nc, iv)
2404  type(mg_box_t), intent(inout) :: box
2405  type(mg_box_t), intent(in) :: box_nb
2406  integer, intent(in) :: nb
2407  integer, intent(in) :: nc
2408  integer, intent(in) :: iv
2409  real(dp) :: gc(nc)
2410 
2411  call box_gc_for_neighbor(box_nb, mg_neighb_rev(nb), nc, iv, gc)
2412  call box_set_gc(box, nb, nc, iv, gc)
2413  end subroutine copy_from_nb
2414 
2415  subroutine buffer_for_nb(mg, box, nc, iv, nb_id, nb_rank, nb, dry_run)
2416  type(mg_t), intent(inout) :: mg
2417  type(mg_box_t), intent(inout) :: box
2418  integer, intent(in) :: nc
2419  integer, intent(in) :: iv
2420  integer, intent(in) :: nb_id
2421  integer, intent(in) :: nb_rank
2422  integer, intent(in) :: nb
2423  logical, intent(in) :: dry_run
2424  integer :: i, dsize
2425  real(dp) :: gc(nc)
2426 
2427  i = mg%buf(nb_rank)%i_send
2428  dsize = nc**(2-1)
2429 
2430  if (.not. dry_run) then
2431  call box_gc_for_neighbor(box, nb, nc, iv, gc)
2432  mg%buf(nb_rank)%send(i+1:i+dsize) = pack(gc, .true.)
2433  end if
2434 
2435  ! Later the buffer is sorted, using the fact that loops go from low to high
2436  ! box id, and we fill ghost cells according to the neighbor order
2437  i = mg%buf(nb_rank)%i_ix
2438  if (.not. dry_run) then
2439  mg%buf(nb_rank)%ix(i+1) = mg_num_neighbors * nb_id + mg_neighb_rev(nb)
2440  end if
2441 
2442  mg%buf(nb_rank)%i_send = mg%buf(nb_rank)%i_send + dsize
2443  mg%buf(nb_rank)%i_ix = mg%buf(nb_rank)%i_ix + 1
2444  end subroutine buffer_for_nb
2445 
2446  subroutine buffer_for_fine_nb(mg, box, nc, iv, fine_id, fine_rank, nb, dry_run)
2447  type(mg_t), intent(inout) :: mg
2448  type(mg_box_t), intent(inout) :: box
2449  integer, intent(in) :: nc
2450  integer, intent(in) :: iv
2451  integer, intent(in) :: fine_id
2452  integer, intent(in) :: fine_rank
2453  integer, intent(in) :: nb
2454  logical, intent(in) :: dry_run
2455  integer :: i, dsize, ix_offset(2)
2456  real(dp) :: gc(nc)
2457 
2458  i = mg%buf(fine_rank)%i_send
2459  dsize = nc**(2-1)
2460 
2461  if (.not. dry_run) then
2462  ix_offset = mg_get_child_offset(mg, fine_id)
2463  call box_gc_for_fine_neighbor(box, nb, ix_offset, nc, iv, gc)
2464  mg%buf(fine_rank)%send(i+1:i+dsize) = pack(gc, .true.)
2465  end if
2466 
2467  ! Later the buffer is sorted, using the fact that loops go from low to high
2468  ! box id, and we fill ghost cells according to the neighbor order
2469  i = mg%buf(fine_rank)%i_ix
2470  if (.not. dry_run) then
2471  mg%buf(fine_rank)%ix(i+1) = mg_num_neighbors * fine_id + &
2472  mg_neighb_rev(nb)
2473  end if
2474 
2475  mg%buf(fine_rank)%i_send = mg%buf(fine_rank)%i_send + dsize
2476  mg%buf(fine_rank)%i_ix = mg%buf(fine_rank)%i_ix + 1
2477  end subroutine buffer_for_fine_nb
2478 
2479  subroutine fill_buffered_nb(mg, box, nb_rank, nb, nc, iv, dry_run)
2480  type(mg_t), intent(inout) :: mg
2481  type(mg_box_t), intent(inout) :: box
2482  integer, intent(in) :: nb_rank
2483  integer, intent(in) :: nb
2484  integer, intent(in) :: nc
2485  integer, intent(in) :: iv
2486  logical, intent(in) :: dry_run
2487  integer :: i, dsize
2488  real(dp) :: gc(nc)
2489 
2490  i = mg%buf(nb_rank)%i_recv
2491  dsize = nc**(2-1)
2492 
2493  if (.not. dry_run) then
2494  gc = reshape(mg%buf(nb_rank)%recv(i+1:i+dsize), shape(gc))
2495  call box_set_gc(box, nb, nc, iv, gc)
2496  end if
2497  mg%buf(nb_rank)%i_recv = mg%buf(nb_rank)%i_recv + dsize
2498 
2499  end subroutine fill_buffered_nb
2500 
2501  subroutine box_gc_for_neighbor(box, nb, nc, iv, gc)
2502  type(mg_box_t), intent(in) :: box
2503  integer, intent(in) :: nb, nc, iv
2504  real(dp), intent(out) :: gc(nc)
2505 
2506  select case (nb)
2507  case (mg_neighb_lowx)
2508  gc = box%cc(1, 1:nc, iv)
2509  case (mg_neighb_highx)
2510  gc = box%cc(nc, 1:nc, iv)
2511  case (mg_neighb_lowy)
2512  gc = box%cc(1:nc, 1, iv)
2513  case (mg_neighb_highy)
2514  gc = box%cc(1:nc, nc, iv)
2515  end select
2516  end subroutine box_gc_for_neighbor
2517 
2518  !> Get ghost cells for a fine neighbor
2519  subroutine box_gc_for_fine_neighbor(box, nb, di, nc, iv, gc)
2520  type(mg_box_t), intent(in) :: box
2521  integer, intent(in) :: nb !< Direction of fine neighbor
2522  integer, intent(in) :: di(2) !< Index offset of fine neighbor
2523  integer, intent(in) :: nc, iv
2524  real(dp), intent(out) :: gc(nc)
2525  real(dp) :: tmp(0:nc/2+1), grad(2-1)
2526  integer :: i, hnc
2527 
2528  hnc = nc/2
2529 
2530  ! First fill a temporary array with data next to the fine grid
2531  select case (nb)
2532  case (mg_neighb_lowx)
2533  tmp = box%cc(1, di(2):di(2)+hnc+1, iv)
2534  case (mg_neighb_highx)
2535  tmp = box%cc(nc, di(2):di(2)+hnc+1, iv)
2536  case (mg_neighb_lowy)
2537  tmp = box%cc(di(1):di(1)+hnc+1, 1, iv)
2538  case (mg_neighb_highy)
2539  tmp = box%cc(di(1):di(1)+hnc+1, nc, iv)
2540  case default
2541  error stop
2542  end select
2543 
2544  ! Now interpolate the coarse grid data to obtain values 'straight' next to
2545  ! the fine grid points
2546  do i = 1, hnc
2547  grad(1) = 0.125_dp * (tmp(i+1) - tmp(i-1))
2548  gc(2*i-1) = tmp(i) - grad(1)
2549  gc(2*i) = tmp(i) + grad(1)
2550  end do
2551  end subroutine box_gc_for_fine_neighbor
2552 
2553  subroutine box_get_gc(box, nb, nc, iv, gc)
2554  type(mg_box_t), intent(in) :: box
2555  integer, intent(in) :: nb, nc, iv
2556  real(dp), intent(out) :: gc(nc)
2557 
2558  select case (nb)
2559  case (mg_neighb_lowx)
2560  gc = box%cc(0, 1:nc, iv)
2561  case (mg_neighb_highx)
2562  gc = box%cc(nc+1, 1:nc, iv)
2563  case (mg_neighb_lowy)
2564  gc = box%cc(1:nc, 0, iv)
2565  case (mg_neighb_highy)
2566  gc = box%cc(1:nc, nc+1, iv)
2567  end select
2568  end subroutine box_get_gc
2569 
2570  subroutine box_set_gc(box, nb, nc, iv, gc)
2571  type(mg_box_t), intent(inout) :: box
2572  integer, intent(in) :: nb, nc, iv
2573  real(dp), intent(in) :: gc(nc)
2574 
2575  select case (nb)
2576  case (mg_neighb_lowx)
2577  box%cc(0, 1:nc, iv) = gc
2578  case (mg_neighb_highx)
2579  box%cc(nc+1, 1:nc, iv) = gc
2580  case (mg_neighb_lowy)
2581  box%cc(1:nc, 0, iv) = gc
2582  case (mg_neighb_highy)
2583  box%cc(1:nc, nc+1, iv) = gc
2584  end select
2585  end subroutine box_set_gc
2586 
2587  subroutine bc_to_gc(mg, id, nc, iv, nb, bc_type)
2588  type(mg_t), intent(inout) :: mg
2589  integer, intent(in) :: id
2590  integer, intent(in) :: nc
2591  integer, intent(in) :: iv
2592  integer, intent(in) :: nb !< Neighbor direction
2593  integer, intent(in) :: bc_type !< Type of b.c.
2594  real(dp) :: c0, c1, c2, dr
2595 
2596  ! If we call the interior point x1, x2 and the ghost point x0, then a
2597  ! Dirichlet boundary value b can be imposed as:
2598  ! x0 = -x1 + 2*b
2599  ! A Neumann b.c. can be imposed as:
2600  ! x0 = x1 +/- dx * b
2601  ! A continuous boundary (same slope) as:
2602  ! x0 = 2 * x1 - x2
2603  ! Below, we set coefficients to handle these cases
2604  select case (bc_type)
2605  case (mg_bc_dirichlet)
2606  c0 = 2
2607  c1 = -1
2608  c2 = 0
2609  case (mg_bc_neumann)
2610  dr = mg%dr(mg_neighb_dim(nb), mg%boxes(id)%lvl)
2611  c0 = dr * mg_neighb_high_pm(nb) ! This gives a + or - sign
2612  c1 = 1
2613  c2 = 0
2614  case (mg_bc_continuous)
2615  c0 = 0
2616  c1 = 2
2617  c2 = -1
2618  case default
2619  error stop "bc_to_gc: unknown boundary condition"
2620  end select
2621 
2622  select case (nb)
2623  case (mg_neighb_lowx)
2624  mg%boxes(id)%cc(0, 1:nc, iv) = &
2625  c0 * mg%boxes(id)%cc(0, 1:nc, iv) + &
2626  c1 * mg%boxes(id)%cc(1, 1:nc, iv) + &
2627  c2 * mg%boxes(id)%cc(2, 1:nc, iv)
2628  case (mg_neighb_highx)
2629  mg%boxes(id)%cc(nc+1, 1:nc, iv) = &
2630  c0 * mg%boxes(id)%cc(nc+1, 1:nc, iv) + &
2631  c1 * mg%boxes(id)%cc(nc, 1:nc, iv) + &
2632  c2 * mg%boxes(id)%cc(nc-1, 1:nc, iv)
2633  case (mg_neighb_lowy)
2634  mg%boxes(id)%cc(1:nc, 0, iv) = &
2635  c0 * mg%boxes(id)%cc(1:nc, 0, iv) + &
2636  c1 * mg%boxes(id)%cc(1:nc, 1, iv) + &
2637  c2 * mg%boxes(id)%cc(1:nc, 2, iv)
2638  case (mg_neighb_highy)
2639  mg%boxes(id)%cc(1:nc, nc+1, iv) = &
2640  c0 * mg%boxes(id)%cc(1:nc, nc+1, iv) + &
2641  c1 * mg%boxes(id)%cc(1:nc, nc, iv) + &
2642  c2 * mg%boxes(id)%cc(1:nc, nc-1, iv)
2643  end select
2644  end subroutine bc_to_gc
2645 
2646  !> Fill ghost cells near refinement boundaries which preserves diffusive fluxes.
2647  subroutine sides_rb(box, nc, iv, nb, gc)
2648  type(mg_box_t), intent(inout) :: box
2649  integer, intent(in) :: nc
2650  integer, intent(in) :: iv
2651  integer, intent(in) :: nb !< Ghost cell direction
2652  !> Interpolated coarse grid ghost cell data (but not yet in the nb direction)
2653  real(dp), intent(in) :: gc(nc)
2654  integer :: di, dj
2655  integer :: i, j, ix, dix
2656 
2657  if (mg_neighb_low(nb)) then
2658  ix = 1
2659  dix = 1
2660  else
2661  ix = nc
2662  dix = -1
2663  end if
2664 
2665  select case (mg_neighb_dim(nb))
2666  case (1)
2667  i = ix
2668  di = dix
2669 
2670  do j = 1, nc
2671  dj = -1 + 2 * iand(j, 1)
2672  box%cc(i-di, j, iv) = 0.5_dp * gc(j) &
2673  + 0.75_dp * box%cc(i, j, iv) &
2674  - 0.25_dp * box%cc(i+di, j, iv)
2675  end do
2676  case (2)
2677  j = ix
2678  dj = dix
2679  do i = 1, nc
2680  di = -1 + 2 * iand(i, 1)
2681  box%cc(i, j-dj, iv) = 0.5_dp * gc(i) &
2682  + 0.75_dp * box%cc(i, j, iv) &
2683  - 0.25_dp * box%cc(i, j+dj, iv)
2684  end do
2685  end select
2686 
2687  end subroutine sides_rb
2688 
2689  !! File ../src/m_prolong.f90
2690 
2691  !> Specify minimum buffer size (per process) for communication
2692  subroutine mg_prolong_buffer_size(mg, n_send, n_recv, dsize)
2693  type(mg_t), intent(inout) :: mg
2694  integer, intent(out) :: n_send(0:mg%n_cpu-1)
2695  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
2696  integer, intent(out) :: dsize
2697  integer :: lvl, min_lvl
2698 
2699  if (.not. allocated(mg%comm_restrict%n_send)) then
2700  error stop "Call restrict_buffer_size before prolong_buffer_size"
2701  end if
2702 
2703  min_lvl = max(mg%first_normal_lvl-1, mg%lowest_lvl)
2704  allocate(mg%comm_prolong%n_send(0:mg%n_cpu-1, &
2705  min_lvl:mg%highest_lvl))
2706  allocate(mg%comm_prolong%n_recv(0:mg%n_cpu-1, &
2707  min_lvl:mg%highest_lvl))
2708 
2709  mg%comm_prolong%n_recv(:, mg%highest_lvl) = 0
2710  mg%comm_prolong%n_send(:, mg%highest_lvl) = 0
2711 
2712  do lvl = min_lvl, mg%highest_lvl-1
2713  mg%comm_prolong%n_recv(:, lvl) = &
2714  mg%comm_restrict%n_send(:, lvl+1)
2715  mg%comm_prolong%n_send(:, lvl) = &
2716  mg%comm_restrict%n_recv(:, lvl+1)
2717  end do
2718 
2719  ! Send fine grid points, because this is more flexible than sending coarse
2720  ! grid points (e.g., when multiple variables are used for interpolation)
2721  dsize = (mg%box_size)**2
2722  n_send = maxval(mg%comm_prolong%n_send, dim=2)
2723  n_recv = maxval(mg%comm_prolong%n_recv, dim=2)
2724  end subroutine mg_prolong_buffer_size
2725 
2726  !> Prolong variable iv from lvl to variable iv_to at lvl+1
2727  subroutine mg_prolong(mg, lvl, iv, iv_to, method, add)
2728 
2729  type(mg_t), intent(inout) :: mg
2730  integer, intent(in) :: lvl !< Level to prolong from
2731  integer, intent(in) :: iv !< Source variable
2732  integer, intent(in) :: iv_to !< Target variable
2733  procedure(mg_box_prolong) :: method !< Prolongation method
2734  logical, intent(in) :: add !< If true, add to current values
2735  integer :: i, id, dsize, nc
2736 
2737  if (lvl == mg%highest_lvl) error stop "cannot prolong highest level"
2738  if (lvl < mg%lowest_lvl) error stop "cannot prolong below lowest level"
2739 
2740  ! Below the first normal level, all boxes are on the same CPU
2741  if (lvl >= mg%first_normal_lvl-1) then
2742  dsize = mg%box_size**2
2743  mg%buf(:)%i_send = 0
2744  mg%buf(:)%i_ix = 0
2745 
2746  do i = 1, size(mg%lvls(lvl)%my_ids)
2747  id = mg%lvls(lvl)%my_ids(i)
2748  call prolong_set_buffer(mg, id, mg%box_size, iv, method)
2749  end do
2750 
2751  mg%buf(:)%i_recv = mg%comm_prolong%n_recv(:, lvl) * dsize
2752  call sort_and_transfer_buffers(mg, dsize)
2753  mg%buf(:)%i_recv = 0
2754  end if
2755 
2756  nc = mg%box_size_lvl(lvl+1)
2757  do i = 1, size(mg%lvls(lvl+1)%my_ids)
2758  id = mg%lvls(lvl+1)%my_ids(i)
2759  call prolong_onto(mg, id, nc, iv, iv_to, add, method)
2760  end do
2761  end subroutine mg_prolong
2762 
2763  !> In case the fine grid is on a different CPU, perform the prolongation and
2764  !> store the fine-grid values in the send buffer.
2765  !>
2766  subroutine prolong_set_buffer(mg, id, nc, iv, method)
2767  type(mg_t), intent(inout) :: mg
2768  integer, intent(in) :: id
2769  integer, intent(in) :: nc
2770  integer, intent(in) :: iv
2771  procedure(mg_box_prolong) :: method
2772  integer :: i, dix(2)
2773  integer :: i_c, c_id, c_rank, dsize
2774  real(dp) :: tmp(nc, nc)
2775 
2776  dsize = nc**2
2777 
2778  do i_c = 1, mg_num_children
2779  c_id = mg%boxes(id)%children(i_c)
2780  if (c_id > mg_no_box) then
2781  c_rank = mg%boxes(c_id)%rank
2782  if (c_rank /= mg%my_rank) then
2783  dix = mg_get_child_offset(mg, c_id)
2784  call method(mg, id, dix, nc, iv, tmp)
2785 
2786  i = mg%buf(c_rank)%i_send
2787  mg%buf(c_rank)%send(i+1:i+dsize) = pack(tmp, .true.)
2788  mg%buf(c_rank)%i_send = mg%buf(c_rank)%i_send + dsize
2789 
2790  i = mg%buf(c_rank)%i_ix
2791  mg%buf(c_rank)%ix(i+1) = c_id
2792  mg%buf(c_rank)%i_ix = mg%buf(c_rank)%i_ix + 1
2793  end if
2794  end if
2795  end do
2796  end subroutine prolong_set_buffer
2797 
2798  !> Prolong onto a child box
2799  subroutine prolong_onto(mg, id, nc, iv, iv_to, add, method)
2800  type(mg_t), intent(inout) :: mg
2801  integer, intent(in) :: id
2802  integer, intent(in) :: nc
2803  integer, intent(in) :: iv !< Prolong from this variable
2804  integer, intent(in) :: iv_to !< Prolong to this variable
2805  logical, intent(in) :: add !< If true, add to current values
2806  procedure(mg_box_prolong) :: method
2807  integer :: hnc, p_id, p_rank, i, dix(2), dsize
2808  real(dp) :: tmp(nc, nc)
2809 
2810  hnc = nc/2
2811  p_id = mg%boxes(id)%parent
2812  p_rank = mg%boxes(p_id)%rank
2813 
2814  if (p_rank == mg%my_rank) then
2815  dix = mg_get_child_offset(mg, id)
2816  call method(mg, p_id, dix, nc, iv, tmp)
2817  else
2818  dsize = nc**2
2819  i = mg%buf(p_rank)%i_recv
2820  tmp = reshape(mg%buf(p_rank)%recv(i+1:i+dsize), [nc, nc])
2821  mg%buf(p_rank)%i_recv = mg%buf(p_rank)%i_recv + dsize
2822  end if
2823 
2824  if (add) then
2825  mg%boxes(id)%cc(1:nc, 1:nc, iv_to) = &
2826  mg%boxes(id)%cc(1:nc, 1:nc, iv_to) + tmp
2827  else
2828  mg%boxes(id)%cc(1:nc, 1:nc, iv_to) = tmp
2829  end if
2830 
2831  end subroutine prolong_onto
2832 
2833  !> Prolong from a parent to a child with index offset dix
2834  subroutine mg_prolong_sparse(mg, p_id, dix, nc, iv, fine)
2835  type(mg_t), intent(inout) :: mg
2836  integer, intent(in) :: p_id !< Id of parent
2837  integer, intent(in) :: dix(2) !< Offset of child in parent grid
2838  integer, intent(in) :: nc !< Child grid size
2839  integer, intent(in) :: iv !< Prolong from this variable
2840  real(dp), intent(out) :: fine(nc, nc) !< Prolonged values
2841 
2842  integer :: i, j, hnc
2843  integer :: ic, jc
2844  real(dp) :: f0, flx, fhx, fly, fhy
2845 
2846  hnc = nc/2
2847 
2848  associate(crs => mg%boxes(p_id)%cc)
2849  do j = 1, hnc
2850  jc = j + dix(2)
2851  do i = 1, hnc
2852  ic = i + dix(1)
2853 
2854  f0 = 0.5_dp * crs(ic, jc, iv)
2855  flx = 0.25_dp * crs(ic-1, jc, iv)
2856  fhx = 0.25_dp * crs(ic+1, jc, iv)
2857  fly = 0.25_dp * crs(ic, jc-1, iv)
2858  fhy = 0.25_dp * crs(ic, jc+1, iv)
2859 
2860  fine(2*i-1, 2*j-1) = f0 + flx + fly
2861  fine(2*i , 2*j-1) = f0 + fhx + fly
2862  fine(2*i-1, 2*j) = f0 + flx + fhy
2863  fine(2*i , 2*j) = f0 + fhx + fhy
2864  end do
2865  end do
2866  end associate
2867  end subroutine mg_prolong_sparse
2868 
2869  !! File ../src/m_helmholtz.f90
2870 
2871  subroutine helmholtz_set_methods(mg)
2872  type(mg_t), intent(inout) :: mg
2873 
2874  mg%subtract_mean = .false.
2875 
2876  select case (mg%geometry_type)
2877  case (mg_cartesian)
2878  mg%box_op => box_helmh
2879 
2880  select case (mg%smoother_type)
2882  mg%box_smoother => box_gs_helmh
2883  case default
2884  error stop "helmholtz_set_methods: unsupported smoother type"
2885  end select
2886  case default
2887  error stop "helmholtz_set_methods: unsupported geometry"
2888  end select
2889 
2890  end subroutine helmholtz_set_methods
2891 
2892  subroutine helmholtz_set_lambda(lambda)
2893  real(dp), intent(in) :: lambda
2894 
2895  if (lambda < 0) &
2896  error stop "helmholtz_set_lambda: lambda < 0 not allowed"
2897 
2898  helmholtz_lambda = lambda
2899  end subroutine helmholtz_set_lambda
2900 
2901  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
2902  subroutine box_gs_helmh(mg, id, nc, redblack_cntr)
2903  type(mg_t), intent(inout) :: mg
2904  integer, intent(in) :: id
2905  integer, intent(in) :: nc
2906  integer, intent(in) :: redblack_cntr !< Iteration counter
2907  integer :: i, j, i0, di
2908  real(dp) :: idr2(2), fac
2909  logical :: redblack
2910 
2911  idr2 = 1/mg%dr(:, mg%boxes(id)%lvl)**2
2912  fac = 1.0_dp / (2 * sum(idr2) + helmholtz_lambda)
2913  i0 = 1
2914 
2915  redblack = (mg%smoother_type == mg_smoother_gsrb)
2916  if (redblack) then
2917  di = 2
2918  else
2919  di = 1
2920  end if
2921 
2922  ! The parity of redblack_cntr determines which cells we use. If
2923  ! redblack_cntr is even, we use the even cells and vice versa.
2924  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
2925  do j = 1, nc
2926  if (redblack) &
2927  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
2928 
2929  do i = i0, nc, di
2930  cc(i, j, n) = fac * ( &
2931  idr2(1) * (cc(i+1, j, n) + cc(i-1, j, n)) + &
2932  idr2(2) * (cc(i, j+1, n) + cc(i, j-1, n)) - &
2933  cc(i, j, mg_irhs))
2934  end do
2935  end do
2936  end associate
2937  end subroutine box_gs_helmh
2938 
2939  !> Perform Helmholtz operator on a box
2940  subroutine box_helmh(mg, id, nc, i_out)
2941  type(mg_t), intent(inout) :: mg
2942  integer, intent(in) :: id
2943  integer, intent(in) :: nc
2944  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
2945  integer :: i, j
2946  real(dp) :: idr2(2)
2947 
2948  idr2 = 1 / mg%dr(:, mg%boxes(id)%lvl)**2
2949 
2950  associate(cc => mg%boxes(id)%cc, n => mg_iphi)
2951  do j = 1, nc
2952  do i = 1, nc
2953  cc(i, j, i_out) = &
2954  idr2(1) * (cc(i-1, j, n) + cc(i+1, j, n) - 2 * cc(i, j, n)) + &
2955  idr2(2) * (cc(i, j-1, n) + cc(i, j+1, n) - 2 * cc(i, j, n)) - &
2956  helmholtz_lambda * cc(i, j, n)
2957  end do
2958  end do
2959  end associate
2960  end subroutine box_helmh
2961 
2962  !! File ../src/m_multigrid.f90
2963 
2964  subroutine mg_set_methods(mg)
2965 
2966  type(mg_t), intent(inout) :: mg
2967 
2968  ! Set default prolongation method (routines below can override this)
2969  mg%box_prolong => mg_prolong_sparse
2970 
2971  select case (mg%operator_type)
2972  case (mg_laplacian)
2973  call laplacian_set_methods(mg)
2974  case (mg_vlaplacian)
2975  call vlaplacian_set_methods(mg)
2976  case (mg_helmholtz)
2977  call helmholtz_set_methods(mg)
2978  case (mg_vhelmholtz)
2979  call vhelmholtz_set_methods(mg)
2980  case (mg_ahelmholtz)
2981  call ahelmholtz_set_methods(mg)
2982  case default
2983  error stop "mg_set_methods: unknown operator"
2984  end select
2985 
2986  ! For red-black, perform two smoothing sub-steps so that all unknowns are
2987  ! updated per cycle
2988  if (mg%smoother_type == mg_smoother_gsrb) then
2989  mg%n_smoother_substeps = 2
2990  else
2991  mg%n_smoother_substeps = 1
2992  end if
2993  end subroutine mg_set_methods
2994 
2995  subroutine check_methods(mg)
2996  type(mg_t), intent(inout) :: mg
2997 
2998  if (.not. associated(mg%box_op) .or. &
2999  .not. associated(mg%box_smoother)) then
3000  call mg_set_methods(mg)
3001  end if
3002 
3003  end subroutine check_methods
3004 
3005  subroutine mg_add_timers(mg)
3006  type(mg_t), intent(inout) :: mg
3007  timer_total_vcycle = mg_add_timer(mg, "mg total V-cycle")
3008  timer_total_fmg = mg_add_timer(mg, "mg total FMG cycle")
3009  timer_smoother = mg_add_timer(mg, "mg smoother")
3010  timer_smoother_gc = mg_add_timer(mg, "mg smoother g.c.")
3011  timer_coarse = mg_add_timer(mg, "mg coarse")
3012  timer_correct = mg_add_timer(mg, "mg correct")
3013  timer_update_coarse = mg_add_timer(mg, "mg update coarse")
3014  end subroutine mg_add_timers
3015 
3016  !> Perform FAS-FMG cycle (full approximation scheme, full multigrid).
3017  subroutine mg_fas_fmg(mg, have_guess, max_res)
3018  type(mg_t), intent(inout) :: mg
3019  logical, intent(in) :: have_guess !< If false, start from phi = 0
3020  real(dp), intent(out), optional :: max_res !< Store max(abs(residual))
3021  integer :: lvl, i, id
3022 
3023  call check_methods(mg)
3024  if (timer_smoother == -1) call mg_add_timers(mg)
3025 
3026  call mg_timer_start(mg%timers(timer_total_fmg))
3027 
3028  if (.not. have_guess) then
3029  do lvl = mg%highest_lvl, mg%lowest_lvl, -1
3030  do i = 1, size(mg%lvls(lvl)%my_ids)
3031  id = mg%lvls(lvl)%my_ids(i)
3032  mg%boxes(id)%cc(:, :, mg_iphi) = 0.0_dp
3033  end do
3034  end do
3035  end if
3036 
3037  ! Ensure ghost cells are filled correctly
3038  call mg_fill_ghost_cells_lvl(mg, mg%highest_lvl, mg_iphi)
3039 
3040  do lvl = mg%highest_lvl, mg%lowest_lvl+1, -1
3041  ! Set rhs on coarse grid and restrict phi
3042  call mg_timer_start(mg%timers(timer_update_coarse))
3043  call update_coarse(mg, lvl)
3044  call mg_timer_end(mg%timers(timer_update_coarse))
3045  end do
3046 
3047  if (mg%subtract_mean) then
3048  ! For fully periodic solutions, the mean source term has to be zero
3049  call subtract_mean(mg, mg_irhs, .false.)
3050  end if
3051 
3052  do lvl = mg%lowest_lvl, mg%highest_lvl
3053  ! Store phi_old
3054  do i = 1, size(mg%lvls(lvl)%my_ids)
3055  id = mg%lvls(lvl)%my_ids(i)
3056  mg%boxes(id)%cc(:, :, mg_iold) = &
3057  mg%boxes(id)%cc(:, :, mg_iphi)
3058  end do
3059 
3060  if (lvl > mg%lowest_lvl) then
3061  ! Correct solution at this lvl using lvl-1 data
3062  ! phi = phi + prolong(phi_coarse - phi_old_coarse)
3063  call mg_timer_start(mg%timers(timer_correct))
3064  call correct_children(mg, lvl-1)
3065  call mg_timer_end(mg%timers(timer_correct))
3066 
3067  ! Update ghost cells
3068  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3069  end if
3070 
3071  ! Perform V-cycle, possibly set residual on last iteration
3072  if (lvl == mg%highest_lvl) then
3073  call mg_fas_vcycle(mg, lvl, max_res, standalone=.false.)
3074  else
3075  call mg_fas_vcycle(mg, lvl, standalone=.false.)
3076  end if
3077  end do
3078 
3079  call mg_timer_end(mg%timers(timer_total_fmg))
3080  end subroutine mg_fas_fmg
3081 
3082  !> Perform FAS V-cycle (full approximation scheme).
3083  subroutine mg_fas_vcycle(mg, highest_lvl, max_res, standalone)
3084  use mpi
3085  type(mg_t), intent(inout) :: mg
3086  integer, intent(in), optional :: highest_lvl !< Maximum level for V-cycle
3087  real(dp), intent(out), optional :: max_res !< Store max(abs(residual))
3088  !> Whether the V-cycle is called by itself (default: true)
3089  logical, intent(in), optional :: standalone
3090  integer :: lvl, min_lvl, i, max_lvl, ierr
3091  real(dp) :: init_res, res
3092  logical :: is_standalone
3093 
3094  is_standalone = .true.
3095  if (present(standalone)) is_standalone = standalone
3096 
3097  call check_methods(mg)
3098  if (timer_smoother == -1) call mg_add_timers(mg)
3099 
3100  if (is_standalone) &
3101  call mg_timer_start(mg%timers(timer_total_vcycle))
3102 
3103  if (mg%subtract_mean .and. .not. present(highest_lvl)) then
3104  ! Assume that this is a stand-alone call. For fully periodic solutions,
3105  ! ensure the mean source term is zero.
3106  call subtract_mean(mg, mg_irhs, .false.)
3107  end if
3108 
3109  min_lvl = mg%lowest_lvl
3110  max_lvl = mg%highest_lvl
3111  if (present(highest_lvl)) max_lvl = highest_lvl
3112 
3113  ! Ensure ghost cells are filled correctly
3114  if (is_standalone) then
3115  call mg_fill_ghost_cells_lvl(mg, max_lvl, mg_iphi)
3116  end if
3117 
3118  do lvl = max_lvl, min_lvl+1, -1
3119  ! Downwards relaxation
3120  call smooth_boxes(mg, lvl, mg%n_cycle_down)
3121 
3122  ! Set rhs on coarse grid, restrict phi, and copy mg_iphi to mg_iold for the
3123  ! correction later
3124  call mg_timer_start(mg%timers(timer_update_coarse))
3125  call update_coarse(mg, lvl)
3126  call mg_timer_end(mg%timers(timer_update_coarse))
3127  end do
3128 
3129  call mg_timer_start(mg%timers(timer_coarse))
3130  if (.not. all(mg%boxes(mg%lvls(min_lvl)%ids)%rank == &
3131  mg%boxes(mg%lvls(min_lvl)%ids(1))%rank)) then
3132  error stop "Multiple CPUs for coarse grid (not implemented yet)"
3133  end if
3134 
3135  init_res = max_residual_lvl(mg, min_lvl)
3136  do i = 1, mg%max_coarse_cycles
3137  call smooth_boxes(mg, min_lvl, mg%n_cycle_up+mg%n_cycle_down)
3138  res = max_residual_lvl(mg, min_lvl)
3139  if (res < mg%residual_coarse_rel * init_res .or. &
3140  res < mg%residual_coarse_abs) exit
3141  end do
3142  call mg_timer_end(mg%timers(timer_coarse))
3143 
3144  ! Do the upwards part of the v-cycle in the tree
3145  do lvl = min_lvl+1, max_lvl
3146  ! Correct solution at this lvl using lvl-1 data
3147  ! phi = phi + prolong(phi_coarse - phi_old_coarse)
3148  call mg_timer_start(mg%timers(timer_correct))
3149  call correct_children(mg, lvl-1)
3150 
3151  ! Have to fill ghost cells after correction
3152  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3153  call mg_timer_end(mg%timers(timer_correct))
3154 
3155  ! Upwards relaxation
3156  call smooth_boxes(mg, lvl, mg%n_cycle_up)
3157  end do
3158 
3159  if (present(max_res)) then
3160  init_res = 0.0_dp
3161  do lvl = min_lvl, max_lvl
3162  res = max_residual_lvl(mg, lvl)
3163  init_res = max(res, init_res)
3164  end do
3165  call mpi_allreduce(init_res, max_res, 1, &
3166  mpi_double, mpi_max, mg%comm, ierr)
3167  end if
3168 
3169  ! Subtract mean(phi) from phi
3170  if (mg%subtract_mean) then
3171  call subtract_mean(mg, mg_iphi, .true.)
3172  end if
3173 
3174  if (is_standalone) &
3175  call mg_timer_end(mg%timers(timer_total_vcycle))
3176  end subroutine mg_fas_vcycle
3177 
3178  subroutine subtract_mean(mg, iv, include_ghostcells)
3179  use mpi
3180  type(mg_t), intent(inout) :: mg
3181  integer, intent(in) :: iv
3182  logical, intent(in) :: include_ghostcells
3183  integer :: i, id, lvl, nc, ierr
3184  real(dp) :: sum_iv, mean_iv, volume
3185 
3186  nc = mg%box_size
3187  sum_iv = get_sum(mg, iv)
3188  call mpi_allreduce(sum_iv, mean_iv, 1, &
3189  mpi_double, mpi_sum, mg%comm, ierr)
3190 
3191  ! Divide by total grid volume to get mean
3192  volume = nc**2 * product(mg%dr(:, 1)) * size(mg%lvls(1)%ids)
3193  mean_iv = mean_iv / volume
3194 
3195  do lvl = mg%lowest_lvl, mg%highest_lvl
3196  nc = mg%box_size_lvl(lvl)
3197 
3198  do i = 1, size(mg%lvls(lvl)%my_ids)
3199  id = mg%lvls(lvl)%my_ids(i)
3200  if (include_ghostcells) then
3201  mg%boxes(id)%cc(:, :, iv) = &
3202  mg%boxes(id)%cc(:, :, iv) - mean_iv
3203  else
3204  mg%boxes(id)%cc(1:nc, 1:nc, iv) = &
3205  mg%boxes(id)%cc(1:nc, 1:nc, iv) - mean_iv
3206  end if
3207  end do
3208  end do
3209  end subroutine subtract_mean
3210 
3211  real(dp) function get_sum(mg, iv)
3212  type(mg_t), intent(in) :: mg
3213  integer, intent(in) :: iv
3214  integer :: lvl, i, id, nc
3215  real(dp) :: w
3216 
3217  get_sum = 0.0_dp
3218  do lvl = 1, mg%highest_lvl
3219  nc = mg%box_size_lvl(lvl)
3220  w = product(mg%dr(:, lvl)) ! Adjust for non-Cartesian cases
3221  do i = 1, size(mg%lvls(lvl)%my_leaves)
3222  id = mg%lvls(lvl)%my_leaves(i)
3223  get_sum = get_sum + w * &
3224  sum(mg%boxes(id)%cc(1:nc, 1:nc, iv))
3225  end do
3226  end do
3227  end function get_sum
3228 
3229  real(dp) function max_residual_lvl(mg, lvl)
3230  type(mg_t), intent(inout) :: mg
3231  integer, intent(in) :: lvl
3232  integer :: i, id, nc
3233  real(dp) :: res
3234 
3235  nc = mg%box_size_lvl(lvl)
3236  max_residual_lvl = 0.0_dp
3237 
3238  do i = 1, size(mg%lvls(lvl)%my_ids)
3239  id = mg%lvls(lvl)%my_ids(i)
3240  call residual_box(mg, id, nc)
3241  res = maxval(abs(mg%boxes(id)%cc(1:nc, 1:nc, mg_ires)))
3242  max_residual_lvl = max(max_residual_lvl, res)
3243  end do
3244  end function max_residual_lvl
3245 
3246  ! subroutine solve_coarse_grid(mg)
3247  ! use m_fishpack
3248  ! type(mg_t), intent(inout) :: mg
3249 
3250  ! real(dp) :: rhs(mg%box_size, mg%box_size)
3251  ! real(dp) :: rmin(2), rmax(2)
3252  ! integer :: nc, nx(2), my_boxes, total_boxes
3253 
3254  ! my_boxes = size(mg%lvls(1)%my_ids)
3255  ! total_boxes = size(mg%lvls(1)%ids)
3256  ! nc = mg%box_size
3257 
3258  ! if (my_boxes == total_boxes) then
3259  ! nx(:) = nc
3260  ! rmin = [0.0_dp, 0.0_dp]
3261  ! rmax = mg%dr(1) * [nc, nc]
3262  ! rhs = mg%boxes(1)%cc(1:nc, 1:nc, mg_irhs)
3263 
3264  ! #if 2 == 2
3265  ! call fishpack_2d(nx, rhs, mg%bc, rmin, rmax)
3266  ! #elif 2 == 3
3267  ! call fishpack_3d(nx, rhs, mg%bc, rmin, rmax)
3268  ! #endif
3269 
3270  ! mg%boxes(1)%cc(1:nc, 1:nc, mg_iphi) = rhs
3271  ! else if (my_boxes > 0) then
3272  ! error stop "Boxes at level 1 at different processors"
3273  ! end if
3274 
3275  ! call fill_ghost_cells_lvl(mg, 1)
3276  ! end subroutine solve_coarse_grid
3277 
3278  ! Set rhs on coarse grid, restrict phi, and copy mg_iphi to mg_iold for the
3279  ! correction later
3280  subroutine update_coarse(mg, lvl)
3281  type(mg_t), intent(inout) :: mg !< Tree containing full grid
3282  integer, intent(in) :: lvl !< Update coarse values at lvl-1
3283  integer :: i, id, nc, nc_c
3284 
3285  nc = mg%box_size_lvl(lvl)
3286  nc_c = mg%box_size_lvl(lvl-1)
3287 
3288  ! Compute residual
3289  do i = 1, size(mg%lvls(lvl)%my_ids)
3290  id = mg%lvls(lvl)%my_ids(i)
3291  call residual_box(mg, id, nc)
3292  end do
3293 
3294  ! Restrict phi and the residual
3295  call mg_restrict_lvl(mg, mg_iphi, lvl)
3296  call mg_restrict_lvl(mg, mg_ires, lvl)
3297 
3298  call mg_fill_ghost_cells_lvl(mg, lvl-1, mg_iphi)
3299 
3300  ! Set rhs_c = laplacian(phi_c) + restrict(res) where it is refined, and
3301  ! store current coarse phi in old.
3302  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3303  id = mg%lvls(lvl-1)%my_parents(i)
3304 
3305  ! Set rhs = L phi
3306  call mg%box_op(mg, id, nc_c, mg_irhs)
3307 
3308  ! Add the fine grid residual to rhs
3309  mg%boxes(id)%cc(1:nc_c, 1:nc_c, mg_irhs) = &
3310  mg%boxes(id)%cc(1:nc_c, 1:nc_c, mg_irhs) + &
3311  mg%boxes(id)%cc(1:nc_c, 1:nc_c, mg_ires)
3312 
3313  ! Story a copy of phi
3314  mg%boxes(id)%cc(:, :, mg_iold) = &
3315  mg%boxes(id)%cc(:, :, mg_iphi)
3316  end do
3317  end subroutine update_coarse
3318 
3319  ! Sets phi = phi + prolong(phi_coarse - phi_old_coarse)
3320  subroutine correct_children(mg, lvl)
3321  type(mg_t), intent(inout) :: mg
3322  integer, intent(in) :: lvl
3323  integer :: i, id
3324 
3325  do i = 1, size(mg%lvls(lvl)%my_parents)
3326  id = mg%lvls(lvl)%my_parents(i)
3327 
3328  ! Store the correction in mg_ires
3329  mg%boxes(id)%cc(:, :, mg_ires) = &
3330  mg%boxes(id)%cc(:, :, mg_iphi) - &
3331  mg%boxes(id)%cc(:, :, mg_iold)
3332  end do
3333 
3334  call mg_prolong(mg, lvl, mg_ires, mg_iphi, mg%box_prolong, add=.true.)
3335  end subroutine correct_children
3336 
3337  subroutine smooth_boxes(mg, lvl, n_cycle)
3338  type(mg_t), intent(inout) :: mg
3339  integer, intent(in) :: lvl
3340  integer, intent(in) :: n_cycle !< Number of cycles to perform
3341  integer :: n, i, id, nc
3342 
3343  nc = mg%box_size_lvl(lvl)
3344 
3345  do n = 1, n_cycle * mg%n_smoother_substeps
3346  call mg_timer_start(mg%timers(timer_smoother))
3347  do i = 1, size(mg%lvls(lvl)%my_ids)
3348  id = mg%lvls(lvl)%my_ids(i)
3349  call mg%box_smoother(mg, id, nc, n)
3350  end do
3351  call mg_timer_end(mg%timers(timer_smoother))
3352 
3353  call mg_timer_start(mg%timers(timer_smoother_gc))
3354  call mg_fill_ghost_cells_lvl(mg, lvl, mg_iphi)
3355  call mg_timer_end(mg%timers(timer_smoother_gc))
3356  end do
3357  end subroutine smooth_boxes
3358 
3359  subroutine residual_box(mg, id, nc)
3360  type(mg_t), intent(inout) :: mg
3361  integer, intent(in) :: id
3362  integer, intent(in) :: nc
3363 
3364  call mg%box_op(mg, id, nc, mg_ires)
3365 
3366  mg%boxes(id)%cc(1:nc, 1:nc, mg_ires) = &
3367  mg%boxes(id)%cc(1:nc, 1:nc, mg_irhs) &
3368  - mg%boxes(id)%cc(1:nc, 1:nc, mg_ires)
3369  end subroutine residual_box
3370 
3371  !> Apply operator to the tree and store in variable i_out
3372  subroutine mg_apply_op(mg, i_out, op)
3373  type(mg_t), intent(inout) :: mg
3374  integer, intent(in) :: i_out
3375  procedure(mg_box_op), optional :: op
3376  integer :: lvl, i, id, nc
3377 
3378  do lvl = mg%lowest_lvl, mg%highest_lvl
3379  nc = mg%box_size_lvl(lvl)
3380  do i = 1, size(mg%lvls(lvl)%my_ids)
3381  id = mg%lvls(lvl)%my_ids(i)
3382  if (present(op)) then
3383  call op(mg, id, nc, i_out)
3384  else
3385  call mg%box_op(mg, id, nc, i_out)
3386  end if
3387  end do
3388  end do
3389  end subroutine mg_apply_op
3390 
3391  !! File ../src/m_restrict.f90
3392 
3393  !> Specify minimum buffer size (per process) for communication
3394  subroutine mg_restrict_buffer_size(mg, n_send, n_recv, dsize)
3395  type(mg_t), intent(inout) :: mg
3396  integer, intent(out) :: n_send(0:mg%n_cpu-1)
3397  integer, intent(out) :: n_recv(0:mg%n_cpu-1)
3398  integer, intent(out) :: dsize
3399  integer :: n_out(0:mg%n_cpu-1, mg%first_normal_lvl:mg%highest_lvl)
3400  integer :: n_in(0:mg%n_cpu-1, mg%first_normal_lvl:mg%highest_lvl)
3401  integer :: lvl, i, id, p_id, p_rank
3402  integer :: i_c, c_id, c_rank, min_lvl
3403 
3404  n_out(:, :) = 0
3405  n_in(:, :) = 0
3406  min_lvl = max(mg%lowest_lvl+1, mg%first_normal_lvl)
3407 
3408  do lvl = min_lvl, mg%highest_lvl
3409  ! Number of messages to receive (at lvl-1)
3410  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3411  id = mg%lvls(lvl-1)%my_parents(i)
3412  do i_c = 1, mg_num_children
3413  c_id = mg%boxes(id)%children(i_c)
3414 
3415  if (c_id > mg_no_box) then
3416  c_rank = mg%boxes(c_id)%rank
3417  if (c_rank /= mg%my_rank) then
3418  n_in(c_rank, lvl) = n_in(c_rank, lvl) + 1
3419  end if
3420  end if
3421  end do
3422  end do
3423 
3424  ! Number of messages to send
3425  do i = 1, size(mg%lvls(lvl)%my_ids)
3426  id = mg%lvls(lvl)%my_ids(i)
3427 
3428  p_id = mg%boxes(id)%parent
3429  p_rank = mg%boxes(p_id)%rank
3430  if (p_rank /= mg%my_rank) then
3431  n_out(p_rank, lvl) = n_out(p_rank, lvl) + 1
3432  end if
3433 
3434  end do
3435  end do
3436 
3437  allocate(mg%comm_restrict%n_send(0:mg%n_cpu-1, &
3438  mg%first_normal_lvl:mg%highest_lvl))
3439  allocate(mg%comm_restrict%n_recv(0:mg%n_cpu-1, &
3440  mg%first_normal_lvl:mg%highest_lvl))
3441  mg%comm_restrict%n_send = n_out
3442  mg%comm_restrict%n_recv = n_in
3443 
3444  dsize = (mg%box_size/2)**2
3445  n_send = maxval(n_out, dim=2)
3446  n_recv = maxval(n_in, dim=2)
3447  end subroutine mg_restrict_buffer_size
3448 
3449  !> Restrict all levels
3450  subroutine mg_restrict(mg, iv)
3451  type(mg_t), intent(inout) :: mg
3452  integer, intent(in) :: iv
3453  integer :: lvl
3454 
3455  do lvl = mg%highest_lvl, mg%lowest_lvl+1, -1
3456  call mg_restrict_lvl(mg, iv, lvl)
3457  end do
3458  end subroutine mg_restrict
3459 
3460  !> Restrict from lvl to lvl-1
3461  subroutine mg_restrict_lvl(mg, iv, lvl)
3462 
3463  type(mg_t), intent(inout) :: mg
3464  integer, intent(in) :: iv
3465  integer, intent(in) :: lvl
3466  integer :: i, id, dsize, nc
3467 
3468  if (lvl <= mg%lowest_lvl) error stop "cannot restrict lvl <= lowest_lvl"
3469 
3470  nc = mg%box_size_lvl(lvl)
3471 
3472  if (lvl >= mg%first_normal_lvl) then
3473  dsize = (nc/2)**2
3474 
3475  mg%buf(:)%i_send = 0
3476  mg%buf(:)%i_ix = 0
3477 
3478  do i = 1, size(mg%lvls(lvl)%my_ids)
3479  id = mg%lvls(lvl)%my_ids(i)
3480  call restrict_set_buffer(mg, id, iv)
3481  end do
3482 
3483  mg%buf(:)%i_recv = mg%comm_restrict%n_recv(:, lvl) * dsize
3484  call sort_and_transfer_buffers(mg, dsize)
3485  mg%buf(:)%i_recv = 0
3486  end if
3487 
3488  do i = 1, size(mg%lvls(lvl-1)%my_parents)
3489  id = mg%lvls(lvl-1)%my_parents(i)
3490  call restrict_onto(mg, id, nc, iv)
3491  end do
3492  end subroutine mg_restrict_lvl
3493 
3494  subroutine restrict_set_buffer(mg, id, iv)
3495  type(mg_t), intent(inout) :: mg
3496  integer, intent(in) :: id
3497  integer, intent(in) :: iv
3498  integer :: i, j, n, hnc, p_id, p_rank
3499  real(dp) :: tmp(mg%box_size/2, mg%box_size/2)
3500 
3501  hnc = mg%box_size/2
3502  p_id = mg%boxes(id)%parent
3503  p_rank = mg%boxes(p_id)%rank
3504 
3505  if (p_rank /= mg%my_rank) then
3506  do j = 1, hnc
3507  do i = 1, hnc
3508  tmp(i, j) = 0.25_dp * &
3509  sum(mg%boxes(id)%cc(2*i-1:2*i, 2*j-1:2*j, iv))
3510  end do
3511  end do
3512 
3513  ! Buffer
3514  n = size(tmp)
3515  i = mg%buf(p_rank)%i_send
3516  mg%buf(p_rank)%send(i+1:i+n) = pack(tmp, .true.)
3517  mg%buf(p_rank)%i_send = mg%buf(p_rank)%i_send + n
3518 
3519  ! To later sort the send buffer according to parent order
3520  i = mg%buf(p_rank)%i_ix
3521  n = mg_ix_to_ichild(mg%boxes(id)%ix)
3522  mg%buf(p_rank)%ix(i+1) = mg_num_children * p_id + n
3523  mg%buf(p_rank)%i_ix = mg%buf(p_rank)%i_ix + 1
3524  end if
3525  end subroutine restrict_set_buffer
3526 
3527  subroutine restrict_onto(mg, id, nc, iv)
3528  type(mg_t), intent(inout) :: mg
3529  integer, intent(in) :: id
3530  integer, intent(in) :: nc
3531  integer, intent(in) :: iv
3532  integer :: i, j, hnc, dsize, i_c, c_id
3533  integer :: c_rank, dix(2)
3534 
3535  hnc = nc/2
3536  dsize = hnc**2
3537 
3538  do i_c = 1, mg_num_children
3539  c_id = mg%boxes(id)%children(i_c)
3540  if (c_id == mg_no_box) cycle ! For coarsened grid
3541  c_rank = mg%boxes(c_id)%rank
3542  dix = mg_get_child_offset(mg, c_id)
3543 
3544  if (c_rank == mg%my_rank) then
3545  do j=1, hnc; do i=1, hnc
3546  mg%boxes(id)%cc(dix(1)+i, dix(2)+j, iv) = 0.25_dp * &
3547  sum(mg%boxes(c_id)%cc(2*i-1:2*i, 2*j-1:2*j, iv))
3548  end do; end do
3549  else
3550  i = mg%buf(c_rank)%i_recv
3551  mg%boxes(id)%cc(dix(1)+1:dix(1)+hnc, &
3552  dix(2)+1:dix(2)+hnc, iv) = &
3553  reshape(mg%buf(c_rank)%recv(i+1:i+dsize), [hnc, hnc])
3554  mg%buf(c_rank)%i_recv = mg%buf(c_rank)%i_recv + dsize
3555  end if
3556  end do
3557 
3558  end subroutine restrict_onto
3559 
3560  !! File ../src/m_mrgrnk.f90
3561 
3562  Subroutine i_mrgrnk (XDONT, IRNGT)
3563  ! __________________________________________________________
3564  ! MRGRNK = Merge-sort ranking of an array
3565  ! For performance reasons, the first 2 passes are taken
3566  ! out of the standard loop, and use dedicated coding.
3567  ! __________________________________________________________
3568  ! __________________________________________________________
3569  Integer, Dimension (:), Intent (In) :: xdont
3570  Integer, Dimension (:), Intent (Out) :: irngt
3571  ! __________________________________________________________
3572  Integer :: xvala, xvalb
3573  !
3574  Integer, Dimension (SIZE(IRNGT)) :: jwrkt
3575  Integer :: lmtna, lmtnc, irng1, irng2
3576  Integer :: nval, iind, iwrkd, iwrk, iwrkf, jinda, iinda, iindb
3577  !
3578  nval = min(SIZE(xdont), SIZE(irngt))
3579  Select Case (nval)
3580  Case (:0)
3581  Return
3582  Case (1)
3583  irngt(1) = 1
3584  Return
3585  Case Default
3586  Continue
3587  End Select
3588  !
3589  ! Fill-in the index array, creating ordered couples
3590  !
3591  Do iind = 2, nval, 2
3592  If (xdont(iind-1) <= xdont(iind)) Then
3593  irngt(iind-1) = iind - 1
3594  irngt(iind) = iind
3595  Else
3596  irngt(iind-1) = iind
3597  irngt(iind) = iind - 1
3598  End If
3599  End Do
3600  If (modulo(nval, 2) /= 0) Then
3601  irngt(nval) = nval
3602  End If
3603  !
3604  ! We will now have ordered subsets A - B - A - B - ...
3605  ! and merge A and B couples into C - C - ...
3606  !
3607  lmtna = 2
3608  lmtnc = 4
3609  !
3610  ! First iteration. The length of the ordered subsets goes from 2 to 4
3611  !
3612  Do
3613  If (nval <= 2) Exit
3614  !
3615  ! Loop on merges of A and B into C
3616  !
3617  Do iwrkd = 0, nval - 1, 4
3618  If ((iwrkd+4) > nval) Then
3619  If ((iwrkd+2) >= nval) Exit
3620  !
3621  ! 1 2 3
3622  !
3623  If (xdont(irngt(iwrkd+2)) <= xdont(irngt(iwrkd+3))) Exit
3624  !
3625  ! 1 3 2
3626  !
3627  If (xdont(irngt(iwrkd+1)) <= xdont(irngt(iwrkd+3))) Then
3628  irng2 = irngt(iwrkd+2)
3629  irngt(iwrkd+2) = irngt(iwrkd+3)
3630  irngt(iwrkd+3) = irng2
3631  !
3632  ! 3 1 2
3633  !
3634  Else
3635  irng1 = irngt(iwrkd+1)
3636  irngt(iwrkd+1) = irngt(iwrkd+3)
3637  irngt(iwrkd+3) = irngt(iwrkd+2)
3638  irngt(iwrkd+2) = irng1
3639  End If
3640  Exit
3641  End If
3642  !
3643  ! 1 2 3 4
3644  !
3645  If (xdont(irngt(iwrkd+2)) <= xdont(irngt(iwrkd+3))) cycle
3646  !
3647  ! 1 3 x x
3648  !
3649  If (xdont(irngt(iwrkd+1)) <= xdont(irngt(iwrkd+3))) Then
3650  irng2 = irngt(iwrkd+2)
3651  irngt(iwrkd+2) = irngt(iwrkd+3)
3652  If (xdont(irng2) <= xdont(irngt(iwrkd+4))) Then
3653  ! 1 3 2 4
3654  irngt(iwrkd+3) = irng2
3655  Else
3656  ! 1 3 4 2
3657  irngt(iwrkd+3) = irngt(iwrkd+4)
3658  irngt(iwrkd+4) = irng2
3659  End If
3660  !
3661  ! 3 x x x
3662  !
3663  Else
3664  irng1 = irngt(iwrkd+1)
3665  irng2 = irngt(iwrkd+2)
3666  irngt(iwrkd+1) = irngt(iwrkd+3)
3667  If (xdont(irng1) <= xdont(irngt(iwrkd+4))) Then
3668  irngt(iwrkd+2) = irng1
3669  If (xdont(irng2) <= xdont(irngt(iwrkd+4))) Then
3670  ! 3 1 2 4
3671  irngt(iwrkd+3) = irng2
3672  Else
3673  ! 3 1 4 2
3674  irngt(iwrkd+3) = irngt(iwrkd+4)
3675  irngt(iwrkd+4) = irng2
3676  End If
3677  Else
3678  ! 3 4 1 2
3679  irngt(iwrkd+2) = irngt(iwrkd+4)
3680  irngt(iwrkd+3) = irng1
3681  irngt(iwrkd+4) = irng2
3682  End If
3683  End If
3684  End Do
3685  !
3686  ! The Cs become As and Bs
3687  !
3688  lmtna = 4
3689  Exit
3690  End Do
3691  !
3692  ! Iteration loop. Each time, the length of the ordered subsets
3693  ! is doubled.
3694  !
3695  Do
3696  If (lmtna >= nval) Exit
3697  iwrkf = 0
3698  lmtnc = 2 * lmtnc
3699  !
3700  ! Loop on merges of A and B into C
3701  !
3702  Do
3703  iwrk = iwrkf
3704  iwrkd = iwrkf + 1
3705  jinda = iwrkf + lmtna
3706  iwrkf = iwrkf + lmtnc
3707  If (iwrkf >= nval) Then
3708  If (jinda >= nval) Exit
3709  iwrkf = nval
3710  End If
3711  iinda = 1
3712  iindb = jinda + 1
3713  !
3714  ! Shortcut for the case when the max of A is smaller
3715  ! than the min of B. This line may be activated when the
3716  ! initial set is already close to sorted.
3717  !
3718  ! IF (XDONT(IRNGT(JINDA)) <= XDONT(IRNGT(IINDB))) CYCLE
3719  !
3720  ! One steps in the C subset, that we build in the final rank array
3721  !
3722  ! Make a copy of the rank array for the merge iteration
3723  !
3724  jwrkt(1:lmtna) = irngt(iwrkd:jinda)
3725  !
3726  xvala = xdont(jwrkt(iinda))
3727  xvalb = xdont(irngt(iindb))
3728  !
3729  Do
3730  iwrk = iwrk + 1
3731  !
3732  ! We still have unprocessed values in both A and B
3733  !
3734  If (xvala > xvalb) Then
3735  irngt(iwrk) = irngt(iindb)
3736  iindb = iindb + 1
3737  If (iindb > iwrkf) Then
3738  ! Only A still with unprocessed values
3739  irngt(iwrk+1:iwrkf) = jwrkt(iinda:lmtna)
3740  Exit
3741  End If
3742  xvalb = xdont(irngt(iindb))
3743  Else
3744  irngt(iwrk) = jwrkt(iinda)
3745  iinda = iinda + 1
3746  If (iinda > lmtna) exit! Only B still with unprocessed values
3747  xvala = xdont(jwrkt(iinda))
3748  End If
3749  !
3750  End Do
3751  End Do
3752  !
3753  ! The Cs become As and Bs
3754  !
3755  lmtna = 2 * lmtna
3756  End Do
3757  !
3758  Return
3759  !
3760  End Subroutine i_mrgrnk
3761 
3762  !! File ../src/m_ahelmholtz.f90
3763 
3764  subroutine ahelmholtz_set_methods(mg)
3765  type(mg_t), intent(inout) :: mg
3766 
3767  if (mg%n_extra_vars == 0 .and. mg%is_allocated) then
3768  error stop "ahelmholtz_set_methods: mg%n_extra_vars == 0"
3769  else
3770  mg%n_extra_vars = max(2, mg%n_extra_vars)
3771  end if
3772 
3773  ! Use Neumann zero boundary conditions for the variable coefficient, since
3774  ! it is needed in ghost cells.
3775  mg%bc(:, mg_iveps1)%bc_type = mg_bc_neumann
3776  mg%bc(:, mg_iveps1)%bc_value = 0.0_dp
3777 
3778  mg%bc(:, mg_iveps2)%bc_type = mg_bc_neumann
3779  mg%bc(:, mg_iveps2)%bc_value = 0.0_dp
3780 
3781 
3782  select case (mg%geometry_type)
3783  case (mg_cartesian)
3784  mg%box_op => box_ahelmh
3785 
3786  select case (mg%smoother_type)
3788  mg%box_smoother => box_gs_ahelmh
3789  case default
3790  error stop "ahelmholtz_set_methods: unsupported smoother type"
3791  end select
3792  case default
3793  error stop "ahelmholtz_set_methods: unsupported geometry"
3794  end select
3795 
3796  end subroutine ahelmholtz_set_methods
3797 
3798  subroutine ahelmholtz_set_lambda(lambda)
3799  real(dp), intent(in) :: lambda
3800 
3801  if (lambda < 0) &
3802  error stop "ahelmholtz_set_lambda: lambda < 0 not allowed"
3803 
3804  ahelmholtz_lambda = lambda
3805  end subroutine ahelmholtz_set_lambda
3806 
3807  !> Perform Gauss-Seidel relaxation on box for a Helmholtz operator
3808  subroutine box_gs_ahelmh(mg, id, nc, redblack_cntr)
3809  type(mg_t), intent(inout) :: mg
3810  integer, intent(in) :: id
3811  integer, intent(in) :: nc
3812  integer, intent(in) :: redblack_cntr !< Iteration counter
3813  integer :: i, j, i0, di
3814  logical :: redblack
3815  real(dp) :: idr2(2*2), u(2*2)
3816  real(dp) :: a0(2*2), a(2*2), c(2*2)
3817 
3818  ! Duplicate 1/dr^2 array to multiply neighbor values
3819  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
3820  idr2(2:2*2:2) = idr2(1:2*2:2)
3821  i0 = 1
3822 
3823  redblack = (mg%smoother_type == mg_smoother_gsrb)
3824  if (redblack) then
3825  di = 2
3826  else
3827  di = 1
3828  end if
3829 
3830  ! The parity of redblack_cntr determines which cells we use. If
3831  ! redblack_cntr is even, we use the even cells and vice versa.
3832  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
3833  i_eps1 => mg_iveps1, &
3834  i_eps2 => mg_iveps2)
3835 
3836  do j = 1, nc
3837  if (redblack) &
3838  i0 = 2 - iand(ieor(redblack_cntr, j), 1)
3839 
3840  do i = i0, nc, di
3841  a0(1:2) = cc(i, j, i_eps1)
3842  a0(3:4) = cc(i, j, i_eps2)
3843  u(1:2) = cc(i-1:i+1:2, j, n)
3844  a(1:2) = cc(i-1:i+1:2, j, i_eps1)
3845  u(3:4) = cc(i, j-1:j+1:2, n)
3846  a(3:4) = cc(i, j-1:j+1:2, i_eps2)
3847  c(:) = 2 * a0(:) * a(:) / (a0(:) + a(:)) * idr2
3848 
3849  cc(i, j, n) = &
3850  (sum(c(:) * u(:)) - cc(i, j, mg_irhs)) / &
3851  (sum(c(:)) + ahelmholtz_lambda)
3852  end do
3853  end do
3854  end associate
3855  end subroutine box_gs_ahelmh
3856 
3857  !> Perform Helmholtz operator on a box
3858  subroutine box_ahelmh(mg, id, nc, i_out)
3859  type(mg_t), intent(inout) :: mg
3860  integer, intent(in) :: id
3861  integer, intent(in) :: nc
3862  integer, intent(in) :: i_out !< Index of variable to store Helmholtz in
3863  integer :: i, j
3864  real(dp) :: idr2(2*2), a0(2*2), u0, u(2*2), a(2*2)
3865 
3866  ! Duplicate 1/dr^2 array to multiply neighbor values
3867  idr2(1:2*2:2) = 1/mg%dr(:, mg%boxes(id)%lvl)**2
3868  idr2(2:2*2:2) = idr2(1:2*2:2)
3869 
3870  associate(cc => mg%boxes(id)%cc, n => mg_iphi, &
3871  i_eps1 => mg_iveps1, &
3872  i_eps2 => mg_iveps2)
3873  do j = 1, nc
3874  do i = 1, nc
3875  a0(1:2) = cc(i, j, i_eps1)
3876  a0(3:4) = cc(i, j, i_eps2)
3877  a(1:2) = cc(i-1:i+1:2, j, i_eps1)
3878  a(3:4) = cc(i, j-1:j+1:2, i_eps2)
3879  u0 = cc(i, j, n)
3880  u(1:2) = cc(i-1:i+1:2, j, n)
3881  u(3:4) = cc(i, j-1:j+1:2, n)
3882 
3883  cc(i, j, i_out) = sum(2 * idr2 * &
3884  a0(:)*a(:)/(a0(:) + a(:)) * (u(:) - u0)) - &
3885  ahelmholtz_lambda * u0
3886  end do
3887  end do
3888  end associate
3889  end subroutine box_ahelmh
3890 
3891 end module m_octree_mg_2d
To fill ghost cells near physical boundaries.
subroutine, public mg_comm_init(mg, comm)
Prolong from a parent to a child with index offset dix. This method could sometimes work better than ...
subroutine, public mg_fas_vcycle(mg, highest_lvl, max_res, standalone)
Perform FAS V-cycle (full approximation scheme).
subroutine, public vhelmholtz_set_methods(mg)
subroutine, public mg_ghost_cell_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
integer, parameter, public mg_num_children
subroutine, public mg_timer_end(timer)
subroutine, public mg_restrict(mg, iv)
Restrict all levels.
integer, parameter, public mg_vhelmholtz
Indicates a variable-coefficient Helmholtz equation.
integer, parameter, public mg_iveps1
Indexes of anisotropic variable coefficients.
integer, parameter, public mg_ahelmholtz
Indicates a anisotropic-coefficient Helmholtz equation.
integer, parameter, public mg_vlaplacian
Indicates a variable-coefficient Laplacian.
integer, dimension(2, 4), parameter, public mg_neighb_dix
integer, parameter, public mg_helmholtz
Indicates a constant-coefficient Helmholtz equation.
integer, parameter, public mg_physical_boundary
Special value that indicates there is a physical boundary.
subroutine, public mg_prolong_sparse(mg, p_id, dix, nc, iv, fine)
Prolong from a parent to a child with index offset dix.
subroutine, public mg_get_face_coords(box, nb, nc, x)
Get coordinates at the face of a box.
subroutine, public sort_and_transfer_buffers(mg, dsize)
integer, parameter, public dp
Type of reals.
pure integer function, public mg_highest_uniform_lvl(mg)
integer function, public mg_ix_to_ichild(ix)
Compute the child index for a box with spatial index ix. With child index we mean the index in the ch...
integer, parameter, public mg_ndim
Problem dimension.
subroutine, public mg_prolong_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
subroutine, public diffusion_solve_acoeff(mg, dt, order, max_res)
Solve a diffusion equation implicitly, assuming anisotropic diffusion coefficient which has been stor...
subroutine, public mg_fas_fmg(mg, have_guess, max_res)
Perform FAS-FMG cycle (full approximation scheme, full multigrid).
integer, parameter, public mg_ires
Index of residual.
integer, parameter, public mg_neighb_highy
subroutine, public mg_set_neighbors_lvl(mg, lvl)
integer, parameter, public mg_iveps3
subroutine, public mg_load_balance_simple(mg)
Load balance all boxes in the multigrid tree, by simply distributing the load per grid level....
subroutine, public helmholtz_set_methods(mg)
integer, parameter, public mg_bc_continuous
Value to indicate a continuous boundary condition.
subroutine, public mg_timers_show(mg)
integer, parameter, public mg_iphi
Index of solution.
subroutine, public mg_add_children(mg, id)
subroutine, public mg_prolong(mg, lvl, iv, iv_to, method, add)
Prolong variable iv from lvl to variable iv_to at lvl+1.
integer, dimension(4), parameter, public mg_neighb_dim
integer, dimension(2, 4), parameter, public mg_child_dix
integer, parameter, public mg_spherical
Spherical coordinate system.
subroutine, public mg_timer_start(timer)
integer(i8) function, public mg_number_of_unknowns(mg)
Determine total number of unknowns (on leaves)
integer, parameter, public mg_neighb_highx
integer, parameter, public mg_bc_dirichlet
Value to indicate a Dirichlet boundary condition.
subroutine, public helmholtz_set_lambda(lambda)
integer, parameter, public mg_laplacian
Indicates a standard Laplacian.
integer, dimension(4), parameter, public mg_neighb_rev
subroutine, public ahelmholtz_set_lambda(lambda)
real(dp), public, protected ahelmholtz_lambda
Module which contains multigrid procedures for a Helmholtz operator of the form: div(D grad(phi)) - l...
integer, parameter, public mg_lvl_hi
Maximum allowed grid level.
integer, dimension(4), parameter, public mg_neighb_high_pm
integer, parameter, public mg_bc_neumann
Value to indicate a Neumann boundary condition.
integer, parameter, public mg_irhs
Index of right-hand side.
logical, dimension(2, 4), parameter, public mg_child_low
integer, parameter, public mg_iveps2
elemental logical function, public mg_has_children(box)
Return .true. if a box has children.
subroutine sort_sendbuf(gc, dsize)
Sort send buffers according to the idbuf array.
subroutine, public diffusion_solve_vcoeff(mg, dt, order, max_res)
Solve a diffusion equation implicitly, assuming a variable diffusion coefficient which has been store...
integer, parameter, public mg_cylindrical
Cylindrical coordinate system.
real(dp), public, protected vhelmholtz_lambda
Module for implicitly solving diffusion equations.
subroutine, public mg_restrict_buffer_size(mg, n_send, n_recv, dsize)
Specify minimum buffer size (per process) for communication.
integer, parameter, public mg_smoother_jacobi
subroutine, public laplacian_set_methods(mg)
subroutine, public mg_deallocate_storage(mg)
Deallocate all allocatable arrays.
subroutine, public ahelmholtz_set_methods(mg)
subroutine, public mg_load_balance(mg)
Load balance all boxes in the multigrid tree. Compared to mg_load_balance_simple, this method does a ...
subroutine, public mg_fill_ghost_cells_lvl(mg, lvl, iv)
Fill ghost cells at a grid level.
subroutine, public vhelmholtz_set_lambda(lambda)
subroutine, public mg_set_next_level_ids(mg, lvl)
integer, parameter, public mg_num_neighbors
integer, parameter, public mg_cartesian
Cartesian coordinate system.
integer, dimension(2, 4), parameter, public mg_child_adj_nb
subroutine, public mg_set_refinement_boundaries(boxes, level)
Create a list of refinement boundaries (from the coarse side)
integer, parameter, public mg_neighb_lowx
integer, parameter, public mg_neighb_lowy
integer, parameter, public mg_num_vars
Number of predefined multigrid variables.
subroutine subtract_mean(mg, iv, include_ghostcells)
integer, parameter, public mg_lvl_lo
Minimum allowed grid level.
subroutine, public mg_set_leaves_parents(boxes, level)
Create a list of leaves and a list of parents for a level.
subroutine, public mg_load_balance_parents(mg)
Load balance the parents (non-leafs). Assign them to the rank that has most children.
subroutine, public mg_fill_ghost_cells(mg, iv)
Fill ghost cells at all grid levels.
subroutine, public mg_restrict_lvl(mg, iv, lvl)
Restrict from lvl to lvl-1.
subroutine, public vlaplacian_set_methods(mg)
integer, parameter, public mg_iveps
Index of the variable coefficient (at cell centers)
subroutine, public mg_allocate_storage(mg)
Allocate communication buffers and local boxes for a tree that has already been created.
real(dp) function get_sum(mg, iv)
subroutine, public mg_build_rectangle(mg, domain_size, box_size, dx, r_min, periodic, n_finer)
subroutine, public mg_apply_op(mg, i_out, op)
Apply operator to the tree and store in variable i_out.
integer, parameter, public i8
Type for 64-bit integers.
subroutine, public mg_set_methods(mg)
integer, parameter, public mg_max_num_vars
Maximum number of variables.
pure integer function, dimension(2), public mg_get_child_offset(mg, id)
Get the offset of a box with respect to its parent (e.g. in 2d, there can be a child at offset 0,...
integer, parameter, public mg_iold
Index of previous solution (used for correction)
logical, dimension(4), parameter, public mg_neighb_low
integer, parameter, public mg_max_timers
Maximum number of timers to use.
integer, parameter, public mg_smoother_gs
real(dp), public, protected helmholtz_lambda
Module for load balancing a tree (that already has been constructed). The load balancing determines w...
subroutine, public diffusion_solve(mg, dt, diffusion_coeff, order, max_res)
Solve a diffusion equation implicitly, assuming a constant diffusion coefficient. The solution at tim...
subroutine, public mg_phi_bc_store(mg)
Store boundary conditions for the solution variable, this can speed up calculations if the same bound...
integer, dimension(4, 2), parameter, public mg_child_rev
integer, parameter, public mg_no_box
Special value that indicates there is no box.
integer, parameter, public mg_smoother_gsrb
integer function, public mg_add_timer(mg, name)
Box data structure.
Buffer type (one is used for each pair of communicating processes)
Lists of blocks per refinement level.