amr_coarsen_refine Subroutine

public subroutine amr_coarsen_refine()

Uses

  • proc~~amr_coarsen_refine~~UsesGraph proc~amr_coarsen_refine amr_coarsen_refine mod_amr_solution_node mod_amr_solution_node proc~amr_coarsen_refine->mod_amr_solution_node module~mod_amr_fct mod_amr_fct proc~amr_coarsen_refine->module~mod_amr_fct module~mod_forest mod_forest proc~amr_coarsen_refine->module~mod_forest module~mod_functions_connectivity mod_functions_connectivity proc~amr_coarsen_refine->module~mod_functions_connectivity module~mod_functions_forest mod_functions_forest proc~amr_coarsen_refine->module~mod_functions_forest module~mod_ghostcells_update mod_ghostcells_update proc~amr_coarsen_refine->module~mod_ghostcells_update module~mod_global_parameters mod_global_parameters proc~amr_coarsen_refine->module~mod_global_parameters module~mod_load_balance mod_load_balance proc~amr_coarsen_refine->module~mod_load_balance module~mod_multigrid_coupling mod_multigrid_coupling proc~amr_coarsen_refine->module~mod_multigrid_coupling module~mod_refine mod_refine proc~amr_coarsen_refine->module~mod_refine module~mod_selectgrids mod_selectgrids proc~amr_coarsen_refine->module~mod_selectgrids module~mod_space_filling_curve mod_space_filling_curve proc~amr_coarsen_refine->module~mod_space_filling_curve module~mod_usr_methods mod_usr_methods proc~amr_coarsen_refine->module~mod_usr_methods module~mod_mpi_wrapper mod_mpi_wrapper module~mod_amr_fct->module~mod_mpi_wrapper module~mod_functions_forest->module~mod_mpi_wrapper module~mod_ghostcells_update->module~mod_mpi_wrapper module~mod_basic_types mod_basic_types module~mod_global_parameters->module~mod_basic_types module~mod_connectivity mod_connectivity module~mod_global_parameters->module~mod_connectivity module~mod_constants mod_constants module~mod_global_parameters->module~mod_constants module~mod_physicaldata mod_physicaldata module~mod_global_parameters->module~mod_physicaldata module~mod_variables mod_variables module~mod_global_parameters->module~mod_variables mpi mpi module~mod_global_parameters->mpi module~mod_load_balance->module~mod_mpi_wrapper module~m_octree_mg_3d m_octree_mg_3d module~mod_multigrid_coupling->module~m_octree_mg_3d module~m_octree_mg_3d->module~mod_mpi_wrapper module~m_octree_mg_3d->mpi module~mod_mpi_wrapper->mpi module~mod_variables->module~mod_basic_types

coarsen and refine blocks to update AMR grid

Arguments

None

Calls

proc~~amr_coarsen_refine~~CallsGraph proc~amr_coarsen_refine amr_coarsen_refine getnode getnode proc~amr_coarsen_refine->getnode mpi_barrier mpi_barrier proc~amr_coarsen_refine->mpi_barrier mpi_waitall mpi_waitall proc~amr_coarsen_refine->mpi_waitall proc~amr_morton_order amr_Morton_order proc~amr_coarsen_refine->proc~amr_morton_order proc~build_connectivity build_connectivity proc~amr_coarsen_refine->proc~build_connectivity proc~coarsen_grid_siblings coarsen_grid_siblings proc~amr_coarsen_refine->proc~coarsen_grid_siblings proc~coarsen_tree_leaf coarsen_tree_leaf proc~amr_coarsen_refine->proc~coarsen_tree_leaf proc~comm_faces comm_faces proc~amr_coarsen_refine->proc~comm_faces proc~end_comm_faces end_comm_faces proc~amr_coarsen_refine->proc~end_comm_faces proc~exchange_coarsened_blocks exchange_coarsened_blocks proc~amr_coarsen_refine->proc~exchange_coarsened_blocks proc~get_level_range get_level_range proc~amr_coarsen_refine->proc~get_level_range proc~getbc getbc proc~amr_coarsen_refine->proc~getbc proc~getigrids getigrids proc~amr_coarsen_refine->proc~getigrids proc~load_balance load_balance proc~amr_coarsen_refine->proc~load_balance proc~mg_update_refinement mg_update_refinement proc~amr_coarsen_refine->proc~mg_update_refinement proc~proper_nesting proper_nesting proc~amr_coarsen_refine->proc~proper_nesting proc~refine_grids refine_grids proc~amr_coarsen_refine->proc~refine_grids proc~refine_tree_leaf refine_tree_leaf proc~amr_coarsen_refine->proc~refine_tree_leaf proc~selectgrids selectgrids proc~amr_coarsen_refine->proc~selectgrids proc~store_faces store_faces proc~amr_coarsen_refine->proc~store_faces putnode putnode proc~amr_coarsen_refine->putnode proc~mpistop mpistop proc~amr_morton_order->proc~mpistop proc~add_to_c nbprocs_info_t%add_to_c proc~build_connectivity->proc~add_to_c proc~add_to_f nbprocs_info_t%add_to_f proc~build_connectivity->proc~add_to_f proc~add_to_srl nbprocs_info_t%add_to_srl proc~build_connectivity->proc~add_to_srl proc~alloc_buffers_c nbprocs_info_t%alloc_buffers_c proc~build_connectivity->proc~alloc_buffers_c proc~alloc_buffers_f nbprocs_info_t%alloc_buffers_f proc~build_connectivity->proc~alloc_buffers_f proc~alloc_buffers_srl nbprocs_info_t%alloc_buffers_srl proc~build_connectivity->proc~alloc_buffers_srl proc~find_neighbor find_neighbor proc~build_connectivity->proc~find_neighbor proc~identifyphysbound_connectivity identifyphysbound_connectivity proc~build_connectivity->proc~identifyphysbound_connectivity proc~reset nbprocs_info_t%reset proc~build_connectivity->proc~reset alloc_node alloc_node proc~coarsen_grid_siblings->alloc_node proc~coarsen_grid coarsen_grid proc~coarsen_grid_siblings->proc~coarsen_grid proc~initial_condition initial_condition proc~coarsen_grid_siblings->proc~initial_condition proc~mpi_irecv_wrapper mpi_irecv_wrapper proc~coarsen_grid_siblings->proc~mpi_irecv_wrapper proc~mpi_isend_wrapper mpi_isend_wrapper proc~coarsen_grid_siblings->proc~mpi_isend_wrapper proc~coarsen_grid_siblings->proc~mpistop proc~add_to_linked_list add_to_linked_list proc~coarsen_tree_leaf->proc~add_to_linked_list proc~delete_from_linked_list delete_from_linked_list proc~coarsen_tree_leaf->proc~delete_from_linked_list proc~comm_faces->mpi_waitall proc~comm_faces->proc~mpi_irecv_wrapper proc~comm_faces->proc~mpi_isend_wrapper proc~exchange_coarsened_blocks->mpi_waitall mpi_get_count mpi_get_count proc~exchange_coarsened_blocks->mpi_get_count proc~layout_runs layout_runs proc~exchange_coarsened_blocks->proc~layout_runs proc~exchange_coarsened_blocks->proc~mpi_irecv_wrapper proc~exchange_coarsened_blocks->proc~mpi_isend_wrapper proc~exchange_coarsened_blocks->proc~mpistop proc~getbc->mpi_waitall mpi_wtime mpi_wtime proc~getbc->mpi_wtime phys_boundary_adjust phys_boundary_adjust proc~getbc->phys_boundary_adjust proc~fill_boundary_after_gc fill_boundary_after_gc proc~getbc->proc~fill_boundary_after_gc proc~fill_boundary_before_gc fill_boundary_before_gc proc~getbc->proc~fill_boundary_before_gc proc~fill_coarse_boundary fill_coarse_boundary proc~getbc->proc~fill_coarse_boundary proc~getintbc getintbc proc~getbc->proc~getintbc proc~idecode~2 idecode proc~getbc->proc~idecode~2 proc~getbc->proc~mpi_irecv_wrapper proc~getbc->proc~mpi_isend_wrapper proc~nvtxendrange~2 nvtxEndRange proc~getbc->proc~nvtxendrange~2 proc~nvtxstartrange~2 nvtxStartRange proc~getbc->proc~nvtxstartrange~2 proc~skip_direction skip_direction proc~getbc->proc~skip_direction proc~load_balance->getnode proc~load_balance->mpi_waitall proc~load_balance->proc~amr_morton_order proc~load_balance->putnode proc~change_ipe_tree_leaf change_ipe_tree_leaf proc~load_balance->proc~change_ipe_tree_leaf proc~exchange_migrated_blocks exchange_migrated_blocks proc~load_balance->proc~exchange_migrated_blocks proc~get_morton_range get_Morton_range proc~load_balance->proc~get_morton_range proc~lb_recv lb_recv proc~load_balance->proc~lb_recv proc~lb_send lb_send proc~load_balance->proc~lb_send proc~mg_deallocate_storage~3 mg_deallocate_storage proc~mg_update_refinement->proc~mg_deallocate_storage~3 proc~mg_tree_from_amrvac mg_tree_from_amrvac proc~mg_update_refinement->proc~mg_tree_from_amrvac mpi_allgather mpi_allgather proc~proper_nesting->mpi_allgather mpi_allreduce mpi_allreduce proc~proper_nesting->mpi_allreduce proc~proper_nesting->proc~find_neighbor proc~refine_grids->alloc_node dealloc_node dealloc_node proc~refine_grids->dealloc_node proc~refine_grids->proc~initial_condition proc~prolong_grid prolong_grid proc~refine_grids->proc~prolong_grid proc~refine_tree_leaf->proc~delete_from_linked_list proc~refine_tree_leaf->proc~find_neighbor proc~init_tree_leaf init_tree_leaf proc~refine_tree_leaf->proc~init_tree_leaf proc~gather_isafety gather_isafety proc~selectgrids->proc~gather_isafety proc~igrid_active igrid_active proc~selectgrids->proc~igrid_active proc~add_igrid_to_c nbprocs_info_t%add_igrid_to_c proc~add_to_c->proc~add_igrid_to_c proc~add_igrid_to_f nbprocs_info_t%add_igrid_to_f proc~add_to_f->proc~add_igrid_to_f proc~add_igrid_to_srl nbprocs_info_t%add_igrid_to_srl proc~add_to_srl->proc~add_igrid_to_srl proc~idecode nbprocs_info_t%idecode proc~alloc_buffers_srl->proc~idecode proc~exchange_migrated_blocks->mpi_waitall proc~exchange_migrated_blocks->mpi_get_count proc~exchange_migrated_blocks->proc~layout_runs proc~exchange_migrated_blocks->proc~mpi_irecv_wrapper proc~exchange_migrated_blocks->proc~mpi_isend_wrapper proc~exchange_migrated_blocks->proc~mpistop proc~bc_phys bc_phys proc~fill_boundary_after_gc->proc~bc_phys proc~fill_boundary_before_gc->proc~bc_phys proc~fill_coarse_boundary->proc~bc_phys proc~find_root_neighbor find_root_neighbor proc~find_neighbor->proc~find_root_neighbor proc~gather_isafety->proc~mpistop mpi_allgatherv mpi_allgatherv proc~gather_isafety->mpi_allgatherv proc~sync_positions_host sync_positions_host proc~getintbc->proc~sync_positions_host proc~init_tree_leaf->proc~add_to_linked_list proc~initial_condition->proc~mpistop proc~initial_condition->proc~sync_positions_host proc~rank_keys rank_keys proc~layout_runs->proc~rank_keys proc~lb_recv->alloc_node proc~lb_recv->proc~mpi_irecv_wrapper proc~lb_recv->proc~mpistop proc~lb_send->proc~mpi_isend_wrapper proc~lb_send->proc~mpistop proc~mg_add_children~3 mg_add_children proc~mg_tree_from_amrvac->proc~mg_add_children~3 proc~mg_allocate_storage~3 mg_allocate_storage proc~mg_tree_from_amrvac->proc~mg_allocate_storage~3 proc~mg_build_rectangle~3 mg_build_rectangle proc~mg_tree_from_amrvac->proc~mg_build_rectangle~3 proc~mg_load_balance_parents~3 mg_load_balance_parents proc~mg_tree_from_amrvac->proc~mg_load_balance_parents~3 proc~mg_set_leaves_parents~3 mg_set_leaves_parents proc~mg_tree_from_amrvac->proc~mg_set_leaves_parents~3 proc~mg_set_neighbors_lvl~3 mg_set_neighbors_lvl proc~mg_tree_from_amrvac->proc~mg_set_neighbors_lvl~3 proc~mg_set_next_level_ids~3 mg_set_next_level_ids proc~mg_tree_from_amrvac->proc~mg_set_next_level_ids~3 proc~mg_set_refinement_boundaries~3 mg_set_refinement_boundaries proc~mg_tree_from_amrvac->proc~mg_set_refinement_boundaries~3 mpi_irecv mpi_irecv proc~mpi_irecv_wrapper->mpi_irecv mpi_isend mpi_isend proc~mpi_isend_wrapper->mpi_isend mpi_abort mpi_abort proc~mpistop->mpi_abort phys_to_conserved phys_to_conserved proc~prolong_grid->phys_to_conserved phys_to_primitive phys_to_primitive proc~prolong_grid->phys_to_primitive proc~old_neighbors old_neighbors proc~prolong_grid->proc~old_neighbors proc~prolong_2nd prolong_2nd proc~prolong_grid->proc~prolong_2nd proc~prolong_grid->proc~sync_positions_host proc~init_cf_info nbinfo_cf_t%init_cf_info proc~add_igrid_to_c->proc~init_cf_info proc~add_igrid_to_f->proc~init_cf_info proc~iencode nbprocs_info_t%iencode proc~add_igrid_to_srl->proc~iencode proc~init_srl_info init_srl_info proc~add_igrid_to_srl->proc~init_srl_info specialbound_usr specialbound_usr proc~bc_phys->specialbound_usr proc~mg_ghost_cell_buffer_size~3 mg_ghost_cell_buffer_size proc~mg_allocate_storage~3->proc~mg_ghost_cell_buffer_size~3 proc~mg_prolong_buffer_size~3 mg_prolong_buffer_size proc~mg_allocate_storage~3->proc~mg_prolong_buffer_size~3 proc~mg_restrict_buffer_size~3 mg_restrict_buffer_size proc~mg_allocate_storage~3->proc~mg_restrict_buffer_size~3 proc~mg_build_rectangle~3->proc~mg_add_children~3 proc~mg_build_rectangle~3->proc~mg_set_leaves_parents~3 proc~mg_build_rectangle~3->proc~mg_set_neighbors_lvl~3 proc~mg_build_rectangle~3->proc~mg_set_next_level_ids~3 proc~add_single_child~3 add_single_child proc~mg_build_rectangle~3->proc~add_single_child~3 proc~most_popular~3 most_popular proc~mg_load_balance_parents~3->proc~most_popular~3 proc~update_lvl_info~3 update_lvl_info proc~mg_load_balance_parents~3->proc~update_lvl_info~3 proc~mg_has_children~3 mg_has_children proc~mg_set_leaves_parents~3->proc~mg_has_children~3 proc~set_neighbs~3 set_neighbs proc~mg_set_neighbors_lvl~3->proc~set_neighbs~3 proc~mg_set_refinement_boundaries~3->proc~mg_has_children~3 proc~prolong_2nd->phys_to_conserved proc~prolong_2nd->proc~sync_positions_host proc~already_fine already_fine proc~prolong_2nd->proc~already_fine proc~prolong_2nd_stg prolong_2nd_stg proc~prolong_2nd->proc~prolong_2nd_stg proc~buffer_ghost_cells~3 buffer_ghost_cells proc~mg_ghost_cell_buffer_size~3->proc~buffer_ghost_cells~3 proc~buffer_refinement_boundaries~3 buffer_refinement_boundaries proc~mg_ghost_cell_buffer_size~3->proc~buffer_refinement_boundaries~3 proc~set_ghost_cells~3 set_ghost_cells proc~mg_ghost_cell_buffer_size~3->proc~set_ghost_cells~3 phys_face_to_center phys_face_to_center proc~prolong_2nd_stg->phys_face_to_center proc~find_neighb~3 find_neighb proc~set_neighbs~3->proc~find_neighb~3 proc~buffer_for_nb~3 buffer_for_nb proc~buffer_ghost_cells~3->proc~buffer_for_nb~3 proc~buffer_refinement_boundaries~3->proc~mg_has_children~3 proc~buffer_for_fine_nb~3 buffer_for_fine_nb proc~buffer_refinement_boundaries~3->proc~buffer_for_fine_nb~3 proc~mg_ix_to_ichild~3 mg_ix_to_ichild proc~find_neighb~3->proc~mg_ix_to_ichild~3 proc~bc_to_gc~3 bc_to_gc proc~set_ghost_cells~3->proc~bc_to_gc~3 proc~box_get_gc~3 box_get_gc proc~set_ghost_cells~3->proc~box_get_gc~3 proc~box_set_gc~3 box_set_gc proc~set_ghost_cells~3->proc~box_set_gc~3 proc~copy_from_nb~3 copy_from_nb proc~set_ghost_cells~3->proc~copy_from_nb~3 proc~fill_buffered_nb~3 fill_buffered_nb proc~set_ghost_cells~3->proc~fill_buffered_nb~3 proc~fill_refinement_bnd~3 fill_refinement_bnd proc~set_ghost_cells~3->proc~fill_refinement_bnd~3 proc~box_gc_for_fine_neighbor~3 box_gc_for_fine_neighbor proc~buffer_for_fine_nb~3->proc~box_gc_for_fine_neighbor~3 proc~mg_get_child_offset~3 mg_get_child_offset proc~buffer_for_fine_nb~3->proc~mg_get_child_offset~3 proc~box_gc_for_neighbor~3 box_gc_for_neighbor proc~buffer_for_nb~3->proc~box_gc_for_neighbor~3 proc~copy_from_nb~3->proc~box_set_gc~3 proc~copy_from_nb~3->proc~box_gc_for_neighbor~3 proc~fill_buffered_nb~3->proc~box_set_gc~3 proc~fill_refinement_bnd~3->proc~box_gc_for_fine_neighbor~3 proc~fill_refinement_bnd~3->proc~mg_get_child_offset~3 proc~sides_rb~3 sides_rb proc~fill_refinement_bnd~3->proc~sides_rb~3

Called by

proc~~amr_coarsen_refine~~CalledByGraph proc~amr_coarsen_refine amr_coarsen_refine proc~resettree resettree proc~resettree->proc~amr_coarsen_refine proc~resettree_convert resettree_convert proc~resettree_convert->proc~amr_coarsen_refine proc~settree settree proc~settree->proc~amr_coarsen_refine proc~main main proc~main->proc~resettree_convert proc~main->proc~settree proc~timeintegration timeintegration proc~main->proc~timeintegration proc~timeintegration->proc~resettree program~agile agile program~agile->proc~main

Source Code

  subroutine amr_coarsen_refine
    use mod_forest
    use mod_global_parameters
    use mod_ghostcells_update
    use mod_usr_methods, only: usr_after_refine
    use mod_amr_fct
    use mod_space_filling_curve
    use mod_load_balance
    use mod_functions_connectivity, only: get_level_range,getigrids,&
         build_connectivity
    use mod_amr_solution_node, only: getnode, putnode
    use mod_functions_forest, only: coarsen_tree_leaf,refine_tree_leaf
    use mod_selectgrids, only: selectgrids
    use mod_refine, only: refine_grids
    use mod_multigrid_coupling


    integer :: iigrid, igrid, ipe, igridCo, ipeCo, level, ic1,ic2,ic3
    integer, dimension(2,2,2) :: igridFi, ipeFi
    integer :: n_coarsen, n_refine
    type(tree_node_ptr) :: tree, sibling
    logical             :: active
    integer             :: ibuff, iw, ix1, ix2, ix3

    call proper_nesting

    if(stagger_grid) then
       call store_faces
       call comm_faces
    end if

    n_coarsen = count(coarsen(:, :))
    n_refine = count(refine(:, :))

    ! to save memory: first coarsen then refine
    irecv=0
    isend=0
    n_cf_snd=0
    n_cf_rcv=0
    if (.not.allocated(recvrequest)) then
       allocate(recvstatus(MPI_STATUS_SIZE,npe),recvrequest(npe),&
            sendstatus(MPI_STATUS_SIZE,npe),sendrequest(npe))
       allocate(cf_send_pe(npe),cf_send_pe_off(npe),cf_send_pe_len(npe),&
            cf_recv_pe(npe),cf_recv_pe_off(npe),cf_recv_pe_len(npe))
    end if
    recvrequest=MPI_REQUEST_NULL
    sendrequest=MPI_REQUEST_NULL

    if(stagger_grid) then
       allocate(recvstatus_stg(MPI_STATUS_SIZE,max_blocks*3),&
            recvrequest_stg(max_blocks*3), sendstatus_stg(MPI_STATUS_SIZE,&
            max_blocks*3),sendrequest_stg(max_blocks*3))
       recvrequest_stg=MPI_REQUEST_NULL
       sendrequest_stg=MPI_REQUEST_NULL
    end if

    ! Allocate the chunk descriptors and the exchange buffers.  Both are
    ! capped by max_buff chunks, which coarsen_grid_siblings checks against.
    if ( .not. allocated(snd_buff_cf) ) then
       allocate( snd_buff_cf((block_nx1/2)*(block_nx2/2)*(block_nx3/2)*nw &
            *max_buff), &
            rcv_buff_cf((block_nx1/2)*(block_nx2/2)*(block_nx3/2)*nw &
            *max_buff) )
       allocate( cf_snd_igrid(max_buff), cf_snd_ibuf(max_buff), &
            cf_snd_dest(max_buff), cf_snd_key(max_buff), &
            cf_rcv_igrid(max_buff), cf_rcv_ibuf(max_buff), &
            cf_rcv_src(max_buff), cf_rcv_key(max_buff), cf_rcv_ic(3,max_buff) )
       
    end if

    do ipe=0,npe-1
       do igrid=1,max_blocks
          if (coarsen(igrid,ipe)) then
             if (.not.associated(igrid_to_node(igrid,ipe)%node)) cycle

             tree%node => igrid_to_node(igrid,ipe)%node%parent%node
             do ic3=1,2
                do ic2=1,2
                   do ic1=1,2
                      sibling%node => tree%node%child(ic1,ic2,ic3)%node
                      ipeFi(ic1,ic2,ic3)=sibling%node%ipe
                      igridFi(ic1,ic2,ic3)=sibling%node%igrid
                   end do
                end do
             end do

             ipeCo=ipeFi(1,1,1)
             igridCo=getnode(ipeCo)

             call coarsen_tree_leaf(igridCo,ipeCo,igridFi,ipeFi,active)

             call coarsen_grid_siblings(igridCo,ipeCo,igridFi,ipeFi,active)

             ! local coarsening done
             do ic3=1,2
                do ic2=1,2
                   do ic1=1,2
                      if (ipeFi(ic1,ic2,ic3)==ipeCo) then
                         call putnode(igridFi(ic1,ic2,ic3),ipeFi(ic1,ic2,ic3))
                         coarsen(igridFi(ic1,ic2,ic3),ipeFi(ic1,ic2,ic3))=.false.
                      end if
                   end do
                end do
             end do
          end if
       end do
    end do

    ! Lay the chunks out per peer, exchange them in one message each, and
    ! apply them.  The walk above only recorded descriptors.
    call exchange_coarsened_blocks

    ! The staggered path still posts per chunk; see the note in
    ! coarsen_grid_siblings.  It is unreachable here (stagger_grid is false and
    ! fix_edges is not called), so it is left as it was found.
    if(stagger_grid) then
       if (irecv>0) call MPI_WAITALL(irecv,recvrequest_stg,recvstatus_stg,&
            ierrmpi)
       if (isend>0) call MPI_WAITALL(isend,sendrequest_stg,sendstatus_stg,&
            ierrmpi)
       deallocate(recvstatus_stg,recvrequest_stg,sendstatus_stg,&
            sendrequest_stg)
    end if

    ! non-local coarsening done
    do ipe=0,npe-1
       do igrid=1,max_blocks
          if (coarsen(igrid,ipe)) then
             !if (ipe==mype) call dealloc_node(igrid) ! do not deallocate node
             ! memory preventing fragmentization of system memory as a result
             ! of frequent allocating and deallocating memory

             ! put the node (igrid number) into unused.
             call putnode(igrid,ipe)
             coarsen(igrid,ipe)=.false.
          end if
       end do
    end do

    do ipe=0,npe-1
       do igrid=1,max_blocks
          if (refine(igrid,ipe)) then

             do ic3=1,2
                do ic2=1,2
                   do ic1=1,2
                      igridFi(ic1,ic2,ic3)=getnode(ipe)
                      ipeFi(ic1,ic2,ic3)=ipe
                   end do
                end do
             end do

             call refine_tree_leaf(igridFi,ipeFi,igrid,ipe,active)

             if (ipe==mype) call refine_grids(igridFi,ipeFi,igrid,ipe,active)

             ! refinement done
             call putnode(igrid,ipe)
             refine(igrid,ipe)=.false.
          end if
       end do
    end do

    ! A crash occurs in later MPI_WAITALL when initial condition comsumes too
    ! much time to filling new blocks with both gfortran and intel fortran compiler.
    ! This barrier cure this problem
    !TODO to find the reason
    if(.not.time_advance) call MPI_BARRIER(icomm,ierrmpi)

    if(stagger_grid) call end_comm_faces

    call get_level_range

    ! Update sfc array: igrid and ipe info in space filling curve
    call amr_Morton_order

    call load_balance

    ! Rebuild tree connectivity
    call getigrids
    call build_connectivity

    ! Update the list of active grids
    call selectgrids
    !  grid structure now complete again.

    ! since we only filled mesh values, and advance assumes filled
    ! ghost cells, do boundary filling for the new levels
    if (time_advance) then
       call getbc(global_time+dt,0.d0,ps,iwstart,nwgc)
    else
       call getbc(global_time,0.d0,ps,iwstart,nwgc)
    end if

    if (use_multigrid) call mg_update_refinement(n_coarsen, n_refine)

    if (associated(usr_after_refine)) then
       call usr_after_refine(n_coarsen, n_refine)
    end if

    

  end subroutine amr_coarsen_refine