subroutine timeintegration(time0)
use mod_nvtx
use mod_timing
use mod_advance, only: advance, process, process_advanced
use mod_amr_grid, only: resettree, settree, resettree_convert
use mod_forest, only: nleafs_active
use mod_global_parameters
use mod_input_output, only: saveamrfile
use mod_input_output_helper, only: save_now
use mod_ghostcells_update
use mod_multigrid_coupling
use mod_dt, only: setdt
use mod_particles
use mod_usr_methods
use mod_geometry, only: sync_positions_host
double precision, intent(in) :: time0
integer :: level, ifile, fixcount, ncells_block, igrid, iigrid, itimelevel
integer(kind=8) ncells_update
logical :: crashall
double precision :: time_last_print, time_write0, time_write,&
time_before_advance, dt_loop
time_in=MPI_WTIME()
time_last_print = -bigdouble
fixcount=1
n_saves(filelog_:fileout_) = snapshotini
do ifile=nfile,1,-1
if(resume_previous_run) then
tsavelast(ifile)=aint((global_time+&
smalldouble)/dtsave(ifile))*dtsave(ifile)
itsavelast(ifile)=it/ditsave(ifile)*ditsave(ifile)
else
tsavelast(ifile)=global_time
itsavelast(ifile)=it
end if
end do
! the next two are used to keep track of the performance during runtime:
itTimeLast=it
timeLast=MPI_WTIME()
! ------ start of integration loop. ------------------
if (mype==0) then
write(*, '(A,ES9.2,A)') ' Start integrating, print status every ',&
time_between_print, ' seconds'
write(*, '(A4,A10,A12,A12,A12)') ' #', 'it', 'time', 'dt', 'wc-time(s)'
end if
timeloop0=MPI_WTIME()
time_bc=0.d0
time_write=0.d0
ncells_block=(ixGhi1-2*nghostcells)*(ixGhi2-2*nghostcells)*(ixGhi3-&
2*nghostcells)
ncells_update=0
dt_loop=0.d0
time_advance=.true.
time_evol : do
time_before_advance=MPI_WTIME()
! set time step
call nvtxStartRange("setdt"); call setdt()
call nvtxEndRange
! Optionally call a user method that can modify the grid variables at the
! beginning of a time step
if (associated(usr_process_grid) .or. associated(usr_process_global)) &
then
call process(it,global_time)
end if
! Check if output needs to be written
do ifile=nfile,1,-1
save_file(ifile) = timetosave(ifile)
end do
timeio0=MPI_WTIME()
if (timeio0 - time_last_print > time_between_print) then
time_last_print = timeio0
if (mype == 0) then
write(*, '(A4,I10,ES12.4,ES12.4,ES12.4)') " #", it, global_time, dt,&
timeio0 - time_in
end if
end if
! output data
if (any(save_file)) then
if(associated(usr_modify_output)) then
! Users can modify or set variables before output is written
! (usr_modify_output reads ps(igrid)%x on the host)
call sync_positions_host()
do iigrid=1,igridstail; igrid=igrids(iigrid);
dxlevel(1)=rnode(rpdx1_,igrid);dxlevel(2)=rnode(rpdx2_,igrid)
dxlevel(3)=rnode(rpdx3_,igrid);
block=>ps(igrid)
call usr_modify_output(ixGlo1,ixGlo2,ixGlo3,ixGhi1,ixGhi2,ixGhi3,&
ixMlo1,ixMlo2,ixMlo3,ixMhi1,ixMhi2,ixMhi3,global_time,&
ps(igrid)%w,ps(igrid)%x)
end do
end if
time_write=0.d0
do ifile=nfile,1,-1
if (save_file(ifile)) then
time_write0=MPI_WTIME()
call saveamrfile(ifile)
time_write=time_write+MPI_WTIME()-time_write0
end if
end do
end if
! output a snapshot when user write a file named 'savenow' in the same
! folder as the executable amrvac
if (mype==0) inquire(file='savenow',exist=save_now)
if (npe>1) call MPI_BCAST(save_now,1,MPI_LOGICAL,0,icomm,ierrmpi)
if (save_now) then
if(mype==0) write(*,'(a,i7,a,i7,a,es12.4)') ' save a snapshot No.',&
snapshotnext,' at it=',it,' global_time=',global_time
call saveamrfile(1)
call saveamrfile(2)
call MPI_FILE_DELETE('savenow',MPI_INFO_NULL,ierrmpi)
end if
timeio_tot=timeio_tot+MPI_WTIME()-timeio0
pass_wall_time=MPI_WTIME()-time0+dt_loop+4.d0*time_write &
>=wall_time_max
! exit time loop if time is up
if (it>=it_max .or. global_time>=time_max .or. pass_wall_time .or. &
final_dt_exit) exit time_evol
! solving equations
call advance(it)
!opedit: FIXME, this seems to take a long time!
! if met unphysical values, output the last good status and stop the run
! call MPI_ALLREDUCE(crash,crashall,1,MPI_LOGICAL,MPI_LOR,icomm,ierrmpi)
! if (crashall) then
! call saveamrfile(1)
! call saveamrfile(2)
! if(mype==0) write(*,*) "Error: small value encountered, run crash."
! call MPI_ABORT(icomm, iigrid, ierrmpi)
! end if
! Optionally call a user method that can modify the grid variables at the
! end of a time step: this is for two-way coupling to PIC, e.g.
if (associated(usr_process_adv_grid) .or. &
associated(usr_process_adv_global)) then
call process_advanced(it,global_time)
end if
! update AMR mesh and tree
timegr0=MPI_WTIME()
if(ditregrid>1) then
if(fixcount<ditregrid) then
fixcount=fixcount+1
else
if (refine_max_level>1 .and. .not.(fixgrid())) call resettree
fixcount=1
end if
else
if (refine_max_level>1 .and. .not.(fixgrid())) call resettree
end if
timegr_tot=timegr_tot+(MPI_WTIME()-timegr0)
! update time variables
it = it + 1
global_time = global_time + dt
if(it>9000000)then
it = slowsteps+it_init
itsavelast(:)=0
end if
! count updated cells
ncells_update=ncells_update+ncells_block*nleafs_active
! time lapses in one loop
dt_loop=MPI_WTIME()-time_before_advance
end do time_evol
if(use_particles) then
call finish_gridvars()
end if
time_advance=.false.
timeloop=MPI_WTIME()-timeloop0
if (mype==0) then
write(*,'(a,f12.3,a)')' Total timeloop took : ',timeloop,' sec'
write(*,'(a,f12.3,a)')' Time spent on AMR : ',timegr_tot,&
' sec'
write(*,'(a,f12.2,a)')' Percentage: ',&
100.0*timegr_tot/timeloop,' %'
write(*,'(a,f12.3,a)')' Time spent on IO in loop : ',timeio_tot,&
' sec'
write(*,'(a,f12.2,a)')' Percentage: ',&
100.0*timeio_tot/timeloop,' %'
write(*,'(a,f12.3,a)')' Time spent on ghost cells : ',time_bc,' sec'
write(*,'(a,f12.2,a)')' Percentage: ',&
100.0*time_bc/timeloop,' %'
write(*,'(a,f12.3,a)')' Time spent on computing : ',&
timeloop-timeio_tot-timegr_tot-time_bc,' sec'
write(*,'(a,f12.2,a)')' Percentage: ',&
100.0*(timeloop-timeio_tot-timegr_tot-time_bc)/timeloop,' %'
write(*,'(a,es12.3 )')' Cells updated / proc / sec : ',&
dble(ncells_update)*dble(nstep)/dble(npe)/(timeloop-timeio_tot)
end if
! output end state
timeio0=MPI_WTIME()
do ifile=nfile,1,-1
if(itsavelast(ifile)<it) call saveamrfile(ifile)
end do
if (mype==0 .and. log_fh > 0) then
call MPI_FILE_CLOSE(log_fh,ierrmpi)
endif
timeio_tot=timeio_tot+(MPI_WTIME()-timeio0)
if (mype==0) then
write(*,'(a,f12.3,a)')' Total time spent on IO : ',timeio_tot,&
' sec'
write(*,'(a,f12.3,a)')' Total timeintegration took : ',&
MPI_WTIME()-time_in,' sec'
write(*, '(A4,I10,ES12.3,ES12.3,ES12.3)') " #", it, global_time, dt,&
timeio0 - time_in
end if
if(use_particles) call time_spent_on_particles
if (use_multigrid) call mg_timers_show(mg)
end subroutine timeintegration