tecplot_mpi Subroutine

public subroutine tecplot_mpi(qunit)

Uses

  • proc~~tecplot_mpi~~UsesGraph proc~tecplot_mpi tecplot_mpi module~mod_calculate_xw mod_calculate_xw proc~tecplot_mpi->module~mod_calculate_xw module~mod_forest mod_forest proc~tecplot_mpi->module~mod_forest module~mod_global_parameters mod_global_parameters proc~tecplot_mpi->module~mod_global_parameters module~mod_basic_types mod_basic_types module~mod_global_parameters->module~mod_basic_types module~mod_connectivity mod_connectivity module~mod_global_parameters->module~mod_connectivity module~mod_constants mod_constants module~mod_global_parameters->module~mod_constants module~mod_physicaldata mod_physicaldata module~mod_global_parameters->module~mod_physicaldata module~mod_variables mod_variables module~mod_global_parameters->module~mod_variables mpi mpi module~mod_global_parameters->mpi module~mod_variables->module~mod_basic_types

Arguments

Type IntentOptional Attributes Name
integer, intent(in) :: qunit

Calls

proc~~tecplot_mpi~~CallsGraph proc~tecplot_mpi tecplot_mpi mpi_allreduce mpi_allreduce proc~tecplot_mpi->mpi_allreduce mpi_barrier mpi_barrier proc~tecplot_mpi->mpi_barrier proc~calc_grid calc_grid proc~tecplot_mpi->proc~calc_grid proc~calc_x calc_x proc~tecplot_mpi->proc~calc_x proc~getheadernames getheadernames proc~tecplot_mpi->proc~getheadernames proc~mpi_recv_wrapper mpi_recv_wrapper proc~tecplot_mpi->proc~mpi_recv_wrapper proc~mpi_send_wrapper mpi_send_wrapper proc~tecplot_mpi->proc~mpi_send_wrapper proc~mpistop mpistop proc~tecplot_mpi->proc~mpistop proc~save_conntec save_conntec proc~tecplot_mpi->proc~save_conntec proc~calc_grid->proc~mpistop phys_to_primitive phys_to_primitive proc~calc_grid->phys_to_primitive proc~to_cartesian to_cartesian proc~calc_grid->proc~to_cartesian proc~getheadernames->proc~mpistop mpi_recv mpi_recv proc~mpi_recv_wrapper->mpi_recv mpi_send mpi_send proc~mpi_send_wrapper->mpi_send mpi_abort mpi_abort proc~mpistop->mpi_abort proc~nodenumbertec3d nodenumbertec3D proc~save_conntec->proc~nodenumbertec3d proc~nodenumbertec3d->proc~mpistop

Called by

proc~~tecplot_mpi~~CalledByGraph proc~tecplot_mpi tecplot_mpi proc~generate_plotfile generate_plotfile proc~generate_plotfile->proc~tecplot_mpi proc~main main proc~main->proc~generate_plotfile proc~timeintegration timeintegration proc~main->proc~timeintegration proc~saveamrfile saveamrfile proc~saveamrfile->proc~generate_plotfile proc~timeintegration->proc~saveamrfile program~agile agile program~agile->proc~main

Source Code

  subroutine tecplot_mpi(qunit)
    ! output for tecplot (ASCII format)
    ! parallel, uses calc_grid to compute nwauxio variables
    ! allows renormalizing using convert factors
    ! the current implementation is such that tecplotmpi and tecplotCCmpi will
    ! create different length output ASCII files when used on 1 versus multiple CPUs
    ! in fact, on 1 CPU, there will be as many zones as there are levels
    ! on multiple CPUs, there will be a number of zones up to the number of
    ! levels times the number of CPUs (can be less, when some level not on a CPU)
    use mod_forest, only: Morton_start, Morton_stop, sfc_to_igrid
    use mod_global_parameters
    use mod_calculate_xw

    integer, intent(in) :: qunit

    integer::               igrid,iigrid,level,igonlevel,iw,idim,ix1,ix2,ix3
    integer::               NumGridsOnLevel(1:nlevelshi)
    integer :: nx1,nx2,nx3,nxC1,nxC2,nxC3,nodesonlevel,elemsonlevel,ixCmin1,&
       ixCmin2,ixCmin3,ixCmax1,ixCmax2,ixCmax3,ixCCmin1,ixCCmin2,ixCCmin3,&
       ixCCmax1,ixCCmax2,ixCCmax3
    integer :: nodesonlevelmype,elemsonlevelmype
    integer ::              nodes, elems
    integer, allocatable :: intstatus(:,:)
    double precision :: x_TEC(ndim), w_TEC(nw+nwauxio)
    double precision, dimension(ixMlo1-1:ixMhi1,ixMlo2-1:ixMhi2,&
       ixMlo3-1:ixMhi3,ndim) :: xC_TMP,xC_TMP_recv
    double precision, dimension(ixMlo1:ixMhi1,ixMlo2:ixMhi2,ixMlo3:ixMhi3,&
       ndim)   :: xCC_TMP,xCC_TMP_recv
    double precision, dimension(ixMlo1-1:ixMhi1,ixMlo2-1:ixMhi2,&
       ixMlo3-1:ixMhi3,ndim) :: xC
    double precision, dimension(ixMlo1:ixMhi1,ixMlo2:ixMhi2,ixMlo3:ixMhi3,&
       ndim)   :: xCC
    double precision, dimension(ixMlo1-1:ixMhi1,ixMlo2-1:ixMhi2,&
       ixMlo3-1:ixMhi3,nw+nwauxio)   :: wC_TMP,wC_TMP_recv
    double precision, dimension(ixMlo1:ixMhi1,ixMlo2:ixMhi2,ixMlo3:ixMhi3,&
       nw+nwauxio)     :: wCC_TMP,wCC_TMP_recv
    double precision, dimension(0:nw+nwauxio)                   :: normconv
    logical :: fileopen,first
    integer :: itag,Morton_no,ipe,levmin_recv,levmax_recv,igrid_recv,&
       level_recv
    integer :: ixrvCmin1,ixrvCmin2,ixrvCmin3,ixrvCmax1,ixrvCmax2,ixrvCmax3,&
       ixrvCCmin1,ixrvCCmin2,ixrvCCmin3,ixrvCCmax1,ixrvCCmax2,ixrvCCmax3
    integer :: ind_send(2*3),ind_recv(2*3),siz_ind,igonlevel_recv
    integer :: NumGridsOnLevel_mype(1:nlevelshi,0:npe-1)
    character(len=80) :: filename
    integer ::           filenr
    character(len=1024) :: tecplothead
    character(len=name_len) :: wnamei(1:nw+nwauxio),&
       xandwnamei(1:ndim+nw+nwauxio)
    character(len=1024) :: outfilehead

    if(nw/=count(w_write(1:nw)))then
     if(mype==0) PRINT *,'tecplot_mpi does not use w_write=F'
     call mpistop('w_write, tecplot')
    end if

    if(nocartesian)then
      if(mype==0) PRINT *,'tecplot_mpi with nocartesian'
    end if

    Master_cpu_open : if (mype == 0) then
      inquire(qunit,opened=fileopen)
      if (.not.fileopen) then
        ! generate filename
         filenr=snapshotini
         if (autoconvert) filenr=snapshotnext
        write(filename,'(a,i4.4,a)') TRIM(base_filename),filenr,".plt"
        open(qunit,file=filename,status='unknown')
      end if
      call getheadernames(wnamei,xandwnamei,outfilehead)
      write(tecplothead,'(a)') "VARIABLES = "//TRIM(outfilehead)
      write(qunit,'(a)') tecplothead(1:len_trim(tecplothead))
    end if  Master_cpu_open

    ! determine overall number of grids per level, and the same info per CPU
    NumGridsOnLevel(1:nlevelshi)=0
    do level=levmin,levmax
      NumGridsOnLevel(level)=0
      do Morton_no=Morton_start(mype),Morton_stop(mype)
        igrid = sfc_to_igrid(Morton_no)
        if (node(plevel_,igrid)/=level) cycle
        NumGridsOnLevel(level)=NumGridsOnLevel(level)+1
      end do
      NumGridsOnLevel_mype(level,0:npe-1)=0
      NumGridsOnLevel_mype(level,mype) = NumGridsOnLevel(level)
      call MPI_ALLREDUCE(MPI_IN_PLACE,NumGridsOnLevel_mype(level,0:npe-1),npe,&
         MPI_INTEGER,MPI_MAX,icomm,ierrmpi)
      call MPI_ALLREDUCE(MPI_IN_PLACE,NumGridsOnLevel(level),1,MPI_INTEGER,&
         MPI_SUM, icomm,ierrmpi)
    end do

    nx1=ixMhi1-ixMlo1+1;nx2=ixMhi2-ixMlo2+1;nx3=ixMhi3-ixMlo3+1;
    nxC1=nx1+1;nxC2=nx2+1;nxC3=nx3+1;

    if(mype==0.and.npe>1) allocate(intstatus(MPI_STATUS_SIZE,1))


    if(mype/=0) then
      itag=1000*Morton_stop(mype)
      call mpi_send_wrapper(levmin,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
      itag=2000*Morton_stop(mype)
      call mpi_send_wrapper(levmax,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
    end if

    do level=levmin,levmax
       nodesonlevelmype=NumGridsOnLevel_mype(level,mype)*nxC1*nxC2*nxC3
       elemsonlevelmype=NumGridsOnLevel_mype(level,mype)*nx1*nx2*nx3
       nodesonlevel=NumGridsOnLevel(level)*nxC1*nxC2*nxC3
       elemsonlevel=NumGridsOnLevel(level)*nx1*nx2*nx3
       ! for all tecplot variants coded up here, we let the TECPLOT ZONES coincide
       ! with the AMR grid LEVEL. Other options would be
       !    let each grid define a zone: inefficient for TECPLOT internal workings
       !       hence not implemented
       !    let entire octree define 1 zone: no difference in interpolation
       !       properties across TECPLOT zones detected as yet, hence not done
       select case(convert_type)
         case('tecplotmpi')
           ! in this option, we store the corner coordinates, as well as the corner
           ! values of all variables (obtained by averaging). This allows POINT packaging,
           ! and thus we can save full grid info by using one call to calc_grid
           if (mype==0.and.(nodesonlevelmype>0.and.&
              elemsonlevelmype>0))write(qunit,&
              "(a,i7,a,a,i7,a,i7,a,f25.16,a,a)") 'ZONE T="',level,'"',', N=',&
              nodesonlevelmype,', E=',elemsonlevelmype, ', SOLUTIONTIME=',&
              global_time*time_convert_factor,', DATAPACKING=POINT, ZONETYPE=',&
                'FEBRICK'
          do Morton_no=Morton_start(mype),Morton_stop(mype)
             igrid = sfc_to_igrid(Morton_no)
             if (mype/=0)then
               itag=Morton_no
               call mpi_send_wrapper(igrid,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
               itag=igrid
               call mpi_send_wrapper(node(plevel_,igrid),1,MPI_INTEGER, 0,itag,icomm,&
                  ierrmpi)
             end if
             if (node(plevel_,igrid)/=level) cycle
             call calc_x(igrid,xC,xCC)
             call calc_grid(qunit,igrid,xC,xCC,xC_TMP,xCC_TMP,wC_TMP,wCC_TMP,&
                normconv,ixCmin1,ixCmin2,ixCmin3,ixCmax1,ixCmax2,ixCmax3,&
                ixCCmin1,ixCCmin2,ixCCmin3,ixCCmax1,ixCCmax2,ixCCmax3,.true.)
             if (mype/=0) then
                itag=Morton_no
                ind_send=(/ ixCmin1,ixCmin2,ixCmin3,ixCmax1,ixCmax2,ixCmax3 /)
                siz_ind=2*3
                call mpi_send_wrapper(ind_send,siz_ind, MPI_INTEGER, 0,itag,icomm,&
                   ierrmpi)
                call mpi_send_wrapper(normconv,nw+nwauxio+1,MPI_DOUBLE_PRECISION, 0,&
                   itag,icomm,ierrmpi)

                call mpi_send_wrapper(wC_TMP,1,type_block_wc_io, 0,itag,icomm,ierrmpi)
                call mpi_send_wrapper(xC_TMP,1,type_block_xc_io, 0,itag,icomm,ierrmpi)
             else
               do ix3=ixCmin3,ixCmax3
               do ix2=ixCmin2,ixCmax2
               do ix1=ixCmin1,ixCmax1
                  x_TEC(1:ndim)=xC_TMP(ix1,ix2,ix3,1:ndim)*normconv(0)
                  w_TEC(1:nw+nwauxio)=wC_TMP(ix1,ix2,ix3,&
                     1:nw+nwauxio)*normconv(1:nw+nwauxio)
                  write(qunit,fmt="(100(e14.6))") x_TEC, w_TEC
               end do
               end do
               end do
             end if
           end do
         case('tecplotCCmpi')
           ! in this option, we store the corner coordinates, and the cell center
           ! values of all variables. Due to this mix of corner/cell center, we must
           ! use BLOCK packaging, and thus we have enormous overhead by using
           ! calc_grid repeatedly to merely fill values of cell corner coordinates
           ! and cell center values per dimension, per variable
           if(ndim+nw+nwauxio>99) call mpistop(&
              "adjust format specification in writeout")
           if(nw+nwauxio==1)then
             ! to make tecplot happy: avoid [ndim+1-ndim+1] in varlocation varset
             ! and just set [ndim+1]
             if (mype==0.and.(nodesonlevelmype>0.and.&
                elemsonlevelmype>0))write(qunit,&
                "(a,i7,a,a,i7,a,i7,a,f25.16,a,i1,a,a)") 'ZONE T="',level,'"',&
                ', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                 ', SOLUTIONTIME=',global_time*time_convert_factor,&
                ', DATAPACKING=BLOCK, VARLOCATION=([', ndim+1,&
                ']=CELLCENTERED), ZONETYPE=',  'FEBRICK'
           else
             if(ndim+nw+nwauxio<10) then
               ! difference only in length of integer format specification for ndim+nw+nwauxio
               if (mype==0.and.(nodesonlevelmype>0.and.&
                  elemsonlevelmype>0))write(qunit,&
                  "(a,i7,a,a,i7,a,i7,a,f25.16,a,i1,a,i1,a,a)") 'ZONE T="',&
                  level,'"',', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                   ', SOLUTIONTIME=',global_time*time_convert_factor,&
                  ', DATAPACKING=BLOCK, VARLOCATION=([', ndim+1,'-',&
                  ndim+nw+nwauxio,']=CELLCENTERED), ZONETYPE=',  'FEBRICK'
             else
               if (mype==0.and.(nodesonlevelmype>0.and.&
                  elemsonlevelmype>0))write(qunit,&
                  "(a,i7,a,a,i7,a,i7,a,f25.16,a,i1,a,i2,a,a)") 'ZONE T="',&
                  level,'"',', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                   ', SOLUTIONTIME=',global_time*time_convert_factor,&
                  ', DATAPACKING=BLOCK, VARLOCATION=([', ndim+1,'-',&
                  ndim+nw+nwauxio,']=CELLCENTERED), ZONETYPE=',  'FEBRICK'
             end if
           end if

           do idim=1,ndim
             first=(idim==1)
             do Morton_no=Morton_start(mype),Morton_stop(mype)
               igrid = sfc_to_igrid(Morton_no)
               if (mype/=0)then
                 itag=Morton_no*idim
                 call mpi_send_wrapper(igrid,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
                 itag=igrid*idim
                 call mpi_send_wrapper(node(plevel_,igrid),1,MPI_INTEGER, 0,itag,icomm,&
                    ierrmpi)
               end if
               if (node(plevel_,igrid)/=level) cycle
               call calc_x(igrid,xC,xCC)
               call calc_grid(qunit,igrid,xC,xCC,xC_TMP,xCC_TMP,wC_TMP,wCC_TMP,&
                  normconv,ixCmin1,ixCmin2,ixCmin3,ixCmax1,ixCmax2,ixCmax3,&
                  ixCCmin1,ixCCmin2,ixCCmin3,ixCCmax1,ixCCmax2,ixCCmax3,first)
               if (mype/=0)then
                 ind_send=(/ ixCmin1,ixCmin2,ixCmin3,ixCmax1,ixCmax2,&
                    ixCmax3 /)
                 siz_ind=2*3
                 itag=igrid*idim
                 call mpi_send_wrapper(ind_send,siz_ind, MPI_INTEGER, 0,itag,icomm,&
                    ierrmpi)
                 call mpi_send_wrapper(normconv,nw+nwauxio+1,MPI_DOUBLE_PRECISION, 0,&
                    itag,icomm,ierrmpi)
                 call mpi_send_wrapper(xC_TMP,1,type_block_xc_io, 0,itag,icomm,&
                    ierrmpi)
               else
                 write(qunit,fmt="(100(e14.6))") xC_TMP(ixCmin1:ixCmax1,&
                    ixCmin2:ixCmax2,ixCmin3:ixCmax3,idim)*normconv(0)
               end if
             end do
           end do
           do iw=1,nw+nwauxio
             do Morton_no=Morton_start(mype),Morton_stop(mype)
               igrid = sfc_to_igrid(Morton_no)
               if(mype/=0)then
                 itag=Morton_no*(ndim+iw)
                 call mpi_send_wrapper(igrid,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
                 itag=igrid*(ndim+iw)
                 call mpi_send_wrapper(node(plevel_,igrid),1,MPI_INTEGER, 0,itag,icomm,&
                    ierrmpi)
               end if
               if (node(plevel_,igrid)/=level) cycle
               call calc_x(igrid,xC,xCC)
               call calc_grid(qunit,igrid,xC,xCC,xC_TMP,xCC_TMP,wC_TMP,wCC_TMP,&
                  normconv,ixCmin1,ixCmin2,ixCmin3,ixCmax1,ixCmax2,ixCmax3,&
                  ixCCmin1,ixCCmin2,ixCCmin3,ixCCmax1,ixCCmax2,ixCCmax3,&
                  .true.)
               if(mype/=0)then
                 ind_send=(/ ixCCmin1,ixCCmin2,ixCCmin3,ixCCmax1,ixCCmax2,&
                    ixCCmax3 /)
                 siz_ind=2*3
                 itag=igrid*(ndim+iw)
                 call mpi_send_wrapper(ind_send,siz_ind, MPI_INTEGER, 0,itag,icomm,&
                    ierrmpi)
                 call mpi_send_wrapper(normconv,nw+nwauxio+1,MPI_DOUBLE_PRECISION, 0,&
                    itag,icomm,ierrmpi)
                 call mpi_send_wrapper(wCC_TMP,1,type_block_wcc_io, 0,itag,icomm,&
                    ierrmpi)
               else
                 write(qunit,fmt="(100(e14.6))") wCC_TMP(ixCCmin1:ixCCmax1,&
                    ixCCmin2:ixCCmax2,ixCCmin3:ixCCmax3,iw)*normconv(iw)
               end if
             end do
           end do
         case default
           call mpistop('no such tecplot type')
       end select

       igonlevel=0
       do Morton_no=Morton_start(mype),Morton_stop(mype)
         igrid = sfc_to_igrid(Morton_no)
         if(mype/=0)then
           itag=Morton_no
           call mpi_send_wrapper(igrid,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
           itag=igrid
           call mpi_send_wrapper(node(plevel_,igrid),1,MPI_INTEGER, 0,itag,icomm,&
              ierrmpi)
         end if
         if(node(plevel_,igrid)/=level) cycle
         igonlevel=igonlevel+1
         if(mype/=0)then
           itag=igrid
           call mpi_send_wrapper(igonlevel,1,MPI_INTEGER, 0,itag,icomm,ierrmpi)
         end if
         if(mype==0)then
           call save_conntec(qunit,igrid,igonlevel)
         end if
       end do
    end do

    if(mype==0 .and.npe>1) then
      do ipe=1,npe-1
        itag=1000*Morton_stop(ipe)
        call mpi_recv_wrapper(levmin_recv,1,MPI_INTEGER, ipe,itag,icomm,intstatus(:,1),&
           ierrmpi)
        itag=2000*Morton_stop(ipe)
        call mpi_recv_wrapper(levmax_recv,1,MPI_INTEGER, ipe,itag,icomm,intstatus(:,1),&
           ierrmpi)
        do level=levmin_recv,levmax_recv
          nodesonlevelmype=NumGridsOnLevel_mype(level,ipe)*nxC1*nxC2*nxC3
          elemsonlevelmype=NumGridsOnLevel_mype(level,ipe)*nx1*nx2*nx3
          nodesonlevel=NumGridsOnLevel(level)*nxC1*nxC2*nxC3
          elemsonlevel=NumGridsOnLevel(level)*nx1*nx2*nx3
          select case(convert_type)
           case('tecplotmpi')
              ! in this option, we store the corner coordinates, as well as the corner
              ! values of all variables (obtained by averaging). This allows POINT packaging,
              ! and thus we can save full grid info by using one call to calc_grid
              if(nodesonlevelmype>0.and.elemsonlevelmype>0) write(qunit,&
                 "(a,i7,a,a,i7,a,i7,a,f25.16,a,a)") 'ZONE T="',level,'"',&
                 ', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                  ', SOLUTIONTIME=',global_time*time_convert_factor,&
                 ', DATAPACKING=POINT, ZONETYPE=',  'FEBRICK'
              do  Morton_no=Morton_start(ipe),Morton_stop(ipe)
               itag=Morton_no
               call mpi_recv_wrapper(igrid_recv,1,MPI_INTEGER, ipe,itag,icomm,&
                  intstatus(:,1),ierrmpi)
               itag=igrid_recv
               call mpi_recv_wrapper(level_recv,1,MPI_INTEGER, ipe,itag,icomm,&
                  intstatus(:,1),ierrmpi)
               if (level_recv/=level) cycle
               itag=Morton_no
               siz_ind=2*3
               call mpi_recv_wrapper(ind_recv,siz_ind, MPI_INTEGER, ipe,itag,icomm,&
                  intstatus(:,1),ierrmpi)
               ixrvCmin1=ind_recv(1);ixrvCmin2=ind_recv(2)
               ixrvCmin3=ind_recv(3);ixrvCmax1=ind_recv(3+1)
               ixrvCmax2=ind_recv(3+2);ixrvCmax3=ind_recv(3+3);
               call mpi_recv_wrapper(normconv,nw+nwauxio+1, MPI_DOUBLE_PRECISION,ipe,&
                  itag,icomm,intstatus(:,1),ierrmpi)
               call mpi_recv_wrapper(wC_TMP_recv,1,type_block_wc_io, ipe,itag,icomm,&
                  intstatus(:,1),ierrmpi)
               call mpi_recv_wrapper(xC_TMP_recv,1,type_block_xc_io, ipe,itag,icomm,&
                  intstatus(:,1),ierrmpi)
               do ix3=ixrvCmin3,ixrvCmax3
               do ix2=ixrvCmin2,ixrvCmax2
               do ix1=ixrvCmin1,ixrvCmax1
                    x_TEC(1:ndim)=xC_TMP_RECV(ix1,ix2,ix3,1:ndim)*normconv(0)
                    w_TEC(1:nw+nwauxio)=wC_TMP_RECV(ix1,ix2,ix3,&
                       1:nw+nwauxio)*normconv(1:nw+nwauxio)
                    write(qunit,fmt="(100(e14.6))") x_TEC, w_TEC
               end do
               end do
               end do
              end do
           case('tecplotCCmpi')
             ! in this option, we store the corner coordinates, and the cell center
             ! values of all variables. Due to this mix of corner/cell center, we must
             ! use BLOCK packaging, and thus we have enormous overhead by using
             ! calc_grid repeatedly to merely fill values of cell corner coordinates
             ! and cell center values per dimension, per variable
             if(ndim+nw+nwauxio>99) call &
                mpistop("adjust format specification in writeout")
             if(nw+nwauxio==1)then
               ! to make tecplot happy: avoid [ndim+1-ndim+1] in varlocation varset
               ! and just set [ndim+1]
               if(nodesonlevelmype>0.and.elemsonlevelmype>0) write(qunit,&
                  "(a,i7,a,a,i7,a,i7,a,f25.16,a,i1,a,a)") 'ZONE T="',level,'"',&
                  ', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                   ', SOLUTIONTIME=',global_time*time_convert_factor,&
                  ', DATAPACKING=BLOCK, VARLOCATION=([', ndim+1,&
                  ']=CELLCENTERED), ZONETYPE=',  'FEBRICK'
             else
               if(ndim+nw+nwauxio<10) then
                 ! difference only in length of integer format specification for ndim+nw+nwauxio
                 if(nodesonlevelmype>0.and.elemsonlevelmype>0) write(qunit,&
                    "(a,i7,a,a,i7,a,i7,a,f25.16,a,i1,a,i1,a,a)") 'ZONE T="',&
                    level,'"',', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                     ', SOLUTIONTIME=',global_time*time_convert_factor,&
                    ', DATAPACKING=BLOCK, VARLOCATION=([', ndim+1,'-',&
                    ndim+nw+nwauxio,']=CELLCENTERED), ZONETYPE=',  'FEBRICK'
               else
                 if(nodesonlevelmype>0.and.elemsonlevelmype>0) write(qunit,&
                    "(a,i7,a,a,i7,a,i7,a,f25.16,a,i1,a,i2,a,a)") 'ZONE T="',&
                    level,'"',', N=',nodesonlevelmype,', E=',elemsonlevelmype,&
                     ', SOLUTIONTIME=',global_time*time_convert_factor,&
                    ', DATAPACKING=BLOCK, VARLOCATION=([', ndim+1,'-',&
                    ndim+nw+nwauxio,']=CELLCENTERED), ZONETYPE=',  'FEBRICK'
               end if
             end if

             do idim=1,ndim
               do  Morton_no=Morton_start(ipe),Morton_stop(ipe)
                 itag=Morton_no*idim
                 call mpi_recv_wrapper(igrid_recv,1,MPI_INTEGER, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 itag=igrid_recv*idim
                 call mpi_recv_wrapper(level_recv,1,MPI_INTEGER, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 if (level_recv/=level) cycle
                 siz_ind=2*3
                 itag=igrid_recv*idim
                 call mpi_recv_wrapper(ind_recv,siz_ind, MPI_INTEGER, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 ixrvCmin1=ind_recv(1);ixrvCmin2=ind_recv(2)
                 ixrvCmin3=ind_recv(3);ixrvCmax1=ind_recv(3+1)
                 ixrvCmax2=ind_recv(3+2);ixrvCmax3=ind_recv(3+3);
                 call mpi_recv_wrapper(normconv,nw+nwauxio+1, MPI_DOUBLE_PRECISION,ipe,&
                    itag,icomm,intstatus(:,1),ierrmpi)
                 call mpi_recv_wrapper(xC_TMP_recv,1,type_block_xc_io, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 write(qunit,fmt="(100(e14.6))") &
                    xC_TMP_recv(ixrvCmin1:ixrvCmax1,ixrvCmin2:ixrvCmax2,&
                    ixrvCmin3:ixrvCmax3,idim)*normconv(0)
               end do
             end do
             do iw=1,nw+nwauxio
               do Morton_no=Morton_start(ipe),Morton_stop(ipe)
                 itag=Morton_no*(ndim+iw)
                 call mpi_recv_wrapper(igrid_recv,1,MPI_INTEGER, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 itag=igrid_recv*(ndim+iw)
                 call mpi_recv_wrapper(level_recv,1,MPI_INTEGER, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 if (level_recv/=level) cycle
                 siz_ind=2*3
                 itag=igrid_recv*(ndim+iw)
                 call mpi_recv_wrapper(ind_recv,siz_ind, MPI_INTEGER, ipe,itag,icomm,&
                    intstatus(:,1),ierrmpi)
                 ixrvCCmin1=ind_recv(1);ixrvCCmin2=ind_recv(2)
                 ixrvCCmin3=ind_recv(3);ixrvCCmax1=ind_recv(3+1)
                 ixrvCCmax2=ind_recv(3+2);ixrvCCmax3=ind_recv(3+3);
                 call mpi_recv_wrapper(normconv,nw+nwauxio+1, MPI_DOUBLE_PRECISION,ipe,&
                    itag,icomm,intstatus(:,1),ierrmpi)
                 call mpi_recv_wrapper(wCC_TMP_recv,1,type_block_wcc_io, ipe,itag,&
                    icomm,intstatus(:,1),ierrmpi)
                 write(qunit,fmt="(100(e14.6))") &
                    wCC_TMP_recv(ixrvCCmin1:ixrvCCmax1,ixrvCCmin2:ixrvCCmax2,&
                    ixrvCCmin3:ixrvCCmax3,iw)*normconv(iw)
               end do
             end do
           case default
             call mpistop('no such tecplot type')
          end select

          do Morton_no=Morton_start(ipe),Morton_stop(ipe)
            itag=Morton_no
            call mpi_recv_wrapper(igrid_recv,1,MPI_INTEGER, ipe,itag,icomm,intstatus(:,&
               1),ierrmpi)
            itag=igrid_recv
            call mpi_recv_wrapper(level_recv,1,MPI_INTEGER, ipe,itag,icomm,intstatus(:,&
               1),ierrmpi)
            if (level_recv/=level) cycle
            itag=igrid_recv
            call mpi_recv_wrapper(igonlevel_recv,1,MPI_INTEGER, ipe,itag,icomm,&
               intstatus(:,1),ierrmpi)
            call save_conntec(qunit,igrid_recv,igonlevel_recv)
          end do ! morton loop
        end do ! level loop
      end do ! ipe loop
    end if ! mype=0 if


    if (npe>1) then
      call MPI_BARRIER(icomm,ierrmpi)
      if(mype==0)deallocate(intstatus)
    end if

  end subroutine tecplot_mpi