      subroutine exchange_ghostline1(a,m,jf,jl,my_rank,nproc,mpi_comm,
     $     irealtype)
      implicit real*8(a-h,o-z)
      include "mpif.h"
c     
c     /* performs the communication of ghost lines on both extremes of */
c     /* the second index of a 2d field between neighboring processors */
c     /* in a 1d cartesian processor "grid". extreme processors do not */
c     /* exchange as their ghost lines correspond to physical bcs.     */
c     /*                                                               */
c     /* uses MPI_SENDRECV, therefore the communication is done in     */
c     /* about nproc steps.                                            */
c     /*                                                               */
c     /* input                                                         */
c     /* a(0:m+1,jf-1:jl+1)   array in local memory                    */
c     /* my_rank              processor rank                           */
c     /* nproc                no of procs in communicator              */
c     /* mpi_comm             mpi communicator                         */
c     /* irealtype            the data-type for the current floats     */
c     /*                                                               */
c     /* output                                                        */
c     /* a(0:m+1,jf-1:jl+1)   array in local memory; the ghost lines   */
c     /*                       have been exchanged with neighbors,     */
c     /*                       except for extreme processors           */
c     /*                                                               */

c     
c     /* mpi related */
      integer istat(MPI_STATUS_SIZE)
c     
c     /* arguments */
      dimension a(0:m+1,jf-1:jl+1)
c
c     /* etc */
      parameter(itag=10)
c-----|---------------------------------------------------------------|
#ifdef DEBUG_COMM
      write(*,*)my_rank,' hello inside exchange_ghostline1'
#endif
      nline=m+2
      if (my_rank.gt.0) then
#ifdef DEBUG_COMM
         write(*,*)my_rank,' about to sendrecv with ',my_rank-1
#endif
         CALL MPI_SENDRECV(
     $        a(0,jf)  ,nline,irealtype,my_rank-1,itag, 
     $        a(0,jf-1),nline,irealtype,my_rank-1,itag,
     $        mpi_comm,istat,ierr)
#ifdef DEBUG_COMM
         write(*,*)my_rank,' finished sendrecv with ',my_rank-1
#endif
      endif
      if (my_rank.lt.nproc-1) then
#ifdef DEBUG_COMM
         write(*,*)my_rank,' about to sendrecv with ',my_rank+1
#endif
         CALL MPI_SENDRECV(
     $        a(0,jl)  ,nline,irealtype,my_rank+1,itag, 
     $        a(0,jl+1),nline,irealtype,my_rank+1,itag,
     $        mpi_comm,istat,ierr)
#ifdef DEBUG_COMM
         write(*,*)my_rank,' finished sendrecv with ',my_rank+1
#endif
      endif      
c     
      return
      end
c-----|---------------------------------------------------------------|
      subroutine exchange_ghostline2(a,m,jf,jl,my_rank,nproc,mpi_comm,
     $     irealtype)
      implicit real*8(a-h,o-z)
      include "mpif.h"
c     
c     /* performs the communication of ghost lines on both extremes of */
c     /* the second index of a 2d field between neighboring processors */
c     /* in a 1d cartesian processor "grid". extreme processors do not */
c     /* exchange as their ghost lines correspond to physical bcs.     */
c     /*                                                               */
c     /* uses MPI_ISEND, MPI_IRECV therefore the communication is done */
c     /* non-blockingly in 4 steps.                                    */
c     /*                                                               */
c     /* input                                                         */
c     /* a(0:m+1,jf-1:jl+1)   array in local memory                    */
c     /* my_rank              processor rank                           */
c     /* nproc                no of procs in communicator              */
c     /* mpi_comm             mpi communicator                         */
c     /* irealtype            the data-type for the current floats     */
c     /*                                                               */
c     /* output                                                        */
c     /* a(0:m+1,jf-1:jl+1)   array in local memory; the ghost lines   */
c     /*                       have been exchanged with neighbors,     */
c     /*                       except for extreme processors           */
c     /*                                                               */
c     /*                                                               */
c     
c     /* mpi related */
      parameter(nmsg=2)
      integer istat(MPI_STATUS_SIZE,2*nmsg),ireq(2*nmsg)
      integer iseq_send(nmsg),iseq_recv(nmsg)
      data iseq_send /-1,+1/ !order of the send operations
      data iseq_recv /+1,-1/ !order of the recv operations
c     
c     /* arguments */
      dimension a(0:m+1,jf-1:jl+1)
c
c     /* etc */
      parameter(itag=10)
c-----|---------------------------------------------------------------|
      nline=m+2
      nreq=0
c     
      do ii=1,nmsg
c     
c     /* send to next in sender-list 'iseq_send' */
c     
         idest=my_rank+iseq_send(ii)
         if (idest.ge.0.and.idest.le.nproc-1) then
            if(iseq_send(ii).eq.+1)then
               jj=jl
            else
               jj=jf
            endif
            nreq=nreq+1
#ifdef DEBUG_COMM
      write(*,*)my_rank,' about to MPI_ISEND ',nline,jj,' to ',idest
#endif
            call MPI_ISEND(a(0,jj),nline,irealtype,
     $           idest,itag,mpi_comm,ireq(nreq),ierr)
#ifdef DEBUG_COMM
      write(*,*)my_rank,' finished MPI_ISEND ',nline,jj,' to ',idest
#endif
         end if
c     
c     /* receive from next in receiver-list 'iseq_recv' */
c     
         iorig=my_rank+iseq_recv(ii)
         if (iorig.ge.0.and.iorig.le.nproc-1) then
            if(iseq_recv(ii).eq.+1)then
               jj=jl+1
            else
               jj=jf-1
            endif
            nreq=nreq+1
#ifdef DEBUG_COMM
      write(*,*)my_rank,' about to MPI_IRECV ',nline,jj,' from ',iorig
#endif
            call MPI_IRECV(a(0,jj),nline,irealtype,
     $           iorig,itag,mpi_comm,ireq(nreq),ierr)
#ifdef DEBUG_COMM
      write(*,*)my_rank,' finished MPI_IRECV ',nline,jj,' from ',iorig
#endif
         end if
c     
      enddo
c     
c     /* wait for all the posted operations to complete */
c     
#ifdef DEBUG_COMM
      write(*,*)my_rank,' waiting at MPI_WAITALL ',nreq
#endif
      call MPI_WAITALL(nreq,ireq,istat,ierr)
#ifdef DEBUG_COMM
      write(*,*)my_rank,' finished MPI_WAITALL ',nreq
#endif
c     
      return
      end
c-----|---------------------------------------------------------------|
      subroutine exchange_ghostline3(a,m,jf,jl,my_rank,nproc,mpi_comm,
     $     irealtype)
      implicit real*8(a-h,o-z)
      include "mpif.h"
c     
c     /* performs the communication of ghost lines on both extremes of */
c     /* the second index of a 2d field between neighboring processors */
c     /* in a 1d cartesian processor "grid". extreme processors do not */
c     /* exchange as their ghost lines correspond to physical bcs.     */
c     /*                                                               */
c     /* uses MPI_SENDRECV. please chak-out this code carefully.       */
c     /* what happens when you vary the number of processors?          */
c     /*                                                               */
c     /* input                                                         */
c     /* a(0:m+1,jf-1:jl+1)   array in local memory                    */
c     /* my_rank              processor rank                           */
c     /* nproc                no of procs in communicator              */
c     /* mpi_comm             mpi communicator                         */
c     /*                                                               */
c     /* output                                                        */
c     /* a(0:m+1,jf-1:jl+1)   array in local memory; the ghost lines   */
c     /*                       have been exchanged with neighbors,     */
c     /*                       except for extreme processors           */
c     /*                                                               */
c     /* written by: itano-san                                         */
c     /*             (only works for nproc ODD !!!)                    */

c     
c     /* mpi related */
      integer istat(MPI_STATUS_SIZE)
c     
c     /* arguments */
      dimension a(0:m+1,jf-1:jl+1)
c
c     /* etc */
      parameter(itag=10)
c-----|---------------------------------------------------------------|
      nline=m+2
 1    format('I am ',i3,',   communicating with',i3)
c     
      if (my_rank.lt.nproc-1) then
         if(mod(my_rank,2).eq.0) then
#ifdef DEBUG_COMM
            write(*,1) my_rank,my_rank+1
#endif
            CALL MPI_SENDRECV(
     $           a(0,jl)  ,nline,irealtype,my_rank+1,itag, 
     $           a(0,jl+1),nline,irealtype,my_rank+1,itag,
     $           mpi_comm,istat,ierr)
         endif      
         if(mod(my_rank,2).eq.1) then
#ifdef DEBUG_COMM
            write(*,1) my_rank,my_rank-1
#endif
            CALL MPI_SENDRECV(
     $           a(0,jf)  ,nline,irealtype,my_rank-1,itag, 
     $           a(0,jf-1),nline,irealtype,my_rank-1,itag,
     $           mpi_comm,istat,ierr)
         endif
      endif
      
#ifdef DEBUG_COMM
      call MPI_BARRIER(MPI_COMM_WORLD,ierr)
      write(*,*) 'timing...'
#endif
c      
      if (my_rank.gt.0) then
         if(mod(my_rank,2).eq.0) then
#ifdef DEBUG_COMM
            write(*,1) my_rank,my_rank-1
#endif
            CALL MPI_SENDRECV(
     $           a(0,jf)  ,nline,irealtype,my_rank-1,itag, 
     $           a(0,jf-1),nline,irealtype,my_rank-1,itag,
     $           mpi_comm,istat,ierr)
         endif
         if(mod(my_rank,2).eq.1) then
#ifdef DEBUG_COMM
            write(*,1) my_rank,my_rank+1
#endif
            CALL MPI_SENDRECV(
     $           a(0,jl)  ,nline,irealtype,my_rank+1,itag, 
     $           a(0,jl+1),nline,irealtype,my_rank+1,itag,
     $           mpi_comm,istat,ierr)
         endif      
      endif
c
      return
      end
c-----|---------------------------------------------------------------|
