Commit 44da542b authored by Francois Tessier's avatar Francois Tessier

TAPIOCA module for S3D-IO

parent 299ea71c
......@@ -22,7 +22,7 @@
# New version of makefile with dependencies auto-inserted using sfmakedepend
#----------------------------------------------------------------------
MACH = BGQ
MACH = HASWELL
include make.$(MACH)
......@@ -72,6 +72,7 @@ SRC_MODULES = \
$(MODULES)mpi_io_m.f90 \
$(MODULES)pnetcdf_m.f90 \
$(MODULES)hdf5_m.f90 \
$(MODULES)tapioca_m.f90 \
$(MODULES)check_io_m.f90
......@@ -316,3 +317,5 @@ work_m.xyz = ../source/modules/work_m.o
../source/modules/hdf5_m.o: ../source/modules/hdf5_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz)
$(COMPILE_F90) $(BUILD_HDF5_PNETCDF) -I$(FSEVEN_HDR) $(HDF5_FLIB) $<
../source/modules/tapioca_m.o: ../source/modules/tapioca_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz)
\ No newline at end of file
......@@ -5,12 +5,12 @@ MODE
==========================================================================================
GRID DIMENSION PARAMETERS
==========================================================================================
256 - global number of grid points in the x-direction (nx_g)
256 - global number of grid points in the y-direction (ny_g)
256 - global number of grid points in the z-direction (nz_g)
8 - number of processors in x-direction (npx)
8 - number of processors in y-direction (npy)
8 - number of processors in z-direction (npz)
128 - global number of grid points in the x-direction (nx_g)
128 - global number of grid points in the y-direction (ny_g)
128 - global number of grid points in the z-direction (nz_g)
4 - number of processors in x-direction (npx)
4 - number of processors in y-direction (npy)
4 - number of processors in z-direction (npz)
==========================================================================================
RUN-TIME PARAMETERS
==========================================================================================
......@@ -70,7 +70,7 @@ REQUIRED REFERENCE VALUES
==========================================================================================
flag to enable/disable MPI I/O
==========================================================================================
1 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5
4 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5, 4:TAPIOCA
==========================================================================================
Output directory
==========================================================================================
......
......@@ -103,7 +103,7 @@ subroutine solve_driver(io)
! wkliao: define MPI I/O file type to be used to define file view
if (io_method .EQ. 1) then
#ifdef MPIIO
call mpi_io_set_filetype(0)
call mpi_io_set_filetype(0)
#endif
endif
......@@ -291,6 +291,7 @@ subroutine solve_driver(io)
#endif
endif
call date_and_time(dat_1,tim_stop)
if(myid==0) then
call write_header(io,'=')
......
......@@ -31,6 +31,7 @@ subroutine read_savefile(io, precsn, dirtime)
use mpi_io_m, only : mpi_file_int_io, mpi_io_set_filetype, &
mpi_io_set_filetype_interp
#endif
use tapioca_m, only : tapioca_io
use pnetcdf_m, only : pnetcdf_read
use hdf5_m, only : hdf5_read
#endif
......@@ -191,7 +192,23 @@ subroutine read_savefile(io, precsn, dirtime)
call terminate_run(io,0)
endif
elseif ( (io_method .EQ. 4.or. io_method.eq.5 .or. io_method.eq.6) ) then
elseif (io_method .EQ. 4 ) then
! Read using TAPIOCA
filename = '../data/'//trim(run_title)//'.'//dirtime2//'.field.tp'
inquire(file=trim(filename), exist=exist)
if(exist) then
if(myid == 0) write(io,*) 'Reading collectively with TAPIOCA'
#ifdef MPIIO
call tapioca_io (filename, 'r')
#endif
else
call MPI_Barrier(gcomm,ierr)
write(io,*) 'the following restart file does not exist'
write(io,*) trim(filename)
call MPI_Barrier(gcomm,ierr)
call terminate_run(io,0)
endif
elseif ( (io_method.eq.5 .or. io_method.eq.6) ) then
! Read using ADIOS
filename = '../data/'//trim(run_title)//'.'//dirtime2//'.field.bp'
if( io_method .eq. 6) filename = trim(run_title)//'.'//dirtime2//'.field.bp'
......
......@@ -12,6 +12,7 @@ subroutine write_savefile(io, precsn)
use param_m, only : output_dir
#ifdef MPIIO
use mpi_io_m, only : mpi_file_io
use tapioca_m, only : tapioca_io
use pnetcdf_m, only : pnetcdf_write
use hdf5_m, only : hdf5_write
#endif
......@@ -56,7 +57,7 @@ subroutine write_savefile(io, precsn)
elseif (io_method .EQ. 3) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.h5'
elseif (io_method .EQ. 4) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.tp'
elseif (io_method .EQ. 5) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
elseif (io_method .EQ. 6) then
......@@ -101,7 +102,7 @@ subroutine write_savefile(io, precsn)
endif
endif
call MPI_Barrier(gcomm, ierr)
if (io_method .EQ. 1.or.io_method .eq. 4) then
if (io_method .EQ. 1) then
#ifdef MPIIO
call mpi_file_io(filename, 'w')
#endif
......@@ -113,7 +114,11 @@ subroutine write_savefile(io, precsn)
#ifdef MPIIO
call hdf5_write(filename)
#endif
elseif (io_method .EQ. 4 .or. io_method .EQ. 5 .or. io_method .EQ. 6) then
elseif (io_method .EQ. 4) then
#ifdef MPIIO
call tapioca_io(filename, 'w')
#endif
elseif (io_method .EQ. 5 .or. io_method .EQ. 6) then
#ifdef ADIOS
call readwrite_savefile_data_adios(filename,'w')
#endif
......@@ -182,7 +187,7 @@ contains
elseif (io_method .EQ. 3) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.h5'
elseif (io_method .EQ. 4) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.tp'
elseif (io_method .EQ. 5) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
else
......
......@@ -9,6 +9,7 @@ subroutine write_savefile(io,aprecsn)
use param_m, only : io_method
#ifdef MPIIO
use mpi_io_m, only : mpi_file_io
use tapioca_m, only : tapioca_io
use pnetcdf_m, only : pnetcdf_write
use hdf5_m, only : hdf5_write
#endif
......@@ -44,7 +45,7 @@ subroutine write_savefile(io,aprecsn)
elseif (io_method .EQ. 3) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.h5'
elseif (io_method .EQ. 4) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.tp'
elseif (io_method .EQ. 5) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
else
......@@ -87,7 +88,7 @@ subroutine write_savefile(io,aprecsn)
! call execute_command( trim(tarcmd) )
! endif
!endif
if (io_method .EQ. 1.or.io_method .eq. 4) then
if (io_method .EQ. 1) then
call MPI_Barrier(gcomm, ierr)
if(myid.eq.0) then
write(io,100) 'writing nw mpi save files for: i_time = ',&
......@@ -153,6 +154,10 @@ subroutine write_savefile(io,aprecsn)
elseif (io_method .EQ. 3) then
#ifdef MPIIO
call hdf5_write(filename)
#endif
elseif (io_method .EQ. 4) then
#ifdef MPIIO
call tapioca_io(filename, 'w')
#endif
elseif (io_method .EQ. 5) then
#ifdef ADIOS
......@@ -252,7 +257,7 @@ contains
elseif (io_method .EQ. 3) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.h5'
elseif (io_method .EQ. 4) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.tp'
elseif (io_method .EQ. 5) then
filename = '../data/'//trim(run_title)//'.'//time_ext//'.field.bp'
else
......
!
! Author: François Tessier
! Argonne National Lab
! ftessier@anl.gov
!
!==============================================================================
module tapioca_m
! module for Read-Write Restart files using TAPIOCA
implicit none
contains
subroutine tapioca_io(filename,rw)
use topology_m, only : gcomm
use param_m, only : nx, ny, nz, nsc
use variables_m, only : temp, pressure, yspecies, u
implicit none
include 'mpif.h'
! declarations passed in
character*1, intent(in) :: rw ! 'r' for read, 'w' for write
character*100, intent(inout) :: filename
! local variables
integer nx_ny_nz
integer fp, ierr, errorclass
integer mstatus(MPI_STATUS_SIZE)
integer(MPI_OFFSET_KIND) iOffset
! TAPIOCA variables
integer*8 chunkcount(4)
integer chunksize(4)
integer*8 chunkoffset(4)
integer*8 buffsize, scansize, hdr
integer typesize
call MPI_Type_size ( MPI_REAL8, typesize, ierr )
nx_ny_nz = nx * ny * nz
buffsize = ( nx_ny_nz * ( nsc + 1 ) + nx_ny_nz + nx_ny_nz + nx_ny_nz * 3 ) * typesize
scansize = 0
hdr = 0
call MPI_Exscan ( buffsize, scansize, 1, MPI_REAL8, MPI_SUM, gcomm, ierr )
! yspecies
chunkcount (1) = nx_ny_nz * ( nsc + 1 )
chunksize (1) = typesize
chunkoffset (1) = scansize
! temp
chunkcount (2) = nx_ny_nz
chunksize (2) = typesize
chunkoffset (2) = chunkoffset (1) + chunkcount (1) * chunksize (1)
! pressure
chunkcount (3) = nx_ny_nz
chunksize (3) = typesize
chunkoffset (3) = chunkoffset (2) + chunkcount (2) * chunksize (2)
! u
chunkcount (4) = nx_ny_nz * 3
chunksize (4) = typesize
chunkoffset (4) = chunkoffset (3) + chunkcount (3) * chunksize (3)
!---- initialize
call tapioca_init ( chunkcount, chunksize, chunkoffset, 4, hdr, gcomm )
!---- write/read array yspecies
if (rw .EQ. 'w') then
call tapioca_write ( chunkoffset (1), yspecies, chunkcount (1), MPI_REAL8, mstatus )
else
call tapioca_read ( chunkoffset (1), yspecies, chunkcount (1), MPI_REAL8, mstatus )
endif
!---- write/read array temp
if (rw .EQ. 'w') then
call tapioca_write ( chunkoffset (2), temp, chunkcount (2), MPI_REAL8, mstatus )
else
call tapioca_read ( chunkoffset (2), temp, chunkcount (2), MPI_REAL8, mstatus )
endif
!---- write/read array pressure
if (rw .EQ. 'w') then
call tapioca_write ( chunkoffset (3), pressure, chunkcount (3), MPI_REAL8, mstatus )
else
call tapioca_read ( chunkoffset (3), pressure, chunkcount (3), MPI_REAL8, mstatus )
endif
!---- write/read array u
if (rw .EQ. 'w') then
call tapioca_write ( chunkoffset (4), u, chunkcount (4), MPI_REAL8, mstatus )
else
call tapioca_read ( chunkoffset (4), u, chunkcount (4), MPI_REAL8, mstatus )
endif
!---- finalize
call tapioca_finalize ( )
return
end subroutine tapioca_io
!==============================================================================
end module tapioca_m
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment