Commit d68c2b08 authored by Francois Tessier's avatar Francois Tessier

Fix S3D-IO. Need to validate with experiments.

parent c94d8326
......@@ -9,7 +9,7 @@ examples/HACC-IO/bin/*
examples/1D-Array/bin/*
examples/C/bin/*
examples/Fortran/bin/*
examples/S3D-IO/run/*.out
examples/S3D-IO/run/writeperf.log
examples/S3D-IO/data/*
examples/S3D-IO/run/s3d_f90.x
examples/S3D-IO/run/output
\ No newline at end of file
......@@ -22,9 +22,10 @@
# New version of makefile with dependencies auto-inserted using sfmakedepend
#----------------------------------------------------------------------
MACH = BGQ
include make.$(MACH)
#MACH = XC40
#
#include make.$(MACH)
include ../../make.inc/Makefile.$(ARCHI)
MYERRMSG =
MAKEDEPEND = ./sfmakedepend
......@@ -318,4 +319,4 @@ work_m.xyz = ../source/modules/work_m.o
../source/modules/hdf5_m.o: ../source/modules/hdf5_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz)
$(COMPILE_F90) $(BUILD_HDF5_PNETCDF) -I$(FSEVEN_HDR) $(HDF5_FLIB) $<
../source/modules/tapioca_m.o: ../source/modules/tapioca_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz)
\ No newline at end of file
../source/modules/tapioca_m.o: ../source/modules/tapioca_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz)
# CPQ: define this and the code will use system(3)
# SAVEFILEINSEPDIR: do not know
directives = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO -I$(HOME)/install/$(ARCHI)/include/ -DDBG -DCOLOR
MPIF90 = ftn
MPIF77 = ftn
MPICC = cc
LDFLAGS = ../../../bindings/F90/tapioca_f90api.o -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca -lstdc++ -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
COMPILE_F90 = ${MPIF90} -g $(directives) -fpp -c -o $@
COMPILE_F77 = ${MPIF77} -g -fpp -c -o $@
COMPILE_C = ${MPIF90} -g -c -o $@
LINK = ${MPIF90} ${LDFLAGS}
......@@ -5,6 +5,6 @@ time period to save fields in restart files (time_save_inc) 1.00000E+05
frequency to monitor min/max files and active (i_time_mon) 10
frequency to check resolution; set < 0 for no check (i_time_res) -1
frequency write current tecplot file (i_time_tec) -1
frequency to filter solution vector (i_time_fil) 31
frequency to filter solution vector (i_time_fil) -1431142048
initial/constant timestep (sec) (tstep_init) 0.00000E+00
timestep frequency to write controller info (i_time_cont) 0
......@@ -5,11 +5,11 @@ MODE
==========================================================================================
GRID DIMENSION PARAMETERS
==========================================================================================
128 - global number of grid points in the x-direction (nx_g)
128 - global number of grid points in the y-direction (ny_g)
128 - global number of grid points in the z-direction (nz_g)
4 - number of processors in x-direction (npx)
4 - number of processors in y-direction (npy)
256 - global number of grid points in the x-direction (nx_g)
256 - global number of grid points in the y-direction (ny_g)
128 - global number of grid points in the z-direction (nz_g)
8 - number of processors in x-direction (npx)
8 - number of processors in y-direction (npy)
4 - number of processors in z-direction (npz)
==========================================================================================
RUN-TIME PARAMETERS
......@@ -70,8 +70,8 @@ REQUIRED REFERENCE VALUES
==========================================================================================
flag to enable/disable MPI I/O
==========================================================================================
4 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5, 4:TAPIOCA
1 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5, 4:TAPIOCA
==========================================================================================
Output directory
==========================================================================================
/projects/visualization/ftessier/S3D
/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA
#!/bin/bash
NODES=16
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA"
DDT="/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT=16
STRIPE_SIZE=8388608
AGGR_MULT=1
cd $HOME/TAPIOCA/examples/S3D-IO/run/
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_NBBUFFERS=2
export TAPIOCA_BUFFERSIZE=16777216
export TAPIOCA_AGGRTIER=DDR
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=false
function setLustreFS ()
{
rm $TARGET/*
lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET
lfs getstripe $TARGET
}
function updateSettings()
{
printenv | egrep "TAPIOCA_"
SETTINGS="-e TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="$SETTINGS -e TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
SETTINGS="$SETTINGS -e TAPIOCA_NBBUFFERS=$TAPIOCA_NBBUFFERS"
SETTINGS="$SETTINGS -e TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
SETTINGS="$SETTINGS -e TAPIOCA_AGGRTIER=$TAPIOCA_AGGRTIER"
SETTINGS="$SETTINGS -e TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS -e TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS -e TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
SETTINGS="$SETTINGS -e TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"
SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
}
rm $TARGET/*
setLustreFS
updateSettings
aprun $SETTINGS -n $NPROCS -N $PPN ./s3d_f90.x
......@@ -125,7 +125,6 @@ subroutine read_savefile(io, precsn, dirtime)
! Read the data to intermediate arrays and interpolate as necessary
! No interpolation / morphing implemented for p-netcdf or hdf5
if (io_method .EQ. 1 ) then
! Read using MPI-IO
filename = '../data/'//trim(run_title)//'.'//dirtime2//'.field.mpi'
......
......@@ -9,10 +9,9 @@ subroutine write_savefile(io, precsn)
use param_m, only : nx, ny, nz, n_spec !quad
use param_m, only : io_method
use param_m, only : output_dir
#ifdef MPIIO
use mpi_io_m, only : mpi_file_io
use tapioca_m, only : tapioca_io
use mpi_io_m, only : mpi_file_io
use tapioca_m, only : tapioca_io
use pnetcdf_m, only : pnetcdf_write
use hdf5_m, only : hdf5_write
#endif
......@@ -28,7 +27,7 @@ subroutine write_savefile(io, precsn)
! other declarations
character*5 :: myid_ext*5, time_ext*10, filename*100, dirname*100
character*5 :: myid_ext*5, time_ext*10, filename*100, dirname*100, outputdir*100
character*100 :: bpfile
integer mz
logical :: exist
......@@ -50,21 +49,34 @@ subroutine write_savefile(io, precsn)
! wkliao: set a different file name for MPI-IO, pnetcdf, and HDF5 methods
! wkliao: no subdirectories are created if io_method > 0
outputdir = '/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA/'
if (io_method .EQ. 1) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.mpi'
if(myid.eq.0) then
write(io,100) 'MPI-IO driver'
endif
filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.mpi'
elseif (io_method .EQ. 2) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.nc'
if(myid.eq.0) then
write(io,100) 'NetCFD driver'
endif
filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.nc'
elseif (io_method .EQ. 3) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.h5'
if(myid.eq.0) then
write(io,100) 'HDF5 driver'
endif
filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.h5'
elseif (io_method .EQ. 4) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.tp'
if(myid.eq.0) then
write(io,100) 'TAPIOCA driver'
endif
filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.tp'
elseif (io_method .EQ. 5) then
filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp'
filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.bp'
elseif (io_method .EQ. 6) then
filename = trim(run_title)//'.'//time_ext//'.field.bp'
else
dirname = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'/'
dirname = '../data/'//trim(run_title)//'.'//time_ext//'/'
if(myid == 0) then
#ifdef SYSTEMCALLWONTWORK
! when system calls do not work, as on infiniband
......
......@@ -60,7 +60,7 @@
integer io_method
character*100 output_dir !output directory
character*100 output_dir
character*8 dat_1, dat_2 !for start and end date (wall clock)
character*10 tim_1, tim_2 !for start and end time (wall clock)
......
......@@ -25,7 +25,7 @@ subroutine tapioca_io(filename,rw)
! local variables
integer nx_ny_nz
integer i, fp, ierr, errorclass
integer i, fp, ierr, errorclass, l
integer mstatus(MPI_STATUS_SIZE)
integer(MPI_OFFSET_KIND) iOffset
......@@ -34,7 +34,7 @@ subroutine tapioca_io(filename,rw)
integer chunksize(4)
integer*8 chunkoffset(4)
integer*8 buffsize, scansize, hdr, filesize, totsize
integer typesize, agg_mem, target_mem, nbuff
integer typesize
call MPI_Type_size ( MPI_REAL8, typesize, ierr )
nx_ny_nz = nx * ny * nz
......@@ -43,10 +43,6 @@ subroutine tapioca_io(filename,rw)
totsize = 0
filesize = 0
hdr = 0
! DDR=0, HBM=1, SSD=2, HDD=3, NVR=4
agg_mem = 0
target_mem = 3
nbuff = 2
call MPI_Exscan ( buffsize, scansize, 1, MPI_REAL8, MPI_SUM, gcomm, ierr )
......@@ -77,10 +73,8 @@ subroutine tapioca_io(filename,rw)
call MPI_Allreduce ( totsize, filesize, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD, ierr )
!---- initialize
call tapioca_init ( chunkcount, chunksize, chunkoffset, 4, hdr, gcomm )
call tapioca_set_aggregation_tier ( nbuff, agg_mem, "" )
call tapioca_set_target_tier ( target_mem, filesize, filename )
call tapioca_init ( chunkcount, chunksize, chunkoffset, 4, hdr, trim(filename)//CHAR(0), gcomm )
!---- write/read array yspecies
if (rw .EQ. 'w') then
call tapioca_write ( chunkoffset (1), yspecies, chunkcount (1), MPI_REAL8, mstatus )
......
MPICXX = CC
MPIF90 = ftn
MPIF77 = ftn
MPICC = cc
MPI_CFLAGS = -g -O3 -I$(HOME)/install/$(ARCHI)/include/ -DDBG -DCOLOR
MPI_LDFLAGS = -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive
MPI_F90_FLAGS = $(MPI_CFLAGS)
MPI_F90_FLAGS = -g -O0 -I$(HOME)/install/$(ARCHI)/include/ -DDBG -DCOLOR -fpp
MPI_F90_LDFLAGS = $(MPI_LDFLAGS) -lstdc++
INSTALL_PATH = $(HOME)/install/$(ARCHI)
###### S3D-IO ONLY ######
# CPQ: define this and the code will use system(3)
# SAVEFILEINSEPDIR: do not know
S3D_F90_FLAGS = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO ${MPI_F90_FLAGS}
S3D_F90_LDFLAGS = ../../../bindings/F90/tapioca_f90api.o ${MPI_F90_LDFLAGS}
COMPILE_F90 = ${MPIF90} -g ${S3D_F90_FLAGS} -c -o $@
COMPILE_F77 = ${MPIF77} -g ${S3D_F90_FLAGS} -c -o $@
COMPILE_C = ${MPICC} -g -c -o $@
LINK = ${MPIF90} ${S3D_F90_LDFLAGS}
###### END ######
###### BT ONLY ######
BT_F90_FLAGS = ${MPI_F90_FLAGS}
BT_F90_LDFLAGS = ../../bindings/F90/tapioca_f90api.o ${MPI_F90_LDFLAGS}
###### END ######
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment