diff --git a/.gitignore b/.gitignore index 6cdb65fa4163545568ff9c1ab76c1d9d0e65aca1..f4dc9600eeb6fea0cde4f52bef8fa989705e7e6e 100644 --- a/.gitignore +++ b/.gitignore @@ -9,7 +9,7 @@ examples/HACC-IO/bin/* examples/1D-Array/bin/* examples/C/bin/* examples/Fortran/bin/* -examples/S3D-IO/run/*.out examples/S3D-IO/run/writeperf.log examples/S3D-IO/data/* examples/S3D-IO/run/s3d_f90.x +examples/S3D-IO/run/output \ No newline at end of file diff --git a/examples/S3D-IO/build/Makefile b/examples/S3D-IO/build/Makefile index 3200e876db39aff12ffd24affa761e8c449a2f45..3c2c45d06771534caeba8780302ed5233bfbbb66 100644 --- a/examples/S3D-IO/build/Makefile +++ b/examples/S3D-IO/build/Makefile @@ -22,9 +22,10 @@ # New version of makefile with dependencies auto-inserted using sfmakedepend #---------------------------------------------------------------------- -MACH = BGQ - -include make.$(MACH) +#MACH = XC40 +# +#include make.$(MACH) +include ../../make.inc/Makefile.$(ARCHI) MYERRMSG = MAKEDEPEND = ./sfmakedepend @@ -318,4 +319,4 @@ work_m.xyz = ../source/modules/work_m.o ../source/modules/hdf5_m.o: ../source/modules/hdf5_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz) $(COMPILE_F90) $(BUILD_HDF5_PNETCDF) -I$(FSEVEN_HDR) $(HDF5_FLIB) $< -../source/modules/tapioca_m.o: ../source/modules/tapioca_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz) \ No newline at end of file +../source/modules/tapioca_m.o: ../source/modules/tapioca_m.f90 $(param_m.xyz) $(topology_m.xyz) $(variables_m.xyz) $(bc_m.xyz) $(runtime_m.xyz) diff --git a/examples/S3D-IO/build/make.XC40 b/examples/S3D-IO/build/make.XC40 new file mode 100644 index 0000000000000000000000000000000000000000..eb9f123dfe189db19c0f72252f9fd3ff5fbddf92 --- /dev/null +++ b/examples/S3D-IO/build/make.XC40 @@ -0,0 +1,15 @@ +# CPQ: define this and the code will use system(3) +# SAVEFILEINSEPDIR: do not know +directives = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO -I$(HOME)/install/$(ARCHI)/include/ -DDBG -DCOLOR + +MPIF90 = ftn +MPIF77 = ftn +MPICC = cc + +LDFLAGS = ../../../bindings/F90/tapioca_f90api.o -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca -lstdc++ -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive + +COMPILE_F90 = ${MPIF90} -g $(directives) -fpp -c -o $@ +COMPILE_F77 = ${MPIF77} -g -fpp -c -o $@ +COMPILE_C = ${MPIF90} -g -c -o $@ + +LINK = ${MPIF90} ${LDFLAGS} diff --git a/examples/S3D-IO/input/active.in b/examples/S3D-IO/input/active.in index 3dde62fca08b81f46e12dd56d82ea22518e7d0b5..acd812beb83fa319efe86e34da559b46c11e3e58 100644 --- a/examples/S3D-IO/input/active.in +++ b/examples/S3D-IO/input/active.in @@ -5,6 +5,6 @@ time period to save fields in restart files (time_save_inc) 1.00000E+05 frequency to monitor min/max files and active (i_time_mon) 10 frequency to check resolution; set < 0 for no check (i_time_res) -1 frequency write current tecplot file (i_time_tec) -1 -frequency to filter solution vector (i_time_fil) 31 +frequency to filter solution vector (i_time_fil) -1431142048 initial/constant timestep (sec) (tstep_init) 0.00000E+00 timestep frequency to write controller info (i_time_cont) 0 diff --git a/examples/S3D-IO/input/s3d.in b/examples/S3D-IO/input/s3d.in index eafeba0b958f7c6e84dcdb6d1350383b13e40101..34fdf1fa3fed35c82e043b5a53a70bfcf4cd5cb3 100644 --- a/examples/S3D-IO/input/s3d.in +++ b/examples/S3D-IO/input/s3d.in @@ -5,11 +5,11 @@ MODE ========================================================================================== GRID DIMENSION PARAMETERS ========================================================================================== -128 - global number of grid points in the x-direction (nx_g) -128 - global number of grid points in the y-direction (ny_g) -128 - global number of grid points in the z-direction (nz_g) -4 - number of processors in x-direction (npx) -4 - number of processors in y-direction (npy) +256 - global number of grid points in the x-direction (nx_g) +256 - global number of grid points in the y-direction (ny_g) +128 - global number of grid points in the z-direction (nz_g) +8 - number of processors in x-direction (npx) +8 - number of processors in y-direction (npy) 4 - number of processors in z-direction (npz) ========================================================================================== RUN-TIME PARAMETERS @@ -70,8 +70,8 @@ REQUIRED REFERENCE VALUES ========================================================================================== flag to enable/disable MPI I/O ========================================================================================== -4 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5, 4:TAPIOCA +1 - I/O method: 0:Fortran I/O, 1:MPI-IO, 2:PnetCDF, 3:HDF5, 4:TAPIOCA ========================================================================================== Output directory ========================================================================================== -/projects/visualization/ftessier/S3D +/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA diff --git a/examples/S3D-IO/run/run_xc40.sh b/examples/S3D-IO/run/run_xc40.sh new file mode 100755 index 0000000000000000000000000000000000000000..e98e023f217ca888ae7d9376a650d297ef4c00ef --- /dev/null +++ b/examples/S3D-IO/run/run_xc40.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +NODES=16 +PPN=16 +NPROCS=$((NODES*PPN)) +TARGET="/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA" +DDT="/soft/debuggers/forge/bin/ddt --connect" +STRIPE_COUNT=16 +STRIPE_SIZE=8388608 +AGGR_MULT=1 + +cd $HOME/TAPIOCA/examples/S3D-IO/run/ + +export TAPIOCA_STRATEGY=TOPOLOGY_AWARE + +export TAPIOCA_NBAGGR=2 +export TAPIOCA_NBBUFFERS=2 +export TAPIOCA_BUFFERSIZE=16777216 +export TAPIOCA_AGGRTIER=DDR + +export TAPIOCA_DEVNULL=false +export TAPIOCA_COMMSPLIT=true +export TAPIOCA_PIPELINING=true +export TAPIOCA_REELECTAGGR=false + +function setLustreFS () +{ + rm $TARGET/* + lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET + lfs getstripe $TARGET +} + + +function updateSettings() +{ + printenv | egrep "TAPIOCA_" + SETTINGS="-e TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY" + + SETTINGS="$SETTINGS -e TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR" + SETTINGS="$SETTINGS -e TAPIOCA_NBBUFFERS=$TAPIOCA_NBBUFFERS" + SETTINGS="$SETTINGS -e TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE" + SETTINGS="$SETTINGS -e TAPIOCA_AGGRTIER=$TAPIOCA_AGGRTIER" + + SETTINGS="$SETTINGS -e TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL" + SETTINGS="$SETTINGS -e TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT" + SETTINGS="$SETTINGS -e TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING" + SETTINGS="$SETTINGS -e TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR" + + SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1" + SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT" +} + +rm $TARGET/* +setLustreFS +updateSettings + +aprun $SETTINGS -n $NPROCS -N $PPN ./s3d_f90.x diff --git a/examples/S3D-IO/source/f90_files/solve/read_savefile.f90 b/examples/S3D-IO/source/f90_files/solve/read_savefile.f90 index 23127df9e205aa3c97e4ef5c9e7312efc4cc8d0e..9ccc50e80575886cd6c098b7c61dd03df40abb2e 100644 --- a/examples/S3D-IO/source/f90_files/solve/read_savefile.f90 +++ b/examples/S3D-IO/source/f90_files/solve/read_savefile.f90 @@ -125,7 +125,6 @@ subroutine read_savefile(io, precsn, dirtime) ! Read the data to intermediate arrays and interpolate as necessary ! No interpolation / morphing implemented for p-netcdf or hdf5 - if (io_method .EQ. 1 ) then ! Read using MPI-IO filename = '../data/'//trim(run_title)//'.'//dirtime2//'.field.mpi' diff --git a/examples/S3D-IO/source/f90_files/solve/write_savefile.f90 b/examples/S3D-IO/source/f90_files/solve/write_savefile.f90 index cca0df43d89be0614bc456cb2b4938000ab551f6..0808efdd9d41bcc8b5181084b32b06f13b6d5cc7 100644 --- a/examples/S3D-IO/source/f90_files/solve/write_savefile.f90 +++ b/examples/S3D-IO/source/f90_files/solve/write_savefile.f90 @@ -9,10 +9,9 @@ subroutine write_savefile(io, precsn) use param_m, only : nx, ny, nz, n_spec !quad use param_m, only : io_method - use param_m, only : output_dir #ifdef MPIIO - use mpi_io_m, only : mpi_file_io - use tapioca_m, only : tapioca_io + use mpi_io_m, only : mpi_file_io + use tapioca_m, only : tapioca_io use pnetcdf_m, only : pnetcdf_write use hdf5_m, only : hdf5_write #endif @@ -28,7 +27,7 @@ subroutine write_savefile(io, precsn) ! other declarations - character*5 :: myid_ext*5, time_ext*10, filename*100, dirname*100 + character*5 :: myid_ext*5, time_ext*10, filename*100, dirname*100, outputdir*100 character*100 :: bpfile integer mz logical :: exist @@ -50,21 +49,34 @@ subroutine write_savefile(io, precsn) ! wkliao: set a different file name for MPI-IO, pnetcdf, and HDF5 methods ! wkliao: no subdirectories are created if io_method > 0 + outputdir = '/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA/' if (io_method .EQ. 1) then - filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.mpi' + if(myid.eq.0) then + write(io,100) 'MPI-IO driver' + endif + filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.mpi' elseif (io_method .EQ. 2) then - filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.nc' + if(myid.eq.0) then + write(io,100) 'NetCFD driver' + endif + filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.nc' elseif (io_method .EQ. 3) then - filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.h5' + if(myid.eq.0) then + write(io,100) 'HDF5 driver' + endif + filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.h5' elseif (io_method .EQ. 4) then - filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.tp' + if(myid.eq.0) then + write(io,100) 'TAPIOCA driver' + endif + filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.tp' elseif (io_method .EQ. 5) then - filename = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'.field.bp' + filename = trim(outputdir)//trim(run_title)//'.'//time_ext//'.field.bp' elseif (io_method .EQ. 6) then filename = trim(run_title)//'.'//time_ext//'.field.bp' else - dirname = trim(output_dir)//'/'//trim(run_title)//'.'//time_ext//'/' + dirname = '../data/'//trim(run_title)//'.'//time_ext//'/' if(myid == 0) then #ifdef SYSTEMCALLWONTWORK ! when system calls do not work, as on infiniband diff --git a/examples/S3D-IO/source/modules/param_m.f90 b/examples/S3D-IO/source/modules/param_m.f90 index 345b6c039aba02465838b1d813424fa6bf005118..be71f15c30933e93875626ca3e967982688174f7 100644 --- a/examples/S3D-IO/source/modules/param_m.f90 +++ b/examples/S3D-IO/source/modules/param_m.f90 @@ -60,7 +60,7 @@ integer io_method - character*100 output_dir !output directory + character*100 output_dir character*8 dat_1, dat_2 !for start and end date (wall clock) character*10 tim_1, tim_2 !for start and end time (wall clock) diff --git a/examples/S3D-IO/source/modules/tapioca_m.f90 b/examples/S3D-IO/source/modules/tapioca_m.f90 index 5ecfa99f4ff4f47f77b4d450fb4fc32635f6f468..c5d58f79d1b950d29293eabac897ba435f407cff 100644 --- a/examples/S3D-IO/source/modules/tapioca_m.f90 +++ b/examples/S3D-IO/source/modules/tapioca_m.f90 @@ -25,7 +25,7 @@ subroutine tapioca_io(filename,rw) ! local variables integer nx_ny_nz - integer i, fp, ierr, errorclass + integer i, fp, ierr, errorclass, l integer mstatus(MPI_STATUS_SIZE) integer(MPI_OFFSET_KIND) iOffset @@ -34,7 +34,7 @@ subroutine tapioca_io(filename,rw) integer chunksize(4) integer*8 chunkoffset(4) integer*8 buffsize, scansize, hdr, filesize, totsize - integer typesize, agg_mem, target_mem, nbuff + integer typesize call MPI_Type_size ( MPI_REAL8, typesize, ierr ) nx_ny_nz = nx * ny * nz @@ -43,10 +43,6 @@ subroutine tapioca_io(filename,rw) totsize = 0 filesize = 0 hdr = 0 - ! DDR=0, HBM=1, SSD=2, HDD=3, NVR=4 - agg_mem = 0 - target_mem = 3 - nbuff = 2 call MPI_Exscan ( buffsize, scansize, 1, MPI_REAL8, MPI_SUM, gcomm, ierr ) @@ -77,10 +73,8 @@ subroutine tapioca_io(filename,rw) call MPI_Allreduce ( totsize, filesize, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD, ierr ) !---- initialize - call tapioca_init ( chunkcount, chunksize, chunkoffset, 4, hdr, gcomm ) - call tapioca_set_aggregation_tier ( nbuff, agg_mem, "" ) - call tapioca_set_target_tier ( target_mem, filesize, filename ) - + call tapioca_init ( chunkcount, chunksize, chunkoffset, 4, hdr, trim(filename)//CHAR(0), gcomm ) + !---- write/read array yspecies if (rw .EQ. 'w') then call tapioca_write ( chunkoffset (1), yspecies, chunkcount (1), MPI_REAL8, mstatus ) diff --git a/examples/make.inc/Makefile.linux-sles_12-x86_64 b/examples/make.inc/Makefile.linux-sles_12-x86_64 index b44be2ff0ca067490145903f7094e2af77feafc0..75dc03828fc56306f207dafe7c8342d2b6b8012d 100644 --- a/examples/make.inc/Makefile.linux-sles_12-x86_64 +++ b/examples/make.inc/Makefile.linux-sles_12-x86_64 @@ -1,11 +1,31 @@ MPICXX = CC MPIF90 = ftn +MPIF77 = ftn +MPICC = cc MPI_CFLAGS = -g -O3 -I$(HOME)/install/$(ARCHI)/include/ -DDBG -DCOLOR MPI_LDFLAGS = -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca -llustreapi -Wl,--whole-archive,-ldmapp,--no-whole-archive -MPI_F90_FLAGS = $(MPI_CFLAGS) +MPI_F90_FLAGS = -g -O0 -I$(HOME)/install/$(ARCHI)/include/ -DDBG -DCOLOR -fpp MPI_F90_LDFLAGS = $(MPI_LDFLAGS) -lstdc++ INSTALL_PATH = $(HOME)/install/$(ARCHI) +###### S3D-IO ONLY ###### +# CPQ: define this and the code will use system(3) +# SAVEFILEINSEPDIR: do not know + +S3D_F90_FLAGS = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO ${MPI_F90_FLAGS} +S3D_F90_LDFLAGS = ../../../bindings/F90/tapioca_f90api.o ${MPI_F90_LDFLAGS} +COMPILE_F90 = ${MPIF90} -g ${S3D_F90_FLAGS} -c -o $@ +COMPILE_F77 = ${MPIF77} -g ${S3D_F90_FLAGS} -c -o $@ +COMPILE_C = ${MPICC} -g -c -o $@ + +LINK = ${MPIF90} ${S3D_F90_LDFLAGS} +###### END ###### + +###### BT ONLY ###### +BT_F90_FLAGS = ${MPI_F90_FLAGS} +BT_F90_LDFLAGS = ../../bindings/F90/tapioca_f90api.o ${MPI_F90_LDFLAGS} +###### END ###### +