Commit c7dedd53 authored by Francois Tessier's avatar Francois Tessier

Set aggregation and target tiers in the S3D-IO Tapioca module. Makefile for...

Set aggregation and target tiers in the S3D-IO Tapioca module. Makefile for IBM BG/Q architecture. Script for BG/Q.
parent a48a3cf6
#!/bin/bash
NODES=32
NODES=8
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/debug"
......@@ -11,10 +11,10 @@ cd $HOME/TAPIOCA/examples/1D-Array
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=4
export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=4194304
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=false
export TAPIOCA_REELECTAGGR=true
function updateSettings()
{
......@@ -45,10 +45,10 @@ function outputFile ()
}
PARTICLES=50000
PARTICLES=40000
for RUN in 1
do
for AGGR in 4
for AGGR in 2
do
export TAPIOCA_NBAGGR=$AGGR
updateSettings
......
......@@ -22,7 +22,7 @@
# New version of makefile with dependencies auto-inserted using sfmakedepend
#----------------------------------------------------------------------
MACH = HASWELL
MACH = BGQ
include make.$(MACH)
......
# CPQ: define this and the code will use system(3)
# SAVEFILEINSEPDIR: do not know
directives = -DCPQ -DSAVEFILEINSEPDIR -DMPIIO
directives = -WF,-DCPQ -WF,-DSAVEFILEINSEPDIR -WF,-DMPIIO -WF,-C! -qsuffix=cpp=f90
COMPILE_F90 = mpif90 -g $(directives) -xf95-cpp-input -c -o $@
COMPILE_F77 = mpif90 -g -xf77-cpp-input -c -o $@
COMPILE_C = mpifcc -g -c -o $@
MPIF90 = mpixlf90
MPIF77 = mpixlf77
MPICC = mpixlc
FFLAGS = -I$(HOME)/install/$(ARCHI)/include/ -WF,-DBGQ -WF,-DDBG -WF,-DCOLOR
FFLAGS += -qextname=tapioca_init \
-qextname=tapioca_set_aggregation_tier \
-qextname=tapioca_set_target_tier \
-qextname=tapioca_read \
-qextname=tapioca_write \
-qextname=tapioca_finalize \
-qextname=flush
LINK = mpif90
LDFLAGS = ../../../bindings/F90/tapioca_f90api.o
LDFLAGS += -L${IBM_MAIN_DIR}/vacpp/bg/12.1/lib64 -L$(HOME)/install/$(ARCHI)/lib/ -ltapioca -lstdc++ -libmc++
COMPILE_F90 = ${MPIF90} -g $(directives) ${FFLAGS} -c -o $@
COMPILE_F77 = ${MPIF77} -g ${FFLAGS} -c -o $@
COMPILE_C = ${MPIF90} -g ${FFLAGS} -c -o $@
LINK = ${MPIF90} ${LDFLAGS}
......@@ -5,6 +5,6 @@ time period to save fields in restart files (time_save_inc) 1.00000E+05
frequency to monitor min/max files and active (i_time_mon) 10
frequency to check resolution; set < 0 for no check (i_time_res) -1
frequency write current tecplot file (i_time_tec) -1
frequency to filter solution vector (i_time_fil) 0
frequency to filter solution vector (i_time_fil) 31
initial/constant timestep (sec) (tstep_init) 0.00000E+00
timestep frequency to write controller info (i_time_cont) 0
VARS="PAMID_VERBOSE=1 BG_SHAREDMEMSIZE=64 PAMID_COLLECTIVES=1 BGLOCKLESSMPIO_F_TYPE=0x47504653 PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
NODES=32
PPN=16
#!/bin/bash
NODES=8
PPN=8
NPROCS=$((NODES*PPN))
TARGET="/projects/visualization/ftessier/S3D"
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_BUFFERSIZE=16777216
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=false
function updateSettings()
{
printenv | egrep "TAPIOCA_"
SETTINGS="TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="$SETTINGS TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
SETTINGS="$SETTINGS TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
SETTINGS="$SETTINGS TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
SETTINGS="$SETTINGS TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"
SETTINGS="$SETTINGS PAMID_VERBOSE=1"
SETTINGS="$SETTINGS PAMID_COLLECTIVES_MEMORY_OPTIMIZED=1"
SETTINGS="$SETTINGS PAMID_COLLECTIVES=1"
SETTINGS="$SETTINGS BG_SHAREDMEMSIZE=64"
SETTINGS="$SETTINGS BGLOCKLESSMPIO_F_TYPE=0x47504653"
SETTINGS="$SETTINGS GPFSMPIO_BALANCECONTIG=1"
SETTINGS="$SETTINGS GPFSMPIO_P2PCONTIG=1"
#SETTINGS="$SETTINGS MPICH_MPIIO_HINTS=*:bg_nodes_pset=$TAPIOCA_NBAGGR"
echo $SETTINGS
}
cd $HOME/TAPIOCA/examples/S3D-IO/run/
rm $TARGET/*
updateSettings
# Run and write out files with 8000pts/node
cp -f ../input/s3d.in.110592_$NPROCS ../input/s3d.in
runjob --block $COBALT_PARTNAME --envs $VARS -p $PPN --np $NPROCS : ./s3d_f90.x > $NPROCS.s3d.out
# cp -f ../input/s3d.in.110592_$NPROCS ../input/s3d.in
runjob --block $COBALT_PARTNAME --envs $SETTINGS -p $PPN --np $NPROCS : ./s3d_f90.x
#> $NPROCS.s3d.out
# # Run and re-read ouput files written previously; check for errors
# cd $NEW_WORKDIR/run
......
......@@ -25,7 +25,7 @@ subroutine tapioca_io(filename,rw)
! local variables
integer nx_ny_nz
integer fp, ierr, errorclass
integer i, fp, ierr, errorclass
integer mstatus(MPI_STATUS_SIZE)
integer(MPI_OFFSET_KIND) iOffset
......@@ -33,14 +33,21 @@ subroutine tapioca_io(filename,rw)
integer*8 chunkcount(4)
integer chunksize(4)
integer*8 chunkoffset(4)
integer*8 buffsize, scansize, hdr
integer typesize
integer*8 buffsize, scansize, hdr, filesize, totsize
integer typesize, agg_mem, target_mem, nbuff
call MPI_Type_size ( MPI_REAL8, typesize, ierr )
nx_ny_nz = nx * ny * nz
buffsize = ( nx_ny_nz * ( nsc + 1 ) + nx_ny_nz + nx_ny_nz + nx_ny_nz * 3 ) * typesize
scansize = 0
hdr = 0
nx_ny_nz = nx * ny * nz
buffsize = ( nx_ny_nz * ( nsc + 1 ) + nx_ny_nz + nx_ny_nz + nx_ny_nz * 3 ) * typesize
scansize = 0
totsize = 0
filesize = 0
hdr = 0
! DDR=0, HBM=1, SSD=2, HDD=3, NVR=4
agg_mem = 0
target_mem = 3
nbuff = 2
call MPI_Exscan ( buffsize, scansize, 1, MPI_REAL8, MPI_SUM, gcomm, ierr )
! yspecies
......@@ -63,8 +70,16 @@ subroutine tapioca_io(filename,rw)
chunksize (4) = typesize
chunkoffset (4) = chunkoffset (3) + chunkcount (3) * chunksize (3)
do i = 1, 4
totsize = totsize + chunkcount (i) * chunksize (i)
end do
call MPI_Allreduce ( totsize, filesize, 1, MPI_LONG_LONG, MPI_SUM, MPI_COMM_WORLD, ierr )
!---- initialize
call tapioca_init ( chunkcount, chunksize, chunkoffset, 4, hdr, gcomm )
call tapioca_set_aggregation_tier ( nbuff, agg_mem, "" )
call tapioca_set_target_tier ( target_mem, filesize, filename )
!---- write/read array yspecies
if (rw .EQ. 'w') then
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment