Commit 194debdc authored by Francois Tessier's avatar Francois Tessier

[NOT WORKING] BT-IO benchmark

parent 2af0cf67
...@@ -3,9 +3,9 @@ BENCHMARK=bt ...@@ -3,9 +3,9 @@ BENCHMARK=bt
BENCHMARKU=BT BENCHMARKU=BT
VEC= VEC=
include ../make.inc/Makefile.$(ARCHI)
include config/make.def include config/make.def
OBJS = bt.o make_set.o initialize.o exact_solution.o exact_rhs.o \ OBJS = bt.o make_set.o initialize.o exact_solution.o exact_rhs.o \
set_constants.o adi.o define.o copy_faces.o rhs.o solve_subs.o \ set_constants.o adi.o define.o copy_faces.o rhs.o solve_subs.o \
x_solve$(VEC).o y_solve$(VEC).o z_solve$(VEC).o add.o error.o \ x_solve$(VEC).o y_solve$(VEC).o z_solve$(VEC).o add.o error.o \
...@@ -112,4 +112,4 @@ epio.o: epio.f mpinpb.h npbparams.h ...@@ -112,4 +112,4 @@ epio.o: epio.f mpinpb.h npbparams.h
clean: clean:
- rm -f *.o *~ mputil* common/*.o - rm -f *.o *~ mputil* common/*.o
- rm -f npbparams.h core btio.full.out - rm -f npbparams.h core btio.full.out
- rm -f ${HOME}/install/${ARCHI}/bin/bt.* - rm -f ${HOME}/install/${ARCHI}/bin/bt.*
\ No newline at end of file
Building Steps
--------------------
- Edit sys/setparams.c and replace the output directory (filepath) /!\ split the string to comply with fortran rules... /!\
- Build setparams using gcc
- Clean the root directory
- make NPROCS=<#procs> CLASS={A,B,C,D} SUBTYPE=<io method>
...@@ -185,7 +185,11 @@ c set default to No-File-Hints with a value of 0 ...@@ -185,7 +185,11 @@ c set default to No-File-Hints with a value of 0
call initialize call initialize
call setup_btio if (iotype .eq. 2) then
call setup_btio (niter, wr_interval)
else
call setup_btio
endif
idump = 0 idump = 0
call lhsinit call lhsinit
......
...@@ -29,30 +29,29 @@ ...@@ -29,30 +29,29 @@
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# This is the fortran compiler used for MPI programs # This is the fortran compiler used for MPI programs
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
MPIF77 = mpif77
# This links MPI fortran programs; usually the same as ${MPIF77} # This links MPI fortran programs; usually the same as ${MPIF77}
FLINK = $(MPIF77) FLINK = ${MPIF77}
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# These macros are passed to the linker to help link with MPI correctly # These macros are passed to the linker to help link with MPI correctly
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
FMPI_LIB = -L/usr/local/lib -lmpi FMPI_LIB =
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# These macros are passed to the compiler to help find 'mpif.h' # These macros are passed to the compiler to help find 'mpif.h'
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
FMPI_INC = -I/usr/local/include FMPI_INC =
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# Global *compile time* flags for Fortran programs # Global *compile time* flags for Fortran programs
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
FFLAGS = -O FFLAGS = $(BT_F90_FLAGS)
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# Global *link time* flags. Flags for increasing maximum executable # Global *link time* flags. Flags for increasing maximum executable
# size usually go here. # size usually go here.
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
FLINKFLAGS = -O FLINKFLAGS = $(BT_F90_LDFLAGS)
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
...@@ -75,30 +74,29 @@ FLINKFLAGS = -O ...@@ -75,30 +74,29 @@ FLINKFLAGS = -O
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# This is the C compiler used for MPI programs # This is the C compiler used for MPI programs
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
MPICC = mpicc
# This links MPI C programs; usually the same as ${MPICC} # This links MPI C programs; usually the same as ${MPICC}
CLINK = $(MPICC) CLINK = $(MPICC)
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# These macros are passed to the linker to help link with MPI correctly # These macros are passed to the linker to help link with MPI correctly
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
CMPI_LIB = -L/usr/local/lib -lmpi CMPI_LIB =
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# These macros are passed to the compiler to help find 'mpi.h' # These macros are passed to the compiler to help find 'mpi.h'
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
CMPI_INC = -I/usr/local/include CMPI_INC =
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# Global *compile time* flags for C programs # Global *compile time* flags for C programs
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
CFLAGS = -O CFLAGS = $(BT_F90_FLAGS)
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
# Global *link time* flags. Flags for increasing maximum executable # Global *link time* flags. Flags for increasing maximum executable
# size usually go here. # size usually go here.
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
CLINKFLAGS = -O CLINKFLAGS = $(BT_F90_LDFLAGS)
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
...@@ -119,7 +117,7 @@ CLINKFLAGS = -O ...@@ -119,7 +117,7 @@ CLINKFLAGS = -O
# this compiler go here also; typically there are few flags required; hence # this compiler go here also; typically there are few flags required; hence
# there are no separate macros provided for such flags. # there are no separate macros provided for such flags.
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
CC = cc -g CC = gcc -g
#--------------------------------------------------------------------------- #---------------------------------------------------------------------------
......
...@@ -17,6 +17,8 @@ c--------------------------------------------------------------------- ...@@ -17,6 +17,8 @@ c---------------------------------------------------------------------
call mpi_bcast(record_length, 1, MPI_INTEGER, call mpi_bcast(record_length, 1, MPI_INTEGER,
> root, comm_setup, ierr) > root, comm_setup, ierr)
if (node .eq. root) print *, 'record length ', record_length
open (unit=99, file=filenm, open (unit=99, file=filenm,
$ form='unformatted', access='direct', $ form='unformatted', access='direct',
$ recl=record_length) $ recl=record_length)
...@@ -52,6 +54,7 @@ c--------------------------------------------------------------------- ...@@ -52,6 +54,7 @@ c---------------------------------------------------------------------
$ PROBLEM_SIZE*idump_sub))) $ PROBLEM_SIZE*idump_sub)))
do ix=0,cell_size(1,cio)-1 do ix=0,cell_size(1,cio)-1
if (node .eq. root) print *, 'write at', iseek+ix+1
write(99, rec=iseek+ix+1) write(99, rec=iseek+ix+1)
$ u(1,ix, jio,kio,cio), $ u(1,ix, jio,kio,cio),
$ u(2,ix, jio,kio,cio), $ u(2,ix, jio,kio,cio),
......
...@@ -19,6 +19,7 @@ c--------------------------------------------------------------------- ...@@ -19,6 +19,7 @@ c---------------------------------------------------------------------
character*20 cb_nodes, cb_size character*20 cb_nodes, cb_size
integer c, m integer c, m
integer cell_disp(maxcells) integer cell_disp(maxcells)
integer tsize
call mpi_bcast(collbuf_nodes, 1, MPI_INTEGER, call mpi_bcast(collbuf_nodes, 1, MPI_INTEGER,
> root, comm_setup, ierr) > root, comm_setup, ierr)
...@@ -146,7 +147,7 @@ c ...@@ -146,7 +147,7 @@ c
stop stop
endif endif
call MPI_File_set_view(fp, iseek, element, call MPI_File_set_view(fp, iseek, element,
$ combined_ftype, 'native', info, ierr) $ combined_ftype, 'native', info, ierr)
if (ierr .ne. MPI_SUCCESS) then if (ierr .ne. MPI_SUCCESS) then
...@@ -154,6 +155,9 @@ c ...@@ -154,6 +155,9 @@ c
stop stop
endif endif
c$$$ call MPI_Type_size(combined_ftype, tsize, ierr)
c$$$ iseek = tsize * node
do m = 1, 5 do m = 1, 5
xce_sub(m) = 0.d0 xce_sub(m) = 0.d0
end do end do
...@@ -175,8 +179,8 @@ c--------------------------------------------------------------------- ...@@ -175,8 +179,8 @@ c---------------------------------------------------------------------
include 'mpinpb.h' include 'mpinpb.h'
integer mstatus(MPI_STATUS_SIZE) integer mstatus(MPI_STATUS_SIZE)
integer ierr integer ierr, disp
call MPI_File_write_at_all(fp, iseek, u, call MPI_File_write_at_all(fp, iseek, u,
$ 1, combined_btype, mstatus, ierr) $ 1, combined_btype, mstatus, ierr)
if (ierr .ne. MPI_SUCCESS) then if (ierr .ne. MPI_SUCCESS) then
...@@ -192,7 +196,7 @@ c--------------------------------------------------------------------- ...@@ -192,7 +196,7 @@ c---------------------------------------------------------------------
if (idump_sub .ge. rd_interval) then if (idump_sub .ge. rd_interval) then
iseek = 0 iseek = 0
call acc_sub_norms(idump+1) c call acc_sub_norms(idump+1)
iseek = 0 iseek = 0
idump_sub = 0 idump_sub = 0
...@@ -293,7 +297,7 @@ c clear the last time step ...@@ -293,7 +297,7 @@ c clear the last time step
c read back the time steps and accumulate norms c read back the time steps and accumulate norms
call acc_sub_norms(idump) c call acc_sub_norms(idump)
call MPI_File_close(fp, ierr) call MPI_File_close(fp, ierr)
......
c NPROCS = 256 CLASS = B SUBTYPE = FORTRAN
c
c
c This file is generated automatically by the setparams utility.
c It sets the number of processors and the class of the NPB
c in this directory. Do not modify it by hand.
c
integer maxcells, problem_size, niter_default
parameter (maxcells=16, problem_size=102, niter_default=200)
double precision dt_default
parameter (dt_default = 0.0003d0)
integer wr_default
parameter (wr_default = 5)
integer iotype
parameter (iotype = 5)
character*(*) tmppath
parameter (tmppath = '/lus/theta-fs0/projects/Performance')
character*(*) filepath
parameter (filepath = trim(tmppath)//'/ftessier/TAPIOCA')
character*(*) filenm
parameter (filenm = trim(filepath)//'/btio.fortran.out')
integer fortran_rec_sz
parameter (fortran_rec_sz = 4)
logical convertdouble
parameter (convertdouble = .false.)
character*11 compiletime
parameter (compiletime='09 Mar 2018')
character*5 npbversion
parameter (npbversion='3.3.1')
character*6 cs1
parameter (cs1='(none)')
character*9 cs2
parameter (cs2='${MPIF77}')
character*6 cs3
parameter (cs3='(none)')
character*6 cs4
parameter (cs4='(none)')
character*15 cs5
parameter (cs5='$(BT_F90_FLAGS)')
character*17 cs6
parameter (cs6='$(BT_F90_LDFLAGS)')
character*6 cs7
parameter (cs7='randi8')
#!/bin/bash
NODES=16
PPN=16
NPROCS=$((NODES*PPN))
TARGET="/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA"
DDT="/soft/debuggers/ddt/bin/ddt --connect"
STRIPE_COUNT=16
STRIPE_SIZE=8388608
AGGR_MULT=1
cd $HOME/install/$ARCHI/bin/
export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
export TAPIOCA_NBAGGR=2
export TAPIOCA_NBBUFFERS=2
export TAPIOCA_BUFFERSIZE=8388608
export TAPIOCA_AGGRTIER=DDR
export TAPIOCA_DEVNULL=false
export TAPIOCA_COMMSPLIT=true
export TAPIOCA_PIPELINING=true
export TAPIOCA_REELECTAGGR=true
function updateSettings()
{
printenv | egrep "TAPIOCA_"
SETTINGS="-e TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
SETTINGS="$SETTINGS -e TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
SETTINGS="$SETTINGS -e TAPIOCA_NBBUFFERS=$TAPIOCA_NBBUFFERS"
SETTINGS="$SETTINGS -e TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
SETTINGS="$SETTINGS -e TAPIOCA_AGGRTIER=$TAPIOCA_AGGRTIER"
SETTINGS="$SETTINGS -e TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
SETTINGS="$SETTINGS -e TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
SETTINGS="$SETTINGS -e TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
SETTINGS="$SETTINGS -e TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"
SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
}
function setLustreFS ()
{
rm $TARGET/*
lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET
lfs getstripe $TARGET
}
CLASS=B
#IO=mpi_io_full
IO=fortran_io
#IO=tapioca
updateSettings
setLustreFS
aprun $SETTINGS -n $NPROCS -N $PPN bt.${CLASS}.${NPROCS}.${IO}
...@@ -13,7 +13,8 @@ all: setparams ...@@ -13,7 +13,8 @@ all: setparams
# was compiled and linked # was compiled and linked
setparams: setparams.c ../config/make.def setparams: setparams.c ../config/make.def
$(CC) ${CONVERTFLAG} -o setparams setparams.c #$(CC) ${CONVERTFLAG} -o setparams setparams.c
gcc ${CONVERTFLAG} -o setparams setparams.c
clean: clean:
......
...@@ -556,9 +556,11 @@ void write_bt_info(FILE *fp, int nprocs, char class, int io) ...@@ -556,9 +556,11 @@ void write_bt_info(FILE *fp, int nprocs, char class, int io)
fprintf(fp, "%sinteger iotype\n", FINDENT); fprintf(fp, "%sinteger iotype\n", FINDENT);
fprintf(fp, "%sparameter (iotype = %d)\n", FINDENT, io); fprintf(fp, "%sparameter (iotype = %d)\n", FINDENT, io);
if (io) { if (io) {
fprintf(fp, "%scharacter*(*) tmppath\n", FINDENT);
fprintf(fp, "%sparameter (tmppath = '/lus/theta-fs0/projects/Performance')\n", FINDENT);
fprintf(fp, "%scharacter*(*) filepath\n", FINDENT); fprintf(fp, "%scharacter*(*) filepath\n", FINDENT);
fprintf(fp, "%sparameter (filepath = '/projects/Performance/ftessier/BT-IO')\n", FINDENT); fprintf(fp, "%sparameter (filepath = trim(tmppath)//'/ftessier/TAPIOCA')\n", FINDENT);
fprintf(fp, "%scharacter*(*) filenm\n", FINDENT); fprintf(fp, "%scharacter*(*) filenm\n", FINDENT);
switch (io) { switch (io) {
case FULL: case FULL:
fprintf(fp, "%sparameter (filenm = trim(filepath)//'/btio.full.out')\n", FINDENT); fprintf(fp, "%sparameter (filenm = trim(filepath)//'/btio.full.out')\n", FINDENT);
......
c
c Author: François Tessier
c Argonne National Lab
c ftessier@anl.gov
c
c--------------------------------------------------------------------- c---------------------------------------------------------------------
c--------------------------------------------------------------------- c---------------------------------------------------------------------
subroutine setup_btio subroutine setup_btio(niter, wr_interval)
c--------------------------------------------------------------------- c---------------------------------------------------------------------
c--------------------------------------------------------------------- c---------------------------------------------------------------------
...@@ -11,146 +15,40 @@ c--------------------------------------------------------------------- ...@@ -11,146 +15,40 @@ c---------------------------------------------------------------------
include 'mpinpb.h' include 'mpinpb.h'
integer ierr integer ierr
integer mstatus(MPI_STATUS_SIZE)
integer sizes(4), starts(4), subsizes(4)
integer cell_btype(maxcells), cell_ftype(maxcells)
integer cell_blength(maxcells)
integer info
character*20 cb_nodes, cb_size
integer c, m integer c, m
integer cell_disp(maxcells) c---------------------------------------------------------------------
c TAPIOCA
call mpi_bcast(collbuf_nodes, 1, MPI_INTEGER, c---------------------------------------------------------------------
> root, comm_setup, ierr) integer niter, wr_interval
integer wr_step
call mpi_bcast(collbuf_size, 1, MPI_INTEGER, integer*8 header
> root, comm_setup, ierr) integer*8 chunkcount(niter/wr_interval)
integer chunksize(niter/wr_interval)
info = MPI_INFO_NULL integer*8 chunkoffset(niter/wr_interval)
call MPI_Type_contiguous(5, MPI_DOUBLE_PRECISION, header = 0
$ element, ierr)
call MPI_Type_commit(element, ierr) chunkcount (1) = 1
call MPI_Type_extent(element, eltext, ierr) chunksize (1) = iosize/eltext
chunkoffset (1) = iseek
do c = 1, ncells
c do wr_step = 2, niter/wr_interval
c Outer array dimensions ar same for every cell chunkcount (wr_step) = 1
c chunksize (wr_step) = iosize/eltext
sizes(1) = IMAX+4 chunkoffset (wr_step) = chunkoffset (wr_step-1) + iosize/eltext
sizes(2) = JMAX+4 enddo
sizes(3) = KMAX+4
c
c 4th dimension is cell number, total of maxcells cells
c
sizes(4) = maxcells
c
c Internal dimensions of cells can differ slightly between cells
c
subsizes(1) = cell_size(1, c)
subsizes(2) = cell_size(2, c)
subsizes(3) = cell_size(3, c)
c
c Cell is 4th dimension, 1 cell per cell type to handle varying
c cell sub-array sizes
c
subsizes(4) = 1
c
c type constructors use 0-based start addresses
c
starts(1) = 2
starts(2) = 2
starts(3) = 2
starts(4) = c-1
c
c Create buftype for a cell
c
call MPI_Type_create_subarray(4, sizes, subsizes,
$ starts, MPI_ORDER_FORTRAN, element,
$ cell_btype(c), ierr)
c
c block length and displacement for joining cells -
c 1 cell buftype per block, cell buftypes have own displacment
c generated from cell number (4th array dimension)
c
cell_blength(c) = 1
cell_disp(c) = 0
enddo
c
c Create combined buftype for all cells
c
call MPI_Type_struct(ncells, cell_blength, cell_disp,
$ cell_btype, combined_btype, ierr)
call MPI_Type_commit(combined_btype, ierr)
do c = 1, ncells
c
c Entire array size
c
sizes(1) = PROBLEM_SIZE
sizes(2) = PROBLEM_SIZE
sizes(3) = PROBLEM_SIZE
c
c Size of c'th cell
c
subsizes(1) = cell_size(1, c)
subsizes(2) = cell_size(2, c)
subsizes(3) = cell_size(3, c)
c
c Starting point in full array of c'th cell
c
starts(1) = cell_low(1,c)
starts(2) = cell_low(2,c)
starts(3) = cell_low(3,c)
call MPI_Type_create_subarray(3, sizes, subsizes,
$ starts, MPI_ORDER_FORTRAN,
$ element, cell_ftype(c), ierr)
cell_blength(c) = 1
cell_disp(c) = 0
enddo
call MPI_Type_struct(ncells, cell_blength, cell_disp,
$ cell_ftype, combined_ftype, ierr)
call MPI_Type_commit(combined_ftype, ierr)
iseek=0
if (node .eq. root) then
call MPI_File_delete(filenm, MPI_INFO_NULL, ierr)
endif
call MPI_Barrier(comm_solve, ierr)
call MPI_File_open(comm_solve,
$ filenm,
$ MPI_MODE_RDWR+MPI_MODE_CREATE,
$ MPI_INFO_NULL, fp, ierr)
if (ierr .ne. MPI_SUCCESS) then
print *, 'Error opening file'
stop
endif
call MPI_File_set_view(fp, iseek, element,
$ combined_ftype, 'native', info, ierr)
if (ierr .ne. MPI_SUCCESS) then
print *, 'Error setting file view'
stop
endif
call tapioca_init ( chunkcount,
$ chunksize,
$ chunkoffset,
$ niter/wr_interval,
$ header, filenm, comm_solve)
do m = 1, 5 do m = 1, 5
xce_sub(m) = 0.d0 xce_sub(m) = 0.d0
end do end do
idump_sub = 0 idump_sub = 0
return return
end end
...@@ -167,14 +65,8 @@ c--------------------------------------------------------------------- ...@@ -167,14 +65,8 @@ c---------------------------------------------------------------------
integer mstatus(MPI_STATUS_SIZE) integer mstatus(MPI_STATUS_SIZE)
integer ierr integer ierr
call MPI_File_write_at_all(fp, iseek, u, call tapioca_write ( iseek, u, 1, combined_btype, mstatus )
$ 1, combined_btype, mstatus, ierr)
if (ierr .ne. MPI_SUCCESS) then
print *, 'Error writing to file'
stop
endif
call MPI_Type_size(combined_btype, iosize, ierr)
iseek = iseek + iosize/eltext iseek = iseek + iosize/eltext
idump_sub = idump_sub + 1 idump_sub = idump_sub + 1
...@@ -182,7 +74,7 @@ c--------------------------------------------------------------------- ...@@ -182,7 +74,7 @@ c---------------------------------------------------------------------
if (idump_sub .ge. rd_interval) then if (idump_sub .ge. rd_interval) then
iseek = 0 iseek = 0
call acc_sub_norms(idump+1) c call acc_sub_norms(idump+1)
iseek = 0 iseek = 0
idump_sub = 0 idump_sub = 0
...@@ -244,8 +136,8 @@ c--------------------------------------------------------------------- ...@@ -244,8 +136,8 @@ c---------------------------------------------------------------------
integer ierr integer ierr
call MPI_File_close(fp, ierr) call tapioca_finalize ( )
return return
end end
...@@ -283,7 +175,7 @@ c clear the last time step ...@@ -283,7 +175,7 @@ c clear the last time step
c read back the time steps and accumulate norms c read back the time steps and accumulate norms
call acc_sub_norms(idump) c call acc_sub_norms(idump)
call MPI_File_close(fp, ierr) call MPI_File_close(fp, ierr)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment