[NOT WORKING] BT-IO benchmark

examples/BT/Makefile

@@ -3,9 +3,9 @@ BENCHMARK=bt
 BENCHMARKU=BT
 VEC=
 
+include ../make.inc/Makefile.$(ARCHI)
 include config/make.def
 
-
 OBJS = bt.o make_set.o initialize.o exact_solution.o exact_rhs.o \
        set_constants.o adi.o define.o copy_faces.o rhs.o solve_subs.o \
        x_solve$(VEC).o y_solve$(VEC).o z_solve$(VEC).o add.o error.o \
@@ -112,4 +112,4 @@ epio.o:           epio.f mpinpb.h npbparams.h
 clean:
 	- rm -f *.o *~ mputil* common/*.o
 	- rm -f  npbparams.h core btio.full.out
-	- rm -f ${HOME}/install/${ARCHI}/bin/bt.*
+	- rm -f ${HOME}/install/${ARCHI}/bin/bt.*
\ No newline at end of file

examples/BT/README

+Building Steps
+--------------------
+- Edit sys/setparams.c and replace the output directory (filepath) /!\ split the string to comply with fortran rules... /!\
+- Build setparams using gcc
+- Clean the root directory
+- make NPROCS=<#procs> CLASS={A,B,C,D} SUBTYPE=<io method>

examples/BT/bt.f

@@ -185,7 +185,11 @@ c             set default to No-File-Hints with a value of 0
 
        call initialize
 
-       call setup_btio
+       if (iotype .eq. 2) then
+          call setup_btio (niter, wr_interval)
+       else
+          call setup_btio
+       endif
        idump = 0
 
        call lhsinit

examples/BT/config/make.def

@@ -29,30 +29,29 @@
 #---------------------------------------------------------------------------
 # This is the fortran compiler used for MPI programs
 #---------------------------------------------------------------------------
-MPIF77 = mpif77
 # This links MPI fortran programs; usually the same as ${MPIF77}
-FLINK	= $(MPIF77)
+FLINK	= ${MPIF77}
 
 #---------------------------------------------------------------------------
 # These macros are passed to the linker to help link with MPI correctly
 #---------------------------------------------------------------------------
-FMPI_LIB  = -L/usr/local/lib -lmpi
+FMPI_LIB  = 
 
 #---------------------------------------------------------------------------
 # These macros are passed to the compiler to help find 'mpif.h'
 #---------------------------------------------------------------------------
-FMPI_INC = -I/usr/local/include
+FMPI_INC = 
 
 #---------------------------------------------------------------------------
 # Global *compile time* flags for Fortran programs
 #---------------------------------------------------------------------------
-FFLAGS	= -O
+FFLAGS	= $(BT_F90_FLAGS)
 
 #---------------------------------------------------------------------------
 # Global *link time* flags. Flags for increasing maximum executable 
 # size usually go here. 
 #---------------------------------------------------------------------------
-FLINKFLAGS = -O
+FLINKFLAGS = $(BT_F90_LDFLAGS)
 
 
 #---------------------------------------------------------------------------
@@ -75,30 +74,29 @@ FLINKFLAGS = -O
 #---------------------------------------------------------------------------
 # This is the C compiler used for MPI programs
 #---------------------------------------------------------------------------
-MPICC = mpicc
 # This links MPI C programs; usually the same as ${MPICC}
 CLINK	= $(MPICC)
 
 #---------------------------------------------------------------------------
 # These macros are passed to the linker to help link with MPI correctly
 #---------------------------------------------------------------------------
-CMPI_LIB  = -L/usr/local/lib -lmpi
+CMPI_LIB  = 
 
 #---------------------------------------------------------------------------
 # These macros are passed to the compiler to help find 'mpi.h'
 #---------------------------------------------------------------------------
-CMPI_INC = -I/usr/local/include
+CMPI_INC = 
 
 #---------------------------------------------------------------------------
 # Global *compile time* flags for C programs
 #---------------------------------------------------------------------------
-CFLAGS	= -O
+CFLAGS	= $(BT_F90_FLAGS)
 
 #---------------------------------------------------------------------------
 # Global *link time* flags. Flags for increasing maximum executable 
 # size usually go here. 
 #---------------------------------------------------------------------------
-CLINKFLAGS = -O
+CLINKFLAGS = $(BT_F90_LDFLAGS)
 
 
 #---------------------------------------------------------------------------
@@ -119,7 +117,7 @@ CLINKFLAGS = -O
 # this compiler go here also; typically there are few flags required; hence 
 # there are no separate macros provided for such flags.
 #---------------------------------------------------------------------------
-CC	= cc -g
+CC	= gcc -g
 
 
 #---------------------------------------------------------------------------

examples/BT/fortran_io.f

@@ -17,6 +17,8 @@ c---------------------------------------------------------------------
       call mpi_bcast(record_length, 1, MPI_INTEGER,
      >                root, comm_setup, ierr)
 
+      if (node .eq. root) print *, 'record length ', record_length
+
       open (unit=99, file=filenm,
      $      form='unformatted', access='direct',
      $      recl=record_length)
@@ -52,6 +54,7 @@ c---------------------------------------------------------------------
      $                   PROBLEM_SIZE*idump_sub)))
 
                   do ix=0,cell_size(1,cio)-1
+                     if (node .eq. root) print *, 'write at', iseek+ix+1
                       write(99, rec=iseek+ix+1)
      $                      u(1,ix, jio,kio,cio),
      $                      u(2,ix, jio,kio,cio),

examples/BT/full_mpiio.f

@@ -19,6 +19,7 @@ c---------------------------------------------------------------------
       character*20 cb_nodes, cb_size
       integer c, m
       integer cell_disp(maxcells)
+      integer tsize
 
        call mpi_bcast(collbuf_nodes, 1, MPI_INTEGER,
      >                root, comm_setup, ierr)
@@ -146,7 +147,7 @@ c
                 stop
        endif
 
-        call MPI_File_set_view(fp, iseek, element, 
+       call MPI_File_set_view(fp, iseek, element, 
      $          combined_ftype, 'native', info, ierr)
 
        if (ierr .ne. MPI_SUCCESS) then
@@ -154,6 +155,9 @@ c
                 stop
        endif
 
+c$$$       call MPI_Type_size(combined_ftype, tsize, ierr)
+c$$$       iseek = tsize * node
+
       do m = 1, 5
          xce_sub(m) = 0.d0
       end do
@@ -175,8 +179,8 @@ c---------------------------------------------------------------------
       include 'mpinpb.h'
 
       integer mstatus(MPI_STATUS_SIZE)
-      integer ierr
-
+      integer ierr, disp
+      
       call MPI_File_write_at_all(fp, iseek, u,
      $                           1, combined_btype, mstatus, ierr)
       if (ierr .ne. MPI_SUCCESS) then
@@ -192,7 +196,7 @@ c---------------------------------------------------------------------
          if (idump_sub .ge. rd_interval) then
 
             iseek = 0
-            call acc_sub_norms(idump+1)
+c     call acc_sub_norms(idump+1)
 
             iseek = 0
             idump_sub = 0
@@ -293,7 +297,7 @@ c     clear the last time step
 
 c     read back the time steps and accumulate norms
 
-      call acc_sub_norms(idump)
+c      call acc_sub_norms(idump)
 
       call MPI_File_close(fp, ierr)
 

examples/BT/npbparams.h

+c NPROCS = 256 CLASS = B SUBTYPE = FORTRAN
+c  
+c  
+c  This file is generated automatically by the setparams utility.
+c  It sets the number of processors and the class of the NPB
+c  in this directory. Do not modify it by hand.
+c  
+        integer maxcells, problem_size, niter_default
+        parameter (maxcells=16, problem_size=102, niter_default=200)
+        double precision dt_default
+        parameter (dt_default = 0.0003d0)
+        integer wr_default
+        parameter (wr_default = 5)
+        integer iotype
+        parameter (iotype = 5)
+        character*(*) tmppath
+        parameter (tmppath = '/lus/theta-fs0/projects/Performance')
+        character*(*) filepath
+        parameter (filepath = trim(tmppath)//'/ftessier/TAPIOCA')
+        character*(*) filenm
+        parameter (filenm = trim(filepath)//'/btio.fortran.out')
+        integer fortran_rec_sz
+        parameter (fortran_rec_sz = 4)
+        logical  convertdouble
+        parameter (convertdouble = .false.)
+        character*11 compiletime
+        parameter (compiletime='09 Mar 2018')
+        character*5 npbversion
+        parameter (npbversion='3.3.1')
+        character*6 cs1
+        parameter (cs1='(none)')
+        character*9 cs2
+        parameter (cs2='${MPIF77}')
+        character*6 cs3
+        parameter (cs3='(none)')
+        character*6 cs4
+        parameter (cs4='(none)')
+        character*15 cs5
+        parameter (cs5='$(BT_F90_FLAGS)')
+        character*17 cs6
+        parameter (cs6='$(BT_F90_LDFLAGS)')
+        character*6 cs7
+        parameter (cs7='randi8')

examples/BT/run_xc40.sh

+#!/bin/bash
+
+NODES=16
+PPN=16
+NPROCS=$((NODES*PPN))
+TARGET="/lus/theta-fs0/projects/Performance/ftessier/TAPIOCA"
+DDT="/soft/debuggers/ddt/bin/ddt --connect"
+STRIPE_COUNT=16
+STRIPE_SIZE=8388608
+AGGR_MULT=1
+
+cd $HOME/install/$ARCHI/bin/
+
+export TAPIOCA_STRATEGY=TOPOLOGY_AWARE
+
+export TAPIOCA_NBAGGR=2
+export TAPIOCA_NBBUFFERS=2
+export TAPIOCA_BUFFERSIZE=8388608
+export TAPIOCA_AGGRTIER=DDR
+
+export TAPIOCA_DEVNULL=false
+export TAPIOCA_COMMSPLIT=true
+export TAPIOCA_PIPELINING=true
+export TAPIOCA_REELECTAGGR=true
+
+function updateSettings()
+{
+    printenv | egrep "TAPIOCA_"
+    SETTINGS="-e TAPIOCA_STRATEGY=$TAPIOCA_STRATEGY"
+
+    SETTINGS="$SETTINGS -e TAPIOCA_NBAGGR=$TAPIOCA_NBAGGR"
+    SETTINGS="$SETTINGS -e TAPIOCA_NBBUFFERS=$TAPIOCA_NBBUFFERS"
+    SETTINGS="$SETTINGS -e TAPIOCA_BUFFERSIZE=$TAPIOCA_BUFFERSIZE"
+    SETTINGS="$SETTINGS -e TAPIOCA_AGGRTIER=$TAPIOCA_AGGRTIER"
+
+    SETTINGS="$SETTINGS -e TAPIOCA_DEVNULL=$TAPIOCA_DEVNULL"
+    SETTINGS="$SETTINGS -e TAPIOCA_COMMSPLIT=$TAPIOCA_COMMSPLIT"
+    SETTINGS="$SETTINGS -e TAPIOCA_PIPELINING=$TAPIOCA_PIPELINING"
+    SETTINGS="$SETTINGS -e TAPIOCA_REELECTAGGR=$TAPIOCA_REELECTAGGR"
+
+    SETTINGS="$SETTINGS -e MPICH_RMA_OVER_DMAPP=1"
+    SETTINGS="$SETTINGS -e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=$AGGR_MULT"
+}
+
+function setLustreFS ()
+{
+    rm $TARGET/*
+    lfs setstripe --stripe-count $STRIPE_COUNT --stripe-size $STRIPE_SIZE $TARGET
+    lfs getstripe $TARGET
+}
+
+CLASS=B
+#IO=mpi_io_full
+IO=fortran_io
+#IO=tapioca
+updateSettings
+setLustreFS
+
+aprun $SETTINGS -n $NPROCS -N $PPN bt.${CLASS}.${NPROCS}.${IO}

examples/BT/sys/Makefile

@@ -13,7 +13,8 @@ all: setparams
 # was compiled and linked
 
 setparams: setparams.c ../config/make.def
-	$(CC) ${CONVERTFLAG} -o setparams setparams.c
+	#$(CC) ${CONVERTFLAG} -o setparams setparams.c
+	gcc ${CONVERTFLAG} -o setparams setparams.c
 
 
 clean: 

examples/BT/sys/setparams.c

@@ -556,9 +556,11 @@ void write_bt_info(FILE *fp, int nprocs, char class, int io)
   fprintf(fp, "%sinteger iotype\n", FINDENT);
   fprintf(fp, "%sparameter (iotype = %d)\n", FINDENT, io);
   if (io) {
+    fprintf(fp, "%scharacter*(*) tmppath\n", FINDENT);
+    fprintf(fp, "%sparameter (tmppath = '/lus/theta-fs0/projects/Performance')\n", FINDENT);
     fprintf(fp, "%scharacter*(*) filepath\n", FINDENT);
-    fprintf(fp, "%sparameter (filepath = '/projects/Performance/ftessier/BT-IO')\n", FINDENT);
-    fprintf(fp, "%scharacter*(*) filenm\n", FINDENT);
+    fprintf(fp, "%sparameter (filepath = trim(tmppath)//'/ftessier/TAPIOCA')\n", FINDENT);
+        fprintf(fp, "%scharacter*(*) filenm\n", FINDENT);
     switch (io) {
 	case FULL:
 	    fprintf(fp, "%sparameter (filenm = trim(filepath)//'/btio.full.out')\n", FINDENT);

examples/BT/tapioca.f

-
+c
+c Author: François Tessier
+c         Argonne National Lab
+c         ftessier@anl.gov
+c
 c---------------------------------------------------------------------
 c---------------------------------------------------------------------
 
-      subroutine setup_btio
+      subroutine setup_btio(niter, wr_interval)
 
 c---------------------------------------------------------------------
 c---------------------------------------------------------------------
@@ -11,146 +15,40 @@ c---------------------------------------------------------------------
       include 'mpinpb.h'
 
       integer ierr
-      integer mstatus(MPI_STATUS_SIZE)
-      integer sizes(4), starts(4), subsizes(4)
-      integer cell_btype(maxcells), cell_ftype(maxcells)
-      integer cell_blength(maxcells)
-      integer info
-      character*20 cb_nodes, cb_size
       integer c, m
-      integer cell_disp(maxcells)
-
-       call mpi_bcast(collbuf_nodes, 1, MPI_INTEGER,
-     >                root, comm_setup, ierr)
-
-       call mpi_bcast(collbuf_size, 1, MPI_INTEGER,
-     >                root, comm_setup, ierr)
-
-       info = MPI_INFO_NULL
-       call MPI_Type_contiguous(5, MPI_DOUBLE_PRECISION,
-     $                          element, ierr)
-       call MPI_Type_commit(element, ierr)
-       call MPI_Type_extent(element, eltext, ierr)
-
-       do  c = 1, ncells
-c
-c Outer array dimensions ar same for every cell
-c
-           sizes(1) = IMAX+4
-           sizes(2) = JMAX+4
-           sizes(3) = KMAX+4
-c
-c 4th dimension is cell number, total of maxcells cells
-c
-           sizes(4) = maxcells
-c
-c Internal dimensions of cells can differ slightly between cells
-c
-           subsizes(1) = cell_size(1, c)
-           subsizes(2) = cell_size(2, c)
-           subsizes(3) = cell_size(3, c)
-c
-c Cell is 4th dimension, 1 cell per cell type to handle varying 
-c cell sub-array sizes
-c
-           subsizes(4) = 1
-
-c
-c type constructors use 0-based start addresses
-c
-           starts(1) = 2 
-           starts(2) = 2
-           starts(3) = 2
-           starts(4) = c-1
-
-c 
-c Create buftype for a cell
-c
-           call MPI_Type_create_subarray(4, sizes, subsizes, 
-     $          starts, MPI_ORDER_FORTRAN, element, 
-     $          cell_btype(c), ierr)
-c
-c block length and displacement for joining cells - 
-c 1 cell buftype per block, cell buftypes have own displacment
-c generated from cell number (4th array dimension)
-c
-           cell_blength(c) = 1
-           cell_disp(c) = 0
-
-       enddo
-c
-c Create combined buftype for all cells
-c
-       call MPI_Type_struct(ncells, cell_blength, cell_disp,
-     $            cell_btype, combined_btype, ierr)
-       call MPI_Type_commit(combined_btype, ierr)
-
-       do  c = 1, ncells
-c
-c Entire array size
-c
-           sizes(1) = PROBLEM_SIZE
-           sizes(2) = PROBLEM_SIZE
-           sizes(3) = PROBLEM_SIZE
-
-c
-c Size of c'th cell
-c
-           subsizes(1) = cell_size(1, c)
-           subsizes(2) = cell_size(2, c)
-           subsizes(3) = cell_size(3, c)
-
-c
-c Starting point in full array of c'th cell
-c
-           starts(1) = cell_low(1,c)
-           starts(2) = cell_low(2,c)
-           starts(3) = cell_low(3,c)
-
-           call MPI_Type_create_subarray(3, sizes, subsizes,
-     $          starts, MPI_ORDER_FORTRAN,
-     $          element, cell_ftype(c), ierr)
-           cell_blength(c) = 1
-           cell_disp(c) = 0
-       enddo
-
-       call MPI_Type_struct(ncells, cell_blength, cell_disp,
-     $            cell_ftype, combined_ftype, ierr)
-       call MPI_Type_commit(combined_ftype, ierr)
-
-       iseek=0
-       if (node .eq. root) then
-          call MPI_File_delete(filenm, MPI_INFO_NULL, ierr)
-       endif
-
-
-      call MPI_Barrier(comm_solve, ierr)
-
-       call MPI_File_open(comm_solve,
-     $          filenm,
-     $          MPI_MODE_RDWR+MPI_MODE_CREATE,
-     $          MPI_INFO_NULL, fp, ierr)
-
-       if (ierr .ne. MPI_SUCCESS) then
-                print *, 'Error opening file'
-                stop
-       endif
-
-        call MPI_File_set_view(fp, iseek, element, 
-     $          combined_ftype, 'native', info, ierr)
-
-       if (ierr .ne. MPI_SUCCESS) then
-                print *, 'Error setting file view'
-                stop
-       endif
+c---------------------------------------------------------------------
+c  TAPIOCA
+c---------------------------------------------------------------------
+      integer niter, wr_interval
+      integer wr_step
+      integer*8 header
+      integer*8 chunkcount(niter/wr_interval)
+      integer chunksize(niter/wr_interval)
+      integer*8 chunkoffset(niter/wr_interval)
+      header = 0
+
+      chunkcount  (1) = 1
+      chunksize   (1) = iosize/eltext
+      chunkoffset (1) = iseek
+      
+      do wr_step = 2, niter/wr_interval
+         chunkcount  (wr_step) = 1
+         chunksize   (wr_step) = iosize/eltext
+         chunkoffset (wr_step) = chunkoffset (wr_step-1) + iosize/eltext
+      enddo
 
+      call tapioca_init ( chunkcount, 
+     $                    chunksize, 
+     $                    chunkoffset, 
+     $                    niter/wr_interval, 
+     $                    header, filenm, comm_solve)
+      
       do m = 1, 5
          xce_sub(m) = 0.d0
       end do
-
+      
       idump_sub = 0
-
-
+      
       return
       end
 
@@ -167,14 +65,8 @@ c---------------------------------------------------------------------
       integer mstatus(MPI_STATUS_SIZE)
       integer ierr
 
-      call MPI_File_write_at_all(fp, iseek, u,
-     $                           1, combined_btype, mstatus, ierr)
-      if (ierr .ne. MPI_SUCCESS) then
-          print *, 'Error writing to file'
-          stop
-      endif
+      call tapioca_write ( iseek, u, 1, combined_btype, mstatus )
 
-      call MPI_Type_size(combined_btype, iosize, ierr)
       iseek = iseek + iosize/eltext
 
       idump_sub = idump_sub + 1
@@ -182,7 +74,7 @@ c---------------------------------------------------------------------
          if (idump_sub .ge. rd_interval) then
 
             iseek = 0
-            call acc_sub_norms(idump+1)
+c     call acc_sub_norms(idump+1)
 
             iseek = 0
             idump_sub = 0
@@ -244,8 +136,8 @@ c---------------------------------------------------------------------
 
       integer ierr
 
-      call MPI_File_close(fp, ierr)
-
+      call tapioca_finalize ( )
+      
       return
       end
 
@@ -283,7 +175,7 @@ c     clear the last time step
 
 c     read back the time steps and accumulate norms
 
-      call acc_sub_norms(idump)
+c      call acc_sub_norms(idump)
 
       call MPI_File_close(fp, ierr)