update regression scripts for cooley

- add bw tests
- switch cci tests to busy-spin only

tests/perf-regression/cooley/margo-p2p-latency-ofi.qsub → tests/perf-regression/cooley/margo-regression-ofi.qsub

@@ -5,6 +5,12 @@
 #COBALT -A radix-io
 #COBALT -q ibleaf3-debug
 
+# NOTE: as of Jan 31 2018, some of the test programs will hang in non-busy
+#       spin mode on cooley with RHEL 6.9.  They work fine with RHEL 7.4 though.
+#
+# NOTE: origin/master of libfabric also will not compile on RHEL 6.9 at this 
+#       either
+
 export FI_FORK_UNSAFE=1
 
 echo "## MPI (one way, double the latency for round trip):"
@@ -12,24 +18,22 @@ mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency
 
 sleep 1 
 
-#echo "## Margo CCI/VERBS (round trip):"
-#mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n cci+verbs://
-#
-#sleep 1
-#
-#echo "## Margo CCI/VERBS (round trip, Hg busy spin):"
-#mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n cci+verbs:// -t 0,0
-#
-#sleep 1
-#
 echo "## Margo OFI/VERBS (round trip, Hg busy spin):"
 mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n ofi+verbs:// -t 0,0
+echo "## Margo OFI/VERBS (bw, Hg busy spin):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n ofi+verbs:// -c 1 -D 20 -t 0,0
+echo "## Margo OFI/VERBS (bw, 8x concurrency, Hg busy spin):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n ofi+verbs:// -c 8 -D 20 -t 0,0
+
+sleep 1
 
-echo "## NOTE: OFI/VERBS HANGS WHEN NOT IN BUSY SPIN MODE AS OF 2018-01-24"
+echo "## Margo OFI/VERBS (round trip):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n ofi+verbs://
+echo "## Margo OFI/VERBS (bw):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n ofi+verbs:// -c 1 -D 20
+echo "## Margo OFI/VERBS (bw, 8x concurrency):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n ofi+verbs:// -c 8 -D 20
 
-# echo "## Margo CCI/VERBS (round trip) with diagnostics:"
-# mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n verbs:// -d $COBALT_JOBID.diag
 
-# TODO: mercury-runner freezes at shutdown w/ cci+verbs -- commenting out mercury finalize routines doesn't seem to help
 #echo "## Mercury-runner CCI/VERBS (round trip):"
 #mpirun -f $COBALT_NODEFILE -n 2 ./mercury-runner -q -c 100000 -l 1 -m c -M -d `pwd` 1 h0=verbs:// h1

tests/perf-regression/cooley/margo-p2p-latency.qsub → tests/perf-regression/cooley/margo-regression.qsub

@@ -8,25 +8,26 @@
 echo "## MPI (one way, double the latency for round trip):"
 mpirun -f $COBALT_NODEFILE -n 2 ./osu_latency
 
-sleep 1 
+# NOTE: the CCI tests sometimes either crash or hang unless run in busy-spin 
+#       mode on Cooley as of 2018-01-31
 
-echo "## Margo CCI/VERBS (round trip):"
-mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n cci+verbs://
-
-sleep 1
+# sleep 1 
+#
+# echo "## Margo CCI/VERBS (round trip):"
+# mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n cci+verbs://
+# echo "## Margo CCI/VERBS (bw):"
+# mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n cci+verbs:// -c 1 -D 20
+# echo "## Margo CCI/VERBS (bw, 8x concurrency):"
+# mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n cci+verbs:// -c 8 -D 20
+# sleep 1
 
 echo "## Margo CCI/VERBS (round trip, Hg busy spin):"
 mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n cci+verbs:// -t 0,0
+echo "## Margo CCI/VERBS (bw, Hg busy spin):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n cci+verbs:// -c 1 -D 20 -t 0,0
+echo "## Margo CCI/VERBS (bw, 8x concurrency, Hg busy spin):"
+mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-bw -x 1048576 -n cci+verbs:// -c 8 -D 20 -t 0,0
 
-#sleep 1
-#
-#echo "## Margo OFI/VERBS (round trip, Hg busy spin):"
-#mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n ofi+verbs:// -t 0,0
-#
-#echo "## NOTE: OFI/VERBS HANGS WHEN NOT IN BUSY SPIN MODE AS OF 2018-01-24"
-
-# echo "## Margo CCI/VERBS (round trip) with diagnostics:"
-# mpirun -f $COBALT_NODEFILE -n 2 ./margo-p2p-latency -i 100000 -n verbs:// -d $COBALT_JOBID.diag
 
 # TODO: mercury-runner freezes at shutdown w/ cci+verbs -- commenting out mercury finalize routines doesn't seem to help
 #echo "## Mercury-runner CCI/VERBS (round trip):"

tests/perf-regression/cooley/run-regression-ofi.sh

@@ -22,7 +22,7 @@ mkdir $SANDBOX
 
 # scratch area for job submission
 mkdir $JOBDIR
-cp margo-p2p-latency-ofi.qsub $JOBDIR
+cp margo-regression-ofi.qsub $JOBDIR
 
 cd $SANDBOX
 git clone https://github.com/carns/argobots.git
@@ -60,8 +60,12 @@ make -j 3
 make install
  
 # libfabric
+# NOTE: check out 1.5.x branch; as of late January 2018, master includes an
+#       initial perf counter implementation that won't work on Cooley's old 
+#       kernel
 echo "=== BUILDING LIBFABRIC ==="
 cd $SANDBOX/libfabric
+# git checkout v1.5.x
 libtoolize
 ./autogen.sh
 mkdir build
@@ -148,17 +152,18 @@ make install
 # set up job to run
 echo "=== SUBMITTING AND WAITING FOR JOB ==="
 cp $SANDBOX/ssg/build/tests/perf-regression/.libs/margo-p2p-latency $JOBDIR
+cp $SANDBOX/ssg/build/tests/perf-regression/.libs/margo-p2p-bw $JOBDIR
 cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR
 cp $PREFIX/bin/mercury-runner $JOBDIR
 cd $JOBDIR
-JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib ./margo-p2p-latency-ofi.qsub`
+JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib ./margo-regression-ofi.qsub`
 cqwait $JOBID
 
 echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
 # gather output, strip out funny characters, mail
 cat $JOBID.* > combined.$JOBID.txt
 dos2unix combined.$JOBID.txt
-mailx -s "margo-p2p-latency (cooley, ofi)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
+mailx -s "margo-regression (cooley, ofi)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
 
 cd /tmp
 rm -rf $SANDBOX

tests/perf-regression/cooley/run-regression.sh

@@ -22,7 +22,7 @@ mkdir $SANDBOX
 
 # scratch area for job submission
 mkdir $JOBDIR
-cp margo-p2p-latency.qsub $JOBDIR
+cp margo-regression.qsub $JOBDIR
 
 cd $SANDBOX
 git clone https://github.com/carns/argobots.git
@@ -148,18 +148,20 @@ make install
 # set up job to run
 echo "=== SUBMITTING AND WAITING FOR JOB ==="
 cp $SANDBOX/ssg/build/tests/perf-regression/.libs/margo-p2p-latency $JOBDIR
+cp $SANDBOX/ssg/build/tests/perf-regression/.libs/margo-p2p-bw $JOBDIR
 cp $PREFIX/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency $JOBDIR
 cp $PREFIX/bin/mercury-runner $JOBDIR
 cd $JOBDIR
-JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib ./margo-p2p-latency.qsub`
+JOBID=`qsub --env LD_LIBRARY_PATH=$PREFIX/lib ./margo-regression.qsub`
 cqwait $JOBID
 
 echo "=== JOB DONE, COLLECTING AND SENDING RESULTS ==="
 # gather output, strip out funny characters, mail
 cat $JOBID.* > combined.$JOBID.txt
 dos2unix combined.$JOBID.txt
-mailx -s "margo-p2p-latency (cooley, cci)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
+mailx -s "margo-regression (cooley, cci)" sds-commits@lists.mcs.anl.gov < combined.$JOBID.txt
 
 cd /tmp
+echo $SANDBOX $PREFIX
 rm -rf $SANDBOX
 rm -rf $PREFIX