bake-kove.qsub 3.53 KB
Newer Older
Philip Carns's avatar
Philip Carns committed
1 2 3 4 5 6 7
#!/bin/bash
#COBALT -n 2
#COBALT -t 30
#COBALT --mode script
#COBALT -A radix-io
#COBALT -q ibleaf3-debug

Philip Carns's avatar
Philip Carns committed
8
# This job tests bake performance on top of a kove volume, alternatively with either
Philip Carns's avatar
Philip Carns committed
9 10 11 12 13 14
# pmdk or the file backend to bake.  Note that it is hard coded to use a specific
# xpd and volume; this job assumes it has exclusive access.

export FI_FORK_UNSAFE=1
export HOME=$SANDBOX
export KDSA_MPIO_MAX_CONN=1
15
export PMDK_PATH="/xmem/xpdha"
Philip Carns's avatar
Philip Carns committed
16 17 18

. $SANDBOX/spack/share/spack/setup-env.sh
BAKE_STABLE_VER=`spack find bake |grep bake |grep -v file-backend`
Philip Carns's avatar
Philip Carns committed
19
spack load -r ssg
Philip Carns's avatar
Philip Carns committed
20 21 22 23 24
spack load -r $BAKE_STABLE_VER
spack load -r mvapich2

module list

25
echo "### NOTE: all results in this output are using Kove XPD volumes"
Philip Carns's avatar
Philip Carns committed
26 27
echo "### NOTE: ALL BAKE RESULTS BELOW are using 2nd socket on cooley nodes: cores 6-11, also using numactl to constrain memory allocations"

28
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
Philip Carns's avatar
Philip Carns committed
29 30 31 32 33

echo "### verbs kove pmdk no threads"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
Philip Carns's avatar
Philip Carns committed
34
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12
Philip Carns's avatar
Philip Carns committed
35

36 37
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
Philip Carns's avatar
Philip Carns committed
38 39 40 41 42 43 44

echo "### verbs kove pmdk 4 rpc threads"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4

45 46
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
Philip Carns's avatar
Philip Carns committed
47 48 49 50 51 52 53

echo "### verbs kove pmdk no threads, pipelined"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -i

54 55
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
Philip Carns's avatar
Philip Carns committed
56 57 58 59 60 61 62

echo "### verbs kove pmdk 4 rpc threads, pipelined"
sleep 1
dd if=/dev/zero of=$PMDK_PATH bs=100M count=1 conv=notrunc
bake-mkpool $PMDK_PATH
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw -x 16777216 -m 34359738368 -n "verbs://" -p $PMDK_PATH -c 12 -r 4 -i

63 64
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
Philip Carns's avatar
Philip Carns committed
65

66 67 68
# reset to use file backend. Note that we use a different name for the
# benchmark executable as well for the one that was linked against this
# backend and has it's rpath set accordingly.
Philip Carns's avatar
Philip Carns committed
69 70 71 72 73
spack unload $BAKE_STABLE_VER
spack load bake@dev-file-backend

echo "### verbs kove filesystem no threads, pipelined"
sleep 1
74 75 76
dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdha oflag=direct
mount /kovefs/xfsa
bake-mkpool /kovefs/xfsa/bake.dat
77
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfsa/bake.dat -c 12 -i
78
umount /kovefs/xfsa
Philip Carns's avatar
Philip Carns committed
79

80 81
kdsa_manage disconnect xpdha
kdsa_manage connect mlx5_0.1/260535867639876.8:124391e4-a843-4cc8-ab28-39a93c7cc9e8 xpdha
Philip Carns's avatar
Philip Carns committed
82 83 84

echo "### verbs kove filesystem 4 rpc threads, pipelined"
sleep 1
85 86 87
dd bs=4M if=/projects/RAN/toonen/mochi-100GiB-xfs.img of=/dev/xpdha oflag=direct
mount /kovefs/xfsa
bake-mkpool /kovefs/xfsa/bake.dat
88
mpirun -f $COBALT_NODEFILE -n 2 numactl -N 1 -m 1 ./bake-p2p-bw-file -x 16777216 -m 34359738368 -n "verbs://" -p /kovefs/xfsa/bake.dat -c 12 -r 4 -i
89
umount /kovefs/xfsa
Philip Carns's avatar
Philip Carns committed
90

91
kdsa_manage disconnect xpdha