From 284bd84088781b9336c5fcbab0ad6cd82fdf49c4 Mon Sep 17 00:00:00 2001
From: Eugen Betke <betke@dkrz.de>
Date: Fri, 23 Nov 2018 05:26:30 +0100
Subject: [PATCH] IME bench working

---
 benchmark/config.sh               | 23 +++++++++++--------
 benchmark/run_v2.sh               | 38 +++++++++++++++++--------------
 benchmark/tool_show_good_nodes.sh | 24 +++++++++++--------
 3 files changed, 49 insertions(+), 36 deletions(-)

diff --git a/benchmark/config.sh b/benchmark/config.sh
index 20bc4a972..e7c62e250 100755
--- a/benchmark/config.sh
+++ b/benchmark/config.sh
@@ -15,11 +15,14 @@ if [[ "isc17" == ${hostname:0:5} ]]; then
     #module list
     export TD="/esfs/jtacquaviva/testfiles"
     export WD="/esfs/jtacquaviva/git/ddn-ime-evaluation/benchmark"
-    export NODES=( \
-        isc17-c02 isc17-c03 isc17-c04 isc17-c05 \
-        isc17-c06 isc17-c07 isc17-c08 isc17-c09 \
-        isc17-c12 isc17-c13 isc17-c14 isc17-c15 \
-        isc17-c18 isc17-c22 isc17-c01 )
+#    export NODES=( \
+#        isc17-c02 isc17-c03 isc17-c04 isc17-c05 \
+#        isc17-c06 isc17-c07 isc17-c08 isc17-c09 \
+#        isc17-c12 isc17-c13 isc17-c14 isc17-c15 \
+#        isc17-c18 isc17-c22 isc17-c01 )
+
+    export NODES=( isc17-c02 isc17-c03 isc17-c04 isc17-c05 isc17-c06 isc17-c08 isc17-c09 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c22 isc17-c23 isc17-c01 )
+ 
 
 elif [[ "m" == ${hostname:0:1} ]]; then
     . /sw/rhel6-x64/tcl/modules-3.2.10/Modules/3.2.10/init/sh
@@ -57,13 +60,13 @@ function hosts() {
 #DATASIZE=$((4800 * 1024 * 1024 * 16))
 DATASIZE=$((48000 * 1024 * 1024 * 16))
 
-COUNT_ARR=( $(seq 3) )
-TYPE_ARR=( "write" "read" )
-API_ARR=( "POSIX") # "MPIIO" ) #
+COUNT_ARR=( $(seq 1) )
+TYPE_ARR=( "write" )
+API_ARR=( "IME") # "MPIIO" ) #
 #API_ARR=( "MPIIO" ) #
 #NN_ARR=( 1 2 4 8 16)
-NN_ARR=( 15 4 8 2 1)
-PPN_ARR=( 8 4 1 )
+NN_ARR=( 1 4 8 2 15)
+PPN_ARR=( 1 4 8 )
 T_ARR=( $((10*1024*1024)) $((1*1024*1024)) $((100*1024)) $((16*1024)) )
 
 #COUNT_ARR=( $(seq 3) )
diff --git a/benchmark/run_v2.sh b/benchmark/run_v2.sh
index b0db3f848..d34f564f0 100755
--- a/benchmark/run_v2.sh
+++ b/benchmark/run_v2.sh
@@ -24,22 +24,26 @@ for API in ${API_ARR[@]}; do
         [ ! -d $OUTDIR ] && mkdir $OUTDIR
         touch $BENCHFILE
 
-        if [[ "read" == $TYPE ]]; then
-            IOR_TYPE_OPTS="-r -E -k"
-            LUSTRE_TESTFILE="$TD/sharedread$NN/file"
-            TESTDIR="$(dirname $LUSTRE_TESTFILE)"
-            lfs getstripe $TESTDIR | tee -a $BENCHFILE
-        elif [[ "write" == $TYPE ]]; then
-            IOR_TYPE_OPTS="-w"
-            LUSTRE_TESTFILE="$TD/sharedwrite$NN/file"
-            TESTDIR="$(dirname $LUSTRE_TESTFILE)"
-            [ -d $TESTDIR ] && rm -r $TESTDIR 
-            mkdir -p $TESTDIR 
-            lfs setstripe	-c $((2 * $NN)) $TESTDIR
-            lfs getstripe $TESTDIR | tee -a $BENCHFILE
+        if [[ $API == "IME" ]]; then
+                TESTFILE="ime:///gsfs/jtacquaviva/file"
+        elif [[ $API == "POSIX" ]]; then
+            if [[ "read" == $TYPE ]]; then
+                IOR_TYPE_OPTS="-r -E -k"
+                TESTFILE="$TD/sharedread$NN/file"
+                TESTDIR="$(dirname $TESTFILE)"
+                lfs getstripe $TESTDIR | tee -a $BENCHFILE
+            elif [[ "write" == $TYPE ]]; then
+                IOR_TYPE_OPTS="-w"
+                TESTFILE="$TD/sharedwrite$NN/file"
+                TESTDIR="$(dirname $TESTFILE)"
+                [ -d $TESTDIR ] && rm -r $TESTDIR 
+                mkdir -p $TESTDIR 
+                lfs setstripe	-c $((2 * $NN)) $TESTDIR
+                lfs getstripe $TESTDIR | tee -a $BENCHFILE
+            fi
         fi
 
-        IOR_PARAMS="-i 1 "
+        IOR_PARAMS="-i 3 "
         IOR_PARAMS+="-s 1 -t $T -b $(($DATASIZE / $PPN)) "
         IOR_PARAMS+="-D 60 -O stoneWallingWearOut=1 "
         IOR_PARAMS+="-a $API "
@@ -48,7 +52,7 @@ for API in ${API_ARR[@]}; do
         IOR_PARAMS+="-g " # intraTestBarriers -- use barriers between open, write/read, and close
         IOR_PARAMS+="-H " # showHints -- show hints 
         IOR_PARAMS+="-z " # randomOffset -- access is to random, not sequential, offsets within a file
-        IOR_PARAMS+="-o $LUSTRE_TESTFILE $IOR_TYPE_OPTS "
+        IOR_PARAMS+="-o $TESTFILE $IOR_TYPE_OPTS "
         IOR_PARAMS+="-O summaryFile=${BENCHFILE}.json "
         IOR_PARAMS+="-O summaryFormat=JSON "
         
@@ -71,8 +75,8 @@ for API in ${API_ARR[@]}; do
 
         (
         set -x
-        $MPIEXEC $MPIEXEC_PARAMS ./ior_wrapper.sh "$IOR_PARAMS" "$NETOUTDIR" | tee -a $BENCHFILE
-        #$MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS | tee -a $BENCHFILE
+        #$MPIEXEC $MPIEXEC_PARAMS ./ior_wrapper.sh "$IOR_PARAMS" "$NETOUTDIR" | tee -a $BENCHFILE
+        $MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS | tee -a $BENCHFILE
         set +x
         ) 2> >(tee -a $BENCHFILE)
     else
diff --git a/benchmark/tool_show_good_nodes.sh b/benchmark/tool_show_good_nodes.sh
index 225555420..946179539 100755
--- a/benchmark/tool_show_good_nodes.sh
+++ b/benchmark/tool_show_good_nodes.sh
@@ -4,20 +4,26 @@ function join_by() {
 	local IFS="$1"; shift; echo "$*"; 
 }
 
-cx4nodes=( isc17-c0{1..9} isc17-c{10..23} )
+#cx4nodes=( isc17-c0{1..9} isc17-c{10..23} )
+cx4nodes=( isc17-c0{1..9} isc17-c{11..23} )
+echo "isc17-c10 is blacklisted"
 node_list=$( join_by , "${cx4nodes[@]}" )
 
 
 IFS='' read -r -d '' script <<"EOF"
-if [ -d /esfs/jtacquaviva ]; then
-	cx4="$(lspci | grep ConnectX-4)"
-	if [[ "" != $cx4 ]]; then
-		echo 'ok'
-	else 
-		echo "no cx4"
-	fi
+if [ -d /gsfs/jtacquaviva ]; then
+    if [ -d /esfs/jtacquaviva ]; then
+        cx4="$(lspci | grep ConnectX-4)"
+        if [[ "" != $cx4 ]]; then
+            echo 'ok'
+        else 
+            echo "no cx4"
+        fi
+    else 
+        echo "/esfs/ is not mounted"
+    fi
 else 
-	echo "/esfs/ is not mounted"
+	echo "/gsfs/ is not mounted"
 fi
 EOF