ddn-ime-evaluation/benchmark/run_v2.sh

#!/bin/bash 

function force_exit {
    echo "Committing suicide (PID $$)"
    exit
}

trap force_exit SIGINT

. ./config.sh



for COUNT in ${COUNT_ARR[@]}; do
for TYPE in ${TYPE_ARR[@]}; do
for NN in ${NN_ARR[@]}; do 
for T in ${T_ARR[@]}; do 
for PPN in ${PPN_ARR[@]}; do 
for API in ${API_ARR[@]}; do 
for FS in ${FS_ARR[@]}; do 
    BENCHFILE="./output_v2/COUNT:$COUNT-NN:$NN-PPN:$PPN-API:$API-FS:$FS-T:$T-TYPE:$TYPE.txt"

    if [ ! -e "${BENCHFILE}" ]; then
        OUTDIR="$(dirname $BENCHFILE)"
        [ ! -d $OUTDIR ] && mkdir $OUTDIR
        touch $BENCHFILE

        if [[ $API == "IME" ]]; then
            if [[ "read" == $TYPE ]]; then
                IOR_TYPE_OPTS="-r -E -k"
                #IOR_TYPE_OPTS="-w -k"
                TESTFILE="ime:///gsfs/jtacquaviva/file_read"
                ime-stat $TESTFILE
                #ime-prestage -b $TESTFILE
                #ime-stat $TESTFILE
            elif [[ "write" == $TYPE ]]; then
                IOR_TYPE_OPTS="-w"
                CLEAN_UP_TESTFILE="/gsfs/jtacquaviva/file_write"
                TESTFILE="ime:///gsfs/jtacquaviva/file_write"
            fi
        elif [[ $API == "MPIIO" ]]; then
            if [[ "read" == $TYPE ]]; then
                IOR_TYPE_OPTS="-r -E -k"
                if [[ "gpfs" == $FS ]]; then
                    TESTFILE="/gsfs/jtacquaviva/file_read_gpfs"
                elif [[ "fuse" == $FS ]]; then
                    TESTFILE="/ime/jtacquaviva/file_read"
                    ime-stat $TESTFILE
                else
                    echo "Unknown FS $FS"
                fi  
                TESTDIR="$(dirname $TESTFILE)"
            elif [[ "write" == $TYPE ]]; then
                IOR_TYPE_OPTS="-w"
                if [[ "gpfs" == $FS ]]; then
                    TESTFILE="/gsfs/jtacquaviva/file_write_gpfs"
                elif [[ "fuse" == $FS ]]; then
                    TESTFILE="/ime/jtacquaviva/file_write"
                else
                    echo "Unknown FS $FS"
                fi
                TESTDIR="$(dirname $TESTFILE)"
            fi
        fi

        IOR_PARAMS="-i 3 "
        IOR_PARAMS+="-s 1 -t $T -b $(($DATASIZE / $PPN)) "
        IOR_PARAMS+="-D 60 "
        IOR_PARAMS+="-O stoneWallingWearOut=1 "
        IOR_PARAMS+="-a $API "
        #IOR_PARAMS+="-p " # preallocate -- preallocate file size
        #IOR_PARAMS+="-e " # fsync -- perform sync operation after each block write
        IOR_PARAMS+="-g " # intraTestBarriers -- use barriers between open, write/read, and close
        IOR_PARAMS+="-H " # showHints -- show hints 
        IOR_PARAMS+="-z " # randomOffset -- access is to random, not sequential, offsets within a file
        IOR_PARAMS+="-o $TESTFILE $IOR_TYPE_OPTS "
        IOR_PARAMS+="-O summaryFile=${BENCHFILE}.json "
        IOR_PARAMS+="-O summaryFormat=JSON "
        
        if [[ "isc17" == ${hostname:0:5} ]]; then
            #$MPIEXEC -np $NN --hosts $(hosts $NN) ./drop_caches.sh
            $MPIEXEC -np $NN --hosts $(hosts $NN) echo 3 > /proc/sys/vm/drop_caches 
            MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) --hosts $(hosts $NN) -wdir $WD "
            MPIEXEC_PARAMS+="-genv MV2_NUM_HCAS 1 "
            MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_LEVEL core "
            MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_POLICY scatter "
            MPIEXEC_PARAMS+="-genv MV2_SHOW_CPU_BINDING 1 "
            MPIEXEC_PARAMS+="-genv RDMAV_FORK_SAFE 1 "
            MPIEXEC_PARAMS+="-genv RDMAV_HUGEPAGES_SAFE 1 "
        elif [[ "m" == ${hostname:0:1} ]]; then
            #MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) -wdir $WD --host $(hosts $NN) "
            MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) -wdir $WD "
        else
            echo "Cluster $hostname is not supported. Quitting."
            exit 1
        fi

        NETOUTDIR="${BENCHFILE}_network"
        [[ ! -d $NETOUTDIR ]] && mkdir $NETOUTDIR || rm $NETOUTDIR/HOST*PID*.txt

        WAIT=true
        declare -a avgload
        while $WAIT; do
            for i in 1 2 3 4; do
                avgload[$i]=$(ssh ime${i} 'cat /proc/loadavg' | cut -f1 -d" ")
                if [[ ${avgload[$i]} > 25 ]]; then
                    WAIT=true
                    echo "ime$i is not ready. Average load is ${avgload[$i]}. Waiting."
                    break
                else
                    WAIT=false
                fi
            done
            sleep 5
        done

        (
        set -x
        $MPIEXEC $MPIEXEC_PARAMS ./ior_wrapper.sh "$IOR_PARAMS" "$NETOUTDIR" | tee -a $BENCHFILE
        #$MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS | tee -a $BENCHFILE

        if [[ "" != $CLEAN_UP_TESTFILE ]]; then
            echo rm $CLEAN_UP_TESTFILE
        fi
        set +x
        ) 2> >(tee -a $BENCHFILE)
    else
        echo "skip $(readlink -f $BENCHFILE), already exists"
    fi

done
done
done
done
done
done
done