#!/bin/bash 


function force_exit {
    echo "Committing suicide (PID $$)"
    exit
}

trap force_exit SIGINT

. ./config.sh

# Provides a list of good hosts (that contains QDR connection)
function hosts() {
    num="$1"
    #HOST_LIST=( isc17-c04 isc17-c05 isc17-c06 isc17-c07 isc17-c08 isc17-c09 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c22 isc17-c01 isc17-c02 isc17-c03 )
    HOST_LIST=( "m11388" )

    hlist=${HOST_LIST[0]}
    for POS in $(seq 1 $(($num - 1))) ; do 
        hlist="$hlist,${HOST_LIST[$POS]}"
    done
    echo $hlist
}


for COUNT in $(seq 1); do
for TYPE in ${TYPE_ARR[@]}; do
for NN in ${NN_ARR[@]}; do 
for T in ${T_ARR[@]}; do 
for PPN in ${PPN_ARR[@]}; do 
for API in ${API_ARR[@]}; do 

    BENCHFILE="./output_v2/COUNT:$COUNT#NN:$NN#PPN:$PPN#API:$API#T:$T#TYPE:$TYPE.txt"

    #if [ ! -e "${BENCHFILE}" ]; then
    if true; then
        OUTDIR="$(dirname $BENCHFILE)"
        [ ! -d $OUTDIR ] && mkdir $OUTDIR
        touch $BENCHFILE

        #$MPIEXEC $MPIEXEC_PARAMS ./drop_caches.sh

        IOR_TYPE_OPTS=""
        if [[ "read" == $TYPE ]]; then
            IOR_TYPE_OPTS="-r"
            LUSTRE_TESTFILE="$TD/sharedread$NN/file"
            TESTDIR="$(dirname $LUSTRE_TESTFILE)"
            lfs getstripe $TESTDIR | tee -a $BENCHFILE

        elif [[ "write" == $TYPE ]]; then
            IOR_TYPE_OPTS="-w"
            LUSTRE_TESTFILE="$TD/sharedwrite/file"
            TESTDIR="$(dirname $LUSTRE_TESTFILE)"
            [ -d $TESTDIR ] && rm -r $TESTDIR || mkdir -p $TESTDIR 
            lfs setstripe	-c $((2 * $NN)) $TESTDIR
            lfs getstripe $TESTDIR | tee -a $BENCHFILE
        fi


        IOR_PARAMS="-i 3 "
        IOR_PARAMS+="-s 1 -t $T -b $((4800 * 1024 * 1024 * 32 / $PPN)) "
        IOR_PARAMS+="-D 60 -O stoneWallingWearOut=1 "
        IOR_PARAMS+="-a $API "
        IOR_PARAMS+="-e -g -z -k -o $LUSTRE_TESTFILE $IOR_TYPE_OPTS "
        
        if [[ "isc17" == ${hostname:0:5} ]]; then
            MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) --hosts $(hosts $NN) -wdir $WD "
            MPIEXEC_PARAMS+="-genv MV2_NUM_HCAS 1 "
            MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_LEVEL core "
            MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_POLICY scatter "
        elif [[ "m" == ${hostname:0:1} ]]; then
            #MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) --host $(hosts $NN)"
            MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) -wdir $WD"
        else
            echo "Cluster $hostname is not supported. Quitting."
            exit 1
        fi

        (
        set -x
        $MPIEXEC $MPIEXEC_PARAMS ./ior_wrapper.sh "$IOR_PARAMS" "$BENCHFILE" | tee -a $BENCHFILE
        #$MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS | tee -a $BENCHFILE
        set +x
        ) 2> >(tee -a $BENCHFILE)
    else
        echo "skip $(readlink -f $BENCHFILE), already exists"
    fi

done
done
done
done
done
done