diff --git a/benchmark/config.sh b/benchmark/config.sh index 9082b5f7b..616d54096 100755 --- a/benchmark/config.sh +++ b/benchmark/config.sh @@ -10,7 +10,7 @@ if [[ "isc17" == ${hostname:0:5} ]]; then module load ddn/mvapich/3.1.4 module load root/hdf5/1.10.4 module load root/ior/git-20181107 - module list + #module list export TD="/esfs/jtacquaviva/testfiles" export WD="/esfs/jtacquaviva/git/ddn-ime-evaluation/benchmark" export NODES=( isc17-c04 isc17-c05 isc17-c02 isc17-c03 isc17-c05 isc17-c06 isc17-c07 isc17-c08 isc17-c09 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c22 isc17-c01 ) @@ -24,23 +24,26 @@ elif [[ "m" == ${hostname:0:1} ]]; then module load bullxmpi_mlx_mt/bullxmpi_mlx_mt-1.2.9.2 module load k202107/hdf5/1.10.4 module load k202107/ior/git-20181108 - module list + #module list export TD="/mnt/lustre01/work/ku0598/k202107/git/ddn-ime-evaluation/benchmark/wd" export WD="/mnt/lustre01/work/ku0598/k202107/git/ddn-ime-evaluation/benchmark" export NODES=() else - echo "Configuration faile: Cluster $hostname is not supported. Quitting." + echo "Configuration failed: Cluster $hostname is not supported. Quitting." exit 1 fi # Lustre cache on DDN cluster ist 32108MB. Test file has to be at least twice as large therefore: DATASIZE = 76800MB -DATASIZE=$((4800 * 1024 * 1024 * 16)) +#DATASIZE=$((4800 * 1024 * 1024 * 16)) +DATASIZE=$((48000 * 1024 * 1024 * 16)) COUNT_ARR=( $(seq 3) ) -TYPE_ARR=( "read" "write" ) -API_ARR=( "MPIIO" "POSIX") -NN_ARR=( 1 2 4 8 16) +TYPE_ARR=( "write" "read" ) +API_ARR=( "POSIX") # "MPIIO" ) # +#API_ARR=( "MPIIO" ) # +#NN_ARR=( 1 2 4 8 16) +NN_ARR=( 2 1) PPN_ARR=( 8 4 1 ) T_ARR=( $((10*1024*1024)) $((1*1024*1024)) $((100*1024)) $((16*1024)) ) diff --git a/benchmark/drop_caches.sh b/benchmark/drop_caches.sh index 4ca42ffdb..571e48db0 100755 --- a/benchmark/drop_caches.sh +++ b/benchmark/drop_caches.sh @@ -1,18 +1,16 @@ #!/bin/bash oscs=( $( find /proc/fs/lustre/osc -mindepth 1 -maxdepth 1 -type d ) ) -echo $oscs +#echo $oscs while [ ! 0 -eq ${#oscs[@]} ]; do - set -x sync echo 3 > /proc/sys/vm/drop_caches - set +x for i in ${!oscs[@]}; do used_mb=$( grep -h used_mb ${oscs[$i]}/osc_cached_mb | cut -d" " -f 2 ) if [ 0 == $used_mb ]; then - echo "remove from list ${oscs[$i]}, because cache is $used_mb MB" + #echo "remove from list ${oscs[$i]}, because cache is $used_mb MB" unset oscs[$i] else echo "skip ${oscs[$i]}, cache is still $used_mb MB" diff --git a/benchmark/ior_wrapper.sh b/benchmark/ior_wrapper.sh index 91220697e..4f5b664a1 100755 --- a/benchmark/ior_wrapper.sh +++ b/benchmark/ior_wrapper.sh @@ -7,11 +7,9 @@ NETOUTDIR=$2 HOST="$hostname" PID=$$ -NETOUTFILE="$NETOUTDIR/HOST:$HOST#PID:$PID.txt" +NETOUTFILE="$NETOUTDIR/HOST:$HOST-PID:$PID-LABEL:" function capture_network_state { - label=$1 - echo "LABEL $label" echo "TIMESTAMP $(date +%s)" perfquery -x cat /proc/net/dev @@ -19,7 +17,8 @@ function capture_network_state { cat /proc/fs/lustre/llite/esfs-*/max_cached_mb } -echo "" > $NETOUTFILE -echo "$( capture_network_state "START" )" &>> $NETOUTFILE +echo "$( capture_network_state )" &> "${NETOUTFILE}start.txt" $IOR $IOR_PARAMS -echo "$( capture_network_state "STOP" )" &>> $NETOUTFILE +echo "$( capture_network_state )" &> "${NETOUTFILE}stop.txt" + +exit 0 diff --git a/benchmark/prepare.sh b/benchmark/prepare.sh index f6c7d91f6..e7b1c3cb5 100755 --- a/benchmark/prepare.sh +++ b/benchmark/prepare.sh @@ -2,30 +2,26 @@ . ./config.sh -NODES='isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22,isc17-c01' +NODES='isc17-c04,isc17-c05,isc17-c02,isc17-c03,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22,isc17-c01' -#for NN in ${NN_ARR[@]}; do - -for NN in 1; do +for NN in ${NN_ARR[@]}; do LUSTRE_TESTFILE="$TD/sharedread${NN}/file" - TESTDIR="$(dirname $LUSTRE_TESTFILE)" - mkdir $TESTDIR - lfs setstripe -c $(($NN * 2)) $TESTDIR + if [ ! -f $LUSTRE_TESTFILE ]; then + TESTDIR="$(dirname $LUSTRE_TESTFILE)" + mkdir $TESTDIR + lfs setstripe -c $(($NN * 2)) $TESTDIR - MPIEXEC_PARAMS="-ppn 8 -np $((8*$NN)) -hosts isc17-c04,isc17-c05 " - MPIEXEC_PARAMS+="-genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter" + MPIEXEC_PARAMS="-ppn 8 -np $((8*$NN)) -hosts $NODES " + MPIEXEC_PARAMS+="-genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter" - IOR_PARAMS="-i 1 -s 1 -t $((16 * 1024 * 1024)) -b $(($DATASIZE / 8)) -o $LUSTRE_TESTFILE -a MPIIO -e -g -k " - IOR_PARAMS+="-D 60 -O stoneWallingWearOut=1 " + IOR_PARAMS="-i 1 -s 1 -t $((16 * 1024 * 1024)) -b $(($DATASIZE / 8)) -o $LUSTRE_TESTFILE -a MPIIO -e -g -k " + #IOR_PARAMS+="-D 60 -O stoneWallingWearOut=1 " - ./drop_caches.sh - set +x - $MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS -w - set -x + #./drop_caches.sh + set +x + $MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS -w + set -x -# ./drop_caches.sh -# set +x -# $MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS -r -# set -x + fi done diff --git a/benchmark/run_v2.sh b/benchmark/run_v2.sh index 7ffad920c..004c423a3 100755 --- a/benchmark/run_v2.sh +++ b/benchmark/run_v2.sh @@ -1,6 +1,5 @@ #!/bin/bash - function force_exit { echo "Committing suicide (PID $$)" exit @@ -27,45 +26,47 @@ for NN in ${NN_ARR[@]}; do for T in ${T_ARR[@]}; do for PPN in ${PPN_ARR[@]}; do for API in ${API_ARR[@]}; do + BENCHFILE="./output_v2/COUNT:$COUNT-NN:$NN-PPN:$PPN-API:$API-T:$T-TYPE:$TYPE.txt" - BENCHFILE="./output_v2/COUNT:$COUNT#NN:$NN#PPN:$PPN#API:$API#T:$T#TYPE:$TYPE.txt" - - #if [ ! -e "${BENCHFILE}" ]; then - if true; then + if [ ! -e "${BENCHFILE}" ]; then OUTDIR="$(dirname $BENCHFILE)" [ ! -d $OUTDIR ] && mkdir $OUTDIR touch $BENCHFILE - IOR_TYPE_OPTS="" if [[ "read" == $TYPE ]]; then - IOR_TYPE_OPTS="-r" + IOR_TYPE_OPTS="-r -E -k" LUSTRE_TESTFILE="$TD/sharedread$NN/file" TESTDIR="$(dirname $LUSTRE_TESTFILE)" lfs getstripe $TESTDIR | tee -a $BENCHFILE - elif [[ "write" == $TYPE ]]; then IOR_TYPE_OPTS="-w" - LUSTRE_TESTFILE="$TD/sharedwrite/file" + LUSTRE_TESTFILE="$TD/sharedwrite$NN/file" TESTDIR="$(dirname $LUSTRE_TESTFILE)" - [ -d $TESTDIR ] && rm -r $TESTDIR || mkdir -p $TESTDIR + [ -d $TESTDIR ] && rm -r $TESTDIR + mkdir -p $TESTDIR lfs setstripe -c $((2 * $NN)) $TESTDIR lfs getstripe $TESTDIR | tee -a $BENCHFILE fi IOR_PARAMS="-i 1 " - IOR_PARAMS+="-s 1 -t $T -b $((4800 * 1024 * 1024 * 32 / $PPN)) " + IOR_PARAMS+="-s 1 -t $T -b $(($DATASIZE / $PPN)) " IOR_PARAMS+="-D 60 -O stoneWallingWearOut=1 " IOR_PARAMS+="-a $API " - IOR_PARAMS+="-e -g -z -k -o $LUSTRE_TESTFILE $IOR_TYPE_OPTS " + #IOR_PARAMS+="-p " # preallocate -- preallocate file size + #IOR_PARAMS+="-e " # fsync -- perform sync operation after each block write + IOR_PARAMS+="-g " # intraTestBarriers -- use barriers between open, write/read, and close + IOR_PARAMS+="-H " # showHints -- show hints + IOR_PARAMS+="-z " # randomOffset -- access is to random, not sequential, offsets within a file + IOR_PARAMS+="-o $LUSTRE_TESTFILE $IOR_TYPE_OPTS " + IOR_PARAMS+="-O summaryFile=${BENCHFILE}.json " + IOR_PARAMS+="-O summaryFormat=JSON " if [[ "isc17" == ${hostname:0:5} ]]; then $MPIEXEC -np $NN --hosts $(hosts $NN) ./drop_caches.sh - MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) --hosts $(hosts $NN) -wdir $WD " MPIEXEC_PARAMS+="-genv MV2_NUM_HCAS 1 " MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_LEVEL core " MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_POLICY scatter " - elif [[ "m" == ${hostname:0:1} ]]; then #MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) -wdir $WD --host $(hosts $NN) " MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) -wdir $WD " @@ -74,12 +75,11 @@ for API in ${API_ARR[@]}; do exit 1 fi + NETOUTDIR="${BENCHFILE}_network" + [[ ! -d $NETOUTDIR ]] && mkdir $NETOUTDIR || rm $NETOUTDIR/HOST*PID*.txt + ( set -x - NETOUTDIR="${BENCHFILE}_network" -echo $NETjOUTDIR - [[ ! -d $NETOUTDIR ]] && mkdir $NETOUTDIR || rm $NETOUTDIR/HOST*PID*.txt - $MPIEXEC $MPIEXEC_PARAMS ./ior_wrapper.sh "$IOR_PARAMS" "$NETOUTDIR" | tee -a $BENCHFILE #$MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS | tee -a $BENCHFILE set +x