diff --git a/benchmark/batch_job.sh b/benchmark/batch_job.sh deleted file mode 100755 index 32e03236f..000000000 --- a/benchmark/batch_job.sh +++ /dev/null @@ -1,320 +0,0 @@ -#!/bin/bash -#SBATCH -w isc17-c[02,03,04,05,08,09,10,12,14,23] -#SBATCH --dependency=singleton -#SBATCH --output=slurm/slurm-%j.txt -#SBATCH --job-name=DDNIME - -##SBATCH -w isc17-c[02-06,08-10,12-14,18] - -#These nodes can be used with DDN IME -##SBATCH --distribution=block - - -SCRIPT="$(readlink -f $0)" -SCRIPTPATH="$(dirname $SCRIPT)" -cd "$SCRIPTPATH" - -function die() { - echo "$1" && exit 1 -} - -export MODULEPATH=/gsfs/betke/software/modules:$MODULEPATH - - -#module purge -#module load ddn/mvapich/3.1.4 -#module load betke/hdf5/1.8.20 -#module load betke/netcdf/4.4.1 -#module load betke/ior/git -#module load betke/netcdf-bench/git-20171206 -#module list - - -module purge -module load ddn/mvapich/3.1.4 -module load betke/hdf5/1.8.20-ddn -module load betke/netcdf/4.4.1-ddn -module load betke/ior/git-ddn -module load betke/netcdf-bench/git-20171206-ddn -module list - -type mpiexec 2>&1 || die "mpiexec not found" -type benchtool 2>&1 || die "benchtool not found" - - -IOR="$(which ior)" # default IOR (path to DDN-IOR is hardcoded in the nested loops) -NCBENCH="$(which benchtool)" -TESTDIR="/ime/isc18/perf" -TIMESTAMP="20171218_162418" -LUSTRE_TESTFILE="/esfs/isc18/perf/file" -FUSE_TESTFILE="/ime/isc18/perf/file" -IME_TESTFILE="ime:///esfs/isc18/perf/file" -OUTPUTDIR="./output" -SLURM_OUTPUT="slurm/slurm-%j.txt" -WORKDIR="/gsfs/betke/git/manny-benchmark/scripts" -cd $WORKDIR - - -# Benchmarks to run -# NCBENCH - NetCDF bench -# IOR - default IOR and DDN-IOR -PROG_ARR=("$IOR" "$NCBENCH") - -# Experiment scaling -# NN_ARR: number of nodes -# PPN_ARR: processes per node -NN_ARR=(10 8 6 4 2 1) -PPN_ARR=(8 6 4 2 1) - -# Data geomentry of int (4 bytes) values -# Attention: it was too small in the previous experiments -# TIME, X, Y, Z: size of dimensizion -# size = time * x * y * z * 4 bytes -TIME_ARR=(100) -XGEOM_ARR=($((8 * 2))) -YGEOM_ARR=($((8 * 8))) -ZGEOM_ARR=(2560 256 25 4) - -# I/O type (only for NetCDF-Bench, ior runs alway in independent mode) -# ind - independent -# coll - collective -TYPE_ARR=("ind" "coll") - -# Interface list -# ime - direct access to IME using DDN-IOR -# mpio - enable MPIO interface in IOR -# posix - enable POSIX interface in IOR -IFACE_ARR=("ime" "mpio" "posix") - -# File systems list -# ime - direct access to IMEs using DDN IOR -# fuse - indirect accesss to IMEs over FUSE -# lustre - Lustre files system -FS_ARR=("ime" "lustre" "fuse") - -# NetCDF4 chunking -# none - disable chunking -# auto - chunking enabled (see netcdf-bench documentation for details) -CHUNKED_ARR=("none" "-c=auto") - -# NetCDF4 unlimited dimensions -# none - disabled unlimited dimensions -# -u - enabled unlimited dimenstions -UNLIMITED_ARR=("none" "-u") # ( "none" "-u" ) - -# NetCDF4 fill value -# none - disable fill value -# -F - enable fill value -FILL_VALUE_ARR=("none") # "-F" ) - - -##### -SLURMDIR="$(dirname $SLURM_OUTPUT)" -mkdir -p $SLURMDIR - -TDIR="$(dirname $IME_TESTFILE)" -srun --nodes=1 --ntasks=1 mkdir -p $TDIR -mkdir -p $OUTPUTDIR - - - -# Run an experiment -function runTest(){ - local l_command="$1" - local l_out="$2" - local l_testdir="$3" - local l_stripes="$4" - - local l_wml="$(which mpiexec) -ppn 1 -np 1" - - local l_timestamp=$(date +"%Y%m%d_%H%M%S") - echo "create $l_out" - touch $l_out - echo "${l_timestamp}" &>> $l_out - echo "FILENAME $l_out" &>> $l_out - - rm -rf $l_testdir - mkdir -p $l_testdir - lfs setstripe -c $l_stripes $l_testdir - echo "$l_command" | tee -a $l_out - #eval "$l_command >> $l_out 2>&1" - $l_command >> $l_out 2>&1 - lfs getstripe $l_testdir >> $l_out 2>&1 - rm -rf $l_testdir -} - - -# Create function names -function makeOutputName() { - local l_outdir="$1" - local l_tag="$2" - local l_count="${3}" - local l_prog="$4" - local l_nn="$5" - local l_ppn="$6" - local l_t="$7" - local l_x="$8" - local l_y="$9" - local l_z="${10}" - local l_type="${11}" - local l_iface="${12}" - local l_fs="${13}" - local l_c="${14}" - local l_u="${15}" - local l_f="${16}" - - [[ "${l_c}" == "none" ]] && l_c="notset" || l_c="auto" - [[ "${l_u}" == "none" ]] && l_u="notset" || l_u="set" - [[ "${l_f}" == "none" ]] && l_f="notset" || l_f="set" - - local l_out="${l_outdir}/COUNT:${l_count}#TAG:${l_tag}#PROG:${l_prog}#NN:${l_nn}#PPN:${l_ppn}#TYPE:${l_type}#IFACE:${l_iface}#FS:${l_fs}#CHUNKED:${l_c}#FILLED:${l_f}#UNLIMITED:${l_u}#output:$l_t:${l_x}:${l_y}:${l_z}.txt" - echo ${l_out} -} - - - - - -# Parameter exploration loops -for COUNT in $(seq 3); do -for PROG in ${PROG_ARR[@]}; do -for NN in ${NN_ARR[@]}; do -for PPN in ${PPN_ARR[@]}; do -for T in ${TIME_ARR[@]}; do -for X in ${XGEOM_ARR[@]}; do -for Y in ${YGEOM_ARR[@]}; do -for Z in ${ZGEOM_ARR[@]}; do -for TYPE in ${TYPE_ARR[@]}; do -for IFACE in ${IFACE_ARR[@]}; do -for FS in ${FS_ARR[@]}; do -for CHUNKED in ${CHUNKED_ARR[@]}; do -for UNLIMITED in ${UNLIMITED_ARR[@]}; do -for FILL_VALUE in ${FILL_VALUE_ARR[@]}; do - BENCHFILE=$(makeOutputName $OUTPUTDIR $TIMESTAMP $COUNT $(basename $PROG) $NN $PPN $T $X $Y $Z $TYPE $IFACE $FS ${CHUNKED} ${UNLIMITED} ${FILL_VALUE}) - STRIPES=$(($NN * 2)) - - if [ ! -e "${BENCHFILE}" ]; then - if [[ $TYPE == "ind" && $UNLIMITED == "-u" ]]; then - echo "skip ${BENCHFILE}, because -t=ind and -u are not supported" - else - - HOST_LIST=isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c08,isc17-c09,isc17-c10,isc17-c12,isc17-c14,isc17-c23 - - - #http://www.prace-ri.eu/best-practice-guide-generic-x86-html/ - #"$ADAPTIVE " \ - #"-genv IM_MONITOR_FILE /dev/shm/native_mon.%p " \ - #"-genv IM_CLIENT_NET_BUFFERS 32 " \ - - # Workload manager configuration - WLM="$(which mpiexec) -ppn ${PPN} -np $(($NN*$PPN)) -hosts $HOST_LIST " - WLM+="-genv IM_CLIENT_NET_BUFFERS 32 " - WLM+="-genv MV2_NUM_HCAS 1 " - WLM+="-genv MV2_CPU_BINDING_LEVEL core " - WLM+="-genv MV2_CPU_BINDING_POLICY scatter " - WLM+="-genv MV2_SHOW_CPU_BINDING 1 " - - #WLM="srun --nodes=$NN --ntasks-per-node=$PPN --ntasks=$(($NN*$PPN))" - #WLM="mpiexec --allow-run-as-root -npernode $(($NN*$PPN)) -N $PPN" # for DDN IME system - #WLM="/opt/ddn/mvapich/bin/mpiexec.hydra -ppn $PPN -np $(($PPN*$NN)) -hosts `nodeset -S "," -e manny[600-607]`" - - case "$PROG" in - $IOR) - echo "RUN $IOR" - #set -x - B=$(($X*$Y*$Z*4)) - S=$T - - if [[ "none" == "${CHUNKED}" ]]; then - if [[ "none" == "${UNLIMITED}" ]]; then - if [[ "none" == "${FILL_VALUE}" ]]; then - - $PARAMS="-e -g -z" - PARAMS="-e -g" - - if [[ "lustre" == "${FS}" ]]; then - if [[ "posix" == "${IFACE}" ]]; then - if [[ "ind" == "${TYPE}" ]]; then - runTest "$WLM $IOR -s $S -t $B -b $B -o $LUSTRE_TESTFILE $PARAMS -f ./tests/POSIX-individual" "${BENCHFILE}" "$(dirname $LUSTRE_TESTFILE)" "$STRIPES" - fi - elif [[ "mpio" == "${IFACE}" ]]; then - if [[ "ind" == "${TYPE}" ]]; then - runTest "$WLM $IOR -s $S -t $B -b $B -o $LUSTRE_TESTFILE $PARAMS -f ./tests/MPI-shared" "${BENCHFILE}" "$(dirname $LUSTRE_TESTFILE)" "$STRIPES" - fi - fi - elif [[ "fuse" == "${FS}" ]]; then - if [[ "posix" == "${IFACE}" ]]; then - if [[ "ind" == "${TYPE}" ]]; then - runTest "$WLM $IOR -s $S -t $B -b $B -o $FUSE_TESTFILE $PARAMS -f ./tests/POSIX-individual" "${BENCHFILE}" "$(dirname $FUSE_TESTFILE)" "$STRIPES" - fi - elif [[ "mpio" == "${IFACE}" ]]; then - if [[ "ind" == "${TYPE}" ]]; then - runTest "$WLM $IOR -s $S -t $B -b $B -o $FUSE_TESTFILE $PARAMS -f ./tests/MPI-shared" "${BENCHFILE}" "$(dirname $FUSE_TESTFILE)" "$STRIPES" - fi - fi - elif [[ "ime" == "${FS}" ]]; then - if [[ "ime" == "${IFACE}" ]]; then - if [[ "ind" == "${TYPE}" ]]; then - runTest "$WLM /opt/ddn/ior/bin/IOR-mvapich -s $S -t $B -b $B -o $IME_TESTFILE $PARAMS -a "IM"" "${BENCHFILE}" "$TESTDIR" "$STRIPES" - fi - fi - fi - fi - fi - fi - - #runTest "$WLM $IOR -a IM -s $S -t $B -b $B -o ime://$TESTFILE -e -f ./tests/MPI-shared" "shared" "${BENCHFILE}" "$TESTDIR" "$STRIPES" - #set +x - ;; - $NCBENCH) - echo "RUN $NCBENCH" - #set -x - - PARAMS="" - if [[ "none" != "${CHUNKED}" ]]; then - PARAMS+="${CHUNKED} " - fi - if [[ "none" != "${UNLIMITED}" ]]; then - PARAMS+="${UNLIMITED} " - fi - if [[ "none" != "${FILL_VALUE}" ]]; then - PARAMS+="${FILL_VALUE} " - fi - - if [[ "lustre" == "${FS}" ]]; then - if [[ "mpio" == "${IFACE}" ]]; then - runTest "$WLM $NCBENCH -f=$LUSTRE_TESTFILE -n=$NN -p=$PPN -d=$T:$(($X*$NN)):$(($Y*$PPN)):$Z -w -r -t=$TYPE $PARAMS" "${BENCHFILE}" "$(dirname ${LUSTRE_TESTFILE})" "$STRIPES" "$DEBUG" - fi - elif [[ "fuse" == "${FS}" ]]; then - if [[ "mpio" == "${IFACE}" ]]; then - runTest "$WLM $NCBENCH -f=$FUSE_TESTFILE -n=$NN -p=$PPN -d=$T:$(($X*$NN)):$(($Y*$PPN)):$Z -w -r -t=$TYPE $PARAMS" "${BENCHFILE}" "$(dirname ${FUSE_TESTFILE})" "$STRIPES" "$DEBUG" - fi - fi - #set +x - ;; - *) - echo "$PROG not supported" - exit 1 - esac - fi - else - echo "skip $(readlink -f $BENCHFILE), already exists" - fi -#exit - -done -done -done -done -done -done -done -done -done -done -done -done -done -done - -exit 0 diff --git a/benchmark/drop_caches.sh b/benchmark/drop_caches.sh index 4b472eb84..3dd958be1 100755 --- a/benchmark/drop_caches.sh +++ b/benchmark/drop_caches.sh @@ -4,7 +4,7 @@ oscs=( $( find /proc/fs/lustre/osc -mindepth 1 -maxdepth 1 -type d ) ) while [ ! 0 -eq ${#oscs[@]} ]; do sync - echo 3 > /proc/sys/vm/drop_caches + #echo 3 > /proc/sys/vm/drop_caches for i in ${!oscs[@]}; do used_mb=$( grep -h used_mb ${oscs[$i]}/osc_cached_mb | cut -d" " -f 2 ) diff --git a/benchmark/run_v2.sh b/benchmark/run_v2.sh index aca014e81..00ad46c94 100755 --- a/benchmark/run_v2.sh +++ b/benchmark/run_v2.sh @@ -1,5 +1,6 @@ #!/bin/bash + function force_exit { echo "Committing suicide (PID $$)" exit @@ -7,13 +8,13 @@ function force_exit { trap force_exit SIGINT +. ./config.sh + # Provides a list of good hosts (that contains QDR connection) function hosts() { num="$1" - #HOST_LIST=( isc17-c04 isc17-c05 isc17-c06 isc17-c18 ) - - #HOST_LIST=( isc17-c04 isc17-c05 isc17-c06 isc17-c18 isc17-c01 isc17-c02 isc17-c03 isc17-c07 isc17-c08 isc17-c09 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c22 ) - HOST_LIST=( isc17-c04 isc17-c05 isc17-c06 isc17-c07 isc17-c08 isc17-c09 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c22 isc17-c01 isc17-c02 isc17-c03 ) + #HOST_LIST=( isc17-c04 isc17-c05 isc17-c06 isc17-c07 isc17-c08 isc17-c09 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c22 isc17-c01 isc17-c02 isc17-c03 ) + HOST_LIST=( "m11388" ) hlist=${HOST_LIST[0]} for POS in $(seq 1 $(($num - 1))) ; do @@ -22,28 +23,6 @@ function hosts() { echo $hlist } -export MODULEPATH=/esfs/jtacquaviva/software/modules:$MODULEPATH - -module purge -module load betke/hdf5/1.8.20-ddn -module load betke/ior/git-ddn -module list - - -LUSTRE_TESTFILE="" - -ITERATIONS=3 -IOR="$(which ior)" -MPIEXEC="/opt/ddn/mvapich/bin/mpiexec" - - -TYPE_ARR=( "read" ) -API_ARR=( "MPIIO" ) -#NN_ARR=( 4 2 1 8 10 16) -NN_ARR=( 1 2 4 8 16) -PPN_ARR=( 8 4 1 6 2) -T_ARR=( $((10*1024*1024)) $((1*1024*1024)) $((100*1024)) $((16*1024)) ) - for COUNT in $(seq 1); do for TYPE in ${TYPE_ARR[@]}; do @@ -54,53 +33,59 @@ for API in ${API_ARR[@]}; do BENCHFILE="./output_v2/COUNT:$COUNT#NN:$NN#PPN:$PPN#API:$API#T:$T#TYPE:$TYPE.txt" - if [ ! -e "${BENCHFILE}" ]; then - OUTDIR="$(dirname $BENCHFILE)" - if [ ! -d $OUTDIR ]; then - mkdir $OUTDIR - fi + #if [ ! -e "${BENCHFILE}" ]; then + if true; then + OUTDIR="$(dirname $BENCHFILE)" + [ ! -d $OUTDIR ] && mkdir $OUTDIR touch $BENCHFILE - IOR_API_OPTS="" + #$MPIEXEC $MPIEXEC_PARAMS ./drop_caches.sh IOR_TYPE_OPTS="" if [[ "read" == $TYPE ]]; then IOR_TYPE_OPTS="-r" - $MPIEXEC $MPIEXEC_PARAMS /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh - if [[ "POSIX" == $API ]]; then - IOR_API_OPTS="-F" - LUSTRE_TESTFILE="/esfs/jtacquaviva/indread$NN/file" - elif [[ "MPIIO" == $API ]]; then - IOR_API_OPTS="" - LUSTRE_TESTFILE="/esfs/jtacquaviva/sharedread$NN/file" - fi + LUSTRE_TESTFILE="$TD/sharedread$NN/file" + TESTDIR="$(dirname $LUSTRE_TESTFILE)" + lfs getstripe $TESTDIR | tee -a $BENCHFILE elif [[ "write" == $TYPE ]]; then IOR_TYPE_OPTS="-w" - LUSTRE_TESTFILE="/esfs/jtacquaviva/ioperf/file_write" - + LUSTRE_TESTFILE="$TD/sharedwrite/file" TESTDIR="$(dirname $LUSTRE_TESTFILE)" - if [ -d $TESTDIR ]; then - rm -r $TESTDIR - fi - mkdir -p $TESTDIR + [ -d $TESTDIR ] && rm -r $TESTDIR || mkdir -p $TESTDIR lfs setstripe -c $((2 * $NN)) $TESTDIR lfs getstripe $TESTDIR | tee -a $BENCHFILE fi - IOR_PARAMS="-i $ITERATIONS -s 1 -t $T -b $((4800 * 1024 * 1024 * 32 / $PPN)) -D $((30)) -a $API $IOR_API_OPTS -e -g -z -k" - ENVVAR="-genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter" - MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) $ENVVAR --hosts $(hosts $NN) " + IOR_PARAMS="-i 3 " + IOR_PARAMS+="-s 1 -t $T -b $((4800 * 1024 * 1024 * 32 / $PPN)) " + IOR_PARAMS+="-D 60 -O stoneWallingWearOut=1 " + IOR_PARAMS+="-a $API " + IOR_PARAMS+="-e -g -z -k -o $LUSTRE_TESTFILE $IOR_TYPE_OPTS " + + if [[ "isc17" == ${hostname:0:5} ]]; then + MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) --hosts $(hosts $NN) -wdir $WD " + MPIEXEC_PARAMS+="-genv MV2_NUM_HCAS 1 " + MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_LEVEL core " + MPIEXEC_PARAMS+="-genv MV2_CPU_BINDING_POLICY scatter " + elif [[ "m" == ${hostname:0:1} ]]; then + #MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) --host $(hosts $NN)" + MPIEXEC_PARAMS=" -ppn $PPN -n $(($NN * $PPN)) -wdir $WD" + else + echo "Cluster $hostname is not supported. Quitting." + exit 1 + fi - ( + ( set -x - $MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS -o $LUSTRE_TESTFILE $IOR_TYPE_OPTS | tee -a $BENCHFILE + $MPIEXEC $MPIEXEC_PARAMS ./ior_wrapper.sh "$IOR_PARAMS" "$BENCHFILE" | tee -a $BENCHFILE + #$MPIEXEC $MPIEXEC_PARAMS $IOR $IOR_PARAMS | tee -a $BENCHFILE set +x - ) 2> >(tee -a $BENCHFILE) - else - echo "skip $(readlink -f $BENCHFILE), already exists" - fi + ) 2> >(tee -a $BENCHFILE) + else + echo "skip $(readlink -f $BENCHFILE), already exists" + fi done done diff --git a/benchmark/tool_micro_run_.py b/benchmark/tool_micro_run_.py deleted file mode 100755 index 99c196be0..000000000 --- a/benchmark/tool_micro_run_.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -export MODULEPATH=/esfs/jtacquaviva/software/modules:$MODULEPATH - -module purge -module load betke/hdf5/1.8.20-ddn -module load betke/ior/git-ddn -module list - -IOTYPE="shared" -NN=16 -NODES='isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22,isc17-c01' - -if [[ "shared" == $IOTYPE ]]; then - LUSTRE_TESTFILE="/esfs/jtacquaviva/sharedread${NN}/file" -elif [[ "posix" == $IOTYPE ]]; then - LUSTRE_TESTFILE="/esfs/jtacquaviva/indread${NN}/file" -fi - -TESTDIR="$(dirname $LUSTRE_TESTFILE)" -mkdir $TESTDIR -lfs setstripe -c $(($NN * 2)) $TESTDIR - -ITERATIONS=1 - -if [[ "shared" == $IOTYPE ]]; then - IOR="$(which ior) -i $ITERATIONS -s 1 -t $((16 * 1024 * 1024)) -b $((4800 * 1024 * 1024 * 32 / 8)) -o $LUSTRE_TESTFILE -a MPIIO -e -g -k -w" -elif [[ "posix" == $IOTYPE ]]; then - IOR="$(which ior) -i $ITERATIONS -s 1 -t $((16 * 1024 * 1024)) -b $((4800 * 1024 * 1024 * 32)) -o $LUSTRE_TESTFILE -a POSIX -F -e -g -k -w" -fi - - -ENVVAR="-genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter" -MPIEXEC="/opt/ddn/mvapich/bin/mpiexec -ppn 8 -np $((8*$NN)) $ENVVAR -hosts isc17-c04,isc17-c05" - -set -x -$MPIEXEC $IOR -#$MPIEXEC $IOR -r -D $((5*60)) -set +x -#rm $LUSTRE_TESTFILE - diff --git a/benchmark/tool_sanity_check.sh b/benchmark/tool_sanity_check.sh index f90b0af40..4401c6614 100755 --- a/benchmark/tool_sanity_check.sh +++ b/benchmark/tool_sanity_check.sh @@ -1,9 +1,10 @@ #!/bin/bash +. ./config.sh -NN_ARR=( 4 2 1 8 10 12 14 16) -PPN_ARR=( 8 6 4 2 1 ) -T_ARR=( $((10*1024*1024)) $((1*1024*1024)) $((100*1024)) $((16*1024)) ) +#NN_ARR=( 4 2 1 8 10 12 14 16) +#PPN_ARR=( 8 6 4 2 1 ) +#T_ARR=( $((10*1024*1024)) $((1*1024*1024)) $((100*1024)) $((16*1024)) ) res="OK"