diff --git a/micro_run.py b/micro_run.py index ca64b5431..1b8eef8b2 100755 --- a/micro_run.py +++ b/micro_run.py @@ -8,10 +8,11 @@ module load betke/ior/git-ddn module list LUSTRE_TESTFILE="/esfs/jtacquaviva/file" -ITERATIONS=1 +ITERATIONS=3 IOR="$(which ior) -i $ITERATIONS -s 1 -t $((1 * 1024 * 1024)) -b $((128 * 1024 * 1024 * 1024)) -o $LUSTRE_TESTFILE -e -g -z -k" MPIEXEC="/opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 1 -hosts isc17-c17" $MPIEXEC $IOR -w $MPIEXEC $IOR -r -D $((5*60)) rm $LUSTRE_TESTFILE + diff --git a/output/COUNT:1#NN:1#PPN:1#API:MPIIO#T:1048576.txt b/output/COUNT:1#NN:1#PPN:1#API:MPIIO#T:1048576.txt new file mode 100644 index 000000000..dbd0d560b --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:1#API:MPIIO#T:1048576.txt @@ -0,0 +1,69 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 19:29:21 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:29:21 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1126.11 137871360 1024.00 0.001397 119.56 0.000280 119.56 0 +write 1178.04 137871360 1024.00 0.027352 114.26 0.000223 114.29 1 +write 1158.85 137871360 1024.00 0.022593 116.14 0.000176 116.18 2 + +Max Write: 1178.04 MiB/sec (1235.27 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1178.04 1126.11 1154.33 21.44 116.67916 0 1 1 3 0 0 1 0 0 1 141180272640 1048576 141180272640 MPIIO 0 + +Finished: Sat Oct 20 19:35:23 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 19:35:23 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:35:23 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 963.96 137871360 1024.00 0.000703 139.67 0.000229 139.67 0 +read 960.85 137871360 1024.00 0.000514 140.12 0.000245 140.13 1 +read 961.40 137871360 1024.00 0.000521 140.04 0.000180 140.05 2 + +Max Read: 963.96 MiB/sec (1010.78 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 963.96 960.85 962.07 1.35 139.94855 0 1 1 3 0 0 1 0 0 1 141180272640 1048576 141180272640 MPIIO 0 + +Finished: Sat Oct 20 19:42:23 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:1#API:MPIIO#T:10485760.txt b/output/COUNT:1#NN:1#PPN:1#API:MPIIO#T:10485760.txt new file mode 100644 index 000000000..92f69c6cf --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:1#API:MPIIO#T:10485760.txt @@ -0,0 +1,69 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 19:55:22 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:55:22 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1189.98 137871360 10240 0.001168 113.14 0.000277 113.14 0 +write 1192.85 137871360 10240 0.001364 112.87 0.000219 112.87 1 +write 1212.95 137871360 10240 0.079144 110.92 0.000185 111.00 2 + +Max Write: 1212.95 MiB/sec (1271.87 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1212.95 1189.98 1198.59 10.22 112.33975 0 1 1 3 0 0 1 0 0 1 141180272640 10485760 141180272640 MPIIO 0 + +Finished: Sat Oct 20 20:01:10 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 20:01:10 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 20:01:10 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 904.24 137871360 10240 0.000657 148.90 0.000240 148.90 0 +read 894.22 137871360 10240 0.000531 150.57 0.000175 150.57 1 +read 873.24 137871360 10240 0.000590 154.18 0.000175 154.18 2 + +Max Read: 904.24 MiB/sec (948.16 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 904.24 873.24 890.57 12.92 151.21664 0 1 1 3 0 0 1 0 0 1 141180272640 10485760 141180272640 MPIIO 0 + +Finished: Sat Oct 20 20:08:44 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:1#API:POSIX#T:1048576.txt b/output/COUNT:1#NN:1#PPN:1#API:POSIX#T:1048576.txt new file mode 100644 index 000000000..f0b101fa9 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:1#API:POSIX#T:1048576.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 19:42:24 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:42:24 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1131.22 137871360 1024.00 0.000220 119.02 0.000148 119.02 0 +write 1165.02 137871360 1024.00 0.000493 115.57 0.000134 115.57 1 +write 1177.53 137871360 1024.00 0.000479 114.34 0.000164 114.34 2 + +Max Write: 1177.53 MiB/sec (1234.73 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1177.53 1131.22 1157.92 19.56 116.31072 0 1 1 3 0 0 1 0 0 1 141180272640 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 19:48:23 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 19:48:23 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:48:23 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 955.45 137871360 1024.00 0.000149 140.92 0.000142 140.92 0 +read 981.05 137871360 1024.00 0.000170 137.24 0.000137 137.24 1 +read 957.89 137871360 1024.00 0.000163 140.56 0.000135 140.56 2 + +Max Read: 981.05 MiB/sec (1028.71 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 981.05 955.45 964.80 11.54 139.57243 0 1 1 3 0 0 1 0 0 1 141180272640 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 19:55:22 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:1#API:POSIX#T:10485760.txt b/output/COUNT:1#NN:1#PPN:1#API:POSIX#T:10485760.txt new file mode 100644 index 000000000..372e0a9e1 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:1#API:POSIX#T:10485760.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 20:08:45 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 20:08:45 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1162.06 137871360 10240 0.000226 115.86 0.000197 115.86 0 +write 1212.11 137871360 10240 0.000506 111.08 0.000177 111.08 1 +write 1195.81 137871360 10240 0.000478 112.59 0.000181 112.59 2 + +Max Write: 1212.11 MiB/sec (1270.99 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1212.11 1162.06 1189.99 20.85 113.17852 0 1 1 3 0 0 1 0 0 1 141180272640 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 20:14:34 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 20:14:34 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 141180272640 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 20:14:34 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 1 (1 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 131.48 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 895.81 137871360 10240 0.000129 150.30 0.000137 150.30 0 +read 879.80 137871360 10240 0.000155 153.03 0.000135 153.03 1 +read 872.57 137871360 10240 0.000162 154.30 0.000130 154.30 2 + +Max Read: 895.81 MiB/sec (939.33 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 895.81 872.57 882.73 9.71 152.54564 0 1 1 3 0 0 1 0 0 1 141180272640 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 20:22:12 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:2#API:MPIIO#T:1048576.txt b/output/COUNT:1#NN:1#PPN:2#API:MPIIO#T:1048576.txt new file mode 100644 index 000000000..8c59b9732 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:2#API:MPIIO#T:1048576.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:53:42 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:53:42 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141295616000. +write 1295.78 68935680 1024.00 0.001372 103.99 0.000276 103.99 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141196001280. +write 1294.03 68935680 1024.00 0.073494 103.98 0.000257 104.06 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141508476928. +write 1291.41 68935680 1024.00 0.008966 104.49 0.000225 104.50 2 + +Max Write: 1295.78 MiB/sec (1358.73 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1295.78 1291.41 1293.74 1.79 104.18338 0 2 2 3 0 0 1 0 0 1 70590136320 1048576 141295616000 MPIIO 0 + +Finished: Sat Oct 20 18:59:06 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:59:07 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:59:07 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141063880704. +read 1762.72 68935680 1024.00 0.000961 76.32 0.000231 76.32 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141363773440. +read 1729.20 68935680 1024.00 0.000582 77.96 0.000271 77.96 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141474922496. +read 1799.67 68935680 1024.00 0.000542 74.97 0.000193 74.97 2 + +Max Read: 1799.67 MiB/sec (1887.09 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 1799.67 1729.20 1763.87 28.78 76.41743 0 2 2 3 0 0 1 0 0 1 70590136320 1048576 141063880704 MPIIO 0 + +Finished: Sat Oct 20 19:02:56 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:2#API:MPIIO#T:10485760.txt b/output/COUNT:1#NN:1#PPN:2#API:MPIIO#T:10485760.txt new file mode 100644 index 000000000..b3ac55935 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:2#API:MPIIO#T:10485760.txt @@ -0,0 +1,84 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 19:12:28 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:12:28 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141924761600. +write 1835.71 68935680 10240 0.001383 73.73 0.000292 73.73 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 139984896000. +write 1863.92 68935680 10240 0.314354 71.31 0.000247 71.62 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141022986240. +write 1907.42 68935680 10240 0.575783 69.93 0.000243 70.51 2 + +Max Write: 1907.42 MiB/sec (2000.08 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1907.42 1835.71 1869.02 29.50 71.95456 0 2 2 3 0 0 1 0 0 1 70590136320 10485760 141924761600 MPIIO 0 + +Finished: Sat Oct 20 19:16:15 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 19:16:15 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:16:15 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140781813760. +read 1568.78 68935680 10240 0.000907 85.58 0.000250 85.58 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140676956160. +read 1524.54 68935680 10240 0.000612 88.00 0.000194 88.00 1 +read 1526.10 68935680 10240 0.000571 88.22 0.000199 88.23 2 + +Max Read: 1568.78 MiB/sec (1644.98 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 1568.78 1524.54 1539.80 20.50 87.26935 0 2 2 3 0 0 1 0 0 1 70590136320 10485760 140781813760 MPIIO 0 + +Finished: Sat Oct 20 19:20:37 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:2#API:POSIX#T:1048576.txt b/output/COUNT:1#NN:1#PPN:2#API:POSIX#T:1048576.txt new file mode 100644 index 000000000..d3ffd7bdf --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:2#API:POSIX#T:1048576.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 19:02:57 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:02:57 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1291.18 68935680 1024.00 0.000445 104.28 0.000133 104.28 0 +write 1281.97 68935680 1024.00 0.000627 105.02 0.000112 105.03 1 +write 1320.00 68935680 1024.00 0.000814 102.00 0.000116 102.00 2 + +Max Write: 1320.00 MiB/sec (1384.12 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1320.00 1281.97 1297.72 16.20 103.76741 0 2 2 3 0 0 1 0 0 1 70590136320 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 19:08:19 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 19:08:20 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:08:20 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 1658.62 68935680 1024.00 0.000287 81.18 0.000135 81.18 0 +read 1641.12 68935680 1024.00 0.000240 82.04 0.000139 82.04 1 +read 1610.13 68935680 1024.00 0.000281 83.62 0.000132 83.62 2 + +Max Read: 1658.62 MiB/sec (1739.19 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 1658.62 1610.13 1636.62 20.05 82.27951 0 2 2 3 0 0 1 0 0 1 70590136320 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 19:12:27 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:2#API:POSIX#T:10485760.txt b/output/COUNT:1#NN:1#PPN:2#API:POSIX#T:10485760.txt new file mode 100644 index 000000000..ab08b9901 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:2#API:POSIX#T:10485760.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 19:20:38 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:20:38 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1726.07 68935680 10240 0.000454 78.00 0.000187 78.00 0 +write 1808.51 68935680 10240 0.000662 74.45 0.000118 74.45 1 +write 1804.30 68935680 10240 0.000878 74.62 0.000115 74.62 2 + +Max Write: 1808.51 MiB/sec (1896.36 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1808.51 1726.07 1779.63 37.91 75.69107 0 2 2 3 0 0 1 0 0 1 70590136320 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 19:24:37 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 19:24:37 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 70590136320 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 19:24:37 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 2 (2 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 65.74 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 1431.71 68935680 10240 0.000273 94.04 0.000141 94.04 0 +read 1428.92 68935680 10240 0.000249 94.22 0.000161 94.23 1 +read 1420.46 68935680 10240 0.000251 94.79 0.000139 94.79 2 + +Max Read: 1431.71 MiB/sec (1501.26 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 1431.71 1420.46 1427.03 4.78 94.35078 0 2 2 3 0 0 1 0 0 1 70590136320 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 19:29:20 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:4#API:MPIIO#T:1048576.txt b/output/COUNT:1#NN:1#PPN:4#API:MPIIO#T:1048576.txt new file mode 100644 index 000000000..1bd07c193 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:4#API:MPIIO#T:1048576.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:26:28 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:26:28 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141087997952. +write 1316.52 34467840 1024.00 0.001603 102.20 0.000241 102.20 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140818513920. +write 1325.90 34467840 1024.00 0.157288 101.13 0.000279 101.29 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141232701440. +write 1343.43 34467840 1024.00 0.079092 100.18 0.000244 100.26 2 + +Max Write: 1343.43 MiB/sec (1408.68 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1343.43 1316.52 1328.61 11.15 101.24915 0 4 4 3 0 0 1 0 0 1 35295068160 1048576 141087997952 MPIIO 0 + +Finished: Sat Oct 20 18:31:43 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:31:44 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:31:44 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141672054784. +read 2887.71 34467840 1024.00 0.003405 46.78 0.000251 46.79 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141026131968. +read 2899.53 34467840 1024.00 0.000729 46.38 0.000282 46.38 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141100580864. +read 2945.94 34467840 1024.00 0.000738 45.68 0.000222 45.68 2 + +Max Read: 2945.94 MiB/sec (3089.04 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 2945.94 2887.71 2911.06 25.13 46.28328 0 4 4 3 0 0 1 0 0 1 35295068160 1048576 141672054784 MPIIO 0 + +Finished: Sat Oct 20 18:34:03 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:4#API:MPIIO#T:10485760.txt b/output/COUNT:1#NN:1#PPN:4#API:MPIIO#T:10485760.txt new file mode 100644 index 000000000..d3257beee --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:4#API:MPIIO#T:10485760.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:41:44 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:41:44 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140110725120. +write 2297.85 34467840 10240 0.001558 58.15 0.000256 58.15 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141348044800. +write 2330.01 34467840 10240 0.215638 57.64 0.000246 57.85 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 142113505280. +write 2311.06 34467840 10240 0.083370 58.56 0.000230 58.64 2 + +Max Write: 2330.01 MiB/sec (2443.19 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 2330.01 2297.85 2312.97 13.20 58.21601 0 4 4 3 0 0 1 0 0 1 35295068160 10485760 140110725120 MPIIO 0 + +Finished: Sat Oct 20 18:44:50 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:44:51 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:44:51 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140781813760. +read 2392.47 34467840 10240 0.031971 56.09 0.000252 56.12 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141012500480. +read 2450.89 34467840 10240 0.000730 54.87 0.000188 54.87 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140855214080. +read 2388.63 34467840 10240 0.000714 56.24 0.000197 56.24 2 + +Max Read: 2450.89 MiB/sec (2569.94 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 2450.89 2388.63 2410.66 28.48 55.74164 0 4 4 3 0 0 1 0 0 1 35295068160 10485760 140781813760 MPIIO 0 + +Finished: Sat Oct 20 18:47:38 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:4#API:POSIX#T:1048576.txt b/output/COUNT:1#NN:1#PPN:4#API:POSIX#T:1048576.txt new file mode 100644 index 000000000..3ba367b58 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:4#API:POSIX#T:1048576.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:34:05 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:34:05 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1313.35 34467840 1024.00 0.000472 102.52 0.000130 102.52 0 +write 1310.34 34467840 1024.00 0.000672 102.75 0.000120 102.75 1 +write 1386.60 34467840 1024.00 0.000665 97.10 0.000129 97.10 2 + +Max Write: 1386.60 MiB/sec (1453.96 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1386.60 1310.34 1336.76 35.26 100.78977 0 4 4 3 0 0 1 0 0 1 35295068160 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:39:18 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:39:20 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:39:20 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 2867.36 34467840 1024.00 0.000485 46.96 0.000141 46.96 0 +read 2804.59 34467840 1024.00 0.000428 48.01 0.000127 48.01 1 +read 2819.50 34467840 1024.00 0.000572 47.75 0.000138 47.75 2 + +Max Read: 2867.36 MiB/sec (3006.64 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 2867.36 2804.59 2830.48 26.77 47.57206 0 4 4 3 0 0 1 0 0 1 35295068160 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:41:42 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:4#API:POSIX#T:10485760.txt b/output/COUNT:1#NN:1#PPN:4#API:POSIX#T:10485760.txt new file mode 100644 index 000000000..352424f5a --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:4#API:POSIX#T:10485760.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:47:40 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:47:40 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 2288.12 34467840 10240 0.000477 58.84 0.000139 58.84 0 +write 2268.09 34467840 10240 0.000650 59.36 0.000128 59.36 1 +write 2290.76 34467840 10240 0.000859 58.77 0.000116 58.78 2 + +Max Write: 2290.76 MiB/sec (2402.03 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 2290.76 2268.09 2282.32 10.12 58.99371 0 4 4 3 0 0 1 0 0 1 35295068160 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:50:47 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:50:49 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 35295068160 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:50:49 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 4 (4 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 32.87 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 2348.20 34467840 10240 0.000502 57.34 0.000148 57.34 0 +read 2343.14 34467840 10240 0.000418 57.46 0.000133 57.46 1 +read 2325.58 34467840 10240 0.000533 57.89 0.000133 57.90 2 + +Max Read: 2348.20 MiB/sec (2462.27 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 2348.20 2325.58 2338.97 9.69 57.56473 0 4 4 3 0 0 1 0 0 1 35295068160 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:53:42 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:6#API:MPIIO#T:1048576.txt b/output/COUNT:1#NN:1#PPN:6#API:MPIIO#T:1048576.txt new file mode 100644 index 000000000..486661b95 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:6#API:MPIIO#T:1048576.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:00:59 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:00:59 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141444513792. +write 1302.51 22978560 1024.00 0.001642 103.56 0.000348 103.56 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141187612672. +write 1338.51 22978560 1024.00 0.276888 100.32 0.000292 100.59 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 141331267584. +write 1315.71 22978560 1024.00 0.261772 102.18 0.000263 102.44 2 + +Max Write: 1338.51 MiB/sec (1403.53 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1338.51 1302.51 1318.91 14.87 102.19986 0 6 6 3 0 0 1 0 0 1 23530045440 1048576 141444513792 MPIIO 0 + +Finished: Sat Oct 20 18:06:16 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:06:18 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:06:18 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 140790202368. +read 3769 22978560 1024.00 0.002009 35.62 0.000244 35.62 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 141249478656. +read 3659 22978560 1024.00 0.000938 36.82 0.000232 36.82 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 141186564096. +read 3555.19 22978560 1024.00 0.000880 37.87 0.000201 37.87 2 + +Max Read: 3769.36 MiB/sec (3952.46 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 3769.36 3555.19 3661.13 87.45 36.77023 0 6 6 3 0 0 1 0 0 1 23530045440 1048576 140790202368 MPIIO 0 + +Finished: Sat Oct 20 18:08:09 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:6#API:MPIIO#T:10485760.txt b/output/COUNT:1#NN:1#PPN:6#API:MPIIO#T:10485760.txt new file mode 100644 index 000000000..801977b2f --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:6#API:MPIIO#T:10485760.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:15:15 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:15:15 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141002014720. +write 1682.10 22978560 10240 0.001670 79.94 0.000294 79.94 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141369016320. +write 2294.22 22978560 10240 0.120886 58.64 0.000227 58.76 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141169786880. +write 2328.02 22978560 10240 0.371375 57.46 0.000262 57.83 2 + +Max Write: 2328.02 MiB/sec (2441.11 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 2328.02 1682.10 2101.45 296.85 65.51239 0 6 6 3 0 0 1 0 0 1 23530045440 10485760 141002014720 MPIIO 0 + +Finished: Sat Oct 20 18:18:44 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 18:18:46 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:18:46 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141400473600. +read 3175.10 22978560 10240 0.002134 42.47 0.000257 42.47 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141138329600. +read 3052.31 22978560 10240 0.000900 44.10 0.000196 44.10 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140823756800. +read 3007.59 22978560 10240 0.000900 44.65 0.000184 44.65 2 + +Max Read: 3175.10 MiB/sec (3329.34 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 3175.10 3007.59 3078.33 70.82 43.74086 0 6 6 3 0 0 1 0 0 1 23530045440 10485760 141400473600 MPIIO 0 + +Finished: Sat Oct 20 18:20:57 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:6#API:POSIX#T:1048576.txt b/output/COUNT:1#NN:1#PPN:6#API:POSIX#T:1048576.txt new file mode 100644 index 000000000..9a27acb39 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:6#API:POSIX#T:1048576.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:08:11 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:08:11 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1300.73 22978560 1024.00 0.000503 103.51 0.000129 103.51 0 +write 1351.58 22978560 1024.00 0.000870 99.62 0.000111 99.62 1 +write 1325.05 22978560 1024.00 0.000860 101.61 0.000124 101.61 2 + +Max Write: 1351.58 MiB/sec (1417.24 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1351.58 1300.73 1325.79 20.77 101.57960 0 6 6 3 0 0 1 0 0 1 23530045440 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:13:26 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:13:28 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:13:28 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 3975 22978560 1024.00 0.000652 33.87 0.000156 33.87 0 +read 3718 22978560 1024.00 0.000590 36.21 0.000129 36.21 1 +read 3918 22978560 1024.00 0.000617 34.24 0.119599 34.36 2 + +Max Read: 3975.29 MiB/sec (4168.40 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 3975.29 3718.16 3870.59 110.27 34.81412 0 6 6 3 0 0 1 0 0 1 23530045440 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:15:13 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:6#API:POSIX#T:10485760.txt b/output/COUNT:1#NN:1#PPN:6#API:POSIX#T:10485760.txt new file mode 100644 index 000000000..568152c10 --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:6#API:POSIX#T:10485760.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:20:59 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:20:59 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 2241.64 22978560 10240 0.000494 60.06 0.000145 60.06 0 +write 2250.56 22978560 10240 0.000685 59.82 0.000127 59.83 1 +write 2226.54 22978560 10240 0.000876 60.47 0.000123 60.47 2 + +Max Write: 2250.56 MiB/sec (2359.88 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 2250.56 2226.54 2239.58 9.92 60.11966 0 6 6 3 0 0 1 0 0 1 23530045440 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:24:10 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 18:24:12 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 23530045440 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 18:24:12 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 6 (6 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 21.91 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 3044.46 22978560 10240 0.000691 44.22 0.000194 44.22 0 +read 3080.73 22978560 10240 0.000546 43.70 0.000129 43.70 1 +read 2900.44 22978560 10240 0.000672 46.42 0.000139 46.42 2 + +Max Read: 3080.73 MiB/sec (3230.38 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 3080.73 2900.44 3008.54 77.86 44.78304 0 6 6 3 0 0 1 0 0 1 23530045440 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:26:27 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:8#API:MPIIO#T:1048576.txt b/output/COUNT:1#NN:1#PPN:8#API:MPIIO#T:1048576.txt new file mode 100644 index 000000000..9535323cb --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:8#API:MPIIO#T:1048576.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 17:34:52 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:34:52 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141318684672. +write 1109.19 17233920 1024.00 0.002362 121.50 0.000341 121.51 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141104775168. +write 825.23 17233920 1024.00 0.273637 162.79 0.000296 163.07 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 141117358080. +write 1125.67 17233920 1024.00 0.292227 119.26 0.000276 119.56 2 + +Max Write: 1125.67 MiB/sec (1180.35 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1125.67 825.23 1020.03 137.91 134.70950 0 8 8 3 0 0 1 0 0 1 17647534080 1048576 141318684672 MPIIO 0 + +Finished: Sat Oct 20 17:41:48 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 17:41:51 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:41:51 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 141240041472. +read 4174 17233920 1024.00 0.011059 32.26 0.000316 32.27 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 140888768512. +read 4329 17233920 1024.00 0.001100 31.03 0.000286 31.04 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141179224064. +WARNING: Using actual aggregate bytes moved = 141495894016. +read 4089 17233920 1024.00 0.001105 33.00 0.000253 33.00 2 + +Max Read: 4329.29 MiB/sec (4539.59 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 4329.29 4089.02 4197.50 99.47 32.10183 0 8 8 3 0 0 1 0 0 1 17647534080 1048576 141240041472 MPIIO 0 + +Finished: Sat Oct 20 17:43:27 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:8#API:MPIIO#T:10485760.txt b/output/COUNT:1#NN:1#PPN:8#API:MPIIO#T:10485760.txt new file mode 100644 index 000000000..064ef152d --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:8#API:MPIIO#T:10485760.txt @@ -0,0 +1,87 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 17:50:34 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:50:34 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141169786880. +WARNING: Using actual aggregate bytes moved = 140771328000. +write 2236.23 17233920 10240 0.001734 60.03 0.000373 60.03 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141159301120. +write 2369.18 17233920 10240 0.534101 56.29 0.000263 56.82 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141389987840. +write 2274.03 17233920 10240 0.193066 58.85 0.253811 59.30 2 + +Max Write: 2369.18 MiB/sec (2484.27 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 2369.18 2236.23 2293.15 55.93 58.71694 0 8 8 3 0 0 1 0 0 1 17647534080 10485760 140771328000 MPIIO 0 + +Finished: Sat Oct 20 17:53:42 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +ior WARNING: fsync() only available in POSIX. Using value of 0. +Began: Sat Oct 20 17:53:44 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a MPIIO -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:53:44 2018 +Summary: + api = MPIIO (version=3, subversion=0) + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141022986240. +read 3457.78 17233920 10240 0.024159 38.87 0.000325 38.89 0 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 141652131840. +read 3447.49 17233920 10240 0.001108 39.18 0.000231 39.19 1 +WARNING: Expected aggregate file size = 141180272640. +WARNING: Stat() of aggregate file size = 141180272640. +WARNING: Using actual aggregate bytes moved = 140918128640. +read 3456.22 17233920 10240 0.001038 38.88 0.000246 38.88 2 + +Max Read: 3457.78 MiB/sec (3625.75 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 3457.78 3447.49 3453.83 4.53 38.98781 0 8 8 3 0 0 1 0 0 1 17647534080 10485760 141022986240 MPIIO 0 + +Finished: Sat Oct 20 17:55:41 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:8#API:POSIX#T:1048576.txt b/output/COUNT:1#NN:1#PPN:8#API:POSIX#T:1048576.txt new file mode 100644 index 000000000..eeee49e2a --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:8#API:POSIX#T:1048576.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 17:43:30 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:43:30 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 1223.18 17233920 1024.00 0.000505 110.07 0.000515 110.07 0 +write 1366.60 17233920 1024.00 0.001174 98.52 0.000117 98.52 1 +write 1312.30 17233920 1024.00 0.001512 102.60 0.000118 102.60 2 + +Max Write: 1366.60 MiB/sec (1432.98 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 1366.60 1223.18 1300.69 59.12 103.73149 0 8 8 3 0 0 1 0 0 1 17647534080 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 17:48:52 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 17:48:54 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:48:54 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 1 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 4254 17233920 1024.00 0.000828 31.65 0.000168 31.65 0 +read 4214 17233920 1024.00 0.000709 31.95 0.000142 31.95 1 +read 3982 17233920 1024.00 0.000727 33.81 0.000173 33.81 2 + +Max Read: 4253.88 MiB/sec (4460.51 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 4253.88 3981.91 4149.97 119.94 32.47124 0 8 8 3 0 0 1 0 0 1 17647534080 1048576 141180272640 POSIX 0 + +Finished: Sat Oct 20 17:50:32 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:1#PPN:8#API:POSIX#T:10485760.txt b/output/COUNT:1#NN:1#PPN:8#API:POSIX#T:10485760.txt new file mode 100644 index 000000000..2eae7a9ba --- /dev/null +++ b/output/COUNT:1#NN:1#PPN:8#API:POSIX#T:10485760.txt @@ -0,0 +1,67 @@ +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 17:55:44 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -w +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:55:44 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +write 2247.59 17233920 10240 0.000514 59.90 0.003747 59.90 0 +write 2261.63 17233920 10240 0.000841 59.53 0.000917 59.53 1 +write 2229.22 17233920 10240 0.004229 60.39 0.000114 60.40 2 + +Max Write: 2261.63 MiB/sec (2371.49 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +write 2261.63 2229.22 2246.14 13.27 59.94482 0 8 8 3 0 0 1 0 0 1 17647534080 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 17:58:55 2018 +IOR-3.0.1: MPI Coordinated Test of Parallel I/O + +Began: Sat Oct 20 17:58:58 2018 +Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 17647534080 -o /esfs/jtacquaviva/iorperf/file -a POSIX -e -g -z -k -r -D 300 +Machine: Linux isc17-c02 + +Test 0 started: Sat Oct 20 17:58:58 2018 +Summary: + api = POSIX + test filename = /esfs/jtacquaviva/iorperf/file + access = single-shared-file + ordering in a file = random offsets + ordering inter file= no tasks offsets + clients = 8 (8 per node) + repetitions = 3 + xfersize = 10 MiB + blocksize = 16.44 GiB + aggregate filesize = 131.48 GiB + Using stonewalling = 300 second(s) + +access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter +------ --------- ---------- --------- -------- -------- -------- -------- ---- +read 3362.12 17233920 10240 0.000914 40.05 0.000192 40.05 0 +read 3446.39 17233920 10240 0.000809 39.07 0.000132 39.07 1 +read 3375.07 17233920 10240 0.001231 39.89 0.000132 39.89 2 + +Max Read: 3446.39 MiB/sec (3613.81 MB/sec) + +Summary of all tests: +Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum +read 3446.39 3362.12 3394.53 37.05 39.66853 0 8 8 3 0 0 1 0 0 1 17647534080 10485760 141180272640 POSIX 0 + +Finished: Sat Oct 20 18:00:57 2018 +/esfs/jtacquaviva/iorperf +stripe_count: 2 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:2#PPN:8#API:MPIIO#T:1048576.txt b/output/COUNT:1#NN:2#PPN:8#API:MPIIO#T:1048576.txt new file mode 100644 index 000000000..e69de29bb diff --git a/output/COUNT:1#NN:2#PPN:8#API:MPIIO#T:10485760.txt b/output/COUNT:1#NN:2#PPN:8#API:MPIIO#T:10485760.txt new file mode 100644 index 000000000..e69de29bb diff --git a/run.py b/run.sh similarity index 61% rename from run.py rename to run.sh index 2a53e7d30..df80e79d2 100755 --- a/run.py +++ b/run.sh @@ -7,11 +7,35 @@ function force_exit { trap force_exit SIGINT +# HOST_LIST=( \ +##isc17-c01 \ +#isc17-c02 \ +#isc17-c03 \ +#isc17-c04 \ +#isc17-c05 \ +#isc17-c06 \ +#isc17-c07 \ +#isc17-c09 \ +#isc17-c11 \ +#isc17-c12 \ +##isc17-c13 \ +##isc17-c14 \ +##isc17-c15 \ +##isc17-c18 \ +##isc17-c20 \ +##isc17-c22 \ +##isc17-c23 \ +#) + +# isc17-c19 offline +# isc17-c10 offline +# isc17-c08 MPI problems + # Provides a list of good hosts (that contains QDR connection) function hosts() { num="$1" - #HOST_LIST=( isc17-c02 isc17-c03 isc17-c04 isc17-c05 isc17-c08 isc17-c09 isc17-c10 isc17-c12 isc17-c14 isc17-c23 ) - HOST_LIST=( isc17-c01 isc17-c02 isc17-c03 isc17-c04 isc17-c05 isc17-c06 isc17-c07 isc17-c08 isc17-c09 isc17-c10 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c19 isc17-c20 isc17-c22 isc17-c23 ) + HOST_LIST=( isc17-c02 isc17-c04 isc17-c03 isc17-c05 isc17-c06 isc17-c07 isc17-c08 isc17-c09 isc17-c11 isc17-c12 isc17-c13 isc17-c14 isc17-c15 isc17-c18 isc17-c22 isc17-c01) + hlist=${HOST_LIST[0]} for POS in $(seq 1 $(($num - 1))) ; do hlist="$hlist,${HOST_LIST[$POS]}" @@ -19,22 +43,23 @@ function hosts() { echo $hlist } - -export MODULEPATH=/gsfs/betke/software/modules:$MODULEPATH +#export MODULEPATH=/gsfs/betke/software/modules:$MODULEPATH +export MODULEPATH=/esfs/jtacquaviva/software/modules:$MODULEPATH module purge module load betke/hdf5/1.8.20-ddn module load betke/ior/git-ddn module list -LUSTRE_TESTFILE="/esfs/jtacquaviva/iorperf/file" +LUSTRE_TESTFILE="/esfs/jtacquaviva/iorperf/file2" ITERATIONS=3 IOR="$(which ior)" MPIEXEC="/opt/ddn/mvapich/bin/mpiexec" -NN_ARR=(1) -PPN_ARR=(8 6 4 2 1) -T_ARR=( $((16*1024)) $((100*1024)) $((1024*1024)) $((10240*1024)) ) +NN_ARR=(2) +PPN_ARR=( 1 8 6 4 2 ) +#T_ARR=( $((16*1024)) $((100*1024)) $((1024*1024)) $((10240*1024)) ) +T_ARR=( $((1024*1024)) $((10240*1024)) ) API_ARR=("MPIIO" "POSIX") for COUNT in $(seq 1); do @@ -52,12 +77,12 @@ for API in ${API_ARR[@]}; do fi touch $BENCHFILE - IOR_PARAMS="-i $ITERATIONS -s 1 -t $T -b $((130 * 1024 * 1024 * 1020 / $PPN)) -o $LUSTRE_TESTFILE -a $API -e -g -z -k" - MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) -hosts $(hosts $NN)" + IOR_PARAMS="-i $ITERATIONS -s 1 -t $T -b $((132 * 1024 * 1024 * 1020 / $PPN)) -o $LUSTRE_TESTFILE -a $API -e -g -z -k" + MPIEXEC_PARAMS=" -ppn $PPN -np $(($NN * $PPN)) --hosts $(hosts $NN)" TESTDIR="$(dirname $LUSTRE_TESTFILE)" if [ -d $TESTDIR ]; then - rm $TESTDIR + rm -r $TESTDIR fi mkdir -p $TESTDIR lfs setstripe -c $((2 * $NN)) $TESTDIR diff --git a/show_node_status.sh b/show_node_status.sh new file mode 100755 index 000000000..d03c99cc0 --- /dev/null +++ b/show_node_status.sh @@ -0,0 +1,63 @@ +#/bin/bash + +function join_by() { + local IFS="$1"; shift; echo "$*"; +} + + +cx4nodes=( \ +isc17-c02 \ +isc17-c03 \ +isc17-c04 \ +isc17-c05 \ +isc17-c06 \ +isc17-c07 \ +isc17-c08 \ +isc17-c09 \ +isc17-c10 \ +isc17-c11 \ +isc17-c12 \ +isc17-c13 \ +isc17-c14 \ +isc17-c15 \ +isc17-c18 \ +isc17-c19 \ +isc17-c20 \ +isc17-c22 \ +isc17-c23 \ +isc17-c01 \ +) + +#cx4nodes=( isc17-c0{1..9} isc17-c{10..22} ) +node_list=$( join_by , "${cx4nodes[@]}" ) + + +IFS='' read -r -d '' script <<"EOF" +if [ -d /esfs/jtacquaviva ]; then + echo 'ok' +else + echo "failed: $(hostname) /esfs/ is not mounted" +fi +EOF + +declare -a good_nodes +for node in ${cx4nodes[@]}; do + #echo "ping $node" + status=$(ping -c1 $node | grep Unreachable) + if [[ "" == $status ]]; then + check=$(ssh $node "$script") + if [[ "ok" == ${check} ]]; then + good_nodes=( ${good_nodes[@]} $node ) + else + echo "$node failed with: $check" + fi + else + echo "$node is unreachable" + fi +done + +echo "${good_nodes[@]}" +echo $( join_by , "${good_nodes[@]}" ) + + +