+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 80530636800 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w + tee -a ./output/COUNT:1#NN:8#PPN:2#API:POSIX#T:102400.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Wed Oct 24 06:09:44 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 80530636800 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w Machine: Linux isc17-c04 Test 0 started: Wed Oct 24 06:09:44 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/ioperf/file_write access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (2 per node) repetitions = 3 xfersize = 102400 bytes blocksize = 75 GiB aggregate filesize = 1200 GiB Using stonewalling = 60 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 1288446668800. WARNING: Using actual aggregate bytes moved = 84816793600. write 906.70 78643200 100.00 0.000991 89.21 0.000457 89.21 0 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 1288423731200. WARNING: Using actual aggregate bytes moved = 74972160000. write 776.88 78643200 100.00 0.001311 92.03 0.001350 92.03 1 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 1288459468800. WARNING: Using actual aggregate bytes moved = 77627289600. write 798.18 78643200 100.00 0.001265 92.75 0.000567 92.75 2 Max Write: 906.70 MiB/sec (950.75 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum write 906.70 776.88 827.25 56.85 91.33173 0 16 2 3 1 0 1 0 0 1 80530636800 102400 84816793600 POSIX 0 Finished: Wed Oct 24 06:14:21 2018 + /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + tee -a ./output/COUNT:1#NN:8#PPN:2#API:POSIX#T:102400.txt + /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 80530636800 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Wed Oct 24 06:14:33 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 80530636800 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r Machine: Linux isc17-c04 Test 0 started: Wed Oct 24 06:14:33 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread8/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (2 per node) repetitions = 3 xfersize = 102400 bytes blocksize = 75 GiB aggregate filesize = 1200 GiB Using stonewalling = 60 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 11614617600. read 184.49 78643200 100.00 0.001346 60.04 0.001002 60.04 0 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 11714048000. read 186.08 78643200 100.00 0.001052 60.04 0.001039 60.04 1 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 11938304000. read 189.62 78643200 100.00 0.001054 60.04 0.001043 60.04 2 Max Read: 189.62 MiB/sec (198.83 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 189.62 184.49 186.73 2.15 60.03849 0 16 2 3 1 0 1 0 0 1 80530636800 102400 11614617600 POSIX 0 Finished: Wed Oct 24 06:17:33 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 16 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12646231 0xc0f757 0 3 12494534 0xbea6c6 0 6 12599842 0xc04222 0 0 12428245 0xbda3d5 0 2 12574594 0xbfdf82 0 5 12386604 0xbd012c 0 7 12464952 0xbe3338 0 4 12516324 0xbefbe4 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12428243 0xbda3d3 0 4 12516317 0xbefbdd 0 6 12599840 0xc04220 0 1 12646230 0xc0f756 0 2 12574590 0xbfdf7e 0 3 12494532 0xbea6c4 0 5 12386599 0xbd0127 0 7 12464947 0xbe3333 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12428244 0xbda3d4 0 3 12494533 0xbea6c5 0 5 12386600 0xbd0128 0 4 12516318 0xbefbde 0 7 12464948 0xbe3334 0 6 12599843 0xc04223 0 1 12646235 0xc0f75b 0 2 12574595 0xbfdf83 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12386602 0xbd012a 0 2 12574592 0xbfdf80 0 0 12428246 0xbda3d6 0 4 12516323 0xbefbe3 0 3 12494538 0xbea6ca 0 7 12464953 0xbe3339 0 6 12599844 0xc04224 0 1 12646237 0xc0f75d 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12386611 0xbd0133 0 6 12599851 0xc0422b 0 0 12428256 0xbda3e0 0 2 12574604 0xbfdf8c 0 4 12516330 0xbefbea 0 7 12464960 0xbe3340 0 1 12646244 0xc0f764 0 3 12494547 0xbea6d3 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12599853 0xc0422d 0 4 12516331 0xbefbeb 0 7 12464961 0xbe3341 0 3 12494546 0xbea6d2 0 0 12428257 0xbda3e1 0 1 12646245 0xc0f765 0 5 12386614 0xbd0136 0 2 12574605 0xbfdf8d 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12646233 0xc0f759 0 2 12574593 0xbfdf81 0 4 12516322 0xbefbe2 0 0 12428249 0xbda3d9 0 5 12386606 0xbd012e 0 7 12464954 0xbe333a 0 6 12599846 0xc04226 0 3 12494539 0xbea6cb 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12599847 0xc04227 0 0 12428251 0xbda3db 0 2 12574598 0xbfdf86 0 4 12516325 0xbefbe5 0 7 12464955 0xbe333b 0 1 12646238 0xc0f75e 0 3 12494540 0xbea6cc 0 5 12386607 0xbd012f 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12516326 0xbefbe6 0 7 12464957 0xbe333d 0 1 12646240 0xc0f760 0 3 12494542 0xbea6ce 0 5 12386609 0xbd0131 0 6 12599849 0xc04229 0 0 12428253 0xbda3dd 0 2 12574600 0xbfdf88 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12428252 0xbda3dc 0 7 12464956 0xbe333c 0 1 12646239 0xc0f75f 0 3 12494541 0xbea6cd 0 5 12386608 0xbd0130 0 6 12599848 0xc04228 0 2 12574599 0xbfdf87 0 4 12516327 0xbefbe7 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12574603 0xbfdf8b 0 1 12646243 0xc0f763 0 5 12386613 0xbd0135 0 4 12516332 0xbefbec 0 3 12494545 0xbea6d1 0 7 12464962 0xbe3342 0 6 12599854 0xc0422e 0 0 12428258 0xbda3e2 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12494543 0xbea6cf 0 6 12599850 0xc0422a 0 2 12574601 0xbfdf89 0 7 12464959 0xbe333f 0 5 12386612 0xbd0134 0 0 12428255 0xbda3df 0 4 12516329 0xbefbe9 0 1 12646242 0xc0f762 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12386601 0xbd0129 0 2 12574591 0xbfdf7f 0 6 12599841 0xc04221 0 1 12646234 0xc0f75a 0 7 12464950 0xbe3336 0 4 12516321 0xbefbe1 0 0 12428247 0xbda3d7 0 3 12494537 0xbea6c9 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12516319 0xbefbdf 0 3 12494535 0xbea6c7 0 7 12464949 0xbe3335 0 1 12646236 0xc0f75c 0 5 12386605 0xbd012d 0 6 12599845 0xc04225 0 2 12574597 0xbfdf85 0 0 12428250 0xbda3da 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12464958 0xbe333e 0 5 12386610 0xbd0132 0 0 12428254 0xbda3de 0 4 12516328 0xbefbe8 0 1 12646241 0xc0f761 0 6 12599852 0xc0422c 0 2 12574602 0xbfdf8a 0 3 12494544 0xbea6d0 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12646232 0xc0f758 0 4 12516320 0xbefbe0 0 7 12464951 0xbe3337 0 3 12494536 0xbea6c8 0 5 12386603 0xbd012b 0 0 12428248 0xbda3d8 0 2 12574596 0xbfdf84 0