+ tee -a ./output/COUNT:1#NN:4#PPN:4#API:POSIX#T:10485760.txt + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 12:19:04 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w Machine: Linux isc17-c04 Test 0 started: Tue Oct 23 12:19:04 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/ioperf/file_write access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (4 per node) repetitions = 3 xfersize = 10 MiB blocksize = 37.50 GiB aggregate filesize = 600 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- write 5780 39321600 10240 0.001069 106.30 0.000287 106.30 0 write 5871 39321600 10240 0.001675 104.65 0.000314 104.65 1 write 5804 39321600 10240 0.004186 105.86 0.000410 105.87 2 Max Write: 5870.76 MiB/sec (6155.94 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum write 5870.76 5779.72 5818.02 38.54 105.60748 0 16 4 3 1 0 1 0 0 1 40265318400 10485760 644245094400 POSIX 0 Finished: Tue Oct 23 12:24:25 2018 + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r + tee -a ./output/COUNT:1#NN:4#PPN:4#API:POSIX#T:10485760.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 12:24:36 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r Machine: Linux isc17-c04 Test 0 started: Tue Oct 23 12:24:36 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread4/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (4 per node) repetitions = 3 xfersize = 10 MiB blocksize = 37.50 GiB aggregate filesize = 600 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 196796743680. read 1562.08 39321600 10240 0.001144 120.15 0.000530 120.15 0 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 243112345600. read 1930.22 39321600 10240 0.000539 120.12 0.000569 120.12 1 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 244790067200. read 1942.28 39321600 10240 0.000613 120.19 0.000658 120.19 2 Max Read: 1942.28 MiB/sec (2036.63 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 1942.28 1562.08 1811.53 176.46 120.15249 0 16 4 3 1 0 1 0 0 1 40265318400 10485760 196796743680 POSIX 0 Finished: Tue Oct 23 12:30:36 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 8 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12641196 0xc0e3ac 0 3 12489497 0xbe9319 0 5 12381560 0xbced78 0 6 12594817 0xc02e81 0 0 12423209 0xbd9029 0 2 12569550 0xbfcbce 0 4 12511287 0xbee837 0 7 12459918 0xbe1f8e 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12423223 0xbd9037 0 2 12569564 0xbfcbdc 0 4 12511300 0xbee844 0 7 12459932 0xbe1f9c 0 1 12641210 0xc0e3ba 0 3 12489511 0xbe9327 0 5 12381574 0xbced86 0 6 12594831 0xc02e8f 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569565 0xbfcbdd 0 4 12511301 0xbee845 0 7 12459933 0xbe1f9d 0 1 12641211 0xc0e3bb 0 3 12489512 0xbe9328 0 5 12381575 0xbced87 0 6 12594832 0xc02e90 0 0 12423224 0xbd9038 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12641205 0xc0e3b5 0 3 12489506 0xbe9322 0 5 12381569 0xbced81 0 6 12594826 0xc02e8a 0 0 12423218 0xbd9032 0 2 12569559 0xbfcbd7 0 4 12511295 0xbee83f 0 7 12459927 0xbe1f97 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12511292 0xbee83c 0 6 12594822 0xc02e86 0 7 12459924 0xbe1f94 0 3 12489503 0xbe931f 0 2 12569555 0xbfcbd3 0 1 12641202 0xc0e3b2 0 5 12381566 0xbced7e 0 0 12423215 0xbd902f 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489502 0xbe931e 0 7 12459923 0xbe1f93 0 5 12381565 0xbced7d 0 6 12594823 0xc02e87 0 1 12641201 0xc0e3b1 0 0 12423214 0xbd902e 0 2 12569556 0xbfcbd4 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12459926 0xbe1f96 0 1 12641204 0xc0e3b4 0 3 12489505 0xbe9321 0 5 12381568 0xbced80 0 6 12594825 0xc02e89 0 0 12423217 0xbd9031 0 2 12569558 0xbfcbd6 0 4 12511294 0xbee83e 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594830 0xc02e8e 0 0 12423222 0xbd9036 0 2 12569563 0xbfcbdb 0 4 12511299 0xbee843 0 7 12459931 0xbe1f9b 0 1 12641209 0xc0e3b9 0 3 12489510 0xbe9326 0 5 12381573 0xbced85 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489507 0xbe9323 0 5 12381570 0xbced82 0 6 12594827 0xc02e8b 0 0 12423219 0xbd9033 0 2 12569560 0xbfcbd8 0 4 12511296 0xbee840 0 7 12459928 0xbe1f98 0 1 12641206 0xc0e3b6 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12511293 0xbee83d 0 7 12459925 0xbe1f95 0 1 12641203 0xc0e3b3 0 3 12489504 0xbe9320 0 5 12381567 0xbced7f 0 6 12594824 0xc02e88 0 0 12423216 0xbd9030 0 2 12569557 0xbfcbd5 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12381572 0xbced84 0 6 12594829 0xc02e8d 0 0 12423221 0xbd9035 0 2 12569562 0xbfcbda 0 4 12511298 0xbee842 0 7 12459930 0xbe1f9a 0 1 12641208 0xc0e3b8 0 3 12489509 0xbe9325 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594818 0xc02e82 0 7 12459919 0xbe1f8f 0 1 12641197 0xc0e3ad 0 0 12423210 0xbd902a 0 3 12489499 0xbe931b 0 2 12569552 0xbfcbd0 0 5 12381563 0xbced7b 0 4 12511290 0xbee83a 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12511291 0xbee83b 0 7 12459922 0xbe1f92 0 5 12381564 0xbced7c 0 6 12594821 0xc02e85 0 0 12423213 0xbd902d 0 1 12641200 0xc0e3b0 0 3 12489501 0xbe931d 0 2 12569554 0xbfcbd2 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12381571 0xbced83 0 6 12594828 0xc02e8c 0 0 12423220 0xbd9034 0 2 12569561 0xbfcbd9 0 4 12511297 0xbee841 0 7 12459929 0xbe1f99 0 1 12641207 0xc0e3b7 0 3 12489508 0xbe9324 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489498 0xbe931a 0 4 12511288 0xbee838 0 5 12381561 0xbced79 0 7 12459920 0xbe1f90 0 6 12594819 0xc02e83 0 1 12641198 0xc0e3ae 0 0 12423211 0xbd902b 0 2 12569553 0xbfcbd1 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569551 0xbfcbcf 0 4 12511289 0xbee839 0 5 12381562 0xbced7a 0 7 12459921 0xbe1f91 0 3 12489500 0xbe931c 0 1 12641199 0xc0e3af 0 6 12594820 0xc02e84 0 0 12423212 0xbd902c 0