+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w + tee -a ./output/COUNT:1#NN:16#PPN:1#API:POSIX#T:1048576.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 10:13:02 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w Machine: Linux isc17-c01 Test 0 started: Tue Oct 23 10:13:02 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/ioperf/file_write access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (1 per node) repetitions = 3 xfersize = 1 MiB blocksize = 131.48 GiB aggregate filesize = 2103.75 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 2258846613504. WARNING: Using actual aggregate bytes moved = 673961738240. write 4819 137871360 1024.00 0.000930 133.37 0.000665 133.37 0 WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 2258867585024. WARNING: Using actual aggregate bytes moved = 617881796608. write 4363 137871360 1024.00 0.006670 135.07 0.000829 135.07 1 WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 2258884362240. WARNING: Using actual aggregate bytes moved = 678933037056. write 4745 137871360 1024.00 0.001755 136.44 0.001155 136.45 2 Max Write: 4819.17 MiB/sec (5053.27 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum write 4819.17 4362.52 4642.34 200.15 134.96339 0 16 1 3 1 0 1 0 0 1 141180272640 1048576 673961738240 POSIX 0 Finished: Tue Oct 23 10:20:00 2018 + /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r + tee -a ./output/COUNT:1#NN:16#PPN:1#API:POSIX#T:1048576.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 10:20:25 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 141180272640 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r Machine: Linux isc17-c01 Test 0 started: Tue Oct 23 10:20:25 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread16/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (1 per node) repetitions = 3 xfersize = 1 MiB blocksize = 131.48 GiB aggregate filesize = 2103.75 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 2258884362240. WARNING: Using actual aggregate bytes moved = 153187516416. read 1217.24 137871360 1024.00 0.000789 120.02 0.000565 120.02 0 WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 2258884362240. WARNING: Using actual aggregate bytes moved = 164603363328. read 1307.91 137871360 1024.00 0.000597 120.02 0.000861 120.02 1 WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 2258884362240. WARNING: Using actual aggregate bytes moved = 178045059072. read 1414.67 137871360 1024.00 0.000901 120.02 0.001137 120.03 2 Max Read: 1414.67 MiB/sec (1483.39 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 1414.67 1217.24 1313.28 80.69 120.02189 0 16 1 3 1 0 1 0 0 1 141180272640 1048576 153187516416 POSIX 0 Finished: Tue Oct 23 10:26:25 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488825 0xbe9079 0 7 12459248 0xbe1cf0 0 0 12422541 0xbd8d8d 0 6 12594150 0xc02be6 0 4 12510618 0xbee59a 0 1 12640530 0xc0e112 0 2 12568887 0xbfc937 0 5 12380894 0xbceade 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12568873 0xbfc929 0 1 12640516 0xc0e104 0 4 12510610 0xbee592 0 5 12380880 0xbcead0 0 7 12459243 0xbe1ceb 0 6 12594142 0xc02bde 0 0 12422532 0xbd8d84 0 3 12488821 0xbe9075 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12568884 0xbfc934 0 4 12510620 0xbee59c 0 7 12459255 0xbe1cf7 0 1 12640531 0xc0e113 0 3 12488834 0xbe9082 0 5 12380895 0xbceadf 0 6 12594155 0xc02beb 0 0 12422546 0xbd8d92 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12568875 0xbfc92b 0 3 12488822 0xbe9076 0 6 12594145 0xc02be1 0 4 12510613 0xbee595 0 1 12640523 0xc0e10b 0 5 12380890 0xbceada 0 0 12422542 0xbd8d8e 0 7 12459253 0xbe1cf5 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594146 0xc02be2 0 5 12380887 0xbcead7 0 2 12568880 0xbfc930 0 4 12510616 0xbee598 0 3 12488830 0xbe907e 0 1 12640529 0xc0e111 0 0 12422545 0xbd8d91 0 7 12459256 0xbe1cf8 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12422535 0xbd8d87 0 1 12640524 0xc0e10c 0 7 12459249 0xbe1cf1 0 6 12594149 0xc02be5 0 3 12488829 0xbe907d 0 2 12568883 0xbfc933 0 5 12380892 0xbceadc 0 4 12510622 0xbee59e 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380885 0xbcead5 0 0 12422537 0xbd8d89 0 1 12640526 0xc0e10e 0 7 12459250 0xbe1cf2 0 3 12488827 0xbe907b 0 6 12594152 0xc02be8 0 2 12568882 0xbfc932 0 4 12510619 0xbee59b 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488824 0xbe9078 0 2 12568879 0xbfc92f 0 5 12380889 0xbcead9 0 1 12640527 0xc0e10f 0 0 12422543 0xbd8d8f 0 6 12594154 0xc02bea 0 4 12510623 0xbee59f 0 7 12459257 0xbe1cf9 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12640517 0xc0e105 0 4 12510612 0xbee594 0 5 12380883 0xbcead3 0 7 12459247 0xbe1cef 0 6 12594143 0xc02bdf 0 2 12568877 0xbfc92d 0 0 12422540 0xbd8d8c 0 3 12488826 0xbe907a 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380884 0xbcead4 0 0 12422534 0xbd8d86 0 6 12594147 0xc02be3 0 1 12640525 0xc0e10d 0 3 12488828 0xbe907c 0 7 12459254 0xbe1cf6 0 4 12510621 0xbee59d 0 2 12568886 0xbfc936 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594141 0xc02bdd 0 4 12510609 0xbee591 0 3 12488819 0xbe9073 0 0 12422531 0xbd8d83 0 5 12380881 0xbcead1 0 7 12459244 0xbe1cec 0 2 12568874 0xbfc92a 0 1 12640518 0xc0e106 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12640521 0xc0e109 0 5 12380886 0xbcead6 0 2 12568881 0xbfc931 0 6 12594151 0xc02be7 0 3 12488832 0xbe9080 0 0 12422544 0xbd8d90 0 4 12510624 0xbee5a0 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12422536 0xbd8d88 0 2 12568878 0xbfc92e 0 4 12510615 0xbee597 0 7 12459251 0xbe1cf3 0 1 12640528 0xc0e110 0 3 12488833 0xbe9081 0 5 12380893 0xbceadd 0 6 12594156 0xc02bec 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488820 0xbe9074 0 4 12510611 0xbee593 0 5 12380882 0xbcead2 0 7 12459246 0xbe1cee 0 6 12594144 0xc02be0 0 1 12640519 0xc0e107 0 0 12422539 0xbd8d8b 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12422533 0xbd8d85 0 3 12488823 0xbe9077 0 1 12640522 0xc0e10a 0 5 12380888 0xbcead8 0 7 12459252 0xbe1cf4 0 4 12510617 0xbee599 0 6 12594153 0xc02be9 0 2 12568885 0xbfc935 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12459245 0xbe1ced 0 2 12568876 0xbfc92c 0 1 12640520 0xc0e108 0 4 12510614 0xbee596 0 0 12422538 0xbd8d8a 0 6 12594148 0xc02be4 0 5 12380891 0xbceadb 0 3 12488831 0xbe907f 0