+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + tee -a ./output/COUNT:1#NN:16#PPN:2#API:POSIX#T:1048576.txt + /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 70590136320 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 09:54:31 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 70590136320 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r Machine: Linux isc17-c01 Test 0 started: Tue Oct 23 09:54:31 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread16/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 32 (2 per node) repetitions = 3 xfersize = 1 MiB blocksize = 65.74 GiB aggregate filesize = 2103.75 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 4517768724480. WARNING: Using actual aggregate bytes moved = 289747763200. read 2301.58 68935680 1024.00 0.004985 120.06 0.004599 120.06 0 WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 4517768724480. WARNING: Using actual aggregate bytes moved = 329581068288. read 2618.67 68935680 1024.00 0.004627 120.03 0.004544 120.03 1 WARNING: Expected aggregate file size = 2258884362240. WARNING: Stat() of aggregate file size = 4517768724480. WARNING: Using actual aggregate bytes moved = 353401569280. read 2808.00 68935680 1024.00 0.004710 120.02 0.004864 120.03 2 Max Read: 2808.00 MiB/sec (2944.40 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 2808.00 2301.58 2576.08 208.93 120.03717 0 32 2 3 1 0 1 0 0 1 70590136320 1048576 289747763200 POSIX 0 Finished: Tue Oct 23 10:00:31 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12459177 0xbe1ca9 0 1 12640453 0xc0e0c5 0 5 12380824 0xbcea98 0 6 12594081 0xc02ba1 0 3 12488763 0xbe903b 0 4 12510552 0xbee558 0 0 12422478 0xbd8d4e 0 2 12568820 0xbfc8f4 0 /esfs/jtacquaviva/ioperf/file_write.00000023 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12568812 0xbfc8ec 0 3 12488759 0xbe9037 0 7 12459183 0xbe1caf 0 0 12422476 0xbd8d4c 0 5 12380829 0xbcea9d 0 1 12640462 0xc0e0ce 0 6 12594087 0xc02ba7 0 4 12510556 0xbee55c 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594094 0xc02bae 0 2 12568831 0xbfc8ff 0 7 12459198 0xbe1cbe 0 3 12488778 0xbe904a 0 0 12422492 0xbd8d5c 0 1 12640477 0xc0e0dd 0 4 12510567 0xbee567 0 5 12380846 0xbceaae 0 /esfs/jtacquaviva/ioperf/file_write.00000026 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12422468 0xbd8d44 0 3 12488755 0xbe9033 0 6 12594076 0xc02b9c 0 5 12380819 0xbcea93 0 1 12640451 0xc0e0c3 0 2 12568811 0xbfc8eb 0 7 12459178 0xbe1caa 0 4 12510551 0xbee557 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594105 0xc02bb9 0 0 12422498 0xbd8d62 0 2 12568840 0xbfc908 0 4 12510574 0xbee56e 0 7 12459205 0xbe1cc5 0 1 12640481 0xc0e0e1 0 3 12488784 0xbe9050 0 5 12380848 0xbceab0 0 /esfs/jtacquaviva/ioperf/file_write.00000020 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12568829 0xbfc8fd 0 7 12459197 0xbe1cbd 0 3 12488774 0xbe9046 0 0 12422493 0xbd8d5d 0 1 12640478 0xc0e0de 0 6 12594103 0xc02bb7 0 4 12510571 0xbee56b 0 5 12380847 0xbceaaf 0 /esfs/jtacquaviva/ioperf/file_write.00000030 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488779 0xbe904b 0 5 12380843 0xbceaab 0 6 12594100 0xc02bb4 0 0 12422494 0xbd8d5e 0 2 12568836 0xbfc904 0 4 12510569 0xbee569 0 7 12459202 0xbe1cc2 0 1 12640479 0xc0e0df 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488770 0xbe9042 0 0 12422485 0xbd8d55 0 4 12510561 0xbee561 0 2 12568826 0xbfc8fa 0 7 12459194 0xbe1cba 0 6 12594092 0xc02bac 0 1 12640468 0xc0e0d4 0 5 12380837 0xbceaa5 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380836 0xbceaa4 0 7 12459193 0xbe1cb9 0 1 12640467 0xc0e0d3 0 2 12568827 0xbfc8fb 0 0 12422486 0xbd8d56 0 4 12510564 0xbee564 0 3 12488772 0xbe9044 0 6 12594097 0xc02bb1 0 /esfs/jtacquaviva/ioperf/file_write.00000018 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594077 0xc02b9d 0 5 12380821 0xbcea95 0 4 12510547 0xbee553 0 0 12422471 0xbd8d47 0 2 12568814 0xbfc8ee 0 7 12459182 0xbe1cae 0 1 12640454 0xc0e0c6 0 3 12488762 0xbe903a 0 /esfs/jtacquaviva/ioperf/file_write.00000031 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594078 0xc02b9e 0 7 12459180 0xbe1cac 0 5 12380826 0xbcea9a 0 1 12640461 0xc0e0cd 0 4 12510554 0xbee55a 0 3 12488766 0xbe903e 0 2 12568821 0xbfc8f5 0 0 12422480 0xbd8d50 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12640470 0xc0e0d6 0 2 12568830 0xbfc8fe 0 4 12510565 0xbee565 0 3 12488777 0xbe9049 0 7 12459206 0xbe1cc6 0 6 12594106 0xc02bba 0 0 12422499 0xbd8d63 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12568813 0xbfc8ed 0 7 12459181 0xbe1cad 0 6 12594083 0xc02ba3 0 1 12640459 0xc0e0cb 0 0 12422477 0xbd8d4d 0 4 12510553 0xbee559 0 5 12380830 0xbcea9e 0 /esfs/jtacquaviva/ioperf/file_write.00000025 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12422488 0xbd8d58 0 1 12640471 0xc0e0d7 0 2 12568832 0xbfc900 0 5 12380845 0xbceaad 0 4 12510570 0xbee56a 0 7 12459201 0xbe1cc1 0 6 12594104 0xc02bb8 0 /esfs/jtacquaviva/ioperf/file_write.00000022 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12640472 0xc0e0d8 0 3 12488775 0xbe9047 0 5 12380842 0xbceaaa 0 6 12594102 0xc02bb6 0 0 12422495 0xbd8d5f 0 2 12568837 0xbfc905 0 4 12510572 0xbee56c 0 7 12459203 0xbe1cc3 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594075 0xc02b9b 0 4 12510544 0xbee550 0 1 12640450 0xc0e0c2 0 0 12422469 0xbd8d45 0 7 12459176 0xbe1ca8 0 2 12568809 0xbfc8e9 0 3 12488756 0xbe9034 0 5 12380822 0xbcea96 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12594093 0xc02bad 0 1 12640469 0xc0e0d5 0 4 12510563 0xbee563 0 3 12488776 0xbe9048 0 7 12459200 0xbe1cc0 0 2 12568833 0xbfc901 0 0 12422496 0xbd8d60 0 /esfs/jtacquaviva/ioperf/file_write.00000028 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12459195 0xbe1cbb 0 4 12510562 0xbee562 0 2 12568828 0xbfc8fc 0 5 12380840 0xbceaa8 0 0 12422489 0xbd8d59 0 3 12488773 0xbe9045 0 1 12640474 0xc0e0da 0 6 12594096 0xc02bb0 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380838 0xbceaa6 0 6 12594095 0xc02baf 0 1 12640473 0xc0e0d9 0 0 12422491 0xbd8d5b 0 4 12510566 0xbee566 0 7 12459199 0xbe1cbf 0 3 12488780 0xbe904c 0 2 12568835 0xbfc903 0 /esfs/jtacquaviva/ioperf/file_write.00000017 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488758 0xbe9036 0 4 12510550 0xbee556 0 5 12380825 0xbcea99 0 0 12422475 0xbd8d4b 0 1 12640456 0xc0e0c8 0 2 12568818 0xbfc8f2 0 6 12594085 0xc02ba5 0 7 12459185 0xbe1cb1 0 /esfs/jtacquaviva/ioperf/file_write.00000029 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380823 0xbcea97 0 3 12488760 0xbe9038 0 6 12594080 0xc02ba0 0 2 12568816 0xbfc8f0 0 7 12459184 0xbe1cb0 0 0 12422479 0xbd8d4f 0 4 12510555 0xbee55b 0 1 12640463 0xc0e0cf 0 /esfs/jtacquaviva/ioperf/file_write.00000016 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380839 0xbceaa7 0 0 12422490 0xbd8d5a 0 1 12640476 0xc0e0dc 0 6 12594099 0xc02bb3 0 4 12510568 0xbee568 0 2 12568838 0xbfc906 0 3 12488782 0xbe904e 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12510548 0xbee554 0 0 12422472 0xbd8d48 0 1 12640457 0xc0e0c9 0 2 12568819 0xbfc8f3 0 6 12594086 0xc02ba6 0 7 12459188 0xbe1cb4 0 3 12488767 0xbe903f 0 5 12380832 0xbceaa0 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380834 0xbceaa2 0 6 12594090 0xc02baa 0 0 12422483 0xbd8d53 0 2 12568824 0xbfc8f8 0 4 12510559 0xbee55f 0 7 12459191 0xbe1cb7 0 1 12640465 0xc0e0d1 0 3 12488769 0xbe9041 0 /esfs/jtacquaviva/ioperf/file_write.00000021 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488757 0xbe9035 0 4 12510549 0xbee555 0 0 12422473 0xbd8d49 0 1 12640455 0xc0e0c7 0 6 12594082 0xc02ba2 0 5 12380828 0xbcea9c 0 2 12568817 0xbfc8f1 0 7 12459187 0xbe1cb3 0 /esfs/jtacquaviva/ioperf/file_write.00000019 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380844 0xbceaac 0 6 12594101 0xc02bb5 0 0 12422497 0xbd8d61 0 2 12568839 0xbfc907 0 4 12510573 0xbee56d 0 7 12459204 0xbe1cc4 0 1 12640480 0xc0e0e0 0 3 12488783 0xbe904f 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12488768 0xbe9040 0 5 12380833 0xbceaa1 0 6 12594089 0xc02ba9 0 0 12422482 0xbd8d52 0 2 12568823 0xbfc8f7 0 4 12510558 0xbee55e 0 7 12459190 0xbe1cb6 0 1 12640464 0xc0e0d0 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12640460 0xc0e0cc 0 3 12488765 0xbe903d 0 5 12380831 0xbcea9f 0 6 12594088 0xc02ba8 0 0 12422481 0xbd8d51 0 2 12568822 0xbfc8f6 0 4 12510557 0xbee55d 0 7 12459189 0xbe1cb5 0 /esfs/jtacquaviva/ioperf/file_write.00000027 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12459196 0xbe1cbc 0 0 12422487 0xbd8d57 0 5 12380841 0xbceaa9 0 1 12640475 0xc0e0db 0 6 12594098 0xc02bb2 0 3 12488781 0xbe904d 0 2 12568834 0xbfc902 0 /esfs/jtacquaviva/ioperf/file_write.00000024 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380820 0xbcea94 0 4 12510546 0xbee552 0 1 12640452 0xc0e0c4 0 7 12459179 0xbe1cab 0 2 12568815 0xbfc8ef 0 0 12422474 0xbd8d4a 0 3 12488764 0xbe903c 0 6 12594084 0xc02ba4 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12380835 0xbceaa3 0 2 12568825 0xbfc8f9 0 6 12594091 0xc02bab 0 3 12488771 0xbe9043 0 0 12422484 0xbd8d54 0 4 12510560 0xbee560 0 7 12459192 0xbe1cb8 0 1 12640466 0xc0e0d2 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12510545 0xbee551 0 2 12568810 0xbfc8ea 0 0 12422470 0xbd8d46 0 6 12594079 0xc02b9f 0 3 12488761 0xbe9039 0 5 12380827 0xbcea9b 0 1 12640458 0xc0e0ca 0 7 12459186 0xbe1cb2 0