+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r + tee -a ./output/COUNT:1#NN:4#PPN:4#API:POSIX#T:1048576.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 14:29:44 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r Machine: Linux isc17-c04 Test 0 started: Tue Oct 23 14:29:44 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread4/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (4 per node) repetitions = 3 xfersize = 1 MiB blocksize = 37.50 GiB aggregate filesize = 600 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 181194981376. read 1439.78 39321600 1024.00 0.001215 120.02 0.000552 120.02 0 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 220225077248. read 1749.91 39321600 1024.00 0.000579 120.02 0.000494 120.02 1 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 227791601664. read 1810.02 39321600 1024.00 0.000698 120.02 0.000670 120.02 2 Max Read: 1810.02 MiB/sec (1897.95 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 1810.02 1439.78 1666.57 162.23 120.01958 0 16 4 3 1 0 1 0 0 1 40265318400 1048576 181194981376 POSIX 0 Finished: Tue Oct 23 14:35:44 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 8 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460195 0xbe20a3 0 1 12641471 0xc0e4bf 0 3 12489772 0xbe942c 0 5 12381838 0xbcee8e 0 6 12595093 0xc02f95 0 0 12423484 0xbd913c 0 2 12569826 0xbfcce2 0 4 12511565 0xbee94d 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595083 0xc02f8b 0 2 12569816 0xbfccd8 0 0 12423474 0xbd9132 0 4 12511555 0xbee943 0 1 12641461 0xc0e4b5 0 5 12381828 0xbcee84 0 7 12460186 0xbe209a 0 3 12489765 0xbe9425 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489766 0xbe9426 0 5 12381832 0xbcee88 0 6 12595087 0xc02f8f 0 0 12423478 0xbd9136 0 2 12569820 0xbfccdc 0 4 12511559 0xbee947 0 7 12460189 0xbe209d 0 1 12641465 0xc0e4b9 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569829 0xbfcce5 0 4 12511568 0xbee950 0 7 12460198 0xbe20a6 0 1 12641475 0xc0e4c3 0 3 12489776 0xbe9430 0 5 12381842 0xbcee92 0 6 12595097 0xc02f99 0 0 12423488 0xbd9140 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595089 0xc02f91 0 0 12423480 0xbd9138 0 2 12569822 0xbfccde 0 4 12511561 0xbee949 0 7 12460191 0xbe209f 0 1 12641467 0xc0e4bb 0 3 12489768 0xbe9428 0 5 12381834 0xbcee8a 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12511570 0xbee952 0 7 12460200 0xbe20a8 0 1 12641476 0xc0e4c4 0 3 12489777 0xbe9431 0 5 12381843 0xbcee93 0 6 12595098 0xc02f9a 0 0 12423489 0xbd9141 0 2 12569831 0xbfcce7 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489762 0xbe9422 0 0 12423475 0xbd9133 0 5 12381829 0xbcee85 0 2 12569819 0xbfccdb 0 7 12460187 0xbe209b 0 4 12511557 0xbee945 0 1 12641464 0xc0e4b8 0 6 12595086 0xc02f8e 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569824 0xbfcce0 0 4 12511563 0xbee94b 0 7 12460193 0xbe20a1 0 1 12641469 0xc0e4bd 0 3 12489770 0xbe942a 0 5 12381836 0xbcee8c 0 6 12595091 0xc02f93 0 0 12423482 0xbd913a 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489774 0xbe942e 0 5 12381840 0xbcee90 0 2 12569828 0xbfcce4 0 4 12511567 0xbee94f 0 1 12641473 0xc0e4c1 0 6 12595095 0xc02f97 0 7 12460199 0xbe20a7 0 0 12423487 0xbd913f 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12511564 0xbee94c 0 7 12460194 0xbe20a2 0 1 12641470 0xc0e4be 0 3 12489771 0xbe942b 0 5 12381837 0xbcee8d 0 6 12595092 0xc02f94 0 0 12423483 0xbd913b 0 2 12569825 0xbfcce1 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460185 0xbe2099 0 3 12489764 0xbe9424 0 2 12569818 0xbfccda 0 1 12641462 0xc0e4b6 0 6 12595085 0xc02f8d 0 0 12423476 0xbd9134 0 5 12381831 0xbcee87 0 4 12511558 0xbee946 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12641472 0xc0e4c0 0 3 12489773 0xbe942d 0 5 12381839 0xbcee8f 0 6 12595094 0xc02f96 0 0 12423485 0xbd913d 0 2 12569827 0xbfcce3 0 4 12511566 0xbee94e 0 7 12460196 0xbe20a4 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12381833 0xbcee89 0 6 12595088 0xc02f90 0 0 12423479 0xbd9137 0 2 12569821 0xbfccdd 0 4 12511560 0xbee948 0 7 12460190 0xbe209e 0 1 12641466 0xc0e4ba 0 3 12489767 0xbe9427 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12423481 0xbd9139 0 2 12569823 0xbfccdf 0 4 12511562 0xbee94a 0 7 12460192 0xbe20a0 0 1 12641468 0xc0e4bc 0 3 12489769 0xbe9429 0 5 12381835 0xbcee8b 0 6 12595090 0xc02f92 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12423486 0xbd913e 0 7 12460197 0xbe20a5 0 3 12489775 0xbe942f 0 5 12381841 0xbcee91 0 2 12569830 0xbfcce6 0 1 12641474 0xc0e4c2 0 4 12511569 0xbee951 0 6 12595096 0xc02f98 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489763 0xbe9423 0 6 12595084 0xc02f8c 0 2 12569817 0xbfccd9 0 4 12511556 0xbee944 0 5 12381830 0xbcee86 0 1 12641463 0xc0e4b7 0 0 12423477 0xbd9135 0 7 12460188 0xbe209c 0