+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 24 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 24 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 26843545600 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r + tee -a ./output/COUNT:1#NN:4#PPN:6#API:POSIX#T:1048576.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 14:04:35 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 26843545600 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r Machine: Linux isc17-c04 Test 0 started: Tue Oct 23 14:04:35 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread4/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 24 (6 per node) repetitions = 3 xfersize = 1 MiB blocksize = 25 GiB aggregate filesize = 600 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 3865470566400. WARNING: Using actual aggregate bytes moved = 269722058752. read 2143.03 26214400 1024.00 0.001394 120.03 0.000856 120.03 0 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 3865470566400. WARNING: Using actual aggregate bytes moved = 322788392960. read 2564.60 26214400 1024.00 0.000829 120.03 0.000856 120.03 1 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 3865470566400. WARNING: Using actual aggregate bytes moved = 329560096768. read 2618.62 26214400 1024.00 0.001044 120.02 0.000911 120.02 2 Max Read: 2618.62 MiB/sec (2745.83 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 2618.62 2143.03 2442.09 212.61 120.02798 0 24 6 3 1 0 1 0 0 1 26843545600 1048576 269722058752 POSIX 0 Finished: Tue Oct 23 14:10:35 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 8 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460139 0xbe206b 0 3 12489718 0xbe93f6 0 6 12595037 0xc02f5d 0 4 12511511 0xbee917 0 5 12381786 0xbcee5a 0 1 12641418 0xc0e48a 0 0 12423431 0xbd9107 0 2 12569773 0xbfccad 0 /esfs/jtacquaviva/ioperf/file_write.00000023 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460142 0xbe206e 0 1 12641419 0xc0e48b 0 3 12489720 0xbe93f8 0 5 12381787 0xbcee5b 0 6 12595040 0xc02f60 0 0 12423432 0xbd9108 0 2 12569774 0xbfccae 0 4 12511513 0xbee919 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489717 0xbe93f5 0 4 12511510 0xbee916 0 7 12460140 0xbe206c 0 1 12641417 0xc0e489 0 5 12381785 0xbcee59 0 6 12595038 0xc02f5e 0 0 12423430 0xbd9106 0 2 12569772 0xbfccac 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12381772 0xbcee4c 0 2 12569759 0xbfcc9f 0 7 12460127 0xbe205f 0 3 12489706 0xbe93ea 0 0 12423418 0xbd90fa 0 4 12511499 0xbee90b 0 6 12595026 0xc02f52 0 1 12641405 0xc0e47d 0 /esfs/jtacquaviva/ioperf/file_write.00000020 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12423417 0xbd90f9 0 4 12511498 0xbee90a 0 1 12641404 0xc0e47c 0 5 12381773 0xbcee4d 0 6 12595025 0xc02f51 0 2 12569760 0xbfcca0 0 7 12460128 0xbe2060 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595034 0xc02f5a 0 0 12423427 0xbd9103 0 2 12569769 0xbfcca9 0 4 12511508 0xbee914 0 7 12460137 0xbe2069 0 1 12641414 0xc0e486 0 3 12489715 0xbe93f3 0 5 12381782 0xbcee56 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460130 0xbe2062 0 2 12569764 0xbfcca4 0 1 12641409 0xc0e481 0 4 12511503 0xbee90f 0 5 12381777 0xbcee51 0 3 12489711 0xbe93ef 0 6 12595031 0xc02f57 0 0 12423423 0xbd90ff 0 /esfs/jtacquaviva/ioperf/file_write.00000018 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12511505 0xbee911 0 7 12460134 0xbe2066 0 1 12641411 0xc0e483 0 3 12489712 0xbe93f0 0 5 12381779 0xbcee53 0 6 12595032 0xc02f58 0 0 12423424 0xbd9100 0 2 12569766 0xbfcca6 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489722 0xbe93fa 0 0 12423434 0xbd910a 0 7 12460144 0xbe2070 0 5 12381789 0xbcee5d 0 2 12569777 0xbfccb1 0 1 12641422 0xc0e48e 0 6 12595043 0xc02f63 0 4 12511516 0xbee91c 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569771 0xbfccab 0 1 12641416 0xc0e488 0 5 12381784 0xbcee58 0 0 12423429 0xbd9105 0 7 12460141 0xbe206d 0 3 12489719 0xbe93f7 0 4 12511512 0xbee918 0 6 12595039 0xc02f5f 0 /esfs/jtacquaviva/ioperf/file_write.00000022 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569762 0xbfcca2 0 3 12489708 0xbe93ec 0 5 12381775 0xbcee4f 0 4 12511502 0xbee90e 0 0 12423421 0xbd90fd 0 1 12641410 0xc0e482 0 7 12460133 0xbe2065 0 6 12595029 0xc02f55 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12641420 0xc0e48c 0 3 12489721 0xbe93f9 0 5 12381788 0xbcee5c 0 6 12595041 0xc02f61 0 0 12423433 0xbd9109 0 2 12569775 0xbfccaf 0 4 12511514 0xbee91a 0 7 12460143 0xbe206f 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595042 0xc02f62 0 2 12569776 0xbfccb0 0 4 12511515 0xbee91b 0 1 12641421 0xc0e48d 0 0 12423435 0xbd910b 0 7 12460145 0xbe2071 0 3 12489723 0xbe93fb 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489713 0xbe93f1 0 6 12595033 0xc02f59 0 2 12569767 0xbfcca7 0 7 12460136 0xbe2068 0 5 12381781 0xbcee55 0 0 12423426 0xbd9102 0 4 12511507 0xbee913 0 1 12641413 0xc0e485 0 /esfs/jtacquaviva/ioperf/file_write.00000017 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12381774 0xbcee4e 0 6 12595027 0xc02f53 0 0 12423419 0xbd90fb 0 2 12569761 0xbfcca1 0 4 12511500 0xbee90c 0 7 12460129 0xbe2061 0 1 12641406 0xc0e47e 0 3 12489707 0xbe93eb 0 /esfs/jtacquaviva/ioperf/file_write.00000016 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489704 0xbe93e8 0 6 12595023 0xc02f4f 0 2 12569757 0xbfcc9d 0 7 12460126 0xbe205e 0 5 12381771 0xbcee4b 0 0 12423416 0xbd90f8 0 4 12511497 0xbee909 0 1 12641403 0xc0e47b 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595028 0xc02f54 0 1 12641407 0xc0e47f 0 0 12423420 0xbd90fc 0 7 12460132 0xbe2064 0 2 12569765 0xbfcca5 0 3 12489710 0xbe93ee 0 4 12511504 0xbee910 0 5 12381778 0xbcee52 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460135 0xbe2067 0 5 12381780 0xbcee54 0 0 12423425 0xbd9101 0 4 12511506 0xbee912 0 1 12641412 0xc0e484 0 6 12595035 0xc02f5b 0 2 12569768 0xbfcca8 0 3 12489714 0xbe93f2 0 /esfs/jtacquaviva/ioperf/file_write.00000021 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12489724 0xbe93fc 0 5 12381790 0xbcee5e 0 6 12595044 0xc02f64 0 0 12423436 0xbd910c 0 2 12569778 0xbfccb2 0 4 12511517 0xbee91d 0 7 12460146 0xbe2072 0 1 12641423 0xc0e48f 0 /esfs/jtacquaviva/ioperf/file_write.00000019 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12423428 0xbd9104 0 2 12569770 0xbfccaa 0 4 12511509 0xbee915 0 7 12460138 0xbe206a 0 1 12641415 0xc0e487 0 3 12489716 0xbe93f4 0 5 12381783 0xbcee57 0 6 12595036 0xc02f5c 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460125 0xbe205d 0 5 12381770 0xbcee4a 0 0 12423415 0xbd90f7 0 4 12511496 0xbee908 0 1 12641402 0xc0e47a 0 6 12595024 0xc02f50 0 2 12569758 0xbfcc9e 0 3 12489705 0xbe93e9 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12569763 0xbfcca3 0 4 12511501 0xbee90d 0 7 12460131 0xbe2063 0 1 12641408 0xc0e480 0 3 12489709 0xbe93ed 0 5 12381776 0xbcee50 0 6 12595030 0xc02f56 0 0 12423422 0xbd90fe 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595046 0xc02f66 0 0 12423438 0xbd910e 0 2 12569780 0xbfccb4 0 4 12511519 0xbee91f 0 7 12460148 0xbe2074 0 1 12641425 0xc0e491 0 3 12489726 0xbe93fe 0 5 12381792 0xbcee60 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12381791 0xbcee5f 0 6 12595045 0xc02f65 0 0 12423437 0xbd910d 0 2 12569779 0xbfccb3 0 4 12511518 0xbee91e 0 7 12460147 0xbe2073 0 1 12641424 0xc0e490 0 3 12489725 0xbe93fd 0