+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 16384 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r + tee -a ./output/COUNT:1#NN:4#PPN:4#API:POSIX#T:16384.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Tue Oct 23 18:45:04 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 16384 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r Machine: Linux isc17-c04 Test 0 started: Tue Oct 23 18:45:04 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread4/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 16 (4 per node) repetitions = 3 xfersize = 16384 bytes blocksize = 37.50 GiB aggregate filesize = 600 GiB Using stonewalling = 120 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 5504909312. read 43.69 39321600 16.00 0.003983 120.15 0.003403 120.15 0 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 5849333760. read 46.43 39321600 16.00 0.003422 120.15 0.003406 120.15 1 WARNING: Expected aggregate file size = 644245094400. WARNING: Stat() of aggregate file size = 2576980377600. WARNING: Using actual aggregate bytes moved = 5870469120. read 46.60 39321600 16.00 0.003577 120.15 0.003422 120.15 2 Max Read: 46.60 MiB/sec (48.86 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 46.60 43.69 45.57 1.33 120.15097 0 16 4 3 1 0 1 0 0 1 40265318400 16384 5504909312 POSIX 0 Finished: Tue Oct 23 18:51:05 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 8 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12512094 0xbeeb5e 0 7 12460725 0xbe22b5 0 1 12642002 0xc0e6d2 0 3 12490303 0xbe963f 0 5 12382369 0xbcf0a1 0 6 12595623 0xc031a7 0 0 12424013 0xbd934d 0 2 12570355 0xbfcef3 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12642009 0xc0e6d9 0 3 12490310 0xbe9646 0 5 12382376 0xbcf0a8 0 6 12595630 0xc031ae 0 0 12424020 0xbd9354 0 2 12570362 0xbfcefa 0 4 12512101 0xbeeb65 0 7 12460732 0xbe22bc 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12490311 0xbe9647 0 5 12382377 0xbcf0a9 0 6 12595631 0xc031af 0 0 12424021 0xbd9355 0 2 12570363 0xbfcefb 0 4 12512102 0xbeeb66 0 7 12460733 0xbe22bd 0 1 12642010 0xc0e6da 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12570359 0xbfcef7 0 4 12512098 0xbeeb62 0 7 12460729 0xbe22b9 0 1 12642006 0xc0e6d6 0 3 12490307 0xbe9643 0 5 12382373 0xbcf0a5 0 6 12595627 0xc031ab 0 0 12424017 0xbd9351 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12382362 0xbcf09a 0 2 12570348 0xbfceec 0 7 12460718 0xbe22ae 0 1 12641995 0xc0e6cb 0 6 12595616 0xc031a0 0 4 12512088 0xbeeb58 0 0 12424007 0xbd9347 0 3 12490298 0xbe963a 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460726 0xbe22b6 0 1 12642003 0xc0e6d3 0 3 12490304 0xbe9640 0 0 12424014 0xbd934e 0 2 12570356 0xbfcef4 0 4 12512095 0xbeeb5f 0 5 12382370 0xbcf0a2 0 6 12595625 0xc031a9 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12460731 0xbe22bb 0 1 12642008 0xc0e6d8 0 3 12490309 0xbe9645 0 5 12382375 0xbcf0a7 0 6 12595629 0xc031ad 0 0 12424019 0xbd9353 0 2 12570361 0xbfcef9 0 4 12512100 0xbeeb64 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12424016 0xbd9350 0 2 12570358 0xbfcef6 0 4 12512097 0xbeeb61 0 7 12460728 0xbe22b8 0 1 12642005 0xc0e6d5 0 3 12490306 0xbe9642 0 5 12382372 0xbcf0a4 0 6 12595626 0xc031aa 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12424006 0xbd9346 0 4 12512087 0xbeeb57 0 3 12490297 0xbe9639 0 2 12570349 0xbfceed 0 7 12460719 0xbe22af 0 1 12641996 0xc0e6cc 0 5 12382363 0xbcf09b 0 6 12595617 0xc031a1 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12595624 0xc031a8 0 7 12460727 0xbe22b7 0 0 12424015 0xbd934f 0 1 12642004 0xc0e6d4 0 2 12570357 0xbfcef5 0 4 12512096 0xbeeb60 0 3 12490305 0xbe9641 0 5 12382371 0xbcf0a3 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12382365 0xbcf09d 0 4 12512091 0xbeeb5b 0 0 12424010 0xbd934a 0 6 12595620 0xc031a4 0 2 12570353 0xbfcef1 0 1 12642000 0xc0e6d0 0 7 12460723 0xbe22b3 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12570350 0xbfceee 0 4 12512089 0xbeeb59 0 7 12460720 0xbe22b0 0 1 12641997 0xc0e6cd 0 3 12490299 0xbe963b 0 5 12382364 0xbcf09c 0 6 12595618 0xc031a2 0 0 12424008 0xbd9348 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12641999 0xc0e6cf 0 4 12512092 0xbeeb5c 0 5 12382368 0xbcf0a0 0 3 12490301 0xbe963d 0 0 12424011 0xbd934b 0 2 12570354 0xbfcef2 0 7 12460724 0xbe22b4 0 6 12595622 0xc031a6 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12512090 0xbeeb5a 0 7 12460721 0xbe22b1 0 1 12641998 0xc0e6ce 0 6 12595619 0xc031a3 0 0 12424009 0xbd9349 0 2 12570351 0xbfceef 0 3 12490300 0xbe963c 0 5 12382367 0xbcf09f 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12382366 0xbcf09e 0 2 12570352 0xbfcef0 0 7 12460722 0xbe22b2 0 3 12490302 0xbe963e 0 1 12642001 0xc0e6d1 0 6 12595621 0xc031a5 0 4 12512093 0xbeeb5d 0 0 12424012 0xbd934c 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12512099 0xbeeb63 0 7 12460730 0xbe22ba 0 1 12642007 0xc0e6d7 0 3 12490308 0xbe9644 0 5 12382374 0xbcf0a6 0 6 12595628 0xc031ac 0 0 12424018 0xbd9352 0 2 12570360 0xbfcef8 0