+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w + tee -a ./output/COUNT:1#NN:8#PPN:4#API:POSIX#T:10485760.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Wed Oct 24 03:12:53 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w Machine: Linux isc17-c04 Test 0 started: Wed Oct 24 03:12:53 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/ioperf/file_write access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 32 (4 per node) repetitions = 3 xfersize = 10 MiB blocksize = 37.50 GiB aggregate filesize = 1200 GiB Using stonewalling = 60 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 1288154644480. WARNING: Using actual aggregate bytes moved = 396697272320. write 5553 39321600 10240 0.001622 68.12 0.000543 68.13 0 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 1284966973440. WARNING: Using actual aggregate bytes moved = 391297105920. write 5525 39321600 10240 0.009448 67.54 0.000672 67.55 1 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 1286602752000. WARNING: Using actual aggregate bytes moved = 391611678720. write 5503 39321600 10240 0.015847 67.86 0.000832 67.87 2 Max Write: 5553.20 MiB/sec (5822.95 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum write 5553.20 5502.54 5526.77 20.74 67.84871 0 32 4 3 1 0 1 0 0 1 40265318400 10485760 396697272320 POSIX 0 Finished: Wed Oct 24 03:16:22 2018 + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh + /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r + tee -a ./output/COUNT:1#NN:8#PPN:4#API:POSIX#T:10485760.txt IOR-3.0.1: MPI Coordinated Test of Parallel I/O Began: Wed Oct 24 03:16:39 2018 Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 40265318400 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r Machine: Linux isc17-c04 Test 0 started: Wed Oct 24 03:16:39 2018 Summary: api = POSIX test filename = /esfs/jtacquaviva/indread8/file access = file-per-process ordering in a file = random offsets ordering inter file= no tasks offsets clients = 32 (4 per node) repetitions = 3 xfersize = 10 MiB blocksize = 37.50 GiB aggregate filesize = 1200 GiB Using stonewalling = 60 second(s) access bw(MiB/s) block(KiB) xfer(KiB) open(s) wr/rd(s) close(s) total(s) iter ------ --------- ---------- --------- -------- -------- -------- -------- ---- WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 5153960755200. WARNING: Using actual aggregate bytes moved = 169198223360. read 2681.48 39321600 10240 0.001316 60.17 0.000816 60.18 0 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 5153960755200. WARNING: Using actual aggregate bytes moved = 191574835200. read 3033.32 39321600 10240 0.000779 60.23 0.001073 60.23 1 WARNING: Expected aggregate file size = 1288490188800. WARNING: Stat() of aggregate file size = 5153960755200. WARNING: Using actual aggregate bytes moved = 211235635200. read 3351.82 39321600 10240 0.000796 60.10 0.000797 60.10 2 Max Read: 3351.82 MiB/sec (3514.63 MB/sec) Summary of all tests: Operation Max(MiB) Min(MiB) Mean(MiB) StdDev Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum read 3351.82 2681.48 3022.21 273.78 60.16949 0 32 4 3 1 0 1 0 0 1 40265318400 10485760 169198223360 POSIX 0 Finished: Wed Oct 24 03:19:40 2018 + set +x /esfs/jtacquaviva/ioperf stripe_count: 16 stripe_size: 1048576 stripe_offset: -1 /esfs/jtacquaviva/ioperf/file_write.00000006 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12493276 0xbea1dc 0 4 12515068 0xbef6fc 0 5 12385340 0xbcfc3c 0 7 12463689 0xbe2e49 0 1 12644976 0xc0f270 0 6 12598591 0xc03d3f 0 0 12426995 0xbd9ef3 0 2 12573333 0xbfda95 0 /esfs/jtacquaviva/ioperf/file_write.00000023 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12515059 0xbef6f3 0 3 12493267 0xbea1d3 0 2 12573324 0xbfda8c 0 6 12598585 0xc03d39 0 5 12385335 0xbcfc37 0 7 12463684 0xbe2e44 0 1 12644972 0xc0f26c 0 0 12426991 0xbd9eef 0 /esfs/jtacquaviva/ioperf/file_write.00000002 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12463704 0xbe2e58 0 1 12644991 0xc0f27f 0 3 12493293 0xbea1ed 0 5 12385355 0xbcfc4b 0 6 12598607 0xc03d4f 0 0 12427011 0xbd9f03 0 2 12573348 0xbfdaa4 0 4 12515084 0xbef70c 0 /esfs/jtacquaviva/ioperf/file_write.00000026 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12426986 0xbd9eea 0 4 12515061 0xbef6f5 0 5 12385334 0xbcfc36 0 1 12644969 0xc0f269 0 3 12493272 0xbea1d8 0 7 12463682 0xbe2e42 0 2 12573328 0xbfda90 0 /esfs/jtacquaviva/ioperf/file_write.00000000 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12463695 0xbe2e4f 0 4 12515073 0xbef701 0 6 12598598 0xc03d46 0 2 12573339 0xbfda9b 0 1 12644983 0xc0f277 0 3 12493287 0xbea1e7 0 5 12385350 0xbcfc46 0 0 12427006 0xbd9efe 0 /esfs/jtacquaviva/ioperf/file_write.00000020 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12598589 0xc03d3d 0 7 12463687 0xbe2e47 0 0 12426993 0xbd9ef1 0 3 12493275 0xbea1db 0 5 12385338 0xbcfc3a 0 2 12573330 0xbfda92 0 1 12644974 0xc0f26e 0 /esfs/jtacquaviva/ioperf/file_write.00000030 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12493297 0xbea1f1 0 5 12385359 0xbcfc4f 0 6 12598611 0xc03d53 0 0 12427015 0xbd9f07 0 2 12573352 0xbfdaa8 0 4 12515088 0xbef710 0 7 12463709 0xbe2e5d 0 1 12644996 0xc0f284 0 /esfs/jtacquaviva/ioperf/file_write.00000004 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12385332 0xbcfc34 0 6 12598583 0xc03d37 0 7 12463680 0xbe2e40 0 1 12644968 0xc0f268 0 3 12493269 0xbea1d5 0 2 12573327 0xbfda8f 0 4 12515064 0xbef6f8 0 0 12426990 0xbd9eee 0 /esfs/jtacquaviva/ioperf/file_write.00000007 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12644993 0xc0f281 0 3 12493294 0xbea1ee 0 0 12427012 0xbd9f04 0 2 12573349 0xbfdaa5 0 4 12515085 0xbef70d 0 5 12385356 0xbcfc4c 0 6 12598609 0xc03d51 0 7 12463707 0xbe2e5b 0 /esfs/jtacquaviva/ioperf/file_write.00000018 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12515069 0xbef6fd 0 5 12385342 0xbcfc3e 0 2 12573334 0xbfda96 0 3 12493280 0xbea1e0 0 7 12463693 0xbe2e4d 0 6 12598594 0xc03d42 0 0 12426999 0xbd9ef7 0 1 12644980 0xc0f274 0 /esfs/jtacquaviva/ioperf/file_write.00000031 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12644995 0xc0f283 0 3 12493296 0xbea1f0 0 5 12385358 0xbcfc4e 0 6 12598610 0xc03d52 0 0 12427014 0xbd9f06 0 2 12573351 0xbfdaa7 0 4 12515087 0xbef70f 0 7 12463708 0xbe2e5c 0 /esfs/jtacquaviva/ioperf/file_write.00000012 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12598590 0xc03d3e 0 4 12515067 0xbef6fb 0 5 12385339 0xbcfc3b 0 1 12644975 0xc0f26f 0 7 12463688 0xbe2e48 0 0 12426994 0xbd9ef2 0 3 12493277 0xbea1dd 0 2 12573331 0xbfda93 0 /esfs/jtacquaviva/ioperf/file_write.00000013 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 4 obdidx objid objid group 4 12515082 0xbef70a 0 7 12463705 0xbe2e59 0 1 12644992 0xc0f280 0 3 12493292 0xbea1ec 0 5 12385354 0xbcfc4a 0 6 12598606 0xc03d4e 0 0 12427010 0xbd9f02 0 2 12573347 0xbfdaa3 0 /esfs/jtacquaviva/ioperf/file_write.00000025 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12493285 0xbea1e5 0 5 12385348 0xbcfc44 0 6 12598602 0xc03d4a 0 0 12427005 0xbd9efd 0 2 12573343 0xbfda9f 0 4 12515079 0xbef707 0 7 12463700 0xbe2e54 0 1 12644987 0xc0f27b 0 /esfs/jtacquaviva/ioperf/file_write.00000022 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12463694 0xbe2e4e 0 5 12385344 0xbcfc40 0 1 12644982 0xc0f276 0 6 12598597 0xc03d45 0 3 12493286 0xbea1e6 0 0 12427002 0xbd9efa 0 2 12573340 0xbfda9c 0 4 12515077 0xbef705 0 /esfs/jtacquaviva/ioperf/file_write.00000003 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12573332 0xbfda94 0 1 12644977 0xc0f271 0 4 12515070 0xbef6fe 0 0 12426997 0xbd9ef5 0 3 12493279 0xbea1df 0 6 12598592 0xc03d40 0 7 12463692 0xbe2e4c 0 /esfs/jtacquaviva/ioperf/file_write.00000008 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12573345 0xbfdaa1 0 4 12515081 0xbef709 0 7 12463703 0xbe2e57 0 1 12644989 0xc0f27d 0 3 12493291 0xbea1eb 0 5 12385353 0xbcfc49 0 6 12598604 0xc03d4c 0 0 12427009 0xbd9f01 0 /esfs/jtacquaviva/ioperf/file_write.00000028 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12385331 0xbcfc33 0 2 12573322 0xbfda8a 0 7 12463679 0xbe2e3f 0 4 12515060 0xbef6f4 0 3 12493268 0xbea1d4 0 0 12426989 0xbd9eed 0 6 12598586 0xc03d3a 0 1 12644971 0xc0f26b 0 /esfs/jtacquaviva/ioperf/file_write.00000010 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12427000 0xbd9ef8 0 3 12493284 0xbea1e4 0 5 12385347 0xbcfc43 0 6 12598600 0xc03d48 0 2 12573341 0xbfda9d 0 4 12515076 0xbef704 0 7 12463698 0xbe2e52 0 1 12644985 0xc0f279 0 /esfs/jtacquaviva/ioperf/file_write.00000017 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12573337 0xbfda99 0 1 12644981 0xc0f275 0 3 12493282 0xbea1e2 0 6 12598596 0xc03d44 0 5 12385345 0xbcfc41 0 0 12427001 0xbd9ef9 0 4 12515075 0xbef703 0 7 12463697 0xbe2e51 0 /esfs/jtacquaviva/ioperf/file_write.00000029 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12385360 0xbcfc50 0 6 12598612 0xc03d54 0 0 12427016 0xbd9f08 0 2 12573353 0xbfdaa9 0 4 12515089 0xbef711 0 7 12463710 0xbe2e5e 0 1 12644997 0xc0f285 0 3 12493298 0xbea1f2 0 /esfs/jtacquaviva/ioperf/file_write.00000016 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12598584 0xc03d38 0 7 12463681 0xbe2e41 0 0 12426988 0xbd9eec 0 2 12573326 0xbfda8e 0 3 12493273 0xbea1d9 0 5 12385336 0xbcfc38 0 4 12515065 0xbef6f9 0 /esfs/jtacquaviva/ioperf/file_write.00000005 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12385346 0xbcfc42 0 6 12598599 0xc03d47 0 0 12427003 0xbd9efb 0 2 12573342 0xbfda9e 0 4 12515078 0xbef706 0 7 12463699 0xbe2e53 0 1 12644986 0xc0f27a 0 3 12493288 0xbea1e8 0 /esfs/jtacquaviva/ioperf/file_write.00000014 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12644967 0xc0f267 0 6 12598582 0xc03d36 0 0 12426985 0xbd9ee9 0 5 12385333 0xbcfc35 0 4 12515062 0xbef6f6 0 2 12573325 0xbfda8d 0 3 12493271 0xbea1d7 0 7 12463683 0xbe2e43 0 /esfs/jtacquaviva/ioperf/file_write.00000021 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 5 obdidx objid objid group 5 12385351 0xbcfc47 0 2 12573344 0xbfdaa0 0 4 12515080 0xbef708 0 1 12644988 0xc0f27c 0 6 12598603 0xc03d4b 0 7 12463702 0xbe2e56 0 3 12493290 0xbea1ea 0 0 12427008 0xbd9f00 0 /esfs/jtacquaviva/ioperf/file_write.00000019 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 0 obdidx objid objid group 0 12427007 0xbd9eff 0 7 12463701 0xbe2e55 0 3 12493289 0xbea1e9 0 5 12385352 0xbcfc48 0 2 12573346 0xbfdaa2 0 1 12644990 0xc0f27e 0 4 12515083 0xbef70b 0 6 12598605 0xc03d4d 0 /esfs/jtacquaviva/ioperf/file_write.00000009 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 7 obdidx objid objid group 7 12463690 0xbe2e4a 0 5 12385341 0xbcfc3d 0 4 12515071 0xbef6ff 0 2 12573335 0xbfda97 0 0 12426998 0xbd9ef6 0 1 12644979 0xc0f273 0 3 12493281 0xbea1e1 0 6 12598595 0xc03d43 0 /esfs/jtacquaviva/ioperf/file_write.00000015 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12573338 0xbfda9a 0 7 12463696 0xbe2e50 0 3 12493283 0xbea1e3 0 4 12515074 0xbef702 0 5 12385349 0xbcfc45 0 6 12598601 0xc03d49 0 0 12427004 0xbd9efc 0 1 12644984 0xc0f278 0 /esfs/jtacquaviva/ioperf/file_write.00000027 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 6 obdidx objid objid group 6 12598608 0xc03d50 0 7 12463706 0xbe2e5a 0 1 12644994 0xc0f282 0 0 12427013 0xbd9f05 0 2 12573350 0xbfdaa6 0 3 12493295 0xbea1ef 0 5 12385357 0xbcfc4d 0 4 12515086 0xbef70e 0 /esfs/jtacquaviva/ioperf/file_write.00000024 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 3 obdidx objid objid group 3 12493278 0xbea1de 0 0 12426996 0xbd9ef4 0 1 12644978 0xc0f272 0 7 12463691 0xbe2e4b 0 6 12598593 0xc03d41 0 4 12515072 0xbef700 0 5 12385343 0xbcfc3f 0 2 12573336 0xbfda98 0 /esfs/jtacquaviva/ioperf/file_write.00000001 lmm_stripe_count: 7 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 2 obdidx objid objid group 2 12573323 0xbfda8b 0 0 12426987 0xbd9eeb 0 4 12515063 0xbef6f7 0 3 12493270 0xbea1d6 0 1 12644970 0xc0f26a 0 7 12463685 0xbe2e45 0 6 12598587 0xc03d3b 0 /esfs/jtacquaviva/ioperf/file_write.00000011 lmm_stripe_count: 8 lmm_stripe_size: 1048576 lmm_pattern: 1 lmm_layout_gen: 0 lmm_stripe_offset: 1 obdidx objid objid group 1 12644973 0xc0f26d 0 3 12493274 0xbea1da 0 5 12385337 0xbcfc39 0 6 12598588 0xc03d3c 0 0 12426992 0xbd9ef0 0 2 12573329 0xbfda91 0 4 12515066 0xbef6fa 0 7 12463686 0xbe2e46 0