+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh
+ tee -a ./output/COUNT:1#NN:8#PPN:2#API:POSIX#T:16384.txt
+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 16384 -b 80530636800 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 07:32:00 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 16384 -b 80530636800 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 07:32:00 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/indread8/file
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 16 (2 per node)
	repetitions        = 3
	xfersize           = 16384 bytes
	blocksize          = 75 GiB
	aggregate filesize = 1200 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 1782382592.
read      28.19      78643200   16.00      0.003095   60.29      0.002781   60.29      0   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 2341502976.
read      37.04      78643200   16.00      0.002794   60.29      0.002829   60.29      1   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 2328739840.
read      36.84      78643200   16.00      0.002892   60.28      0.002882   60.28      2   

Max Read:  37.04 MiB/sec (38.84 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read           37.04      28.19      34.03       4.12   60.28596 0 16 2 3 1 0 1 0 0 1 80530636800 16384 1782382592 POSIX 0

Finished: Wed Oct 24 07:35:01 2018
+ set +x
/esfs/jtacquaviva/ioperf
stripe_count:  16 stripe_size:   1048576 stripe_offset: -1
/esfs/jtacquaviva/ioperf/file_write.00000006
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12646748	     0xc0f95c	             0
	     4	      12516833	     0xbefde1	             0
	     3	      12495053	     0xbea8cd	             0
	     6	      12600358	     0xc04426	             0
	     0	      12428764	     0xbda5dc	             0
	     2	      12575110	     0xbfe186	             0
	     7	      12465472	     0xbe3540	             0
	     5	      12387116	     0xbd032c	             0

/esfs/jtacquaviva/ioperf/file_write.00000002
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12387119	     0xbd032f	             0
	     6	      12600364	     0xc0442c	             0
	     4	      12516841	     0xbefde9	             0
	     7	      12465476	     0xbe3544	             0
	     1	      12646756	     0xc0f964	             0
	     0	      12428768	     0xbda5e0	             0
	     2	      12575115	     0xbfe18b	             0
	     3	      12495062	     0xbea8d6	             0

/esfs/jtacquaviva/ioperf/file_write.00000000
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12646749	     0xc0f95d	             0
	     0	      12428762	     0xbda5da	             0
	     4	      12516836	     0xbefde4	             0
	     7	      12465471	     0xbe353f	             0
	     5	      12387115	     0xbd032b	             0
	     3	      12495057	     0xbea8d1	             0
	     6	      12600361	     0xc04429	             0

/esfs/jtacquaviva/ioperf/file_write.00000004
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12428767	     0xbda5df	             0
	     1	      12646757	     0xc0f965	             0
	     3	      12495061	     0xbea8d5	             0
	     4	      12516842	     0xbefdea	             0
	     5	      12387121	     0xbd0331	             0
	     6	      12600366	     0xc0442e	             0
	     2	      12575116	     0xbfe18c	             0
	     7	      12465478	     0xbe3546	             0

/esfs/jtacquaviva/ioperf/file_write.00000007
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12600369	     0xc04431	             0
	     0	      12428772	     0xbda5e4	             0
	     2	      12575119	     0xbfe18f	             0
	     4	      12516846	     0xbefdee	             0
	     7	      12465481	     0xbe3549	             0
	     1	      12646761	     0xc0f969	             0
	     3	      12495065	     0xbea8d9	             0
	     5	      12387124	     0xbd0334	             0

/esfs/jtacquaviva/ioperf/file_write.00000012
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12575105	     0xbfe181	             0
	     1	      12646747	     0xc0f95b	             0
	     5	      12387110	     0xbd0326	             0
	     3	      12495051	     0xbea8cb	             0
	     4	      12516831	     0xbefddf	             0
	     6	      12600354	     0xc04422	             0
	     7	      12465466	     0xbe353a	             0
	     0	      12428758	     0xbda5d6	             0

/esfs/jtacquaviva/ioperf/file_write.00000013
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12495058	     0xbea8d2	             0
	     0	      12428766	     0xbda5de	             0
	     7	      12465475	     0xbe3543	             0
	     2	      12575113	     0xbfe189	             0
	     5	      12387118	     0xbd032e	             0
	     6	      12600363	     0xc0442b	             0
	     4	      12516840	     0xbefde8	             0
	     1	      12646755	     0xc0f963	             0

/esfs/jtacquaviva/ioperf/file_write.00000003
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12600357	     0xc04425	             0
	     7	      12465469	     0xbe353d	             0
	     1	      12646751	     0xc0f95f	             0
	     5	      12387114	     0xbd032a	             0
	     2	      12575109	     0xbfe185	             0
	     4	      12516837	     0xbefde5	             0
	     0	      12428765	     0xbda5dd	             0

/esfs/jtacquaviva/ioperf/file_write.00000008
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12575112	     0xbfe188	             0
	     4	      12516839	     0xbefde7	             0
	     7	      12465474	     0xbe3542	             0
	     5	      12387117	     0xbd032d	             0
	     6	      12600362	     0xc0442a	             0
	     1	      12646754	     0xc0f962	             0
	     3	      12495059	     0xbea8d3	             0

/esfs/jtacquaviva/ioperf/file_write.00000010
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12575114	     0xbfe18a	             0
	     3	      12495060	     0xbea8d4	             0
	     0	      12428769	     0xbda5e1	             0
	     6	      12600365	     0xc0442d	             0
	     5	      12387120	     0xbd0330	             0
	     4	      12516843	     0xbefdeb	             0
	     7	      12465477	     0xbe3545	             0
	     1	      12646758	     0xc0f966	             0

/esfs/jtacquaviva/ioperf/file_write.00000005
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 4
	obdidx		 objid		 objid		 group
	     4	      12516834	     0xbefde2	             0
	     5	      12387113	     0xbd0329	             0
	     6	      12600359	     0xc04427	             0
	     0	      12428763	     0xbda5db	             0
	     2	      12575108	     0xbfe184	             0
	     3	      12495056	     0xbea8d0	             0
	     1	      12646753	     0xc0f961	             0
	     7	      12465473	     0xbe3541	             0

/esfs/jtacquaviva/ioperf/file_write.00000014
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12428760	     0xbda5d8	             0
	     5	      12387112	     0xbd0328	             0
	     6	      12600356	     0xc04424	             0
	     1	      12646750	     0xc0f95e	             0
	     3	      12495054	     0xbea8ce	             0
	     7	      12465470	     0xbe353e	             0
	     2	      12575111	     0xbfe187	             0
	     4	      12516838	     0xbefde6	             0

/esfs/jtacquaviva/ioperf/file_write.00000009
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12465467	     0xbe353b	             0
	     3	      12495052	     0xbea8cc	             0
	     4	      12516832	     0xbefde0	             0
	     0	      12428759	     0xbda5d7	             0
	     5	      12387111	     0xbd0327	             0
	     2	      12575106	     0xbfe182	             0
	     6	      12600355	     0xc04423	             0

/esfs/jtacquaviva/ioperf/file_write.00000015
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12495063	     0xbea8d7	             0
	     5	      12387122	     0xbd0332	             0
	     6	      12600367	     0xc0442f	             0
	     0	      12428770	     0xbda5e2	             0
	     2	      12575117	     0xbfe18d	             0
	     4	      12516844	     0xbefdec	             0
	     7	      12465479	     0xbe3547	             0
	     1	      12646759	     0xc0f967	             0

/esfs/jtacquaviva/ioperf/file_write.00000001
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12387123	     0xbd0333	             0
	     6	      12600368	     0xc04430	             0
	     0	      12428771	     0xbda5e3	             0
	     2	      12575118	     0xbfe18e	             0
	     4	      12516845	     0xbefded	             0
	     7	      12465480	     0xbe3548	             0
	     1	      12646760	     0xc0f968	             0
	     3	      12495064	     0xbea8d8	             0

/esfs/jtacquaviva/ioperf/file_write.00000011
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12428761	     0xbda5d9	             0
	     2	      12575107	     0xbfe183	             0
	     7	      12465468	     0xbe353c	             0
	     4	      12516835	     0xbefde3	             0
	     6	      12600360	     0xc04428	             0
	     3	      12495055	     0xbea8cf	             0
	     1	      12646752	     0xc0f960	             0

