+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh
+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
+ tee -a ./output/COUNT:1#NN:8#PPN:1#API:POSIX#T:1048576.txt
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 04:37:13 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 1048576 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 04:37:13 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/indread8/file
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 8 (1 per node)
	repetitions        = 3
	xfersize           = 1 MiB
	blocksize          = 150 GiB
	aggregate filesize = 1200 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 39415971840.
read      626.28     157286400  1024.00    0.000842   60.02      0.000558   60.02      0   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 40653291520.
read      645.99     157286400  1024.00    0.000575   60.02      0.000583   60.02      1   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 41420849152.
read      658.16     157286400  1024.00    0.000618   60.02      0.000540   60.02      2   

Max Read:  658.16 MiB/sec (690.14 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read          658.16     626.28     643.48      13.14   60.01856 0 8 1 3 1 0 1 0 0 1 161061273600 1048576 39415971840 POSIX 0

Finished: Wed Oct 24 04:40:13 2018
+ set +x
/esfs/jtacquaviva/ioperf
stripe_count:  16 stripe_size:   1048576 stripe_offset: -1
/esfs/jtacquaviva/ioperf/file_write.00000006
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12464243	     0xbe3073	             0
	     5	      12385897	     0xbcfe69	             0
	     1	      12645532	     0xc0f49c	             0
	     4	      12515621	     0xbef925	             0
	     2	      12573890	     0xbfdcc2	             0
	     0	      12427550	     0xbda11e	             0
	     6	      12599144	     0xc03f68	             0
	     3	      12493831	     0xbea407	             0

/esfs/jtacquaviva/ioperf/file_write.00000002
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12427551	     0xbda11f	             0
	     5	      12385900	     0xbcfe6c	             0
	     4	      12515623	     0xbef927	             0
	     2	      12573892	     0xbfdcc4	             0
	     1	      12645535	     0xc0f49f	             0
	     6	      12599147	     0xc03f6b	             0
	     3	      12493835	     0xbea40b	             0

/esfs/jtacquaviva/ioperf/file_write.00000000
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12599142	     0xc03f66	             0
	     1	      12645531	     0xc0f49b	             0
	     4	      12515620	     0xbef924	             0
	     2	      12573889	     0xbfdcc1	             0
	     0	      12427549	     0xbda11d	             0
	     5	      12385899	     0xbcfe6b	             0
	     7	      12464248	     0xbe3078	             0
	     3	      12493833	     0xbea409	             0

/esfs/jtacquaviva/ioperf/file_write.00000004
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12427552	     0xbda120	             0
	     3	      12493832	     0xbea408	             0
	     7	      12464249	     0xbe3079	             0
	     5	      12385903	     0xbcfe6f	             0
	     6	      12599146	     0xc03f6a	             0
	     2	      12573895	     0xbfdcc7	             0
	     1	      12645538	     0xc0f4a2	             0
	     4	      12515626	     0xbef92a	             0

/esfs/jtacquaviva/ioperf/file_write.00000007
lmm_stripe_count:  6
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12645533	     0xc0f49d	             0
	     7	      12464244	     0xbe3074	             0
	     3	      12493830	     0xbea406	             0
	     4	      12515622	     0xbef926	             0
	     5	      12385901	     0xbcfe6d	             0
	     2	      12573893	     0xbfdcc5	             0

/esfs/jtacquaviva/ioperf/file_write.00000003
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12573888	     0xbfdcc0	             0
	     3	      12493829	     0xbea405	             0
	     7	      12464245	     0xbe3075	             0
	     5	      12385902	     0xbcfe6e	             0
	     6	      12599145	     0xc03f69	             0
	     1	      12645537	     0xc0f4a1	             0
	     4	      12515625	     0xbef929	             0
	     0	      12427554	     0xbda122	             0

/esfs/jtacquaviva/ioperf/file_write.00000005
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12464246	     0xbe3076	             0
	     2	      12573891	     0xbfdcc3	             0
	     0	      12427553	     0xbda121	             0
	     1	      12645536	     0xc0f4a0	             0
	     6	      12599148	     0xc03f6c	             0
	     5	      12385904	     0xbcfe70	             0
	     3	      12493836	     0xbea40c	             0

/esfs/jtacquaviva/ioperf/file_write.00000001
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12427548	     0xbda11c	             0
	     5	      12385898	     0xbcfe6a	             0
	     1	      12645534	     0xc0f49e	             0
	     6	      12599143	     0xc03f67	             0
	     7	      12464247	     0xbe3077	             0
	     3	      12493834	     0xbea40a	             0
	     4	      12515624	     0xbef928	             0
	     2	      12573894	     0xbfdcc6	             0

