+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w
+ tee -a ./output/COUNT:1#NN:8#PPN:1#API:POSIX#T:102400.txt
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 05:41:43 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 05:41:43 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/ioperf/file_write
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 8 (1 per node)
	repetitions        = 3
	xfersize           = 102400 bytes
	blocksize          = 150 GiB
	aggregate filesize = 1200 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288465817600.
WARNING: Using actual aggregate bytes moved = 83866828800.
write     881.60     157286400  100.00     0.000598   90.72      0.000482   90.72      0   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288488550400.
WARNING: Using actual aggregate bytes moved = 80570163200.
write     808.36     157286400  100.00     0.000983   95.05      0.000788   95.05      1   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288469094400.
WARNING: Using actual aggregate bytes moved = 83156172800.
write     857.98     157286400  100.00     0.000992   92.43      0.000664   92.43      2   

Max Write: 881.60 MiB/sec (924.43 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
write         881.60     808.36     849.31      30.52   92.73608 0 8 1 3 1 0 1 0 0 1 161061273600 102400 83866828800 POSIX 0

Finished: Wed Oct 24 05:46:25 2018
+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh
+ tee -a ./output/COUNT:1#NN:8#PPN:1#API:POSIX#T:102400.txt
+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 05:46:36 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 05:46:36 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/indread8/file
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 8 (1 per node)
	repetitions        = 3
	xfersize           = 102400 bytes
	blocksize          = 150 GiB
	aggregate filesize = 1200 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 4965376000.
read      78.83      157286400  100.00     0.000876   60.07      0.000693   60.07      0   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 5366579200.
read      85.21      157286400  100.00     0.000662   60.06      0.000757   60.06      1   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 5337292800.
read      84.75      157286400  100.00     0.000726   60.06      0.000836   60.06      2   

Max Read:  85.21 MiB/sec (89.35 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read           85.21      78.83      82.93       2.90   60.06219 0 8 1 3 1 0 1 0 0 1 161061273600 102400 4965376000 POSIX 0

Finished: Wed Oct 24 05:49:36 2018
+ set +x
/esfs/jtacquaviva/ioperf
stripe_count:  16 stripe_size:   1048576 stripe_offset: -1
/esfs/jtacquaviva/ioperf/file_write.00000006
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12574408	     0xbfdec8	             0
	     4	      12516137	     0xbefb29	             0
	     7	      12464768	     0xbe3280	             0
	     1	      12646049	     0xc0f6a1	             0
	     3	      12494348	     0xbea60c	             0
	     5	      12386419	     0xbd0073	             0
	     6	      12599663	     0xc0416f	             0
	     0	      12428062	     0xbda31e	             0

/esfs/jtacquaviva/ioperf/file_write.00000002
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12574405	     0xbfdec5	             0
	     1	      12646046	     0xc0f69e	             0
	     7	      12464765	     0xbe327d	             0
	     5	      12386417	     0xbd0071	             0
	     4	      12516135	     0xbefb27	             0
	     3	      12494347	     0xbea60b	             0
	     0	      12428060	     0xbda31c	             0
	     6	      12599662	     0xc0416e	             0

/esfs/jtacquaviva/ioperf/file_write.00000000
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12428058	     0xbda31a	             0
	     7	      12464764	     0xbe327c	             0
	     2	      12574406	     0xbfdec6	             0
	     3	      12494345	     0xbea609	             0
	     4	      12516134	     0xbefb26	             0
	     1	      12646048	     0xc0f6a0	             0
	     5	      12386418	     0xbd0072	             0
	     6	      12599661	     0xc0416d	             0

/esfs/jtacquaviva/ioperf/file_write.00000004
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12464763	     0xbe327b	             0
	     3	      12494344	     0xbea608	             0
	     1	      12646047	     0xc0f69f	             0
	     5	      12386416	     0xbd0070	             0
	     6	      12599660	     0xc0416c	             0
	     4	      12516138	     0xbefb2a	             0
	     0	      12428061	     0xbda31d	             0

/esfs/jtacquaviva/ioperf/file_write.00000007
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12646043	     0xc0f69b	             0
	     6	      12599658	     0xc0416a	             0
	     0	      12428057	     0xbda319	             0
	     2	      12574404	     0xbfdec4	             0
	     3	      12494342	     0xbea606	             0
	     5	      12386415	     0xbd006f	             0
	     4	      12516133	     0xbefb25	             0
	     7	      12464766	     0xbe327e	             0

/esfs/jtacquaviva/ioperf/file_write.00000003
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12646042	     0xc0f69a	             0
	     6	      12599656	     0xc04168	             0
	     2	      12574402	     0xbfdec2	             0
	     4	      12516131	     0xbefb23	             0
	     3	      12494341	     0xbea605	             0
	     0	      12428056	     0xbda318	             0
	     5	      12386413	     0xbd006d	             0
	     7	      12464762	     0xbe327a	             0

/esfs/jtacquaviva/ioperf/file_write.00000005
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12386412	     0xbd006c	             0
	     0	      12428055	     0xbda317	             0
	     7	      12464761	     0xbe3279	             0
	     6	      12599657	     0xc04169	             0
	     2	      12574403	     0xbfdec3	             0
	     1	      12646044	     0xc0f69c	             0
	     3	      12494343	     0xbea607	             0
	     4	      12516132	     0xbefb24	             0

/esfs/jtacquaviva/ioperf/file_write.00000001
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12386414	     0xbd006e	             0
	     1	      12646045	     0xc0f69d	             0
	     6	      12599659	     0xc0416b	             0
	     2	      12574407	     0xbfdec7	             0
	     3	      12494346	     0xbea60a	             0
	     4	      12516136	     0xbefb28	             0
	     0	      12428059	     0xbda31b	             0
	     7	      12464767	     0xbe327f	             0

