+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22,isc17-c01,isc17-c02,isc17-c03 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w
+ tee -a ./output/COUNT:1#NN:16#PPN:1#API:POSIX#T:102400.txt
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 00:07:38 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 00:07:38 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/ioperf/file_write
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 16 (1 per node)
	repetitions        = 3
	xfersize           = 102400 bytes
	blocksize          = 150 GiB
	aggregate filesize = 2400 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 2576980377600.
WARNING: Stat() of aggregate file size      = 2576906240000.
WARNING: Using actual aggregate bytes moved = 113326899200.
write     921.17     157286400  100.00     0.000784   117.32     0.000425   117.33     0   
WARNING: Expected aggregate file size       = 2576980377600.
WARNING: Stat() of aggregate file size      = 2576955801600.
WARNING: Using actual aggregate bytes moved = 98036326400.
write     776.57     157286400  100.00     0.014977   120.38     0.000757   120.39     1   
WARNING: Expected aggregate file size       = 2576980377600.
WARNING: Stat() of aggregate file size      = 2576922009600.
WARNING: Using actual aggregate bytes moved = 104338432000.
write     779.62     157286400  100.00     0.001920   127.63     0.000862   127.63     2   

Max Write: 921.17 MiB/sec (965.92 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
write         921.17     776.57     825.79      67.46  121.78398 0 16 1 3 1 0 1 0 0 1 161061273600 102400 113326899200 POSIX 0

Finished: Wed Oct 24 00:13:46 2018
+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22,isc17-c01,isc17-c02,isc17-c03 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh
+ tee -a ./output/COUNT:1#NN:16#PPN:1#API:POSIX#T:102400.txt
+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22,isc17-c01,isc17-c02,isc17-c03 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 00:13:51 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 00:13:51 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/indread16/file
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 16 (1 per node)
	repetitions        = 3
	xfersize           = 102400 bytes
	blocksize          = 150 GiB
	aggregate filesize = 2400 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 2576980377600.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 10354176000.
read      164.23     157286400  100.00     0.000855   60.12      0.000512   60.12      0   
WARNING: Expected aggregate file size       = 2576980377600.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 10616524800.
read      168.57     157286400  100.00     0.000525   60.06      0.000531   60.06      1   
WARNING: Expected aggregate file size       = 2576980377600.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 10593689600.
read      168.21     157286400  100.00     0.000705   60.06      0.000526   60.06      2   

Max Read:  168.57 MiB/sec (176.76 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read          168.57     164.23     167.00       1.96   60.08273 0 16 1 3 1 0 1 0 0 1 161061273600 102400 10354176000 POSIX 0

Finished: Wed Oct 24 00:16:51 2018
+ set +x
/esfs/jtacquaviva/ioperf
stripe_count:  32 stripe_size:   1048576 stripe_offset: -1
/esfs/jtacquaviva/ioperf/file_write.00000006
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12572838	     0xbfd8a6	             0
	     3	      12492781	     0xbe9fed	             0
	     6	      12598098	     0xc03b52	             0
	     4	      12514577	     0xbef511	             0
	     5	      12384841	     0xbcfa49	             0
	     0	      12426504	     0xbd9d08	             0
	     7	      12463203	     0xbe2c63	             0
	     1	      12644489	     0xc0f089	             0

/esfs/jtacquaviva/ioperf/file_write.00000002
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12492776	     0xbe9fe8	             0
	     1	      12644480	     0xc0f080	             0
	     5	      12384836	     0xbcfa44	             0
	     2	      12572834	     0xbfd8a2	             0
	     6	      12598094	     0xc03b4e	             0
	     0	      12426498	     0xbd9d02	             0
	     4	      12514574	     0xbef50e	             0
	     7	      12463200	     0xbe2c60	             0

/esfs/jtacquaviva/ioperf/file_write.00000000
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12492783	     0xbe9fef	             0
	     2	      12572842	     0xbfd8aa	             0
	     1	      12644492	     0xc0f08c	             0
	     6	      12598104	     0xc03b58	             0
	     5	      12384847	     0xbcfa4f	             0
	     4	      12514586	     0xbef51a	             0
	     7	      12463208	     0xbe2c68	             0
	     0	      12426509	     0xbd9d0d	             0

/esfs/jtacquaviva/ioperf/file_write.00000004
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12644487	     0xc0f087	             0
	     4	      12514580	     0xbef514	             0
	     5	      12384849	     0xbcfa51	             0
	     2	      12572846	     0xbfd8ae	             0
	     6	      12598106	     0xc03b5a	             0
	     0	      12426510	     0xbd9d0e	             0
	     3	      12492790	     0xbe9ff6	             0
	     7	      12463210	     0xbe2c6a	             0

/esfs/jtacquaviva/ioperf/file_write.00000007
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12463197	     0xbe2c5d	             0
	     3	      12492777	     0xbe9fe9	             0
	     4	      12514575	     0xbef50f	             0
	     1	      12644482	     0xc0f082	             0
	     0	      12426500	     0xbd9d04	             0
	     2	      12572836	     0xbfd8a4	             0
	     5	      12384839	     0xbcfa47	             0
	     6	      12598096	     0xc03b50	             0

/esfs/jtacquaviva/ioperf/file_write.00000012
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12426502	     0xbd9d06	             0
	     4	      12514578	     0xbef512	             0
	     3	      12492782	     0xbe9fee	             0
	     5	      12384842	     0xbcfa4a	             0
	     6	      12598101	     0xc03b55	             0
	     7	      12463205	     0xbe2c65	             0
	     1	      12644491	     0xc0f08b	             0
	     2	      12572844	     0xbfd8ac	             0

/esfs/jtacquaviva/ioperf/file_write.00000013
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12572837	     0xbfd8a5	             0
	     4	      12514576	     0xbef510	             0
	     3	      12492780	     0xbe9fec	             0
	     5	      12384840	     0xbcfa48	             0
	     1	      12644484	     0xc0f084	             0
	     7	      12463201	     0xbe2c61	             0
	     0	      12426501	     0xbd9d05	             0
	     6	      12598097	     0xc03b51	             0

/esfs/jtacquaviva/ioperf/file_write.00000003
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12644485	     0xc0f085	             0
	     0	      12426503	     0xbd9d07	             0
	     4	      12514579	     0xbef513	             0
	     7	      12463202	     0xbe2c62	             0
	     2	      12572840	     0xbfd8a8	             0
	     6	      12598102	     0xc03b56	             0
	     3	      12492785	     0xbe9ff1	             0
	     5	      12384844	     0xbcfa4c	             0

/esfs/jtacquaviva/ioperf/file_write.00000008
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12644479	     0xc0f07f	             0
	     3	      12492775	     0xbe9fe7	             0
	     5	      12384835	     0xbcfa43	             0
	     6	      12598092	     0xc03b4c	             0
	     0	      12426496	     0xbd9d00	             0
	     2	      12572832	     0xbfd8a0	             0
	     4	      12514571	     0xbef50b	             0
	     7	      12463196	     0xbe2c5c	             0

/esfs/jtacquaviva/ioperf/file_write.00000010
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12598099	     0xc03b53	             0
	     2	      12572839	     0xbfd8a7	             0
	     5	      12384843	     0xbcfa4b	             0
	     7	      12463204	     0xbe2c64	             0
	     1	      12644490	     0xc0f08a	             0
	     4	      12514581	     0xbef515	             0
	     3	      12492788	     0xbe9ff4	             0
	     0	      12426507	     0xbd9d0b	             0

/esfs/jtacquaviva/ioperf/file_write.00000005
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12572833	     0xbfd8a1	             0
	     4	      12514572	     0xbef50c	             0
	     0	      12426499	     0xbd9d03	             0
	     7	      12463199	     0xbe2c5f	             0
	     1	      12644483	     0xc0f083	             0
	     5	      12384838	     0xbcfa46	             0
	     6	      12598095	     0xc03b4f	             0
	     3	      12492779	     0xbe9feb	             0

/esfs/jtacquaviva/ioperf/file_write.00000014
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12644486	     0xc0f086	             0
	     7	      12463206	     0xbe2c66	             0
	     3	      12492786	     0xbe9ff2	             0
	     6	      12598105	     0xc03b59	             0
	     4	      12514585	     0xbef519	             0
	     5	      12384848	     0xbcfa50	             0
	     0	      12426508	     0xbd9d0c	             0
	     2	      12572847	     0xbfd8af	             0

/esfs/jtacquaviva/ioperf/file_write.00000009
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12598100	     0xc03b54	             0
	     2	      12572841	     0xbfd8a9	             0
	     7	      12463207	     0xbe2c67	             0
	     3	      12492787	     0xbe9ff3	             0
	     1	      12644493	     0xc0f08d	             0
	     5	      12384846	     0xbcfa4e	             0
	     4	      12514582	     0xbef516	             0
	     0	      12426506	     0xbd9d0a	             0

/esfs/jtacquaviva/ioperf/file_write.00000015
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12598093	     0xc03b4d	             0
	     0	      12426497	     0xbd9d01	             0
	     7	      12463198	     0xbe2c5e	             0
	     2	      12572835	     0xbfd8a3	             0
	     4	      12514573	     0xbef50d	             0
	     1	      12644481	     0xc0f081	             0
	     3	      12492778	     0xbe9fea	             0
	     5	      12384837	     0xbcfa45	             0

/esfs/jtacquaviva/ioperf/file_write.00000001
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12572845	     0xbfd8ad	             0
	     4	      12514584	     0xbef518	             0
	     7	      12463209	     0xbe2c69	             0
	     1	      12644494	     0xc0f08e	             0
	     3	      12492789	     0xbe9ff5	             0
	     5	      12384850	     0xbcfa52	             0
	     6	      12598107	     0xc03b5b	             0
	     0	      12426511	     0xbd9d0f	             0

/esfs/jtacquaviva/ioperf/file_write.00000011
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12644488	     0xc0f088	             0
	     3	      12492784	     0xbe9ff0	             0
	     6	      12598103	     0xc03b57	             0
	     5	      12384845	     0xbcfa4d	             0
	     2	      12572843	     0xbfd8ab	             0
	     0	      12426505	     0xbd9d09	             0
	     4	      12514583	     0xbef517	             0

