+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh
+ tee -a ./output/COUNT:1#NN:8#PPN:1#API:POSIX#T:10485760.txt
+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 8 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Wed Oct 24 03:30:04 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 10485760 -b 161061273600 -D 60 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread8/file -r
Machine: Linux isc17-c04

Test 0 started: Wed Oct 24 03:30:04 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/indread8/file
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 8 (1 per node)
	repetitions        = 3
	xfersize           = 10 MiB
	blocksize          = 150 GiB
	aggregate filesize = 1200 GiB
	Using stonewalling = 60 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 45329940480.
read      719.44     157286400  10240      0.000782   60.09      0.000533   60.09      0   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 46472888320.
read      737.10     157286400  10240      0.000535   60.13      0.000640   60.13      1   
WARNING: Expected aggregate file size       = 1288490188800.
WARNING: Stat() of aggregate file size      = 1288490188800.
WARNING: Using actual aggregate bytes moved = 49377443840.
read      783.26     157286400  10240      0.000899   60.12      0.000746   60.12      2   

Max Read:  783.26 MiB/sec (821.30 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read          783.26     719.44     746.60      26.90   60.11220 0 8 1 3 1 0 1 0 0 1 161061273600 10485760 45329940480 POSIX 0

Finished: Wed Oct 24 03:33:05 2018
+ set +x
/esfs/jtacquaviva/ioperf
stripe_count:  16 stripe_size:   1048576 stripe_offset: -1
/esfs/jtacquaviva/ioperf/file_write.00000006
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12598633	     0xc03d69	             0
	     0	      12427039	     0xbd9f1f	             0
	     2	      12573378	     0xbfdac2	             0
	     7	      12463736	     0xbe2e78	             0
	     4	      12515115	     0xbef72b	             0
	     5	      12385388	     0xbcfc6c	             0
	     1	      12645024	     0xc0f2a0	             0

/esfs/jtacquaviva/ioperf/file_write.00000002
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12463730	     0xbe2e72	             0
	     0	      12427035	     0xbd9f1b	             0
	     4	      12515109	     0xbef725	             0
	     3	      12493319	     0xbea207	             0
	     6	      12598632	     0xc03d68	             0
	     1	      12645018	     0xc0f29a	             0
	     5	      12385383	     0xbcfc67	             0
	     2	      12573374	     0xbfdabe	             0

/esfs/jtacquaviva/ioperf/file_write.00000000
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12573376	     0xbfdac0	             0
	     4	      12515112	     0xbef728	             0
	     1	      12645022	     0xc0f29e	             0
	     5	      12385387	     0xbcfc6b	             0
	     6	      12598638	     0xc03d6e	             0
	     3	      12493324	     0xbea20c	             0
	     0	      12427041	     0xbd9f21	             0

/esfs/jtacquaviva/ioperf/file_write.00000004
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12463734	     0xbe2e76	             0
	     1	      12645020	     0xc0f29c	             0
	     3	      12493321	     0xbea209	             0
	     4	      12515114	     0xbef72a	             0
	     5	      12385386	     0xbcfc6a	             0
	     6	      12598636	     0xc03d6c	             0
	     0	      12427040	     0xbd9f20	             0

/esfs/jtacquaviva/ioperf/file_write.00000007
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12493318	     0xbea206	             0
	     2	      12573372	     0xbfdabc	             0
	     1	      12645017	     0xc0f299	             0
	     0	      12427036	     0xbd9f1c	             0
	     4	      12515110	     0xbef726	             0
	     5	      12385384	     0xbcfc68	             0
	     7	      12463733	     0xbe2e75	             0
	     6	      12598634	     0xc03d6a	             0

/esfs/jtacquaviva/ioperf/file_write.00000003
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12493320	     0xbea208	             0
	     5	      12385382	     0xbcfc66	             0
	     7	      12463732	     0xbe2e74	             0
	     0	      12427037	     0xbd9f1d	             0
	     2	      12573375	     0xbfdabf	             0
	     6	      12598635	     0xc03d6b	             0
	     1	      12645021	     0xc0f29d	             0
	     4	      12515113	     0xbef729	             0

/esfs/jtacquaviva/ioperf/file_write.00000005
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12573377	     0xbfdac1	             0
	     5	      12385385	     0xbcfc69	             0
	     7	      12463735	     0xbe2e77	             0
	     3	      12493323	     0xbea20b	             0
	     6	      12598637	     0xc03d6d	             0
	     1	      12645023	     0xc0f29f	             0
	     4	      12515116	     0xbef72c	             0
	     0	      12427042	     0xbd9f22	             0

/esfs/jtacquaviva/ioperf/file_write.00000001
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12598631	     0xc03d67	             0
	     7	      12463731	     0xbe2e73	             0
	     5	      12385381	     0xbcfc65	             0
	     2	      12573373	     0xbfdabd	             0
	     4	      12515111	     0xbef727	             0
	     0	      12427038	     0xbd9f1e	             0
	     1	      12645019	     0xc0f29b	             0
	     3	      12493322	     0xbea20a	             0

