+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh
+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c04,isc17-c05,isc17-c06,isc17-c07 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 16384 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r
+ tee -a ./output/COUNT:1#NN:4#PPN:4#API:POSIX#T:16384.txt
IOR-3.0.1: MPI Coordinated Test of Parallel I/O

Began: Tue Oct 23 18:45:04 2018
Command line used: /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 16384 -b 40265318400 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread4/file -r
Machine: Linux isc17-c04

Test 0 started: Tue Oct 23 18:45:04 2018
Summary:
	api                = POSIX
	test filename      = /esfs/jtacquaviva/indread4/file
	access             = file-per-process
	ordering in a file = random offsets
	ordering inter file= no tasks offsets
	clients            = 16 (4 per node)
	repetitions        = 3
	xfersize           = 16384 bytes
	blocksize          = 37.50 GiB
	aggregate filesize = 600 GiB
	Using stonewalling = 120 second(s)

access    bw(MiB/s)  block(KiB) xfer(KiB)  open(s)    wr/rd(s)   close(s)   total(s)   iter
------    ---------  ---------- ---------  --------   --------   --------   --------   ----
WARNING: Expected aggregate file size       = 644245094400.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 5504909312.
read      43.69      39321600   16.00      0.003983   120.15     0.003403   120.15     0   
WARNING: Expected aggregate file size       = 644245094400.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 5849333760.
read      46.43      39321600   16.00      0.003422   120.15     0.003406   120.15     1   
WARNING: Expected aggregate file size       = 644245094400.
WARNING: Stat() of aggregate file size      = 2576980377600.
WARNING: Using actual aggregate bytes moved = 5870469120.
read      46.60      39321600   16.00      0.003577   120.15     0.003422   120.15     2   

Max Read:  46.60 MiB/sec (48.86 MB/sec)

Summary of all tests:
Operation   Max(MiB)   Min(MiB)  Mean(MiB)     StdDev    Mean(s) Test# #Tasks tPN reps fPP reord reordoff reordrand seed segcnt blksiz xsize aggsize API RefNum
read           46.60      43.69      45.57       1.33  120.15097 0 16 4 3 1 0 1 0 0 1 40265318400 16384 5504909312 POSIX 0

Finished: Tue Oct 23 18:51:05 2018
+ set +x
/esfs/jtacquaviva/ioperf
stripe_count:  8 stripe_size:   1048576 stripe_offset: -1
/esfs/jtacquaviva/ioperf/file_write.00000006
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 4
	obdidx		 objid		 objid		 group
	     4	      12512094	     0xbeeb5e	             0
	     7	      12460725	     0xbe22b5	             0
	     1	      12642002	     0xc0e6d2	             0
	     3	      12490303	     0xbe963f	             0
	     5	      12382369	     0xbcf0a1	             0
	     6	      12595623	     0xc031a7	             0
	     0	      12424013	     0xbd934d	             0
	     2	      12570355	     0xbfcef3	             0

/esfs/jtacquaviva/ioperf/file_write.00000002
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12642009	     0xc0e6d9	             0
	     3	      12490310	     0xbe9646	             0
	     5	      12382376	     0xbcf0a8	             0
	     6	      12595630	     0xc031ae	             0
	     0	      12424020	     0xbd9354	             0
	     2	      12570362	     0xbfcefa	             0
	     4	      12512101	     0xbeeb65	             0
	     7	      12460732	     0xbe22bc	             0

/esfs/jtacquaviva/ioperf/file_write.00000000
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 3
	obdidx		 objid		 objid		 group
	     3	      12490311	     0xbe9647	             0
	     5	      12382377	     0xbcf0a9	             0
	     6	      12595631	     0xc031af	             0
	     0	      12424021	     0xbd9355	             0
	     2	      12570363	     0xbfcefb	             0
	     4	      12512102	     0xbeeb66	             0
	     7	      12460733	     0xbe22bd	             0
	     1	      12642010	     0xc0e6da	             0

/esfs/jtacquaviva/ioperf/file_write.00000004
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12570359	     0xbfcef7	             0
	     4	      12512098	     0xbeeb62	             0
	     7	      12460729	     0xbe22b9	             0
	     1	      12642006	     0xc0e6d6	             0
	     3	      12490307	     0xbe9643	             0
	     5	      12382373	     0xbcf0a5	             0
	     6	      12595627	     0xc031ab	             0
	     0	      12424017	     0xbd9351	             0

/esfs/jtacquaviva/ioperf/file_write.00000007
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12382362	     0xbcf09a	             0
	     2	      12570348	     0xbfceec	             0
	     7	      12460718	     0xbe22ae	             0
	     1	      12641995	     0xc0e6cb	             0
	     6	      12595616	     0xc031a0	             0
	     4	      12512088	     0xbeeb58	             0
	     0	      12424007	     0xbd9347	             0
	     3	      12490298	     0xbe963a	             0

/esfs/jtacquaviva/ioperf/file_write.00000012
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12460726	     0xbe22b6	             0
	     1	      12642003	     0xc0e6d3	             0
	     3	      12490304	     0xbe9640	             0
	     0	      12424014	     0xbd934e	             0
	     2	      12570356	     0xbfcef4	             0
	     4	      12512095	     0xbeeb5f	             0
	     5	      12382370	     0xbcf0a2	             0
	     6	      12595625	     0xc031a9	             0

/esfs/jtacquaviva/ioperf/file_write.00000013
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 7
	obdidx		 objid		 objid		 group
	     7	      12460731	     0xbe22bb	             0
	     1	      12642008	     0xc0e6d8	             0
	     3	      12490309	     0xbe9645	             0
	     5	      12382375	     0xbcf0a7	             0
	     6	      12595629	     0xc031ad	             0
	     0	      12424019	     0xbd9353	             0
	     2	      12570361	     0xbfcef9	             0
	     4	      12512100	     0xbeeb64	             0

/esfs/jtacquaviva/ioperf/file_write.00000003
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12424016	     0xbd9350	             0
	     2	      12570358	     0xbfcef6	             0
	     4	      12512097	     0xbeeb61	             0
	     7	      12460728	     0xbe22b8	             0
	     1	      12642005	     0xc0e6d5	             0
	     3	      12490306	     0xbe9642	             0
	     5	      12382372	     0xbcf0a4	             0
	     6	      12595626	     0xc031aa	             0

/esfs/jtacquaviva/ioperf/file_write.00000008
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 0
	obdidx		 objid		 objid		 group
	     0	      12424006	     0xbd9346	             0
	     4	      12512087	     0xbeeb57	             0
	     3	      12490297	     0xbe9639	             0
	     2	      12570349	     0xbfceed	             0
	     7	      12460719	     0xbe22af	             0
	     1	      12641996	     0xc0e6cc	             0
	     5	      12382363	     0xbcf09b	             0
	     6	      12595617	     0xc031a1	             0

/esfs/jtacquaviva/ioperf/file_write.00000010
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 6
	obdidx		 objid		 objid		 group
	     6	      12595624	     0xc031a8	             0
	     7	      12460727	     0xbe22b7	             0
	     0	      12424015	     0xbd934f	             0
	     1	      12642004	     0xc0e6d4	             0
	     2	      12570357	     0xbfcef5	             0
	     4	      12512096	     0xbeeb60	             0
	     3	      12490305	     0xbe9641	             0
	     5	      12382371	     0xbcf0a3	             0

/esfs/jtacquaviva/ioperf/file_write.00000005
lmm_stripe_count:  7
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12382365	     0xbcf09d	             0
	     4	      12512091	     0xbeeb5b	             0
	     0	      12424010	     0xbd934a	             0
	     6	      12595620	     0xc031a4	             0
	     2	      12570353	     0xbfcef1	             0
	     1	      12642000	     0xc0e6d0	             0
	     7	      12460723	     0xbe22b3	             0

/esfs/jtacquaviva/ioperf/file_write.00000014
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 2
	obdidx		 objid		 objid		 group
	     2	      12570350	     0xbfceee	             0
	     4	      12512089	     0xbeeb59	             0
	     7	      12460720	     0xbe22b0	             0
	     1	      12641997	     0xc0e6cd	             0
	     3	      12490299	     0xbe963b	             0
	     5	      12382364	     0xbcf09c	             0
	     6	      12595618	     0xc031a2	             0
	     0	      12424008	     0xbd9348	             0

/esfs/jtacquaviva/ioperf/file_write.00000009
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 1
	obdidx		 objid		 objid		 group
	     1	      12641999	     0xc0e6cf	             0
	     4	      12512092	     0xbeeb5c	             0
	     5	      12382368	     0xbcf0a0	             0
	     3	      12490301	     0xbe963d	             0
	     0	      12424011	     0xbd934b	             0
	     2	      12570354	     0xbfcef2	             0
	     7	      12460724	     0xbe22b4	             0
	     6	      12595622	     0xc031a6	             0

/esfs/jtacquaviva/ioperf/file_write.00000015
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 4
	obdidx		 objid		 objid		 group
	     4	      12512090	     0xbeeb5a	             0
	     7	      12460721	     0xbe22b1	             0
	     1	      12641998	     0xc0e6ce	             0
	     6	      12595619	     0xc031a3	             0
	     0	      12424009	     0xbd9349	             0
	     2	      12570351	     0xbfceef	             0
	     3	      12490300	     0xbe963c	             0
	     5	      12382367	     0xbcf09f	             0

/esfs/jtacquaviva/ioperf/file_write.00000001
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 5
	obdidx		 objid		 objid		 group
	     5	      12382366	     0xbcf09e	             0
	     2	      12570352	     0xbfcef0	             0
	     7	      12460722	     0xbe22b2	             0
	     3	      12490302	     0xbe963e	             0
	     1	      12642001	     0xc0e6d1	             0
	     6	      12595621	     0xc031a5	             0
	     4	      12512093	     0xbeeb5d	             0
	     0	      12424012	     0xbd934c	             0

/esfs/jtacquaviva/ioperf/file_write.00000011
lmm_stripe_count:  8
lmm_stripe_size:   1048576
lmm_pattern:       1
lmm_layout_gen:    0
lmm_stripe_offset: 4
	obdidx		 objid		 objid		 group
	     4	      12512099	     0xbeeb63	             0
	     7	      12460730	     0xbe22ba	             0
	     1	      12642007	     0xc0e6d7	             0
	     3	      12490308	     0xbe9644	             0
	     5	      12382374	     0xbcf0a6	             0
	     6	      12595628	     0xc031ac	             0
	     0	      12424018	     0xbd9352	             0
	     2	      12570360	     0xbfcef8	             0

