+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 4 --hosts isc17-c02,isc17-c03,isc17-c04,isc17-c05 -wdir /gsfs/jtacquaviva/git/ddn-ime-evaluation/benchmark -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter -genv MV2_SHOW_CPU_BINDING 1 -genv RDMAV_FORK_SAFE 1 -genv RDMAV_HUGEPAGES_SAFE 1 ./ior_wrapper.sh '-i 3 -s 1 -t 16384 -b 805306368000 -D 60 -O stoneWallingWearOut=1 -a IME -g -H -z -o ime:///gsfs/jtacquaviva/file_write -w -O summaryFile=./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt.json -O summaryFormat=JSON ' ./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt_network
+ tee -a ./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt
Writing output to ./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:1-API:IME-T:16384-TYPE:write.txt.json
-------------CPU AFFINITY-------------
RANK:0  CPU_SET:   0
-------------------------------------
[0002] 2018-12-06 22:52:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 22:52:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:52:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 22:52:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:52:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 22:52:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:52:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 22:52:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:55:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0003] 2018-12-06 22:55:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:55:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0000] 2018-12-06 22:55:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:55:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0001] 2018-12-06 22:55:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:55:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (240 sec)
[0002] 2018-12-06 22:55:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 1, partial write, 0 of 16384 bytes at offset 2395021623296
WARNING: Task 3, partial write, 0 of 16384 bytes at offset 2749437870080
WARNING: Task 0, partial write, 0 of 16384 bytes at offset 2362155876352
WARNING: Task 2, partial write, 0 of 16384 bytes at offset 394948231168
[0003] 2018-12-06 22:58:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0003] 2018-12-06 22:58:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:58:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0000] 2018-12-06 22:58:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:58:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0002] 2018-12-06 22:58:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:58:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0001] 2018-12-06 22:58:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 259852 max: 263951 -- min data: 4.0 GiB mean data: 4.0 GiB time: 587.5s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221224226816.
WARNING: Using actual aggregate bytes moved = 17298292736.
WARNING: maybe caused by deadlineForStonewalling
stonewalling pairs accessed min: 144718 max: 146133 -- min data: 2.2 GiB mean data: 2.2 GiB time: 87.8s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221219966976.
WARNING: Using actual aggregate bytes moved = 9576972288.
WARNING: maybe caused by deadlineForStonewalling
[0001] 2018-12-06 23:03:55 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 23:03:55 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 23:04:02 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 23:04:02 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 23:04:02 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 23:04:02 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 23:03:59 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 23:03:59 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 16371 max: 16378 -- min data: 0.2 GiB mean data: 0.2 GiB time: 66.0s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221212643328.
WARNING: Using actual aggregate bytes moved = 1073348608.
WARNING: maybe caused by deadlineForStonewalling
Pairs deadlineForStonewallingaccessed: 263951
+ [[ '' != /gsfs/jtacquaviva/file_write ]]
+ echo rm /gsfs/jtacquaviva/file_write
+ set +x
