+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 16 --hosts isc17-c02,isc17-c03 -wdir /gsfs/jtacquaviva/git/ddn-ime-evaluation/benchmark -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter -genv MV2_SHOW_CPU_BINDING 1 -genv RDMAV_FORK_SAFE 1 -genv RDMAV_HUGEPAGES_SAFE 1 ./ior_wrapper.sh '-i 3 -s 1 -t 102400 -b 100663296000 -D 60 -O stoneWallingWearOut=1 -a IME -g -H -z -o ime:///gsfs/jtacquaviva/file_write -w -O summaryFile=./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json -O summaryFormat=JSON ' ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt_network
+ tee -a ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:2-PPN:8-API:IME-T:102400-TYPE:write.txt.json
-------------CPU AFFINITY-------------
RANK:0  CPU_SET:   0
RANK:1  CPU_SET:   1
RANK:2  CPU_SET:   2
RANK:3  CPU_SET:   3
RANK:4  CPU_SET:   4
RANK:5  CPU_SET:   5
RANK:6  CPU_SET:   6
RANK:7  CPU_SET:   7
-------------------------------------
[0010] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0010] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0013] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0008] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0004] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0006] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0005] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0007] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0011] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0009] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 19:24:36 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0015] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0014] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0012] 2018-12-06 19:24:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 19:27:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0008] 2018-12-06 19:27:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 19:27:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0010] 2018-12-06 19:27:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 19:27:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0013] 2018-12-06 19:27:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0007] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0003] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0004] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0002] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0006] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0000] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0001] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0005] 2018-12-06 19:27:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (240 sec)
[0011] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (240 sec)
[0009] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (240 sec)
[0015] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (240 sec)
[0014] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (240 sec)
[0012] 2018-12-06 19:27:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 15, partial write, 53248 of 102400 bytes at offset 148860416000
WARNING: Task 13, partial write, 8192 of 102400 bytes at offset 1487513497600
WARNING: Task 9, partial write, 12288 of 102400 bytes at offset 292018380800
WARNING: Task 3, partial write, 4096 of 102400 bytes at offset 1013639372800
WARNING: Task 12, partial write, 73728 of 102400 bytes at offset 467855667200
WARNING: Task 7, partial write, 98304 of 102400 bytes at offset 413264896000
WARNING: Task 14, partial write, 32768 of 102400 bytes at offset 164486553600
WARNING: Task 11, partial write, 32768 of 102400 bytes at offset 1170629120000
WARNING: Task 2, partial write, 0 of 102400 bytes at offset 764549120000
WARNING: Task 10, partial write, 16384 of 102400 bytes at offset 949446758400
WARNING: Task 4, partial write, 73728 of 102400 bytes at offset 468595507200
WARNING: Task 1, partial write, 98304 of 102400 bytes at offset 875961753600
WARNING: Task 6, partial write, 49152 of 102400 bytes at offset 1108862361600
WARNING: Task 5, partial write, 4096 of 102400 bytes at offset 314434560000
WARNING: Task 8, partial write, 86016 of 102400 bytes at offset 87604531200
WARNING: Task 0, partial write, 73728 of 102400 bytes at offset 1368421376000
[0010] 2018-12-06 19:30:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (418 sec)
[0010] 2018-12-06 19:30:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 19:30:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (418 sec)
[0008] 2018-12-06 19:30:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 19:30:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (418 sec)
[0013] 2018-12-06 19:30:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0007] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0003] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0004] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0002] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0006] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0000] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0001] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0005] 2018-12-06 19:30:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0011] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0009] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0015] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0014] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0012] 2018-12-06 19:30:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 19:33:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0010] 2018-12-06 19:33:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 19:33:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0008] 2018-12-06 19:33:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 19:33:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0013] 2018-12-06 19:33:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0003] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0004] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0002] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0006] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0000] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0001] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0005] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0007] 2018-12-06 19:33:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0011] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0009] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0015] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0014] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0012] 2018-12-06 19:33:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 15, partial write, 0 of 49152 bytes at offset 148860469248
WARNING: Task 13, partial write, 0 of 94208 bytes at offset 1487513505792
WARNING: Task 3, partial write, 0 of 98304 bytes at offset 1013639376896
WARNING: Task 9, partial write, 0 of 90112 bytes at offset 292018393088
WARNING: Task 7, partial write, 0 of 4096 bytes at offset 413264994304
WARNING: Task 12, partial write, 0 of 28672 bytes at offset 467855740928
WARNING: Task 2, partial write, 0 of 102400 bytes at offset 764549120000
WARNING: Task 10, partial write, 0 of 86016 bytes at offset 949446774784
WARNING: Task 11, partial write, 0 of 69632 bytes at offset 1170629152768
WARNING: Task 14, partial write, 0 of 69632 bytes at offset 164486586368
WARNING: Task 1, partial write, 0 of 4096 bytes at offset 875961851904
WARNING: Task 4, partial write, 0 of 28672 bytes at offset 468595580928
WARNING: Task 6, partial write, 0 of 53248 bytes at offset 1108862410752
WARNING: Task 5, partial write, 0 of 98304 bytes at offset 314434564096
WARNING: Task 8, partial write, 0 of 16384 bytes at offset 87604617216
WARNING: Task 0, partial write, 0 of 28672 bytes at offset 1368421449728
stonewalling pairs accessed min: 40731 max: 42912 -- min data: 3.9 GiB mean data: 4.0 GiB time: 665.5s
WARNING: Expected aggregate file size       = 1610612736000.
WARNING: Stat() of aggregate file size      = 1610609868800.
WARNING: Using actual aggregate bytes moved = 70307020800.
WARNING: maybe caused by deadlineForStonewalling
[0010] 2018-12-06 19:36:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0010] 2018-12-06 19:36:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 19:36:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0008] 2018-12-06 19:36:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 19:36:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0013] 2018-12-06 19:36:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0004] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0002] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0006] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0000] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0001] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0005] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0007] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (143 sec)
[0003] 2018-12-06 19:36:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (144 sec)
[0015] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (144 sec)
[0014] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (144 sec)
[0012] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (144 sec)
[0011] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (144 sec)
[0009] 2018-12-06 19:36:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 16769 max: 17362 -- min data: 1.6 GiB mean data: 1.6 GiB time: 250.2s
WARNING: Expected aggregate file size       = 1610612736000.
WARNING: Stat() of aggregate file size      = 1610605465600.
WARNING: Using actual aggregate bytes moved = 28445900800.
WARNING: maybe caused by deadlineForStonewalling
[0013] 2018-12-06 19:39:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (72 sec)
[0013] 2018-12-06 19:39:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 19:39:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (72 sec)
[0010] 2018-12-06 19:39:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 19:39:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (72 sec)
[0008] 2018-12-06 19:39:28 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (72 sec)
[0000] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (72 sec)
[0001] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (72 sec)
[0005] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0007] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0003] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0004] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0002] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0006] 2018-12-06 19:39:35 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0009] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0011] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0015] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0014] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (73 sec)
[0012] 2018-12-06 19:39:29 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 10748 max: 11240 -- min data: 1.0 GiB mean data: 1.1 GiB time: 107.4s
WARNING: Expected aggregate file size       = 1610612736000.
WARNING: Stat() of aggregate file size      = 1610606387200.
WARNING: Using actual aggregate bytes moved = 18415616000.
WARNING: maybe caused by deadlineForStonewalling
Pairs deadlineForStonewallingaccessed: 42912
+ [[ '' != /gsfs/jtacquaviva/file_write ]]
+ echo rm /gsfs/jtacquaviva/file_write
+ set +x
