+ tee -a ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt
+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 --hosts isc17-c02,isc17-c03,isc17-c04,isc17-c05 -wdir /gsfs/jtacquaviva/git/ddn-ime-evaluation/benchmark -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter -genv MV2_SHOW_CPU_BINDING 1 -genv RDMAV_FORK_SAFE 1 -genv RDMAV_HUGEPAGES_SAFE 1 ./ior_wrapper.sh '-i 3 -s 1 -t 1048576 -b 201326592000 -D 60 -O stoneWallingWearOut=1 -a IME -g -H -z -o ime:///gsfs/jtacquaviva/file_write -w -O summaryFile=./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json -O summaryFormat=JSON ' ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt_network
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:1048576-TYPE:write.txt.json
-------------CPU AFFINITY-------------
RANK:0  CPU_SET:   0
RANK:1  CPU_SET:   1
RANK:2  CPU_SET:   2
RANK:3  CPU_SET:   3
-------------------------------------
stonewalling pairs accessed min: 84484 max: 93848 -- min data: 82.5 GiB mean data: 86.2 GiB time: 60.0s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221225472000.
WARNING: Using actual aggregate bytes moved = 1574508167168.
WARNING: maybe caused by deadlineForStonewalling
stonewalling pairs accessed min: 55867 max: 61857 -- min data: 54.6 GiB mean data: 57.1 GiB time: 60.0s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221222326272.
WARNING: Using actual aggregate bytes moved = 1037788250112.
WARNING: maybe caused by deadlineForStonewalling
[0012] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0012] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0014] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0005] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0006] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0004] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0013] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0015] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0007] 2018-12-06 22:22:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0010] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0008] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0009] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0011] 2018-12-06 22:22:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 22:22:16 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0012] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0014] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0002] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0013] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0015] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0000] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0003] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0001] 2018-12-06 22:25:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0006] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0004] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0005] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0010] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0008] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0007] 2018-12-06 22:25:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0009] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (239 sec)
[0011] 2018-12-06 22:25:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 14, partial write, 655360 of 1048576 bytes at offset 1312225755136
WARNING: Task 0, partial write, 0 of 1048576 bytes at offset 425616998400
WARNING: Task 8, partial write, 917504 of 1048576 bytes at offset 790872719360
WARNING: Task 10, partial write, 524288 of 1048576 bytes at offset 1441011859456
WARNING: Task 11, partial write, 0 of 1048576 bytes at offset 2376341651456
WARNING: Task 1, partial write, 917504 of 1048576 bytes at offset 414461198336
WARNING: Task 12, partial write, 262144 of 1048576 bytes at offset 2567865106432
WARNING: Task 9, partial write, 655360 of 1048576 bytes at offset 1338823933952
WARNING: Task 4, partial write, 917504 of 1048576 bytes at offset 697072353280
WARNING: Task 3, partial write, 393216 of 1048576 bytes at offset 1477898665984
WARNING: Task 15, partial write, 524288 of 1048576 bytes at offset 332720504832
WARNING: Task 6, partial write, 917504 of 1048576 bytes at offset 2047468371968
WARNING: Task 13, partial write, 917504 of 1048576 bytes at offset 287633833984
WARNING: Task 7, partial write, 524288 of 1048576 bytes at offset 23604494336
WARNING: Task 2, partial write, 0 of 1048576 bytes at offset 444999925760
WARNING: Task 5, partial write, 0 of 1048576 bytes at offset 190934155264
[0012] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0012] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0014] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0002] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0013] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0000] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0015] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0003] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0001] 2018-12-06 22:28:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0006] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0004] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0005] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0010] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0008] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0007] 2018-12-06 22:28:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0011] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (419 sec)
[0009] 2018-12-06 22:28:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0014] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0002] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0000] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0013] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0015] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0003] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0001] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0012] 2018-12-06 22:31:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0006] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0004] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0005] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0010] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0008] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0007] 2018-12-06 22:31:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0011] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (599 sec)
[0009] 2018-12-06 22:31:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 0, partial write, 0 of 1048576 bytes at offset 425616998400
WARNING: Task 1, partial write, 0 of 131072 bytes at offset 414462115840
WARNING: Task 14, partial write, 0 of 393216 bytes at offset 1312226410496
WARNING: Task 8, partial write, 0 of 131072 bytes at offset 790873636864
WARNING: Task 10, partial write, 0 of 524288 bytes at offset 1441012383744
WARNING: Task 11, partial write, 0 of 1048576 bytes at offset 2376341651456
WARNING: Task 3, partial write, 0 of 655360 bytes at offset 1477899059200
WARNING: Task 12, partial write, 0 of 786432 bytes at offset 2567865368576
WARNING: Task 9, partial write, 0 of 393216 bytes at offset 1338824589312
WARNING: Task 4, partial write, 0 of 131072 bytes at offset 697073270784
WARNING: Task 15, partial write, 0 of 524288 bytes at offset 332721029120
WARNING: Task 6, partial write, 0 of 131072 bytes at offset 2047469289472
WARNING: Task 13, partial write, 0 of 131072 bytes at offset 287634751488
WARNING: Task 7, partial write, 0 of 524288 bytes at offset 23605018624
WARNING: Task 2, partial write, 0 of 1048576 bytes at offset 444999925760
WARNING: Task 5, partial write, 0 of 1048576 bytes at offset 190934155264
[0003] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0003] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0001] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0012] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0014] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0002] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0000] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0013] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0015] 2018-12-06 22:34:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0004] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0005] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0007] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0009] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0011] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0006] 2018-12-06 22:34:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0010] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (779 sec)
[0008] 2018-12-06 22:34:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 14, partial write, 0 of 393216 bytes at offset 1312226410496
WARNING: Task 1, partial write, 0 of 131072 bytes at offset 414462115840
WARNING: Task 0, partial write, 0 of 1048576 bytes at offset 425616998400
WARNING: Task 8, partial write, 0 of 131072 bytes at offset 790873636864
WARNING: Task 10, partial write, 0 of 524288 bytes at offset 1441012383744
WARNING: Task 11, partial write, 0 of 1048576 bytes at offset 2376341651456
WARNING: Task 12, partial write, 0 of 786432 bytes at offset 2567865368576
WARNING: Task 3, partial write, 0 of 655360 bytes at offset 1477899059200
WARNING: Task 9, partial write, 0 of 393216 bytes at offset 1338824589312
WARNING: Task 15, partial write, 0 of 524288 bytes at offset 332721029120
WARNING: Task 4, partial write, 0 of 131072 bytes at offset 697073270784
WARNING: Task 13, partial write, 0 of 131072 bytes at offset 287634751488
WARNING: Task 6, partial write, 0 of 131072 bytes at offset 2047469289472
WARNING: Task 2, partial write, 0 of 1048576 bytes at offset 444999925760
WARNING: Task 7, partial write, 0 of 524288 bytes at offset 23605018624
WARNING: Task 5, partial write, 0 of 1048576 bytes at offset 190934155264
[0001] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0001] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0012] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0014] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0002] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0000] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0013] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0015] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0003] 2018-12-06 22:37:15 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0007] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0006] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0010] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0008] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0004] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0005] 2018-12-06 22:37:09 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0009] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (959 sec)
[0011] 2018-12-06 22:37:12 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 2119 max: 2315 -- min data: 2.1 GiB mean data: 2.2 GiB time: 1018.4s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221202403328.
WARNING: Using actual aggregate bytes moved = 38839255040.
WARNING: maybe caused by deadlineForStonewalling
Pairs deadlineForStonewallingaccessed: 93848
+ [[ '' != /gsfs/jtacquaviva/file_write ]]
+ echo rm /gsfs/jtacquaviva/file_write
+ set +x
