+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 32 --hosts isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c08,isc17-c09,isc17-c11 -wdir /gsfs/jtacquaviva/git/ddn-ime-evaluation/benchmark -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter -genv MV2_SHOW_CPU_BINDING 1 -genv RDMAV_FORK_SAFE 1 -genv RDMAV_HUGEPAGES_SAFE 1 ./ior_wrapper.sh '-i 3 -s 1 -t 16384 -b 201326592000 -D 60 -O stoneWallingWearOut=1 -a IME -g -H -z -o ime:///gsfs/jtacquaviva/file_write -w -O summaryFile=./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json -O summaryFormat=JSON ' ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt_network
+ tee -a ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:8-PPN:4-API:IME-T:16384-TYPE:write.txt.json
-------------CPU AFFINITY-------------
RANK:0  CPU_SET:   0
RANK:1  CPU_SET:   1
RANK:2  CPU_SET:   2
RANK:3  CPU_SET:   3
-------------------------------------
[0010] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0010] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0011] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0009] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0029] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0029] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0007] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0025] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0025] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0005] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0026] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0026] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0027] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0027] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0030] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0030] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0024] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0024] 2018-12-06 21:17:46 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0028] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0028] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0031] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0031] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0020] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0020] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0022] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0022] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0023] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0023] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0021] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0021] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0019] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0019] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0015] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0018] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0018] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0014] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0013] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0008] 2018-12-06 21:17:43 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0016] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0016] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0012] 2018-12-06 21:17:47 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0004] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0006] 2018-12-06 21:17:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0017] 2018-12-06 21:17:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0017] 2018-12-06 21:17:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 20206 max: 24235 -- min data: 0.3 GiB mean data: 0.4 GiB time: 71.8s
WARNING: Expected aggregate file size       = 6442450944000.
WARNING: Stat() of aggregate file size      = 6442440032256.
WARNING: Using actual aggregate bytes moved = 12706119680.
WARNING: maybe caused by deadlineForStonewalling
[0022] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0022] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0023] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0023] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0019] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0019] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0015] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0018] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0018] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0013] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0014] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0008] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0004] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0012] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0010] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0016] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0016] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0011] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0009] 2018-12-06 21:21:17 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0025] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0025] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0027] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0027] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0007] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0026] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0026] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0024] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0024] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0005] 2018-12-06 21:21:13 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0029] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0029] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0028] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0028] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0030] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0030] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0031] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0031] 2018-12-06 21:21:20 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0020] 2018-12-06 21:21:21 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0020] 2018-12-06 21:21:21 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0021] 2018-12-06 21:21:21 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0021] 2018-12-06 21:21:21 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0017] 2018-12-06 21:21:22 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0017] 2018-12-06 21:21:22 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 22375 max: 23904 -- min data: 0.3 GiB mean data: 0.4 GiB time: 73.7s
WARNING: Expected aggregate file size       = 6442450944000.
WARNING: Stat() of aggregate file size      = 6442447962112.
WARNING: Using actual aggregate bytes moved = 12532580352.
WARNING: maybe caused by deadlineForStonewalling
[0020] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0020] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0021] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0021] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0031] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0031] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0022] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0022] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0023] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0023] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0019] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0019] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0013] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0015] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0014] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0018] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0018] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0008] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0004] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0012] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0010] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0006] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0011] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0016] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0009] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0016] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:24:37 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0025] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0025] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0007] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0024] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0024] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0027] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0027] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0005] 2018-12-06 21:24:33 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0026] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0026] 2018-12-06 21:24:39 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0028] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0028] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0030] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0030] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0029] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0029] 2018-12-06 21:24:40 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0017] 2018-12-06 21:24:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0017] 2018-12-06 21:24:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
stonewalling pairs accessed min: 23197 max: 27406 -- min data: 0.4 GiB mean data: 0.4 GiB time: 86.2s
WARNING: Expected aggregate file size       = 6442450944000.
WARNING: Stat() of aggregate file size      = 6442444439552.
WARNING: Using actual aggregate bytes moved = 14368636928.
WARNING: maybe caused by deadlineForStonewalling
Pairs deadlineForStonewallingaccessed: 24235
+ [[ '' != /gsfs/jtacquaviva/file_write ]]
+ echo rm /gsfs/jtacquaviva/file_write
+ set +x
