+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 16 --hosts isc17-c02,isc17-c03,isc17-c04,isc17-c05 -wdir /gsfs/jtacquaviva/git/ddn-ime-evaluation/benchmark -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter -genv MV2_SHOW_CPU_BINDING 1 -genv RDMAV_FORK_SAFE 1 -genv RDMAV_HUGEPAGES_SAFE 1 ./ior_wrapper.sh '-i 3 -s 1 -t 10485760 -b 201326592000 -D 60 -O stoneWallingWearOut=1 -a IME -g -H -z -o ime:///gsfs/jtacquaviva/file_write -w -O summaryFile=./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json -O summaryFormat=JSON ' ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt_network
+ tee -a ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
Writing output to ./output_v2/COUNT:1-NN:4-PPN:4-API:IME-T:10485760-TYPE:write.txt.json
-------------CPU AFFINITY-------------
RANK:0  CPU_SET:   0
RANK:1  CPU_SET:   1
RANK:2  CPU_SET:   2
RANK:3  CPU_SET:   3
-------------------------------------
stonewalling pairs accessed min: 2558 max: 2777 -- min data: 25.0 GiB mean data: 26.1 GiB time: 65.9s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221015756800.
WARNING: Using actual aggregate bytes moved = 465903288320.
WARNING: maybe caused by deadlineForStonewalling
[0008] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0008] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0007] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0011] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0004] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0014] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0015] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0010] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0000] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0009] 2018-12-06 21:38:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0013] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0012] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0006] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0002] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0005] 2018-12-06 21:38:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0001] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (60 sec)
[0003] 2018-12-06 21:38:49 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 21:41:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (236 sec)
[0008] 2018-12-06 21:41:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:41:41 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0007] 2018-12-06 21:41:41 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0014] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0000] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0015] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0012] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0013] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0002] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0001] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0003] 2018-12-06 21:41:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:41:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0004] 2018-12-06 21:41:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:41:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0011] 2018-12-06 21:41:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 21:41:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (237 sec)
[0010] 2018-12-06 21:41:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:41:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (238 sec)
[0006] 2018-12-06 21:41:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:41:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (238 sec)
[0009] 2018-12-06 21:41:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:41:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (238 sec)
[0005] 2018-12-06 21:41:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 9, partial write, 7340032 of 10485760 bytes at offset 1904685875200
WARNING: Task 0, partial write, 0 of 10485760 bytes at offset 1112654479360
WARNING: Task 14, partial write, 5242880 of 10485760 bytes at offset 3004925214720
WARNING: Task 8, partial write, 7340032 of 10485760 bytes at offset 2780100034560
WARNING: Task 10, partial write, 9437184 of 10485760 bytes at offset 2247937228800
WARNING: Task 7, partial write, 3145728 of 10485760 bytes at offset 2985023242240
WARNING: Task 11, partial write, 9437184 of 10485760 bytes at offset 633046302720
WARNING: Task 6, partial write, 3145728 of 10485760 bytes at offset 383170641920
WARNING: Task 3, partial write, 1048576 of 10485760 bytes at offset 540928901120
WARNING: Task 1, partial write, 5242880 of 10485760 bytes at offset 367253258240
WARNING: Task 2, partial write, 5242880 of 10485760 bytes at offset 2662837780480
WARNING: Task 13, partial write, 1048576 of 10485760 bytes at offset 3109646499840
WARNING: Task 15, partial write, 3145728 of 10485760 bytes at offset 899164405760
WARNING: Task 12, partial write, 7340032 of 10485760 bytes at offset 1852487761920
WARNING: Task 4, partial write, 5242880 of 10485760 bytes at offset 2937449349120
WARNING: Task 5, partial write, 5242880 of 10485760 bytes at offset 1334638018560
[0008] 2018-12-06 21:44:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (416 sec)
[0008] 2018-12-06 21:44:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:44:41 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (416 sec)
[0007] 2018-12-06 21:44:41 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0002] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0013] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0001] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0003] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0000] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0014] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0015] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0012] 2018-12-06 21:44:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:44:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0004] 2018-12-06 21:44:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:44:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0006] 2018-12-06 21:44:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:44:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0005] 2018-12-06 21:44:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:44:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (417 sec)
[0009] 2018-12-06 21:44:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 21:44:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (418 sec)
[0010] 2018-12-06 21:44:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:44:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (418 sec)
[0011] 2018-12-06 21:44:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0007] 2018-12-06 21:47:41 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (596 sec)
[0007] 2018-12-06 21:47:41 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0008] 2018-12-06 21:47:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0008] 2018-12-06 21:47:44 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0001] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0001] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0003] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0003] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0000] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0000] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0014] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0014] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0015] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0015] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0012] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0012] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0002] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0002] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0013] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0013] 2018-12-06 21:47:48 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0005] 2018-12-06 21:47:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0005] 2018-12-06 21:47:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0009] 2018-12-06 21:47:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (597 sec)
[0009] 2018-12-06 21:47:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0004] 2018-12-06 21:47:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0004] 2018-12-06 21:47:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0010] 2018-12-06 21:47:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0010] 2018-12-06 21:47:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0006] 2018-12-06 21:47:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0006] 2018-12-06 21:47:42 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
[0011] 2018-12-06 21:47:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00573) Some written data can't reach IME servers for long time (598 sec)
[0011] 2018-12-06 21:47:45 [ warn] (     ime_inode_imcl_buf_flush_try_lru_cb:00582) Requesting cluster geometry update
WARNING: Task 9, partial write, 0 of 3145728 bytes at offset 1904693215232
WARNING: Task 8, partial write, 0 of 3145728 bytes at offset 2780107374592
WARNING: Task 0, partial write, 0 of 10485760 bytes at offset 1112654479360
WARNING: Task 10, partial write, 0 of 1048576 bytes at offset 2247946665984
WARNING: Task 14, partial write, 0 of 5242880 bytes at offset 3004930457600
WARNING: Task 7, partial write, 0 of 7340032 bytes at offset 2985026387968
WARNING: Task 11, partial write, 0 of 1048576 bytes at offset 633055739904
WARNING: Task 6, partial write, 0 of 7340032 bytes at offset 383173787648
WARNING: Task 1, partial write, 0 of 5242880 bytes at offset 367258501120
WARNING: Task 2, partial write, 0 of 5242880 bytes at offset 2662843023360
WARNING: Task 3, partial write, 0 of 9437184 bytes at offset 540929949696
WARNING: Task 13, partial write, 0 of 9437184 bytes at offset 3109647548416
WARNING: Task 12, partial write, 0 of 3145728 bytes at offset 1852495101952
WARNING: Task 4, partial write, 0 of 5242880 bytes at offset 2937454592000
WARNING: Task 5, partial write, 0 of 5242880 bytes at offset 1334643261440
WARNING: Task 15, partial write, 0 of 7340032 bytes at offset 899167551488
stonewalling pairs accessed min: 28 max: 39 -- min data: 0.3 GiB mean data: 0.3 GiB time: 730.7s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3210613882880.
WARNING: Using actual aggregate bytes moved = 6543114240.
WARNING: maybe caused by deadlineForStonewalling
stonewalling pairs accessed min: 3946 max: 4327 -- min data: 38.5 GiB mean data: 40.1 GiB time: 60.5s
WARNING: Expected aggregate file size       = 3221225472000.
WARNING: Stat() of aggregate file size      = 3221214986240.
WARNING: Using actual aggregate bytes moved = 725950136320.
WARNING: maybe caused by deadlineForStonewalling
Pairs deadlineForStonewallingaccessed: 2777
+ [[ '' != /gsfs/jtacquaviva/file_write ]]
+ echo rm /gsfs/jtacquaviva/file_write
+ set +x
