From 7895dde428382497ce7691c906033e8d9ba59de1 Mon Sep 17 00:00:00 2001 From: Eugen Betke Date: Tue, 23 Oct 2018 11:14:00 +0200 Subject: [PATCH] Result for NN:16 purged --- ...COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt | 137 ---- ...COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt | 243 ------- ...COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt | 234 ------- ...COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt | 445 ------------ ...COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt | 438 ------------ ...COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt | 628 ----------------- ...COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt | 632 ------------------ ...COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt | 560 ---------------- ...COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt | 501 -------------- 9 files changed, 3818 deletions(-) delete mode 100644 output/COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt delete mode 100644 output/COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt diff --git a/output/COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt b/output/COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt deleted file mode 100644 index 7886e60d1..000000000 --- a/output/COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt +++ /dev/null @@ -1,137 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 141180272640 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_8]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_2]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_5]: [cli_14]: [cli_15]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_10]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 48355 RUNNING AT isc17-c07 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:3@isc17-c04] HYDU_sock_write (utils/sock/sock.c:286): write error (Broken pipe) -[proxy:0:3@isc17-c04] main (pm/pmiserv/pmip.c:265): unable to send EXIT_STATUS command upstream -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:1#API:MPIIO#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 1 -np 16 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 141180272640 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/file_read -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_8]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_7]: [cli_0]: [cli_14]: [cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_12]: [cli_4]: [cli_5]: [cli_9]: [cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 35109 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:9@isc17-c11] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:9@isc17-c11] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:9@isc17-c11] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt b/output/COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt deleted file mode 100644 index 70b352272..000000000 --- a/output/COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt +++ /dev/null @@ -1,243 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 70590136320 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_12]: [cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_8]: [cli_22]: [cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_29]: [cli_28]: [cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34923 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:10@isc17-c12] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:10@isc17-c12] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:10@isc17-c12] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:5@isc17-c06] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:5@isc17-c06] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:5@isc17-c06] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:6@isc17-c07] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:6@isc17-c07] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:6@isc17-c07] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:2#API:MPIIO#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 70590136320 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/file_read -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: [cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_9]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_28]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_10]: [cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_14]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 16409 RUNNING AT isc17-c15 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:5@isc17-c06] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:5@isc17-c06] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:5@isc17-c06] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:12@isc17-c14] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:12@isc17-c14] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:12@isc17-c14] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt b/output/COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt deleted file mode 100644 index a8d399055..000000000 --- a/output/COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt +++ /dev/null @@ -1,234 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 70590136320 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_14]: [cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_8]: [cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_24]: [cli_28]: [cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34847 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:12@isc17-c14] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:12@isc17-c14] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:12@isc17-c14] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:2#API:POSIX#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 2 -np 32 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 70590136320 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_13]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_12]: [cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_8]: [cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_23]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_24]: [cli_22]: [cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_4]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_7]: [cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34897 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:6@isc17-c07] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:6@isc17-c07] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:6@isc17-c07] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt b/output/COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt deleted file mode 100644 index edf957f95..000000000 --- a/output/COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt +++ /dev/null @@ -1,445 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 64 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 35295068160 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_12]: [cli_0]: [cli_9]: [cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -tee: standard output[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -: Resource temporarily unavailable -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34756 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:12@isc17-c14] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:12@isc17-c14] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:12@isc17-c14] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:7@isc17-c08] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:7@isc17-c08] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:7@isc17-c08] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:1@isc17-c02] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:1@isc17-c02] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:1@isc17-c02] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:15@isc17-c22] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:15@isc17-c22] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:15@isc17-c22] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:6@isc17-c07] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:6@isc17-c07] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:6@isc17-c07] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:5@isc17-c06] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:5@isc17-c06] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:5@isc17-c06] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -tee: write error -+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 64 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:4#API:MPIIO#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 64 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 35295068160 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/file_read -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_44]: [cli_0]: tee: standard output[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -: Resource temporarily unavailable -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_16]: [cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_31]: [cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_48]: [cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_57]: ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_56]: [cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_13]: [cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34816 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:2@isc17-c03] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:2@isc17-c03] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:2@isc17-c03] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -tee: write error -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt b/output/COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt deleted file mode 100644 index 04d34d7a5..000000000 --- a/output/COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt +++ /dev/null @@ -1,438 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 64 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 35295068160 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_0]: [cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_25]: [cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_45]: [cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_56]: [cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_8]: [cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_46]: [cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34002 RUNNING AT isc17-c11 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:3@isc17-c04] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:3@isc17-c04] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:3@isc17-c04] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:15@isc17-c22] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:15@isc17-c22] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:15@isc17-c22] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 64 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:4#API:POSIX#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 4 -np 64 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 35295068160 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_40]: [cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_21]: [cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_16]: [cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_20]: [cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34724 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:0@isc17-c01] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:0@isc17-c01] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:0@isc17-c01] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:14@isc17-c18] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:14@isc17-c18] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:14@isc17-c18] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:4@isc17-c05] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:4@isc17-c05] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:4@isc17-c05] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:5@isc17-c06] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:5@isc17-c06] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:5@isc17-c06] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt b/output/COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt deleted file mode 100644 index 9e571aef0..000000000 --- a/output/COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt +++ /dev/null @@ -1,628 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 96 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 23530045440 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -tee: standard output[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -: Resource temporarily unavailable -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_67]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 67 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_72]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_66]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 66 -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_73]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 73 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_68]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 68 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -[cli_65]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 65 -[cli_74]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_69]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 69 -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_75]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 75 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -[cli_70]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 70 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_76]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 76 -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_71]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 71 -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -[cli_77]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 77 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_6]: [cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_78]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 78 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_79]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 79 -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -[cli_81]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 81 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_5]: [cli_82]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 82 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -[cli_83]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 83 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_84]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 84 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_85]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 85 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_86]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 86 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_87]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 87 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_89]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 89 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_24]: [cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -[cli_91]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 91 -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -[cli_93]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 93 -[cli_94]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 94 -[cli_95]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 95 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 15953 RUNNING AT isc17-c15 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:3@isc17-c04] HYDU_sock_write (utils/sock/sock.c:286): write error (Broken pipe) -[proxy:0:3@isc17-c04] main (pm/pmiserv/pmip.c:265): unable to send EXIT_STATUS command upstream -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -tee: write error -+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 96 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:6#API:MPIIO#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 96 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 23530045440 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/file_read -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_3]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_69]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 69 -[cli_71]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 71 -[cli_67]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 67 -[cli_68]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 68 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_70]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 70 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_66]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 66 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_65]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 65 -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_72]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -[cli_73]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 73 -[cli_74]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_75]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 75 -[cli_76]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 76 -[cli_77]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 77 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_84]: [cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_85]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 85 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_25]: [cli_86]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 86 -[cli_36]: [cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_24]: [cli_87]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 87 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_89]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 89 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 84 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_79]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 79 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_81]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 81 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_83]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 83 -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_82]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 82 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_78]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 78 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_6]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_91]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 91 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_93]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 93 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_94]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 94 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_95]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 95 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 16028 RUNNING AT isc17-c15 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:9@isc17-c11] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:9@isc17-c11] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:9@isc17-c11] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:8@isc17-c09] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:8@isc17-c09] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:8@isc17-c09] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:2@isc17-c03] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:2@isc17-c03] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:2@isc17-c03] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:0@isc17-c01] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:0@isc17-c01] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:0@isc17-c01] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt b/output/COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt deleted file mode 100644 index 21ced231a..000000000 --- a/output/COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt +++ /dev/null @@ -1,632 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 96 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 23530045440 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_84]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 84 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_12]: [cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_8]: [cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_75]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 75 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -tee: [cli_86]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 86 -standard output[cli_78]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 78 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_33]: [cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_72]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_68]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 68 -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -[cli_89]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 89 -[cli_79]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 79 -: Resource temporarily unavailable -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_73]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 73 -[cli_50]: [cli_70]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 70 -[cli_91]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 91 -[cli_87]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 87 -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_24]: [cli_74]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -[cli_66]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 66 -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -[cli_82]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 82 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_76]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 76 -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_67]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 67 -[cli_93]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 93 -[cli_83]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 83 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_65]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 65 -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_77]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 77 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_71]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 71 -[cli_94]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 94 -[cli_85]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 85 -[cli_81]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 81 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_69]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 69 -[cli_95]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 95 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 15828 RUNNING AT isc17-c15 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:8@isc17-c09] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:8@isc17-c09] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:8@isc17-c09] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:9@isc17-c11] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:9@isc17-c11] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:9@isc17-c11] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:14@isc17-c18] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:14@isc17-c18] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:14@isc17-c18] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -tee: write error -+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 96 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:6#API:POSIX#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 6 -np 96 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 23530045440 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_66]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 66 -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_72]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_68]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 68 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_74]: [cli_70]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 70 -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -[cli_76]: [cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 76 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_78]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -[cli_82]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 82 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 78 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_81]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 81 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_77]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 77 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_67]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 67 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_84]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 84 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_65]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 65 -[cli_75]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 75 -[cli_69]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 69 -[cli_86]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 86 -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_83]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 83 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_73]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 73 -[cli_79]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 79 -[cli_71]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 71 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_37]: [cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_85]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 85 -[cli_30]: [cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_42]: [cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_6]: [cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_87]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 87 -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_89]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 89 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -[cli_93]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 93 -[cli_94]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 94 -[cli_95]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 95 -[cli_91]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 91 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34458 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:11@isc17-c13] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:11@isc17-c13] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:11@isc17-c13] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:7@isc17-c08] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:7@isc17-c08] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:7@isc17-c08] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:6@isc17-c07] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:6@isc17-c07] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:6@isc17-c07] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt b/output/COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt deleted file mode 100644 index 86038a113..000000000 --- a/output/COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt +++ /dev/null @@ -1,560 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 128 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 17647534080 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_96]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 96 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_112]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 112 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_120]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 120 -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_72]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_104]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 104 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -[cli_74]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_106]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 106 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -tee: [cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -standard output[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -: Resource temporarily unavailable -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 3230 RUNNING AT isc17-c22 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:13@isc17-c15] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:13@isc17-c15] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:13@isc17-c15] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -tee: write error -+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 128 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 128 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 17647534080 -D 120 -a MPIIO -e -g -z -k -o /esfs/jtacquaviva/file_read -r -+ tee -a ./output/COUNT:1#NN:16#PPN:8#API:MPIIO#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -tee: [cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -standard output[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -: Resource temporarily unavailable -[cli_7]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_68]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 68 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_70]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 70 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_69]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 69 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_100]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 100 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_112]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 112 -[cli_71]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 71 -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_101]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 101 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_116]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 116 -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -[cli_102]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 102 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_104]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 104 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_118]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 118 -[cli_65]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 65 -[cli_93]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 93 -[cli_103]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 103 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_84]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 84 -[cli_108]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 108 -[cli_117]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 117 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_94]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 94 -[cli_96]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 96 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_86]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 86 -[cli_109]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 109 -[cli_119]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 119 -[cli_95]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 95 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -[cli_85]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 85 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_110]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 110 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_97]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 97 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_87]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 87 -[cli_111]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 111 -[cli_113]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 113 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_81]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 81 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_124]: [cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -[cli_126]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 126 -[cli_72]: [cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_127]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 127 -[cli_76]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 76 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_120]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 120 -[cli_77]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 77 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 124 -[cli_78]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 78 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_125]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 125 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_79]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 79 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_121]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 121 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_105]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 105 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_98]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 98 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_73]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 73 -[cli_66]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 66 -[cli_89]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 89 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_99]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 99 -[cli_114]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 114 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_67]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 67 -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_122]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 122 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -[cli_106]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 106 -[cli_115]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 115 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_91]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 91 -[cli_123]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 123 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_59]: [cli_74]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_107]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 107 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_75]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 75 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_82]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 82 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -[cli_83]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 83 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 15767 RUNNING AT isc17-c15 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:7@isc17-c08] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:7@isc17-c08] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:7@isc17-c08] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:9@isc17-c11] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:9@isc17-c11] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:9@isc17-c11] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:10@isc17-c12] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:10@isc17-c12] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:10@isc17-c12] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:11@isc17-c13] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:11@isc17-c13] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:11@isc17-c13] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:12@isc17-c14] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:12@isc17-c14] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:12@isc17-c14] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:8@isc17-c09] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:8@isc17-c09] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:8@isc17-c09] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:14@isc17-c18] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:14@isc17-c18] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:14@isc17-c18] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:15@isc17-c22] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:15@isc17-c22] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:15@isc17-c22] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -tee: write error -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1 diff --git a/output/COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt b/output/COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt deleted file mode 100644 index a6e1a0725..000000000 --- a/output/COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt +++ /dev/null @@ -1,501 +0,0 @@ -+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 128 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 17647534080 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/ioperf/file_write -w -+ tee -a ./output/COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -[cli_96]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 96 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_48]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -[cli_112]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 112 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -[cli_72]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -[cli_104]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 104 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -[cli_120]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 120 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 34042 RUNNING AT isc17-c09 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:7@isc17-c08] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:7@isc17-c08] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:7@isc17-c08] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:11@isc17-c13] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:11@isc17-c13] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:11@isc17-c13] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:9@isc17-c11] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:9@isc17-c11] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:9@isc17-c11] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:10@isc17-c12] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:10@isc17-c12] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:10@isc17-c12] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:12@isc17-c14] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:12@isc17-c14] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:12@isc17-c14] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:14@isc17-c18] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:14@isc17-c18] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:14@isc17-c18] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:15@isc17-c22] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:15@isc17-c22] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:15@isc17-c22] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:2@isc17-c03] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:2@isc17-c03] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:2@isc17-c03] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[proxy:0:4@isc17-c05] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:4@isc17-c05] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:4@isc17-c05] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 128 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/git/ime-evaluation/drop_caches.sh -+ tee -a ./output/COUNT:1#NN:16#PPN:8#API:POSIX#T:102400.txt -+ /opt/ddn/mvapich/bin/mpiexec -ppn 8 -np 128 -genv MV2_NUM_HCAS 1 -genv MV2_CPU_BINDING_LEVEL core -genv MV2_CPU_BINDING_POLICY scatter --hosts isc17-c01,isc17-c02,isc17-c03,isc17-c04,isc17-c05,isc17-c06,isc17-c07,isc17-c08,isc17-c09,isc17-c11,isc17-c12,isc17-c13,isc17-c14,isc17-c15,isc17-c18,isc17-c22 /esfs/jtacquaviva/software/install/ior/git-ddn/bin/ior -i 3 -s 1 -t 102400 -b 17647534080 -D 120 -a POSIX -F -e -g -z -k -o /esfs/jtacquaviva/indread16/file -r -IOR-3.0.1: MPI Coordinated Test of Parallel I/O - -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_0]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 0 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_64]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 64 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_32]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 32 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_80]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 80 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_48]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 48 -[cli_96]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 96 -[cli_112]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 112 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_16]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 16 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_88]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 88 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_72]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 72 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_120]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 120 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_40]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 40 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_44]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 44 -[cli_124]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 124 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_56]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 56 -[cli_84]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 84 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_116]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 116 -[cli_36]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 36 -[cli_4]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 4 -[cli_20]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 20 -[cli_76]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 76 -[cli_100]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 100 -[cli_92]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 92 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_60]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 60 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_52]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 52 -[cli_42]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 42 -[cli_122]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 122 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_82]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 82 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_114]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 114 -[cli_34]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 34 -[cli_50]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 50 -[cli_2]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 2 -[cli_18]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 18 -[cli_74]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 74 -[cli_98]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 98 -[cli_90]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 90 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_58]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 58 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_8]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 8 -[cli_46]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 46 -[cli_126]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 126 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_86]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 86 -[cli_118]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 118 -[cli_38]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 38 -[cli_54]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 54 -[cli_6]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 6 -[cli_12]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 12 -[cli_22]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 22 -[cli_78]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 78 -[cli_102]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 102 -[cli_94]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 94 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_62]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 62 -[cli_10]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 10 -[cli_106]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 106 -[cli_87]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 87 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_55]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 55 -[cli_23]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 23 -[cli_47]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 47 -[cli_127]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 127 -[cli_110]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 110 -[cli_119]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 119 -[cli_39]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 39 -[cli_7]: [cli_14]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 14 -[cli_43]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 43 -[cli_123]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 123 -[cli_79]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 79 -[cli_103]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 103 -[cli_95]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 95 -[cli_104]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 104 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -[cli_63]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 63 -[cli_35]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 35 -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 7 -[cli_75]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 75 -[cli_99]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 99 -[cli_91]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 91 -[cli_108]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 108 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_59]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 59 -[cli_83]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 83 -[cli_115]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 115 -[cli_51]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 51 -[cli_3]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 3 -[cli_19]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 19 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_11]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 11 -[cli_41]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 41 -[cli_121]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 121 -[cli_85]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 85 -[cli_117]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 117 -[cli_37]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 37 -[cli_53]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 53 -[cli_15]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 15 -[cli_21]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 21 -[cli_45]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 45 -[cli_125]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 125 -[cli_73]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 73 -[cli_97]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 97 -[cli_89]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 89 -[cli_111]: ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 0, Success (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_57]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 57 -[cli_81]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 81 -[cli_113]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 113 -[cli_33]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 33 -[cli_49]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 49 -[cli_5]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 5 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_17]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 17 -[cli_77]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 77 -[cli_101]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 101 -[cli_93]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 93 -aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 111 -[cli_61]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 61 -[cli_1]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 1 -[cli_107]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 107 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_9]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 9 -[cli_13]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 13 -ior ERROR: block size must be a multiple of transfer size, errno 2, No such file or directory (ior.c:2293) -[cli_105]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 105 -[cli_109]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 109 -[cli_26]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 26 -[cli_28]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 28 -[cli_30]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 30 -[cli_31]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 31 -[cli_24]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 24 -[cli_25]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 25 -[cli_27]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 27 -[cli_29]: aborting job: -application called MPI_Abort(MPI_COMM_WORLD, -1) - process 29 - -=================================================================================== -= BAD TERMINATION OF ONE OF YOUR APPLICATION PROCESSES -= PID 15529 RUNNING AT isc17-c15 -= EXIT CODE: 255 -= CLEANING UP REMAINING PROCESSES -= YOU CAN IGNORE THE BELOW CLEANUP MESSAGES -=================================================================================== -[proxy:0:7@isc17-c08] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:7@isc17-c08] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:7@isc17-c08] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:9@isc17-c11] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:9@isc17-c11] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:9@isc17-c11] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:10@isc17-c12] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:10@isc17-c12] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:10@isc17-c12] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:11@isc17-c13] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:11@isc17-c13] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:11@isc17-c13] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:12@isc17-c14] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:12@isc17-c14] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:12@isc17-c14] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:8@isc17-c09] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:8@isc17-c09] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:8@isc17-c09] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:14@isc17-c18] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:14@isc17-c18] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:14@isc17-c18] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:15@isc17-c22] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:15@isc17-c22] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:15@isc17-c22] main (pm/pmiserv/pmip.c:206): demux engine error waiting for event -[proxy:0:0@isc17-c01] HYD_pmcd_pmip_control_cmd_cb (pm/pmiserv/pmip_cb.c:912): assert (!closed) failed -[proxy:0:0@isc17-c01] HYDT_dmxu_poll_wait_for_event (tools/demux/demux_poll.c:76): callback returned error status -[proxy:0:0@isc17-c01] main (pm/pmiserv/pmip.c:256): demux engine error waiting for event -[mpiexec@isc17-c04] HYDT_bscu_wait_for_completion (tools/bootstrap/utils/bscu_wait.c:76): one of the processes terminated badly; aborting -[mpiexec@isc17-c04] HYDT_bsci_wait_for_completion (tools/bootstrap/src/bsci_wait.c:23): launcher returned error waiting for completion -[mpiexec@isc17-c04] HYD_pmci_wait_for_completion (pm/pmiserv/pmiserv_pmci.c:218): launcher returned error waiting for completion -[mpiexec@isc17-c04] main (ui/mpich/mpiexec.c:344): process manager error waiting for completion -+ set +x -/esfs/jtacquaviva/ioperf -stripe_count: 32 stripe_size: 1048576 stripe_offset: -1