In case my previous e-mail is too vague for anyone to address, here's a
backtrace from my application. This version, compiled with Intel
11.1.064 (OpenMPI 1.4.2 w/ gcc 4.4.2), hangs during MPI_Alltoall
instead. Running on 16 CPUs, Opteron 2427, Mellanox Technologies
MT25418 w/ OFED 1.5
strace on all ranks repeatedly shows:
poll([{fd=4, events=POLLIN}, {fd=5, events=POLLIN}, {fd=6,
events=POLLIN}, {fd=7, events=POLLIN}, {fd=10, events=POLLIN}, {fd=22,
events=POLLIN}, {fd=23, events=POLLIN}], 7, 0) = 0 (Timeout)
...
gdb --pid=<pid>
(gdb) bt
#0 sm_fifo_read () at btl_sm.h:267
#1 mca_btl_sm_component_progress () at btl_sm_component.c:391
#2 0x00002b00085116ea in opal_progress () at
runtime/opal_progress.c:207
#3 0x00002b0007def215 in opal_condition_wait (count=2,
requests=0x7fffd27802a0, statuses=0x7fffd2780270)
at ../opal/threads/condition.h:99
#4 ompi_request_default_wait_all (count=2, requests=0x7fffd27802a0,
statuses=0x7fffd2780270) at request/req_wait.c:262
#5 0x00002b0007e805b7 in ompi_coll_tuned_sendrecv_actual
(sendbuf=0x2aaac2c4c210, scount=28000,
sdatatype=0x2b0008198ea0, dest=6, stag=-13, recvbuf=<value optimized
out>, rcount=28000, rdatatype=0x2b0008198ea0,
source=10, rtag=-13, comm=0x16ad7420, status=0x0) at
coll_tuned_util.c:55
#6 0x00002b0007e8705f in ompi_coll_tuned_sendrecv (sbuf=0x2aaac2b04010,
scount=28000, sdtype=0x2b0008198ea0,
rbuf=0x2aaac99a2010, rcount=28000, rdtype=0x2b0008198ea0,
comm=0x16ad7420, module=0x16ad8450)
at coll_tuned_util.h:60
#7 ompi_coll_tuned_alltoall_intra_pairwise (sbuf=0x2aaac2b04010,
scount=28000, sdtype=0x2b0008198ea0,
rbuf=0x2aaac99a2010, rcount=28000, rdtype=0x2b0008198ea0,
comm=0x16ad7420, module=0x16ad8450)
at coll_tuned_alltoall.c:70
#8 0x00002b0007e0a71f in PMPI_Alltoall (sendbuf=0x2aaac2b04010,
sendcount=28000, sendtype=0x2b0008198ea0,
recvbuf=0x2aaac99a2010, recvcount=28000, recvtype=0x2b0008198ea0,
comm=0x16ad7420) at palltoall.c:84
#9 0x00002b0007b8bc86 in mpi_alltoall_f (sendbuf=0x2aaac2b04010 "",
sendcount=0x7fffd27806a0,
sendtype=<value optimized out>,
recvbuf=0x2aaac99a2010 "6%\177e\373\354\306>\346\226z\262\347\350
\260>\032ya(\303\003\272\276\231\343\322\363zjþ\230\247i\232\307PԾ(\304
\373\321D\261ľ\204֜Εh־H\266H\342l2\245\276\231C7]\003\250Ǿ`\277\231\272
\265E\261>j\213ѓ\370\002\263>НØx.\254>}\332-\313\371\326\320>\346\245f
\304\f\214\262\276\070\222zf#'\321>\024\066̆\026\227ɾ.T\277\266}\366
\270>h|\323L\330\fƾ^z\214!q*\277\276pQ?O\346\067\270>~\006\300",
recvcount=0x7fffd27806a4, recvtype=0xb67490,
comm=0x12d9ba0, ierr=0x7fffd27806a8) at palltoall_f.c:76
#10 0x00000000004634cc in m_sumf_d_ ()
#11 0x0000000000463072 in m_sum_z_ ()
#12 0x00000000004c8a8b in mrg_grid_rc_ ()
#13 0x00000000004ffc5e in rhosym_ ()
#14 0x0000000000610dc6 in us_mp_set_charge_ ()
#15 0x0000000000771c43 in elmin_ ()
#16 0x0000000000453853 in MAIN__ ()
#17 0x000000000042f15c in main ()
On other processes:
(gdb) bt
#0 0x0000003692a0b725 in pthread_spin_lock ()
from /lib64/libpthread.so.0
#1 0x00002aaaaacdfa7b in ibv_cmd_create_qp ()
from /usr/lib64/libmlx4-rdmav2.so
#2 0x00002b9dc1db3ff8 in progress_one_device ()
at /usr/include/infiniband/verbs.h:884
#3 btl_openib_component_progress () at btl_openib_component.c:3451
#4 0x00002b9dc24736ea in opal_progress () at
runtime/opal_progress.c:207
#5 0x00002b9dc1d51215 in opal_condition_wait (count=2,
requests=0x7fffece3cc20, statuses=0x7fffece3cbf0)
at ../opal/threads/condition.h:99
#6 ompi_request_default_wait_all (count=2, requests=0x7fffece3cc20,
statuses=0x7fffece3cbf0) at request/req_wait.c:262
#7 0x00002b9dc1de25b7 in ompi_coll_tuned_sendrecv_actual
(sendbuf=0x2aaac2c4c210, scount=28000,
sdatatype=0x2b9dc20faea0, dest=6, stag=-13, recvbuf=<value optimized
out>, rcount=28000, rdatatype=0x2b9dc20faea0,
source=10, rtag=-13, comm=0x1745b420, status=0x0) at
coll_tuned_util.c:55
#8 0x00002b9dc1de905f in ompi_coll_tuned_sendrecv (sbuf=0x2aaac2b04010,
scount=28000, sdtype=0x2b9dc20faea0,
rbuf=0x2aaac99a2010, rcount=28000, rdtype=0x2b9dc20faea0,
comm=0x1745b420, module=0x1745c450)
at coll_tuned_util.h:60
#9 ompi_coll_tuned_alltoall_intra_pairwise (sbuf=0x2aaac2b04010,
scount=28000, sdtype=0x2b9dc20faea0,
rbuf=0x2aaac99a2010, rcount=28000, rdtype=0x2b9dc20faea0,
comm=0x1745b420, module=0x1745c450)
at coll_tuned_alltoall.c:70
#10 0x00002b9dc1d6c71f in PMPI_Alltoall (sendbuf=0x2aaac2b04010,
sendcount=28000, sendtype=0x2b9dc20faea0,
recvbuf=0x2aaac99a2010, recvcount=28000, recvtype=0x2b9dc20faea0,
comm=0x1745b420) at palltoall.c:84
#11 0x00002b9dc1aedc86 in mpi_alltoall_f (sendbuf=0x2aaac2b04010 "",
sendcount=0x7fffece3d020,
sendtype=<value optimized out>,
recvbuf=0x2aaac99a2010 "6%\177e\373\354\306>\346\226z\262\347\350
\260>\032ya(\303\003\272\276\231\343\322\363zjþ\230\247i\232\307PԾ(\304
\373\321D\261ľ\204֜Εh־H\266H\342l2\245\276\231C7]\003\250Ǿ`\277\231\272
\265E\261>j\213ѓ\370\002\263>НØx.\254>}\332-\313\371\326\320>\346\245f
\304\f\214\262\276\070\222zf#'\321>\024\066̆\026\227ɾ.T\277\266}\366
\270>h|\323L\330\fƾ^z\214!q*\277\276pQ?O\346\067\270>~\006\300",
recvcount=0x7fffece3d024, recvtype=0xb67490,
comm=0x12d9ba0, ierr=0x7fffece3d028) at palltoall_f.c:76
#12 0x00000000004634cc in m_sumf_d_ ()
#13 0x0000000000463072 in m_sum_z_ ()
#14 0x00000000004c8a8b in mrg_grid_rc_ ()
#15 0x00000000004ffc5e in rhosym_ ()
#16 0x0000000000610dc6 in us_mp_set_charge_ ()
#17 0x0000000000771c43 in elmin_ ()
#18 0x0000000000453853 in MAIN__ ()
#19 0x000000000042f15c in main ()
I set up padb to collect a full report on the process and I've attached
it to this message. Let me know if I can provide anything further.
Thanks,
-Brian
On Wed, 2010-07-21 at 10:07 -0400, Brian Smith wrote:
> Hi, All,
>
> A couple of applications that I'm using -- VASP and Charmm -- end up
> "stuck" (for lack of a better word) during a waitall call after some
> non-blocking send/recv action. This only happens when utilizing the
> openib btl. I've followed a couple of bugs where this seemed to happen
> in some previous revisions and tried the work-arounds provided, but to
> no avail. I'm going to try running against a previous version to see if
> its a regression of some sort, but this problem didn't seem to exist in
> 1.4.1 until our systems were updated to OFED >= 1.4. Any suggestions
> besides the obvious, "well, down-grade from >= 1.4"? What additional
> info can I provide to help?
>
> Thanks,
> -Brian
>
padb version 3.2 (Revision 389)
full job report for job 3436
----------------
[0]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '0'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '0'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '0'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '0'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '0'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '0'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 0
comm8: Rank: local 1 global 1
comm8: Rank: local 2 global 2
comm8: Rank: local 3 global 3
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '0'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 0
comm9: Rank: local 1 global 4
comm9: Rank: local 2 global 8
comm9: Rank: local 3 global 12
----------------
[1]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '1'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '1'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '1'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '1'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '1'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '1'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 0
comm8: Rank: local 1 global 1
comm8: Rank: local 2 global 2
comm8: Rank: local 3 global 3
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '0'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 1
comm9: Rank: local 1 global 5
comm9: Rank: local 2 global 9
comm9: Rank: local 3 global 13
----------------
[2]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '2'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '2'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '2'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '2'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '2'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '2'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 0
comm8: Rank: local 1 global 1
comm8: Rank: local 2 global 2
comm8: Rank: local 3 global 3
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '0'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 2
comm9: Rank: local 1 global 6
comm9: Rank: local 2 global 10
comm9: Rank: local 3 global 14
----------------
[3]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '3'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '3'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '3'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '3'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '3'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '3'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 0
comm8: Rank: local 1 global 1
comm8: Rank: local 2 global 2
comm8: Rank: local 3 global 3
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '0'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 3
comm9: Rank: local 1 global 7
comm9: Rank: local 2 global 11
comm9: Rank: local 3 global 15
----------------
[4]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '4'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '4'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '4'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '4'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '4'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '0'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 4
comm8: Rank: local 1 global 5
comm8: Rank: local 2 global 6
comm8: Rank: local 3 global 7
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '1'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 0
comm9: Rank: local 1 global 4
comm9: Rank: local 2 global 8
comm9: Rank: local 3 global 12
----------------
[5]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '5'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '5'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '5'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '5'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '5'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '1'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 4
comm8: Rank: local 1 global 5
comm8: Rank: local 2 global 6
comm8: Rank: local 3 global 7
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '1'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 1
comm9: Rank: local 1 global 5
comm9: Rank: local 2 global 9
comm9: Rank: local 3 global 13
----------------
[6]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '6'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '6'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '6'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '6'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '6'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '2'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 4
comm8: Rank: local 1 global 5
comm8: Rank: local 2 global 6
comm8: Rank: local 3 global 7
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '1'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 2
comm9: Rank: local 1 global 6
comm9: Rank: local 2 global 10
comm9: Rank: local 3 global 14
----------------
[7]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '7'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '7'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '7'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '7'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '7'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '3'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 4
comm8: Rank: local 1 global 5
comm8: Rank: local 2 global 6
comm8: Rank: local 3 global 7
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '1'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 3
comm9: Rank: local 1 global 7
comm9: Rank: local 2 global 11
comm9: Rank: local 3 global 15
----------------
[8]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '8'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '8'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '8'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '8'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '8'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '0'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 8
comm8: Rank: local 1 global 9
comm8: Rank: local 2 global 10
comm8: Rank: local 3 global 11
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '2'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 0
comm9: Rank: local 1 global 4
comm9: Rank: local 2 global 8
comm9: Rank: local 3 global 12
----------------
[9]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '9'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '9'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '9'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '9'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '9'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '1'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 8
comm8: Rank: local 1 global 9
comm8: Rank: local 2 global 10
comm8: Rank: local 3 global 11
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '2'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 1
comm9: Rank: local 1 global 5
comm9: Rank: local 2 global 9
comm9: Rank: local 3 global 13
----------------
[10]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '10'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '10'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '10'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '10'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '10'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '2'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 8
comm8: Rank: local 1 global 9
comm8: Rank: local 2 global 10
comm8: Rank: local 3 global 11
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '2'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 2
comm9: Rank: local 1 global 6
comm9: Rank: local 2 global 10
comm9: Rank: local 3 global 14
----------------
[11]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '11'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '11'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '11'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '11'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '11'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '3'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 8
comm8: Rank: local 1 global 9
comm8: Rank: local 2 global 10
comm8: Rank: local 3 global 11
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '2'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 3
comm9: Rank: local 1 global 7
comm9: Rank: local 2 global 11
comm9: Rank: local 3 global 15
----------------
[12]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '12'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '12'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '12'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '12'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '12'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '0'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 12
comm8: Rank: local 1 global 13
comm8: Rank: local 2 global 14
comm8: Rank: local 3 global 15
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '3'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 0
comm9: Rank: local 1 global 4
comm9: Rank: local 2 global 8
comm9: Rank: local 3 global 12
----------------
[13]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '13'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '13'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '13'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '13'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '13'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '1'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 12
comm8: Rank: local 1 global 13
comm8: Rank: local 2 global 14
comm8: Rank: local 3 global 15
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '3'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 1
comm9: Rank: local 1 global 5
comm9: Rank: local 2 global 9
comm9: Rank: local 3 global 13
----------------
[14]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '14'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '14'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '14'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '14'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '14'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '2'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 12
comm8: Rank: local 1 global 13
comm8: Rank: local 2 global 14
comm8: Rank: local 3 global 15
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '3'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 2
comm9: Rank: local 1 global 6
comm9: Rank: local 2 global 10
comm9: Rank: local 3 global 14
----------------
[15]
----------------
comm0: name: 'MPI_COMM_WORLD'
comm0: rank: '15'
comm0: size: '16'
comm0: id: '0'
comm0: Rank: local 0 global 0
comm0: Rank: local 1 global 1
comm0: Rank: local 2 global 2
comm0: Rank: local 3 global 3
comm0: Rank: local 4 global 4
comm0: Rank: local 5 global 5
comm0: Rank: local 6 global 6
comm0: Rank: local 7 global 7
comm0: Rank: local 8 global 8
comm0: Rank: local 9 global 9
comm0: Rank: local 10 global 10
comm0: Rank: local 11 global 11
comm0: Rank: local 12 global 12
comm0: Rank: local 13 global 13
comm0: Rank: local 14 global 14
comm0: Rank: local 15 global 15
comm1: name: 'MPI_COMM_SELF'
comm1: rank: '0'
comm1: size: '1'
comm1: id: '0x1'
comm2: name: 'MPI_COMM_NULL'
comm2: size: '0'
comm2: id: '0x2'
comm3: name: 'MPI_COMMUNICATOR 3'
comm3: rank: '15'
comm3: size: '16'
comm3: id: '0x3'
comm3: Rank: local 0 global 0
comm3: Rank: local 1 global 1
comm3: Rank: local 2 global 2
comm3: Rank: local 3 global 3
comm3: Rank: local 4 global 4
comm3: Rank: local 5 global 5
comm3: Rank: local 6 global 6
comm3: Rank: local 7 global 7
comm3: Rank: local 8 global 8
comm3: Rank: local 9 global 9
comm3: Rank: local 10 global 10
comm3: Rank: local 11 global 11
comm3: Rank: local 12 global 12
comm3: Rank: local 13 global 13
comm3: Rank: local 14 global 14
comm3: Rank: local 15 global 15
comm4: name: 'MPI COMMUNICATOR 4 SPLIT FROM 3'
comm4: rank: '0'
comm4: size: '1'
comm4: id: '0x4'
comm5: name: 'MPI COMMUNICATOR 5 SPLIT FROM 3'
comm5: rank: '15'
comm5: size: '16'
comm5: id: '0x5'
comm5: Rank: local 0 global 0
comm5: Rank: local 1 global 1
comm5: Rank: local 2 global 2
comm5: Rank: local 3 global 3
comm5: Rank: local 4 global 4
comm5: Rank: local 5 global 5
comm5: Rank: local 6 global 6
comm5: Rank: local 7 global 7
comm5: Rank: local 8 global 8
comm5: Rank: local 9 global 9
comm5: Rank: local 10 global 10
comm5: Rank: local 11 global 11
comm5: Rank: local 12 global 12
comm5: Rank: local 13 global 13
comm5: Rank: local 14 global 14
comm5: Rank: local 15 global 15
comm6: name: 'MPI COMMUNICATOR 6 CREATE FROM 0'
comm6: rank: '15'
comm6: size: '16'
comm6: id: '0x6'
comm6: Rank: local 0 global 0
comm6: Rank: local 1 global 1
comm6: Rank: local 2 global 2
comm6: Rank: local 3 global 3
comm6: Rank: local 4 global 4
comm6: Rank: local 5 global 5
comm6: Rank: local 6 global 6
comm6: Rank: local 7 global 7
comm6: Rank: local 8 global 8
comm6: Rank: local 9 global 9
comm6: Rank: local 10 global 10
comm6: Rank: local 11 global 11
comm6: Rank: local 12 global 12
comm6: Rank: local 13 global 13
comm6: Rank: local 14 global 14
comm6: Rank: local 15 global 15
comm7: name: 'MPI COMMUNICATOR 7 DUP FROM 6'
comm7: rank: '15'
comm7: size: '16'
comm7: id: '0x7'
comm7: Rank: local 0 global 0
comm7: Rank: local 1 global 1
comm7: Rank: local 2 global 2
comm7: Rank: local 3 global 3
comm7: Rank: local 4 global 4
comm7: Rank: local 5 global 5
comm7: Rank: local 6 global 6
comm7: Rank: local 7 global 7
comm7: Rank: local 8 global 8
comm7: Rank: local 9 global 9
comm7: Rank: local 10 global 10
comm7: Rank: local 11 global 11
comm7: Rank: local 12 global 12
comm7: Rank: local 13 global 13
comm7: Rank: local 14 global 14
comm7: Rank: local 15 global 15
comm8: name: 'MPI COMMUNICATOR 8 SPLIT FROM 6'
comm8: rank: '3'
comm8: size: '4'
comm8: id: '0x8'
comm8: Rank: local 0 global 12
comm8: Rank: local 1 global 13
comm8: Rank: local 2 global 14
comm8: Rank: local 3 global 15
comm9: name: 'MPI COMMUNICATOR 9 SPLIT FROM 6'
comm9: rank: '3'
comm9: size: '4'
comm9: id: '0x9'
comm9: Rank: local 0 global 3
comm9: Rank: local 1 global 7
comm9: Rank: local 2 global 11
comm9: Rank: local 3 global 15
Total: 46 communicators of which 0 are in use.
No data was recorded for 160 communicators
Warning: errors reported by some ranks
========
[6-11]: Critical error: (Failed to extrace square braces from
{name="recvbuf",type="char *",value="0x2aaac7efc010
\"6%\\177e\\373\\354\\306>\\346\\226z\\262\\347\\350\\260>\\032ya(\\303\\003\\272\\276\\231\\343\\322\\363zj\303\276\\230\\247i\\232\\307P\324\276(\\304\\373\\321D\\261\304\276\\204\326\234\316\225h\326\276H\\266H\\342l2\\245\\276\\231C7
at /home/b/brs/padb/bin/padb line 6343)
[12-13,15]: Critical error: (Failed to extrace square braces from
{name="recvbuf",type="char *",value="0x2aaac824d010
\"6%\\177e\\373\\354\\306>\\346\\226z\\262\\347\\350\\260>\\032ya(\\303\\003\\272\\276\\231\\343\\322\\363zj\303\276\\230\\247i\\232\\307P\324\276(\\304\\373\\321D\\261\304\276\\204\326\234\316\225h\326\276H\\266H\\342l2\\245\\276\\231C7
at /home/b/brs/padb/bin/padb line 6343)
[14]: Critical error: (Failed to extrace square braces from
{name="recvbuf",type="char *",value="0x2aaac8361010
\"6%\\177e\\373\\354\\306>\\346\\226z\\262\\347\\350\\260>\\032ya(\\303\\003\\272\\276\\231\\343\\322\\363zj\303\276\\230\\247i\\232\\307P\324\276(\\304\\373\\321D\\261\304\276\\204\326\234\316\225h\326\276H\\266H\\342l2\\245\\276\\231C7
at /home/b/brs/padb/bin/padb line 6343)
[0]: Critical error: (Failed to extrace square braces from
{name="recvbuf",type="char *",value="0x2aaac99a2010
\"6%\\177e\\373\\354\\306>\\346\\226z\\262\\347\\350\\260>\\032ya(\\303\\003\\272\\276\\231\\343\\322\\363zj\303\276\\230\\247i\\232\\307P\324\276(\\304\\373\\321D\\261\304\276\\204\326\234\316\225h\326\276H\\266H\\342l2\\245\\276\\231C7
at /home/b/brs/padb/bin/padb line 6343)
========
Stack trace(s) for thread: 1
-----------------
[1-5] (5 processes)
-----------------
main() at ?:?
MAIN__() at ?:?
elmin_() at ?:?
us_mp_set_charge_() at ?:?
rhosym_() at ?:?
mrg_grid_rc_() at ?:?
m_sum_z_() at ?:?
m_sumf_d_() at ?:?
mpi_alltoall_f() at palltoall_f.c:76
params
char * sendbuf = '0x2aaac105e010 ""' [1-5]
int * sendcount: more than 3 distinct values
int * sendtype = 'value optimized out' [1-5]
char * recvbuf = '0x2aaac7efc010 ""' [1-5]
int * recvcount = 'valid pointer perm=rwxp ([stack])'
[1-5]
int * recvtype = 'valid pointer perm=r-xp
(/home/b/brs/src/vasp-5.2.2/bin/vasp)' [1-5]
int * comm = '19766176' [1-5]
int * ierr: more than 3 distinct values
locals
MPI_Comm c_comm = '' [1-5]
MPI_Datatype c_recvtype = '' [1-5]
MPI_Datatype c_sendtype = '' [1-5]
-----------------
[1-5] (5 processes)
-----------------
PMPI_Alltoall() at palltoall.c:84
params
void * sendbuf = 'valid pointer perm=rwxp'
[1-5]
int sendcount = '28000' [1-5]
MPI_Datatype sendtype: more than 3 distinct values
void * recvbuf = 'valid pointer perm=rwxp'
[1-5]
int recvcount = '28000' [1-5]
MPI_Datatype recvtype: more than 3 distinct values
MPI_Comm comm: more than 3 distinct values
locals
int err = 'value optimized out' [1-5]
-----------------
[1-5] (5 processes)
-----------------
ompi_coll_tuned_alltoall_intra_pairwise() at
coll_tuned_alltoall.c:70
params
void * sbuf = 'valid pointer
perm=rwxp' [1-5]
int scount = '28000' [1-5]
struct ompi_datatype_t * sdtype =
'MPI_DOUBLE_PRECISION' [1-5]
void * rbuf = 'valid pointer
perm=rwxp' [1-5]
int rcount = '28000' [1-5]
struct ompi_datatype_t * rdtype =
'MPI_DOUBLE_PRECISION' [1-5]
struct ompi_communicator_t * comm =
'MPI_COMMUNICATOR 3' [1-5]
mca_coll_base_module_t * module = 'valid pointer
perm=rwxp' [1-5]
locals
int err = 'value optimized out' [1-5]
ptrdiff_t lb = 'value optimized out' [1-5]
int rank: more than 3 distinct values
int recvfrom = 'value optimized out' [1-5]
int sendto: more than 3 distinct values
int size = '16' [1-5]
int step:
'4' [1]
'5' [2-5]
void * tmprecv = 'null pointer' [1-5]
void * tmpsend = '0x2 (Invalid pointer)' [1-5]
-----------------
[1-5] (5 processes)
-----------------
ompi_coll_tuned_sendrecv() at coll_tuned_util.h:60
ompi_coll_tuned_sendrecv_actual() at
coll_tuned_util.c:55
params
void * sendbuf = 'valid
pointer perm=rwxp' [1-5]
int scount = '28000'
[1-5]
ompi_datatype_t * sdatatype =
'MPI_DOUBLE_PRECISION' [1-5]
int dest: more than 3
distinct values
int stag = '-13' [1-5]
void * recvbuf = 'value
optimized out' [1-5]
int rcount = '28000'
[1-5]
ompi_datatype_t * rdatatype =
'MPI_DOUBLE_PRECISION' [1-5]
int source: more than 3
distinct values
int rtag = '-13' [1-5]
struct ompi_communicator_t * comm = 'valid
pointer perm=rwxp' [1-5]
ompi_status_public_t * status = 'null
pointer' [1-5]
locals
int err = '0' [1-5]
int line: more than 3
distinct values
ompi_request_t *[2] reqs = '{, }' [1-5]
ompi_status_public_t [2] statuses = 'value too
long to display' [1-5]
-----------------
[1-5] (5 processes)
-----------------
ompi_request_default_wait_all() at
request/req_wait.c:262
params
size_t count = '2' [1-5]
ompi_request_t ** requests: more than 3
distinct values
ompi_status_public_t * statuses = 'valid
pointer perm=rwxp ([stack])' [1-5]
locals
char [30] __PRETTY_FUNCTION__ =
'"ompi_request_default_wait_all"' [1-5]
size_t completed = 'value
optimized out' [1-5]
size_t i: more than 3
distinct values
int mpi_error = 'value
optimized out' [1-5]
size_t pending = '2' [1-5]
ompi_request_t * request = 'value
optimized out' [1-5]
ompi_request_t ** rptr = 'value
optimized out' [1-5]
size_t start: more than 3
distinct values
-----------------
[1-5] (5 processes)
-----------------
opal_condition_wait() at
../opal/threads/condition.h:99
opal_progress() at runtime/opal_progress.c:207
locals
int events = '0' [1-5]
size_t i:
'1' [1,3-5]
'2' [2]
-----------------
[1,4-5] (3 processes)
-----------------
mca_btl_sm_component_progress() at
btl_sm_component.c:391
locals
mca_btl_sm_frag_t Frag = 'value too
long to display' [1,4-5]
mca_btl_sm_fifo_t * fifo = 'value too
long to display' [1,4-5]
mca_btl_sm_frag_t * frag = '0x2
(Invalid pointer)' [1,4-5]
mca_btl_sm_hdr_t * hdr = '0x2
(Invalid pointer)' [1,4-5]
int j:
'10' [4]
'11' [1]
'4' [5]
int my_smp_rank:
'1' [1]
'4' [4]
'5' [5]
int peer_smp_rank = 'value
optimized out' [1,4-5]
int rc = '0' [1,4-5]
-----------------
[1,5] (2 processes)
-----------------
sm_fifo_read() at btl_sm.h:272
locals
void * value = '0xfffffffffffffffe
(Invalid pointer)' [1,5]
-----------------
[4] (1 processes)
-----------------
sm_fifo_read() at btl_sm.h:267
locals
void * value = '0xfffffffffffffffe
(Invalid pointer)' [4]
-----------------
[2] (1 processes)
-----------------
btl_openib_component_progress() at
btl_openib_component.c:3437
locals
int count = 'value optimized out' [2]
int i = 'value optimized out' [2]
-----------------
[3] (1 processes)
-----------------
mca_btl_sm_component_progress() at
btl_sm_component.c:357
locals
mca_btl_sm_frag_t Frag = 'value too
long to display' [3]
mca_btl_sm_fifo_t * fifo = 'value
optimized out' [3]
mca_btl_sm_frag_t * frag = 'value
optimized out' [3]
mca_btl_sm_hdr_t * hdr = 'value
optimized out' [3]
int j = '2' [3]
int my_smp_rank = '0' [3]
int peer_smp_rank = 'value
optimized out' [3]
int rc = '0' [3]
Stack trace(s) for thread: 2
-----------------
[1-5] (5 processes)
-----------------
start_thread() at ?:?
btl_openib_async_thread() at btl_openib_async.c:344
params
void * async = 'value optimized out' [1-5]
locals
char [24] __func__ = '"btl_openib_async_thread"' [1-5]
int i = '2' [1-5]
int rc = '-4' [1-5]
-----------------
[1-5] (5 processes)
-----------------
poll() at ?:?
Stack trace(s) for thread: 3
-----------------
[1-5] (5 processes)
-----------------
start_thread() at ?:?
service_thread_start() at btl_openib_fd.c:427
params
void * context = 'value optimized out' [1-5]
locals
int flags = '0' [1-5]
opal_list_item_t * item = 'valid pointer perm=rwxp' [1-5]
int rc = '0' [1-5]
fd_set read_fds_copy = '{fds_bits = {16640, 0 <repeats 15
times>}}' [1-5]
fd_set write_fds_copy = '{fds_bits = {0 <repeats 16 times>}}' [1-5]
-----------------
[1-5] (5 processes)
-----------------
select() at ?:?
