[OMPI users] Problem with MPI_Comm_split and intercomms

[Bert Wesarg]

Hello list,

i get some erroneous results from calls to MPI_Comm_split with an
intercommunicator as communicator.

this is a brokendown testcase, the source is attached:

test_split.c:

mpi_comm_spwan(world, "test_split_client", 1, &inter)
for i=1:np
  if rank < i
    mpi_comm_split(inter, 0, 0, &inter2)
    /* print new rank and size */
    mpi_comm_free(&inter2)
  else
    mpi_comm_split(inter, MPI_UNDEFINED, 0, &inter2)

test_split_client.c:

mpi_comm_get_parent(&inter)
while true
  mpi_comm_split(inter, 0, 0, &inter2)
  /* print new remote size */
  mpi_comm_free(&inter2)
mpi_comm_free(&inter)


first, my expected results:
np = 2:

        test_split:0     test_split:1     test_split_client:0
i = 1:
        new rank = 0                      new remote size = 1
        new size = 1

i = 2:
        new rank = 0     new rank = 1     new remote size = 2
        new size = 2     new size = 2
...

now some actually results:
a)
        test_split:0     test_split:1     test_split_client:0
i = 1:
        new rank = 0                      new remote size = 1
        new size = 2

i = 2:
        new rank = 0     new rank = 1     new remote size = 2
        new size = 2     new size = 2

b)
        test_split:0     test_split:1     test_split_client:0
i = 1:
        new rank = 0                      new remote size = 1
        new size = 1

i = 2:
        new rank = 0     new rank = 1     new remote size = 2
        new size = 1     new size = 2

c)
        test_split:0     test_split:1     test_split_client:0
i = 1:
        new rank = 0                      new remote size = 1
        new size = 2

i = 2:
        new rank = 0     new rank = 1     new remote size = 2
        new size = 1     new size = 2

something with the new size in rank 0 is wrong or the remote size

can someone please pointout if my code is wrong, or something with open-mpi?

greatings
bert wesarg

ps: these test were run on an dual smp machine and open-mpi 1.0.2
pps: i attached also an split test with intracommunicators, these results
are all expected



#define _GNU_SOURCE
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <math.h>
#include <getopt.h>

#include <mpi.h>

int
main(int ac, char *av[])
{
    MPI_Comm inter, inter2;
    int r, s, err, i, r2, s2, rs, isinter;

    MPI_Init(&ac, &av);
    MPI_Comm_size(MPI_COMM_WORLD, &s);
    MPI_Comm_rank(MPI_COMM_WORLD, &r);

    MPI_Comm_spawn(
        "./test_split_client", MPI_ARGV_NULL, 1, MPI_INFO_NULL, 0,
        MPI_COMM_WORLD, &inter, &err
    );

    for (i = 1; i <= s; i++) {

        MPI_Barrier(inter);

        MPI_Comm_split(
            inter, (r < i) ? 0 : MPI_UNDEFINED, 0, &inter2
        );
        printf(
            "%d:%d: split color %d, key %d\n",
            i, r, (r < i) ? 0 : MPI_UNDEFINED, 0
        );
        MPI_Barrier(inter);

        if (r < i) {
            err = MPI_Comm_test_inter(inter2, &isinter);
            printf(
                "%d:%d: is inter %s\n",
                i, r, isinter ? "true" : "false"
            );

            err = MPI_Comm_size(inter2, &s2);

            err = MPI_Comm_remote_size(inter2, &rs);

            err = MPI_Comm_rank(inter2, &r2);

            printf(
                "%d:%d: new rank %d, new size %d, remote size %d\n",
                i, r, r2, s2, rs
            );

            MPI_Barrier(inter2);

            err = MPI_Comm_free(&inter2);
        }
        MPI_Barrier(inter);
    }

    MPI_Barrier(inter);

    MPI_Comm_free(&inter);

    MPI_Finalize();
    return 0;
}

#define _GNU_SOURCE
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <math.h>
#include <getopt.h>

#include <mpi.h>

int
main(int ac, char *av[])
{
    MPI_Comm inter, inter2;
    int r, s, rs;

    MPI_Init(&ac, &av);
    MPI_Comm_size(MPI_COMM_WORLD, &s);
    MPI_Comm_rank(MPI_COMM_WORLD, &r);

    MPI_Comm_get_parent(&inter);

    while (true) {
        MPI_Barrier(inter);

        MPI_Comm_split(inter, 0, 0, &inter2);

        MPI_Barrier(inter);

        MPI_Comm_remote_size(inter2, &rs);
        printf("c:%d: remote size %d\n", r, rs);
        MPI_Barrier(inter2);
        MPI_Comm_free(&inter2);

        MPI_Barrier(inter);
    }
    MPI_Barrier(inter);

    MPI_Comm_free(&inter);
    MPI_Finalize();
    return 0;
}

#define _GNU_SOURCE
#include <unistd.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <stdint.h>
#include <stdbool.h>
#include <math.h>
#include <getopt.h>

#include <mpi.h>

int
main(int ac, char *av[])
{
    MPI_Comm split;
    int r, s, i, r2, s2;
    int send;

    MPI_Init(&ac, &av);
    MPI_Comm_size(MPI_COMM_WORLD, &s);
    MPI_Comm_rank(MPI_COMM_WORLD, &r);

    send = 1;
    for (i = 1; i <= s; i++) {
        MPI_Barrier(MPI_COMM_WORLD);

        printf(
            "%d:%d: split color %d, key %d\n",
            i, r, (r < i) ? 0 : MPI_UNDEFINED, 0
        );
        MPI_Comm_split(
            MPI_COMM_WORLD, (r < i) ? 0 : MPI_UNDEFINED, 0, &split
        );
        if (r < i) {
            MPI_Comm_size(split, &s2);
            MPI_Comm_rank(split, &r2);
            printf(
                "%d:%d: new rank %d, new size %d\n", i, r, r2, s2
            );
            MPI_Barrier(split);
            MPI_Comm_free(&split);
        }
        MPI_Barrier(MPI_COMM_WORLD);
    }

    MPI_Finalize();
    return 0;
}