#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h> /*  getpid() */

#define NUM_SPAWNS 3

int main(int argc, char *argv[])  {
    int numtasks, rank, task1=1, task2=2, rc, count,len,dest,source,buffer[2];  
    char inmsg[30], outmsg0[]="Hello Other Task.",outmsg1[]="You are Welcome Other Task.";
    char name[MPI_MAX_PROCESSOR_NAME];
    int np =NUM_SPAWNS;
    int errcodes[NUM_SPAWNS];
    MPI_Comm parentcomm, intercomm,intracomm;
    MPI_Init( &argc, &argv );
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_get_parent( &parentcomm );
    if (parentcomm == MPI_COMM_NULL) {
        buffer[0]=atoi(argv[1]); buffer[1]=atoi(argv[2]);
        printf("Next the Spawn pid=%d \n",getpid());
        /* Create 3 more processes - Need to use the actual executables name. */
        MPI_Comm_spawn( "mpi_spawn", MPI_ARGV_NULL, np, MPI_INFO_NULL, 0, MPI_COMM_WORLD, &intercomm, errcodes );
        printf("After the Spawn\n");
        MPI_Get_processor_name(name, &len);
        MPI_Intercomm_merge(intercomm, 0, &intracomm);   /* The 0 for the merge is important here*/
        printf("I'm the parent process pid=%d  and rank= %i, runing on %s\n",getpid(),rank,name);
        MPI_Bcast(buffer, 2, MPI_INT, 0, intracomm);
    } else {
/*MPI_Comm_get_parent - Now returns the parent intercommunicator of current spawned process. */
        MPI_Comm_get_parent( &parentcomm );
        MPI_Get_processor_name(name, &len);
        MPI_Intercomm_merge(parentcomm, 1, &intracomm);   /* The 1 for the merge is important here*/
        MPI_Bcast(buffer, 2, MPI_INT, 0, intracomm);   
        MPI_Comm_rank(intracomm, &rank);//MPI_COMM_WORLD
        printf("I'm a spawned process pid=%d and rank= %i, runing on %s\n",getpid(),rank,name);
    }
    if (rank ==buffer[1]) {
        dest = buffer[0];
        source = buffer[0];
        MPI_Send(&outmsg0,strlen(outmsg0), MPI_CHAR, dest, 0,intracomm);
        printf("\nTask %i has sent its  message to Task %i. \n",rank, dest);
        MPI_Recv(&inmsg, 30, MPI_CHAR, dest, 1, intracomm,  MPI_STATUS_IGNORE);
        printf("Task %i received this  message from Task %i   %s\n\n",rank, dest, inmsg);
    } else if (rank ==buffer[0]) {
        dest = buffer[1];
        source =buffer[1];
        MPI_Recv(&inmsg,30, MPI_CHAR, source, 0, intracomm,  MPI_STATUS_IGNORE);
        printf("Task %i received this  message:   %s\n", rank,inmsg);
        MPI_Send(&outmsg1,strlen(outmsg1), MPI_CHAR, source, 1, intracomm);    
    }
    MPI_Barrier(  intracomm ) ;
    printf("Got here %i\n",rank);
    MPI_Finalize();
    return 0;
}

Two Runs

[siegelj@tc-login MPI_Revisited]$ mpi_spawn 1 2
Next the Spawn pid=26645
After the Spawn
I'm the parent process pid=26645  and rank= 0, runing on tc-login.red.lan
I'm a spawned process pid=26649 and rank= 1, runing on tc-login.red.lan
I'm a spawned process pid=26652 and rank= 3, runing on tc-login.red.lan
I'm a spawned process pid=26650 and rank= 2, runing on tc-login.red.lan

Task 2 has sent its  message to Task 1.
Task 1 received this  message:   Hello Other Task  
Task 2 received this  message from Task 1   You are Welcome Other Task

Got here 3
Got here 1
Got here 0
Got here 2
[siegelj@tc-login MPI_Revisited]$ mpi_spawn 3 0
Next the Spawn pid=26677
After the Spawn
I'm the parent process pid=26677  and rank= 0, runing on tc-login.red.lan
I'm a spawned process pid=26681 and rank= 1, runing on tc-login.red.lan
I'm a spawned process pid=26684 and rank= 3, runing on tc-login.red.lan
I'm a spawned process pid=26682 and rank= 2, runing on tc-login.red.lan

Task 0 has sent its  message to Task 3.
Task 3 received this  message:   Hello Other Task. 
Task 0 received this  message from Task 3   You are Welcome Other Task.

Got here 1
Got here 3
Got here 0
Got here 2



SLURM


[siegelj@tc-login MPI_Revisited]$ cat spawntest_2722.out
I'm a spawned process pid=3764 and rank= 1, runing on tc-node159.red.lan
I'm a spawned process pid=9401 and rank= 3, runing on tc-node161.red.lan
Task 1 received this  message:   Hello Other Task.
I'm a spawned process pid=3745 and rank= 2, runing on tc-node160.red.lan

Task 2 has sent its  message to Task 1.
Got here 1
Task 2 received this  message from Task 1   You are Welcome Other Task.

Got here 2
Got here 3
Next the Spawn pid=3745
After the Spawn
I'm the parent process pid=3745  and rank= 0, runing on tc-node159.red.lan
Got here 0
[siegelj@tc-login MPI_Revisited]$