1: static const char help[] = "Test overlapped communication on a single star forest (PetscSF)\n\n";

  3: #include <petscvec.h>
  4: #include <petscsf.h>

  6: int main(int argc, char **argv)
  7: {
  8:   PetscSF      sf;
  9:   Vec          A, Aout;
 10:   Vec          B, Bout;
 11:   PetscScalar *bufA;
 12:   PetscScalar *bufAout;
 13:   PetscScalar *bufB;
 14:   PetscScalar *bufBout;
 15:   PetscMPIInt  rank, size;
 16:   PetscInt     nroots, nleaves;
 17:   PetscInt    *ilocal;
 18:   PetscSFNode *iremote;

 20:   PetscFunctionBeginUser;
 21:   PetscCall(PetscInitialize(&argc, &argv, NULL, help));
 22:   PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
 23:   PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));

 25:   PetscCheck(size == 2, PETSC_COMM_WORLD, PETSC_ERR_USER, "Only coded for two MPI processes");

 27:   PetscCall(PetscSFCreate(PETSC_COMM_WORLD, &sf));
 28:   PetscCall(PetscSFSetFromOptions(sf));

 30:   nleaves = 2;
 31:   nroots  = 1;
 32:   PetscCall(PetscMalloc1(nleaves, &ilocal));

 34:   for (PetscInt i = 0; i < nleaves; i++) ilocal[i] = i;

 36:   PetscCall(PetscMalloc1(nleaves, &iremote));
 37:   if (rank == 0) {
 38:     iremote[0].rank  = 0;
 39:     iremote[0].index = 0;
 40:     iremote[1].rank  = 1;
 41:     iremote[1].index = 0;
 42:   } else {
 43:     iremote[0].rank  = 1;
 44:     iremote[0].index = 0;
 45:     iremote[1].rank  = 0;
 46:     iremote[1].index = 0;
 47:   }
 48:   PetscCall(PetscSFSetGraph(sf, nroots, nleaves, ilocal, PETSC_OWN_POINTER, iremote, PETSC_OWN_POINTER));
 49:   PetscCall(PetscSFSetUp(sf));
 50:   PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
 51:   PetscCall(VecCreate(PETSC_COMM_WORLD, &A));
 52:   PetscCall(VecSetSizes(A, 2, PETSC_DETERMINE));
 53:   PetscCall(VecSetFromOptions(A));
 54:   PetscCall(VecSetUp(A));

 56:   PetscCall(VecDuplicate(A, &B));
 57:   PetscCall(VecDuplicate(A, &Aout));
 58:   PetscCall(VecDuplicate(A, &Bout));
 59:   PetscCall(VecGetArray(A, &bufA));
 60:   PetscCall(VecGetArray(B, &bufB));
 61:   for (PetscInt i = 0; i < 2; i++) {
 62:     bufA[i] = (PetscScalar)rank;
 63:     bufB[i] = (PetscScalar)rank + 10.0;
 64:   }
 65:   PetscCall(VecRestoreArray(A, &bufA));
 66:   PetscCall(VecRestoreArray(B, &bufB));

 68:   PetscCall(VecGetArrayRead(A, (const PetscScalar **)&bufA));
 69:   PetscCall(VecGetArrayRead(B, (const PetscScalar **)&bufB));
 70:   PetscCall(VecGetArray(Aout, &bufAout));
 71:   PetscCall(VecGetArray(Bout, &bufBout));
 72:   PetscCall(PetscSFBcastBegin(sf, MPIU_SCALAR, (const void *)bufA, (void *)bufAout, MPI_REPLACE));
 73:   PetscCall(PetscSFBcastBegin(sf, MPIU_SCALAR, (const void *)bufB, (void *)bufBout, MPI_REPLACE));
 74:   PetscCall(PetscSFBcastEnd(sf, MPIU_SCALAR, (const void *)bufA, (void *)bufAout, MPI_REPLACE));
 75:   PetscCall(PetscSFBcastEnd(sf, MPIU_SCALAR, (const void *)bufB, (void *)bufBout, MPI_REPLACE));
 76:   PetscCall(VecRestoreArrayRead(A, (const PetscScalar **)&bufA));
 77:   PetscCall(VecRestoreArrayRead(B, (const PetscScalar **)&bufB));
 78:   PetscCall(VecRestoreArray(Aout, &bufAout));
 79:   PetscCall(VecRestoreArray(Bout, &bufBout));

 81:   PetscCall(VecView(Aout, PETSC_VIEWER_STDOUT_WORLD));
 82:   PetscCall(VecView(Bout, PETSC_VIEWER_STDOUT_WORLD));
 83:   PetscCall(VecDestroy(&A));
 84:   PetscCall(VecDestroy(&B));
 85:   PetscCall(VecDestroy(&Aout));
 86:   PetscCall(VecDestroy(&Bout));
 87:   PetscCall(PetscSFDestroy(&sf));

 89:   PetscCall(PetscFinalize());
 90:   return 0;
 91: }

 93: /*TEST

 95:    test:
 96:       suffix: basic
 97:       nsize: 2
 98:       filter: grep -v "type" | grep -v "sort"
 99:       args: -sf_type basic

101:    test:
102:       suffix: window
103:       nsize: 2
104:       filter: grep -v "type" | grep -v "sort"
105:       output_file: output/ex2_basic.out
106:       args: -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
107:       requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)

109:    # The nightly test suite with MPICH uses ch3:sock, which is broken when winsize == 0 in some of the processes
110:    test:
111:       suffix: window_shared
112:       nsize: 2
113:       filter: grep -v "type" | grep -v "sort"
114:       output_file: output/ex2_basic.out
115:       args: -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor shared
116:       requires: defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY) !defined(PETSC_HAVE_MPICH) defined(PETSC_HAVE_MPI_ONE_SIDED)

118: TEST*/