Actual source code: ex8.c

  1: static char help[] = "Test VecScatterCreateToZero, VecScatterCreateToAll\n\n";

  3: #include <petscvec.h>
  4: int main(int argc, char **argv)
  5: {
  6:   PetscInt    i, N = 10, n = PETSC_DECIDE, low, high, onlylocal = -1;
  7:   PetscMPIInt size, rank;
  8:   Vec         x, y;
  9:   VecScatter  vscat;

 11:   PetscFunctionBeginUser;
 12:   PetscCall(PetscInitialize(&argc, &argv, NULL, help));
 13:   PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
 14:   PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
 15:   PetscCall(PetscOptionsGetInt(NULL, NULL, "-n", &N, NULL));

 17:   /* Trigger special case in VecScatterCreateToAll to deal with the one-to-all pattern */
 18:   PetscCall(PetscOptionsGetInt(NULL, NULL, "-onlylocal", &onlylocal, NULL));
 19:   if (onlylocal >= 0 && onlylocal < size) n = (rank == onlylocal ? N : 0);

 21:   PetscCall(VecCreate(PETSC_COMM_WORLD, &x));
 22:   PetscCall(VecSetFromOptions(x));
 23:   PetscCall(VecSetSizes(x, n, N));
 24:   PetscCall(VecGetOwnershipRange(x, &low, &high));
 25:   PetscCall(PetscObjectSetName((PetscObject)x, "x"));

 27:   /*-------------------------------------*/
 28:   /*       VecScatterCreateToZero        */
 29:   /*-------------------------------------*/

 31:   /* MPI vec x = [0, 1, 2, .., N-1] */
 32:   for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i, INSERT_VALUES));
 33:   PetscCall(VecAssemblyBegin(x));
 34:   PetscCall(VecAssemblyEnd(x));

 36:   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting VecScatterCreateToZero\n"));
 37:   PetscCall(VecScatterCreateToZero(x, &vscat, &y));
 38:   PetscCall(PetscObjectSetName((PetscObject)y, "y"));

 40:   /* Test PetscSFBcastAndOp with op = MPI_REPLACE, which does y = x on rank 0 */
 41:   PetscCall(VecScatterBegin(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
 42:   PetscCall(VecScatterEnd(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
 43:   if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));

 45:   /* Test PetscSFBcastAndOp with op = MPI_SUM, which does y += x */
 46:   PetscCall(VecScatterBegin(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
 47:   PetscCall(VecScatterEnd(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
 48:   if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));

 50:   /* Test PetscSFReduce with op = MPI_REPLACE, which does x = y */
 51:   PetscCall(VecScatterBegin(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
 52:   PetscCall(VecScatterEnd(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
 53:   PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));

 55:   /* Test PetscSFReduce with op = MPI_SUM, which does x += y on x's local part on rank 0*/
 56:   PetscCall(VecScatterBegin(vscat, y, x, ADD_VALUES, SCATTER_REVERSE_LOCAL));
 57:   PetscCall(VecScatterEnd(vscat, y, x, ADD_VALUES, SCATTER_REVERSE_LOCAL));
 58:   PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));

 60:   PetscCall(VecDestroy(&y));
 61:   PetscCall(VecScatterDestroy(&vscat));

 63:   /*-------------------------------------*/
 64:   /*       VecScatterCreateToAll         */
 65:   /*-------------------------------------*/
 66:   for (i = low; i < high; i++) PetscCall(VecSetValue(x, i, (PetscScalar)i, INSERT_VALUES));
 67:   PetscCall(VecAssemblyBegin(x));
 68:   PetscCall(VecAssemblyEnd(x));

 70:   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\nTesting VecScatterCreateToAll\n"));

 72:   PetscCall(VecScatterCreateToAll(x, &vscat, &y));
 73:   PetscCall(PetscObjectSetName((PetscObject)y, "y"));

 75:   /* Test PetscSFBcastAndOp with op = MPI_REPLACE, which does y = x on all ranks */
 76:   PetscCall(VecScatterBegin(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
 77:   PetscCall(VecScatterEnd(vscat, x, y, INSERT_VALUES, SCATTER_FORWARD));
 78:   if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));

 80:   /* Test PetscSFBcastAndOp with op = MPI_SUM, which does y += x */
 81:   PetscCall(VecScatterBegin(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
 82:   PetscCall(VecScatterEnd(vscat, x, y, ADD_VALUES, SCATTER_FORWARD));
 83:   if (rank == 0) PetscCall(VecView(y, PETSC_VIEWER_STDOUT_SELF));

 85:   /* Test PetscSFReduce with op = MPI_REPLACE, which does x = y */
 86:   PetscCall(VecScatterBegin(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
 87:   PetscCall(VecScatterEnd(vscat, y, x, INSERT_VALUES, SCATTER_REVERSE));
 88:   PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));

 90:   /* Test PetscSFReduce with op = MPI_SUM, which does x += size*y */
 91:   PetscCall(VecScatterBegin(vscat, y, x, ADD_VALUES, SCATTER_REVERSE));
 92:   PetscCall(VecScatterEnd(vscat, y, x, ADD_VALUES, SCATTER_REVERSE));
 93:   PetscCall(VecView(x, PETSC_VIEWER_STDOUT_WORLD));
 94:   PetscCall(VecDestroy(&x));
 95:   PetscCall(VecDestroy(&y));
 96:   PetscCall(VecScatterDestroy(&vscat));

 98:   PetscCall(PetscFinalize());
 99:   return 0;
100: }

102: /*TEST

104:    testset:
105:       # N=10 is divisible by nsize, to trigger Allgather/Gather in SF
106:       nsize: 2
107:       # Exact numbers really matter here
108:       diff_args: -j
109:       filter: grep -v "type"
110:       output_file: output/ex8_1.out

112:       test:
113:         suffix: 1_standard

115:       test:
116:         suffix: 1_cuda
117:         # sf_backend cuda is not needed if compiling only with cuda
118:         args: -vec_type cuda -sf_backend cuda
119:         requires: cuda

121:       test:
122:         suffix: 1_hip
123:         args: -vec_type hip -sf_backend hip
124:         requires: hip

126:       test:
127:         suffix: 1_kokkos
128:         args: -vec_type kokkos
129:         requires: kokkos_kernels

131:       test:
132:         suffix: 1_kokkos_sf
133:         args: -vec_type kokkos -sf_backend kokkos
134:         requires: kokkos_kernels

136:       test:
137:         suffix: 1_cuda_aware_mpi
138:         # sf_backend cuda is not needed if compiling only with cuda
139:         args: -vec_type cuda -sf_backend cuda
140:         requires: cuda defined(PETSC_HAVE_MPI_GPU_AWARE)

142:    testset:
143:       # N=10 is not divisible by nsize, to trigger Allgatherv/Gatherv in SF
144:       nsize: 3
145:       # Exact numbers really matter here
146:       diff_args: -j
147:       filter: grep -v "type" | grep -v "Process "
148:       output_file: output/ex8_2.out

150:       test:
151:         suffix: 2_standard

153:       test:
154:         suffix: 2_cuda
155:         # sf_backend cuda is not needed if compiling only with cuda
156:         args: -vec_type cuda -sf_backend cuda
157:         requires: cuda

159:       test:
160:         suffix: 2_hip
161:         # sf_backend hip is not needed if compiling only with hip
162:         args: -vec_type hip -sf_backend hip
163:         requires: hip

165:       test:
166:         suffix: 2_cuda_aware_mpi
167:         args: -vec_type cuda
168:         requires: cuda defined(PETSC_HAVE_MPI_GPU_AWARE)

170:    testset:
171:       # trigger one-to-all pattern in Allgatherv
172:       nsize: 3
173:       diff_args: -j
174:       filter: grep -v "type" | grep -v "Process "
175:       output_file: output/ex8_3.out
176:       args: -onlylocal 1

178:       test:
179:         suffix: 2_standard_onetoall

181:       test:
182:         suffix: 2_cuda_onetoall
183:         # sf_backend cuda is not needed if compiling only with cuda
184:         args: -vec_type cuda -sf_backend cuda
185:         requires: cuda

187:       test:
188:         suffix: 2_hip_onetoall
189:         # sf_backend hip is not needed if compiling only with hip
190:         args: -vec_type hip -sf_backend hip
191:         requires: hip

193:       test:
194:         suffix: 2_cuda_aware_mpi_onetoall
195:         args: -vec_type cuda
196:         requires: cuda defined(PETSC_HAVE_MPI_GPU_AWARE)

198: TEST*/