Actual source code: ex14.c
1: static char help[] = "Test event log of VecScatter with various block sizes\n\n";
3: #include <petscvec.h>
5: int main(int argc, char **argv)
6: {
7: PetscInt i, j, low, high, n = 256, N, tot_errors;
8: PetscInt bs = 1, ix[2], iy[2];
9: PetscMPIInt nproc, rank;
10: PetscScalar *xval;
11: const PetscScalar *yval;
12: Vec x, y;
13: IS isx, isy;
14: VecScatter ctx;
15: const PetscInt niter = 10;
16: PetscLogStage stage1, stage2;
17: PetscLogEvent event1, event2;
19: PetscFunctionBegin;
20: PetscFunctionBeginUser;
21: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
22: PetscCall(PetscLogDefaultBegin());
23: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &nproc));
24: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
26: PetscCall(PetscLogStageRegister("Scatter(bs=1)", &stage1));
27: PetscCall(PetscLogEventRegister("VecScatter(bs=1)", PETSC_OBJECT_CLASSID, &event1));
28: PetscCall(PetscLogStageRegister("Scatter(bs=4)", &stage2));
29: PetscCall(PetscLogEventRegister("VecScatter(bs=4)", PETSC_OBJECT_CLASSID, &event2));
31: /* Create a parallel vector x and a sequential vector y */
32: PetscCall(VecCreate(PETSC_COMM_WORLD, &x));
33: PetscCall(VecSetSizes(x, n, PETSC_DECIDE));
34: PetscCall(VecSetFromOptions(x));
35: PetscCall(VecGetOwnershipRange(x, &low, &high));
36: PetscCall(VecGetSize(x, &N));
37: PetscCall(VecCreateSeq(PETSC_COMM_SELF, n, &y));
39: /*=======================================
40: test VecScatter with bs = 1
41: ======================================*/
43: /* the code works as if we are going to do 3-point stencil computations on a 1D domain x,
44: which has periodic boundary conditions but the two ghosts are scatterred to beginning of y.
45: */
46: bs = 1;
47: ix[0] = rank ? low - 1 : N - 1; /* ix[] contains global indices of the two ghost points */
48: ix[1] = (rank != nproc - 1) ? high : 0;
49: iy[0] = 0;
50: iy[1] = 1;
52: PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2, ix, PETSC_COPY_VALUES, &isx));
53: PetscCall(ISCreateGeneral(PETSC_COMM_SELF, 2, iy, PETSC_COPY_VALUES, &isy));
54: PetscCall(VecScatterCreate(x, isx, y, isy, &ctx));
55: PetscCall(VecScatterSetUp(ctx));
57: PetscCall(PetscLogStagePush(stage1));
58: PetscCall(PetscLogEventBegin(event1, 0, 0, 0, 0));
59: tot_errors = 0;
60: for (i = 0; i < niter; i++) {
61: /* set x = 0+i, 1+i, 2+i, ..., N-1+i */
62: PetscCall(VecGetArray(x, &xval));
63: for (j = 0; j < n; j++) xval[j] = (PetscScalar)(low + j + i);
64: PetscCall(VecRestoreArray(x, &xval));
65: /* scatter the ghosts to y */
66: PetscCall(VecScatterBegin(ctx, x, y, INSERT_VALUES, SCATTER_FORWARD));
67: PetscCall(VecScatterEnd(ctx, x, y, INSERT_VALUES, SCATTER_FORWARD));
68: /* check if y has correct values */
69: PetscCall(VecGetArrayRead(y, &yval));
70: if ((PetscInt)PetscRealPart(yval[0]) != ix[0] + i) tot_errors++;
71: if ((PetscInt)PetscRealPart(yval[1]) != ix[1] + i) tot_errors++;
72: PetscCall(VecRestoreArrayRead(y, &yval));
73: }
74: PetscCall(PetscLogEventEnd(event1, 0, 0, 0, 0));
75: PetscCall(PetscLogStagePop());
77: /* check if we found wrong values on any processors */
78: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &tot_errors, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD));
79: if (tot_errors) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Error: wrong values were scatterred in vecscatter with bs = %" PetscInt_FMT "\n", bs));
81: /* print out event log of VecScatter(bs=1) */
82: if (PetscDefined(USE_LOG)) {
83: PetscEventPerfInfo eventInfo;
84: PetscInt tot_msg, tot_len, avg_len;
86: PetscCall(PetscLogEventGetPerfInfo(stage1, event1, &eventInfo));
87: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &eventInfo.numMessages, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, PETSC_COMM_WORLD));
88: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &eventInfo.messageLength, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, PETSC_COMM_WORLD));
89: tot_msg = (PetscInt)eventInfo.numMessages * 0.5; /* two MPI calls (Send & Recv) per message */
90: tot_len = (PetscInt)eventInfo.messageLength * 0.5;
91: avg_len = tot_msg ? (PetscInt)(eventInfo.messageLength / eventInfo.numMessages) : 0;
92: /* when nproc > 2, tot_msg = 2*nproc*niter, tot_len = tot_msg*sizeof(PetscScalar)*bs */
93: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "VecScatter(bs=%" PetscInt_FMT ") has sent out %" PetscInt_FMT " messages, total %" PetscInt_FMT " bytes, with average length %" PetscInt_FMT " bytes\n", bs, tot_msg, tot_len, avg_len));
94: }
96: PetscCall(ISDestroy(&isx));
97: PetscCall(ISDestroy(&isy));
98: PetscCall(VecScatterDestroy(&ctx));
100: /*=======================================
101: test VecScatter with bs = 4
102: ======================================*/
104: /* similar to the 3-point stencil above, except that this time a ghost is a block */
105: bs = 4; /* n needs to be a multiple of bs to make the following code work */
106: ix[0] = rank ? low / bs - 1 : N / bs - 1; /* ix[] contains global indices of the two ghost blocks */
107: ix[1] = (rank != nproc - 1) ? high / bs : 0;
108: iy[0] = 0;
109: iy[1] = 1;
111: PetscCall(ISCreateBlock(PETSC_COMM_SELF, bs, 2, ix, PETSC_COPY_VALUES, &isx));
112: PetscCall(ISCreateBlock(PETSC_COMM_SELF, bs, 2, iy, PETSC_COPY_VALUES, &isy));
114: PetscCall(VecScatterCreate(x, isx, y, isy, &ctx));
115: /* Call SetUp explicitly, otherwise messages in implicit SetUp will be counted in events below */
116: PetscCall(VecScatterSetUp(ctx));
118: PetscCall(PetscLogStagePush(stage2));
119: PetscCall(PetscLogEventBegin(event2, 0, 0, 0, 0));
120: tot_errors = 0;
121: for (i = 0; i < niter; i++) {
122: /* set x = 0+i, 1+i, 2+i, ..., N-1+i */
123: PetscCall(VecGetArray(x, &xval));
124: for (j = 0; j < n; j++) xval[j] = (PetscScalar)(low + j + i);
125: PetscCall(VecRestoreArray(x, &xval));
126: /* scatter the ghost blocks to y */
127: PetscCall(VecScatterBegin(ctx, x, y, INSERT_VALUES, SCATTER_FORWARD));
128: PetscCall(VecScatterEnd(ctx, x, y, INSERT_VALUES, SCATTER_FORWARD));
129: /* check if y has correct values */
130: PetscCall(VecGetArrayRead(y, &yval));
131: if ((PetscInt)PetscRealPart(yval[0]) != ix[0] * bs + i) tot_errors++;
132: if ((PetscInt)PetscRealPart(yval[bs]) != ix[1] * bs + i) tot_errors++;
133: PetscCall(VecRestoreArrayRead(y, &yval));
134: }
135: PetscCall(PetscLogEventEnd(event2, 0, 0, 0, 0));
136: PetscCall(PetscLogStagePop());
138: /* check if we found wrong values on any processors */
139: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &tot_errors, 1, MPIU_INT, MPI_SUM, PETSC_COMM_WORLD));
140: if (tot_errors) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Error: wrong values were scatterred in vecscatter with bs = %" PetscInt_FMT "\n", bs));
142: /* print out event log of VecScatter(bs=4) */
143: if (PetscDefined(USE_LOG)) {
144: PetscEventPerfInfo eventInfo;
145: PetscInt tot_msg, tot_len, avg_len;
147: PetscCall(PetscLogEventGetPerfInfo(stage2, event2, &eventInfo));
148: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &eventInfo.numMessages, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, PETSC_COMM_WORLD));
149: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &eventInfo.messageLength, 1, MPIU_PETSCLOGDOUBLE, MPI_SUM, PETSC_COMM_WORLD));
150: tot_msg = (PetscInt)eventInfo.numMessages * 0.5; /* two MPI calls (Send & Recv) per message */
151: tot_len = (PetscInt)eventInfo.messageLength * 0.5;
152: avg_len = tot_msg ? (PetscInt)(eventInfo.messageLength / eventInfo.numMessages) : 0;
153: /* when nproc > 2, tot_msg = 2*nproc*niter, tot_len = tot_msg*sizeof(PetscScalar)*bs */
154: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "VecScatter(bs=%" PetscInt_FMT ") has sent out %" PetscInt_FMT " messages, total %" PetscInt_FMT " bytes, with average length %" PetscInt_FMT " bytes\n", bs, tot_msg, tot_len, avg_len));
155: }
157: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Program finished\n"));
158: PetscCall(ISDestroy(&isx));
159: PetscCall(ISDestroy(&isy));
160: PetscCall(VecScatterDestroy(&ctx));
161: PetscCall(VecDestroy(&x));
162: PetscCall(VecDestroy(&y));
163: PetscCall(PetscFinalize());
164: return 0;
165: }
167: /*TEST
169: test:
170: nsize: 4
171: args:
172: requires: double defined(PETSC_USE_LOG)
174: TEST*/