Actual source code: ex1.c
1: static const char help[] = "Test star forest communication (PetscSF)\n\n";
3: /*
4: Description: A star is a simple tree with one root and zero or more leaves.
5: A star forest is a union of disjoint stars.
6: Many common communication patterns can be expressed as updates of rootdata using leafdata and vice-versa.
7: This example creates a star forest, communicates values using the graph (see options for types of communication), views the graph, then destroys it.
8: */
10: /*
11: Include petscsf.h so we can use PetscSF objects. Note that this automatically
12: includes petscsys.h.
13: */
14: #include <petscsf.h>
15: #include <petscviewer.h>
17: /* like PetscSFView() but with alternative array of local indices */
18: static PetscErrorCode PetscSFViewCustomLocals_Private(PetscSF sf, const PetscInt locals[], PetscViewer viewer)
19: {
20: const PetscSFNode *iremote;
21: PetscInt i, nroots, nleaves;
22: PetscMPIInt rank, nranks;
24: PetscFunctionBeginUser;
25: PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)sf), &rank));
26: PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, NULL, &iremote));
27: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, NULL, NULL, NULL));
28: PetscCall(PetscViewerASCIIPushTab(viewer));
29: PetscCall(PetscViewerASCIIPushSynchronized(viewer));
30: PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Number of roots=%" PetscInt_FMT ", leaves=%" PetscInt_FMT ", remote ranks=%d\n", rank, nroots, nleaves, nranks));
31: for (i = 0; i < nleaves; i++) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] %" PetscInt_FMT " <- (%d,%" PetscInt_FMT ")\n", rank, locals[i], (PetscMPIInt)iremote[i].rank, iremote[i].index));
32: PetscCall(PetscViewerFlush(viewer));
33: PetscCall(PetscViewerASCIIPopTab(viewer));
34: PetscCall(PetscViewerASCIIPopSynchronized(viewer));
35: PetscFunctionReturn(PETSC_SUCCESS);
36: }
38: int main(int argc, char **argv)
39: {
40: PetscInt i, bs = 2, nroots, nrootsalloc, nleaves, nleavesalloc, *mine, stride;
41: PetscSFNode *remote;
42: PetscMPIInt rank, size;
43: PetscSF sf, vsf;
44: PetscBool test_all, test_bcast, test_bcastop, test_reduce, test_degree, test_fetchandop, test_gather, test_scatter, test_embed, test_invert, test_sf_distribute, test_char, test_vector = PETSC_FALSE;
45: MPI_Op mop = MPI_OP_NULL; /* initialize to prevent compiler warnings with cxx_quad build */
46: char opstring[256];
47: PetscBool strflg;
49: PetscFunctionBeginUser;
50: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
51: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
52: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
54: PetscOptionsBegin(PETSC_COMM_WORLD, "", "PetscSF Test Options", "none");
55: test_all = PETSC_FALSE;
56: PetscCall(PetscOptionsBool("-test_all", "Test all SF communications", "", test_all, &test_all, NULL));
57: test_bcast = test_all;
58: PetscCall(PetscOptionsBool("-test_bcast", "Test broadcast", "", test_bcast, &test_bcast, NULL));
59: test_bcastop = test_all;
60: PetscCall(PetscOptionsBool("-test_bcastop", "Test broadcast and reduce", "", test_bcastop, &test_bcastop, NULL));
61: test_reduce = test_all;
62: PetscCall(PetscOptionsBool("-test_reduce", "Test reduction", "", test_reduce, &test_reduce, NULL));
63: test_char = test_all;
64: PetscCall(PetscOptionsBool("-test_char", "Test signed char, unsigned char, and char", "", test_char, &test_char, NULL));
65: mop = MPI_SUM;
66: PetscCall(PetscStrncpy(opstring, "sum", sizeof(opstring)));
67: PetscCall(PetscOptionsString("-test_op", "Designate which MPI_Op to use", "", opstring, opstring, sizeof(opstring), NULL));
68: PetscCall(PetscStrcmp("sum", opstring, &strflg));
69: if (strflg) mop = MPIU_SUM;
70: PetscCall(PetscStrcmp("prod", opstring, &strflg));
71: if (strflg) mop = MPI_PROD;
72: PetscCall(PetscStrcmp("max", opstring, &strflg));
73: if (strflg) mop = MPI_MAX;
74: PetscCall(PetscStrcmp("min", opstring, &strflg));
75: if (strflg) mop = MPI_MIN;
76: PetscCall(PetscStrcmp("land", opstring, &strflg));
77: if (strflg) mop = MPI_LAND;
78: PetscCall(PetscStrcmp("band", opstring, &strflg));
79: if (strflg) mop = MPI_BAND;
80: PetscCall(PetscStrcmp("lor", opstring, &strflg));
81: if (strflg) mop = MPI_LOR;
82: PetscCall(PetscStrcmp("bor", opstring, &strflg));
83: if (strflg) mop = MPI_BOR;
84: PetscCall(PetscStrcmp("lxor", opstring, &strflg));
85: if (strflg) mop = MPI_LXOR;
86: PetscCall(PetscStrcmp("bxor", opstring, &strflg));
87: if (strflg) mop = MPI_BXOR;
88: test_degree = test_all;
89: PetscCall(PetscOptionsBool("-test_degree", "Test computation of vertex degree", "", test_degree, &test_degree, NULL));
90: test_fetchandop = test_all;
91: PetscCall(PetscOptionsBool("-test_fetchandop", "Test atomic Fetch-And-Op", "", test_fetchandop, &test_fetchandop, NULL));
92: test_gather = test_all;
93: PetscCall(PetscOptionsBool("-test_gather", "Test point gather", "", test_gather, &test_gather, NULL));
94: test_scatter = test_all;
95: PetscCall(PetscOptionsBool("-test_scatter", "Test point scatter", "", test_scatter, &test_scatter, NULL));
96: test_embed = test_all;
97: PetscCall(PetscOptionsBool("-test_embed", "Test point embed", "", test_embed, &test_embed, NULL));
98: test_invert = test_all;
99: PetscCall(PetscOptionsBool("-test_invert", "Test point invert", "", test_invert, &test_invert, NULL));
100: stride = 1;
101: PetscCall(PetscOptionsInt("-stride", "Stride for leaf and root data", "", stride, &stride, NULL));
102: test_sf_distribute = PETSC_FALSE;
103: PetscCall(PetscOptionsBool("-test_sf_distribute", "Create an SF that 'distributes' to each process, like an alltoall", "", test_sf_distribute, &test_sf_distribute, NULL));
104: PetscCall(PetscOptionsString("-test_op", "Designate which MPI_Op to use", "", opstring, opstring, sizeof(opstring), NULL));
105: PetscCall(PetscOptionsInt("-bs", "Block size for vectorial SF", "", bs, &bs, NULL));
106: PetscCall(PetscOptionsBool("-test_vector", "Run tests using the vectorial SF", "", test_vector, &test_vector, NULL));
107: PetscOptionsEnd();
109: if (test_sf_distribute) {
110: nroots = size;
111: nrootsalloc = size;
112: nleaves = size;
113: nleavesalloc = size;
114: mine = NULL;
115: PetscCall(PetscMalloc1(nleaves, &remote));
116: for (PetscMPIInt ii = 0; ii < size; ii++) {
117: remote[ii].rank = ii;
118: remote[ii].index = rank;
119: }
120: } else {
121: nroots = 2 + (PetscInt)(rank == 0);
122: nrootsalloc = nroots * stride;
123: nleaves = 2 + (PetscInt)(rank > 0);
124: nleavesalloc = nleaves * stride;
125: mine = NULL;
126: if (stride > 1) {
127: PetscInt i;
129: PetscCall(PetscMalloc1(nleaves, &mine));
130: for (i = 0; i < nleaves; i++) mine[i] = stride * i;
131: }
132: PetscCall(PetscMalloc1(nleaves, &remote));
133: /* Left periodic neighbor */
134: remote[0].rank = (rank + size - 1) % size;
135: remote[0].index = 1 * stride;
136: /* Right periodic neighbor */
137: remote[1].rank = (rank + 1) % size;
138: remote[1].index = 0 * stride;
139: if (rank > 0) { /* All processes reference rank 0, index 1 */
140: remote[2].rank = 0;
141: remote[2].index = 2 * stride;
142: }
143: }
145: /* Create a star forest for communication */
146: PetscCall(PetscSFCreate(PETSC_COMM_WORLD, &sf));
147: PetscCall(PetscSFSetFromOptions(sf));
148: PetscCall(PetscSFSetGraph(sf, nrootsalloc, nleaves, mine, PETSC_OWN_POINTER, remote, PETSC_OWN_POINTER));
149: PetscCall(PetscSFSetUp(sf));
151: /* We can also obtain a strided SF to communicate interleaved blocks of data */
152: PetscCall(PetscSFCreateStridedSF(sf, bs, PETSC_DECIDE, PETSC_DECIDE, &vsf));
153: if (test_vector) { /* perform all tests below using the vectorial SF */
154: PetscSF t = sf;
156: sf = vsf;
157: vsf = t;
159: nroots *= bs;
160: nleaves *= bs;
161: nrootsalloc *= bs;
162: nleavesalloc *= bs;
163: }
164: PetscCall(PetscSFDestroy(&vsf));
166: /* View graph, mostly useful for debugging purposes. */
167: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_INFO_DETAIL));
168: PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
169: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
171: if (test_bcast) { /* broadcast rootdata into leafdata */
172: PetscInt *rootdata, *leafdata;
173: /* Allocate space for send and receive buffers. This example communicates PetscInt, but other types, including
174: * user-defined structures, could also be used. */
175: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
176: /* Set rootdata buffer to be broadcast */
177: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
178: for (i = 0; i < nroots; i++) rootdata[i * stride] = 100 * (rank + 1) + i;
179: /* Initialize local buffer, these values are never used. */
180: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
181: /* Broadcast entries from rootdata to leafdata. Computation or other communication can be performed between the begin and end calls. */
182: // test persistent communication code paths by repeated bcast several times
183: PetscCall(PetscSFRegisterPersistent(sf, MPIU_INT, rootdata, leafdata));
184: for (PetscInt i = 0; i < 10; i++) {
185: PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootdata, leafdata, MPI_REPLACE));
186: PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootdata, leafdata, MPI_REPLACE));
187: }
188: PetscCall(PetscSFDeregisterPersistent(sf, MPIU_INT, rootdata, leafdata));
189: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Rootdata\n"));
190: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
191: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Leafdata\n"));
192: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
193: PetscCall(PetscFree2(rootdata, leafdata));
194: }
196: if (test_bcast && test_char) { /* Bcast with char */
197: PetscInt len;
198: char buf[256];
199: char *rootdata, *leafdata;
200: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
201: /* Set rootdata buffer to be broadcast */
202: for (i = 0; i < nrootsalloc; i++) rootdata[i] = '*';
203: for (i = 0; i < nroots; i++) rootdata[i * stride] = (char)('A' + rank * 3 + i); /* rank is very small, so it is fine to compute a char */
204: /* Initialize local buffer, these values are never used. */
205: for (i = 0; i < nleavesalloc; i++) leafdata[i] = '?';
207: PetscCall(PetscSFBcastBegin(sf, MPI_CHAR, rootdata, leafdata, MPI_REPLACE));
208: PetscCall(PetscSFBcastEnd(sf, MPI_CHAR, rootdata, leafdata, MPI_REPLACE));
210: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Rootdata in type of char\n"));
211: len = 0;
212: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
213: len += 5;
214: for (i = 0; i < nrootsalloc; i++) {
215: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5c", rootdata[i]));
216: len += 5;
217: }
218: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
219: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
221: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Leafdata in type of char\n"));
222: len = 0;
223: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
224: len += 5;
225: for (i = 0; i < nleavesalloc; i++) {
226: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5c", leafdata[i]));
227: len += 5;
228: }
229: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
230: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
232: PetscCall(PetscFree2(rootdata, leafdata));
233: }
235: if (test_bcastop) { /* Reduce rootdata into leafdata */
236: PetscInt *rootdata, *leafdata;
237: /* Allocate space for send and receive buffers. This example communicates PetscInt, but other types, including
238: * user-defined structures, could also be used. */
239: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
240: /* Set rootdata buffer to be broadcast */
241: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
242: for (i = 0; i < nroots; i++) rootdata[i * stride] = 100 * (rank + 1) + i;
243: /* Set leaf values to reduce with */
244: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -10 * (rank + 1) - i;
245: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-BcastAndOp Leafdata\n"));
246: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
247: /* Broadcast entries from rootdata to leafdata. Computation or other communication can be performed between the begin and end calls. */
248: PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootdata, leafdata, mop));
249: PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootdata, leafdata, mop));
250: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## BcastAndOp Rootdata\n"));
251: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
252: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## BcastAndOp Leafdata\n"));
253: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
254: PetscCall(PetscFree2(rootdata, leafdata));
255: }
257: if (test_reduce) { /* Reduce leafdata into rootdata */
258: PetscInt *rootdata, *leafdata;
259: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
260: /* Initialize rootdata buffer in which the result of the reduction will appear. */
261: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
262: for (i = 0; i < nroots; i++) rootdata[i * stride] = 100 * (rank + 1) + i;
263: /* Set leaf values to reduce. */
264: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
265: for (i = 0; i < nleaves; i++) leafdata[i * stride] = 1000 * (rank + 1) + 10 * i;
266: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-Reduce Rootdata\n"));
267: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
268: /* Perform reduction. Computation or other communication can be performed between the begin and end calls.
269: * This example sums the values, but other MPI_Ops can be used (e.g MPI_MAX, MPI_PROD). */
270: PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafdata, rootdata, mop));
271: PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafdata, rootdata, mop));
272: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Leafdata\n"));
273: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
274: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Rootdata\n"));
275: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
276: PetscCall(PetscFree2(rootdata, leafdata));
277: }
279: if (test_reduce && test_char) { /* Reduce with signed char */
280: PetscInt len;
281: char buf[256];
282: signed char *rootdata, *leafdata;
283: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
284: /* Initialize rootdata buffer in which the result of the reduction will appear. */
285: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
286: for (i = 0; i < nroots; i++) rootdata[i * stride] = (signed char)(10 * (rank + 1) + i);
287: /* Set leaf values to reduce. */
288: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
289: for (i = 0; i < nleaves; i++) leafdata[i * stride] = (signed char)(50 * (rank + 1) + 10 * i);
290: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-Reduce Rootdata in type of signed char\n"));
292: len = 0;
293: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
294: len += 5;
295: for (i = 0; i < nrootsalloc; i++) {
296: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5d", rootdata[i]));
297: len += 5;
298: }
299: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
300: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
302: /* Using MPI_CHAR should trigger an error since MPI standard does not support reduction on MPI_CHAR.
303: Testing with -test_op max, one can see the sign does take effect in MPI_MAX.
304: */
305: PetscCall(PetscSFReduceBegin(sf, MPI_SIGNED_CHAR, leafdata, rootdata, mop));
306: PetscCall(PetscSFReduceEnd(sf, MPI_SIGNED_CHAR, leafdata, rootdata, mop));
308: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Leafdata in type of signed char\n"));
309: len = 0;
310: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
311: len += 5;
312: for (i = 0; i < nleavesalloc; i++) {
313: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5d", leafdata[i]));
314: len += 5;
315: }
316: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
317: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
319: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Rootdata in type of signed char\n"));
320: len = 0;
321: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
322: len += 5;
323: for (i = 0; i < nrootsalloc; i++) {
324: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5d", rootdata[i]));
325: len += 5;
326: }
327: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
328: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
330: PetscCall(PetscFree2(rootdata, leafdata));
331: }
333: if (test_reduce && test_char) { /* Reduce with unsigned char */
334: PetscInt len;
335: char buf[256];
336: unsigned char *rootdata, *leafdata;
337: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
338: /* Initialize rootdata buffer in which the result of the reduction will appear. */
339: for (i = 0; i < nrootsalloc; i++) rootdata[i] = 0;
340: for (i = 0; i < nroots; i++) rootdata[i * stride] = (unsigned char)(10 * (rank + 1) + i);
341: /* Set leaf values to reduce. */
342: for (i = 0; i < nleavesalloc; i++) leafdata[i] = 0;
343: for (i = 0; i < nleaves; i++) leafdata[i * stride] = (unsigned char)(50 * (rank + 1) + 10 * i);
344: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-Reduce Rootdata in type of unsigned char\n"));
346: len = 0;
347: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
348: len += 5;
349: for (i = 0; i < nrootsalloc; i++) {
350: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5u", rootdata[i]));
351: len += 5;
352: }
353: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
354: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
356: /* Using MPI_CHAR should trigger an error since MPI standard does not support reduction on MPI_CHAR.
357: Testing with -test_op max, one can see the sign does take effect in MPI_MAX.
358: */
359: PetscCall(PetscSFReduceBegin(sf, MPI_UNSIGNED_CHAR, leafdata, rootdata, mop));
360: PetscCall(PetscSFReduceEnd(sf, MPI_UNSIGNED_CHAR, leafdata, rootdata, mop));
362: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Leafdata in type of unsigned char\n"));
363: len = 0;
364: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
365: len += 5;
366: for (i = 0; i < nleavesalloc; i++) {
367: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5u", leafdata[i]));
368: len += 5;
369: }
370: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
371: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
373: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Rootdata in type of unsigned char\n"));
374: len = 0;
375: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
376: len += 5;
377: for (i = 0; i < nrootsalloc; i++) {
378: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5u", rootdata[i]));
379: len += 5;
380: }
381: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
382: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
384: PetscCall(PetscFree2(rootdata, leafdata));
385: }
387: if (test_degree) {
388: const PetscInt *degree;
389: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
390: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
391: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Root degrees\n"));
392: PetscCall(PetscIntView(nrootsalloc, degree, PETSC_VIEWER_STDOUT_WORLD));
393: }
395: if (test_fetchandop) {
396: /* Cannot use text compare here because token ordering is not deterministic */
397: PetscInt *leafdata, *leafupdate, *rootdata;
398: PetscCall(PetscMalloc3(nleavesalloc, &leafdata, nleavesalloc, &leafupdate, nrootsalloc, &rootdata));
399: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
400: for (i = 0; i < nleaves; i++) leafdata[i * stride] = 1;
401: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
402: for (i = 0; i < nroots; i++) rootdata[i * stride] = 0;
403: PetscCall(PetscSFFetchAndOpBegin(sf, MPIU_INT, rootdata, leafdata, leafupdate, mop));
404: PetscCall(PetscSFFetchAndOpEnd(sf, MPIU_INT, rootdata, leafdata, leafupdate, mop));
405: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Rootdata (sum of 1 from each leaf)\n"));
406: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
407: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Leafupdate (value at roots prior to my atomic update)\n"));
408: PetscCall(PetscIntView(nleavesalloc, leafupdate, PETSC_VIEWER_STDOUT_WORLD));
409: PetscCall(PetscFree3(leafdata, leafupdate, rootdata));
410: }
412: if (test_gather) {
413: const PetscInt *degree;
414: PetscInt inedges, *indata, *outdata;
415: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
416: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
417: for (i = 0, inedges = 0; i < nrootsalloc; i++) inedges += degree[i];
418: PetscCall(PetscMalloc2(inedges, &indata, nleavesalloc, &outdata));
419: for (i = 0; i < nleavesalloc; i++) outdata[i] = -1;
420: for (i = 0; i < nleaves; i++) outdata[i * stride] = 1000 * (rank + 1) + i;
421: PetscCall(PetscSFGatherBegin(sf, MPIU_INT, outdata, indata));
422: PetscCall(PetscSFGatherEnd(sf, MPIU_INT, outdata, indata));
423: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Gathered data at multi-roots from leaves\n"));
424: PetscCall(PetscIntView(inedges, indata, PETSC_VIEWER_STDOUT_WORLD));
425: PetscCall(PetscFree2(indata, outdata));
426: }
428: if (test_scatter) {
429: const PetscInt *degree;
430: PetscInt j, count, inedges, *indata, *outdata;
431: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
432: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
433: for (i = 0, inedges = 0; i < nrootsalloc; i++) inedges += degree[i];
434: PetscCall(PetscMalloc2(inedges, &indata, nleavesalloc, &outdata));
435: for (i = 0; i < nleavesalloc; i++) outdata[i] = -1;
436: for (i = 0, count = 0; i < nrootsalloc; i++) {
437: for (j = 0; j < degree[i]; j++) indata[count++] = 1000 * (rank + 1) + 100 * i + j;
438: }
439: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Data at multi-roots, to scatter to leaves\n"));
440: PetscCall(PetscIntView(inedges, indata, PETSC_VIEWER_STDOUT_WORLD));
442: PetscCall(PetscSFScatterBegin(sf, MPIU_INT, indata, outdata));
443: PetscCall(PetscSFScatterEnd(sf, MPIU_INT, indata, outdata));
444: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Scattered data at leaves\n"));
445: PetscCall(PetscIntView(nleavesalloc, outdata, PETSC_VIEWER_STDOUT_WORLD));
446: PetscCall(PetscFree2(indata, outdata));
447: }
449: if (test_embed) {
450: const PetscInt nroots = 1 + (PetscInt)(rank == 0);
451: PetscInt selected[2];
452: PetscSF esf;
454: selected[0] = stride;
455: selected[1] = 2 * stride;
456: PetscCall(PetscSFCreateEmbeddedRootSF(sf, nroots, selected, &esf));
457: PetscCall(PetscSFSetUp(esf));
458: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Embedded PetscSF\n"));
459: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_INFO_DETAIL));
460: PetscCall(PetscSFView(esf, PETSC_VIEWER_STDOUT_WORLD));
461: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
462: PetscCall(PetscSFDestroy(&esf));
463: }
465: if (test_invert) {
466: const PetscInt *degree;
467: PetscInt *mRootsOrigNumbering;
468: PetscInt inedges;
469: PetscSF msf, imsf;
471: PetscCall(PetscSFGetMultiSF(sf, &msf));
472: PetscCall(PetscSFCreateInverseSF(msf, &imsf));
473: PetscCall(PetscSFSetUp(msf));
474: PetscCall(PetscSFSetUp(imsf));
475: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Multi-SF\n"));
476: PetscCall(PetscSFView(msf, PETSC_VIEWER_STDOUT_WORLD));
477: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Multi-SF roots indices in original SF roots numbering\n"));
478: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
479: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
480: PetscCall(PetscSFComputeMultiRootOriginalNumbering(sf, degree, &inedges, &mRootsOrigNumbering));
481: PetscCall(PetscIntView(inedges, mRootsOrigNumbering, PETSC_VIEWER_STDOUT_WORLD));
482: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Inverse of Multi-SF\n"));
483: PetscCall(PetscSFView(imsf, PETSC_VIEWER_STDOUT_WORLD));
484: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Inverse of Multi-SF, original numbering\n"));
485: PetscCall(PetscSFViewCustomLocals_Private(imsf, mRootsOrigNumbering, PETSC_VIEWER_STDOUT_WORLD));
486: PetscCall(PetscSFDestroy(&imsf));
487: PetscCall(PetscFree(mRootsOrigNumbering));
488: }
490: /* Clean storage for star forest. */
491: PetscCall(PetscSFDestroy(&sf));
492: PetscCall(PetscFinalize());
493: return 0;
494: }
496: /*TEST
498: test:
499: nsize: 4
500: filter: grep -v "type" | grep -v "sort"
501: args: -test_bcast -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
502: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
504: test:
505: suffix: 2
506: nsize: 4
507: filter: grep -v "type" | grep -v "sort"
508: args: -test_reduce -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
509: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
511: test:
512: suffix: 2_basic
513: nsize: 4
514: args: -test_reduce -sf_type basic
516: test:
517: suffix: 3
518: nsize: 4
519: filter: grep -v "type" | grep -v "sort"
520: args: -test_degree -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
521: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
523: test:
524: suffix: 3_basic
525: nsize: 4
526: args: -test_degree -sf_type basic
528: test:
529: suffix: 4
530: nsize: 4
531: filter: grep -v "type" | grep -v "sort"
532: args: -test_gather -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
533: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
535: test:
536: suffix: 4_basic
537: nsize: 4
538: args: -test_gather -sf_type basic
540: test:
541: suffix: 4_stride
542: nsize: 4
543: args: -test_gather -sf_type basic -stride 2
545: test:
546: suffix: 5
547: nsize: 4
548: filter: grep -v "type" | grep -v "sort"
549: args: -test_scatter -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
550: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
552: test:
553: suffix: 5_basic
554: nsize: 4
555: args: -test_scatter -sf_type basic
557: test:
558: suffix: 5_stride
559: nsize: 4
560: args: -test_scatter -sf_type basic -stride 2
562: test:
563: suffix: 6
564: nsize: 4
565: filter: grep -v "type" | grep -v "sort"
566: # No -sf_window_flavor dynamic due to bug https://gitlab.com/petsc/petsc/issues/555
567: args: -test_embed -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create allocate}}
568: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
570: test:
571: suffix: 6_basic
572: nsize: 4
573: args: -test_embed -sf_type basic
575: test:
576: suffix: 7
577: nsize: 4
578: filter: grep -v "type" | grep -v "sort"
579: args: -test_invert -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
580: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
582: test:
583: suffix: 7_basic
584: nsize: 4
585: args: -test_invert -sf_type basic
587: test:
588: suffix: basic
589: nsize: 4
590: args: -test_bcast -sf_type basic
591: output_file: output/ex1_1_basic.out
593: test:
594: suffix: bcastop_basic
595: nsize: 4
596: args: -test_bcastop -sf_type basic
597: output_file: output/ex1_bcastop_basic.out
599: test:
600: suffix: 8
601: nsize: 3
602: filter: grep -v "type" | grep -v "sort"
603: args: -test_bcast -test_sf_distribute -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
604: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
606: test:
607: suffix: 8_basic
608: nsize: 3
609: args: -test_bcast -test_sf_distribute -sf_type basic
611: test:
612: suffix: 9_char
613: nsize: 4
614: args: -sf_type basic -test_bcast -test_reduce -test_op max -test_char
616: # Here we do not test -sf_window_flavor dynamic since it is designed for repeated SFs with few different rootdata pointers
617: test:
618: suffix: 10
619: filter: grep -v "type" | grep -v "sort"
620: nsize: 4
621: args: -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create allocate}} -test_all -test_bcastop 0 -test_fetchandop 0
622: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
624: # The nightly test suite with MPICH uses ch3:sock, which is broken when winsize == 0 in some of the processes
625: test:
626: suffix: 10_shared
627: output_file: output/ex1_10.out
628: filter: grep -v "type" | grep -v "sort"
629: nsize: 4
630: args: -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor shared -test_all -test_bcastop 0 -test_fetchandop 0
631: requires: defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY) !defined(PETSC_HAVE_MPICH) defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
633: test:
634: suffix: 10_basic
635: nsize: 4
636: args: -sf_type basic -test_all -test_bcastop 0 -test_fetchandop 0
638: test:
639: suffix: 10_basic_vector
640: nsize: 4
641: args: -sf_type basic -test_all -test_bcastop 0 -test_fetchandop 0 -test_vector
643: TEST*/