Actual source code: ex1.c
1: static const char help[] = "Test star forest communication (PetscSF)\n\n";
3: /*
4: Description: A star is a simple tree with one root and zero or more leaves.
5: A star forest is a union of disjoint stars.
6: Many common communication patterns can be expressed as updates of rootdata using leafdata and vice-versa.
7: This example creates a star forest, communicates values using the graph (see options for types of communication), views the graph, then destroys it.
8: */
10: /*
11: Include petscsf.h so we can use PetscSF objects. Note that this automatically
12: includes petscsys.h.
13: */
14: #include <petscsf.h>
16: /* like PetscSFView() but with alternative array of local indices */
17: static PetscErrorCode PetscSFViewCustomLocals_Private(PetscSF sf, const PetscInt locals[], PetscViewer viewer)
18: {
19: const PetscSFNode *iremote;
20: PetscInt i, nroots, nleaves;
21: PetscMPIInt rank, nranks;
23: PetscFunctionBeginUser;
24: PetscCallMPI(MPI_Comm_rank(PetscObjectComm((PetscObject)sf), &rank));
25: PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, NULL, &iremote));
26: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, NULL, NULL, NULL));
27: PetscCall(PetscViewerASCIIPushTab(viewer));
28: PetscCall(PetscViewerASCIIPushSynchronized(viewer));
29: PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] Number of roots=%" PetscInt_FMT ", leaves=%" PetscInt_FMT ", remote ranks=%d\n", rank, nroots, nleaves, nranks));
30: for (i = 0; i < nleaves; i++) PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, "[%d] %" PetscInt_FMT " <- (%d,%" PetscInt_FMT ")\n", rank, locals[i], (PetscMPIInt)iremote[i].rank, iremote[i].index));
31: PetscCall(PetscViewerFlush(viewer));
32: PetscCall(PetscViewerASCIIPopTab(viewer));
33: PetscCall(PetscViewerASCIIPopSynchronized(viewer));
34: PetscFunctionReturn(PETSC_SUCCESS);
35: }
37: int main(int argc, char **argv)
38: {
39: PetscInt i, bs = 2, nroots, nrootsalloc, nleaves, nleavesalloc, *mine, stride;
40: PetscSFNode *remote;
41: PetscMPIInt rank, size;
42: PetscSF sf, vsf;
43: PetscBool test_all, test_bcast, test_bcastop, test_reduce, test_degree, test_fetchandop, test_gather, test_scatter, test_embed, test_invert, test_sf_distribute, test_char, test_vector = PETSC_FALSE;
44: MPI_Op mop = MPI_OP_NULL; /* initialize to prevent compiler warnings with cxx_quad build */
45: char opstring[256];
46: PetscBool strflg;
48: PetscFunctionBeginUser;
49: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
50: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
51: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
53: PetscOptionsBegin(PETSC_COMM_WORLD, "", "PetscSF Test Options", "none");
54: test_all = PETSC_FALSE;
55: PetscCall(PetscOptionsBool("-test_all", "Test all SF communications", "", test_all, &test_all, NULL));
56: test_bcast = test_all;
57: PetscCall(PetscOptionsBool("-test_bcast", "Test broadcast", "", test_bcast, &test_bcast, NULL));
58: test_bcastop = test_all;
59: PetscCall(PetscOptionsBool("-test_bcastop", "Test broadcast and reduce", "", test_bcastop, &test_bcastop, NULL));
60: test_reduce = test_all;
61: PetscCall(PetscOptionsBool("-test_reduce", "Test reduction", "", test_reduce, &test_reduce, NULL));
62: test_char = test_all;
63: PetscCall(PetscOptionsBool("-test_char", "Test signed char, unsigned char, and char", "", test_char, &test_char, NULL));
64: mop = MPI_SUM;
65: PetscCall(PetscStrncpy(opstring, "sum", sizeof(opstring)));
66: PetscCall(PetscOptionsString("-test_op", "Designate which MPI_Op to use", "", opstring, opstring, sizeof(opstring), NULL));
67: PetscCall(PetscStrcmp("sum", opstring, &strflg));
68: if (strflg) mop = MPIU_SUM;
69: PetscCall(PetscStrcmp("prod", opstring, &strflg));
70: if (strflg) mop = MPI_PROD;
71: PetscCall(PetscStrcmp("max", opstring, &strflg));
72: if (strflg) mop = MPI_MAX;
73: PetscCall(PetscStrcmp("min", opstring, &strflg));
74: if (strflg) mop = MPI_MIN;
75: PetscCall(PetscStrcmp("land", opstring, &strflg));
76: if (strflg) mop = MPI_LAND;
77: PetscCall(PetscStrcmp("band", opstring, &strflg));
78: if (strflg) mop = MPI_BAND;
79: PetscCall(PetscStrcmp("lor", opstring, &strflg));
80: if (strflg) mop = MPI_LOR;
81: PetscCall(PetscStrcmp("bor", opstring, &strflg));
82: if (strflg) mop = MPI_BOR;
83: PetscCall(PetscStrcmp("lxor", opstring, &strflg));
84: if (strflg) mop = MPI_LXOR;
85: PetscCall(PetscStrcmp("bxor", opstring, &strflg));
86: if (strflg) mop = MPI_BXOR;
87: test_degree = test_all;
88: PetscCall(PetscOptionsBool("-test_degree", "Test computation of vertex degree", "", test_degree, &test_degree, NULL));
89: test_fetchandop = test_all;
90: PetscCall(PetscOptionsBool("-test_fetchandop", "Test atomic Fetch-And-Op", "", test_fetchandop, &test_fetchandop, NULL));
91: test_gather = test_all;
92: PetscCall(PetscOptionsBool("-test_gather", "Test point gather", "", test_gather, &test_gather, NULL));
93: test_scatter = test_all;
94: PetscCall(PetscOptionsBool("-test_scatter", "Test point scatter", "", test_scatter, &test_scatter, NULL));
95: test_embed = test_all;
96: PetscCall(PetscOptionsBool("-test_embed", "Test point embed", "", test_embed, &test_embed, NULL));
97: test_invert = test_all;
98: PetscCall(PetscOptionsBool("-test_invert", "Test point invert", "", test_invert, &test_invert, NULL));
99: stride = 1;
100: PetscCall(PetscOptionsInt("-stride", "Stride for leaf and root data", "", stride, &stride, NULL));
101: test_sf_distribute = PETSC_FALSE;
102: PetscCall(PetscOptionsBool("-test_sf_distribute", "Create an SF that 'distributes' to each process, like an alltoall", "", test_sf_distribute, &test_sf_distribute, NULL));
103: PetscCall(PetscOptionsString("-test_op", "Designate which MPI_Op to use", "", opstring, opstring, sizeof(opstring), NULL));
104: PetscCall(PetscOptionsInt("-bs", "Block size for vectorial SF", "", bs, &bs, NULL));
105: PetscCall(PetscOptionsBool("-test_vector", "Run tests using the vectorial SF", "", test_vector, &test_vector, NULL));
106: PetscOptionsEnd();
108: if (test_sf_distribute) {
109: nroots = size;
110: nrootsalloc = size;
111: nleaves = size;
112: nleavesalloc = size;
113: mine = NULL;
114: PetscCall(PetscMalloc1(nleaves, &remote));
115: for (PetscMPIInt ii = 0; ii < size; ii++) {
116: remote[ii].rank = ii;
117: remote[ii].index = rank;
118: }
119: } else {
120: nroots = 2 + (PetscInt)(rank == 0);
121: nrootsalloc = nroots * stride;
122: nleaves = 2 + (PetscInt)(rank > 0);
123: nleavesalloc = nleaves * stride;
124: mine = NULL;
125: if (stride > 1) {
126: PetscCall(PetscMalloc1(nleaves, &mine));
127: for (PetscInt i = 0; i < nleaves; i++) mine[i] = stride * i;
128: }
129: PetscCall(PetscMalloc1(nleaves, &remote));
130: /* Left periodic neighbor */
131: remote[0].rank = (rank + size - 1) % size;
132: remote[0].index = 1 * stride;
133: /* Right periodic neighbor */
134: remote[1].rank = (rank + 1) % size;
135: remote[1].index = 0 * stride;
136: if (rank > 0) { /* All processes reference rank 0, index 1 */
137: remote[2].rank = 0;
138: remote[2].index = 2 * stride;
139: }
140: }
142: /* Create a star forest for communication */
143: PetscCall(PetscSFCreate(PETSC_COMM_WORLD, &sf));
144: PetscCall(PetscSFSetFromOptions(sf));
145: PetscCall(PetscSFSetGraph(sf, nrootsalloc, nleaves, mine, PETSC_OWN_POINTER, remote, PETSC_OWN_POINTER));
146: PetscCall(PetscSFSetUp(sf));
148: /* We can also obtain a strided SF to communicate interleaved blocks of data */
149: PetscCall(PetscSFCreateStridedSF(sf, bs, PETSC_DECIDE, PETSC_DECIDE, &vsf));
150: if (test_vector) { /* perform all tests below using the vectorial SF */
151: PetscSF t = sf;
153: sf = vsf;
154: vsf = t;
156: nroots *= bs;
157: nleaves *= bs;
158: nrootsalloc *= bs;
159: nleavesalloc *= bs;
160: }
161: PetscCall(PetscSFDestroy(&vsf));
163: /* View graph, mostly useful for debugging purposes. */
164: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_INFO_DETAIL));
165: PetscCall(PetscSFView(sf, PETSC_VIEWER_STDOUT_WORLD));
166: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
168: if (test_bcast) { /* broadcast rootdata into leafdata */
169: PetscInt *rootdata, *leafdata;
170: /* Allocate space for send and receive buffers. This example communicates PetscInt, but other types, including
171: * user-defined structures, could also be used. */
172: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
173: /* Set rootdata buffer to be broadcast */
174: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
175: for (i = 0; i < nroots; i++) rootdata[i * stride] = 100 * (rank + 1) + i;
176: /* Initialize local buffer, these values are never used. */
177: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
178: /* Broadcast entries from rootdata to leafdata. Computation or other communication can be performed between the begin and end calls. */
179: // test persistent communication code paths by repeated bcast several times
180: PetscCall(PetscSFRegisterPersistent(sf, MPIU_INT, rootdata, leafdata));
181: for (PetscInt i = 0; i < 10; i++) {
182: PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootdata, leafdata, MPI_REPLACE));
183: PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootdata, leafdata, MPI_REPLACE));
184: }
185: PetscCall(PetscSFDeregisterPersistent(sf, MPIU_INT, rootdata, leafdata));
186: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Rootdata\n"));
187: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
188: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Leafdata\n"));
189: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
190: PetscCall(PetscFree2(rootdata, leafdata));
191: }
193: if (test_bcast && test_char) { /* Bcast with char */
194: PetscInt len;
195: char buf[256];
196: char *rootdata, *leafdata;
197: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
198: /* Set rootdata buffer to be broadcast */
199: for (i = 0; i < nrootsalloc; i++) rootdata[i] = '*';
200: for (i = 0; i < nroots; i++) rootdata[i * stride] = (char)('A' + rank * 3 + i); /* rank is very small, so it is fine to compute a char */
201: /* Initialize local buffer, these values are never used. */
202: for (i = 0; i < nleavesalloc; i++) leafdata[i] = '?';
204: PetscCall(PetscSFBcastBegin(sf, MPI_CHAR, rootdata, leafdata, MPI_REPLACE));
205: PetscCall(PetscSFBcastEnd(sf, MPI_CHAR, rootdata, leafdata, MPI_REPLACE));
207: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Rootdata in type of char\n"));
208: len = 0;
209: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
210: len += 5;
211: for (i = 0; i < nrootsalloc; i++) {
212: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5c", rootdata[i]));
213: len += 5;
214: }
215: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
216: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
218: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Bcast Leafdata in type of char\n"));
219: len = 0;
220: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
221: len += 5;
222: for (i = 0; i < nleavesalloc; i++) {
223: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5c", leafdata[i]));
224: len += 5;
225: }
226: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
227: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
229: PetscCall(PetscFree2(rootdata, leafdata));
230: }
232: if (test_bcastop) { /* Reduce rootdata into leafdata */
233: PetscInt *rootdata, *leafdata;
234: /* Allocate space for send and receive buffers. This example communicates PetscInt, but other types, including
235: * user-defined structures, could also be used. */
236: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
237: /* Set rootdata buffer to be broadcast */
238: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
239: for (i = 0; i < nroots; i++) rootdata[i * stride] = 100 * (rank + 1) + i;
240: /* Set leaf values to reduce with */
241: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -10 * (rank + 1) - i;
242: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-BcastAndOp Leafdata\n"));
243: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
244: /* Broadcast entries from rootdata to leafdata. Computation or other communication can be performed between the begin and end calls. */
245: PetscCall(PetscSFBcastBegin(sf, MPIU_INT, rootdata, leafdata, mop));
246: PetscCall(PetscSFBcastEnd(sf, MPIU_INT, rootdata, leafdata, mop));
247: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## BcastAndOp Rootdata\n"));
248: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
249: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## BcastAndOp Leafdata\n"));
250: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
251: PetscCall(PetscFree2(rootdata, leafdata));
252: }
254: if (test_reduce) { /* Reduce leafdata into rootdata */
255: PetscInt *rootdata, *leafdata;
256: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
257: /* Initialize rootdata buffer in which the result of the reduction will appear. */
258: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
259: for (i = 0; i < nroots; i++) rootdata[i * stride] = 100 * (rank + 1) + i;
260: /* Set leaf values to reduce. */
261: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
262: for (i = 0; i < nleaves; i++) leafdata[i * stride] = 1000 * (rank + 1) + 10 * i;
263: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-Reduce Rootdata\n"));
264: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
265: /* Perform reduction. Computation or other communication can be performed between the begin and end calls.
266: * This example sums the values, but other MPI_Ops can be used (e.g MPI_MAX, MPI_PROD). */
267: PetscCall(PetscSFReduceBegin(sf, MPIU_INT, leafdata, rootdata, mop));
268: PetscCall(PetscSFReduceEnd(sf, MPIU_INT, leafdata, rootdata, mop));
269: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Leafdata\n"));
270: PetscCall(PetscIntView(nleavesalloc, leafdata, PETSC_VIEWER_STDOUT_WORLD));
271: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Rootdata\n"));
272: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
273: PetscCall(PetscFree2(rootdata, leafdata));
274: }
276: if (test_reduce && test_char) { /* Reduce with signed char */
277: PetscInt len;
278: char buf[256];
279: signed char *rootdata, *leafdata;
280: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
281: /* Initialize rootdata buffer in which the result of the reduction will appear. */
282: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
283: for (i = 0; i < nroots; i++) rootdata[i * stride] = (signed char)(10 * (rank + 1) + i);
284: /* Set leaf values to reduce. */
285: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
286: for (i = 0; i < nleaves; i++) leafdata[i * stride] = (signed char)(50 * (rank + 1) + 10 * i);
287: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-Reduce Rootdata in type of signed char\n"));
289: len = 0;
290: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
291: len += 5;
292: for (i = 0; i < nrootsalloc; i++) {
293: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5d", rootdata[i]));
294: len += 5;
295: }
296: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
297: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
299: /* Using MPI_CHAR should trigger an error since MPI standard does not support reduction on MPI_CHAR.
300: Testing with -test_op max, one can see the sign does take effect in MPI_MAX.
301: */
302: PetscCall(PetscSFReduceBegin(sf, MPI_SIGNED_CHAR, leafdata, rootdata, mop));
303: PetscCall(PetscSFReduceEnd(sf, MPI_SIGNED_CHAR, leafdata, rootdata, mop));
305: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Leafdata in type of signed char\n"));
306: len = 0;
307: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
308: len += 5;
309: for (i = 0; i < nleavesalloc; i++) {
310: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5d", leafdata[i]));
311: len += 5;
312: }
313: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
314: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
316: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Rootdata in type of signed char\n"));
317: len = 0;
318: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
319: len += 5;
320: for (i = 0; i < nrootsalloc; i++) {
321: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5d", rootdata[i]));
322: len += 5;
323: }
324: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
325: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
327: PetscCall(PetscFree2(rootdata, leafdata));
328: }
330: if (test_reduce && test_char) { /* Reduce with unsigned char */
331: PetscInt len;
332: char buf[256];
333: unsigned char *rootdata, *leafdata;
334: PetscCall(PetscMalloc2(nrootsalloc, &rootdata, nleavesalloc, &leafdata));
335: /* Initialize rootdata buffer in which the result of the reduction will appear. */
336: for (i = 0; i < nrootsalloc; i++) rootdata[i] = 0;
337: for (i = 0; i < nroots; i++) rootdata[i * stride] = (unsigned char)(10 * (rank + 1) + i);
338: /* Set leaf values to reduce. */
339: for (i = 0; i < nleavesalloc; i++) leafdata[i] = 0;
340: for (i = 0; i < nleaves; i++) leafdata[i * stride] = (unsigned char)(50 * (rank + 1) + 10 * i);
341: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Pre-Reduce Rootdata in type of unsigned char\n"));
343: len = 0;
344: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
345: len += 5;
346: for (i = 0; i < nrootsalloc; i++) {
347: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5u", rootdata[i]));
348: len += 5;
349: }
350: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
351: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
353: /* Using MPI_CHAR should trigger an error since MPI standard does not support reduction on MPI_CHAR.
354: Testing with -test_op max, one can see the sign does take effect in MPI_MAX.
355: */
356: PetscCall(PetscSFReduceBegin(sf, MPI_UNSIGNED_CHAR, leafdata, rootdata, mop));
357: PetscCall(PetscSFReduceEnd(sf, MPI_UNSIGNED_CHAR, leafdata, rootdata, mop));
359: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Leafdata in type of unsigned char\n"));
360: len = 0;
361: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
362: len += 5;
363: for (i = 0; i < nleavesalloc; i++) {
364: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5u", leafdata[i]));
365: len += 5;
366: }
367: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
368: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
370: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Reduce Rootdata in type of unsigned char\n"));
371: len = 0;
372: PetscCall(PetscSNPrintf(buf, 256, "%4d:", rank));
373: len += 5;
374: for (i = 0; i < nrootsalloc; i++) {
375: PetscCall(PetscSNPrintf(buf + len, 256 - len, "%5u", rootdata[i]));
376: len += 5;
377: }
378: PetscCall(PetscSynchronizedPrintf(PETSC_COMM_WORLD, "%s\n", buf));
379: PetscCall(PetscSynchronizedFlush(PETSC_COMM_WORLD, PETSC_STDOUT));
381: PetscCall(PetscFree2(rootdata, leafdata));
382: }
384: if (test_degree) {
385: const PetscInt *degree;
386: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
387: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
388: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Root degrees\n"));
389: PetscCall(PetscIntView(nrootsalloc, degree, PETSC_VIEWER_STDOUT_WORLD));
390: }
392: if (test_fetchandop) {
393: /* Cannot use text compare here because token ordering is not deterministic */
394: PetscInt *leafdata, *leafupdate, *rootdata;
395: PetscCall(PetscMalloc3(nleavesalloc, &leafdata, nleavesalloc, &leafupdate, nrootsalloc, &rootdata));
396: for (i = 0; i < nleavesalloc; i++) leafdata[i] = -1;
397: for (i = 0; i < nleaves; i++) leafdata[i * stride] = 1;
398: for (i = 0; i < nrootsalloc; i++) rootdata[i] = -1;
399: for (i = 0; i < nroots; i++) rootdata[i * stride] = 0;
400: PetscCall(PetscSFFetchAndOpBegin(sf, MPIU_INT, rootdata, leafdata, leafupdate, mop));
401: PetscCall(PetscSFFetchAndOpEnd(sf, MPIU_INT, rootdata, leafdata, leafupdate, mop));
402: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Rootdata (sum of 1 from each leaf)\n"));
403: PetscCall(PetscIntView(nrootsalloc, rootdata, PETSC_VIEWER_STDOUT_WORLD));
404: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Leafupdate (value at roots prior to my atomic update)\n"));
405: PetscCall(PetscIntView(nleavesalloc, leafupdate, PETSC_VIEWER_STDOUT_WORLD));
406: PetscCall(PetscFree3(leafdata, leafupdate, rootdata));
407: }
409: if (test_gather) {
410: const PetscInt *degree;
411: PetscInt inedges, *indata, *outdata;
412: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
413: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
414: for (i = 0, inedges = 0; i < nrootsalloc; i++) inedges += degree[i];
415: PetscCall(PetscMalloc2(inedges, &indata, nleavesalloc, &outdata));
416: for (i = 0; i < nleavesalloc; i++) outdata[i] = -1;
417: for (i = 0; i < nleaves; i++) outdata[i * stride] = 1000 * (rank + 1) + i;
418: PetscCall(PetscSFGatherBegin(sf, MPIU_INT, outdata, indata));
419: PetscCall(PetscSFGatherEnd(sf, MPIU_INT, outdata, indata));
420: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Gathered data at multi-roots from leaves\n"));
421: PetscCall(PetscIntView(inedges, indata, PETSC_VIEWER_STDOUT_WORLD));
422: PetscCall(PetscFree2(indata, outdata));
423: }
425: if (test_scatter) {
426: const PetscInt *degree;
427: PetscInt j, count, inedges, *indata, *outdata;
428: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
429: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
430: for (i = 0, inedges = 0; i < nrootsalloc; i++) inedges += degree[i];
431: PetscCall(PetscMalloc2(inedges, &indata, nleavesalloc, &outdata));
432: for (i = 0; i < nleavesalloc; i++) outdata[i] = -1;
433: for (i = 0, count = 0; i < nrootsalloc; i++) {
434: for (j = 0; j < degree[i]; j++) indata[count++] = 1000 * (rank + 1) + 100 * i + j;
435: }
436: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Data at multi-roots, to scatter to leaves\n"));
437: PetscCall(PetscIntView(inedges, indata, PETSC_VIEWER_STDOUT_WORLD));
439: PetscCall(PetscSFScatterBegin(sf, MPIU_INT, indata, outdata));
440: PetscCall(PetscSFScatterEnd(sf, MPIU_INT, indata, outdata));
441: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Scattered data at leaves\n"));
442: PetscCall(PetscIntView(nleavesalloc, outdata, PETSC_VIEWER_STDOUT_WORLD));
443: PetscCall(PetscFree2(indata, outdata));
444: }
446: if (test_embed) {
447: const PetscInt nroots = 1 + (PetscInt)(rank == 0);
448: PetscInt selected[2];
449: PetscSF esf;
451: selected[0] = stride;
452: selected[1] = 2 * stride;
453: PetscCall(PetscSFCreateEmbeddedRootSF(sf, nroots, selected, &esf));
454: PetscCall(PetscSFSetUp(esf));
455: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Embedded PetscSF\n"));
456: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_INFO_DETAIL));
457: PetscCall(PetscSFView(esf, PETSC_VIEWER_STDOUT_WORLD));
458: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
459: PetscCall(PetscSFDestroy(&esf));
460: }
462: if (test_invert) {
463: const PetscInt *degree;
464: PetscInt *mRootsOrigNumbering;
465: PetscInt inedges;
466: PetscSF msf, imsf;
468: PetscCall(PetscSFGetMultiSF(sf, &msf));
469: PetscCall(PetscSFCreateInverseSF(msf, &imsf));
470: PetscCall(PetscSFSetUp(msf));
471: PetscCall(PetscSFSetUp(imsf));
472: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Multi-SF\n"));
473: PetscCall(PetscSFView(msf, PETSC_VIEWER_STDOUT_WORLD));
474: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Multi-SF roots indices in original SF roots numbering\n"));
475: PetscCall(PetscSFComputeDegreeBegin(sf, °ree));
476: PetscCall(PetscSFComputeDegreeEnd(sf, °ree));
477: PetscCall(PetscSFComputeMultiRootOriginalNumbering(sf, degree, &inedges, &mRootsOrigNumbering));
478: PetscCall(PetscIntView(inedges, mRootsOrigNumbering, PETSC_VIEWER_STDOUT_WORLD));
479: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Inverse of Multi-SF\n"));
480: PetscCall(PetscSFView(imsf, PETSC_VIEWER_STDOUT_WORLD));
481: PetscCall(PetscViewerASCIIPrintf(PETSC_VIEWER_STDOUT_WORLD, "## Inverse of Multi-SF, original numbering\n"));
482: PetscCall(PetscSFViewCustomLocals_Private(imsf, mRootsOrigNumbering, PETSC_VIEWER_STDOUT_WORLD));
483: PetscCall(PetscSFDestroy(&imsf));
484: PetscCall(PetscFree(mRootsOrigNumbering));
485: }
487: /* Clean storage for star forest. */
488: PetscCall(PetscSFDestroy(&sf));
489: PetscCall(PetscFinalize());
490: return 0;
491: }
493: /*TEST
495: test:
496: nsize: 4
497: filter: grep -v "type" | grep -v "sort"
498: args: -test_bcast -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
499: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
501: test:
502: suffix: 2
503: nsize: 4
504: filter: grep -v "type" | grep -v "sort"
505: args: -test_reduce -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
506: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
508: test:
509: suffix: 2_basic
510: nsize: 4
511: args: -test_reduce -sf_type basic
513: test:
514: suffix: 3
515: nsize: 4
516: filter: grep -v "type" | grep -v "sort"
517: args: -test_degree -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
518: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
520: test:
521: suffix: 3_basic
522: nsize: 4
523: args: -test_degree -sf_type basic
525: test:
526: suffix: 4
527: nsize: 4
528: filter: grep -v "type" | grep -v "sort"
529: args: -test_gather -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
530: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
532: test:
533: suffix: 4_basic
534: nsize: 4
535: args: -test_gather -sf_type basic
537: test:
538: suffix: 4_stride
539: nsize: 4
540: args: -test_gather -sf_type basic -stride 2
542: test:
543: suffix: 5
544: nsize: 4
545: filter: grep -v "type" | grep -v "sort"
546: args: -test_scatter -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
547: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
549: test:
550: suffix: 5_basic
551: nsize: 4
552: args: -test_scatter -sf_type basic
554: test:
555: suffix: 5_stride
556: nsize: 4
557: args: -test_scatter -sf_type basic -stride 2
559: test:
560: suffix: 6
561: nsize: 4
562: filter: grep -v "type" | grep -v "sort"
563: # No -sf_window_flavor dynamic due to bug https://gitlab.com/petsc/petsc/issues/555
564: args: -test_embed -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create allocate}}
565: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
567: test:
568: suffix: 6_basic
569: nsize: 4
570: args: -test_embed -sf_type basic
572: test:
573: suffix: 7
574: nsize: 4
575: filter: grep -v "type" | grep -v "sort"
576: args: -test_invert -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
577: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
579: test:
580: suffix: 7_basic
581: nsize: 4
582: args: -test_invert -sf_type basic
584: test:
585: suffix: basic
586: nsize: 4
587: args: -test_bcast -sf_type basic
588: output_file: output/ex1_1_basic.out
590: test:
591: suffix: bcastop_basic
592: nsize: 4
593: args: -test_bcastop -sf_type basic
594: output_file: output/ex1_bcastop_basic.out
596: test:
597: suffix: 8
598: nsize: 3
599: filter: grep -v "type" | grep -v "sort"
600: args: -test_bcast -test_sf_distribute -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create dynamic allocate}}
601: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
603: test:
604: suffix: 8_basic
605: nsize: 3
606: args: -test_bcast -test_sf_distribute -sf_type basic
608: test:
609: suffix: 9_char
610: nsize: 4
611: args: -sf_type basic -test_bcast -test_reduce -test_op max -test_char
613: # Here we do not test -sf_window_flavor dynamic since it is designed for repeated SFs with few different rootdata pointers
614: test:
615: suffix: 10
616: filter: grep -v "type" | grep -v "sort"
617: nsize: 4
618: args: -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor {{create allocate}} -test_all -test_bcastop 0 -test_fetchandop 0
619: requires: defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
621: # The nightly test suite with MPICH uses ch3:sock, which is broken when winsize == 0 in some of the processes
622: test:
623: suffix: 10_shared
624: output_file: output/ex1_10.out
625: filter: grep -v "type" | grep -v "sort"
626: nsize: 4
627: args: -sf_type window -sf_window_sync {{fence active lock}} -sf_window_flavor shared -test_all -test_bcastop 0 -test_fetchandop 0
628: requires: defined(PETSC_HAVE_MPI_PROCESS_SHARED_MEMORY) !defined(PETSC_HAVE_MPICH) defined(PETSC_HAVE_MPI_ONE_SIDED) defined(PETSC_HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
630: test:
631: suffix: 10_basic
632: nsize: 4
633: args: -sf_type basic -test_all -test_bcastop 0 -test_fetchandop 0
635: test:
636: suffix: 10_basic_vector
637: nsize: 4
638: args: -sf_type basic -test_all -test_bcastop 0 -test_fetchandop 0 -test_vector
640: TEST*/