Actual source code: subcomm.c
1: /*
2: Provides utility routines for split MPI communicator.
3: */
4: #include <petscsys.h>
5: #include <petscviewer.h>
7: const char *const PetscSubcommTypes[] = {"GENERAL", "CONTIGUOUS", "INTERLACED", "PetscSubcommType", "PETSC_SUBCOMM_", NULL};
9: static PetscErrorCode PetscSubcommCreate_contiguous(PetscSubcomm);
10: static PetscErrorCode PetscSubcommCreate_interlaced(PetscSubcomm);
12: /*@
13: PetscSubcommSetFromOptions - Allows setting options for a `PetscSubcomm`
15: Collective
17: Input Parameter:
18: . psubcomm - `PetscSubcomm` context
20: Level: beginner
22: .seealso: `PetscSubcomm`, `PetscSubcommCreate()`
23: @*/
24: PetscErrorCode PetscSubcommSetFromOptions(PetscSubcomm psubcomm)
25: {
26: PetscSubcommType type;
27: PetscBool flg;
29: PetscFunctionBegin;
30: PetscCheck(psubcomm, PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "Must call PetscSubcommCreate first");
32: PetscOptionsBegin(psubcomm->parent, psubcomm->subcommprefix, "Options for PetscSubcomm", NULL);
33: PetscCall(PetscOptionsEnum("-psubcomm_type", NULL, NULL, PetscSubcommTypes, (PetscEnum)psubcomm->type, (PetscEnum *)&type, &flg));
34: if (flg && psubcomm->type != type) {
35: /* free old structures */
36: PetscCall(PetscCommDestroy(&(psubcomm)->dupparent));
37: PetscCall(PetscCommDestroy(&(psubcomm)->child));
38: PetscCall(PetscFree((psubcomm)->subsize));
39: switch (type) {
40: case PETSC_SUBCOMM_GENERAL:
41: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Runtime option PETSC_SUBCOMM_GENERAL is not supported, use PetscSubcommSetTypeGeneral()");
42: case PETSC_SUBCOMM_CONTIGUOUS:
43: PetscCall(PetscSubcommCreate_contiguous(psubcomm));
44: break;
45: case PETSC_SUBCOMM_INTERLACED:
46: PetscCall(PetscSubcommCreate_interlaced(psubcomm));
47: break;
48: default:
49: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "PetscSubcommType %s is not supported yet", PetscSubcommTypes[type]);
50: }
51: }
53: PetscCall(PetscOptionsName("-psubcomm_view", "Triggers display of PetscSubcomm context", "PetscSubcommView", &flg));
54: if (flg) PetscCall(PetscSubcommView(psubcomm, PETSC_VIEWER_STDOUT_(psubcomm->parent)));
55: PetscOptionsEnd();
56: PetscFunctionReturn(PETSC_SUCCESS);
57: }
59: /*@
60: PetscSubcommSetOptionsPrefix - Sets the prefix used for searching for options in the options database for this object
62: Logically Collective
64: Level: intermediate
66: Input Parameters:
67: + psubcomm - `PetscSubcomm` context
68: - pre - the prefix to prepend all `PetscSubcomm` item names with.
70: .seealso: `PetscSubcomm`, `PetscSubcommCreate()`
71: @*/
72: PetscErrorCode PetscSubcommSetOptionsPrefix(PetscSubcomm psubcomm, const char pre[])
73: {
74: PetscFunctionBegin;
75: if (!pre) {
76: PetscCall(PetscFree(psubcomm->subcommprefix));
77: } else {
78: PetscCheck(pre[0] != '-', PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Options prefix should not begin with a hyphen");
79: PetscCall(PetscFree(psubcomm->subcommprefix));
80: PetscCall(PetscStrallocpy(pre, &psubcomm->subcommprefix));
81: }
82: PetscFunctionReturn(PETSC_SUCCESS);
83: }
85: /*@
86: PetscSubcommView - Views a `PetscSubcomm`
88: Collective
90: Input Parameters:
91: + psubcomm - `PetscSubcomm` context
92: - viewer - `PetscViewer` to display the information
94: Level: beginner
96: .seealso: `PetscSubcomm`, `PetscSubcommCreate()`, `PetscViewer`
97: @*/
98: PetscErrorCode PetscSubcommView(PetscSubcomm psubcomm, PetscViewer viewer)
99: {
100: PetscBool iascii;
101: PetscViewerFormat format;
103: PetscFunctionBegin;
104: PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii));
105: if (iascii) {
106: PetscCall(PetscViewerGetFormat(viewer, &format));
107: if (format == PETSC_VIEWER_DEFAULT) {
108: MPI_Comm comm = psubcomm->parent;
109: PetscMPIInt rank, size, subsize, subrank, duprank;
111: PetscCallMPI(MPI_Comm_size(comm, &size));
112: PetscCall(PetscViewerASCIIPrintf(viewer, "PetscSubcomm type %s with total %d MPI processes:\n", PetscSubcommTypes[psubcomm->type], size));
113: PetscCallMPI(MPI_Comm_rank(comm, &rank));
114: PetscCallMPI(MPI_Comm_size(psubcomm->child, &subsize));
115: PetscCallMPI(MPI_Comm_rank(psubcomm->child, &subrank));
116: PetscCallMPI(MPI_Comm_rank(psubcomm->dupparent, &duprank));
117: PetscCall(PetscViewerASCIIPushSynchronized(viewer));
118: PetscCall(PetscViewerASCIISynchronizedPrintf(viewer, " [%d], color %d, sub-size %d, sub-rank %d, duprank %d\n", rank, psubcomm->color, subsize, subrank, duprank));
119: PetscCall(PetscViewerFlush(viewer));
120: PetscCall(PetscViewerASCIIPopSynchronized(viewer));
121: }
122: } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Not supported yet");
123: PetscFunctionReturn(PETSC_SUCCESS);
124: }
126: /*@
127: PetscSubcommSetNumber - Set total number of subcommunicators desired in the given `PetscSubcomm`
129: Collective
131: Input Parameters:
132: + psubcomm - `PetscSubcomm` context
133: - nsubcomm - the total number of subcommunicators in psubcomm
135: Level: advanced
137: .seealso: `PetscSubcomm`, `PetscSubcommCreate()`, `PetscSubcommDestroy()`, `PetscSubcommSetType()`, `PetscSubcommSetTypeGeneral()`
138: @*/
139: PetscErrorCode PetscSubcommSetNumber(PetscSubcomm psubcomm, PetscInt nsubcomm)
140: {
141: MPI_Comm comm = psubcomm->parent;
142: PetscMPIInt msub, size;
144: PetscFunctionBegin;
145: PetscCheck(psubcomm, PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "PetscSubcomm is not created. Call PetscSubcommCreate() first");
146: PetscCallMPI(MPI_Comm_size(comm, &size));
147: PetscCall(PetscMPIIntCast(nsubcomm, &msub));
148: PetscCheck(msub >= 1 && msub <= size, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Num of subcommunicators %d cannot be < 1 or > input comm size %d", msub, size);
150: psubcomm->n = msub;
151: PetscFunctionReturn(PETSC_SUCCESS);
152: }
154: /*@
155: PetscSubcommSetType - Set the way the original MPI communicator is divided up in the `PetscSubcomm`
157: Collective
159: Input Parameters:
160: + psubcomm - `PetscSubcomm` context
161: - subcommtype - `PetscSubcommType` `PETSC_SUBCOMM_CONTIGUOUS` or `PETSC_SUBCOMM_INTERLACED`
163: Level: advanced
165: .seealso: `PetscSubcommType`, `PETSC_SUBCOMM_CONTIGUOUS`, `PETSC_SUBCOMM_INTERLACED`,
166: `PetscSubcommCreate()`, `PetscSubcommDestroy()`, `PetscSubcommSetNumber()`, `PetscSubcommSetTypeGeneral()`
167: @*/
168: PetscErrorCode PetscSubcommSetType(PetscSubcomm psubcomm, PetscSubcommType subcommtype)
169: {
170: PetscFunctionBegin;
171: PetscCheck(psubcomm, PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "PetscSubcomm is not created. Call PetscSubcommCreate()");
172: PetscCheck(psubcomm->n >= 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "number of subcommunicators %d is incorrect. Call PetscSubcommSetNumber()", psubcomm->n);
174: if (subcommtype == PETSC_SUBCOMM_CONTIGUOUS) {
175: PetscCall(PetscSubcommCreate_contiguous(psubcomm));
176: } else if (subcommtype == PETSC_SUBCOMM_INTERLACED) {
177: PetscCall(PetscSubcommCreate_interlaced(psubcomm));
178: } else SETERRQ(psubcomm->parent, PETSC_ERR_SUP, "PetscSubcommType %s is not supported yet", PetscSubcommTypes[subcommtype]);
179: PetscFunctionReturn(PETSC_SUCCESS);
180: }
182: /*@
183: PetscSubcommSetTypeGeneral - Divides up a communicator based on a specific user's specification
185: Collective
187: Input Parameters:
188: + psubcomm - `PetscSubcomm` context
189: . color - control of subset assignment (nonnegative integer). Processes with the same color are in the same subcommunicator.
190: - subrank - rank in the subcommunicator
192: Level: advanced
194: .seealso: `PetscSubcommType`, `PETSC_SUBCOMM_CONTIGUOUS`, `PETSC_SUBCOMM_INTERLACED`, `PetscSubcommCreate()`, `PetscSubcommDestroy()`, `PetscSubcommSetNumber()`, `PetscSubcommSetType()`
195: @*/
196: PetscErrorCode PetscSubcommSetTypeGeneral(PetscSubcomm psubcomm, PetscMPIInt color, PetscMPIInt subrank)
197: {
198: MPI_Comm subcomm = 0, dupcomm = 0, comm = psubcomm->parent;
199: PetscMPIInt size, icolor, duprank, *recvbuf, sendbuf[3], mysubsize, rank, *subsize;
200: PetscMPIInt i, nsubcomm = psubcomm->n;
202: PetscFunctionBegin;
203: PetscCheck(psubcomm, PETSC_COMM_SELF, PETSC_ERR_ARG_NULL, "PetscSubcomm is not created. Call PetscSubcommCreate()");
204: PetscCheck(nsubcomm >= 1, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "number of subcommunicators %d is incorrect. Call PetscSubcommSetNumber()", nsubcomm);
206: PetscCallMPI(MPI_Comm_split(comm, color, subrank, &subcomm));
208: /* create dupcomm with same size as comm, but its rank, duprank, maps subcomm's contiguously into dupcomm */
209: /* TODO: this can be done in an ostensibly scalale way (i.e., without allocating an array of size 'size') as is done in PetscObjectsCreateGlobalOrdering(). */
210: PetscCallMPI(MPI_Comm_size(comm, &size));
211: PetscCall(PetscMalloc1(2 * size, &recvbuf));
213: PetscCallMPI(MPI_Comm_rank(comm, &rank));
214: PetscCallMPI(MPI_Comm_size(subcomm, &mysubsize));
216: sendbuf[0] = color;
217: sendbuf[1] = mysubsize;
218: PetscCallMPI(MPI_Allgather(sendbuf, 2, MPI_INT, recvbuf, 2, MPI_INT, comm));
220: PetscCall(PetscCalloc1(nsubcomm, &subsize));
221: for (i = 0; i < 2 * size; i += 2) subsize[recvbuf[i]] = recvbuf[i + 1];
222: PetscCall(PetscFree(recvbuf));
224: duprank = 0;
225: for (icolor = 0; icolor < nsubcomm; icolor++) {
226: if (icolor != color) { /* not color of this process */
227: duprank += subsize[icolor];
228: } else {
229: duprank += subrank;
230: break;
231: }
232: }
233: PetscCallMPI(MPI_Comm_split(comm, 0, duprank, &dupcomm));
235: PetscCall(PetscCommDuplicate(dupcomm, &psubcomm->dupparent, NULL));
236: PetscCall(PetscCommDuplicate(subcomm, &psubcomm->child, NULL));
237: PetscCallMPI(MPI_Comm_free(&dupcomm));
238: PetscCallMPI(MPI_Comm_free(&subcomm));
240: psubcomm->color = color;
241: psubcomm->subsize = subsize;
242: psubcomm->type = PETSC_SUBCOMM_GENERAL;
243: PetscFunctionReturn(PETSC_SUCCESS);
244: }
246: /*@
247: PetscSubcommDestroy - Destroys a `PetscSubcomm` object
249: Collective
251: Input Parameter:
252: . psubcomm - the `PetscSubcomm` context
254: Level: advanced
256: .seealso: `PetscSubcommCreate()`, `PetscSubcommSetType()`
257: @*/
258: PetscErrorCode PetscSubcommDestroy(PetscSubcomm *psubcomm)
259: {
260: PetscFunctionBegin;
261: if (!*psubcomm) PetscFunctionReturn(PETSC_SUCCESS);
262: PetscCall(PetscCommDestroy(&(*psubcomm)->dupparent));
263: PetscCall(PetscCommDestroy(&(*psubcomm)->child));
264: PetscCall(PetscFree((*psubcomm)->subsize));
265: if ((*psubcomm)->subcommprefix) PetscCall(PetscFree((*psubcomm)->subcommprefix));
266: PetscCall(PetscFree(*psubcomm));
267: PetscFunctionReturn(PETSC_SUCCESS);
268: }
270: /*@
271: PetscSubcommCreate - Create a `PetscSubcomm` context. This object is used to manage the division of a `MPI_Comm` into subcommunicators
273: Collective
275: Input Parameter:
276: . comm - MPI communicator
278: Output Parameter:
279: . psubcomm - location to store the `PetscSubcomm` context
281: Level: advanced
283: .seealso: `PetscSubcomm`, `PetscSubcommDestroy()`, `PetscSubcommSetTypeGeneral()`, `PetscSubcommSetFromOptions()`, `PetscSubcommSetType()`,
284: `PetscSubcommSetNumber()`
285: @*/
286: PetscErrorCode PetscSubcommCreate(MPI_Comm comm, PetscSubcomm *psubcomm)
287: {
288: PetscMPIInt rank, size;
290: PetscFunctionBegin;
291: PetscCall(PetscNew(psubcomm));
293: /* set defaults */
294: PetscCallMPI(MPI_Comm_rank(comm, &rank));
295: PetscCallMPI(MPI_Comm_size(comm, &size));
297: (*psubcomm)->parent = comm;
298: (*psubcomm)->dupparent = comm;
299: (*psubcomm)->child = PETSC_COMM_SELF;
300: (*psubcomm)->n = size;
301: (*psubcomm)->color = rank;
302: (*psubcomm)->subsize = NULL;
303: (*psubcomm)->type = PETSC_SUBCOMM_INTERLACED;
304: PetscFunctionReturn(PETSC_SUCCESS);
305: }
307: /*@C
308: PetscSubcommGetParent - Gets the communicator that was used to create the `PetscSubcomm`
310: Collective
312: Input Parameter:
313: . scomm - the `PetscSubcomm`
315: Output Parameter:
316: . pcomm - location to store the parent communicator
318: Level: intermediate
320: .seealso: `PetscSubcommDestroy()`, `PetscSubcommSetTypeGeneral()`, `PetscSubcommSetFromOptions()`, `PetscSubcommSetType()`,
321: `PetscSubcommSetNumber()`, `PetscSubcommGetChild()`, `PetscSubcommContiguousParent()`
322: @*/
323: PetscErrorCode PetscSubcommGetParent(PetscSubcomm scomm, MPI_Comm *pcomm)
324: {
325: *pcomm = PetscSubcommParent(scomm);
326: return PETSC_SUCCESS;
327: }
329: /*@C
330: PetscSubcommGetContiguousParent - Gets a communicator that is a duplicate of the parent but has the ranks
331: reordered by the order they are in the children
333: Collective
335: Input Parameter:
336: . scomm - the `PetscSubcomm`
338: Output Parameter:
339: . pcomm - location to store the parent communicator
341: Level: intermediate
343: .seealso: `PetscSubcommDestroy()`, `PetscSubcommSetTypeGeneral()`, `PetscSubcommSetFromOptions()`, `PetscSubcommSetType()`,
344: `PetscSubcommSetNumber()`, `PetscSubcommGetChild()`, `PetscSubcommContiguousParent()`
345: @*/
346: PetscErrorCode PetscSubcommGetContiguousParent(PetscSubcomm scomm, MPI_Comm *pcomm)
347: {
348: *pcomm = PetscSubcommContiguousParent(scomm);
349: return PETSC_SUCCESS;
350: }
352: /*@C
353: PetscSubcommGetChild - Gets the communicator created by the `PetscSubcomm`. This is part of one of the subcommunicators created by the `PetscSubcomm`
355: Collective
357: Input Parameter:
358: . scomm - the `PetscSubcomm`
360: Output Parameter:
361: . ccomm - location to store the child communicator
363: Level: intermediate
365: .seealso: `PetscSubcommDestroy()`, `PetscSubcommSetTypeGeneral()`, `PetscSubcommSetFromOptions()`, `PetscSubcommSetType()`,
366: `PetscSubcommSetNumber()`, `PetscSubcommGetParent()`, `PetscSubcommContiguousParent()`
367: @*/
368: PetscErrorCode PetscSubcommGetChild(PetscSubcomm scomm, MPI_Comm *ccomm)
369: {
370: *ccomm = PetscSubcommChild(scomm);
371: return PETSC_SUCCESS;
372: }
374: static PetscErrorCode PetscSubcommCreate_contiguous(PetscSubcomm psubcomm)
375: {
376: PetscMPIInt rank, size, *subsize, duprank = -1, subrank = -1;
377: PetscMPIInt np_subcomm, nleftover, i, color = -1, rankstart, nsubcomm = psubcomm->n;
378: MPI_Comm subcomm = 0, dupcomm = 0, comm = psubcomm->parent;
380: PetscFunctionBegin;
381: PetscCallMPI(MPI_Comm_rank(comm, &rank));
382: PetscCallMPI(MPI_Comm_size(comm, &size));
384: /* get size of each subcommunicator */
385: PetscCall(PetscMalloc1(1 + nsubcomm, &subsize));
387: np_subcomm = size / nsubcomm;
388: nleftover = size - nsubcomm * np_subcomm;
389: for (i = 0; i < nsubcomm; i++) {
390: subsize[i] = np_subcomm;
391: if (i < nleftover) subsize[i]++;
392: }
394: /* get color and subrank of this proc */
395: rankstart = 0;
396: for (i = 0; i < nsubcomm; i++) {
397: if (rank >= rankstart && rank < rankstart + subsize[i]) {
398: color = i;
399: subrank = rank - rankstart;
400: duprank = rank;
401: break;
402: } else rankstart += subsize[i];
403: }
405: PetscCallMPI(MPI_Comm_split(comm, color, subrank, &subcomm));
407: /* create dupcomm with same size as comm, but its rank, duprank, maps subcomm's contiguously into dupcomm */
408: PetscCallMPI(MPI_Comm_split(comm, 0, duprank, &dupcomm));
409: PetscCall(PetscCommDuplicate(dupcomm, &psubcomm->dupparent, NULL));
410: PetscCall(PetscCommDuplicate(subcomm, &psubcomm->child, NULL));
411: PetscCallMPI(MPI_Comm_free(&dupcomm));
412: PetscCallMPI(MPI_Comm_free(&subcomm));
414: psubcomm->color = color;
415: psubcomm->subsize = subsize;
416: psubcomm->type = PETSC_SUBCOMM_CONTIGUOUS;
417: PetscFunctionReturn(PETSC_SUCCESS);
418: }
420: /*
421: Note:
422: In PCREDUNDANT, to avoid data scattering from subcomm back to original comm, we create subcommunicators
423: by iteratively taking a process into a subcommunicator.
424: Example: size=4, nsubcomm=(*psubcomm)->n=3
425: comm=(*psubcomm)->parent:
426: rank: [0] [1] [2] [3]
427: color: 0 1 2 0
429: subcomm=(*psubcomm)->comm:
430: subrank: [0] [0] [0] [1]
432: dupcomm=(*psubcomm)->dupparent:
433: duprank: [0] [2] [3] [1]
435: Here, subcomm[color = 0] has subsize=2, owns process [0] and [3]
436: subcomm[color = 1] has subsize=1, owns process [1]
437: subcomm[color = 2] has subsize=1, owns process [2]
438: dupcomm has same number of processes as comm, and its duprank maps
439: processes in subcomm contiguously into a 1d array:
440: duprank: [0] [1] [2] [3]
441: rank: [0] [3] [1] [2]
442: subcomm[0] subcomm[1] subcomm[2]
443: */
445: static PetscErrorCode PetscSubcommCreate_interlaced(PetscSubcomm psubcomm)
446: {
447: PetscMPIInt rank, size, *subsize, duprank, subrank;
448: PetscMPIInt np_subcomm, nleftover, i, j, color, nsubcomm = psubcomm->n;
449: MPI_Comm subcomm = 0, dupcomm = 0, comm = psubcomm->parent;
451: PetscFunctionBegin;
452: PetscCallMPI(MPI_Comm_rank(comm, &rank));
453: PetscCallMPI(MPI_Comm_size(comm, &size));
455: /* get size of each subcommunicator */
456: PetscCall(PetscMalloc1(1 + nsubcomm, &subsize));
458: np_subcomm = size / nsubcomm;
459: nleftover = size - nsubcomm * np_subcomm;
460: for (i = 0; i < nsubcomm; i++) {
461: subsize[i] = np_subcomm;
462: if (i < nleftover) subsize[i]++;
463: }
465: /* find color for this proc */
466: color = rank % nsubcomm;
467: subrank = rank / nsubcomm;
469: PetscCallMPI(MPI_Comm_split(comm, color, subrank, &subcomm));
471: j = 0;
472: duprank = 0;
473: for (i = 0; i < nsubcomm; i++) {
474: if (j == color) {
475: duprank += subrank;
476: break;
477: }
478: duprank += subsize[i];
479: j++;
480: }
482: /* create dupcomm with same size as comm, but its rank, duprank, maps subcomm's contiguously into dupcomm */
483: PetscCallMPI(MPI_Comm_split(comm, 0, duprank, &dupcomm));
484: PetscCall(PetscCommDuplicate(dupcomm, &psubcomm->dupparent, NULL));
485: PetscCall(PetscCommDuplicate(subcomm, &psubcomm->child, NULL));
486: PetscCallMPI(MPI_Comm_free(&dupcomm));
487: PetscCallMPI(MPI_Comm_free(&subcomm));
489: psubcomm->color = color;
490: psubcomm->subsize = subsize;
491: psubcomm->type = PETSC_SUBCOMM_INTERLACED;
492: PetscFunctionReturn(PETSC_SUCCESS);
493: }