Actual source code: telescope.c
1: #include <petsc/private/petscimpl.h>
2: #include <petsc/private/matimpl.h>
3: #include <petsc/private/pcimpl.h>
4: #include <petscksp.h>
5: #include <petscdm.h>
6: #include "../src/ksp/pc/impls/telescope/telescope.h"
8: static PetscBool cited = PETSC_FALSE;
9: static const char citation[] = "@inproceedings{MaySananRuppKnepleySmith2016,\n"
10: " title = {Extreme-Scale Multigrid Components within PETSc},\n"
11: " author = {Dave A. May and Patrick Sanan and Karl Rupp and Matthew G. Knepley and Barry F. Smith},\n"
12: " booktitle = {Proceedings of the Platform for Advanced Scientific Computing Conference},\n"
13: " series = {PASC '16},\n"
14: " isbn = {978-1-4503-4126-4},\n"
15: " location = {Lausanne, Switzerland},\n"
16: " pages = {5:1--5:12},\n"
17: " articleno = {5},\n"
18: " numpages = {12},\n"
19: " url = {https://doi.acm.org/10.1145/2929908.2929913},\n"
20: " doi = {10.1145/2929908.2929913},\n"
21: " acmid = {2929913},\n"
22: " publisher = {ACM},\n"
23: " address = {New York, NY, USA},\n"
24: " keywords = {GPU, HPC, agglomeration, coarse-level solver, multigrid, parallel computing, preconditioning},\n"
25: " year = {2016}\n"
26: "}\n";
28: /*
29: default setup mode
31: [1a] scatter to (FORWARD)
32: x(comm) -> xtmp(comm)
33: [1b] local copy (to) ranks with color = 0
34: xred(subcomm) <- xtmp
36: [2] solve on sub KSP to obtain yred(subcomm)
38: [3a] local copy (from) ranks with color = 0
39: yred(subcomm) --> xtmp
40: [2b] scatter from (REVERSE)
41: xtmp(comm) -> y(comm)
42: */
44: /*
45: Collective[comm_f]
46: Notes
47: * Using comm_f = MPI_COMM_NULL will result in an error
48: * Using comm_c = MPI_COMM_NULL is valid. If all instances of comm_c are NULL the subcomm is not valid.
49: * If any non NULL comm_c communicator cannot map any of its ranks to comm_f, the subcomm is not valid.
50: */
51: static PetscErrorCode PCTelescopeTestValidSubcomm(MPI_Comm comm_f, MPI_Comm comm_c, PetscBool *isvalid)
52: {
53: PetscInt valid = 1;
54: MPI_Group group_f, group_c;
55: PetscMPIInt count, k, size_f = 0, size_c = 0, size_c_sum = 0;
56: PetscMPIInt *ranks_f, *ranks_c;
58: PetscFunctionBegin;
59: PetscCheck(comm_f != MPI_COMM_NULL, PETSC_COMM_SELF, PETSC_ERR_SUP, "comm_f cannot be MPI_COMM_NULL");
61: PetscCallMPI(MPI_Comm_group(comm_f, &group_f));
62: if (comm_c != MPI_COMM_NULL) PetscCallMPI(MPI_Comm_group(comm_c, &group_c));
64: PetscCallMPI(MPI_Comm_size(comm_f, &size_f));
65: if (comm_c != MPI_COMM_NULL) PetscCallMPI(MPI_Comm_size(comm_c, &size_c));
67: /* check not all comm_c's are NULL */
68: size_c_sum = size_c;
69: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &size_c_sum, 1, MPI_INT, MPI_SUM, comm_f));
70: if (size_c_sum == 0) valid = 0;
72: /* check we can map at least 1 rank in comm_c to comm_f */
73: PetscCall(PetscMalloc1(size_f, &ranks_f));
74: PetscCall(PetscMalloc1(size_c, &ranks_c));
75: for (k = 0; k < size_f; k++) ranks_f[k] = MPI_UNDEFINED;
76: for (k = 0; k < size_c; k++) ranks_c[k] = k;
78: /*
79: MPI_Group_translate_ranks() returns a non-zero exit code if any rank cannot be translated.
80: I do not want the code to terminate immediately if this occurs, rather I want to throw
81: the error later (during PCSetUp_Telescope()) via SETERRQ() with a message indicating
82: that comm_c is not a valid sub-communicator.
83: Hence I purposefully do not call PetscCall() after MPI_Group_translate_ranks().
84: */
85: count = 0;
86: if (comm_c != MPI_COMM_NULL) {
87: (void)MPI_Group_translate_ranks(group_c, size_c, ranks_c, group_f, ranks_f);
88: for (k = 0; k < size_f; k++) {
89: if (ranks_f[k] == MPI_UNDEFINED) count++;
90: }
91: }
92: if (count == size_f) valid = 0;
94: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &valid, 1, MPIU_INT, MPI_MIN, comm_f));
95: if (valid == 1) *isvalid = PETSC_TRUE;
96: else *isvalid = PETSC_FALSE;
98: PetscCall(PetscFree(ranks_f));
99: PetscCall(PetscFree(ranks_c));
100: PetscCallMPI(MPI_Group_free(&group_f));
101: if (comm_c != MPI_COMM_NULL) PetscCallMPI(MPI_Group_free(&group_c));
102: PetscFunctionReturn(PETSC_SUCCESS);
103: }
105: static DM private_PCTelescopeGetSubDM(PC_Telescope sred)
106: {
107: DM subdm = NULL;
109: if (!PCTelescope_isActiveRank(sred)) {
110: subdm = NULL;
111: } else {
112: switch (sred->sr_type) {
113: case TELESCOPE_DEFAULT:
114: subdm = NULL;
115: break;
116: case TELESCOPE_DMDA:
117: subdm = ((PC_Telescope_DMDACtx *)sred->dm_ctx)->dmrepart;
118: break;
119: case TELESCOPE_DMPLEX:
120: subdm = NULL;
121: break;
122: case TELESCOPE_COARSEDM:
123: if (sred->ksp) PetscCallAbort(PETSC_COMM_SELF, KSPGetDM(sred->ksp, &subdm));
124: break;
125: }
126: }
127: return subdm;
128: }
130: static PetscErrorCode PCTelescopeSetUp_default(PC pc, PC_Telescope sred)
131: {
132: PetscInt m, M, bs, st, ed;
133: Vec x, xred, yred, xtmp;
134: Mat B;
135: MPI_Comm comm, subcomm;
136: VecScatter scatter;
137: IS isin;
138: VecType vectype;
140: PetscFunctionBegin;
141: PetscCall(PetscInfo(pc, "PCTelescope: setup (default)\n"));
142: comm = PetscSubcommParent(sred->psubcomm);
143: subcomm = PetscSubcommChild(sred->psubcomm);
145: PetscCall(PCGetOperators(pc, NULL, &B));
146: PetscCall(MatGetSize(B, &M, NULL));
147: PetscCall(MatGetBlockSize(B, &bs));
148: PetscCall(MatCreateVecs(B, &x, NULL));
149: PetscCall(MatGetVecType(B, &vectype)); /* Use the vectype of the matrix used to construct the preconditioner by default */
151: xred = NULL;
152: m = 0;
153: if (PCTelescope_isActiveRank(sred)) {
154: PetscCall(VecCreate(subcomm, &xred));
155: PetscCall(VecSetSizes(xred, PETSC_DECIDE, M));
156: PetscCall(VecSetBlockSize(xred, bs));
157: PetscCall(VecSetType(xred, vectype));
158: PetscCall(VecSetFromOptions(xred));
159: PetscCall(VecGetLocalSize(xred, &m));
160: }
162: yred = NULL;
163: if (PCTelescope_isActiveRank(sred)) PetscCall(VecDuplicate(xred, &yred));
165: PetscCall(VecCreate(comm, &xtmp));
166: PetscCall(VecSetSizes(xtmp, m, PETSC_DECIDE));
167: PetscCall(VecSetBlockSize(xtmp, bs));
168: PetscCall(VecSetType(xtmp, vectype));
170: if (PCTelescope_isActiveRank(sred)) {
171: PetscCall(VecGetOwnershipRange(xred, &st, &ed));
172: PetscCall(ISCreateStride(comm, ed - st, st, 1, &isin));
173: } else {
174: PetscCall(VecGetOwnershipRange(x, &st, &ed));
175: PetscCall(ISCreateStride(comm, 0, st, 1, &isin));
176: }
177: PetscCall(ISSetBlockSize(isin, bs));
179: PetscCall(VecScatterCreate(x, isin, xtmp, NULL, &scatter));
181: sred->isin = isin;
182: sred->scatter = scatter;
183: sred->xred = xred;
184: sred->yred = yred;
185: sred->xtmp = xtmp;
186: PetscCall(VecDestroy(&x));
187: PetscFunctionReturn(PETSC_SUCCESS);
188: }
190: static PetscErrorCode PCTelescopeMatCreate_default(PC pc, PC_Telescope sred, MatReuse reuse, Mat *A)
191: {
192: MPI_Comm comm, subcomm;
193: Mat Bred, B;
194: PetscInt nr, nc, bs;
195: IS isrow, iscol;
196: Mat Blocal, *_Blocal;
198: PetscFunctionBegin;
199: PetscCall(PetscInfo(pc, "PCTelescope: updating the redundant preconditioned operator (default)\n"));
200: PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
201: subcomm = PetscSubcommChild(sred->psubcomm);
202: PetscCall(PCGetOperators(pc, NULL, &B));
203: PetscCall(MatGetSize(B, &nr, &nc));
204: isrow = sred->isin;
205: PetscCall(ISCreateStride(PETSC_COMM_SELF, nc, 0, 1, &iscol));
206: PetscCall(ISSetIdentity(iscol));
207: PetscCall(MatGetBlockSizes(B, NULL, &bs));
208: PetscCall(ISSetBlockSize(iscol, bs));
209: PetscCall(MatSetOption(B, MAT_SUBMAT_SINGLEIS, PETSC_TRUE));
210: PetscCall(MatCreateSubMatrices(B, 1, &isrow, &iscol, MAT_INITIAL_MATRIX, &_Blocal));
211: Blocal = *_Blocal;
212: PetscCall(PetscFree(_Blocal));
213: Bred = NULL;
214: if (PCTelescope_isActiveRank(sred)) {
215: PetscInt mm;
217: if (reuse != MAT_INITIAL_MATRIX) Bred = *A;
219: PetscCall(MatGetSize(Blocal, &mm, NULL));
220: PetscCall(MatCreateMPIMatConcatenateSeqMat(subcomm, Blocal, mm, reuse, &Bred));
221: PetscCall(MatPropagateSymmetryOptions(B, Bred));
222: }
223: *A = Bred;
224: PetscCall(ISDestroy(&iscol));
225: PetscCall(MatDestroy(&Blocal));
226: PetscFunctionReturn(PETSC_SUCCESS);
227: }
229: static PetscErrorCode PCTelescopeSubNullSpaceCreate_Telescope(PC pc, PC_Telescope sred, MatNullSpace nullspace, MatNullSpace *sub_nullspace)
230: {
231: PetscBool has_const;
232: const Vec *vecs;
233: Vec *sub_vecs = NULL;
234: PetscInt i, k, n = 0;
235: MPI_Comm subcomm;
237: PetscFunctionBegin;
238: subcomm = PetscSubcommChild(sred->psubcomm);
239: PetscCall(MatNullSpaceGetVecs(nullspace, &has_const, &n, &vecs));
241: if (PCTelescope_isActiveRank(sred)) {
242: if (n) PetscCall(VecDuplicateVecs(sred->xred, n, &sub_vecs));
243: }
245: /* copy entries */
246: for (k = 0; k < n; k++) {
247: const PetscScalar *x_array;
248: PetscScalar *LA_sub_vec;
249: PetscInt st, ed;
251: /* pull in vector x->xtmp */
252: PetscCall(VecScatterBegin(sred->scatter, vecs[k], sred->xtmp, INSERT_VALUES, SCATTER_FORWARD));
253: PetscCall(VecScatterEnd(sred->scatter, vecs[k], sred->xtmp, INSERT_VALUES, SCATTER_FORWARD));
254: if (sub_vecs) {
255: /* copy vector entries into xred */
256: PetscCall(VecGetArrayRead(sred->xtmp, &x_array));
257: if (sub_vecs[k]) {
258: PetscCall(VecGetOwnershipRange(sub_vecs[k], &st, &ed));
259: PetscCall(VecGetArray(sub_vecs[k], &LA_sub_vec));
260: for (i = 0; i < ed - st; i++) LA_sub_vec[i] = x_array[i];
261: PetscCall(VecRestoreArray(sub_vecs[k], &LA_sub_vec));
262: }
263: PetscCall(VecRestoreArrayRead(sred->xtmp, &x_array));
264: }
265: }
267: if (PCTelescope_isActiveRank(sred)) {
268: /* create new (near) nullspace for redundant object */
269: PetscCall(MatNullSpaceCreate(subcomm, has_const, n, sub_vecs, sub_nullspace));
270: PetscCall(VecDestroyVecs(n, &sub_vecs));
271: PetscCheck(!nullspace->remove, PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "Propagation of custom remove callbacks not supported when propagating (near) nullspaces with PCTelescope");
272: PetscCheck(!nullspace->rmctx, PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "Propagation of custom remove callback context not supported when propagating (near) nullspaces with PCTelescope");
273: }
274: PetscFunctionReturn(PETSC_SUCCESS);
275: }
277: static PetscErrorCode PCTelescopeMatNullSpaceCreate_default(PC pc, PC_Telescope sred, Mat sub_mat)
278: {
279: Mat B;
281: PetscFunctionBegin;
282: PetscCall(PCGetOperators(pc, NULL, &B));
283: /* Propagate the nullspace if it exists */
284: {
285: MatNullSpace nullspace, sub_nullspace;
286: PetscCall(MatGetNullSpace(B, &nullspace));
287: if (nullspace) {
288: PetscCall(PetscInfo(pc, "PCTelescope: generating nullspace (default)\n"));
289: PetscCall(PCTelescopeSubNullSpaceCreate_Telescope(pc, sred, nullspace, &sub_nullspace));
290: if (PCTelescope_isActiveRank(sred)) {
291: PetscCall(MatSetNullSpace(sub_mat, sub_nullspace));
292: PetscCall(MatNullSpaceDestroy(&sub_nullspace));
293: }
294: }
295: }
296: /* Propagate the near nullspace if it exists */
297: {
298: MatNullSpace nearnullspace, sub_nearnullspace;
299: PetscCall(MatGetNearNullSpace(B, &nearnullspace));
300: if (nearnullspace) {
301: PetscCall(PetscInfo(pc, "PCTelescope: generating near nullspace (default)\n"));
302: PetscCall(PCTelescopeSubNullSpaceCreate_Telescope(pc, sred, nearnullspace, &sub_nearnullspace));
303: if (PCTelescope_isActiveRank(sred)) {
304: PetscCall(MatSetNearNullSpace(sub_mat, sub_nearnullspace));
305: PetscCall(MatNullSpaceDestroy(&sub_nearnullspace));
306: }
307: }
308: }
309: PetscFunctionReturn(PETSC_SUCCESS);
310: }
312: static PetscErrorCode PCView_Telescope(PC pc, PetscViewer viewer)
313: {
314: PC_Telescope sred = (PC_Telescope)pc->data;
315: PetscBool isascii, isstring;
316: PetscViewer subviewer;
318: PetscFunctionBegin;
319: PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &isascii));
320: PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERSTRING, &isstring));
321: if (isascii) {
322: {
323: MPI_Comm comm, subcomm;
324: PetscMPIInt comm_size, subcomm_size;
325: DM dm = NULL, subdm = NULL;
327: PetscCall(PCGetDM(pc, &dm));
328: subdm = private_PCTelescopeGetSubDM(sred);
330: if (sred->psubcomm) {
331: comm = PetscSubcommParent(sred->psubcomm);
332: subcomm = PetscSubcommChild(sred->psubcomm);
333: PetscCallMPI(MPI_Comm_size(comm, &comm_size));
334: PetscCallMPI(MPI_Comm_size(subcomm, &subcomm_size));
336: PetscCall(PetscViewerASCIIPushTab(viewer));
337: PetscCall(PetscViewerASCIIPrintf(viewer, "PETSc subcomm: parent comm size reduction factor = %" PetscInt_FMT "\n", sred->redfactor));
338: PetscCall(PetscViewerASCIIPrintf(viewer, "PETSc subcomm: parent_size = %d , subcomm_size = %d\n", comm_size, subcomm_size));
339: switch (sred->subcommtype) {
340: case PETSC_SUBCOMM_INTERLACED:
341: PetscCall(PetscViewerASCIIPrintf(viewer, "PETSc subcomm: type = %s\n", PetscSubcommTypes[sred->subcommtype]));
342: break;
343: case PETSC_SUBCOMM_CONTIGUOUS:
344: PetscCall(PetscViewerASCIIPrintf(viewer, "PETSc subcomm type = %s\n", PetscSubcommTypes[sred->subcommtype]));
345: break;
346: default:
347: SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "General subcomm type not supported by PCTelescope");
348: }
349: PetscCall(PetscViewerASCIIPopTab(viewer));
350: } else {
351: PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
352: subcomm = sred->subcomm;
353: if (!PCTelescope_isActiveRank(sred)) subcomm = PETSC_COMM_SELF;
355: PetscCall(PetscViewerASCIIPushTab(viewer));
356: PetscCall(PetscViewerASCIIPrintf(viewer, "subcomm: using user provided sub-communicator\n"));
357: PetscCall(PetscViewerASCIIPopTab(viewer));
358: }
360: PetscCall(PetscViewerGetSubViewer(viewer, subcomm, &subviewer));
361: if (PCTelescope_isActiveRank(sred)) {
362: PetscCall(PetscViewerASCIIPushTab(subviewer));
364: if (dm && sred->ignore_dm) PetscCall(PetscViewerASCIIPrintf(subviewer, "ignoring DM\n"));
365: if (sred->ignore_kspcomputeoperators) PetscCall(PetscViewerASCIIPrintf(subviewer, "ignoring KSPComputeOperators\n"));
366: switch (sred->sr_type) {
367: case TELESCOPE_DEFAULT:
368: PetscCall(PetscViewerASCIIPrintf(subviewer, "setup type: default\n"));
369: break;
370: case TELESCOPE_DMDA:
371: PetscCall(PetscViewerASCIIPrintf(subviewer, "setup type: DMDA auto-repartitioning\n"));
372: PetscCall(DMView_DA_Short(subdm, subviewer));
373: break;
374: case TELESCOPE_DMPLEX:
375: PetscCall(PetscViewerASCIIPrintf(subviewer, "setup type: DMPLEX auto-repartitioning\n"));
376: break;
377: case TELESCOPE_COARSEDM:
378: PetscCall(PetscViewerASCIIPrintf(subviewer, "setup type: coarse DM\n"));
379: break;
380: }
382: if (dm) {
383: PetscObject obj = (PetscObject)dm;
384: PetscCall(PetscViewerASCIIPrintf(subviewer, "Parent DM object:"));
385: PetscCall(PetscViewerASCIIUseTabs(subviewer, PETSC_FALSE));
386: if (obj->type_name) PetscCall(PetscViewerASCIIPrintf(subviewer, " type = %s;", obj->type_name));
387: if (obj->name) PetscCall(PetscViewerASCIIPrintf(subviewer, " name = %s;", obj->name));
388: if (obj->prefix) PetscCall(PetscViewerASCIIPrintf(subviewer, " prefix = %s", obj->prefix));
389: PetscCall(PetscViewerASCIIPrintf(subviewer, "\n"));
390: PetscCall(PetscViewerASCIIUseTabs(subviewer, PETSC_TRUE));
391: } else {
392: PetscCall(PetscViewerASCIIPrintf(subviewer, "Parent DM object: NULL\n"));
393: }
394: if (subdm) {
395: PetscObject obj = (PetscObject)subdm;
396: PetscCall(PetscViewerASCIIPrintf(subviewer, "Sub DM object:"));
397: PetscCall(PetscViewerASCIIUseTabs(subviewer, PETSC_FALSE));
398: if (obj->type_name) PetscCall(PetscViewerASCIIPrintf(subviewer, " type = %s;", obj->type_name));
399: if (obj->name) PetscCall(PetscViewerASCIIPrintf(subviewer, " name = %s;", obj->name));
400: if (obj->prefix) PetscCall(PetscViewerASCIIPrintf(subviewer, " prefix = %s", obj->prefix));
401: PetscCall(PetscViewerASCIIPrintf(subviewer, "\n"));
402: PetscCall(PetscViewerASCIIUseTabs(subviewer, PETSC_TRUE));
403: } else {
404: PetscCall(PetscViewerASCIIPrintf(subviewer, "Sub DM object: NULL\n"));
405: }
407: PetscCall(KSPView(sred->ksp, subviewer));
408: PetscCall(PetscViewerASCIIPopTab(subviewer));
409: }
410: PetscCall(PetscViewerRestoreSubViewer(viewer, subcomm, &subviewer));
411: }
412: }
413: PetscFunctionReturn(PETSC_SUCCESS);
414: }
416: static PetscErrorCode PCSetUp_Telescope(PC pc)
417: {
418: PC_Telescope sred = (PC_Telescope)pc->data;
419: MPI_Comm comm, subcomm = 0;
420: PCTelescopeType sr_type;
422: PetscFunctionBegin;
423: PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
425: /* Determine type of setup/update */
426: if (!pc->setupcalled) {
427: PetscBool has_dm, same;
428: DM dm;
430: sr_type = TELESCOPE_DEFAULT;
431: has_dm = PETSC_FALSE;
432: PetscCall(PCGetDM(pc, &dm));
433: if (dm) has_dm = PETSC_TRUE;
434: if (has_dm) {
435: /* check for dmda */
436: PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMDA, &same));
437: if (same) {
438: PetscCall(PetscInfo(pc, "PCTelescope: found DMDA\n"));
439: sr_type = TELESCOPE_DMDA;
440: }
441: /* check for dmplex */
442: PetscCall(PetscObjectTypeCompare((PetscObject)dm, DMPLEX, &same));
443: if (same) {
444: PetscCall(PetscInfo(pc, "PCTelescope: found DMPLEX\n"));
445: sr_type = TELESCOPE_DMPLEX;
446: }
448: if (sred->use_coarse_dm) {
449: PetscCall(PetscInfo(pc, "PCTelescope: using coarse DM\n"));
450: sr_type = TELESCOPE_COARSEDM;
451: }
453: if (sred->ignore_dm) {
454: PetscCall(PetscInfo(pc, "PCTelescope: ignoring DM\n"));
455: sr_type = TELESCOPE_DEFAULT;
456: }
457: }
458: sred->sr_type = sr_type;
459: } else {
460: sr_type = sred->sr_type;
461: }
463: /* set function pointers for repartition setup, matrix creation/update, matrix (near) nullspace, and reset functionality */
464: switch (sr_type) {
465: case TELESCOPE_DEFAULT:
466: sred->pctelescope_setup_type = PCTelescopeSetUp_default;
467: sred->pctelescope_matcreate_type = PCTelescopeMatCreate_default;
468: sred->pctelescope_matnullspacecreate_type = PCTelescopeMatNullSpaceCreate_default;
469: sred->pctelescope_reset_type = NULL;
470: break;
471: case TELESCOPE_DMDA:
472: pc->ops->apply = PCApply_Telescope_dmda;
473: pc->ops->applyrichardson = PCApplyRichardson_Telescope_dmda;
474: sred->pctelescope_setup_type = PCTelescopeSetUp_dmda;
475: sred->pctelescope_matcreate_type = PCTelescopeMatCreate_dmda;
476: sred->pctelescope_matnullspacecreate_type = PCTelescopeMatNullSpaceCreate_dmda;
477: sred->pctelescope_reset_type = PCReset_Telescope_dmda;
478: break;
479: case TELESCOPE_DMPLEX:
480: SETERRQ(comm, PETSC_ERR_SUP, "Support for DMPLEX is currently not available");
481: case TELESCOPE_COARSEDM:
482: pc->ops->apply = PCApply_Telescope_CoarseDM;
483: pc->ops->applyrichardson = PCApplyRichardson_Telescope_CoarseDM;
484: sred->pctelescope_setup_type = PCTelescopeSetUp_CoarseDM;
485: sred->pctelescope_matcreate_type = NULL;
486: sred->pctelescope_matnullspacecreate_type = NULL; /* PCTelescopeMatNullSpaceCreate_CoarseDM; */
487: sred->pctelescope_reset_type = PCReset_Telescope_CoarseDM;
488: break;
489: default:
490: SETERRQ(comm, PETSC_ERR_SUP, "Support only provided for: repartitioning an operator; repartitioning a DMDA; or using a coarse DM");
491: }
493: /* subcomm definition */
494: if (!pc->setupcalled) {
495: if ((sr_type == TELESCOPE_DEFAULT) || (sr_type == TELESCOPE_DMDA)) {
496: if (!sred->psubcomm) {
497: PetscCall(PetscSubcommCreate(comm, &sred->psubcomm));
498: PetscCall(PetscSubcommSetNumber(sred->psubcomm, sred->redfactor));
499: PetscCall(PetscSubcommSetType(sred->psubcomm, sred->subcommtype));
500: sred->subcomm = PetscSubcommChild(sred->psubcomm);
501: }
502: } else { /* query PC for DM, check communicators */
503: DM dm, dm_coarse_partition = NULL;
504: MPI_Comm comm_fine, comm_coarse_partition = MPI_COMM_NULL;
505: PetscMPIInt csize_fine = 0, csize_coarse_partition = 0, cs[2], csg[2], cnt = 0;
506: PetscBool isvalidsubcomm = PETSC_TRUE;
508: PetscCall(PCGetDM(pc, &dm));
509: comm_fine = PetscObjectComm((PetscObject)dm);
510: PetscCall(DMGetCoarseDM(dm, &dm_coarse_partition));
511: if (dm_coarse_partition) cnt = 1;
512: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &cnt, 1, MPI_INT, MPI_SUM, comm_fine));
513: PetscCheck(cnt != 0, comm_fine, PETSC_ERR_SUP, "Zero instances of a coarse DM were found");
515: PetscCallMPI(MPI_Comm_size(comm_fine, &csize_fine));
516: if (dm_coarse_partition) {
517: comm_coarse_partition = PetscObjectComm((PetscObject)dm_coarse_partition);
518: PetscCallMPI(MPI_Comm_size(comm_coarse_partition, &csize_coarse_partition));
519: }
521: cs[0] = csize_fine;
522: cs[1] = csize_coarse_partition;
523: PetscCallMPI(MPIU_Allreduce(cs, csg, 2, MPI_INT, MPI_MAX, comm_fine));
524: PetscCheck(csg[0] != csg[1], comm_fine, PETSC_ERR_SUP, "Coarse DM uses the same size communicator as the parent DM attached to the PC");
526: PetscCall(PCTelescopeTestValidSubcomm(comm_fine, comm_coarse_partition, &isvalidsubcomm));
527: PetscCheck(isvalidsubcomm, comm_fine, PETSC_ERR_SUP, "Coarse DM communicator is not a sub-communicator of parentDM->comm");
528: sred->subcomm = comm_coarse_partition;
529: }
530: }
531: subcomm = sred->subcomm;
533: /* internal KSP */
534: if (!pc->setupcalled) {
535: const char *prefix;
537: if (PCTelescope_isActiveRank(sred)) {
538: PetscCall(KSPCreate(subcomm, &sred->ksp));
539: PetscCall(KSPSetNestLevel(sred->ksp, pc->kspnestlevel));
540: PetscCall(KSPSetErrorIfNotConverged(sred->ksp, pc->erroriffailure));
541: PetscCall(PetscObjectIncrementTabLevel((PetscObject)sred->ksp, (PetscObject)pc, 1));
542: PetscCall(KSPSetType(sred->ksp, KSPPREONLY));
543: PetscCall(PCGetOptionsPrefix(pc, &prefix));
544: PetscCall(KSPSetOptionsPrefix(sred->ksp, prefix));
545: PetscCall(KSPAppendOptionsPrefix(sred->ksp, "telescope_"));
546: }
547: }
549: /* setup */
550: if (!pc->setupcalled && sred->pctelescope_setup_type) PetscCall(sred->pctelescope_setup_type(pc, sred));
551: /* update */
552: if (!pc->setupcalled) {
553: if (sred->pctelescope_matcreate_type) PetscCall(sred->pctelescope_matcreate_type(pc, sred, MAT_INITIAL_MATRIX, &sred->Bred));
554: if (sred->pctelescope_matnullspacecreate_type) PetscCall(sred->pctelescope_matnullspacecreate_type(pc, sred, sred->Bred));
555: } else {
556: if (sred->pctelescope_matcreate_type) PetscCall(sred->pctelescope_matcreate_type(pc, sred, MAT_REUSE_MATRIX, &sred->Bred));
557: }
559: /* common - no construction */
560: if (PCTelescope_isActiveRank(sred)) {
561: PetscCall(KSPSetOperators(sred->ksp, sred->Bred, sred->Bred));
562: if (pc->setfromoptionscalled && !pc->setupcalled) PetscCall(KSPSetFromOptions(sred->ksp));
563: }
564: PetscFunctionReturn(PETSC_SUCCESS);
565: }
567: static PetscErrorCode PCApply_Telescope(PC pc, Vec x, Vec y)
568: {
569: PC_Telescope sred = (PC_Telescope)pc->data;
570: Vec xtmp, xred, yred;
571: PetscInt i, st, ed;
572: VecScatter scatter;
573: PetscScalar *array;
574: const PetscScalar *x_array;
576: PetscFunctionBegin;
577: PetscCall(PetscCitationsRegister(citation, &cited));
579: xtmp = sred->xtmp;
580: scatter = sred->scatter;
581: xred = sred->xred;
582: yred = sred->yred;
584: /* pull in vector x->xtmp */
585: PetscCall(VecScatterBegin(scatter, x, xtmp, INSERT_VALUES, SCATTER_FORWARD));
586: PetscCall(VecScatterEnd(scatter, x, xtmp, INSERT_VALUES, SCATTER_FORWARD));
588: /* copy vector entries into xred */
589: PetscCall(VecGetArrayRead(xtmp, &x_array));
590: if (xred) {
591: PetscScalar *LA_xred;
592: PetscCall(VecGetOwnershipRange(xred, &st, &ed));
593: PetscCall(VecGetArray(xred, &LA_xred));
594: for (i = 0; i < ed - st; i++) LA_xred[i] = x_array[i];
595: PetscCall(VecRestoreArray(xred, &LA_xred));
596: }
597: PetscCall(VecRestoreArrayRead(xtmp, &x_array));
598: /* solve */
599: if (PCTelescope_isActiveRank(sred)) {
600: PetscCall(KSPSolve(sred->ksp, xred, yred));
601: PetscCall(KSPCheckSolve(sred->ksp, pc, yred));
602: }
603: /* return vector */
604: PetscCall(VecGetArray(xtmp, &array));
605: if (yred) {
606: const PetscScalar *LA_yred;
607: PetscCall(VecGetOwnershipRange(yred, &st, &ed));
608: PetscCall(VecGetArrayRead(yred, &LA_yred));
609: for (i = 0; i < ed - st; i++) array[i] = LA_yred[i];
610: PetscCall(VecRestoreArrayRead(yred, &LA_yred));
611: }
612: PetscCall(VecRestoreArray(xtmp, &array));
613: PetscCall(VecScatterBegin(scatter, xtmp, y, INSERT_VALUES, SCATTER_REVERSE));
614: PetscCall(VecScatterEnd(scatter, xtmp, y, INSERT_VALUES, SCATTER_REVERSE));
615: PetscFunctionReturn(PETSC_SUCCESS);
616: }
618: static PetscErrorCode PCApplyRichardson_Telescope(PC pc, Vec x, Vec y, Vec w, PetscReal rtol, PetscReal abstol, PetscReal dtol, PetscInt its, PetscBool zeroguess, PetscInt *outits, PCRichardsonConvergedReason *reason)
619: {
620: PC_Telescope sred = (PC_Telescope)pc->data;
621: Vec xtmp, yred;
622: PetscInt i, st, ed;
623: VecScatter scatter;
624: const PetscScalar *x_array;
625: PetscBool default_init_guess_value;
627: PetscFunctionBegin;
628: xtmp = sred->xtmp;
629: scatter = sred->scatter;
630: yred = sred->yred;
632: PetscCheck(its <= 1, PetscObjectComm((PetscObject)pc), PETSC_ERR_SUP, "PCApplyRichardson_Telescope only supports max_it = 1");
633: *reason = (PCRichardsonConvergedReason)0;
635: if (!zeroguess) {
636: PetscCall(PetscInfo(pc, "PCTelescope: Scattering y for non-zero initial guess\n"));
637: /* pull in vector y->xtmp */
638: PetscCall(VecScatterBegin(scatter, y, xtmp, INSERT_VALUES, SCATTER_FORWARD));
639: PetscCall(VecScatterEnd(scatter, y, xtmp, INSERT_VALUES, SCATTER_FORWARD));
641: /* copy vector entries into xred */
642: PetscCall(VecGetArrayRead(xtmp, &x_array));
643: if (yred) {
644: PetscScalar *LA_yred;
645: PetscCall(VecGetOwnershipRange(yred, &st, &ed));
646: PetscCall(VecGetArray(yred, &LA_yred));
647: for (i = 0; i < ed - st; i++) LA_yred[i] = x_array[i];
648: PetscCall(VecRestoreArray(yred, &LA_yred));
649: }
650: PetscCall(VecRestoreArrayRead(xtmp, &x_array));
651: }
653: if (PCTelescope_isActiveRank(sred)) {
654: PetscCall(KSPGetInitialGuessNonzero(sred->ksp, &default_init_guess_value));
655: if (!zeroguess) PetscCall(KSPSetInitialGuessNonzero(sred->ksp, PETSC_TRUE));
656: }
658: PetscCall(PCApply_Telescope(pc, x, y));
660: if (PCTelescope_isActiveRank(sred)) PetscCall(KSPSetInitialGuessNonzero(sred->ksp, default_init_guess_value));
662: if (!*reason) *reason = PCRICHARDSON_CONVERGED_ITS;
663: *outits = 1;
664: PetscFunctionReturn(PETSC_SUCCESS);
665: }
667: static PetscErrorCode PCReset_Telescope(PC pc)
668: {
669: PC_Telescope sred = (PC_Telescope)pc->data;
671: PetscFunctionBegin;
672: PetscCall(ISDestroy(&sred->isin));
673: PetscCall(VecScatterDestroy(&sred->scatter));
674: PetscCall(VecDestroy(&sred->xred));
675: PetscCall(VecDestroy(&sred->yred));
676: PetscCall(VecDestroy(&sred->xtmp));
677: PetscCall(MatDestroy(&sred->Bred));
678: PetscCall(KSPReset(sred->ksp));
679: if (sred->pctelescope_reset_type) PetscCall(sred->pctelescope_reset_type(pc));
680: PetscFunctionReturn(PETSC_SUCCESS);
681: }
683: static PetscErrorCode PCDestroy_Telescope(PC pc)
684: {
685: PC_Telescope sred = (PC_Telescope)pc->data;
687: PetscFunctionBegin;
688: PetscCall(PCReset_Telescope(pc));
689: PetscCall(KSPDestroy(&sred->ksp));
690: PetscCall(PetscSubcommDestroy(&sred->psubcomm));
691: PetscCall(PetscFree(sred->dm_ctx));
692: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetKSP_C", NULL));
693: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetSubcommType_C", NULL));
694: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetSubcommType_C", NULL));
695: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetReductionFactor_C", NULL));
696: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetReductionFactor_C", NULL));
697: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetIgnoreDM_C", NULL));
698: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetIgnoreDM_C", NULL));
699: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetIgnoreKSPComputeOperators_C", NULL));
700: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetIgnoreKSPComputeOperators_C", NULL));
701: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetDM_C", NULL));
702: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetUseCoarseDM_C", NULL));
703: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetUseCoarseDM_C", NULL));
704: PetscCall(PetscFree(pc->data));
705: PetscFunctionReturn(PETSC_SUCCESS);
706: }
708: static PetscErrorCode PCSetFromOptions_Telescope(PC pc, PetscOptionItems PetscOptionsObject)
709: {
710: PC_Telescope sred = (PC_Telescope)pc->data;
711: MPI_Comm comm;
712: PetscMPIInt size;
713: PetscBool flg;
714: PetscSubcommType subcommtype;
716: PetscFunctionBegin;
717: PetscCall(PetscObjectGetComm((PetscObject)pc, &comm));
718: PetscCallMPI(MPI_Comm_size(comm, &size));
719: PetscOptionsHeadBegin(PetscOptionsObject, "Telescope options");
720: PetscCall(PetscOptionsEnum("-pc_telescope_subcomm_type", "Subcomm type (interlaced or contiguous)", "PCTelescopeSetSubcommType", PetscSubcommTypes, (PetscEnum)sred->subcommtype, (PetscEnum *)&subcommtype, &flg));
721: if (flg) PetscCall(PCTelescopeSetSubcommType(pc, subcommtype));
722: PetscCall(PetscOptionsInt("-pc_telescope_reduction_factor", "Factor to reduce comm size by", "PCTelescopeSetReductionFactor", sred->redfactor, &sred->redfactor, NULL));
723: PetscCheck(sred->redfactor <= size, comm, PETSC_ERR_ARG_WRONG, "-pc_telescope_reduction_factor <= comm size");
724: PetscCall(PetscOptionsBool("-pc_telescope_ignore_dm", "Ignore any DM attached to the PC", "PCTelescopeSetIgnoreDM", sred->ignore_dm, &sred->ignore_dm, NULL));
725: PetscCall(PetscOptionsBool("-pc_telescope_ignore_kspcomputeoperators", "Ignore method used to compute A", "PCTelescopeSetIgnoreKSPComputeOperators", sred->ignore_kspcomputeoperators, &sred->ignore_kspcomputeoperators, NULL));
726: PetscCall(PetscOptionsBool("-pc_telescope_use_coarse_dm", "Define sub-communicator from the coarse DM", "PCTelescopeSetUseCoarseDM", sred->use_coarse_dm, &sred->use_coarse_dm, NULL));
727: PetscOptionsHeadEnd();
728: PetscFunctionReturn(PETSC_SUCCESS);
729: }
731: /* PC implementation specific API's */
733: static PetscErrorCode PCTelescopeGetKSP_Telescope(PC pc, KSP *ksp)
734: {
735: PC_Telescope red = (PC_Telescope)pc->data;
737: PetscFunctionBegin;
738: if (ksp) *ksp = red->ksp;
739: PetscFunctionReturn(PETSC_SUCCESS);
740: }
742: static PetscErrorCode PCTelescopeGetSubcommType_Telescope(PC pc, PetscSubcommType *subcommtype)
743: {
744: PC_Telescope red = (PC_Telescope)pc->data;
746: PetscFunctionBegin;
747: if (subcommtype) *subcommtype = red->subcommtype;
748: PetscFunctionReturn(PETSC_SUCCESS);
749: }
751: static PetscErrorCode PCTelescopeSetSubcommType_Telescope(PC pc, PetscSubcommType subcommtype)
752: {
753: PC_Telescope red = (PC_Telescope)pc->data;
755: PetscFunctionBegin;
756: PetscCheck(!pc->setupcalled, PetscObjectComm((PetscObject)pc), PETSC_ERR_ARG_WRONGSTATE, "You cannot change the subcommunicator type for PCTelescope after it has been set up.");
757: red->subcommtype = subcommtype;
758: PetscFunctionReturn(PETSC_SUCCESS);
759: }
761: static PetscErrorCode PCTelescopeGetReductionFactor_Telescope(PC pc, PetscInt *fact)
762: {
763: PC_Telescope red = (PC_Telescope)pc->data;
765: PetscFunctionBegin;
766: if (fact) *fact = red->redfactor;
767: PetscFunctionReturn(PETSC_SUCCESS);
768: }
770: static PetscErrorCode PCTelescopeSetReductionFactor_Telescope(PC pc, PetscInt fact)
771: {
772: PC_Telescope red = (PC_Telescope)pc->data;
773: PetscMPIInt size;
775: PetscFunctionBegin;
776: PetscCallMPI(MPI_Comm_size(PetscObjectComm((PetscObject)pc), &size));
777: PetscCheck(fact > 0, PetscObjectComm((PetscObject)pc), PETSC_ERR_ARG_WRONG, "Reduction factor of telescoping PC %" PetscInt_FMT " must be positive", fact);
778: PetscCheck(fact <= size, PetscObjectComm((PetscObject)pc), PETSC_ERR_ARG_WRONG, "Reduction factor of telescoping PC %" PetscInt_FMT " must be <= comm.size", fact);
779: red->redfactor = fact;
780: PetscFunctionReturn(PETSC_SUCCESS);
781: }
783: static PetscErrorCode PCTelescopeGetIgnoreDM_Telescope(PC pc, PetscBool *v)
784: {
785: PC_Telescope red = (PC_Telescope)pc->data;
787: PetscFunctionBegin;
788: if (v) *v = red->ignore_dm;
789: PetscFunctionReturn(PETSC_SUCCESS);
790: }
792: static PetscErrorCode PCTelescopeSetIgnoreDM_Telescope(PC pc, PetscBool v)
793: {
794: PC_Telescope red = (PC_Telescope)pc->data;
796: PetscFunctionBegin;
797: red->ignore_dm = v;
798: PetscFunctionReturn(PETSC_SUCCESS);
799: }
801: static PetscErrorCode PCTelescopeGetUseCoarseDM_Telescope(PC pc, PetscBool *v)
802: {
803: PC_Telescope red = (PC_Telescope)pc->data;
805: PetscFunctionBegin;
806: if (v) *v = red->use_coarse_dm;
807: PetscFunctionReturn(PETSC_SUCCESS);
808: }
810: static PetscErrorCode PCTelescopeSetUseCoarseDM_Telescope(PC pc, PetscBool v)
811: {
812: PC_Telescope red = (PC_Telescope)pc->data;
814: PetscFunctionBegin;
815: red->use_coarse_dm = v;
816: PetscFunctionReturn(PETSC_SUCCESS);
817: }
819: static PetscErrorCode PCTelescopeGetIgnoreKSPComputeOperators_Telescope(PC pc, PetscBool *v)
820: {
821: PC_Telescope red = (PC_Telescope)pc->data;
823: PetscFunctionBegin;
824: if (v) *v = red->ignore_kspcomputeoperators;
825: PetscFunctionReturn(PETSC_SUCCESS);
826: }
828: static PetscErrorCode PCTelescopeSetIgnoreKSPComputeOperators_Telescope(PC pc, PetscBool v)
829: {
830: PC_Telescope red = (PC_Telescope)pc->data;
832: PetscFunctionBegin;
833: red->ignore_kspcomputeoperators = v;
834: PetscFunctionReturn(PETSC_SUCCESS);
835: }
837: static PetscErrorCode PCTelescopeGetDM_Telescope(PC pc, DM *dm)
838: {
839: PC_Telescope red = (PC_Telescope)pc->data;
841: PetscFunctionBegin;
842: *dm = private_PCTelescopeGetSubDM(red);
843: PetscFunctionReturn(PETSC_SUCCESS);
844: }
846: /*@
847: PCTelescopeGetKSP - Gets the `KSP` created by the telescoping `PC`.
849: Not Collective
851: Input Parameter:
852: . pc - the preconditioner context
854: Output Parameter:
855: . subksp - the `KSP` defined on the smaller set of processes
857: Level: advanced
859: .seealso: [](ch_ksp), `PC`, `KSP`, `PCTELESCOPE`
860: @*/
861: PetscErrorCode PCTelescopeGetKSP(PC pc, KSP *subksp)
862: {
863: PetscFunctionBegin;
864: PetscUseMethod(pc, "PCTelescopeGetKSP_C", (PC, KSP *), (pc, subksp));
865: PetscFunctionReturn(PETSC_SUCCESS);
866: }
868: /*@
869: PCTelescopeGetReductionFactor - Gets the factor by which the original number of MPI processes has been reduced by that was set by
870: `PCTelescopeSetReductionFactor()`
872: Not Collective
874: Input Parameter:
875: . pc - the preconditioner context
877: Output Parameter:
878: . fact - the reduction factor
880: Level: advanced
882: .seealso: [](ch_ksp), `PC`, `PCTELESCOPE`, `PCTelescopeSetReductionFactor()`
883: @*/
884: PetscErrorCode PCTelescopeGetReductionFactor(PC pc, PetscInt *fact)
885: {
886: PetscFunctionBegin;
887: PetscUseMethod(pc, "PCTelescopeGetReductionFactor_C", (PC, PetscInt *), (pc, fact));
888: PetscFunctionReturn(PETSC_SUCCESS);
889: }
891: /*@
892: PCTelescopeSetReductionFactor - Sets the factor by which the original number of MPI processes will been reduced by when
893: constructing the subcommunicator to be used with the `PCTELESCOPE`.
895: Not Collective
897: Input Parameter:
898: . pc - the preconditioner context
900: Output Parameter:
901: . fact - the reduction factor
903: Level: advanced
905: .seealso: [](ch_ksp), `PCTELESCOPE`, `PCTelescopeGetReductionFactor()`
906: @*/
907: PetscErrorCode PCTelescopeSetReductionFactor(PC pc, PetscInt fact)
908: {
909: PetscFunctionBegin;
910: PetscTryMethod(pc, "PCTelescopeSetReductionFactor_C", (PC, PetscInt), (pc, fact));
911: PetscFunctionReturn(PETSC_SUCCESS);
912: }
914: /*@
915: PCTelescopeGetIgnoreDM - Get the flag indicating if any `DM` attached to the `PC` will be used in constructing the `PC` on the
916: reduced number of MPI processes
918: Not Collective
920: Input Parameter:
921: . pc - the preconditioner context
923: Output Parameter:
924: . v - the flag
926: Level: advanced
928: .seealso: [](ch_ksp), `DM`, `PCTELESCOPE`, `PCTelescopeSetIgnoreDM()`
929: @*/
930: PetscErrorCode PCTelescopeGetIgnoreDM(PC pc, PetscBool *v)
931: {
932: PetscFunctionBegin;
933: PetscUseMethod(pc, "PCTelescopeGetIgnoreDM_C", (PC, PetscBool *), (pc, v));
934: PetscFunctionReturn(PETSC_SUCCESS);
935: }
937: /*@
938: PCTelescopeSetIgnoreDM - Set a flag to ignore any `DM` attached to the `PC` when constructing the `PC` on the
939: reduced number of MPI processes
941: Not Collective
943: Input Parameter:
944: . pc - the preconditioner context
946: Output Parameter:
947: . v - Use `PETSC_TRUE` to ignore any `DM`
949: Level: advanced
951: .seealso: [](ch_ksp), `DM`, `PCTELESCOPE`, `PCTelescopeGetIgnoreDM()`
952: @*/
953: PetscErrorCode PCTelescopeSetIgnoreDM(PC pc, PetscBool v)
954: {
955: PetscFunctionBegin;
956: PetscTryMethod(pc, "PCTelescopeSetIgnoreDM_C", (PC, PetscBool), (pc, v));
957: PetscFunctionReturn(PETSC_SUCCESS);
958: }
960: /*@
961: PCTelescopeGetUseCoarseDM - Get the flag indicating if the coarse `DM` attached to `DM` associated with the `PC` will be used in constructing
962: the `PC` on the reduced number of MPI processes
964: Not Collective
966: Input Parameter:
967: . pc - the preconditioner context
969: Output Parameter:
970: . v - the flag
972: Level: advanced
974: .seealso: [](ch_ksp), `DM`, `PCTELESCOPE`, `PCTelescopeSetIgnoreDM()`, `PCTelescopeSetUseCoarseDM()`
975: @*/
976: PetscErrorCode PCTelescopeGetUseCoarseDM(PC pc, PetscBool *v)
977: {
978: PetscFunctionBegin;
979: PetscUseMethod(pc, "PCTelescopeGetUseCoarseDM_C", (PC, PetscBool *), (pc, v));
980: PetscFunctionReturn(PETSC_SUCCESS);
981: }
983: /*@
984: PCTelescopeSetUseCoarseDM - Set a flag to query the `DM` attached to the `PC` if it also has a coarse `DM` and utilize that `DM`
985: in constructing the `PC` on the reduced number of MPI processes
987: Not Collective
989: Input Parameter:
990: . pc - the preconditioner context
992: Output Parameter:
993: . v - Use `PETSC_FALSE` to ignore any coarse `DM`
995: Level: advanced
997: Notes:
998: When you have specified to use a coarse `DM`, the communicator used to create the sub-`KSP` within `PCTELESCOPE`
999: will be that of the coarse `DM`. Hence the flags `-pc_telescope_reduction_factor` and
1000: `-pc_telescope_subcomm_type` will not be used.
1002: It is required that the communicator associated with the parent (fine) and the coarse `DM` are of different sizes.
1003: An error will occur of the size if the communicator associated with the coarse `DM` is the same as that of the parent `DM`.
1004: Furthermore, it is required that the communicator on the coarse `DM` is a sub-communicator of the parent.
1005: This will be checked at the time the preconditioner is setup and an error will occur if
1006: the coarse `DM` does not define a sub-communicator of that used by the parent `DM`.
1008: The particular Telescope setup invoked when using a coarse `DM` is agnostic with respect to the type of
1009: the `DM` used (e.g. it supports `DMSHELL`, `DMPLEX`, etc).
1011: Support is currently only provided for the case when you are using `KSPSetComputeOperators()`
1013: The user is required to compose a function with the parent `DM` to facilitate the transfer of fields (`Vec`)
1014: between the different decompositions defined by the fine and coarse `DM`s.
1015: In the user code, this is achieved via
1016: .vb
1017: {
1018: DM dm_fine;
1019: PetscObjectCompose((PetscObject)dm_fine,"PCTelescopeFieldScatter",your_field_scatter_method);
1020: }
1021: .ve
1022: The signature of the user provided field scatter method is
1023: .vb
1024: PetscErrorCode your_field_scatter_method(DM dm_fine,Vec x_fine,ScatterMode mode,DM dm_coarse,Vec x_coarse);
1025: .ve
1026: The user must provide support for both mode `SCATTER_FORWARD` and mode `SCATTER_REVERSE`.
1027: `SCATTER_FORWARD` implies the direction of transfer is from the parent (fine) `DM` to the coarse `DM`.
1029: Optionally, the user may also compose a function with the parent `DM` to facilitate the transfer
1030: of state variables between the fine and coarse `DM`s.
1031: In the context of a finite element discretization, an example state variable might be
1032: values associated with quadrature points within each element.
1033: A user provided state scatter method is composed via
1034: .vb
1035: {
1036: DM dm_fine;
1037: PetscObjectCompose((PetscObject)dm_fine,"PCTelescopeStateScatter",your_state_scatter_method);
1038: }
1039: .ve
1040: The signature of the user provided state scatter method is
1041: .vb
1042: PetscErrorCode your_state_scatter_method(DM dm_fine,ScatterMode mode,DM dm_coarse);
1043: .ve
1044: `SCATTER_FORWARD` implies the direction of transfer is from the fine `DM` to the coarse `DM`.
1045: The user is only required to support mode = `SCATTER_FORWARD`.
1046: No assumption is made about the data type of the state variables.
1047: These must be managed by the user and must be accessible from the `DM`.
1049: Care must be taken in defining the user context passed to `KSPSetComputeOperators()` which is to be
1050: associated with the sub-`KSP` residing within `PCTELESCOPE`.
1051: In general, `PCTELESCOPE` assumes that the context on the fine and coarse `DM` used with
1052: `KSPSetComputeOperators()` should be "similar" in type or origin.
1053: Specifically the following rules are used to infer what context on the sub-`KSP` should be.
1055: First the contexts from the `KSP` and the fine and coarse `DM`s are retrieved.
1056: Note that the special case of a `DMSHELL` context is queried.
1058: .vb
1059: DMKSPGetComputeOperators(dm_fine,&dmfine_kspfunc,&dmfine_kspctx);
1060: DMGetApplicationContext(dm_fine,&dmfine_appctx);
1061: DMShellGetContext(dm_fine,&dmfine_shellctx);
1063: DMGetApplicationContext(dm_coarse,&dmcoarse_appctx);
1064: DMShellGetContext(dm_coarse,&dmcoarse_shellctx);
1065: .ve
1067: The following rules are then enforced\:
1069: 1. If `dmfine_kspctx` = `NULL`, then we provide a `NULL` pointer as the context for the sub-`KSP`\:
1070: `KSPSetComputeOperators`(`sub_ksp`,`dmfine_kspfunc`,`NULL`);
1072: 2. If `dmfine_kspctx` != `NULL` and `dmfine_kspctx` == `dmfine_appctx`,
1074: check that `dmcoarse_appctx` is also non-`NULL`. If this is true, then\:
1075: `KSPSetComputeOperators`(`sub_ksp`,`dmfine_kspfunc`,`dmcoarse_appctx`);
1077: 3. If `dmfine_kspctx` != `NULL` and `dmfine_kspctx` == `dmfine_shellctx`,
1079: check that `dmcoarse_shellctx` is also non-`NULL`. If this is true, then\:
1080: `KSPSetComputeOperators`(`sub_ksp`,`dmfine_kspfunc`,`dmcoarse_shellctx`);
1082: If neither of the above three tests passed, then `PCTELESCOPE` cannot safely determine what
1083: context should be provided to `KSPSetComputeOperators()` for use with the sub-`KSP`.
1084: In this case, an additional mechanism is provided via a composed function which will return
1085: the actual context to be used. To use this feature you must compose the "getter" function
1086: with the coarse `DM`, e.g.
1087: .vb
1088: {
1089: DM dm_coarse;
1090: PetscObjectCompose((PetscObject)dm_coarse,"PCTelescopeGetCoarseDMKSPContext",your_coarse_context_getter);
1091: }
1092: .ve
1093: The signature of the user provided method is
1094: .vb
1095: PetscErrorCode your_coarse_context_getter(DM dm_coarse,void **your_kspcontext);
1096: .ve
1098: .seealso: [](ch_ksp), `DM`, `PCTELESCOPE`, `PCTelescopeSetIgnoreDM()`
1099: @*/
1100: PetscErrorCode PCTelescopeSetUseCoarseDM(PC pc, PetscBool v)
1101: {
1102: PetscFunctionBegin;
1103: PetscTryMethod(pc, "PCTelescopeSetUseCoarseDM_C", (PC, PetscBool), (pc, v));
1104: PetscFunctionReturn(PETSC_SUCCESS);
1105: }
1107: /*@
1108: PCTelescopeGetIgnoreKSPComputeOperators - Get the flag indicating if `KSPComputeOperators()` will be used to construct
1109: the matrix on the reduced number of MPI processes
1111: Not Collective
1113: Input Parameter:
1114: . pc - the preconditioner context
1116: Output Parameter:
1117: . v - the flag
1119: Level: advanced
1121: .seealso: [](ch_ksp), `PCTELESCOPE`, `PCTelescopeSetIgnoreDM()`, `PCTelescopeSetUseCoarseDM()`, `PCTelescopeSetIgnoreKSPComputeOperators()`
1122: @*/
1123: PetscErrorCode PCTelescopeGetIgnoreKSPComputeOperators(PC pc, PetscBool *v)
1124: {
1125: PetscFunctionBegin;
1126: PetscUseMethod(pc, "PCTelescopeGetIgnoreKSPComputeOperators_C", (PC, PetscBool *), (pc, v));
1127: PetscFunctionReturn(PETSC_SUCCESS);
1128: }
1130: /*@
1131: PCTelescopeSetIgnoreKSPComputeOperators - Set a flag to have `PCTELESCOPE` ignore the function provided to `KSPComputeOperators()` in
1132: constructint the matrix on the reduced number of MPI processes
1134: Not Collective
1136: Input Parameter:
1137: . pc - the preconditioner context
1139: Output Parameter:
1140: . v - Use `PETSC_TRUE` to ignore the function (if defined) set via `KSPSetComputeOperators()` on `pc`
1142: Level: advanced
1144: .seealso: [](ch_ksp), `PCTELESCOPE`, `PCTelescopeSetIgnoreDM()`, `PCTelescopeSetUseCoarseDM()`, `PCTelescopeGetIgnoreKSPComputeOperators()`
1145: @*/
1146: PetscErrorCode PCTelescopeSetIgnoreKSPComputeOperators(PC pc, PetscBool v)
1147: {
1148: PetscFunctionBegin;
1149: PetscTryMethod(pc, "PCTelescopeSetIgnoreKSPComputeOperators_C", (PC, PetscBool), (pc, v));
1150: PetscFunctionReturn(PETSC_SUCCESS);
1151: }
1153: /*@
1154: PCTelescopeGetDM - Get the re-partitioned `DM` attached to the sub-`KSP`.
1156: Not Collective
1158: Input Parameter:
1159: . pc - the preconditioner context
1161: Output Parameter:
1162: . subdm - The re-partitioned `DM`
1164: Level: advanced
1166: .seealso: [](ch_ksp), `DM`, `PCTELESCOPE`, `PCTelescopeSetIgnoreDM()`, `PCTelescopeSetUseCoarseDM()`, `PCTelescopeGetIgnoreKSPComputeOperators()`
1167: @*/
1168: PetscErrorCode PCTelescopeGetDM(PC pc, DM *subdm)
1169: {
1170: PetscFunctionBegin;
1171: PetscUseMethod(pc, "PCTelescopeGetDM_C", (PC, DM *), (pc, subdm));
1172: PetscFunctionReturn(PETSC_SUCCESS);
1173: }
1175: /*@
1176: PCTelescopeSetSubcommType - set subcommunicator type `PetscSubcommType` (interlaced or contiguous) to be used when
1177: the subcommunicator is generated from the given `PC`
1179: Logically Collective
1181: Input Parameters:
1182: + pc - the preconditioner context
1183: - subcommtype - the subcommunicator type (see `PetscSubcommType`)
1185: Level: advanced
1187: .seealso: [](ch_ksp), `PetscSubcommType`, `PetscSubcomm`, `PCTELESCOPE`, `PCTelescopeGetSubcommType()`
1188: @*/
1189: PetscErrorCode PCTelescopeSetSubcommType(PC pc, PetscSubcommType subcommtype)
1190: {
1191: PetscFunctionBegin;
1192: PetscTryMethod(pc, "PCTelescopeSetSubcommType_C", (PC, PetscSubcommType), (pc, subcommtype));
1193: PetscFunctionReturn(PETSC_SUCCESS);
1194: }
1196: /*@
1197: PCTelescopeGetSubcommType - Get the subcommunicator type `PetscSubcommType` (interlaced or contiguous) set with `PCTelescopeSetSubcommType()`
1199: Not Collective
1201: Input Parameter:
1202: . pc - the preconditioner context
1204: Output Parameter:
1205: . subcommtype - the subcommunicator type (see `PetscSubcommType`)
1207: Level: advanced
1209: .seealso: [](ch_ksp), `PetscSubcomm`, `PetscSubcommType`, `PCTELESCOPE`, `PCTelescopeSetSubcommType()`
1210: @*/
1211: PetscErrorCode PCTelescopeGetSubcommType(PC pc, PetscSubcommType *subcommtype)
1212: {
1213: PetscFunctionBegin;
1214: PetscUseMethod(pc, "PCTelescopeGetSubcommType_C", (PC, PetscSubcommType *), (pc, subcommtype));
1215: PetscFunctionReturn(PETSC_SUCCESS);
1216: }
1218: /*MC
1219: PCTELESCOPE - Runs a `KSP` solver on a sub-communicator {cite}`maysananruppknepleysmith2016` of the communicator used by the original `KSP`.
1220: MPI processes not in the sub-communicator are idle during the solve. Usually used to solve the smaller coarser grid problems in multigrid
1221: (`PCMG`) that could not be efficiently solved on the entire communication
1223: Options Database Keys:
1224: + -pc_telescope_reduction_factor <r> - factor to reduce the communicator size by. e.g. with 64 MPI ranks and r=4, the new sub-communicator will have 64/4 = 16 ranks.
1225: . -pc_telescope_ignore_dm - flag to indicate whether an attached `DM` should be ignored in constructing the new `PC`
1226: . -pc_telescope_subcomm_type <interlaced,contiguous> - defines the selection of MPI processes on the sub-communicator. see `PetscSubcomm` for more information.
1227: . -pc_telescope_ignore_kspcomputeoperators - flag to indicate whether `KSPSetComputeOperators()` should be used on the sub-`KSP`.
1228: - -pc_telescope_use_coarse_dm - flag to indicate whether the coarse `DM` should be used to define the sub-communicator.
1230: Level: advanced
1232: Notes:
1233: Assuming that the parent preconditioner `PC` is defined on a communicator c, this implementation
1234: creates a child sub-communicator (c') containing fewer MPI processes than the original parent preconditioner `PC`.
1235: The preconditioner is deemed telescopic as it only calls `KSPSolve()` on a single
1236: sub-communicator, in contrast with `PCREDUNDANT` which calls `KSPSolve()` on N sub-communicators.
1237: This means there will be MPI processes which will be idle during the application of this preconditioner.
1238: Additionally, in comparison with `PCREDUNDANT`, `PCTELESCOPE` can utilize an attached `DM` to construct `DM` dependent preconditioner, such as `PCMG`
1240: The default type `KSPType` of the sub `KSP` (the `KSP` defined on c') is `KSPPREONLY`.
1242: There are three setup mechanisms for `PCTELESCOPE`. Features support by each type are described below.
1243: In the following, we will refer to the operators B and B', these are the `Bmat` provided to the `KSP` on the
1244: communicators c and c' respectively.
1246: [1] Default setup
1247: The sub-communicator c' is created via `PetscSubcommCreate()`.
1248: Any explicitly defined nullspace and near nullspace vectors attached to B with `MatSetNullSpace()` and `MatSetNearNullSpace()` are transferred to B'.
1249: Currently there is no support for nullspaces provided with `MatNullSpaceSetFunction()`).
1250: No support is provided for `KSPSetComputeOperators()`.
1251: Currently there is no support for the flag `-pc_use_amat`.
1253: [2] `DM` aware setup
1254: The sub-communicator c' is created via `PetscSubcommCreate()`.
1255: If a `DM` is attached to the `PC`, it is re-partitioned on the sub-communicator c'.
1256: Both the `Bmat` operator and the right-hand side vector are permuted into the new DOF ordering defined by the re-partitioned `DM`.
1257: Currently only support for re-partitioning a `DMDA` is provided.
1258: Any explicitly defined nullspace or near nullspace vectors attached to the original B with `MatSetNullSpace()`
1259: and `MatSetNearNullSpace()` are extracted, re-partitioned and set on B'
1260: (currently there is no support for nullspaces provided with `MatNullSpaceSetFunction()`).
1261: Support is provided for `KSPSetComputeOperators()`. The user provided function and context is propagated to the sub `KSP`.
1262: This is fragile since the user must ensure that their user context is valid for use on c'.
1263: Currently there is no support for the flag `-pc_use_amat`.
1265: [3] Coarse `DM` setup
1266: If a `DM` (dmfine) is attached to the `PC`, dmfine is queried for a "coarse" `DM` (call this dmcoarse) via `DMGetCoarseDM()`.
1267: `PCTELESCOPE` will interpret the coarse `DM` as being defined on a sub-communicator of c.
1268: The communicator associated with dmcoarse will define the c' to be used within `PCTELESCOPE`.
1269: `PCTELESCOPE` will check that c' is in fact a sub-communicator of c. If it is not, an error will be reported.
1270: The intention of this setup type is that `PCTELESCOPE` will use an existing (e.g. user defined) communicator hierarchy, say as would be
1271: available with using multi-grid on unstructured meshes.
1272: This setup will not use the command line options `-pc_telescope_reduction_factor` or `-pc_telescope_subcomm_type`.
1273: Any explicitly defined nullspace or near nullspace vectors attached to the B are extracted, scattered into the correct ordering consistent
1274: with dmcoarse and set on B'
1275: (currently there is no support for nullspaces provided with `MatNullSpaceSetFunction()`).
1276: There is no general method to permute field orderings, hence only `KSPSetComputeOperators()` is supported.
1277: The user must use `PetscObjectComposeFunction()` with dmfine to define the method to scatter fields from dmfine to dmcoarse.
1278: Propagation of the user context for `KSPSetComputeOperators()` on the sub `KSP` is attempted by querying the `DM` contexts associated with
1279: dmfine and dmcoarse. Alternatively, the user may use `PetscObjectComposeFunction()` with dmcoarse to define a method which will return the appropriate user context for `KSPSetComputeOperators()`.
1280: Currently there is no support for the flag `-pc_use_amat`.
1281: This setup can be invoked by the option `-pc_telescope_use_coarse_dm` or by calling `PCTelescopeSetUseCoarseDM`(pc,`PETSC_TRUE`);
1282: Further information about the user-provided methods required by this setup type are described here `PCTelescopeSetUseCoarseDM()`.
1284: Developer Notes:
1285: During `PCSetUp()`, the B operator is scattered onto c'.
1286: Within `PCApply()`, the RHS vector (x) is scattered into a redundant vector, xred (defined on c').
1287: Then, `KSPSolve()` is executed on the c' communicator.
1289: The communicator used within the telescoping preconditioner is defined by a `PetscSubcomm` using the INTERLACED
1290: creation routine by default (this can be changed with `-pc_telescope_subcomm_type`). We run the sub `KSP` on only
1291: the ranks within the communicator which have a color equal to zero.
1293: The telescoping preconditioner is aware of nullspaces and near nullspaces which are attached to the B operator.
1294: In the case where B has a (near) nullspace attached, the (near) nullspace vectors are extracted from B and mapped into
1295: a new (near) nullspace, defined on the sub-communicator, which is attached to B' (the B operator which was scattered to c')
1297: The telescoping preconditioner can re-partition an attached `DM` if it is a `DMDA` (2D or 3D -
1298: support for 1D `DMDA`s is not provided). If a `DMDA` is found, a topologically equivalent `DMDA` is created on c'
1299: and this new `DM` is attached the sub `KSP`. The design of telescope is such that it should be possible to extend support
1300: for re-partitioning other to `DM`'s (e.g. `DMPLEX`). The user can supply a flag to ignore attached DMs.
1301: Alternatively, user-provided re-partitioned `DM`s can be used via `-pc_telescope_use_coarse_dm`.
1303: With the default setup mode, B' is defined by fusing rows (in order) associated with MPI processes common to c and c'.
1305: When a `DMDA` is attached to the parent preconditioner, B' is defined by: (i) performing a symmetric permutation of B
1306: into the ordering defined by the `DMDA` on c', (ii) extracting the local chunks via `MatCreateSubMatrices()`, (iii) fusing the
1307: locally (sequential) matrices defined on the ranks common to c and c' into B' using `MatCreateMPIMatConcatenateSeqMat()`
1309: Limitations/improvements include the following.
1310: `VecPlaceArray()` could be used within `PCApply()` to improve efficiency and reduce memory usage.
1311: A unified mechanism to query for user contexts as required by `KSPSetComputeOperators()` and `MatNullSpaceSetFunction()`.
1313: The symmetric permutation used when a `DMDA` is encountered is performed via explicitly assembling a permutation matrix P,
1314: and performing P^T.A.P. Possibly it might be more efficient to use `MatPermute()`. We opted to use P^T.A.P as it appears
1315: `VecPermute()` does not support the use case required here. By computing P, one can permute both the operator and RHS in a
1316: consistent manner.
1318: Mapping of vectors (default setup mode) is performed in the following way.
1319: Suppose the parent communicator size was 4, and we set a reduction factor of 2; this would give a comm size on c' of 2.
1320: Using the interlaced creation routine, the ranks in c with color = 0 will be rank 0 and 2.
1321: We perform the scatter to the sub-communicator in the following way.
1322: [1] Given a vector x defined on communicator c
1324: .vb
1325: rank(c) local values of x
1326: ------- ----------------------------------------
1327: 0 [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0 ]
1328: 1 [ 6.0, 7.0, 8.0, 9.0, 10.0, 11.0 ]
1329: 2 [ 12.0, 13.0, 14.0, 15.0, 16.0, 17.0 ]
1330: 3 [ 18.0, 19.0, 20.0, 21.0, 22.0, 23.0 ]
1331: .ve
1333: scatter into xtmp defined also on comm c, so that we have the following values
1335: .vb
1336: rank(c) local values of xtmp
1337: ------- ----------------------------------------------------------------------------
1338: 0 [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0 ]
1339: 1 [ ]
1340: 2 [ 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0 ]
1341: 3 [ ]
1342: .ve
1344: The entries on rank 1 and 3 (ranks which do not have a color = 0 in c') have no values
1346: [2] Copy the values from ranks 0, 2 (indices with respect to comm c) into the vector xred which is defined on communicator c'.
1347: Ranks 0 and 2 are the only ranks in the subcomm which have a color = 0.
1349: .vb
1350: rank(c') local values of xred
1351: -------- ----------------------------------------------------------------------------
1352: 0 [ 0.0, 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0, 10.0, 11.0 ]
1353: 1 [ 12.0, 13.0, 14.0, 15.0, 16.0, 17.0, 18.0, 19.0, 20.0, 21.0, 22.0, 23.0 ]
1354: .ve
1356: Contributed by:
1357: Dave May
1359: .seealso: [](ch_ksp), `PCTelescopeGetKSP()`, `PCTelescopeGetDM()`, `PCTelescopeGetReductionFactor()`, `PCTelescopeSetReductionFactor()`, `PCTelescopeGetIgnoreDM()`, `PCTelescopeSetIgnoreDM()`, `PCREDUNDANT`
1360: M*/
1361: PETSC_EXTERN PetscErrorCode PCCreate_Telescope(PC pc)
1362: {
1363: struct _PC_Telescope *sred;
1365: PetscFunctionBegin;
1366: PetscCall(PetscNew(&sred));
1367: sred->psubcomm = NULL;
1368: sred->subcommtype = PETSC_SUBCOMM_INTERLACED;
1369: sred->subcomm = MPI_COMM_NULL;
1370: sred->redfactor = 1;
1371: sred->ignore_dm = PETSC_FALSE;
1372: sred->ignore_kspcomputeoperators = PETSC_FALSE;
1373: sred->use_coarse_dm = PETSC_FALSE;
1374: pc->data = (void *)sred;
1376: pc->ops->apply = PCApply_Telescope;
1377: pc->ops->applytranspose = NULL;
1378: pc->ops->applyrichardson = PCApplyRichardson_Telescope;
1379: pc->ops->setup = PCSetUp_Telescope;
1380: pc->ops->destroy = PCDestroy_Telescope;
1381: pc->ops->reset = PCReset_Telescope;
1382: pc->ops->setfromoptions = PCSetFromOptions_Telescope;
1383: pc->ops->view = PCView_Telescope;
1385: sred->pctelescope_setup_type = PCTelescopeSetUp_default;
1386: sred->pctelescope_matcreate_type = PCTelescopeMatCreate_default;
1387: sred->pctelescope_matnullspacecreate_type = PCTelescopeMatNullSpaceCreate_default;
1388: sred->pctelescope_reset_type = NULL;
1390: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetKSP_C", PCTelescopeGetKSP_Telescope));
1391: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetSubcommType_C", PCTelescopeGetSubcommType_Telescope));
1392: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetSubcommType_C", PCTelescopeSetSubcommType_Telescope));
1393: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetReductionFactor_C", PCTelescopeGetReductionFactor_Telescope));
1394: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetReductionFactor_C", PCTelescopeSetReductionFactor_Telescope));
1395: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetIgnoreDM_C", PCTelescopeGetIgnoreDM_Telescope));
1396: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetIgnoreDM_C", PCTelescopeSetIgnoreDM_Telescope));
1397: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetIgnoreKSPComputeOperators_C", PCTelescopeGetIgnoreKSPComputeOperators_Telescope));
1398: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetIgnoreKSPComputeOperators_C", PCTelescopeSetIgnoreKSPComputeOperators_Telescope));
1399: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetDM_C", PCTelescopeGetDM_Telescope));
1400: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeGetUseCoarseDM_C", PCTelescopeGetUseCoarseDM_Telescope));
1401: PetscCall(PetscObjectComposeFunction((PetscObject)pc, "PCTelescopeSetUseCoarseDM_C", PCTelescopeSetUseCoarseDM_Telescope));
1402: PetscFunctionReturn(PETSC_SUCCESS);
1403: }