Actual source code: ex22.c
1: static char help[] = "Extract a 2D slice in natural ordering from a 3D vector, Command line options : \n\
2: Mx/My/Mz - set the dimensions of the parent vector \n\
3: sliceaxis - string describing the axis along which the sice will be selected : X, Y, Z \n\
4: gp - global grid point number along the sliceaxis direction where the slice will be extracted from the parent vector \n";
6: /*
7: This example shows to extract a 2D slice in natural ordering
8: from a 3D DMDA vector (first by extracting the slice and then
9: by converting it to natural ordering)
10: */
12: #include <petscdmda.h>
14: const char *const sliceaxes[] = {"X", "Y", "Z", "sliceaxis", "DM_", NULL};
16: int main(int argc, char **argv)
17: {
18: DM da3D; /* 3D DMDA object */
19: DM da2D; /* 2D DMDA object */
20: Vec vec_full; /* Parent vector */
21: Vec vec_extracted; /* Extracted slice vector (in DMDA ordering) */
22: Vec vec_slice; /* vector in natural ordering */
23: Vec vec_slice_g; /* aliased vector in natural ordering */
24: IS patchis_3d; /* IS to select slice and extract subvector */
25: IS patchis_2d; /* Patch IS for 2D vector, will be converted to application ordering */
26: IS scatis_extracted_slice; /* PETSc indexed IS for extracted slice */
27: IS scatis_natural_slice; /* natural/application ordered IS for slice*/
28: IS scatis_natural_slice_g; /* aliased natural/application ordered IS for slice */
29: VecScatter vscat; /* scatter slice in DMDA ordering <-> slice in column major ordering */
30: AO da2D_ao; /* AO associated with 2D DMDA */
31: MPI_Comm subset_mpi_comm = MPI_COMM_NULL; /* MPI communicator where the slice lives */
32: PetscScalar ***vecdata3d; /* Pointer to access 3d parent vector */
33: const PetscScalar *array; /* pointer to create aliased Vec */
34: PetscInt Mx = 4, My = 4, Mz = 4; /* Dimensions for 3D DMDA */
35: const PetscInt *l1, *l2; /* 3D DMDA layout */
36: PetscInt M1 = -1, M2 = -1; /* Dimensions for 2D DMDA */
37: PetscInt m1 = -1, m2 = -1; /* Layouts for 2D DMDA */
38: PetscInt gp = 2; /* grid point along sliceaxis to pick the slice */
39: DMDirection sliceaxis = DM_X; /* Select axis along which the slice will be extracted */
40: PetscInt i, j, k; /* Iteration indices */
41: PetscInt ixs, iys, izs; /* Corner indices for 3D vector */
42: PetscInt ixm, iym, izm; /* Widths of parent vector */
43: PetscInt low, high; /* ownership range indices */
44: PetscInt in; /* local size index for IS*/
45: PetscInt vn; /* local size index */
46: const PetscInt *is_array; /* pointer to create aliased IS */
47: MatStencil lower, upper; /* Stencils to select slice for Vec */
48: PetscBool patchis_offproc = PETSC_FALSE; /* flag to DMDACreatePatchIS indicating that off-proc values are to be ignored */
49: PetscMPIInt rank, size; /* MPI rank and size */
51: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
52: Initialize program and set problem parameters
53: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
54: PetscFunctionBeginUser;
55: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
56: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
57: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
59: PetscOptionsBegin(PETSC_COMM_WORLD, "", "ex22 DMDA tutorial example options", "DMDA");
60: PetscCall(PetscOptionsRangeInt("-Mx", "dimension along x-axis", "ex22.c", Mx, &Mx, NULL, 0, PETSC_INT_MAX));
61: PetscCall(PetscOptionsRangeInt("-My", "dimension along y-axis", "ex22.c", My, &My, NULL, 0, PETSC_INT_MAX));
62: PetscCall(PetscOptionsRangeInt("-Mz", "dimension along z-axis", "ex22.c", Mz, &Mz, NULL, 0, PETSC_INT_MAX));
63: PetscCall(PetscOptionsEnum("-sliceaxis", "axis along which 2D slice is extracted from : X, Y, Z", "", sliceaxes, (PetscEnum)sliceaxis, (PetscEnum *)&sliceaxis, NULL));
64: PetscCall(PetscOptionsRangeInt("-gp", "index along sliceaxis at which 2D slice is extracted", "ex22.c", gp, &gp, NULL, 0, PETSC_INT_MAX));
65: PetscOptionsEnd();
67: /* Ensure that the requested slice is not out of bounds for the selected axis */
68: if (sliceaxis == DM_X) {
69: PetscCheck(gp <= Mx, PETSC_COMM_WORLD, PETSC_ERR_USER_INPUT, "grid point along sliceaxis is larger than largest index!");
70: } else if (sliceaxis == DM_Y) {
71: PetscCheck(gp <= My, PETSC_COMM_WORLD, PETSC_ERR_USER_INPUT, "grid point along sliceaxis is larger than largest index!");
72: } else if (sliceaxis == DM_Z) {
73: PetscCheck(gp <= Mz, PETSC_COMM_WORLD, PETSC_ERR_USER_INPUT, "grid point along sliceaxis is larger than largest index!");
74: }
76: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
77: Create 3D DMDA object.
78: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/
79: PetscCall(DMDACreate3d(PETSC_COMM_WORLD, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DMDA_STENCIL_STAR, Mx, My, Mz, PETSC_DECIDE, PETSC_DECIDE, PETSC_DECIDE, 1, 1, NULL, NULL, NULL, &da3D));
80: PetscCall(DMSetFromOptions(da3D));
81: PetscCall(DMSetUp(da3D));
83: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
84: Create the parent vector
85: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/
86: PetscCall(DMCreateGlobalVector(da3D, &vec_full));
87: PetscCall(PetscObjectSetName((PetscObject)vec_full, "full_vector"));
89: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
90: Populate the 3D vector
91: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
92: PetscCall(DMDAGetCorners(da3D, &ixs, &iys, &izs, &ixm, &iym, &izm));
93: PetscCall(DMDAVecGetArray(da3D, vec_full, &vecdata3d));
94: for (k = izs; k < izs + izm; k++) {
95: for (j = iys; j < iys + iym; j++) {
96: for (i = ixs; i < ixs + ixm; i++) vecdata3d[k][j][i] = ((i - Mx / 2.0) * (j + Mx / 2.0)) + k * 100;
97: }
98: }
99: PetscCall(DMDAVecRestoreArray(da3D, vec_full, &vecdata3d));
101: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
102: Get an IS corresponding to a 2D slice
103: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
104: if (sliceaxis == DM_X) {
105: lower.i = gp;
106: lower.j = 0;
107: lower.k = 0;
108: upper.i = gp;
109: upper.j = My;
110: upper.k = Mz;
111: } else if (sliceaxis == DM_Y) {
112: lower.i = 0;
113: lower.j = gp;
114: lower.k = 0;
115: upper.i = Mx;
116: upper.j = gp;
117: upper.k = Mz;
118: } else if (sliceaxis == DM_Z) {
119: lower.i = 0;
120: lower.j = 0;
121: lower.k = gp;
122: upper.i = Mx;
123: upper.j = My;
124: upper.k = gp;
125: }
126: PetscCall(DMDACreatePatchIS(da3D, &lower, &upper, &patchis_3d, patchis_offproc));
127: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\n IS to select slice from 3D DMDA vector : \n"));
128: PetscCall(ISView(patchis_3d, PETSC_VIEWER_STDOUT_WORLD));
130: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
131: Use the obtained IS to extract the slice as a subvector
132: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
133: PetscCall(VecGetSubVector(vec_full, patchis_3d, &vec_extracted));
135: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
136: View the extracted subvector
137: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
138: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_DENSE));
139: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\n Extracted slice vector, in DMDA ordering : \n"));
140: PetscCall(VecView(vec_extracted, PETSC_VIEWER_STDOUT_WORLD));
141: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
143: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
144: Query 3D DMDA layout, get the subset MPI communicator
145: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/
146: if (sliceaxis == DM_X) {
147: PetscCall(DMDAGetInfo(da3D, NULL, NULL, NULL, NULL, NULL, &m1, &m2, NULL, NULL, NULL, NULL, NULL, NULL));
148: PetscCall(DMDAGetOwnershipRanges(da3D, NULL, &l1, &l2));
149: M1 = My;
150: M2 = Mz;
151: PetscCall(DMDAGetProcessorSubset(da3D, DM_X, gp, &subset_mpi_comm));
152: } else if (sliceaxis == DM_Y) {
153: PetscCall(DMDAGetInfo(da3D, NULL, NULL, NULL, NULL, &m1, NULL, &m2, NULL, NULL, NULL, NULL, NULL, NULL));
154: PetscCall(DMDAGetOwnershipRanges(da3D, &l1, NULL, &l2));
155: M1 = Mx;
156: M2 = Mz;
157: PetscCall(DMDAGetProcessorSubset(da3D, DM_Y, gp, &subset_mpi_comm));
158: } else if (sliceaxis == DM_Z) {
159: PetscCall(DMDAGetInfo(da3D, NULL, NULL, NULL, NULL, &m1, &m2, NULL, NULL, NULL, NULL, NULL, NULL, NULL));
160: PetscCall(DMDAGetOwnershipRanges(da3D, &l1, &l2, NULL));
161: M1 = Mx;
162: M2 = My;
163: PetscCall(DMDAGetProcessorSubset(da3D, DM_Z, gp, &subset_mpi_comm));
164: }
166: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
167: Create 2D DMDA object,
168: vector (that will hold the slice as a column major flattened array) &
169: index set (that will be used for scattering to the column major
170: indexed slice vector)
171: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -*/
172: if (subset_mpi_comm != MPI_COMM_NULL) {
173: PetscCallMPI(MPI_Comm_size(subset_mpi_comm, &size));
174: PetscCall(PetscSynchronizedPrintf(subset_mpi_comm, "subset MPI subcomm size is : %d, includes global rank : %d \n", size, rank));
175: PetscCall(PetscSynchronizedFlush(subset_mpi_comm, PETSC_STDOUT));
176: PetscCall(DMDACreate2d(subset_mpi_comm, DM_BOUNDARY_NONE, DM_BOUNDARY_NONE, DMDA_STENCIL_STAR, M1, M2, m1, m2, 1, 1, l1, l2, &da2D));
177: PetscCall(DMSetFromOptions(da2D));
178: PetscCall(DMSetUp(da2D));
180: /* Create a 2D patch IS for the slice */
181: lower.i = 0;
182: lower.j = 0;
183: upper.i = M1;
184: upper.j = M2;
185: PetscCall(DMDACreatePatchIS(da2D, &lower, &upper, &patchis_2d, patchis_offproc));
187: /* Convert the 2D patch IS to natural indexing (column major flattened) */
188: PetscCall(ISDuplicate(patchis_2d, &scatis_natural_slice));
189: PetscCall(DMDAGetAO(da2D, &da2D_ao));
190: PetscCall(AOPetscToApplicationIS(da2D_ao, scatis_natural_slice));
191: PetscCall(ISGetIndices(scatis_natural_slice, &is_array));
192: PetscCall(ISGetLocalSize(scatis_natural_slice, &in));
194: /* Create an aliased IS on the 3D DMDA's communicator */
195: PetscCall(ISCreateGeneral(PETSC_COMM_WORLD, in, is_array, PETSC_USE_POINTER, &scatis_natural_slice_g));
196: PetscCall(ISRestoreIndices(scatis_natural_slice, &is_array));
198: /* Create a 2D DMDA global vector */
199: PetscCall(DMCreateGlobalVector(da2D, &vec_slice));
200: PetscCall(PetscObjectSetName((PetscObject)vec_slice, "slice_vector_natural"));
201: PetscCall(VecGetLocalSize(vec_slice, &vn));
202: PetscCall(VecGetArrayRead(vec_slice, &array));
204: /* Create an aliased version of the above on the 3D DMDA's communicator */
205: PetscCall(VecCreateMPIWithArray(PETSC_COMM_WORLD, 1, vn, M1 * M2, array, &vec_slice_g));
206: PetscCall(VecRestoreArrayRead(vec_slice, &array));
207: } else {
208: /* Ranks not part of the subset MPI communicator provide no entries, but the routines for creating
209: the IS and Vec on the 3D DMDA's communicator still need to called, since they are collective routines */
210: PetscCall(ISCreateGeneral(PETSC_COMM_WORLD, 0, NULL, PETSC_USE_POINTER, &scatis_natural_slice_g));
211: PetscCall(VecCreateMPIWithArray(PETSC_COMM_WORLD, 1, 0, M1 * M2, NULL, &vec_slice_g));
212: }
213: PetscCall(PetscBarrier(NULL));
215: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
216: Create IS that maps from the extracted slice vector
217: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
218: PetscCall(VecGetOwnershipRange(vec_extracted, &low, &high));
219: PetscCall(ISCreateStride(PETSC_COMM_WORLD, high - low, low, 1, &scatis_extracted_slice));
221: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
222: Scatter extracted subvector -> natural 2D slice vector
223: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
224: PetscCall(VecScatterCreate(vec_extracted, scatis_extracted_slice, vec_slice_g, scatis_natural_slice_g, &vscat));
225: PetscCall(VecScatterBegin(vscat, vec_extracted, vec_slice_g, INSERT_VALUES, SCATTER_FORWARD));
226: PetscCall(VecScatterEnd(vscat, vec_extracted, vec_slice_g, INSERT_VALUES, SCATTER_FORWARD));
228: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
229: View the natural 2D slice vector
230: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
231: PetscCall(PetscViewerPushFormat(PETSC_VIEWER_STDOUT_WORLD, PETSC_VIEWER_ASCII_DENSE));
232: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "\n Extracted slice vector, in natural ordering : \n"));
233: PetscCall(VecView(vec_slice_g, PETSC_VIEWER_STDOUT_WORLD));
234: PetscCall(PetscViewerPopFormat(PETSC_VIEWER_STDOUT_WORLD));
236: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
237: Restore subvector
238: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
239: PetscCall(VecRestoreSubVector(vec_full, patchis_3d, &vec_extracted));
241: /* - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
242: Destroy data structures and exit.
243: - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - */
244: PetscCall(VecDestroy(&vec_full));
245: PetscCall(VecScatterDestroy(&vscat));
246: PetscCall(ISDestroy(&scatis_extracted_slice));
247: PetscCall(ISDestroy(&scatis_natural_slice_g));
248: PetscCall(VecDestroy(&vec_slice_g));
249: PetscCall(ISDestroy(&patchis_3d));
250: PetscCall(DMDestroy(&da3D));
252: if (subset_mpi_comm != MPI_COMM_NULL) {
253: PetscCall(ISDestroy(&patchis_2d));
254: PetscCall(ISDestroy(&scatis_natural_slice));
255: PetscCall(VecDestroy(&vec_slice));
256: PetscCall(DMDestroy(&da2D));
257: PetscCallMPI(MPI_Comm_free(&subset_mpi_comm));
258: }
260: PetscCall(PetscFinalize());
261: return 0;
262: }
264: /*TEST
266: test:
267: nsize: 1
268: args: -sliceaxis X -gp 0
270: test:
271: suffix: 2
272: nsize: 2
273: args: -sliceaxis Y -gp 1
274: filter: grep -v "subset MPI subcomm"
276: test:
277: suffix: 3
278: nsize: 3
279: args: -sliceaxis Z -gp 2
280: filter: grep -v "subset MPI subcomm"
282: test:
283: suffix: 4
284: nsize: 4
285: args: -sliceaxis X -gp 2
286: filter: grep -v "subset MPI subcomm"
288: test:
289: suffix: 5
290: nsize: 4
291: args: -sliceaxis Z -gp 1
292: filter: grep -v "subset MPI subcomm"
294: TEST*/