Actual source code: ex1.c
1: static const char help[] = "Performance Tests for FE Integration";
3: #include <petscdmplex.h>
4: #include <petscfe.h>
5: #include <petscds.h>
7: typedef struct {
8: PetscInt dim; /* The topological dimension */
9: PetscBool simplex; /* True for simplices, false for hexes */
10: PetscInt its; /* Number of replications for timing */
11: PetscInt cbs; /* Number of cells in an integration block */
12: } AppCtx;
14: static PetscErrorCode ProcessOptions(MPI_Comm comm, AppCtx *options)
15: {
16: PetscFunctionBeginUser;
17: options->dim = 2;
18: options->simplex = PETSC_TRUE;
19: options->its = 1;
20: options->cbs = 8;
22: PetscOptionsBegin(comm, "", "FE Integration Performance Options", "PETSCFE");
23: PetscCall(PetscOptionsInt("-dim", "The topological dimension", "ex1.c", options->dim, &options->dim, NULL));
24: PetscCall(PetscOptionsBool("-simplex", "Simplex or hex cells", "ex1.c", options->simplex, &options->simplex, NULL));
25: PetscCall(PetscOptionsInt("-its", "The number of replications for timing", "ex1.c", options->its, &options->its, NULL));
26: PetscCall(PetscOptionsInt("-cbs", "The number of cells in an integration block", "ex1.c", options->cbs, &options->cbs, NULL));
27: PetscOptionsEnd();
28: PetscFunctionReturn(PETSC_SUCCESS);
29: }
31: static PetscErrorCode trig_u(PetscInt dim, PetscReal time, const PetscReal x[], PetscInt Nc, PetscScalar *u, PetscCtx ctx)
32: {
33: PetscInt d;
34: *u = 0.0;
35: for (d = 0; d < dim; ++d) *u += PetscSinReal(2.0 * PETSC_PI * x[d]);
36: return PETSC_SUCCESS;
37: }
39: static void f0_trig_u(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f0[])
40: {
41: for (PetscInt d = 0; d < dim; ++d) f0[0] += -4.0 * PetscSqr(PETSC_PI) * PetscSinReal(2.0 * PETSC_PI * x[d]);
42: }
44: static void f1_u(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar f1[])
45: {
46: for (PetscInt d = 0; d < dim; ++d) f1[d] = u_x[d];
47: }
49: static void g3_uu(PetscInt dim, PetscInt Nf, PetscInt NfAux, const PetscInt uOff[], const PetscInt uOff_x[], const PetscScalar u[], const PetscScalar u_t[], const PetscScalar u_x[], const PetscInt aOff[], const PetscInt aOff_x[], const PetscScalar a[], const PetscScalar a_t[], const PetscScalar a_x[], PetscReal t, PetscReal u_tShift, const PetscReal x[], PetscInt numConstants, const PetscScalar constants[], PetscScalar g3[])
50: {
51: for (PetscInt d = 0; d < dim; ++d) g3[d * dim + d] = 1.0;
52: }
54: static PetscErrorCode SetupPrimalProblem(DM dm, AppCtx *user)
55: {
56: PetscDS prob;
57: DMLabel label;
58: const PetscInt id = 1;
60: PetscFunctionBeginUser;
61: PetscCall(DMGetDS(dm, &prob));
62: PetscCall(PetscDSSetResidual(prob, 0, f0_trig_u, f1_u));
63: PetscCall(PetscDSSetJacobian(prob, 0, 0, NULL, NULL, NULL, g3_uu));
64: PetscCall(PetscDSSetExactSolution(prob, 0, trig_u, user));
65: PetscCall(DMGetLabel(dm, "marker", &label));
66: PetscCall(DMAddBoundary(dm, DM_BC_ESSENTIAL, "wall", label, 1, &id, 0, 0, NULL, (PetscVoidFn *)trig_u, NULL, user, NULL));
67: PetscFunctionReturn(PETSC_SUCCESS);
68: }
70: static PetscErrorCode SetupDiscretization(DM dm, const char name[], PetscErrorCode (*setup)(DM, AppCtx *), AppCtx *user)
71: {
72: DM cdm = dm;
73: PetscFE fe;
74: char prefix[PETSC_MAX_PATH_LEN];
76: PetscFunctionBeginUser;
77: /* Create finite element */
78: PetscCall(PetscSNPrintf(prefix, PETSC_MAX_PATH_LEN, "%s_", name));
79: PetscCall(PetscFECreateDefault(PetscObjectComm((PetscObject)dm), user->dim, 1, user->simplex, name ? prefix : NULL, -1, &fe));
80: PetscCall(PetscObjectSetName((PetscObject)fe, name));
81: /* Set discretization and boundary conditions for each mesh */
82: PetscCall(DMSetField(dm, 0, NULL, (PetscObject)fe));
83: PetscCall(DMCreateDS(dm));
84: PetscCall((*setup)(dm, user));
85: while (cdm) {
86: PetscCall(DMCopyDisc(dm, cdm));
87: /* TODO: Check whether the boundary of coarse meshes is marked */
88: PetscCall(DMGetCoarseDM(cdm, &cdm));
89: }
90: PetscCall(PetscFEDestroy(&fe));
91: PetscFunctionReturn(PETSC_SUCCESS);
92: }
94: /* PetscObjectContainerCompose() compose requires void ** signature on destructor */
95: static PetscErrorCode PetscFEGeomDestroy_Void(PetscCtxRt ctx)
96: {
97: return PetscFEGeomDestroy((PetscFEGeom **)ctx);
98: }
100: PetscErrorCode CellRangeGetFEGeom(IS cellIS, DMField coordField, PetscQuadrature quad, PetscFEGeomMode mode, PetscFEGeom **geom)
101: {
102: char composeStr[33] = {0};
103: PetscObjectId id;
104: PetscContainer container;
106: PetscFunctionBegin;
107: PetscCall(PetscObjectGetId((PetscObject)quad, &id));
108: PetscCall(PetscSNPrintf(composeStr, 32, "CellRangeGetFEGeom_%" PetscInt64_FMT "\n", id));
109: PetscCall(PetscObjectQuery((PetscObject)cellIS, composeStr, (PetscObject *)&container));
110: if (container) {
111: PetscCall(PetscContainerGetPointer(container, geom));
112: } else {
113: PetscCall(DMFieldCreateFEGeom(coordField, cellIS, quad, mode, geom));
114: PetscCall(PetscObjectContainerCompose((PetscObject)cellIS, composeStr, *geom, PetscFEGeomDestroy_Void));
115: }
116: PetscFunctionReturn(PETSC_SUCCESS);
117: }
119: PetscErrorCode CellRangeRestoreFEGeom(IS cellIS, DMField coordField, PetscQuadrature quad, PetscBool faceData, PetscFEGeom **geom)
120: {
121: PetscFunctionBegin;
122: *geom = NULL;
123: PetscFunctionReturn(PETSC_SUCCESS);
124: }
126: static PetscErrorCode CreateFEGeometry(DM dm, PetscDS ds, IS cellIS, PetscQuadrature *affineQuad, PetscFEGeom **affineGeom, PetscQuadrature **quads, PetscFEGeom ***geoms)
127: {
128: DMField coordField;
129: PetscInt Nf, f, maxDegree;
131: PetscFunctionBeginUser;
132: *affineQuad = NULL;
133: *affineGeom = NULL;
134: *quads = NULL;
135: *geoms = NULL;
136: PetscCall(PetscDSGetNumFields(ds, &Nf));
137: PetscCall(DMGetCoordinateField(dm, &coordField));
138: PetscCall(DMFieldGetDegree(coordField, cellIS, NULL, &maxDegree));
139: if (maxDegree <= 1) {
140: PetscCall(DMFieldCreateDefaultQuadrature(coordField, cellIS, affineQuad));
141: if (*affineQuad) PetscCall(CellRangeGetFEGeom(cellIS, coordField, *affineQuad, PETSC_FEGEOM_BASIC, affineGeom));
142: } else {
143: PetscCall(PetscCalloc2(Nf, quads, Nf, geoms));
144: for (f = 0; f < Nf; ++f) {
145: PetscFE fe;
147: PetscCall(PetscDSGetDiscretization(ds, f, (PetscObject *)&fe));
148: PetscCall(PetscFEGetQuadrature(fe, &(*quads)[f]));
149: PetscCall(PetscObjectReference((PetscObject)(*quads)[f]));
150: PetscCall(CellRangeGetFEGeom(cellIS, coordField, (*quads)[f], PETSC_FEGEOM_BASIC, &(*geoms)[f]));
151: }
152: }
153: PetscFunctionReturn(PETSC_SUCCESS);
154: }
156: static PetscErrorCode DestroyFEGeometry(DM dm, PetscDS ds, IS cellIS, PetscQuadrature *affineQuad, PetscFEGeom **affineGeom, PetscQuadrature **quads, PetscFEGeom ***geoms)
157: {
158: DMField coordField;
159: PetscInt Nf, f;
161: PetscFunctionBeginUser;
162: PetscCall(PetscDSGetNumFields(ds, &Nf));
163: PetscCall(DMGetCoordinateField(dm, &coordField));
164: if (*affineQuad) {
165: PetscCall(CellRangeRestoreFEGeom(cellIS, coordField, *affineQuad, PETSC_FALSE, affineGeom));
166: PetscCall(PetscQuadratureDestroy(affineQuad));
167: } else {
168: for (f = 0; f < Nf; ++f) {
169: PetscCall(CellRangeRestoreFEGeom(cellIS, coordField, (*quads)[f], PETSC_FALSE, &(*geoms)[f]));
170: PetscCall(PetscQuadratureDestroy(&(*quads)[f]));
171: }
172: PetscCall(PetscFree2(*quads, *geoms));
173: }
174: PetscFunctionReturn(PETSC_SUCCESS);
175: }
177: static PetscErrorCode TestIntegration(DM dm, PetscInt cbs, PetscInt its)
178: {
179: PetscDS ds;
180: PetscFEGeom *chunkGeom = NULL;
181: PetscQuadrature affineQuad, *quads = NULL;
182: PetscFEGeom *affineGeom, **geoms = NULL;
183: PetscScalar *u, *elemVec;
184: IS cellIS;
185: PetscInt depth, cStart, cEnd, cell, chunkSize = cbs, Nch = 0, Nf, f, totDim, i, k;
186: PetscLogStage stage;
187: PetscLogEvent event;
189: PetscFunctionBeginUser;
190: PetscCall(PetscLogStageRegister("PetscFE Residual Integration Test", &stage));
191: PetscCall(PetscLogEventRegister("FEIntegRes", PETSCFE_CLASSID, &event));
192: PetscCall(PetscLogStagePush(stage));
193: PetscCall(DMPlexGetDepth(dm, &depth));
194: PetscCall(DMGetStratumIS(dm, "depth", depth, &cellIS));
195: PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
196: PetscCall(DMGetCellDS(dm, cStart, &ds, NULL));
197: PetscCall(PetscDSGetNumFields(ds, &Nf));
198: PetscCall(PetscDSGetTotalDimension(ds, &totDim));
199: PetscCall(CreateFEGeometry(dm, ds, cellIS, &affineQuad, &affineGeom, &quads, &geoms));
200: PetscCall(PetscMalloc2(chunkSize * totDim, &u, chunkSize * totDim, &elemVec));
201: /* Assumptions:
202: - Single field
203: - No input data
204: - No auxiliary data
205: - No time-dependence
206: */
207: for (i = 0; i < its; ++i) {
208: for (cell = cStart; cell < cEnd; cell += chunkSize, ++Nch) {
209: const PetscInt cS = cell, cE = PetscMin(cS + chunkSize, cEnd), Ne = cE - cS;
211: PetscCall(PetscArrayzero(elemVec, chunkSize * totDim));
212: /* TODO Replace with DMPlexGetCellFields() */
213: for (k = 0; k < chunkSize * totDim; ++k) u[k] = 1.0;
214: for (f = 0; f < Nf; ++f) {
215: PetscFormKey key;
216: PetscFEGeom *geom = affineGeom ? affineGeom : geoms[f];
217: /* PetscQuadrature quad = affineQuad ? affineQuad : quads[f]; */
219: key.label = NULL;
220: key.value = 0;
221: key.field = f;
222: key.part = 0;
223: PetscCall(PetscFEGeomGetChunk(geom, cS, cE, &chunkGeom));
224: PetscCall(PetscLogEventBegin(event, 0, 0, 0, 0));
225: PetscCall(PetscFEIntegrateResidual(ds, key, Ne, chunkGeom, u, NULL, NULL, NULL, 0.0, elemVec));
226: PetscCall(PetscLogEventEnd(event, 0, 0, 0, 0));
227: }
228: }
229: }
230: PetscCall(PetscFEGeomRestoreChunk(affineGeom, cStart, cEnd, &chunkGeom));
231: PetscCall(DestroyFEGeometry(dm, ds, cellIS, &affineQuad, &affineGeom, &quads, &geoms));
232: PetscCall(ISDestroy(&cellIS));
233: PetscCall(PetscFree2(u, elemVec));
234: PetscCall(PetscLogStagePop());
235: if (PetscDefined(USE_LOG)) {
236: const char *title = "PETSc FE Residual Integration";
237: PetscEventPerfInfo eventInfo;
238: PetscInt N = (cEnd - cStart) * Nf * its;
239: PetscReal flopRate, cellRate;
241: PetscCall(PetscLogEventGetPerfInfo(stage, event, &eventInfo));
242: flopRate = eventInfo.time != 0.0 ? eventInfo.flops / eventInfo.time : 0.0;
243: cellRate = eventInfo.time != 0.0 ? N / eventInfo.time : 0.0;
244: PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "%s: %" PetscInt_FMT " integrals %" PetscInt_FMT " chunks %" PetscInt_FMT " reps\n Cell rate: %.2f/s flop rate: %.2f MF/s\n", title, N, Nch, its, (double)cellRate, (double)(flopRate / 1.e6)));
245: }
246: PetscFunctionReturn(PETSC_SUCCESS);
247: }
249: static PetscErrorCode TestIntegration2(DM dm, PetscInt cbs, PetscInt its)
250: {
251: Vec X, F;
252: PetscLogStage stage;
254: PetscFunctionBeginUser;
255: PetscCall(PetscLogStageRegister("DMPlex Residual Integration Test", &stage));
256: PetscCall(PetscLogStagePush(stage));
257: PetscCall(DMGetLocalVector(dm, &X));
258: PetscCall(DMGetLocalVector(dm, &F));
259: for (PetscInt i = 0; i < its; ++i) PetscCall(DMPlexSNESComputeResidualFEM(dm, X, F, NULL));
260: PetscCall(DMRestoreLocalVector(dm, &X));
261: PetscCall(DMRestoreLocalVector(dm, &F));
262: PetscCall(PetscLogStagePop());
263: if (PetscDefined(USE_LOG)) {
264: const char *title = "DMPlex Residual Integration";
265: PetscEventPerfInfo eventInfo;
266: PetscReal flopRate, cellRate;
267: PetscInt cStart, cEnd, Nf, N;
268: PetscLogEvent event;
270: PetscCall(DMPlexGetHeightStratum(dm, 0, &cStart, &cEnd));
271: PetscCall(DMGetNumFields(dm, &Nf));
272: PetscCall(PetscLogEventGetId("DMPlexResidualFE", &event));
273: PetscCall(PetscLogEventGetPerfInfo(stage, event, &eventInfo));
274: N = (cEnd - cStart) * Nf * eventInfo.count;
275: flopRate = eventInfo.time != 0.0 ? eventInfo.flops / eventInfo.time : 0.0;
276: cellRate = eventInfo.time != 0.0 ? N / eventInfo.time : 0.0;
277: PetscCall(PetscPrintf(PetscObjectComm((PetscObject)dm), "%s: %" PetscInt_FMT " integrals %d reps\n Cell rate: %.2f/s flop rate: %.2f MF/s\n", title, N, eventInfo.count, (double)cellRate, (double)(flopRate / 1.e6)));
278: }
279: PetscFunctionReturn(PETSC_SUCCESS);
280: }
282: int main(int argc, char **argv)
283: {
284: DM dm;
285: AppCtx ctx;
286: PetscMPIInt size;
288: PetscFunctionBeginUser;
289: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
290: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
291: PetscCheck(size == 1, PETSC_COMM_WORLD, PETSC_ERR_WRONG_MPI_SIZE, "This is a uniprocessor example only.");
292: PetscCall(ProcessOptions(PETSC_COMM_WORLD, &ctx));
293: PetscCall(PetscLogDefaultBegin());
294: PetscCall(DMCreate(PETSC_COMM_WORLD, &dm));
295: PetscCall(DMSetType(dm, DMPLEX));
296: PetscCall(DMSetFromOptions(dm));
297: PetscCall(PetscObjectSetName((PetscObject)dm, "Mesh"));
298: PetscCall(PetscObjectViewFromOptions((PetscObject)dm, NULL, "-dm_view"));
299: PetscCall(SetupDiscretization(dm, "potential", SetupPrimalProblem, &ctx));
300: PetscCall(TestIntegration(dm, ctx.cbs, ctx.its));
301: PetscCall(TestIntegration2(dm, ctx.cbs, ctx.its));
302: PetscCall(DMDestroy(&dm));
303: PetscCall(PetscFinalize());
304: return 0;
305: }
307: /*TEST
308: test:
309: suffix: 0
310: requires: triangle
311: args: -dm_view
313: test:
314: suffix: 1
315: requires: triangle
316: args: -dm_view -potential_petscspace_degree 1
318: test:
319: suffix: 2
320: requires: triangle
321: args: -dm_view -potential_petscspace_degree 2
323: test:
324: suffix: 3
325: requires: triangle
326: args: -dm_view -potential_petscspace_degree 3
327: TEST*/