Actual source code: ex51.c
1: static char help[] = "This example solves a linear system in parallel with KSP. The matrix\n\
2: uses arbitrary order polynomials for finite elements on the unit square. To test the parallel\n\
3: matrix assembly, the matrix is intentionally laid out across processors\n\
4: differently from the way it is assembled. Input arguments are:\n\
5: -m <size> -p <order> : mesh size and polynomial order\n\n";
7: /* Contributed by Travis Austin <austin@txcorp.com>, 2010.
8: based on src/ksp/ksp/tutorials/ex3.c
9: */
11: #include <petscksp.h>
13: /* Declare user-defined routines */
14: static PetscReal src(PetscReal, PetscReal);
15: static PetscReal ubdy(PetscReal, PetscReal);
16: static PetscReal polyBasisFunc(PetscInt, PetscInt, PetscReal *, PetscReal);
17: static PetscReal derivPolyBasisFunc(PetscInt, PetscInt, PetscReal *, PetscReal);
18: static PetscErrorCode Form1DElementMass(PetscReal, PetscInt, PetscReal *, PetscReal *, PetscScalar *);
19: static PetscErrorCode Form1DElementStiffness(PetscReal, PetscInt, PetscReal *, PetscReal *, PetscScalar *);
20: static PetscErrorCode Form2DElementMass(PetscInt, PetscScalar *, PetscScalar *);
21: static PetscErrorCode Form2DElementStiffness(PetscInt, PetscScalar *, PetscScalar *, PetscScalar *);
22: static PetscErrorCode FormNodalRhs(PetscInt, PetscReal, PetscReal, PetscReal, PetscReal *, PetscScalar *);
23: static PetscErrorCode FormNodalSoln(PetscInt, PetscReal, PetscReal, PetscReal, PetscReal *, PetscScalar *);
24: static void leggaulob(PetscReal, PetscReal, PetscReal[], PetscReal[], PetscInt);
25: static void qAndLEvaluation(PetscInt, PetscReal, PetscReal *, PetscReal *, PetscReal *);
27: int main(int argc, char **args)
28: {
29: PetscInt p = 2, m = 5;
30: PetscInt num1Dnodes, num2Dnodes;
31: PetscScalar *Ke1D, *Ke2D, *Me1D, *Me2D;
32: PetscScalar *r, *ue, val;
33: Vec u, ustar, b, q;
34: Mat A, Mass;
35: KSP ksp;
36: PetscInt M, N;
37: PetscMPIInt rank, size;
38: PetscReal x, y, h, norm;
39: PetscInt *idx, indx, count, *rows, i, j, k, start, end, its;
40: PetscReal *rowsx, *rowsy;
41: PetscReal *gllNode, *gllWgts;
43: PetscFunctionBeginUser;
44: PetscCall(PetscInitialize(&argc, &args, NULL, help));
45: PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "Options for p-FEM", "");
46: PetscCall(PetscOptionsInt("-m", "Number of elements in each direction", "None", m, &m, NULL));
47: PetscCall(PetscOptionsInt("-p", "Order of each element (tensor product basis)", "None", p, &p, NULL));
48: PetscOptionsEnd();
49: PetscCheck(p > 0, PETSC_COMM_SELF, PETSC_ERR_USER, "Option -p value should be greater than zero");
50: N = (p * m + 1) * (p * m + 1); /* dimension of matrix */
51: M = m * m; /* number of elements */
52: h = 1.0 / m; /* mesh width */
53: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
54: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
56: /* Create stiffness matrix */
57: PetscCall(MatCreate(PETSC_COMM_WORLD, &A));
58: PetscCall(MatSetSizes(A, PETSC_DECIDE, PETSC_DECIDE, N, N));
59: PetscCall(MatSetFromOptions(A));
60: PetscCall(MatSetUp(A));
62: /* Create matrix */
63: PetscCall(MatCreate(PETSC_COMM_WORLD, &Mass));
64: PetscCall(MatSetSizes(Mass, PETSC_DECIDE, PETSC_DECIDE, N, N));
65: PetscCall(MatSetFromOptions(Mass));
66: PetscCall(MatSetUp(Mass));
67: start = rank * (M / size) + ((M % size) < rank ? (M % size) : rank);
68: end = start + M / size + ((M % size) > rank);
70: /* Allocate element stiffness matrices */
71: num1Dnodes = (p + 1);
72: num2Dnodes = num1Dnodes * num1Dnodes;
74: PetscCall(PetscMalloc1(num1Dnodes * num1Dnodes, &Me1D));
75: PetscCall(PetscMalloc1(num1Dnodes * num1Dnodes, &Ke1D));
76: PetscCall(PetscMalloc1(num2Dnodes * num2Dnodes, &Me2D));
77: PetscCall(PetscMalloc1(num2Dnodes * num2Dnodes, &Ke2D));
78: PetscCall(PetscMalloc1(num2Dnodes, &idx));
79: PetscCall(PetscMalloc1(num2Dnodes, &r));
80: PetscCall(PetscMalloc1(num2Dnodes, &ue));
82: /* Allocate quadrature and create stiffness matrices */
83: PetscCall(PetscMalloc1(p + 1, &gllNode));
84: PetscCall(PetscMalloc1(p + 1, &gllWgts));
85: leggaulob(0.0, 1.0, gllNode, gllWgts, p); /* Get GLL nodes and weights */
86: PetscCall(Form1DElementMass(h, p, gllNode, gllWgts, Me1D));
87: PetscCall(Form1DElementStiffness(h, p, gllNode, gllWgts, Ke1D));
88: PetscCall(Form2DElementMass(p, Me1D, Me2D));
89: PetscCall(Form2DElementStiffness(p, Ke1D, Me1D, Ke2D));
91: /* Assemble matrix */
92: for (i = start; i < end; i++) {
93: indx = 0;
94: for (k = 0; k < (p + 1); ++k) {
95: for (j = 0; j < (p + 1); ++j) idx[indx++] = p * (p * m + 1) * (i / m) + p * (i % m) + k * (p * m + 1) + j;
96: }
97: PetscCall(MatSetValues(A, num2Dnodes, idx, num2Dnodes, idx, Ke2D, ADD_VALUES));
98: PetscCall(MatSetValues(Mass, num2Dnodes, idx, num2Dnodes, idx, Me2D, ADD_VALUES));
99: }
100: PetscCall(MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY));
101: PetscCall(MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY));
102: PetscCall(MatAssemblyBegin(Mass, MAT_FINAL_ASSEMBLY));
103: PetscCall(MatAssemblyEnd(Mass, MAT_FINAL_ASSEMBLY));
105: PetscCall(PetscFree(Me1D));
106: PetscCall(PetscFree(Ke1D));
107: PetscCall(PetscFree(Me2D));
108: PetscCall(PetscFree(Ke2D));
110: /* Create right-hand side and solution vectors */
111: PetscCall(VecCreate(PETSC_COMM_WORLD, &u));
112: PetscCall(VecSetSizes(u, PETSC_DECIDE, N));
113: PetscCall(VecSetFromOptions(u));
114: PetscCall(PetscObjectSetName((PetscObject)u, "Approx. Solution"));
115: PetscCall(VecDuplicate(u, &b));
116: PetscCall(PetscObjectSetName((PetscObject)b, "Right hand side"));
117: PetscCall(VecDuplicate(u, &q));
118: PetscCall(PetscObjectSetName((PetscObject)q, "Right hand side 2"));
119: PetscCall(VecDuplicate(b, &ustar));
121: /* Assemble nodal right-hand side and soln vector */
122: for (i = start; i < end; i++) {
123: x = h * (i % m);
124: y = h * (i / m);
125: indx = 0;
126: for (k = 0; k < (p + 1); ++k) {
127: for (j = 0; j < (p + 1); ++j) idx[indx++] = p * (p * m + 1) * (i / m) + p * (i % m) + k * (p * m + 1) + j;
128: }
129: PetscCall(FormNodalRhs(p, x, y, h, gllNode, r));
130: PetscCall(FormNodalSoln(p, x, y, h, gllNode, ue));
131: PetscCall(VecSetValues(q, num2Dnodes, idx, r, INSERT_VALUES));
132: PetscCall(VecSetValues(ustar, num2Dnodes, idx, ue, INSERT_VALUES));
133: }
134: PetscCall(VecAssemblyBegin(q));
135: PetscCall(VecAssemblyEnd(q));
136: PetscCall(VecAssemblyBegin(ustar));
137: PetscCall(VecAssemblyEnd(ustar));
139: PetscCall(PetscFree(idx));
140: PetscCall(PetscFree(r));
141: PetscCall(PetscFree(ue));
143: /* Get FE right-hand side vector */
144: PetscCall(MatMult(Mass, q, b));
146: /* Modify matrix and right-hand side for Dirichlet boundary conditions */
147: PetscCall(PetscMalloc1(4 * p * m, &rows));
148: PetscCall(PetscMalloc1(4 * p * m, &rowsx));
149: PetscCall(PetscMalloc1(4 * p * m, &rowsy));
150: for (i = 0; i < p * m + 1; i++) {
151: rows[i] = i; /* bottom */
152: rowsx[i] = (i / p) * h + gllNode[i % p] * h;
153: rowsy[i] = 0.0;
154: rows[3 * p * m - 1 + i] = (p * m) * (p * m + 1) + i; /* top */
155: rowsx[3 * p * m - 1 + i] = (i / p) * h + gllNode[i % p] * h;
156: rowsy[3 * p * m - 1 + i] = 1.0;
157: }
158: count = p * m + 1; /* left side */
159: indx = 1;
160: for (i = p * m + 1; i < (p * m) * (p * m + 1); i += (p * m + 1)) {
161: rows[count] = i;
162: rowsx[count] = 0.0;
163: rowsy[count++] = (indx / p) * h + gllNode[indx % p] * h;
164: indx++;
165: }
166: count = 2 * p * m; /* right side */
167: indx = 1;
168: for (i = 2 * p * m + 1; i < (p * m) * (p * m + 1); i += (p * m + 1)) {
169: rows[count] = i;
170: rowsx[count] = 1.0;
171: rowsy[count++] = (indx / p) * h + gllNode[indx % p] * h;
172: indx++;
173: }
174: for (i = 0; i < 4 * p * m; i++) {
175: x = rowsx[i];
176: y = rowsy[i];
177: val = ubdy(x, y);
178: PetscCall(VecSetValues(b, 1, &rows[i], &val, INSERT_VALUES));
179: PetscCall(VecSetValues(u, 1, &rows[i], &val, INSERT_VALUES));
180: }
181: PetscCall(MatZeroRows(A, 4 * p * m, rows, 1.0, 0, 0));
182: PetscCall(PetscFree(rows));
183: PetscCall(PetscFree(rowsx));
184: PetscCall(PetscFree(rowsy));
186: PetscCall(VecAssemblyBegin(u));
187: PetscCall(VecAssemblyEnd(u));
188: PetscCall(VecAssemblyBegin(b));
189: PetscCall(VecAssemblyEnd(b));
191: /* Solve linear system */
192: PetscCall(KSPCreate(PETSC_COMM_WORLD, &ksp));
193: PetscCall(KSPSetOperators(ksp, A, A));
194: PetscCall(KSPSetInitialGuessNonzero(ksp, PETSC_TRUE));
195: PetscCall(KSPSetFromOptions(ksp));
196: PetscCall(KSPSolve(ksp, b, u));
198: /* Check error */
199: PetscCall(VecAXPY(u, -1.0, ustar));
200: PetscCall(VecNorm(u, NORM_2, &norm));
201: PetscCall(KSPGetIterationNumber(ksp, &its));
202: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Norm of error %g Iterations %" PetscInt_FMT "\n", (double)(norm * h), its));
204: PetscCall(PetscFree(gllNode));
205: PetscCall(PetscFree(gllWgts));
207: PetscCall(KSPDestroy(&ksp));
208: PetscCall(VecDestroy(&u));
209: PetscCall(VecDestroy(&b));
210: PetscCall(VecDestroy(&q));
211: PetscCall(VecDestroy(&ustar));
212: PetscCall(MatDestroy(&A));
213: PetscCall(MatDestroy(&Mass));
215: PetscCall(PetscFinalize());
216: return 0;
217: }
219: /* --------------------------------------------------------------------- */
221: /* 1d element stiffness mass matrix */
222: static PetscErrorCode Form1DElementMass(PetscReal H, PetscInt P, PetscReal *gqn, PetscReal *gqw, PetscScalar *Me1D)
223: {
224: PetscInt i, j, k;
225: PetscInt indx;
227: PetscFunctionBeginUser;
228: for (j = 0; j < (P + 1); ++j) {
229: for (i = 0; i < (P + 1); ++i) {
230: indx = j * (P + 1) + i;
231: Me1D[indx] = 0.0;
232: for (k = 0; k < (P + 1); ++k) Me1D[indx] += H * gqw[k] * polyBasisFunc(P, i, gqn, gqn[k]) * polyBasisFunc(P, j, gqn, gqn[k]);
233: }
234: }
235: PetscFunctionReturn(PETSC_SUCCESS);
236: }
238: /* --------------------------------------------------------------------- */
240: /* 1d element stiffness matrix for derivative */
241: static PetscErrorCode Form1DElementStiffness(PetscReal H, PetscInt P, PetscReal *gqn, PetscReal *gqw, PetscScalar *Ke1D)
242: {
243: PetscInt i, j, k;
244: PetscInt indx;
246: PetscFunctionBeginUser;
247: for (j = 0; j < (P + 1); ++j) {
248: for (i = 0; i < (P + 1); ++i) {
249: indx = j * (P + 1) + i;
250: Ke1D[indx] = 0.0;
251: for (k = 0; k < (P + 1); ++k) Ke1D[indx] += (1. / H) * gqw[k] * derivPolyBasisFunc(P, i, gqn, gqn[k]) * derivPolyBasisFunc(P, j, gqn, gqn[k]);
252: }
253: }
254: PetscFunctionReturn(PETSC_SUCCESS);
255: }
257: /* --------------------------------------------------------------------- */
259: /* element mass matrix */
260: static PetscErrorCode Form2DElementMass(PetscInt P, PetscScalar *Me1D, PetscScalar *Me2D)
261: {
262: PetscInt i1, j1, i2, j2;
263: PetscInt indx1, indx2, indx3;
265: PetscFunctionBeginUser;
266: for (j2 = 0; j2 < (P + 1); ++j2) {
267: for (i2 = 0; i2 < (P + 1); ++i2) {
268: for (j1 = 0; j1 < (P + 1); ++j1) {
269: for (i1 = 0; i1 < (P + 1); ++i1) {
270: indx1 = j1 * (P + 1) + i1;
271: indx2 = j2 * (P + 1) + i2;
272: indx3 = (j2 * (P + 1) + j1) * (P + 1) * (P + 1) + (i2 * (P + 1) + i1);
273: Me2D[indx3] = Me1D[indx1] * Me1D[indx2];
274: }
275: }
276: }
277: }
278: PetscFunctionReturn(PETSC_SUCCESS);
279: }
281: /* --------------------------------------------------------------------- */
283: /* element stiffness for Laplacian */
284: static PetscErrorCode Form2DElementStiffness(PetscInt P, PetscScalar *Ke1D, PetscScalar *Me1D, PetscScalar *Ke2D)
285: {
286: PetscInt i1, j1, i2, j2;
287: PetscInt indx1, indx2, indx3;
289: PetscFunctionBeginUser;
290: for (j2 = 0; j2 < (P + 1); ++j2) {
291: for (i2 = 0; i2 < (P + 1); ++i2) {
292: for (j1 = 0; j1 < (P + 1); ++j1) {
293: for (i1 = 0; i1 < (P + 1); ++i1) {
294: indx1 = j1 * (P + 1) + i1;
295: indx2 = j2 * (P + 1) + i2;
296: indx3 = (j2 * (P + 1) + j1) * (P + 1) * (P + 1) + (i2 * (P + 1) + i1);
297: Ke2D[indx3] = Ke1D[indx1] * Me1D[indx2] + Me1D[indx1] * Ke1D[indx2];
298: }
299: }
300: }
301: }
302: PetscFunctionReturn(PETSC_SUCCESS);
303: }
305: /* --------------------------------------------------------------------- */
307: static PetscErrorCode FormNodalRhs(PetscInt P, PetscReal x, PetscReal y, PetscReal H, PetscReal *nds, PetscScalar *r)
308: {
309: PetscInt i, j, indx;
311: PetscFunctionBeginUser;
312: indx = 0;
313: for (j = 0; j < (P + 1); ++j) {
314: for (i = 0; i < (P + 1); ++i) {
315: r[indx] = src(x + H * nds[i], y + H * nds[j]);
316: indx++;
317: }
318: }
319: PetscFunctionReturn(PETSC_SUCCESS);
320: }
322: /* --------------------------------------------------------------------- */
324: static PetscErrorCode FormNodalSoln(PetscInt P, PetscReal x, PetscReal y, PetscReal H, PetscReal *nds, PetscScalar *u)
325: {
326: PetscInt i, j, indx;
328: PetscFunctionBeginUser;
329: indx = 0;
330: for (j = 0; j < (P + 1); ++j) {
331: for (i = 0; i < (P + 1); ++i) {
332: u[indx] = ubdy(x + H * nds[i], y + H * nds[j]);
333: indx++;
334: }
335: }
336: PetscFunctionReturn(PETSC_SUCCESS);
337: }
339: /* --------------------------------------------------------------------- */
341: static PetscReal polyBasisFunc(PetscInt order, PetscInt basis, PetscReal *xLocVal, PetscReal xval)
342: {
343: PetscReal denominator = 1.;
344: PetscReal numerator = 1.;
345: PetscInt i = 0;
347: for (i = 0; i < (order + 1); i++) {
348: if (i != basis) {
349: numerator *= (xval - xLocVal[i]);
350: denominator *= (xLocVal[basis] - xLocVal[i]);
351: }
352: }
353: return numerator / denominator;
354: }
356: /* --------------------------------------------------------------------- */
358: static PetscReal derivPolyBasisFunc(PetscInt order, PetscInt basis, PetscReal *xLocVal, PetscReal xval)
359: {
360: PetscReal denominator;
361: PetscReal numerator;
362: PetscReal numtmp;
363: PetscInt i = 0, j = 0;
365: denominator = 1.;
366: for (i = 0; i < (order + 1); i++) {
367: if (i != basis) denominator *= (xLocVal[basis] - xLocVal[i]);
368: }
369: numerator = 0.;
370: for (j = 0; j < (order + 1); ++j) {
371: if (j != basis) {
372: numtmp = 1.;
373: for (i = 0; i < (order + 1); ++i) {
374: if (i != basis && j != i) numtmp *= (xval - xLocVal[i]);
375: }
376: numerator += numtmp;
377: }
378: }
380: return numerator / denominator;
381: }
383: /* --------------------------------------------------------------------- */
385: static PetscReal ubdy(PetscReal x, PetscReal y)
386: {
387: return x * x * y * y;
388: }
390: static PetscReal src(PetscReal x, PetscReal y)
391: {
392: return -2. * y * y - 2. * x * x;
393: }
394: /* --------------------------------------------------------------------- */
396: static void leggaulob(PetscReal x1, PetscReal x2, PetscReal x[], PetscReal w[], PetscInt n)
397: /*******************************************************************************
398: Given the lower and upper limits of integration x1 and x2, and given n, this
399: routine returns arrays x[0..n-1] and w[0..n-1] of length n, containing the abscissas
400: and weights of the Gauss-Lobatto-Legendre n-point quadrature formula.
401: *******************************************************************************/
402: {
403: PetscInt m;
404: PetscReal z1, z, xm, xl, q, qp, Ln, scale;
405: if (n == 1) {
406: x[0] = x1; /* Scale the root to the desired interval, */
407: x[1] = x2; /* and put in its symmetric counterpart. */
408: w[0] = 1.; /* Compute the weight */
409: w[1] = 1.; /* and its symmetric counterpart. */
410: } else {
411: x[0] = x1; /* Scale the root to the desired interval, */
412: x[n] = x2; /* and put in its symmetric counterpart. */
413: w[0] = 2. / (n * (n + 1)); /* Compute the weight */
414: w[n] = 2. / (n * (n + 1)); /* and its symmetric counterpart. */
415: m = (n + 1) / 2; /* The roots are symmetric, so we only find half of them. */
416: xm = 0.5 * (x2 + x1);
417: xl = 0.5 * (x2 - x1);
418: for (PetscInt j = 1; j <= (m - 1); j++) { /* Loop over the desired roots. */
419: z = -1.0 * PetscCosReal((PETSC_PI * (j + 0.25) / (n)) - (3.0 / (8.0 * n * PETSC_PI)) * (1.0 / (j + 0.25)));
420: /* Starting with the above approximation to the ith root, we enter */
421: /* the main loop of refinement by Newton's method. */
422: do {
423: qAndLEvaluation(n, z, &q, &qp, &Ln);
424: z1 = z;
425: z = z1 - q / qp; /* Newton's method. */
426: } while (PetscAbsReal(z - z1) > 3.0e-11);
427: qAndLEvaluation(n, z, &q, &qp, &Ln);
428: x[j] = xm + xl * z; /* Scale the root to the desired interval, */
429: x[n - j] = xm - xl * z; /* and put in its symmetric counterpart. */
430: w[j] = 2.0 / (n * (n + 1) * Ln * Ln); /* Compute the weight */
431: w[n - j] = w[j]; /* and its symmetric counterpart. */
432: }
433: }
434: if (n % 2 == 0) {
435: qAndLEvaluation(n, 0.0, &q, &qp, &Ln);
436: x[n / 2] = (x2 - x1) / 2.0;
437: w[n / 2] = 2.0 / (n * (n + 1) * Ln * Ln);
438: }
439: /* scale the weights according to mapping from [-1,1] to [0,1] */
440: scale = (x2 - x1) / 2.0;
441: for (PetscInt j = 0; j <= n; ++j) w[j] = w[j] * scale;
442: }
444: /******************************************************************************/
445: static void qAndLEvaluation(PetscInt n, PetscReal x, PetscReal *q, PetscReal *qp, PetscReal *Ln)
446: /*******************************************************************************
447: Compute the polynomial qn(x) = L_{N+1}(x) - L_{n-1}(x) and its derivative in
448: addition to L_N(x) as these are needed for the GLL points. See the book titled
449: "Implementing Spectral Methods for Partial Differential Equations: Algorithms
450: for Scientists and Engineers" by David A. Kopriva.
451: *******************************************************************************/
452: {
453: PetscInt k;
455: PetscReal Lnp;
456: PetscReal Lnp1, Lnp1p;
457: PetscReal Lnm1, Lnm1p;
458: PetscReal Lnm2, Lnm2p;
460: Lnm1 = 1.0;
461: *Ln = x;
462: Lnm1p = 0.0;
463: Lnp = 1.0;
465: for (k = 2; k <= n; ++k) {
466: Lnm2 = Lnm1;
467: Lnm1 = *Ln;
468: Lnm2p = Lnm1p;
469: Lnm1p = Lnp;
470: *Ln = (2. * k - 1.) / (1.0 * k) * x * Lnm1 - (k - 1.) / (1.0 * k) * Lnm2;
471: Lnp = Lnm2p + (2.0 * k - 1) * Lnm1;
472: }
473: k = n + 1;
474: Lnp1 = (2. * k - 1.) / (1.0 * k) * x * (*Ln) - (k - 1.) / (1.0 * k) * Lnm1;
475: Lnp1p = Lnm1p + (2.0 * k - 1) * (*Ln);
476: *q = Lnp1 - Lnm1;
477: *qp = Lnp1p - Lnm1p;
478: }
480: /*TEST
482: test:
483: nsize: 2
484: args: -ksp_monitor
486: TEST*/