Actual source code: ex92.c

  1: #include <petscksp.h>

  3: static char help[] = "Checks per-RHS convergence of HPDDM pseudo-block and block Krylov methods.\n";

  5: int main(int argc, char **argv)
  6: {
  7:   Mat                A, B, X;
  8:   KSP                ksp;
  9:   PC                 pc;
 10:   Vec                b, x, r;
 11:   PetscReal          rtol, atol, bnorm, rnorm, reference, tolerance[4];
 12:   PetscInt           start, end, m, n, nrhs, nsolve, its;
 13:   PetscBool          nonzero_guess = PETSC_FALSE, extended = PETSC_FALSE, initial_norm = PETSC_FALSE, min_initial_norm = PETSC_FALSE, check_converged_maxits = PETSC_FALSE, zero_rhs = PETSC_FALSE, exact_guess = PETSC_FALSE;
 14:   KSPConvergedReason reason;

 16:   PetscFunctionBeginUser;
 17:   PetscCall(PetscInitialize(&argc, &argv, NULL, help));
 18:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-nonzero_guess", &nonzero_guess, NULL));
 19:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-extended", &extended, NULL));
 20:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-check_converged_maxits", &check_converged_maxits, NULL));
 21:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-zero_rhs", &zero_rhs, NULL));
 22:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-exact_guess", &exact_guess, NULL));
 23:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-ksp_converged_use_initial_residual_norm", &initial_norm, NULL));
 24:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-ksp_converged_use_min_initial_residual_norm", &min_initial_norm, NULL));
 25:   n      = extended == PETSC_TRUE ? 6 : 2;
 26:   nrhs   = extended == PETSC_TRUE ? 4 : 2;
 27:   nsolve = zero_rhs == PETSC_TRUE ? 5 : (extended == PETSC_TRUE ? 2 : 1);
 28:   if (extended == PETSC_TRUE) nonzero_guess = PETSC_TRUE;
 29:   PetscCall(MatCreateAIJ(PETSC_COMM_WORLD, PETSC_DECIDE, PETSC_DECIDE, n, n, 1, NULL, 0, NULL, &A));
 30:   PetscCall(MatGetOwnershipRange(A, &start, &end));
 31:   for (PetscInt i = start; i < end; ++i) PetscCall(MatSetValue(A, i, i, extended == PETSC_FALSE && nonzero_guess == PETSC_TRUE ? 1.0 : i + 1.0, INSERT_VALUES));
 32:   PetscCall(MatAssemblyBegin(A, MAT_FINAL_ASSEMBLY));
 33:   PetscCall(MatAssemblyEnd(A, MAT_FINAL_ASSEMBLY));
 34:   PetscCall(MatGetLocalSize(A, &m, NULL));
 35:   PetscCall(MatCreateDense(PETSC_COMM_WORLD, m, PETSC_DECIDE, n, nrhs, NULL, &B));
 36:   PetscCall(MatCreateDense(PETSC_COMM_WORLD, m, PETSC_DECIDE, n, nrhs, NULL, &X));
 37:   PetscCall(MatCreateVecs(A, NULL, &r));

 39:   PetscCall(KSPCreate(PETSC_COMM_WORLD, &ksp));
 40:   PetscCall(KSPSetType(ksp, KSPHPDDM));
 41:   PetscCall(KSPHPDDMSetType(ksp, KSP_HPDDM_TYPE_GMRES));
 42:   PetscCall(KSPSetOperators(ksp, A, A));
 43:   PetscCall(KSPGetPC(ksp, &pc));
 44:   PetscCall(PCSetType(pc, PCNONE));
 45:   PetscCall(KSPSetTolerances(ksp, extended == PETSC_TRUE ? 1.e-8 : 1.e-4, 1.e-12, PETSC_CURRENT, extended == PETSC_TRUE ? 100 : 10));
 46:   PetscCall(KSPSetInitialGuessNonzero(ksp, (PetscBool)(nonzero_guess == PETSC_TRUE || zero_rhs == PETSC_TRUE || exact_guess == PETSC_TRUE)));
 47:   PetscCall(KSPSetFromOptions(ksp));
 48:   PetscCall(KSPGetTolerances(ksp, &rtol, &atol, NULL, NULL));

 50:   for (PetscInt solve = 0; solve < nsolve; ++solve) {
 51:     PetscCall(MatZeroEntries(B));
 52:     PetscCall(MatZeroEntries(X));
 53:     if (zero_rhs == PETSC_TRUE) {
 54:       // move the zero column, then solve zero RHS from zero and nonzero initial guesses
 55:       for (PetscInt j = 0; j < nrhs; ++j) {
 56:         PetscBool zero = (PetscBool)(solve == 2 || solve == 4 || j == (solve == 1 ? 1 : 0));

 58:         for (PetscInt i = start; i < end; ++i) {
 59:           if (zero == PETSC_FALSE) PetscCall(MatSetValue(B, i, j, i + 1.0, INSERT_VALUES));
 60:           if (zero == PETSC_TRUE && solve >= 3) PetscCall(MatSetValue(X, i, j, 1.0, INSERT_VALUES));
 61:         }
 62:       }
 63:     } else if (extended == PETSC_TRUE) {
 64:       // reverse the RHS scales on the second solve to exercise reused convergence and recycling state
 65:       for (PetscInt j = 0; j < 2; ++j) {
 66:         PetscReal scale = j == solve ? 10000.0 : 1.0, factor = j == 0 ? 10.0 : 0.01;

 68:         for (PetscInt i = start; i < end; ++i) {
 69:           PetscCall(MatSetValue(B, i, j, scale * (i + 1.0), INSERT_VALUES));
 70:           PetscCall(MatSetValue(X, i, j, scale * (1.0 - factor), INSERT_VALUES));
 71:         }
 72:       }
 73:       // RHS #2 is 0, and RHS #3 starts with an exact solution
 74:       for (PetscInt i = start; i < end; ++i) {
 75:         PetscCall(MatSetValue(B, i, 3, i == 0 ? 1.0 : 0.0, INSERT_VALUES));
 76:         PetscCall(MatSetValue(X, i, 3, i == 0 ? 1.0 : 0.0, INSERT_VALUES));
 77:       }
 78:     } else if (nonzero_guess == PETSC_TRUE) {
 79:       // each initial residual exceeds its own tolerance but is below rtol times the Frobenius norm of B
 80:       for (PetscInt i = start; i < end; ++i) {
 81:         PetscCall(MatSetValue(B, i, i, 1.0, INSERT_VALUES));
 82:         PetscCall(MatSetValue(X, i, i, 1.0 - 1.2e-4, INSERT_VALUES));
 83:       }
 84:     } else {
 85:       // the first RHS is solved in one iteration; the smaller second RHS needs another
 86:       for (PetscInt i = start; i < end; ++i) {
 87:         PetscCall(MatSetValue(B, i, 0, i == 0 ? 10000.0 : 0.0, INSERT_VALUES));
 88:         PetscCall(MatSetValue(B, i, 1, i == 0 ? 1.0 : 2.0, INSERT_VALUES));
 89:       }
 90:     }
 91:     PetscCall(MatAssemblyBegin(B, MAT_FINAL_ASSEMBLY));
 92:     PetscCall(MatAssemblyEnd(B, MAT_FINAL_ASSEMBLY));
 93:     PetscCall(MatAssemblyBegin(X, MAT_FINAL_ASSEMBLY));
 94:     PetscCall(MatAssemblyEnd(X, MAT_FINAL_ASSEMBLY));
 95:     if (exact_guess == PETSC_TRUE) {
 96:       PetscCall(MatGetDiagonal(A, r));
 97:       for (PetscInt j = 0; j < nrhs; ++j) {
 98:         PetscCall(MatDenseGetColumnVecRead(B, j, &b));
 99:         PetscCall(MatDenseGetColumnVecWrite(X, j, &x));
100:         PetscCall(VecPointwiseDivide(x, b, r));
101:         PetscCall(MatDenseRestoreColumnVecWrite(X, j, &x));
102:         PetscCall(MatDenseRestoreColumnVecRead(B, j, &b));
103:       }
104:     }
105:     for (PetscInt j = 0; j < nrhs; ++j) {
106:       PetscCall(MatDenseGetColumnVecRead(B, j, &b));
107:       PetscCall(MatDenseGetColumnVecRead(X, j, &x));
108:       PetscCall(MatMult(A, x, r));
109:       PetscCall(VecAYPX(r, -1.0, b));
110:       PetscCall(VecNorm(b, NORM_2, &bnorm));
111:       PetscCall(VecNorm(r, NORM_2, &rnorm));
112:       reference = bnorm;
113:       if (initial_norm == PETSC_TRUE || bnorm == 0.0) reference = rnorm;
114:       else if (min_initial_norm == PETSC_TRUE) reference = PetscMin(bnorm, rnorm);
115:       tolerance[j] = PetscMax(atol, rtol * reference);
116:       PetscCall(MatDenseRestoreColumnVecRead(X, j, &x));
117:       PetscCall(MatDenseRestoreColumnVecRead(B, j, &b));
118:     }
119:     PetscCall(KSPMatSolve(ksp, B, X));
120:     PetscCall(KSPGetConvergedReason(ksp, &reason));
121:     PetscCheck(reason > 0, PETSC_COMM_WORLD, PETSC_ERR_PLIB, "Solve #%" PetscInt_FMT " did not converge: %s", solve, KSPConvergedReasons[reason]);
122:     PetscCheck(check_converged_maxits == PETSC_FALSE || reason == KSP_CONVERGED_ITS, PETSC_COMM_WORLD, PETSC_ERR_PLIB, "Expected convergence at the iteration limit, got %s", KSPConvergedReasons[reason]);
123:     if (exact_guess == PETSC_TRUE || (zero_rhs == PETSC_TRUE && solve == 2)) {
124:       PetscCall(KSPGetIterationNumber(ksp, &its));
125:       PetscCheck(its == 0, PETSC_COMM_WORLD, PETSC_ERR_PLIB, "Solve #%" PetscInt_FMT " took %" PetscInt_FMT " iterations with an exact initial solution", solve, its);
126:     }
127:     for (PetscInt j = 0; j < nrhs; ++j) {
128:       PetscCall(MatDenseGetColumnVecRead(B, j, &b));
129:       PetscCall(MatDenseGetColumnVecRead(X, j, &x));
130:       PetscCall(MatMult(A, x, r));
131:       PetscCall(VecAYPX(r, -1.0, b));
132:       PetscCall(VecNorm(r, NORM_2, &rnorm));
133:       PetscCheck(rnorm <= 1.01 * tolerance[j], PETSC_COMM_WORLD, PETSC_ERR_PLIB, "Solve #%" PetscInt_FMT " RHS #%" PetscInt_FMT " has residual %g above its tolerance %g", solve, j, (double)rnorm, (double)tolerance[j]);
134:       PetscCall(MatDenseRestoreColumnVecRead(X, j, &x));
135:       PetscCall(MatDenseRestoreColumnVecRead(B, j, &b));
136:     }
137:   }
138:   PetscCall(KSPDestroy(&ksp));
139:   PetscCall(VecDestroy(&r));
140:   PetscCall(MatDestroy(&X));
141:   PetscCall(MatDestroy(&B));
142:   PetscCall(MatDestroy(&A));
143:   PetscCall(PetscFinalize());
144:   return 0;
145: }

147: /*TEST

149:   build:
150:     requires: hpddm double

152:   testset:
153:     nsize: 2
154:     output_file: output/empty.out
155:     test:
156:       suffix: 1
157:       args: -ksp_hpddm_variant {{left right flexible}shared output}
158:     test:
159:       suffix: nonzero_guess
160:       requires: !complex !defined(PETSC_USE_64BIT_INDICES)
161:       args: -nonzero_guess -ksp_hpddm_variant {{left right flexible}shared output}
162:     test:
163:       suffix: nonzero_guess_bcg
164:       requires: !complex !defined(PETSC_USE_64BIT_INDICES)
165:       args: -nonzero_guess -ksp_hpddm_type bcg

167:   test:
168:     suffix: converged_maxits
169:     requires: !complex !defined(PETSC_USE_64BIT_INDICES)
170:     nsize: 2
171:     args: -ksp_converged_maxits -ksp_max_it 2 -check_converged_maxits
172:     output_file: output/empty.out

174:   testset:
175:     suffix: nonzero_guess_block
176:     requires: !complex !defined(PETSC_USE_64BIT_INDICES)
177:     nsize: 2
178:     args: -nonzero_guess -ksp_gmres_restart 4
179:     output_file: output/empty.out
180:     test:
181:       suffix: gmres
182:       args: -ksp_hpddm_type bgmres
183:     test:
184:       suffix: gcrodr
185:       args: -ksp_hpddm_type bgcrodr -ksp_hpddm_recycle 1

187:   test:
188:     suffix: cg
189:     nsize: 2
190:     args: -ksp_hpddm_type cg -ksp_hpddm_variant {{left flexible}shared output}
191:     output_file: output/empty.out

193:   test:
194:     suffix: gcrodr
195:     nsize: 2
196:     args: -ksp_hpddm_type gcrodr -ksp_hpddm_variant right -ksp_gmres_restart 3 -ksp_hpddm_recycle 1
197:     output_file: output/empty.out

199:   test:
200:     suffix: extended
201:     nsize: 2
202:     args: -extended -ksp_gmres_restart 3 -ksp_matsolve_batch_size {{4 3}shared output}
203:     output_file: output/empty.out

205:   testset:
206:     suffix: cg_extended
207:     nsize: 2
208:     args: -extended -ksp_hpddm_type cg -ksp_hpddm_variant {{left flexible}shared output}
209:     output_file: output/empty.out
210:     test:
211:       suffix: uir
212:       args: -ksp_converged_use_initial_residual_norm
213:     test:
214:       suffix: umir
215:       args: -ksp_converged_use_min_initial_residual_norm

217:   testset:
218:     suffix: gcrodr_extended
219:     nsize: 2
220:     args: -extended -ksp_hpddm_type gcrodr -ksp_hpddm_variant {{right flexible}shared output} -ksp_gmres_restart 4 -ksp_hpddm_recycle 2 -ksp_matsolve_batch_size 3
221:     output_file: output/empty.out
222:     test:
223:       suffix: uir
224:       args: -ksp_converged_use_initial_residual_norm
225:     test:
226:       suffix: umir
227:       args: -ksp_converged_use_min_initial_residual_norm

229:   testset:
230:     suffix: gcrodr_extended
231:     nsize: 2
232:     args: -extended -ksp_hpddm_type gcrodr -ksp_hpddm_recycle 2
233:     output_file: output/empty.out
234:     test:
235:       suffix: umir_full_batch
236:       args: -ksp_matsolve_batch_size 4 -ksp_gmres_restart 4 -ksp_hpddm_variant right -ksp_converged_use_min_initial_residual_norm
237:     test:
238:       suffix: large_restart
239:       requires: !complex !defined(PETSC_USE_64BIT_INDICES)
240:       args: -ksp_matsolve_batch_size 3 -ksp_gmres_restart 30
241:     test:
242:       suffix: mixed_precision
243:       args: -ksp_matsolve_batch_size 3 -ksp_gmres_restart 4 -ksp_hpddm_variant {{right flexible}shared output} -ksp_hpddm_precision single -ksp_rtol 1e-5
244:     test:
245:       suffix: gcrodr_extended_strategy_b
246:       requires: !complex !defined(PETSC_USE_64BIT_INDICES)
247:       args: -ksp_hpddm_variant flexible -ksp_gmres_restart 4 -ksp_hpddm_recycle_strategy B -ksp_converged_use_min_initial_residual_norm

249:   testset:
250:     suffix: zero_rhs
251:     nsize: 2
252:     args: -zero_rhs
253:     output_file: output/empty.out
254:     test:
255:       suffix: pseudo
256:       args: -ksp_hpddm_type {{gmres cg}shared output}
257:     test:
258:       suffix: recycle
259:       args: -ksp_hpddm_type gcrodr -ksp_gmres_restart 4 -ksp_hpddm_recycle 1
260:     test:
261:       suffix: block
262:       args: -ksp_hpddm_type {{bgmres bfbcg}shared output} -ksp_hpddm_deflation_tol 1e-12
263:     test:
264:       suffix: block_recycle
265:       args: -ksp_hpddm_type bgcrodr -ksp_gmres_restart 4 -ksp_hpddm_recycle 1 -ksp_hpddm_deflation_tol 1e-12

267:   testset:
268:     suffix: exact_guess
269:     requires: !complex !defined(PETSC_USE_64BIT_INDICES)
270:     nsize: 2
271:     args: -exact_guess
272:     output_file: output/empty.out
273:     test:
274:       suffix: zero_iterations
275:       args: -ksp_hpddm_type {{gmres cg gcrodr}shared output} -ksp_max_it 0

277: TEST*/