Actual source code: pinit.c
1: #define PETSC_DESIRE_FEATURE_TEST_MACROS
2: /*
3: This file defines the initialization of PETSc, including PetscInitialize()
4: */
5: #include <petsc/private/petscimpl.h>
6: #include <petsc/private/logimpl.h>
7: #include <petscviewer.h>
8: #include <petsc/private/garbagecollector.h>
10: #if !defined(PETSC_HAVE_WINDOWS_COMPILERS)
11: #include <petsc/private/valgrind/valgrind.h>
12: #endif
14: #if defined(PETSC_USE_FORTRAN_BINDINGS)
15: #include <petsc/private/fortranimpl.h>
16: #endif
18: #if PetscDefined(USE_COVERAGE)
19: EXTERN_C_BEGIN
20: #if defined(PETSC_HAVE___GCOV_DUMP)
22: #endif
23: void __gcov_flush(void);
24: EXTERN_C_END
25: #endif
27: #if defined(PETSC_SERIALIZE_FUNCTIONS)
28: PETSC_INTERN PetscFPT PetscFPTData;
29: PetscFPT PetscFPTData = 0;
30: #endif
32: #if PetscDefined(HAVE_SAWS)
33: #include <petscviewersaws.h>
34: #endif
36: PETSC_INTERN FILE *petsc_history;
38: PETSC_INTERN PetscErrorCode PetscInitialize_DynamicLibraries(void);
39: PETSC_INTERN PetscErrorCode PetscFinalize_DynamicLibraries(void);
40: PETSC_INTERN PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm, int);
41: PETSC_INTERN PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm, int);
42: PETSC_INTERN PetscErrorCode PetscCloseHistoryFile(FILE **);
44: /* user may set these BEFORE calling PetscInitialize() */
45: MPI_Comm PETSC_COMM_WORLD = MPI_COMM_NULL;
46: #if PetscDefined(HAVE_MPI_INIT_THREAD)
47: PetscMPIInt PETSC_MPI_THREAD_REQUIRED = PETSC_DECIDE;
48: #else
49: PetscMPIInt PETSC_MPI_THREAD_REQUIRED = MPI_THREAD_SINGLE;
50: #endif
52: PetscMPIInt Petsc_Counter_keyval = MPI_KEYVAL_INVALID;
53: PetscMPIInt Petsc_InnerComm_keyval = MPI_KEYVAL_INVALID;
54: PetscMPIInt Petsc_OuterComm_keyval = MPI_KEYVAL_INVALID;
55: PetscMPIInt Petsc_ShmComm_keyval = MPI_KEYVAL_INVALID;
56: PetscMPIInt Petsc_CreationIdx_keyval = MPI_KEYVAL_INVALID;
57: PetscMPIInt Petsc_Garbage_HMap_keyval = MPI_KEYVAL_INVALID;
59: PetscMPIInt Petsc_SharedWD_keyval = MPI_KEYVAL_INVALID;
60: PetscMPIInt Petsc_SharedTmp_keyval = MPI_KEYVAL_INVALID;
62: /*
63: Declare and set all the string names of the PETSc enums
64: */
65: const char *const PetscBools[] = {"FALSE", "TRUE", "PetscBool", "PETSC_", NULL};
66: const char *const PetscCopyModes[] = {"COPY_VALUES", "OWN_POINTER", "USE_POINTER", "PetscCopyMode", "PETSC_", NULL};
68: PetscBool PetscPreLoadingUsed = PETSC_FALSE;
69: PetscBool PetscPreLoadingOn = PETSC_FALSE;
71: PetscInt PetscHotRegionDepth;
73: PetscBool PETSC_RUNNING_ON_VALGRIND = PETSC_FALSE;
75: #if defined(PETSC_HAVE_THREADSAFETY)
76: PetscSpinlock PetscViewerASCIISpinLockOpen;
77: PetscSpinlock PetscViewerASCIISpinLockStdout;
78: PetscSpinlock PetscViewerASCIISpinLockStderr;
79: PetscSpinlock PetscCommSpinLock;
80: #endif
82: extern PetscInt PetscNumBLASThreads;
84: /*@C
85: PetscInitializeNoPointers - Calls PetscInitialize() from C/C++ without the pointers to argc and args
87: Collective
89: Input Parameters:
90: + argc - number of args
91: . args - array of command line arguments
92: . filename - optional name of the program file, pass `NULL` to ignore
93: - help - optional help, pass `NULL` to ignore
95: Level: advanced
97: Notes:
98: this is called only by the PETSc Julia interface. Even though it might start MPI it sets the flag to
99: indicate that it did NOT start MPI so that the PetscFinalize() does not end MPI, thus allowing PetscInitialize() to
100: be called multiple times from Julia without the problem of trying to initialize MPI more than once.
102: Developer Notes:
103: Turns off PETSc signal handling to allow Julia to manage signals
105: .seealso: `PetscInitialize()`, `PetscInitializeFortran()`, `PetscInitializeNoArguments()`
106: */
107: PetscErrorCode PetscInitializeNoPointers(int argc, char **args, const char *filename, const char *help)
108: {
109: int myargc = argc;
110: char **myargs = args;
112: PetscFunctionBegin;
113: PetscCall(PetscInitialize(&myargc, &myargs, filename, help));
114: PetscCall(PetscPopSignalHandler());
115: PetscBeganMPI = PETSC_FALSE;
116: PetscFunctionReturn(PETSC_SUCCESS);
117: }
119: /*@C
120: PetscInitializeNoArguments - Calls `PetscInitialize()` from C/C++ without
121: the command line arguments.
123: Collective
125: Level: advanced
127: .seealso: `PetscInitialize()`, `PetscInitializeFortran()`
128: @*/
129: PetscErrorCode PetscInitializeNoArguments(void)
130: {
131: int argc = 0;
132: char **args = NULL;
134: PetscFunctionBegin;
135: PetscCall(PetscInitialize(&argc, &args, NULL, NULL));
136: PetscFunctionReturn(PETSC_SUCCESS);
137: }
139: /*@
140: PetscInitialized - Determine whether PETSc is initialized.
142: Output Parameter:
143: . isInitialized - `PETSC_TRUE` if PETSc is initialized, `PETSC_FALSE` otherwise
145: Level: beginner
147: .seealso: `PetscInitialize()`, `PetscInitializeNoArguments()`, `PetscInitializeFortran()`
148: @*/
149: PetscErrorCode PetscInitialized(PetscBool *isInitialized)
150: {
151: PetscFunctionBegin;
152: if (PetscInitializeCalled) PetscAssertPointer(isInitialized, 1);
153: *isInitialized = PetscInitializeCalled;
154: PetscFunctionReturn(PETSC_SUCCESS);
155: }
157: /*@
158: PetscFinalized - Determine whether `PetscFinalize()` has been called yet
160: Output Parameter:
161: . isFinalized - `PETSC_TRUE` if PETSc is finalized, `PETSC_FALSE` otherwise
163: Level: developer
165: .seealso: `PetscInitialize()`, `PetscInitializeNoArguments()`, `PetscInitializeFortran()`
166: @*/
167: PetscErrorCode PetscFinalized(PetscBool *isFinalized)
168: {
169: PetscFunctionBegin;
170: if (!PetscFinalizeCalled) PetscAssertPointer(isFinalized, 1);
171: *isFinalized = PetscFinalizeCalled;
172: PetscFunctionReturn(PETSC_SUCCESS);
173: }
175: PETSC_INTERN PetscErrorCode PetscOptionsCheckInitial_Private(const char[]);
177: /*
178: This function is the MPI reduction operation used to compute the sum of the
179: first half of the datatype and the max of the second half.
180: */
181: MPI_Op MPIU_MAXSUM_OP = 0;
182: MPI_Op Petsc_Garbage_SetIntersectOp = 0;
184: PETSC_INTERN void MPIAPI MPIU_MaxSum_Local(void *in, void *out, int *cnt, MPI_Datatype *datatype)
185: {
186: PetscInt *xin = (PetscInt *)in, *xout = (PetscInt *)out, i, count = *cnt;
188: PetscFunctionBegin;
189: if (*datatype != MPIU_2INT) {
190: PetscErrorCode ierr = (*PetscErrorPrintf)("Can only handle MPIU_2INT data types");
191: (void)ierr;
192: PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG);
193: }
195: for (i = 0; i < count; i++) {
196: xout[2 * i] = PetscMax(xout[2 * i], xin[2 * i]);
197: xout[2 * i + 1] += xin[2 * i + 1];
198: }
199: PetscFunctionReturnVoid();
200: }
202: /*
203: Returns the max of the first entry owned by this processor and the
204: sum of the second entry.
206: The reason sizes[2*i] contains lengths sizes[2*i+1] contains flag of 1 if length is nonzero
207: is so that the MPIU_MAXSUM_OP() can set TWO values, if we passed in only sizes[i] with lengths
208: there would be no place to store the both needed results.
209: */
210: PetscErrorCode PetscMaxSum(MPI_Comm comm, const PetscInt sizes[], PetscInt *max, PetscInt *sum)
211: {
212: PetscFunctionBegin;
213: #if defined(PETSC_HAVE_MPI_REDUCE_SCATTER_BLOCK)
214: {
215: struct {
216: PetscInt max, sum;
217: } work;
218: PetscCallMPI(MPI_Reduce_scatter_block((void *)sizes, &work, 1, MPIU_2INT, MPIU_MAXSUM_OP, comm));
219: *max = work.max;
220: *sum = work.sum;
221: }
222: #else
223: {
224: PetscMPIInt size, rank;
225: struct {
226: PetscInt max, sum;
227: } *work;
228: PetscCallMPI(MPI_Comm_size(comm, &size));
229: PetscCallMPI(MPI_Comm_rank(comm, &rank));
230: PetscCall(PetscMalloc1(size, &work));
231: PetscCall(MPIU_Allreduce((void *)sizes, work, size, MPIU_2INT, MPIU_MAXSUM_OP, comm));
232: *max = work[rank].max;
233: *sum = work[rank].sum;
234: PetscCall(PetscFree(work));
235: }
236: #endif
237: PetscFunctionReturn(PETSC_SUCCESS);
238: }
240: #if defined(PETSC_HAVE_REAL___FLOAT128) || defined(PETSC_HAVE_REAL___FP16)
241: #if defined(PETSC_HAVE_REAL___FLOAT128)
242: #include <quadmath.h>
243: #endif
244: MPI_Op MPIU_SUM___FP16___FLOAT128 = 0;
245: #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
246: MPI_Op MPIU_SUM = 0;
247: #endif
249: PETSC_EXTERN void MPIAPI PetscSum_Local(void *in, void *out, PetscMPIInt *cnt, MPI_Datatype *datatype)
250: {
251: PetscInt i, count = *cnt;
253: PetscFunctionBegin;
254: if (*datatype == MPIU_REAL) {
255: PetscReal *xin = (PetscReal *)in, *xout = (PetscReal *)out;
256: for (i = 0; i < count; i++) xout[i] += xin[i];
257: }
258: #if defined(PETSC_HAVE_COMPLEX)
259: else if (*datatype == MPIU_COMPLEX) {
260: PetscComplex *xin = (PetscComplex *)in, *xout = (PetscComplex *)out;
261: for (i = 0; i < count; i++) xout[i] += xin[i];
262: }
263: #endif
264: #if defined(PETSC_HAVE_REAL___FLOAT128)
265: else if (*datatype == MPIU___FLOAT128) {
266: __float128 *xin = (__float128 *)in, *xout = (__float128 *)out;
267: for (i = 0; i < count; i++) xout[i] += xin[i];
268: #if defined(PETSC_HAVE_COMPLEX)
269: } else if (*datatype == MPIU___COMPLEX128) {
270: __complex128 *xin = (__complex128 *)in, *xout = (__complex128 *)out;
271: for (i = 0; i < count; i++) xout[i] += xin[i];
272: #endif
273: }
274: #endif
275: #if defined(PETSC_HAVE_REAL___FP16)
276: else if (*datatype == MPIU___FP16) {
277: __fp16 *xin = (__fp16 *)in, *xout = (__fp16 *)out;
278: for (i = 0; i < count; i++) xout[i] += xin[i];
279: }
280: #endif
281: else {
282: #if !defined(PETSC_HAVE_REAL___FLOAT128) && !defined(PETSC_HAVE_REAL___FP16)
283: PetscCallAbort(MPI_COMM_SElF, (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types"));
284: #elif !defined(PETSC_HAVE_REAL___FP16)
285: PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL, MPIU_COMPLEX, MPIU___FLOAT128, or MPIU___COMPLEX128 data types"));
286: #elif !defined(PETSC_HAVE_REAL___FLOAT128)
287: PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL, MPIU_COMPLEX, or MPIU___FP16 data types"));
288: #else
289: PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL, MPIU_COMPLEX, MPIU___FLOAT128, MPIU___COMPLEX128, or MPIU___FP16 data types"));
290: #endif
291: PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG);
292: }
293: PetscFunctionReturnVoid();
294: }
295: #endif
297: #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
298: MPI_Op MPIU_MAX = 0;
299: MPI_Op MPIU_MIN = 0;
301: PETSC_EXTERN void MPIAPI PetscMax_Local(void *in, void *out, PetscMPIInt *cnt, MPI_Datatype *datatype)
302: {
303: PetscInt i, count = *cnt;
305: PetscFunctionBegin;
306: if (*datatype == MPIU_REAL) {
307: PetscReal *xin = (PetscReal *)in, *xout = (PetscReal *)out;
308: for (i = 0; i < count; i++) xout[i] = PetscMax(xout[i], xin[i]);
309: }
310: #if defined(PETSC_HAVE_COMPLEX)
311: else if (*datatype == MPIU_COMPLEX) {
312: PetscComplex *xin = (PetscComplex *)in, *xout = (PetscComplex *)out;
313: for (i = 0; i < count; i++) xout[i] = PetscRealPartComplex(xout[i]) < PetscRealPartComplex(xin[i]) ? xin[i] : xout[i];
314: }
315: #endif
316: else {
317: PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_COMPLEX data types"));
318: PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG);
319: }
320: PetscFunctionReturnVoid();
321: }
323: PETSC_EXTERN void MPIAPI PetscMin_Local(void *in, void *out, PetscMPIInt *cnt, MPI_Datatype *datatype)
324: {
325: PetscInt i, count = *cnt;
327: PetscFunctionBegin;
328: if (*datatype == MPIU_REAL) {
329: PetscReal *xin = (PetscReal *)in, *xout = (PetscReal *)out;
330: for (i = 0; i < count; i++) xout[i] = PetscMin(xout[i], xin[i]);
331: }
332: #if defined(PETSC_HAVE_COMPLEX)
333: else if (*datatype == MPIU_COMPLEX) {
334: PetscComplex *xin = (PetscComplex *)in, *xout = (PetscComplex *)out;
335: for (i = 0; i < count; i++) xout[i] = PetscRealPartComplex(xout[i]) > PetscRealPartComplex(xin[i]) ? xin[i] : xout[i];
336: }
337: #endif
338: else {
339: PetscCallAbort(MPI_COMM_SELF, (*PetscErrorPrintf)("Can only handle MPIU_REAL or MPIU_SCALAR data (i.e. double or complex) types"));
340: PETSCABORT(MPI_COMM_SELF, PETSC_ERR_ARG_WRONG);
341: }
342: PetscFunctionReturnVoid();
343: }
344: #endif
346: /*
347: Private routine to delete internal tag/name counter storage when a communicator is freed.
349: This is called by MPI, not by users. This is called by MPI_Comm_free() when the communicator that has this data as an attribute is freed.
351: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
353: */
354: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_Counter_Attr_DeleteFn(MPI_Comm comm, PetscMPIInt keyval, void *count_val, void *extra_state)
355: {
356: PetscCommCounter *counter = (PetscCommCounter *)count_val;
357: struct PetscCommStash *comms = counter->comms, *pcomm;
359: PetscFunctionBegin;
360: PetscCallMPI(PetscInfo(NULL, "Deleting counter data in an MPI_Comm %ld\n", (long)comm));
361: PetscCallMPI(PetscFree(counter->iflags));
362: while (comms) {
363: PetscCallMPI(MPI_Comm_free(&comms->comm));
364: pcomm = comms;
365: comms = comms->next;
366: PetscCall(PetscFree(pcomm));
367: }
368: PetscCallMPI(PetscFree(counter));
369: PetscFunctionReturn(MPI_SUCCESS);
370: }
372: /*
373: This is invoked on the outer comm as a result of either PetscCommDestroy() (via MPI_Comm_delete_attr) or when the user
374: calls MPI_Comm_free().
376: This is the only entry point for breaking the links between inner and outer comms.
378: This is called by MPI, not by users. This is called when MPI_Comm_free() is called on the communicator.
380: Note: this is declared extern "C" because it is passed to MPI_Comm_create_keyval()
382: */
383: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_InnerComm_Attr_DeleteFn(MPI_Comm comm, PetscMPIInt keyval, void *attr_val, void *extra_state)
384: {
385: union
386: {
387: MPI_Comm comm;
388: void *ptr;
389: } icomm;
391: PetscFunctionBegin;
392: if (keyval != Petsc_InnerComm_keyval) SETERRMPI(PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Unexpected keyval");
393: icomm.ptr = attr_val;
394: if (PetscDefined(USE_DEBUG)) {
395: /* Error out if the inner/outer comms are not correctly linked through their Outer/InnterComm attributes */
396: PetscMPIInt flg;
397: union
398: {
399: MPI_Comm comm;
400: void *ptr;
401: } ocomm;
402: PetscCallMPI(MPI_Comm_get_attr(icomm.comm, Petsc_OuterComm_keyval, &ocomm, &flg));
403: if (!flg) SETERRMPI(PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Inner comm does not have OuterComm attribute");
404: if (ocomm.comm != comm) SETERRMPI(PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Inner comm's OuterComm attribute does not point to outer PETSc comm");
405: }
406: PetscCallMPI(MPI_Comm_delete_attr(icomm.comm, Petsc_OuterComm_keyval));
407: PetscCallMPI(PetscInfo(NULL, "User MPI_Comm %ld is being unlinked from inner PETSc comm %ld\n", (long)comm, (long)icomm.comm));
408: PetscFunctionReturn(MPI_SUCCESS);
409: }
411: /*
412: * This is invoked on the inner comm when Petsc_InnerComm_Attr_DeleteFn calls MPI_Comm_delete_attr(). It should not be reached any other way.
413: */
414: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_OuterComm_Attr_DeleteFn(MPI_Comm comm, PetscMPIInt keyval, void *attr_val, void *extra_state)
415: {
416: PetscFunctionBegin;
417: PetscCallMPI(PetscInfo(NULL, "Removing reference to PETSc communicator embedded in a user MPI_Comm %ld\n", (long)comm));
418: PetscFunctionReturn(MPI_SUCCESS);
419: }
421: PETSC_EXTERN PetscMPIInt MPIAPI Petsc_ShmComm_Attr_DeleteFn(MPI_Comm, PetscMPIInt, void *, void *);
423: #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
424: PETSC_EXTERN PetscMPIInt PetscDataRep_extent_fn(MPI_Datatype, MPI_Aint *, void *);
425: PETSC_EXTERN PetscMPIInt PetscDataRep_read_conv_fn(void *, MPI_Datatype, PetscMPIInt, void *, MPI_Offset, void *);
426: PETSC_EXTERN PetscMPIInt PetscDataRep_write_conv_fn(void *, MPI_Datatype, PetscMPIInt, void *, MPI_Offset, void *);
427: #endif
429: PetscMPIInt PETSC_MPI_ERROR_CLASS = MPI_ERR_LASTCODE, PETSC_MPI_ERROR_CODE;
431: PETSC_INTERN int PetscGlobalArgc;
432: PETSC_INTERN char **PetscGlobalArgs;
433: int PetscGlobalArgc = 0;
434: char **PetscGlobalArgs = NULL;
435: PetscSegBuffer PetscCitationsList;
437: PetscErrorCode PetscCitationsInitialize(void)
438: {
439: PetscFunctionBegin;
440: PetscCall(PetscSegBufferCreate(1, 10000, &PetscCitationsList));
442: PetscCall(PetscCitationsRegister("@TechReport{petsc-user-ref,\n\
443: Author = {Satish Balay and Shrirang Abhyankar and Mark~F. Adams and Steven Benson and Jed Brown\n\
444: and Peter Brune and Kris Buschelman and Emil Constantinescu and Lisandro Dalcin and Alp Dener\n\
445: and Victor Eijkhout and Jacob Faibussowitsch and William~D. Gropp and V\'{a}clav Hapla and Tobin Isaac and Pierre Jolivet\n\
446: and Dmitry Karpeev and Dinesh Kaushik and Matthew~G. Knepley and Fande Kong and Scott Kruger\n\
447: and Dave~A. May and Lois Curfman McInnes and Richard Tran Mills and Lawrence Mitchell and Todd Munson\n\
448: and Jose~E. Roman and Karl Rupp and Patrick Sanan and Jason Sarich and Barry~F. Smith\n\
449: and Stefano Zampini and Hong Zhang and Hong Zhang and Junchao Zhang},\n\
450: Title = {{PETSc/TAO} Users Manual},\n\
451: Number = {ANL-21/39 - Revision 3.21},\n\
452: Doi = {10.2172/2205494},\n\
453: Institution = {Argonne National Laboratory},\n\
454: Year = {2024}\n}\n",
455: NULL));
457: PetscCall(PetscCitationsRegister("@InProceedings{petsc-efficient,\n\
458: Author = {Satish Balay and William D. Gropp and Lois Curfman McInnes and Barry F. Smith},\n\
459: Title = {Efficient Management of Parallelism in Object Oriented Numerical Software Libraries},\n\
460: Booktitle = {Modern Software Tools in Scientific Computing},\n\
461: Editor = {E. Arge and A. M. Bruaset and H. P. Langtangen},\n\
462: Pages = {163--202},\n\
463: Publisher = {Birkh{\\\"{a}}user Press},\n\
464: Year = {1997}\n}\n",
465: NULL));
466: PetscFunctionReturn(PETSC_SUCCESS);
467: }
469: static char programname[PETSC_MAX_PATH_LEN] = ""; /* HP includes entire path in name */
471: PetscErrorCode PetscSetProgramName(const char name[])
472: {
473: PetscFunctionBegin;
474: PetscCall(PetscStrncpy(programname, name, sizeof(programname)));
475: PetscFunctionReturn(PETSC_SUCCESS);
476: }
478: /*@C
479: PetscGetProgramName - Gets the name of the running program.
481: Not Collective
483: Input Parameter:
484: . len - length of the string name
486: Output Parameter:
487: . name - the name of the running program, provide a string of length `PETSC_MAX_PATH_LEN`
489: Level: advanced
491: .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArguments()`, `PetscInitialize()`
492: @*/
493: PetscErrorCode PetscGetProgramName(char name[], size_t len)
494: {
495: PetscFunctionBegin;
496: PetscCall(PetscStrncpy(name, programname, len));
497: PetscFunctionReturn(PETSC_SUCCESS);
498: }
500: /*@C
501: PetscGetArgs - Allows you to access the raw command line arguments anywhere
502: after PetscInitialize() is called but before `PetscFinalize()`.
504: Not Collective
506: Output Parameters:
507: + argc - count of number of command line arguments
508: - args - the command line arguments
510: Level: intermediate
512: Notes:
513: This is usually used to pass the command line arguments into other libraries
514: that are called internally deep in PETSc or the application.
516: The first argument contains the program name as is normal for C arguments.
518: .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArguments()`, `PetscInitialize()`
519: @*/
520: PetscErrorCode PetscGetArgs(int *argc, char ***args)
521: {
522: PetscFunctionBegin;
523: PetscCheck(PetscInitializeCalled || !PetscFinalizeCalled, PETSC_COMM_SELF, PETSC_ERR_ORDER, "You must call after PetscInitialize() but before PetscFinalize()");
524: *argc = PetscGlobalArgc;
525: *args = PetscGlobalArgs;
526: PetscFunctionReturn(PETSC_SUCCESS);
527: }
529: /*@C
530: PetscGetArguments - Allows you to access the command line arguments anywhere
531: after `PetscInitialize()` is called but before `PetscFinalize()`.
533: Not Collective
535: Output Parameter:
536: . args - the command line arguments
538: Level: intermediate
540: Note:
541: This does NOT start with the program name and IS `NULL` terminated (final arg is void)
543: .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscFreeArguments()`, `PetscInitialize()`
544: @*/
545: PetscErrorCode PetscGetArguments(char ***args)
546: {
547: PetscInt i, argc = PetscGlobalArgc;
549: PetscFunctionBegin;
550: PetscCheck(PetscInitializeCalled || !PetscFinalizeCalled, PETSC_COMM_SELF, PETSC_ERR_ORDER, "You must call after PetscInitialize() but before PetscFinalize()");
551: if (!argc) {
552: *args = NULL;
553: PetscFunctionReturn(PETSC_SUCCESS);
554: }
555: PetscCall(PetscMalloc1(argc, args));
556: for (i = 0; i < argc - 1; i++) PetscCall(PetscStrallocpy(PetscGlobalArgs[i + 1], &(*args)[i]));
557: (*args)[argc - 1] = NULL;
558: PetscFunctionReturn(PETSC_SUCCESS);
559: }
561: /*@C
562: PetscFreeArguments - Frees the memory obtained with `PetscGetArguments()`
564: Not Collective
566: Output Parameter:
567: . args - the command line arguments
569: Level: intermediate
571: .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscGetArguments()`
572: @*/
573: PetscErrorCode PetscFreeArguments(char **args)
574: {
575: PetscFunctionBegin;
576: if (args) {
577: PetscInt i = 0;
579: while (args[i]) PetscCall(PetscFree(args[i++]));
580: PetscCall(PetscFree(args));
581: }
582: PetscFunctionReturn(PETSC_SUCCESS);
583: }
585: #if PetscDefined(HAVE_SAWS)
586: #include <petscconfiginfo.h>
588: PETSC_INTERN PetscErrorCode PetscInitializeSAWs(const char help[])
589: {
590: PetscFunctionBegin;
591: if (!PetscGlobalRank) {
592: char cert[PETSC_MAX_PATH_LEN], root[PETSC_MAX_PATH_LEN], *intro, programname[64], *appline, *options, version[64];
593: int port;
594: PetscBool flg, rootlocal = PETSC_FALSE, flg2, selectport = PETSC_FALSE;
595: size_t applinelen, introlen;
596: char sawsurl[256];
598: PetscCall(PetscOptionsHasName(NULL, NULL, "-saws_log", &flg));
599: if (flg) {
600: char sawslog[PETSC_MAX_PATH_LEN];
602: PetscCall(PetscOptionsGetString(NULL, NULL, "-saws_log", sawslog, sizeof(sawslog), NULL));
603: if (sawslog[0]) {
604: PetscCallSAWs(SAWs_Set_Use_Logfile, (sawslog));
605: } else {
606: PetscCallSAWs(SAWs_Set_Use_Logfile, (NULL));
607: }
608: }
609: PetscCall(PetscOptionsGetString(NULL, NULL, "-saws_https", cert, sizeof(cert), &flg));
610: if (flg) PetscCallSAWs(SAWs_Set_Use_HTTPS, (cert));
611: PetscCall(PetscOptionsGetBool(NULL, NULL, "-saws_port_auto_select", &selectport, NULL));
612: if (selectport) {
613: PetscCallSAWs(SAWs_Get_Available_Port, (&port));
614: PetscCallSAWs(SAWs_Set_Port, (port));
615: } else {
616: PetscCall(PetscOptionsGetInt(NULL, NULL, "-saws_port", &port, &flg));
617: if (flg) PetscCallSAWs(SAWs_Set_Port, (port));
618: }
619: PetscCall(PetscOptionsGetString(NULL, NULL, "-saws_root", root, sizeof(root), &flg));
620: if (flg) {
621: PetscCallSAWs(SAWs_Set_Document_Root, (root));
622: PetscCall(PetscStrcmp(root, ".", &rootlocal));
623: } else {
624: PetscCall(PetscOptionsHasName(NULL, NULL, "-saws_options", &flg));
625: if (flg) {
626: PetscCall(PetscStrreplace(PETSC_COMM_WORLD, "${PETSC_DIR}/share/petsc/saws", root, sizeof(root)));
627: PetscCallSAWs(SAWs_Set_Document_Root, (root));
628: }
629: }
630: PetscCall(PetscOptionsHasName(NULL, NULL, "-saws_local", &flg2));
631: if (flg2) {
632: char jsdir[PETSC_MAX_PATH_LEN];
633: PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_SUP, "-saws_local option requires -saws_root option");
634: PetscCall(PetscSNPrintf(jsdir, sizeof(jsdir), "%s/js", root));
635: PetscCall(PetscTestDirectory(jsdir, 'r', &flg));
636: PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_FILE_READ, "-saws_local option requires js directory in root directory");
637: PetscCallSAWs(SAWs_Push_Local_Header, ());
638: }
639: PetscCall(PetscGetProgramName(programname, sizeof(programname)));
640: PetscCall(PetscStrlen(help, &applinelen));
641: introlen = 4096 + applinelen;
642: applinelen += 1024;
643: PetscCall(PetscMalloc(applinelen, &appline));
644: PetscCall(PetscMalloc(introlen, &intro));
646: if (rootlocal) {
647: PetscCall(PetscSNPrintf(appline, applinelen, "%s.c.html", programname));
648: PetscCall(PetscTestFile(appline, 'r', &rootlocal));
649: }
650: PetscCall(PetscOptionsGetAll(NULL, &options));
651: if (rootlocal && help) {
652: PetscCall(PetscSNPrintf(appline, applinelen, "<center> Running <a href=\"%s.c.html\">%s</a> %s</center><br><center><pre>%s</pre></center><br>\n", programname, programname, options, help));
653: } else if (help) {
654: PetscCall(PetscSNPrintf(appline, applinelen, "<center>Running %s %s</center><br><center><pre>%s</pre></center><br>", programname, options, help));
655: } else {
656: PetscCall(PetscSNPrintf(appline, applinelen, "<center> Running %s %s</center><br>\n", programname, options));
657: }
658: PetscCall(PetscFree(options));
659: PetscCall(PetscGetVersion(version, sizeof(version)));
660: PetscCall(PetscSNPrintf(intro, introlen,
661: "<body>\n"
662: "<center><h2> <a href=\"https://petsc.org/\">PETSc</a> Application Web server powered by <a href=\"https://bitbucket.org/saws/saws\">SAWs</a> </h2></center>\n"
663: "<center>This is the default PETSc application dashboard, from it you can access any published PETSc objects or logging data</center><br><center>%s configured with %s</center><br>\n"
664: "%s",
665: version, petscconfigureoptions, appline));
666: PetscCallSAWs(SAWs_Push_Body, ("index.html", 0, intro));
667: PetscCall(PetscFree(intro));
668: PetscCall(PetscFree(appline));
669: if (selectport) {
670: PetscBool silent;
672: /* another process may have grabbed the port so keep trying */
673: while (SAWs_Initialize()) {
674: PetscCallSAWs(SAWs_Get_Available_Port, (&port));
675: PetscCallSAWs(SAWs_Set_Port, (port));
676: }
678: PetscCall(PetscOptionsGetBool(NULL, NULL, "-saws_port_auto_select_silent", &silent, NULL));
679: if (!silent) {
680: PetscCallSAWs(SAWs_Get_FullURL, (sizeof(sawsurl), sawsurl));
681: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Point your browser to %s for SAWs\n", sawsurl));
682: }
683: } else {
684: PetscCallSAWs(SAWs_Initialize, ());
685: }
686: PetscCall(PetscCitationsRegister("@TechReport{ saws,\n"
687: " Author = {Matt Otten and Jed Brown and Barry Smith},\n"
688: " Title = {Scientific Application Web Server (SAWs) Users Manual},\n"
689: " Institution = {Argonne National Laboratory},\n"
690: " Year = 2013\n}\n",
691: NULL));
692: }
693: PetscFunctionReturn(PETSC_SUCCESS);
694: }
695: #endif
697: /* Things must be done before MPI_Init() when MPI is not yet initialized, and can be shared between C init and Fortran init */
698: PETSC_INTERN PetscErrorCode PetscPreMPIInit_Private(void)
699: {
700: PetscFunctionBegin;
701: #if defined(PETSC_HAVE_HWLOC_SOLARIS_BUG)
702: /* see MPI.py for details on this bug */
703: (void)setenv("HWLOC_COMPONENTS", "-x86", 1);
704: #endif
705: PetscFunctionReturn(PETSC_SUCCESS);
706: }
708: #if PetscDefined(HAVE_ADIOS)
709: #include <adios.h>
710: #include <adios_read.h>
711: int64_t Petsc_adios_group;
712: #endif
713: #if PetscDefined(HAVE_OPENMP)
714: #include <omp.h>
715: PetscInt PetscNumOMPThreads;
716: #endif
718: #include <petsc/private/deviceimpl.h>
719: #if PetscDefined(HAVE_CUDA)
720: #include <petscdevice_cuda.h>
721: // REMOVE ME
722: cudaStream_t PetscDefaultCudaStream = NULL;
723: #endif
724: #if PetscDefined(HAVE_HIP)
725: #include <petscdevice_hip.h>
726: // REMOVE ME
727: hipStream_t PetscDefaultHipStream = NULL;
728: #endif
730: #if PetscDefined(HAVE_DLFCN_H)
731: #include <dlfcn.h>
732: #endif
733: PETSC_INTERN PetscErrorCode PetscLogInitialize(void);
734: #if PetscDefined(HAVE_VIENNACL)
735: PETSC_EXTERN PetscErrorCode PetscViennaCLInit(void);
736: PetscBool PetscViennaCLSynchronize = PETSC_FALSE;
737: #endif
739: PetscBool PetscCIEnabled = PETSC_FALSE, PetscCIEnabledPortableErrorOutput = PETSC_FALSE;
741: /*
742: PetscInitialize_Common - shared code between C and Fortran initialization
744: prog: program name
745: file: optional PETSc database file name. Might be in Fortran string format when 'ftn' is true
746: help: program help message
747: ftn: is it called from Fortran initialization (petscinitializef_)?
748: readarguments,len: used when fortran is true
749: */
750: PETSC_INTERN PetscErrorCode PetscInitialize_Common(const char *prog, const char *file, const char *help, PetscBool ftn, PetscBool readarguments, PetscInt len)
751: {
752: PetscMPIInt size;
753: PetscBool flg = PETSC_TRUE;
754: char hostname[256];
755: PetscBool blas_view_flag = PETSC_FALSE;
757: PetscFunctionBegin;
758: if (PetscInitializeCalled) PetscFunctionReturn(PETSC_SUCCESS);
759: /* these must be initialized in a routine, not as a constant declaration */
760: PETSC_STDOUT = stdout;
761: PETSC_STDERR = stderr;
763: /* PetscCall can be used from now */
764: PetscErrorHandlingInitialized = PETSC_TRUE;
766: /*
767: The checking over compatible runtime libraries is complicated by the MPI ABI initiative
768: https://wiki.mpich.org/mpich/index.php/ABI_Compatibility_Initiative which started with
769: MPICH v3.1 (Released February 2014)
770: IBM MPI v2.1 (December 2014)
771: Intel MPI Library v5.0 (2014)
772: Cray MPT v7.0.0 (June 2014)
773: As of July 31, 2017 the ABI number still appears to be 12, that is all of the versions
774: listed above and since that time are compatible.
776: Unfortunately the MPI ABI initiative has not defined a way to determine the ABI number
777: at compile time or runtime. Thus we will need to systematically track the allowed versions
778: and how they are represented in the mpi.h and MPI_Get_library_version() output in order
779: to perform the checking.
781: Currently we only check for pre MPI ABI versions (and packages that do not follow the MPI ABI).
783: Questions:
785: Should the checks for ABI incompatibility be only on the major version number below?
786: Presumably the output to stderr will be removed before a release.
787: */
789: #if defined(PETSC_HAVE_MPI_GET_LIBRARY_VERSION)
790: {
791: char mpilibraryversion[MPI_MAX_LIBRARY_VERSION_STRING];
792: PetscMPIInt mpilibraryversionlength;
794: PetscCallMPI(MPI_Get_library_version(mpilibraryversion, &mpilibraryversionlength));
795: /* check for MPICH versions before MPI ABI initiative */
796: #if defined(MPICH_VERSION)
797: #if MPICH_NUMVERSION < 30100000
798: {
799: char *ver, *lf;
800: PetscBool flg = PETSC_FALSE;
802: PetscCall(PetscStrstr(mpilibraryversion, "MPICH Version:", &ver));
803: if (ver) {
804: PetscCall(PetscStrchr(ver, '\n', &lf));
805: if (lf) {
806: *lf = 0;
807: PetscCall(PetscStrendswith(ver, MPICH_VERSION, &flg));
808: }
809: }
810: if (!flg) {
811: PetscCall(PetscInfo(NULL, "PETSc warning --- MPICH library version \n%s does not match what PETSc was compiled with %s.\n", mpilibraryversion, MPICH_VERSION));
812: flg = PETSC_TRUE;
813: }
814: }
815: #endif
816: /* check for Open MPI version, it is not part of the MPI ABI initiative (is it part of another initiative that needs to be handled?) */
817: #elif defined(OMPI_MAJOR_VERSION)
818: {
819: char *ver, bs[MPI_MAX_LIBRARY_VERSION_STRING], *bsf;
820: PetscBool flg = PETSC_FALSE;
821: #define PSTRSZ 2
822: char ompistr1[PSTRSZ][MPI_MAX_LIBRARY_VERSION_STRING] = {"Open MPI", "FUJITSU MPI"};
823: char ompistr2[PSTRSZ][MPI_MAX_LIBRARY_VERSION_STRING] = {"v", "Library "};
824: int i;
825: for (i = 0; i < PSTRSZ; i++) {
826: PetscCall(PetscStrstr(mpilibraryversion, ompistr1[i], &ver));
827: if (ver) {
828: PetscCall(PetscSNPrintf(bs, MPI_MAX_LIBRARY_VERSION_STRING, "%s%d.%d", ompistr2[i], OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION));
829: PetscCall(PetscStrstr(ver, bs, &bsf));
830: if (bsf) flg = PETSC_TRUE;
831: break;
832: }
833: }
834: if (!flg) {
835: PetscCall(PetscInfo(NULL, "PETSc warning --- Open MPI library version \n%s does not match what PETSc was compiled with %d.%d.\n", mpilibraryversion, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION));
836: flg = PETSC_TRUE;
837: }
838: }
839: #endif
840: }
841: #endif
843: #if defined(PETSC_HAVE_DLADDR) && !(defined(__cray__) && defined(__clang__))
844: /* These symbols are currently in the Open MPI and MPICH libraries; they may not always be, in that case the test will simply not detect the problem */
845: PetscCheck(!dlsym(RTLD_DEFAULT, "ompi_mpi_init") || !dlsym(RTLD_DEFAULT, "MPID_Abort"), PETSC_COMM_SELF, PETSC_ERR_MPI_LIB_INCOMP, "Application was linked against both Open MPI and MPICH based MPI libraries and will not run correctly");
846: #endif
848: /* on Windows - set printf to default to printing 2 digit exponents */
849: #if defined(PETSC_HAVE__SET_OUTPUT_FORMAT)
850: _set_output_format(_TWO_DIGIT_EXPONENT);
851: #endif
853: PetscCall(PetscOptionsCreateDefault());
855: PetscFinalizeCalled = PETSC_FALSE;
857: PetscCall(PetscSetProgramName(prog));
858: PetscCall(PetscSpinlockCreate(&PetscViewerASCIISpinLockOpen));
859: PetscCall(PetscSpinlockCreate(&PetscViewerASCIISpinLockStdout));
860: PetscCall(PetscSpinlockCreate(&PetscViewerASCIISpinLockStderr));
861: PetscCall(PetscSpinlockCreate(&PetscCommSpinLock));
863: if (PETSC_COMM_WORLD == MPI_COMM_NULL) PETSC_COMM_WORLD = MPI_COMM_WORLD;
864: PetscCallMPI(MPI_Comm_set_errhandler(PETSC_COMM_WORLD, MPI_ERRORS_RETURN));
866: if (PETSC_MPI_ERROR_CLASS == MPI_ERR_LASTCODE) {
867: PetscCallMPI(MPI_Add_error_class(&PETSC_MPI_ERROR_CLASS));
868: PetscCallMPI(MPI_Add_error_code(PETSC_MPI_ERROR_CLASS, &PETSC_MPI_ERROR_CODE));
869: }
871: /* Done after init due to a bug in MPICH-GM? */
872: PetscCall(PetscErrorPrintfInitialize());
874: PetscCallMPI(MPI_Comm_rank(MPI_COMM_WORLD, &PetscGlobalRank));
875: PetscCallMPI(MPI_Comm_size(MPI_COMM_WORLD, &PetscGlobalSize));
877: MPIU_BOOL = MPI_INT;
878: MPIU_ENUM = MPI_INT;
879: MPIU_FORTRANADDR = (sizeof(void *) == sizeof(int)) ? MPI_INT : MPIU_INT64;
880: if (sizeof(size_t) == sizeof(unsigned)) MPIU_SIZE_T = MPI_UNSIGNED;
881: else if (sizeof(size_t) == sizeof(unsigned long)) MPIU_SIZE_T = MPI_UNSIGNED_LONG;
882: #if defined(PETSC_SIZEOF_LONG_LONG)
883: else if (sizeof(size_t) == sizeof(unsigned long long)) MPIU_SIZE_T = MPI_UNSIGNED_LONG_LONG;
884: #endif
885: else SETERRQ(PETSC_COMM_WORLD, PETSC_ERR_SUP_SYS, "Could not find MPI type for size_t");
887: /*
888: Initialized the global complex variable; this is because with
889: shared libraries the constructors for global variables
890: are not called; at least on IRIX.
891: */
892: #if defined(PETSC_HAVE_COMPLEX)
893: {
894: #if defined(PETSC_CLANGUAGE_CXX) && !defined(PETSC_USE_REAL___FLOAT128)
895: PetscComplex ic(0.0, 1.0);
896: PETSC_i = ic;
897: #else
898: PETSC_i = _Complex_I;
899: #endif
900: }
901: #endif /* PETSC_HAVE_COMPLEX */
903: /*
904: Create the PETSc MPI reduction operator that sums of the first
905: half of the entries and maxes the second half.
906: */
907: PetscCallMPI(MPI_Op_create(MPIU_MaxSum_Local, 1, &MPIU_MAXSUM_OP));
909: #if defined(PETSC_HAVE_REAL___FLOAT128)
910: PetscCallMPI(MPI_Type_contiguous(2, MPI_DOUBLE, &MPIU___FLOAT128));
911: PetscCallMPI(MPI_Type_commit(&MPIU___FLOAT128));
912: #if defined(PETSC_HAVE_COMPLEX)
913: PetscCallMPI(MPI_Type_contiguous(4, MPI_DOUBLE, &MPIU___COMPLEX128));
914: PetscCallMPI(MPI_Type_commit(&MPIU___COMPLEX128));
915: #endif
916: #endif
917: #if defined(PETSC_HAVE_REAL___FP16)
918: PetscCallMPI(MPI_Type_contiguous(2, MPI_CHAR, &MPIU___FP16));
919: PetscCallMPI(MPI_Type_commit(&MPIU___FP16));
920: #endif
922: #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
923: PetscCallMPI(MPI_Op_create(PetscSum_Local, 1, &MPIU_SUM));
924: PetscCallMPI(MPI_Op_create(PetscMax_Local, 1, &MPIU_MAX));
925: PetscCallMPI(MPI_Op_create(PetscMin_Local, 1, &MPIU_MIN));
926: #elif defined(PETSC_HAVE_REAL___FLOAT128) || defined(PETSC_HAVE_REAL___FP16)
927: PetscCallMPI(MPI_Op_create(PetscSum_Local, 1, &MPIU_SUM___FP16___FLOAT128));
928: #endif
930: PetscCallMPI(MPI_Type_contiguous(2, MPIU_SCALAR, &MPIU_2SCALAR));
931: PetscCallMPI(MPI_Op_create(PetscGarbageKeySortedIntersect, 1, &Petsc_Garbage_SetIntersectOp));
932: PetscCallMPI(MPI_Type_commit(&MPIU_2SCALAR));
934: /* create datatypes used by MPIU_MAXLOC, MPIU_MINLOC and PetscSplitReduction_Op */
935: #if !defined(PETSC_HAVE_MPIUNI)
936: {
937: PetscMPIInt blockSizes[2] = {1, 1};
938: MPI_Aint blockOffsets[2] = {offsetof(struct petsc_mpiu_real_int, v), offsetof(struct petsc_mpiu_real_int, i)};
939: MPI_Datatype blockTypes[2] = {MPIU_REAL, MPIU_INT}, tmpStruct;
941: PetscCallMPI(MPI_Type_create_struct(2, blockSizes, blockOffsets, blockTypes, &tmpStruct));
942: PetscCallMPI(MPI_Type_create_resized(tmpStruct, 0, sizeof(struct petsc_mpiu_real_int), &MPIU_REAL_INT));
943: PetscCallMPI(MPI_Type_free(&tmpStruct));
944: PetscCallMPI(MPI_Type_commit(&MPIU_REAL_INT));
945: }
946: {
947: PetscMPIInt blockSizes[2] = {1, 1};
948: MPI_Aint blockOffsets[2] = {offsetof(struct petsc_mpiu_scalar_int, v), offsetof(struct petsc_mpiu_scalar_int, i)};
949: MPI_Datatype blockTypes[2] = {MPIU_SCALAR, MPIU_INT}, tmpStruct;
951: PetscCallMPI(MPI_Type_create_struct(2, blockSizes, blockOffsets, blockTypes, &tmpStruct));
952: PetscCallMPI(MPI_Type_create_resized(tmpStruct, 0, sizeof(struct petsc_mpiu_scalar_int), &MPIU_SCALAR_INT));
953: PetscCallMPI(MPI_Type_free(&tmpStruct));
954: PetscCallMPI(MPI_Type_commit(&MPIU_SCALAR_INT));
955: }
956: #endif
958: #if defined(PETSC_USE_64BIT_INDICES)
959: PetscCallMPI(MPI_Type_contiguous(2, MPIU_INT, &MPIU_2INT));
960: PetscCallMPI(MPI_Type_commit(&MPIU_2INT));
961: #endif
962: PetscCallMPI(MPI_Type_contiguous(4, MPI_INT, &MPI_4INT));
963: PetscCallMPI(MPI_Type_commit(&MPI_4INT));
964: PetscCallMPI(MPI_Type_contiguous(4, MPIU_INT, &MPIU_4INT));
965: PetscCallMPI(MPI_Type_commit(&MPIU_4INT));
967: /*
968: Attributes to be set on PETSc communicators
969: */
970: PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_Counter_Attr_DeleteFn, &Petsc_Counter_keyval, (void *)0));
971: PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_InnerComm_Attr_DeleteFn, &Petsc_InnerComm_keyval, (void *)0));
972: PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_OuterComm_Attr_DeleteFn, &Petsc_OuterComm_keyval, (void *)0));
973: PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, Petsc_ShmComm_Attr_DeleteFn, &Petsc_ShmComm_keyval, (void *)0));
974: PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, MPI_COMM_NULL_DELETE_FN, &Petsc_CreationIdx_keyval, (void *)0));
975: PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, MPI_COMM_NULL_DELETE_FN, &Petsc_Garbage_HMap_keyval, (void *)0));
977: #if defined(PETSC_USE_FORTRAN_BINDINGS)
978: if (ftn) PetscCall(PetscInitFortran_Private(readarguments, file, len));
979: else
980: #endif
981: PetscCall(PetscOptionsInsert(NULL, &PetscGlobalArgc, &PetscGlobalArgs, file));
983: /* call a second time so it can look in the options database */
984: PetscCall(PetscErrorPrintfInitialize());
986: /*
987: Check system options and print help
988: */
989: PetscCall(PetscOptionsCheckInitial_Private(help));
991: /*
992: Creates the logging data structures; this is enabled even if logging is not turned on
993: This is the last thing we do before returning to the user code to prevent having the
994: logging numbers contaminated by any startup time associated with MPI
995: */
996: PetscCall(PetscLogInitialize());
998: /*
999: Initialize PetscDevice and PetscDeviceContext
1001: Note to any future devs thinking of moving this, proper initialization requires:
1002: 1. MPI initialized
1003: 2. Options DB initialized
1004: 3. Petsc error handling initialized, specifically signal handlers. This expects to set up
1005: its own SIGSEV handler via the push/pop interface.
1006: 4. Logging initialized
1007: */
1008: PetscCall(PetscDeviceInitializeFromOptions_Internal(PETSC_COMM_WORLD));
1010: #if PetscDefined(HAVE_VIENNACL)
1011: flg = PETSC_FALSE;
1012: PetscCall(PetscOptionsHasName(NULL, NULL, "-log_view", &flg));
1013: if (!flg) PetscCall(PetscOptionsGetBool(NULL, NULL, "-viennacl_synchronize", &flg, NULL));
1014: PetscViennaCLSynchronize = flg;
1015: PetscCall(PetscViennaCLInit());
1016: #endif
1018: PetscCall(PetscCitationsInitialize());
1020: #if defined(PETSC_HAVE_SAWS)
1021: PetscCall(PetscInitializeSAWs(ftn ? NULL : help));
1022: flg = PETSC_FALSE;
1023: PetscCall(PetscOptionsHasName(NULL, NULL, "-stack_view", &flg));
1024: if (flg) PetscCall(PetscStackViewSAWs());
1025: #endif
1027: /*
1028: Load the dynamic libraries (on machines that support them), this registers all
1029: the solvers etc. (On non-dynamic machines this initializes the PetscDraw and PetscViewer classes)
1030: */
1031: PetscCall(PetscInitialize_DynamicLibraries());
1033: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
1034: PetscCall(PetscInfo(NULL, "PETSc successfully started: number of processors = %d\n", size));
1035: PetscCall(PetscGetHostName(hostname, sizeof(hostname)));
1036: PetscCall(PetscInfo(NULL, "Running on machine: %s\n", hostname));
1037: #if defined(PETSC_HAVE_OPENMP)
1038: {
1039: PetscBool omp_view_flag;
1040: char *threads = getenv("OMP_NUM_THREADS");
1042: if (threads) {
1043: PetscCall(PetscInfo(NULL, "Number of OpenMP threads %s (as given by OMP_NUM_THREADS)\n", threads));
1044: (void)sscanf(threads, "%" PetscInt_FMT, &PetscNumOMPThreads);
1045: } else {
1046: PetscNumOMPThreads = (PetscInt)omp_get_max_threads();
1047: PetscCall(PetscInfo(NULL, "Number of OpenMP threads %" PetscInt_FMT " (as given by omp_get_max_threads())\n", PetscNumOMPThreads));
1048: }
1049: PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "OpenMP options", "Sys");
1050: PetscCall(PetscOptionsInt("-omp_num_threads", "Number of OpenMP threads to use (can also use environmental variable OMP_NUM_THREADS", "None", PetscNumOMPThreads, &PetscNumOMPThreads, &flg));
1051: PetscCall(PetscOptionsName("-omp_view", "Display OpenMP number of threads", NULL, &omp_view_flag));
1052: PetscOptionsEnd();
1053: if (flg) {
1054: PetscCall(PetscInfo(NULL, "Number of OpenMP threads %" PetscInt_FMT " (given by -omp_num_threads)\n", PetscNumOMPThreads));
1055: omp_set_num_threads((int)PetscNumOMPThreads);
1056: }
1057: if (omp_view_flag) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "OpenMP: number of threads %" PetscInt_FMT "\n", PetscNumOMPThreads));
1058: }
1059: #endif
1061: PetscOptionsBegin(PETSC_COMM_WORLD, NULL, "BLAS options", "Sys");
1062: PetscCall(PetscOptionsName("-blas_view", "Display number of threads to use for BLAS operations", NULL, &blas_view_flag));
1063: #if defined(PETSC_HAVE_BLI_THREAD_SET_NUM_THREADS) || defined(PETSC_HAVE_MKL_SET_NUM_THREADS) || defined(PETSC_HAVE_OPENBLAS_SET_NUM_THREADS)
1064: {
1065: char *threads = NULL;
1067: /* determine any default number of threads requested in the environment; TODO: Apple libraries? */
1068: #if defined(PETSC_HAVE_BLI_THREAD_SET_NUM_THREADS)
1069: threads = getenv("BLIS_NUM_THREADS");
1070: if (threads) PetscCall(PetscInfo(NULL, "BLAS: Environment number of BLIS threads %s given by BLIS_NUM_THREADS\n", threads));
1071: if (!threads) {
1072: threads = getenv("OMP_NUM_THREADS");
1073: if (threads) PetscCall(PetscInfo(NULL, "BLAS: Environment number of BLIS threads %s given by OMP_NUM_THREADS\n", threads));
1074: }
1075: #elif defined(PETSC_HAVE_MKL_SET_NUM_THREADS)
1076: threads = getenv("MKL_NUM_THREADS");
1077: if (threads) PetscCall(PetscInfo(NULL, "BLAS: Environment number of MKL threads %s given by MKL_NUM_THREADS\n", threads));
1078: #elif defined(PETSC_HAVE_OPENBLAS_SET_NUM_THREADS)
1079: threads = getenv("OPENBLAS_NUM_THREADS");
1080: if (threads) PetscCall(PetscInfo(NULL, "BLAS: Environment number of OpenBLAS threads %s given by OPENBLAS_NUM_THREADS\n", threads));
1081: if (!threads) {
1082: threads = getenv("OMP_NUM_THREADS");
1083: if (threads) PetscCall(PetscInfo(NULL, "BLAS: Environment number of OpenBLAS threads %s given by OMP_NUM_THREADS\n", threads));
1084: }
1085: #endif
1086: if (threads) (void)sscanf(threads, "%" PetscInt_FMT, &PetscNumBLASThreads);
1087: PetscCall(PetscOptionsInt("-blas_num_threads", "Number of threads to use for BLAS operations", "None", PetscNumBLASThreads, &PetscNumBLASThreads, &flg));
1088: PetscCall(PetscBLASSetNumThreads(PetscNumBLASThreads));
1089: if (blas_view_flag) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "BLAS: number of threads %" PetscInt_FMT "\n", PetscNumBLASThreads));
1090: }
1091: #elif defined(PETSC_HAVE_APPLE_ACCELERATE)
1092: PetscCall(PetscInfo(NULL, "BLAS: Apple Accelerate library, thread support with no user control\n"));
1093: if (blas_view_flag) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "BLAS: Apple Accelerate library, thread support with no user control\n"));
1094: #else
1095: if (blas_view_flag) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "BLAS: no thread support\n"));
1096: #endif
1097: PetscOptionsEnd();
1099: #if defined(PETSC_USE_PETSC_MPI_EXTERNAL32)
1100: /*
1101: Tell MPI about our own data representation converter, this would/should be used if extern32 is not supported by the MPI
1103: Currently not used because it is not supported by MPICH.
1104: */
1105: if (!PetscBinaryBigEndian()) PetscCallMPI(MPI_Register_datarep((char *)"petsc", PetscDataRep_read_conv_fn, PetscDataRep_write_conv_fn, PetscDataRep_extent_fn, NULL));
1106: #endif
1108: #if defined(PETSC_SERIALIZE_FUNCTIONS)
1109: PetscCall(PetscFPTCreate(10000));
1110: #endif
1112: #if defined(PETSC_HAVE_HWLOC)
1113: {
1114: PetscViewer viewer;
1115: PetscCall(PetscOptionsGetViewer(PETSC_COMM_WORLD, NULL, NULL, "-process_view", &viewer, NULL, &flg));
1116: if (flg) {
1117: PetscCall(PetscProcessPlacementView(viewer));
1118: PetscCall(PetscOptionsRestoreViewer(&viewer));
1119: }
1120: }
1121: #endif
1123: flg = PETSC_TRUE;
1124: PetscCall(PetscOptionsGetBool(NULL, NULL, "-viewfromoptions", &flg, NULL));
1125: if (!flg) PetscCall(PetscOptionsPushGetViewerOff(PETSC_TRUE));
1127: #if defined(PETSC_HAVE_ADIOS)
1128: PetscCallExternal(adios_init_noxml, PETSC_COMM_WORLD);
1129: PetscCallExternal(adios_declare_group, &Petsc_adios_group, "PETSc", "", adios_stat_default);
1130: PetscCallExternal(adios_select_method, Petsc_adios_group, "MPI", "", "");
1131: PetscCallExternal(adios_read_init_method, ADIOS_READ_METHOD_BP, PETSC_COMM_WORLD, "");
1132: #endif
1134: #if defined(__VALGRIND_H)
1135: PETSC_RUNNING_ON_VALGRIND = RUNNING_ON_VALGRIND ? PETSC_TRUE : PETSC_FALSE;
1136: #if defined(PETSC_USING_DARWIN) && defined(PETSC_BLASLAPACK_SDOT_RETURNS_DOUBLE)
1137: if (PETSC_RUNNING_ON_VALGRIND) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "WARNING: Running valgrind with the macOS native BLAS and LAPACK can fail. If it fails, try configuring with --download-fblaslapack or --download-f2cblaslapack"));
1138: #endif
1139: #endif
1140: /*
1141: Set flag that we are completely initialized
1142: */
1143: PetscInitializeCalled = PETSC_TRUE;
1145: PetscCall(PetscOptionsHasName(NULL, NULL, "-python", &flg));
1146: if (flg) PetscCall(PetscPythonInitialize(NULL, NULL));
1148: PetscCall(PetscOptionsHasName(NULL, NULL, "-mpi_linear_solver_server", &flg));
1149: if (PetscDefined(USE_SINGLE_LIBRARY) && flg) PetscCall(PCMPIServerBegin());
1150: else PetscCheck(!flg, PETSC_COMM_WORLD, PETSC_ERR_SUP, "PETSc configured using -with-single-library=0; -mpi_linear_solver_server not supported in that case");
1151: PetscFunctionReturn(PETSC_SUCCESS);
1152: }
1154: // "Unknown section 'Environmental Variables'"
1155: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
1156: /*@C
1157: PetscInitialize - Initializes the PETSc database and MPI.
1158: `PetscInitialize()` calls MPI_Init() if that has yet to be called,
1159: so this routine should always be called near the beginning of
1160: your program -- usually the very first line!
1162: Collective on `MPI_COMM_WORLD` or `PETSC_COMM_WORLD` if it has been set
1164: Input Parameters:
1165: + argc - count of number of command line arguments
1166: . args - the command line arguments
1167: . file - [optional] PETSc database file, append ":yaml" to filename to specify YAML options format.
1168: Use NULL or empty string to not check for code specific file.
1169: Also checks ~/.petscrc, .petscrc and petscrc.
1170: Use -skip_petscrc in the code specific file (or command line) to skip ~/.petscrc, .petscrc and petscrc files.
1171: - help - [optional] Help message to print, use NULL for no message
1173: If you wish PETSc code to run ONLY on a subcommunicator of `MPI_COMM_WORLD`, create that
1174: communicator first and assign it to `PETSC_COMM_WORLD` BEFORE calling `PetscInitialize()`. Thus if you are running a
1175: four process job and two processes will run PETSc and have `PetscInitialize()` and PetscFinalize() and two process will not,
1176: then do this. If ALL processes in the job are using `PetscInitialize()` and `PetscFinalize()` then you don't need to do this, even
1177: if different subcommunicators of the job are doing different things with PETSc.
1179: Options Database Keys:
1180: + -help [intro] - prints help method for each option; if intro is given the program stops after printing the introductory help message
1181: . -start_in_debugger [noxterm,dbx,xdb,gdb,...] - Starts program in debugger
1182: . -on_error_attach_debugger [noxterm,dbx,xdb,gdb,...] - Starts debugger when error detected
1183: . -on_error_emacs <machinename> - causes emacsclient to jump to error file
1184: . -on_error_abort - calls `abort()` when error detected (no traceback)
1185: . -on_error_mpiabort - calls `MPI_abort()` when error detected
1186: . -error_output_stdout - prints PETSc error messages to stdout instead of the default stderr
1187: . -error_output_none - does not print the error messages (but handles errors in the same way as if this was not called)
1188: . -debugger_ranks [rank1,rank2,...] - Indicates ranks to start in debugger
1189: . -debugger_pause [sleeptime] (in seconds) - Pauses debugger
1190: . -stop_for_debugger - Print message on how to attach debugger manually to
1191: process and wait (-debugger_pause) seconds for attachment
1192: . -malloc_dump - prints a list of all unfreed memory at the end of the run
1193: . -malloc_test - like -malloc_dump -malloc_debug, but only active for debugging builds, ignored in optimized build. May want to set in PETSC_OPTIONS environmental variable
1194: . -malloc_view - show a list of all allocated memory during `PetscFinalize()`
1195: . -malloc_view_threshold <t> - only list memory allocations of size greater than t with -malloc_view
1196: . -malloc_requested_size - malloc logging will record the requested size rather than size after alignment
1197: . -fp_trap - Stops on floating point exceptions
1198: . -no_signal_handler - Indicates not to trap error signals
1199: . -shared_tmp - indicates /tmp directory is shared by all processors
1200: . -not_shared_tmp - each processor has own /tmp
1201: . -tmp - alternative name of /tmp directory
1202: . -get_total_flops - returns total flops done by all processors
1203: - -memory_view - Print memory usage at end of run
1205: Options Database Keys for Option Database:
1206: + -skip_petscrc - skip the default option files ~/.petscrc, .petscrc, petscrc
1207: . -options_monitor - monitor all set options to standard output for the whole program run
1208: - -options_monitor_cancel - cancel options monitoring hard-wired using `PetscOptionsMonitorSet()`
1210: Options -options_monitor_{all,cancel} are
1211: position-independent and apply to all options set since the PETSc start.
1212: They can be used also in option files.
1214: See `PetscOptionsMonitorSet()` to do monitoring programmatically.
1216: Options Database Keys for Profiling:
1217: See Users-Manual: ch_profiling for details.
1218: + -info [filename][:[~]<list,of,classnames>[:[~]self]] - Prints verbose information. See `PetscInfo()`.
1219: . -log_sync - Enable barrier synchronization for all events. This option is useful to debug imbalance within each event,
1220: however it slows things down and gives a distorted view of the overall runtime.
1221: . -log_trace [filename] - Print traces of all PETSc calls to the screen (useful to determine where a program
1222: hangs without running in the debugger). See `PetscLogTraceBegin()`.
1223: . -log_view [:filename:format][,[:filename:format]...] - Prints summary of flop and timing information to screen or file, see `PetscLogView()` (up to 4 viewers)
1224: . -log_view_memory - Includes in the summary from -log_view the memory used in each event, see `PetscLogView()`.
1225: . -log_view_gpu_time - Includes in the summary from -log_view the time used in each GPU kernel, see `PetscLogView().
1226: . -log_exclude: <vec,mat,pc,ksp,snes> - excludes subset of object classes from logging
1227: . -log [filename] - Logs profiling information in a dump file, see `PetscLogDump()`.
1228: . -log_all [filename] - Same as `-log`.
1229: . -log_mpe [filename] - Creates a logfile viewable by the utility Jumpshot (in MPICH distribution)
1230: . -log_perfstubs - Starts a log handler with the perfstubs interface (which is used by TAU)
1231: . -log_nvtx - Starts an nvtx log handler for use with Nsight
1232: . -viewfromoptions on,off - Enable or disable `XXXSetFromOptions()` calls, for applications with many small solves turn this off
1233: - -check_pointer_intensity 0,1,2 - if pointers are checked for validity (debug version only), using 0 will result in faster code
1235: Options Database Keys for SAWs:
1236: + -saws_port <portnumber> - port number to publish SAWs data, default is 8080
1237: . -saws_port_auto_select - have SAWs select a new unique port number where it publishes the data, the URL is printed to the screen
1238: this is useful when you are running many jobs that utilize SAWs at the same time
1239: . -saws_log <filename> - save a log of all SAWs communication
1240: . -saws_https <certificate file> - have SAWs use HTTPS instead of HTTP
1241: - -saws_root <directory> - allow SAWs to have access to the given directory to search for requested resources and files
1243: Environmental Variables:
1244: + `PETSC_TMP` - alternative tmp directory
1245: . `PETSC_SHARED_TMP` - tmp is shared by all processes
1246: . `PETSC_NOT_SHARED_TMP` - each process has its own private tmp
1247: . `PETSC_OPTIONS` - a string containing additional options for petsc in the form of command line "-key value" pairs
1248: . `PETSC_OPTIONS_YAML` - (requires configuring PETSc to use libyaml) a string containing additional options for petsc in the form of a YAML document
1249: . `PETSC_VIEWER_SOCKET_PORT` - socket number to use for socket viewer
1250: - `PETSC_VIEWER_SOCKET_MACHINE` - machine to use for socket viewer to connect to
1252: Level: beginner
1254: Note:
1255: If for some reason you must call `MPI_Init()` separately, call
1256: it before `PetscInitialize()`.
1258: Fortran Notes:
1259: In Fortran this routine can be called with
1260: .vb
1261: call PetscInitialize(ierr)
1262: call PetscInitialize(file,ierr) or
1263: call PetscInitialize(file,help,ierr)
1264: .ve
1266: If your main program is C but you call Fortran code that also uses PETSc you need to call `PetscInitializeFortran()` soon after
1267: calling `PetscInitialize()`.
1269: Options Database Key for Developers:
1270: . -checkfunctionlist - automatically checks that function lists associated with objects are correctly cleaned up. Produces messages of the form:
1271: "function name: MatInodeGetInodeSizes_C" if they are not cleaned up. This flag is always set for the test harness (in framework.py)
1273: .seealso: `PetscFinalize()`, `PetscInitializeFortran()`, `PetscGetArgs()`, `PetscInitializeNoArguments()`, `PetscLogGpuTime()`
1274: @*/
1275: PetscErrorCode PetscInitialize(int *argc, char ***args, const char file[], const char help[])
1276: {
1277: PetscMPIInt flag;
1278: const char *prog = "Unknown Name", *mpienv;
1280: PetscFunctionBegin;
1281: if (PetscInitializeCalled) PetscFunctionReturn(PETSC_SUCCESS);
1282: PetscCallMPI(MPI_Initialized(&flag));
1283: if (!flag) {
1284: PetscCheck(PETSC_COMM_WORLD == MPI_COMM_NULL, PETSC_COMM_SELF, PETSC_ERR_SUP, "You cannot set PETSC_COMM_WORLD if you have not initialized MPI first");
1285: PetscCall(PetscPreMPIInit_Private());
1286: #if defined(PETSC_HAVE_MPI_INIT_THREAD)
1287: {
1288: PetscMPIInt provided;
1289: PetscCallMPI(MPI_Init_thread(argc, args, PETSC_MPI_THREAD_REQUIRED == PETSC_DECIDE ? MPI_THREAD_FUNNELED : PETSC_MPI_THREAD_REQUIRED, &provided));
1290: PetscCheck(PETSC_MPI_THREAD_REQUIRED == PETSC_DECIDE || provided >= PETSC_MPI_THREAD_REQUIRED, PETSC_COMM_SELF, PETSC_ERR_MPI, "The MPI implementation's provided thread level is less than what you required");
1291: if (PETSC_MPI_THREAD_REQUIRED == PETSC_DECIDE) PETSC_MPI_THREAD_REQUIRED = MPI_THREAD_FUNNELED; // assign it a valid value after check-up
1292: }
1293: #else
1294: PetscCallMPI(MPI_Init(argc, args));
1295: #endif
1296: if (PetscDefined(HAVE_MPIUNI)) {
1297: mpienv = getenv("PMI_SIZE");
1298: if (!mpienv) mpienv = getenv("OMPI_COMM_WORLD_SIZE");
1299: if (mpienv) {
1300: PetscInt isize;
1301: PetscCall(PetscOptionsStringToInt(mpienv, &isize));
1302: if (isize != 1) printf("You are using an MPI-uni (sequential) install of PETSc but trying to launch parallel jobs; you need full MPI version of PETSc\n");
1303: PetscCheck(isize == 1, MPI_COMM_SELF, PETSC_ERR_MPI, "You are using an MPI-uni (sequential) install of PETSc but trying to launch parallel jobs; you need full MPI version of PETSc");
1304: }
1305: }
1306: PetscBeganMPI = PETSC_TRUE;
1307: }
1309: if (argc && *argc) prog = **args;
1310: if (argc && args) {
1311: PetscGlobalArgc = *argc;
1312: PetscGlobalArgs = *args;
1313: }
1314: PetscCall(PetscInitialize_Common(prog, file, help, PETSC_FALSE, PETSC_FALSE, 0));
1315: PetscFunctionReturn(PETSC_SUCCESS);
1316: }
1318: PETSC_INTERN PetscObject *PetscObjects;
1319: PETSC_INTERN PetscInt PetscObjectsCounts;
1320: PETSC_INTERN PetscInt PetscObjectsMaxCounts;
1321: PETSC_INTERN PetscBool PetscObjectsLog;
1323: /*
1324: Frees all the MPI types and operations that PETSc may have created
1325: */
1326: PetscErrorCode PetscFreeMPIResources(void)
1327: {
1328: PetscFunctionBegin;
1329: #if defined(PETSC_HAVE_REAL___FLOAT128)
1330: PetscCallMPI(MPI_Type_free(&MPIU___FLOAT128));
1331: #if defined(PETSC_HAVE_COMPLEX)
1332: PetscCallMPI(MPI_Type_free(&MPIU___COMPLEX128));
1333: #endif
1334: #endif
1335: #if defined(PETSC_HAVE_REAL___FP16)
1336: PetscCallMPI(MPI_Type_free(&MPIU___FP16));
1337: #endif
1339: #if defined(PETSC_USE_REAL___FLOAT128) || defined(PETSC_USE_REAL___FP16)
1340: PetscCallMPI(MPI_Op_free(&MPIU_SUM));
1341: PetscCallMPI(MPI_Op_free(&MPIU_MAX));
1342: PetscCallMPI(MPI_Op_free(&MPIU_MIN));
1343: #elif defined(PETSC_HAVE_REAL___FLOAT128) || defined(PETSC_HAVE_REAL___FP16)
1344: PetscCallMPI(MPI_Op_free(&MPIU_SUM___FP16___FLOAT128));
1345: #endif
1347: PetscCallMPI(MPI_Type_free(&MPIU_2SCALAR));
1348: PetscCallMPI(MPI_Type_free(&MPIU_REAL_INT));
1349: PetscCallMPI(MPI_Type_free(&MPIU_SCALAR_INT));
1350: #if defined(PETSC_USE_64BIT_INDICES)
1351: PetscCallMPI(MPI_Type_free(&MPIU_2INT));
1352: #endif
1353: PetscCallMPI(MPI_Type_free(&MPI_4INT));
1354: PetscCallMPI(MPI_Type_free(&MPIU_4INT));
1355: PetscCallMPI(MPI_Op_free(&MPIU_MAXSUM_OP));
1356: PetscCallMPI(MPI_Op_free(&Petsc_Garbage_SetIntersectOp));
1357: PetscFunctionReturn(PETSC_SUCCESS);
1358: }
1360: PETSC_INTERN PetscErrorCode PetscLogFinalize(void);
1362: /*@C
1363: PetscFinalize - Checks for options to be called at the conclusion
1364: of the program. `MPI_Finalize()` is called only if the user had not
1365: called `MPI_Init()` before calling `PetscInitialize()`.
1367: Collective on `PETSC_COMM_WORLD`
1369: Options Database Keys:
1370: + -options_view - Calls `PetscOptionsView()`
1371: . -options_left - Prints unused options that remain in the database
1372: . -objects_dump [all] - Prints list of objects allocated by the user that have not been freed, the option all cause all outstanding objects to be listed
1373: . -mpidump - Calls PetscMPIDump()
1374: . -malloc_dump <optional filename> - Calls `PetscMallocDump()`, displays all memory allocated that has not been freed
1375: . -memory_view - Prints total memory usage
1376: - -malloc_view <optional filename> - Prints list of all memory allocated and in what functions
1378: Level: beginner
1380: Note:
1381: See `PetscInitialize()` for other runtime options.
1383: .seealso: `PetscInitialize()`, `PetscOptionsView()`, `PetscMallocDump()`, `PetscMPIDump()`, `PetscEnd()`
1384: @*/
1385: PetscErrorCode PetscFinalize(void)
1386: {
1387: PetscMPIInt rank;
1388: PetscInt nopt;
1389: PetscBool flg1 = PETSC_FALSE, flg2 = PETSC_FALSE, flg3 = PETSC_FALSE;
1390: PetscBool flg;
1391: char mname[PETSC_MAX_PATH_LEN];
1393: PetscFunctionBegin;
1394: PetscCheck(PetscInitializeCalled, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "PetscInitialize() must be called before PetscFinalize()");
1395: PetscCall(PetscInfo(NULL, "PetscFinalize() called\n"));
1397: PetscCall(PetscOptionsHasName(NULL, NULL, "-mpi_linear_solver_server", &flg));
1398: if (PetscDefined(USE_SINGLE_LIBRARY) && flg) PetscCall(PCMPIServerEnd());
1400: /* Clean up Garbage automatically on COMM_SELF and COMM_WORLD at finalize */
1401: {
1402: union
1403: {
1404: MPI_Comm comm;
1405: void *ptr;
1406: } ucomm;
1407: PetscMPIInt flg;
1408: void *tmp;
1410: PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_SELF, Petsc_InnerComm_keyval, &ucomm, &flg));
1411: if (flg) PetscCallMPI(MPI_Comm_get_attr(ucomm.comm, Petsc_Garbage_HMap_keyval, &tmp, &flg));
1412: if (flg) PetscCall(PetscGarbageCleanup(PETSC_COMM_SELF));
1413: PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_WORLD, Petsc_InnerComm_keyval, &ucomm, &flg));
1414: if (flg) PetscCallMPI(MPI_Comm_get_attr(ucomm.comm, Petsc_Garbage_HMap_keyval, &tmp, &flg));
1415: if (flg) PetscCall(PetscGarbageCleanup(PETSC_COMM_WORLD));
1416: }
1418: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
1419: #if defined(PETSC_HAVE_ADIOS)
1420: PetscCallExternal(adios_read_finalize_method, ADIOS_READ_METHOD_BP_AGGREGATE);
1421: PetscCallExternal(adios_finalize, rank);
1422: #endif
1423: PetscCall(PetscOptionsHasName(NULL, NULL, "-citations", &flg));
1424: if (flg) {
1425: char *cits, filename[PETSC_MAX_PATH_LEN];
1426: FILE *fd = PETSC_STDOUT;
1428: PetscCall(PetscOptionsGetString(NULL, NULL, "-citations", filename, sizeof(filename), NULL));
1429: if (filename[0]) PetscCall(PetscFOpen(PETSC_COMM_WORLD, filename, "w", &fd));
1430: PetscCall(PetscSegBufferGet(PetscCitationsList, 1, &cits));
1431: cits[0] = 0;
1432: PetscCall(PetscSegBufferExtractAlloc(PetscCitationsList, &cits));
1433: PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "If you publish results based on this computation please cite the following:\n"));
1434: PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "===========================================================================\n"));
1435: PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "%s", cits));
1436: PetscCall(PetscFPrintf(PETSC_COMM_WORLD, fd, "===========================================================================\n"));
1437: PetscCall(PetscFClose(PETSC_COMM_WORLD, fd));
1438: PetscCall(PetscFree(cits));
1439: }
1440: PetscCall(PetscSegBufferDestroy(&PetscCitationsList));
1442: #if defined(PETSC_SERIALIZE_FUNCTIONS)
1443: PetscCall(PetscFPTDestroy());
1444: #endif
1446: #if defined(PETSC_HAVE_SAWS)
1447: flg = PETSC_FALSE;
1448: PetscCall(PetscOptionsGetBool(NULL, NULL, "-saw_options", &flg, NULL));
1449: if (flg) PetscCall(PetscOptionsSAWsDestroy());
1450: #endif
1452: #if defined(PETSC_HAVE_X)
1453: flg1 = PETSC_FALSE;
1454: PetscCall(PetscOptionsGetBool(NULL, NULL, "-x_virtual", &flg1, NULL));
1455: if (flg1) {
1456: /* this is a crude hack, but better than nothing */
1457: PetscCall(PetscPOpen(PETSC_COMM_WORLD, NULL, "pkill -9 Xvfb", "r", NULL));
1458: }
1459: #endif
1461: #if !defined(PETSC_HAVE_THREADSAFETY)
1462: PetscCall(PetscOptionsGetBool(NULL, NULL, "-memory_view", &flg2, NULL));
1463: if (flg2) PetscCall(PetscMemoryView(PETSC_VIEWER_STDOUT_WORLD, "Summary of Memory Usage in PETSc\n"));
1464: #endif
1466: if (PetscDefined(USE_LOG)) {
1467: flg1 = PETSC_FALSE;
1468: PetscCall(PetscOptionsGetBool(NULL, NULL, "-get_total_flops", &flg1, NULL));
1469: if (flg1) {
1470: PetscLogDouble flops = 0;
1471: PetscCallMPI(MPI_Reduce(&petsc_TotalFlops, &flops, 1, MPI_DOUBLE, MPI_SUM, 0, PETSC_COMM_WORLD));
1472: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Total flops over all processors %g\n", flops));
1473: }
1474: }
1476: if (PetscDefined(USE_LOG) && PetscDefined(HAVE_MPE)) {
1477: mname[0] = 0;
1478: PetscCall(PetscOptionsGetString(NULL, NULL, "-log_mpe", mname, sizeof(mname), &flg1));
1479: if (flg1) PetscCall(PetscLogMPEDump(mname[0] ? mname : NULL));
1480: }
1482: // Free all objects registered with PetscObjectRegisterDestroy() such as PETSC_VIEWER_XXX_().
1483: PetscCall(PetscObjectRegisterDestroyAll());
1485: if (PetscDefined(USE_LOG)) {
1486: PetscCall(PetscOptionsPushGetViewerOff(PETSC_FALSE));
1487: PetscCall(PetscLogViewFromOptions());
1488: PetscCall(PetscOptionsPopGetViewerOff());
1489: // It should be turned on with PetscLogGpuTime() and never turned off except in this place
1490: PetscLogGpuTimeFlag = PETSC_FALSE;
1492: // Free any objects created by the last block of code.
1493: PetscCall(PetscObjectRegisterDestroyAll());
1495: mname[0] = 0;
1496: PetscCall(PetscOptionsGetString(NULL, NULL, "-log_all", mname, sizeof(mname), &flg1));
1497: PetscCall(PetscOptionsGetString(NULL, NULL, "-log", mname, sizeof(mname), &flg2));
1498: if (flg1 || flg2) PetscCall(PetscLogDump(mname));
1499: }
1501: flg1 = PETSC_FALSE;
1502: PetscCall(PetscOptionsGetBool(NULL, NULL, "-no_signal_handler", &flg1, NULL));
1503: if (!flg1) PetscCall(PetscPopSignalHandler());
1504: flg1 = PETSC_FALSE;
1505: PetscCall(PetscOptionsGetBool(NULL, NULL, "-mpidump", &flg1, NULL));
1506: if (flg1) PetscCall(PetscMPIDump(stdout));
1507: flg1 = PETSC_FALSE;
1508: flg2 = PETSC_FALSE;
1509: /* preemptive call to avoid listing this option in options table as unused */
1510: PetscCall(PetscOptionsHasName(NULL, NULL, "-malloc_dump", &flg1));
1511: PetscCall(PetscOptionsHasName(NULL, NULL, "-objects_dump", &flg1));
1512: PetscCall(PetscOptionsGetBool(NULL, NULL, "-options_view", &flg2, NULL));
1514: if (flg2) { PetscCall(PetscOptionsView(NULL, PETSC_VIEWER_STDOUT_WORLD)); }
1516: /* to prevent PETSc -options_left from warning */
1517: PetscCall(PetscOptionsHasName(NULL, NULL, "-nox", &flg1));
1518: PetscCall(PetscOptionsHasName(NULL, NULL, "-nox_warning", &flg1));
1520: flg3 = PETSC_FALSE; /* default value is required */
1521: PetscCall(PetscOptionsGetBool(NULL, NULL, "-options_left", &flg3, &flg1));
1522: if (!flg1) flg3 = PETSC_TRUE;
1523: if (flg3) {
1524: if (!flg2 && flg1) { /* have not yet printed the options */
1525: PetscCall(PetscOptionsView(NULL, PETSC_VIEWER_STDOUT_WORLD));
1526: }
1527: PetscCall(PetscOptionsAllUsed(NULL, &nopt));
1528: if (nopt) {
1529: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "WARNING! There are options you set that were not used!\n"));
1530: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "WARNING! could be spelling mistake, etc!\n"));
1531: if (nopt == 1) {
1532: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "There is one unused database option. It is:\n"));
1533: } else {
1534: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "There are %" PetscInt_FMT " unused database options. They are:\n", nopt));
1535: }
1536: } else if (flg3 && flg1) {
1537: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "There are no unused options.\n"));
1538: }
1539: PetscCall(PetscOptionsLeft(NULL));
1540: }
1542: #if defined(PETSC_HAVE_SAWS)
1543: if (!PetscGlobalRank) {
1544: PetscCall(PetscStackSAWsViewOff());
1545: PetscCallSAWs(SAWs_Finalize, ());
1546: }
1547: #endif
1549: /*
1550: List all objects the user may have forgot to free
1551: */
1552: if (PetscDefined(USE_LOG) && PetscObjectsLog) {
1553: PetscCall(PetscOptionsHasName(NULL, NULL, "-objects_dump", &flg1));
1554: if (flg1) {
1555: MPI_Comm local_comm;
1556: char string[64];
1558: PetscCall(PetscOptionsGetString(NULL, NULL, "-objects_dump", string, sizeof(string), NULL));
1559: PetscCallMPI(MPI_Comm_dup(PETSC_COMM_WORLD, &local_comm));
1560: PetscCall(PetscSequentialPhaseBegin_Private(local_comm, 1));
1561: PetscCall(PetscObjectsDump(stdout, (string[0] == 'a') ? PETSC_TRUE : PETSC_FALSE));
1562: PetscCall(PetscSequentialPhaseEnd_Private(local_comm, 1));
1563: PetscCallMPI(MPI_Comm_free(&local_comm));
1564: }
1565: }
1567: PetscObjectsCounts = 0;
1568: PetscObjectsMaxCounts = 0;
1569: PetscCall(PetscFree(PetscObjects));
1571: /*
1572: Destroy any packages that registered a finalize
1573: */
1574: PetscCall(PetscRegisterFinalizeAll());
1576: PetscCall(PetscLogFinalize());
1578: /*
1579: Print PetscFunctionLists that have not been properly freed
1580: */
1581: if (PetscPrintFunctionList) PetscCall(PetscFunctionListPrintAll());
1583: if (petsc_history) {
1584: PetscCall(PetscCloseHistoryFile(&petsc_history));
1585: petsc_history = NULL;
1586: }
1587: PetscCall(PetscOptionsHelpPrintedDestroy(&PetscOptionsHelpPrintedSingleton));
1588: PetscCall(PetscInfoDestroy());
1590: #if !defined(PETSC_HAVE_THREADSAFETY)
1591: if (!(PETSC_RUNNING_ON_VALGRIND)) {
1592: char fname[PETSC_MAX_PATH_LEN];
1593: char sname[PETSC_MAX_PATH_LEN];
1594: FILE *fd;
1595: int err;
1597: flg2 = PETSC_FALSE;
1598: flg3 = PETSC_FALSE;
1599: if (PetscDefined(USE_DEBUG)) PetscCall(PetscOptionsGetBool(NULL, NULL, "-malloc_test", &flg2, NULL));
1600: PetscCall(PetscOptionsGetBool(NULL, NULL, "-malloc_debug", &flg3, NULL));
1601: fname[0] = 0;
1602: PetscCall(PetscOptionsGetString(NULL, NULL, "-malloc_dump", fname, sizeof(fname), &flg1));
1603: if (flg1 && fname[0]) {
1604: PetscCall(PetscSNPrintf(sname, sizeof(sname), "%s_%d", fname, rank));
1605: fd = fopen(sname, "w");
1606: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Cannot open log file: %s", sname);
1607: PetscCall(PetscMallocDump(fd));
1608: err = fclose(fd);
1609: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
1610: } else if (flg1 || flg2 || flg3) {
1611: MPI_Comm local_comm;
1613: PetscCallMPI(MPI_Comm_dup(PETSC_COMM_WORLD, &local_comm));
1614: PetscCall(PetscSequentialPhaseBegin_Private(local_comm, 1));
1615: PetscCall(PetscMallocDump(stdout));
1616: PetscCall(PetscSequentialPhaseEnd_Private(local_comm, 1));
1617: PetscCallMPI(MPI_Comm_free(&local_comm));
1618: }
1619: fname[0] = 0;
1620: PetscCall(PetscOptionsGetString(NULL, NULL, "-malloc_view", fname, sizeof(fname), &flg1));
1621: if (flg1 && fname[0]) {
1622: PetscCall(PetscSNPrintf(sname, sizeof(sname), "%s_%d", fname, rank));
1623: fd = fopen(sname, "w");
1624: PetscCheck(fd, PETSC_COMM_SELF, PETSC_ERR_FILE_OPEN, "Cannot open log file: %s", sname);
1625: PetscCall(PetscMallocView(fd));
1626: err = fclose(fd);
1627: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_SYS, "fclose() failed on file");
1628: } else if (flg1) {
1629: MPI_Comm local_comm;
1631: PetscCallMPI(MPI_Comm_dup(PETSC_COMM_WORLD, &local_comm));
1632: PetscCall(PetscSequentialPhaseBegin_Private(local_comm, 1));
1633: PetscCall(PetscMallocView(stdout));
1634: PetscCall(PetscSequentialPhaseEnd_Private(local_comm, 1));
1635: PetscCallMPI(MPI_Comm_free(&local_comm));
1636: }
1637: }
1638: #endif
1640: /*
1641: Close any open dynamic libraries
1642: */
1643: PetscCall(PetscFinalize_DynamicLibraries());
1645: /* Can be destroyed only after all the options are used */
1646: PetscCall(PetscOptionsDestroyDefault());
1648: PetscGlobalArgc = 0;
1649: PetscGlobalArgs = NULL;
1651: #if defined(PETSC_HAVE_KOKKOS)
1652: if (PetscKokkosInitialized) {
1653: PetscCall(PetscKokkosFinalize_Private());
1654: PetscKokkosInitialized = PETSC_FALSE;
1655: }
1656: #endif
1658: #if defined(PETSC_HAVE_NVSHMEM)
1659: if (PetscBeganNvshmem) {
1660: PetscCall(PetscNvshmemFinalize());
1661: PetscBeganNvshmem = PETSC_FALSE;
1662: }
1663: #endif
1665: PetscCall(PetscFreeMPIResources());
1667: /*
1668: Destroy any known inner MPI_Comm's and attributes pointing to them
1669: Note this will not destroy any new communicators the user has created.
1671: If all PETSc objects were not destroyed those left over objects will have hanging references to
1672: the MPI_Comms that were freed; but that is ok because those PETSc objects will never be used again
1673: */
1674: {
1675: PetscCommCounter *counter;
1676: PetscMPIInt flg;
1677: MPI_Comm icomm;
1678: union
1679: {
1680: MPI_Comm comm;
1681: void *ptr;
1682: } ucomm;
1683: PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_SELF, Petsc_InnerComm_keyval, &ucomm, &flg));
1684: if (flg) {
1685: icomm = ucomm.comm;
1686: PetscCallMPI(MPI_Comm_get_attr(icomm, Petsc_Counter_keyval, &counter, &flg));
1687: PetscCheck(flg, PETSC_COMM_SELF, PETSC_ERR_ARG_CORRUPT, "Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1689: PetscCallMPI(MPI_Comm_delete_attr(PETSC_COMM_SELF, Petsc_InnerComm_keyval));
1690: PetscCallMPI(MPI_Comm_delete_attr(icomm, Petsc_Counter_keyval));
1691: PetscCallMPI(MPI_Comm_free(&icomm));
1692: }
1693: PetscCallMPI(MPI_Comm_get_attr(PETSC_COMM_WORLD, Petsc_InnerComm_keyval, &ucomm, &flg));
1694: if (flg) {
1695: icomm = ucomm.comm;
1696: PetscCallMPI(MPI_Comm_get_attr(icomm, Petsc_Counter_keyval, &counter, &flg));
1697: PetscCheck(flg, PETSC_COMM_WORLD, PETSC_ERR_ARG_CORRUPT, "Inner MPI_Comm does not have expected tag/name counter, problem with corrupted memory");
1699: PetscCallMPI(MPI_Comm_delete_attr(PETSC_COMM_WORLD, Petsc_InnerComm_keyval));
1700: PetscCallMPI(MPI_Comm_delete_attr(icomm, Petsc_Counter_keyval));
1701: PetscCallMPI(MPI_Comm_free(&icomm));
1702: }
1703: }
1705: PetscCallMPI(MPI_Comm_free_keyval(&Petsc_Counter_keyval));
1706: PetscCallMPI(MPI_Comm_free_keyval(&Petsc_InnerComm_keyval));
1707: PetscCallMPI(MPI_Comm_free_keyval(&Petsc_OuterComm_keyval));
1708: PetscCallMPI(MPI_Comm_free_keyval(&Petsc_ShmComm_keyval));
1709: PetscCallMPI(MPI_Comm_free_keyval(&Petsc_CreationIdx_keyval));
1710: PetscCallMPI(MPI_Comm_free_keyval(&Petsc_Garbage_HMap_keyval));
1712: // Free keyvals which may be silently created by some routines
1713: if (Petsc_SharedWD_keyval != MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_free_keyval(&Petsc_SharedWD_keyval));
1714: if (Petsc_SharedTmp_keyval != MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_free_keyval(&Petsc_SharedTmp_keyval));
1716: PetscCall(PetscSpinlockDestroy(&PetscViewerASCIISpinLockOpen));
1717: PetscCall(PetscSpinlockDestroy(&PetscViewerASCIISpinLockStdout));
1718: PetscCall(PetscSpinlockDestroy(&PetscViewerASCIISpinLockStderr));
1719: PetscCall(PetscSpinlockDestroy(&PetscCommSpinLock));
1721: if (PetscBeganMPI) {
1722: PetscMPIInt flag;
1723: PetscCallMPI(MPI_Finalized(&flag));
1724: PetscCheck(!flag, PETSC_COMM_SELF, PETSC_ERR_LIB, "MPI_Finalize() has already been called, even though MPI_Init() was called by PetscInitialize()");
1725: /* wait until the very last moment to disable error handling */
1726: PetscErrorHandlingInitialized = PETSC_FALSE;
1727: PetscCallMPI(MPI_Finalize());
1728: } else PetscErrorHandlingInitialized = PETSC_FALSE;
1730: /*
1732: Note: In certain cases PETSC_COMM_WORLD is never MPI_Comm_free()ed because
1733: the communicator has some outstanding requests on it. Specifically if the
1734: flag PETSC_HAVE_BROKEN_REQUEST_FREE is set (for IBM MPI implementation). See
1735: src/vec/utils/vpscat.c. Due to this the memory allocated in PetscCommDuplicate()
1736: is never freed as it should be. Thus one may obtain messages of the form
1737: [ 1] 8 bytes PetscCommDuplicate() line 645 in src/sys/mpiu.c indicating the
1738: memory was not freed.
1740: */
1741: PetscCall(PetscMallocClear());
1742: PetscCall(PetscStackReset());
1744: PetscInitializeCalled = PETSC_FALSE;
1745: PetscFinalizeCalled = PETSC_TRUE;
1746: #if defined(PETSC_USE_COVERAGE)
1747: /*
1748: flush gcov, otherwise during CI the flushing continues into the next pipeline resulting in git not being able to delete directories since the
1749: gcov files are still being added to the directories as git tries to remove the directories.
1750: */
1751: __gcov_flush();
1752: #endif
1753: /* To match PetscFunctionBegin() at the beginning of this function */
1754: PetscStackClearTop;
1755: return PETSC_SUCCESS;
1756: }
1758: #if defined(PETSC_MISSING_LAPACK_lsame_)
1759: PETSC_EXTERN int lsame_(char *a, char *b)
1760: {
1761: if (*a == *b) return 1;
1762: if (*a + 32 == *b) return 1;
1763: if (*a - 32 == *b) return 1;
1764: return 0;
1765: }
1766: #endif
1768: #if defined(PETSC_MISSING_LAPACK_lsame)
1769: PETSC_EXTERN int lsame(char *a, char *b)
1770: {
1771: if (*a == *b) return 1;
1772: if (*a + 32 == *b) return 1;
1773: if (*a - 32 == *b) return 1;
1774: return 0;
1775: }
1776: #endif