Actual source code: garbage.c
1: #include <petsc/private/garbagecollector.h>
3: /* Fetches garbage hashmap from communicator */
4: static PetscErrorCode GarbageGetHMap_Private(MPI_Comm comm, PetscGarbage *garbage)
5: {
6: PetscMPIInt iflg;
7: PetscHMapObj garbage_map;
9: PetscFunctionBegin;
10: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_Garbage_HMap_keyval, garbage, &iflg));
11: if (!iflg) {
12: /* No garbage, create one */
13: PetscCall(PetscHMapObjCreate(&garbage_map));
14: garbage->map = garbage_map;
15: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_Garbage_HMap_keyval, garbage->ptr));
16: }
17: PetscFunctionReturn(PETSC_SUCCESS);
18: }
20: /*@
21: PetscObjectDelayedDestroy - Adds an object to a data structure for
22: later destruction.
24: Not Collective
26: Input Parameter:
27: . obj - object to be destroyed
29: Level: developer
31: Notes:
32: Analogue to `PetscObjectDestroy()` for use in managed languages.
34: A PETSc object is given a creation index at initialisation based on
35: the communicator it was created on and the order in which it is
36: created. When this function is passed a PETSc object, a pointer to
37: the object is stashed on a garbage dictionary (`PetscHMapObj`) which is
38: keyed by its creation index.
40: Objects stashed on this garbage dictionary can later be destroyed
41: with a call to `PetscGarbageCleanup()`.
43: This function is intended for use with managed languages such as
44: Python or Julia, which may not destroy objects in a deterministic
45: order.
47: Serial objects (that have a communicator with size 1) are destroyed
48: eagerly since deadlocks cannot occur.
50: .seealso: `PetscGarbageCleanup()`, `PetscObjectDestroy()`
51: @*/
52: PetscErrorCode PetscObjectDelayedDestroy(PetscObject *obj)
53: {
54: MPI_Comm comm;
55: PetscMPIInt size;
56: PetscInt count;
57: PetscGarbage garbage;
59: PetscFunctionBegin;
60: PetscAssertPointer(obj, 1);
61: /* Don't stash NULL pointers */
62: if (*obj != NULL) {
63: /* Elaborate check for getting non-cyclic reference counts */
64: if (!(*obj)->non_cyclic_references) {
65: count = --(*obj)->refct;
66: } else {
67: PetscCall((*obj)->non_cyclic_references(*obj, &count));
68: --count;
69: --(*obj)->refct;
70: }
71: /* Only stash if the (non-cyclic) reference count hits 0 */
72: if (count == 0) {
73: (*obj)->refct = 1;
74: PetscCall(PetscObjectGetComm(*obj, &comm));
75: PetscCallMPI(MPI_Comm_size(comm, &size));
76: /* Eagerly destroy serial objects */
77: if (size == 1) {
78: PetscCall(PetscObjectDestroy(obj));
79: } else {
80: PetscCall(GarbageGetHMap_Private(comm, &garbage));
81: PetscCall(PetscHMapObjSet(garbage.map, (*obj)->cidx, *obj));
82: }
83: }
84: }
85: *obj = NULL;
86: PetscFunctionReturn(PETSC_SUCCESS);
87: }
89: /* Performs the intersection of 2 sorted arrays seta and setb of lengths
90: lena and lenb respectively,returning the result in seta and lena
91: This is an O(n) operation */
92: static PetscErrorCode GarbageKeySortedIntersect_Private(PetscInt64 seta[], PetscInt *lena, PetscInt64 setb[], PetscInt lenb)
93: {
94: /* The arrays seta and setb MUST be sorted! */
95: PetscInt ii, jj = 0, counter = 0;
97: PetscFunctionBegin;
98: if (PetscDefined(USE_DEBUG)) {
99: PetscBool sorted = PETSC_FALSE;
100: /* In debug mode check whether the array are sorted */
101: PetscCall(PetscSortedInt64(*lena, seta, &sorted));
102: PetscCheck(sorted, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Provided array in argument 1 is not sorted");
103: PetscCall(PetscSortedInt64(lenb, setb, &sorted));
104: PetscCheck(sorted, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Provided array in argument 3 is not sorted");
105: }
106: for (ii = 0; ii < *lena; ii++) {
107: while (jj < lenb && seta[ii] > setb[jj]) jj++;
108: if (jj >= lenb) break;
109: if (seta[ii] == setb[jj]) {
110: seta[counter] = seta[ii];
111: counter++;
112: }
113: }
115: *lena = counter;
116: PetscFunctionReturn(PETSC_SUCCESS);
117: }
119: /* Wrapper to create MPI reduce operator for set intersection */
120: void PetscGarbageKeySortedIntersect(void *inset, void *inoutset, PetscMPIInt *length, MPI_Datatype *dtype)
121: {
122: PetscInt64 *seta, *setb;
123: PetscInt lena = 0, lenb = 0;
125: seta = (PetscInt64 *)inoutset;
126: setb = (PetscInt64 *)inset;
128: PetscCallAbort(PETSC_COMM_SELF, PetscIntCast(seta[0], &lena));
129: PetscCallAbort(PETSC_COMM_SELF, PetscIntCast(setb[0], &lenb));
130: PetscCallAbort(PETSC_COMM_SELF, GarbageKeySortedIntersect_Private(seta + 1, &lena, setb + 1, lenb));
131: seta[0] = lena;
132: }
134: /* Performs a collective allreduce intersection of one array per rank */
135: PetscErrorCode GarbageKeyAllReduceIntersect_Private(MPI_Comm comm, PetscInt64 *set, PetscInt *entries)
136: {
137: PetscInt ii, max_entries;
138: PetscInt64 *recvset;
139: MPI_Datatype keyset_type;
140: PetscMPIInt imax_entries;
142: PetscFunctionBegin;
143: /* Sort keys first for use with `GarbageKeySortedIntersect_Private()`*/
144: PetscCall(PetscSortInt64(*entries, set));
146: /* Get the maximum size of all key sets */
147: PetscCallMPI(MPIU_Allreduce(entries, &max_entries, 1, MPIU_INT, MPI_MAX, comm));
148: PetscCall(PetscMalloc1(max_entries + 1, &recvset));
149: recvset[0] = *entries;
150: for (ii = 1; ii < *entries + 1; ii++) recvset[ii] = set[ii - 1];
152: /* Create a custom data type to hold the set */
153: PetscCall(PetscMPIIntCast(max_entries, &imax_entries));
154: PetscCallMPI(MPI_Type_contiguous(imax_entries + 1, MPIU_INT64, &keyset_type));
155: /* PetscCallMPI(MPI_Type_set_name(keyset_type,"PETSc garbage key set type")); */
156: PetscCallMPI(MPI_Type_commit(&keyset_type));
158: /* Perform custom intersect reduce operation over sets */
159: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, recvset, 1, keyset_type, Petsc_Garbage_SetIntersectOp, comm));
161: PetscCallMPI(MPI_Type_free(&keyset_type));
163: PetscCall(PetscIntCast(recvset[0], entries));
164: for (ii = 0; ii < *entries; ii++) set[ii] = recvset[ii + 1];
166: PetscCall(PetscFree(recvset));
167: PetscFunctionReturn(PETSC_SUCCESS);
168: }
170: /*@
171: PetscGarbageCleanup - Destroys objects placed in the garbage by
172: `PetscObjectDelayedDestroy()`.
174: Collective
176: Input Parameter:
177: . comm - MPI communicator over which to perform collective cleanup
179: Level: developer
181: Notes:
182: Implements a collective garbage collection.
183: A per- MPI communicator garbage dictionary is created to store
184: references to objects destroyed using `PetscObjectDelayedDestroy()`.
185: Objects that appear in this dictionary on all MPI processes can be destroyed
186: by calling `PetscGarbageCleanup()`.
188: This is done as follows\:
189: 1. Keys of the garbage dictionary, which correspond to the creation
190: indices of the objects stashed, are sorted.
191: 2. A collective intersection of dictionary keys is performed by all
192: ranks in the communicator.
193: 3. The intersection is broadcast back to all ranks in the
194: communicator.
195: 4. The objects in the intersection are collectively destroyed in
196: creation index order using a call to `PetscObjectDestroy()`.
197: 5. Objects queued during destruction are added to the dictionary, and
198: the process repeats until the collective intersection is empty.
200: Objects queued on only some ranks remain in the dictionary for a subsequent
201: call to `PetscGarbageCleanup()`.
203: This function is intended for use with managed languages such as
204: Python or Julia, which may not destroy objects in a deterministic
205: order.
207: .seealso: `PetscObjectDelayedDestroy()`
208: @*/
209: PetscErrorCode PetscGarbageCleanup(MPI_Comm comm)
210: {
211: PetscInt ii, entries, offset;
212: PetscMPIInt flg;
213: PetscInt64 *keys;
214: PetscObject obj;
215: PetscGarbage garbage, pending;
217: PetscFunctionBegin;
218: /* Duplicate comm to prevent it being cleaned up by PetscObjectDestroy() */
219: PetscCall(PetscCommDuplicate(comm, &comm, NULL));
221: /* Grab garbage from comm and remove it
222: this avoids calling PetscCommDestroy() and endlessly recursing */
223: PetscCall(GarbageGetHMap_Private(comm, &garbage));
224: PetscCallMPI(MPI_Comm_delete_attr(comm, Petsc_Garbage_HMap_keyval));
226: do {
227: /* Get keys from garbage hash map */
228: PetscCall(PetscHMapObjGetSize(garbage.map, &entries));
229: PetscCall(PetscMalloc1(entries, &keys));
230: offset = 0;
231: PetscCall(PetscHMapObjGetKeys(garbage.map, &offset, keys));
233: /* Gather and intersect */
234: PetscCall(GarbageKeyAllReduceIntersect_Private(comm, keys, &entries));
236: /* Collectively destroy objects that appear in garbage in
237: creation index order */
238: for (ii = 0; ii < entries; ii++) {
239: PetscCall(PetscHMapObjGet(garbage.map, keys[ii], &obj));
240: PetscCall(PetscObjectDestroy(&obj));
241: PetscCall(PetscFree(obj));
242: PetscCall(PetscHMapObjDel(garbage.map, keys[ii]));
243: }
244: PetscCall(PetscFree(keys));
246: // Destructors may have queued more objects on the communicator.
247: PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_Garbage_HMap_keyval, &pending, &flg));
248: if (flg) {
249: PetscHashIter it;
250: PetscInt64 key;
252: PetscCallMPI(MPI_Comm_delete_attr(comm, Petsc_Garbage_HMap_keyval));
253: PetscHashIterBegin(pending.map, it);
254: while (!PetscHashIterAtEnd(pending.map, it)) {
255: PetscHashIterGetKey(pending.map, it, key);
256: PetscHashIterGetVal(pending.map, it, obj);
257: PetscCall(PetscHMapObjSet(garbage.map, key, obj));
258: PetscHashIterNext(pending.map, it);
259: }
260: PetscCall(PetscHMapObjDestroy(&pending.map));
261: }
262: } while (entries);
264: /* Put garbage back */
265: PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_Garbage_HMap_keyval, garbage.ptr));
266: PetscCall(PetscCommDestroy(&comm));
267: PetscFunctionReturn(PETSC_SUCCESS);
268: }
270: /* Utility function for printing the contents of the garbage on a given comm */
271: PetscErrorCode PetscGarbageView(MPI_Comm comm, PetscViewer viewer)
272: {
273: char text[64];
274: PetscInt ii, entries, offset;
275: PetscInt64 *keys;
276: PetscObject obj;
277: PetscGarbage garbage;
278: PetscMPIInt rank;
280: PetscFunctionBegin;
281: PetscCall(PetscPrintf(comm, "PETSc garbage on "));
282: if (comm == PETSC_COMM_WORLD) {
283: PetscCall(PetscPrintf(comm, "PETSC_COMM_WORLD\n"));
284: } else if (comm == PETSC_COMM_SELF) {
285: PetscCall(PetscPrintf(comm, "PETSC_COMM_SELF\n"));
286: } else {
287: PetscCall(PetscPrintf(comm, "UNKNOWN_COMM\n"));
288: }
289: PetscCall(PetscCommDuplicate(comm, &comm, NULL));
290: PetscCall(GarbageGetHMap_Private(comm, &garbage));
292: /* Get keys from garbage hash map and sort */
293: PetscCall(PetscHMapObjGetSize(garbage.map, &entries));
294: PetscCall(PetscMalloc1(entries, &keys));
295: offset = 0;
296: PetscCall(PetscHMapObjGetKeys(garbage.map, &offset, keys));
298: /* Pretty print entries in a table */
299: PetscCallMPI(MPI_Comm_rank(comm, &rank));
300: PetscCall(PetscSynchronizedPrintf(comm, "Rank %i:: ", rank));
301: PetscCall(PetscFormatConvert("Total entries: %" PetscInt_FMT "\n", text));
302: PetscCall(PetscSynchronizedPrintf(comm, text, entries));
303: if (entries) {
304: PetscCall(PetscSynchronizedPrintf(comm, "| Key | Type | Name | Object ID |\n"));
305: PetscCall(PetscSynchronizedPrintf(comm, "|-------|------------------------|----------------------------------|-----------|\n"));
306: }
307: for (ii = 0; ii < entries; ii++) {
308: PetscCall(PetscHMapObjGet(garbage.map, keys[ii], &obj));
309: PetscCall(PetscFormatConvert("| %5" PetscInt64_FMT " | %-22s | %-32s | %6" PetscInt_FMT " |\n", text));
310: PetscCall(PetscSynchronizedPrintf(comm, text, keys[ii], obj->class_name, obj->description, obj->id));
311: }
312: PetscCall(PetscSynchronizedFlush(comm, PETSC_STDOUT));
314: PetscCall(PetscFree(keys));
315: PetscCall(PetscCommDestroy(&comm));
316: PetscFunctionReturn(PETSC_SUCCESS);
317: }