Actual source code: mal.c
1: /*
2: Code that allows a user to dictate what malloc() PETSc uses.
3: */
4: #define PETSC_DESIRE_FEATURE_TEST_MACROS /* for posix_memalign() */
5: #include <petscsys.h>
6: #include <stdarg.h>
7: #if defined(PETSC_HAVE_MALLOC_H)
8: #include <malloc.h>
9: #endif
10: #if defined(PETSC_HAVE_MEMKIND)
11: #include <errno.h>
12: #include <memkind.h>
13: typedef enum {
14: PETSC_MK_DEFAULT = 0,
15: PETSC_MK_HBW_PREFERRED = 1
16: } PetscMemkindType;
17: PetscMemkindType currentmktype = PETSC_MK_HBW_PREFERRED;
18: PetscMemkindType previousmktype = PETSC_MK_HBW_PREFERRED;
19: #endif
20: /*
21: We want to make sure that all mallocs of double or complex numbers are complex aligned.
22: 1) on systems with memalign() we call that routine to get an aligned memory location
23: 2) on systems without memalign() we
24: - allocate one sizeof(PetscScalar) extra space
25: - we shift the pointer up slightly if needed to get PetscScalar aligned
26: - if shifted we store at ptr[-1] the amount of shift (plus a classid)
27: */
28: #define SHIFT_CLASSID 456123
30: PETSC_EXTERN PetscErrorCode PetscMallocAlign(size_t mem, PetscBool clear, int line, const char func[], const char file[], void **result)
31: {
32: if (!mem) {
33: *result = NULL;
34: return PETSC_SUCCESS;
35: }
36: #if PetscDefined(HAVE_MEMKIND)
37: {
38: int err = memkind_posix_memalign(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT, result, PETSC_MEMALIGN, mem);
39: PetscCheck(err != EINVAL, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
40: if (err == ENOMEM) PetscInfo(NULL, "Memkind: fail to request HBW memory %.0f, falling back to normal memory\n", (PetscLogDouble)mem);
41: PetscCheck(*result, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memory requested %.0f", (PetscLogDouble)mem);
42: if (clear) PetscCall(PetscMemzero(*result, mem));
43: }
44: #else /* PetscDefined(HAVE_MEMKIND) */
45: #if PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)
46: if (clear) *result = calloc(1 + mem / sizeof(int), sizeof(int));
47: else *result = malloc(mem);
49: PetscCheck(*result, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memory requested %.0f", (PetscLogDouble)mem);
50: if (PetscLogMemory) PetscCall(PetscMemzero(*result, mem));
51: #elif PetscDefined(HAVE_POSIX_MEMALIGN)
52: int ret = posix_memalign(result, PETSC_MEMALIGN, mem);
53: PetscCheck(ret == 0, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memory requested %.0f", (PetscLogDouble)mem);
54: if (clear || PetscLogMemory) PetscCall(PetscMemzero(*result, mem));
55: #else /* PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) || PetscDefined(HAVE_POSIX_MEMALIGN) */
56: {
57: int *ptr, shift;
58: /*
59: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
60: */
61: if (clear) {
62: ptr = (int *)calloc(1 + (mem + 2 * PETSC_MEMALIGN) / sizeof(int), sizeof(int));
63: } else {
64: ptr = (int *)malloc(mem + 2 * PETSC_MEMALIGN);
65: }
66: PetscCheck(ptr, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memory requested %.0f", (PetscLogDouble)mem);
67: shift = (int)(((PETSC_UINTPTR_T)ptr) % PETSC_MEMALIGN);
68: shift = (2 * PETSC_MEMALIGN - shift) / sizeof(int);
69: ptr[shift - 1] = shift + SHIFT_CLASSID;
70: ptr += shift;
71: *result = (void *)ptr;
72: if (PetscLogMemory) PetscCall(PetscMemzero(*result, mem));
73: }
74: #endif /* PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) || PetscDefined(HAVE_POSIX_MEMALIGN) */
75: #endif /* PetscDefined(HAVE_MEMKIND) */
76: return PETSC_SUCCESS;
77: }
79: PETSC_EXTERN PetscErrorCode PetscFreeAlign(void *ptr, int line, const char func[], const char file[])
80: {
81: if (!ptr) return PETSC_SUCCESS;
82: #if PetscDefined(HAVE_MEMKIND)
83: memkind_free(0, ptr); /* specify the kind to 0 so that memkind will look up for the right type */
84: #else /* PetscDefined(HAVE_MEMKIND) */
85: #if (!(PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !PetscDefined(HAVE_POSIX_MEMALIGN))
86: {
87: /*
88: Previous int tells us how many ints the pointer has been shifted from
89: the original address provided by the system malloc().
90: */
91: const int shift = *((int *)ptr - 1) - SHIFT_CLASSID;
93: PetscCheck(shift <= PETSC_MEMALIGN - 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Likely memory corruption in heap");
94: PetscCheck(shift >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Likely memory corruption in heap");
95: ptr = (void *)((int *)ptr - shift);
96: }
97: #endif
99: #if PetscDefined(HAVE_FREE_RETURN_INT)
100: int err = free(ptr);
101: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_PLIB, "System free returned error %d", err);
102: #else
103: free(ptr);
104: #endif
105: #endif
106: return PETSC_SUCCESS;
107: }
109: PETSC_EXTERN PetscErrorCode PetscReallocAlign(size_t mem, int line, const char func[], const char file[], void **result)
110: {
111: if (!mem) {
112: PetscCall(PetscFreeAlign(*result, line, func, file));
113: *result = NULL;
114: return PETSC_SUCCESS;
115: }
116: #if PetscDefined(HAVE_MEMKIND)
117: *result = memkind_realloc(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT, *result, mem);
118: #else
119: #if (!(PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) && !PetscDefined(HAVE_POSIX_MEMALIGN))
120: {
121: /*
122: Previous int tells us how many ints the pointer has been shifted from
123: the original address provided by the system malloc().
124: */
125: int shift = *(((int *)*result) - 1) - SHIFT_CLASSID;
126: PetscCheck(shift <= PETSC_MEMALIGN - 1, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Likely memory corruption in heap");
127: PetscCheck(shift >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Likely memory corruption in heap");
128: *result = (void *)(((int *)*result) - shift);
129: }
130: #endif
132: #if (PetscDefined(HAVE_DOUBLE_ALIGN_MALLOC) && (PETSC_MEMALIGN == 8)) || PetscDefined(HAVE_POSIX_MEMALIGN)
133: *result = realloc(*result, mem);
134: #else
135: {
136: /*
137: malloc space for two extra chunks and shift ptr 1 + enough to get it PetscScalar aligned
138: */
139: int *ptr = (int *)realloc(*result, mem + 2 * PETSC_MEMALIGN);
140: if (ptr) {
141: int shift = (int)(((PETSC_UINTPTR_T)ptr) % PETSC_MEMALIGN);
142: shift = (2 * PETSC_MEMALIGN - shift) / sizeof(int);
143: ptr[shift - 1] = shift + SHIFT_CLASSID;
144: ptr += shift;
145: *result = (void *)ptr;
146: } else {
147: *result = NULL;
148: }
149: }
150: #endif
151: #endif
152: PetscCheck(*result, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memory requested %.0f", (PetscLogDouble)mem);
153: #if PetscDefined(HAVE_POSIX_MEMALIGN)
154: /* There are no standard guarantees that realloc() maintains the alignment of memalign(), so I think we have to
155: * realloc and, if the alignment is wrong, malloc/copy/free. */
156: if (((size_t)*result) % PETSC_MEMALIGN) {
157: void *newResult;
158: #if PetscDefined(HAVE_MEMKIND)
159: {
160: int err = memkind_posix_memalign(currentmktype ? MEMKIND_HBW_PREFERRED : MEMKIND_DEFAULT, &newResult, PETSC_MEMALIGN, mem);
161: PetscCheck(err != EINVAL, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memkind: invalid 3rd or 4th argument of memkind_posix_memalign()");
162: if (err == ENOMEM) PetscInfo(NULL, "Memkind: fail to request HBW memory %.0f, falling back to normal memory\n", (PetscLogDouble)mem);
163: }
164: PetscCheck(newResult, PETSC_COMM_SELF, PETSC_ERR_MEM, "Memory requested %.0f", (PetscLogDouble)mem);
165: #else
166: int ret = posix_memalign(&newResult, PETSC_MEMALIGN, mem);
167: PetscCheck(ret == 0, PETSC_COMM_SELF, PETSC_ERR_LIB, "posix_memalign() failed with error code %d, memory requested %.0f", ret, (PetscLogDouble)mem);
168: #endif
169: PetscCall(PetscMemcpy(newResult, *result, mem));
170: #if PetscDefined(HAVE_FREE_RETURN_INT)
171: {
172: int err = free(*result);
173: PetscCheck(!err, PETSC_COMM_SELF, PETSC_ERR_PLIB, "System free returned error %d", err);
174: }
175: #else
176: #if defined(PETSC_HAVE_MEMKIND)
177: memkind_free(0, *result);
178: #else
179: free(*result);
180: #endif
181: #endif
182: *result = newResult;
183: }
184: #endif
185: return PETSC_SUCCESS;
186: }
188: PetscErrorCode (*PetscTrMalloc)(size_t, PetscBool, int, const char[], const char[], void **) = PetscMallocAlign;
189: PetscErrorCode (*PetscTrFree)(void *, int, const char[], const char[]) = PetscFreeAlign;
190: PetscErrorCode (*PetscTrRealloc)(size_t, int, const char[], const char[], void **) = PetscReallocAlign;
192: PETSC_INTERN PetscBool petscsetmallocvisited;
193: PetscBool petscsetmallocvisited = PETSC_FALSE;
195: /*@C
196: PetscMallocSet - Sets the underlying allocation routines used by `PetscMalloc()` and `PetscFree()`
198: Not Collective, No Fortran Support
200: Input Parameters:
201: + imalloc - the routine that provides the `malloc()` implementation (also provides `calloc()`, which is used depending on the second argument)
202: . ifree - the routine that provides the `free()` implementation
203: - iralloc - the routine that provides the `realloc()` implementation
205: Level: developer
207: Note:
208: This routine MUST be called before `PetscInitialize()` and may be
209: called only once.
211: .seealso: `PetscMallocClear()`, `PetscInitialize()`, `PetscMalloc()`, `PetscFree()`
212: @*/
213: PetscErrorCode PetscMallocSet(PetscErrorCode (*imalloc)(size_t, PetscBool, int, const char[], const char[], void **), PetscErrorCode (*ifree)(void *, int, const char[], const char[]), PetscErrorCode (*iralloc)(size_t, int, const char[], const char[], void **))
214: {
215: PetscFunctionBegin;
216: PetscCheck(!petscsetmallocvisited || !(imalloc != PetscTrMalloc || ifree != PetscTrFree), PETSC_COMM_SELF, PETSC_ERR_SUP, "cannot call multiple times");
217: PetscTrMalloc = imalloc;
218: PetscTrFree = ifree;
219: PetscTrRealloc = iralloc;
220: petscsetmallocvisited = PETSC_TRUE;
221: PetscFunctionReturn(PETSC_SUCCESS);
222: }
224: /*@
225: PetscMallocClear - Resets the routines used by `PetscMalloc()` and `PetscFree()`
227: Not Collective
229: Level: developer
231: Notes:
232: In general one should never run a PETSc program with different `malloc()` and
233: `free()` settings for different parts; this is because one NEVER wants to
234: `free()` an address that was malloced by a different memory management system
236: Called in `PetscFinalize()` so that if `PetscInitialize()` is called again it starts with a fresh slate of allocation information
238: .seealso: `PetscMallocSet()`, `PetscMalloc()`, `PetscFree()`
239: @*/
240: PetscErrorCode PetscMallocClear(void)
241: {
242: PetscFunctionBegin;
243: PetscTrMalloc = PetscMallocAlign;
244: PetscTrFree = PetscFreeAlign;
245: PetscTrRealloc = PetscReallocAlign;
246: petscsetmallocvisited = PETSC_FALSE;
247: PetscFunctionReturn(PETSC_SUCCESS);
248: }
250: /*@C
251: PetscMemoryTrace - Print the current and high-water memory usage and the delta since the last call, tagged with a user-supplied label
253: Collective on `PETSC_COMM_WORLD`; No Fortran Support
255: Input Parameter:
256: . label - short string used to tag the printed line so successive calls can be distinguished
258: Level: developer
260: Note:
261: Useful for tracking down memory growth between major phases of an application. Uses
262: `PetscMemoryGetCurrentUsage()` and `PetscMallocGetCurrentUsage()` internally.
264: .seealso: `PetscMemoryGetCurrentUsage()`, `PetscMallocGetCurrentUsage()`, `PetscMallocDump()`
265: @*/
266: PetscErrorCode PetscMemoryTrace(const char label[])
267: {
268: PetscLogDouble mem, mal;
269: static PetscLogDouble oldmem = 0, oldmal = 0;
271: PetscFunctionBegin;
272: PetscCall(PetscMemoryGetCurrentUsage(&mem));
273: PetscCall(PetscMallocGetCurrentUsage(&mal));
275: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "%s High water %8.3f MB increase %8.3f MB Current %8.3f MB increase %8.3f MB\n", label, mem * 1e-6, (mem - oldmem) * 1e-6, mal * 1e-6, (mal - oldmal) * 1e-6));
276: oldmem = mem;
277: oldmal = mal;
278: PetscFunctionReturn(PETSC_SUCCESS);
279: }
281: static PetscErrorCode (*PetscTrMallocOld)(size_t, PetscBool, int, const char[], const char[], void **) = PetscMallocAlign;
282: static PetscErrorCode (*PetscTrReallocOld)(size_t, int, const char[], const char[], void **) = PetscReallocAlign;
283: static PetscErrorCode (*PetscTrFreeOld)(void *, int, const char[], const char[]) = PetscFreeAlign;
285: /*@
286: PetscMallocSetDRAM - Set `PetscMalloc()` to use DRAM.
287: If memkind is available, change the memkind type. Otherwise, switch the
288: current malloc and free routines to the `PetscMallocAlign()` and
289: `PetscFreeAlign()` (PETSc default).
291: Not Collective
293: Level: developer
295: Note:
296: This provides a way to do the allocation on DRAM temporarily. One
297: can switch back to the previous choice by calling `PetscMallocReset()`.
299: .seealso: `PetscMallocReset()`, `PetscMalloc()`, `PetscFree()`
300: @*/
301: PetscErrorCode PetscMallocSetDRAM(void)
302: {
303: PetscFunctionBegin;
304: if (PetscTrMalloc == PetscMallocAlign) {
305: #if defined(PETSC_HAVE_MEMKIND)
306: previousmktype = currentmktype;
307: currentmktype = PETSC_MK_DEFAULT;
308: #endif
309: } else {
310: /* Save the previous choice */
311: PetscTrMallocOld = PetscTrMalloc;
312: PetscTrReallocOld = PetscTrRealloc;
313: PetscTrFreeOld = PetscTrFree;
314: PetscTrMalloc = PetscMallocAlign;
315: PetscTrFree = PetscFreeAlign;
316: PetscTrRealloc = PetscReallocAlign;
317: }
318: PetscFunctionReturn(PETSC_SUCCESS);
319: }
321: /*@
322: PetscMallocResetDRAM - Reset the changes made by `PetscMallocSetDRAM()`
324: Not Collective
326: Level: developer
328: .seealso: `PetscMallocSetDRAM()`
329: @*/
330: PetscErrorCode PetscMallocResetDRAM(void)
331: {
332: PetscFunctionBegin;
333: if (PetscTrMalloc == PetscMallocAlign) {
334: #if defined(PETSC_HAVE_MEMKIND)
335: currentmktype = previousmktype;
336: #endif
337: } else {
338: /* Reset to the previous choice */
339: PetscTrMalloc = PetscTrMallocOld;
340: PetscTrRealloc = PetscTrReallocOld;
341: PetscTrFree = PetscTrFreeOld;
342: }
343: PetscFunctionReturn(PETSC_SUCCESS);
344: }
346: static PetscBool petscmalloccoalesce = PetscDefined(USE_MALLOC_COALESCED) ? PETSC_TRUE : PETSC_FALSE;
348: /*@
349: PetscMallocSetCoalesce - Use coalesced `PetscMalloc()` when allocating groups of objects, that is when using `PetscMallocN()`
351: Not Collective
353: Input Parameter:
354: . coalesce - `PETSC_TRUE` to use coalesced malloc for multi-memory allocation.
356: Options Database Key:
357: . -malloc_coalesce - turn coalesced `PetscMallocN()` on or off
359: Level: developer
361: Notes:
362: PETSc uses coalesced `PetscMallocN()` by default for optimized builds and not for debugging builds.
364: This default can be changed via the command-line option `-malloc_coalesce` or by calling this function.
366: This function can only be called immediately after `PetscInitialize()`
368: .seealso: `PetscMallocA()`, `PetscMalloc()`, `PetscFree()`
369: @*/
370: PetscErrorCode PetscMallocSetCoalesce(PetscBool coalesce)
371: {
372: PetscFunctionBegin;
373: petscmalloccoalesce = coalesce;
374: PetscFunctionReturn(PETSC_SUCCESS);
375: }
377: /*@C
378: PetscMallocA - Allocate and optionally clear one or more memory locations, possibly using coalesced malloc
380: Not Collective, No Fortran Support
382: Input Parameters:
383: + n - number of objects to allocate (at least 1)
384: . clear - use `calloc()` to allocate space initialized to zero
385: . lineno - line number to attribute allocation (typically `__LINE__`)
386: . function - function to attribute allocation (typically `PETSC_FUNCTION_NAME`)
387: . filename - file name to attribute allocation (typically `__FILE__`)
388: - bytes0 - first of `n` object sizes
390: Output Parameter:
391: . ptr0 - first of `n` pointers to allocate
393: Level: developer
395: Note:
396: This function is not normally called directly by users, but rather via the macros `PetscMalloc1()`, `PetscMalloc2()`, or `PetscCalloc1()`, etc.
398: .seealso: `PetscMallocAlign()`, `PetscMallocSet()`, `PetscMalloc1()`, `PetscMalloc2()`, `PetscMalloc3()`, `PetscMalloc4()`, `PetscMalloc5()`, `PetscMalloc6()`, `PetscMalloc7()`,
399: `PetscCalloc1()`, `PetscCalloc2()`, `PetscCalloc3()`, `PetscCalloc4()`, `PetscCalloc5()`, `PetscCalloc6()`, `PetscCalloc7()`, `PetscFreeA()`
400: @*/
401: PetscErrorCode PetscMallocA(int n, PetscBool clear, int lineno, const char *function, const char *filename, size_t bytes0, void *ptr0, ...)
402: {
403: va_list Argp;
404: size_t bytes[8], sumbytes;
405: void **ptr[8];
406: int i;
408: PetscFunctionBegin;
409: PetscCheck(n <= 8, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Attempt to allocate %d objects but only 8 supported", n);
410: bytes[0] = bytes0;
411: ptr[0] = (void **)ptr0;
412: sumbytes = (bytes0 + PETSC_MEMALIGN - 1) & ~(PETSC_MEMALIGN - 1);
413: va_start(Argp, ptr0);
414: for (i = 1; i < n; i++) {
415: bytes[i] = va_arg(Argp, size_t);
416: ptr[i] = va_arg(Argp, void **);
417: sumbytes += (bytes[i] + PETSC_MEMALIGN - 1) & ~(PETSC_MEMALIGN - 1);
418: }
419: va_end(Argp);
420: if (petscmalloccoalesce) {
421: char *p;
422: PetscCall((*PetscTrMalloc)(sumbytes, clear, lineno, function, filename, (void **)&p));
423: if (p == NULL) {
424: for (i = 0; i < n; i++) *ptr[i] = NULL;
425: } else {
426: for (i = 0; i < n; i++) {
427: *ptr[i] = bytes[i] ? p : NULL;
428: p = (char *)PetscAddrAlign(p + bytes[i]);
429: }
430: }
431: } else {
432: for (i = 0; i < n; i++) PetscCall((*PetscTrMalloc)(bytes[i], clear, lineno, function, filename, ptr[i]));
433: }
434: PetscFunctionReturn(PETSC_SUCCESS);
435: }
437: /*@C
438: PetscFreeA - Free one or more memory locations, possibly allocated using coalesced `PetscMallocN()`
440: Not Collective, No Fortran Support
442: Input Parameters:
443: + n - number of objects to free (at least 1)
444: . lineno - line number to attribute deallocation (typically `__LINE__`)
445: . function - function to attribute deallocation (typically `PETSC_FUNCTION_NAME`)
446: . filename - file name to attribute deallocation (typically `__FILE__`)
447: - ptr0 - first of `n` pointers to free
449: Level: developer
451: Notes:
452: This function is not normally called directly by users, but rather via the macros `PetscFree()`, `PetscFree2()`, etc.
454: The pointers are zeroed to prevent users from accidentally reusing space that has been freed.
456: If the arguments were obtained via `PetscMallocA()`, `PetscMalloc2()`, `PetscMalloc3()`, etc., then the arguments must be passed in the same order to the corresponding `PetscFreeA()`, `PetscFree2()`, `PetscFree3()`, respectively.
458: .seealso: `PetscMallocAlign()`, `PetscMallocSet()`, `PetscMallocA()`, `PetscFree()`, `PetscFree2()`, `PetscFree3()`, `PetscFree4()`, `PetscFree5()`, `PetscFree6()`, `PetscFree7()`
459: @*/
460: PetscErrorCode PetscFreeA(int n, int lineno, const char *function, const char *filename, void *ptr0, ...)
461: {
462: va_list Argp;
463: void **ptr[8];
464: int i;
466: PetscFunctionBegin;
467: PetscCheck((n >= 1) && (n <= 8), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Attempt to allocate %d objects but only up to 8 supported", n);
468: ptr[0] = (void **)ptr0;
469: va_start(Argp, ptr0);
470: for (i = 1; i < n; i++) ptr[i] = va_arg(Argp, void **);
471: va_end(Argp);
472: if (petscmalloccoalesce) {
473: for (i = 0; i < n; i++) { /* Find first nonempty allocation */
474: if (*ptr[i]) break;
475: }
476: while (--n > i) *ptr[n] = NULL;
477: PetscCall((*PetscTrFree)(*ptr[n], lineno, function, filename));
478: *ptr[n] = NULL;
479: } else {
480: while (--n >= 0) {
481: PetscCall((*PetscTrFree)(*ptr[n], lineno, function, filename));
482: *ptr[n] = NULL;
483: }
484: }
485: PetscFunctionReturn(PETSC_SUCCESS);
486: }