Actual source code: ex7.c
1: static const char help[] = "Tests device allocation, alignment, and asynchronous memory operations.\n\n";
3: #include "petscdevicetestcommon.h"
4: #include <petscdevice_cuda.h>
5: #include <petscdevice_hip.h>
7: #define DebugPrintf(comm, ...) PetscPrintf((comm), "[DEBUG OUTPUT] " __VA_ARGS__)
9: static PetscErrorCode IncrementSize(PetscRandom rand, PetscInt *value)
10: {
11: PetscReal rval;
13: PetscFunctionBeginUser;
14: // set the interval such that *value += rval never goes below 0 or above 500
15: PetscCall(PetscRandomSetInterval(rand, -(*value), 500 - (*value)));
16: PetscCall(PetscRandomGetValueReal(rand, &rval));
17: *value += (PetscInt)rval;
18: PetscCall(DebugPrintf(PetscObjectComm((PetscObject)rand), "n: %" PetscInt_FMT "\n", *value));
19: PetscFunctionReturn(PETSC_SUCCESS);
20: }
22: static PetscErrorCode TestAllocate(PetscDeviceContext dctx, PetscRandom rand, PetscMemType mtype)
23: {
24: PetscScalar *ptr, *tmp_ptr;
25: PetscInt n = 10;
27: PetscFunctionBeginUser;
28: if (PetscMemTypeDevice(mtype)) {
29: PetscDeviceType dtype;
31: PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
32: // host device context cannot handle this
33: if (dtype == PETSC_DEVICE_HOST) PetscFunctionReturn(PETSC_SUCCESS);
34: }
35: // test basic allocation, deallocation
36: PetscCall(IncrementSize(rand, &n));
37: PetscCall(PetscDeviceMalloc(dctx, mtype, n, PETSC_DECIDE, &ptr));
38: PetscCheck(ptr, PETSC_COMM_SELF, PETSC_ERR_POINTER, "PetscDeviceMalloc() return NULL pointer for %s allocation size %" PetscInt_FMT, PetscMemTypeToString(mtype), n);
39: // this ensures the host pointer is at least valid
40: if (PetscMemTypeHost(mtype)) {
41: PetscCall(PetscDeviceContextSynchronize(dctx));
42: for (PetscInt i = 0; i < n; ++i) ptr[i] = (PetscScalar)i;
43: }
44: PetscCall(PetscDeviceFree(dctx, ptr));
46: // test alignment of various types
47: {
48: char *char_ptr;
49: short *short_ptr;
50: int *int_ptr;
51: double *double_ptr;
52: long int *long_int_ptr;
54: PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &char_ptr));
55: PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &short_ptr));
56: PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &int_ptr));
57: PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &double_ptr));
58: PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &long_int_ptr));
60: // if an error occurs here, it means the alignment system is broken!
61: PetscCall(PetscDeviceFree(dctx, char_ptr));
62: PetscCall(PetscDeviceFree(dctx, short_ptr));
63: PetscCall(PetscDeviceFree(dctx, int_ptr));
64: PetscCall(PetscDeviceFree(dctx, double_ptr));
65: PetscCall(PetscDeviceFree(dctx, long_int_ptr));
66: }
68: // test that calloc() produces cleared memory
69: PetscCall(IncrementSize(rand, &n));
70: PetscCall(PetscDeviceCalloc(dctx, mtype, n, PETSC_DECIDE, &ptr));
71: PetscCheck(ptr, PETSC_COMM_SELF, PETSC_ERR_POINTER, "PetscDeviceCalloc() returned NULL pointer for %s allocation size %" PetscInt_FMT, PetscMemTypeToString(mtype), n);
72: if (PetscMemTypeHost(mtype)) {
73: tmp_ptr = ptr;
74: } else {
75: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &tmp_ptr));
76: PetscCall(PetscDeviceArrayCopy(dctx, tmp_ptr, ptr, n));
77: }
78: PetscCall(PetscDeviceContextSynchronize(dctx));
79: for (PetscInt i = 0; i < n; ++i) PetscCheck(tmp_ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceCalloc() returned memory that was not cleared, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(tmp_ptr[i]));
80: if (tmp_ptr == ptr) {
81: tmp_ptr = NULL;
82: } else {
83: PetscCall(PetscDeviceFree(dctx, tmp_ptr));
84: }
85: PetscCall(PetscDeviceFree(dctx, ptr));
87: // test that devicearrayzero produces cleared memory
88: PetscCall(IncrementSize(rand, &n));
89: PetscCall(PetscDeviceMalloc(dctx, mtype, n, PETSC_DECIDE, &ptr));
90: PetscCall(PetscDeviceArrayZero(dctx, ptr, n));
91: PetscCall(PetscMalloc1(n, &tmp_ptr));
92: PetscCall(PetscDeviceRegisterMemory(tmp_ptr, PETSC_MEMTYPE_HOST, n * sizeof(*tmp_ptr)));
93: for (PetscInt i = 0; i < n; ++i) tmp_ptr[i] = (PetscScalar)i;
94: PetscCall(PetscDeviceArrayCopy(dctx, tmp_ptr, ptr, n));
95: PetscCall(PetscDeviceContextSynchronize(dctx));
96: for (PetscInt i = 0; i < n; ++i) PetscCheck(tmp_ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceArrayZero() did not clear memory, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(tmp_ptr[i]));
97: PetscCall(PetscDeviceFree(dctx, tmp_ptr));
98: PetscCall(PetscDeviceFree(dctx, ptr));
99: PetscFunctionReturn(PETSC_SUCCESS);
100: }
102: static PetscErrorCode TestAsyncCoherence(PetscDeviceContext dctx, PetscRandom rand)
103: {
104: const PetscInt nsub = 2;
105: const PetscInt n = 1024;
106: PetscScalar *ptr, *tmp_ptr;
107: PetscDeviceType dtype;
108: PetscDeviceContext *sub;
110: PetscFunctionBeginUser;
111: PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
112: // ensure the streams are nonblocking
113: PetscCall(PetscDeviceContextForkWithStreamType(dctx, PETSC_STREAM_NONBLOCKING, nsub, &sub));
114: // do a warmup to ensure each context acquires any necessary data structures
115: for (PetscInt i = 0; i < nsub; ++i) {
116: PetscCall(PetscDeviceMalloc(sub[i], PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &ptr));
117: PetscCall(PetscDeviceFree(sub[i], ptr));
118: if (dtype != PETSC_DEVICE_HOST) {
119: PetscCall(PetscDeviceMalloc(sub[i], PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &ptr));
120: PetscCall(PetscDeviceFree(sub[i], ptr));
121: }
122: }
124: // allocate on one
125: PetscCall(PetscDeviceMalloc(sub[0], PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &ptr));
126: // free on the other
127: PetscCall(PetscDeviceFree(sub[1], ptr));
129: // allocate on one
130: PetscCall(PetscDeviceMalloc(sub[0], PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &ptr));
131: // zero on the other
132: PetscCall(PetscDeviceArrayZero(sub[1], ptr, n));
133: PetscCall(PetscDeviceContextSynchronize(sub[1]));
134: for (PetscInt i = 0; i < n; ++i) {
135: for (PetscInt i = 0; i < n; ++i) PetscCheck(ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceArrayZero() was not properly serialized, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(ptr[i]));
136: }
137: PetscCall(PetscDeviceFree(sub[1], ptr));
139: // test the transfers are serialized
140: if (dtype != PETSC_DEVICE_HOST) {
141: PetscCall(PetscDeviceCalloc(dctx, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &ptr));
142: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &tmp_ptr));
143: PetscCall(PetscDeviceArrayCopy(sub[0], tmp_ptr, ptr, n));
144: PetscCall(PetscDeviceContextSynchronize(sub[0]));
145: for (PetscInt i = 0; i < n; ++i) {
146: for (PetscInt i = 0; i < n; ++i) PetscCheck(tmp_ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceArrayCopt() was not properly serialized, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(tmp_ptr[i]));
147: }
148: PetscCall(PetscDeviceFree(sub[1], ptr));
149: }
151: PetscCall(PetscDeviceContextJoin(dctx, nsub, PETSC_DEVICE_CONTEXT_JOIN_DESTROY, &sub));
152: PetscFunctionReturn(PETSC_SUCCESS);
153: }
155: static PetscErrorCode CheckAlignment(const void *ptr, size_t alignment)
156: {
157: PetscFunctionBeginUser;
158: PetscCheck(!((PETSC_UINTPTR_T)ptr % alignment), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Pointer %p does not satisfy %zu-byte alignment", ptr, alignment);
159: PetscFunctionReturn(PETSC_SUCCESS);
160: }
162: static PetscErrorCode CheckBarrierCompletion(PetscDeviceContext dctx, const char operation[])
163: {
164: PetscStreamType stype;
165: PetscBool idle;
167: PetscFunctionBeginUser;
168: PetscCall(PetscDeviceContextGetStreamType(dctx, &stype));
169: PetscCall(PetscDeviceContextQueryIdle(dctx, &idle));
170: PetscCheck(idle || (stype != PETSC_STREAM_DEFAULT_WITH_BARRIER && stype != PETSC_STREAM_NONBLOCKING_WITH_BARRIER), PETSC_COMM_SELF, PETSC_ERR_PLIB, "%s returned with pending work on stream type %s", operation, PetscStreamTypes[stype]);
171: PetscFunctionReturn(PETSC_SUCCESS);
172: }
174: static PetscErrorCode TestMemoryStreamTypes(PetscDeviceContext dctx, PetscMemType mtype)
175: {
176: const PetscStreamType types[] = {PETSC_STREAM_DEFAULT, PETSC_STREAM_NONBLOCKING, PETSC_STREAM_DEFAULT_WITH_BARRIER, PETSC_STREAM_NONBLOCKING_WITH_BARRIER};
177: const PetscInt n = 4097;
178: PetscDeviceType dtype;
180: PetscFunctionBeginUser;
181: PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
182: if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
183: for (size_t t = 0; t < PETSC_STATIC_ARRAY_LENGTH(types); ++t) {
184: PetscDeviceContext ctx;
185: unsigned char *ptr, *host;
187: PetscCall(PetscDeviceContextDuplicate(dctx, &ctx));
188: PetscCall(PetscDeviceContextSetStreamType(ctx, types[t]));
189: PetscCall(PetscDeviceContextSetUp(ctx));
190: PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &host));
191: // Warm the pool so allocation cannot hide a missing barrier through runtime initialization.
192: PetscCall(PetscDeviceMalloc(ctx, mtype, n, 256, &ptr));
193: PetscCall(PetscDeviceFree(ctx, ptr));
194: PetscCall(PetscDeviceContextSynchronize(ctx));
195: for (PetscInt clear = 0; clear < 2; ++clear) {
196: PetscCall(PetscDeviceContextDelay(ctx, 0.02));
197: if (clear) PetscCall(PetscDeviceCalloc(ctx, mtype, n, 256, &ptr));
198: else PetscCall(PetscDeviceMalloc(ctx, mtype, n, 256, &ptr));
199: PetscCall(CheckBarrierCompletion(ctx, clear ? "PetscDeviceCalloc()" : "PetscDeviceMalloc()"));
200: PetscCall(CheckAlignment(ptr, 256));
201: PetscCall(PetscDeviceContextSynchronize(ctx));
202: if (clear) {
203: PetscCall(PetscDeviceArrayCopy(ctx, host, ptr, n));
204: PetscCall(PetscDeviceContextSynchronize(ctx));
205: for (PetscInt i = 0; i < n; ++i) PetscCheck(!host[i], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Calloc did not clear byte %" PetscInt_FMT, i);
206: }
207: PetscCall(PetscDeviceContextDelay(ctx, 0.02));
208: PetscCall(PetscDeviceMemset(ctx, ptr, 19 + clear, n));
209: PetscCall(CheckBarrierCompletion(ctx, "PetscDeviceMemset()"));
210: PetscCall(PetscDeviceContextSynchronize(ctx));
211: PetscCall(PetscDeviceContextDelay(ctx, 0.02));
212: PetscCall(PetscDeviceArrayCopy(ctx, host, ptr, n));
213: PetscCall(CheckBarrierCompletion(ctx, "PetscDeviceMemcpy()"));
214: PetscCall(PetscDeviceContextSynchronize(ctx));
215: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 19 + clear, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect value at byte %" PetscInt_FMT, i);
216: PetscCall(PetscDeviceContextDelay(ctx, 0.02));
217: PetscCall(PetscDeviceFree(ctx, ptr));
218: PetscCall(CheckBarrierCompletion(ctx, "PetscDeviceFree()"));
219: PetscCall(PetscDeviceContextSynchronize(ctx));
220: }
221: PetscCall(PetscDeviceFree(ctx, host));
222: PetscCall(PetscDeviceContextSynchronize(ctx));
223: PetscCall(PetscDeviceContextDestroy(&ctx));
224: }
225: PetscFunctionReturn(PETSC_SUCCESS);
226: }
228: static PetscErrorCode TestPinnedAllocationReuse(PetscDeviceContext dctx)
229: {
230: const PetscInt n = 32769;
231: PetscDeviceContext ctx;
232: unsigned char *input, *device, *result;
234: PetscFunctionBeginUser;
235: PetscCall(PetscDeviceContextDuplicate(dctx, &ctx));
236: PetscCall(PetscDeviceContextSetStreamType(ctx, PETSC_STREAM_NONBLOCKING));
237: PetscCall(PetscDeviceContextSetUp(ctx));
238: PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &result));
239: PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &device));
240: PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &input));
241: for (PetscInt pass = 0; pass < 4; ++pass) {
242: PetscCall(PetscDeviceMemset(ctx, input, 37 + pass, n));
243: PetscCall(PetscDeviceContextSynchronize(ctx));
244: PetscCall(PetscDeviceContextDelay(ctx, 0.05));
245: PetscCall(PetscDeviceArrayCopy(ctx, device, input, n));
246: PetscCall(PetscDeviceFree(ctx, input));
247: // Reallocation, including debug initialization, must follow the pending upload.
248: PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &input));
249: PetscCall(PetscDeviceArrayCopy(ctx, result, device, n));
250: PetscCall(PetscDeviceContextSynchronize(ctx));
251: for (PetscInt i = 0; i < n; ++i) PetscCheck(result[i] == 37 + pass, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Pinned allocation reuse corrupted byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)result[i], 37 + pass);
252: }
253: PetscCall(PetscDeviceFree(ctx, input));
254: PetscCall(PetscDeviceFree(ctx, device));
255: PetscCall(PetscDeviceFree(ctx, result));
256: PetscCall(PetscDeviceContextSynchronize(ctx));
257: PetscCall(PetscDeviceContextDestroy(&ctx));
258: PetscFunctionReturn(PETSC_SUCCESS);
259: }
261: static PetscErrorCode TestMemoryAccessOrdering(PetscDeviceContext dctx)
262: {
263: const PetscInt n = 32769;
264: PetscDeviceContext producer, consumer;
265: unsigned char *src[2], *dest, *host;
267: PetscFunctionBeginUser;
268: PetscCall(PetscDeviceContextDuplicate(dctx, &producer));
269: PetscCall(PetscDeviceContextSetStreamType(producer, PETSC_STREAM_NONBLOCKING));
270: PetscCall(PetscDeviceContextSetUp(producer));
271: PetscCall(PetscDeviceContextDuplicate(producer, &consumer));
272: PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &src[0]));
273: PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &src[1]));
274: PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &dest));
275: PetscCall(PetscDeviceMalloc(consumer, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &host));
276: PetscCall(PetscDeviceContextSynchronize(producer));
277: PetscCall(PetscDeviceContextSynchronize(consumer));
278: for (PetscInt pass = 0; pass < 4; ++pass) {
279: const PetscInt value = 37 + pass;
281: // The consumer must wait for the most recent memset, including repeated writes.
282: PetscCall(PetscDeviceMemset(producer, dest, 17, n));
283: PetscCall(PetscDeviceContextDelay(producer, 0.05));
284: PetscCall(PetscDeviceMemset(producer, dest, value, n));
285: PetscCall(PetscDeviceArrayCopy(consumer, host, dest, n));
286: PetscCall(PetscDeviceContextSynchronize(consumer));
287: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Consumer read stale memset byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)host[i], value);
288: PetscCall(PetscDeviceContextSynchronize(producer));
290: PetscCall(PetscDeviceMemset(producer, src[0], value, n));
291: PetscCall(PetscDeviceMemset(producer, src[1], value + 32, n));
292: PetscCall(PetscDeviceContextSynchronize(producer));
293: // The destination dependency must cover the latest copy.
294: PetscCall(PetscDeviceArrayCopy(producer, dest, src[0], n));
295: PetscCall(PetscDeviceContextDelay(producer, 0.05));
296: PetscCall(PetscDeviceArrayCopy(producer, dest, src[1], n));
297: PetscCall(PetscDeviceArrayCopy(consumer, host, dest, n));
298: PetscCall(PetscDeviceContextSynchronize(consumer));
299: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value + 32, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Consumer read stale copy byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)host[i], value + 32);
300: PetscCall(PetscDeviceContextSynchronize(producer));
302: // The source dependency must protect a pending read from a later overwrite.
303: PetscCall(PetscDeviceArrayCopy(producer, dest, src[0], n));
304: PetscCall(PetscDeviceContextDelay(producer, 0.05));
305: PetscCall(PetscDeviceArrayCopy(producer, dest, src[0], n));
306: PetscCall(PetscDeviceMemset(consumer, src[0], value + 64, n));
307: PetscCall(PetscDeviceArrayCopy(producer, host, dest, n));
308: PetscCall(PetscDeviceContextSynchronize(producer));
309: PetscCall(PetscDeviceContextSynchronize(consumer));
310: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Source overwritten before its copy at byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)host[i], value);
311: }
312: PetscCall(PetscDeviceFree(producer, src[0]));
313: PetscCall(PetscDeviceFree(producer, src[1]));
314: PetscCall(PetscDeviceFree(producer, dest));
315: PetscCall(PetscDeviceFree(consumer, host));
316: PetscCall(PetscDeviceContextSynchronize(producer));
317: PetscCall(PetscDeviceContextSynchronize(consumer));
318: PetscCall(PetscDeviceContextDestroy(&producer));
319: PetscCall(PetscDeviceContextDestroy(&consumer));
320: PetscFunctionReturn(PETSC_SUCCESS);
321: }
323: static PetscErrorCode TestTypeAlignment(PetscDeviceContext dctx)
324: {
325: char *prefix;
326: short *s;
327: double *d;
328: PetscScalar *v;
329: struct Triple {
330: double x, y, z;
331: } *triple;
333: PetscFunctionBeginUser;
334: for (PetscInt pass = 0; pass < 8; ++pass) {
335: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, pass + 1, PETSC_DECIDE, &prefix));
336: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 3, PETSC_DECIDE, &s));
337: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 3, 1, &d));
338: PetscCall(PetscDeviceCalloc(dctx, PETSC_MEMTYPE_HOST, 3, PETSC_DECIDE, &v));
339: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 3, PETSC_DECIDE, &triple));
340: PetscCall(CheckAlignment(s, PETSC_DEVICE_ALIGNOF(*s)));
341: PetscCall(CheckAlignment(d, PETSC_DEVICE_ALIGNOF(*d)));
342: PetscCall(CheckAlignment(v, PETSC_DEVICE_ALIGNOF(*v)));
343: PetscCall(CheckAlignment(triple, PETSC_DEVICE_ALIGNOF(*triple)));
344: PetscCall(PetscDeviceContextSynchronize(dctx));
345: for (PetscInt i = 0; i < 3; ++i) {
346: PetscCheck(v[i] == 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Scalar calloc did not clear entry %" PetscInt_FMT, i);
347: s[i] = (short)i;
348: d[i] = i + 0.5;
349: triple[i].x = d[i];
350: triple[i].y = s[i];
351: triple[i].z = triple[i].x + triple[i].y;
352: PetscCheck(triple[i].z == 2 * i + 0.5, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect structure entry");
353: }
354: PetscCall(PetscDeviceFree(dctx, d));
355: PetscCall(PetscDeviceFree(dctx, prefix));
356: PetscCall(PetscDeviceFree(dctx, triple));
357: PetscCall(PetscDeviceFree(dctx, v));
358: PetscCall(PetscDeviceFree(dctx, s));
359: }
360: PetscFunctionReturn(PETSC_SUCCESS);
361: }
363: static PetscErrorCode TestAlignmentStress(PetscDeviceContext dctx, PetscMemType mtype)
364: {
365: const size_t alignments[] = {(size_t)PETSC_DECIDE, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 4096};
366: const PetscInt sizes[] = {0, 1, 3, 7, 17, 255, 256, 257, 2047, 2048, 2049, 4097};
367: PetscInt lengths[12] = {0}, values[12] = {0}, sequence = 0;
368: unsigned char *buffers[12] = {NULL}, *host;
369: PetscDeviceContext ctx[2];
370: PetscDeviceType dtype;
371: PetscBool cupm;
373: PetscFunctionBeginUser;
374: PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
375: cupm = (PetscBool)(dtype == PETSC_DEVICE_CUDA || dtype == PETSC_DEVICE_HIP);
376: if (!cupm && PetscMemTypeDevice(mtype)) PetscFunctionReturn(PETSC_SUCCESS);
377: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 4097, PETSC_DECIDE, &host));
378: for (PetscInt c = 0; c < 2; ++c) {
379: PetscCall(PetscDeviceContextDuplicate(dctx, &ctx[c]));
380: PetscCall(PetscDeviceContextSetStreamType(ctx[c], c ? PETSC_STREAM_NONBLOCKING_WITH_BARRIER : PETSC_STREAM_NONBLOCKING));
381: PetscCall(PetscDeviceContextSetUp(ctx[c]));
382: }
383: for (PetscInt pass = 0; pass < 2; ++pass) {
384: for (size_t a = 0; a < PETSC_STATIC_ARRAY_LENGTH(alignments); ++a) {
385: size_t alignment = alignments[a];
386: size_t expected = alignment == (size_t)PETSC_DECIDE ? PETSC_DEVICE_ALIGNOF(*host) : alignment;
388: if (!cupm && alignment != (size_t)PETSC_DECIDE && PETSC_MEMALIGN % alignment) continue;
389: for (PetscInt clear = 0; clear < 2; ++clear) {
390: for (size_t k = 0; k < PETSC_STATIC_ARRAY_LENGTH(sizes); ++k, ++sequence) {
391: PetscInt slot = sequence % 12, n = sizes[(k + pass) % PETSC_STATIC_ARRAY_LENGTH(sizes)];
392: PetscDeviceContext alloc = ctx[sequence % 2], use = ctx[(sequence + 1) % 2];
394: PetscCall(PetscDeviceFree(use, buffers[slot]));
395: if (clear) PetscCall(PetscDeviceCalloc(alloc, mtype, n, alignment, &buffers[slot]));
396: else PetscCall(PetscDeviceMalloc(alloc, mtype, n, alignment, &buffers[slot]));
397: lengths[slot] = n;
398: values[slot] = 1 + sequence % 251;
399: if (!n) {
400: PetscCheck(!buffers[slot], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Zero-sized allocation did not return NULL");
401: continue;
402: }
403: PetscCall(CheckAlignment(buffers[slot], expected));
404: if (clear) {
405: PetscCall(PetscDeviceArrayCopy(use, host, buffers[slot], n));
406: PetscCall(PetscDeviceContextSynchronize(use));
407: for (PetscInt j = 0; j < n; ++j) PetscCheck(!host[j], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Calloc did not clear byte %" PetscInt_FMT, j);
408: }
409: PetscCall(PetscDeviceMemset(use, buffers[slot], values[slot], n));
410: // Check all live allocations to detect overlap or corruption during pool reuse.
411: for (PetscInt b = 0; b < 12; ++b) {
412: if (!lengths[b]) continue;
413: PetscCall(PetscDeviceArrayCopy(alloc, host, buffers[b], lengths[b]));
414: PetscCall(PetscDeviceContextSynchronize(alloc));
415: for (PetscInt j = 0; j < lengths[b]; ++j) PetscCheck(host[j] == values[b], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Allocation %" PetscInt_FMT " byte %" PetscInt_FMT " was corrupted", b, j);
416: }
417: }
418: }
419: }
420: }
421: for (PetscInt b = 11; b >= 0; --b) PetscCall(PetscDeviceFree(ctx[b % 2], buffers[b]));
422: for (PetscInt c = 0; c < 2; ++c) {
423: PetscCall(PetscDeviceContextSynchronize(ctx[c]));
424: PetscCall(PetscDeviceContextDestroy(&ctx[c]));
425: }
426: PetscCall(PetscDeviceFree(dctx, host));
427: PetscCall(PetscDeviceContextSynchronize(dctx));
428: PetscFunctionReturn(PETSC_SUCCESS);
429: }
431: static PetscScalar ScalarValue(PetscInt i)
432: {
433: #if PetscDefined(USE_COMPLEX)
434: return PetscCMPLX(i + 1, 2 * i + 1);
435: #else
436: return i + 1;
437: #endif
438: }
440: static PetscErrorCode TestScalarAlignment(PetscDeviceContext dctx)
441: {
442: PetscDeviceType dtype;
443: PetscScalar *values[8], *host;
444: char *prefix;
446: PetscFunctionBeginUser;
447: PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
448: if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
449: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 7, sizeof(PetscScalar), &host));
450: PetscCall(PetscDeviceContextSynchronize(dctx));
451: for (PetscInt i = 0; i < 7; ++i) host[i] = ScalarValue(i);
452: for (PetscInt pass = 0; pass < 4; ++pass) {
453: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_DEVICE, 3, 1, &prefix));
454: for (PetscInt b = 0; b < 8; ++b) {
455: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_DEVICE, 7, sizeof(PetscScalar), &values[b]));
456: PetscCall(CheckAlignment(values[b], sizeof(PetscScalar)));
457: PetscCall(PetscDeviceArrayCopy(dctx, values[b], host, 7));
458: #if PetscDefined(HAVE_CUDA)
459: if (dtype == PETSC_DEVICE_CUDA) {
460: cublasHandle_t handle;
461: cublasPointerMode_t mode;
462: const PetscReal alpha = 2;
464: PetscCall(PetscDeviceContextGetBLASHandle_Internal(dctx, &handle));
465: PetscCallCUBLAS(cublasGetPointerMode(handle, &mode));
466: PetscCallCUBLAS(cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST));
467: #if PetscDefined(USE_COMPLEX)
468: #if PetscDefined(USE_REAL_SINGLE)
469: PetscCallCUBLAS(cublasCsscal(handle, 7, &alpha, (cuComplex *)values[b], 1));
470: #else
471: PetscCallCUBLAS(cublasZdscal(handle, 7, &alpha, (cuDoubleComplex *)values[b], 1));
472: #endif
473: #elif PetscDefined(USE_REAL_SINGLE)
474: PetscCallCUBLAS(cublasSscal(handle, 7, &alpha, values[b], 1));
475: #else
476: PetscCallCUBLAS(cublasDscal(handle, 7, &alpha, values[b], 1));
477: #endif
478: PetscCallCUBLAS(cublasSetPointerMode(handle, mode));
479: }
480: #endif
481: #if PetscDefined(HAVE_HIP)
482: if (dtype == PETSC_DEVICE_HIP) {
483: hipblasHandle_t handle;
484: hipblasPointerMode_t mode;
485: const PetscReal alpha = 2;
487: PetscCall(PetscDeviceContextGetBLASHandle_Internal(dctx, &handle));
488: PetscCallHIPBLAS(hipblasGetPointerMode(handle, &mode));
489: PetscCallHIPBLAS(hipblasSetPointerMode(handle, HIPBLAS_POINTER_MODE_HOST));
490: #if PetscDefined(USE_COMPLEX)
491: #if PetscDefined(USE_REAL_SINGLE)
492: PetscCallHIPBLAS(hipblasCsscal(handle, 7, &alpha, (hipFloatComplex *)values[b], 1));
493: #else
494: PetscCallHIPBLAS(hipblasZdscal(handle, 7, &alpha, (hipDoubleComplex *)values[b], 1));
495: #endif
496: #elif PetscDefined(USE_REAL_SINGLE)
497: PetscCallHIPBLAS(hipblasSscal(handle, 7, &alpha, values[b], 1));
498: #else
499: PetscCallHIPBLAS(hipblasDscal(handle, 7, &alpha, values[b], 1));
500: #endif
501: PetscCallHIPBLAS(hipblasSetPointerMode(handle, mode));
502: }
503: #endif
504: }
505: PetscCall(PetscDeviceContextSynchronize(dctx));
506: for (PetscInt b = 7; b >= 0; --b) {
507: PetscCall(PetscDeviceArrayCopy(dctx, host, values[b], 7));
508: PetscCall(PetscDeviceContextSynchronize(dctx));
509: for (PetscInt i = 0; i < 7; ++i) PetscCheck(host[i] == 2 * ScalarValue(i), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect scaled scalar at entry %" PetscInt_FMT, i);
510: PetscCall(PetscDeviceFree(dctx, values[b]));
511: }
512: PetscCall(PetscDeviceFree(dctx, prefix));
513: for (PetscInt i = 0; i < 7; ++i) host[i] = ScalarValue(i);
514: }
515: PetscCall(PetscDeviceFree(dctx, host));
516: PetscCall(PetscDeviceContextSynchronize(dctx));
517: PetscFunctionReturn(PETSC_SUCCESS);
518: }
520: static PetscErrorCode TestAlignmentOrdering(PetscDeviceContext dctx, PetscMemType mtype)
521: {
522: const PetscInt n = 4097;
523: PetscDeviceContext ctx[2], current;
524: PetscDeviceType dtype;
525: unsigned char *output[8], *buffer;
527: PetscFunctionBeginUser;
528: PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
529: if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
530: for (PetscInt c = 0; c < 2; ++c) {
531: PetscCall(PetscDeviceContextDuplicate(dctx, &ctx[c]));
532: PetscCall(PetscDeviceContextSetStreamType(ctx[c], PETSC_STREAM_NONBLOCKING));
533: PetscCall(PetscDeviceContextSetUp(ctx[c]));
534: }
535: for (PetscInt k = 0; k < 8; ++k) PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n, 128, &output[k]));
536: // Warm the pool before delaying allocation, initialization, and reuse.
537: PetscCall(PetscDeviceCalloc(ctx[0], mtype, n, 128, &buffer));
538: PetscCall(PetscDeviceFree(ctx[0], buffer));
539: PetscCall(PetscDeviceContextSynchronize(ctx[0]));
540: for (PetscInt k = 0; k < 8; ++k) {
541: PetscDeviceContext producer = ctx[k % 2], consumer = ctx[(k + 1) % 2];
543: PetscCall(PetscDeviceContextDelay(producer, 0.02));
544: PetscCall(PetscDeviceCalloc(producer, mtype, n, 128, &buffer));
545: PetscCall(CheckAlignment(buffer, 128));
546: if (k % 2) PetscCall(PetscDeviceMemset(producer, buffer, k + 1, n));
547: PetscCall(PetscDeviceArrayCopy(consumer, output[k], buffer, n));
548: // Free must wait for the other stream's read before this chunk can be reused.
549: PetscCall(PetscDeviceFree(producer, buffer));
550: }
551: for (PetscInt c = 0; c < 2; ++c) PetscCall(PetscDeviceContextSynchronize(ctx[c]));
552: for (PetscInt k = 0; k < 8; ++k) {
553: for (PetscInt j = 0; j < n; ++j)
554: PetscCheck(output[k][j] == (k % 2 ? k + 1 : 0), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Stream handoff %" PetscInt_FMT " corrupted byte %" PetscInt_FMT " (got %u, memory type %s)", k, j, (unsigned)output[k][j], PetscMemTypeToString(mtype));
555: PetscCall(PetscDeviceFree(dctx, output[k]));
556: }
557: if (PetscMemTypeHost(mtype)) {
558: const PetscInt nreuse = 32769;
559: unsigned char *input, *device, *result;
561: PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, nreuse, 128, &result));
562: PetscCall(PetscDeviceMalloc(ctx[0], PETSC_MEMTYPE_DEVICE, nreuse, 128, &device));
563: PetscCall(PetscDeviceMalloc(ctx[0], PETSC_MEMTYPE_HOST, nreuse, 128, &input));
564: PetscCall(PetscDeviceMemset(ctx[0], input, 37, nreuse));
565: PetscCall(PetscDeviceContextSynchronize(ctx[0]));
566: PetscCall(PetscDeviceContextDelay(ctx[0], 0.05));
567: PetscCall(PetscDeviceArrayCopy(ctx[0], device, input, nreuse));
568: PetscCall(PetscDeviceFree(ctx[0], input));
569: // Reallocation, including debug initialization, must follow the pending upload.
570: PetscCall(PetscDeviceMalloc(ctx[0], PETSC_MEMTYPE_HOST, nreuse, 128, &input));
571: PetscCall(PetscDeviceArrayCopy(ctx[1], result, device, nreuse));
572: PetscCall(PetscDeviceContextSynchronize(ctx[1]));
573: for (PetscInt j = 0; j < nreuse; ++j) PetscCheck(result[j] == 37, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Pinned allocation reuse corrupted byte %" PetscInt_FMT " (got %u)", j, (unsigned)result[j]);
574: PetscCall(PetscDeviceFree(ctx[0], input));
575: PetscCall(PetscDeviceFree(ctx[1], device));
576: PetscCall(PetscDeviceFree(dctx, result));
577: PetscCall(PetscDeviceContextSynchronize(ctx[0]));
578: PetscCall(PetscDeviceContextSynchronize(ctx[1]));
579: }
580: // Typed accesses must also use the explicit context while another remains current.
581: if (PetscMemTypeDevice(mtype)) PetscCall(TestScalarAlignment(ctx[0]));
582: PetscCall(PetscDeviceContextGetCurrentContext(¤t));
583: PetscCheck(current == dctx, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Explicit-context allocation changed the current context");
584: for (PetscInt c = 0; c < 2; ++c) PetscCall(PetscDeviceContextDestroy(&ctx[c]));
585: PetscCall(PetscDeviceContextSynchronize(dctx));
586: PetscFunctionReturn(PETSC_SUCCESS);
587: }
589: /*
590: Both helpers call the context's backend methods directly to leave memory accesses untracked.
591: PetscDeviceMemset() and PetscDeviceMemcpy() would register these accesses and make later
592: operations wait automatically, hiding missing explicit PetscDeviceContextWaitForContext()
593: calls in TestContextOrdering().
594: */
595: static PetscErrorCode SetDeviceBytes(PetscDeviceContext dctx, void *ptr, PetscInt value, size_t n)
596: {
597: PetscFunctionBeginUser;
598: PetscUseTypeMethod(dctx, memset, PETSC_MEMTYPE_DEVICE, ptr, value, n);
599: PetscFunctionReturn(PETSC_SUCCESS);
600: }
602: static PetscErrorCode CopyDeviceBytesToHost(PetscDeviceContext dctx, void *host, const void *ptr, size_t n)
603: {
604: PetscFunctionBeginUser;
605: PetscUseTypeMethod(dctx, memcopy, host, ptr, n, PETSC_DEVICE_COPY_DTOH);
606: PetscFunctionReturn(PETSC_SUCCESS);
607: }
609: static PetscErrorCode TestContextOrdering(PetscDeviceContext saved)
610: {
611: const PetscInt n = 32769;
612: PetscDeviceContext current, other, standard, contexts[3];
613: PetscDeviceType dtype;
614: unsigned char *src, *dest, *host;
616: PetscFunctionBeginUser;
617: PetscCall(PetscDeviceContextGetDeviceType(saved, &dtype));
618: if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
619: PetscCall(PetscDeviceContextDuplicate(saved, ¤t));
620: PetscCall(PetscDeviceContextSetStreamType(current, PETSC_STREAM_NONBLOCKING));
621: PetscCall(PetscDeviceContextSetUp(current));
622: PetscCall(PetscDeviceContextDuplicate(current, &other));
623: standard = PetscDeviceContextDefault;
624: PetscCall(PetscDeviceContextSetCurrentContext(current));
625: contexts[0] = standard;
626: contexts[1] = current;
627: contexts[2] = other;
628: PetscCall(PetscDeviceMalloc(current, PETSC_MEMTYPE_HOST, n, 128, &host));
630: // Memory operations must use the supplied context even when another context is current.
631: PetscCall(PetscDeviceCalloc(standard, PETSC_MEMTYPE_HOST, n, 128, &src));
632: PetscCall(PetscDeviceContextSynchronize(standard));
633: PetscCall(PetscDeviceContextSynchronize(current));
634: PetscCall(PetscDeviceContextDelay(current, 0.05));
635: PetscCall(PetscDeviceMemset(standard, src, 73, n));
636: PetscCall(PetscDeviceContextSynchronize(standard));
637: for (PetscInt i = 0; i < n; ++i) PetscCheck(src[i] == 73, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Synchronization did not complete the explicit-context memory operation at byte %" PetscInt_FMT " (got %u)", i, (unsigned)src[i]);
638: PetscCall(PetscDeviceContextSynchronize(current));
639: PetscCall(PetscDeviceFree(standard, src));
641: // Track dependencies between a default-stream context and two nonblocking contexts.
642: for (PetscInt p = 0; p < 3; ++p) {
643: for (PetscInt c = 0; c < 3; ++c) {
644: PetscDeviceContext alloc = contexts[(p + c) % 3], producer = contexts[p], consumer = contexts[c];
645: PetscInt value = 17 + 3 * p + c;
647: PetscCall(PetscDeviceCalloc(alloc, PETSC_MEMTYPE_DEVICE, n, 128, &src));
648: PetscCall(PetscDeviceMalloc(alloc, PETSC_MEMTYPE_DEVICE, n, 128, &dest));
649: PetscCall(PetscDeviceContextSynchronize(alloc));
650: PetscCall(PetscDeviceContextDelay(producer, 0.02));
651: PetscCall(PetscDeviceMemset(producer, src, value, n));
652: PetscCall(PetscDeviceContextDelay(consumer, 0.02));
653: PetscCall(PetscDeviceArrayCopy(consumer, dest, src, n));
654: PetscCall(PetscDeviceFree(producer, src));
655: PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, 128, &src));
656: PetscCall(PetscDeviceMemset(producer, src, 99, n));
657: PetscCall(PetscDeviceArrayCopy(consumer, host, dest, n));
658: PetscCall(PetscDeviceContextSynchronize(consumer));
659: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Tracked transfer %" PetscInt_FMT " -> %" PetscInt_FMT " corrupted byte %" PetscInt_FMT " (got %u)", p, c, i, (unsigned)host[i]);
660: PetscCall(PetscDeviceFree(alloc, src));
661: PetscCall(PetscDeviceFree(alloc, dest));
662: }
663: }
664: for (PetscInt c = 0; c < 3; ++c) PetscCall(PetscDeviceContextSynchronize(contexts[c]));
666: // Warm reuse before testing allocation on standard followed by an untracked write on current.
667: PetscCall(PetscDeviceMalloc(standard, PETSC_MEMTYPE_DEVICE, n, 128, &src));
668: PetscCall(PetscDeviceFree(standard, src));
669: PetscCall(PetscDeviceContextSynchronize(standard));
670: PetscCall(PetscDeviceContextDelay(standard, 0.05));
671: PetscCall(PetscDeviceCalloc(standard, PETSC_MEMTYPE_DEVICE, n, 128, &src));
672: PetscCall(PetscDeviceContextWaitForContext(current, standard));
673: PetscCall(SetDeviceBytes(current, src, 63, n));
674: PetscCall(PetscDeviceArrayCopy(current, host, src, n));
675: PetscCall(PetscDeviceContextSynchronize(current));
676: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 63, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Allocation-to-raw-write ordering corrupted byte %" PetscInt_FMT " (got %u)", i, (unsigned)host[i]);
678: // A raw producer is not registered with memory tracking, so copying on standard needs a wait.
679: PetscCall(PetscDeviceContextDelay(current, 0.05));
680: PetscCall(SetDeviceBytes(current, src, 37, n));
681: PetscCall(PetscDeviceContextWaitForContext(standard, current));
682: PetscCall(PetscDeviceArrayCopy(standard, host, src, n));
683: PetscCall(PetscDeviceContextSynchronize(standard));
684: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 37, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Raw-write-to-copy ordering corrupted byte %" PetscInt_FMT " (got %u)", i, (unsigned)host[i]);
686: // Releasing and reusing on standard must also follow an untracked read on current.
687: PetscCall(PetscDeviceContextDelay(current, 0.05));
688: PetscCall(CopyDeviceBytesToHost(current, host, src, n));
689: PetscCall(PetscDeviceContextWaitForContext(standard, current));
690: PetscCall(PetscDeviceFree(standard, src));
691: PetscCall(PetscDeviceMalloc(standard, PETSC_MEMTYPE_DEVICE, n, 128, &src));
692: PetscCall(PetscDeviceMemset(standard, src, 99, n));
693: PetscCall(PetscDeviceContextSynchronize(standard));
694: PetscCall(PetscDeviceContextSynchronize(current));
695: for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 37, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Raw-read-to-reuse ordering corrupted byte %" PetscInt_FMT " (got %u)", i, (unsigned)host[i]);
696: PetscCall(PetscDeviceFree(standard, src));
697: PetscCall(PetscDeviceFree(current, host));
698: for (PetscInt c = 0; c < 3; ++c) PetscCall(PetscDeviceContextSynchronize(contexts[c]));
699: PetscCall(PetscDeviceContextSetCurrentContext(saved));
700: PetscCall(PetscDeviceContextDestroy(&other));
701: PetscCall(PetscDeviceContextDestroy(¤t));
702: PetscFunctionReturn(PETSC_SUCCESS);
703: }
705: int main(int argc, char *argv[])
706: {
707: PetscDeviceContext dctx;
708: PetscRandom rand;
709: PetscBool test_alignment = PETSC_FALSE, test_context_policy = PETSC_FALSE, test_stream_types = PETSC_FALSE, test_pinned_reuse = PETSC_FALSE, test_memory_access = PETSC_FALSE;
711: PetscFunctionBeginUser;
712: PetscCall(PetscInitialize(&argc, &argv, NULL, help));
714: PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_alignment", &test_alignment, NULL));
715: PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_context_policy", &test_context_policy, NULL));
716: PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_stream_types", &test_stream_types, NULL));
717: PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_pinned_reuse", &test_pinned_reuse, NULL));
718: PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_memory_access", &test_memory_access, NULL));
719: if (test_stream_types) {
720: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
721: PetscCall(TestMemoryStreamTypes(dctx, PETSC_MEMTYPE_HOST));
722: PetscCall(TestMemoryStreamTypes(dctx, PETSC_MEMTYPE_DEVICE));
723: } else if (test_pinned_reuse) {
724: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
725: PetscCall(TestPinnedAllocationReuse(dctx));
726: } else if (test_memory_access) {
727: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
728: PetscCall(TestMemoryAccessOrdering(dctx));
729: } else if (test_context_policy) {
730: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
731: PetscCall(TestContextOrdering(dctx));
732: } else if (test_alignment) {
733: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
734: PetscCall(TestTypeAlignment(dctx));
735: PetscCall(TestAlignmentStress(dctx, PETSC_MEMTYPE_HOST));
736: PetscCall(TestAlignmentStress(dctx, PETSC_MEMTYPE_DEVICE));
737: PetscCall(TestAlignmentOrdering(dctx, PETSC_MEMTYPE_HOST));
738: PetscCall(TestAlignmentOrdering(dctx, PETSC_MEMTYPE_DEVICE));
739: PetscCall(TestTypeAlignment(PetscDeviceContextDefault));
740: PetscCall(PetscDeviceContextSynchronize(PetscDeviceContextDefault));
741: } else {
742: // A vile hack. The -info output is used to test correctness in this test which prints --
743: // among other things -- the PetscObjectId of the PetscDevicContext and the allocated memory.
744: //
745: // Due to device and host creating slightly different number of objects on startup there will
746: // be a mismatch in the ID's. So for the tests involving the host we sit here creating
747: // PetscContainers (and incrementing the global PetscObjectId counter) until it reaches some
748: // arbitrarily high number to ensure that our first PetscDeviceContext has the same ID across
749: // systems.
750: {
751: PetscObjectId prev_id = 0;
753: do {
754: PetscContainer c;
755: PetscObjectId id;
757: PetscCall(PetscContainerCreate(PETSC_COMM_WORLD, &c));
758: PetscCall(PetscObjectGetId((PetscObject)c, &id));
759: // sanity check, in case PetscContainer ever stops being a PetscObject
760: PetscCheck(id > prev_id, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscObjectIds are not increasing for successively created PetscContainers! current: %" PetscInt64_FMT ", previous: %" PetscInt64_FMT, id, prev_id);
761: prev_id = id;
762: PetscCall(PetscContainerDestroy(&c));
763: } while (prev_id < 50);
764: }
765: PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
767: PetscCall(PetscRandomCreate(PETSC_COMM_WORLD, &rand));
768: // this seed just so happens to keep the allocation size increasing
769: PetscCall(PetscRandomSetSeed(rand, 123));
770: PetscCall(PetscRandomSeed(rand));
771: PetscCall(PetscRandomSetFromOptions(rand));
773: PetscCall(TestAllocate(dctx, rand, PETSC_MEMTYPE_HOST));
774: PetscCall(TestAllocate(dctx, rand, PETSC_MEMTYPE_DEVICE));
775: PetscCall(TestAsyncCoherence(dctx, rand));
777: PetscCall(PetscRandomDestroy(&rand));
778: }
779: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "EXIT_SUCCESS\n"));
780: PetscCall(PetscFinalize());
781: return 0;
782: }
784: /*TEST
786: testset:
787: args: -test_memory_access
788: output_file: output/ExitSuccess.out
789: test:
790: suffix: memory_access_cuda
791: requires: cuda
792: args: -default_device_type cuda
793: test:
794: suffix: memory_access_hip
795: requires: hip
796: args: -default_device_type hip
798: testset:
799: requires: defined(PETSC_USE_DEBUG)
800: args: -test_pinned_reuse
801: output_file: output/ExitSuccess.out
802: test:
803: suffix: pinned_reuse_cuda
804: requires: cuda
805: args: -default_device_type cuda
806: test:
807: suffix: pinned_reuse_hip
808: requires: hip
809: args: -default_device_type hip
811: testset:
812: args: -test_stream_types
813: output_file: output/ExitSuccess.out
814: test:
815: suffix: stream_types_cuda
816: requires: cuda
817: args: -default_device_type cuda
818: test:
819: suffix: stream_types_hip
820: requires: hip
821: args: -default_device_type hip
823: testset:
824: args: -test_context_policy
825: output_file: output/ExitSuccess.out
826: test:
827: suffix: context_policy_cuda
828: requires: cuda
829: args: -default_device_type cuda
830: test:
831: suffix: context_policy_hip
832: requires: hip
833: args: -default_device_type hip
835: testset:
836: args: -test_alignment
837: output_file: output/ExitSuccess.out
838: test:
839: suffix: alignment_host
840: args: -default_device_type host -options_left 0
841: test:
842: suffix: alignment_cuda
843: requires: cuda
844: args: -default_device_type cuda
845: test:
846: suffix: alignment_hip
847: requires: hip
848: args: -default_device_type hip
850: testset:
851: requires: defined(PETSC_USE_INFO) defined(PETSC_USE_DEBUG) defined(PETSC_DEVICELANGUAGE_CXX)
852: args: -info :device
853: suffix: with_info
854: test:
855: requires: !device
856: suffix: host_no_device
857: test:
858: requires: device
859: args: -default_device_type host
860: filter: sed -e 's/host/IMPL/g' -e 's/cuda/IMPL/g' -e 's/hip/IMPL/g' -e 's/sycl/IMPL/g'
861: suffix: host_with_device
862: test:
863: requires: cuda
864: args: -default_device_type cuda
865: suffix: cuda
866: test:
867: requires: hip
868: args: -default_device_type hip
869: suffix: hip
870: test:
871: requires: sycl
872: TODO: unclear if it is needed
873: args: -default_device_type sycl
874: suffix: sycl
876: testset:
877: output_file: output/ExitSuccess.out
878: requires: !defined(PETSC_USE_DEBUG) defined(PETSC_DEVICELANGUAGE_CXX)
879: filter: grep -v "\[DEBUG OUTPUT\]"
880: suffix: no_info
881: test:
882: requires: !device
883: suffix: host_no_device
884: test:
885: requires: device
886: args: -default_device_type host
887: suffix: host_with_device
888: test:
889: requires: cuda
890: args: -default_device_type cuda
891: suffix: cuda
892: test:
893: requires: hip
894: args: -default_device_type hip
895: suffix: hip
896: test:
897: requires: sycl
898: TODO: unclear if it is needed
899: args: -default_device_type sycl
900: suffix: sycl
902: test:
903: requires: !defined(PETSC_DEVICELANGUAGE_CXX)
904: output_file: output/ExitSuccess.out
905: filter: grep -v "\[DEBUG OUTPUT\]"
906: suffix: no_cxx
908: TEST*/