Actual source code: ex7.c

  1: static const char help[] = "Tests device allocation, alignment, and asynchronous memory operations.\n\n";

  3: #include "petscdevicetestcommon.h"
  4: #include <petscdevice_cuda.h>
  5: #include <petscdevice_hip.h>

  7: #define DebugPrintf(comm, ...) PetscPrintf((comm), "[DEBUG OUTPUT] " __VA_ARGS__)

  9: static PetscErrorCode IncrementSize(PetscRandom rand, PetscInt *value)
 10: {
 11:   PetscReal rval;

 13:   PetscFunctionBeginUser;
 14:   // set the interval such that *value += rval never goes below 0 or above 500
 15:   PetscCall(PetscRandomSetInterval(rand, -(*value), 500 - (*value)));
 16:   PetscCall(PetscRandomGetValueReal(rand, &rval));
 17:   *value += (PetscInt)rval;
 18:   PetscCall(DebugPrintf(PetscObjectComm((PetscObject)rand), "n: %" PetscInt_FMT "\n", *value));
 19:   PetscFunctionReturn(PETSC_SUCCESS);
 20: }

 22: static PetscErrorCode TestAllocate(PetscDeviceContext dctx, PetscRandom rand, PetscMemType mtype)
 23: {
 24:   PetscScalar *ptr, *tmp_ptr;
 25:   PetscInt     n = 10;

 27:   PetscFunctionBeginUser;
 28:   if (PetscMemTypeDevice(mtype)) {
 29:     PetscDeviceType dtype;

 31:     PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
 32:     // host device context cannot handle this
 33:     if (dtype == PETSC_DEVICE_HOST) PetscFunctionReturn(PETSC_SUCCESS);
 34:   }
 35:   // test basic allocation, deallocation
 36:   PetscCall(IncrementSize(rand, &n));
 37:   PetscCall(PetscDeviceMalloc(dctx, mtype, n, PETSC_DECIDE, &ptr));
 38:   PetscCheck(ptr, PETSC_COMM_SELF, PETSC_ERR_POINTER, "PetscDeviceMalloc() return NULL pointer for %s allocation size %" PetscInt_FMT, PetscMemTypeToString(mtype), n);
 39:   // this ensures the host pointer is at least valid
 40:   if (PetscMemTypeHost(mtype)) {
 41:     PetscCall(PetscDeviceContextSynchronize(dctx));
 42:     for (PetscInt i = 0; i < n; ++i) ptr[i] = (PetscScalar)i;
 43:   }
 44:   PetscCall(PetscDeviceFree(dctx, ptr));

 46:   // test alignment of various types
 47:   {
 48:     char     *char_ptr;
 49:     short    *short_ptr;
 50:     int      *int_ptr;
 51:     double   *double_ptr;
 52:     long int *long_int_ptr;

 54:     PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &char_ptr));
 55:     PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &short_ptr));
 56:     PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &int_ptr));
 57:     PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &double_ptr));
 58:     PetscCall(PetscDeviceMalloc(dctx, mtype, 1, PETSC_DECIDE, &long_int_ptr));

 60:     // if an error occurs here, it means the alignment system is broken!
 61:     PetscCall(PetscDeviceFree(dctx, char_ptr));
 62:     PetscCall(PetscDeviceFree(dctx, short_ptr));
 63:     PetscCall(PetscDeviceFree(dctx, int_ptr));
 64:     PetscCall(PetscDeviceFree(dctx, double_ptr));
 65:     PetscCall(PetscDeviceFree(dctx, long_int_ptr));
 66:   }

 68:   // test that calloc() produces cleared memory
 69:   PetscCall(IncrementSize(rand, &n));
 70:   PetscCall(PetscDeviceCalloc(dctx, mtype, n, PETSC_DECIDE, &ptr));
 71:   PetscCheck(ptr, PETSC_COMM_SELF, PETSC_ERR_POINTER, "PetscDeviceCalloc() returned NULL pointer for %s allocation size %" PetscInt_FMT, PetscMemTypeToString(mtype), n);
 72:   if (PetscMemTypeHost(mtype)) {
 73:     tmp_ptr = ptr;
 74:   } else {
 75:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &tmp_ptr));
 76:     PetscCall(PetscDeviceArrayCopy(dctx, tmp_ptr, ptr, n));
 77:   }
 78:   PetscCall(PetscDeviceContextSynchronize(dctx));
 79:   for (PetscInt i = 0; i < n; ++i) PetscCheck(tmp_ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceCalloc() returned memory that was not cleared, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(tmp_ptr[i]));
 80:   if (tmp_ptr == ptr) {
 81:     tmp_ptr = NULL;
 82:   } else {
 83:     PetscCall(PetscDeviceFree(dctx, tmp_ptr));
 84:   }
 85:   PetscCall(PetscDeviceFree(dctx, ptr));

 87:   // test that devicearrayzero produces cleared memory
 88:   PetscCall(IncrementSize(rand, &n));
 89:   PetscCall(PetscDeviceMalloc(dctx, mtype, n, PETSC_DECIDE, &ptr));
 90:   PetscCall(PetscDeviceArrayZero(dctx, ptr, n));
 91:   PetscCall(PetscMalloc1(n, &tmp_ptr));
 92:   PetscCall(PetscDeviceRegisterMemory(tmp_ptr, PETSC_MEMTYPE_HOST, n * sizeof(*tmp_ptr)));
 93:   for (PetscInt i = 0; i < n; ++i) tmp_ptr[i] = (PetscScalar)i;
 94:   PetscCall(PetscDeviceArrayCopy(dctx, tmp_ptr, ptr, n));
 95:   PetscCall(PetscDeviceContextSynchronize(dctx));
 96:   for (PetscInt i = 0; i < n; ++i) PetscCheck(tmp_ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceArrayZero() did not clear memory, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(tmp_ptr[i]));
 97:   PetscCall(PetscDeviceFree(dctx, tmp_ptr));
 98:   PetscCall(PetscDeviceFree(dctx, ptr));
 99:   PetscFunctionReturn(PETSC_SUCCESS);
100: }

102: static PetscErrorCode TestAsyncCoherence(PetscDeviceContext dctx, PetscRandom rand)
103: {
104:   const PetscInt      nsub = 2;
105:   const PetscInt      n    = 1024;
106:   PetscScalar        *ptr, *tmp_ptr;
107:   PetscDeviceType     dtype;
108:   PetscDeviceContext *sub;

110:   PetscFunctionBeginUser;
111:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
112:   // ensure the streams are nonblocking
113:   PetscCall(PetscDeviceContextForkWithStreamType(dctx, PETSC_STREAM_NONBLOCKING, nsub, &sub));
114:   // do a warmup to ensure each context acquires any necessary data structures
115:   for (PetscInt i = 0; i < nsub; ++i) {
116:     PetscCall(PetscDeviceMalloc(sub[i], PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &ptr));
117:     PetscCall(PetscDeviceFree(sub[i], ptr));
118:     if (dtype != PETSC_DEVICE_HOST) {
119:       PetscCall(PetscDeviceMalloc(sub[i], PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &ptr));
120:       PetscCall(PetscDeviceFree(sub[i], ptr));
121:     }
122:   }

124:   // allocate on one
125:   PetscCall(PetscDeviceMalloc(sub[0], PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &ptr));
126:   // free on the other
127:   PetscCall(PetscDeviceFree(sub[1], ptr));

129:   // allocate on one
130:   PetscCall(PetscDeviceMalloc(sub[0], PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &ptr));
131:   // zero on the other
132:   PetscCall(PetscDeviceArrayZero(sub[1], ptr, n));
133:   PetscCall(PetscDeviceContextSynchronize(sub[1]));
134:   for (PetscInt i = 0; i < n; ++i) {
135:     for (PetscInt i = 0; i < n; ++i) PetscCheck(ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceArrayZero() was not properly serialized, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(ptr[i]));
136:   }
137:   PetscCall(PetscDeviceFree(sub[1], ptr));

139:   // test the transfers are serialized
140:   if (dtype != PETSC_DEVICE_HOST) {
141:     PetscCall(PetscDeviceCalloc(dctx, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &ptr));
142:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &tmp_ptr));
143:     PetscCall(PetscDeviceArrayCopy(sub[0], tmp_ptr, ptr, n));
144:     PetscCall(PetscDeviceContextSynchronize(sub[0]));
145:     for (PetscInt i = 0; i < n; ++i) {
146:       for (PetscInt i = 0; i < n; ++i) PetscCheck(tmp_ptr[i] == (PetscScalar)0.0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscDeviceArrayCopt() was not properly serialized, ptr[%" PetscInt_FMT "] %g != 0", i, (double)PetscAbsScalar(tmp_ptr[i]));
147:     }
148:     PetscCall(PetscDeviceFree(sub[1], ptr));
149:   }

151:   PetscCall(PetscDeviceContextJoin(dctx, nsub, PETSC_DEVICE_CONTEXT_JOIN_DESTROY, &sub));
152:   PetscFunctionReturn(PETSC_SUCCESS);
153: }

155: static PetscErrorCode CheckAlignment(const void *ptr, size_t alignment)
156: {
157:   PetscFunctionBeginUser;
158:   PetscCheck(!((PETSC_UINTPTR_T)ptr % alignment), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Pointer %p does not satisfy %zu-byte alignment", ptr, alignment);
159:   PetscFunctionReturn(PETSC_SUCCESS);
160: }

162: static PetscErrorCode CheckBarrierCompletion(PetscDeviceContext dctx, const char operation[])
163: {
164:   PetscStreamType stype;
165:   PetscBool       idle;

167:   PetscFunctionBeginUser;
168:   PetscCall(PetscDeviceContextGetStreamType(dctx, &stype));
169:   PetscCall(PetscDeviceContextQueryIdle(dctx, &idle));
170:   PetscCheck(idle || (stype != PETSC_STREAM_DEFAULT_WITH_BARRIER && stype != PETSC_STREAM_NONBLOCKING_WITH_BARRIER), PETSC_COMM_SELF, PETSC_ERR_PLIB, "%s returned with pending work on stream type %s", operation, PetscStreamTypes[stype]);
171:   PetscFunctionReturn(PETSC_SUCCESS);
172: }

174: static PetscErrorCode TestMemoryStreamTypes(PetscDeviceContext dctx, PetscMemType mtype)
175: {
176:   const PetscStreamType types[] = {PETSC_STREAM_DEFAULT, PETSC_STREAM_NONBLOCKING, PETSC_STREAM_DEFAULT_WITH_BARRIER, PETSC_STREAM_NONBLOCKING_WITH_BARRIER};
177:   const PetscInt        n       = 4097;
178:   PetscDeviceType       dtype;

180:   PetscFunctionBeginUser;
181:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
182:   if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
183:   for (size_t t = 0; t < PETSC_STATIC_ARRAY_LENGTH(types); ++t) {
184:     PetscDeviceContext ctx;
185:     unsigned char     *ptr, *host;

187:     PetscCall(PetscDeviceContextDuplicate(dctx, &ctx));
188:     PetscCall(PetscDeviceContextSetStreamType(ctx, types[t]));
189:     PetscCall(PetscDeviceContextSetUp(ctx));
190:     PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &host));
191:     // Warm the pool so allocation cannot hide a missing barrier through runtime initialization.
192:     PetscCall(PetscDeviceMalloc(ctx, mtype, n, 256, &ptr));
193:     PetscCall(PetscDeviceFree(ctx, ptr));
194:     PetscCall(PetscDeviceContextSynchronize(ctx));
195:     for (PetscInt clear = 0; clear < 2; ++clear) {
196:       PetscCall(PetscDeviceContextDelay(ctx, 0.02));
197:       if (clear) PetscCall(PetscDeviceCalloc(ctx, mtype, n, 256, &ptr));
198:       else PetscCall(PetscDeviceMalloc(ctx, mtype, n, 256, &ptr));
199:       PetscCall(CheckBarrierCompletion(ctx, clear ? "PetscDeviceCalloc()" : "PetscDeviceMalloc()"));
200:       PetscCall(CheckAlignment(ptr, 256));
201:       PetscCall(PetscDeviceContextSynchronize(ctx));
202:       if (clear) {
203:         PetscCall(PetscDeviceArrayCopy(ctx, host, ptr, n));
204:         PetscCall(PetscDeviceContextSynchronize(ctx));
205:         for (PetscInt i = 0; i < n; ++i) PetscCheck(!host[i], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Calloc did not clear byte %" PetscInt_FMT, i);
206:       }
207:       PetscCall(PetscDeviceContextDelay(ctx, 0.02));
208:       PetscCall(PetscDeviceMemset(ctx, ptr, 19 + clear, n));
209:       PetscCall(CheckBarrierCompletion(ctx, "PetscDeviceMemset()"));
210:       PetscCall(PetscDeviceContextSynchronize(ctx));
211:       PetscCall(PetscDeviceContextDelay(ctx, 0.02));
212:       PetscCall(PetscDeviceArrayCopy(ctx, host, ptr, n));
213:       PetscCall(CheckBarrierCompletion(ctx, "PetscDeviceMemcpy()"));
214:       PetscCall(PetscDeviceContextSynchronize(ctx));
215:       for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 19 + clear, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect value at byte %" PetscInt_FMT, i);
216:       PetscCall(PetscDeviceContextDelay(ctx, 0.02));
217:       PetscCall(PetscDeviceFree(ctx, ptr));
218:       PetscCall(CheckBarrierCompletion(ctx, "PetscDeviceFree()"));
219:       PetscCall(PetscDeviceContextSynchronize(ctx));
220:     }
221:     PetscCall(PetscDeviceFree(ctx, host));
222:     PetscCall(PetscDeviceContextSynchronize(ctx));
223:     PetscCall(PetscDeviceContextDestroy(&ctx));
224:   }
225:   PetscFunctionReturn(PETSC_SUCCESS);
226: }

228: static PetscErrorCode TestPinnedAllocationReuse(PetscDeviceContext dctx)
229: {
230:   const PetscInt     n = 32769;
231:   PetscDeviceContext ctx;
232:   unsigned char     *input, *device, *result;

234:   PetscFunctionBeginUser;
235:   PetscCall(PetscDeviceContextDuplicate(dctx, &ctx));
236:   PetscCall(PetscDeviceContextSetStreamType(ctx, PETSC_STREAM_NONBLOCKING));
237:   PetscCall(PetscDeviceContextSetUp(ctx));
238:   PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &result));
239:   PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &device));
240:   PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &input));
241:   for (PetscInt pass = 0; pass < 4; ++pass) {
242:     PetscCall(PetscDeviceMemset(ctx, input, 37 + pass, n));
243:     PetscCall(PetscDeviceContextSynchronize(ctx));
244:     PetscCall(PetscDeviceContextDelay(ctx, 0.05));
245:     PetscCall(PetscDeviceArrayCopy(ctx, device, input, n));
246:     PetscCall(PetscDeviceFree(ctx, input));
247:     // Reallocation, including debug initialization, must follow the pending upload.
248:     PetscCall(PetscDeviceMalloc(ctx, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &input));
249:     PetscCall(PetscDeviceArrayCopy(ctx, result, device, n));
250:     PetscCall(PetscDeviceContextSynchronize(ctx));
251:     for (PetscInt i = 0; i < n; ++i) PetscCheck(result[i] == 37 + pass, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Pinned allocation reuse corrupted byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)result[i], 37 + pass);
252:   }
253:   PetscCall(PetscDeviceFree(ctx, input));
254:   PetscCall(PetscDeviceFree(ctx, device));
255:   PetscCall(PetscDeviceFree(ctx, result));
256:   PetscCall(PetscDeviceContextSynchronize(ctx));
257:   PetscCall(PetscDeviceContextDestroy(&ctx));
258:   PetscFunctionReturn(PETSC_SUCCESS);
259: }

261: static PetscErrorCode TestMemoryAccessOrdering(PetscDeviceContext dctx)
262: {
263:   const PetscInt     n = 32769;
264:   PetscDeviceContext producer, consumer;
265:   unsigned char     *src[2], *dest, *host;

267:   PetscFunctionBeginUser;
268:   PetscCall(PetscDeviceContextDuplicate(dctx, &producer));
269:   PetscCall(PetscDeviceContextSetStreamType(producer, PETSC_STREAM_NONBLOCKING));
270:   PetscCall(PetscDeviceContextSetUp(producer));
271:   PetscCall(PetscDeviceContextDuplicate(producer, &consumer));
272:   PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &src[0]));
273:   PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &src[1]));
274:   PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, PETSC_DECIDE, &dest));
275:   PetscCall(PetscDeviceMalloc(consumer, PETSC_MEMTYPE_HOST, n, PETSC_DECIDE, &host));
276:   PetscCall(PetscDeviceContextSynchronize(producer));
277:   PetscCall(PetscDeviceContextSynchronize(consumer));
278:   for (PetscInt pass = 0; pass < 4; ++pass) {
279:     const PetscInt value = 37 + pass;

281:     // The consumer must wait for the most recent memset, including repeated writes.
282:     PetscCall(PetscDeviceMemset(producer, dest, 17, n));
283:     PetscCall(PetscDeviceContextDelay(producer, 0.05));
284:     PetscCall(PetscDeviceMemset(producer, dest, value, n));
285:     PetscCall(PetscDeviceArrayCopy(consumer, host, dest, n));
286:     PetscCall(PetscDeviceContextSynchronize(consumer));
287:     for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Consumer read stale memset byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)host[i], value);
288:     PetscCall(PetscDeviceContextSynchronize(producer));

290:     PetscCall(PetscDeviceMemset(producer, src[0], value, n));
291:     PetscCall(PetscDeviceMemset(producer, src[1], value + 32, n));
292:     PetscCall(PetscDeviceContextSynchronize(producer));
293:     // The destination dependency must cover the latest copy.
294:     PetscCall(PetscDeviceArrayCopy(producer, dest, src[0], n));
295:     PetscCall(PetscDeviceContextDelay(producer, 0.05));
296:     PetscCall(PetscDeviceArrayCopy(producer, dest, src[1], n));
297:     PetscCall(PetscDeviceArrayCopy(consumer, host, dest, n));
298:     PetscCall(PetscDeviceContextSynchronize(consumer));
299:     for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value + 32, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Consumer read stale copy byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)host[i], value + 32);
300:     PetscCall(PetscDeviceContextSynchronize(producer));

302:     // The source dependency must protect a pending read from a later overwrite.
303:     PetscCall(PetscDeviceArrayCopy(producer, dest, src[0], n));
304:     PetscCall(PetscDeviceContextDelay(producer, 0.05));
305:     PetscCall(PetscDeviceArrayCopy(producer, dest, src[0], n));
306:     PetscCall(PetscDeviceMemset(consumer, src[0], value + 64, n));
307:     PetscCall(PetscDeviceArrayCopy(producer, host, dest, n));
308:     PetscCall(PetscDeviceContextSynchronize(producer));
309:     PetscCall(PetscDeviceContextSynchronize(consumer));
310:     for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Source overwritten before its copy at byte %" PetscInt_FMT " (got %u, expected %" PetscInt_FMT ")", i, (unsigned)host[i], value);
311:   }
312:   PetscCall(PetscDeviceFree(producer, src[0]));
313:   PetscCall(PetscDeviceFree(producer, src[1]));
314:   PetscCall(PetscDeviceFree(producer, dest));
315:   PetscCall(PetscDeviceFree(consumer, host));
316:   PetscCall(PetscDeviceContextSynchronize(producer));
317:   PetscCall(PetscDeviceContextSynchronize(consumer));
318:   PetscCall(PetscDeviceContextDestroy(&producer));
319:   PetscCall(PetscDeviceContextDestroy(&consumer));
320:   PetscFunctionReturn(PETSC_SUCCESS);
321: }

323: static PetscErrorCode TestTypeAlignment(PetscDeviceContext dctx)
324: {
325:   char        *prefix;
326:   short       *s;
327:   double      *d;
328:   PetscScalar *v;
329:   struct Triple {
330:     double x, y, z;
331:   } *triple;

333:   PetscFunctionBeginUser;
334:   for (PetscInt pass = 0; pass < 8; ++pass) {
335:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, pass + 1, PETSC_DECIDE, &prefix));
336:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 3, PETSC_DECIDE, &s));
337:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 3, 1, &d));
338:     PetscCall(PetscDeviceCalloc(dctx, PETSC_MEMTYPE_HOST, 3, PETSC_DECIDE, &v));
339:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 3, PETSC_DECIDE, &triple));
340:     PetscCall(CheckAlignment(s, PETSC_DEVICE_ALIGNOF(*s)));
341:     PetscCall(CheckAlignment(d, PETSC_DEVICE_ALIGNOF(*d)));
342:     PetscCall(CheckAlignment(v, PETSC_DEVICE_ALIGNOF(*v)));
343:     PetscCall(CheckAlignment(triple, PETSC_DEVICE_ALIGNOF(*triple)));
344:     PetscCall(PetscDeviceContextSynchronize(dctx));
345:     for (PetscInt i = 0; i < 3; ++i) {
346:       PetscCheck(v[i] == 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Scalar calloc did not clear entry %" PetscInt_FMT, i);
347:       s[i]        = (short)i;
348:       d[i]        = i + 0.5;
349:       triple[i].x = d[i];
350:       triple[i].y = s[i];
351:       triple[i].z = triple[i].x + triple[i].y;
352:       PetscCheck(triple[i].z == 2 * i + 0.5, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect structure entry");
353:     }
354:     PetscCall(PetscDeviceFree(dctx, d));
355:     PetscCall(PetscDeviceFree(dctx, prefix));
356:     PetscCall(PetscDeviceFree(dctx, triple));
357:     PetscCall(PetscDeviceFree(dctx, v));
358:     PetscCall(PetscDeviceFree(dctx, s));
359:   }
360:   PetscFunctionReturn(PETSC_SUCCESS);
361: }

363: static PetscErrorCode TestAlignmentStress(PetscDeviceContext dctx, PetscMemType mtype)
364: {
365:   const size_t       alignments[] = {(size_t)PETSC_DECIDE, 1, 2, 4, 8, 16, 32, 64, 128, 256, 512, 1024, 4096};
366:   const PetscInt     sizes[]      = {0, 1, 3, 7, 17, 255, 256, 257, 2047, 2048, 2049, 4097};
367:   PetscInt           lengths[12] = {0}, values[12] = {0}, sequence = 0;
368:   unsigned char     *buffers[12] = {NULL}, *host;
369:   PetscDeviceContext ctx[2];
370:   PetscDeviceType    dtype;
371:   PetscBool          cupm;

373:   PetscFunctionBeginUser;
374:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
375:   cupm = (PetscBool)(dtype == PETSC_DEVICE_CUDA || dtype == PETSC_DEVICE_HIP);
376:   if (!cupm && PetscMemTypeDevice(mtype)) PetscFunctionReturn(PETSC_SUCCESS);
377:   PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 4097, PETSC_DECIDE, &host));
378:   for (PetscInt c = 0; c < 2; ++c) {
379:     PetscCall(PetscDeviceContextDuplicate(dctx, &ctx[c]));
380:     PetscCall(PetscDeviceContextSetStreamType(ctx[c], c ? PETSC_STREAM_NONBLOCKING_WITH_BARRIER : PETSC_STREAM_NONBLOCKING));
381:     PetscCall(PetscDeviceContextSetUp(ctx[c]));
382:   }
383:   for (PetscInt pass = 0; pass < 2; ++pass) {
384:     for (size_t a = 0; a < PETSC_STATIC_ARRAY_LENGTH(alignments); ++a) {
385:       size_t alignment = alignments[a];
386:       size_t expected  = alignment == (size_t)PETSC_DECIDE ? PETSC_DEVICE_ALIGNOF(*host) : alignment;

388:       if (!cupm && alignment != (size_t)PETSC_DECIDE && PETSC_MEMALIGN % alignment) continue;
389:       for (PetscInt clear = 0; clear < 2; ++clear) {
390:         for (size_t k = 0; k < PETSC_STATIC_ARRAY_LENGTH(sizes); ++k, ++sequence) {
391:           PetscInt           slot = sequence % 12, n = sizes[(k + pass) % PETSC_STATIC_ARRAY_LENGTH(sizes)];
392:           PetscDeviceContext alloc = ctx[sequence % 2], use = ctx[(sequence + 1) % 2];

394:           PetscCall(PetscDeviceFree(use, buffers[slot]));
395:           if (clear) PetscCall(PetscDeviceCalloc(alloc, mtype, n, alignment, &buffers[slot]));
396:           else PetscCall(PetscDeviceMalloc(alloc, mtype, n, alignment, &buffers[slot]));
397:           lengths[slot] = n;
398:           values[slot]  = 1 + sequence % 251;
399:           if (!n) {
400:             PetscCheck(!buffers[slot], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Zero-sized allocation did not return NULL");
401:             continue;
402:           }
403:           PetscCall(CheckAlignment(buffers[slot], expected));
404:           if (clear) {
405:             PetscCall(PetscDeviceArrayCopy(use, host, buffers[slot], n));
406:             PetscCall(PetscDeviceContextSynchronize(use));
407:             for (PetscInt j = 0; j < n; ++j) PetscCheck(!host[j], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Calloc did not clear byte %" PetscInt_FMT, j);
408:           }
409:           PetscCall(PetscDeviceMemset(use, buffers[slot], values[slot], n));
410:           // Check all live allocations to detect overlap or corruption during pool reuse.
411:           for (PetscInt b = 0; b < 12; ++b) {
412:             if (!lengths[b]) continue;
413:             PetscCall(PetscDeviceArrayCopy(alloc, host, buffers[b], lengths[b]));
414:             PetscCall(PetscDeviceContextSynchronize(alloc));
415:             for (PetscInt j = 0; j < lengths[b]; ++j) PetscCheck(host[j] == values[b], PETSC_COMM_SELF, PETSC_ERR_PLIB, "Allocation %" PetscInt_FMT " byte %" PetscInt_FMT " was corrupted", b, j);
416:           }
417:         }
418:       }
419:     }
420:   }
421:   for (PetscInt b = 11; b >= 0; --b) PetscCall(PetscDeviceFree(ctx[b % 2], buffers[b]));
422:   for (PetscInt c = 0; c < 2; ++c) {
423:     PetscCall(PetscDeviceContextSynchronize(ctx[c]));
424:     PetscCall(PetscDeviceContextDestroy(&ctx[c]));
425:   }
426:   PetscCall(PetscDeviceFree(dctx, host));
427:   PetscCall(PetscDeviceContextSynchronize(dctx));
428:   PetscFunctionReturn(PETSC_SUCCESS);
429: }

431: static PetscScalar ScalarValue(PetscInt i)
432: {
433: #if PetscDefined(USE_COMPLEX)
434:   return PetscCMPLX(i + 1, 2 * i + 1);
435: #else
436:   return i + 1;
437: #endif
438: }

440: static PetscErrorCode TestScalarAlignment(PetscDeviceContext dctx)
441: {
442:   PetscDeviceType dtype;
443:   PetscScalar    *values[8], *host;
444:   char           *prefix;

446:   PetscFunctionBeginUser;
447:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
448:   if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
449:   PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, 7, sizeof(PetscScalar), &host));
450:   PetscCall(PetscDeviceContextSynchronize(dctx));
451:   for (PetscInt i = 0; i < 7; ++i) host[i] = ScalarValue(i);
452:   for (PetscInt pass = 0; pass < 4; ++pass) {
453:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_DEVICE, 3, 1, &prefix));
454:     for (PetscInt b = 0; b < 8; ++b) {
455:       PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_DEVICE, 7, sizeof(PetscScalar), &values[b]));
456:       PetscCall(CheckAlignment(values[b], sizeof(PetscScalar)));
457:       PetscCall(PetscDeviceArrayCopy(dctx, values[b], host, 7));
458: #if PetscDefined(HAVE_CUDA)
459:       if (dtype == PETSC_DEVICE_CUDA) {
460:         cublasHandle_t      handle;
461:         cublasPointerMode_t mode;
462:         const PetscReal     alpha = 2;

464:         PetscCall(PetscDeviceContextGetBLASHandle_Internal(dctx, &handle));
465:         PetscCallCUBLAS(cublasGetPointerMode(handle, &mode));
466:         PetscCallCUBLAS(cublasSetPointerMode(handle, CUBLAS_POINTER_MODE_HOST));
467:   #if PetscDefined(USE_COMPLEX)
468:     #if PetscDefined(USE_REAL_SINGLE)
469:         PetscCallCUBLAS(cublasCsscal(handle, 7, &alpha, (cuComplex *)values[b], 1));
470:     #else
471:         PetscCallCUBLAS(cublasZdscal(handle, 7, &alpha, (cuDoubleComplex *)values[b], 1));
472:     #endif
473:   #elif PetscDefined(USE_REAL_SINGLE)
474:         PetscCallCUBLAS(cublasSscal(handle, 7, &alpha, values[b], 1));
475:   #else
476:         PetscCallCUBLAS(cublasDscal(handle, 7, &alpha, values[b], 1));
477:   #endif
478:         PetscCallCUBLAS(cublasSetPointerMode(handle, mode));
479:       }
480: #endif
481: #if PetscDefined(HAVE_HIP)
482:       if (dtype == PETSC_DEVICE_HIP) {
483:         hipblasHandle_t      handle;
484:         hipblasPointerMode_t mode;
485:         const PetscReal      alpha = 2;

487:         PetscCall(PetscDeviceContextGetBLASHandle_Internal(dctx, &handle));
488:         PetscCallHIPBLAS(hipblasGetPointerMode(handle, &mode));
489:         PetscCallHIPBLAS(hipblasSetPointerMode(handle, HIPBLAS_POINTER_MODE_HOST));
490:   #if PetscDefined(USE_COMPLEX)
491:     #if PetscDefined(USE_REAL_SINGLE)
492:         PetscCallHIPBLAS(hipblasCsscal(handle, 7, &alpha, (hipFloatComplex *)values[b], 1));
493:     #else
494:         PetscCallHIPBLAS(hipblasZdscal(handle, 7, &alpha, (hipDoubleComplex *)values[b], 1));
495:     #endif
496:   #elif PetscDefined(USE_REAL_SINGLE)
497:         PetscCallHIPBLAS(hipblasSscal(handle, 7, &alpha, values[b], 1));
498:   #else
499:         PetscCallHIPBLAS(hipblasDscal(handle, 7, &alpha, values[b], 1));
500:   #endif
501:         PetscCallHIPBLAS(hipblasSetPointerMode(handle, mode));
502:       }
503: #endif
504:     }
505:     PetscCall(PetscDeviceContextSynchronize(dctx));
506:     for (PetscInt b = 7; b >= 0; --b) {
507:       PetscCall(PetscDeviceArrayCopy(dctx, host, values[b], 7));
508:       PetscCall(PetscDeviceContextSynchronize(dctx));
509:       for (PetscInt i = 0; i < 7; ++i) PetscCheck(host[i] == 2 * ScalarValue(i), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Incorrect scaled scalar at entry %" PetscInt_FMT, i);
510:       PetscCall(PetscDeviceFree(dctx, values[b]));
511:     }
512:     PetscCall(PetscDeviceFree(dctx, prefix));
513:     for (PetscInt i = 0; i < 7; ++i) host[i] = ScalarValue(i);
514:   }
515:   PetscCall(PetscDeviceFree(dctx, host));
516:   PetscCall(PetscDeviceContextSynchronize(dctx));
517:   PetscFunctionReturn(PETSC_SUCCESS);
518: }

520: static PetscErrorCode TestAlignmentOrdering(PetscDeviceContext dctx, PetscMemType mtype)
521: {
522:   const PetscInt     n = 4097;
523:   PetscDeviceContext ctx[2], current;
524:   PetscDeviceType    dtype;
525:   unsigned char     *output[8], *buffer;

527:   PetscFunctionBeginUser;
528:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
529:   if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
530:   for (PetscInt c = 0; c < 2; ++c) {
531:     PetscCall(PetscDeviceContextDuplicate(dctx, &ctx[c]));
532:     PetscCall(PetscDeviceContextSetStreamType(ctx[c], PETSC_STREAM_NONBLOCKING));
533:     PetscCall(PetscDeviceContextSetUp(ctx[c]));
534:   }
535:   for (PetscInt k = 0; k < 8; ++k) PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, n, 128, &output[k]));
536:   // Warm the pool before delaying allocation, initialization, and reuse.
537:   PetscCall(PetscDeviceCalloc(ctx[0], mtype, n, 128, &buffer));
538:   PetscCall(PetscDeviceFree(ctx[0], buffer));
539:   PetscCall(PetscDeviceContextSynchronize(ctx[0]));
540:   for (PetscInt k = 0; k < 8; ++k) {
541:     PetscDeviceContext producer = ctx[k % 2], consumer = ctx[(k + 1) % 2];

543:     PetscCall(PetscDeviceContextDelay(producer, 0.02));
544:     PetscCall(PetscDeviceCalloc(producer, mtype, n, 128, &buffer));
545:     PetscCall(CheckAlignment(buffer, 128));
546:     if (k % 2) PetscCall(PetscDeviceMemset(producer, buffer, k + 1, n));
547:     PetscCall(PetscDeviceArrayCopy(consumer, output[k], buffer, n));
548:     // Free must wait for the other stream's read before this chunk can be reused.
549:     PetscCall(PetscDeviceFree(producer, buffer));
550:   }
551:   for (PetscInt c = 0; c < 2; ++c) PetscCall(PetscDeviceContextSynchronize(ctx[c]));
552:   for (PetscInt k = 0; k < 8; ++k) {
553:     for (PetscInt j = 0; j < n; ++j)
554:       PetscCheck(output[k][j] == (k % 2 ? k + 1 : 0), PETSC_COMM_SELF, PETSC_ERR_PLIB, "Stream handoff %" PetscInt_FMT " corrupted byte %" PetscInt_FMT " (got %u, memory type %s)", k, j, (unsigned)output[k][j], PetscMemTypeToString(mtype));
555:     PetscCall(PetscDeviceFree(dctx, output[k]));
556:   }
557:   if (PetscMemTypeHost(mtype)) {
558:     const PetscInt nreuse = 32769;
559:     unsigned char *input, *device, *result;

561:     PetscCall(PetscDeviceMalloc(dctx, PETSC_MEMTYPE_HOST, nreuse, 128, &result));
562:     PetscCall(PetscDeviceMalloc(ctx[0], PETSC_MEMTYPE_DEVICE, nreuse, 128, &device));
563:     PetscCall(PetscDeviceMalloc(ctx[0], PETSC_MEMTYPE_HOST, nreuse, 128, &input));
564:     PetscCall(PetscDeviceMemset(ctx[0], input, 37, nreuse));
565:     PetscCall(PetscDeviceContextSynchronize(ctx[0]));
566:     PetscCall(PetscDeviceContextDelay(ctx[0], 0.05));
567:     PetscCall(PetscDeviceArrayCopy(ctx[0], device, input, nreuse));
568:     PetscCall(PetscDeviceFree(ctx[0], input));
569:     // Reallocation, including debug initialization, must follow the pending upload.
570:     PetscCall(PetscDeviceMalloc(ctx[0], PETSC_MEMTYPE_HOST, nreuse, 128, &input));
571:     PetscCall(PetscDeviceArrayCopy(ctx[1], result, device, nreuse));
572:     PetscCall(PetscDeviceContextSynchronize(ctx[1]));
573:     for (PetscInt j = 0; j < nreuse; ++j) PetscCheck(result[j] == 37, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Pinned allocation reuse corrupted byte %" PetscInt_FMT " (got %u)", j, (unsigned)result[j]);
574:     PetscCall(PetscDeviceFree(ctx[0], input));
575:     PetscCall(PetscDeviceFree(ctx[1], device));
576:     PetscCall(PetscDeviceFree(dctx, result));
577:     PetscCall(PetscDeviceContextSynchronize(ctx[0]));
578:     PetscCall(PetscDeviceContextSynchronize(ctx[1]));
579:   }
580:   // Typed accesses must also use the explicit context while another remains current.
581:   if (PetscMemTypeDevice(mtype)) PetscCall(TestScalarAlignment(ctx[0]));
582:   PetscCall(PetscDeviceContextGetCurrentContext(&current));
583:   PetscCheck(current == dctx, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Explicit-context allocation changed the current context");
584:   for (PetscInt c = 0; c < 2; ++c) PetscCall(PetscDeviceContextDestroy(&ctx[c]));
585:   PetscCall(PetscDeviceContextSynchronize(dctx));
586:   PetscFunctionReturn(PETSC_SUCCESS);
587: }

589: /*
590:   Both helpers call the context's backend methods directly to leave memory accesses untracked.
591:   PetscDeviceMemset() and PetscDeviceMemcpy() would register these accesses and make later
592:   operations wait automatically, hiding missing explicit PetscDeviceContextWaitForContext()
593:   calls in TestContextOrdering().
594: */
595: static PetscErrorCode SetDeviceBytes(PetscDeviceContext dctx, void *ptr, PetscInt value, size_t n)
596: {
597:   PetscFunctionBeginUser;
598:   PetscUseTypeMethod(dctx, memset, PETSC_MEMTYPE_DEVICE, ptr, value, n);
599:   PetscFunctionReturn(PETSC_SUCCESS);
600: }

602: static PetscErrorCode CopyDeviceBytesToHost(PetscDeviceContext dctx, void *host, const void *ptr, size_t n)
603: {
604:   PetscFunctionBeginUser;
605:   PetscUseTypeMethod(dctx, memcopy, host, ptr, n, PETSC_DEVICE_COPY_DTOH);
606:   PetscFunctionReturn(PETSC_SUCCESS);
607: }

609: static PetscErrorCode TestContextOrdering(PetscDeviceContext saved)
610: {
611:   const PetscInt     n = 32769;
612:   PetscDeviceContext current, other, standard, contexts[3];
613:   PetscDeviceType    dtype;
614:   unsigned char     *src, *dest, *host;

616:   PetscFunctionBeginUser;
617:   PetscCall(PetscDeviceContextGetDeviceType(saved, &dtype));
618:   if (dtype != PETSC_DEVICE_CUDA && dtype != PETSC_DEVICE_HIP) PetscFunctionReturn(PETSC_SUCCESS);
619:   PetscCall(PetscDeviceContextDuplicate(saved, &current));
620:   PetscCall(PetscDeviceContextSetStreamType(current, PETSC_STREAM_NONBLOCKING));
621:   PetscCall(PetscDeviceContextSetUp(current));
622:   PetscCall(PetscDeviceContextDuplicate(current, &other));
623:   standard = PetscDeviceContextDefault;
624:   PetscCall(PetscDeviceContextSetCurrentContext(current));
625:   contexts[0] = standard;
626:   contexts[1] = current;
627:   contexts[2] = other;
628:   PetscCall(PetscDeviceMalloc(current, PETSC_MEMTYPE_HOST, n, 128, &host));

630:   // Memory operations must use the supplied context even when another context is current.
631:   PetscCall(PetscDeviceCalloc(standard, PETSC_MEMTYPE_HOST, n, 128, &src));
632:   PetscCall(PetscDeviceContextSynchronize(standard));
633:   PetscCall(PetscDeviceContextSynchronize(current));
634:   PetscCall(PetscDeviceContextDelay(current, 0.05));
635:   PetscCall(PetscDeviceMemset(standard, src, 73, n));
636:   PetscCall(PetscDeviceContextSynchronize(standard));
637:   for (PetscInt i = 0; i < n; ++i) PetscCheck(src[i] == 73, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Synchronization did not complete the explicit-context memory operation at byte %" PetscInt_FMT " (got %u)", i, (unsigned)src[i]);
638:   PetscCall(PetscDeviceContextSynchronize(current));
639:   PetscCall(PetscDeviceFree(standard, src));

641:   // Track dependencies between a default-stream context and two nonblocking contexts.
642:   for (PetscInt p = 0; p < 3; ++p) {
643:     for (PetscInt c = 0; c < 3; ++c) {
644:       PetscDeviceContext alloc = contexts[(p + c) % 3], producer = contexts[p], consumer = contexts[c];
645:       PetscInt           value = 17 + 3 * p + c;

647:       PetscCall(PetscDeviceCalloc(alloc, PETSC_MEMTYPE_DEVICE, n, 128, &src));
648:       PetscCall(PetscDeviceMalloc(alloc, PETSC_MEMTYPE_DEVICE, n, 128, &dest));
649:       PetscCall(PetscDeviceContextSynchronize(alloc));
650:       PetscCall(PetscDeviceContextDelay(producer, 0.02));
651:       PetscCall(PetscDeviceMemset(producer, src, value, n));
652:       PetscCall(PetscDeviceContextDelay(consumer, 0.02));
653:       PetscCall(PetscDeviceArrayCopy(consumer, dest, src, n));
654:       PetscCall(PetscDeviceFree(producer, src));
655:       PetscCall(PetscDeviceMalloc(producer, PETSC_MEMTYPE_DEVICE, n, 128, &src));
656:       PetscCall(PetscDeviceMemset(producer, src, 99, n));
657:       PetscCall(PetscDeviceArrayCopy(consumer, host, dest, n));
658:       PetscCall(PetscDeviceContextSynchronize(consumer));
659:       for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == value, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Tracked transfer %" PetscInt_FMT " -> %" PetscInt_FMT " corrupted byte %" PetscInt_FMT " (got %u)", p, c, i, (unsigned)host[i]);
660:       PetscCall(PetscDeviceFree(alloc, src));
661:       PetscCall(PetscDeviceFree(alloc, dest));
662:     }
663:   }
664:   for (PetscInt c = 0; c < 3; ++c) PetscCall(PetscDeviceContextSynchronize(contexts[c]));

666:   // Warm reuse before testing allocation on standard followed by an untracked write on current.
667:   PetscCall(PetscDeviceMalloc(standard, PETSC_MEMTYPE_DEVICE, n, 128, &src));
668:   PetscCall(PetscDeviceFree(standard, src));
669:   PetscCall(PetscDeviceContextSynchronize(standard));
670:   PetscCall(PetscDeviceContextDelay(standard, 0.05));
671:   PetscCall(PetscDeviceCalloc(standard, PETSC_MEMTYPE_DEVICE, n, 128, &src));
672:   PetscCall(PetscDeviceContextWaitForContext(current, standard));
673:   PetscCall(SetDeviceBytes(current, src, 63, n));
674:   PetscCall(PetscDeviceArrayCopy(current, host, src, n));
675:   PetscCall(PetscDeviceContextSynchronize(current));
676:   for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 63, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Allocation-to-raw-write ordering corrupted byte %" PetscInt_FMT " (got %u)", i, (unsigned)host[i]);

678:   // A raw producer is not registered with memory tracking, so copying on standard needs a wait.
679:   PetscCall(PetscDeviceContextDelay(current, 0.05));
680:   PetscCall(SetDeviceBytes(current, src, 37, n));
681:   PetscCall(PetscDeviceContextWaitForContext(standard, current));
682:   PetscCall(PetscDeviceArrayCopy(standard, host, src, n));
683:   PetscCall(PetscDeviceContextSynchronize(standard));
684:   for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 37, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Raw-write-to-copy ordering corrupted byte %" PetscInt_FMT " (got %u)", i, (unsigned)host[i]);

686:   // Releasing and reusing on standard must also follow an untracked read on current.
687:   PetscCall(PetscDeviceContextDelay(current, 0.05));
688:   PetscCall(CopyDeviceBytesToHost(current, host, src, n));
689:   PetscCall(PetscDeviceContextWaitForContext(standard, current));
690:   PetscCall(PetscDeviceFree(standard, src));
691:   PetscCall(PetscDeviceMalloc(standard, PETSC_MEMTYPE_DEVICE, n, 128, &src));
692:   PetscCall(PetscDeviceMemset(standard, src, 99, n));
693:   PetscCall(PetscDeviceContextSynchronize(standard));
694:   PetscCall(PetscDeviceContextSynchronize(current));
695:   for (PetscInt i = 0; i < n; ++i) PetscCheck(host[i] == 37, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Raw-read-to-reuse ordering corrupted byte %" PetscInt_FMT " (got %u)", i, (unsigned)host[i]);
696:   PetscCall(PetscDeviceFree(standard, src));
697:   PetscCall(PetscDeviceFree(current, host));
698:   for (PetscInt c = 0; c < 3; ++c) PetscCall(PetscDeviceContextSynchronize(contexts[c]));
699:   PetscCall(PetscDeviceContextSetCurrentContext(saved));
700:   PetscCall(PetscDeviceContextDestroy(&other));
701:   PetscCall(PetscDeviceContextDestroy(&current));
702:   PetscFunctionReturn(PETSC_SUCCESS);
703: }

705: int main(int argc, char *argv[])
706: {
707:   PetscDeviceContext dctx;
708:   PetscRandom        rand;
709:   PetscBool          test_alignment = PETSC_FALSE, test_context_policy = PETSC_FALSE, test_stream_types = PETSC_FALSE, test_pinned_reuse = PETSC_FALSE, test_memory_access = PETSC_FALSE;

711:   PetscFunctionBeginUser;
712:   PetscCall(PetscInitialize(&argc, &argv, NULL, help));

714:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_alignment", &test_alignment, NULL));
715:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_context_policy", &test_context_policy, NULL));
716:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_stream_types", &test_stream_types, NULL));
717:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_pinned_reuse", &test_pinned_reuse, NULL));
718:   PetscCall(PetscOptionsGetBool(NULL, NULL, "-test_memory_access", &test_memory_access, NULL));
719:   if (test_stream_types) {
720:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
721:     PetscCall(TestMemoryStreamTypes(dctx, PETSC_MEMTYPE_HOST));
722:     PetscCall(TestMemoryStreamTypes(dctx, PETSC_MEMTYPE_DEVICE));
723:   } else if (test_pinned_reuse) {
724:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
725:     PetscCall(TestPinnedAllocationReuse(dctx));
726:   } else if (test_memory_access) {
727:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
728:     PetscCall(TestMemoryAccessOrdering(dctx));
729:   } else if (test_context_policy) {
730:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
731:     PetscCall(TestContextOrdering(dctx));
732:   } else if (test_alignment) {
733:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
734:     PetscCall(TestTypeAlignment(dctx));
735:     PetscCall(TestAlignmentStress(dctx, PETSC_MEMTYPE_HOST));
736:     PetscCall(TestAlignmentStress(dctx, PETSC_MEMTYPE_DEVICE));
737:     PetscCall(TestAlignmentOrdering(dctx, PETSC_MEMTYPE_HOST));
738:     PetscCall(TestAlignmentOrdering(dctx, PETSC_MEMTYPE_DEVICE));
739:     PetscCall(TestTypeAlignment(PetscDeviceContextDefault));
740:     PetscCall(PetscDeviceContextSynchronize(PetscDeviceContextDefault));
741:   } else {
742:     // A vile hack. The -info output is used to test correctness in this test which prints --
743:     // among other things -- the PetscObjectId of the PetscDevicContext and the allocated memory.
744:     //
745:     // Due to device and host creating slightly different number of objects on startup there will
746:     // be a mismatch in the ID's. So for the tests involving the host we sit here creating
747:     // PetscContainers (and incrementing the global PetscObjectId counter) until it reaches some
748:     // arbitrarily high number to ensure that our first PetscDeviceContext has the same ID across
749:     // systems.
750:     {
751:       PetscObjectId prev_id = 0;

753:       do {
754:         PetscContainer c;
755:         PetscObjectId  id;

757:         PetscCall(PetscContainerCreate(PETSC_COMM_WORLD, &c));
758:         PetscCall(PetscObjectGetId((PetscObject)c, &id));
759:         // sanity check, in case PetscContainer ever stops being a PetscObject
760:         PetscCheck(id > prev_id, PETSC_COMM_SELF, PETSC_ERR_PLIB, "PetscObjectIds are not increasing for successively created PetscContainers! current: %" PetscInt64_FMT ", previous: %" PetscInt64_FMT, id, prev_id);
761:         prev_id = id;
762:         PetscCall(PetscContainerDestroy(&c));
763:       } while (prev_id < 50);
764:     }
765:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));

767:     PetscCall(PetscRandomCreate(PETSC_COMM_WORLD, &rand));
768:     // this seed just so happens to keep the allocation size increasing
769:     PetscCall(PetscRandomSetSeed(rand, 123));
770:     PetscCall(PetscRandomSeed(rand));
771:     PetscCall(PetscRandomSetFromOptions(rand));

773:     PetscCall(TestAllocate(dctx, rand, PETSC_MEMTYPE_HOST));
774:     PetscCall(TestAllocate(dctx, rand, PETSC_MEMTYPE_DEVICE));
775:     PetscCall(TestAsyncCoherence(dctx, rand));

777:     PetscCall(PetscRandomDestroy(&rand));
778:   }
779:   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "EXIT_SUCCESS\n"));
780:   PetscCall(PetscFinalize());
781:   return 0;
782: }

784: /*TEST

786:   testset:
787:     args: -test_memory_access
788:     output_file: output/ExitSuccess.out
789:     test:
790:       suffix: memory_access_cuda
791:       requires: cuda
792:       args: -default_device_type cuda
793:     test:
794:       suffix: memory_access_hip
795:       requires: hip
796:       args: -default_device_type hip

798:   testset:
799:     requires: defined(PETSC_USE_DEBUG)
800:     args: -test_pinned_reuse
801:     output_file: output/ExitSuccess.out
802:     test:
803:       suffix: pinned_reuse_cuda
804:       requires: cuda
805:       args: -default_device_type cuda
806:     test:
807:       suffix: pinned_reuse_hip
808:       requires: hip
809:       args: -default_device_type hip

811:   testset:
812:     args: -test_stream_types
813:     output_file: output/ExitSuccess.out
814:     test:
815:       suffix: stream_types_cuda
816:       requires: cuda
817:       args: -default_device_type cuda
818:     test:
819:       suffix: stream_types_hip
820:       requires: hip
821:       args: -default_device_type hip

823:   testset:
824:     args: -test_context_policy
825:     output_file: output/ExitSuccess.out
826:     test:
827:       suffix: context_policy_cuda
828:       requires: cuda
829:       args: -default_device_type cuda
830:     test:
831:       suffix: context_policy_hip
832:       requires: hip
833:       args: -default_device_type hip

835:   testset:
836:     args: -test_alignment
837:     output_file: output/ExitSuccess.out
838:     test:
839:       suffix: alignment_host
840:       args: -default_device_type host -options_left 0
841:     test:
842:       suffix: alignment_cuda
843:       requires: cuda
844:       args: -default_device_type cuda
845:     test:
846:       suffix: alignment_hip
847:       requires: hip
848:       args: -default_device_type hip

850:   testset:
851:     requires: defined(PETSC_USE_INFO) defined(PETSC_USE_DEBUG) defined(PETSC_DEVICELANGUAGE_CXX)
852:     args: -info :device
853:     suffix: with_info
854:     test:
855:       requires: !device
856:       suffix: host_no_device
857:     test:
858:       requires: device
859:       args: -default_device_type host
860:       filter: sed -e 's/host/IMPL/g' -e 's/cuda/IMPL/g' -e 's/hip/IMPL/g' -e 's/sycl/IMPL/g'
861:       suffix: host_with_device
862:     test:
863:       requires: cuda
864:       args: -default_device_type cuda
865:       suffix: cuda
866:     test:
867:       requires: hip
868:       args: -default_device_type hip
869:       suffix: hip
870:     test:
871:       requires: sycl
872:       TODO: unclear if it is needed
873:       args: -default_device_type sycl
874:       suffix: sycl

876:   testset:
877:     output_file: output/ExitSuccess.out
878:     requires: !defined(PETSC_USE_DEBUG) defined(PETSC_DEVICELANGUAGE_CXX)
879:     filter: grep -v "\[DEBUG OUTPUT\]"
880:     suffix: no_info
881:     test:
882:       requires: !device
883:       suffix: host_no_device
884:     test:
885:       requires: device
886:       args: -default_device_type host
887:       suffix: host_with_device
888:     test:
889:       requires: cuda
890:       args: -default_device_type cuda
891:       suffix: cuda
892:     test:
893:       requires: hip
894:       args: -default_device_type hip
895:       suffix: hip
896:     test:
897:       requires: sycl
898:       TODO: unclear if it is needed
899:       args: -default_device_type sycl
900:       suffix: sycl

902:   test:
903:     requires: !defined(PETSC_DEVICELANGUAGE_CXX)
904:     output_file: output/ExitSuccess.out
905:     filter: grep -v "\[DEBUG OUTPUT\]"
906:     suffix: no_cxx

908: TEST*/