Actual source code: dcontext.cxx

  1: #include "petscdevice_interface_internal.hpp" /*I <petscdevice.h> I*/
  2: #include <petsc/private/viewerimpl.h>

  4: #include <petsc/private/cpp/array.hpp>
  5: #include <petsc/private/cpp/object_pool.hpp>
  6: #include <petsc/private/cpp/utility.hpp>

  8: #include <string> // std::to_string among other things
  9: #include <cmath>
 10: #include <vector>

 12: static _p_PetscDeviceContext defaultContextPlaceholder;
 13: PetscDeviceContext const     PetscDeviceContextDefault = &defaultContextPlaceholder;

 15: /* Define the allocator */
 16: class PetscDeviceContextConstructor : public Petsc::ConstructorInterface<_p_PetscDeviceContext, PetscDeviceContextConstructor> {
 17: public:
 18:   PetscErrorCode construct_(PetscDeviceContext dctx) const noexcept
 19:   {
 20:     PetscFunctionBegin;
 21:     PetscCall(PetscArrayzero(dctx, 1));
 22:     PetscCall(PetscHeaderCreate_Private((PetscObject)dctx, PETSC_DEVICE_CONTEXT_CLASSID, "PetscDeviceContext", "PetscDeviceContext", "Sys", PETSC_COMM_SELF, (PetscObjectDestroyFn *)PetscDeviceContextDestroy, (PetscObjectViewFn *)PetscDeviceContextView));
 23:     PetscCall(PetscLogObjectCreate((PetscObject)dctx));

 25:     PetscCallCXX(PetscObjectCast(dctx)->cpp = new CxxData{dctx});
 26:     PetscCall(underlying().reset(dctx, false));
 27:     PetscFunctionReturn(PETSC_SUCCESS);
 28:   }

 30:   static PetscErrorCode destroy_(PetscDeviceContext dctx) noexcept
 31:   {
 32:     PetscFunctionBegin;
 33:     PetscAssert(!dctx->numChildren, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Device context still has %" PetscInt_FMT " un-joined children, must call PetscDeviceContextJoin() with all children before destroying", dctx->numChildren);
 34:     PetscTryTypeMethod(dctx, destroy);
 35:     PetscCall(PetscDeviceDestroy(&dctx->device));
 36:     PetscCall(PetscFree(dctx->childIDs));
 37:     delete CxxDataCast(dctx);
 38:     PetscCall(PetscHeaderDestroy_Private(PetscObjectCast(dctx), PETSC_FALSE));
 39:     PetscFunctionReturn(PETSC_SUCCESS);
 40:   }

 42:   static PetscErrorCode reset_(PetscDeviceContext dctx, bool zero = true) noexcept
 43:   {
 44:     PetscFunctionBegin;
 45:     if (zero) {
 46:       // reset the device if the user set it
 47:       if (Petsc::util::exchange(dctx->usersetdevice, PETSC_FALSE)) {
 48:         PetscTryTypeMethod(dctx, destroy);
 49:         PetscCall(PetscDeviceDestroy(&dctx->device));
 50:         PetscCall(PetscArrayzero(dctx->ops, 1));
 51:         dctx->data = nullptr;
 52:       }
 53:       PetscCall(PetscHeaderReset_Internal(PetscObjectCast(dctx)));
 54:       dctx->numChildren = 0;
 55:       dctx->setup       = PETSC_FALSE;
 56:       // don't deallocate the child array, rather just zero it out
 57:       PetscCall(PetscArrayzero(dctx->childIDs, dctx->maxNumChildren));
 58:       PetscCall(CxxDataCast(dctx)->clear());
 59:       PetscCall(CxxDataCast(dctx)->reset_self(dctx));
 60:     }
 61:     dctx->streamType = PETSC_STREAM_DEFAULT;
 62:     PetscFunctionReturn(PETSC_SUCCESS);
 63:   }

 65:   static PetscErrorCode invalidate_(PetscDeviceContext dctx) noexcept
 66:   {
 67:     PetscFunctionBegin;
 68:     PetscCall(CxxDataCast(dctx)->reset_self(dctx));
 69:     PetscFunctionReturn(PETSC_SUCCESS);
 70:   }
 71: };

 73: static Petsc::ObjectPool<_p_PetscDeviceContext, PetscDeviceContextConstructor> contextPool;

 75: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
 76: /*@
 77:   PetscDeviceContextCreate - Creates a `PetscDeviceContext`

 79:   Not Collective

 81:   Output Parameter:
 82: . dctx - The `PetscDeviceContext`

 84:   Level: beginner

 86:   Note:
 87:   Unlike almost every other PETSc class it is advised that most users use
 88:   `PetscDeviceContextDuplicate()` rather than this routine to create new contexts. Contexts of
 89:   different types are incompatible with one another; using `PetscDeviceContextDuplicate()`
 90:   ensures compatible types.

 92:   DAG representation:
 93: .vb
 94:   time ->

 96:   |= CALL =| - dctx ->
 97: .ve

 99: .N ASYNC_API

101: .seealso: `PetscDeviceContextDuplicate()`, `PetscDeviceContextSetDevice()`,
102: `PetscDeviceContextSetStreamType()`, `PetscDeviceContextSetUp()`,
103: `PetscDeviceContextSetFromOptions()`, `PetscDeviceContextView()`, `PetscDeviceContextDestroy()`
104: @*/
105: PetscErrorCode PetscDeviceContextCreate(PetscDeviceContext *dctx)
106: {
107:   PetscFunctionBegin;
108:   PetscAssertPointer(dctx, 1);
109:   PetscCall(PetscDeviceInitializePackage());
110:   PetscCall(PetscLogEventBegin(DCONTEXT_Create, nullptr, nullptr, nullptr, nullptr));
111:   PetscCall(contextPool.allocate(dctx));
112:   PetscCall(PetscLogEventEnd(DCONTEXT_Create, nullptr, nullptr, nullptr, nullptr));
113:   PetscFunctionReturn(PETSC_SUCCESS);
114: }

116: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
117: /*@
118:   PetscDeviceContextDestroy - Frees a `PetscDeviceContext`

120:   Not Collective

122:   Input Parameter:
123: . dctx - The `PetscDeviceContext`

125:   Level: beginner

127:   Notes:
128:   If `*dctx` is `NULL`, this routine does nothing. The pointer `dctx` itself must be valid.

130:   No implicit synchronization occurs due to this routine, all resources are released completely
131:   asynchronously w.r.t. the host. If one needs to guarantee access to the data produced on
132:   `dctx`'s stream the user is responsible for calling `PetscDeviceContextSynchronize()` before
133:   calling this routine.

135:   DAG representation:
136: .vb
137:   time ->

139:   -> dctx - |= CALL =|
140: .ve

142:   Developer Notes:
143:   `dctx` is never actually "destroyed" in the classical sense. It is returned to an ever
144:   growing pool of `PetscDeviceContext`s. There are currently no limits on the size of the pool,
145:   this should perhaps be implemented.

147: .N ASYNC_API

149: .seealso: `PetscDeviceContextCreate()`, `PetscDeviceContextSetDevice()`,
150: `PetscDeviceContextSetUp()`, `PetscDeviceContextSynchronize()`
151: @*/
152: PetscErrorCode PetscDeviceContextDestroy(PetscDeviceContext *dctx)
153: {
154:   PetscFunctionBegin;
155:   PetscAssertPointer(dctx, 1);
156:   if (!*dctx) PetscFunctionReturn(PETSC_SUCCESS);
158:   PetscCall(PetscLogEventBegin(DCONTEXT_Destroy, nullptr, nullptr, nullptr, nullptr));
159:   if (--(PetscObjectCast(*dctx)->refct) <= 0) {
160:     PetscCall(PetscDeviceContextCheckNotOrphaned_Internal(*dctx));
161:     PetscCall(contextPool.deallocate(dctx));
162:   }
163:   PetscCall(PetscLogEventEnd(DCONTEXT_Destroy, nullptr, nullptr, nullptr, nullptr));
164:   *dctx = nullptr;
165:   PetscFunctionReturn(PETSC_SUCCESS);
166: }

168: /*@
169:   PetscDeviceContextSetStreamType - Set the implementation type of the underlying stream for a
170:   `PetscDeviceContext`

172:   Not Collective

174:   Input Parameters:
175: + dctx - The `PetscDeviceContext`
176: - type - The `PetscStreamType`

178:   Level: intermediate

180:   Note:
181:   See `PetscStreamType` in `include/petscdevicetypes.h` for more information on the available
182:   types and their interactions. If the `PetscDeviceContext` was previously set up and stream
183:   type was changed, you must call `PetscDeviceContextSetUp()` again after this routine.

185: .seealso: `PetscStreamType`, `PetscDeviceContextGetStreamType()`, `PetscDeviceContextCreate()`,
186: `PetscDeviceContextSetUp()`, `PetscDeviceContextSetFromOptions()`
187: @*/
188: PetscErrorCode PetscDeviceContextSetStreamType(PetscDeviceContext dctx, PetscStreamType type)
189: {
190:   PetscFunctionBegin;
193:   // only need to do complex swapping if the object has already been setup
194:   if (dctx->setup && (dctx->streamType != type)) {
195:     dctx->setup = PETSC_FALSE;
196:     PetscCall(PetscLogEventBegin(DCONTEXT_ChangeStream, dctx, nullptr, nullptr, nullptr));
197:     PetscUseTypeMethod(dctx, changestreamtype, type);
198:     PetscCall(PetscLogEventEnd(DCONTEXT_ChangeStream, dctx, nullptr, nullptr, nullptr));
199:   }
200:   dctx->streamType = type;
201:   PetscFunctionReturn(PETSC_SUCCESS);
202: }

204: /*@
205:   PetscDeviceContextGetStreamType - Get the implementation type of the underlying stream for a
206:   `PetscDeviceContext`

208:   Not Collective

210:   Input Parameter:
211: . dctx - The `PetscDeviceContext`

213:   Output Parameter:
214: . type - The `PetscStreamType`

216:   Level: intermediate

218:   Note:
219:   See `PetscStreamType` in `include/petscdevicetypes.h` for more information on the available
220:   types and their interactions

222: .seealso: `PetscDeviceContextSetStreamType()`, `PetscDeviceContextCreate()`,
223: `PetscDeviceContextSetFromOptions()`
224: @*/
225: PetscErrorCode PetscDeviceContextGetStreamType(PetscDeviceContext dctx, PetscStreamType *type)
226: {
227:   PetscFunctionBegin;
228:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
230:   PetscAssertPointer(type, 2);
231:   *type = dctx->streamType;
232:   PetscFunctionReturn(PETSC_SUCCESS);
233: }

235: /*
236:   Actual function to set the device.

238:   1. Repeatedly destroying and recreating internal data structures (like streams and events)
239:      for recycled PetscDeviceContexts is not free. If done often, it does add up.
240:   2. The vast majority of PetscDeviceContexts are created by PETSc either as children or
241:      default contexts. The default contexts *never* change type, and the children are extremely
242:      unlikely to (chances are if you fork once, you will fork again very soon).
243:   3. The only time this calculus changes is if the user themselves sets the device type. In
244:      this case we do not know what the user has changed, so must always wipe the slate clean.

246:   Thus we need to keep track whether the user explicitly sets the device contexts device.
247: */
248: static PetscErrorCode PetscDeviceContextSetDevice_Private(PetscDeviceContext dctx, PetscDevice device, PetscBool user_set)
249: {
250:   PetscFunctionBegin;
253:   if (dctx->device && (dctx->device->id == device->id)) PetscFunctionReturn(PETSC_SUCCESS);
254:   PetscCall(PetscLogEventBegin(DCONTEXT_SetDevice, dctx, nullptr, nullptr, nullptr));
255:   PetscTryTypeMethod(dctx, destroy);
256:   PetscCall(PetscDeviceDestroy(&dctx->device));
257:   PetscCall(PetscMemzero(dctx->ops, sizeof(*dctx->ops)));
258:   PetscCall(PetscDeviceReference_Internal(device));
259:   // set it before calling the method
260:   dctx->device = device;
261:   PetscCall((*device->ops->createcontext)(dctx));
262:   PetscCall(PetscLogEventEnd(DCONTEXT_SetDevice, dctx, nullptr, nullptr, nullptr));
263:   dctx->setup         = PETSC_FALSE;
264:   dctx->usersetdevice = user_set;
265:   PetscFunctionReturn(PETSC_SUCCESS);
266: }

268: PetscErrorCode PetscDeviceContextSetDefaultDeviceForType_Internal(PetscDeviceContext dctx, PetscDeviceType type)
269: {
270:   PetscDevice device;

272:   PetscFunctionBegin;
273:   PetscCall(PetscDeviceGetDefaultForType_Internal(type, &device));
274:   PetscCall(PetscDeviceContextSetDevice_Private(dctx, device, PETSC_FALSE));
275:   PetscFunctionReturn(PETSC_SUCCESS);
276: }

278: /*@
279:   PetscDeviceContextSetDevice - Set the underlying `PetscDevice` for a `PetscDeviceContext`

281:   Not Collective

283:   Input Parameters:
284: + dctx   - The `PetscDeviceContext`
285: - device - The `PetscDevice`

287:   Level: intermediate

289:   Notes:
290:   This routine is effectively `PetscDeviceContext`'s "set-type" (so every `PetscDeviceContext` must
291:   also have an attached `PetscDevice`). Unlike the usual set-type semantics, it is not strictly
292:   necessary to set a contexts device to enable usage, any created `PetscDeviceContext`s will
293:   always come equipped with the "default" device.

295:   This routine is a no-op if `device` is already attached to `dctx`.

297:   This routine may (but is very unlikely to) initialize the backend device and may incur
298:   synchronization.

300: .seealso: `PetscDeviceCreate()`, `PetscDeviceConfigure()`, `PetscDeviceContextGetDevice()`,
301: `PetscDeviceContextGetDeviceType()`
302: @*/
303: PetscErrorCode PetscDeviceContextSetDevice(PetscDeviceContext dctx, PetscDevice device)
304: {
305:   PetscFunctionBegin;
306:   PetscCall(PetscDeviceContextSetDevice_Private(dctx, device, PETSC_TRUE));
307:   PetscFunctionReturn(PETSC_SUCCESS);
308: }

310: /*@
311:   PetscDeviceContextGetDevice - Get the underlying `PetscDevice` for a `PetscDeviceContext`

313:   Not Collective

315:   Input Parameter:
316: . dctx - the `PetscDeviceContext`

318:   Output Parameter:
319: . device - The `PetscDevice`

321:   Level: intermediate

323:   Note:
324:   This is a borrowed reference, the user should not destroy `device`.

326: .seealso: `PetscDeviceContextSetDevice()`, `PetscDevice`, `PetscDeviceContextGetDeviceType()`
327: @*/
328: PetscErrorCode PetscDeviceContextGetDevice(PetscDeviceContext dctx, PetscDevice *device)
329: {
330:   PetscFunctionBegin;
331:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
333:   PetscAssertPointer(device, 2);
334:   PetscAssert(dctx->device, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "PetscDeviceContext %" PetscInt64_FMT " has no attached PetscDevice to get", PetscObjectCast(dctx)->id);
335:   *device = dctx->device;
336:   PetscFunctionReturn(PETSC_SUCCESS);
337: }

339: /*@
340:   PetscDeviceContextGetDeviceType - Get the `PetscDeviceType` for a `PetscDeviceContext`

342:   Not Collective

344:   Input Parameter:
345: . dctx - The `PetscDeviceContext`

347:   Output Parameter:
348: . type - The `PetscDeviceType`

350:   Level: intermediate

352:   Note:
353:   This routine is a convenience shorthand for `PetscDeviceContextGetDevice()` ->
354:   `PetscDeviceGetType()`.

356: .seealso: `PetscDeviceType`, `PetscDeviceContextGetDevice()`, `PetscDeviceGetType()`, `PetscDevice`
357: @*/
358: PetscErrorCode PetscDeviceContextGetDeviceType(PetscDeviceContext dctx, PetscDeviceType *type)
359: {
360:   PetscDevice device = nullptr;

362:   PetscFunctionBegin;
363:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
365:   PetscAssertPointer(type, 2);
366:   PetscCall(PetscDeviceContextGetDevice(dctx, &device));
367:   PetscCall(PetscDeviceGetType(device, type));
368:   PetscFunctionReturn(PETSC_SUCCESS);
369: }

371: /*@
372:   PetscDeviceContextSetUp - Prepares a `PetscDeviceContext` for use

374:   Not Collective

376:   Input Parameter:
377: . dctx - The `PetscDeviceContext`

379:   Level: beginner

381:   Developer Notes:
382:   This routine is usually the stage where a `PetscDeviceContext` acquires device-side data
383:   structures such as streams, events, and (possibly) handles.

385: .seealso: `PetscDeviceContextCreate()`, `PetscDeviceContextSetDevice()`,
386: `PetscDeviceContextDestroy()`, `PetscDeviceContextSetFromOptions()`
387: @*/
388: PetscErrorCode PetscDeviceContextSetUp(PetscDeviceContext dctx)
389: {
390:   PetscFunctionBegin;
391:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
393:   if (dctx->setup) PetscFunctionReturn(PETSC_SUCCESS);
394:   if (!dctx->device) {
395:     const auto default_dtype = PETSC_DEVICE_DEFAULT();

397:     PetscCall(PetscInfo(dctx, "PetscDeviceContext %" PetscInt64_FMT " did not have an explicitly attached PetscDevice, using default with type %s\n", PetscObjectCast(dctx)->id, PetscDeviceTypes[default_dtype]));
398:     PetscCall(PetscDeviceContextSetDefaultDeviceForType_Internal(dctx, default_dtype));
399:   }
400:   PetscCall(PetscLogEventBegin(DCONTEXT_SetUp, dctx, nullptr, nullptr, nullptr));
401:   PetscUseTypeMethod(dctx, setup);
402:   PetscCall(PetscLogEventEnd(DCONTEXT_SetUp, dctx, nullptr, nullptr, nullptr));
403:   dctx->setup = PETSC_TRUE;
404:   PetscFunctionReturn(PETSC_SUCCESS);
405: }

407: static PetscErrorCode PetscDeviceContextDuplicate_Private(PetscDeviceContext dctx, PetscStreamType stype, PetscDeviceContext *dctxdup)
408: {
409:   PetscFunctionBegin;
410:   PetscCall(PetscLogEventBegin(DCONTEXT_Duplicate, dctx, nullptr, nullptr, nullptr));
411:   PetscCall(PetscDeviceContextCreate(dctxdup));
412:   PetscCall(PetscDeviceContextSetStreamType(*dctxdup, stype));
413:   if (const auto device = dctx->device) PetscCall(PetscDeviceContextSetDevice_Private(*dctxdup, device, dctx->usersetdevice));
414:   PetscCall(PetscDeviceContextSetUp(*dctxdup));
415:   PetscCall(PetscLogEventEnd(DCONTEXT_Duplicate, dctx, nullptr, nullptr, nullptr));
416:   PetscFunctionReturn(PETSC_SUCCESS);
417: }

419: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
420: /*@
421:   PetscDeviceContextDuplicate - Duplicates a `PetscDeviceContext` object

423:   Not Collective

425:   Input Parameter:
426: . dctx - The `PetscDeviceContext` to duplicate

428:   Output Parameter:
429: . dctxdup - The duplicated `PetscDeviceContext`

431:   Level: beginner

433:   Notes:
434:   This is a shorthand method for creating a `PetscDeviceContext` with the exact same settings as
435:   another. Note however that `dctxdup` does not share any of the underlying data with `dctx`,
436:   (including its current stream-state) they are completely separate objects.

438:   There is no implied ordering between `dctx` or `dctxdup`.

440:   DAG representation:
441: .vb
442:   time ->

444:   -> dctx - |= CALL =| - dctx ---->
445:                        - dctxdup ->
446: .ve

448: .seealso: `PetscDeviceContextCreate()`, `PetscDeviceContextSetDevice()`,
449: `PetscDeviceContextSetStreamType()`
450: @*/
451: PetscErrorCode PetscDeviceContextDuplicate(PetscDeviceContext dctx, PetscDeviceContext *dctxdup)
452: {
453:   auto stype = PETSC_STREAM_DEFAULT;

455:   PetscFunctionBegin;
456:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
458:   PetscAssertPointer(dctxdup, 2);
459:   PetscCall(PetscDeviceContextGetStreamType(dctx, &stype));
460:   PetscCall(PetscDeviceContextDuplicate_Private(dctx, stype, dctxdup));
461:   PetscFunctionReturn(PETSC_SUCCESS);
462: }

464: /*@
465:   PetscDeviceContextQueryIdle - Returns whether or not a `PetscDeviceContext` is idle

467:   Not Collective

469:   Input Parameter:
470: . dctx - The `PetscDeviceContext`

472:   Output Parameter:
473: . idle - `PETSC_TRUE` if `dctx` has NO work, `PETSC_FALSE` if it has work

475:   Level: beginner

477:   Note:
478:   This routine only refers a singular context and does NOT take any of its children into
479:   account. That is, if `dctx` is idle but has dependents who do have work this routine still
480:   returns `PETSC_TRUE`.

482: .seealso: `PetscDeviceContextCreate()`, `PetscDeviceContextWaitForContext()`, `PetscDeviceContextFork()`
483: @*/
484: PetscErrorCode PetscDeviceContextQueryIdle(PetscDeviceContext dctx, PetscBool *idle)
485: {
486:   PetscFunctionBegin;
487:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
489:   PetscAssertPointer(idle, 2);
490:   PetscCall(PetscLogEventBegin(DCONTEXT_QueryIdle, dctx, nullptr, nullptr, nullptr));
491:   PetscUseTypeMethod(dctx, query, idle);
492:   PetscCall(PetscLogEventEnd(DCONTEXT_QueryIdle, dctx, nullptr, nullptr, nullptr));
493:   PetscCall(PetscInfo(dctx, "PetscDeviceContext ('%s', id %" PetscInt64_FMT ") %s idle\n", PetscObjectCast(dctx)->name ? PetscObjectCast(dctx)->name : "unnamed", PetscObjectCast(dctx)->id, *idle ? "was" : "was not"));
494:   PetscFunctionReturn(PETSC_SUCCESS);
495: }

497: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
498: /*@
499:   PetscDeviceContextWaitForContext - Make one context wait for another context to finish

501:   Not Collective

503:   Input Parameters:
504: + dctxa - The `PetscDeviceContext` object that is waiting
505: - dctxb - The `PetscDeviceContext` object that is being waited on

507:   Level: advanced

509:   Notes:
510:   Serializes two `PetscDeviceContext`s. Serialization is performed asynchronously; the host
511:   does not wait for the serialization to actually occur.

513:   This routine uses only the state of `dctxb` at the moment this routine was called, so any
514:   future work queued will not affect `dctxa`. It is safe to pass the same context to both
515:   arguments (in which case this routine does nothing).

517:   DAG representation:
518: .vb
519:   time ->

521:   -> dctxa ---/- |= CALL =| - dctxa ->
522:              /
523:   -> dctxb -/------------------------>
524: .ve

526: .N ASYNC_API

528: .seealso: `PetscDeviceContextCreate()`, `PetscDeviceContextQueryIdle()`, `PetscDeviceContextJoin()`
529: @*/
530: PetscErrorCode PetscDeviceContextWaitForContext(PetscDeviceContext dctxa, PetscDeviceContext dctxb)
531: {
532:   PetscObjectId bid;

534:   PetscFunctionBegin;
537:   if (dctxa == dctxb) PetscFunctionReturn(PETSC_SUCCESS);
538:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctxa));
539:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctxb));
540:   PetscCheckCompatibleDeviceContexts(dctxa, 1, dctxb, 2);
541:   if (dctxa == dctxb) PetscFunctionReturn(PETSC_SUCCESS);
542:   bid = PetscObjectCast(dctxb)->id;
543:   PetscCall(PetscLogEventBegin(DCONTEXT_WaitForCtx, dctxa, dctxb, nullptr, nullptr));
544:   PetscUseTypeMethod(dctxa, waitforcontext, dctxb);
545:   PetscCallCXX(CxxDataCast(dctxa)->upstream()[bid] = CxxDataCast(dctxb)->weak_snapshot());
546:   PetscCall(PetscLogEventEnd(DCONTEXT_WaitForCtx, dctxa, dctxb, nullptr, nullptr));
547:   PetscCall(PetscInfo(dctxa, "dctx %" PetscInt64_FMT " waiting on dctx %" PetscInt64_FMT "\n", PetscObjectCast(dctxa)->id, bid));
548:   PetscCall(PetscObjectStateIncrease(PetscObjectCast(dctxa)));
549:   PetscFunctionReturn(PETSC_SUCCESS);
550: }

552: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
553: /*@
554:   PetscDeviceContextForkWithStreamType - Create a set of dependent child contexts from a parent
555:   context with a prescribed `PetscStreamType`

557:   Not Collective, Asynchronous

559:   Input Parameters:
560: + dctx  - The parent `PetscDeviceContext`
561: . stype - The prescribed `PetscStreamType`
562: - n     - The number of children to create

564:   Output Parameter:
565: . dsub - The created child context(s)

567:   Level: advanced

569:   Notes:
570:   This routine creates `n` edges of a DAG from a source node which are causally dependent on the
571:   source node. This causal dependency is established as-if by calling
572:   `PetscDeviceContextWaitForContext()` on every child.

574:   `dsub` is allocated by this routine and has its lifetime bounded by `dctx`. That is, `dctx`
575:   expects to free `dsub` (via `PetscDeviceContextJoin()`) before it itself is destroyed.

577:   This routine only accounts for work queued on `dctx` up until calling this routine, any
578:   subsequent work enqueued on `dctx` has no effect on `dsub`.

580:   The `PetscStreamType` of `dctx` does not have to equal `stype`. In fact, it is often the case
581:   that they are different. This is useful in cases where a routine can locally exploit stream
582:   parallelism without needing to worry about what stream type the incoming `PetscDeviceContext`
583:   carries.

585:   DAG representation:
586: .vb
587:   time ->

589:   -> dctx - |= CALL =| -\----> dctx ------>
590:                          \---> dsub[0] --->
591:                           \--> ... ------->
592:                            \-> dsub[n-1] ->
593: .ve

595: .N ASYNC_API

597: .seealso: `PetscDeviceContextJoin()`, `PetscDeviceContextSynchronize()`,
598: `PetscDeviceContextQueryIdle()`, `PetscDeviceContextWaitForContext()`
599: @*/
600: PetscErrorCode PetscDeviceContextForkWithStreamType(PetscDeviceContext dctx, PetscStreamType stype, PetscInt n, PetscDeviceContext **dsub)
601: {
602:   // debugging only
603:   std::string idList;
604:   auto        ninput = n;

606:   PetscFunctionBegin;
607:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
609:   PetscAssert(n >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Number of contexts requested %" PetscInt_FMT " < 0", n);
610:   PetscAssertPointer(dsub, 4);
611:   *dsub = nullptr;
612:   /* reserve 4 chars per id, 2 for number and 2 for ', ' separator */
613:   if (PetscDefined(USE_DEBUG_AND_INFO)) PetscCallCXX(idList.reserve(4 * n));
614:   PetscCall(PetscLogEventBegin(DCONTEXT_Fork, dctx, nullptr, nullptr, nullptr));
615:   /* update child totals */
616:   dctx->numChildren += n;
617:   /* now to find out if we have room */
618:   if (dctx->numChildren > dctx->maxNumChildren) {
619:     const auto numChildren    = dctx->numChildren;
620:     auto      &maxNumChildren = dctx->maxNumChildren;
621:     auto       numAllocated   = numChildren;

623:     /* no room, either from having too many kids or not having any */
624:     if (auto &childIDs = dctx->childIDs) {
625:       // the difference is backwards because we have not updated maxNumChildren yet
626:       numAllocated -= maxNumChildren;
627:       /* have existing children, must reallocate them */
628:       PetscCall(PetscRealloc(numChildren * sizeof(*childIDs), &childIDs));
629:       /* clear the extra memory since realloc doesn't do it for us */
630:       PetscCall(PetscArrayzero(std::next(childIDs, maxNumChildren), numAllocated));
631:     } else {
632:       /* have no children */
633:       PetscCall(PetscCalloc1(numChildren, &childIDs));
634:     }
635:     /* update total number of children */
636:     maxNumChildren = numChildren;
637:   }
638:   PetscCall(PetscMalloc1(n, dsub));
639:   for (PetscInt i = 0; ninput && (i < dctx->numChildren); ++i) {
640:     auto &childID = dctx->childIDs[i];
641:     /* empty child slot */
642:     if (!childID) {
643:       auto &childctx = (*dsub)[i];

645:       /* create the child context in the image of its parent */
646:       PetscCall(PetscDeviceContextDuplicate_Private(dctx, stype, &childctx));
647:       PetscCall(PetscDeviceContextWaitForContext(childctx, dctx));
648:       /* register the child with its parent */
649:       PetscCall(PetscObjectGetId(PetscObjectCast(childctx), &childID));
650:       if (PetscDefined(USE_DEBUG_AND_INFO)) {
651:         PetscCallCXX(idList += std::to_string(childID));
652:         if (ninput != 1) PetscCallCXX(idList += ", ");
653:       }
654:       --ninput;
655:     }
656:   }
657:   PetscCall(PetscLogEventEnd(DCONTEXT_Fork, dctx, nullptr, nullptr, nullptr));
658:   PetscCall(PetscDebugInfo(dctx, "Forked %" PetscInt_FMT " children from parent %" PetscInt64_FMT " with IDs: %s\n", n, PetscObjectCast(dctx)->id, idList.c_str()));
659:   PetscFunctionReturn(PETSC_SUCCESS);
660: }

662: /*@
663:   PetscDeviceContextFork - Create a set of dependent child contexts from a parent context

665:   Not Collective, Asynchronous

667:   Input Parameters:
668: + dctx - The parent `PetscDeviceContext`
669: - n    - The number of children to create

671:   Output Parameter:
672: . dsub - The created child context(s)

674:   Level: advanced

676:   Notes:
677:   Behaves identically to `PetscDeviceContextForkWithStreamType()` except that the prescribed
678:   `PetscStreamType` is taken from `dctx`. In effect this routine is shorthand for\:

680: .vb
681:   PetscStreamType stype;

683:   PetscDeviceContextGetStreamType(dctx, &stype);
684:   PetscDeviceContextForkWithStreamType(dctx, stype, ...);
685: .ve

687: .N ASYNC_API

689: .seealso: `PetscDeviceContextForkWithStreamType()`, `PetscDeviceContextJoin()`,
690: `PetscDeviceContextSynchronize()`, `PetscDeviceContextQueryIdle()`
691: @*/
692: PetscErrorCode PetscDeviceContextFork(PetscDeviceContext dctx, PetscInt n, PetscDeviceContext **dsub)
693: {
694:   auto stype = PETSC_STREAM_DEFAULT;

696:   PetscFunctionBegin;
697:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
699:   PetscCall(PetscDeviceContextGetStreamType(dctx, &stype));
700:   PetscCall(PetscDeviceContextForkWithStreamType(dctx, stype, n, dsub));
701:   PetscFunctionReturn(PETSC_SUCCESS);
702: }

704: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
705: /*@
706:   PetscDeviceContextJoin - Converge a set of child contexts

708:   Not Collective, Asynchronous

710:   Input Parameters:
711: + dctx     - A `PetscDeviceContext` to converge on
712: . n        - The number of sub contexts to converge
713: . joinMode - The type of join to perform
714: - dsub     - The sub contexts to converge

716:   Level: advanced

718:   Notes:
719:   If `PetscDeviceContextFork()` creates `n` edges from a source node which all depend on the source
720:   node, then this routine is the exact mirror. That is, it creates a node (represented in `dctx`)
721:   which receives `n` edges (and optionally destroys them) which is dependent on the completion
722:   of all incoming edges.

724:   If `joinMode` is `PETSC_DEVICE_CONTEXT_JOIN_DESTROY`. All contexts in `dsub` will be
725:   destroyed by this routine. Thus all sub contexts must have been created with the `dctx`
726:   passed to this routine.

728:   If `joinMode` is `PETSC_DEVICE_CONTEXT_JOIN_SYNC`. All sub contexts will additionally wait on
729:   `dctx` after converging. This has the effect of "synchronizing" the outgoing edges. Note the
730:   sync suffix does NOT refer to the host, i.e. this routine does NOT call
731:   `PetscDeviceSynchronize()`.

733:   If `joinMode` is `PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC`. `dctx` waits for all sub contexts but
734:   the sub contexts do not wait for one another or `dctx` afterwards.

736:   DAG representations:
737:   If `joinMode` is `PETSC_DEVICE_CONTEXT_JOIN_DESTROY`
738: .vb
739:   time ->

741:   -> dctx ---------/- |= CALL =| - dctx ->
742:   -> dsub[0] -----/
743:   ->  ... -------/
744:   -> dsub[n-1] -/
745: .ve
746:   If `joinMode` is `PETSC_DEVICE_CONTEXT_JOIN_SYNC`
747: .vb
748:   time ->

750:   -> dctx ---------/- |= CALL =| -\----> dctx ------>
751:   -> dsub[0] -----/                \---> dsub[0] --->
752:   ->  ... -------/                  \--> ... ------->
753:   -> dsub[n-1] -/                    \-> dsub[n-1] ->
754: .ve
755:   If `joinMode` is `PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC`
756: .vb
757:   time ->

759:   -> dctx ----------/- |= CALL =| - dctx ->
760:   -> dsub[0] ------/----------------------->
761:   ->  ... --------/------------------------>
762:   -> dsub[n-1] --/------------------------->
763: .ve

765: .N ASYNC_API

767: .seealso: `PetscDeviceContextFork()`, `PetscDeviceContextForkWithStreamType()`,
768: `PetscDeviceContextSynchronize()`, `PetscDeviceContextJoinMode`
769: @*/
770: PetscErrorCode PetscDeviceContextJoin(PetscDeviceContext dctx, PetscInt n, PetscDeviceContextJoinMode joinMode, PetscDeviceContext **dsub)
771: {
772:   // debugging only
773:   std::string idList;

775:   PetscFunctionBegin;
776:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
778:   PetscAssertPointer(dsub, 4);
779:   PetscAssert(n >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Number of contexts merged %" PetscInt_FMT " < 0", n);
780:   /* reserve 4 chars per id, 2 for number and 2 for ', ' separator */
781:   if (PetscDefined(USE_DEBUG_AND_INFO)) PetscCallCXX(idList.reserve(4 * n));
782:   /* first dctx waits on all the incoming edges */
783:   PetscCall(PetscLogEventBegin(DCONTEXT_Join, dctx, nullptr, nullptr, nullptr));
784:   for (PetscInt i = 0; i < n; ++i) {
785:     PetscCheckCompatibleDeviceContexts(dctx, 1, (*dsub)[i], 4);
786:     PetscCall(PetscDeviceContextWaitForContext(dctx, (*dsub)[i]));
787:     if (PetscDefined(USE_DEBUG_AND_INFO)) {
788:       PetscCallCXX(idList += std::to_string(PetscObjectCast((*dsub)[i])->id));
789:       if (i + 1 < n) PetscCallCXX(idList += ", ");
790:     }
791:   }

793:   /* now we handle the aftermath */
794:   switch (joinMode) {
795:   case PETSC_DEVICE_CONTEXT_JOIN_DESTROY: {
796:     const auto children = dctx->childIDs;
797:     const auto maxchild = dctx->maxNumChildren;
798:     auto      &nchild   = dctx->numChildren;
799:     PetscInt   j        = 0;

801:     PetscCheck(n <= nchild, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Trying to destroy %" PetscInt_FMT " children of a parent context that only has %" PetscInt_FMT " children, likely trying to restore to wrong parent", n, nchild);
802:     /* update child count while it's still fresh in memory */
803:     nchild -= n;
804:     for (PetscInt i = 0; i < maxchild; ++i) {
805:       if (children[i] && (children[i] == PetscObjectCast((*dsub)[j])->id)) {
806:         /* child is one of ours, can destroy it */
807:         PetscCall(PetscDeviceContextDestroy((*dsub) + j));
808:         /* reset the child slot */
809:         children[i] = 0;
810:         if (++j == n) break;
811:       }
812:     }
813:     /* gone through the loop but did not find every child */
814:     PetscCheck(j == n, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "%" PetscInt_FMT " contexts still remain after destroy, this may be because you are trying to restore to the wrong parent context, or the device contexts are not in the same order as they were checked out in", n - j);
815:     PetscCall(PetscFree(*dsub));
816:   } break;
817:   case PETSC_DEVICE_CONTEXT_JOIN_SYNC:
818:     for (PetscInt i = 0; i < n; ++i) PetscCall(PetscDeviceContextWaitForContext((*dsub)[i], dctx));
819:   case PETSC_DEVICE_CONTEXT_JOIN_NO_SYNC:
820:     break;
821:   default:
822:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Unknown PetscDeviceContextJoinMode given");
823:   }
824:   PetscCall(PetscLogEventEnd(DCONTEXT_Join, dctx, nullptr, nullptr, nullptr));

826:   PetscCall(PetscDebugInfo(dctx, "Joined %" PetscInt_FMT " ctxs to ctx %" PetscInt64_FMT ", mode %s with IDs: %s\n", n, PetscObjectCast(dctx)->id, PetscDeviceContextJoinModes[joinMode], idList.c_str()));
827:   PetscFunctionReturn(PETSC_SUCCESS);
828: }

830: // PetscClangLinter pragma disable: -fdoc-section-header-unknown
831: /*@
832:   PetscDeviceContextSynchronize - Block the host until all work queued on a
833:   `PetscDeviceContext` has finished

835:   Not Collective

837:   Input Parameter:
838: . dctx - The `PetscDeviceContext` to synchronize

840:   Level: beginner

842:   Notes:
843:   The host will not return from this routine until `dctx` is idle. Any and all memory
844:   operations queued on or otherwise associated with (either explicitly or implicitly via
845:   dependencies) are guaranteed to have finished and be globally visible on return.

847:   In effect, this routine serves as memory and execution barrier.

849:   DAG representation:
850: .vb
851:   time ->

853:   -> dctx - |= CALL =| - dctx ->
854: .ve

856: .seealso: `PetscDeviceContextFork()`, `PetscDeviceContextJoin()`, `PetscDeviceContextQueryIdle()`
857: @*/
858: PetscErrorCode PetscDeviceContextSynchronize(PetscDeviceContext dctx)
859: {
860:   PetscFunctionBegin;
861:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
863:   PetscCall(PetscLogEventBegin(DCONTEXT_Sync, dctx, nullptr, nullptr, nullptr));
864:   /* if it isn't setup there is nothing to sync on */
865:   if (dctx->setup) {
866:     PetscUseTypeMethod(dctx, synchronize);
867:     PetscCall(PetscDeviceContextSyncClearMap_Internal(dctx));
868:   }
869:   PetscCall(PetscLogEventEnd(DCONTEXT_Sync, dctx, nullptr, nullptr, nullptr));
870:   PetscFunctionReturn(PETSC_SUCCESS);
871: }

873: /*@
874:   PetscDeviceContextDelay - Queue a delay on a device context for testing stream ordering

876:   Not Collective; Asynchronous

878:   Input Parameters:
879: + dctx    - the device context
880: - seconds - length of the delay in seconds, which must be finite and nonnegative

882:   Level: developer

884:   Notes:
885:   This routine is intended for tests of asynchronous stream operations. On CUDA and HIP, it
886:   queues a host callback after earlier work on `dctx` without waiting for the callback to
887:   complete. Work queued later on the same context waits for the callback. Use
888:   `PetscDeviceContextSynchronize()` to wait for the delay to finish.

890:   A host context sleeps before returning. SYCL contexts are not yet supported.

892: .seealso: `PetscDeviceContextSynchronize()`, `PetscDeviceContextQueryIdle()`, `PetscDeviceContextGetStreamHandle()`
893: @*/
894: PetscErrorCode PetscDeviceContextDelay(PetscDeviceContext dctx, PetscReal seconds)
895: {
896:   PetscFunctionBegin;
898:   PetscCheck(seconds >= 0 && std::isfinite(static_cast<double>(seconds)), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Delay must be finite and nonnegative, got %g", static_cast<double>(seconds));
899:   if (!seconds) PetscFunctionReturn(PETSC_SUCCESS);
900:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
901:   PetscCall(PetscDeviceContextSetUp(dctx));
902:   PetscUseTypeMethod(dctx, delay, seconds);
903:   PetscFunctionReturn(PETSC_SUCCESS);
904: }

906: /* Cache one default-stream context per backend and device ID. */
907: static auto defaultContexts          = std::array<std::vector<PetscDeviceContext>, PETSC_DEVICE_MAX>{};
908: static auto defaultContextsFinalizer = false;

910: static PetscErrorCode PetscDeviceContextGetDefaultContextForDevice_Private(PetscBool user_set_device, PetscDevice device, PetscDeviceContext *dctx)
911: {
912:   PetscInt        devid;
913:   PetscDeviceType dtype;

915:   PetscFunctionBegin;
917:   PetscAssertPointer(dctx, 3);
918:   if (PetscUnlikely(!defaultContextsFinalizer)) {
919:     defaultContextsFinalizer = true;
920:     PetscCall(PetscRegisterFinalize([] {
921:       PetscFunctionBegin;
922:       for (auto &&dvec : defaultContexts) {
923:         for (auto &&dctx : dvec) PetscCall(PetscDeviceContextDestroy(&dctx));
924:         PetscCallCXX(dvec.clear());
925:       }
926:       defaultContextsFinalizer = false;
927:       PetscFunctionReturn(PETSC_SUCCESS);
928:     }));
929:   }
930:   PetscCall(PetscDeviceGetDeviceId(device, &devid));
931:   PetscCall(PetscDeviceGetType(device, &dtype));
932:   {
933:     auto &ctxlist = defaultContexts[dtype];

935:     PetscCheck(devid >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Device ID (%" PetscInt_FMT ") must be positive", devid);
936:     if (static_cast<std::size_t>(devid) >= ctxlist.size()) PetscCallCXX(ctxlist.resize(devid + 1));
937:     if (PetscUnlikely(!ctxlist[devid])) {
938:       // we have not seen this device before
939:       PetscCall(PetscDeviceContextCreate(dctx));
940:       PetscCall(PetscInfo(*dctx, "Initializing default PetscDeviceContext (of type %s) for device %" PetscInt_FMT "\n", PetscDeviceTypes[dtype], devid));
941:       {
942:         const auto pobj   = PetscObjectCast(*dctx);
943:         const auto name   = "default context " + std::to_string(devid);
944:         const auto prefix = "default_context_" + std::to_string(devid) + '_';

946:         PetscCall(PetscObjectSetName(pobj, name.c_str()));
947:         PetscCall(PetscObjectSetOptionsPrefix(pobj, prefix.c_str()));
948:       }
949:       PetscCall(PetscDeviceContextSetStreamType(*dctx, PETSC_STREAM_DEFAULT));
950:       PetscCall(PetscDeviceContextSetDevice_Private(*dctx, device, user_set_device));
951:       PetscCall(PetscDeviceContextSetUp(*dctx));
952:       ctxlist[devid] = *dctx;
953:     } else *dctx = ctxlist[devid];
954:   }
955:   PetscFunctionReturn(PETSC_SUCCESS);
956: }

958: /*@
959:   PetscDeviceContextGetDefaultContext - Get the default-stream context for the current device

961:   Not Collective

963:   Output Parameter:
964: . dctx - The borrowed default-stream context

966:   Level: beginner

968:   Notes:
969:   The context uses `PETSC_STREAM_DEFAULT` on the device attached to the current context.
970:   It is created lazily and owned by PETSc. Do not destroy or reconfigure the borrowed context.
971:   Duplicate it to obtain an independently owned context.

973:   The returned handle captures a particular context and remains valid until `PetscFinalize()`, even
974:   if the current device changes. In contrast, `PetscDeviceContextDefault` is resolved again at each
975:   accepting API call. In the C-only device-interface fallback, this routine returns a null handle.

977: .seealso: `PetscDeviceContextDefault`, `PetscDeviceContextGetCurrentContext()`, `PetscDeviceContextDuplicate()`
978: @*/
979: PetscErrorCode PetscDeviceContextGetDefaultContext(PetscDeviceContext *dctx)
980: {
981:   PetscDeviceContext current;
982:   PetscDevice        device;

984:   PetscFunctionBegin;
985:   PetscAssertPointer(dctx, 1);
986:   PetscCall(PetscDeviceContextGetCurrentContext(&current));
987:   PetscCall(PetscDeviceContextGetDevice(current, &device));
988:   PetscCall(PetscDeviceContextGetDefaultContextForDevice_Private(current->usersetdevice, device, dctx));
989:   PetscFunctionReturn(PETSC_SUCCESS);
990: }

992: /*@
993:   PetscDeviceContextSetFromOptions - Configure a `PetscDeviceContext` from the options database

995:   Collective on `comm` or `dctx`

997:   Input Parameters:
998: + comm - MPI communicator on which to query the options database (optional)
999: - dctx - The `PetscDeviceContext` to configure

1001:   Output Parameter:
1002: . dctx - The `PetscDeviceContext`

1004:   Options Database Keys:
1005: + -device_context_stream_type - type of stream to create inside the `PetscDeviceContext` -
1006:    `PetscDeviceContextSetStreamType()`
1007: - -device_context_device_type - the type of `PetscDevice` to attach by default - `PetscDeviceType`

1009:   Level: beginner

1011:   Note:
1012:   The user may pass `MPI_COMM_NULL` for `comm` in which case the communicator of `dctx` is
1013:   used (which is always `PETSC_COMM_SELF`).

1015: .seealso: `PetscDeviceContextSetStreamType()`, `PetscDeviceContextSetDevice()`,
1016: `PetscDeviceContextView()`
1017: @*/
1018: PetscErrorCode PetscDeviceContextSetFromOptions(MPI_Comm comm, PetscDeviceContext dctx)
1019: {
1020:   const auto pobj     = PetscObjectCast(dctx);
1021:   auto       dtype    = std::make_pair(PETSC_DEVICE_DEFAULT(), PETSC_FALSE);
1022:   auto       stype    = std::make_pair(PETSC_DEVICE_CONTEXT_DEFAULT_STREAM_TYPE, PETSC_FALSE);
1023:   MPI_Comm   old_comm = PETSC_COMM_SELF;

1025:   PetscFunctionBegin;
1027:   /* set the device type first */
1028:   if (const auto device = dctx->device) PetscCall(PetscDeviceGetType(device, &dtype.first));
1029:   PetscCall(PetscDeviceContextGetStreamType(dctx, &stype.first));

1031:   if (comm == MPI_COMM_NULL) PetscCall(PetscObjectGetComm(pobj, &comm));
1032:   else {
1033:     // briefly set the communicator for dctx (it is always PETSC_COMM_SELF) so
1034:     // PetscObjectOptionsBegin() behaves as if dctx had comm
1035:     old_comm = Petsc::util::exchange(pobj->comm, comm);
1036:   }

1038:   PetscObjectOptionsBegin(pobj);
1039:   PetscCall(PetscDeviceContextQueryOptions_Internal(PetscOptionsObject, dtype, stype));
1040:   PetscOptionsEnd();
1041:   // reset the comm (should be PETSC_COMM_SELF)
1042:   if (comm != MPI_COMM_NULL) pobj->comm = old_comm;
1043:   if (dtype.second) PetscCall(PetscDeviceContextSetDefaultDeviceForType_Internal(dctx, dtype.first));
1044:   if (stype.second) PetscCall(PetscDeviceContextSetStreamType(dctx, stype.first));
1045:   PetscCall(PetscDeviceContextSetUp(dctx));
1046:   PetscFunctionReturn(PETSC_SUCCESS);
1047: }

1049: /*@
1050:   PetscDeviceContextView - View a `PetscDeviceContext`

1052:   Collective on `viewer`

1054:   Input Parameters:
1055: + dctx   - The `PetscDeviceContext`
1056: - viewer - The `PetscViewer` to view `dctx` with (may be `NULL`)

1058:   Level: beginner

1060:   Note:
1061:   If `viewer` is `NULL`, `PETSC_VIEWER_STDOUT_WORLD` is used instead, in which case this
1062:   routine is collective on `PETSC_COMM_WORLD`.

1064: .seealso: `PetscDeviceContextViewFromOptions()`, `PetscDeviceView()`, `PETSC_VIEWER_STDOUT_WORLD`, `PetscDeviceContextCreate()`
1065: @*/
1066: PetscErrorCode PetscDeviceContextView(PetscDeviceContext dctx, PetscViewer viewer)
1067: {
1068:   PetscBool isascii;

1070:   PetscFunctionBegin;
1071:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
1073:   if (!viewer) PetscCall(PetscViewerASCIIGetStdout(PETSC_COMM_WORLD, &viewer));
1075:   PetscCall(PetscObjectTypeCompare(PetscObjectCast(viewer), PETSCVIEWERASCII, &isascii));
1076:   if (isascii) {
1077:     auto        stype = PETSC_STREAM_DEFAULT;
1078:     PetscViewer sub;

1080:     PetscCall(PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sub));
1081:     PetscCall(PetscObjectPrintClassNamePrefixType(PetscObjectCast(dctx), sub));
1082:     PetscCall(PetscViewerASCIIPushTab(sub));
1083:     PetscCall(PetscDeviceContextGetStreamType(dctx, &stype));
1084:     PetscCall(PetscViewerASCIIPrintf(sub, "stream type: %s\n", PetscStreamTypes[stype]));
1085:     PetscCall(PetscViewerASCIIPrintf(sub, "children: %" PetscInt_FMT "\n", dctx->numChildren));
1086:     if (const auto nchild = dctx->numChildren) {
1087:       PetscCall(PetscViewerASCIIPushTab(sub));
1088:       for (PetscInt i = 0; i < nchild; ++i) {
1089:         if (i == nchild - 1) {
1090:           PetscCall(PetscViewerASCIIPrintf(sub, "%" PetscInt64_FMT, dctx->childIDs[i]));
1091:         } else {
1092:           PetscCall(PetscViewerASCIIPrintf(sub, "%" PetscInt64_FMT ", ", dctx->childIDs[i]));
1093:         }
1094:       }
1095:     }
1096:     PetscCall(PetscViewerASCIIPopTab(sub));
1097:     PetscCall(PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sub));
1098:     PetscCall(PetscViewerASCIIPushTab(viewer));
1099:   }
1100:   if (const auto device = dctx->device) PetscCall(PetscDeviceView(device, viewer));
1101:   if (isascii) PetscCall(PetscViewerASCIIPopTab(viewer));
1102:   PetscFunctionReturn(PETSC_SUCCESS);
1103: }

1105: /*@
1106:   PetscDeviceContextViewFromOptions - View a `PetscDeviceContext` from options

1108:   Input Parameters:
1109: + dctx - the `PetscDeviceContext` to view
1110: . obj  - optional `PetscObject` to provide the options prefix, pass `NULL` to use the options prefix of `dctx`
1111: - name - the command line option

1113:   Options Database Key:
1114: . -name viewer_specification - See `PetscOptionsCreateViewer()` for the values of `viewer_specification`

1116:   Level: beginner

1118:   Note:
1119:   This checks the options database, creates the viewer on-the-fly, uses it and then destroys it. Hence it should not be called in heavily used routines,
1120:   rather `PetscOptionsCreateViewer()` should be used to construct the viewer once which can then be utilized in the heavily used routine.

1122: .seealso: `PetscDeviceContextView()`, `PetscObjectViewFromOptions()`, `PetscDeviceContextCreate()`, `PetscOptionsCreateViewer()`
1123: @*/
1124: PetscErrorCode PetscDeviceContextViewFromOptions(PetscDeviceContext dctx, PetscObject obj, const char name[])
1125: {
1126:   PetscFunctionBegin;
1127:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
1130:   PetscAssertPointer(name, 3);
1131:   PetscCall(PetscObjectViewFromOptions(PetscObjectCast(dctx), obj, name));
1132:   PetscFunctionReturn(PETSC_SUCCESS);
1133: }

1135: /*@
1136:   PetscDeviceContextGetStreamHandle - Return a handle to the underlying stream of a device context

1138:   Input Parameter:
1139: . dctx - The `PetscDeviceContext` to get the stream from

1141:   Output Parameter:
1142: . handle - A pointer to the handle to the stream

1144:   Level: advanced

1146:   Note:
1147:   This routine is dangerous. It exists only for the most experienced users and
1148:   internal PETSc development.

1150:   There is no way for PETSc's auto-dependency system to track what the caller does with the
1151:   stream.

1153:   If the user uses the stream to copy memory that was previously modified by PETSc, or launches
1154:   kernels that modify memory with the stream, it is the users responsibility to inform PETSc of
1155:   their actions via `PetscDeviceContextMarkIntentFromID()`. Failure to do so may introduce a
1156:   race condition. This race condition may manifest in nondeterministic ways.

1158:   Alternatively, the user may synchronize the stream immediately before and after use. This is
1159:   the safest option.

1161:   Example Usage:
1162: .vb
1163:   PetscDeviceContext dctx;
1164:   PetscDeviceType    type;
1165:   void               *handle;

1167:   PetscDeviceContextGetCurrentContext(&dctx);
1168:   PetscDeviceContextGetStreamHandle(dctx, &handle);
1169:   PetscDeviceContextGetDeviceType(dctx, &type);

1171:   if (type == PETSC_DEVICE_CUDA) {
1172:     cudaStream_t stream = *(cudaStream_t *)handle;

1174:     my_cuda_kernel<<<1, 2, 3, stream>>>();
1175:   }
1176: .ve
1177:   Alternatively, if type of `PetscDeviceContext` is known (for example `PETSC_DEVICE_HIP`), the
1178:   user may pass in a pointer to stream handle directly\:
1179: .vb
1180:   hipStream_t *stream;

1182:   // note the cast to void **
1183:   PetscDeviceContextGetStreamHandle(dctx, (void **)&stream);
1184:   // note the dereference
1185:   my_hip_kernel<<<1, 2, 3, *stream>>>();
1186: .ve

1188: .N ASYNC_API

1190: .seealso: `PetscDeviceContext`
1191: @*/
1192: PetscErrorCode PetscDeviceContextGetStreamHandle(PetscDeviceContext dctx, void **handle)
1193: {
1194:   PetscFunctionBegin;
1195:   PetscCall(PetscDeviceContextResolveDefault_Private(&dctx));
1197:   PetscAssertPointer(handle, 2);
1198:   PetscCall(PetscDeviceContextGetStreamHandle_Internal(dctx, handle));
1199:   PetscFunctionReturn(PETSC_SUCCESS);
1200: }