Actual source code: device.cxx

  1: #include "petscdevice_interface_internal.hpp" /*I <petscdevice.h> I*/

  3: #include <petsc/private/cpp/register_finalize.hpp>

  5: #include "../impls/host/hostdevice.hpp"
  6: #if PetscDefined(HAVE_CUPM)
  7:   #include "../impls/cupm/cupmdevice.hpp"
  8: #endif
  9: #if PetscDefined(HAVE_SYCL)
 10:   #include "../impls/sycl/sycldevice.hpp"
 11: #endif

 13: #include <utility> // std::make_pair

 15: using namespace Petsc::device;

 17: #if PetscDefined(HAVE_CUPM)
 18: int PetscDeviceCUPMRuntimeArch = 0;
 19: #endif

 21: namespace
 22: {

 24: /*
 25:   note to anyone adding more classes, the name must be ALL_CAPS_SHORT_NAME + Device exactly to
 26:   be picked up by the switch-case macros below
 27: */
 28: host::Device HOSTDevice{PetscDeviceContextCreate_HOST};
 29: #if PetscDefined(HAVE_CUDA)
 30: cupm::Device<cupm::DeviceType::CUDA> CUDADevice{PetscDeviceContextCreate_CUDA};
 31: #endif
 32: #if PetscDefined(HAVE_HIP)
 33: cupm::Device<cupm::DeviceType::HIP> HIPDevice{PetscDeviceContextCreate_HIP};
 34: #endif
 35: #if PetscDefined(HAVE_SYCL)
 36: sypm::Device SYCLDevice{PetscDeviceContextCreate_SYCL};
 37: #endif

 39: } // namespace

 41: #define PETSC_DEVICE_CASE(IMPLS, func, ...) \
 42:   case PetscConcat_(PETSC_DEVICE_, IMPLS): { \
 43:     PetscCall(PetscConcat_(IMPLS, Device).func(__VA_ARGS__)); \
 44:   } break

 46: #define PETSC_VOID_0(...) ((void)0)

 48: /*
 49:   Suppose you have:

 51:   CUDADevice.myFunction(arg1,arg2)

 53:   that you would like to conditionally define and call in a switch-case:

 55:   switch(PetscDeviceType) {
 56:   #if PetscDefined(HAVE_CUDA)
 57:   case PETSC_DEVICE_CUDA: {
 58:     PetscCall(CUDADevice.myFunction(arg1,arg2));
 59:   } break;
 60:   #endif
 61:   }

 63:   then calling this macro:

 65:   PETSC_DEVICE_CASE_IF_PETSC_DEFINED(CUDA,myFunction,arg1,arg2)

 67:   will expand to the following case statement:

 69:   case PETSC_DEVICE_CUDA: {
 70:     PetscCall(CUDADevice.myFunction(arg1,arg2));
 71:   } break

 73:   if PetscDefined(HAVE_CUDA) evaluates to 1, and expand to nothing otherwise
 74: */
 75: #define PETSC_DEVICE_CASE_IF_PETSC_DEFINED(IMPLS, func, ...) PetscIfPetscDefined(HAVE_##IMPLS, PETSC_DEVICE_CASE, PETSC_VOID_0)(IMPLS, func, __VA_ARGS__)

 77: /*@
 78:   PetscDeviceCreate - Get a new handle for a particular device (often a GPU) type

 80:   Not Collective

 82:   Input Parameters:
 83: + type  - The type of `PetscDevice`
 84: - devid - The device selection strategy or the numeric ID# of the device

 86:   Output Parameter:
 87: . device - The `PetscDevice`

 89:   Options Database Key:
 90: . -device_select N - The device selection strategy or the numeric ID# of the device

 92:   Level: beginner

 94:   Notes:
 95:   This routine may initialize `PetscDevice`. If this is the case, it may cause some sort of
 96:   device synchronization.

 98:   Available device selection strategies\:

100:   | Value | Behavior                                                      |
101:   |-------|---------------------------------------------------------------|
102:   | -1    |  Use the configured default device (`PETSC_DECIDE`)           |
103:   | -2    |  Topology-aware allocation (CUDA/HIP with hwloc 2.0 or later) |
104:   | >=0   |  Select a specific GPU device by index                        |

106:   SYCL gives -2 its own meaning, the host device, so in a CUPM and SYCL build
107:   `-device_select -2` is topology-aware for CUDA or HIP but the host device for SYCL.

109:   When `PETSC_DECIDE` (-1) is passed to `-device_select` and to this function, either
110:   a round-robin allocation across all available GPUs is used, or if `torchrun` was used to
111:   launch the application, the value of the `LOCAL_RANK` environment variable set by
112:   `torchrun` is selected. When the "Topology-aware allocation" strategy is selected, PETSc
113:   uses `hwloc` to inspect the hardware layout of the current system and determine which GPU
114:   is closest to the CPU core(s) the current process has been bound to. Each process must
115:   actually be bound (e.g. `mpiexec -bind-to core`, `srun --cpu-bind=cores`); an unbound process
116:   has no locality to use and falls back to `PETSC_DECIDE`. For example, the LUMI, Frontier and
117:   Setonix HPC systems all have nodes comprised of 64-core AMD EPYC "Trento" CPUs connected
118:   to 4 AMD MI250X GPUs, each of which is comprised of two Graphics Compute Die (GCD) that
119:   appears to the operating system as a distinct GPU device. The CPU is divided into four NUMA
120:   nodes, each of which is divided into two chiplets with a direct connection to each GCD. Each
121:   GCD is assigned an index from 0 to 7 by the GPU runtime. The diagram below shows how `hwloc`
122:   sees one of these nodes.

124:   ![Node diagram](/images/manual/gpu_topo.svg)

126:   In order to construct a mapping between the GCD index and the devices found by `hwloc`, we
127:   inspect the PCIe Bus ID for each GCD. The results of this are shown in the table below

129:   | PCIe Bus ID | GCD Index |
130:   |-------------|----------:|
131:   | d1:00.0     |         4 |
132:   | d6:00.0     |         5 |
133:   | c9:00.0     |         2 |
134:   | ce:00.0     |         3 |
135:   | d9:00.0     |         6 |
136:   | de:00.0     |         7 |
137:   | c1:00.0     |         0 |
138:   | c6:00.0     |         1 |

140:   By cross-referencing the figure and the table, we can see that the 'closest' GPU to e.g. core 0 has the PCIe
141:   Bus ID `d1:00.0`, which corresponds to GCD 4.

143:   `hwloc` uses the notion of "depth" of devices to describe the different layers of the topology it
144:   discovers. The top layer is always Machine and is assigned a value 0. Beneath that is the Package
145:   layer, which is assigned depth 1 and generally corresponds to CPU sockets. In order to determine which GCD
146:   is topologically closest, PETSc uses `hwloc` to determine the depth value of the nearest common ancestor
147:   for the CPU the current process is bound to and each device on the system. The higher the depth value,
148:   the closer the hardware controller for the device is to CPU core 0, and therefore the closer the GCD
149:   is to CPU core 0. The table below shows the nearest common ancestor between CPU core 0 and each of the
150:   GCDs identified on the system

152:   | PCIe Bus ID | Nearest Common Ancestor to Core 0 | Depth |
153:   |-------------|----------------------------------:|------:|
154:   | d1:00.0     | L3 (chiplet)                      |     3 |
155:   | d6:00.0     | Group (NUMA Node)                 |     2 |
156:   | c9:00.0     | Package (CPU socket)              |     1 |
157:   | ce:00.0     | Package (CPU socket)              |     1 |
158:   | d9:00.0     | Package (CPU socket)              |     1 |
159:   | de:00.0     | Package (CPU socket)              |     1 |
160:   | c1:00.0     | Package (CPU socket)              |     1 |
161:   | c6:00.0     | Package (CPU socket)              |     1 |

163:   Choosing the device with the highest depth value gives the intuitive result that CPU
164:   core 0 is closest to the GCD on PCIe Bus ID d1:00.0, or GCD 4. In the case where multiple devices
165:   are found at the same depth, the logical ordering of the CPU cores under the device at that depth
166:   is used to determine which device to select. For instance, a system with 2 GPUs at the same depth
167:   and 16 cores will assign device 0 to processes on cores 0-7 and device 1 to processes on cores 8-15.
168:   A summary of the topologically aware device selection method is as follows.

170:   - If there is a single GPU on the system, select it and return
171:   - Enumerate all devices on the system and map PCIe Bus ID to device ID
172:   - Discover system topology with hwloc
173:   - Determine current process's CPU binding
174:   - Find GPU devices by PCIe Bus ID in discovered topology
175:   - Find the depth of nearest common ancestor between current CPU core and every GPU device
176:   - If there is a single device at a maximum depth, select that device and return
177:   - If there are multiple devices at the same depth, select between those devices based on logical CPU core ordering

179: .seealso: `PetscDevice`, `PetscDeviceInitType`,
180: `PetscDeviceInitialize()`, `PetscDeviceInitialized()`, `PetscDeviceConfigure()`,
181: `PetscDeviceView()`, `PetscDeviceDestroy()`
182: @*/
183: PetscErrorCode PetscDeviceCreate(PetscDeviceType type, PetscInt devid, PetscDevice *device)
184: {
185:   static PetscInt PetscDeviceCounter = 0;

187:   PetscFunctionBegin;
189:   PetscAssertPointer(device, 3);
190:   PetscCall(PetscDeviceInitializePackage());

192:   PetscCall(PetscNew(device));
193:   (*device)->id     = PetscDeviceCounter++;
194:   (*device)->type   = type;
195:   (*device)->refcnt = 1;
196:   /*
197:     if you are adding a device, you also need to add its initialization in
198:     PetscDeviceInitializeTypeFromOptions_Private() below
199:   */
200:   switch (type) {
201:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HOST, getDevice, *device, devid);
202:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(CUDA, getDevice, *device, devid);
203:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HIP, getDevice, *device, devid);
204:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(SYCL, getDevice, *device, devid);
205:   default:
206:     /* in case the above macros expand to nothing this silences any unused variable warnings */
207:     (void)devid;
208:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "PETSc was seemingly configured for PetscDeviceType %s but we've fallen through all cases in a switch", PetscDeviceTypes[type]);
209:   }
210:   PetscFunctionReturn(PETSC_SUCCESS);
211: }

213: /*@
214:   PetscDeviceDestroy - Free a `PetscDevice`

216:   Not Collective

218:   Input Parameter:
219: . device - The `PetscDevice`

221:   Level: beginner

223: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceConfigure()`, `PetscDeviceView()`,
224: `PetscDeviceGetType()`, `PetscDeviceGetDeviceId()`
225: @*/
226: PetscErrorCode PetscDeviceDestroy(PetscDevice *device)
227: {
228:   PetscFunctionBegin;
229:   PetscAssertPointer(device, 1);
230:   if (!*device) PetscFunctionReturn(PETSC_SUCCESS);
232:   PetscCall(PetscDeviceDereference_Internal(*device));
233:   if ((*device)->refcnt) {
234:     *device = nullptr;
235:     PetscFunctionReturn(PETSC_SUCCESS);
236:   }
237:   PetscCall(PetscFree((*device)->data));
238:   PetscCall(PetscFree(*device));
239:   PetscFunctionReturn(PETSC_SUCCESS);
240: }

242: /*@
243:   PetscDeviceConfigure - Configure a particular `PetscDevice`

245:   Not Collective

247:   Input Parameter:
248: . device - The `PetscDevice` to configure

250:   Level: beginner

252:   Notes:
253:   The user should not assume that this is a cheap operation.

255: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceView()`, `PetscDeviceDestroy()`,
256: `PetscDeviceGetType()`, `PetscDeviceGetDeviceId()`
257: @*/
258: PetscErrorCode PetscDeviceConfigure(PetscDevice device)
259: {
260:   PetscFunctionBegin;
262:   /*
263:     if no available configuration is available, this cascades all the way down to default
264:     and error
265:   */
266:   switch (const auto dtype = device->type) {
267:   case PETSC_DEVICE_HOST:
268:     if (PetscDefined(HAVE_HOST)) break; // always true
269:   case PETSC_DEVICE_CUDA:
270:     if (PetscDefined(HAVE_CUDA)) break;
271:     goto error;
272:   case PETSC_DEVICE_HIP:
273:     if (PetscDefined(HAVE_HIP)) break;
274:     goto error;
275:   case PETSC_DEVICE_SYCL:
276:     if (PetscDefined(HAVE_SYCL)) break;
277:     goto error;
278:   default:
279:   error:
280:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "PETSc was not configured for PetscDeviceType %s", PetscDeviceTypes[dtype]);
281:   }
282:   PetscUseTypeMethod(device, configure);
283:   PetscFunctionReturn(PETSC_SUCCESS);
284: }

286: /*@
287:   PetscDeviceView - View a `PetscDevice`

289:   Collective on viewer

291:   Input Parameters:
292: + device - The `PetscDevice` to view
293: - viewer - The `PetscViewer` to view the device with (`NULL` for `PETSC_VIEWER_STDOUT_WORLD`)

295:   Level: beginner

297: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceConfigure()`,
298: `PetscDeviceDestroy()`, `PetscDeviceGetType()`, `PetscDeviceGetDeviceId()`
299: @*/
300: PetscErrorCode PetscDeviceView(PetscDevice device, PetscViewer viewer)
301: {
302:   auto      sub = viewer;
303:   PetscBool isascii;

305:   PetscFunctionBegin;
307:   if (viewer) {
309:     PetscCall(PetscObjectTypeCompare(PetscObjectCast(viewer), PETSCVIEWERASCII, &isascii));
310:   } else {
311:     PetscCall(PetscViewerASCIIGetStdout(PETSC_COMM_WORLD, &viewer));
312:     isascii = PETSC_TRUE;
313:   }

315:   if (isascii) {
316:     auto        dtype = PETSC_DEVICE_HOST;
317:     MPI_Comm    comm;
318:     PetscMPIInt size;
319:     PetscInt    id = 0;

321:     PetscCall(PetscObjectGetComm(PetscObjectCast(viewer), &comm));
322:     PetscCallMPI(MPI_Comm_size(comm, &size));

324:     PetscCall(PetscDeviceGetDeviceId(device, &id));
325:     PetscCall(PetscDeviceGetType(device, &dtype));
326:     PetscCall(PetscViewerGetSubViewer(viewer, PETSC_COMM_SELF, &sub));
327:     PetscCall(PetscViewerASCIIPrintf(sub, "PetscDevice Object: %d MPI %s\n", size, size == 1 ? "process" : "processes"));
328:     PetscCall(PetscViewerASCIIPushTab(sub));
329:     PetscCall(PetscViewerASCIIPrintf(sub, "type: %s\n", PetscDeviceTypes[dtype]));
330:     PetscCall(PetscViewerASCIIPrintf(sub, "id: %" PetscInt_FMT "\n", id));
331:   }

333:   // see if impls has extra viewer stuff
334:   PetscTryTypeMethod(device, view, sub);

336:   if (isascii) {
337:     // undo the ASCII specific stuff
338:     PetscCall(PetscViewerASCIIPopTab(sub));
339:     PetscCall(PetscViewerRestoreSubViewer(viewer, PETSC_COMM_SELF, &sub));
340:   }
341:   PetscFunctionReturn(PETSC_SUCCESS);
342: }

344: /*@
345:   PetscDeviceGetType - Get the type of device

347:   Not Collective

349:   Input Parameter:
350: . device - The `PetscDevice`

352:   Output Parameter:
353: . type - The `PetscDeviceType`

355:   Level: intermediate

357: .seealso: `PetscDevice`, `PetscDeviceType`, `PetscDeviceSetDefaultDeviceType()`,
358: `PetscDeviceCreate()`, `PetscDeviceConfigure()`, `PetscDeviceDestroy()`,
359: `PetscDeviceGetDeviceId()`, `PETSC_DEVICE_DEFAULT()`
360: @*/
361: PetscErrorCode PetscDeviceGetType(PetscDevice device, PetscDeviceType *type)
362: {
363:   PetscFunctionBegin;
365:   PetscAssertPointer(type, 2);
366:   *type = device->type;
367:   PetscFunctionReturn(PETSC_SUCCESS);
368: }

370: /*@
371:   PetscDeviceGetDeviceId - Get the device ID for a `PetscDevice`

373:   Not Collective

375:   Input Parameter:
376: . device - The `PetscDevice`

378:   Output Parameter:
379: . id - The id

381:   Level: intermediate

383:   Notes:
384:   The returned ID may have been assigned by the underlying device backend. For example if the
385:   backend is CUDA then `id` is exactly the value returned by `cudaGetDevice()` at the time when
386:   this device was configured.

388: .seealso: `PetscDevice`, `PetscDeviceCreate()`, `PetscDeviceGetType()`
389: @*/
390: PetscErrorCode PetscDeviceGetDeviceId(PetscDevice device, PetscInt *id)
391: {
392:   PetscFunctionBegin;
394:   PetscAssertPointer(id, 2);
395:   *id = device->deviceId;
396:   PetscFunctionReturn(PETSC_SUCCESS);
397: }

399: namespace
400: {

402: struct DefaultDeviceType : public Petsc::RegisterFinalizeable<DefaultDeviceType> {
403:   PetscDeviceType type = PETSC_DEVICE_HARDWARE_DEFAULT_TYPE;

405:   PetscErrorCode finalize_() noexcept
406:   {
407:     PetscFunctionBegin;
408:     type = PETSC_DEVICE_HARDWARE_DEFAULT_TYPE;
409:     PetscFunctionReturn(PETSC_SUCCESS);
410:   }
411: };

413: auto default_device_type = DefaultDeviceType();

415: } // namespace

417: /*@
418:   PETSC_DEVICE_DEFAULT - Retrieve the current default `PetscDeviceType`

420:   Not Collective

422:   Level: intermediate

424:   Notes:
425:   Unless selected by the user, the default device is selected in the following order\:
426:   `PETSC_DEVICE_HIP`, `PETSC_DEVICE_CUDA`, `PETSC_DEVICE_SYCL`, `PETSC_DEVICE_HOST`.

428: .seealso: `PetscDeviceType`, `PetscDeviceSetDefaultDeviceType()`, `PetscDeviceGetType()`
429: @*/
430: PetscDeviceType PETSC_DEVICE_DEFAULT(void)
431: {
432:   return default_device_type.type;
433: }

435: /*@
436:   PetscDeviceSetDefaultDeviceType - Set the default device type for `PetscDevice`

438:   Not Collective

440:   Input Parameter:
441: . type - the new default device type

443:   Level: intermediate

445:   Notes:
446:   This sets the `PetscDeviceType` returned by `PETSC_DEVICE_DEFAULT()`.

448: .seealso: `PetscDeviceType`, `PetscDeviceGetType`
449: @*/
450: PetscErrorCode PetscDeviceSetDefaultDeviceType(PetscDeviceType type)
451: {
452:   PetscFunctionBegin;
454:   if (default_device_type.type != type) {
455:     // no need to waste a PetscRegisterFinalize() slot if we don't change it
456:     default_device_type.type = type;
457:     PetscCall(default_device_type.register_finalize());
458:   }
459:   PetscFunctionReturn(PETSC_SUCCESS);
460: }

462: namespace
463: {

465: std::array<std::pair<PetscDevice, bool>, PETSC_DEVICE_MAX> defaultDevices = {};

467: /*
468:   Actual initialization function; any functions claiming to initialize PetscDevice or
469:   PetscDeviceContext will have to run through this one
470: */
471: PetscErrorCode PetscDeviceInitializeDefaultDevice_Internal(PetscDeviceType type, PetscInt defaultDeviceId)
472: {
473:   PetscFunctionBegin;
475:   if (PetscUnlikely(!PetscDeviceInitialized(type))) {
476:     auto &dev  = defaultDevices[type].first;
477:     auto &init = defaultDevices[type].second;

479:     PetscAssert(!dev, PETSC_COMM_SELF, PETSC_ERR_MEM, "Trying to overwrite existing default device of type %s", PetscDeviceTypes[type]);
480:     PetscCall(PetscDeviceCreate(type, defaultDeviceId, &dev));
481:     PetscCall(PetscDeviceConfigure(dev));
482:     init = true;
483:   }
484:   PetscFunctionReturn(PETSC_SUCCESS);
485: }

487: } // namespace

489: /*@
490:   PetscDeviceInitialize - Initialize `PetscDevice`

492:   Not Collective

494:   Input Parameter:
495: . type - The `PetscDeviceType` to initialize

497:   Level: intermediate

499:   Notes:
500:   Eagerly initializes the corresponding `PetscDeviceType` if needed. If this is the case it may
501:   result in device synchronization.

503: .seealso: `PetscDevice`, `PetscDeviceInitType`, `PetscDeviceInitialized()`,
504: `PetscDeviceCreate()`, `PetscDeviceDestroy()`
505: @*/
506: PetscErrorCode PetscDeviceInitialize(PetscDeviceType type)
507: {
508:   PetscFunctionBegin;
510:   PetscCall(PetscDeviceInitializeDefaultDevice_Internal(type, PETSC_DECIDE));
511:   PetscFunctionReturn(PETSC_SUCCESS);
512: }

514: /*@
515:   PetscDeviceInitialized - Determines whether `PetscDevice` is initialized for a particular
516:   `PetscDeviceType`

518:   Not Collective

520:   Input Parameter:
521: . type - The `PetscDeviceType` to check

523:   Level: intermediate

525:   Notes:
526:   Returns `PETSC_TRUE` if `type` is initialized, `PETSC_FALSE` otherwise.

528:   If one has not configured PETSc for a particular `PetscDeviceType` then this routine will
529:   return `PETSC_FALSE` for that `PetscDeviceType`.

531: .seealso: `PetscDevice`, `PetscDeviceInitType`, `PetscDeviceInitialize()`,
532: `PetscDeviceCreate()`, `PetscDeviceDestroy()`
533: @*/
534: PetscBool PetscDeviceInitialized(PetscDeviceType type)
535: {
536:   return static_cast<PetscBool>(PetscDeviceConfiguredFor_Internal(type) && defaultDevices[type].second);
537: }

539: /* Get the default PetscDevice for a particular type and constructs them if lazily initialized. */
540: PetscErrorCode PetscDeviceGetDefaultForType_Internal(PetscDeviceType type, PetscDevice *device)
541: {
542:   PetscFunctionBegin;
543:   PetscAssertPointer(device, 2);
544:   PetscCall(PetscDeviceInitialize(type));
545:   *device = defaultDevices[type].first;
546:   PetscFunctionReturn(PETSC_SUCCESS);
547: }

549: /*@
550:   PetscDeviceGetAttribute - Query a particular attribute of a `PetscDevice`

552:   Not Collective

554:   Input Parameters:
555: + device - The `PetscDevice`
556: - attr   - The attribute

558:   Output Parameter:
559: . value - The value of the attribute

561:   Level: developer

563:   Notes:
564:   Since different attributes are often different types `value` is a `void *` to accommodate
565:   them all. The underlying type of the attribute is therefore included in the name of the
566:   `PetscDeviceAttribute` responsible for querying it. For example,
567:   `PETSC_DEVICE_ATTR_SIZE_T_SHARED_MEM_PER_BLOCK` is of type `size_t`.

569: .seealso: `PetscDeviceAtrtibute`, `PetscDeviceConfigure()`, `PetscDevice`
570: @*/
571: PetscErrorCode PetscDeviceGetAttribute(PetscDevice device, PetscDeviceAttribute attr, void *value)
572: {
573:   PetscFunctionBegin;
576:   PetscAssertPointer(value, 3);
577:   PetscUseTypeMethod(device, getattribute, attr, value);
578:   PetscFunctionReturn(PETSC_SUCCESS);
579: }

581: namespace
582: {

584: PetscErrorCode PetscDeviceInitializeTypeFromOptions_Private(MPI_Comm comm, PetscDeviceType type, PetscInt defaultDeviceId, PetscBool defaultView, PetscDeviceInitType *defaultInitType)
585: {
586:   PetscFunctionBegin;
587:   if (!PetscDeviceConfiguredFor_Internal(type)) {
588:     PetscCall(PetscInfo(nullptr, "PetscDeviceType %s not available\n", PetscDeviceTypes[type]));
589:     defaultDevices[type].first = nullptr;
590:     PetscFunctionReturn(PETSC_SUCCESS);
591:   }
592:   PetscCall(PetscInfo(nullptr, "PetscDeviceType %s available, initializing\n", PetscDeviceTypes[type]));
593:   /* ugly switch needed to pick the right global variable... could maybe do this as a union? */
594:   switch (type) {
595:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HOST, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
596:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(CUDA, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
597:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(HIP, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
598:     PETSC_DEVICE_CASE_IF_PETSC_DEFINED(SYCL, initialize, comm, &defaultDeviceId, &defaultView, defaultInitType);
599:   default:
600:     SETERRQ(comm, PETSC_ERR_PLIB, "PETSc was seemingly configured for PetscDeviceType %s but we've fallen through all cases in a switch", PetscDeviceTypes[type]);
601:   }
602:   PetscCall(PetscInfo(nullptr, "PetscDevice %s initialized, default device id %" PetscInt_FMT ", view %s, init type %s\n", PetscDeviceTypes[type], defaultDeviceId, PetscBools[defaultView], PetscDeviceInitTypes[Petsc::util::to_underlying(*defaultInitType)]));
603:   /*
604:     defaultInitType, defaultView  and defaultDeviceId now represent what the individual TYPES
605:     have decided to initialize as
606:   */
607:   if ((*defaultInitType == PETSC_DEVICE_INIT_EAGER) || defaultView) {
608:     PetscCall(PetscInfo(nullptr, "Eagerly initializing %s PetscDevice\n", PetscDeviceTypes[type]));
609:     PetscCall(PetscDeviceInitializeDefaultDevice_Internal(type, defaultDeviceId));
610:     if (defaultView) PetscCall(PetscDeviceView(defaultDevices[type].first, nullptr));
611:   }
612:   PetscFunctionReturn(PETSC_SUCCESS);
613: }

615: PetscErrorCode PetscDeviceInitializeQueryOptions_Private(MPI_Comm comm, PetscDeviceType *deviceContextInitDevice, PetscDeviceInitType *defaultInitType, PetscInt *defaultDeviceId, PetscBool *defaultDeviceIdSet, PetscBool *defaultView)
616: {
617:   PetscInt initIdx       = PETSC_DEVICE_INIT_LAZY;
618:   auto     initDeviceIdx = static_cast<PetscInt>(*deviceContextInitDevice);
619:   auto     flg           = PETSC_FALSE;

621:   PetscFunctionBegin;
622:   PetscCall(PetscOptionsHasName(nullptr, nullptr, "-log_view_gpu_time", &flg));
623:   if (flg) PetscCall(PetscLogGpuTime());
624:   PetscCall(PetscOptionsHasName(nullptr, nullptr, "-log_view_gpu_energy_meter", &flg));
625:   if (flg) PetscCall(PetscLogGpuEnergyMeter());
626:   PetscCall(PetscOptionsHasName(nullptr, nullptr, "-log_view_gpu_energy", &flg));
627:   if (flg) PetscCall(PetscLogGpuEnergy());

629:   PetscOptionsBegin(comm, nullptr, "PetscDevice Options", "Sys");
630:   PetscCall(PetscOptionsEList("-device_enable", "How (or whether) to initialize PetscDevices", "PetscDeviceInitialize", PetscDeviceInitTypes, 3, PetscDeviceInitTypes[initIdx], &initIdx, nullptr));
631:   PetscCall(PetscOptionsEList("-default_device_type", "Set the PetscDeviceType returned by PETSC_DEVICE_DEFAULT()", "PetscDeviceSetDefaultDeviceType", PetscDeviceTypes, PETSC_DEVICE_MAX, PetscDeviceTypes[initDeviceIdx], &initDeviceIdx, defaultDeviceIdSet));
632:   PetscCall(PetscOptionsRangeInt("-device_select", PETSC_DEVICE_SELECT_HELP, "PetscDeviceCreate", *defaultDeviceId, defaultDeviceId, nullptr, PETSC_DEVICE_SELECT_LOWER_BOUND, PETSC_DEVICE_MAX_DEVICES));
633:   PetscCall(PetscOptionsBool("-device_view", "Display device information and assignments (forces eager initialization)", "PetscDeviceView", *defaultView, defaultView, &flg));
634:   PetscOptionsEnd();

636:   if (initIdx == PETSC_DEVICE_INIT_NONE) {
637:     /* disabled all device initialization if devices are globally disabled */
638:     PetscCheck(*defaultDeviceId == PETSC_DECIDE, comm, PETSC_ERR_USER_INPUT, "You have disabled devices but also specified a particular device to use, these options are mutually exclusive");
639:     *defaultView  = PETSC_FALSE;
640:     initDeviceIdx = PETSC_DEVICE_HOST;
641:   } else {
642:     *defaultView = static_cast<PetscBool>(*defaultView && flg);
643:     if (*defaultView) initIdx = PETSC_DEVICE_INIT_EAGER;
644:   }
645:   *defaultInitType         = PetscDeviceInitTypeCast(initIdx);
646:   *deviceContextInitDevice = PetscDeviceTypeCast(initDeviceIdx);
647:   PetscFunctionReturn(PETSC_SUCCESS);
648: }

650: /* called from PetscFinalize() do not call yourself! */
651: PetscErrorCode PetscDeviceFinalize_Private()
652: {
653:   PetscFunctionBegin;
654:   if (PetscDefined(USE_DEBUG)) {
655:     /*
656:       you might be thinking, why on earth are you registered yet another finalizer in a
657:       function already called during PetscRegisterFinalizeAll()? If this seems stupid it's
658:       because it is.

660:       The crux of the problem is that the initializer (and therefore the ~finalizer~) of
661:       PetscDeviceContext is guaranteed to run after PetscDevice's. So if the global context had
662:       a default PetscDevice attached, that PetscDevice will have a reference count >0 and hence
663:       won't be destroyed yet. So we need to repeat the check that all devices have been
664:       destroyed again ~after~ the global context is destroyed. In summary:

666:       1. This finalizer runs and destroys all devices, except it may not because the global
667:          context may still hold a reference!
668:       2. The global context finalizer runs and does the final reference count decrement
669:          required, which actually destroys the held device.
670:       3. Our newly added finalizer runs and checks that all is well.
671:     */
672:     PetscCall(PetscRegisterFinalize([] {
673:       PetscFunctionBegin;
674:       for (auto &&device : defaultDevices) {
675:         const auto dev = device.first;

677:         PetscCheck(!dev, PETSC_COMM_WORLD, PETSC_ERR_COR, "Device of type '%s' had reference count %" PetscInt_FMT " and was not fully destroyed during PetscFinalize()", PetscDeviceTypes[dev->type], dev->refcnt);
678:       }
679:       PetscFunctionReturn(PETSC_SUCCESS);
680:     }));
681:   }
682:   for (auto &&device : defaultDevices) {
683:     PetscCall(PetscDeviceDestroy(&device.first));
684:     device.second = false;
685:   }
686:   PetscFunctionReturn(PETSC_SUCCESS);
687: }

689: } // namespace

691: /*
692:   Begins the init proceeedings for the entire PetscDevice stack. there are 3 stages of
693:   initialization types:

695:   1. defaultInitType - how does PetscDevice as a whole expect to initialize?
696:   2. subTypeDefaultInitType - how does each PetscDevice implementation expect to initialize?
697:      e.g. you may want to blanket disable PetscDevice init (and disable say Kokkos init), but
698:      have all CUDA devices still initialize.

700:   All told the following happens:

702:   0. defaultInitType -> LAZY
703:   1. Check for log_view/log_summary, if yes defaultInitType -> EAGER
704:   2. PetscDevice initializes each sub type with deviceDefaultInitType.
705:   2.1 Each enabled PetscDevice sub-type then does the above disable or view check in addition
706:       to checking for specific device init. if view or specific device init
707:       subTypeDefaultInitType -> EAGER. disabled once again overrides all.
708: */

710: PetscErrorCode PetscDeviceInitializeFromOptions_Internal(MPI_Comm comm)
711: {
712:   auto defaultView                    = PETSC_FALSE;
713:   auto initializeDeviceContextEagerly = PETSC_FALSE;
714:   auto defaultDeviceIdSet             = PETSC_FALSE;
715:   auto defaultDeviceId                = PetscInt{PETSC_DECIDE};
716:   auto deviceContextInitDevice        = PETSC_DEVICE_DEFAULT();
717:   auto defaultInitType                = PETSC_DEVICE_INIT_LAZY;

719:   PetscFunctionBegin;
720:   if (PetscDefined(USE_DEBUG)) {
721:     int result;

723:     PetscCallMPI(MPI_Comm_compare(comm, PETSC_COMM_WORLD, &result));
724:     /* in order to accurately assign ranks to gpus we need to get the MPI_Comm_rank of the
725:      * global space */
726:     if (PetscUnlikely(result != MPI_IDENT)) {
727:       char name[MPI_MAX_OBJECT_NAME] = {};
728:       int  len; /* unused */

730:       PetscCallMPI(MPI_Comm_get_name(comm, name, &len));
731:       SETERRQ(comm, PETSC_ERR_MPI, "Default devices being initialized on MPI_Comm '%s' not PETSC_COMM_WORLD", name);
732:     }
733:   }
734:   comm = PETSC_COMM_WORLD; /* from this point on we assume we're on PETSC_COMM_WORLD */
735:   PetscCall(PetscRegisterFinalize(PetscDeviceFinalize_Private));

737:   PetscCall(PetscDeviceInitializeQueryOptions_Private(comm, &deviceContextInitDevice, &defaultInitType, &defaultDeviceId, &defaultDeviceIdSet, &defaultView));

739:   // the precise values don't matter here, so long as they are sequential
740:   static_assert(Petsc::util::to_underlying(PETSC_DEVICE_HOST) == 0, "");
741:   static_assert(Petsc::util::to_underlying(PETSC_DEVICE_CUDA) == 1, "");
742:   static_assert(Petsc::util::to_underlying(PETSC_DEVICE_HIP) == 2, "");
743:   static_assert(Petsc::util::to_underlying(PETSC_DEVICE_SYCL) == 3, "");
744:   static_assert(Petsc::util::to_underlying(PETSC_DEVICE_MAX) == 4, "");
745:   for (int i = PETSC_DEVICE_HOST; i < PETSC_DEVICE_MAX; ++i) {
746:     const auto deviceType = PetscDeviceTypeCast(i);
747:     auto       initType   = defaultInitType;

749:     PetscCall(PetscDeviceInitializeTypeFromOptions_Private(comm, deviceType, defaultDeviceId, defaultView, &initType));
750:     if (PetscDeviceConfiguredFor_Internal(deviceType)) {
751:       if (initType == PETSC_DEVICE_INIT_EAGER) {
752:         initializeDeviceContextEagerly = PETSC_TRUE;
753:         // only update the default device if the user hasn't set it previously
754:         if (!defaultDeviceIdSet) {
755:           deviceContextInitDevice = deviceType;
756:           PetscCall(PetscInfo(nullptr, "PetscDevice %s set as default device type due to eager initialization\n", PetscDeviceTypes[deviceType]));
757:         }
758:       } else if (initType == PETSC_DEVICE_INIT_NONE) {
759:         if (deviceType != PETSC_DEVICE_HOST) PetscCheck(!defaultDeviceIdSet || (deviceType != deviceContextInitDevice), comm, PETSC_ERR_USER_INPUT, "Cannot explicitly disable the device set as default device type (%s)", PetscDeviceTypes[deviceType]);
760:       }
761:     }
762:   }

764:   PetscCall(PetscDeviceSetDefaultDeviceType(deviceContextInitDevice));
765:   PetscCall(PetscDeviceContextSetRootDeviceType_Internal(PETSC_DEVICE_DEFAULT()));
766:   /*                       PetscDevice is now fully initialized                          */
767:   {
768:     /*
769:       query the options db to get the root settings from the user (if any).

771:       This section is a bit of a hack. We have to reach across to dcontext.cxx to all but call
772:       PetscDeviceContextSetFromOptions() before we even have one, then set a few static
773:       variables in that file with the results.
774:     */
775:     auto dtype = std::make_pair(PETSC_DEVICE_DEFAULT(), PETSC_FALSE);
776:     auto stype = std::make_pair(PETSC_DEVICE_CONTEXT_DEFAULT_STREAM_TYPE, PETSC_FALSE);

778:     PetscOptionsBegin(comm, "root_", "Root PetscDeviceContext Options", "Sys");
779:     PetscCall(PetscDeviceContextQueryOptions_Internal(PetscOptionsObject, dtype, stype));
780:     PetscOptionsEnd();

782:     if (dtype.second) PetscCall(PetscDeviceContextSetRootDeviceType_Internal(dtype.first));
783:     if (stype.second) PetscCall(PetscDeviceContextSetRootStreamType_Internal(stype.first));
784:   }

786:   if (initializeDeviceContextEagerly) {
787:     PetscDeviceContext dctx;

789:     PetscCall(PetscInfo(nullptr, "Eagerly initializing PetscDeviceContext with %s device\n", PetscDeviceTypes[deviceContextInitDevice]));
790:     /* instantiates the device context */
791:     PetscCall(PetscDeviceContextGetCurrentContext(&dctx));
792:     PetscCall(PetscDeviceContextSetUp(dctx));
793:   }
794:   PetscFunctionReturn(PETSC_SUCCESS);
795: }