Actual source code: ex11.cxx

  1: static const char help[] = "Tests PetscDeviceContextMarkIntentFromID().\n\n";

  3: #include "petscdevicetestcommon.h"

  5: #include <petsc/private/cpp/type_traits.hpp>
  6: #include <petsc/private/cpp/array.hpp>

  8: #include <cstdarg>       // std::va_list
  9: #include <vector>        // std:vector
 10: #include <unordered_map> // std::take_a_wild_guess
 11: #include <algorithm>     // std::find
 12: #include <iterator>      // std::distance, std::next

 14: #if PETSC_CPP_VERSION > 14
 15: struct Marker {
 16:   PetscMemoryAccessMode mode{};

 18:   PetscErrorCode operator()(PetscDeviceContext dctx, PetscContainer cont) const noexcept
 19:   {
 20:     const auto    obj  = reinterpret_cast<PetscObject>(cont);
 21:     PetscObjectId id   = 0;
 22:     const char   *name = nullptr;

 24:     PetscFunctionBegin;
 25:     PetscCall(PetscObjectGetId(obj, &id));
 26:     PetscCall(PetscObjectGetName(obj, &name));
 27:     PetscCall(PetscDeviceContextMarkIntentFromID(dctx, id, this->mode, name));
 28:     PetscFunctionReturn(PETSC_SUCCESS);
 29:   }
 30: };

 32: static constexpr auto mem_read       = Marker{PETSC_MEMORY_ACCESS_READ};
 33: static constexpr auto mem_write      = Marker{PETSC_MEMORY_ACCESS_WRITE};
 34: static constexpr auto mem_read_write = Marker{PETSC_MEMORY_ACCESS_READ_WRITE};
 35: static constexpr auto mark_funcs     = Petsc::util::make_array(mem_read, mem_write, mem_read_write);

 37: static PetscErrorCode TestDeepStack(PetscDeviceContext dctx, PetscContainer cont, int remaining)
 38: {
 39:   PetscFunctionBeginUser;
 40:   if (remaining) PetscCall(TestDeepStack(dctx, cont, remaining - 1));
 41:   else {
 42:     PetscCall(mem_read(dctx, cont));
 43:     PetscCall(PetscDeviceContextSynchronize(dctx));
 44:   }
 45:   PetscFunctionReturn(PETSC_SUCCESS);
 46: }

 48: static PetscErrorCode MarkedObjectMapView(PetscViewer vwr, std::size_t nkeys, const PetscObjectId *keys, const PetscMemoryAccessMode *modes, const std::size_t *ndeps, const PetscEvent **dependencies)
 49: {
 50:   PetscFunctionBegin;
 51:   if (!vwr) PetscCall(PetscViewerASCIIGetStdout(PETSC_COMM_WORLD, &vwr));
 52:   PetscCall(PetscViewerFlush(vwr));
 53:   PetscCall(PetscViewerASCIIPushSynchronized(vwr));
 54:   PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "Marked Object Map:\n"));
 55:   PetscCall(PetscViewerASCIIPushTab(vwr));
 56:   PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "size: %zu\n", nkeys));
 57:   PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "entries:\n"));
 58:   PetscCall(PetscViewerASCIIPushTab(vwr));
 59:   for (std::size_t i = 0; i < nkeys; ++i) {
 60:     PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "id %" PetscInt64_FMT " -> {\n", keys[i]));
 61:     PetscCall(PetscViewerASCIIPushTab(vwr));
 62:     PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "mode: %s\n", PetscMemoryAccessModeToString(modes[i])));
 63:     PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "dependencies:\n"));
 64:     PetscCall(PetscViewerASCIIPushTab(vwr));
 65:     for (std::size_t j = 0; j < ndeps[i]; ++j) {
 66:       const auto event = dependencies[i][j];

 68:       PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "event %zu {dtype: %s, dctx_id: %" PetscInt64_FMT ", dctx_state: %" PetscInt64_FMT ", data: %p, destroy: %p}\n", j, PetscDeviceTypes[event->dtype], event->dctx_id, event->dctx_state, event->data,
 69:                                                    reinterpret_cast<void *>(event->destroy)));
 70:     }
 71:     PetscCall(PetscViewerASCIIPopTab(vwr));
 72:     PetscCall(PetscViewerASCIIPopTab(vwr));
 73:     PetscCall(PetscViewerASCIISynchronizedPrintf(vwr, "}\n"));
 74:   }
 75:   PetscCall(PetscViewerASCIIPopTab(vwr));
 76:   PetscCall(PetscViewerASCIIPopTab(vwr));
 77:   PetscCall(PetscViewerFlush(vwr));
 78:   PetscCall(PetscViewerASCIIPopSynchronized(vwr));
 79:   PetscFunctionReturn(PETSC_SUCCESS);
 80: }

 82: PETSC_ATTRIBUTE_FORMAT(10, 11) static PetscErrorCode CheckMarkedObjectMap_Private(PetscBool cond, const char cond_str[], MPI_Comm comm, PetscDeviceContext dctx, std::size_t nkeys, const PetscObjectId *keys, const PetscMemoryAccessMode *modes, const std::size_t *ndeps, const PetscEvent **dependencies, const char *format, ...)
 83: {
 84:   PetscFunctionBegin;
 85:   if (PetscUnlikely(!cond)) {
 86:     std::array<char, 2048> buf;
 87:     std::va_list           argp;
 88:     std::size_t            len;
 89:     PetscViewer            vwr;

 91:     PetscCallCXX(buf.fill(0));
 92:     va_start(argp, format);
 93:     PetscCall(PetscVSNPrintf(buf.data(), buf.size(), format, &len, argp));
 94:     va_end(argp);
 95:     PetscCall(PetscViewerASCIIGetStdout(comm, &vwr));
 96:     if (dctx) PetscCall(PetscDeviceContextView(dctx, vwr));
 97:     PetscCall(MarkedObjectMapView(vwr, nkeys, keys, modes, ndeps, dependencies));
 98:     SETERRQ(comm, PETSC_ERR_PLIB, "Condition '%s' failed, marked object map in corrupt state: %s", cond_str, buf.data());
 99:   }
100:   PetscFunctionReturn(PETSC_SUCCESS);
101: }

103:   #define CheckMarkedObjectMap(__cond__, ...) CheckMarkedObjectMap_Private((PetscBool)(!!(__cond__)), PetscStringize(__cond__), PETSC_COMM_SELF, dctx, nkeys, keys, modes, ndeps, const_cast<const PetscEvent **>(dependencies), __VA_ARGS__);

105: static PetscErrorCode TestAllCombinations(PetscDeviceContext dctx, const std::vector<PetscContainer> &cont)
106: {
107:   std::vector<PetscObjectId> cont_ids;
108:   PetscObjectId              dctx_id;
109:   PetscDeviceType            dtype;

111:   PetscFunctionBegin;
112:   PetscCallCXX(cont_ids.reserve(cont.size()));
113:   for (auto &&c : cont) {
114:     PetscObjectId id;

116:     PetscCall(PetscObjectGetId((PetscObject)c, &id));
117:     PetscCallCXX(cont_ids.emplace_back(id));
118:   }
119:   PetscCall(PetscObjectGetId(PetscObjectCast(dctx), &dctx_id));
120:   PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
121:   for (auto &&func_i : mark_funcs) {
122:     for (auto &&func_j : mark_funcs) {
123:       for (auto it = cont.cbegin(), next = std::next(it); it != cont.cend(); ++it, ++next) {
124:         std::vector<int>       found_keys;
125:         std::size_t            nkeys;
126:         PetscObjectId         *keys;
127:         PetscMemoryAccessMode *modes;
128:         std::size_t           *ndeps;
129:         PetscEvent           **dependencies;

131:         if (next >= cont.cend()) next = cont.cbegin();
132:         PetscCall(func_i(dctx, *it));
133:         PetscCall(func_j(dctx, *next));
134:         PetscCall(PetscGetMarkedObjectMap_Internal(&nkeys, &keys, &modes, &ndeps, &dependencies));
135:         PetscCallCXX(found_keys.resize(nkeys));
136:         {
137:           // The underlying marked object map is *unordered*, and hence the order in which we
138:           // get the keys is not necessarily the same as the order of operations. This is
139:           // confounded by the fact that k and knext are not necessarily "linear", i.e. k could
140:           // be 2 while knext is 0. So we need to map these back to linear space so we can loop
141:           // over them.
142:           const auto keys_end           = keys + nkeys;
143:           const auto num_expected_keys  = std::min(cont.size(), static_cast<std::size_t>(2));
144:           const auto check_applied_mode = [&](PetscContainer container, PetscMemoryAccessMode mode) {
145:             std::ptrdiff_t key_idx = 0;
146:             PetscObjectId  actual_key;

148:             PetscFunctionBegin;
149:             PetscCall(PetscObjectGetId((PetscObject)container, &actual_key));
150:             // search the list of keys from the map for the selected key
151:             key_idx = std::distance(keys, std::find(keys, keys_end, actual_key));
152:             PetscCheck(key_idx >= 0, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Key index %" PetscCount_FMT " < 0, this indicates keys_begin > keys_end?", key_idx);
153:             found_keys[key_idx]++;
154:             PetscCall(CheckMarkedObjectMap(key_idx < std::distance(keys, keys_end), "marked object map could not find expected key %" PetscInt64_FMT, actual_key));
155:             // OK found it, now check the rest of the entries are as we expect them to be
156:             PetscCall(CheckMarkedObjectMap(modes[key_idx] == mode, "unexpected mode %s, expected %s", PetscMemoryAccessModeToString(modes[key_idx]), PetscMemoryAccessModeToString(mode)));
157:             PetscCall(CheckMarkedObjectMap(ndeps[key_idx] == 1, "unexpected number of dependencies %zu, expected 1", ndeps[key_idx]));
158:             PetscCall(CheckMarkedObjectMap(dependencies[key_idx][0]->dtype == dtype, "unexpected device type on event: %s, expected %s", PetscDeviceTypes[dependencies[key_idx][0]->dtype], PetscDeviceTypes[dtype]));
159:             PetscFunctionReturn(PETSC_SUCCESS);
160:           };

162:           // if it == next, then even though we might num_expected_keys keys we never "look
163:           // for" the missing key
164:           PetscCheck(cont.size() == 1 || it != next, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Test assumes different inputs, otherwise key check may fail (cont.size(): %zu, it != next: %s)", cont.size(), it != next ? "true" : "false");
165:           PetscCall(CheckMarkedObjectMap(nkeys == num_expected_keys, "marked object map has %zu keys expected %zu", nkeys, num_expected_keys));
166:           // check that each function properly applied its mode, it == next if cont.size() = 1,
167:           // i.e. testing identity
168:           if (it != next) PetscCall(check_applied_mode(*it, func_i.mode));
169:           PetscCall(check_applied_mode(*next, func_j.mode));
170:         }
171:         // Check that the map contained only keys we were looking for. Any extra keys will have
172:         // zero find count
173:         for (auto it = found_keys.cbegin(); it != found_keys.cend(); ++it) PetscCall(CheckMarkedObjectMap(*it > 0, "Marked Object Map has extra object entry: id %" PetscInt64_FMT, keys[std::distance(found_keys.cbegin(), it)]));

175:         PetscCall(PetscRestoreMarkedObjectMap_Internal(nkeys, &keys, &modes, &ndeps, &dependencies));

177:         PetscCall(PetscDeviceContextSynchronize(dctx));
178:         PetscCall(PetscGetMarkedObjectMap_Internal(&nkeys, &keys, &modes, &ndeps, &dependencies));
179:         PetscCall(CheckMarkedObjectMap(nkeys == 0, "synchronizing device context did not empty dependency map, have %zu keys", nkeys));
180:         PetscCall(PetscRestoreMarkedObjectMap_Internal(nkeys, &keys, &modes, &ndeps, &dependencies));
181:       }
182:     }
183:   }
184:   PetscCall(PetscDeviceContextSynchronize(dctx));
185:   PetscFunctionReturn(PETSC_SUCCESS);
186: }

188: template <typename... T>
189: PETSC_NODISCARD static std::pair<PetscObjectId, std::pair<PetscMemoryAccessMode, std::vector<PetscDeviceContext>>> make_map_entry(PetscObjectId id, PetscMemoryAccessMode mode, T &&...dctxs)
190: {
191:   return {
192:     id, {mode, {std::forward<T>(dctxs)...}}
193:   };
194: }

196: static PetscErrorCode CheckMapEqual(std::unordered_map<PetscObjectId, std::pair<PetscMemoryAccessMode, std::vector<PetscDeviceContext>>> expected_map)
197: {
198:   std::size_t            nkeys;
199:   PetscObjectId         *keys;
200:   PetscMemoryAccessMode *modes;
201:   std::size_t           *ndeps;
202:   PetscEvent           **dependencies;
203:   PetscDeviceContext     dctx = nullptr;

205:   PetscFunctionBegin;
206:   PetscCall(PetscGetMarkedObjectMap_Internal(&nkeys, &keys, &modes, &ndeps, &dependencies));
207:   {
208:     const auto key_end = keys + nkeys;
209:     auto       mode_it = modes;
210:     auto       ndep_it = ndeps;
211:     auto       dep_it  = dependencies;

213:     for (auto key_it = keys; key_it != key_end; ++key_it, ++mode_it, ++ndep_it, ++dep_it) {
214:       const auto found_it = expected_map.find(*key_it);

216:       PetscCall(CheckMarkedObjectMap(found_it != expected_map.cend(), "marked object map did not contain key %" PetscInt64_FMT, *key_it));
217:       {
218:         // must do these here since found_it may be expected_map.cend()
219:         const auto &expected_mode  = found_it->second.first;
220:         const auto &expected_dctxs = found_it->second.second;
221:         auto        sub_dep_it     = *dep_it;

223:         PetscCall(CheckMarkedObjectMap(expected_mode == *mode_it, "unexpected mode %s, expected %s", PetscMemoryAccessModeToString(expected_mode), PetscMemoryAccessModeToString(*mode_it)));
224:         PetscCall(CheckMarkedObjectMap(expected_dctxs.size() == *ndep_it, "unexpected number of dependencies %zu, expected %zu", *ndep_it, expected_dctxs.size()));
225:         // purposefully hide "dctx" with the loop variable, so we get more detailed output in
226:         // the error message
227:         for (auto &&dctx : expected_dctxs) {
228:           const auto      event = *sub_dep_it;
229:           PetscDeviceType dtype;
230:           PetscObjectId   id;

232:           PetscCall(PetscDeviceContextGetDeviceType(dctx, &dtype));
233:           PetscCall(PetscObjectGetId(PetscObjectCast(dctx), &id));
234:           PetscCall(CheckMarkedObjectMap(event->dtype == dtype, "unexpected device type on event: %s, expected %s", PetscDeviceTypes[event->dtype], PetscDeviceTypes[dtype]));
235:           PetscCall(CheckMarkedObjectMap(event->dctx_id == id, "unexpected dctx id on event: %" PetscInt64_FMT ", expected %" PetscInt64_FMT, event->dctx_id, id));
236:           ++sub_dep_it;
237:         }
238:       }
239:       // remove the found iterator from the map, this ensure we either run out of map (which is
240:       // caught by the first check in the loop), or we run out of keys to check, which is
241:       // caught in the end of the loop
242:       PetscCallCXX(expected_map.erase(found_it));
243:     }
244:   }
245:   PetscCall(CheckMarkedObjectMap(expected_map.empty(), "Not all keys in marked object map accounted for!"));
246:   PetscCall(PetscRestoreMarkedObjectMap_Internal(nkeys, &keys, &modes, &ndeps, &dependencies));
247:   PetscFunctionReturn(PETSC_SUCCESS);
248: }

250: int main(int argc, char *argv[])
251: {
252:   PetscContainer     x, y, z;
253:   PetscObjectId      x_id, y_id, z_id;
254:   PetscDeviceContext dctx_a, dctx_b, dctx_c;
255:   auto               container_view   = PETSC_FALSE;
256:   const auto         create_container = [&](PetscContainer *c, const char name[], PetscObjectId *id) {
257:     PetscFunctionBegin;
258:     PetscCall(PetscContainerCreate(PETSC_COMM_WORLD, c));
259:     PetscCall(PetscObjectSetName((PetscObject)*c, name));
260:     PetscCall(PetscObjectGetId((PetscObject)*c, id));
261:     if (container_view) PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Container '%s' -> id %" PetscInt64_FMT "\n", name, *id));
262:     PetscFunctionReturn(PETSC_SUCCESS);
263:   };
264:   const auto sync_all = [&] {
265:     PetscFunctionBegin;
266:     for (auto &&ctx : {dctx_a, dctx_b, dctx_c}) PetscCall(PetscDeviceContextSynchronize(ctx));
267:     PetscFunctionReturn(PETSC_SUCCESS);
268:   };

270:   PetscFunctionBeginUser;
271:   PetscCall(PetscInitialize(&argc, &argv, nullptr, help));

273:   PetscOptionsBegin(PETSC_COMM_WORLD, nullptr, "Test Options", "Sys");
274:   PetscCall(PetscOptionsBool("-container_view", "View container names and ID's", nullptr, container_view, &container_view, nullptr));
275:   PetscOptionsEnd();

277:   PetscCall(create_container(&x, "x", &x_id));
278:   PetscCall(create_container(&y, "y", &y_id));
279:   PetscCall(create_container(&z, "z", &z_id));

281:   PetscCall(PetscDeviceContextCreate(&dctx_a));
282:   PetscCall(PetscObjectSetName(PetscObjectCast(dctx_a), "dctx_a"));
283:   PetscCall(PetscDeviceContextSetStreamType(dctx_a, PETSC_STREAM_DEFAULT));
284:   PetscCall(PetscDeviceContextSetFromOptions(PETSC_COMM_WORLD, dctx_a));
285:   PetscCall(PetscDeviceContextDuplicate(dctx_a, &dctx_b));
286:   PetscCall(PetscObjectSetName(PetscObjectCast(dctx_b), "dctx_b"));
287:   PetscCall(PetscDeviceContextDuplicate(dctx_a, &dctx_c));
288:   PetscCall(PetscObjectSetName(PetscObjectCast(dctx_c), "dctx_c"));
289:   PetscCall(PetscDeviceContextViewFromOptions(dctx_a, nullptr, "-dctx_a_view"));
290:   PetscCall(PetscDeviceContextViewFromOptions(dctx_b, nullptr, "-dctx_b_view"));
291:   PetscCall(PetscDeviceContextViewFromOptions(dctx_c, nullptr, "-dctx_c_view"));

293:   // ensure they are all idle
294:   PetscCall(sync_all());
295:   PetscCall(CheckMapEqual({}));

297:   PetscCall(TestDeepStack(dctx_a, x, PETSCSTACKSIZE + 2));
298:   PetscCall(CheckMapEqual({}));

300:   // do the bulk combination tests, these test only the very basic combinations for simple
301:   // correctness
302:   PetscCall(TestAllCombinations(dctx_a, {x}));
303:   PetscCall(TestAllCombinations(dctx_a, {x, y, z}));

305:   // Now do some specific tests, these should test more complicated scenarios. First and
306:   // foremost, ensure they are all idle, and that it does not change the map
307:   PetscCall(sync_all());
308:   // Map should be empty
309:   PetscCall(CheckMapEqual({}));

311:   // Syncing again shouldn't magically fill the map back up
312:   PetscCall(sync_all());
313:   PetscCall(CheckMapEqual({}));

315:   const auto test_multiple_readers = [&](std::array<PetscDeviceContext, 2> readers, std::size_t sync_idx) {
316:     // the reader which synchronizes
317:     const auto sync_reader = readers[sync_idx];
318:     // the reader that will remain in the map after sync_reader synchronizes
319:     const auto remain_idx    = sync_idx + 1 >= readers.size() ? 0 : sync_idx + 1;
320:     const auto remain_reader = readers[remain_idx];

322:     PetscFunctionBegin;
323:     for (auto &&ctx : readers) PetscCall(mem_read(ctx, x));
324:     for (auto &&ctx : readers) PetscCall(mem_read(ctx, y));
325:     PetscCall(CheckMapEqual({
326:       make_map_entry(x_id, PETSC_MEMORY_ACCESS_READ, readers[0], readers[1]),
327:       make_map_entry(y_id, PETSC_MEMORY_ACCESS_READ, readers[0], readers[1]),
328:     }));
329:     // synchronizing sync_reader should remove it from the dependency list -- but leave remain_reader
330:     // intact
331:     PetscCall(PetscDeviceContextSynchronize(sync_reader));
332:     PetscCall(CheckMapEqual({
333:       make_map_entry(x_id, PETSC_MEMORY_ACCESS_READ, remain_reader),
334:       make_map_entry(y_id, PETSC_MEMORY_ACCESS_READ, remain_reader),
335:     }));
336:     PetscCall(PetscDeviceContextSynchronize(remain_reader));
337:     PetscCall(CheckMapEqual({}));
338:     PetscFunctionReturn(PETSC_SUCCESS);
339:   };

341:   // Test that multiple readers can simultaneously read -- even if one of them is synchronized
342:   PetscCall(test_multiple_readers({dctx_a, dctx_b}, 0));
343:   PetscCall(test_multiple_readers({dctx_a, dctx_b}, 1));

345:   // Test that sync of unrelated ctx does not affect the map
346:   PetscCall(mem_read(dctx_a, x));
347:   PetscCall(mem_read(dctx_b, y));
348:   PetscCall(PetscDeviceContextSynchronize(dctx_c));
349:   PetscCall(CheckMapEqual({make_map_entry(x_id, PETSC_MEMORY_ACCESS_READ, dctx_a), make_map_entry(y_id, PETSC_MEMORY_ACCESS_READ, dctx_b)}));
350:   PetscCall(PetscDeviceContextSynchronize(dctx_a));
351:   PetscCall(PetscDeviceContextSynchronize(dctx_b));
352:   // Now the map is empty again
353:   PetscCall(CheckMapEqual({}));

355:   // Test another context writing over two reads
356:   PetscCall(mem_read(dctx_a, x));
357:   PetscCall(mem_read(dctx_b, x));
358:   // C writing should kick out both A and B
359:   PetscCall(mem_write(dctx_c, x));
360:   PetscCall(CheckMapEqual({make_map_entry(x_id, PETSC_MEMORY_ACCESS_WRITE, dctx_c)}));
361:   PetscCall(PetscDeviceContextSynchronize(dctx_c));
362:   PetscCall(CheckMapEqual({}));

364:   // Test that write and synchronize does not interfere with unrelated read
365:   PetscCall(mem_read_write(dctx_a, x));
366:   PetscCall(mem_read(dctx_a, y));
367:   PetscCall(mem_read_write(dctx_b, x));
368:   PetscCall(mem_read(dctx_b, y));
369:   // Synchronizing B here must clear everything *but* A's read on Y!
370:   PetscCall(PetscDeviceContextSynchronize(dctx_b));
371:   PetscCall(CheckMapEqual({make_map_entry(y_id, PETSC_MEMORY_ACCESS_READ, dctx_a)}));
372:   PetscCall(PetscDeviceContextSynchronize(dctx_a));
373:   // Now the map is empty again
374:   PetscCall(CheckMapEqual({}));

376:   // Test that implicit stream-dependencies are properly tracked
377:   PetscCall(mem_read(dctx_a, x));
378:   PetscCall(mem_read(dctx_b, y));
379:   // A waits for B
380:   PetscCall(PetscDeviceContextWaitForContext(dctx_a, dctx_b));
381:   // Because A waits on B, synchronizing A implicitly implies B read must have finished so the
382:   // map must be empty
383:   PetscCall(PetscDeviceContextSynchronize(dctx_a));
384:   PetscCall(CheckMapEqual({}));

386:   PetscCall(mem_write(dctx_a, x));
387:   PetscCall(CheckMapEqual({make_map_entry(x_id, PETSC_MEMORY_ACCESS_WRITE, dctx_a)}));
388:   PetscCall(PetscDeviceContextWaitForContext(dctx_b, dctx_a));
389:   PetscCall(PetscDeviceContextWaitForContext(dctx_c, dctx_b));
390:   // We have created the chain C -> B -> A, so synchronizing C should trickle down to synchronize and
391:   // remove A from the map
392:   PetscCall(PetscDeviceContextSynchronize(dctx_c));
393:   PetscCall(CheckMapEqual({}));

395:   // Test that superfluous stream-dependencies are properly ignored
396:   PetscCall(mem_read(dctx_a, x));
397:   PetscCall(mem_read(dctx_b, y));
398:   PetscCall(PetscDeviceContextWaitForContext(dctx_c, dctx_b));
399:   // C waited on B, so synchronizing C should remove B from the map but *not* remove A
400:   PetscCall(PetscDeviceContextSynchronize(dctx_c));
401:   PetscCall(CheckMapEqual({make_map_entry(x_id, PETSC_MEMORY_ACCESS_READ, dctx_a)}));
402:   PetscCall(PetscDeviceContextSynchronize(dctx_a));
403:   PetscCall(CheckMapEqual({}));

405:   // Test that read->write correctly wipes out the map
406:   PetscCall(mem_read(dctx_a, x));
407:   PetscCall(mem_read(dctx_b, x));
408:   PetscCall(mem_read(dctx_c, x));
409:   PetscCall(CheckMapEqual({make_map_entry(x_id, PETSC_MEMORY_ACCESS_READ, dctx_a, dctx_b, dctx_c)}));
410:   PetscCall(mem_write(dctx_a, x));
411:   PetscCall(CheckMapEqual({make_map_entry(x_id, PETSC_MEMORY_ACCESS_WRITE, dctx_a)}));
412:   PetscCall(PetscDeviceContextSynchronize(dctx_a));
413:   PetscCall(CheckMapEqual({}));

415:   PetscCall(PetscDeviceContextDestroy(&dctx_a));
416:   PetscCall(PetscDeviceContextDestroy(&dctx_b));
417:   PetscCall(PetscDeviceContextDestroy(&dctx_c));

419:   PetscCall(PetscContainerDestroy(&x));
420:   PetscCall(PetscContainerDestroy(&y));
421:   PetscCall(PetscContainerDestroy(&z));
422:   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "EXIT_SUCCESS\n"));
423:   PetscCall(PetscFinalize());
424:   return 0;
425: }
426: #else // PETSC_CPP_VERSION > 11
427: int main(int argc, char *argv[])
428: {
429:   PetscFunctionBeginUser;
430:   PetscCall(PetscInitialize(&argc, &argv, nullptr, help));
431:   PetscCall(PetscPrintf(PETSC_COMM_WORLD, "EXIT_SUCCESS\n"));
432:   PetscCall(PetscFinalize());
433:   return 0;
434: }
435: #endif

437: /*TEST

439:   testset:
440:     requires: defined(PETSC_DEVICELANGUAGE_CXX)
441:     output_file: output/ExitSuccess.out
442:     test:
443:       requires: !device
444:       suffix: host_no_device
445:     test:
446:       requires: device
447:       args: -default_device_type host
448:       suffix: host_with_device
449:     test:
450:       requires: cuda
451:       args: -default_device_type cuda
452:       suffix: cuda
453:     test:
454:       requires: hip
455:       args: -default_device_type hip
456:       suffix: hip
457:     test:
458:       requires: sycl
459:       TODO: unclear if it is needed
460:       args: -default_device_type sycl
461:       suffix: sycl

463: TEST*/