Actual source code: sfwindow.c
1: #include <petsc/private/sfimpl.h>
3: typedef struct _n_PetscSFDataLink *PetscSFDataLink;
4: typedef struct _n_PetscSFWinLink *PetscSFWinLink;
6: typedef struct {
7: PetscSFWindowSyncType sync; /* FENCE, LOCK, or ACTIVE synchronization */
8: PetscSFDataLink link; /* List of MPI data types, lazily constructed for each data type */
9: PetscSFWinLink wins; /* List of active windows */
10: PetscSFWindowFlavorType flavor; /* Current PETSCSF_WINDOW_FLAVOR_ */
11: PetscSF dynsf;
12: MPI_Info info;
13: MPI_Comm window_comm;
14: PetscBool is_empty;
15: PetscMPIInt *wcommranks;
16: } PetscSF_Window;
18: struct _n_PetscSFDataLink {
19: MPI_Datatype unit;
20: MPI_Datatype *mine;
21: MPI_Datatype *remote;
22: PetscSFDataLink next;
23: };
25: struct _n_PetscSFWinLink {
26: PetscBool inuse;
27: MPI_Aint bytes;
28: void *addr;
29: void *rootdata;
30: void *leafdata;
31: MPI_Win win;
32: MPI_Request *reqs;
33: PetscSFWindowFlavorType flavor;
34: MPI_Aint *dyn_target_addr;
35: PetscBool epoch;
36: PetscBool persistent;
37: PetscSFWinLink next;
38: };
40: const char *const PetscSFWindowSyncTypes[] = {"FENCE", "LOCK", "ACTIVE", "PetscSFWindowSyncType", "PETSCSF_WINDOW_SYNC_", NULL};
41: const char *const PetscSFWindowFlavorTypes[] = {"CREATE", "DYNAMIC", "ALLOCATE", "SHARED", "PetscSFWindowFlavorType", "PETSCSF_WINDOW_FLAVOR_", NULL};
43: /* Built-in MPI_Ops act elementwise inside MPI_Accumulate, but cannot be used with composite types inside collectives (MPI_Allreduce) */
44: static PetscErrorCode PetscSFWindowOpTranslate(MPI_Op *op)
45: {
46: PetscFunctionBegin;
47: if (*op == MPIU_SUM) *op = MPI_SUM;
48: else if (*op == MPIU_MAX) *op = MPI_MAX;
49: else if (*op == MPIU_MIN) *op = MPI_MIN;
50: PetscFunctionReturn(PETSC_SUCCESS);
51: }
53: /*
54: PetscSFWindowGetDataTypes - gets composite local and remote data types for each rank
56: Not Collective
58: Input Parameters:
59: + sf - star forest of type `PETSCSFWINDOW`
60: - unit - data type for each node
62: Output Parameters:
63: + localtypes - types describing part of local leaf buffer referencing each remote rank
64: - remotetypes - types describing part of remote root buffer referenced for each remote rank
66: Level: developer
68: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetGraph()`, `PetscSFView()`
69: @*/
70: static PetscErrorCode PetscSFWindowGetDataTypes(PetscSF sf, MPI_Datatype unit, const MPI_Datatype **localtypes, const MPI_Datatype **remotetypes)
71: {
72: PetscSF_Window *w = (PetscSF_Window *)sf->data;
73: PetscSFDataLink link;
74: PetscMPIInt nranks;
75: const PetscInt *roffset;
77: PetscFunctionBegin;
78: /* Look for types in cache */
79: for (link = w->link; link; link = link->next) {
80: PetscBool match;
82: PetscCall(MPIPetsc_Type_compare(unit, link->unit, &match));
83: if (match) {
84: *localtypes = link->mine;
85: *remotetypes = link->remote;
86: PetscFunctionReturn(PETSC_SUCCESS);
87: }
88: }
90: /* Create new composite types for each send rank */
91: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, &roffset, NULL, NULL));
92: PetscCall(PetscNew(&link));
93: PetscCallMPI(MPI_Type_dup(unit, &link->unit));
94: PetscCall(PetscMalloc2(nranks, &link->mine, nranks, &link->remote));
95: for (PetscMPIInt i = 0; i < nranks; i++) {
96: PetscMPIInt rcount;
97: PetscMPIInt *rmine, *rremote;
99: PetscCall(PetscMPIIntCast(roffset[i + 1] - roffset[i], &rcount));
100: #if !PetscDefined(USE_64BIT_INDICES)
101: rmine = sf->rmine + sf->roffset[i];
102: rremote = sf->rremote + sf->roffset[i];
103: #else
104: PetscCall(PetscMalloc2(rcount, &rmine, rcount, &rremote));
105: for (PetscInt j = 0; j < rcount; j++) {
106: PetscCall(PetscMPIIntCast(sf->rmine[sf->roffset[i] + j], &rmine[j]));
107: PetscCall(PetscMPIIntCast(sf->rremote[sf->roffset[i] + j], &rremote[j]));
108: }
109: #endif
111: PetscCallMPI(MPI_Type_create_indexed_block(rcount, 1, rmine, link->unit, &link->mine[i]));
112: PetscCallMPI(MPI_Type_create_indexed_block(rcount, 1, rremote, link->unit, &link->remote[i]));
113: #if PetscDefined(USE_64BIT_INDICES)
114: PetscCall(PetscFree2(rmine, rremote));
115: #endif
116: PetscCallMPI(MPI_Type_commit(&link->mine[i]));
117: PetscCallMPI(MPI_Type_commit(&link->remote[i]));
118: }
119: link->next = w->link;
120: w->link = link;
122: *localtypes = link->mine;
123: *remotetypes = link->remote;
124: PetscFunctionReturn(PETSC_SUCCESS);
125: }
127: /*@
128: PetscSFWindowSetFlavorType - Set flavor type for `MPI_Win` creation
130: Logically Collective
132: Input Parameters:
133: + sf - star forest for communication of type `PETSCSFWINDOW`
134: - flavor - flavor type
136: Options Database Key:
137: . -sf_window_flavor flavor - sets the flavor type CREATE, DYNAMIC, ALLOCATE or SHARED (see `PetscSFWindowFlavorType`)
139: Level: advanced
141: Notes:
142: Windows reuse follows these rules\:
143: .vb
144: PETSCSF_WINDOW_FLAVOR_CREATE: creates a new window every time, uses MPI_Win_create
146: PETSCSF_WINDOW_FLAVOR_DYNAMIC: uses MPI_Win_create_dynamic/MPI_Win_attach and tries to reuse windows by comparing the root array. Intended to be used on repeated applications of the same SF, e.g.
147: PetscSFRegisterPersistent(sf,rootdata1,leafdata);
148: for i=1 to K
149: PetscSFOperationBegin(sf,rootdata1,leafdata);
150: PetscSFOperationEnd(sf,rootdata1,leafdata);
151: ...
152: PetscSFOperationBegin(sf,rootdata1,leafdata);
153: PetscSFOperationEnd(sf,rootdata1,leafdata);
154: endfor
155: PetscSFDeregisterPersistent(sf,rootdata1,leafdata);
157: The following pattern will instead raise an error
158: PetscSFOperationBegin(sf,rootdata1,leafdata);
159: PetscSFOperationEnd(sf,rootdata1,leafdata);
160: PetscSFOperationBegin(sf,rank ? rootdata1 : rootdata2,leafdata);
161: PetscSFOperationEnd(sf,rank ? rootdata1 : rootdata2,leafdata);
163: PETSCSF_WINDOW_FLAVOR_ALLOCATE: uses MPI_Win_allocate, reuses any pre-existing window which fits the data and it is not in use
165: PETSCSF_WINDOW_FLAVOR_SHARED: uses MPI_Win_allocate_shared, reusage policy as for PETSCSF_WINDOW_FLAVOR_ALLOCATE
166: .ve
168: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetFromOptions()`, `PetscSFWindowGetFlavorType()`
169: @*/
170: PetscErrorCode PetscSFWindowSetFlavorType(PetscSF sf, PetscSFWindowFlavorType flavor)
171: {
172: PetscFunctionBegin;
175: PetscTryMethod(sf, "PetscSFWindowSetFlavorType_C", (PetscSF, PetscSFWindowFlavorType), (sf, flavor));
176: PetscFunctionReturn(PETSC_SUCCESS);
177: }
179: static PetscErrorCode PetscSFWindowSetFlavorType_Window(PetscSF sf, PetscSFWindowFlavorType flavor)
180: {
181: PetscSF_Window *w = (PetscSF_Window *)sf->data;
183: PetscFunctionBegin;
184: w->flavor = flavor;
185: PetscFunctionReturn(PETSC_SUCCESS);
186: }
188: /*@
189: PetscSFWindowGetFlavorType - Get `PETSCSFWINDOW` flavor type for `PetscSF` communication
191: Logically Collective
193: Input Parameter:
194: . sf - star forest for communication of type `PETSCSFWINDOW`
196: Output Parameter:
197: . flavor - flavor type
199: Level: advanced
201: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetFromOptions()`, `PetscSFWindowSetFlavorType()`
202: @*/
203: PetscErrorCode PetscSFWindowGetFlavorType(PetscSF sf, PetscSFWindowFlavorType *flavor)
204: {
205: PetscFunctionBegin;
207: PetscAssertPointer(flavor, 2);
208: PetscUseMethod(sf, "PetscSFWindowGetFlavorType_C", (PetscSF, PetscSFWindowFlavorType *), (sf, flavor));
209: PetscFunctionReturn(PETSC_SUCCESS);
210: }
212: static PetscErrorCode PetscSFWindowGetFlavorType_Window(PetscSF sf, PetscSFWindowFlavorType *flavor)
213: {
214: PetscSF_Window *w = (PetscSF_Window *)sf->data;
216: PetscFunctionBegin;
217: *flavor = w->flavor;
218: PetscFunctionReturn(PETSC_SUCCESS);
219: }
221: /*@
222: PetscSFWindowSetSyncType - Set synchronization type for `PetscSF` communication of type `PETSCSFWINDOW`
224: Logically Collective
226: Input Parameters:
227: + sf - star forest for communication
228: - sync - synchronization type
230: Options Database Key:
231: . -sf_window_sync sync - sets the synchronization type FENCE, LOCK, or ACTIVE (see `PetscSFWindowSyncType`)
233: Level: advanced
235: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetFromOptions()`, `PetscSFWindowGetSyncType()`, `PetscSFWindowSyncType`
236: @*/
237: PetscErrorCode PetscSFWindowSetSyncType(PetscSF sf, PetscSFWindowSyncType sync)
238: {
239: PetscFunctionBegin;
242: PetscTryMethod(sf, "PetscSFWindowSetSyncType_C", (PetscSF, PetscSFWindowSyncType), (sf, sync));
243: PetscFunctionReturn(PETSC_SUCCESS);
244: }
246: static PetscErrorCode PetscSFWindowSetSyncType_Window(PetscSF sf, PetscSFWindowSyncType sync)
247: {
248: PetscSF_Window *w = (PetscSF_Window *)sf->data;
250: PetscFunctionBegin;
251: w->sync = sync;
252: PetscFunctionReturn(PETSC_SUCCESS);
253: }
255: /*@
256: PetscSFWindowGetSyncType - Get synchronization type for `PetscSF` communication of type `PETSCSFWINDOW`
258: Logically Collective
260: Input Parameter:
261: . sf - star forest for communication
263: Output Parameter:
264: . sync - synchronization type
266: Level: advanced
268: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetFromOptions()`, `PetscSFWindowSetSyncType()`, `PetscSFWindowSyncType`
269: @*/
270: PetscErrorCode PetscSFWindowGetSyncType(PetscSF sf, PetscSFWindowSyncType *sync)
271: {
272: PetscFunctionBegin;
274: PetscAssertPointer(sync, 2);
275: PetscUseMethod(sf, "PetscSFWindowGetSyncType_C", (PetscSF, PetscSFWindowSyncType *), (sf, sync));
276: PetscFunctionReturn(PETSC_SUCCESS);
277: }
279: static PetscErrorCode PetscSFWindowGetSyncType_Window(PetscSF sf, PetscSFWindowSyncType *sync)
280: {
281: PetscSF_Window *w = (PetscSF_Window *)sf->data;
283: PetscFunctionBegin;
284: *sync = w->sync;
285: PetscFunctionReturn(PETSC_SUCCESS);
286: }
288: /*@C
289: PetscSFWindowSetInfo - Set the `MPI_Info` handle that will be used for subsequent windows allocation
291: Logically Collective
293: Input Parameters:
294: + sf - star forest for communication
295: - info - `MPI_Info` handle
297: Level: advanced
299: Note:
300: The info handle is duplicated with a call to `MPI_Info_dup()` unless info = `MPI_INFO_NULL`.
302: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetFromOptions()`, `PetscSFWindowGetInfo()`
303: @*/
304: PetscErrorCode PetscSFWindowSetInfo(PetscSF sf, MPI_Info info)
305: {
306: PetscFunctionBegin;
308: PetscTryMethod(sf, "PetscSFWindowSetInfo_C", (PetscSF, MPI_Info), (sf, info));
309: PetscFunctionReturn(PETSC_SUCCESS);
310: }
312: static PetscErrorCode PetscSFWindowSetInfo_Window(PetscSF sf, MPI_Info info)
313: {
314: PetscSF_Window *w = (PetscSF_Window *)sf->data;
316: PetscFunctionBegin;
317: if (w->info != MPI_INFO_NULL) PetscCallMPI(MPI_Info_free(&w->info));
318: if (info != MPI_INFO_NULL) PetscCallMPI(MPI_Info_dup(info, &w->info));
319: PetscFunctionReturn(PETSC_SUCCESS);
320: }
322: /*@C
323: PetscSFWindowGetInfo - Get the `MPI_Info` handle used for windows allocation
325: Logically Collective
327: Input Parameter:
328: . sf - star forest for communication
330: Output Parameter:
331: . info - `MPI_Info` handle
333: Level: advanced
335: Note:
336: If `PetscSFWindowSetInfo()` has not be called, this returns `MPI_INFO_NULL`
338: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFSetFromOptions()`, `PetscSFWindowSetInfo()`
339: @*/
340: PetscErrorCode PetscSFWindowGetInfo(PetscSF sf, MPI_Info *info)
341: {
342: PetscFunctionBegin;
344: PetscAssertPointer(info, 2);
345: PetscUseMethod(sf, "PetscSFWindowGetInfo_C", (PetscSF, MPI_Info *), (sf, info));
346: PetscFunctionReturn(PETSC_SUCCESS);
347: }
349: static PetscErrorCode PetscSFWindowGetInfo_Window(PetscSF sf, MPI_Info *info)
350: {
351: PetscSF_Window *w = (PetscSF_Window *)sf->data;
353: PetscFunctionBegin;
354: *info = w->info;
355: PetscFunctionReturn(PETSC_SUCCESS);
356: }
358: static PetscErrorCode PetscSFWindowCreateDynamicSF(PetscSF sf, PetscSF *dynsf)
359: {
360: PetscSFNode *remotes;
362: PetscFunctionBegin;
363: PetscCall(PetscMalloc1(sf->nranks, &remotes));
364: for (PetscInt i = 0; i < sf->nranks; i++) {
365: remotes[i].rank = sf->ranks[i];
366: remotes[i].index = 0;
367: }
368: PetscCall(PetscSFDuplicate(sf, PETSCSF_DUPLICATE_RANKS, dynsf));
369: PetscCall(PetscSFSetType(*dynsf, PETSCSFBASIC)); /* break recursion */
370: PetscCall(PetscSFSetGraph(*dynsf, 1, sf->nranks, NULL, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
371: PetscFunctionReturn(PETSC_SUCCESS);
372: }
374: static PetscErrorCode PetscSFWindowAttach(PetscSF sf, PetscSFWinLink link, void *rootdata, size_t wsize)
375: {
376: PetscFunctionBegin;
377: #if PetscDefined(HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
378: {
379: PetscSF_Window *w = (PetscSF_Window *)sf->data;
380: MPI_Comm wcomm;
381: MPI_Aint winaddr;
382: void *addr = rootdata;
383: PetscMPIInt nranks;
384: // some Open MPI versions do not support MPI_Win_attach(win,NULL,0);
385: wcomm = w->window_comm;
386: if (addr != NULL) PetscCallMPI(MPI_Win_attach(link->win, addr, wsize));
387: link->addr = addr;
388: PetscCheck(w->dynsf, wcomm, PETSC_ERR_ORDER, "Must call PetscSFSetUp()");
389: PetscCall(PetscSFGetRootRanks(w->dynsf, &nranks, NULL, NULL, NULL, NULL));
390: PetscCallMPI(MPI_Get_address(addr, &winaddr));
391: if (!link->dyn_target_addr) PetscCall(PetscMalloc1(nranks, &link->dyn_target_addr));
392: PetscCall(PetscSFBcastBegin(w->dynsf, MPI_AINT, &winaddr, link->dyn_target_addr, MPI_REPLACE));
393: PetscCall(PetscSFBcastEnd(w->dynsf, MPI_AINT, &winaddr, link->dyn_target_addr, MPI_REPLACE));
394: }
395: #else
396: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_PLIB, "dynamic windows not supported");
397: #endif
398: PetscFunctionReturn(PETSC_SUCCESS);
399: }
401: /*
402: PetscSFGetWindow - Get a window for use with a given data type
404: Collective
406: Input Parameters:
407: + sf - star forest
408: . unit - data type
409: . rootdata - array to be sent
410: . leafdata - only used to help uniquely identify windows
411: . sync - type of synchronization `PetscSFWindowSyncType`
412: . epoch - `PETSC_TRUE` to acquire the window and start an epoch, `PETSC_FALSE` to just acquire the window
413: . fenceassert - assert parameter for call to `MPI_Win_fence()`, if sync == `PETSCSF_WINDOW_SYNC_FENCE`
414: . postassert - assert parameter for call to `MPI_Win_post()`, if sync == `PETSCSF_WINDOW_SYNC_ACTIVE`
415: - startassert - assert parameter for call to `MPI_Win_start()`, if sync == `PETSCSF_WINDOW_SYNC_ACTIVE`
417: Output Parameters:
418: + target_disp - target_disp argument for RMA calls (significative for `PETSCSF_WINDOW_FLAVOR_DYNAMIC` only)
419: + reqs - array of requests (significative for sync == `PETSCSF_WINDOW_SYNC_LOCK` only)
420: - win - window
422: Level: developer
424: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFGetRootRanks()`, `PetscSFWindowGetDataTypes()`
425: */
427: static PetscErrorCode PetscSFGetWindow(PetscSF sf, MPI_Datatype unit, void *rootdata, void *leafdata, PetscSFWindowSyncType sync, PetscBool epoch, PetscMPIInt fenceassert, PetscMPIInt postassert, PetscMPIInt startassert, const MPI_Aint **target_disp, MPI_Request **reqs, MPI_Win *win)
428: {
429: PetscSF_Window *w = (PetscSF_Window *)sf->data;
430: MPI_Aint bytes;
431: PetscSFWinLink link;
432: PetscBool reuse = PETSC_FALSE, update = PETSC_FALSE;
433: MPI_Aint wsize;
434: MPI_Comm wcomm;
435: PetscBool is_empty;
437: PetscFunctionBegin;
438: PetscCall(PetscSFGetDatatypeSize_Internal(PetscObjectComm((PetscObject)sf), unit, &bytes));
439: wsize = (MPI_Aint)(bytes * sf->nroots);
440: wcomm = w->window_comm;
441: is_empty = w->is_empty;
442: if (is_empty) {
443: if (target_disp) *target_disp = NULL;
444: if (reqs) *reqs = NULL;
445: *win = MPI_WIN_NULL;
446: PetscFunctionReturn(PETSC_SUCCESS);
447: }
448: if (w->flavor != PETSCSF_WINDOW_FLAVOR_CREATE) reuse = PETSC_TRUE;
449: if (PetscDefined(HAVE_MPI_FEATURE_DYNAMIC_WINDOW) && w->flavor == PETSCSF_WINDOW_FLAVOR_DYNAMIC) {
450: // first search for a persistent window
451: for (link = w->wins; reuse && link; link = link->next) {
452: PetscBool match;
454: if (!link->persistent) continue;
455: match = (link->flavor == w->flavor && link->rootdata == rootdata && link->leafdata == leafdata) ? PETSC_TRUE : PETSC_FALSE;
456: if (PetscDefined(USE_DEBUG)) {
457: PetscInt all_matches[2];
459: all_matches[0] = match ? 1 : 0;
460: all_matches[1] = match ? -1 : 0;
461: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, all_matches, 2, MPIU_INT, MPI_MAX, wcomm));
462: all_matches[1] = -all_matches[1];
463: PetscCheck(all_matches[0] == all_matches[1], wcomm, PETSC_ERR_ARG_INCOMP,
464: "Inconsistent use across MPI processes of persistent leaf and root data registered with PetscSFRegisterPersistent().\n"
465: "Either the persistent data was changed on a subset of processes (which is not allowed),\n"
466: "or persistent data was not deregistered with PetscSFDeregisterPersistent() before being deallocated");
467: }
468: if (match) {
469: PetscCheck(!link->inuse, wcomm, PETSC_ERR_ARG_WRONGSTATE, "Communication already in progress on persistent root and leaf data");
470: PetscCheck(!epoch || !link->epoch, wcomm, PETSC_ERR_ARG_WRONGSTATE, "Communication epoch already open for window");
471: PetscCheck(bytes == link->bytes, wcomm, PETSC_ERR_ARG_WRONGSTATE, "Wrong data type for persistent root and leaf data");
472: *win = link->win;
473: goto found;
474: }
475: }
476: }
477: for (link = w->wins; reuse && link; link = link->next) {
478: if (w->flavor != link->flavor) continue;
479: /* an existing window can be used (1) if it is not in use, (2) if we are
480: not asking to start an epoch or it does not have an already started
481: epoch, and (3) if it is the right size */
482: if (!link->inuse && (!epoch || !link->epoch) && bytes == (MPI_Aint)link->bytes) {
483: if (w->flavor == PETSCSF_WINDOW_FLAVOR_DYNAMIC) {
484: PetscCall(PetscSFWindowAttach(sf, link, rootdata, wsize));
485: } else {
486: update = PETSC_TRUE;
487: }
488: link->rootdata = rootdata;
489: link->leafdata = leafdata;
490: PetscCall(PetscInfo(sf, "Reusing window %" PETSC_INTPTR_T_FMT " of flavor %d for comm %" PETSC_INTPTR_T_FMT "\n", (PETSC_INTPTR_T)link->win, link->flavor, (PETSC_INTPTR_T)wcomm));
491: *win = link->win;
492: goto found;
493: }
494: }
496: PetscCall(PetscNew(&link));
497: link->bytes = bytes;
498: link->next = w->wins;
499: link->flavor = w->flavor;
500: link->dyn_target_addr = NULL;
501: link->reqs = NULL;
502: w->wins = link;
503: link->rootdata = rootdata;
504: link->leafdata = leafdata;
505: if (sync == PETSCSF_WINDOW_SYNC_LOCK) {
506: PetscCall(PetscMalloc1(sf->nranks, &link->reqs));
507: for (PetscMPIInt i = 0; i < sf->nranks; i++) link->reqs[i] = MPI_REQUEST_NULL;
508: }
509: switch (w->flavor) {
510: case PETSCSF_WINDOW_FLAVOR_CREATE:
511: PetscCallMPI(MPI_Win_create(rootdata, wsize, (PetscMPIInt)bytes, w->info, wcomm, &link->win));
512: link->addr = rootdata;
513: break;
514: #if PetscDefined(HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
515: case PETSCSF_WINDOW_FLAVOR_DYNAMIC:
516: PetscCallMPI(MPI_Win_create_dynamic(w->info, wcomm, &link->win));
517: PetscCall(PetscSFWindowAttach(sf, link, rootdata, wsize));
518: break;
519: case PETSCSF_WINDOW_FLAVOR_ALLOCATE:
520: PetscCallMPI(MPI_Win_allocate(wsize, (PetscMPIInt)bytes, w->info, wcomm, &link->addr, &link->win));
521: update = PETSC_TRUE;
522: break;
523: #endif
524: #if PetscDefined(HAVE_MPI_PROCESS_SHARED_MEMORY)
525: case PETSCSF_WINDOW_FLAVOR_SHARED:
526: PetscCallMPI(MPI_Win_allocate_shared(wsize, (PetscMPIInt)bytes, w->info, wcomm, &link->addr, &link->win));
527: update = PETSC_TRUE;
528: break;
529: #endif
530: default:
531: SETERRQ(wcomm, PETSC_ERR_SUP, "No support for flavor %s", PetscSFWindowFlavorTypes[w->flavor]);
532: }
533: PetscCall(PetscInfo(sf, "New window %" PETSC_INTPTR_T_FMT " of flavor %d for comm %" PETSC_INTPTR_T_FMT "\n", (PETSC_INTPTR_T)link->win, link->flavor, (PETSC_INTPTR_T)wcomm));
534: *win = link->win;
536: found:
538: if (target_disp) *target_disp = link->dyn_target_addr;
539: if (reqs) *reqs = link->reqs;
540: if (update) { /* locks are needed for the "separate" memory model only, the fence guarantees memory-synchronization */
541: PetscMPIInt rank;
543: PetscCallMPI(MPI_Comm_rank(wcomm, &rank));
544: if (sync == PETSCSF_WINDOW_SYNC_LOCK) PetscCallMPI(MPI_Win_lock(MPI_LOCK_EXCLUSIVE, rank, MPI_MODE_NOCHECK, *win));
545: PetscCall(PetscMemcpy(link->addr, rootdata, sf->nroots * bytes));
546: if (sync == PETSCSF_WINDOW_SYNC_LOCK) {
547: PetscCallMPI(MPI_Win_unlock(rank, *win));
548: PetscCallMPI(MPI_Win_fence(0, *win));
549: }
550: }
551: link->inuse = PETSC_TRUE;
552: link->epoch = epoch;
553: if (epoch) {
554: switch (sync) {
555: case PETSCSF_WINDOW_SYNC_FENCE:
556: PetscCallMPI(MPI_Win_fence(fenceassert, *win));
557: break;
558: case PETSCSF_WINDOW_SYNC_LOCK: /* Handled outside */
559: break;
560: case PETSCSF_WINDOW_SYNC_ACTIVE: {
561: MPI_Group ingroup, outgroup;
562: PetscMPIInt isize, osize;
564: /* Open MPI 4.0.2 with btl=vader does not like calling
565: - MPI_Win_complete when ogroup is empty
566: - MPI_Win_wait when igroup is empty
567: So, we do not even issue the corresponding start and post calls
568: The MPI standard (Sec. 11.5.2 of MPI 3.1) only requires that
569: start(outgroup) has a matching post(ingroup)
570: and this is guaranteed by PetscSF
571: */
572: PetscCall(PetscSFGetGroups(sf, &ingroup, &outgroup));
573: PetscCallMPI(MPI_Group_size(ingroup, &isize));
574: PetscCallMPI(MPI_Group_size(outgroup, &osize));
575: if (isize) PetscCallMPI(MPI_Win_post(ingroup, postassert, *win));
576: if (osize) PetscCallMPI(MPI_Win_start(outgroup, startassert, *win));
577: } break;
578: default:
579: SETERRQ(wcomm, PETSC_ERR_PLIB, "Unknown synchronization type");
580: }
581: }
582: PetscFunctionReturn(PETSC_SUCCESS);
583: }
585: /*
586: PetscSFFindWindow - Finds a window that is already in use
588: Not Collective
590: Input Parameters:
591: + sf - star forest
592: . unit - data type
593: . rootdata - array with which the window is associated
594: - leafdata - only used to help uniquely identify windows
596: Output Parameters:
597: + win - window
598: - reqs - outstanding requests associated to the window
600: Level: developer
602: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFGetWindow()`, `PetscSFRestoreWindow()`
603: */
604: static PetscErrorCode PetscSFFindWindow(PetscSF sf, MPI_Datatype unit, const void *rootdata, const void *leafdata, MPI_Win *win, MPI_Request **reqs)
605: {
606: PetscSF_Window *w = (PetscSF_Window *)sf->data;
607: PetscSFWinLink link;
608: PetscBool is_empty;
609: MPI_Aint bytes;
611: PetscFunctionBegin;
612: PetscCall(PetscSFGetDatatypeSize_Internal(PetscObjectComm((PetscObject)sf), unit, &bytes));
613: *win = MPI_WIN_NULL;
614: is_empty = w->is_empty;
615: if (is_empty) {
616: *reqs = NULL;
617: *win = MPI_WIN_NULL;
618: PetscFunctionReturn(PETSC_SUCCESS);
619: }
620: for (link = w->wins; link; link = link->next) {
621: if (rootdata == link->rootdata && leafdata == link->leafdata && bytes == link->bytes) {
622: PetscCall(PetscInfo(sf, "Window %" PETSC_INTPTR_T_FMT " of flavor %d for comm %" PETSC_INTPTR_T_FMT "\n", (PETSC_INTPTR_T)link->win, link->flavor, (PETSC_INTPTR_T)w->window_comm));
623: *win = link->win;
624: *reqs = link->reqs;
625: PetscFunctionReturn(PETSC_SUCCESS);
626: }
627: }
628: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Requested window not in use");
629: }
631: /*
632: PetscSFRestoreWindow - Restores a window obtained with `PetscSFGetWindow()`
634: Collective
636: Input Parameters:
637: + sf - star forest
638: . unit - data type
639: . array - array associated with window
640: . sync - type of synchronization `PetscSFWindowSyncType`
641: . epoch - close an epoch, must match argument to `PetscSFGetWindow()`
642: . update - if we have to update the local window array
643: - win - window
645: Level: developer
647: .seealso: `PetscSF`, `PETSCSFWINDOW`, `PetscSFFindWindow()`
648: */
649: static PetscErrorCode PetscSFRestoreWindow(PetscSF sf, MPI_Datatype unit, void *array, PetscSFWindowSyncType sync, PetscBool epoch, PetscMPIInt fenceassert, PetscBool update, MPI_Win *win)
650: {
651: PetscSF_Window *w = (PetscSF_Window *)sf->data;
652: PetscSFWinLink *p, link;
653: PetscBool reuse = PETSC_FALSE;
654: PetscSFWindowFlavorType flavor;
655: void *laddr;
656: MPI_Aint bytes;
657: MPI_Comm wcomm;
659: PetscFunctionBegin;
660: if (*win == MPI_WIN_NULL) PetscFunctionReturn(PETSC_SUCCESS);
661: wcomm = w->window_comm;
662: for (p = &w->wins; *p; p = &(*p)->next) {
663: link = *p;
664: if (*win == link->win) {
665: PetscCheck(array == link->rootdata, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Matched window, but not array");
666: if (epoch != link->epoch) {
667: PetscCheck(!epoch, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "No epoch to end");
668: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Restoring window without ending epoch");
669: }
670: laddr = link->addr;
671: flavor = link->flavor;
672: bytes = link->bytes;
673: if (flavor != PETSCSF_WINDOW_FLAVOR_CREATE) reuse = PETSC_TRUE;
674: else {
675: *p = link->next;
676: update = PETSC_FALSE;
677: } /* remove from list */
678: goto found;
679: }
680: }
681: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Requested window not in use");
683: found:
684: PetscCall(PetscInfo(sf, "Window %" PETSC_INTPTR_T_FMT " of flavor %d for comm %" PETSC_INTPTR_T_FMT "\n", (PETSC_INTPTR_T)link->win, link->flavor, (PETSC_INTPTR_T)wcomm));
685: if (epoch) {
686: switch (sync) {
687: case PETSCSF_WINDOW_SYNC_FENCE:
688: PetscCallMPI(MPI_Win_fence(fenceassert, *win));
689: break;
690: case PETSCSF_WINDOW_SYNC_LOCK: /* Handled outside */
691: break;
692: case PETSCSF_WINDOW_SYNC_ACTIVE: {
693: MPI_Group ingroup, outgroup;
694: PetscMPIInt isize, osize;
696: /* Open MPI 4.0.2 with btl=wader does not like calling
697: - MPI_Win_complete when ogroup is empty
698: - MPI_Win_wait when igroup is empty
699: The MPI standard (Sec. 11.5.2 of MPI 3.1) only requires that
700: - each process who issues a call to MPI_Win_start issues a call to MPI_Win_Complete
701: - each process who issues a call to MPI_Win_post issues a call to MPI_Win_Wait
702: */
703: PetscCall(PetscSFGetGroups(sf, &ingroup, &outgroup));
704: PetscCallMPI(MPI_Group_size(ingroup, &isize));
705: PetscCallMPI(MPI_Group_size(outgroup, &osize));
706: if (osize) PetscCallMPI(MPI_Win_complete(*win));
707: if (isize) PetscCallMPI(MPI_Win_wait(*win));
708: } break;
709: default:
710: SETERRQ(wcomm, PETSC_ERR_PLIB, "Unknown synchronization type");
711: }
712: }
713: #if PetscDefined(HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
714: if (link->flavor == PETSCSF_WINDOW_FLAVOR_DYNAMIC && !link->persistent) {
715: if (link->addr != NULL) PetscCallMPI(MPI_Win_detach(link->win, link->addr));
716: link->addr = NULL;
717: }
718: #endif
719: if (update) {
720: if (sync == PETSCSF_WINDOW_SYNC_LOCK) PetscCallMPI(MPI_Win_fence(MPI_MODE_NOPUT | MPI_MODE_NOSUCCEED, *win));
721: PetscCall(PetscMemcpy(array, laddr, sf->nroots * bytes));
722: }
723: link->epoch = PETSC_FALSE;
724: link->inuse = PETSC_FALSE;
725: if (!link->persistent) {
726: link->rootdata = NULL;
727: link->leafdata = NULL;
728: }
729: if (!reuse) {
730: PetscCall(PetscFree(link->dyn_target_addr));
731: PetscCall(PetscFree(link->reqs));
732: PetscCallMPI(MPI_Win_free(&link->win));
733: PetscCall(PetscFree(link));
734: *win = MPI_WIN_NULL;
735: }
736: PetscFunctionReturn(PETSC_SUCCESS);
737: }
739: static PetscErrorCode PetscSFSetUp_Window(PetscSF sf)
740: {
741: PetscSF_Window *w = (PetscSF_Window *)sf->data;
742: MPI_Group ingroup, outgroup;
743: MPI_Comm comm;
745: PetscFunctionBegin;
746: PetscCall(PetscSFSetUpRanks(sf, MPI_GROUP_EMPTY));
747: PetscCall(PetscObjectGetComm((PetscObject)sf, &comm));
748: if (w->window_comm == MPI_COMM_NULL) {
749: PetscInt nroots, nleaves, nranks;
750: PetscBool has_empty;
751: PetscMPIInt wcommrank;
752: PetscSF dynsf_full = NULL;
754: if (w->flavor == PETSCSF_WINDOW_FLAVOR_DYNAMIC) PetscCall(PetscSFWindowCreateDynamicSF(sf, &dynsf_full));
756: PetscCall(PetscSFGetGraph(sf, &nroots, &nleaves, NULL, NULL));
757: has_empty = (nroots == 0 && nleaves == 0) ? PETSC_TRUE : PETSC_FALSE;
758: nranks = sf->nranks;
759: PetscCall(PetscMalloc1(nranks, &w->wcommranks));
760: w->is_empty = has_empty;
761: PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, &has_empty, 1, MPI_C_BOOL, MPI_LOR, comm));
762: if (has_empty) {
763: PetscMPIInt rank;
764: MPI_Comm raw_comm;
765: PetscSFNode *remotes;
767: PetscCallMPI(MPI_Comm_rank(comm, &rank));
768: PetscCallMPI(MPI_Comm_split(comm, w->is_empty ? 1 : 0, rank, &raw_comm));
769: PetscCall(PetscCommDuplicate(raw_comm, &w->window_comm, NULL));
770: PetscCallMPI(MPI_Comm_free(&raw_comm));
772: PetscCallMPI(MPI_Comm_rank(w->window_comm, &wcommrank));
773: if (!dynsf_full) PetscCall(PetscSFWindowCreateDynamicSF(sf, &dynsf_full));
774: PetscCall(PetscSFBcastBegin(dynsf_full, MPI_INT, &wcommrank, w->wcommranks, MPI_REPLACE));
775: PetscCall(PetscSFBcastEnd(dynsf_full, MPI_INT, &wcommrank, w->wcommranks, MPI_REPLACE));
777: if (w->flavor == PETSCSF_WINDOW_FLAVOR_DYNAMIC) {
778: PetscCall(PetscSFCreate(w->window_comm, &w->dynsf));
779: PetscCall(PetscSFSetType(w->dynsf, PETSCSFBASIC)); /* break recursion */
780: PetscCall(PetscMalloc1(sf->nranks, &remotes));
781: for (PetscInt i = 0; i < sf->nranks; i++) {
782: remotes[i].rank = w->wcommranks[i];
783: remotes[i].index = 0;
784: }
785: PetscCall(PetscSFSetGraph(w->dynsf, 1, sf->nranks, NULL, PETSC_OWN_POINTER, remotes, PETSC_OWN_POINTER));
786: }
787: } else {
788: PetscCall(PetscCommDuplicate(PetscObjectComm((PetscObject)sf), &w->window_comm, NULL));
789: PetscCall(PetscArraycpy(w->wcommranks, sf->ranks, nranks));
790: PetscCall(PetscObjectReference((PetscObject)dynsf_full));
791: w->dynsf = dynsf_full;
792: }
793: if (w->dynsf) PetscCall(PetscSFSetUp(w->dynsf));
794: PetscCall(PetscSFDestroy(&dynsf_full));
795: }
796: switch (w->sync) {
797: case PETSCSF_WINDOW_SYNC_ACTIVE:
798: PetscCall(PetscSFGetGroups(sf, &ingroup, &outgroup));
799: default:
800: break;
801: }
802: PetscFunctionReturn(PETSC_SUCCESS);
803: }
805: static PetscErrorCode PetscSFSetFromOptions_Window(PetscSF sf, PetscOptionItems PetscOptionsObject)
806: {
807: PetscSF_Window *w = (PetscSF_Window *)sf->data;
808: PetscSFWindowFlavorType flavor = w->flavor;
810: PetscFunctionBegin;
811: PetscOptionsHeadBegin(PetscOptionsObject, "PetscSF Window options");
812: PetscCall(PetscOptionsEnum("-sf_window_sync", "synchronization type to use for PetscSF Window communication", "PetscSFWindowSetSyncType", PetscSFWindowSyncTypes, (PetscEnum)w->sync, (PetscEnum *)&w->sync, NULL));
813: PetscCall(PetscOptionsEnum("-sf_window_flavor", "flavor to use for PetscSF Window creation", "PetscSFWindowSetFlavorType", PetscSFWindowFlavorTypes, (PetscEnum)flavor, (PetscEnum *)&flavor, NULL));
814: PetscCall(PetscSFWindowSetFlavorType(sf, flavor));
815: PetscOptionsHeadEnd();
816: PetscFunctionReturn(PETSC_SUCCESS);
817: }
819: static PetscErrorCode PetscSFReset_Window(PetscSF sf)
820: {
821: PetscSF_Window *w = (PetscSF_Window *)sf->data;
822: PetscSFDataLink link, next;
823: PetscSFWinLink wlink, wnext;
824: MPI_Comm wcomm;
825: PetscBool is_empty;
827: PetscFunctionBegin;
828: for (link = w->link; link; link = next) {
829: next = link->next;
830: PetscCallMPI(MPI_Type_free(&link->unit));
831: for (PetscInt i = 0; i < sf->nranks; i++) {
832: PetscCallMPI(MPI_Type_free(&link->mine[i]));
833: PetscCallMPI(MPI_Type_free(&link->remote[i]));
834: }
835: PetscCall(PetscFree2(link->mine, link->remote));
836: PetscCall(PetscFree(link));
837: }
838: w->link = NULL;
839: wcomm = w->window_comm;
840: is_empty = w->is_empty;
841: if (!is_empty) {
842: for (wlink = w->wins; wlink; wlink = wnext) {
843: wnext = wlink->next;
844: PetscCheck(!wlink->inuse, wcomm, PETSC_ERR_ARG_WRONGSTATE, "Window still in use with address %p", (void *)wlink->addr);
845: PetscCall(PetscFree(wlink->dyn_target_addr));
846: PetscCall(PetscFree(wlink->reqs));
847: PetscCallMPI(MPI_Win_free(&wlink->win));
848: PetscCall(PetscFree(wlink));
849: }
850: }
851: w->wins = NULL;
852: PetscCall(PetscSFDestroy(&w->dynsf));
853: if (w->info != MPI_INFO_NULL) PetscCallMPI(MPI_Info_free(&w->info));
854: PetscCall(PetscCommDestroy(&w->window_comm));
855: PetscCall(PetscFree(w->wcommranks));
856: PetscFunctionReturn(PETSC_SUCCESS);
857: }
859: static PetscErrorCode PetscSFRegisterPersistent_Window(PetscSF sf, MPI_Datatype unit, const void *rootdata, const void *leafdata)
860: {
861: PetscSF_Window *w = (PetscSF_Window *)sf->data;
862: MPI_Aint bytes, wsize;
863: PetscBool is_empty;
864: PetscSFWinLink link;
866: PetscFunctionBegin;
867: PetscCall(PetscSFSetUp(sf));
868: if (w->flavor != PETSCSF_WINDOW_FLAVOR_DYNAMIC) PetscFunctionReturn(PETSC_SUCCESS);
869: PetscCall(PetscSFGetDatatypeSize_Internal(PetscObjectComm((PetscObject)sf), unit, &bytes));
870: wsize = (MPI_Aint)(bytes * sf->nroots);
871: is_empty = w->is_empty;
872: if (is_empty) PetscFunctionReturn(PETSC_SUCCESS);
873: PetscCall(PetscNew(&link));
874: link->flavor = w->flavor;
875: link->next = w->wins;
876: #if PetscDefined(HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
877: {
878: MPI_Comm wcomm = w->window_comm;
879: PetscCallMPI(MPI_Win_create_dynamic(w->info, wcomm, &link->win));
880: }
881: #endif
882: PetscCall(PetscSFWindowAttach(sf, link, (void *)rootdata, wsize));
883: link->rootdata = (void *)rootdata;
884: link->leafdata = (void *)leafdata;
885: link->bytes = bytes;
886: link->epoch = PETSC_FALSE;
887: link->inuse = PETSC_FALSE;
888: link->persistent = PETSC_TRUE;
889: w->wins = link;
890: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) {
891: PetscCall(PetscMalloc1(sf->nranks, &link->reqs));
892: for (PetscInt i = 0; i < sf->nranks; i++) link->reqs[i] = MPI_REQUEST_NULL;
893: }
894: PetscFunctionReturn(PETSC_SUCCESS);
895: }
897: static PetscErrorCode PetscSFDeregisterPersistent_Window(PetscSF sf, MPI_Datatype unit, const void *rootdata, const void *leafdata)
898: {
899: PetscSF_Window *w = (PetscSF_Window *)sf->data;
900: MPI_Aint bytes;
901: MPI_Comm wcomm;
902: PetscBool is_empty;
903: PetscSFWinLink *p;
905: PetscFunctionBegin;
906: PetscCall(PetscSFSetUp(sf));
907: if (w->flavor != PETSCSF_WINDOW_FLAVOR_DYNAMIC) PetscFunctionReturn(PETSC_SUCCESS);
908: PetscCall(PetscSFGetDatatypeSize_Internal(PetscObjectComm((PetscObject)sf), unit, &bytes));
909: wcomm = w->window_comm;
910: is_empty = w->is_empty;
911: if (is_empty) PetscFunctionReturn(PETSC_SUCCESS);
912: for (p = &w->wins; *p; p = &(*p)->next) {
913: PetscSFWinLink link = *p;
914: if (link->flavor == w->flavor && link->persistent && link->rootdata == rootdata && link->leafdata == leafdata && link->bytes == bytes) {
915: PetscCheck(!link->inuse, wcomm, PETSC_ERR_ARG_WRONGSTATE, "Deregistering a window when communication is still in progress");
916: PetscCheck(!link->epoch, wcomm, PETSC_ERR_ARG_WRONGSTATE, "Deregistering a window with an unconcluded epoch");
917: #if PetscDefined(HAVE_MPI_FEATURE_DYNAMIC_WINDOW)
918: PetscCallMPI(MPI_Win_detach(link->win, link->addr));
919: link->addr = NULL;
920: #endif
921: PetscCall(PetscFree(link->dyn_target_addr));
922: PetscCall(PetscFree(link->reqs));
923: PetscCallMPI(MPI_Win_free(&link->win));
924: *p = link->next;
925: PetscCall(PetscFree(link));
926: break;
927: }
928: }
929: PetscFunctionReturn(PETSC_SUCCESS);
930: }
932: static PetscErrorCode PetscSFDestroy_Window(PetscSF sf)
933: {
934: PetscFunctionBegin;
935: PetscCall(PetscSFReset_Window(sf));
936: PetscCall(PetscFree(sf->data));
937: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowSetSyncType_C", NULL));
938: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowGetSyncType_C", NULL));
939: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowSetFlavorType_C", NULL));
940: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowGetFlavorType_C", NULL));
941: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowSetInfo_C", NULL));
942: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowGetInfo_C", NULL));
943: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFRegisterPersistent_C", NULL));
944: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFDeregisterPersistent_C", NULL));
945: PetscFunctionReturn(PETSC_SUCCESS);
946: }
948: static PetscErrorCode PetscSFView_Window(PetscSF sf, PetscViewer viewer)
949: {
950: PetscSF_Window *w = (PetscSF_Window *)sf->data;
951: PetscBool isascii;
952: PetscViewerFormat format;
954: PetscFunctionBegin;
955: PetscCall(PetscViewerGetFormat(viewer, &format));
956: PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &isascii));
957: if (isascii) {
958: PetscCall(PetscViewerASCIIPrintf(viewer, " current flavor=%s synchronization=%s MultiSF sort=%s\n", PetscSFWindowFlavorTypes[w->flavor], PetscSFWindowSyncTypes[w->sync], sf->rankorder ? "rank-order" : "unordered"));
959: if (format == PETSC_VIEWER_ASCII_INFO_DETAIL) {
960: if (w->info != MPI_INFO_NULL) {
961: PetscMPIInt k, nkeys;
962: char key[MPI_MAX_INFO_KEY], value[MPI_MAX_INFO_VAL];
964: PetscCallMPI(MPI_Info_get_nkeys(w->info, &nkeys));
965: PetscCall(PetscViewerASCIIPrintf(viewer, " current info with %d keys. Ordered key-value pairs follow:\n", nkeys));
966: for (k = 0; k < nkeys; k++) {
967: PetscMPIInt flag;
969: PetscCallMPI(MPI_Info_get_nthkey(w->info, k, key));
970: PetscCallMPI(MPI_Info_get(w->info, key, MPI_MAX_INFO_VAL, value, &flag));
971: PetscCheck(flag, PETSC_COMM_SELF, PETSC_ERR_PLIB, "Missing key %s", key);
972: PetscCall(PetscViewerASCIIPrintf(viewer, " %s = %s\n", key, value));
973: }
974: } else {
975: PetscCall(PetscViewerASCIIPrintf(viewer, " current info=MPI_INFO_NULL\n"));
976: }
977: }
978: }
979: PetscFunctionReturn(PETSC_SUCCESS);
980: }
982: static PetscErrorCode PetscSFDuplicate_Window(PetscSF sf, PetscSFDuplicateOption opt, PetscSF newsf)
983: {
984: PetscSF_Window *w = (PetscSF_Window *)sf->data;
985: PetscSFWindowSyncType synctype;
987: PetscFunctionBegin;
988: synctype = w->sync;
989: /* HACK: Must use FENCE or LOCK when called from PetscSFGetGroups() because ACTIVE here would cause recursion. */
990: if (!sf->setupcalled) synctype = PETSCSF_WINDOW_SYNC_LOCK;
991: PetscCall(PetscSFWindowSetSyncType(newsf, synctype));
992: PetscCall(PetscSFWindowSetFlavorType(newsf, w->flavor));
993: PetscCall(PetscSFWindowSetInfo(newsf, w->info));
994: PetscFunctionReturn(PETSC_SUCCESS);
995: }
997: static PetscErrorCode PetscSFBcastBegin_Window(PetscSF sf, MPI_Datatype unit, PetscMemType rootmtype, const void *rootdata, PetscMemType leafmtype, void *leafdata, MPI_Op op)
998: {
999: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1000: PetscMPIInt nranks;
1001: const PetscMPIInt *ranks;
1002: const MPI_Aint *target_disp;
1003: const MPI_Datatype *mine, *remote;
1004: MPI_Request *reqs;
1005: MPI_Win win;
1007: PetscFunctionBegin;
1008: PetscCheck(op == MPI_REPLACE, PetscObjectComm((PetscObject)sf), PETSC_ERR_SUP, "PetscSFBcastBegin_Window with op!=MPI_REPLACE has not been implemented");
1009: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, NULL, NULL, NULL));
1010: PetscCall(PetscSFWindowGetDataTypes(sf, unit, &mine, &remote));
1011: PetscCall(PetscSFGetWindow(sf, unit, (void *)rootdata, leafdata, w->sync, PETSC_TRUE, MPI_MODE_NOPUT | MPI_MODE_NOPRECEDE, MPI_MODE_NOPUT, 0, &target_disp, &reqs, &win));
1012: ranks = w->wcommranks;
1013: for (PetscMPIInt i = 0; i < nranks; i++) {
1014: MPI_Aint tdp = target_disp ? target_disp[i] : 0;
1015: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) {
1016: PetscCallMPI(MPI_Win_lock(MPI_LOCK_SHARED, ranks[i], MPI_MODE_NOCHECK, win));
1017: #if PetscDefined(HAVE_MPI_RGET)
1018: PetscCallMPI(MPI_Rget(leafdata, 1, mine[i], ranks[i], tdp, 1, remote[i], win, &reqs[i]));
1019: #else
1020: PetscCallMPI(MPI_Get(leafdata, 1, mine[i], ranks[i], tdp, 1, remote[i], win));
1021: #endif
1022: } else {
1023: CHKMEMQ;
1024: PetscCallMPI(MPI_Get(leafdata, 1, mine[i], ranks[i], tdp, 1, remote[i], win));
1025: CHKMEMQ;
1026: }
1027: }
1028: PetscFunctionReturn(PETSC_SUCCESS);
1029: }
1031: static PetscErrorCode PetscSFBcastEnd_Window(PetscSF sf, MPI_Datatype unit, const void *rootdata, void *leafdata, MPI_Op op)
1032: {
1033: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1034: MPI_Win win;
1035: MPI_Request *reqs = NULL;
1037: PetscFunctionBegin;
1038: PetscCall(PetscSFFindWindow(sf, unit, rootdata, leafdata, &win, &reqs));
1039: if (reqs) PetscCallMPI(MPI_Waitall(sf->nranks, reqs, MPI_STATUSES_IGNORE));
1040: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) {
1041: PetscMPIInt nranks;
1042: const PetscMPIInt *ranks;
1044: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, NULL, NULL, NULL));
1045: ranks = w->wcommranks;
1046: for (PetscMPIInt i = 0; i < nranks; i++) PetscCallMPI(MPI_Win_unlock(ranks[i], win));
1047: }
1048: PetscCall(PetscSFRestoreWindow(sf, unit, (void *)rootdata, w->sync, PETSC_TRUE, MPI_MODE_NOSTORE | MPI_MODE_NOSUCCEED, PETSC_FALSE, &win));
1049: PetscFunctionReturn(PETSC_SUCCESS);
1050: }
1052: static PetscErrorCode PetscSFReduceBegin_Window(PetscSF sf, MPI_Datatype unit, PetscMemType leafmtype, const void *leafdata, PetscMemType rootmtype, void *rootdata, MPI_Op op)
1053: {
1054: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1055: PetscMPIInt nranks;
1056: const PetscMPIInt *ranks;
1057: const MPI_Aint *target_disp;
1058: const MPI_Datatype *mine, *remote;
1059: MPI_Win win;
1061: PetscFunctionBegin;
1062: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, NULL, NULL, NULL));
1063: PetscCall(PetscSFWindowGetDataTypes(sf, unit, &mine, &remote));
1064: PetscCall(PetscSFWindowOpTranslate(&op));
1065: PetscCall(PetscSFGetWindow(sf, unit, rootdata, (void *)leafdata, w->sync, PETSC_TRUE, MPI_MODE_NOPRECEDE, 0, 0, &target_disp, NULL, &win));
1066: ranks = w->wcommranks;
1067: for (PetscMPIInt i = 0; i < nranks; i++) {
1068: MPI_Aint tdp = target_disp ? target_disp[i] : 0;
1070: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) PetscCallMPI(MPI_Win_lock(MPI_LOCK_SHARED, ranks[i], MPI_MODE_NOCHECK, win));
1071: PetscCallMPI(MPI_Accumulate((void *)leafdata, 1, mine[i], ranks[i], tdp, 1, remote[i], op, win));
1072: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) PetscCallMPI(MPI_Win_unlock(ranks[i], win));
1073: }
1074: PetscFunctionReturn(PETSC_SUCCESS);
1075: }
1077: static PetscErrorCode PetscSFReduceEnd_Window(PetscSF sf, MPI_Datatype unit, const void *leafdata, void *rootdata, MPI_Op op)
1078: {
1079: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1080: MPI_Win win;
1081: MPI_Request *reqs = NULL;
1083: PetscFunctionBegin;
1084: PetscCall(PetscSFFindWindow(sf, unit, rootdata, leafdata, &win, &reqs));
1085: if (reqs) PetscCallMPI(MPI_Waitall(sf->nranks, reqs, MPI_STATUSES_IGNORE));
1086: PetscCall(PetscSFRestoreWindow(sf, unit, rootdata, w->sync, PETSC_TRUE, MPI_MODE_NOSUCCEED, PETSC_TRUE, &win));
1087: PetscFunctionReturn(PETSC_SUCCESS);
1088: }
1090: static PetscErrorCode PetscSFFetchAndOpBegin_Window(PetscSF sf, MPI_Datatype unit, PetscMemType rootmtype, void *rootdata, PetscMemType leafmtype, const void *leafdata, void *leafupdate, MPI_Op op)
1091: {
1092: PetscMPIInt nranks;
1093: const PetscMPIInt *ranks;
1094: const MPI_Datatype *mine, *remote;
1095: const MPI_Aint *target_disp;
1096: MPI_Win win;
1097: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1098: #if !PetscDefined(HAVE_MPI_GET_ACCUMULATE)
1099: PetscSFWindowFlavorType oldf;
1100: #endif
1102: PetscFunctionBegin;
1103: PetscCall(PetscSFGetRootRanks(sf, &nranks, NULL, NULL, NULL, NULL));
1104: PetscCall(PetscSFWindowGetDataTypes(sf, unit, &mine, &remote));
1105: PetscCall(PetscSFWindowOpTranslate(&op));
1106: #if !PetscDefined(HAVE_MPI_GET_ACCUMULATE)
1107: /* FetchAndOp without MPI_Get_Accumulate requires locking.
1108: we create a new window every time to not interfere with user-defined MPI_Info which may have used "no_locks"="true" */
1109: oldf = w->flavor;
1110: w->flavor = PETSCSF_WINDOW_FLAVOR_CREATE;
1111: PetscCall(PetscSFGetWindow(sf, unit, rootdata, (void *)leafdata, PETSCSF_WINDOW_SYNC_LOCK, PETSC_FALSE, 0, 0, 0, &target_disp, NULL, &win));
1112: #else
1113: PetscCall(PetscSFGetWindow(sf, unit, rootdata, (void *)leafdata, w->sync, PETSC_TRUE, MPI_MODE_NOPRECEDE, 0, 0, &target_disp, NULL, &win));
1114: #endif
1115: ranks = w->wcommranks;
1116: for (PetscMPIInt i = 0; i < nranks; i++) {
1117: MPI_Aint tdp = target_disp ? target_disp[i] : 0;
1119: #if !PetscDefined(HAVE_MPI_GET_ACCUMULATE)
1120: PetscCallMPI(MPI_Win_lock(MPI_LOCK_EXCLUSIVE, ranks[i], 0, win));
1121: PetscCallMPI(MPI_Get(leafupdate, 1, mine[i], ranks[i], tdp, 1, remote[i], win));
1122: PetscCallMPI(MPI_Accumulate((void *)leafdata, 1, mine[i], ranks[i], tdp, 1, remote[i], op, win));
1123: PetscCallMPI(MPI_Win_unlock(ranks[i], win));
1124: #else
1125: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) PetscCallMPI(MPI_Win_lock(MPI_LOCK_SHARED, ranks[i], 0, win));
1126: PetscCallMPI(MPI_Get_accumulate((void *)leafdata, 1, mine[i], leafupdate, 1, mine[i], ranks[i], tdp, 1, remote[i], op, win));
1127: if (w->sync == PETSCSF_WINDOW_SYNC_LOCK) PetscCallMPI(MPI_Win_unlock(ranks[i], win));
1128: #endif
1129: }
1130: #if !PetscDefined(HAVE_MPI_GET_ACCUMULATE)
1131: w->flavor = oldf;
1132: #endif
1133: PetscFunctionReturn(PETSC_SUCCESS);
1134: }
1136: static PetscErrorCode PetscSFFetchAndOpEnd_Window(PetscSF sf, MPI_Datatype unit, void *rootdata, const void *leafdata, void *leafupdate, MPI_Op op)
1137: {
1138: MPI_Win win;
1139: #if PetscDefined(HAVE_MPI_GET_ACCUMULATE)
1140: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1141: #endif
1142: MPI_Request *reqs = NULL;
1144: PetscFunctionBegin;
1145: PetscCall(PetscSFFindWindow(sf, unit, rootdata, leafdata, &win, &reqs));
1146: if (reqs) PetscCallMPI(MPI_Waitall(sf->nranks, reqs, MPI_STATUSES_IGNORE));
1147: #if PetscDefined(HAVE_MPI_GET_ACCUMULATE)
1148: PetscCall(PetscSFRestoreWindow(sf, unit, rootdata, w->sync, PETSC_TRUE, MPI_MODE_NOSUCCEED, PETSC_TRUE, &win));
1149: #else
1150: PetscCall(PetscSFRestoreWindow(sf, unit, rootdata, PETSCSF_WINDOW_SYNC_LOCK, PETSC_FALSE, 0, PETSC_TRUE, &win));
1151: #endif
1152: PetscFunctionReturn(PETSC_SUCCESS);
1153: }
1155: PETSC_INTERN PetscErrorCode PetscSFCreate_Window(PetscSF sf)
1156: {
1157: PetscSF_Window *w = (PetscSF_Window *)sf->data;
1159: PetscFunctionBegin;
1160: sf->ops->SetUp = PetscSFSetUp_Window;
1161: sf->ops->SetFromOptions = PetscSFSetFromOptions_Window;
1162: sf->ops->Reset = PetscSFReset_Window;
1163: sf->ops->Destroy = PetscSFDestroy_Window;
1164: sf->ops->View = PetscSFView_Window;
1165: sf->ops->Duplicate = PetscSFDuplicate_Window;
1166: sf->ops->BcastBegin = PetscSFBcastBegin_Window;
1167: sf->ops->BcastEnd = PetscSFBcastEnd_Window;
1168: sf->ops->ReduceBegin = PetscSFReduceBegin_Window;
1169: sf->ops->ReduceEnd = PetscSFReduceEnd_Window;
1170: sf->ops->FetchAndOpBegin = PetscSFFetchAndOpBegin_Window;
1171: sf->ops->FetchAndOpEnd = PetscSFFetchAndOpEnd_Window;
1173: PetscCall(PetscNew(&w));
1174: sf->data = (void *)w;
1175: w->sync = PETSCSF_WINDOW_SYNC_FENCE;
1176: w->flavor = PETSCSF_WINDOW_FLAVOR_CREATE;
1177: w->info = MPI_INFO_NULL;
1178: w->window_comm = MPI_COMM_NULL;
1180: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowSetSyncType_C", PetscSFWindowSetSyncType_Window));
1181: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowGetSyncType_C", PetscSFWindowGetSyncType_Window));
1182: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowSetFlavorType_C", PetscSFWindowSetFlavorType_Window));
1183: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowGetFlavorType_C", PetscSFWindowGetFlavorType_Window));
1184: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowSetInfo_C", PetscSFWindowSetInfo_Window));
1185: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFWindowGetInfo_C", PetscSFWindowGetInfo_Window));
1186: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFRegisterPersistent_C", PetscSFRegisterPersistent_Window));
1187: PetscCall(PetscObjectComposeFunction((PetscObject)sf, "PetscSFDeregisterPersistent_C", PetscSFDeregisterPersistent_Window));
1189: #if PetscDefined(HAVE_OPENMPI)
1190: #if PETSC_PKG_OPENMPI_VERSION_LE(1, 6, 0)
1191: {
1192: PetscBool ackbug = PETSC_FALSE;
1194: PetscCall(PetscOptionsGetBool(NULL, NULL, "-acknowledge_ompi_onesided_bug", &ackbug, NULL));
1195: PetscCheck(ackbug, PetscObjectComm((PetscObject)sf), PETSC_ERR_LIB, "Open MPI is known to be buggy (https://svn.open-mpi.org/trac/ompi/ticket/1905 and 2656), use -acknowledge_ompi_onesided_bug to proceed");
1196: PetscCall(PetscInfo(sf, "Acknowledged Open MPI bug, proceeding anyway. Expect memory corruption.\n"));
1197: }
1198: #endif
1199: #endif
1200: PetscFunctionReturn(PETSC_SUCCESS);
1201: }