1: #include <petsc/private/petscimpl.h>
  2: /*
  3:     Note that tag of 0 is ok because comm is a private communicator
  4:   generated below just for these routines.
  5: */

  7: PETSC_INTERN PetscErrorCode PetscSequentialPhaseBegin_Private(MPI_Comm comm, int ng)
  8: {
  9:   PetscMPIInt rank, size, tag = 0;
 10:   MPI_Status  status;

 12:   PetscFunctionBegin;
 13:   PetscCallMPI(MPI_Comm_size(comm, &size));
 14:   if (size == 1) PetscFunctionReturn(PETSC_SUCCESS);
 15:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
 16:   if (rank) PetscCallMPI(MPI_Recv(NULL, 0, MPI_INT, rank - 1, tag, comm, &status));
 17:   /* Send to the next process in the group unless we are the last process */
 18:   if ((rank % ng) < ng - 1 && rank != size - 1) PetscCallMPI(MPI_Send(NULL, 0, MPI_INT, rank + 1, tag, comm));
 19:   PetscFunctionReturn(PETSC_SUCCESS);
 20: }

 22: PETSC_INTERN PetscErrorCode PetscSequentialPhaseEnd_Private(MPI_Comm comm, int ng)
 23: {
 24:   PetscMPIInt rank, size, tag = 0;
 25:   MPI_Status  status;

 27:   PetscFunctionBegin;
 28:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
 29:   PetscCallMPI(MPI_Comm_size(comm, &size));
 30:   if (size == 1) PetscFunctionReturn(PETSC_SUCCESS);

 32:   /* Send to the first process in the next group */
 33:   if ((rank % ng) == ng - 1 || rank == size - 1) PetscCallMPI(MPI_Send(NULL, 0, MPI_INT, (rank + 1) % size, tag, comm));
 34:   if (rank == 0) PetscCallMPI(MPI_Recv(NULL, 0, MPI_INT, size - 1, tag, comm, &status));
 35:   PetscFunctionReturn(PETSC_SUCCESS);
 36: }

 38: /*
 39:     The variable Petsc_Seq_keyval is used to indicate an MPI attribute that
 40:   is attached to a communicator that manages the sequential phase code below.
 41: */
 42: PetscMPIInt Petsc_Seq_keyval = MPI_KEYVAL_INVALID;

 44: /*@
 45:   PetscSequentialPhaseBegin - Begins a sequential section of code.

 47:   Collective

 49:   Input Parameters:
 50: + comm - Communicator to sequentialize over
 51: - ng   - Number in processor group.  This many processes are allowed to execute
 52:    at the same time (usually 1)

 54:   Level: intermediate

 56:   Notes:
 57:   `PetscSequentialPhaseBegin()` and `PetscSequentialPhaseEnd()` provide a
 58:   way to force a section of code to be executed by the processes in
 59:   rank order.  Typically, this is done with
 60: .vb
 61:       PetscSequentialPhaseBegin(comm, 1);
 62:       <code to be executed sequentially>
 63:       PetscSequentialPhaseEnd(comm, 1);
 64: .ve

 66:   You should use `PetscSynchronizedPrintf()` to ensure output between MPI ranks is properly order and not these routines.

 68: .seealso: `PetscSequentialPhaseEnd()`, `PetscSynchronizedPrintf()`
 69: @*/
 70: PetscErrorCode PetscSequentialPhaseBegin(MPI_Comm comm, int ng)
 71: {
 72:   PetscMPIInt size;
 73:   MPI_Comm    local_comm, *addr_local_comm;

 75:   PetscFunctionBegin;
 76:   PetscCall(PetscSysInitializePackage());
 77:   PetscCallMPI(MPI_Comm_size(comm, &size));
 78:   if (size == 1) PetscFunctionReturn(PETSC_SUCCESS);

 80:   /* Get the private communicator for the sequential operations */
 81:   if (Petsc_Seq_keyval == MPI_KEYVAL_INVALID) PetscCallMPI(MPI_Comm_create_keyval(MPI_COMM_NULL_COPY_FN, MPI_COMM_NULL_DELETE_FN, &Petsc_Seq_keyval, NULL));

 83:   PetscCallMPI(MPI_Comm_dup(comm, &local_comm));
 84:   PetscCall(PetscMalloc1(1, &addr_local_comm));

 86:   *addr_local_comm = local_comm;

 88:   PetscCallMPI(MPI_Comm_set_attr(comm, Petsc_Seq_keyval, (void *)addr_local_comm));
 89:   PetscCall(PetscSequentialPhaseBegin_Private(local_comm, ng));
 90:   PetscFunctionReturn(PETSC_SUCCESS);
 91: }

 93: /*@
 94:   PetscSequentialPhaseEnd - Ends a sequential section of code.

 96:   Collective

 98:   Input Parameters:
 99: + comm - Communicator to sequentialize.
100: - ng   - Number in processor group.  This many processes are allowed to execute
101:    at the same time (usually 1)

103:   Level: intermediate

105:   Note:
106:   See `PetscSequentialPhaseBegin()` for more details.

108: .seealso: `PetscSequentialPhaseBegin()`
109: @*/
110: PetscErrorCode PetscSequentialPhaseEnd(MPI_Comm comm, int ng)
111: {
112:   PetscMPIInt size, iflg;
113:   MPI_Comm    local_comm, *addr_local_comm;

115:   PetscFunctionBegin;
116:   PetscCallMPI(MPI_Comm_size(comm, &size));
117:   if (size == 1) PetscFunctionReturn(PETSC_SUCCESS);

119:   PetscCallMPI(MPI_Comm_get_attr(comm, Petsc_Seq_keyval, (void **)&addr_local_comm, &iflg));
120:   PetscCheck(iflg, PETSC_COMM_SELF, PETSC_ERR_ARG_INCOMP, "Wrong MPI communicator; must pass in one used with PetscSequentialPhaseBegin()");
121:   local_comm = *addr_local_comm;

123:   PetscCall(PetscSequentialPhaseEnd_Private(local_comm, ng));

125:   PetscCall(PetscFree(addr_local_comm));
126:   PetscCallMPI(MPI_Comm_free(&local_comm));
127:   PetscCallMPI(MPI_Comm_delete_attr(comm, Petsc_Seq_keyval));
128:   PetscFunctionReturn(PETSC_SUCCESS);
129: }

131: /*@
132:   PetscGlobalMinMaxInt - Get the global min/max from local min/max input

134:   Collective

136:   Input Parameters:
137: + comm      - The MPI communicator to reduce with
138: - minMaxVal - An array with the local min and max

140:   Output Parameter:
141: . minMaxValGlobal - An array with the global min and max

143:   Level: beginner

145: .seealso: `PetscSplitOwnership()`, `PetscGlobalMinMaxReal()`
146: @*/
147: PetscErrorCode PetscGlobalMinMaxInt(MPI_Comm comm, const PetscInt minMaxVal[2], PetscInt minMaxValGlobal[2])
148: {
149:   PetscInt  buf[3];
150:   PetscBool hasminint = (PetscBool)(minMaxVal[0] == PETSC_MIN_INT);

152:   PetscFunctionBegin;
153:   buf[0] = hasminint ? PETSC_MIN_INT : -minMaxVal[0]; /* Note that -PETSC_INT_MIN = PETSC_INT_MIN: ternary to suppress sanitizer warnings */
154:   buf[1] = minMaxVal[1];
155:   buf[2] = hasminint ? 1 : 0; /* Are there PETSC_INT_MIN in minMaxVal[0]? */
156:   PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, buf, 3, MPIU_INT, MPI_MAX, comm));
157:   minMaxValGlobal[0] = buf[2] ? PETSC_INT_MIN : -buf[0];
158:   minMaxValGlobal[1] = buf[1];
159:   PetscFunctionReturn(PETSC_SUCCESS);
160: }

162: /*@
163:   PetscGlobalMinMaxReal - Get the global min/max from local min/max input

165:   Collective

167:   Input Parameters:
168: + comm      - The MPI communicator to reduce with
169: - minMaxVal - An array with the local min and max

171:   Output Parameter:
172: . minMaxValGlobal - An array with the global min and max

174:   Level: beginner

176: .seealso: `PetscSplitOwnership()`, `PetscGlobalMinMaxInt()`
177: @*/
178: PetscErrorCode PetscGlobalMinMaxReal(MPI_Comm comm, const PetscReal minMaxVal[2], PetscReal minMaxValGlobal[2])
179: {
180:   PetscFunctionBegin;
181:   minMaxValGlobal[0] = -minMaxVal[0];
182:   minMaxValGlobal[1] = minMaxVal[1];
183:   PetscCallMPI(MPIU_Allreduce(MPI_IN_PLACE, minMaxValGlobal, 2, MPIU_REAL, MPIU_MAX, comm));
184:   minMaxValGlobal[0] = -minMaxValGlobal[0];
185:   PetscFunctionReturn(PETSC_SUCCESS);
186: }