1: static char help[] = "Partition tiny grid using hierarchical partitioning and increase overlap using MatIncreaseOverlapSplit.\n\n";

  3: /*
  4:   Include "petscmat.h" so that we can use matrices.  Note that this file
  5:   automatically includes:
  6:      petscsys.h - base PETSc routines
  7:      petscvec.h - vectors
  8:      petscis.h  - index sets
  9: */
 10: #include <petscmat.h>

 12: int main(int argc, char **args)
 13: {
 14:   Mat             A, B;
 15:   PetscMPIInt     rank, size, membershipKey;
 16:   PetscInt       *ia, *ja, *indices_sc, isrows_localsize;
 17:   const PetscInt *indices;
 18:   MatPartitioning part;
 19:   IS              is, isrows, isrows_sc;
 20:   IS              coarseparts, fineparts;
 21:   MPI_Comm        comm, scomm;

 23:   PetscFunctionBeginUser;
 24:   PetscCall(PetscInitialize(&argc, &args, NULL, help));
 25:   comm = PETSC_COMM_WORLD;
 26:   PetscCallMPI(MPI_Comm_size(comm, &size));
 27:   PetscCheck(size == 4, comm, PETSC_ERR_WRONG_MPI_SIZE, "Must run with 4 processors ");
 28:   PetscCallMPI(MPI_Comm_rank(comm, &rank));
 29:   /*set a small matrix */
 30:   PetscCall(PetscMalloc1(5, &ia));
 31:   PetscCall(PetscMalloc1(16, &ja));
 32:   if (rank == 0) {
 33:     ja[0]         = 1;
 34:     ja[1]         = 4;
 35:     ja[2]         = 0;
 36:     ja[3]         = 2;
 37:     ja[4]         = 5;
 38:     ja[5]         = 1;
 39:     ja[6]         = 3;
 40:     ja[7]         = 6;
 41:     ja[8]         = 2;
 42:     ja[9]         = 7;
 43:     ia[0]         = 0;
 44:     ia[1]         = 2;
 45:     ia[2]         = 5;
 46:     ia[3]         = 8;
 47:     ia[4]         = 10;
 48:     membershipKey = 0;
 49:   } else if (rank == 1) {
 50:     ja[0]         = 0;
 51:     ja[1]         = 5;
 52:     ja[2]         = 8;
 53:     ja[3]         = 1;
 54:     ja[4]         = 4;
 55:     ja[5]         = 6;
 56:     ja[6]         = 9;
 57:     ja[7]         = 2;
 58:     ja[8]         = 5;
 59:     ja[9]         = 7;
 60:     ja[10]        = 10;
 61:     ja[11]        = 3;
 62:     ja[12]        = 6;
 63:     ja[13]        = 11;
 64:     ia[0]         = 0;
 65:     ia[1]         = 3;
 66:     ia[2]         = 7;
 67:     ia[3]         = 11;
 68:     ia[4]         = 14;
 69:     membershipKey = 0;
 70:   } else if (rank == 2) {
 71:     ja[0]         = 4;
 72:     ja[1]         = 9;
 73:     ja[2]         = 12;
 74:     ja[3]         = 5;
 75:     ja[4]         = 8;
 76:     ja[5]         = 10;
 77:     ja[6]         = 13;
 78:     ja[7]         = 6;
 79:     ja[8]         = 9;
 80:     ja[9]         = 11;
 81:     ja[10]        = 14;
 82:     ja[11]        = 7;
 83:     ja[12]        = 10;
 84:     ja[13]        = 15;
 85:     ia[0]         = 0;
 86:     ia[1]         = 3;
 87:     ia[2]         = 7;
 88:     ia[3]         = 11;
 89:     ia[4]         = 14;
 90:     membershipKey = 1;
 91:   } else {
 92:     ja[0]         = 8;
 93:     ja[1]         = 13;
 94:     ja[2]         = 9;
 95:     ja[3]         = 12;
 96:     ja[4]         = 14;
 97:     ja[5]         = 10;
 98:     ja[6]         = 13;
 99:     ja[7]         = 15;
100:     ja[8]         = 11;
101:     ja[9]         = 14;
102:     ia[0]         = 0;
103:     ia[1]         = 2;
104:     ia[2]         = 5;
105:     ia[3]         = 8;
106:     ia[4]         = 10;
107:     membershipKey = 1;
108:   }
109:   PetscCall(MatCreateMPIAdj(comm, 4, 16, ia, ja, NULL, &A));
110:   PetscCall(MatView(A, PETSC_VIEWER_STDOUT_WORLD));
111:   /*
112:    Partition the graph of the matrix
113:   */
114:   PetscCall(MatPartitioningCreate(comm, &part));
115:   PetscCall(MatPartitioningSetAdjacency(part, A));
116:   PetscCall(MatPartitioningSetType(part, MATPARTITIONINGHIERARCH));
117:   PetscCall(MatPartitioningHierarchicalSetNcoarseparts(part, 2));
118:   PetscCall(MatPartitioningHierarchicalSetNfineparts(part, 2));
119:   PetscCall(MatPartitioningSetFromOptions(part));
120:   /* get new processor owner number of each vertex */
121:   PetscCall(MatPartitioningApply(part, &is));
122:   /* coarse parts */
123:   PetscCall(MatPartitioningHierarchicalGetCoarseparts(part, &coarseparts));
124:   PetscCall(ISView(coarseparts, PETSC_VIEWER_STDOUT_WORLD));
125:   /* fine parts */
126:   PetscCall(MatPartitioningHierarchicalGetFineparts(part, &fineparts));
127:   PetscCall(ISView(fineparts, PETSC_VIEWER_STDOUT_WORLD));
128:   /* partitioning */
129:   PetscCall(ISView(is, PETSC_VIEWER_STDOUT_WORLD));
130:   /* compute coming rows */
131:   PetscCall(ISBuildTwoSided(is, NULL, &isrows));
132:   PetscCall(ISView(isrows, PETSC_VIEWER_STDOUT_WORLD));
133:   /*create a sub-communicator */
134:   PetscCallMPI(MPI_Comm_split(comm, membershipKey, rank, &scomm));
135:   PetscCall(ISGetLocalSize(isrows, &isrows_localsize));
136:   PetscCall(PetscMalloc1(isrows_localsize, &indices_sc));
137:   PetscCall(ISGetIndices(isrows, &indices));
138:   PetscCall(PetscArraycpy(indices_sc, indices, isrows_localsize));
139:   PetscCall(ISRestoreIndices(isrows, &indices));
140:   PetscCall(ISDestroy(&is));
141:   PetscCall(ISDestroy(&coarseparts));
142:   PetscCall(ISDestroy(&fineparts));
143:   PetscCall(ISDestroy(&isrows));
144:   PetscCall(MatPartitioningDestroy(&part));
145:   /*create a sub-IS on the sub communicator  */
146:   PetscCall(ISCreateGeneral(scomm, isrows_localsize, indices_sc, PETSC_OWN_POINTER, &isrows_sc));
147:   PetscCall(MatConvert(A, MATMPIAIJ, MAT_INITIAL_MATRIX, &B));
148: #if 1
149:   PetscCall(MatView(B, PETSC_VIEWER_STDOUT_WORLD));
150: #endif
151:   /*increase overlap */
152:   PetscCall(MatIncreaseOverlapSplit(B, 1, &isrows_sc, 1));
153:   PetscCall(ISView(isrows_sc, NULL));
154:   PetscCall(ISDestroy(&isrows_sc));
155:   /*
156:     Free work space.  All PETSc objects should be destroyed when they
157:     are no longer needed.
158:   */
159:   PetscCall(MatDestroy(&A));
160:   PetscCall(MatDestroy(&B));
161:   PetscCall(PetscFinalize());
162:   return 0;
163: }