Actual source code: ex73.c
1: static char help[] = "Reads a PETSc matrix from a file partitions it\n\n";
3: /*
4: Include "petscmat.h" so that we can use matrices. Note that this file
5: automatically includes:
6: petscsys.h - base PETSc routines
7: petscvec.h - vectors
8: petscis.h - index sets
10: Example of usage:
11: mpiexec -n 3 ex73 -f <matfile> -mat_partitioning_type parmetis/scotch -viewer_binary_skip_info -nox
12: */
13: #include <petscmat.h>
15: int main(int argc, char **args)
16: {
17: MatType mtype = MATMPIAIJ; /* matrix format */
18: Mat A, B; /* matrix */
19: PetscViewer fd; /* viewer */
20: char file[PETSC_MAX_PATH_LEN]; /* input file name */
21: PetscBool flg, viewMats, viewIS, viewVecs, useND, noVecLoad = PETSC_FALSE;
22: PetscInt *nlocal, m, n;
23: PetscMPIInt rank, size;
24: MatPartitioning part;
25: IS is, isn;
26: Vec xin, xout;
27: VecScatter scat;
29: PetscFunctionBeginUser;
30: PetscCall(PetscInitialize(&argc, &args, NULL, help));
31: PetscCallMPI(MPI_Comm_size(PETSC_COMM_WORLD, &size));
32: PetscCallMPI(MPI_Comm_rank(PETSC_COMM_WORLD, &rank));
33: PetscCall(PetscOptionsHasName(NULL, NULL, "-view_mats", &viewMats));
34: PetscCall(PetscOptionsHasName(NULL, NULL, "-view_is", &viewIS));
35: PetscCall(PetscOptionsHasName(NULL, NULL, "-view_vecs", &viewVecs));
36: PetscCall(PetscOptionsHasName(NULL, NULL, "-use_nd", &useND));
37: PetscCall(PetscOptionsHasName(NULL, NULL, "-novec_load", &noVecLoad));
39: /*
40: Determine file from which we read the matrix
41: */
42: PetscCall(PetscOptionsGetString(NULL, NULL, "-f", file, sizeof(file), &flg));
44: /*
45: Open binary file. Note that we use FILE_MODE_READ to indicate
46: reading from this file.
47: */
48: PetscCall(PetscViewerBinaryOpen(PETSC_COMM_WORLD, file, FILE_MODE_READ, &fd));
50: /*
51: Load the matrix and vector; then destroy the viewer.
52: */
53: PetscCall(MatCreate(PETSC_COMM_WORLD, &A));
54: PetscCall(MatSetType(A, mtype));
55: PetscCall(MatLoad(A, fd));
56: if (!noVecLoad) {
57: PetscCall(VecCreate(PETSC_COMM_WORLD, &xin));
58: PetscCall(VecLoad(xin, fd));
59: } else {
60: PetscCall(MatCreateVecs(A, &xin, NULL));
61: PetscCall(VecSetRandom(xin, NULL));
62: }
63: PetscCall(PetscViewerDestroy(&fd));
64: if (viewMats) {
65: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Original matrix:\n"));
66: PetscCall(MatView(A, PETSC_VIEWER_DRAW_WORLD));
67: }
68: if (viewVecs) {
69: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Original vector:\n"));
70: PetscCall(VecView(xin, PETSC_VIEWER_STDOUT_WORLD));
71: }
73: /* Partition the graph of the matrix */
74: PetscCall(MatPartitioningCreate(PETSC_COMM_WORLD, &part));
75: PetscCall(MatPartitioningSetAdjacency(part, A));
76: PetscCall(MatPartitioningSetFromOptions(part));
78: /* get new processor owner number of each vertex */
79: if (useND) {
80: PetscCall(MatPartitioningApplyND(part, &is));
81: } else {
82: PetscCall(MatPartitioningApply(part, &is));
83: }
84: if (viewIS) {
85: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "IS1 - new processor ownership:\n"));
86: PetscCall(ISView(is, PETSC_VIEWER_STDOUT_WORLD));
87: }
89: /* get new global number of each old global number */
90: PetscCall(ISPartitioningToNumbering(is, &isn));
91: if (viewIS) {
92: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "IS2 - new global numbering:\n"));
93: PetscCall(ISView(isn, PETSC_VIEWER_STDOUT_WORLD));
94: }
96: /* get number of new vertices for each processor */
97: PetscCall(PetscMalloc1(size, &nlocal));
98: PetscCall(ISPartitioningCount(is, size, nlocal));
99: PetscCall(ISDestroy(&is));
101: /* get old global number of each new global number */
102: PetscCall(ISInvertPermutation(isn, useND ? PETSC_DECIDE : nlocal[rank], &is));
103: if (viewIS) {
104: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "IS3=inv(IS2) - old global number of each new global number:\n"));
105: PetscCall(ISView(is, PETSC_VIEWER_STDOUT_WORLD));
106: }
108: /* move the matrix rows to the new processes they have been assigned to by the permutation */
109: PetscCall(MatCreateSubMatrix(A, is, is, MAT_INITIAL_MATRIX, &B));
110: PetscCall(PetscFree(nlocal));
111: PetscCall(ISDestroy(&isn));
112: PetscCall(MatDestroy(&A));
113: PetscCall(MatPartitioningDestroy(&part));
114: if (viewMats) {
115: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Partitioned matrix:\n"));
116: PetscCall(MatView(B, PETSC_VIEWER_DRAW_WORLD));
117: }
119: /* move the vector rows to the new processes they have been assigned to */
120: PetscCall(MatGetLocalSize(B, &m, &n));
121: PetscCall(VecCreateFromOptions(PETSC_COMM_WORLD, NULL, 1, m, PETSC_DECIDE, &xout));
122: PetscCall(VecScatterCreate(xin, is, xout, NULL, &scat));
123: PetscCall(VecScatterBegin(scat, xin, xout, INSERT_VALUES, SCATTER_FORWARD));
124: PetscCall(VecScatterEnd(scat, xin, xout, INSERT_VALUES, SCATTER_FORWARD));
125: PetscCall(VecScatterDestroy(&scat));
126: if (viewVecs) {
127: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Mapped vector:\n"));
128: PetscCall(VecView(xout, PETSC_VIEWER_STDOUT_WORLD));
129: }
130: PetscCall(VecDestroy(&xout));
131: PetscCall(ISDestroy(&is));
133: {
134: PetscInt rstart, i, *nzd, *nzo, nzl, nzmax = 0, *ncols, nrow, j;
135: Mat J;
136: const PetscInt *cols;
137: const PetscScalar *vals;
138: PetscScalar *nvals;
140: PetscCall(MatGetOwnershipRange(B, &rstart, NULL));
141: PetscCall(PetscCalloc2(2 * m, &nzd, 2 * m, &nzo));
142: for (i = 0; i < m; i++) {
143: PetscCall(MatGetRow(B, i + rstart, &nzl, &cols, NULL));
144: for (j = 0; j < nzl; j++) {
145: if (cols[j] >= rstart && cols[j] < rstart + n) {
146: nzd[2 * i] += 2;
147: nzd[2 * i + 1] += 2;
148: } else {
149: nzo[2 * i] += 2;
150: nzo[2 * i + 1] += 2;
151: }
152: }
153: nzmax = PetscMax(nzmax, nzd[2 * i] + nzo[2 * i]);
154: PetscCall(MatRestoreRow(B, i + rstart, &nzl, &cols, NULL));
155: }
156: PetscCall(MatCreateAIJ(PETSC_COMM_WORLD, 2 * m, 2 * m, PETSC_DECIDE, PETSC_DECIDE, 0, nzd, 0, nzo, &J));
157: PetscCall(PetscInfo(0, "Created empty Jacobian matrix\n"));
158: PetscCall(PetscFree2(nzd, nzo));
159: PetscCall(PetscMalloc2(nzmax, &ncols, nzmax, &nvals));
160: PetscCall(PetscArrayzero(nvals, nzmax));
161: for (i = 0; i < m; i++) {
162: PetscCall(MatGetRow(B, i + rstart, &nzl, &cols, &vals));
163: for (j = 0; j < nzl; j++) {
164: ncols[2 * j] = 2 * cols[j];
165: ncols[2 * j + 1] = 2 * cols[j] + 1;
166: }
167: nrow = 2 * (i + rstart);
168: PetscCall(MatSetValues(J, 1, &nrow, 2 * nzl, ncols, nvals, INSERT_VALUES));
169: nrow = 2 * (i + rstart) + 1;
170: PetscCall(MatSetValues(J, 1, &nrow, 2 * nzl, ncols, nvals, INSERT_VALUES));
171: PetscCall(MatRestoreRow(B, i + rstart, &nzl, &cols, &vals));
172: }
173: PetscCall(MatAssemblyBegin(J, MAT_FINAL_ASSEMBLY));
174: PetscCall(MatAssemblyEnd(J, MAT_FINAL_ASSEMBLY));
175: if (viewMats) {
176: PetscCall(PetscPrintf(PETSC_COMM_WORLD, "Jacobian matrix nonzero structure:\n"));
177: PetscCall(MatView(J, PETSC_VIEWER_DRAW_WORLD));
178: }
179: PetscCall(MatDestroy(&J));
180: PetscCall(PetscFree2(ncols, nvals));
181: }
183: /*
184: Free work space. All PETSc objects should be destroyed when they
185: are no longer needed.
186: */
187: PetscCall(MatDestroy(&B));
188: PetscCall(VecDestroy(&xin));
189: PetscCall(PetscFinalize());
190: return 0;
191: }
193: /*TEST
195: test:
196: nsize: 3
197: requires: parmetis datafilespath !complex double !defined(PETSC_USE_64BIT_INDICES)
198: args: -nox -f ${DATAFILESPATH}/matrices/arco1 -mat_partitioning_type parmetis -viewer_binary_skip_info -novec_load
199: output_file: output/empty.out
201: test:
202: requires: parmetis !complex double !defined(PETSC_USE_64BIT_INDICES)
203: output_file: output/empty.out
204: suffix: parmetis_nd_32
205: nsize: 3
206: args: -nox -f ${wPETSC_DIR}/share/petsc/datafiles/matrices/spd-real-int32-float64 -mat_partitioning_type parmetis -viewer_binary_skip_info -use_nd -novec_load
208: test:
209: requires: parmetis !complex double defined(PETSC_USE_64BIT_INDICES)
210: output_file: output/empty.out
211: suffix: parmetis_nd_64
212: nsize: 3
213: args: -nox -f ${wPETSC_DIR}/share/petsc/datafiles/matrices/spd-real-int64-float64 -mat_partitioning_type parmetis -viewer_binary_skip_info -use_nd -novec_load
215: test:
216: requires: ptscotch !complex double !defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
217: output_file: output/empty.out
218: suffix: ptscotch_nd_32
219: nsize: 4
220: args: -nox -f ${wPETSC_DIR}/share/petsc/datafiles/matrices/spd-real-int32-float64 -mat_partitioning_type ptscotch -viewer_binary_skip_info -use_nd -novec_load
222: test:
223: requires: ptscotch !complex double defined(PETSC_USE_64BIT_INDICES) defined(PETSC_HAVE_SCOTCH_PARMETIS_V3_NODEND)
224: output_file: output/empty.out
225: suffix: ptscotch_nd_64
226: nsize: 4
227: args: -nox -f ${wPETSC_DIR}/share/petsc/datafiles/matrices/spd-real-int64-float64 -mat_partitioning_type ptscotch -viewer_binary_skip_info -use_nd -novec_load
229: TEST*/