Actual source code: aijmkl.c


  2: /*
  3:   Defines basic operations for the MATSEQAIJMKL matrix class.
  4:   This class is derived from the MATSEQAIJ class and retains the
  5:   compressed row storage (aka Yale sparse matrix format) but uses
  6:   sparse BLAS operations from the Intel Math Kernel Library (MKL)
  7:   wherever possible.
  8: */

 10: #include <../src/mat/impls/aij/seq/aij.h>
 11: #include <../src/mat/impls/aij/seq/aijmkl/aijmkl.h>
 12: #include <mkl_spblas.h>

 14: typedef struct {
 15:   PetscBool           no_SpMV2;  /* If PETSC_TRUE, then don't use the MKL SpMV2 inspector-executor routines. */
 16:   PetscBool           eager_inspection; /* If PETSC_TRUE, then call mkl_sparse_optimize() in MatDuplicate()/MatAssemblyEnd(). */
 17:   PetscBool           sparse_optimized; /* If PETSC_TRUE, then mkl_sparse_optimize() has been called. */
 18:   PetscObjectState    state;
 19: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
 20:   sparse_matrix_t     csrA; /* "Handle" used by SpMV2 inspector-executor routines. */
 21:   struct matrix_descr descr;
 22: #endif
 23: } Mat_SeqAIJMKL;

 25: extern PetscErrorCode MatAssemblyEnd_SeqAIJ(Mat,MatAssemblyType);

 27: PETSC_INTERN PetscErrorCode MatConvert_SeqAIJMKL_SeqAIJ(Mat A,MatType type,MatReuse reuse,Mat *newmat)
 28: {
 29:   /* This routine is only called to convert a MATAIJMKL to its base PETSc type, */
 30:   /* so we will ignore 'MatType type'. */
 32:   Mat            B       = *newmat;
 33: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
 34:   Mat_SeqAIJMKL  *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
 35: #endif

 38:   if (reuse == MAT_INITIAL_MATRIX) {
 39:     MatDuplicate(A,MAT_COPY_VALUES,&B);
 40:   }

 42:   /* Reset the original function pointers. */
 43:   B->ops->duplicate               = MatDuplicate_SeqAIJ;
 44:   B->ops->assemblyend             = MatAssemblyEnd_SeqAIJ;
 45:   B->ops->destroy                 = MatDestroy_SeqAIJ;
 46:   B->ops->mult                    = MatMult_SeqAIJ;
 47:   B->ops->multtranspose           = MatMultTranspose_SeqAIJ;
 48:   B->ops->multadd                 = MatMultAdd_SeqAIJ;
 49:   B->ops->multtransposeadd        = MatMultTransposeAdd_SeqAIJ;
 50:   B->ops->productsetfromoptions   = MatProductSetFromOptions_SeqAIJ;
 51:   B->ops->matmultsymbolic         = MatMatMultSymbolic_SeqAIJ_SeqAIJ;
 52:   B->ops->matmultnumeric          = MatMatMultNumeric_SeqAIJ_SeqAIJ;
 53:   B->ops->mattransposemultnumeric = MatMatTransposeMultNumeric_SeqAIJ_SeqAIJ;
 54:   B->ops->transposematmultnumeric = MatTransposeMatMultNumeric_SeqAIJ_SeqAIJ;
 55:   B->ops->ptapnumeric             = MatPtAPNumeric_SeqAIJ_SeqAIJ;

 57:   PetscObjectComposeFunction((PetscObject)B,"MatConvert_seqaijmkl_seqaij_C",NULL);

 59: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
 60:   /* Free everything in the Mat_SeqAIJMKL data structure. Currently, this
 61:    * simply involves destroying the MKL sparse matrix handle and then freeing
 62:    * the spptr pointer. */
 63:   if (reuse == MAT_INITIAL_MATRIX) aijmkl = (Mat_SeqAIJMKL*)B->spptr;

 65:   if (aijmkl->sparse_optimized) {
 66:     sparse_status_t stat;
 67:     stat = mkl_sparse_destroy(aijmkl->csrA);
 68:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to set hints/complete mkl_sparse_optimize()");
 69:   }
 70: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */
 71:   PetscFree(B->spptr);

 73:   /* Change the type of B to MATSEQAIJ. */
 74:   PetscObjectChangeTypeName((PetscObject)B, MATSEQAIJ);

 76:   *newmat = B;
 77:   return(0);
 78: }

 80: PetscErrorCode MatDestroy_SeqAIJMKL(Mat A)
 81: {
 83:   Mat_SeqAIJMKL  *aijmkl = (Mat_SeqAIJMKL*) A->spptr;


 87:   /* If MatHeaderMerge() was used, then this SeqAIJMKL matrix will not have an spptr pointer. */
 88:   if (aijmkl) {
 89:     /* Clean up everything in the Mat_SeqAIJMKL data structure, then free A->spptr. */
 90: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
 91:     if (aijmkl->sparse_optimized) {
 92:       sparse_status_t stat = SPARSE_STATUS_SUCCESS;
 93:       stat = mkl_sparse_destroy(aijmkl->csrA);
 94:       if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_destroy()");
 95:     }
 96: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */
 97:     PetscFree(A->spptr);
 98:   }

100:   /* Change the type of A back to SEQAIJ and use MatDestroy_SeqAIJ()
101:    * to destroy everything that remains. */
102:   PetscObjectChangeTypeName((PetscObject)A, MATSEQAIJ);
103:   /* Note that I don't call MatSetType().  I believe this is because that
104:    * is only to be called when *building* a matrix.  I could be wrong, but
105:    * that is how things work for the SuperLU matrix class. */
106:   MatDestroy_SeqAIJ(A);
107:   return(0);
108: }

110: /* MatSeqAIJMKL_create_mkl_handle(), if called with an AIJMKL matrix that has not had mkl_sparse_optimize() called for it,
111:  * creates an MKL sparse matrix handle from the AIJ arrays and calls mkl_sparse_optimize().
112:  * If called with an AIJMKL matrix for which aijmkl->sparse_optimized == PETSC_TRUE, then it destroys the old matrix
113:  * handle, creates a new one, and then calls mkl_sparse_optimize().
114:  * Although in normal MKL usage it is possible to have a valid matrix handle on which mkl_sparse_optimize() has not been
115:  * called, for AIJMKL the handle creation and optimization step always occur together, so we don't handle the case of
116:  * an unoptimized matrix handle here. */
117: PETSC_INTERN PetscErrorCode MatSeqAIJMKL_create_mkl_handle(Mat A)
118: {
119: #if !defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
120:   /* If the MKL library does not have mkl_sparse_optimize(), then this routine
121:    * does nothing. We make it callable anyway in this case because it cuts
122:    * down on littering the code with #ifdefs. */
124:   return(0);
125: #else
126:   Mat_SeqAIJ       *a = (Mat_SeqAIJ*)A->data;
127:   Mat_SeqAIJMKL    *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
128:   PetscInt         m,n;
129:   MatScalar        *aa;
130:   PetscInt         *aj,*ai;
131:   sparse_status_t  stat;
132:   PetscErrorCode   ierr;

135: #if !defined(PETSC_MKL_SPBLAS_DEPRECATED)
136:   /* For MKL versions that still support the old, non-inspector-executor interfaces versions, we simply exit here if the no_SpMV2
137:    * option has been specified. For versions that have deprecated the old interfaces (version 18, update 2 and later), we must
138:    * use the new inspector-executor interfaces, but we can still use the old, non-inspector-executor code by not calling
139:    * mkl_sparse_optimize() later. */
140:   if (aijmkl->no_SpMV2) return(0);
141: #endif

143:   if (aijmkl->sparse_optimized) {
144:     /* Matrix has been previously assembled and optimized. Must destroy old
145:      * matrix handle before running the optimization step again. */
146:     stat = mkl_sparse_destroy(aijmkl->csrA);
147:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_destroy()");
148:   }
149:   aijmkl->sparse_optimized = PETSC_FALSE;

151:   /* Now perform the SpMV2 setup and matrix optimization. */
152:   aijmkl->descr.type = SPARSE_MATRIX_TYPE_GENERAL;
153:   aijmkl->descr.mode = SPARSE_FILL_MODE_LOWER;
154:   aijmkl->descr.diag = SPARSE_DIAG_NON_UNIT;
155:   m = A->rmap->n;
156:   n = A->cmap->n;
157:   aj   = a->j;  /* aj[k] gives column index for element aa[k]. */
158:   aa   = a->a;  /* Nonzero elements stored row-by-row. */
159:   ai   = a->i;  /* ai[k] is the position in aa and aj where row k starts. */
160:   if (a->nz && aa && !A->structure_only) {
161:     /* Create a new, optimized sparse matrix handle only if the matrix has nonzero entries.
162:      * The MKL sparse-inspector executor routines don't like being passed an empty matrix. */
163:     stat = mkl_sparse_x_create_csr(&aijmkl->csrA,SPARSE_INDEX_BASE_ZERO,m,n,ai,ai+1,aj,aa);
164:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to create matrix handle, mkl_sparse_x_create_csr()");
165:     stat = mkl_sparse_set_mv_hint(aijmkl->csrA,SPARSE_OPERATION_NON_TRANSPOSE,aijmkl->descr,1000);
166:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_set_mv_hint()");
167:     stat = mkl_sparse_set_memory_hint(aijmkl->csrA,SPARSE_MEMORY_AGGRESSIVE);
168:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_set_memory_hint()");
169:     if (!aijmkl->no_SpMV2) {
170:       stat = mkl_sparse_optimize(aijmkl->csrA);
171:       if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to complete mkl_sparse_optimize()");
172:     }
173:     aijmkl->sparse_optimized = PETSC_TRUE;
174:     PetscObjectStateGet((PetscObject)A,&(aijmkl->state));
175:   } else {
176:     aijmkl->csrA = PETSC_NULL;
177:   }

179:   return(0);
180: #endif
181: }

183: #if defined(PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE)
184: /* Take an already created but empty matrix and set up the nonzero structure from an MKL sparse matrix handle. */
185: static PetscErrorCode MatSeqAIJMKL_setup_structure_from_mkl_handle(MPI_Comm comm,sparse_matrix_t csrA,PetscInt nrows,PetscInt ncols,Mat A)
186: {
187:   PetscErrorCode      ierr;
188:   sparse_status_t     stat;
189:   sparse_index_base_t indexing;
190:   PetscInt            m,n;
191:   PetscInt            *aj,*ai,*dummy;
192:   MatScalar           *aa;
193:   Mat_SeqAIJMKL       *aijmkl;

196:   if (csrA) {
197:     /* Note: Must pass in &dummy below since MKL can't accept NULL for this output array we don't actually want. */
198:     stat = mkl_sparse_x_export_csr(csrA,&indexing,&m,&n,&ai,&dummy,&aj,&aa);
199:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to complete mkl_sparse_x_export_csr()");
200:     if ((m != nrows) || (n != ncols)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Number of rows/columns does not match those from mkl_sparse_x_export_csr()");
201:   } else {
202:     aj = ai = PETSC_NULL;
203:     aa = PETSC_NULL;
204:   }

206:   MatSetType(A,MATSEQAIJ);
207:   MatSetSizes(A,PETSC_DECIDE,PETSC_DECIDE,nrows,ncols);
208:   /* We use MatSeqAIJSetPreallocationCSR() instead of MatCreateSeqAIJWithArrays() because we must copy the arrays exported
209:    * from MKL; MKL developers tell us that modifying the arrays may cause unexpected results when using the MKL handle, and
210:    * they will be destroyed when the MKL handle is destroyed.
211:    * (In the interest of reducing memory consumption in future, can we figure out good ways to deal with this?) */
212:   if (csrA) {
213:     MatSeqAIJSetPreallocationCSR(A,ai,aj,NULL);
214:   } else {
215:     /* Since MatSeqAIJSetPreallocationCSR does initial set up and assembly begin/end, we must do that ourselves here. */
216:     MatSetUp(A);
217:     MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
218:     MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);
219:   }

221:   /* We now have an assembled sequential AIJ matrix created from copies of the exported arrays from the MKL matrix handle.
222:    * Now turn it into a MATSEQAIJMKL. */
223:   MatConvert_SeqAIJ_SeqAIJMKL(A,MATSEQAIJMKL,MAT_INPLACE_MATRIX,&A);

225:   aijmkl = (Mat_SeqAIJMKL*) A->spptr;
226:   aijmkl->csrA = csrA;

228:   /* The below code duplicates much of what is in MatSeqAIJKL_create_mkl_handle(). I dislike this code duplication, but
229:    * MatSeqAIJMKL_create_mkl_handle() cannot be used because we don't need to create a handle -- we've already got one,
230:    * and just need to be able to run the MKL optimization step. */
231:   aijmkl->descr.type = SPARSE_MATRIX_TYPE_GENERAL;
232:   aijmkl->descr.mode = SPARSE_FILL_MODE_LOWER;
233:   aijmkl->descr.diag = SPARSE_DIAG_NON_UNIT;
234:   if (csrA) {
235:     stat = mkl_sparse_set_mv_hint(aijmkl->csrA,SPARSE_OPERATION_NON_TRANSPOSE,aijmkl->descr,1000);
236:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_set_mv_hint()");
237:     stat = mkl_sparse_set_memory_hint(aijmkl->csrA,SPARSE_MEMORY_AGGRESSIVE);
238:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_set_memory_hint()");
239:   }
240:   PetscObjectStateGet((PetscObject)A,&(aijmkl->state));
241:   return(0);
242: }
243: #endif /* PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE */

245: /* MatSeqAIJMKL_update_from_mkl_handle() updates the matrix values array from the contents of the associated MKL sparse matrix handle.
246:  * This is needed after mkl_sparse_sp2m() with SPARSE_STAGE_FINALIZE_MULT has been used to compute new values of the matrix in
247:  * MatMatMultNumeric(). */
248: #if defined(PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE)
249: static PetscErrorCode MatSeqAIJMKL_update_from_mkl_handle(Mat A)
250: {
251:   PetscInt            i;
252:   PetscInt            nrows,ncols;
253:   PetscInt            nz;
254:   PetscInt            *ai,*aj,*dummy;
255:   PetscScalar         *aa;
256:   PetscErrorCode      ierr;
257:   Mat_SeqAIJMKL       *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
258:   sparse_status_t     stat;
259:   sparse_index_base_t indexing;

262:   /* Exit immediately in case of the MKL matrix handle being NULL; this will be the case for empty matrices (zero rows or columns). */
263:   if (!aijmkl->csrA) return(0);

265:   /* Note: Must pass in &dummy below since MKL can't accept NULL for this output array we don't actually want. */
266:   stat = mkl_sparse_x_export_csr(aijmkl->csrA,&indexing,&nrows,&ncols,&ai,&dummy,&aj,&aa);
267:   if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to complete mkl_sparse_x_export_csr()");

269:   /* We can't just do a copy from the arrays exported by MKL to those used for the PETSc AIJ storage, because the MKL and PETSc
270:    * representations differ in small ways (e.g., more explicit nonzeros per row due to preallocation). */
271:   for (i=0; i<nrows; i++) {
272:     nz = ai[i+1] - ai[i];
273:     MatSetValues_SeqAIJ(A, 1, &i, nz, aj+ai[i], aa+ai[i], INSERT_VALUES);
274:   }

276:   MatAssemblyBegin(A,MAT_FINAL_ASSEMBLY);
277:   MatAssemblyEnd(A,MAT_FINAL_ASSEMBLY);

279:   PetscObjectStateGet((PetscObject)A,&(aijmkl->state));
280:   /* At this point our matrix has a valid MKL handle, the contents of which match the PETSc AIJ representation.
281:    * The MKL handle has *not* had mkl_sparse_optimize() called on it, though -- the MKL developers have confirmed
282:    * that the matrix inspection/optimization step is not performed when matrix-matrix multiplication is finalized. */
283:   aijmkl->sparse_optimized = PETSC_FALSE;
284:   return(0);
285: }
286: #endif /* PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE */

288: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
289: PETSC_INTERN PetscErrorCode MatSeqAIJMKL_view_mkl_handle(Mat A,PetscViewer viewer)
290: {
291:   PetscInt            i,j,k;
292:   PetscInt            nrows,ncols;
293:   PetscInt            nz;
294:   PetscInt            *ai,*aj,*dummy;
295:   PetscScalar         *aa;
296:   PetscErrorCode      ierr;
297:   Mat_SeqAIJMKL       *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
298:   sparse_status_t     stat;
299:   sparse_index_base_t indexing;

302:   PetscViewerASCIIPrintf(viewer,"Contents of MKL sparse matrix handle for MATSEQAIJMKL object:\n");

304:   /* Exit immediately in case of the MKL matrix handle being NULL; this will be the case for empty matrices (zero rows or columns). */
305:   if (!aijmkl->csrA) {
306:     PetscViewerASCIIPrintf(viewer,"MKL matrix handle is NULL\n");
307:     return(0);
308:   }

310:   /* Note: Must pass in &dummy below since MKL can't accept NULL for this output array we don't actually want. */
311:   stat = mkl_sparse_x_export_csr(aijmkl->csrA,&indexing,&nrows,&ncols,&ai,&dummy,&aj,&aa);
312:   if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to complete mkl_sparse_x_export_csr()");

314:   k = 0;
315:   for (i=0; i<nrows; i++) {
316:     PetscViewerASCIIPrintf(viewer,"row %D: ",i);
317:     nz = ai[i+1] - ai[i];
318:     for (j=0; j<nz; j++) {
319:       if (aa) {
320:         PetscViewerASCIIPrintf(viewer,"(%D, %g)  ",aj[k],PetscRealPart(aa[k]));
321:       } else {
322:         PetscViewerASCIIPrintf(viewer,"(%D, NULL)",aj[k]);
323:       }
324:       k++;
325:     }
326:     PetscViewerASCIIPrintf(viewer,"\n");
327:   }
328:   return(0);
329: }
330: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */

332: PetscErrorCode MatDuplicate_SeqAIJMKL(Mat A, MatDuplicateOption op, Mat *M)
333: {
335:   Mat_SeqAIJMKL  *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
336:   Mat_SeqAIJMKL  *aijmkl_dest;

339:   MatDuplicate_SeqAIJ(A,op,M);
340:   aijmkl_dest = (Mat_SeqAIJMKL*)(*M)->spptr;
341:   PetscArraycpy(aijmkl_dest,aijmkl,1);
342:   aijmkl_dest->sparse_optimized = PETSC_FALSE;
343:   if (aijmkl->eager_inspection) {
344:     MatSeqAIJMKL_create_mkl_handle(A);
345:   }
346:   return(0);
347: }

349: PetscErrorCode MatAssemblyEnd_SeqAIJMKL(Mat A, MatAssemblyType mode)
350: {
351:   PetscErrorCode  ierr;
352:   Mat_SeqAIJ      *a = (Mat_SeqAIJ*)A->data;
353:   Mat_SeqAIJMKL   *aijmkl;

356:   if (mode == MAT_FLUSH_ASSEMBLY) return(0);

358:   /* Since a MATSEQAIJMKL matrix is really just a MATSEQAIJ with some
359:    * extra information and some different methods, call the AssemblyEnd
360:    * routine for a MATSEQAIJ.
361:    * I'm not sure if this is the best way to do this, but it avoids
362:    * a lot of code duplication. */
363:   a->inode.use = PETSC_FALSE;  /* Must disable: otherwise the MKL routines won't get used. */
364:   MatAssemblyEnd_SeqAIJ(A, mode);

366:   /* If the user has requested "eager" inspection, create the optimized MKL sparse handle (if needed; the function checks).
367:    * (The default is to do "lazy" inspection, deferring this until something like MatMult() is called.) */
368:   aijmkl = (Mat_SeqAIJMKL*)A->spptr;
369:   if (aijmkl->eager_inspection) {
370:     MatSeqAIJMKL_create_mkl_handle(A);
371:   }

373:   return(0);
374: }

376: #if !defined(PETSC_MKL_SPBLAS_DEPRECATED)
377: PetscErrorCode MatMult_SeqAIJMKL(Mat A,Vec xx,Vec yy)
378: {
379:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
380:   const PetscScalar *x;
381:   PetscScalar       *y;
382:   const MatScalar   *aa;
383:   PetscErrorCode    ierr;
384:   PetscInt          m = A->rmap->n;
385:   PetscInt          n = A->cmap->n;
386:   PetscScalar       alpha = 1.0;
387:   PetscScalar       beta = 0.0;
388:   const PetscInt    *aj,*ai;
389:   char              matdescra[6];

391:   /* Variables not in MatMult_SeqAIJ. */
392:   char transa = 'n';  /* Used to indicate to MKL that we are not computing the transpose product. */

395:   matdescra[0] = 'g';  /* Indicates to MKL that we using a general CSR matrix. */
396:   matdescra[3] = 'c';  /* Indicates to MKL that we use C-style (0-based) indexing. */
397:   VecGetArrayRead(xx,&x);
398:   VecGetArray(yy,&y);
399:   aj   = a->j;  /* aj[k] gives column index for element aa[k]. */
400:   aa   = a->a;  /* Nonzero elements stored row-by-row. */
401:   ai   = a->i;  /* ai[k] is the position in aa and aj where row k starts. */

403:   /* Call MKL sparse BLAS routine to do the MatMult. */
404:   mkl_xcsrmv(&transa,&m,&n,&alpha,matdescra,aa,aj,ai,ai+1,x,&beta,y);

406:   PetscLogFlops(2.0*a->nz - a->nonzerorowcnt);
407:   VecRestoreArrayRead(xx,&x);
408:   VecRestoreArray(yy,&y);
409:   return(0);
410: }
411: #endif

413: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
414: PetscErrorCode MatMult_SeqAIJMKL_SpMV2(Mat A,Vec xx,Vec yy)
415: {
416:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
417:   Mat_SeqAIJMKL     *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
418:   const PetscScalar *x;
419:   PetscScalar       *y;
420:   PetscErrorCode    ierr;
421:   sparse_status_t   stat = SPARSE_STATUS_SUCCESS;
422:   PetscObjectState  state;


426:   /* If there are no nonzero entries, zero yy and return immediately. */
427:   if (!a->nz) {
428:     PetscInt i;
429:     PetscInt m=A->rmap->n;
430:     VecGetArray(yy,&y);
431:     for (i=0; i<m; i++) {
432:       y[i] = 0.0;
433:     }
434:     VecRestoreArray(yy,&y);
435:     return(0);
436:   }

438:   VecGetArrayRead(xx,&x);
439:   VecGetArray(yy,&y);

441:   /* In some cases, we get to this point without mkl_sparse_optimize() having been called, so we check and then call
442:    * it if needed. Eventually, when everything in PETSc is properly updating the matrix state, we should probably
443:    * take a "lazy" approach to creation/updating of the MKL matrix handle and plan to always do it here (when needed). */
444:   PetscObjectStateGet((PetscObject)A,&state);
445:   if (!aijmkl->sparse_optimized || aijmkl->state != state) {
446:     MatSeqAIJMKL_create_mkl_handle(A);
447:   }

449:   /* Call MKL SpMV2 executor routine to do the MatMult. */
450:   stat = mkl_sparse_x_mv(SPARSE_OPERATION_NON_TRANSPOSE,1.0,aijmkl->csrA,aijmkl->descr,x,0.0,y);
451:   if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_x_mv()");

453:   PetscLogFlops(2.0*a->nz - a->nonzerorowcnt);
454:   VecRestoreArrayRead(xx,&x);
455:   VecRestoreArray(yy,&y);
456:   return(0);
457: }
458: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */

460: #if !defined(PETSC_MKL_SPBLAS_DEPRECATED)
461: PetscErrorCode MatMultTranspose_SeqAIJMKL(Mat A,Vec xx,Vec yy)
462: {
463:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
464:   const PetscScalar *x;
465:   PetscScalar       *y;
466:   const MatScalar   *aa;
467:   PetscErrorCode    ierr;
468:   PetscInt          m = A->rmap->n;
469:   PetscInt          n = A->cmap->n;
470:   PetscScalar       alpha = 1.0;
471:   PetscScalar       beta = 0.0;
472:   const PetscInt    *aj,*ai;
473:   char              matdescra[6];

475:   /* Variables not in MatMultTranspose_SeqAIJ. */
476:   char transa = 't';  /* Used to indicate to MKL that we are computing the transpose product. */

479:   matdescra[0] = 'g';  /* Indicates to MKL that we using a general CSR matrix. */
480:   matdescra[3] = 'c';  /* Indicates to MKL that we use C-style (0-based) indexing. */
481:   VecGetArrayRead(xx,&x);
482:   VecGetArray(yy,&y);
483:   aj   = a->j;  /* aj[k] gives column index for element aa[k]. */
484:   aa   = a->a;  /* Nonzero elements stored row-by-row. */
485:   ai   = a->i;  /* ai[k] is the position in aa and aj where row k starts. */

487:   /* Call MKL sparse BLAS routine to do the MatMult. */
488:   mkl_xcsrmv(&transa,&m,&n,&alpha,matdescra,aa,aj,ai,ai+1,x,&beta,y);

490:   PetscLogFlops(2.0*a->nz - a->nonzerorowcnt);
491:   VecRestoreArrayRead(xx,&x);
492:   VecRestoreArray(yy,&y);
493:   return(0);
494: }
495: #endif

497: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
498: PetscErrorCode MatMultTranspose_SeqAIJMKL_SpMV2(Mat A,Vec xx,Vec yy)
499: {
500:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
501:   Mat_SeqAIJMKL     *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
502:   const PetscScalar *x;
503:   PetscScalar       *y;
504:   PetscErrorCode    ierr;
505:   sparse_status_t   stat;
506:   PetscObjectState  state;


510:   /* If there are no nonzero entries, zero yy and return immediately. */
511:   if (!a->nz) {
512:     PetscInt i;
513:     PetscInt n=A->cmap->n;
514:     VecGetArray(yy,&y);
515:     for (i=0; i<n; i++) {
516:       y[i] = 0.0;
517:     }
518:     VecRestoreArray(yy,&y);
519:     return(0);
520:   }

522:   VecGetArrayRead(xx,&x);
523:   VecGetArray(yy,&y);

525:   /* In some cases, we get to this point without mkl_sparse_optimize() having been called, so we check and then call
526:    * it if needed. Eventually, when everything in PETSc is properly updating the matrix state, we should probably
527:    * take a "lazy" approach to creation/updating of the MKL matrix handle and plan to always do it here (when needed). */
528:   PetscObjectStateGet((PetscObject)A,&state);
529:   if (!aijmkl->sparse_optimized || aijmkl->state != state) {
530:     MatSeqAIJMKL_create_mkl_handle(A);
531:   }

533:   /* Call MKL SpMV2 executor routine to do the MatMultTranspose. */
534:   stat = mkl_sparse_x_mv(SPARSE_OPERATION_TRANSPOSE,1.0,aijmkl->csrA,aijmkl->descr,x,0.0,y);
535:   if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_x_mv()");

537:   PetscLogFlops(2.0*a->nz - a->nonzerorowcnt);
538:   VecRestoreArrayRead(xx,&x);
539:   VecRestoreArray(yy,&y);
540:   return(0);
541: }
542: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */

544: #if !defined(PETSC_MKL_SPBLAS_DEPRECATED)
545: PetscErrorCode MatMultAdd_SeqAIJMKL(Mat A,Vec xx,Vec yy,Vec zz)
546: {
547:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
548:   const PetscScalar *x;
549:   PetscScalar       *y,*z;
550:   const MatScalar   *aa;
551:   PetscErrorCode    ierr;
552:   PetscInt          m = A->rmap->n;
553:   PetscInt          n = A->cmap->n;
554:   const PetscInt    *aj,*ai;
555:   PetscInt          i;

557:   /* Variables not in MatMultAdd_SeqAIJ. */
558:   char              transa = 'n';  /* Used to indicate to MKL that we are not computing the transpose product. */
559:   PetscScalar       alpha = 1.0;
560:   PetscScalar       beta;
561:   char              matdescra[6];

564:   matdescra[0] = 'g';  /* Indicates to MKL that we using a general CSR matrix. */
565:   matdescra[3] = 'c';  /* Indicates to MKL that we use C-style (0-based) indexing. */

567:   VecGetArrayRead(xx,&x);
568:   VecGetArrayPair(yy,zz,&y,&z);
569:   aj   = a->j;  /* aj[k] gives column index for element aa[k]. */
570:   aa   = a->a;  /* Nonzero elements stored row-by-row. */
571:   ai   = a->i;  /* ai[k] is the position in aa and aj where row k starts. */

573:   /* Call MKL sparse BLAS routine to do the MatMult. */
574:   if (zz == yy) {
575:     /* If zz and yy are the same vector, we can use MKL's mkl_xcsrmv(), which calculates y = alpha*A*x + beta*y. */
576:     beta = 1.0;
577:     mkl_xcsrmv(&transa,&m,&n,&alpha,matdescra,aa,aj,ai,ai+1,x,&beta,z);
578:   } else {
579:     /* zz and yy are different vectors, so call MKL's mkl_xcsrmv() with beta=0, then add the result to z.
580:      * MKL sparse BLAS does not have a MatMultAdd equivalent. */
581:     beta = 0.0;
582:     mkl_xcsrmv(&transa,&m,&n,&alpha,matdescra,aa,aj,ai,ai+1,x,&beta,z);
583:     for (i=0; i<m; i++) {
584:       z[i] += y[i];
585:     }
586:   }

588:   PetscLogFlops(2.0*a->nz);
589:   VecRestoreArrayRead(xx,&x);
590:   VecRestoreArrayPair(yy,zz,&y,&z);
591:   return(0);
592: }
593: #endif

595: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
596: PetscErrorCode MatMultAdd_SeqAIJMKL_SpMV2(Mat A,Vec xx,Vec yy,Vec zz)
597: {
598:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
599:   Mat_SeqAIJMKL     *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
600:   const PetscScalar *x;
601:   PetscScalar       *y,*z;
602:   PetscErrorCode    ierr;
603:   PetscInt          m = A->rmap->n;
604:   PetscInt          i;

606:   /* Variables not in MatMultAdd_SeqAIJ. */
607:   sparse_status_t   stat = SPARSE_STATUS_SUCCESS;
608:   PetscObjectState  state;


612:   /* If there are no nonzero entries, set zz = yy and return immediately. */
613:   if (!a->nz) {
614:     PetscInt i;
615:     VecGetArrayPair(yy,zz,&y,&z);
616:     for (i=0; i<m; i++) {
617:       z[i] = y[i];
618:     }
619:     VecRestoreArrayPair(yy,zz,&y,&z);
620:     return(0);
621:   }

623:   VecGetArrayRead(xx,&x);
624:   VecGetArrayPair(yy,zz,&y,&z);

626:   /* In some cases, we get to this point without mkl_sparse_optimize() having been called, so we check and then call
627:    * it if needed. Eventually, when everything in PETSc is properly updating the matrix state, we should probably
628:    * take a "lazy" approach to creation/updating of the MKL matrix handle and plan to always do it here (when needed). */
629:   PetscObjectStateGet((PetscObject)A,&state);
630:   if (!aijmkl->sparse_optimized || aijmkl->state != state) {
631:     MatSeqAIJMKL_create_mkl_handle(A);
632:   }

634:   /* Call MKL sparse BLAS routine to do the MatMult. */
635:   if (zz == yy) {
636:     /* If zz and yy are the same vector, we can use mkl_sparse_x_mv, which calculates y = alpha*A*x + beta*y,
637:      * with alpha and beta both set to 1.0. */
638:     stat = mkl_sparse_x_mv(SPARSE_OPERATION_NON_TRANSPOSE,1.0,aijmkl->csrA,aijmkl->descr,x,1.0,z);
639:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_x_mv()");
640:   } else {
641:     /* zz and yy are different vectors, so we call mkl_sparse_x_mv with alpha=1.0 and beta=0.0, and then
642:      * we add the contents of vector yy to the result; MKL sparse BLAS does not have a MatMultAdd equivalent. */
643:     stat = mkl_sparse_x_mv(SPARSE_OPERATION_NON_TRANSPOSE,1.0,aijmkl->csrA,aijmkl->descr,x,0.0,z);
644:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_x_mv()");
645:     for (i=0; i<m; i++) {
646:       z[i] += y[i];
647:     }
648:   }

650:   PetscLogFlops(2.0*a->nz);
651:   VecRestoreArrayRead(xx,&x);
652:   VecRestoreArrayPair(yy,zz,&y,&z);
653:   return(0);
654: }
655: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */

657: #if !defined(PETSC_MKL_SPBLAS_DEPRECATED)
658: PetscErrorCode MatMultTransposeAdd_SeqAIJMKL(Mat A,Vec xx,Vec yy,Vec zz)
659: {
660:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
661:   const PetscScalar *x;
662:   PetscScalar       *y,*z;
663:   const MatScalar   *aa;
664:   PetscErrorCode    ierr;
665:   PetscInt          m = A->rmap->n;
666:   PetscInt          n = A->cmap->n;
667:   const PetscInt    *aj,*ai;
668:   PetscInt          i;

670:   /* Variables not in MatMultTransposeAdd_SeqAIJ. */
671:   char transa = 't';  /* Used to indicate to MKL that we are computing the transpose product. */
672:   PetscScalar       alpha = 1.0;
673:   PetscScalar       beta;
674:   char              matdescra[6];

677:   matdescra[0] = 'g';  /* Indicates to MKL that we using a general CSR matrix. */
678:   matdescra[3] = 'c';  /* Indicates to MKL that we use C-style (0-based) indexing. */

680:   VecGetArrayRead(xx,&x);
681:   VecGetArrayPair(yy,zz,&y,&z);
682:   aj   = a->j;  /* aj[k] gives column index for element aa[k]. */
683:   aa   = a->a;  /* Nonzero elements stored row-by-row. */
684:   ai   = a->i;  /* ai[k] is the position in aa and aj where row k starts. */

686:   /* Call MKL sparse BLAS routine to do the MatMult. */
687:   if (zz == yy) {
688:     /* If zz and yy are the same vector, we can use MKL's mkl_xcsrmv(), which calculates y = alpha*A*x + beta*y. */
689:     beta = 1.0;
690:     mkl_xcsrmv(&transa,&m,&n,&alpha,matdescra,aa,aj,ai,ai+1,x,&beta,z);
691:   } else {
692:     /* zz and yy are different vectors, so call MKL's mkl_xcsrmv() with beta=0, then add the result to z.
693:      * MKL sparse BLAS does not have a MatMultAdd equivalent. */
694:     beta = 0.0;
695:     mkl_xcsrmv(&transa,&m,&n,&alpha,matdescra,aa,aj,ai,ai+1,x,&beta,z);
696:     for (i=0; i<n; i++) {
697:       z[i] += y[i];
698:     }
699:   }

701:   PetscLogFlops(2.0*a->nz);
702:   VecRestoreArrayRead(xx,&x);
703:   VecRestoreArrayPair(yy,zz,&y,&z);
704:   return(0);
705: }
706: #endif

708: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
709: PetscErrorCode MatMultTransposeAdd_SeqAIJMKL_SpMV2(Mat A,Vec xx,Vec yy,Vec zz)
710: {
711:   Mat_SeqAIJ        *a = (Mat_SeqAIJ*)A->data;
712:   Mat_SeqAIJMKL     *aijmkl = (Mat_SeqAIJMKL*)A->spptr;
713:   const PetscScalar *x;
714:   PetscScalar       *y,*z;
715:   PetscErrorCode    ierr;
716:   PetscInt          n = A->cmap->n;
717:   PetscInt          i;
718:   PetscObjectState  state;

720:   /* Variables not in MatMultTransposeAdd_SeqAIJ. */
721:   sparse_status_t stat = SPARSE_STATUS_SUCCESS;


725:   /* If there are no nonzero entries, set zz = yy and return immediately. */
726:   if (!a->nz) {
727:     PetscInt i;
728:     VecGetArrayPair(yy,zz,&y,&z);
729:     for (i=0; i<n; i++) {
730:       z[i] = y[i];
731:     }
732:     VecRestoreArrayPair(yy,zz,&y,&z);
733:     return(0);
734:   }

736:   VecGetArrayRead(xx,&x);
737:   VecGetArrayPair(yy,zz,&y,&z);

739:   /* In some cases, we get to this point without mkl_sparse_optimize() having been called, so we check and then call
740:    * it if needed. Eventually, when everything in PETSc is properly updating the matrix state, we should probably
741:    * take a "lazy" approach to creation/updating of the MKL matrix handle and plan to always do it here (when needed). */
742:   PetscObjectStateGet((PetscObject)A,&state);
743:   if (!aijmkl->sparse_optimized || aijmkl->state != state) {
744:     MatSeqAIJMKL_create_mkl_handle(A);
745:   }

747:   /* Call MKL sparse BLAS routine to do the MatMult. */
748:   if (zz == yy) {
749:     /* If zz and yy are the same vector, we can use mkl_sparse_x_mv, which calculates y = alpha*A*x + beta*y,
750:      * with alpha and beta both set to 1.0. */
751:     stat = mkl_sparse_x_mv(SPARSE_OPERATION_TRANSPOSE,1.0,aijmkl->csrA,aijmkl->descr,x,1.0,z);
752:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_x_mv()");
753:   } else {
754:     /* zz and yy are different vectors, so we call mkl_sparse_x_mv with alpha=1.0 and beta=0.0, and then
755:      * we add the contents of vector yy to the result; MKL sparse BLAS does not have a MatMultAdd equivalent. */
756:     stat = mkl_sparse_x_mv(SPARSE_OPERATION_TRANSPOSE,1.0,aijmkl->csrA,aijmkl->descr,x,0.0,z);
757:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: failure in mkl_sparse_x_mv()");
758:     for (i=0; i<n; i++) {
759:       z[i] += y[i];
760:     }
761:   }

763:   PetscLogFlops(2.0*a->nz);
764:   VecRestoreArrayRead(xx,&x);
765:   VecRestoreArrayPair(yy,zz,&y,&z);
766:   return(0);
767: }
768: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */

770: /* -------------------------- MatProduct code -------------------------- */
771: #if defined(PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE)
772: static PetscErrorCode MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL_Private(Mat A,const sparse_operation_t transA,Mat B,const sparse_operation_t transB,Mat C)
773: {
774:   Mat_SeqAIJMKL       *a = (Mat_SeqAIJMKL*)A->spptr,*b = (Mat_SeqAIJMKL*)B->spptr;
775:   sparse_matrix_t     csrA,csrB,csrC;
776:   PetscInt            nrows,ncols;
777:   PetscErrorCode      ierr;
778:   sparse_status_t     stat = SPARSE_STATUS_SUCCESS;
779:   struct matrix_descr descr_type_gen;
780:   PetscObjectState    state;

783:   /* Determine the number of rows and columns that the result matrix C will have. We have to do this ourselves because MKL does
784:    * not handle sparse matrices with zero rows or columns. */
785:   if (transA == SPARSE_OPERATION_NON_TRANSPOSE) nrows = A->rmap->N;
786:   else nrows = A->cmap->N;
787:   if (transB == SPARSE_OPERATION_NON_TRANSPOSE) ncols = B->cmap->N;
788:   else ncols = B->rmap->N;

790:   PetscObjectStateGet((PetscObject)A,&state);
791:   if (!a->sparse_optimized || a->state != state) {
792:     MatSeqAIJMKL_create_mkl_handle(A);
793:   }
794:   PetscObjectStateGet((PetscObject)B,&state);
795:   if (!b->sparse_optimized || b->state != state) {
796:     MatSeqAIJMKL_create_mkl_handle(B);
797:   }
798:   csrA = a->csrA;
799:   csrB = b->csrA;
800:   descr_type_gen.type = SPARSE_MATRIX_TYPE_GENERAL;

802:   if (csrA && csrB) {
803:     stat = mkl_sparse_sp2m(transA,descr_type_gen,csrA,transB,descr_type_gen,csrB,SPARSE_STAGE_FULL_MULT_NO_VAL,&csrC);
804:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to complete symbolic stage of sparse matrix-matrix multiply in mkl_sparse_sp2m()");
805:   } else {
806:     csrC = PETSC_NULL;
807:   }

809:   MatSeqAIJMKL_setup_structure_from_mkl_handle(PETSC_COMM_SELF,csrC,nrows,ncols,C);

811:   return(0);
812: }

814: PetscErrorCode MatMatMultNumeric_SeqAIJMKL_SeqAIJMKL_Private(Mat A,const sparse_operation_t transA,Mat B,const sparse_operation_t transB,Mat C)
815: {
816:   Mat_SeqAIJMKL       *a = (Mat_SeqAIJMKL*)A->spptr,*b = (Mat_SeqAIJMKL*)B->spptr,*c = (Mat_SeqAIJMKL*)C->spptr;
817:   sparse_matrix_t     csrA, csrB, csrC;
818:   PetscErrorCode      ierr;
819:   sparse_status_t     stat = SPARSE_STATUS_SUCCESS;
820:   struct matrix_descr descr_type_gen;
821:   PetscObjectState    state;

824:   PetscObjectStateGet((PetscObject)A,&state);
825:   if (!a->sparse_optimized || a->state != state) {
826:     MatSeqAIJMKL_create_mkl_handle(A);
827:   }
828:   PetscObjectStateGet((PetscObject)B,&state);
829:   if (!b->sparse_optimized || b->state != state) {
830:     MatSeqAIJMKL_create_mkl_handle(B);
831:   }
832:   csrA = a->csrA;
833:   csrB = b->csrA;
834:   csrC = c->csrA;
835:   descr_type_gen.type = SPARSE_MATRIX_TYPE_GENERAL;

837:   if (csrA && csrB) {
838:     stat = mkl_sparse_sp2m(transA,descr_type_gen,csrA,transB,descr_type_gen,csrB,SPARSE_STAGE_FINALIZE_MULT,&csrC);
839:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to complete numerical stage of sparse matrix-matrix multiply in mkl_sparse_sp2m()");
840:   } else {
841:     csrC = PETSC_NULL;
842:   }

844:   /* Have to update the PETSc AIJ representation for matrix C from contents of MKL handle. */
845:   MatSeqAIJMKL_update_from_mkl_handle(C);

847:   return(0);
848: }

850: PetscErrorCode MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL(Mat A,Mat B,PetscReal fill,Mat C)
851: {

855:   MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL_Private(A,SPARSE_OPERATION_NON_TRANSPOSE,B,SPARSE_OPERATION_NON_TRANSPOSE,C);
856:   return(0);
857: }

859: PetscErrorCode MatMatMultNumeric_SeqAIJMKL_SeqAIJMKL(Mat A,Mat B,Mat C)
860: {

864:   MatMatMultNumeric_SeqAIJMKL_SeqAIJMKL_Private(A,SPARSE_OPERATION_NON_TRANSPOSE,B,SPARSE_OPERATION_NON_TRANSPOSE,C);
865:   return(0);
866: }

868: PetscErrorCode MatTransposeMatMultNumeric_SeqAIJMKL_SeqAIJMKL(Mat A,Mat B,Mat C)
869: {

873:   MatMatMultNumeric_SeqAIJMKL_SeqAIJMKL_Private(A,SPARSE_OPERATION_TRANSPOSE,B,SPARSE_OPERATION_NON_TRANSPOSE,C);
874:   return(0);
875: }

877: PetscErrorCode MatTransposeMatMultSymbolic_SeqAIJMKL_SeqAIJMKL(Mat A,Mat B,PetscReal fill,Mat C)
878: {

882:   MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL_Private(A,SPARSE_OPERATION_TRANSPOSE,B,SPARSE_OPERATION_NON_TRANSPOSE,C);
883:   return(0);
884: }

886: PetscErrorCode MatMatTransposeMultSymbolic_SeqAIJMKL_SeqAIJMKL(Mat A,Mat B,PetscReal fill,Mat C)
887: {

891:   MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL_Private(A,SPARSE_OPERATION_NON_TRANSPOSE,B,SPARSE_OPERATION_TRANSPOSE,C);
892:   return(0);
893: }

895: PetscErrorCode MatMatTransposeMultNumeric_SeqAIJMKL_SeqAIJMKL(Mat A,Mat B,Mat C)
896: {

900:   MatMatMultNumeric_SeqAIJMKL_SeqAIJMKL_Private(A,SPARSE_OPERATION_NON_TRANSPOSE,B,SPARSE_OPERATION_TRANSPOSE,C);
901:   return(0);
902: }

904: static PetscErrorCode MatProductNumeric_AtB_SeqAIJMKL_SeqAIJMKL(Mat C)
905: {
907:   Mat_Product    *product = C->product;
908:   Mat            A = product->A,B = product->B;

911:   MatTransposeMatMultNumeric_SeqAIJMKL_SeqAIJMKL(A,B,C);
912:   return(0);
913: }

915: static PetscErrorCode MatProductSymbolic_AtB_SeqAIJMKL_SeqAIJMKL(Mat C)
916: {
918:   Mat_Product    *product = C->product;
919:   Mat            A = product->A,B = product->B;
920:   PetscReal      fill = product->fill;

923:   MatTransposeMatMultSymbolic_SeqAIJMKL_SeqAIJMKL(A,B,fill,C);
924:   C->ops->productnumeric = MatProductNumeric_AtB_SeqAIJMKL_SeqAIJMKL;
925:   return(0);
926: }

928: PetscErrorCode MatPtAPNumeric_SeqAIJMKL_SeqAIJMKL_SymmetricReal(Mat A,Mat P,Mat C)
929: {
930:   Mat                 Ct;
931:   Vec                 zeros;
932:   Mat_SeqAIJMKL       *a = (Mat_SeqAIJMKL*)A->spptr,*p = (Mat_SeqAIJMKL*)P->spptr,*c = (Mat_SeqAIJMKL*)C->spptr;
933:   sparse_matrix_t     csrA, csrP, csrC;
934:   PetscBool           set, flag;
935:   sparse_status_t     stat = SPARSE_STATUS_SUCCESS;
936:   struct matrix_descr descr_type_sym;
937:   PetscObjectState    state;
938:   PetscErrorCode      ierr;

941:   MatIsSymmetricKnown(A,&set,&flag);
942:   if (!set || (set && !flag)) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_PLIB,"MatPtAPNumeric_SeqAIJMKL_SeqAIJMKL_SymmetricReal() called on matrix A not marked as symmetric");

944:   PetscObjectStateGet((PetscObject)A,&state);
945:   if (!a->sparse_optimized || a->state != state) {
946:     MatSeqAIJMKL_create_mkl_handle(A);
947:   }
948:   PetscObjectStateGet((PetscObject)P,&state);
949:   if (!p->sparse_optimized || p->state != state) {
950:     MatSeqAIJMKL_create_mkl_handle(P);
951:   }
952:   csrA = a->csrA;
953:   csrP = p->csrA;
954:   csrC = c->csrA;
955:   descr_type_sym.type = SPARSE_MATRIX_TYPE_SYMMETRIC;
956:   descr_type_sym.mode = SPARSE_FILL_MODE_UPPER;
957:   descr_type_sym.diag = SPARSE_DIAG_NON_UNIT;

959:   /* Note that the call below won't work for complex matrices. (We protect this when pointers are assigned in MatConvert.) */
960:   stat = mkl_sparse_sypr(SPARSE_OPERATION_TRANSPOSE,csrP,csrA,descr_type_sym,&csrC,SPARSE_STAGE_FINALIZE_MULT);
961:   if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to finalize mkl_sparse_sypr()");

963:   /* Update the PETSc AIJ representation for matrix C from contents of MKL handle.
964:    * This is more complicated than it should be: it turns out that, though mkl_sparse_sypr() will accept a full AIJ/CSR matrix,
965:    * the output matrix only contains the upper or lower triangle (we arbitrarily have chosen upper) of the symmetric matrix.
966:    * We have to fill in the missing portion, which we currently do below by forming the tranpose and performing at MatAXPY
967:    * operation. This may kill any performance benefit of using the optimized mkl_sparse_sypr() routine. Performance might
968:    * improve if we come up with a more efficient way to do this, or we can convince the MKL team to provide an option to output
969:    * the full matrix. */
970:   MatSeqAIJMKL_update_from_mkl_handle(C);
971:   MatTranspose(C,MAT_INITIAL_MATRIX,&Ct);
972:   MatCreateVecs(C,&zeros,NULL);
973:   VecSetFromOptions(zeros);
974:   VecZeroEntries(zeros);
975:   MatDiagonalSet(Ct,zeros,INSERT_VALUES);
976:   MatAXPY(C,1.0,Ct,DIFFERENT_NONZERO_PATTERN);
977:   /* Note: The MatAXPY() call destroys the MatProduct, so we must recreate it. */
978:   MatProductCreateWithMat(A,P,PETSC_NULL,C);
979:   MatProductSetType(C,MATPRODUCT_PtAP);
980:   MatSeqAIJMKL_create_mkl_handle(C);
981:   VecDestroy(&zeros);
982:   MatDestroy(&Ct);
983:   return(0);
984: }

986: PetscErrorCode MatProductSymbolic_PtAP_SeqAIJMKL_SeqAIJMKL_SymmetricReal(Mat C)
987: {
988:   Mat_Product         *product = C->product;
989:   Mat                 A = product->A,P = product->B;
990:   Mat_SeqAIJMKL       *a = (Mat_SeqAIJMKL*)A->spptr,*p = (Mat_SeqAIJMKL*)P->spptr;
991:   sparse_matrix_t     csrA,csrP,csrC;
992:   sparse_status_t     stat = SPARSE_STATUS_SUCCESS;
993:   struct matrix_descr descr_type_sym;
994:   PetscObjectState    state;
995:   PetscErrorCode      ierr;

998:   PetscObjectStateGet((PetscObject)A,&state);
999:   if (!a->sparse_optimized || a->state != state) {
1000:     MatSeqAIJMKL_create_mkl_handle(A);
1001:   }
1002:   PetscObjectStateGet((PetscObject)P,&state);
1003:   if (!p->sparse_optimized || p->state != state) {
1004:     MatSeqAIJMKL_create_mkl_handle(P);
1005:   }
1006:   csrA = a->csrA;
1007:   csrP = p->csrA;
1008:   descr_type_sym.type = SPARSE_MATRIX_TYPE_SYMMETRIC;
1009:   descr_type_sym.mode = SPARSE_FILL_MODE_UPPER;
1010:   descr_type_sym.diag = SPARSE_DIAG_NON_UNIT;

1012:   /* Note that the call below won't work for complex matrices. (We protect this when pointers are assigned in MatConvert.) */
1013:   if (csrP && csrA) {
1014:     stat = mkl_sparse_sypr(SPARSE_OPERATION_TRANSPOSE,csrP,csrA,descr_type_sym,&csrC,SPARSE_STAGE_FULL_MULT_NO_VAL);
1015:     if (stat != SPARSE_STATUS_SUCCESS) SETERRQ(PETSC_COMM_SELF,PETSC_ERR_LIB,"Intel MKL error: unable to perform symbolic mkl_sparse_sypr()");
1016:   } else {
1017:     csrC = PETSC_NULL;
1018:   }

1020:   /* Update the I and J arrays of the PETSc AIJ representation for matrix C from contents of MKL handle.
1021:    * Note that, because mkl_sparse_sypr() only computes one triangle of the symmetric matrix, this representation will only contain
1022:    * the upper triangle of the symmetric matrix. We fix this in MatPtAPNumeric_SeqAIJMKL_SeqAIJMKL_SymmetricReal(). I believe that
1023:    * leaving things in this incomplete state is OK because the numeric product should follow soon after, but am not certain if this
1024:    * is guaranteed. */
1025:   MatSeqAIJMKL_setup_structure_from_mkl_handle(PETSC_COMM_SELF,csrC,P->cmap->N,P->cmap->N,C);

1027:   C->ops->productnumeric = MatProductNumeric_PtAP;
1028:   return(0);
1029: }

1031: static PetscErrorCode MatProductSetFromOptions_SeqAIJMKL_AB(Mat C)
1032: {
1034:   C->ops->productsymbolic = MatProductSymbolic_AB;
1035:   C->ops->matmultsymbolic = MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL;
1036:   return(0);
1037: }

1039: static PetscErrorCode MatProductSetFromOptions_SeqAIJMKL_AtB(Mat C)
1040: {
1042:   C->ops->productsymbolic = MatProductSymbolic_AtB_SeqAIJMKL_SeqAIJMKL;
1043:   return(0);
1044: }

1046: static PetscErrorCode MatProductSetFromOptions_SeqAIJMKL_ABt(Mat C)
1047: {
1049:   C->ops->mattransposemultsymbolic = MatMatTransposeMultSymbolic_SeqAIJ_SeqAIJ;
1050:   C->ops->productsymbolic          = MatProductSymbolic_ABt;
1051:   return(0);
1052: }

1054: static PetscErrorCode MatProductSetFromOptions_SeqAIJMKL_PtAP(Mat C)
1055: {
1057:   Mat_Product    *product = C->product;
1058:   Mat            A = product->A;
1059:   PetscBool      set, flag;

1062:   if (PetscDefined(USE_COMPLEX)) {
1063:     /* By setting C->ops->productsymbolic to NULL, we ensure that MatProductSymbolic_Unsafe() will be used.
1064:      * We do this in several other locations in this file. This works for the time being, but these
1065:      * routines are considered unsafe and may be removed from the MatProduct code in the future.
1066:      * TODO: Add proper MATSEQAIJMKL implementations */
1067:     C->ops->productsymbolic = NULL;
1068:   } else {
1069:     /* AIJMKL only has an optimized routine for PtAP when A is symmetric and real. */
1070:     MatIsSymmetricKnown(A,&set,&flag);
1071:     if (set && flag) C->ops->productsymbolic = MatProductSymbolic_PtAP_SeqAIJMKL_SeqAIJMKL_SymmetricReal;
1072:     else C->ops->productsymbolic = NULL; /* MatProductSymbolic_Unsafe() will be used. */
1073:     /* Note that we don't set C->ops->productnumeric here, as this must happen in MatProductSymbolic_PtAP_XXX(),
1074:      * depending on whether the algorithm for the general case vs. the real symmetric one is used. */
1075:   }
1076:   return(0);
1077: }

1079: static PetscErrorCode MatProductSetFromOptions_SeqAIJMKL_RARt(Mat C)
1080: {
1082:   C->ops->productsymbolic = NULL; /* MatProductSymbolic_Unsafe() will be used. */
1083:   return(0);
1084: }

1086: static PetscErrorCode MatProductSetFromOptions_SeqAIJMKL_ABC(Mat C)
1087: {
1089:   C->ops->productsymbolic = NULL; /* MatProductSymbolic_Unsafe() will be used. */
1090:   return(0);
1091: }

1093: PetscErrorCode MatProductSetFromOptions_SeqAIJMKL(Mat C)
1094: {
1096:   Mat_Product    *product = C->product;

1099:   switch (product->type) {
1100:   case MATPRODUCT_AB:
1101:     MatProductSetFromOptions_SeqAIJMKL_AB(C);
1102:     break;
1103:   case MATPRODUCT_AtB:
1104:     MatProductSetFromOptions_SeqAIJMKL_AtB(C);
1105:     break;
1106:   case MATPRODUCT_ABt:
1107:     MatProductSetFromOptions_SeqAIJMKL_ABt(C);
1108:     break;
1109:   case MATPRODUCT_PtAP:
1110:     MatProductSetFromOptions_SeqAIJMKL_PtAP(C);
1111:     break;
1112:   case MATPRODUCT_RARt:
1113:     MatProductSetFromOptions_SeqAIJMKL_RARt(C);
1114:     break;
1115:   case MATPRODUCT_ABC:
1116:     MatProductSetFromOptions_SeqAIJMKL_ABC(C);
1117:     break;
1118:   default:
1119:     break;
1120:   }
1121:   return(0);
1122: }
1123: #endif /* PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE */
1124: /* ------------------------ End MatProduct code ------------------------ */

1126: /* MatConvert_SeqAIJ_SeqAIJMKL converts a SeqAIJ matrix into a
1127:  * SeqAIJMKL matrix.  This routine is called by the MatCreate_SeqAIJMKL()
1128:  * routine, but can also be used to convert an assembled SeqAIJ matrix
1129:  * into a SeqAIJMKL one. */
1130: PETSC_INTERN PetscErrorCode MatConvert_SeqAIJ_SeqAIJMKL(Mat A,MatType type,MatReuse reuse,Mat *newmat)
1131: {
1133:   Mat            B = *newmat;
1134:   Mat_SeqAIJMKL  *aijmkl;
1135:   PetscBool      set;
1136:   PetscBool      sametype;

1139:   if (reuse == MAT_INITIAL_MATRIX) {
1140:     MatDuplicate(A,MAT_COPY_VALUES,&B);
1141:   }

1143:   PetscObjectTypeCompare((PetscObject)A,type,&sametype);
1144:   if (sametype) return(0);

1146:   PetscNewLog(B,&aijmkl);
1147:   B->spptr = (void*) aijmkl;

1149:   /* Set function pointers for methods that we inherit from AIJ but override.
1150:    * We also parse some command line options below, since those determine some of the methods we point to. */
1151:   B->ops->duplicate        = MatDuplicate_SeqAIJMKL;
1152:   B->ops->assemblyend      = MatAssemblyEnd_SeqAIJMKL;
1153:   B->ops->destroy          = MatDestroy_SeqAIJMKL;

1155:   aijmkl->sparse_optimized = PETSC_FALSE;
1156: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
1157:   aijmkl->no_SpMV2 = PETSC_FALSE;  /* Default to using the SpMV2 routines if our MKL supports them. */
1158: #else
1159:   aijmkl->no_SpMV2 = PETSC_TRUE;
1160: #endif
1161:   aijmkl->eager_inspection = PETSC_FALSE;

1163:   /* Parse command line options. */
1164:   PetscOptionsBegin(PetscObjectComm((PetscObject)A),((PetscObject)A)->prefix,"AIJMKL Options","Mat");
1165:   PetscOptionsBool("-mat_aijmkl_no_spmv2","Disable use of inspector-executor (SpMV 2) routines","None",(PetscBool)aijmkl->no_SpMV2,(PetscBool*)&aijmkl->no_SpMV2,&set);
1166:   PetscOptionsBool("-mat_aijmkl_eager_inspection","Run inspection at matrix assembly time, instead of waiting until needed by an operation","None",(PetscBool)aijmkl->eager_inspection,(PetscBool*)&aijmkl->eager_inspection,&set);
1167:   PetscOptionsEnd();
1168: #if !defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
1169:   if (!aijmkl->no_SpMV2) {
1170:     PetscInfo(B,"User requested use of MKL SpMV2 routines, but MKL version does not support mkl_sparse_optimize();  defaulting to non-SpMV2 routines.\n");
1171:     aijmkl->no_SpMV2 = PETSC_TRUE;
1172:   }
1173: #endif

1175: #if defined(PETSC_HAVE_MKL_SPARSE_OPTIMIZE)
1176:   B->ops->mult                    = MatMult_SeqAIJMKL_SpMV2;
1177:   B->ops->multtranspose           = MatMultTranspose_SeqAIJMKL_SpMV2;
1178:   B->ops->multadd                 = MatMultAdd_SeqAIJMKL_SpMV2;
1179:   B->ops->multtransposeadd        = MatMultTransposeAdd_SeqAIJMKL_SpMV2;
1180: # if defined(PETSC_HAVE_MKL_SPARSE_SP2M_FEATURE)
1181:   B->ops->productsetfromoptions   = MatProductSetFromOptions_SeqAIJMKL;
1182:   B->ops->matmultsymbolic         = MatMatMultSymbolic_SeqAIJMKL_SeqAIJMKL;
1183:   B->ops->matmultnumeric          = MatMatMultNumeric_SeqAIJMKL_SeqAIJMKL;
1184:   B->ops->mattransposemultnumeric = MatMatTransposeMultNumeric_SeqAIJMKL_SeqAIJMKL;
1185:   B->ops->transposematmultnumeric = MatTransposeMatMultNumeric_SeqAIJMKL_SeqAIJMKL;
1186: #   if !defined(PETSC_USE_COMPLEX)
1187:   B->ops->ptapnumeric             = MatPtAPNumeric_SeqAIJMKL_SeqAIJMKL_SymmetricReal;
1188: #   else
1189:   B->ops->ptapnumeric             = NULL;
1190: #   endif
1191: # endif
1192: #endif /* PETSC_HAVE_MKL_SPARSE_OPTIMIZE */

1194: #if !defined(PETSC_MKL_SPBLAS_DEPRECATED)
1195:   /* In MKL version 18, update 2, the old sparse BLAS interfaces were marked as deprecated. If "no_SpMV2" has been specified by the
1196:    * user and the old SpBLAS interfaces are deprecated in our MKL version, we use the new _SpMV2 routines (set above), but do not
1197:    * call mkl_sparse_optimize(), which results in the old numerical kernels (without the inspector-executor model) being used. For
1198:    * versions in which the older interface has not been deprecated, we use the old interface. */
1199:   if (aijmkl->no_SpMV2) {
1200:     B->ops->mult             = MatMult_SeqAIJMKL;
1201:     B->ops->multtranspose    = MatMultTranspose_SeqAIJMKL;
1202:     B->ops->multadd          = MatMultAdd_SeqAIJMKL;
1203:     B->ops->multtransposeadd = MatMultTransposeAdd_SeqAIJMKL;
1204:   }
1205: #endif

1207:   PetscObjectComposeFunction((PetscObject)B,"MatConvert_seqaijmkl_seqaij_C",MatConvert_SeqAIJMKL_SeqAIJ);

1209:   PetscObjectChangeTypeName((PetscObject)B,MATSEQAIJMKL);
1210:   *newmat = B;
1211:   return(0);
1212: }

1214: /*@C
1215:    MatCreateSeqAIJMKL - Creates a sparse matrix of type SEQAIJMKL.
1216:    This type inherits from AIJ and is largely identical, but uses sparse BLAS
1217:    routines from Intel MKL whenever possible.
1218:    If the installed version of MKL supports the "SpMV2" sparse
1219:    inspector-executor routines, then those are used by default.
1220:    MatMult, MatMultAdd, MatMultTranspose, MatMultTransposeAdd, MatMatMult, MatTransposeMatMult, and MatPtAP (for
1221:    symmetric A) operations are currently supported.
1222:    Note that MKL version 18, update 2 or later is required for MatPtAP/MatPtAPNumeric and MatMatMultNumeric.

1224:    Collective

1226:    Input Parameters:
1227: +  comm - MPI communicator, set to PETSC_COMM_SELF
1228: .  m - number of rows
1229: .  n - number of columns
1230: .  nz - number of nonzeros per row (same for all rows)
1231: -  nnz - array containing the number of nonzeros in the various rows
1232:          (possibly different for each row) or NULL

1234:    Output Parameter:
1235: .  A - the matrix

1237:    Options Database Keys:
1238: +  -mat_aijmkl_no_spmv2 - disable use of the SpMV2 inspector-executor routines
1239: -  -mat_aijmkl_eager_inspection - perform MKL "inspection" phase upon matrix assembly; default is to do "lazy" inspection, performing this step the first time the matrix is applied

1241:    Notes:
1242:    If nnz is given then nz is ignored

1244:    Level: intermediate

1246: .seealso: MatCreate(), MatCreateMPIAIJMKL(), MatSetValues()
1247: @*/
1248: PetscErrorCode  MatCreateSeqAIJMKL(MPI_Comm comm,PetscInt m,PetscInt n,PetscInt nz,const PetscInt nnz[],Mat *A)
1249: {

1253:   MatCreate(comm,A);
1254:   MatSetSizes(*A,m,n,m,n);
1255:   MatSetType(*A,MATSEQAIJMKL);
1256:   MatSeqAIJSetPreallocation_SeqAIJ(*A,nz,nnz);
1257:   return(0);
1258: }

1260: PETSC_EXTERN PetscErrorCode MatCreate_SeqAIJMKL(Mat A)
1261: {

1265:   MatSetType(A,MATSEQAIJ);
1266:   MatConvert_SeqAIJ_SeqAIJMKL(A,MATSEQAIJMKL,MAT_INPLACE_MATRIX,&A);
1267:   return(0);
1268: }