Actual source code: aijcusparse.cu

  1: /*
  2:   Defines the basic matrix operations for the AIJ (compressed row)
  3:   matrix storage format using the CUSPARSE library,
  4: */
  5: #define PETSC_SKIP_IMMINTRIN_H_CUDAWORKAROUND 1

  7: #include <../src/mat/impls/aij/seq/aij.h>
  8: #include <petscpkg_version.h>
  9: #include <../src/mat/impls/sbaij/seq/sbaij.h>
 10: #include <../src/vec/vec/impls/dvecimpl.h>
 11: #include <petsc/private/vecimpl.h>
 12: #undef VecType
 13: #include <../src/mat/impls/aij/seq/seqcusparse/cusparsematimpl.h>
 14: #include <../src/mat/impls/aij/seq/cupm/aijcupm.hpp>
 15: #include <thrust/adjacent_difference.h>
 16: #include <thrust/iterator/constant_iterator.h>
 17: #include <thrust/remove.h>
 18: #include <thrust/sort.h>
 19: #include <thrust/tuple.h>
 20: #include <thrust/unique.h>
 21: #include <thrust/gather.h>
 22: #include <thrust/binary_search.h> // for thrust::lower_bound
 23: #include <thrust/version.h>
 24: #if THRUST_VERSION >= 200800
 25:   #include <cuda/std/functional>
 26: #endif
 27: #if CCCL_VERSION >= 3004000
 28:   #include <cuda/iterator>
 29: #endif

 31: const char *const MatCUSPARSEStorageFormats[] = {"CSR", "ELL", "HYB", "MatCUSPARSEStorageFormat", "MAT_CUSPARSE_", 0};
 32: /*
 33:   The following are copied from cusparse.h in CUDA-11.0. In MatCUSPARSESpMVAlgorithms[] etc, we copy them in
 34:   0-based integer value order, since we want to use PetscOptionsEnum() to parse user command line options for them.
 35: */
 36: const char *const MatCUSPARSESpMVAlgorithms[]    = {"MV_ALG_DEFAULT", "COOMV_ALG", "CSRMV_ALG1", "CSRMV_ALG2", "cusparseSpMVAlg_t", "CUSPARSE_", 0};
 37: const char *const MatCUSPARSESpMMAlgorithms[]    = {"ALG_DEFAULT", "COO_ALG1", "COO_ALG2", "COO_ALG3", "CSR_ALG1", "COO_ALG4", "CSR_ALG2", "cusparseSpMMAlg_t", "CUSPARSE_SPMM_", 0};
 38: const char *const MatCUSPARSECsr2CscAlgorithms[] = {"INVALID" /*cusparse does not have enum 0! We created one*/, "ALG1", "ALG2", "cusparseCsr2CscAlg_t", "CUSPARSE_CSR2CSC_", 0};

 40: static PetscErrorCode MatICCFactorSymbolic_SeqAIJCUSPARSE(Mat, Mat, IS, const MatFactorInfo *);
 41: static PetscErrorCode MatCholeskyFactorSymbolic_SeqAIJCUSPARSE(Mat, Mat, IS, const MatFactorInfo *);
 42: static PetscErrorCode MatCholeskyFactorNumeric_SeqAIJCUSPARSE(Mat, Mat, const MatFactorInfo *);
 43: static PetscErrorCode MatILUFactorSymbolic_SeqAIJCUSPARSE(Mat, Mat, IS, IS, const MatFactorInfo *);
 44: static PetscErrorCode MatSetFromOptions_SeqAIJCUSPARSE(Mat, PetscOptionItems PetscOptionsObject);
 45: static PetscErrorCode MatAXPY_SeqAIJCUSPARSE(Mat, PetscScalar, Mat, MatStructure);
 46: static PetscErrorCode MatScale_SeqAIJCUSPARSE(Mat, PetscScalar);
 47: static PetscErrorCode MatDiagonalScale_SeqAIJCUSPARSE(Mat, Vec, Vec);
 48: static PetscErrorCode MatMult_SeqAIJCUSPARSE(Mat, Vec, Vec);
 49: static PetscErrorCode MatMultAdd_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec);
 50: static PetscErrorCode MatMultTranspose_SeqAIJCUSPARSE(Mat, Vec, Vec);
 51: static PetscErrorCode MatMultTransposeAdd_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec);
 52: static PetscErrorCode MatMultHermitianTranspose_SeqAIJCUSPARSE(Mat, Vec, Vec);
 53: static PetscErrorCode MatMultHermitianTransposeAdd_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec);
 54: static PetscErrorCode MatMultAddKernel_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec, PetscBool, PetscBool);

 56: static PetscErrorCode CsrMatrix_Destroy(CsrMatrix **);
 57: static PetscErrorCode MatSeqAIJCUSPARSEMultStruct_Destroy(Mat_SeqAIJCUSPARSEMultStruct **, MatCUSPARSEStorageFormat);
 58: static PetscErrorCode MatSeqAIJCUSPARSETriFactors_Destroy(Mat_SeqAIJCUSPARSETriFactors **);
 59: static PetscErrorCode MatSeqAIJCUSPARSE_Destroy(Mat);

 61: static PetscErrorCode MatSeqAIJCUSPARSECopyFromGPU(Mat);
 62: static PetscErrorCode MatSeqAIJCUSPARSEInvalidateTranspose(Mat, PetscBool);

 64: static PetscErrorCode       MatSeqAIJCopySubArray_SeqAIJCUSPARSE(Mat, PetscInt, const PetscInt[], PetscScalar[]);
 65: static PetscErrorCode       MatSetPreallocationCOO_SeqAIJCUSPARSE(Mat, PetscCount, PetscInt[], PetscInt[]);
 66: static PetscErrorCode       MatSetValuesCOO_SeqAIJCUSPARSE(Mat, const PetscScalar[], InsertMode);
 67: PETSC_INTERN PetscErrorCode MatConvert_SeqAIJ_SeqAIJCUSPARSE(Mat, MatType, MatReuse, Mat *);

 69: // cusparseCreateCsr() separates types for row offsets and column indices in prototype, but requires them to have the same type at runtime!
 70: const cusparseIndexType_t csrRowOffsetsType = PetscDefined(USE_64BIT_INDICES) ? CUSPARSE_INDEX_64I : CUSPARSE_INDEX_32I;
 71: const cusparseIndexType_t csrColIndType     = PetscDefined(USE_64BIT_INDICES) ? CUSPARSE_INDEX_64I : CUSPARSE_INDEX_32I;

 73: /* Policy struct for MatSeqAIJCUSPARSE_CUPM shared template (CUDA specialisation) */
 74: struct MatSeqAIJCUSPARSE_Policy {
 75:   typedef Mat_SeqAIJCUSPARSE           mat_struct_type;
 76:   typedef Mat_SeqAIJCUSPARSEMultStruct mult_struct_type;

 78:   static int storage_format_csr() { return (int)MAT_CUSPARSE_CSR; }
 79:   static int storage_format_ell() { return (int)MAT_CUSPARSE_ELL; }
 80:   static int storage_format_hyb() { return (int)MAT_CUSPARSE_HYB; }

 82:   static PetscErrorCode CopyToGPU(Mat A) { return MatSeqAIJCUSPARSECopyToGPU(A); }
 83:   static PetscErrorCode CopyFromGPU(Mat A) { return MatSeqAIJCUSPARSECopyFromGPU(A); }
 84:   static PetscErrorCode InvalidateTranspose(Mat A, PetscBool d) { return MatSeqAIJCUSPARSEInvalidateTranspose(A, d); }
 85:   static PetscErrorCode ConvertFromSeqAIJ(Mat B, MatType t, MatReuse r, Mat *C) { return MatConvert_SeqAIJ_SeqAIJCUSPARSE(B, t, r, C); }
 86:   static const char    *mat_type_name;

 88:   static PetscErrorCode Destroy(Mat A) { return MatSeqAIJCUSPARSE_Destroy(A); }
 89:   static PetscErrorCode TriFactorsDestroy(void **spptr) { return MatSeqAIJCUSPARSETriFactors_Destroy((Mat_SeqAIJCUSPARSETriFactors **)spptr); }
 90:   static const char    *set_format_c;
 91:   static const char    *set_use_cpu_solve_c;
 92:   static const char    *product_seqdense_device_c;
 93:   static const char    *product_seqdense_c;
 94:   static const char    *product_self_c;
 95:   static const char    *seq_convert_hypre_c;

 97:   static PetscErrorCode VecGetArrayRead(Vec v, const PetscScalar **a) { return VecCUDAGetArrayRead(v, a); }
 98:   static PetscErrorCode VecRestoreArrayRead(Vec v, const PetscScalar **a) { return VecCUDARestoreArrayRead(v, a); }
 99:   static PetscErrorCode VecGetArrayWrite(Vec v, PetscScalar **a) { return VecCUDAGetArrayWrite(v, a); }
100:   static PetscErrorCode VecRestoreArrayWrite(Vec v, PetscScalar **a) { return VecCUDARestoreArrayWrite(v, a); }
101: };
102: const char *MatSeqAIJCUSPARSE_Policy::mat_type_name             = MATSEQAIJCUSPARSE;
103: const char *MatSeqAIJCUSPARSE_Policy::set_format_c              = "MatCUSPARSESetFormat_C";
104: const char *MatSeqAIJCUSPARSE_Policy::set_use_cpu_solve_c       = "MatCUSPARSESetUseCPUSolve_C";
105: const char *MatSeqAIJCUSPARSE_Policy::product_seqdense_device_c = "MatProductSetFromOptions_seqaijcusparse_seqdensecuda_C";
106: const char *MatSeqAIJCUSPARSE_Policy::product_seqdense_c        = "MatProductSetFromOptions_seqaijcusparse_seqdense_C";
107: const char *MatSeqAIJCUSPARSE_Policy::product_self_c            = "MatProductSetFromOptions_seqaijcusparse_seqaijcusparse_C";
108: const char *MatSeqAIJCUSPARSE_Policy::seq_convert_hypre_c       = "MatConvert_seqaijcusparse_hypre_C";

110: using MatSeqAIJCUSPARSE_CUPM_t = Petsc::mat::aij::cupm::impl::MatSeqAIJCUSPARSE_CUPM<Petsc::device::cupm::DeviceType::CUDA, MatSeqAIJCUSPARSE_Policy>;

112: using SparsePointerModeGuard = Petsc::device::cupm::impl::CUPMSparsePointerModeGuard<Petsc::device::cupm::DeviceType::CUDA>;

114: using Csr2coo        = Petsc::mat::aij::cupm::impl::Csr2coo;
115: using PetscIntToCInt = Petsc::mat::aij::cupm::impl::PetscIntToCInt;

117: PETSC_INTERN PetscErrorCode MatCUSPARSESetFormat_SeqAIJCUSPARSE(Mat A, MatCUSPARSEFormatOperation op, MatCUSPARSEStorageFormat format)
118: {
119:   Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;

121:   PetscFunctionBegin;
122:   switch (op) {
123:   case MAT_CUSPARSE_MULT:
124:     cusparsestruct->format = format;
125:     break;
126:   case MAT_CUSPARSE_ALL:
127:     cusparsestruct->format = format;
128:     break;
129:   default:
130:     SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "unsupported operation %d for MatCUSPARSEFormatOperation. MAT_CUSPARSE_MULT and MAT_CUSPARSE_ALL are currently supported.", op);
131:   }
132:   PetscFunctionReturn(PETSC_SUCCESS);
133: }

135: /*@
136:   MatCUSPARSESetFormat - Sets the storage format of `MATSEQCUSPARSE` matrices for a particular
137:   operation. Only the `MatMult()` operation can use different GPU storage formats

139:   Not Collective

141:   Input Parameters:
142: + A      - Matrix of type `MATSEQAIJCUSPARSE`
143: . op     - `MatCUSPARSEFormatOperation`. `MATSEQAIJCUSPARSE` matrices support `MAT_CUSPARSE_MULT` and `MAT_CUSPARSE_ALL`.
144:            `MATMPIAIJCUSPARSE` matrices support `MAT_CUSPARSE_MULT_DIAG`,`MAT_CUSPARSE_MULT_OFFDIAG`, and `MAT_CUSPARSE_ALL`.
145: - format - `MatCUSPARSEStorageFormat` (one of `MAT_CUSPARSE_CSR`, `MAT_CUSPARSE_ELL`, `MAT_CUSPARSE_HYB`.)

147:   Level: intermediate

149: .seealso: [](ch_matrices), `Mat`, `MATSEQAIJCUSPARSE`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
150: @*/
151: PetscErrorCode MatCUSPARSESetFormat(Mat A, MatCUSPARSEFormatOperation op, MatCUSPARSEStorageFormat format)
152: {
153:   PetscFunctionBegin;
155:   PetscTryMethod(A, "MatCUSPARSESetFormat_C", (Mat, MatCUSPARSEFormatOperation, MatCUSPARSEStorageFormat), (A, op, format));
156:   PetscFunctionReturn(PETSC_SUCCESS);
157: }

159: PETSC_INTERN PetscErrorCode MatCUSPARSESetUseCPUSolve_SeqAIJCUSPARSE(Mat A, PetscBool use_cpu)
160: {
161:   Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;

163:   PetscFunctionBegin;
164:   cusparsestruct->use_cpu_solve = use_cpu;
165:   PetscFunctionReturn(PETSC_SUCCESS);
166: }

168: /*@
169:   MatCUSPARSESetUseCPUSolve - Sets to use CPU `MatSolve()`.

171:   Input Parameters:
172: + A       - Matrix of type `MATSEQAIJCUSPARSE`
173: - use_cpu - set flag for using the built-in CPU `MatSolve()`

175:   Level: intermediate

177:   Note:
178:   The NVIDIA cuSPARSE LU solver currently computes the factors with the built-in CPU method
179:   and moves the factors to the GPU for the solve. We have observed better performance keeping the data on the CPU and performing the solve there.
180:   This method to specify if the solve is done on the CPU or GPU (GPU is the default).

182: .seealso: [](ch_matrices), `Mat`, `MatSolve()`, `MATSEQAIJCUSPARSE`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
183: @*/
184: PetscErrorCode MatCUSPARSESetUseCPUSolve(Mat A, PetscBool use_cpu)
185: {
186:   PetscFunctionBegin;
188:   PetscTryMethod(A, "MatCUSPARSESetUseCPUSolve_C", (Mat, PetscBool), (A, use_cpu));
189:   PetscFunctionReturn(PETSC_SUCCESS);
190: }

192: static PetscErrorCode MatSetOption_SeqAIJCUSPARSE(Mat A, MatOption op, PetscBool flg)
193: {
194:   PetscFunctionBegin;
195:   switch (op) {
196:   case MAT_FORM_EXPLICIT_TRANSPOSE:
197:     /* need to destroy the transpose matrix if present to prevent from logic errors if flg is set to true later */
198:     if (A->form_explicit_transpose && !flg) PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_TRUE));
199:     A->form_explicit_transpose = flg;
200:     break;
201:   default:
202:     PetscCall(MatSetOption_SeqAIJ(A, op, flg));
203:     break;
204:   }
205:   PetscFunctionReturn(PETSC_SUCCESS);
206: }

208: static PetscErrorCode MatSetFromOptions_SeqAIJCUSPARSE(Mat A, PetscOptionItems PetscOptionsObject)
209: {
210:   MatCUSPARSEStorageFormat format;
211:   PetscBool                flg;
212:   Mat_SeqAIJCUSPARSE      *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;

214:   PetscFunctionBegin;
215:   PetscOptionsHeadBegin(PetscOptionsObject, "SeqAIJCUSPARSE options");
216:   if (A->factortype == MAT_FACTOR_NONE) {
217:     PetscCall(PetscOptionsEnum("-mat_cusparse_mult_storage_format", "sets storage format of (seq)aijcusparse gpu matrices for SpMV", "MatCUSPARSESetFormat", MatCUSPARSEStorageFormats, (PetscEnum)cusparsestruct->format, (PetscEnum *)&format, &flg));
218:     if (flg) PetscCall(MatCUSPARSESetFormat(A, MAT_CUSPARSE_MULT, format));

220:     PetscCall(PetscOptionsEnum("-mat_cusparse_storage_format", "sets storage format of (seq)aijcusparse gpu matrices for SpMV and TriSolve", "MatCUSPARSESetFormat", MatCUSPARSEStorageFormats, (PetscEnum)cusparsestruct->format, (PetscEnum *)&format, &flg));
221:     if (flg) PetscCall(MatCUSPARSESetFormat(A, MAT_CUSPARSE_ALL, format));
222:     PetscCall(PetscOptionsBool("-mat_cusparse_use_cpu_solve", "Use CPU (I)LU solve", "MatCUSPARSESetUseCPUSolve", cusparsestruct->use_cpu_solve, &cusparsestruct->use_cpu_solve, &flg));
223:     if (flg) PetscCall(MatCUSPARSESetUseCPUSolve(A, cusparsestruct->use_cpu_solve));
224:     PetscCall(PetscOptionsEnum("-mat_cusparse_spmv_alg", "sets cuSPARSE algorithm used in sparse-mat dense-vector multiplication (SpMV)", "cusparseSpMVAlg_t", MatCUSPARSESpMVAlgorithms, (PetscEnum)cusparsestruct->spmvAlg, (PetscEnum *)&cusparsestruct->spmvAlg, &flg));
225:     /* If user did use this option, check its consistency with cuSPARSE, since PetscOptionsEnum() sets enum values based on their position in MatCUSPARSESpMVAlgorithms[] */
226:     PetscCheck(!flg || CUSPARSE_SPMV_CSR_ALG1 == 2, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE enum cusparseSpMVAlg_t has been changed but PETSc has not been updated accordingly");
227:     PetscCall(PetscOptionsEnum("-mat_cusparse_spmm_alg", "sets cuSPARSE algorithm used in sparse-mat dense-mat multiplication (SpMM)", "cusparseSpMMAlg_t", MatCUSPARSESpMMAlgorithms, (PetscEnum)cusparsestruct->spmmAlg, (PetscEnum *)&cusparsestruct->spmmAlg, &flg));
228:     PetscCheck(!flg || CUSPARSE_SPMM_CSR_ALG1 == 4, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE enum cusparseSpMMAlg_t has been changed but PETSc has not been updated accordingly");
229:     PetscCall(
230:       PetscOptionsEnum("-mat_cusparse_csr2csc_alg", "sets cuSPARSE algorithm used in converting CSR matrices to CSC matrices", "cusparseCsr2CscAlg_t", MatCUSPARSECsr2CscAlgorithms, (PetscEnum)cusparsestruct->csr2cscAlg, (PetscEnum *)&cusparsestruct->csr2cscAlg, &flg));
231:     PetscCheck(!flg || CUSPARSE_CSR2CSC_ALG1 == 1, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE enum cusparseCsr2CscAlg_t has been changed but PETSc has not been updated accordingly");
232:   }
233:   PetscOptionsHeadEnd();
234:   PetscFunctionReturn(PETSC_SUCCESS);
235: }

237: static PetscErrorCode MatSeqAIJCUSPARSEBuildFactoredMatrix_LU(Mat A)
238: {
239:   Mat_SeqAIJ                   *a  = static_cast<Mat_SeqAIJ *>(A->data);
240:   PetscInt                      m  = A->rmap->n;
241:   Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
242:   const PetscInt               *Ai = a->i, *Aj = a->j, *adiag;
243:   const MatScalar              *Aa = a->a;
244:   PetscInt                     *Mi, *Mj, Mnz;
245:   PetscScalar                  *Ma;
246:   PetscDeviceContext            dctx;
247:   cudaStream_t                  stream;
248:   cusparseHandle_t              handle;

250:   PetscFunctionBegin;
251:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
252:   {
253:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

255:     PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, &adiag, NULL));
256:     if (A->offloadmask == PETSC_OFFLOAD_CPU) { // A's latest factors are on CPU
257:       if (!fs->csrRowPtr) {                    // Is this the first time we are doing setup? Use csrRowPtr since it is not null even when m=0
258:         // Re-arrange the (skewed) factored matrix and put the result into M, a regular csr matrix on host
259:         Mnz = (Ai[m] - Ai[0]) + (adiag[0] - adiag[m]); // Lnz (without the unit diagonal) + Unz (with the non-unit diagonal)
260:         PetscCall(PetscMalloc1(m + 1, &Mi));
261:         PetscCall(PetscMalloc1(Mnz, &Mj)); // Mj is temp
262:         PetscCall(PetscMalloc1(Mnz, &Ma));
263:         Mi[0] = 0;
264:         for (PetscInt i = 0; i < m; i++) {
265:           PetscInt llen = Ai[i + 1] - Ai[i];
266:           PetscInt ulen = adiag[i] - adiag[i + 1];
267:           PetscCall(PetscArraycpy(Mj + Mi[i], Aj + Ai[i], llen));                           // entries of L
268:           Mj[Mi[i] + llen] = i;                                                             // diagonal entry
269:           PetscCall(PetscArraycpy(Mj + Mi[i] + llen + 1, Aj + adiag[i + 1] + 1, ulen - 1)); // entries of U on the right of the diagonal
270:           Mi[i + 1] = Mi[i] + llen + ulen;
271:         }
272:         // Copy M (L,U) from host to device
273:         PetscCallCUDA(cudaMalloc(&fs->csrRowPtr, sizeof(*fs->csrRowPtr) * (m + 1)));
274:         PetscCallCUDA(cudaMalloc(&fs->csrColIdx, sizeof(*fs->csrColIdx) * Mnz));
275:         PetscCallCUDA(cudaMalloc(&fs->csrVal, sizeof(*fs->csrVal) * Mnz));
276:         PetscCallCUDA(cudaMemcpyAsync(fs->csrRowPtr, Mi, sizeof(*fs->csrRowPtr) * (m + 1), cudaMemcpyHostToDevice, stream));
277:         PetscCallCUDA(cudaMemcpyAsync(fs->csrColIdx, Mj, sizeof(*fs->csrColIdx) * Mnz, cudaMemcpyHostToDevice, stream));
278:         PetscCall(PetscDeviceContextSynchronize(dctx));

280:         // Create descriptors for L, U. See https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
281:         // cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
282:         // assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
283:         // all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
284:         // assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
285:         cusparseFillMode_t fillMode = CUSPARSE_FILL_MODE_LOWER;
286:         cusparseDiagType_t diagType = CUSPARSE_DIAG_TYPE_UNIT;

288:         PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_L, m, m, Mnz, fs->csrRowPtr, fs->csrColIdx, fs->csrVal, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
289:         PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
290:         PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));

292:         fillMode = CUSPARSE_FILL_MODE_UPPER;
293:         diagType = CUSPARSE_DIAG_TYPE_NON_UNIT;
294:         PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_U, m, m, Mnz, fs->csrRowPtr, fs->csrColIdx, fs->csrVal, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
295:         PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
296:         PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));

298:         // Allocate work vectors in SpSv
299:         PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(*fs->X) * m));
300:         PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(*fs->Y) * m));

302:         PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
303:         PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));

305:         // Query buffer sizes for SpSV and then allocate buffers, temporarily assuming opA = CUSPARSE_OPERATION_NON_TRANSPOSE
306:         PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_L));
307:         PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, &fs->spsvBufferSize_L));
308:         PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_U));
309:         PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, &fs->spsvBufferSize_U));
310:         PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_U, fs->spsvBufferSize_U));
311:         PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_L, fs->spsvBufferSize_L));

313:         // Record for reuse
314:         fs->csrRowPtr_h = Mi;
315:         fs->csrVal_h    = Ma;
316:         PetscCall(PetscFree(Mj));
317:       }
318:       // Copy the value
319:       Mi  = fs->csrRowPtr_h;
320:       Ma  = fs->csrVal_h;
321:       Mnz = Mi[m];
322:       for (PetscInt i = 0; i < m; i++) {
323:         PetscInt llen = Ai[i + 1] - Ai[i];
324:         PetscInt ulen = adiag[i] - adiag[i + 1];
325:         PetscCall(PetscArraycpy(Ma + Mi[i], Aa + Ai[i], llen));                           // entries of L
326:         Ma[Mi[i] + llen] = (MatScalar)1.0 / Aa[adiag[i]];                                 // recover the diagonal entry
327:         PetscCall(PetscArraycpy(Ma + Mi[i] + llen + 1, Aa + adiag[i + 1] + 1, ulen - 1)); // entries of U on the right of the diagonal
328:       }
329:       PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Ma, sizeof(*Ma) * Mnz, cudaMemcpyHostToDevice, stream));
330:       PetscCall(PetscDeviceContextSynchronize(dctx));

332: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
333:       if (fs->updatedSpSVAnalysis) { // have done cusparseSpSV_analysis before, and only matrix values changed?
334:         // Otherwise cusparse would error out: "On entry to cusparseSpSV_updateMatrix() parameter number 3 (newValues) had an illegal value: NULL pointer"
335:         if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_L, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
336:         if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_U, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
337:       } else
338: #endif
339:       {
340:         // Do cusparseSpSV_analysis(), which is numeric and requires valid and up-to-date matrix values
341:         PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, fs->spsvBuffer_L));

343:         PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, fs->spsvBuffer_U));
344:         fs->updatedSpSVAnalysis          = PETSC_TRUE;
345:         fs->updatedTransposeSpSVAnalysis = PETSC_FALSE;
346:       }
347:     }
348:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
349:   }
350:   PetscFunctionReturn(PETSC_SUCCESS);
351: }

353: static PetscErrorCode MatSeqAIJCUSPARSEILUAnalysisAndCopyToGPU(Mat A)
354: {
355:   Mat_SeqAIJ                   *a                  = (Mat_SeqAIJ *)A->data;
356:   Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)A->spptr;
357:   IS                            isrow = a->row, isicol = a->icol;
358:   PetscBool                     row_identity, col_identity;
359:   PetscInt                      n = A->rmap->n;
360:   PetscDeviceContext            dctx;
361:   cudaStream_t                  stream;

363:   PetscFunctionBegin;
364:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
365:   PetscCheck(cusparseTriFactors, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing cusparseTriFactors");
366:   PetscCall(MatSeqAIJCUSPARSEBuildFactoredMatrix_LU(A));

368:   cusparseTriFactors->nnz = a->nz;

370:   A->offloadmask = PETSC_OFFLOAD_BOTH; // factored matrix is sync'ed to GPU
371:   /* lower triangular indices */
372:   PetscCall(ISIdentity(isrow, &row_identity));
373:   if (!row_identity && !cusparseTriFactors->rpermIndices) {
374:     const PetscInt *r;

376:     PetscCall(ISGetIndices(isrow, &r));
377:     cusparseTriFactors->rpermIndices = new THRUSTINTARRAY(n);
378:     PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->rpermIndices->data().get(), r, (n) * sizeof(*r), cudaMemcpyHostToDevice, stream));
379:     PetscCall(PetscDeviceContextSynchronize(dctx));
380:     PetscCall(ISRestoreIndices(isrow, &r));
381:     PetscCall(PetscLogCpuToGpu(n * sizeof(PetscInt)));
382:   }

384:   /* upper triangular indices */
385:   PetscCall(ISIdentity(isicol, &col_identity));
386:   if (!col_identity && !cusparseTriFactors->cpermIndices) {
387:     const PetscInt *c;

389:     PetscCall(ISGetIndices(isicol, &c));
390:     cusparseTriFactors->cpermIndices = new THRUSTINTARRAY(n);
391:     PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->cpermIndices->data().get(), c, (n) * sizeof(*c), cudaMemcpyHostToDevice, stream));
392:     PetscCall(PetscDeviceContextSynchronize(dctx));
393:     PetscCall(ISRestoreIndices(isicol, &c));
394:     PetscCall(PetscLogCpuToGpu(n * sizeof(PetscInt)));
395:   }
396:   PetscFunctionReturn(PETSC_SUCCESS);
397: }

399: static PetscErrorCode MatSeqAIJCUSPARSEBuildFactoredMatrix_Cholesky(Mat A)
400: {
401:   Mat_SeqAIJ                   *a  = static_cast<Mat_SeqAIJ *>(A->data);
402:   PetscInt                      m  = A->rmap->n;
403:   Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
404:   const PetscInt               *Ai = a->i, *Aj = a->j, *adiag;
405:   const MatScalar              *Aa = a->a;
406:   PetscInt                     *Mj, Mnz;
407:   PetscScalar                  *Ma, *D;
408:   PetscDeviceContext            dctx;
409:   cudaStream_t                  stream;
410:   cusparseHandle_t              handle;

412:   PetscFunctionBegin;
413:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
414:   {
415:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

417:     PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, &adiag, NULL));
418:     if (A->offloadmask == PETSC_OFFLOAD_CPU) { // A's latest factors are on CPU
419:       if (!fs->csrRowPtr) {                    // Is this the first time we are doing setup? Use csrRowPtr since it is not null even m=0
420:         // Re-arrange the (skewed) factored matrix and put the result into M, a regular csr matrix on host.
421:         // See comments at MatICCFactorSymbolic_SeqAIJ() on the layout of the factored matrix (U) on host.
422:         Mnz = Ai[m]; // Unz (with the unit diagonal)
423:         PetscCall(PetscMalloc1(Mnz, &Ma));
424:         PetscCall(PetscMalloc1(Mnz, &Mj)); // Mj[] is temp
425:         PetscCall(PetscMalloc1(m, &D));    // the diagonal
426:         for (PetscInt i = 0; i < m; i++) {
427:           PetscInt ulen = Ai[i + 1] - Ai[i];
428:           Mj[Ai[i]]     = i;                                              // diagonal entry
429:           PetscCall(PetscArraycpy(Mj + Ai[i] + 1, Aj + Ai[i], ulen - 1)); // entries of U on the right of the diagonal
430:         }
431:         // Copy M (U) from host to device
432:         PetscCallCUDA(cudaMalloc(&fs->csrRowPtr, sizeof(*fs->csrRowPtr) * (m + 1)));
433:         PetscCallCUDA(cudaMalloc(&fs->csrColIdx, sizeof(*fs->csrColIdx) * Mnz));
434:         PetscCallCUDA(cudaMalloc(&fs->csrVal, sizeof(*fs->csrVal) * Mnz));
435:         PetscCallCUDA(cudaMalloc(&fs->diag, sizeof(*fs->diag) * m));
436:         PetscCallCUDA(cudaMemcpyAsync(fs->csrRowPtr, Ai, sizeof(*Ai) * (m + 1), cudaMemcpyHostToDevice, stream));
437:         PetscCallCUDA(cudaMemcpyAsync(fs->csrColIdx, Mj, sizeof(*Mj) * Mnz, cudaMemcpyHostToDevice, stream));
438:         PetscCall(PetscDeviceContextSynchronize(dctx));

440:         // Create descriptors for L, U. See https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
441:         // cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
442:         // assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
443:         // all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
444:         // assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
445:         cusparseFillMode_t fillMode = CUSPARSE_FILL_MODE_UPPER;
446:         cusparseDiagType_t diagType = CUSPARSE_DIAG_TYPE_UNIT; // U is unit diagonal

448:         PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_U, m, m, Mnz, fs->csrRowPtr, fs->csrColIdx, fs->csrVal, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
449:         PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
450:         PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));

452:         // Allocate work vectors in SpSv
453:         PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(*fs->X) * m));
454:         PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(*fs->Y) * m));

456:         PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
457:         PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));

459:         // Query buffer sizes for SpSV and then allocate buffers
460:         PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_U));
461:         PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, &fs->spsvBufferSize_U));
462:         PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_U, fs->spsvBufferSize_U));

464:         PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Ut)); // Ut solve uses the same matrix (spMatDescr_U), but different descr and buffer
465:         PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Ut, &fs->spsvBufferSize_Ut));
466:         PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Ut, fs->spsvBufferSize_Ut));

468:         // Record for reuse
469:         fs->csrVal_h = Ma;
470:         fs->diag_h   = D;
471:         PetscCall(PetscFree(Mj));
472:       }
473:       // Copy the value
474:       Ma  = fs->csrVal_h;
475:       D   = fs->diag_h;
476:       Mnz = Ai[m];
477:       for (PetscInt i = 0; i < m; i++) {
478:         D[i]      = Aa[adiag[i]];   // actually Aa[adiag[i]] is the inverse of the diagonal
479:         Ma[Ai[i]] = (MatScalar)1.0; // set the unit diagonal, which is cosmetic since cusparse does not really read it given CUSPARSE_DIAG_TYPE_UNIT
480:         for (PetscInt k = 0; k < Ai[i + 1] - Ai[i] - 1; k++) Ma[Ai[i] + 1 + k] = -Aa[Ai[i] + k];
481:       }
482:       PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Ma, sizeof(*Ma) * Mnz, cudaMemcpyHostToDevice, stream));
483:       PetscCallCUDA(cudaMemcpyAsync(fs->diag, D, sizeof(*D) * m, cudaMemcpyHostToDevice, stream));
484:       PetscCall(PetscDeviceContextSynchronize(dctx));

486: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
487:       if (fs->updatedSpSVAnalysis) {
488:         if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_U, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
489:         if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_Ut, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
490:       } else
491: #endif
492:       {
493:         // Do cusparseSpSV_analysis(), which is numeric and requires valid and up-to-date matrix values
494:         PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, fs->spsvBuffer_U));
495:         PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Ut, fs->spsvBuffer_Ut));
496:         fs->updatedSpSVAnalysis = PETSC_TRUE;
497:       }
498:     }
499:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
500:   }
501:   PetscFunctionReturn(PETSC_SUCCESS);
502: }

504: // Solve Ut D U x = b
505: static PetscErrorCode MatSolve_SeqAIJCUSPARSE_Cholesky(Mat A, Vec b, Vec x)
506: {
507:   Mat_SeqAIJCUSPARSETriFactors         *fs  = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
508:   Mat_SeqAIJ                           *aij = static_cast<Mat_SeqAIJ *>(A->data);
509:   const PetscScalar                    *barray;
510:   PetscScalar                          *xarray;
511:   thrust::device_ptr<const PetscScalar> bGPU;
512:   thrust::device_ptr<PetscScalar>       xGPU;
513:   const cusparseSpSVAlg_t               alg = CUSPARSE_SPSV_ALG_DEFAULT;
514:   PetscInt                              m   = A->rmap->n;
515:   PetscDeviceContext                    dctx;
516:   cudaStream_t                          stream;
517:   cusparseHandle_t                      handle;

519:   PetscFunctionBegin;
520:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
521:   {
522:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

524:     PetscCall(PetscLogGpuTimeBegin());
525:     PetscCall(VecCUDAGetArrayWrite(x, &xarray));
526:     PetscCall(VecCUDAGetArrayRead(b, &barray));
527:     xGPU = thrust::device_pointer_cast(xarray);
528:     bGPU = thrust::device_pointer_cast(barray);

530:     // Reorder b with the row permutation if needed, and wrap the result in fs->X
531:     if (fs->rpermIndices) {
532:       PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->begin()), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->end()), thrust::device_pointer_cast(fs->X)));
533:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
534:     } else {
535:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
536:     }

538:     // Solve Ut Y = X
539:     PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
540:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut));

542:     // Solve diag(D) Z = Y. Actually just do Y = Y*D since D is already inverted in MatCholeskyFactorNumeric_SeqAIJ().
543:     // It is basically a vector element-wise multiplication, but cublas does not have it!
544: #if CCCL_VERSION >= 3001000
545:     auto multiplies = cuda::std::multiplies<PetscScalar>();
546: #else
547:     auto multiplies = thrust::multiplies<PetscScalar>();
548: #endif
549:     PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), thrust::device_pointer_cast(fs->Y), thrust::device_pointer_cast(fs->Y + m), thrust::device_pointer_cast(fs->diag), thrust::device_pointer_cast(fs->Y), multiplies));

551:     // Solve U X = Y
552:     if (fs->cpermIndices) { // if need to permute, we need to use the intermediate buffer X
553:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
554:     } else {
555:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
556:     }
557:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, alg, fs->spsvDescr_U));

559:     // Reorder X with the column permutation if needed, and put the result back to x
560:     if (fs->cpermIndices) {
561:       PetscCallThrust(
562:         thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X), fs->cpermIndices->begin()), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X + m), fs->cpermIndices->end()), xGPU));
563:     }

565:     PetscCall(VecCUDARestoreArrayRead(b, &barray));
566:     PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
567:     PetscCall(PetscLogGpuTimeEnd());
568:     PetscCall(PetscLogGpuFlops(4.0 * aij->nz - A->rmap->n));
569:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
570:   }
571:   PetscFunctionReturn(PETSC_SUCCESS);
572: }

574: static PetscErrorCode MatSeqAIJCUSPARSEICCAnalysisAndCopyToGPU(Mat A)
575: {
576:   Mat_SeqAIJ                   *a                  = (Mat_SeqAIJ *)A->data;
577:   Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)A->spptr;
578:   IS                            ip                 = a->row;
579:   PetscBool                     perm_identity;
580:   PetscInt                      n = A->rmap->n;
581:   PetscDeviceContext            dctx;
582:   cudaStream_t                  stream;

584:   PetscFunctionBegin;
585:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
586:   PetscCheck(cusparseTriFactors, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing cusparseTriFactors");

588:   PetscCall(MatSeqAIJCUSPARSEBuildFactoredMatrix_Cholesky(A));
589:   cusparseTriFactors->nnz = (a->nz - n) * 2 + n;

591:   A->offloadmask = PETSC_OFFLOAD_BOTH;

593:   /* lower triangular indices */
594:   PetscCall(ISIdentity(ip, &perm_identity));
595:   if (!perm_identity) {
596:     IS              iip;
597:     const PetscInt *irip, *rip;

599:     PetscCall(ISInvertPermutation(ip, PETSC_DECIDE, &iip));
600:     PetscCall(ISGetIndices(iip, &irip));
601:     PetscCall(ISGetIndices(ip, &rip));
602:     cusparseTriFactors->rpermIndices = new THRUSTINTARRAY(n);
603:     PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->rpermIndices->data().get(), rip, (n) * sizeof(*rip), cudaMemcpyHostToDevice, stream));
604:     cusparseTriFactors->cpermIndices = new THRUSTINTARRAY(n);
605:     PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->cpermIndices->data().get(), irip, (n) * sizeof(*irip), cudaMemcpyHostToDevice, stream));
606:     PetscCall(PetscDeviceContextSynchronize(dctx));
607:     PetscCall(ISRestoreIndices(iip, &irip));
608:     PetscCall(ISDestroy(&iip));
609:     PetscCall(ISRestoreIndices(ip, &rip));
610:     PetscCall(PetscLogCpuToGpu(2. * n * sizeof(PetscInt)));
611:   }
612:   PetscFunctionReturn(PETSC_SUCCESS);
613: }

615: static PetscErrorCode MatCholeskyFactorNumeric_SeqAIJCUSPARSE(Mat B, Mat A, const MatFactorInfo *info)
616: {
617:   PetscFunctionBegin;
618:   PetscCall(MatSeqAIJCUSPARSECopyFromGPU(A));
619:   PetscCall(MatCholeskyFactorNumeric_SeqAIJ(B, A, info));
620:   B->offloadmask            = PETSC_OFFLOAD_CPU;
621:   B->ops->solve             = MatSolve_SeqAIJCUSPARSE_Cholesky;
622:   B->ops->solvetranspose    = MatSolve_SeqAIJCUSPARSE_Cholesky; // since symmetric
623:   B->ops->matsolve          = NULL;
624:   B->ops->matsolvetranspose = NULL;
625:   /* get the triangular factors */
626:   PetscCall(MatSeqAIJCUSPARSEICCAnalysisAndCopyToGPU(B));
627:   PetscFunctionReturn(PETSC_SUCCESS);
628: }

630: static PetscErrorCode MatSeqAIJCUSPARSEFormExplicitTranspose(Mat A)
631: {
632:   Mat_SeqAIJCUSPARSE           *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
633:   Mat_SeqAIJCUSPARSEMultStruct *matstruct, *matstructT;
634:   Mat_SeqAIJ                   *a = (Mat_SeqAIJ *)A->data;
635:   cusparseIndexBase_t           indexBase;
636:   PetscDeviceContext            dctx;
637:   cudaStream_t                  stream;

639:   PetscFunctionBegin;
640:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
641:   PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
642:   matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
643:   PetscCheck(matstruct, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing mat struct");
644:   matstructT = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->matTranspose;
645:   PetscCheck(!A->transupdated || matstructT, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing matTranspose struct");
646:   if (A->transupdated) PetscFunctionReturn(PETSC_SUCCESS);
647:   PetscCall(PetscLogEventBegin(MAT_CUSPARSEGenerateTranspose, A, 0, 0, 0));
648:   PetscCall(PetscLogGpuTimeBegin());
649:   if (cusparsestruct->format != MAT_CUSPARSE_CSR) PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_TRUE));
650:   if (!cusparsestruct->matTranspose) { /* create cusparse matrix */
651:     matstructT = new Mat_SeqAIJCUSPARSEMultStruct;
652:     PetscCallCUSPARSE(cusparseCreateMatDescr(&matstructT->descr));
653:     indexBase = cusparseGetMatIndexBase(matstruct->descr);
654:     PetscCallCUSPARSE(cusparseSetMatIndexBase(matstructT->descr, indexBase));
655:     PetscCallCUSPARSE(cusparseSetMatType(matstructT->descr, CUSPARSE_MATRIX_TYPE_GENERAL));

657:     /* set alpha and beta */
658:     PetscCallCUDA(cudaMalloc((void **)&matstructT->alpha_one, sizeof(PetscScalar)));
659:     PetscCallCUDA(cudaMalloc((void **)&matstructT->beta_zero, sizeof(PetscScalar)));
660:     PetscCallCUDA(cudaMalloc((void **)&matstructT->beta_one, sizeof(PetscScalar)));
661:     PetscCallCUDA(cudaMemcpyAsync(matstructT->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
662:     PetscCallCUDA(cudaMemcpyAsync(matstructT->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
663:     PetscCallCUDA(cudaMemcpyAsync(matstructT->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));

665:     if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
666:       CsrMatrix *matrixT      = new CsrMatrix;
667:       matstructT->mat         = matrixT;
668:       matrixT->num_rows       = A->cmap->n;
669:       matrixT->num_cols       = A->rmap->n;
670:       matrixT->num_entries    = a->nz;
671:       matrixT->row_offsets    = new THRUSTINTARRAY(matrixT->num_rows + 1);
672:       matrixT->column_indices = new THRUSTINTARRAY(a->nz);
673:       matrixT->values         = new THRUSTARRAY(a->nz);

675:       if (!cusparsestruct->rowoffsets_gpu) cusparsestruct->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
676:       PetscCallCUDA(cudaMemcpyAsync(cusparsestruct->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
677:       PetscCall(PetscDeviceContextSynchronize(dctx));
678:       PetscCallCUSPARSE(cusparseCreateCsr(&matstructT->matDescr, matrixT->num_rows, matrixT->num_cols, matrixT->num_entries, matrixT->row_offsets->data().get(), matrixT->column_indices->data().get(), matrixT->values->data().get(), csrRowOffsetsType, csrColIndType, indexBase, cusparse_scalartype));
679:     } else if (cusparsestruct->format == MAT_CUSPARSE_ELL || cusparsestruct->format == MAT_CUSPARSE_HYB) {
680:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
681:     }
682:   }
683:   if (cusparsestruct->format == MAT_CUSPARSE_CSR) { /* transpose mat struct may be already present, update data */
684:     CsrMatrix *matrix  = (CsrMatrix *)matstruct->mat;
685:     CsrMatrix *matrixT = (CsrMatrix *)matstructT->mat;
686:     PetscCheck(matrix, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix");
687:     PetscCheck(matrix->row_offsets, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix rows");
688:     PetscCheck(matrix->column_indices, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix cols");
689:     PetscCheck(matrix->values, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix values");
690:     PetscCheck(matrixT, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT");
691:     PetscCheck(matrixT->row_offsets, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT rows");
692:     PetscCheck(matrixT->column_indices, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT cols");
693:     PetscCheck(matrixT->values, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT values");
694:     if (!cusparsestruct->rowoffsets_gpu) { /* this may be absent when we did not construct the transpose with csr2csc */
695:       cusparsestruct->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
696:       PetscCallCUDA(cudaMemcpyAsync(cusparsestruct->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
697:       PetscCall(PetscDeviceContextSynchronize(dctx));
698:       PetscCall(PetscLogCpuToGpu((A->rmap->n + 1) * sizeof(PetscInt)));
699:     }
700:     if (!cusparsestruct->csr2csc_i) { // not using cusparseCsr2cscEx2() because it requires 32-bit indices
701:       THRUSTINTARRAY row_indices(matrix->num_entries);

703:       // Transpose the matrix via COO, i.e., by putting the row indices in column_indices[] and the column indices in row_indices[]
704:       cusparsestruct->csr2csc_i = new THRUSTINTARRAY(matrix->num_entries); // will store the matrix to matrixT permutation, i.e., entry matrixT[i] is matrix[csr2csc_i[i]]
705:       PetscCallThrust(thrust::sequence(thrust::cuda::par.on(stream), cusparsestruct->csr2csc_i->begin(), cusparsestruct->csr2csc_i->end()));
706:       PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(A->rmap->n), Csr2coo(cusparsestruct->rowoffsets_gpu->data().get(), matrixT->column_indices->data().get())));
707:       PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), matrix->column_indices->begin(), matrix->column_indices->end(), row_indices.begin()));
708:       // Sort the COO by row then column, and get the permutation csr2csc_i[]
709:       PetscCallThrust(thrust::sort_by_key(thrust::cuda::par.on(stream), thrust::make_zip_iterator(thrust::make_tuple(row_indices.begin(), matrixT->column_indices->begin())),
710:                                           thrust::make_zip_iterator(thrust::make_tuple(row_indices.end(), matrixT->column_indices->end())), cusparsestruct->csr2csc_i->begin()));
711:       // Finalize matrixT's row_offsets by looking up row_indices[]
712:       PetscCallThrust(thrust::lower_bound(thrust::cuda::par.on(stream), row_indices.begin(), row_indices.end(), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(A->cmap->n + 1), matrixT->row_offsets->begin()));
713:     }
714:     PetscCallThrust(thrust::gather(thrust::cuda::par.on(stream), cusparsestruct->csr2csc_i->begin(), cusparsestruct->csr2csc_i->end(), matrix->values->begin(), matrixT->values->begin()));
715:   }
716:   PetscCall(PetscLogGpuTimeEnd());
717:   PetscCall(PetscLogEventEnd(MAT_CUSPARSEGenerateTranspose, A, 0, 0, 0));
718:   /* the compressed row indices is not used for matTranspose */
719:   matstructT->cprowIndices = NULL;
720:   /* assign the pointer */
721:   ((Mat_SeqAIJCUSPARSE *)A->spptr)->matTranspose = matstructT;
722:   A->transupdated                                = PETSC_TRUE;
723:   PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
724:   PetscFunctionReturn(PETSC_SUCCESS);
725: }

727: static PetscErrorCode MatSolve_SeqAIJCUSPARSE_LU(Mat A, Vec b, Vec x)
728: {
729:   const PetscScalar                    *barray;
730:   PetscScalar                          *xarray;
731:   thrust::device_ptr<const PetscScalar> bGPU;
732:   thrust::device_ptr<PetscScalar>       xGPU;
733:   Mat_SeqAIJCUSPARSETriFactors         *fs  = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
734:   const Mat_SeqAIJ                     *aij = static_cast<Mat_SeqAIJ *>(A->data);
735:   const cusparseOperation_t             op  = CUSPARSE_OPERATION_NON_TRANSPOSE;
736:   const cusparseSpSVAlg_t               alg = CUSPARSE_SPSV_ALG_DEFAULT;
737:   PetscInt                              m   = A->rmap->n;
738:   PetscDeviceContext                    dctx;
739:   cudaStream_t                          stream;
740:   cusparseHandle_t                      handle;

742:   PetscFunctionBegin;
743:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
744:   {
745:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

747:     PetscCall(PetscLogGpuTimeBegin());
748:     PetscCall(VecCUDAGetArrayWrite(x, &xarray));
749:     PetscCall(VecCUDAGetArrayRead(b, &barray));
750:     xGPU = thrust::device_pointer_cast(xarray);
751:     bGPU = thrust::device_pointer_cast(barray);

753:     // Reorder b with the row permutation if needed, and wrap the result in fs->X
754:     if (fs->rpermIndices) {
755:       PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->begin()), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->end()), thrust::device_pointer_cast(fs->X)));
756:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
757:     } else {
758:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
759:     }

761:     // Solve L Y = X
762:     PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
763:     // Note that cusparseSpSV_solve() secretly uses the external buffer used in cusparseSpSV_analysis()!
764:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, op, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_L));

766:     // Solve U X = Y
767:     if (fs->cpermIndices) {
768:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
769:     } else {
770:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
771:     }
772:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, op, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, alg, fs->spsvDescr_U));

774:     // Reorder X with the column permutation if needed, and put the result back to x
775:     if (fs->cpermIndices) {
776:       PetscCallThrust(
777:         thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X), fs->cpermIndices->begin()), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X + m), fs->cpermIndices->end()), xGPU));
778:     }
779:     PetscCall(VecCUDARestoreArrayRead(b, &barray));
780:     PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
781:     PetscCall(PetscLogGpuTimeEnd());
782:     PetscCall(PetscLogGpuFlops(2.0 * aij->nz - m));
783:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
784:   }
785:   PetscFunctionReturn(PETSC_SUCCESS);
786: }

788: static PetscErrorCode MatSolveTranspose_SeqAIJCUSPARSE_LU(Mat A, Vec b, Vec x)
789: {
790:   Mat_SeqAIJCUSPARSETriFactors         *fs  = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
791:   Mat_SeqAIJ                           *aij = static_cast<Mat_SeqAIJ *>(A->data);
792:   const PetscScalar                    *barray;
793:   PetscScalar                          *xarray;
794:   thrust::device_ptr<const PetscScalar> bGPU;
795:   thrust::device_ptr<PetscScalar>       xGPU;
796:   const cusparseOperation_t             opA = CUSPARSE_OPERATION_TRANSPOSE;
797:   const cusparseSpSVAlg_t               alg = CUSPARSE_SPSV_ALG_DEFAULT;
798:   PetscInt                              m   = A->rmap->n;
799:   PetscDeviceContext                    dctx;
800:   cudaStream_t                          stream;
801:   cusparseHandle_t                      handle;

803:   PetscFunctionBegin;
804:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
805:   {
806:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

808:     PetscCall(PetscLogGpuTimeBegin());
809:     if (!fs->createdTransposeSpSVDescr) { // Call MatSolveTranspose() for the first time
810:       PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Lt));
811:       PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, /* The matrix is still L. We only do transpose solve with it */
812:                                                 fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Lt, &fs->spsvBufferSize_Lt));

814:       PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Ut));
815:       PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut, &fs->spsvBufferSize_Ut));
816:       PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Lt, fs->spsvBufferSize_Lt));
817:       PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Ut, fs->spsvBufferSize_Ut));
818:       fs->createdTransposeSpSVDescr = PETSC_TRUE;
819:     }

821:     if (!fs->updatedTransposeSpSVAnalysis) {
822:       PetscCallCUSPARSE(cusparseSpSV_analysis(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Lt, fs->spsvBuffer_Lt));

824:       PetscCallCUSPARSE(cusparseSpSV_analysis(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut, fs->spsvBuffer_Ut));
825:       fs->updatedTransposeSpSVAnalysis = PETSC_TRUE;
826:     }

828:     PetscCall(VecCUDAGetArrayWrite(x, &xarray));
829:     PetscCall(VecCUDAGetArrayRead(b, &barray));
830:     xGPU = thrust::device_pointer_cast(xarray);
831:     bGPU = thrust::device_pointer_cast(barray);

833:     // Reorder b with the row permutation if needed, and wrap the result in fs->X
834:     if (fs->rpermIndices) {
835:       PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->begin()), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->end()), thrust::device_pointer_cast(fs->X)));
836:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
837:     } else {
838:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
839:     }

841:     // Solve Ut Y = X
842:     PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
843:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut));

845:     // Solve Lt X = Y
846:     if (fs->cpermIndices) { // if need to permute, we need to use the intermediate buffer X
847:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
848:     } else {
849:       PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
850:     }
851:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, alg, fs->spsvDescr_Lt));

853:     // Reorder X with the column permutation if needed, and put the result back to x
854:     if (fs->cpermIndices) {
855:       PetscCallThrust(
856:         thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X), fs->cpermIndices->begin()), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X + m), fs->cpermIndices->end()), xGPU));
857:     }

859:     PetscCall(VecCUDARestoreArrayRead(b, &barray));
860:     PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
861:     PetscCall(PetscLogGpuTimeEnd());
862:     PetscCall(PetscLogGpuFlops(2.0 * aij->nz - A->rmap->n));
863:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
864:   }
865:   PetscFunctionReturn(PETSC_SUCCESS);
866: }

868: static PetscErrorCode MatILUFactorNumeric_SeqAIJCUSPARSE_ILU0(Mat fact, Mat A, const MatFactorInfo *)
869: {
870:   Mat_SeqAIJCUSPARSETriFactors *fs    = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
871:   Mat_SeqAIJ                   *aij   = (Mat_SeqAIJ *)fact->data;
872:   Mat_SeqAIJCUSPARSE           *Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
873:   CsrMatrix                    *Acsr;
874:   PetscInt                      m, nz;
875:   PetscBool                     flg;
876:   PetscDeviceContext            dctx;
877:   cudaStream_t                  stream;
878:   cusparseHandle_t              handle;

880:   PetscFunctionBegin;
881:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
882:   {
883:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

885:     if (PetscDefined(USE_DEBUG)) {
886:       PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
887:       PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
888:     }

890:     /* Copy A's value to fact */
891:     m  = fact->rmap->n;
892:     nz = aij->nz;
893:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
894:     Acsr = (CsrMatrix *)Acusp->mat->mat;
895:     PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Acsr->values->data().get(), sizeof(PetscScalar) * nz, cudaMemcpyDeviceToDevice, stream));

897:     PetscCall(PetscLogGpuTimeBegin());
898:     /* Factorize fact inplace */
899:     if (m)
900:       PetscCallCUSPARSE(cusparseXcsrilu02(handle, m, nz, /* cusparseXcsrilu02 errors out with empty matrices (m=0) */
901:                                           fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ilu0Info_M, fs->policy_M, fs->factBuffer_M));
902:     if (PetscDefined(USE_DEBUG)) {
903:       int              numerical_zero;
904:       cusparseStatus_t status;
905:       status = cusparseXcsrilu02_zeroPivot(handle, fs->ilu0Info_M, &numerical_zero);
906:       PetscAssert(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Numerical zero pivot detected in csrilu02: A(%d,%d) is zero", numerical_zero, numerical_zero);
907:     }

909: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
910:     if (fs->updatedSpSVAnalysis) {
911:       if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_L, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
912:       if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_U, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
913:     } else
914: #endif
915:     {
916:       /* cusparseSpSV_analysis() is numeric, i.e., it requires valid matrix values, therefore, we do it after cusparseXcsrilu02()
917:        See discussion at https://github.com/NVIDIA/CUDALibrarySamples/issues/78
918:       */
919:       PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, fs->spsvBuffer_L));

921:       PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, fs->spsvBuffer_U));

923:       fs->updatedSpSVAnalysis = PETSC_TRUE;
924:       /* L, U values have changed, reset the flag to indicate we need to redo cusparseSpSV_analysis() for transpose solve */
925:       fs->updatedTransposeSpSVAnalysis = PETSC_FALSE;
926:     }

928:     fact->offloadmask            = PETSC_OFFLOAD_GPU;
929:     fact->ops->solve             = MatSolve_SeqAIJCUSPARSE_LU; // spMatDescr_L/U uses 32-bit indices, but cusparseSpSV_solve() supports both 32 and 64. The info is encoded in cusparseSpMatDescr_t.
930:     fact->ops->solvetranspose    = MatSolveTranspose_SeqAIJCUSPARSE_LU;
931:     fact->ops->matsolve          = NULL;
932:     fact->ops->matsolvetranspose = NULL;
933:     PetscCall(PetscLogGpuTimeEnd());
934:     PetscCall(PetscLogGpuFlops(fs->numericFactFlops));
935:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
936:   }
937:   PetscFunctionReturn(PETSC_SUCCESS);
938: }

940: static PetscErrorCode MatILUFactorSymbolic_SeqAIJCUSPARSE_ILU0(Mat fact, Mat A, IS, IS, const MatFactorInfo *info)
941: {
942:   Mat_SeqAIJCUSPARSETriFactors *fs  = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
943:   Mat_SeqAIJ                   *aij = (Mat_SeqAIJ *)fact->data;
944:   PetscInt                      m, nz;
945:   PetscDeviceContext            dctx;
946:   cudaStream_t                  stream;
947:   cusparseHandle_t              handle;

949:   PetscFunctionBegin;
950:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
951:   {
952:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

954:     if (PetscDefined(USE_DEBUG)) {
955:       PetscBool flg, diagDense;

957:       PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
958:       PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
959:       PetscCheck(A->rmap->n == A->cmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Must be square matrix, rows %" PetscInt_FMT " columns %" PetscInt_FMT, A->rmap->n, A->cmap->n);
960:       PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, NULL, &diagDense));
961:       PetscCheck(diagDense, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Matrix is missing a diagonal entry");
962:     }

964:     /* Free the old stale stuff */
965:     PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&fs));

967:     /* Copy over A's meta data to fact. Note that we also allocated fact's i,j,a on host,
968:        but they will not be used. Allocate them just for easy debugging.
969:      */
970:     PetscCall(MatDuplicateNoCreate_SeqAIJ(fact, A, MAT_DO_NOT_COPY_VALUES, PETSC_TRUE /*malloc*/));

972:     fact->offloadmask            = PETSC_OFFLOAD_BOTH;
973:     fact->factortype             = MAT_FACTOR_ILU;
974:     fact->info.factor_mallocs    = 0;
975:     fact->info.fill_ratio_given  = info->fill;
976:     fact->info.fill_ratio_needed = 1.0;

978:     aij->row = NULL;
979:     aij->col = NULL;

981:     /* ====================================================================== */
982:     /* Copy A's i, j to fact and also allocate the value array of fact.       */
983:     /* We'll do in-place factorization on fact                                */
984:     /* ====================================================================== */
985:     const PetscInt *Ai, *Aj;

987:     m  = fact->rmap->n;
988:     nz = aij->nz;

990:     PetscCallCUDA(cudaMalloc((void **)&fs->csrRowPtr32, sizeof(*fs->csrRowPtr32) * (m + 1)));
991:     PetscCallCUDA(cudaMalloc((void **)&fs->csrColIdx32, sizeof(*fs->csrColIdx32) * nz));
992:     PetscCallCUDA(cudaMalloc((void **)&fs->csrVal, sizeof(*fs->csrVal) * nz));
993:     PetscCall(MatSeqAIJCUSPARSEGetIJ(A, PETSC_FALSE, &Ai, &Aj)); // Ai is uncompressed

995:     PetscCheck(nz <= INT_MAX && m <= INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "nnz %" PetscInt_FMT " and rows %" PetscInt_FMT " overflow C int", nz, m);
996:     PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Ai, Ai + m + 1, fs->csrRowPtr32, PetscIntToCInt()));
997:     PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Aj, Aj + nz, fs->csrColIdx32, PetscIntToCInt()));

999:     /* ====================================================================== */
1000:     /* Create descriptors for M, L, U                                         */
1001:     /* ====================================================================== */
1002:     cusparseFillMode_t fillMode;
1003:     cusparseDiagType_t diagType;

1005:     PetscCallCUSPARSE(cusparseCreateMatDescr(&fs->matDescr_M));
1006:     PetscCallCUSPARSE(cusparseSetMatIndexBase(fs->matDescr_M, CUSPARSE_INDEX_BASE_ZERO));
1007:     PetscCallCUSPARSE(cusparseSetMatType(fs->matDescr_M, CUSPARSE_MATRIX_TYPE_GENERAL));

1009:     /* https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
1010:       cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
1011:       assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
1012:       all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
1013:       assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
1014:     */
1015:     fillMode = CUSPARSE_FILL_MODE_LOWER;
1016:     diagType = CUSPARSE_DIAG_TYPE_UNIT;
1017:     PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_L, m, m, nz, fs->csrRowPtr32, fs->csrColIdx32, fs->csrVal, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1018:     PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
1019:     PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));

1021:     fillMode = CUSPARSE_FILL_MODE_UPPER;
1022:     diagType = CUSPARSE_DIAG_TYPE_NON_UNIT;
1023:     PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_U, m, m, nz, fs->csrRowPtr32, fs->csrColIdx32, fs->csrVal, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1024:     PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
1025:     PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));

1027:     /* ========================================================================= */
1028:     /* Query buffer sizes for csrilu0, SpSV and allocate buffers                 */
1029:     /* ========================================================================= */
1030:     PetscCallCUSPARSE(cusparseCreateCsrilu02Info(&fs->ilu0Info_M));
1031:     if (m)
1032:       PetscCallCUSPARSE(cusparseXcsrilu02_bufferSize(handle, m, nz, /* cusparseXcsrilu02 errors out with empty matrices (m=0) */
1033:                                                      fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ilu0Info_M, &fs->factBufferSize_M));

1035:     PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(PetscScalar) * m));
1036:     PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(PetscScalar) * m));

1038:     PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
1039:     PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));

1041:     PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_L));
1042:     PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, &fs->spsvBufferSize_L));

1044:     PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_U));
1045:     PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, &fs->spsvBufferSize_U));

1047:     /* From my experiment with the example at https://github.com/NVIDIA/CUDALibrarySamples/tree/master/cuSPARSE/bicgstab,
1048:        and discussion at https://github.com/NVIDIA/CUDALibrarySamples/issues/77,
1049:        spsvBuffer_L/U can not be shared (i.e., the same) for our case, but factBuffer_M can share with either of spsvBuffer_L/U.
1050:        To save memory, we make factBuffer_M share with the bigger of spsvBuffer_L/U.
1051:      */
1052:     if (fs->spsvBufferSize_L > fs->spsvBufferSize_U) {
1053:       PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_L, (size_t)fs->factBufferSize_M)));
1054:       fs->spsvBuffer_L = fs->factBuffer_M;
1055:       PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_U, fs->spsvBufferSize_U));
1056:     } else {
1057:       PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_U, (size_t)fs->factBufferSize_M)));
1058:       fs->spsvBuffer_U = fs->factBuffer_M;
1059:       PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_L, fs->spsvBufferSize_L));
1060:     }

1062:     /* ========================================================================== */
1063:     /* Perform analysis of ilu0 on M, SpSv on L and U                             */
1064:     /* The lower(upper) triangular part of M has the same sparsity pattern as L(U)*/
1065:     /* ========================================================================== */
1066:     int              structural_zero;
1067:     cusparseStatus_t status;

1069:     fs->policy_M = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
1070:     if (m)
1071:       PetscCallCUSPARSE(cusparseXcsrilu02_analysis(handle, m, nz, /* cusparseXcsrilu02 errors out with empty matrices (m=0) */
1072:                                                    fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ilu0Info_M, fs->policy_M, fs->factBuffer_M));
1073:     if (PetscDefined(USE_DEBUG)) {
1074:       /* cusparseXcsrilu02_zeroPivot() is a blocking call. It calls cudaDeviceSynchronize() to make sure all previous kernels are done. */
1075:       status = cusparseXcsrilu02_zeroPivot(handle, fs->ilu0Info_M, &structural_zero);
1076:       PetscCheck(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Structural zero pivot detected in csrilu02: A(%d,%d) is missing", structural_zero, structural_zero);
1077:     }

1079:     /* Estimate FLOPs of the numeric factorization */
1080:     {
1081:       Mat_SeqAIJ     *Aseq = (Mat_SeqAIJ *)A->data;
1082:       PetscInt       *Ai, nzRow, nzLeft;
1083:       const PetscInt *adiag;
1084:       PetscLogDouble  flops = 0.0;

1086:       PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, &adiag, NULL));
1087:       Ai = Aseq->i;
1088:       for (PetscInt i = 0; i < m; i++) {
1089:         if (Ai[i] < adiag[i] && adiag[i] < Ai[i + 1]) { /* There are nonzeros left to the diagonal of row i */
1090:           nzRow  = Ai[i + 1] - Ai[i];
1091:           nzLeft = adiag[i] - Ai[i];
1092:           /* We want to eliminate nonzeros left to the diagonal one by one. Assume each time, nonzeros right
1093:             and include the eliminated one will be updated, which incurs a multiplication and an addition.
1094:           */
1095:           nzLeft = (nzRow - 1) / 2;
1096:           flops += nzLeft * (2.0 * nzRow - nzLeft + 1);
1097:         }
1098:       }
1099:       fs->numericFactFlops = flops;
1100:     }
1101:     fact->ops->lufactornumeric = MatILUFactorNumeric_SeqAIJCUSPARSE_ILU0;
1102:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1103:   }
1104:   PetscFunctionReturn(PETSC_SUCCESS);
1105: }

1107: static PetscErrorCode MatSolve_SeqAIJCUSPARSE_ICC0(Mat fact, Vec b, Vec x)
1108: {
1109:   Mat_SeqAIJCUSPARSETriFactors *fs  = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
1110:   Mat_SeqAIJ                   *aij = (Mat_SeqAIJ *)fact->data;
1111:   const PetscScalar            *barray;
1112:   PetscScalar                  *xarray;
1113:   PetscDeviceContext            dctx;
1114:   cusparseHandle_t              handle;

1116:   PetscFunctionBegin;
1117:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle));
1118:   {
1119:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

1121:     PetscCall(VecCUDAGetArrayWrite(x, &xarray));
1122:     PetscCall(VecCUDAGetArrayRead(b, &barray));
1123:     PetscCall(PetscLogGpuTimeBegin());

1125:     /* Solve L*y = b */
1126:     PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
1127:     PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
1128:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, /* L Y = X */
1129:                                          fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L));

1131:     /* Solve Lt*x = y */
1132:     PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
1133:     PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, /* Lt X = Y */
1134:                                          fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Lt));

1136:     PetscCall(VecCUDARestoreArrayRead(b, &barray));
1137:     PetscCall(VecCUDARestoreArrayWrite(x, &xarray));

1139:     PetscCall(PetscLogGpuTimeEnd());
1140:     PetscCall(PetscLogGpuFlops(2.0 * aij->nz - fact->rmap->n));
1141:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1142:   }
1143:   PetscFunctionReturn(PETSC_SUCCESS);
1144: }

1146: static PetscErrorCode MatICCFactorNumeric_SeqAIJCUSPARSE_ICC0(Mat fact, Mat A, const MatFactorInfo *)
1147: {
1148:   Mat_SeqAIJCUSPARSETriFactors *fs    = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
1149:   Mat_SeqAIJ                   *aij   = (Mat_SeqAIJ *)fact->data;
1150:   Mat_SeqAIJCUSPARSE           *Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1151:   CsrMatrix                    *Acsr;
1152:   PetscInt                      m, nz;
1153:   PetscBool                     flg;
1154:   PetscDeviceContext            dctx;
1155:   cudaStream_t                  stream;
1156:   cusparseHandle_t              handle;

1158:   PetscFunctionBegin;
1159:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
1160:   {
1161:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

1163:     if (PetscDefined(USE_DEBUG)) {
1164:       PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1165:       PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
1166:     }

1168:     /* Copy A's value to fact */
1169:     m  = fact->rmap->n;
1170:     nz = aij->nz;
1171:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
1172:     Acsr = (CsrMatrix *)Acusp->mat->mat;
1173:     PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Acsr->values->data().get(), sizeof(PetscScalar) * nz, cudaMemcpyDeviceToDevice, stream));

1175:     /* Factorize fact inplace */
1176:     /* https://docs.nvidia.com/cuda/cusparse/index.html#csric02_solve
1177:        csric02() only takes the lower triangular part of matrix A to perform factorization.
1178:        The matrix type must be CUSPARSE_MATRIX_TYPE_GENERAL, the fill mode and diagonal type are ignored,
1179:        and the strictly upper triangular part is ignored and never touched. It does not matter if A is Hermitian or not.
1180:        In other words, from the point of view of csric02() A is Hermitian and only the lower triangular part is provided.
1181:      */
1182:     if (m) PetscCallCUSPARSE(cusparseXcsric02(handle, m, nz, fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ic0Info_M, fs->policy_M, fs->factBuffer_M));
1183:     if (PetscDefined(USE_DEBUG)) {
1184:       int              numerical_zero;
1185:       cusparseStatus_t status;
1186:       status = cusparseXcsric02_zeroPivot(handle, fs->ic0Info_M, &numerical_zero);
1187:       PetscAssert(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Numerical zero pivot detected in csric02: A(%d,%d) is zero", numerical_zero, numerical_zero);
1188:     }

1190: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
1191:     if (fs->updatedSpSVAnalysis) {
1192:       if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_L, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
1193:       if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_Lt, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
1194:     } else
1195: #endif
1196:     {
1197:       PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, fs->spsvBuffer_L));

1199:       /* Note that cusparse reports this error if we use double and CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE
1200:       ** On entry to cusparseSpSV_analysis(): conjugate transpose (opA) is not supported for matA data type, current -> CUDA_R_64F
1201:     */
1202:       PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Lt, fs->spsvBuffer_Lt));
1203:       fs->updatedSpSVAnalysis = PETSC_TRUE;
1204:     }

1206:     fact->offloadmask            = PETSC_OFFLOAD_GPU;
1207:     fact->ops->solve             = MatSolve_SeqAIJCUSPARSE_ICC0;
1208:     fact->ops->solvetranspose    = MatSolve_SeqAIJCUSPARSE_ICC0;
1209:     fact->ops->matsolve          = NULL;
1210:     fact->ops->matsolvetranspose = NULL;
1211:     PetscCall(PetscLogGpuFlops(fs->numericFactFlops));
1212:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1213:   }
1214:   PetscFunctionReturn(PETSC_SUCCESS);
1215: }

1217: static PetscErrorCode MatICCFactorSymbolic_SeqAIJCUSPARSE_ICC0(Mat fact, Mat A, IS, const MatFactorInfo *info)
1218: {
1219:   Mat_SeqAIJCUSPARSETriFactors *fs  = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
1220:   Mat_SeqAIJ                   *aij = (Mat_SeqAIJ *)fact->data;
1221:   PetscInt                      m, nz;
1222:   PetscDeviceContext            dctx;
1223:   cudaStream_t                  stream;
1224:   cusparseHandle_t              handle;

1226:   PetscFunctionBegin;
1227:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
1228:   {
1229:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

1231:     if (PetscDefined(USE_DEBUG)) {
1232:       PetscBool flg, diagDense;

1234:       PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1235:       PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
1236:       PetscCheck(A->rmap->n == A->cmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Must be square matrix, rows %" PetscInt_FMT " columns %" PetscInt_FMT, A->rmap->n, A->cmap->n);
1237:       PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, NULL, &diagDense));
1238:       PetscCheck(diagDense, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Matrix is missing diagonal entries");
1239:     }

1241:     /* Free the old stale stuff */
1242:     PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&fs));

1244:     /* Copy over A's meta data to fact. Note that we also allocated fact's i,j,a on host,
1245:        but they will not be used. Allocate them just for easy debugging.
1246:      */
1247:     PetscCall(MatDuplicateNoCreate_SeqAIJ(fact, A, MAT_DO_NOT_COPY_VALUES, PETSC_TRUE /*malloc*/));

1249:     fact->offloadmask            = PETSC_OFFLOAD_BOTH;
1250:     fact->factortype             = MAT_FACTOR_ICC;
1251:     fact->info.factor_mallocs    = 0;
1252:     fact->info.fill_ratio_given  = info->fill;
1253:     fact->info.fill_ratio_needed = 1.0;

1255:     aij->row = NULL;
1256:     aij->col = NULL;

1258:     /* ====================================================================== */
1259:     /* Copy A's i, j to fact and also allocate the value array of fact.       */
1260:     /* We'll do in-place factorization on fact                                */
1261:     /* ====================================================================== */
1262:     const PetscInt *Ai, *Aj;

1264:     m  = fact->rmap->n;
1265:     nz = aij->nz;

1267:     PetscCallCUDA(cudaMalloc((void **)&fs->csrRowPtr32, sizeof(*fs->csrRowPtr32) * (m + 1)));
1268:     PetscCallCUDA(cudaMalloc((void **)&fs->csrColIdx32, sizeof(*fs->csrColIdx32) * nz));
1269:     PetscCallCUDA(cudaMalloc((void **)&fs->csrVal, sizeof(PetscScalar) * nz));
1270:     PetscCall(MatSeqAIJCUSPARSEGetIJ(A, PETSC_FALSE, &Ai, &Aj)); // Ai is uncompressed

1272:     PetscCheck(nz <= INT_MAX && m <= INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "nnz %" PetscInt_FMT " and rows %" PetscInt_FMT " overflow C int", nz, m);
1273:     PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Ai, Ai + m + 1, fs->csrRowPtr32, PetscIntToCInt()));
1274:     PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Aj, Aj + nz, fs->csrColIdx32, PetscIntToCInt()));

1276:     /* ====================================================================== */
1277:     /* Create mat descriptors for M, L                                        */
1278:     /* ====================================================================== */
1279:     cusparseFillMode_t fillMode;
1280:     cusparseDiagType_t diagType;

1282:     PetscCallCUSPARSE(cusparseCreateMatDescr(&fs->matDescr_M));
1283:     PetscCallCUSPARSE(cusparseSetMatIndexBase(fs->matDescr_M, CUSPARSE_INDEX_BASE_ZERO));
1284:     PetscCallCUSPARSE(cusparseSetMatType(fs->matDescr_M, CUSPARSE_MATRIX_TYPE_GENERAL));

1286:     /* https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
1287:       cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
1288:       assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
1289:       all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
1290:       assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
1291:     */
1292:     fillMode = CUSPARSE_FILL_MODE_LOWER;
1293:     diagType = CUSPARSE_DIAG_TYPE_NON_UNIT;
1294:     PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_L, m, m, nz, fs->csrRowPtr32, fs->csrColIdx32, fs->csrVal, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1295:     PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
1296:     PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));

1298:     /* ========================================================================= */
1299:     /* Query buffer sizes for csric0, SpSV of L and Lt, and allocate buffers     */
1300:     /* ========================================================================= */
1301:     PetscCallCUSPARSE(cusparseCreateCsric02Info(&fs->ic0Info_M));
1302:     if (m) PetscCallCUSPARSE(cusparseXcsric02_bufferSize(handle, m, nz, fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ic0Info_M, &fs->factBufferSize_M));

1304:     PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(PetscScalar) * m));
1305:     PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(PetscScalar) * m));

1307:     PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
1308:     PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));

1310:     PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_L));
1311:     PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, &fs->spsvBufferSize_L));

1313:     PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Lt));
1314:     PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Lt, &fs->spsvBufferSize_Lt));

1316:     /* To save device memory, we make the factorization buffer share with one of the solver buffer.
1317:        See also comments in MatILUFactorSymbolic_SeqAIJCUSPARSE_ILU0().
1318:      */
1319:     if (fs->spsvBufferSize_L > fs->spsvBufferSize_Lt) {
1320:       PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_L, (size_t)fs->factBufferSize_M)));
1321:       fs->spsvBuffer_L = fs->factBuffer_M;
1322:       PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Lt, fs->spsvBufferSize_Lt));
1323:     } else {
1324:       PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_Lt, (size_t)fs->factBufferSize_M)));
1325:       fs->spsvBuffer_Lt = fs->factBuffer_M;
1326:       PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_L, fs->spsvBufferSize_L));
1327:     }

1329:     /* ========================================================================== */
1330:     /* Perform analysis of ic0 on M                                               */
1331:     /* The lower triangular part of M has the same sparsity pattern as L          */
1332:     /* ========================================================================== */
1333:     int              structural_zero;
1334:     cusparseStatus_t status;

1336:     fs->policy_M = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
1337:     if (m) PetscCallCUSPARSE(cusparseXcsric02_analysis(handle, m, nz, fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ic0Info_M, fs->policy_M, fs->factBuffer_M));
1338:     if (PetscDefined(USE_DEBUG)) {
1339:       /* cusparseXcsric02_zeroPivot() is a blocking call. It calls cudaDeviceSynchronize() to make sure all previous kernels are done. */
1340:       status = cusparseXcsric02_zeroPivot(handle, fs->ic0Info_M, &structural_zero);
1341:       PetscCheck(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Structural zero pivot detected in csric02: A(%d,%d) is missing", structural_zero, structural_zero);
1342:     }

1344:     /* Estimate FLOPs of the numeric factorization */
1345:     {
1346:       Mat_SeqAIJ    *Aseq = (Mat_SeqAIJ *)A->data;
1347:       PetscInt      *Ai, nzRow, nzLeft;
1348:       PetscLogDouble flops = 0.0;

1350:       Ai = Aseq->i;
1351:       for (PetscInt i = 0; i < m; i++) {
1352:         nzRow = Ai[i + 1] - Ai[i];
1353:         if (nzRow > 1) {
1354:           /* We want to eliminate nonzeros left to the diagonal one by one. Assume each time, nonzeros right
1355:             and include the eliminated one will be updated, which incurs a multiplication and an addition.
1356:           */
1357:           nzLeft = (nzRow - 1) / 2;
1358:           flops += nzLeft * (2.0 * nzRow - nzLeft + 1);
1359:         }
1360:       }
1361:       fs->numericFactFlops = flops;
1362:     }
1363:     fact->ops->choleskyfactornumeric = MatICCFactorNumeric_SeqAIJCUSPARSE_ICC0;
1364:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1365:   }
1366:   PetscFunctionReturn(PETSC_SUCCESS);
1367: }

1369: static PetscErrorCode MatLUFactorNumeric_SeqAIJCUSPARSE(Mat B, Mat A, const MatFactorInfo *info)
1370: {
1371:   // use_cpu_solve is a field in Mat_SeqAIJCUSPARSE. B, a factored matrix, uses Mat_SeqAIJCUSPARSETriFactors.
1372:   Mat_SeqAIJCUSPARSE *cusparsestruct = static_cast<Mat_SeqAIJCUSPARSE *>(A->spptr);

1374:   PetscFunctionBegin;
1375:   PetscCall(MatSeqAIJCUSPARSECopyFromGPU(A));
1376:   PetscCall(MatLUFactorNumeric_SeqAIJ(B, A, info));
1377:   B->offloadmask = PETSC_OFFLOAD_CPU;

1379:   if (!cusparsestruct->use_cpu_solve) {
1380:     B->ops->solve          = MatSolve_SeqAIJCUSPARSE_LU;
1381:     B->ops->solvetranspose = MatSolveTranspose_SeqAIJCUSPARSE_LU;
1382:   }
1383:   B->ops->matsolve          = NULL;
1384:   B->ops->matsolvetranspose = NULL;

1386:   /* get the triangular factors */
1387:   if (!cusparsestruct->use_cpu_solve) PetscCall(MatSeqAIJCUSPARSEILUAnalysisAndCopyToGPU(B));
1388:   PetscFunctionReturn(PETSC_SUCCESS);
1389: }

1391: static PetscErrorCode MatLUFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS isrow, IS iscol, const MatFactorInfo *info)
1392: {
1393:   Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(B->spptr);

1395:   PetscFunctionBegin;
1396:   PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1397:   PetscCall(MatLUFactorSymbolic_SeqAIJ(B, A, isrow, iscol, info));
1398:   B->ops->lufactornumeric = MatLUFactorNumeric_SeqAIJCUSPARSE;
1399:   PetscFunctionReturn(PETSC_SUCCESS);
1400: }

1402: static PetscErrorCode MatILUFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS isrow, IS iscol, const MatFactorInfo *info)
1403: {
1404:   Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)B->spptr;

1406:   PetscFunctionBegin;
1407:   PetscBool row_identity = PETSC_FALSE, col_identity = PETSC_FALSE;
1408:   if (!info->factoronhost) {
1409:     PetscCall(ISIdentity(isrow, &row_identity));
1410:     PetscCall(ISIdentity(iscol, &col_identity));
1411:   }
1412:   if (!info->levels && row_identity && col_identity) {
1413:     PetscCall(MatILUFactorSymbolic_SeqAIJCUSPARSE_ILU0(B, A, isrow, iscol, info));
1414:   } else {
1415:     PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1416:     PetscCall(MatILUFactorSymbolic_SeqAIJ(B, A, isrow, iscol, info));
1417:     B->ops->lufactornumeric = MatLUFactorNumeric_SeqAIJCUSPARSE;
1418:   }
1419:   PetscFunctionReturn(PETSC_SUCCESS);
1420: }

1422: static PetscErrorCode MatICCFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS perm, const MatFactorInfo *info)
1423: {
1424:   Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)B->spptr;

1426:   PetscFunctionBegin;
1427:   PetscBool perm_identity = PETSC_FALSE;
1428:   if (!info->factoronhost) PetscCall(ISIdentity(perm, &perm_identity));
1429:   if (!info->levels && perm_identity) {
1430:     PetscCall(MatICCFactorSymbolic_SeqAIJCUSPARSE_ICC0(B, A, perm, info));
1431:   } else {
1432:     PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1433:     PetscCall(MatICCFactorSymbolic_SeqAIJ(B, A, perm, info));
1434:     B->ops->choleskyfactornumeric = MatCholeskyFactorNumeric_SeqAIJCUSPARSE;
1435:   }
1436:   PetscFunctionReturn(PETSC_SUCCESS);
1437: }

1439: static PetscErrorCode MatCholeskyFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS perm, const MatFactorInfo *info)
1440: {
1441:   Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)B->spptr;

1443:   PetscFunctionBegin;
1444:   PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1445:   PetscCall(MatCholeskyFactorSymbolic_SeqAIJ(B, A, perm, info));
1446:   B->ops->choleskyfactornumeric = MatCholeskyFactorNumeric_SeqAIJCUSPARSE;
1447:   PetscFunctionReturn(PETSC_SUCCESS);
1448: }

1450: static PetscErrorCode MatFactorGetSolverType_seqaij_cusparse(Mat, MatSolverType *type)
1451: {
1452:   PetscFunctionBegin;
1453:   *type = MATSOLVERCUSPARSE;
1454:   PetscFunctionReturn(PETSC_SUCCESS);
1455: }

1457: /*MC
1458:   MATSOLVERCUSPARSE = "cusparse" - A matrix type providing triangular solvers for seq matrices
1459:   on a single GPU of type, `MATSEQAIJCUSPARSE`. Currently supported
1460:   algorithms are ILU(k) and ICC(k). Typically, deeper factorizations (larger k) results in poorer
1461:   performance in the triangular solves. Full LU, and Cholesky decompositions can be solved through the
1462:   CuSPARSE triangular solve algorithm. However, the performance can be quite poor and thus these
1463:   algorithms are not recommended. This class does NOT support direct solver operations.

1465:   Level: beginner

1467: .seealso: [](ch_matrices), `Mat`, `MATSEQAIJCUSPARSE`, `PCFactorSetMatSolverType()`, `MatSolverType`, `MatCreateSeqAIJCUSPARSE()`,
1468:           `MATAIJCUSPARSE`, `MatCreateAIJCUSPARSE()`, `MatCUSPARSESetFormat()`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
1469: M*/

1471: PETSC_EXTERN PetscErrorCode MatGetFactor_seqaijcusparse_cusparse(Mat A, MatFactorType ftype, Mat *B)
1472: {
1473:   PetscInt n = A->rmap->n;

1475:   PetscFunctionBegin;
1476:   PetscCall(MatCreate(PetscObjectComm((PetscObject)A), B));
1477:   PetscCall(MatSetSizes(*B, n, n, n, n));
1478:   (*B)->factortype = ftype; // factortype makes MatSetType() allocate spptr of type Mat_SeqAIJCUSPARSETriFactors
1479:   PetscCall(MatSetType(*B, MATSEQAIJCUSPARSE));

1481:   if (A->boundtocpu && A->bindingpropagates) PetscCall(MatBindToCPU(*B, PETSC_TRUE));
1482:   if (ftype == MAT_FACTOR_LU || ftype == MAT_FACTOR_ILU || ftype == MAT_FACTOR_ILUDT) {
1483:     PetscCall(MatSetBlockSizesFromMats(*B, A, A));
1484:     if (!A->boundtocpu) {
1485:       (*B)->ops->ilufactorsymbolic = MatILUFactorSymbolic_SeqAIJCUSPARSE;
1486:       (*B)->ops->lufactorsymbolic  = MatLUFactorSymbolic_SeqAIJCUSPARSE;
1487:     } else {
1488:       (*B)->ops->ilufactorsymbolic = MatILUFactorSymbolic_SeqAIJ;
1489:       (*B)->ops->lufactorsymbolic  = MatLUFactorSymbolic_SeqAIJ;
1490:     }
1491:     PetscCall(PetscStrallocpy(MATORDERINGND, (char **)&(*B)->preferredordering[MAT_FACTOR_LU]));
1492:     PetscCall(PetscStrallocpy(MATORDERINGNATURAL, (char **)&(*B)->preferredordering[MAT_FACTOR_ILU]));
1493:     PetscCall(PetscStrallocpy(MATORDERINGNATURAL, (char **)&(*B)->preferredordering[MAT_FACTOR_ILUDT]));
1494:   } else if (ftype == MAT_FACTOR_CHOLESKY || ftype == MAT_FACTOR_ICC) {
1495:     if (!A->boundtocpu) {
1496:       (*B)->ops->iccfactorsymbolic      = MatICCFactorSymbolic_SeqAIJCUSPARSE;
1497:       (*B)->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SeqAIJCUSPARSE;
1498:     } else {
1499:       (*B)->ops->iccfactorsymbolic      = MatICCFactorSymbolic_SeqAIJ;
1500:       (*B)->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SeqAIJ;
1501:     }
1502:     PetscCall(PetscStrallocpy(MATORDERINGND, (char **)&(*B)->preferredordering[MAT_FACTOR_CHOLESKY]));
1503:     PetscCall(PetscStrallocpy(MATORDERINGNATURAL, (char **)&(*B)->preferredordering[MAT_FACTOR_ICC]));
1504:   } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Factor type not supported for CUSPARSE Matrix Types");

1506:   PetscCall(MatSeqAIJSetPreallocation(*B, MAT_SKIP_ALLOCATION, NULL));
1507:   (*B)->canuseordering = PETSC_TRUE;
1508:   PetscCall(PetscObjectComposeFunction((PetscObject)*B, "MatFactorGetSolverType_C", MatFactorGetSolverType_seqaij_cusparse));
1509:   PetscFunctionReturn(PETSC_SUCCESS);
1510: }

1512: static PetscErrorCode MatSeqAIJCUSPARSECopyFromGPU(Mat A)
1513: {
1514:   Mat_SeqAIJ                   *a    = (Mat_SeqAIJ *)A->data;
1515:   Mat_SeqAIJCUSPARSE           *cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1516:   Mat_SeqAIJCUSPARSETriFactors *fs   = (Mat_SeqAIJCUSPARSETriFactors *)A->spptr;
1517:   PetscDeviceContext            dctx;
1518:   cudaStream_t                  stream;

1520:   PetscFunctionBegin;
1521:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
1522:   if (A->offloadmask == PETSC_OFFLOAD_GPU) {
1523:     PetscCall(PetscLogEventBegin(MAT_CUSPARSECopyFromGPU, A, 0, 0, 0));
1524:     if (A->factortype == MAT_FACTOR_NONE) {
1525:       CsrMatrix *matrix = (CsrMatrix *)cusp->mat->mat;
1526:       PetscCallCUDA(cudaMemcpyAsync(a->a, matrix->values->data().get(), a->nz * sizeof(PetscScalar), cudaMemcpyDeviceToHost, stream));
1527:     } else if (fs->csrVal) {
1528:       /* We have a factorized matrix on device and are able to copy it to host */
1529:       PetscCallCUDA(cudaMemcpyAsync(a->a, fs->csrVal, a->nz * sizeof(PetscScalar), cudaMemcpyDeviceToHost, stream));
1530:     } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "No support for copying this type of factorized matrix from device to host");
1531:     PetscCall(PetscDeviceContextSynchronize(dctx));
1532:     PetscCall(PetscLogGpuToCpu(a->nz * sizeof(PetscScalar)));
1533:     PetscCall(PetscLogEventEnd(MAT_CUSPARSECopyFromGPU, A, 0, 0, 0));
1534:     A->offloadmask = PETSC_OFFLOAD_BOTH;
1535:   }
1536:   PetscFunctionReturn(PETSC_SUCCESS);
1537: }

1539: static PetscErrorCode MatSeqAIJGetArray_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1540: {
1541:   return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJGetArray(A, array);
1542: }

1544: static PetscErrorCode MatSeqAIJRestoreArray_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1545: {
1546:   return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJRestoreArray(A, array);
1547: }

1549: static PetscErrorCode MatSeqAIJGetArrayRead_SeqAIJCUSPARSE(Mat A, const PetscScalar *array[])
1550: {
1551:   return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJGetArrayRead(A, array);
1552: }

1554: static PetscErrorCode MatSeqAIJRestoreArrayRead_SeqAIJCUSPARSE(Mat A, const PetscScalar *array[])
1555: {
1556:   return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJRestoreArrayRead(A, array);
1557: }

1559: static PetscErrorCode MatSeqAIJGetArrayWrite_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1560: {
1561:   return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJGetArrayWrite(A, array);
1562: }

1564: static PetscErrorCode MatSeqAIJRestoreArrayWrite_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1565: {
1566:   return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJRestoreArrayWrite(A, array);
1567: }

1569: static PetscErrorCode MatSeqAIJGetCSRAndMemType_SeqAIJCUSPARSE(Mat A, const PetscInt **i, const PetscInt **j, PetscScalar **a, PetscMemType *mtype)
1570: {
1571:   Mat_SeqAIJCUSPARSE *cusp;
1572:   CsrMatrix          *matrix;

1574:   PetscFunctionBegin;
1575:   PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
1576:   PetscCheck(A->factortype == MAT_FACTOR_NONE, PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONGSTATE, "Not for factored matrix");
1577:   cusp = static_cast<Mat_SeqAIJCUSPARSE *>(A->spptr);
1578:   PetscCheck(cusp != NULL, PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONGSTATE, "cusp is NULL");
1579:   matrix = (CsrMatrix *)cusp->mat->mat;

1581:   if (i) *i = matrix->row_offsets->data().get();
1582:   if (j) *j = matrix->column_indices->data().get();
1583:   if (a) *a = matrix->values->data().get();
1584:   if (mtype) *mtype = PETSC_MEMTYPE_CUDA;
1585:   PetscFunctionReturn(PETSC_SUCCESS);
1586: }

1588: PETSC_INTERN PetscErrorCode MatSeqAIJCUSPARSECopyToGPU(Mat A)
1589: {
1590:   Mat_SeqAIJCUSPARSE           *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
1591:   Mat_SeqAIJCUSPARSEMultStruct *matstruct      = cusparsestruct->mat;
1592:   Mat_SeqAIJ                   *a              = (Mat_SeqAIJ *)A->data;
1593:   PetscInt                      m              = A->rmap->n, *ii, *ridx, tmp;
1594:   PetscBool                     both           = PETSC_TRUE;
1595:   PetscDeviceContext            dctx;
1596:   cudaStream_t                  stream;

1598:   PetscFunctionBegin;
1599:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
1600:   PetscCheck(!A->boundtocpu, PETSC_COMM_SELF, PETSC_ERR_GPU, "Cannot copy to GPU");
1601:   if (A->offloadmask == PETSC_OFFLOAD_UNALLOCATED || A->offloadmask == PETSC_OFFLOAD_CPU) {
1602:     if (A->nonzerostate == cusparsestruct->nonzerostate && cusparsestruct->format == MAT_CUSPARSE_CSR) { /* Copy values only */
1603:       CsrMatrix *matrix;
1604:       matrix = (CsrMatrix *)cusparsestruct->mat->mat;

1606:       PetscCheck(!a->nz || a->a, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CSR values");
1607:       PetscCall(PetscLogEventBegin(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1608:       PetscCallCUDA(cudaMemcpyAsync(matrix->values->data().get(), a->a, a->nz * sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1609:       PetscCall(PetscDeviceContextSynchronize(dctx));
1610:       PetscCall(PetscLogCpuToGpu(a->nz * sizeof(PetscScalar)));
1611:       PetscCall(PetscLogEventEnd(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1612:       PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_FALSE));
1613:     } else {
1614:       PetscInt nnz;
1615:       PetscCall(PetscLogEventBegin(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1616:       PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusparsestruct->mat, cusparsestruct->format));
1617:       PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_TRUE));
1618:       delete cusparsestruct->workVector;
1619:       delete cusparsestruct->rowoffsets_gpu;
1620:       cusparsestruct->workVector     = NULL;
1621:       cusparsestruct->rowoffsets_gpu = NULL;
1622:       try {
1623:         if (a->compressedrow.use) {
1624:           m    = a->compressedrow.nrows;
1625:           ii   = a->compressedrow.i;
1626:           ridx = a->compressedrow.rindex;
1627:         } else {
1628:           m    = A->rmap->n;
1629:           ii   = a->i;
1630:           ridx = NULL;
1631:         }
1632:         PetscCheck(ii, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CSR row data");
1633:         if (!a->a) {
1634:           nnz  = ii[m];
1635:           both = PETSC_FALSE;
1636:         } else nnz = a->nz;
1637:         PetscCheck(!nnz || a->j, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CSR column data");

1639:         /* create cusparse matrix */
1640:         cusparsestruct->nrows = m;
1641:         matstruct             = new Mat_SeqAIJCUSPARSEMultStruct;
1642:         PetscCallCUSPARSE(cusparseCreateMatDescr(&matstruct->descr));
1643:         PetscCallCUSPARSE(cusparseSetMatIndexBase(matstruct->descr, CUSPARSE_INDEX_BASE_ZERO));
1644:         PetscCallCUSPARSE(cusparseSetMatType(matstruct->descr, CUSPARSE_MATRIX_TYPE_GENERAL));

1646:         PetscCallCUDA(cudaMalloc((void **)&matstruct->alpha_one, sizeof(PetscScalar)));
1647:         PetscCallCUDA(cudaMalloc((void **)&matstruct->beta_zero, sizeof(PetscScalar)));
1648:         PetscCallCUDA(cudaMalloc((void **)&matstruct->beta_one, sizeof(PetscScalar)));
1649:         PetscCallCUDA(cudaMemcpyAsync(matstruct->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1650:         PetscCallCUDA(cudaMemcpyAsync(matstruct->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1651:         PetscCallCUDA(cudaMemcpyAsync(matstruct->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));

1653:         /* Build a hybrid/ellpack matrix if this option is chosen for the storage */
1654:         if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
1655:           /* set the matrix */
1656:           CsrMatrix *mat   = new CsrMatrix;
1657:           mat->num_rows    = m;
1658:           mat->num_cols    = A->cmap->n;
1659:           mat->num_entries = nnz;
1660:           PetscCallCXX(mat->row_offsets = new THRUSTINTARRAY(m + 1));
1661:           PetscCallCUDA(cudaMemcpyAsync(mat->row_offsets->data().get(), ii, (m + 1) * sizeof(*ii), cudaMemcpyHostToDevice, stream));
1662:           PetscCallCXX(mat->column_indices = new THRUSTINTARRAY(nnz));
1663:           PetscCallCUDA(cudaMemcpyAsync(mat->column_indices->data().get(), a->j, (nnz) * sizeof(*a->j), cudaMemcpyHostToDevice, stream));

1665:           PetscCallCXX(mat->values = new THRUSTARRAY(nnz));
1666:           if (a->a) PetscCallCUDA(cudaMemcpyAsync(mat->values->data().get(), a->a, (nnz) * sizeof(*a->a), cudaMemcpyHostToDevice, stream));

1668:           /* assign the pointer */
1669:           matstruct->mat = mat;
1670:           if (mat->num_rows) { /* cusparse errors on empty matrices! */
1671:             PetscCallCUSPARSE(cusparseCreateCsr(&matstruct->matDescr, mat->num_rows, mat->num_cols, mat->num_entries, mat->row_offsets->data().get(), mat->column_indices->data().get(), mat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1672:           }
1673:         } else if (cusparsestruct->format == MAT_CUSPARSE_ELL || cusparsestruct->format == MAT_CUSPARSE_HYB) {
1674:           SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
1675:         }

1677:         /* assign the compressed row indices */
1678:         if (a->compressedrow.use) {
1679:           PetscCallCXX(cusparsestruct->workVector = new THRUSTARRAY(m));
1680:           PetscCallCXX(matstruct->cprowIndices = new THRUSTINTARRAY(m));
1681:           PetscCallCUDA(cudaMemcpyAsync(matstruct->cprowIndices->data().get(), ridx, (m) * sizeof(*ridx), cudaMemcpyHostToDevice, stream));
1682:           tmp = m;
1683:         } else {
1684:           cusparsestruct->workVector = NULL;
1685:           matstruct->cprowIndices    = NULL;
1686:           tmp                        = 0;
1687:         }
1688:         PetscCall(PetscLogCpuToGpu(((m + 1) + (a->nz)) * sizeof(int) + tmp * sizeof(PetscInt) + (3 + (a->nz)) * sizeof(PetscScalar)));

1690:         /* assign the pointer */
1691:         cusparsestruct->mat = matstruct;
1692:       } catch (char *ex) {
1693:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "CUSPARSE error: %s", ex);
1694:       }
1695:       PetscCall(PetscDeviceContextSynchronize(dctx));
1696:       PetscCall(PetscLogEventEnd(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1697:       cusparsestruct->nonzerostate = A->nonzerostate;
1698:     }
1699:     if (both) A->offloadmask = PETSC_OFFLOAD_BOTH;
1700:   }
1701:   PetscFunctionReturn(PETSC_SUCCESS);
1702: }

1704: struct VecCUDAPlusEquals {
1705:   template <typename Tuple>
1706:   __host__ __device__ void operator()(Tuple t)
1707:   {
1708:     thrust::get<1>(t) = thrust::get<1>(t) + thrust::get<0>(t);
1709:   }
1710: };

1712: struct VecCUDAEquals {
1713:   template <typename Tuple>
1714:   __host__ __device__ void operator()(Tuple t)
1715:   {
1716:     thrust::get<1>(t) = thrust::get<0>(t);
1717:   }
1718: };

1720: struct VecCUDAEqualsReverse {
1721:   template <typename Tuple>
1722:   __host__ __device__ void operator()(Tuple t)
1723:   {
1724:     thrust::get<0>(t) = thrust::get<1>(t);
1725:   }
1726: };

1728: struct MatProductCtx_MatMatCusparse {
1729:   PetscBool      cisdense;
1730:   PetscScalar   *Bt;
1731:   Mat            X;
1732:   PetscBool      reusesym; /* Cusparse does not have split symbolic and numeric phases for sparse matmat operations */
1733:   PetscLogDouble flops;
1734:   CsrMatrix     *Bcsr;

1736:   cusparseSpMatDescr_t  matSpBDescr;
1737:   PetscBool             initialized; /* C = alpha op(A) op(B) + beta C */
1738:   cusparseDnMatDescr_t  matBDescr;
1739:   cusparseDnMatDescr_t  matCDescr;
1740:   PetscInt              Blda, Clda; /* Record leading dimensions of B and C here to detect changes*/
1741:   void                 *dBuffer4;
1742:   void                 *dBuffer5;
1743:   size_t                mmBufferSize;
1744:   void                 *mmBuffer;
1745:   void                 *mmBuffer2; /* SpGEMM WorkEstimation buffer */
1746:   cusparseSpGEMMDescr_t spgemmDesc;
1747: };

1749: static PetscErrorCode MatProductCtxDestroy_MatMatCusparse(PetscCtxRt data)
1750: {
1751:   MatProductCtx_MatMatCusparse *mmdata = *(MatProductCtx_MatMatCusparse **)data;

1753:   PetscFunctionBegin;
1754:   PetscCallCUDA(cudaFree(mmdata->Bt));
1755:   delete mmdata->Bcsr;
1756:   if (mmdata->matSpBDescr) PetscCallCUSPARSE(cusparseDestroySpMat(mmdata->matSpBDescr));
1757:   if (mmdata->matBDescr) PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matBDescr));
1758:   if (mmdata->matCDescr) PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matCDescr));
1759:   if (mmdata->spgemmDesc) PetscCallCUSPARSE(cusparseSpGEMM_destroyDescr(mmdata->spgemmDesc));
1760:   PetscCallCUDA(cudaFree(mmdata->dBuffer4));
1761:   PetscCallCUDA(cudaFree(mmdata->dBuffer5));
1762:   PetscCallCUDA(cudaFree(mmdata->mmBuffer));
1763:   PetscCallCUDA(cudaFree(mmdata->mmBuffer2));
1764:   PetscCall(MatDestroy(&mmdata->X));
1765:   PetscCall(PetscFree(mmdata));
1766:   PetscFunctionReturn(PETSC_SUCCESS);
1767: }

1769: #include <../src/mat/impls/dense/seq/dense.h>

1771: static PetscErrorCode MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA(Mat C)
1772: {
1773:   Mat_Product                  *product = C->product;
1774:   Mat                           A, B;
1775:   PetscInt                      m, n, blda, clda;
1776:   PetscBool                     flg, biscuda, compressed;
1777:   Mat_SeqAIJ                   *a;
1778:   Mat_SeqAIJCUSPARSE           *cusp;
1779:   cusparseOperation_t           opA;
1780:   const PetscScalar            *barray;
1781:   PetscScalar                  *carray;
1782:   MatProductCtx_MatMatCusparse *mmdata;
1783:   Mat_SeqAIJCUSPARSEMultStruct *mat;
1784:   CsrMatrix                    *csrmat;
1785:   PetscDeviceContext            dctx;
1786:   cudaStream_t                  stream;
1787:   cusparseHandle_t              handle;

1789:   PetscFunctionBegin;
1790:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
1791:   {
1792:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};

1794:     MatCheckProduct(C, 1);
1795:     PetscCheck(C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data empty");
1796:     mmdata = (MatProductCtx_MatMatCusparse *)product->data;
1797:     A      = product->A;
1798:     B      = product->B;
1799:     PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1800:     PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
1801:     /* currently CopyToGpu does not copy if the matrix is bound to CPU
1802:        Instead of silently accepting the wrong answer, I prefer to raise the error */
1803:     PetscCheck(!A->boundtocpu, PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Cannot bind to CPU a CUSPARSE matrix between MatProductSymbolic and MatProductNumeric phases");
1804:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
1805:     a    = (Mat_SeqAIJ *)A->data;
1806:     cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1807:     switch (product->type) {
1808:     case MATPRODUCT_AB:
1809:     case MATPRODUCT_PtAP:
1810:       mat = cusp->mat;
1811:       opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
1812:       m   = A->rmap->n;
1813:       n   = B->cmap->n;
1814:       break;
1815:     case MATPRODUCT_AtB:
1816:       if (!A->form_explicit_transpose) {
1817:         mat = cusp->mat;
1818:         opA = CUSPARSE_OPERATION_TRANSPOSE;
1819:       } else {
1820:         PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
1821:         mat = cusp->matTranspose;
1822:         opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
1823:       }
1824:       m = A->cmap->n;
1825:       n = B->cmap->n;
1826:       break;
1827:     case MATPRODUCT_ABt:
1828:     case MATPRODUCT_RARt:
1829:       mat = cusp->mat;
1830:       opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
1831:       m   = A->rmap->n;
1832:       n   = B->rmap->n;
1833:       break;
1834:     default:
1835:       SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
1836:     }
1837:     PetscCheck(mat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing Mat_SeqAIJCUSPARSEMultStruct");
1838:     csrmat = (CsrMatrix *)mat->mat;
1839:     /* when the rows of A are compressed on the device, csrmat holds only the nonempty rows of A, so the
1840:        SpMM descriptor below must be built with the full row offsets instead of those of csrmat */
1841:     compressed = (PetscBool)(mat->cprowIndices != NULL);
1842:     /* if the user passed a CPU matrix, copy the data to the GPU */
1843:     PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQDENSECUDA, &biscuda));
1844:     if (!biscuda) PetscCall(MatConvert(B, MATSEQDENSECUDA, MAT_INPLACE_MATRIX, &B));
1845:     PetscCall(MatDenseGetArrayReadAndMemType(B, &barray, nullptr));

1847:     PetscCall(MatDenseGetLDA(B, &blda));
1848:     if (product->type == MATPRODUCT_RARt || product->type == MATPRODUCT_PtAP) {
1849:       PetscCall(MatDenseGetArrayWriteAndMemType(mmdata->X, &carray, nullptr));
1850:       PetscCall(MatDenseGetLDA(mmdata->X, &clda));
1851:     } else {
1852:       PetscCall(MatDenseGetArrayWriteAndMemType(C, &carray, nullptr));
1853:       PetscCall(MatDenseGetLDA(C, &clda));
1854:     }

1856:     PetscCall(PetscLogGpuTimeBegin());
1857:     cusparseOperation_t opB = (product->type == MATPRODUCT_ABt || product->type == MATPRODUCT_RARt) ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE;
1858: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0)
1859:     cusparseSpMatDescr_t &matADescr = mat->matDescr_SpMM[opA];
1860: #else
1861:     /* mat->matDescr is also used by the SpGEMM code, which relies on its compressed dimensions, so when A is
1862:        compressed the SpMM needs a descriptor of its own */
1863:     cusparseSpMatDescr_t &matADescr = compressed ? mat->matDescr_SpMM[opA] : mat->matDescr;
1864: #endif

1866:     /* (re)allocate mmBuffer if not initialized or LDAs are different */
1867:     if (!mmdata->initialized || mmdata->Blda != blda || mmdata->Clda != clda) {
1868:       size_t mmBufferSize;
1869:       if (mmdata->initialized && mmdata->Blda != blda) {
1870:         PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matBDescr));
1871:         mmdata->matBDescr = NULL;
1872:       }
1873:       if (!mmdata->matBDescr) {
1874:         PetscCallCUSPARSE(cusparseCreateDnMat(&mmdata->matBDescr, B->rmap->n, B->cmap->n, blda, (void *)barray, cusparse_scalartype, CUSPARSE_ORDER_COL));
1875:         mmdata->Blda = blda;
1876:       }

1878:       if (mmdata->initialized && mmdata->Clda != clda) {
1879:         PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matCDescr));
1880:         mmdata->matCDescr = NULL;
1881:       }
1882:       if (!mmdata->matCDescr) { /* matCDescr is for C or mmdata->X */
1883:         PetscCallCUSPARSE(cusparseCreateDnMat(&mmdata->matCDescr, m, n, clda, (void *)carray, cusparse_scalartype, CUSPARSE_ORDER_COL));
1884:         mmdata->Clda = clda;
1885:       }

1887: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0) // tested up to 12.6.0
1888:       if (matADescr) {
1889:         PetscCallCUSPARSE(cusparseDestroySpMat(matADescr)); // Because I find I could not reuse matADescr. It could be a cusparse bug
1890:         matADescr = NULL;
1891:       }
1892: #endif

1894:       if (!matADescr) {
1895:         if (compressed) {
1896:           if (!cusp->rowoffsets_gpu) { /* the full row offsets may be absent when we did not construct the transpose with csr2csc */
1897:             cusp->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
1898:             PetscCallCUDA(cudaMemcpyAsync(cusp->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
1899:             PetscCall(PetscDeviceContextSynchronize(dctx));
1900:             PetscCall(PetscLogCpuToGpu((A->rmap->n + 1) * sizeof(PetscInt)));
1901:           }
1902:           PetscCallCUSPARSE(cusparseCreateCsr(&matADescr, A->rmap->n, csrmat->num_cols, csrmat->num_entries, cusp->rowoffsets_gpu->data().get(), csrmat->column_indices->data().get(), csrmat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1903:         } else {
1904:           PetscCallCUSPARSE(cusparseCreateCsr(&matADescr, csrmat->num_rows, csrmat->num_cols, csrmat->num_entries, csrmat->row_offsets->data().get(), csrmat->column_indices->data().get(), csrmat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1905:         }
1906:       }

1908:       PetscCallCUSPARSE(cusparseSpMM_bufferSize(handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalartype, cusp->spmmAlg, &mmBufferSize));

1910:       if ((mmdata->mmBuffer && mmdata->mmBufferSize < mmBufferSize) || !mmdata->mmBuffer) {
1911:         PetscCallCUDA(cudaFree(mmdata->mmBuffer));
1912:         PetscCallCUDA(cudaMalloc(&mmdata->mmBuffer, mmBufferSize));
1913:         mmdata->mmBufferSize = mmBufferSize;
1914:       }

1916: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0) // the _preprocess was added in 11.2.1, but PETSc worked without it until 12.4.0
1917:       PetscCallCUSPARSE(cusparseSpMM_preprocess(handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalartype, cusp->spmmAlg, mmdata->mmBuffer));
1918: #endif

1920:       mmdata->initialized = PETSC_TRUE;
1921:     } else {
1922:       /* to be safe, always update pointers of the mats */
1923:       PetscCallCUSPARSE(cusparseSpMatSetValues(matADescr, csrmat->values->data().get()));
1924:       PetscCallCUSPARSE(cusparseDnMatSetValues(mmdata->matBDescr, (void *)barray));
1925:       PetscCallCUSPARSE(cusparseDnMatSetValues(mmdata->matCDescr, (void *)carray));
1926:     }

1928:     /* do cusparseSpMM, which supports transpose on B */
1929:     PetscCallCUSPARSE(cusparseSpMM(handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalartype, cusp->spmmAlg, mmdata->mmBuffer));

1931:     PetscCall(PetscLogGpuTimeEnd());
1932:     PetscCall(PetscLogGpuFlops(n * 2.0 * csrmat->num_entries));
1933:     PetscCall(MatDenseRestoreArrayReadAndMemType(B, &barray));
1934:     if (product->type == MATPRODUCT_RARt) {
1935:       PetscCall(MatDenseRestoreArrayWriteAndMemType(mmdata->X, &carray));
1936:       PetscCall(MatMatMultNumeric_SeqDenseCUDA_SeqDenseCUDA_Internal(B, mmdata->X, C, PETSC_FALSE, PETSC_FALSE));
1937:     } else if (product->type == MATPRODUCT_PtAP) {
1938:       PetscCall(MatDenseRestoreArrayWriteAndMemType(mmdata->X, &carray));
1939:       PetscCall(MatMatMultNumeric_SeqDenseCUDA_SeqDenseCUDA_Internal(B, mmdata->X, C, PETSC_TRUE, PETSC_FALSE));
1940:     } else {
1941:       PetscCall(MatDenseRestoreArrayWriteAndMemType(C, &carray));
1942:     }
1943:     if (mmdata->cisdense) PetscCall(MatConvert(C, MATSEQDENSE, MAT_INPLACE_MATRIX, &C));
1944:     if (!biscuda) PetscCall(MatConvert(B, MATSEQDENSE, MAT_INPLACE_MATRIX, &B));
1945:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1946:   }
1947:   PetscFunctionReturn(PETSC_SUCCESS);
1948: }

1950: static PetscErrorCode MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA(Mat C)
1951: {
1952:   Mat_Product                  *product = C->product;
1953:   Mat                           A, B;
1954:   PetscInt                      m, n;
1955:   PetscBool                     cisdense, flg;
1956:   MatProductCtx_MatMatCusparse *mmdata;
1957:   Mat_SeqAIJCUSPARSE           *cusp;

1959:   PetscFunctionBegin;
1960:   MatCheckProduct(C, 1);
1961:   PetscCheck(!C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data not empty");
1962:   A = product->A;
1963:   B = product->B;
1964:   PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1965:   PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
1966:   cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1967:   PetscCheck(cusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
1968:   switch (product->type) {
1969:   case MATPRODUCT_AB:
1970:     m = A->rmap->n;
1971:     n = B->cmap->n;
1972:     PetscCall(MatSetBlockSizesFromMats(C, A, B));
1973:     break;
1974:   case MATPRODUCT_AtB:
1975:     m = A->cmap->n;
1976:     n = B->cmap->n;
1977:     if (A->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, A->cmap->bs));
1978:     if (B->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->cmap->bs));
1979:     break;
1980:   case MATPRODUCT_ABt:
1981:     m = A->rmap->n;
1982:     n = B->rmap->n;
1983:     if (A->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, A->rmap->bs));
1984:     if (B->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->rmap->bs));
1985:     break;
1986:   case MATPRODUCT_PtAP:
1987:     m = B->cmap->n;
1988:     n = B->cmap->n;
1989:     if (B->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, B->cmap->bs));
1990:     if (B->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->cmap->bs));
1991:     break;
1992:   case MATPRODUCT_RARt:
1993:     m = B->rmap->n;
1994:     n = B->rmap->n;
1995:     if (B->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, B->rmap->bs));
1996:     if (B->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->rmap->bs));
1997:     break;
1998:   default:
1999:     SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
2000:   }
2001:   PetscCall(MatSetSizes(C, m, n, m, n));
2002:   /* if C is of type MATSEQDENSE (CPU), perform the operation on the GPU and then copy on the CPU */
2003:   PetscCall(PetscObjectTypeCompare((PetscObject)C, MATSEQDENSE, &cisdense));
2004:   PetscCall(MatSetType(C, MATSEQDENSECUDA));
2005:   /* keep the VecType of B, e.g. VECKOKKOS from MatCreateDenseFromVecType(), but only if B is a MATSEQDENSECUDA like C, since the VecType of a MATSEQDENSE B, e.g. VECSTANDARD, is not valid for C */
2006:   PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQDENSECUDA, &flg));
2007:   if (flg) PetscCall(MatSetVecType(C, B->defaultvectype));

2009:   /* product data */
2010:   PetscCall(PetscNew(&mmdata));
2011:   mmdata->cisdense = cisdense;
2012:   /* for these products we need intermediate storage */
2013:   if (product->type == MATPRODUCT_RARt || product->type == MATPRODUCT_PtAP) {
2014:     PetscCall(MatCreate(PetscObjectComm((PetscObject)C), &mmdata->X));
2015:     PetscCall(MatSetType(mmdata->X, MATSEQDENSECUDA));
2016:     if (product->type == MATPRODUCT_RARt) { /* do not preallocate, since the first call to MatDenseCUDAGetArray will preallocate on the GPU for us */
2017:       PetscCall(MatSetSizes(mmdata->X, A->rmap->n, B->rmap->n, A->rmap->n, B->rmap->n));
2018:     } else {
2019:       PetscCall(MatSetSizes(mmdata->X, A->rmap->n, B->cmap->n, A->rmap->n, B->cmap->n));
2020:     }
2021:   }
2022:   C->product->data    = mmdata;
2023:   C->product->destroy = MatProductCtxDestroy_MatMatCusparse;

2025:   C->ops->productnumeric = MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA;
2026:   PetscFunctionReturn(PETSC_SUCCESS);
2027: }

2029: static PetscErrorCode MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE(Mat C)
2030: {
2031:   Mat_Product                  *product = C->product;
2032:   Mat                           A, B;
2033:   Mat_SeqAIJCUSPARSE           *Acusp, *Bcusp, *Ccusp;
2034:   Mat_SeqAIJ                   *c = (Mat_SeqAIJ *)C->data;
2035:   Mat_SeqAIJCUSPARSEMultStruct *Amat, *Bmat, *Cmat;
2036:   CsrMatrix                    *Acsr, *Bcsr, *Ccsr;
2037:   PetscBool                     flg;
2038:   MatProductType                ptype;
2039:   MatProductCtx_MatMatCusparse *mmdata;
2040:   cusparseSpMatDescr_t          BmatSpDescr;
2041:   cusparseOperation_t           opA = CUSPARSE_OPERATION_NON_TRANSPOSE, opB = CUSPARSE_OPERATION_NON_TRANSPOSE; /* cuSPARSE spgemm doesn't support transpose yet */
2042:   PetscDeviceContext            dctx;
2043:   cusparseHandle_t              handle;

2045:   PetscFunctionBegin;
2046:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle));
2047:   {
2048:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};

2050:     MatCheckProduct(C, 1);
2051:     PetscCheck(C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data empty");
2052:     PetscCall(PetscObjectTypeCompare((PetscObject)C, MATSEQAIJCUSPARSE, &flg));
2053:     PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for C of type %s", ((PetscObject)C)->type_name);
2054:     mmdata = (MatProductCtx_MatMatCusparse *)C->product->data;
2055:     A      = product->A;
2056:     B      = product->B;
2057:     if (mmdata->reusesym) { /* this happens when api_user is true, meaning that the matrix values have been already computed in the MatProductSymbolic phase */
2058:       mmdata->reusesym = PETSC_FALSE;
2059:       Ccusp            = (Mat_SeqAIJCUSPARSE *)C->spptr;
2060:       PetscCheck(Ccusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2061:       Cmat = Ccusp->mat;
2062:       PetscCheck(Cmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C mult struct for product type %s", MatProductTypes[C->product->type]);
2063:       Ccsr = (CsrMatrix *)Cmat->mat;
2064:       PetscCheck(Ccsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C CSR struct");
2065:       goto finalize;
2066:     }
2067:     if (!c->nz) goto finalize;
2068:     PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
2069:     PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
2070:     PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQAIJCUSPARSE, &flg));
2071:     PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for B of type %s", ((PetscObject)B)->type_name);
2072:     PetscCheck(!A->boundtocpu, PetscObjectComm((PetscObject)C), PETSC_ERR_ARG_WRONG, "Cannot bind to CPU a CUSPARSE matrix between MatProductSymbolic and MatProductNumeric phases");
2073:     PetscCheck(!B->boundtocpu, PetscObjectComm((PetscObject)C), PETSC_ERR_ARG_WRONG, "Cannot bind to CPU a CUSPARSE matrix between MatProductSymbolic and MatProductNumeric phases");
2074:     Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
2075:     Bcusp = (Mat_SeqAIJCUSPARSE *)B->spptr;
2076:     Ccusp = (Mat_SeqAIJCUSPARSE *)C->spptr;
2077:     PetscCheck(Acusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2078:     PetscCheck(Bcusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2079:     PetscCheck(Ccusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2080:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
2081:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));

2083:     ptype = product->type;
2084:     if (A->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_AtB) {
2085:       ptype = MATPRODUCT_AB;
2086:       PetscCheck(product->symbolic_used_the_fact_A_is_symmetric, PetscObjectComm((PetscObject)C), PETSC_ERR_PLIB, "Symbolic should have been built using the fact that A is symmetric");
2087:     }
2088:     if (B->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_ABt) {
2089:       ptype = MATPRODUCT_AB;
2090:       PetscCheck(product->symbolic_used_the_fact_B_is_symmetric, PetscObjectComm((PetscObject)C), PETSC_ERR_PLIB, "Symbolic should have been built using the fact that B is symmetric");
2091:     }
2092:     switch (ptype) {
2093:     case MATPRODUCT_AB:
2094:       Amat = Acusp->mat;
2095:       Bmat = Bcusp->mat;
2096:       break;
2097:     case MATPRODUCT_AtB:
2098:       PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
2099:       Amat = Acusp->matTranspose;
2100:       Bmat = Bcusp->mat;
2101:       break;
2102:     case MATPRODUCT_ABt:
2103:       Amat = Acusp->mat;
2104:       PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(B));
2105:       Bmat = Bcusp->matTranspose;
2106:       break;
2107:     default:
2108:       SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
2109:     }
2110:     Cmat = Ccusp->mat;
2111:     PetscCheck(Amat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A mult struct for product type %s", MatProductTypes[ptype]);
2112:     PetscCheck(Bmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B mult struct for product type %s", MatProductTypes[ptype]);
2113:     PetscCheck(Cmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C mult struct for product type %s", MatProductTypes[ptype]);
2114:     Acsr = (CsrMatrix *)Amat->mat;
2115:     Bcsr = mmdata->Bcsr ? mmdata->Bcsr : (CsrMatrix *)Bmat->mat; /* B may be in compressed row storage */
2116:     Ccsr = (CsrMatrix *)Cmat->mat;
2117:     PetscCheck(Acsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A CSR struct");
2118:     PetscCheck(Bcsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B CSR struct");
2119:     PetscCheck(Ccsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C CSR struct");
2120:     PetscCall(PetscLogGpuTimeBegin());
2121:     BmatSpDescr = mmdata->Bcsr ? mmdata->matSpBDescr : Bmat->matDescr; /* B may be in compressed row storage */
2122:     PetscCallCUSPARSE(cusparseSpGEMM_compute(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, mmdata->mmBuffer));
2123:     PetscCallCUSPARSE(cusparseSpGEMM_copy(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc));
2124:     PetscCall(PetscLogGpuFlops(mmdata->flops));
2125:     PetscCall(PetscDeviceContextSynchronize(dctx));
2126:     PetscCall(PetscLogGpuTimeEnd());
2127:     C->offloadmask = PETSC_OFFLOAD_GPU;
2128:   finalize:
2129:     /* shorter version of MatAssemblyEnd_SeqAIJ */
2130:     PetscCall(PetscInfo(C, "Matrix size: %" PetscInt_FMT " X %" PetscInt_FMT "; storage space: 0 unneeded, %" PetscInt_FMT " used\n", C->rmap->n, C->cmap->n, c->nz));
2131:     PetscCall(PetscInfo(C, "Number of mallocs during MatSetValues() is 0\n"));
2132:     PetscCall(PetscInfo(C, "Maximum nonzeros in any row is %" PetscInt_FMT "\n", c->rmax));
2133:     c->reallocs = 0;
2134:     C->info.mallocs += 0;
2135:     C->info.nz_unneeded = 0;
2136:     C->assembled = C->was_assembled = PETSC_TRUE;
2137:     C->num_ass++;
2138:   }
2139:   PetscFunctionReturn(PETSC_SUCCESS);
2140: }

2142: static PetscErrorCode MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE(Mat C)
2143: {
2144:   Mat_Product                  *product = C->product;
2145:   Mat                           A, B;
2146:   Mat_SeqAIJCUSPARSE           *Acusp, *Bcusp, *Ccusp;
2147:   Mat_SeqAIJ                   *a, *b, *c;
2148:   Mat_SeqAIJCUSPARSEMultStruct *Amat, *Bmat, *Cmat;
2149:   CsrMatrix                    *Acsr, *Bcsr, *Ccsr;
2150:   PetscInt                      i, j, m, n, k;
2151:   PetscBool                     flg;
2152:   MatProductType                ptype;
2153:   MatProductCtx_MatMatCusparse *mmdata;
2154:   PetscLogDouble                flops;
2155:   PetscBool                     biscompressed, ciscompressed;
2156:   int64_t                       C_num_rows1, C_num_cols1, C_nnz1;
2157:   cusparseSpMatDescr_t          BmatSpDescr;
2158:   cusparseOperation_t           opA = CUSPARSE_OPERATION_NON_TRANSPOSE, opB = CUSPARSE_OPERATION_NON_TRANSPOSE; /* cuSPARSE spgemm doesn't support transpose yet */
2159:   size_t                        bufSize2;
2160:   PetscDeviceContext            dctx;
2161:   cudaStream_t                  stream;
2162:   cusparseHandle_t              handle;

2164:   PetscFunctionBegin;
2165:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
2166:   {
2167:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};

2169:     MatCheckProduct(C, 1);
2170:     PetscCheck(!C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data not empty");
2171:     A = product->A;
2172:     B = product->B;
2173:     PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
2174:     PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
2175:     PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQAIJCUSPARSE, &flg));
2176:     PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for B of type %s", ((PetscObject)B)->type_name);
2177:     a = (Mat_SeqAIJ *)A->data;
2178:     b = (Mat_SeqAIJ *)B->data;
2179:     /* product data */
2180:     PetscCall(PetscNew(&mmdata));
2181:     C->product->data    = mmdata;
2182:     C->product->destroy = MatProductCtxDestroy_MatMatCusparse;

2184:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
2185:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
2186:     Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr; /* Access spptr after MatSeqAIJCUSPARSECopyToGPU, not before */
2187:     Bcusp = (Mat_SeqAIJCUSPARSE *)B->spptr;
2188:     PetscCheck(Acusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2189:     PetscCheck(Bcusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");

2191:     ptype = product->type;
2192:     if (A->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_AtB) {
2193:       ptype                                          = MATPRODUCT_AB;
2194:       product->symbolic_used_the_fact_A_is_symmetric = PETSC_TRUE;
2195:     }
2196:     if (B->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_ABt) {
2197:       ptype                                          = MATPRODUCT_AB;
2198:       product->symbolic_used_the_fact_B_is_symmetric = PETSC_TRUE;
2199:     }
2200:     biscompressed = PETSC_FALSE;
2201:     ciscompressed = PETSC_FALSE;
2202:     switch (ptype) {
2203:     case MATPRODUCT_AB:
2204:       m    = A->rmap->n;
2205:       n    = B->cmap->n;
2206:       k    = A->cmap->n;
2207:       Amat = Acusp->mat;
2208:       Bmat = Bcusp->mat;
2209:       if (a->compressedrow.use) ciscompressed = PETSC_TRUE;
2210:       if (b->compressedrow.use) biscompressed = PETSC_TRUE;
2211:       break;
2212:     case MATPRODUCT_AtB:
2213:       m = A->cmap->n;
2214:       n = B->cmap->n;
2215:       k = A->rmap->n;
2216:       PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
2217:       Amat = Acusp->matTranspose;
2218:       Bmat = Bcusp->mat;
2219:       if (b->compressedrow.use) biscompressed = PETSC_TRUE;
2220:       break;
2221:     case MATPRODUCT_ABt:
2222:       m = A->rmap->n;
2223:       n = B->rmap->n;
2224:       k = A->cmap->n;
2225:       PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(B));
2226:       Amat = Acusp->mat;
2227:       Bmat = Bcusp->matTranspose;
2228:       if (a->compressedrow.use) ciscompressed = PETSC_TRUE;
2229:       break;
2230:     default:
2231:       SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
2232:     }

2234:     /* create cusparse matrix */
2235:     PetscCall(MatSetSizes(C, m, n, m, n));
2236:     PetscCall(MatSetType(C, MATSEQAIJCUSPARSE));
2237:     c     = (Mat_SeqAIJ *)C->data;
2238:     Ccusp = (Mat_SeqAIJCUSPARSE *)C->spptr;
2239:     Cmat  = new Mat_SeqAIJCUSPARSEMultStruct;
2240:     Ccsr  = new CsrMatrix;

2242:     c->compressedrow.use = ciscompressed;
2243:     if (c->compressedrow.use) { /* if a is in compressed row, than c will be in compressed row format */
2244:       c->compressedrow.nrows = a->compressedrow.nrows;
2245:       PetscCall(PetscMalloc2(c->compressedrow.nrows + 1, &c->compressedrow.i, c->compressedrow.nrows, &c->compressedrow.rindex));
2246:       PetscCall(PetscArraycpy(c->compressedrow.rindex, a->compressedrow.rindex, c->compressedrow.nrows));
2247:       Ccusp->workVector  = new THRUSTARRAY(c->compressedrow.nrows);
2248:       Cmat->cprowIndices = new THRUSTINTARRAY(c->compressedrow.nrows);
2249:       PetscCallCUDA(cudaMemcpyAsync(Cmat->cprowIndices->data().get(), c->compressedrow.rindex, (c->compressedrow.nrows) * sizeof(*c->compressedrow.rindex), cudaMemcpyHostToDevice, stream));
2250:     } else {
2251:       c->compressedrow.nrows  = 0;
2252:       c->compressedrow.i      = NULL;
2253:       c->compressedrow.rindex = NULL;
2254:       Ccusp->workVector       = NULL;
2255:       Cmat->cprowIndices      = NULL;
2256:     }
2257:     Ccusp->nrows      = ciscompressed ? c->compressedrow.nrows : m;
2258:     Ccusp->mat        = Cmat;
2259:     Ccusp->mat->mat   = Ccsr;
2260:     Ccsr->num_rows    = Ccusp->nrows;
2261:     Ccsr->num_cols    = n;
2262:     Ccsr->row_offsets = new THRUSTINTARRAY(Ccusp->nrows + 1);
2263:     PetscCallCUSPARSE(cusparseCreateMatDescr(&Cmat->descr));
2264:     PetscCallCUSPARSE(cusparseSetMatIndexBase(Cmat->descr, CUSPARSE_INDEX_BASE_ZERO));
2265:     PetscCallCUSPARSE(cusparseSetMatType(Cmat->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
2266:     PetscCallCUDA(cudaMalloc((void **)&Cmat->alpha_one, sizeof(PetscScalar)));
2267:     PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_zero, sizeof(PetscScalar)));
2268:     PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_one, sizeof(PetscScalar)));
2269:     PetscCallCUDA(cudaMemcpyAsync(Cmat->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
2270:     PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
2271:     PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
2272:     if (!Ccsr->num_rows || !Ccsr->num_cols || !a->nz || !b->nz) { /* cusparse raise errors in different calls when matrices have zero rows/columns! */
2273:       PetscCallThrust(thrust::fill(thrust::cuda::par.on(stream), Ccsr->row_offsets->begin(), Ccsr->row_offsets->end(), 0));
2274:       c->nz                = 0;
2275:       Ccsr->column_indices = new THRUSTINTARRAY(c->nz);
2276:       Ccsr->values         = new THRUSTARRAY(c->nz);
2277:       goto finalizesym;
2278:     }

2280:     PetscCheck(Amat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A mult struct for product type %s", MatProductTypes[ptype]);
2281:     PetscCheck(Bmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B mult struct for product type %s", MatProductTypes[ptype]);
2282:     Acsr = (CsrMatrix *)Amat->mat;
2283:     if (!biscompressed) {
2284:       Bcsr        = (CsrMatrix *)Bmat->mat;
2285:       BmatSpDescr = Bmat->matDescr;
2286:     } else { /* we need to use row offsets for the full matrix */
2287:       CsrMatrix *cBcsr     = (CsrMatrix *)Bmat->mat;
2288:       Bcsr                 = new CsrMatrix;
2289:       Bcsr->num_rows       = B->rmap->n;
2290:       Bcsr->num_cols       = cBcsr->num_cols;
2291:       Bcsr->num_entries    = cBcsr->num_entries;
2292:       Bcsr->column_indices = cBcsr->column_indices;
2293:       Bcsr->values         = cBcsr->values;
2294:       if (!Bcusp->rowoffsets_gpu) {
2295:         Bcusp->rowoffsets_gpu = new THRUSTINTARRAY(B->rmap->n + 1);
2296:         PetscCallCUDA(cudaMemcpyAsync(Bcusp->rowoffsets_gpu->data().get(), b->i, (B->rmap->n + 1) * sizeof(*b->i), cudaMemcpyHostToDevice, stream));
2297:         PetscCall(PetscLogCpuToGpu((B->rmap->n + 1) * sizeof(PetscInt)));
2298:       }
2299:       Bcsr->row_offsets = Bcusp->rowoffsets_gpu;
2300:       mmdata->Bcsr      = Bcsr;
2301:       if (Bcsr->num_rows && Bcsr->num_cols) {
2302:         PetscCallCUSPARSE(cusparseCreateCsr(&mmdata->matSpBDescr, Bcsr->num_rows, Bcsr->num_cols, Bcsr->num_entries, Bcsr->row_offsets->data().get(), Bcsr->column_indices->data().get(), Bcsr->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
2303:       }
2304:       BmatSpDescr = mmdata->matSpBDescr;
2305:     }
2306:     PetscCheck(Acsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A CSR struct");
2307:     PetscCheck(Bcsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B CSR struct");
2308:     /* precompute flops count */
2309:     if (ptype == MATPRODUCT_AB) {
2310:       for (i = 0, flops = 0; i < A->rmap->n; i++) {
2311:         const PetscInt st = a->i[i];
2312:         const PetscInt en = a->i[i + 1];
2313:         for (j = st; j < en; j++) {
2314:           const PetscInt brow = a->j[j];
2315:           flops += 2. * (b->i[brow + 1] - b->i[brow]);
2316:         }
2317:       }
2318:     } else if (ptype == MATPRODUCT_AtB) {
2319:       for (i = 0, flops = 0; i < A->rmap->n; i++) {
2320:         const PetscInt anzi = a->i[i + 1] - a->i[i];
2321:         const PetscInt bnzi = b->i[i + 1] - b->i[i];
2322:         flops += (2. * anzi) * bnzi;
2323:       }
2324:     } else { /* TODO */
2325:       flops = 0.;
2326:     }

2328:     mmdata->flops = flops;
2329:     PetscCall(PetscLogGpuTimeBegin());

2331:     // cuda-12.2 requires non-null csrRowOffsets
2332:     PetscCallCUSPARSE(cusparseCreateCsr(&Cmat->matDescr, Ccsr->num_rows, Ccsr->num_cols, 0, Ccsr->row_offsets->data().get(), NULL, NULL, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
2333:     PetscCallCUSPARSE(cusparseSpGEMM_createDescr(&mmdata->spgemmDesc));
2334:     // Note that cusparseSpGEMMreuse is deprecated in CUDA 13.2.1

2336:     PetscCheck(!PetscDefined(USE_64BIT_INDICES) || PETSC_PKG_CUDA_VERSION_GE(13, 0, 0), PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "cusparseSpGEMM did not support 64-bit indices before CUDA 13.0. Update your CUDA installation.");
2337:     /* ask bufferSize bytes for external memory */
2338:     PetscCallCUSPARSE(cusparseSpGEMM_workEstimation(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufSize2, NULL));
2339:     PetscCallCUDA(cudaMalloc((void **)&mmdata->mmBuffer2, bufSize2));
2340:     /* inspect the matrices A and B to understand the memory requirement for the next step */
2341:     PetscCallCUSPARSE(cusparseSpGEMM_workEstimation(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufSize2, mmdata->mmBuffer2));
2342:     /* ask bufferSize again bytes for external memory */
2343:     PetscCallCUSPARSE(cusparseSpGEMM_compute(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, NULL));
2344:     /* The CUSPARSE documentation is not clear, nor the API
2345:        We need both buffers to perform the operations properly!
2346:        mmdata->mmBuffer2 does not appear anywhere in the compute/copy API
2347:        it only appears for the workEstimation stuff, but it seems it is needed in compute, so probably the address
2348:        is stored in the descriptor! What a messy API... */
2349:     PetscCallCUDA(cudaMalloc((void **)&mmdata->mmBuffer, mmdata->mmBufferSize));
2350:     /* compute the intermediate product of A * B */
2351:     PetscCallCUSPARSE(cusparseSpGEMM_compute(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, mmdata->mmBuffer));
2352:     /* get matrix C non-zero entries C_nnz1 */
2353:     PetscCallCUSPARSE(cusparseSpMatGetSize(Cmat->matDescr, &C_num_rows1, &C_num_cols1, &C_nnz1));
2354:     PetscCall(PetscIntCast(C_nnz1, &c->nz));
2355:     PetscCall(PetscInfo(C, "Buffer sizes for type %s, result %" PetscInt_FMT " x %" PetscInt_FMT " (k %" PetscInt_FMT ", nzA %" PetscInt_FMT ", nzB %" PetscInt_FMT ", nzC %" PetscInt_FMT ") are: %ldKB %ldKB\n", MatProductTypes[ptype], m, n, k, a->nz, b->nz, c->nz, bufSize2 / 1024,
2356:                         mmdata->mmBufferSize / 1024));
2357:     Ccsr->column_indices = new THRUSTINTARRAY(c->nz);
2358:     PetscCallCUDA(cudaPeekAtLastError()); /* catch out of memory errors */
2359:     Ccsr->values = new THRUSTARRAY(c->nz);
2360:     PetscCallCUDA(cudaPeekAtLastError()); /* catch out of memory errors */
2361:     if (c->nz) PetscCallCUSPARSE(cusparseCsrSetPointers(Cmat->matDescr, Ccsr->row_offsets->data().get(), Ccsr->column_indices->data().get(), Ccsr->values->data().get()));
2362:     PetscCallCUSPARSE(cusparseSpGEMM_copy(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc));
2363:     PetscCall(PetscLogGpuFlops(mmdata->flops));
2364:     PetscCall(PetscLogGpuTimeEnd());
2365:   finalizesym:
2366:     c->free_a = PETSC_TRUE;
2367:     PetscCall(PetscShmgetAllocateArray(c->nz, sizeof(PetscInt), (void **)&c->j));
2368:     PetscCall(PetscShmgetAllocateArray(m + 1, sizeof(PetscInt), (void **)&c->i));
2369:     c->free_ij = PETSC_TRUE;

2371:     PetscInt *d_i = c->i;
2372:     if (ciscompressed) d_i = c->compressedrow.i;
2373:     PetscCallCUDA(cudaMemcpyAsync(d_i, Ccsr->row_offsets->data().get(), Ccsr->row_offsets->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
2374:     PetscCallCUDA(cudaMemcpyAsync(c->j, Ccsr->column_indices->data().get(), Ccsr->column_indices->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
2375:     PetscCall(PetscDeviceContextSynchronize(dctx));
2376:     if (ciscompressed) { /* need to expand host row offsets */
2377:       PetscInt r = 0;
2378:       c->i[0]    = 0;
2379:       for (k = 0; k < c->compressedrow.nrows; k++) {
2380:         const PetscInt next = c->compressedrow.rindex[k];
2381:         const PetscInt old  = c->compressedrow.i[k];
2382:         for (; r < next; r++) c->i[r + 1] = old;
2383:       }
2384:       for (; r < m; r++) c->i[r + 1] = c->compressedrow.i[c->compressedrow.nrows];
2385:     }
2386:     PetscCall(PetscLogGpuToCpu((Ccsr->column_indices->size() + Ccsr->row_offsets->size()) * sizeof(PetscInt)));
2387:     PetscCall(PetscMalloc1(m, &c->ilen));
2388:     PetscCall(PetscMalloc1(m, &c->imax));
2389:     c->maxnz         = c->nz;
2390:     c->nonzerorowcnt = 0;
2391:     c->rmax          = 0;
2392:     for (k = 0; k < m; k++) {
2393:       const PetscInt nn = c->i[k + 1] - c->i[k];
2394:       c->ilen[k] = c->imax[k] = nn;
2395:       c->nonzerorowcnt += (PetscInt)!!nn;
2396:       c->rmax = PetscMax(c->rmax, nn);
2397:     }
2398:     PetscCall(PetscMalloc1(c->nz, &c->a));
2399:     Ccsr->num_entries = c->nz;

2401:     C->nonzerostate++;
2402:     PetscCall(PetscLayoutSetUp(C->rmap));
2403:     PetscCall(PetscLayoutSetUp(C->cmap));
2404:     Ccusp->nonzerostate = C->nonzerostate;
2405:     C->offloadmask      = PETSC_OFFLOAD_UNALLOCATED;
2406:     C->preallocated     = PETSC_TRUE;
2407:     C->assembled        = PETSC_FALSE;
2408:     C->was_assembled    = PETSC_FALSE;
2409:     if (product->api_user && A->offloadmask == PETSC_OFFLOAD_BOTH && B->offloadmask == PETSC_OFFLOAD_BOTH) { /* flag the matrix C values as computed, so that the numeric phase will only call MatAssembly */
2410:       mmdata->reusesym = PETSC_TRUE;
2411:       C->offloadmask   = PETSC_OFFLOAD_GPU;
2412:     }
2413:     C->ops->productnumeric = MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE;
2414:   }
2415:   PetscFunctionReturn(PETSC_SUCCESS);
2416: }

2418: PETSC_INTERN PetscErrorCode MatProductSetFromOptions_SeqAIJ_SeqDense(Mat);

2420: /* handles sparse or dense B */
2421: static PetscErrorCode MatProductSetFromOptions_SeqAIJCUSPARSE(Mat mat)
2422: {
2423:   Mat_Product *product = mat->product;
2424:   PetscBool    isdense = PETSC_FALSE, Biscusp = PETSC_FALSE, Ciscusp = PETSC_TRUE;

2426:   PetscFunctionBegin;
2427:   MatCheckProduct(mat, 1);
2428:   PetscCall(PetscObjectBaseTypeCompare((PetscObject)product->B, MATSEQDENSE, &isdense));
2429:   if (!product->A->boundtocpu && !product->B->boundtocpu) PetscCall(PetscObjectTypeCompare((PetscObject)product->B, MATSEQAIJCUSPARSE, &Biscusp));
2430:   if (product->type == MATPRODUCT_ABC) {
2431:     Ciscusp = PETSC_FALSE;
2432:     if (!product->C->boundtocpu) PetscCall(PetscObjectTypeCompare((PetscObject)product->C, MATSEQAIJCUSPARSE, &Ciscusp));
2433:   }
2434:   if (Biscusp && Ciscusp) { /* we can always select the CPU backend */
2435:     PetscBool usecpu = PETSC_FALSE;
2436:     switch (product->type) {
2437:     case MATPRODUCT_AB:
2438:       if (product->api_user) {
2439:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatMatMult", "Mat");
2440:         PetscCall(PetscOptionsBool("-matmatmult_backend_cpu", "Use CPU code", "MatMatMult", usecpu, &usecpu, NULL));
2441:         PetscOptionsEnd();
2442:       } else {
2443:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_AB", "Mat");
2444:         PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatMatMult", usecpu, &usecpu, NULL));
2445:         PetscOptionsEnd();
2446:       }
2447:       break;
2448:     case MATPRODUCT_AtB:
2449:       if (product->api_user) {
2450:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatTransposeMatMult", "Mat");
2451:         PetscCall(PetscOptionsBool("-mattransposematmult_backend_cpu", "Use CPU code", "MatTransposeMatMult", usecpu, &usecpu, NULL));
2452:         PetscOptionsEnd();
2453:       } else {
2454:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_AtB", "Mat");
2455:         PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatTransposeMatMult", usecpu, &usecpu, NULL));
2456:         PetscOptionsEnd();
2457:       }
2458:       break;
2459:     case MATPRODUCT_PtAP:
2460:       if (product->api_user) {
2461:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatPtAP", "Mat");
2462:         PetscCall(PetscOptionsBool("-matptap_backend_cpu", "Use CPU code", "MatPtAP", usecpu, &usecpu, NULL));
2463:         PetscOptionsEnd();
2464:       } else {
2465:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_PtAP", "Mat");
2466:         PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatPtAP", usecpu, &usecpu, NULL));
2467:         PetscOptionsEnd();
2468:       }
2469:       break;
2470:     case MATPRODUCT_RARt:
2471:       if (product->api_user) {
2472:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatRARt", "Mat");
2473:         PetscCall(PetscOptionsBool("-matrart_backend_cpu", "Use CPU code", "MatRARt", usecpu, &usecpu, NULL));
2474:         PetscOptionsEnd();
2475:       } else {
2476:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_RARt", "Mat");
2477:         PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatRARt", usecpu, &usecpu, NULL));
2478:         PetscOptionsEnd();
2479:       }
2480:       break;
2481:     case MATPRODUCT_ABC:
2482:       if (product->api_user) {
2483:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatMatMatMult", "Mat");
2484:         PetscCall(PetscOptionsBool("-matmatmatmult_backend_cpu", "Use CPU code", "MatMatMatMult", usecpu, &usecpu, NULL));
2485:         PetscOptionsEnd();
2486:       } else {
2487:         PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_ABC", "Mat");
2488:         PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatMatMatMult", usecpu, &usecpu, NULL));
2489:         PetscOptionsEnd();
2490:       }
2491:       break;
2492:     default:
2493:       break;
2494:     }
2495:     if (usecpu) Biscusp = Ciscusp = PETSC_FALSE;
2496:   }
2497:   /* dispatch */
2498:   if (isdense) {
2499:     switch (product->type) {
2500:     case MATPRODUCT_AB:
2501:     case MATPRODUCT_AtB:
2502:     case MATPRODUCT_ABt:
2503:     case MATPRODUCT_PtAP:
2504:     case MATPRODUCT_RARt:
2505:       if (product->A->boundtocpu) {
2506:         PetscCall(MatProductSetFromOptions_SeqAIJ_SeqDense(mat));
2507:       } else {
2508:         mat->ops->productsymbolic = MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA;
2509:       }
2510:       break;
2511:     case MATPRODUCT_ABC:
2512:       mat->ops->productsymbolic = MatProductSymbolic_ABC_Basic;
2513:       break;
2514:     default:
2515:       break;
2516:     }
2517:   } else if (Biscusp && Ciscusp) {
2518:     switch (product->type) {
2519:     case MATPRODUCT_AB:
2520:     case MATPRODUCT_AtB:
2521:     case MATPRODUCT_ABt:
2522:       mat->ops->productsymbolic = MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE;
2523:       break;
2524:     case MATPRODUCT_PtAP:
2525:     case MATPRODUCT_RARt:
2526:     case MATPRODUCT_ABC:
2527:       mat->ops->productsymbolic = MatProductSymbolic_ABC_Basic;
2528:       break;
2529:     default:
2530:       break;
2531:     }
2532:   } else { /* fallback for AIJ */
2533:     PetscCall(MatProductSetFromOptions_SeqAIJ(mat));
2534:   }
2535:   PetscFunctionReturn(PETSC_SUCCESS);
2536: }

2538: static PetscErrorCode MatMult_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy)
2539: {
2540:   PetscFunctionBegin;
2541:   PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, NULL, yy, PETSC_FALSE, PETSC_FALSE));
2542:   PetscFunctionReturn(PETSC_SUCCESS);
2543: }

2545: static PetscErrorCode MatMultAdd_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz)
2546: {
2547:   PetscFunctionBegin;
2548:   PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, yy, zz, PETSC_FALSE, PETSC_FALSE));
2549:   PetscFunctionReturn(PETSC_SUCCESS);
2550: }

2552: static PetscErrorCode MatMultHermitianTranspose_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy)
2553: {
2554:   PetscFunctionBegin;
2555:   PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, NULL, yy, PETSC_TRUE, PETSC_TRUE));
2556:   PetscFunctionReturn(PETSC_SUCCESS);
2557: }

2559: static PetscErrorCode MatMultHermitianTransposeAdd_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz)
2560: {
2561:   PetscFunctionBegin;
2562:   PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, yy, zz, PETSC_TRUE, PETSC_TRUE));
2563:   PetscFunctionReturn(PETSC_SUCCESS);
2564: }

2566: static PetscErrorCode MatMultTranspose_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy)
2567: {
2568:   PetscFunctionBegin;
2569:   PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, NULL, yy, PETSC_TRUE, PETSC_FALSE));
2570:   PetscFunctionReturn(PETSC_SUCCESS);
2571: }

2573: __global__ static void ScatterAdd(PetscInt n, PetscInt *idx, const PetscScalar *x, PetscScalar *y)
2574: {
2575:   int i = blockIdx.x * blockDim.x + threadIdx.x;
2576:   if (i < n) y[idx[i]] += x[i];
2577: }

2579: /* z = op(A) x + y. If trans & !herm, op = ^T; if trans & herm, op = ^H; if !trans, op = no-op */
2580: static PetscErrorCode MatMultAddKernel_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz, PetscBool trans, PetscBool herm)
2581: {
2582:   Mat_SeqAIJ                   *a              = (Mat_SeqAIJ *)A->data;
2583:   Mat_SeqAIJCUSPARSE           *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
2584:   Mat_SeqAIJCUSPARSEMultStruct *matstruct;
2585:   PetscScalar                  *xarray, *zarray, *dptr, *beta, *xptr;
2586:   cusparseOperation_t           opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
2587:   PetscBool                     compressed;
2588:   PetscInt                      nx, ny;
2589:   PetscDeviceContext            dctx;
2590:   cudaStream_t                  stream;
2591:   cusparseHandle_t              handle;

2593:   PetscFunctionBegin;
2594:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
2595:   {
2596:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};

2598:     PetscCheck(!herm || trans, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Hermitian and not transpose not supported");
2599:     if (!a->nz) {
2600:       if (yy) PetscCall(VecSeq_CUDA::Copy(yy, zz));
2601:       else PetscCall(VecSeq_CUDA::Set(zz, 0));
2602:       PetscFunctionReturn(PETSC_SUCCESS);
2603:     }
2604:     /* The line below is necessary due to the operations that modify the matrix on the CPU (axpy, scale, etc) */
2605:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
2606:     if (!trans) {
2607:       matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
2608:       PetscCheck(matstruct, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "SeqAIJCUSPARSE does not have a 'mat' (need to fix)");
2609:     } else {
2610:       if (herm || !A->form_explicit_transpose) {
2611:         opA       = herm ? CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE : CUSPARSE_OPERATION_TRANSPOSE;
2612:         matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
2613:       } else {
2614:         if (!A->transupdated) PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
2615:         matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->matTranspose;
2616:       }
2617:     }
2618:     /* Does the matrix use compressed rows (i.e., drop zero rows)? */
2619:     compressed = matstruct->cprowIndices ? PETSC_TRUE : PETSC_FALSE;

2621:     try {
2622:       PetscCall(VecCUDAGetArrayRead(xx, (const PetscScalar **)&xarray));
2623:       if (yy == zz) PetscCall(VecCUDAGetArray(zz, &zarray)); /* read & write zz, so need to get up-to-date zarray on GPU */
2624:       else PetscCall(VecCUDAGetArrayWrite(zz, &zarray));     /* write zz, so no need to init zarray on GPU */

2626:       PetscCall(PetscLogGpuTimeBegin());
2627:       if (opA == CUSPARSE_OPERATION_NON_TRANSPOSE) {
2628:         /* z = A x + beta y.
2629:            If A is compressed (with less rows), then Ax is shorter than the full z, so we need a work vector to store Ax.
2630:            When A is non-compressed, and z = y, we can set beta=1 to compute y = Ax + y in one call.
2631:         */
2632:         xptr = xarray;
2633:         dptr = compressed ? cusparsestruct->workVector->data().get() : zarray;
2634:         beta = (yy == zz && !compressed) ? matstruct->beta_one : matstruct->beta_zero;
2635:         /* Get length of x, y for y=Ax. ny might be shorter than the work vector's allocated length, since the work vector is
2636:             allocated to accommodate different uses. So we get the length info directly from mat.
2637:          */
2638:         if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
2639:           CsrMatrix *mat = (CsrMatrix *)matstruct->mat;
2640:           nx             = mat->num_cols; // since y = Ax
2641:           ny             = mat->num_rows;
2642:         }
2643:       } else {
2644:         /* z = A^T x + beta y
2645:            If A is compressed, then we need a work vector as the shorter version of x to compute A^T x.
2646:            Note A^Tx is of full length, so we set beta to 1.0 if y exists.
2647:          */
2648:         xptr = compressed ? cusparsestruct->workVector->data().get() : xarray;
2649:         dptr = zarray;
2650:         beta = yy ? matstruct->beta_one : matstruct->beta_zero;
2651:         if (compressed) { /* Scatter x to work vector */
2652:           thrust::device_ptr<PetscScalar> xarr = thrust::device_pointer_cast(xarray);

2654:           PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::make_zip_iterator(thrust::make_tuple(cusparsestruct->workVector->begin(), thrust::make_permutation_iterator(xarr, matstruct->cprowIndices->begin()))),
2655:                                            thrust::make_zip_iterator(thrust::make_tuple(cusparsestruct->workVector->begin(), thrust::make_permutation_iterator(xarr, matstruct->cprowIndices->begin()))) + matstruct->cprowIndices->size(), VecCUDAEqualsReverse()));
2656:         }
2657:         if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
2658:           CsrMatrix *mat = (CsrMatrix *)matstruct->mat;
2659:           nx             = mat->num_rows; // since y = A^T x
2660:           ny             = mat->num_cols;
2661:         }
2662:       }

2664:       /* csr_spmv does y = alpha op(A) x + beta y */
2665:       if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
2666:         PetscCheck(opA >= 0 && opA <= 2, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE ABI on cusparseOperation_t has changed and PETSc has not been updated accordingly");

2668:         if (!matstruct->cuSpMV[opA].initialized) { /* built on demand */
2669:           CsrMatrix *mat = (CsrMatrix *)matstruct->mat;
2670:           PetscCallCUSPARSE(cusparseCreateCsr(&matstruct->cuSpMV[opA].matDescr, mat->num_rows, mat->num_cols, mat->num_entries, mat->row_offsets->data().get(), mat->column_indices->data().get(), mat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
2671:           PetscCallCUSPARSE(cusparseCreateDnVec(&matstruct->cuSpMV[opA].vecXDescr, nx, xptr, cusparse_scalartype));
2672:           PetscCallCUSPARSE(cusparseCreateDnVec(&matstruct->cuSpMV[opA].vecYDescr, ny, dptr, cusparse_scalartype));
2673:           PetscCallCUSPARSE(
2674:             cusparseSpMV_bufferSize(handle, opA, matstruct->alpha_one, matstruct->cuSpMV[opA].matDescr, matstruct->cuSpMV[opA].vecXDescr, beta, matstruct->cuSpMV[opA].vecYDescr, cusparse_scalartype, cusparsestruct->spmvAlg, &matstruct->cuSpMV[opA].spmvBufferSize));
2675:           PetscCallCUDA(cudaMalloc(&matstruct->cuSpMV[opA].spmvBuffer, matstruct->cuSpMV[opA].spmvBufferSize));
2676: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0) // cusparseSpMV_preprocess is added in 12.4
2677:           PetscCallCUSPARSE(
2678:             cusparseSpMV_preprocess(handle, opA, matstruct->alpha_one, matstruct->cuSpMV[opA].matDescr, matstruct->cuSpMV[opA].vecXDescr, beta, matstruct->cuSpMV[opA].vecYDescr, cusparse_scalartype, cusparsestruct->spmvAlg, matstruct->cuSpMV[opA].spmvBuffer));
2679: #endif
2680:           matstruct->cuSpMV[opA].initialized = PETSC_TRUE;
2681:         } else {
2682:           /* x, y's value pointers might change between calls, but their shape is kept, so we just update pointers */
2683:           PetscCallCUSPARSE(cusparseDnVecSetValues(matstruct->cuSpMV[opA].vecXDescr, xptr));
2684:           PetscCallCUSPARSE(cusparseDnVecSetValues(matstruct->cuSpMV[opA].vecYDescr, dptr));
2685:         }

2687:         PetscCallCUSPARSE(cusparseSpMV(handle, opA, matstruct->alpha_one, matstruct->cuSpMV[opA].matDescr, matstruct->cuSpMV[opA].vecXDescr, beta, matstruct->cuSpMV[opA].vecYDescr, cusparse_scalartype, cusparsestruct->spmvAlg, matstruct->cuSpMV[opA].spmvBuffer));

2689:       } else {
2690:         if (cusparsestruct->nrows) {
2691:           SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
2692:         }
2693:       }
2694:       PetscCall(PetscLogGpuTimeEnd());

2696:       if (opA == CUSPARSE_OPERATION_NON_TRANSPOSE) {
2697:         if (yy) {                                      /* MatMultAdd: zz = A*xx + yy */
2698:           if (compressed) {                            /* A is compressed. We first copy yy to zz, then ScatterAdd the work vector to zz */
2699:             PetscCall(VecSeq_CUDA::Copy(yy, zz));      /* zz = yy */
2700:           } else if (zz != yy) {                       /* A is not compressed. zz already contains A*xx, and we just need to add yy */
2701:             PetscCall(VecSeq_CUDA::AXPY(zz, 1.0, yy)); /* zz += yy */
2702:           }
2703:         } else if (compressed) { /* MatMult: zz = A*xx. A is compressed, so we zero zz first, then ScatterAdd the work vector to zz */
2704:           PetscCall(VecSeq_CUDA::Set(zz, 0));
2705:         }

2707:         /* ScatterAdd the result from work vector into the full vector when A is compressed */
2708:         if (compressed) {
2709:           PetscCall(PetscLogGpuTimeBegin());
2710:           PetscInt n = (PetscInt)matstruct->cprowIndices->size();
2711:           ScatterAdd<<<(int)((n + 255) / 256), 256, 0, stream>>>(n, matstruct->cprowIndices->data().get(), cusparsestruct->workVector->data().get(), zarray);
2712:           PetscCall(PetscLogGpuTimeEnd());
2713:         }
2714:       } else {
2715:         if (yy && yy != zz) PetscCall(VecSeq_CUDA::AXPY(zz, 1.0, yy)); /* zz += yy */
2716:       }
2717:       PetscCall(VecCUDARestoreArrayRead(xx, (const PetscScalar **)&xarray));
2718:       if (yy == zz) PetscCall(VecCUDARestoreArray(zz, &zarray));
2719:       else PetscCall(VecCUDARestoreArrayWrite(zz, &zarray));
2720:     } catch (char *ex) {
2721:       SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "CUSPARSE error: %s", ex);
2722:     }
2723:     if (yy) PetscCall(PetscLogGpuFlops(2.0 * a->nz));
2724:     else PetscCall(PetscLogGpuFlops(2.0 * a->nz - a->nonzerorowcnt));
2725:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
2726:   }
2727:   PetscFunctionReturn(PETSC_SUCCESS);
2728: }

2730: static PetscErrorCode MatMultTransposeAdd_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz)
2731: {
2732:   PetscFunctionBegin;
2733:   PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, yy, zz, PETSC_TRUE, PETSC_FALSE));
2734:   PetscFunctionReturn(PETSC_SUCCESS);
2735: }

2737: static PetscErrorCode MatGetDiagonal_SeqAIJCUSPARSE(Mat A, Vec diag)
2738: {
2739:   PetscFunctionBegin;
2740:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetDiagonal(A, diag));
2741:   PetscFunctionReturn(PETSC_SUCCESS);
2742: }

2744: static PetscErrorCode MatGetRowSumAbs_SeqAIJCUSPARSE(Mat A, Vec v)
2745: {
2746:   PetscFunctionBegin;
2747:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetRowSumAbs(A, v));
2748:   PetscFunctionReturn(PETSC_SUCCESS);
2749: }

2751: static PetscErrorCode MatDiagonalScale_SeqAIJCUSPARSE(Mat A, Vec ll, Vec rr)
2752: {
2753:   PetscFunctionBegin;
2754:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::DiagonalScale(A, ll, rr));
2755:   PetscFunctionReturn(PETSC_SUCCESS);
2756: }

2758: static PetscErrorCode MatAssemblyEnd_SeqAIJCUSPARSE(Mat A, MatAssemblyType mode)
2759: {
2760:   PetscFunctionBegin;
2761:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::AssemblyEnd(A, mode));
2762:   PetscFunctionReturn(PETSC_SUCCESS);
2763: }

2765: /*@
2766:   MatCreateSeqAIJCUSPARSE - Creates a sparse matrix in `MATAIJCUSPARSE` (compressed row) format for use on NVIDIA GPUs

2768:   Collective

2770:   Input Parameters:
2771: + comm - MPI communicator, set to `PETSC_COMM_SELF`
2772: . m    - number of rows
2773: . n    - number of columns
2774: . nz   - number of nonzeros per row (same for all rows), ignored if `nnz` is provide
2775: - nnz  - array containing the number of nonzeros in the various rows (possibly different for each row) or `NULL`

2777:   Output Parameter:
2778: . A - the matrix

2780:   Level: intermediate

2782:   Notes:
2783:   This matrix will ultimately pushed down to NVIDIA GPUs and use the CuSPARSE library for
2784:   calculations. For good matrix assembly performance the user should preallocate the matrix
2785:   storage by setting the parameter `nz` (or the array `nnz`).

2787:   It is recommended that one use the `MatCreate()`, `MatSetType()` and/or `MatSetFromOptions()`,
2788:   MatXXXXSetPreallocation() paradgm instead of this routine directly.
2789:   [MatXXXXSetPreallocation() is, for example, `MatSeqAIJSetPreallocation()`]

2791:   The AIJ format, also called
2792:   compressed row storage, is fully compatible with standard Fortran
2793:   storage.  That is, the stored row and column indices can begin at
2794:   either one (as in Fortran) or zero.

2796:   Specify the preallocated storage with either nz or nnz (not both).
2797:   Set `nz` = `PETSC_DEFAULT` and `nnz` = `NULL` for PETSc to control dynamic memory
2798:   allocation.

2800:   When working with matrices for GPUs, it is often better to use the `MatSetPreallocationCOO()` and `MatSetValuesCOO()` paradigm rather than using this routine and `MatSetValues()`

2802: .seealso: [](ch_matrices), `Mat`, `MATSEQAIJCUSPARSE`, `MatCreate()`, `MatCreateAIJ()`, `MatSetValues()`, `MatSeqAIJSetColumnIndices()`, `MatCreateSeqAIJWithArrays()`, `MATAIJCUSPARSE`,
2803:           `MatSetPreallocationCOO()`, `MatSetValuesCOO()`
2804: @*/
2805: PetscErrorCode MatCreateSeqAIJCUSPARSE(MPI_Comm comm, PetscInt m, PetscInt n, PetscInt nz, const PetscInt nnz[], Mat *A)
2806: {
2807:   return MatSeqAIJCUSPARSE_CUPM_t::CreateSeqAIJ(comm, m, n, nz, nnz, A);
2808: }

2810: static PetscErrorCode MatDestroy_SeqAIJCUSPARSE(Mat A)
2811: {
2812:   return MatSeqAIJCUSPARSE_CUPM_t::Destroy(A);
2813: }

2815: static PetscErrorCode MatDuplicate_SeqAIJCUSPARSE(Mat A, MatDuplicateOption cpvalues, Mat *B)
2816: {
2817:   PetscFunctionBegin;
2818:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::Duplicate(A, cpvalues, B));
2819:   PetscFunctionReturn(PETSC_SUCCESS);
2820: }

2822: static PetscErrorCode MatAXPY_SeqAIJCUSPARSE(Mat Y, PetscScalar a, Mat X, MatStructure str)
2823: {
2824:   Mat_SeqAIJ         *x = (Mat_SeqAIJ *)X->data, *y = (Mat_SeqAIJ *)Y->data;
2825:   Mat_SeqAIJCUSPARSE *cy;
2826:   Mat_SeqAIJCUSPARSE *cx;
2827:   CsrMatrix          *csry, *csrx;
2828:   PetscDeviceContext  dctx;
2829:   cudaStream_t        stream;
2830:   cusparseHandle_t    handle;

2832:   PetscFunctionBegin;
2833:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
2834:   {
2835:     const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};

2837:     cy = (Mat_SeqAIJCUSPARSE *)Y->spptr;
2838:     cx = (Mat_SeqAIJCUSPARSE *)X->spptr;
2839:     if (X->ops->axpy != Y->ops->axpy) {
2840:       PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(Y, PETSC_FALSE));
2841:       PetscCall(MatAXPY_SeqAIJ(Y, a, X, str));
2842:       PetscFunctionReturn(PETSC_SUCCESS);
2843:     }
2844:     /* if we are here, it means both matrices are bound to GPU */
2845:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(Y));
2846:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(X));
2847:     PetscCheck(cy->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)Y), PETSC_ERR_GPU, "only MAT_CUSPARSE_CSR supported");
2848:     PetscCheck(cx->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)X), PETSC_ERR_GPU, "only MAT_CUSPARSE_CSR supported");
2849:     csry = (CsrMatrix *)cy->mat->mat;
2850:     csrx = (CsrMatrix *)cx->mat->mat;
2851:     /* see if we can turn this into a cublas axpy */
2852:     if (str != SAME_NONZERO_PATTERN && x->nz == y->nz && !x->compressedrow.use && !y->compressedrow.use) {
2853:       bool eq;

2855:       PetscCallThrust(eq = thrust::equal(thrust::cuda::par.on(stream), csry->row_offsets->begin(), csry->row_offsets->end(), csrx->row_offsets->begin()));
2856:       if (eq) PetscCallThrust(eq = thrust::equal(thrust::cuda::par.on(stream), csry->column_indices->begin(), csry->column_indices->end(), csrx->column_indices->begin()));
2857:       if (eq) str = SAME_NONZERO_PATTERN;
2858:     }
2859:     /* spgeam is buggy with one column */
2860:     if (Y->cmap->n == 1 && str != SAME_NONZERO_PATTERN) str = DIFFERENT_NONZERO_PATTERN;

2862: #if !PetscDefined(USE_64BIT_INDICES) // cusparseScsrgeam2 etc. do not support 64bit indices
2863:     if (str == SUBSET_NONZERO_PATTERN) {
2864:       PetscScalar       *ay, b = 1.0;
2865:       const PetscScalar *ax;
2866:       size_t             bufferSize;
2867:       void              *buffer;

2869:       PetscCall(MatSeqAIJCUSPARSEGetArrayRead(X, &ax));
2870:       PetscCall(MatSeqAIJCUSPARSEGetArray(Y, &ay));
2871:       PetscCallCUSPARSE(cusparse_csr_spgeam_bufferSize(handle, Y->rmap->n, Y->cmap->n, &a, cx->mat->descr, x->nz, ax, csrx->row_offsets->data().get(), csrx->column_indices->data().get(), &b, cy->mat->descr, y->nz, ay, csry->row_offsets->data().get(),
2872:                                                        csry->column_indices->data().get(), cy->mat->descr, ay, csry->row_offsets->data().get(), csry->column_indices->data().get(), &bufferSize));
2873:       PetscCallCUDA(cudaMalloc(&buffer, bufferSize));
2874:       PetscCall(PetscLogGpuTimeBegin());
2875:       PetscCallCUSPARSE(cusparse_csr_spgeam(handle, Y->rmap->n, Y->cmap->n, &a, cx->mat->descr, x->nz, ax, csrx->row_offsets->data().get(), csrx->column_indices->data().get(), &b, cy->mat->descr, y->nz, ay, csry->row_offsets->data().get(),
2876:                                             csry->column_indices->data().get(), cy->mat->descr, ay, csry->row_offsets->data().get(), csry->column_indices->data().get(), buffer));
2877:       PetscCall(PetscLogGpuFlops(x->nz + y->nz));
2878:       PetscCall(PetscLogGpuTimeEnd());
2879:       PetscCallCUDA(cudaFree(buffer));

2881:       PetscCall(MatSeqAIJCUSPARSERestoreArrayRead(X, &ax));
2882:       PetscCall(MatSeqAIJCUSPARSERestoreArray(Y, &ay));
2883:     } else
2884: #endif
2885:       if (str == SAME_NONZERO_PATTERN) {
2886:       PetscCall(MatSeqAIJCUSPARSE_CUPM_t::AXPY_SameNZ(Y, a, X));
2887:     } else {
2888:       PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(Y, PETSC_FALSE));
2889:       PetscCall(MatAXPY_SeqAIJ(Y, a, X, str));
2890:     }
2891:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
2892:   }
2893:   PetscFunctionReturn(PETSC_SUCCESS);
2894: }

2896: static PetscErrorCode MatScale_SeqAIJCUSPARSE(Mat Y, PetscScalar a)
2897: {
2898:   PetscFunctionBegin;
2899:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::Scale(Y, a));
2900:   PetscFunctionReturn(PETSC_SUCCESS);
2901: }

2903: static PetscErrorCode MatZeroEntries_SeqAIJCUSPARSE(Mat A)
2904: {
2905:   PetscFunctionBegin;
2906:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::ZeroEntries(A));
2907:   PetscFunctionReturn(PETSC_SUCCESS);
2908: }

2910: static PetscErrorCode MatGetCurrentMemType_SeqAIJCUSPARSE(Mat A, PetscMemType *m)
2911: {
2912:   PetscFunctionBegin;
2913:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetCurrentMemType(A, m));
2914:   PetscFunctionReturn(PETSC_SUCCESS);
2915: }

2917: static PetscErrorCode MatBindToCPU_SeqAIJCUSPARSE(Mat A, PetscBool flg)
2918: {
2919:   Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;

2921:   PetscFunctionBegin;
2922:   if (A->factortype != MAT_FACTOR_NONE) {
2923:     A->boundtocpu = flg;
2924:     PetscFunctionReturn(PETSC_SUCCESS);
2925:   }
2926:   if (flg) {
2927:     PetscCall(MatSeqAIJCUSPARSECopyFromGPU(A));

2929:     A->ops->scale                     = MatScale_SeqAIJ;
2930:     A->ops->getdiagonal               = MatGetDiagonal_SeqAIJ;
2931:     A->ops->getrowsumabs              = MatGetRowSumAbs_SeqAIJ;
2932:     A->ops->diagonalscale             = MatDiagonalScale_SeqAIJ;
2933:     A->ops->axpy                      = MatAXPY_SeqAIJ;
2934:     A->ops->zeroentries               = MatZeroEntries_SeqAIJ;
2935:     A->ops->mult                      = MatMult_SeqAIJ;
2936:     A->ops->multadd                   = MatMultAdd_SeqAIJ;
2937:     A->ops->multtranspose             = MatMultTranspose_SeqAIJ;
2938:     A->ops->multtransposeadd          = MatMultTransposeAdd_SeqAIJ;
2939:     A->ops->multhermitiantranspose    = NULL;
2940:     A->ops->multhermitiantransposeadd = NULL;
2941:     A->ops->productsetfromoptions     = MatProductSetFromOptions_SeqAIJ;
2942:     A->ops->getcurrentmemtype         = NULL;
2943:     PetscCall(PetscMemzero(a->ops, sizeof(Mat_SeqAIJOps)));
2944:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSeqAIJCopySubArray_C", NULL));
2945:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdensecuda_C", NULL));
2946:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdense_C", NULL));
2947:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetPreallocationCOO_C", NULL));
2948:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetValuesCOO_C", NULL));
2949:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqaijcusparse_C", NULL));
2950:   } else {
2951:     A->ops->scale                     = MatScale_SeqAIJCUSPARSE;
2952:     A->ops->getdiagonal               = MatGetDiagonal_SeqAIJCUSPARSE;
2953:     A->ops->getrowsumabs              = MatGetRowSumAbs_SeqAIJCUSPARSE;
2954:     A->ops->diagonalscale             = MatDiagonalScale_SeqAIJCUSPARSE;
2955:     A->ops->axpy                      = MatAXPY_SeqAIJCUSPARSE;
2956:     A->ops->zeroentries               = MatZeroEntries_SeqAIJCUSPARSE;
2957:     A->ops->mult                      = MatMult_SeqAIJCUSPARSE;
2958:     A->ops->multadd                   = MatMultAdd_SeqAIJCUSPARSE;
2959:     A->ops->multtranspose             = MatMultTranspose_SeqAIJCUSPARSE;
2960:     A->ops->multtransposeadd          = MatMultTransposeAdd_SeqAIJCUSPARSE;
2961:     A->ops->multhermitiantranspose    = MatMultHermitianTranspose_SeqAIJCUSPARSE;
2962:     A->ops->multhermitiantransposeadd = MatMultHermitianTransposeAdd_SeqAIJCUSPARSE;
2963:     A->ops->productsetfromoptions     = MatProductSetFromOptions_SeqAIJCUSPARSE;
2964:     A->ops->getcurrentmemtype         = MatGetCurrentMemType_SeqAIJCUSPARSE;
2965:     a->ops->getarray                  = MatSeqAIJGetArray_SeqAIJCUSPARSE;
2966:     a->ops->restorearray              = MatSeqAIJRestoreArray_SeqAIJCUSPARSE;
2967:     a->ops->getarrayread              = MatSeqAIJGetArrayRead_SeqAIJCUSPARSE;
2968:     a->ops->restorearrayread          = MatSeqAIJRestoreArrayRead_SeqAIJCUSPARSE;
2969:     a->ops->getarraywrite             = MatSeqAIJGetArrayWrite_SeqAIJCUSPARSE;
2970:     a->ops->restorearraywrite         = MatSeqAIJRestoreArrayWrite_SeqAIJCUSPARSE;
2971:     a->ops->getcsrandmemtype          = MatSeqAIJGetCSRAndMemType_SeqAIJCUSPARSE;

2973:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSeqAIJCopySubArray_C", MatSeqAIJCopySubArray_SeqAIJCUSPARSE));
2974:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdensecuda_C", MatProductSetFromOptions_SeqAIJCUSPARSE));
2975:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdense_C", MatProductSetFromOptions_SeqAIJCUSPARSE));
2976:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetPreallocationCOO_C", MatSetPreallocationCOO_SeqAIJCUSPARSE));
2977:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetValuesCOO_C", MatSetValuesCOO_SeqAIJCUSPARSE));
2978:     PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqaijcusparse_C", MatProductSetFromOptions_SeqAIJCUSPARSE));
2979:   }
2980:   A->boundtocpu = flg;
2981:   a->inode.use  = (flg && a->inode.size_csr) ? PETSC_TRUE : PETSC_FALSE;
2982:   PetscFunctionReturn(PETSC_SUCCESS);
2983: }

2985: PETSC_INTERN PetscErrorCode MatConvert_SeqAIJ_SeqAIJCUSPARSE(Mat A, MatType, MatReuse reuse, Mat *newmat)
2986: {
2987:   Mat B;

2989:   PetscFunctionBegin;
2990:   PetscCall(PetscDeviceInitialize(PETSC_DEVICE_CUDA)); /* first use of CUSPARSE may be via MatConvert */
2991:   if (reuse == MAT_INITIAL_MATRIX) {
2992:     PetscCall(MatDuplicate(A, MAT_COPY_VALUES, newmat));
2993:   } else if (reuse == MAT_REUSE_MATRIX) {
2994:     PetscCall(MatCopy(A, *newmat, SAME_NONZERO_PATTERN));
2995:   }
2996:   B = *newmat;

2998:   PetscCall(PetscFree(B->defaultvectype));
2999:   PetscCall(PetscStrallocpy(VECCUDA, &B->defaultvectype));

3001:   if (reuse != MAT_REUSE_MATRIX && !B->spptr) {
3002:     if (B->factortype == MAT_FACTOR_NONE) {
3003:       Mat_SeqAIJCUSPARSE *spptr;
3004:       PetscCall(PetscNew(&spptr));
3005:       spptr->format     = MAT_CUSPARSE_CSR;
3006:       spptr->spmvAlg    = CUSPARSE_SPMV_CSR_ALG1; /* default, since we only support csr */
3007:       spptr->spmmAlg    = CUSPARSE_SPMM_CSR_ALG1; /* default, only support column-major dense matrix B */
3008:       spptr->csr2cscAlg = CUSPARSE_CSR2CSC_ALG1;
3009:       B->spptr          = spptr;
3010:     } else {
3011:       Mat_SeqAIJCUSPARSETriFactors *spptr;

3013:       PetscCall(PetscNew(&spptr));
3014:       B->spptr = spptr;
3015:     }
3016:     B->offloadmask = PETSC_OFFLOAD_UNALLOCATED;
3017:   }
3018:   B->ops->assemblyend       = MatAssemblyEnd_SeqAIJCUSPARSE;
3019:   B->ops->destroy           = MatDestroy_SeqAIJCUSPARSE;
3020:   B->ops->setoption         = MatSetOption_SeqAIJCUSPARSE;
3021:   B->ops->setfromoptions    = MatSetFromOptions_SeqAIJCUSPARSE;
3022:   B->ops->bindtocpu         = MatBindToCPU_SeqAIJCUSPARSE;
3023:   B->ops->duplicate         = MatDuplicate_SeqAIJCUSPARSE;
3024:   B->ops->getcurrentmemtype = MatGetCurrentMemType_SeqAIJCUSPARSE;

3026:   PetscCall(MatBindToCPU_SeqAIJCUSPARSE(B, PETSC_FALSE));
3027:   PetscCall(PetscObjectChangeTypeName((PetscObject)B, MATSEQAIJCUSPARSE));
3028:   PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatCUSPARSESetFormat_C", MatCUSPARSESetFormat_SeqAIJCUSPARSE));
3029: #if PetscDefined(HAVE_HYPRE)
3030:   PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatConvert_seqaijcusparse_hypre_C", MatConvert_AIJ_HYPRE));
3031: #endif
3032:   PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatCUSPARSESetUseCPUSolve_C", MatCUSPARSESetUseCPUSolve_SeqAIJCUSPARSE));
3033:   PetscFunctionReturn(PETSC_SUCCESS);
3034: }

3036: PETSC_EXTERN PetscErrorCode MatCreate_SeqAIJCUSPARSE(Mat B)
3037: {
3038:   PetscFunctionBegin;
3039:   PetscCall(MatCreate_SeqAIJ(B));
3040:   PetscCall(MatConvert_SeqAIJ_SeqAIJCUSPARSE(B, MATSEQAIJCUSPARSE, MAT_INPLACE_MATRIX, &B));
3041:   PetscFunctionReturn(PETSC_SUCCESS);
3042: }

3044: /*MC
3045:    MATSEQAIJCUSPARSE - MATAIJCUSPARSE = "(seq)aijcusparse" - A matrix type to be used for sparse matrices on NVIDIA GPUs.

3047:    Options Database Keys:
3048: +  -mat_type aijcusparse                           - Sets the matrix type to `MATSEQAIJCUSPARSE` during a call to `MatSetFromOptions()`
3049: .  -mat_cusparse_storage_format (csr|ell|hyb)      - Sets the storage format of matrices (for `MatMult()` and factors in `MatSolve()`).
3050: .  -mat_cusparse_mult_storage_format (csr|ell|hyb) - Sets the storage format of matrices (for `MatMult()`).
3051: -  -mat_cusparse_use_cpu_solve (true|false)        - Performs the `MatSolve()` on the CPU.

3053:   Level: beginner

3055:   Notes:
3056:   These matrices can be in either CSR, ELL, or HYB format.

3058:   All matrix calculations are performed on NVIDIA GPUs using the cuSPARSE library.

3060:   Uses 32-bit integers internally. If PETSc is configured `--with-64-bit-indices`, the integer row and column indices are stored on the GPU with `int`. It is unclear what happens
3061:   if some integer values passed in do not fit in `int`.

3063: .seealso: [](ch_matrices), `Mat`, `MatCreateSeqAIJCUSPARSE()`, `MatCUSPARSESetUseCPUSolve()`, `MATAIJCUSPARSE`, `MatCreateAIJCUSPARSE()`, `MatCUSPARSESetFormat()`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
3064: M*/

3066: PETSC_INTERN PetscErrorCode MatSolverTypeRegister_CUSPARSE(void)
3067: {
3068:   PetscFunctionBegin;
3069:   PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_LU, MatGetFactor_seqaijcusparse_cusparse));
3070:   PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_CHOLESKY, MatGetFactor_seqaijcusparse_cusparse));
3071:   PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_ILU, MatGetFactor_seqaijcusparse_cusparse));
3072:   PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_ICC, MatGetFactor_seqaijcusparse_cusparse));
3073:   PetscFunctionReturn(PETSC_SUCCESS);
3074: }

3076: static PetscErrorCode MatSeqAIJCUSPARSE_Destroy(Mat mat)
3077: {
3078:   Mat_SeqAIJCUSPARSE *cusp = static_cast<Mat_SeqAIJCUSPARSE *>(mat->spptr);

3080:   PetscFunctionBegin;
3081:   if (cusp) {
3082:     PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusp->mat, cusp->format));
3083:     PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusp->matTranspose, cusp->format));
3084:     delete cusp->workVector;
3085:     delete cusp->rowoffsets_gpu;
3086:     delete cusp->csr2csc_i;
3087:     delete cusp->coords;
3088:     PetscCall(PetscFree(mat->spptr));
3089:   }
3090:   PetscFunctionReturn(PETSC_SUCCESS);
3091: }

3093: static PetscErrorCode CsrMatrix_Destroy(CsrMatrix **mat)
3094: {
3095:   PetscFunctionBegin;
3096:   if (*mat) {
3097:     delete (*mat)->values;
3098:     delete (*mat)->column_indices;
3099:     delete (*mat)->row_offsets;
3100:     delete *mat;
3101:     *mat = 0;
3102:   }
3103:   PetscFunctionReturn(PETSC_SUCCESS);
3104: }

3106: static PetscErrorCode MatSeqAIJCUSPARSEMultStruct_Destroy(Mat_SeqAIJCUSPARSEMultStruct **matstruct, MatCUSPARSEStorageFormat format)
3107: {
3108:   CsrMatrix *mat;

3110:   PetscFunctionBegin;
3111:   if (*matstruct) {
3112:     if ((*matstruct)->mat) {
3113:       if (format == MAT_CUSPARSE_ELL || format == MAT_CUSPARSE_HYB) {
3114:         SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
3115:       } else {
3116:         mat = (CsrMatrix *)(*matstruct)->mat;
3117:         PetscCall(CsrMatrix_Destroy(&mat));
3118:       }
3119:     }
3120:     if ((*matstruct)->descr) PetscCallCUSPARSE(cusparseDestroyMatDescr((*matstruct)->descr));
3121:     delete (*matstruct)->cprowIndices;
3122:     PetscCallCUDA(cudaFree((*matstruct)->alpha_one));
3123:     PetscCallCUDA(cudaFree((*matstruct)->beta_zero));
3124:     PetscCallCUDA(cudaFree((*matstruct)->beta_one));

3126:     Mat_SeqAIJCUSPARSEMultStruct *mdata = *matstruct;
3127:     if (mdata->matDescr) PetscCallCUSPARSE(cusparseDestroySpMat(mdata->matDescr));

3129:     for (int i = 0; i < 3; i++) {
3130:       if (mdata->cuSpMV[i].initialized) {
3131:         PetscCallCUDA(cudaFree(mdata->cuSpMV[i].spmvBuffer));
3132:         PetscCallCUSPARSE(cusparseDestroyDnVec(mdata->cuSpMV[i].vecXDescr));
3133:         PetscCallCUSPARSE(cusparseDestroyDnVec(mdata->cuSpMV[i].vecYDescr));
3134:         PetscCallCUSPARSE(cusparseDestroySpMat(mdata->cuSpMV[i].matDescr));
3135:       }
3136:       if (mdata->matDescr_SpMM[i]) PetscCallCUSPARSE(cusparseDestroySpMat(mdata->matDescr_SpMM[i]));
3137:     }
3138:     delete *matstruct;
3139:     *matstruct = NULL;
3140:   }
3141:   PetscFunctionReturn(PETSC_SUCCESS);
3142: }

3144: PetscErrorCode MatSeqAIJCUSPARSETriFactors_Reset(Mat_SeqAIJCUSPARSETriFactors_p *trifactors)
3145: {
3146:   Mat_SeqAIJCUSPARSETriFactors *fs = *trifactors;

3148:   PetscFunctionBegin;
3149:   if (fs) {
3150:     delete fs->rpermIndices;
3151:     delete fs->cpermIndices;
3152:     fs->rpermIndices  = NULL;
3153:     fs->cpermIndices  = NULL;
3154:     fs->init_dev_prop = PETSC_FALSE;
3155:     PetscCallCUDA(cudaFree(fs->csrRowPtr));
3156:     PetscCallCUDA(cudaFree(fs->csrColIdx));
3157:     PetscCallCUDA(cudaFree(fs->csrRowPtr32));
3158:     PetscCallCUDA(cudaFree(fs->csrColIdx32));
3159:     PetscCallCUDA(cudaFree(fs->csrVal));
3160:     PetscCallCUDA(cudaFree(fs->diag));
3161:     PetscCallCUDA(cudaFree(fs->X));
3162:     PetscCallCUDA(cudaFree(fs->Y));
3163:     // PetscCallCUDA(cudaFree(fs->factBuffer_M)); /* No needed since factBuffer_M shares with one of spsvBuffer_L/U */
3164:     PetscCallCUDA(cudaFree(fs->spsvBuffer_L));
3165:     PetscCallCUDA(cudaFree(fs->spsvBuffer_U));
3166:     PetscCallCUDA(cudaFree(fs->spsvBuffer_Lt));
3167:     PetscCallCUDA(cudaFree(fs->spsvBuffer_Ut));
3168:     PetscCallCUSPARSE(cusparseDestroyMatDescr(fs->matDescr_M));
3169:     if (fs->spMatDescr_L) PetscCallCUSPARSE(cusparseDestroySpMat(fs->spMatDescr_L));
3170:     if (fs->spMatDescr_U) PetscCallCUSPARSE(cusparseDestroySpMat(fs->spMatDescr_U));
3171:     PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_L));
3172:     PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_Lt));
3173:     PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_U));
3174:     PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_Ut));
3175:     if (fs->dnVecDescr_X) PetscCallCUSPARSE(cusparseDestroyDnVec(fs->dnVecDescr_X));
3176:     if (fs->dnVecDescr_Y) PetscCallCUSPARSE(cusparseDestroyDnVec(fs->dnVecDescr_Y));
3177:     PetscCallCUSPARSE(cusparseDestroyCsrilu02Info(fs->ilu0Info_M));
3178:     PetscCallCUSPARSE(cusparseDestroyCsric02Info(fs->ic0Info_M));
3179:     PetscCall(PetscFree(fs->csrRowPtr_h));
3180:     PetscCall(PetscFree(fs->csrVal_h));
3181:     PetscCall(PetscFree(fs->diag_h));
3182:     fs->createdTransposeSpSVDescr    = PETSC_FALSE;
3183:     fs->updatedTransposeSpSVAnalysis = PETSC_FALSE;
3184:   }
3185:   PetscFunctionReturn(PETSC_SUCCESS);
3186: }

3188: static PetscErrorCode MatSeqAIJCUSPARSETriFactors_Destroy(Mat_SeqAIJCUSPARSETriFactors **trifactors)
3189: {
3190:   PetscFunctionBegin;
3191:   if (*trifactors) {
3192:     PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(trifactors));
3193:     PetscCall(PetscFree(*trifactors));
3194:   }
3195:   PetscFunctionReturn(PETSC_SUCCESS);
3196: }

3198: static PetscErrorCode MatSeqAIJCUSPARSEInvalidateTranspose(Mat A, PetscBool destroy)
3199: {
3200:   Mat_SeqAIJCUSPARSE *cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;

3202:   PetscFunctionBegin;
3203:   PetscCheckTypeName(A, MATSEQAIJCUSPARSE);
3204:   if (!cusp) PetscFunctionReturn(PETSC_SUCCESS);
3205:   if (destroy) {
3206:     PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusp->matTranspose, cusp->format));
3207:     delete cusp->csr2csc_i;
3208:     cusp->csr2csc_i = NULL;
3209:   }
3210:   A->transupdated = PETSC_FALSE;
3211:   PetscFunctionReturn(PETSC_SUCCESS);
3212: }

3214: static PetscErrorCode MatSetPreallocationCOO_SeqAIJCUSPARSE(Mat mat, PetscCount coo_n, PetscInt coo_i[], PetscInt coo_j[])
3215: {
3216:   PetscFunctionBegin;
3217:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::SetPreallocationCOO(mat, coo_n, coo_i, coo_j));
3218:   PetscFunctionReturn(PETSC_SUCCESS);
3219: }

3221: static PetscErrorCode MatSetValuesCOO_SeqAIJCUSPARSE(Mat A, const PetscScalar v[], InsertMode imode)
3222: {
3223:   PetscFunctionBegin;
3224:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::SetValuesCOO(A, v, imode));
3225:   PetscFunctionReturn(PETSC_SUCCESS);
3226: }

3228: /*@
3229:   MatSeqAIJCUSPARSEGetIJ - returns the device row storage `i` and `j` indices for `MATSEQAIJCUSPARSE` matrices.

3231:   Not Collective

3233:   Input Parameters:
3234: + A          - the matrix
3235: - compressed - `PETSC_TRUE` or `PETSC_FALSE` indicating the matrix data structure should be always returned in compressed form

3237:   Output Parameters:
3238: + i - the CSR row pointers
3239: - j - the CSR column indices

3241:   Level: developer

3243:   Note:
3244:   When compressed is true, the CSR structure does not contain empty rows

3246: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSERestoreIJ()`, `MatSeqAIJCUSPARSEGetArrayRead()`
3247: @*/
3248: PetscErrorCode MatSeqAIJCUSPARSEGetIJ(Mat A, PetscBool compressed, const PetscInt *i[], const PetscInt *j[])
3249: {
3250:   PetscFunctionBegin;
3251:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetIJ(A, compressed, i, j));
3252:   PetscFunctionReturn(PETSC_SUCCESS);
3253: }

3255: /*@
3256:   MatSeqAIJCUSPARSERestoreIJ - restore the device row storage `i` and `j` indices obtained with `MatSeqAIJCUSPARSEGetIJ()`

3258:   Not Collective

3260:   Input Parameters:
3261: + A          - the matrix
3262: . compressed - `PETSC_TRUE` or `PETSC_FALSE` indicating the matrix data structure should be always returned in compressed form
3263: . i          - the CSR row pointers
3264: - j          - the CSR column indices

3266:   Level: developer

3268: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetIJ()`
3269: @*/
3270: PetscErrorCode MatSeqAIJCUSPARSERestoreIJ(Mat A, PetscBool compressed, const PetscInt *i[], const PetscInt *j[])
3271: {
3272:   PetscFunctionBegin;
3273:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::RestoreIJ(A, compressed, i, j));
3274:   PetscFunctionReturn(PETSC_SUCCESS);
3275: }

3277: /*@
3278:   MatSeqAIJCUSPARSEGetArrayRead - gives read-only access to the array where the device data for a `MATSEQAIJCUSPARSE` matrix nonzero entries are stored

3280:   Not Collective

3282:   Input Parameter:
3283: . A - a `MATSEQAIJCUSPARSE` matrix

3285:   Output Parameter:
3286: . a - pointer to the device data

3288:   Level: developer

3290:   Note:
3291:   Will trigger host-to-device copies if the most up-to-date matrix data is on the host

3293: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArray()`, `MatSeqAIJCUSPARSEGetArrayWrite()`, `MatSeqAIJCUSPARSERestoreArrayRead()`
3294: @*/
3295: PetscErrorCode MatSeqAIJCUSPARSEGetArrayRead(Mat A, const PetscScalar **a)
3296: {
3297:   return MatSeqAIJCUSPARSE_CUPM_t::GetArrayRead(A, a);
3298: }

3300: /*@
3301:   MatSeqAIJCUSPARSERestoreArrayRead - restore the read-only access array obtained from `MatSeqAIJCUSPARSEGetArrayRead()`

3303:   Not Collective

3305:   Input Parameters:
3306: + A - a `MATSEQAIJCUSPARSE` matrix
3307: - a - pointer to the device data

3309:   Level: developer

3311: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArrayRead()`
3312: @*/
3313: PetscErrorCode MatSeqAIJCUSPARSERestoreArrayRead(Mat A, const PetscScalar **a)
3314: {
3315:   return MatSeqAIJCUSPARSE_CUPM_t::RestoreArrayRead(A, a);
3316: }

3318: /*@
3319:   MatSeqAIJCUSPARSEGetArray - gives read-write access to the array where the device data for a `MATSEQAIJCUSPARSE` matrix is stored

3321:   Not Collective

3323:   Input Parameter:
3324: . A - a `MATSEQAIJCUSPARSE` matrix

3326:   Output Parameter:
3327: . a - pointer to the device data

3329:   Level: developer

3331:   Note:
3332:   Will trigger host-to-device copies if the most up-to-date matrix data is on the host

3334: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArrayRead()`, `MatSeqAIJCUSPARSEGetArrayWrite()`, `MatSeqAIJCUSPARSERestoreArray()`
3335: @*/
3336: PetscErrorCode MatSeqAIJCUSPARSEGetArray(Mat A, PetscScalar **a)
3337: {
3338:   return MatSeqAIJCUSPARSE_CUPM_t::GetArray(A, a);
3339: }
3340: /*@
3341:   MatSeqAIJCUSPARSERestoreArray - restore the read-write access array obtained from `MatSeqAIJCUSPARSEGetArray()`

3343:   Not Collective

3345:   Input Parameters:
3346: + A - a `MATSEQAIJCUSPARSE` matrix
3347: - a - pointer to the device data

3349:   Level: developer

3351: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArray()`
3352: @*/
3353: PetscErrorCode MatSeqAIJCUSPARSERestoreArray(Mat A, PetscScalar **a)
3354: {
3355:   return MatSeqAIJCUSPARSE_CUPM_t::RestoreArray(A, a);
3356: }

3358: /*@
3359:   MatSeqAIJCUSPARSEGetArrayWrite - gives write access to the array where the device data for a `MATSEQAIJCUSPARSE` matrix is stored

3361:   Not Collective

3363:   Input Parameter:
3364: . A - a `MATSEQAIJCUSPARSE` matrix

3366:   Output Parameter:
3367: . a - pointer to the device data

3369:   Level: developer

3371:   Note:
3372:   Does not trigger any host to device copies.

3374:   It marks the data GPU valid so users must set all the values in `a` to ensure out-of-date data is not considered current

3376: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArray()`, `MatSeqAIJCUSPARSEGetArrayRead()`, `MatSeqAIJCUSPARSERestoreArrayWrite()`
3377: @*/
3378: PetscErrorCode MatSeqAIJCUSPARSEGetArrayWrite(Mat A, PetscScalar **a)
3379: {
3380:   return MatSeqAIJCUSPARSE_CUPM_t::GetArrayWrite(A, a);
3381: }

3383: /*@
3384:   MatSeqAIJCUSPARSERestoreArrayWrite - restore the write-only access array obtained from `MatSeqAIJCUSPARSEGetArrayWrite()`

3386:   Not Collective

3388:   Input Parameters:
3389: + A - a `MATSEQAIJCUSPARSE` matrix
3390: - a - pointer to the device data

3392:   Level: developer

3394: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArrayWrite()`
3395: @*/
3396: PetscErrorCode MatSeqAIJCUSPARSERestoreArrayWrite(Mat A, PetscScalar **a)
3397: {
3398:   return MatSeqAIJCUSPARSE_CUPM_t::RestoreArrayWrite(A, a);
3399: }

3401: struct IJCompare4 {
3402:   __host__ __device__ inline bool operator()(const thrust::tuple<PetscInt, PetscInt, PetscScalar, PetscInt> &t1, const thrust::tuple<PetscInt, PetscInt, PetscScalar, PetscInt> &t2)
3403:   {
3404:     if (thrust::get<0>(t1) < thrust::get<0>(t2)) return true;
3405:     if (thrust::get<0>(t1) == thrust::get<0>(t2)) return thrust::get<1>(t1) < thrust::get<1>(t2);
3406:     return false;
3407:   }
3408: };

3410: struct Shift {
3411:   PetscInt _shift;

3413:   Shift(PetscInt shift) : _shift(shift) { }
3414:   __host__ __device__ inline PetscInt operator()(const PetscInt &c) { return c + _shift; }
3415: };

3417: /* merges two SeqAIJCUSPARSE matrices A, B by concatenating their rows. [A';B']' operation in MATLAB notation */
3418: PetscErrorCode MatSeqAIJCUSPARSEMergeMats(Mat A, Mat B, MatReuse reuse, Mat *C)
3419: {
3420:   Mat_SeqAIJ                   *a = (Mat_SeqAIJ *)A->data, *b = (Mat_SeqAIJ *)B->data, *c;
3421:   Mat_SeqAIJCUSPARSE           *Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr, *Bcusp = (Mat_SeqAIJCUSPARSE *)B->spptr, *Ccusp;
3422:   Mat_SeqAIJCUSPARSEMultStruct *Cmat;
3423:   CsrMatrix                    *Acsr, *Bcsr, *Ccsr;
3424:   PetscInt                      Annz, Bnnz;
3425:   PetscInt                      i, m, n, zero = 0;
3426:   PetscDeviceContext            dctx;
3427:   cudaStream_t                  stream;

3429:   PetscFunctionBegin;
3432:   PetscAssertPointer(C, 4);
3433:   PetscCheckTypeName(A, MATSEQAIJCUSPARSE);
3434:   PetscCheckTypeName(B, MATSEQAIJCUSPARSE);
3435:   PetscCheck(A->rmap->n == B->rmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Invalid number or rows %" PetscInt_FMT " != %" PetscInt_FMT, A->rmap->n, B->rmap->n);
3436:   PetscCheck(reuse != MAT_INPLACE_MATRIX, PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_INPLACE_MATRIX not supported");
3437:   PetscCheck(Acusp->format != MAT_CUSPARSE_ELL && Acusp->format != MAT_CUSPARSE_HYB, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
3438:   PetscCheck(Bcusp->format != MAT_CUSPARSE_ELL && Bcusp->format != MAT_CUSPARSE_HYB, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
3439:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
3440:   if (reuse == MAT_INITIAL_MATRIX) {
3441:     m = A->rmap->n;
3442:     n = A->cmap->n + B->cmap->n;
3443:     PetscCall(MatCreate(PETSC_COMM_SELF, C));
3444:     PetscCall(MatSetSizes(*C, m, n, m, n));
3445:     PetscCall(MatSetType(*C, MATSEQAIJCUSPARSE));
3446:     c                       = (Mat_SeqAIJ *)(*C)->data;
3447:     Ccusp                   = (Mat_SeqAIJCUSPARSE *)(*C)->spptr;
3448:     Cmat                    = new Mat_SeqAIJCUSPARSEMultStruct;
3449:     Ccsr                    = new CsrMatrix;
3450:     Cmat->cprowIndices      = NULL;
3451:     c->compressedrow.use    = PETSC_FALSE;
3452:     c->compressedrow.nrows  = 0;
3453:     c->compressedrow.i      = NULL;
3454:     c->compressedrow.rindex = NULL;
3455:     Ccusp->workVector       = NULL;
3456:     Ccusp->nrows            = m;
3457:     Ccusp->mat              = Cmat;
3458:     Ccusp->mat->mat         = Ccsr;
3459:     Ccsr->num_rows          = m;
3460:     Ccsr->num_cols          = n;
3461:     PetscCallCUSPARSE(cusparseCreateMatDescr(&Cmat->descr));
3462:     PetscCallCUSPARSE(cusparseSetMatIndexBase(Cmat->descr, CUSPARSE_INDEX_BASE_ZERO));
3463:     PetscCallCUSPARSE(cusparseSetMatType(Cmat->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
3464:     PetscCallCUDA(cudaMalloc((void **)&Cmat->alpha_one, sizeof(PetscScalar)));
3465:     PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_zero, sizeof(PetscScalar)));
3466:     PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_one, sizeof(PetscScalar)));
3467:     PetscCallCUDA(cudaMemcpyAsync(Cmat->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3468:     PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3469:     PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3470:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
3471:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
3472:     PetscCheck(Acusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3473:     PetscCheck(Bcusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");

3475:     Acsr                 = (CsrMatrix *)Acusp->mat->mat;
3476:     Bcsr                 = (CsrMatrix *)Bcusp->mat->mat;
3477:     Annz                 = (PetscInt)Acsr->column_indices->size();
3478:     Bnnz                 = (PetscInt)Bcsr->column_indices->size();
3479:     c->nz                = Annz + Bnnz;
3480:     Ccsr->row_offsets    = new THRUSTINTARRAY(m + 1);
3481:     Ccsr->column_indices = new THRUSTINTARRAY(c->nz);
3482:     Ccsr->values         = new THRUSTARRAY(c->nz);
3483:     Ccsr->num_entries    = c->nz;
3484:     Ccusp->coords        = new THRUSTINTARRAY(c->nz);
3485:     if (c->nz) {
3486:       auto            Acoo = new THRUSTINTARRAY(Annz); // initialized with zeros
3487:       auto            Bcoo = new THRUSTINTARRAY(Bnnz);
3488:       auto            Ccoo = new THRUSTINTARRAY(c->nz);
3489:       THRUSTINTARRAY *Aroff, *Broff;

3491:       if (a->compressedrow.use) { /* need full row offset */
3492:         if (!Acusp->rowoffsets_gpu) {
3493:           Acusp->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
3494:           PetscCallCUDA(cudaMemcpyAsync(Acusp->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
3495:           PetscCall(PetscLogCpuToGpu((A->rmap->n + 1) * sizeof(PetscInt)));
3496:         }
3497:         Aroff = Acusp->rowoffsets_gpu;
3498:       } else Aroff = Acsr->row_offsets;
3499:       if (b->compressedrow.use) { /* need full row offset */
3500:         if (!Bcusp->rowoffsets_gpu) {
3501:           Bcusp->rowoffsets_gpu = new THRUSTINTARRAY(B->rmap->n + 1);
3502:           PetscCallCUDA(cudaMemcpyAsync(Bcusp->rowoffsets_gpu->data().get(), b->i, (B->rmap->n + 1) * sizeof(*b->i), cudaMemcpyHostToDevice, stream));
3503:           PetscCall(PetscLogCpuToGpu((B->rmap->n + 1) * sizeof(PetscInt)));
3504:         }
3505:         Broff = Bcusp->rowoffsets_gpu;
3506:       } else Broff = Bcsr->row_offsets;
3507:       PetscCall(PetscLogGpuTimeBegin());
3508:       // Implement cusparseXcsr2coo() with Thrust, as the former doesn't support 64-bit indices.
3509:       PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(m), Csr2coo(Aroff->data().get(), Acoo->data().get())));
3510:       PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(m), Csr2coo(Broff->data().get(), Bcoo->data().get())));

3512:       /* Issues when using bool with large matrices on SUMMIT 10.2.89 */
3513: #if CCCL_VERSION >= 3004000
3514:       auto Aperm = cuda::make_constant_iterator(1);
3515:       auto Bperm = cuda::make_constant_iterator(0);
3516: #else
3517:       auto Aperm = thrust::make_constant_iterator(1);
3518:       auto Bperm = thrust::make_constant_iterator(0);
3519: #endif
3520:       auto Bcib  = thrust::make_transform_iterator(Bcsr->column_indices->begin(), Shift(A->cmap->n));
3521:       auto Bcie  = thrust::make_transform_iterator(Bcsr->column_indices->end(), Shift(A->cmap->n));
3522:       auto wPerm = new THRUSTINTARRAY(Annz + Bnnz);
3523:       auto Azb   = thrust::make_zip_iterator(thrust::make_tuple(Acoo->begin(), Acsr->column_indices->begin(), Acsr->values->begin(), Aperm));
3524:       auto Aze   = thrust::make_zip_iterator(thrust::make_tuple(Acoo->end(), Acsr->column_indices->end(), Acsr->values->end(), Aperm));
3525:       auto Bzb   = thrust::make_zip_iterator(thrust::make_tuple(Bcoo->begin(), Bcib, Bcsr->values->begin(), Bperm)); // Use B column indices shifted by A->cmap->n
3526:       auto Bze   = thrust::make_zip_iterator(thrust::make_tuple(Bcoo->end(), Bcie, Bcsr->values->end(), Bperm));
3527:       auto Czb   = thrust::make_zip_iterator(thrust::make_tuple(Ccoo->begin(), Ccsr->column_indices->begin(), Ccsr->values->begin(), wPerm->begin()));
3528:       auto p1    = Ccusp->coords->begin();
3529:       auto p2    = Ccusp->coords->begin();
3530: #if CCCL_VERSION >= 3001000
3531:       cuda::std::advance(p2, Annz);
3532: #else
3533:       thrust::advance(p2, Annz);
3534: #endif
3535:       PetscCallThrust(thrust::merge(thrust::cuda::par.on(stream), Azb, Aze, Bzb, Bze, Czb, IJCompare4())); // put nonzeros in A and B to C in sorted order (by row and then by column)
3536:       auto cci = thrust::make_counting_iterator(zero);
3537:       auto cce = thrust::make_counting_iterator(c->nz);
3538: #if THRUST_VERSION < 200800
3539:       auto pred = thrust::identity<int>();
3540: #else
3541:       auto pred = cuda::std::identity();
3542: #endif
3543:       PetscCallThrust(thrust::copy_if(thrust::cuda::par.on(stream), cci, cce, wPerm->begin(), p1, pred));
3544:       PetscCallThrust(thrust::remove_copy_if(thrust::cuda::par.on(stream), cci, cce, wPerm->begin(), p2, pred));
3545:       // Implement a simplified cusparseXcoo2csr() with Thrust (assuming the row indices are already sorted), as the former doesn't support 64-bit indices.
3546:       PetscCallThrust(thrust::lower_bound(thrust::cuda::par.on(stream), Ccoo->begin(), Ccoo->end(), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(m + 1), Ccsr->row_offsets->begin()));
3547:       PetscCall(PetscLogGpuTimeEnd());
3548:       delete wPerm;
3549:       delete Acoo;
3550:       delete Bcoo;
3551:       delete Ccoo;
3552:       PetscCallCUSPARSE(cusparseCreateCsr(&Cmat->matDescr, Ccsr->num_rows, Ccsr->num_cols, Ccsr->num_entries, Ccsr->row_offsets->data().get(), Ccsr->column_indices->data().get(), Ccsr->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
3553:       if (A->form_explicit_transpose && B->form_explicit_transpose) { /* if A and B have the transpose, generate C transpose too */
3554:         PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
3555:         PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(B));
3556:         PetscBool                     AT = Acusp->matTranspose ? PETSC_TRUE : PETSC_FALSE, BT = Bcusp->matTranspose ? PETSC_TRUE : PETSC_FALSE;
3557:         Mat_SeqAIJCUSPARSEMultStruct *CmatT = new Mat_SeqAIJCUSPARSEMultStruct;
3558:         CsrMatrix                    *CcsrT = new CsrMatrix;
3559:         CsrMatrix                    *AcsrT = AT ? (CsrMatrix *)Acusp->matTranspose->mat : NULL;
3560:         CsrMatrix                    *BcsrT = BT ? (CsrMatrix *)Bcusp->matTranspose->mat : NULL;

3562:         (*C)->form_explicit_transpose = PETSC_TRUE;
3563:         (*C)->transupdated            = PETSC_TRUE;
3564:         Ccusp->rowoffsets_gpu         = NULL;
3565:         CmatT->cprowIndices           = NULL;
3566:         CmatT->mat                    = CcsrT;
3567:         CcsrT->num_rows               = n;
3568:         CcsrT->num_cols               = m;
3569:         CcsrT->num_entries            = c->nz;

3571:         CcsrT->row_offsets    = new THRUSTINTARRAY(n + 1);
3572:         CcsrT->column_indices = new THRUSTINTARRAY(c->nz);
3573:         CcsrT->values         = new THRUSTARRAY(c->nz);

3575:         PetscCall(PetscLogGpuTimeBegin());
3576:         auto rT = CcsrT->row_offsets->begin();
3577:         if (AT) {
3578:           PetscCallThrust(rT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->row_offsets->begin(), AcsrT->row_offsets->end(), rT));
3579: #if CCCL_VERSION >= 3001000
3580:           cuda::std::advance(rT, -1);
3581: #else
3582:           thrust::advance(rT, -1);
3583: #endif
3584:         }
3585:         if (BT) {
3586:           auto titb = thrust::make_transform_iterator(BcsrT->row_offsets->begin(), Shift(a->nz));
3587:           auto tite = thrust::make_transform_iterator(BcsrT->row_offsets->end(), Shift(a->nz));
3588:           PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), titb, tite, rT));
3589:         }
3590:         auto cT = CcsrT->column_indices->begin();
3591:         if (AT) PetscCallThrust(cT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->column_indices->begin(), AcsrT->column_indices->end(), cT));
3592:         if (BT) PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), BcsrT->column_indices->begin(), BcsrT->column_indices->end(), cT));
3593:         auto vT = CcsrT->values->begin();
3594:         if (AT) PetscCallThrust(vT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->values->begin(), AcsrT->values->end(), vT));
3595:         if (BT) PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), BcsrT->values->begin(), BcsrT->values->end(), vT));
3596:         PetscCall(PetscLogGpuTimeEnd());

3598:         PetscCallCUSPARSE(cusparseCreateMatDescr(&CmatT->descr));
3599:         PetscCallCUSPARSE(cusparseSetMatIndexBase(CmatT->descr, CUSPARSE_INDEX_BASE_ZERO));
3600:         PetscCallCUSPARSE(cusparseSetMatType(CmatT->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
3601:         PetscCallCUDA(cudaMalloc((void **)&CmatT->alpha_one, sizeof(PetscScalar)));
3602:         PetscCallCUDA(cudaMalloc((void **)&CmatT->beta_zero, sizeof(PetscScalar)));
3603:         PetscCallCUDA(cudaMalloc((void **)&CmatT->beta_one, sizeof(PetscScalar)));
3604:         PetscCallCUDA(cudaMemcpyAsync(CmatT->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3605:         PetscCallCUDA(cudaMemcpyAsync(CmatT->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3606:         PetscCallCUDA(cudaMemcpyAsync(CmatT->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3607:         PetscCallCUSPARSE(cusparseCreateCsr(&CmatT->matDescr, CcsrT->num_rows, CcsrT->num_cols, CcsrT->num_entries, CcsrT->row_offsets->data().get(), CcsrT->column_indices->data().get(), CcsrT->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
3608:         Ccusp->matTranspose = CmatT;
3609:       }
3610:     }

3612:     c->free_a = PETSC_TRUE;
3613:     PetscCall(PetscShmgetAllocateArray(c->nz, sizeof(PetscInt), (void **)&c->j));
3614:     PetscCall(PetscShmgetAllocateArray(m + 1, sizeof(PetscInt), (void **)&c->i));
3615:     c->free_ij = PETSC_TRUE;
3616:     PetscCallCUDA(cudaMemcpyAsync(c->i, Ccsr->row_offsets->data().get(), Ccsr->row_offsets->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
3617:     PetscCallCUDA(cudaMemcpyAsync(c->j, Ccsr->column_indices->data().get(), Ccsr->column_indices->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
3618:     PetscCall(PetscDeviceContextSynchronize(dctx));
3619:     PetscCall(PetscLogGpuToCpu((Ccsr->column_indices->size() + Ccsr->row_offsets->size()) * sizeof(PetscInt)));
3620:     PetscCall(PetscMalloc1(m, &c->ilen));
3621:     PetscCall(PetscMalloc1(m, &c->imax));
3622:     c->maxnz         = c->nz;
3623:     c->nonzerorowcnt = 0;
3624:     c->rmax          = 0;
3625:     for (i = 0; i < m; i++) {
3626:       const PetscInt nn = c->i[i + 1] - c->i[i];
3627:       c->ilen[i] = c->imax[i] = nn;
3628:       c->nonzerorowcnt += (PetscInt)!!nn;
3629:       c->rmax = PetscMax(c->rmax, nn);
3630:     }
3631:     PetscCall(PetscMalloc1(c->nz, &c->a));
3632:     (*C)->nonzerostate++;
3633:     PetscCall(PetscLayoutSetUp((*C)->rmap));
3634:     PetscCall(PetscLayoutSetUp((*C)->cmap));
3635:     Ccusp->nonzerostate = (*C)->nonzerostate;
3636:     (*C)->preallocated  = PETSC_TRUE;
3637:   } else {
3638:     PetscCheck((*C)->rmap->n == B->rmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Invalid number or rows %" PetscInt_FMT " != %" PetscInt_FMT, (*C)->rmap->n, B->rmap->n);
3639:     c = (Mat_SeqAIJ *)(*C)->data;
3640:     if (c->nz) {
3641:       Ccusp = (Mat_SeqAIJCUSPARSE *)(*C)->spptr;
3642:       PetscCheck(Ccusp->coords, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing coords");
3643:       PetscCheck(Ccusp->format != MAT_CUSPARSE_ELL && Ccusp->format != MAT_CUSPARSE_HYB, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
3644:       PetscCheck(Ccusp->nonzerostate == (*C)->nonzerostate, PETSC_COMM_SELF, PETSC_ERR_COR, "Wrong nonzerostate");
3645:       PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
3646:       PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
3647:       PetscCheck(Acusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3648:       PetscCheck(Bcusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3649:       Acsr = (CsrMatrix *)Acusp->mat->mat;
3650:       Bcsr = (CsrMatrix *)Bcusp->mat->mat;
3651:       Ccsr = (CsrMatrix *)Ccusp->mat->mat;
3652:       PetscCheck(Acsr->num_entries == (PetscInt)Acsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "A nnz %" PetscInt_FMT " != %" PetscInt_FMT, Acsr->num_entries, (PetscInt)Acsr->values->size());
3653:       PetscCheck(Bcsr->num_entries == (PetscInt)Bcsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "B nnz %" PetscInt_FMT " != %" PetscInt_FMT, Bcsr->num_entries, (PetscInt)Bcsr->values->size());
3654:       PetscCheck(Ccsr->num_entries == (PetscInt)Ccsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "C nnz %" PetscInt_FMT " != %" PetscInt_FMT, Ccsr->num_entries, (PetscInt)Ccsr->values->size());
3655:       PetscCheck(Ccsr->num_entries == Acsr->num_entries + Bcsr->num_entries, PETSC_COMM_SELF, PETSC_ERR_COR, "C nnz %" PetscInt_FMT " != %" PetscInt_FMT " + %" PetscInt_FMT, Ccsr->num_entries, Acsr->num_entries, Bcsr->num_entries);
3656:       PetscCheck(Ccusp->coords->size() == Ccsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "permSize %" PetscInt_FMT " != %" PetscInt_FMT, (PetscInt)Ccusp->coords->size(), (PetscInt)Ccsr->values->size());
3657:       auto pmid = Ccusp->coords->begin();
3658: #if CCCL_VERSION >= 3001000
3659:       cuda::std::advance(pmid, Acsr->num_entries);
3660: #else
3661:       thrust::advance(pmid, Acsr->num_entries);
3662: #endif
3663:       PetscCall(PetscLogGpuTimeBegin());
3664:       auto zibait = thrust::make_zip_iterator(thrust::make_tuple(Acsr->values->begin(), thrust::make_permutation_iterator(Ccsr->values->begin(), Ccusp->coords->begin())));
3665:       auto zieait = thrust::make_zip_iterator(thrust::make_tuple(Acsr->values->end(), thrust::make_permutation_iterator(Ccsr->values->begin(), pmid)));
3666:       PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), zibait, zieait, VecCUDAEquals()));
3667:       auto zibbit = thrust::make_zip_iterator(thrust::make_tuple(Bcsr->values->begin(), thrust::make_permutation_iterator(Ccsr->values->begin(), pmid)));
3668:       auto ziebit = thrust::make_zip_iterator(thrust::make_tuple(Bcsr->values->end(), thrust::make_permutation_iterator(Ccsr->values->begin(), Ccusp->coords->end())));
3669:       PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), zibbit, ziebit, VecCUDAEquals()));
3670:       PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(*C, PETSC_FALSE));
3671:       if (A->form_explicit_transpose && B->form_explicit_transpose && (*C)->form_explicit_transpose) {
3672:         PetscCheck(Ccusp->matTranspose, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing transpose Mat_SeqAIJCUSPARSEMultStruct");
3673:         PetscBool  AT = Acusp->matTranspose ? PETSC_TRUE : PETSC_FALSE, BT = Bcusp->matTranspose ? PETSC_TRUE : PETSC_FALSE;
3674:         CsrMatrix *AcsrT = AT ? (CsrMatrix *)Acusp->matTranspose->mat : NULL;
3675:         CsrMatrix *BcsrT = BT ? (CsrMatrix *)Bcusp->matTranspose->mat : NULL;
3676:         CsrMatrix *CcsrT = (CsrMatrix *)Ccusp->matTranspose->mat;
3677:         auto       vT    = CcsrT->values->begin();
3678:         if (AT) PetscCallThrust(vT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->values->begin(), AcsrT->values->end(), vT));
3679:         if (BT) PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), BcsrT->values->begin(), BcsrT->values->end(), vT));
3680:         (*C)->transupdated = PETSC_TRUE;
3681:       }
3682:       PetscCall(PetscLogGpuTimeEnd());
3683:     }
3684:     PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
3685:   }
3686:   PetscCall(PetscObjectStateIncrease((PetscObject)*C));
3687:   (*C)->assembled     = PETSC_TRUE;
3688:   (*C)->was_assembled = PETSC_FALSE;
3689:   (*C)->offloadmask   = PETSC_OFFLOAD_GPU;
3690:   PetscFunctionReturn(PETSC_SUCCESS);
3691: }

3693: static PetscErrorCode MatSeqAIJCopySubArray_SeqAIJCUSPARSE(Mat A, PetscInt n, const PetscInt idx[], PetscScalar v[])
3694: {
3695:   PetscFunctionBegin;
3696:   PetscCall(MatSeqAIJCUSPARSE_CUPM_t::CopySubArray(A, n, idx, v));
3697:   PetscFunctionReturn(PETSC_SUCCESS);
3698: }