Actual source code: aijcusparse.cu
1: /*
2: Defines the basic matrix operations for the AIJ (compressed row)
3: matrix storage format using the CUSPARSE library,
4: */
5: #define PETSC_SKIP_IMMINTRIN_H_CUDAWORKAROUND 1
7: #include <../src/mat/impls/aij/seq/aij.h>
8: #include <petscpkg_version.h>
9: #include <../src/mat/impls/sbaij/seq/sbaij.h>
10: #include <../src/vec/vec/impls/dvecimpl.h>
11: #include <petsc/private/vecimpl.h>
12: #undef VecType
13: #include <../src/mat/impls/aij/seq/seqcusparse/cusparsematimpl.h>
14: #include <../src/mat/impls/aij/seq/cupm/aijcupm.hpp>
15: #include <thrust/adjacent_difference.h>
16: #include <thrust/iterator/constant_iterator.h>
17: #include <thrust/remove.h>
18: #include <thrust/sort.h>
19: #include <thrust/tuple.h>
20: #include <thrust/unique.h>
21: #include <thrust/gather.h>
22: #include <thrust/binary_search.h> // for thrust::lower_bound
23: #include <thrust/version.h>
24: #if THRUST_VERSION >= 200800
25: #include <cuda/std/functional>
26: #endif
27: #if CCCL_VERSION >= 3004000
28: #include <cuda/iterator>
29: #endif
31: const char *const MatCUSPARSEStorageFormats[] = {"CSR", "ELL", "HYB", "MatCUSPARSEStorageFormat", "MAT_CUSPARSE_", 0};
32: /*
33: The following are copied from cusparse.h in CUDA-11.0. In MatCUSPARSESpMVAlgorithms[] etc, we copy them in
34: 0-based integer value order, since we want to use PetscOptionsEnum() to parse user command line options for them.
35: */
36: const char *const MatCUSPARSESpMVAlgorithms[] = {"MV_ALG_DEFAULT", "COOMV_ALG", "CSRMV_ALG1", "CSRMV_ALG2", "cusparseSpMVAlg_t", "CUSPARSE_", 0};
37: const char *const MatCUSPARSESpMMAlgorithms[] = {"ALG_DEFAULT", "COO_ALG1", "COO_ALG2", "COO_ALG3", "CSR_ALG1", "COO_ALG4", "CSR_ALG2", "cusparseSpMMAlg_t", "CUSPARSE_SPMM_", 0};
38: const char *const MatCUSPARSECsr2CscAlgorithms[] = {"INVALID" /*cusparse does not have enum 0! We created one*/, "ALG1", "ALG2", "cusparseCsr2CscAlg_t", "CUSPARSE_CSR2CSC_", 0};
40: static PetscErrorCode MatICCFactorSymbolic_SeqAIJCUSPARSE(Mat, Mat, IS, const MatFactorInfo *);
41: static PetscErrorCode MatCholeskyFactorSymbolic_SeqAIJCUSPARSE(Mat, Mat, IS, const MatFactorInfo *);
42: static PetscErrorCode MatCholeskyFactorNumeric_SeqAIJCUSPARSE(Mat, Mat, const MatFactorInfo *);
43: static PetscErrorCode MatILUFactorSymbolic_SeqAIJCUSPARSE(Mat, Mat, IS, IS, const MatFactorInfo *);
44: static PetscErrorCode MatSetFromOptions_SeqAIJCUSPARSE(Mat, PetscOptionItems PetscOptionsObject);
45: static PetscErrorCode MatAXPY_SeqAIJCUSPARSE(Mat, PetscScalar, Mat, MatStructure);
46: static PetscErrorCode MatScale_SeqAIJCUSPARSE(Mat, PetscScalar);
47: static PetscErrorCode MatDiagonalScale_SeqAIJCUSPARSE(Mat, Vec, Vec);
48: static PetscErrorCode MatMult_SeqAIJCUSPARSE(Mat, Vec, Vec);
49: static PetscErrorCode MatMultAdd_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec);
50: static PetscErrorCode MatMultTranspose_SeqAIJCUSPARSE(Mat, Vec, Vec);
51: static PetscErrorCode MatMultTransposeAdd_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec);
52: static PetscErrorCode MatMultHermitianTranspose_SeqAIJCUSPARSE(Mat, Vec, Vec);
53: static PetscErrorCode MatMultHermitianTransposeAdd_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec);
54: static PetscErrorCode MatMultAddKernel_SeqAIJCUSPARSE(Mat, Vec, Vec, Vec, PetscBool, PetscBool);
56: static PetscErrorCode CsrMatrix_Destroy(CsrMatrix **);
57: static PetscErrorCode MatSeqAIJCUSPARSEMultStruct_Destroy(Mat_SeqAIJCUSPARSEMultStruct **, MatCUSPARSEStorageFormat);
58: static PetscErrorCode MatSeqAIJCUSPARSETriFactors_Destroy(Mat_SeqAIJCUSPARSETriFactors **);
59: static PetscErrorCode MatSeqAIJCUSPARSE_Destroy(Mat);
61: static PetscErrorCode MatSeqAIJCUSPARSECopyFromGPU(Mat);
62: static PetscErrorCode MatSeqAIJCUSPARSEInvalidateTranspose(Mat, PetscBool);
64: static PetscErrorCode MatSeqAIJCopySubArray_SeqAIJCUSPARSE(Mat, PetscInt, const PetscInt[], PetscScalar[]);
65: static PetscErrorCode MatSetPreallocationCOO_SeqAIJCUSPARSE(Mat, PetscCount, PetscInt[], PetscInt[]);
66: static PetscErrorCode MatSetValuesCOO_SeqAIJCUSPARSE(Mat, const PetscScalar[], InsertMode);
67: PETSC_INTERN PetscErrorCode MatConvert_SeqAIJ_SeqAIJCUSPARSE(Mat, MatType, MatReuse, Mat *);
69: // cusparseCreateCsr() separates types for row offsets and column indices in prototype, but requires them to have the same type at runtime!
70: const cusparseIndexType_t csrRowOffsetsType = PetscDefined(USE_64BIT_INDICES) ? CUSPARSE_INDEX_64I : CUSPARSE_INDEX_32I;
71: const cusparseIndexType_t csrColIndType = PetscDefined(USE_64BIT_INDICES) ? CUSPARSE_INDEX_64I : CUSPARSE_INDEX_32I;
73: /* Policy struct for MatSeqAIJCUSPARSE_CUPM shared template (CUDA specialisation) */
74: struct MatSeqAIJCUSPARSE_Policy {
75: typedef Mat_SeqAIJCUSPARSE mat_struct_type;
76: typedef Mat_SeqAIJCUSPARSEMultStruct mult_struct_type;
78: static int storage_format_csr() { return (int)MAT_CUSPARSE_CSR; }
79: static int storage_format_ell() { return (int)MAT_CUSPARSE_ELL; }
80: static int storage_format_hyb() { return (int)MAT_CUSPARSE_HYB; }
82: static PetscErrorCode CopyToGPU(Mat A) { return MatSeqAIJCUSPARSECopyToGPU(A); }
83: static PetscErrorCode CopyFromGPU(Mat A) { return MatSeqAIJCUSPARSECopyFromGPU(A); }
84: static PetscErrorCode InvalidateTranspose(Mat A, PetscBool d) { return MatSeqAIJCUSPARSEInvalidateTranspose(A, d); }
85: static PetscErrorCode ConvertFromSeqAIJ(Mat B, MatType t, MatReuse r, Mat *C) { return MatConvert_SeqAIJ_SeqAIJCUSPARSE(B, t, r, C); }
86: static const char *mat_type_name;
88: static PetscErrorCode Destroy(Mat A) { return MatSeqAIJCUSPARSE_Destroy(A); }
89: static PetscErrorCode TriFactorsDestroy(void **spptr) { return MatSeqAIJCUSPARSETriFactors_Destroy((Mat_SeqAIJCUSPARSETriFactors **)spptr); }
90: static const char *set_format_c;
91: static const char *set_use_cpu_solve_c;
92: static const char *product_seqdense_device_c;
93: static const char *product_seqdense_c;
94: static const char *product_self_c;
95: static const char *seq_convert_hypre_c;
97: static PetscErrorCode VecGetArrayRead(Vec v, const PetscScalar **a) { return VecCUDAGetArrayRead(v, a); }
98: static PetscErrorCode VecRestoreArrayRead(Vec v, const PetscScalar **a) { return VecCUDARestoreArrayRead(v, a); }
99: static PetscErrorCode VecGetArrayWrite(Vec v, PetscScalar **a) { return VecCUDAGetArrayWrite(v, a); }
100: static PetscErrorCode VecRestoreArrayWrite(Vec v, PetscScalar **a) { return VecCUDARestoreArrayWrite(v, a); }
101: };
102: const char *MatSeqAIJCUSPARSE_Policy::mat_type_name = MATSEQAIJCUSPARSE;
103: const char *MatSeqAIJCUSPARSE_Policy::set_format_c = "MatCUSPARSESetFormat_C";
104: const char *MatSeqAIJCUSPARSE_Policy::set_use_cpu_solve_c = "MatCUSPARSESetUseCPUSolve_C";
105: const char *MatSeqAIJCUSPARSE_Policy::product_seqdense_device_c = "MatProductSetFromOptions_seqaijcusparse_seqdensecuda_C";
106: const char *MatSeqAIJCUSPARSE_Policy::product_seqdense_c = "MatProductSetFromOptions_seqaijcusparse_seqdense_C";
107: const char *MatSeqAIJCUSPARSE_Policy::product_self_c = "MatProductSetFromOptions_seqaijcusparse_seqaijcusparse_C";
108: const char *MatSeqAIJCUSPARSE_Policy::seq_convert_hypre_c = "MatConvert_seqaijcusparse_hypre_C";
110: using MatSeqAIJCUSPARSE_CUPM_t = Petsc::mat::aij::cupm::impl::MatSeqAIJCUSPARSE_CUPM<Petsc::device::cupm::DeviceType::CUDA, MatSeqAIJCUSPARSE_Policy>;
112: using SparsePointerModeGuard = Petsc::device::cupm::impl::CUPMSparsePointerModeGuard<Petsc::device::cupm::DeviceType::CUDA>;
114: using Csr2coo = Petsc::mat::aij::cupm::impl::Csr2coo;
115: using PetscIntToCInt = Petsc::mat::aij::cupm::impl::PetscIntToCInt;
117: PETSC_INTERN PetscErrorCode MatCUSPARSESetFormat_SeqAIJCUSPARSE(Mat A, MatCUSPARSEFormatOperation op, MatCUSPARSEStorageFormat format)
118: {
119: Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
121: PetscFunctionBegin;
122: switch (op) {
123: case MAT_CUSPARSE_MULT:
124: cusparsestruct->format = format;
125: break;
126: case MAT_CUSPARSE_ALL:
127: cusparsestruct->format = format;
128: break;
129: default:
130: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "unsupported operation %d for MatCUSPARSEFormatOperation. MAT_CUSPARSE_MULT and MAT_CUSPARSE_ALL are currently supported.", op);
131: }
132: PetscFunctionReturn(PETSC_SUCCESS);
133: }
135: /*@
136: MatCUSPARSESetFormat - Sets the storage format of `MATSEQCUSPARSE` matrices for a particular
137: operation. Only the `MatMult()` operation can use different GPU storage formats
139: Not Collective
141: Input Parameters:
142: + A - Matrix of type `MATSEQAIJCUSPARSE`
143: . op - `MatCUSPARSEFormatOperation`. `MATSEQAIJCUSPARSE` matrices support `MAT_CUSPARSE_MULT` and `MAT_CUSPARSE_ALL`.
144: `MATMPIAIJCUSPARSE` matrices support `MAT_CUSPARSE_MULT_DIAG`,`MAT_CUSPARSE_MULT_OFFDIAG`, and `MAT_CUSPARSE_ALL`.
145: - format - `MatCUSPARSEStorageFormat` (one of `MAT_CUSPARSE_CSR`, `MAT_CUSPARSE_ELL`, `MAT_CUSPARSE_HYB`.)
147: Level: intermediate
149: .seealso: [](ch_matrices), `Mat`, `MATSEQAIJCUSPARSE`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
150: @*/
151: PetscErrorCode MatCUSPARSESetFormat(Mat A, MatCUSPARSEFormatOperation op, MatCUSPARSEStorageFormat format)
152: {
153: PetscFunctionBegin;
155: PetscTryMethod(A, "MatCUSPARSESetFormat_C", (Mat, MatCUSPARSEFormatOperation, MatCUSPARSEStorageFormat), (A, op, format));
156: PetscFunctionReturn(PETSC_SUCCESS);
157: }
159: PETSC_INTERN PetscErrorCode MatCUSPARSESetUseCPUSolve_SeqAIJCUSPARSE(Mat A, PetscBool use_cpu)
160: {
161: Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
163: PetscFunctionBegin;
164: cusparsestruct->use_cpu_solve = use_cpu;
165: PetscFunctionReturn(PETSC_SUCCESS);
166: }
168: /*@
169: MatCUSPARSESetUseCPUSolve - Sets to use CPU `MatSolve()`.
171: Input Parameters:
172: + A - Matrix of type `MATSEQAIJCUSPARSE`
173: - use_cpu - set flag for using the built-in CPU `MatSolve()`
175: Level: intermediate
177: Note:
178: The NVIDIA cuSPARSE LU solver currently computes the factors with the built-in CPU method
179: and moves the factors to the GPU for the solve. We have observed better performance keeping the data on the CPU and performing the solve there.
180: This method to specify if the solve is done on the CPU or GPU (GPU is the default).
182: .seealso: [](ch_matrices), `Mat`, `MatSolve()`, `MATSEQAIJCUSPARSE`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
183: @*/
184: PetscErrorCode MatCUSPARSESetUseCPUSolve(Mat A, PetscBool use_cpu)
185: {
186: PetscFunctionBegin;
188: PetscTryMethod(A, "MatCUSPARSESetUseCPUSolve_C", (Mat, PetscBool), (A, use_cpu));
189: PetscFunctionReturn(PETSC_SUCCESS);
190: }
192: static PetscErrorCode MatSetOption_SeqAIJCUSPARSE(Mat A, MatOption op, PetscBool flg)
193: {
194: PetscFunctionBegin;
195: switch (op) {
196: case MAT_FORM_EXPLICIT_TRANSPOSE:
197: /* need to destroy the transpose matrix if present to prevent from logic errors if flg is set to true later */
198: if (A->form_explicit_transpose && !flg) PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_TRUE));
199: A->form_explicit_transpose = flg;
200: break;
201: default:
202: PetscCall(MatSetOption_SeqAIJ(A, op, flg));
203: break;
204: }
205: PetscFunctionReturn(PETSC_SUCCESS);
206: }
208: static PetscErrorCode MatSetFromOptions_SeqAIJCUSPARSE(Mat A, PetscOptionItems PetscOptionsObject)
209: {
210: MatCUSPARSEStorageFormat format;
211: PetscBool flg;
212: Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
214: PetscFunctionBegin;
215: PetscOptionsHeadBegin(PetscOptionsObject, "SeqAIJCUSPARSE options");
216: if (A->factortype == MAT_FACTOR_NONE) {
217: PetscCall(PetscOptionsEnum("-mat_cusparse_mult_storage_format", "sets storage format of (seq)aijcusparse gpu matrices for SpMV", "MatCUSPARSESetFormat", MatCUSPARSEStorageFormats, (PetscEnum)cusparsestruct->format, (PetscEnum *)&format, &flg));
218: if (flg) PetscCall(MatCUSPARSESetFormat(A, MAT_CUSPARSE_MULT, format));
220: PetscCall(PetscOptionsEnum("-mat_cusparse_storage_format", "sets storage format of (seq)aijcusparse gpu matrices for SpMV and TriSolve", "MatCUSPARSESetFormat", MatCUSPARSEStorageFormats, (PetscEnum)cusparsestruct->format, (PetscEnum *)&format, &flg));
221: if (flg) PetscCall(MatCUSPARSESetFormat(A, MAT_CUSPARSE_ALL, format));
222: PetscCall(PetscOptionsBool("-mat_cusparse_use_cpu_solve", "Use CPU (I)LU solve", "MatCUSPARSESetUseCPUSolve", cusparsestruct->use_cpu_solve, &cusparsestruct->use_cpu_solve, &flg));
223: if (flg) PetscCall(MatCUSPARSESetUseCPUSolve(A, cusparsestruct->use_cpu_solve));
224: PetscCall(PetscOptionsEnum("-mat_cusparse_spmv_alg", "sets cuSPARSE algorithm used in sparse-mat dense-vector multiplication (SpMV)", "cusparseSpMVAlg_t", MatCUSPARSESpMVAlgorithms, (PetscEnum)cusparsestruct->spmvAlg, (PetscEnum *)&cusparsestruct->spmvAlg, &flg));
225: /* If user did use this option, check its consistency with cuSPARSE, since PetscOptionsEnum() sets enum values based on their position in MatCUSPARSESpMVAlgorithms[] */
226: PetscCheck(!flg || CUSPARSE_SPMV_CSR_ALG1 == 2, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE enum cusparseSpMVAlg_t has been changed but PETSc has not been updated accordingly");
227: PetscCall(PetscOptionsEnum("-mat_cusparse_spmm_alg", "sets cuSPARSE algorithm used in sparse-mat dense-mat multiplication (SpMM)", "cusparseSpMMAlg_t", MatCUSPARSESpMMAlgorithms, (PetscEnum)cusparsestruct->spmmAlg, (PetscEnum *)&cusparsestruct->spmmAlg, &flg));
228: PetscCheck(!flg || CUSPARSE_SPMM_CSR_ALG1 == 4, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE enum cusparseSpMMAlg_t has been changed but PETSc has not been updated accordingly");
229: PetscCall(
230: PetscOptionsEnum("-mat_cusparse_csr2csc_alg", "sets cuSPARSE algorithm used in converting CSR matrices to CSC matrices", "cusparseCsr2CscAlg_t", MatCUSPARSECsr2CscAlgorithms, (PetscEnum)cusparsestruct->csr2cscAlg, (PetscEnum *)&cusparsestruct->csr2cscAlg, &flg));
231: PetscCheck(!flg || CUSPARSE_CSR2CSC_ALG1 == 1, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE enum cusparseCsr2CscAlg_t has been changed but PETSc has not been updated accordingly");
232: }
233: PetscOptionsHeadEnd();
234: PetscFunctionReturn(PETSC_SUCCESS);
235: }
237: static PetscErrorCode MatSeqAIJCUSPARSEBuildFactoredMatrix_LU(Mat A)
238: {
239: Mat_SeqAIJ *a = static_cast<Mat_SeqAIJ *>(A->data);
240: PetscInt m = A->rmap->n;
241: Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
242: const PetscInt *Ai = a->i, *Aj = a->j, *adiag;
243: const MatScalar *Aa = a->a;
244: PetscInt *Mi, *Mj, Mnz;
245: PetscScalar *Ma;
246: PetscDeviceContext dctx;
247: cudaStream_t stream;
248: cusparseHandle_t handle;
250: PetscFunctionBegin;
251: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
252: {
253: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
255: PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, &adiag, NULL));
256: if (A->offloadmask == PETSC_OFFLOAD_CPU) { // A's latest factors are on CPU
257: if (!fs->csrRowPtr) { // Is this the first time we are doing setup? Use csrRowPtr since it is not null even when m=0
258: // Re-arrange the (skewed) factored matrix and put the result into M, a regular csr matrix on host
259: Mnz = (Ai[m] - Ai[0]) + (adiag[0] - adiag[m]); // Lnz (without the unit diagonal) + Unz (with the non-unit diagonal)
260: PetscCall(PetscMalloc1(m + 1, &Mi));
261: PetscCall(PetscMalloc1(Mnz, &Mj)); // Mj is temp
262: PetscCall(PetscMalloc1(Mnz, &Ma));
263: Mi[0] = 0;
264: for (PetscInt i = 0; i < m; i++) {
265: PetscInt llen = Ai[i + 1] - Ai[i];
266: PetscInt ulen = adiag[i] - adiag[i + 1];
267: PetscCall(PetscArraycpy(Mj + Mi[i], Aj + Ai[i], llen)); // entries of L
268: Mj[Mi[i] + llen] = i; // diagonal entry
269: PetscCall(PetscArraycpy(Mj + Mi[i] + llen + 1, Aj + adiag[i + 1] + 1, ulen - 1)); // entries of U on the right of the diagonal
270: Mi[i + 1] = Mi[i] + llen + ulen;
271: }
272: // Copy M (L,U) from host to device
273: PetscCallCUDA(cudaMalloc(&fs->csrRowPtr, sizeof(*fs->csrRowPtr) * (m + 1)));
274: PetscCallCUDA(cudaMalloc(&fs->csrColIdx, sizeof(*fs->csrColIdx) * Mnz));
275: PetscCallCUDA(cudaMalloc(&fs->csrVal, sizeof(*fs->csrVal) * Mnz));
276: PetscCallCUDA(cudaMemcpyAsync(fs->csrRowPtr, Mi, sizeof(*fs->csrRowPtr) * (m + 1), cudaMemcpyHostToDevice, stream));
277: PetscCallCUDA(cudaMemcpyAsync(fs->csrColIdx, Mj, sizeof(*fs->csrColIdx) * Mnz, cudaMemcpyHostToDevice, stream));
278: PetscCall(PetscDeviceContextSynchronize(dctx));
280: // Create descriptors for L, U. See https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
281: // cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
282: // assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
283: // all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
284: // assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
285: cusparseFillMode_t fillMode = CUSPARSE_FILL_MODE_LOWER;
286: cusparseDiagType_t diagType = CUSPARSE_DIAG_TYPE_UNIT;
288: PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_L, m, m, Mnz, fs->csrRowPtr, fs->csrColIdx, fs->csrVal, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
289: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
290: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));
292: fillMode = CUSPARSE_FILL_MODE_UPPER;
293: diagType = CUSPARSE_DIAG_TYPE_NON_UNIT;
294: PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_U, m, m, Mnz, fs->csrRowPtr, fs->csrColIdx, fs->csrVal, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
295: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
296: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));
298: // Allocate work vectors in SpSv
299: PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(*fs->X) * m));
300: PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(*fs->Y) * m));
302: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
303: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));
305: // Query buffer sizes for SpSV and then allocate buffers, temporarily assuming opA = CUSPARSE_OPERATION_NON_TRANSPOSE
306: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_L));
307: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, &fs->spsvBufferSize_L));
308: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_U));
309: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, &fs->spsvBufferSize_U));
310: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_U, fs->spsvBufferSize_U));
311: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_L, fs->spsvBufferSize_L));
313: // Record for reuse
314: fs->csrRowPtr_h = Mi;
315: fs->csrVal_h = Ma;
316: PetscCall(PetscFree(Mj));
317: }
318: // Copy the value
319: Mi = fs->csrRowPtr_h;
320: Ma = fs->csrVal_h;
321: Mnz = Mi[m];
322: for (PetscInt i = 0; i < m; i++) {
323: PetscInt llen = Ai[i + 1] - Ai[i];
324: PetscInt ulen = adiag[i] - adiag[i + 1];
325: PetscCall(PetscArraycpy(Ma + Mi[i], Aa + Ai[i], llen)); // entries of L
326: Ma[Mi[i] + llen] = (MatScalar)1.0 / Aa[adiag[i]]; // recover the diagonal entry
327: PetscCall(PetscArraycpy(Ma + Mi[i] + llen + 1, Aa + adiag[i + 1] + 1, ulen - 1)); // entries of U on the right of the diagonal
328: }
329: PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Ma, sizeof(*Ma) * Mnz, cudaMemcpyHostToDevice, stream));
330: PetscCall(PetscDeviceContextSynchronize(dctx));
332: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
333: if (fs->updatedSpSVAnalysis) { // have done cusparseSpSV_analysis before, and only matrix values changed?
334: // Otherwise cusparse would error out: "On entry to cusparseSpSV_updateMatrix() parameter number 3 (newValues) had an illegal value: NULL pointer"
335: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_L, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
336: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_U, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
337: } else
338: #endif
339: {
340: // Do cusparseSpSV_analysis(), which is numeric and requires valid and up-to-date matrix values
341: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, fs->spsvBuffer_L));
343: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, fs->spsvBuffer_U));
344: fs->updatedSpSVAnalysis = PETSC_TRUE;
345: fs->updatedTransposeSpSVAnalysis = PETSC_FALSE;
346: }
347: }
348: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
349: }
350: PetscFunctionReturn(PETSC_SUCCESS);
351: }
353: static PetscErrorCode MatSeqAIJCUSPARSEILUAnalysisAndCopyToGPU(Mat A)
354: {
355: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
356: Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)A->spptr;
357: IS isrow = a->row, isicol = a->icol;
358: PetscBool row_identity, col_identity;
359: PetscInt n = A->rmap->n;
360: PetscDeviceContext dctx;
361: cudaStream_t stream;
363: PetscFunctionBegin;
364: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
365: PetscCheck(cusparseTriFactors, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing cusparseTriFactors");
366: PetscCall(MatSeqAIJCUSPARSEBuildFactoredMatrix_LU(A));
368: cusparseTriFactors->nnz = a->nz;
370: A->offloadmask = PETSC_OFFLOAD_BOTH; // factored matrix is sync'ed to GPU
371: /* lower triangular indices */
372: PetscCall(ISIdentity(isrow, &row_identity));
373: if (!row_identity && !cusparseTriFactors->rpermIndices) {
374: const PetscInt *r;
376: PetscCall(ISGetIndices(isrow, &r));
377: cusparseTriFactors->rpermIndices = new THRUSTINTARRAY(n);
378: PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->rpermIndices->data().get(), r, (n) * sizeof(*r), cudaMemcpyHostToDevice, stream));
379: PetscCall(PetscDeviceContextSynchronize(dctx));
380: PetscCall(ISRestoreIndices(isrow, &r));
381: PetscCall(PetscLogCpuToGpu(n * sizeof(PetscInt)));
382: }
384: /* upper triangular indices */
385: PetscCall(ISIdentity(isicol, &col_identity));
386: if (!col_identity && !cusparseTriFactors->cpermIndices) {
387: const PetscInt *c;
389: PetscCall(ISGetIndices(isicol, &c));
390: cusparseTriFactors->cpermIndices = new THRUSTINTARRAY(n);
391: PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->cpermIndices->data().get(), c, (n) * sizeof(*c), cudaMemcpyHostToDevice, stream));
392: PetscCall(PetscDeviceContextSynchronize(dctx));
393: PetscCall(ISRestoreIndices(isicol, &c));
394: PetscCall(PetscLogCpuToGpu(n * sizeof(PetscInt)));
395: }
396: PetscFunctionReturn(PETSC_SUCCESS);
397: }
399: static PetscErrorCode MatSeqAIJCUSPARSEBuildFactoredMatrix_Cholesky(Mat A)
400: {
401: Mat_SeqAIJ *a = static_cast<Mat_SeqAIJ *>(A->data);
402: PetscInt m = A->rmap->n;
403: Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
404: const PetscInt *Ai = a->i, *Aj = a->j, *adiag;
405: const MatScalar *Aa = a->a;
406: PetscInt *Mj, Mnz;
407: PetscScalar *Ma, *D;
408: PetscDeviceContext dctx;
409: cudaStream_t stream;
410: cusparseHandle_t handle;
412: PetscFunctionBegin;
413: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
414: {
415: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
417: PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, &adiag, NULL));
418: if (A->offloadmask == PETSC_OFFLOAD_CPU) { // A's latest factors are on CPU
419: if (!fs->csrRowPtr) { // Is this the first time we are doing setup? Use csrRowPtr since it is not null even m=0
420: // Re-arrange the (skewed) factored matrix and put the result into M, a regular csr matrix on host.
421: // See comments at MatICCFactorSymbolic_SeqAIJ() on the layout of the factored matrix (U) on host.
422: Mnz = Ai[m]; // Unz (with the unit diagonal)
423: PetscCall(PetscMalloc1(Mnz, &Ma));
424: PetscCall(PetscMalloc1(Mnz, &Mj)); // Mj[] is temp
425: PetscCall(PetscMalloc1(m, &D)); // the diagonal
426: for (PetscInt i = 0; i < m; i++) {
427: PetscInt ulen = Ai[i + 1] - Ai[i];
428: Mj[Ai[i]] = i; // diagonal entry
429: PetscCall(PetscArraycpy(Mj + Ai[i] + 1, Aj + Ai[i], ulen - 1)); // entries of U on the right of the diagonal
430: }
431: // Copy M (U) from host to device
432: PetscCallCUDA(cudaMalloc(&fs->csrRowPtr, sizeof(*fs->csrRowPtr) * (m + 1)));
433: PetscCallCUDA(cudaMalloc(&fs->csrColIdx, sizeof(*fs->csrColIdx) * Mnz));
434: PetscCallCUDA(cudaMalloc(&fs->csrVal, sizeof(*fs->csrVal) * Mnz));
435: PetscCallCUDA(cudaMalloc(&fs->diag, sizeof(*fs->diag) * m));
436: PetscCallCUDA(cudaMemcpyAsync(fs->csrRowPtr, Ai, sizeof(*Ai) * (m + 1), cudaMemcpyHostToDevice, stream));
437: PetscCallCUDA(cudaMemcpyAsync(fs->csrColIdx, Mj, sizeof(*Mj) * Mnz, cudaMemcpyHostToDevice, stream));
438: PetscCall(PetscDeviceContextSynchronize(dctx));
440: // Create descriptors for L, U. See https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
441: // cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
442: // assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
443: // all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
444: // assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
445: cusparseFillMode_t fillMode = CUSPARSE_FILL_MODE_UPPER;
446: cusparseDiagType_t diagType = CUSPARSE_DIAG_TYPE_UNIT; // U is unit diagonal
448: PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_U, m, m, Mnz, fs->csrRowPtr, fs->csrColIdx, fs->csrVal, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
449: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
450: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));
452: // Allocate work vectors in SpSv
453: PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(*fs->X) * m));
454: PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(*fs->Y) * m));
456: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
457: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));
459: // Query buffer sizes for SpSV and then allocate buffers
460: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_U));
461: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, &fs->spsvBufferSize_U));
462: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_U, fs->spsvBufferSize_U));
464: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Ut)); // Ut solve uses the same matrix (spMatDescr_U), but different descr and buffer
465: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Ut, &fs->spsvBufferSize_Ut));
466: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Ut, fs->spsvBufferSize_Ut));
468: // Record for reuse
469: fs->csrVal_h = Ma;
470: fs->diag_h = D;
471: PetscCall(PetscFree(Mj));
472: }
473: // Copy the value
474: Ma = fs->csrVal_h;
475: D = fs->diag_h;
476: Mnz = Ai[m];
477: for (PetscInt i = 0; i < m; i++) {
478: D[i] = Aa[adiag[i]]; // actually Aa[adiag[i]] is the inverse of the diagonal
479: Ma[Ai[i]] = (MatScalar)1.0; // set the unit diagonal, which is cosmetic since cusparse does not really read it given CUSPARSE_DIAG_TYPE_UNIT
480: for (PetscInt k = 0; k < Ai[i + 1] - Ai[i] - 1; k++) Ma[Ai[i] + 1 + k] = -Aa[Ai[i] + k];
481: }
482: PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Ma, sizeof(*Ma) * Mnz, cudaMemcpyHostToDevice, stream));
483: PetscCallCUDA(cudaMemcpyAsync(fs->diag, D, sizeof(*D) * m, cudaMemcpyHostToDevice, stream));
484: PetscCall(PetscDeviceContextSynchronize(dctx));
486: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
487: if (fs->updatedSpSVAnalysis) {
488: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_U, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
489: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_Ut, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
490: } else
491: #endif
492: {
493: // Do cusparseSpSV_analysis(), which is numeric and requires valid and up-to-date matrix values
494: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, fs->spsvBuffer_U));
495: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Ut, fs->spsvBuffer_Ut));
496: fs->updatedSpSVAnalysis = PETSC_TRUE;
497: }
498: }
499: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
500: }
501: PetscFunctionReturn(PETSC_SUCCESS);
502: }
504: // Solve Ut D U x = b
505: static PetscErrorCode MatSolve_SeqAIJCUSPARSE_Cholesky(Mat A, Vec b, Vec x)
506: {
507: Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
508: Mat_SeqAIJ *aij = static_cast<Mat_SeqAIJ *>(A->data);
509: const PetscScalar *barray;
510: PetscScalar *xarray;
511: thrust::device_ptr<const PetscScalar> bGPU;
512: thrust::device_ptr<PetscScalar> xGPU;
513: const cusparseSpSVAlg_t alg = CUSPARSE_SPSV_ALG_DEFAULT;
514: PetscInt m = A->rmap->n;
515: PetscDeviceContext dctx;
516: cudaStream_t stream;
517: cusparseHandle_t handle;
519: PetscFunctionBegin;
520: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
521: {
522: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
524: PetscCall(PetscLogGpuTimeBegin());
525: PetscCall(VecCUDAGetArrayWrite(x, &xarray));
526: PetscCall(VecCUDAGetArrayRead(b, &barray));
527: xGPU = thrust::device_pointer_cast(xarray);
528: bGPU = thrust::device_pointer_cast(barray);
530: // Reorder b with the row permutation if needed, and wrap the result in fs->X
531: if (fs->rpermIndices) {
532: PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->begin()), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->end()), thrust::device_pointer_cast(fs->X)));
533: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
534: } else {
535: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
536: }
538: // Solve Ut Y = X
539: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
540: PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut));
542: // Solve diag(D) Z = Y. Actually just do Y = Y*D since D is already inverted in MatCholeskyFactorNumeric_SeqAIJ().
543: // It is basically a vector element-wise multiplication, but cublas does not have it!
544: #if CCCL_VERSION >= 3001000
545: auto multiplies = cuda::std::multiplies<PetscScalar>();
546: #else
547: auto multiplies = thrust::multiplies<PetscScalar>();
548: #endif
549: PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), thrust::device_pointer_cast(fs->Y), thrust::device_pointer_cast(fs->Y + m), thrust::device_pointer_cast(fs->diag), thrust::device_pointer_cast(fs->Y), multiplies));
551: // Solve U X = Y
552: if (fs->cpermIndices) { // if need to permute, we need to use the intermediate buffer X
553: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
554: } else {
555: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
556: }
557: PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, alg, fs->spsvDescr_U));
559: // Reorder X with the column permutation if needed, and put the result back to x
560: if (fs->cpermIndices) {
561: PetscCallThrust(
562: thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X), fs->cpermIndices->begin()), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X + m), fs->cpermIndices->end()), xGPU));
563: }
565: PetscCall(VecCUDARestoreArrayRead(b, &barray));
566: PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
567: PetscCall(PetscLogGpuTimeEnd());
568: PetscCall(PetscLogGpuFlops(4.0 * aij->nz - A->rmap->n));
569: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
570: }
571: PetscFunctionReturn(PETSC_SUCCESS);
572: }
574: static PetscErrorCode MatSeqAIJCUSPARSEICCAnalysisAndCopyToGPU(Mat A)
575: {
576: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
577: Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)A->spptr;
578: IS ip = a->row;
579: PetscBool perm_identity;
580: PetscInt n = A->rmap->n;
581: PetscDeviceContext dctx;
582: cudaStream_t stream;
584: PetscFunctionBegin;
585: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
586: PetscCheck(cusparseTriFactors, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing cusparseTriFactors");
588: PetscCall(MatSeqAIJCUSPARSEBuildFactoredMatrix_Cholesky(A));
589: cusparseTriFactors->nnz = (a->nz - n) * 2 + n;
591: A->offloadmask = PETSC_OFFLOAD_BOTH;
593: /* lower triangular indices */
594: PetscCall(ISIdentity(ip, &perm_identity));
595: if (!perm_identity) {
596: IS iip;
597: const PetscInt *irip, *rip;
599: PetscCall(ISInvertPermutation(ip, PETSC_DECIDE, &iip));
600: PetscCall(ISGetIndices(iip, &irip));
601: PetscCall(ISGetIndices(ip, &rip));
602: cusparseTriFactors->rpermIndices = new THRUSTINTARRAY(n);
603: PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->rpermIndices->data().get(), rip, (n) * sizeof(*rip), cudaMemcpyHostToDevice, stream));
604: cusparseTriFactors->cpermIndices = new THRUSTINTARRAY(n);
605: PetscCallCUDA(cudaMemcpyAsync(cusparseTriFactors->cpermIndices->data().get(), irip, (n) * sizeof(*irip), cudaMemcpyHostToDevice, stream));
606: PetscCall(PetscDeviceContextSynchronize(dctx));
607: PetscCall(ISRestoreIndices(iip, &irip));
608: PetscCall(ISDestroy(&iip));
609: PetscCall(ISRestoreIndices(ip, &rip));
610: PetscCall(PetscLogCpuToGpu(2. * n * sizeof(PetscInt)));
611: }
612: PetscFunctionReturn(PETSC_SUCCESS);
613: }
615: static PetscErrorCode MatCholeskyFactorNumeric_SeqAIJCUSPARSE(Mat B, Mat A, const MatFactorInfo *info)
616: {
617: PetscFunctionBegin;
618: PetscCall(MatSeqAIJCUSPARSECopyFromGPU(A));
619: PetscCall(MatCholeskyFactorNumeric_SeqAIJ(B, A, info));
620: B->offloadmask = PETSC_OFFLOAD_CPU;
621: B->ops->solve = MatSolve_SeqAIJCUSPARSE_Cholesky;
622: B->ops->solvetranspose = MatSolve_SeqAIJCUSPARSE_Cholesky; // since symmetric
623: B->ops->matsolve = NULL;
624: B->ops->matsolvetranspose = NULL;
625: /* get the triangular factors */
626: PetscCall(MatSeqAIJCUSPARSEICCAnalysisAndCopyToGPU(B));
627: PetscFunctionReturn(PETSC_SUCCESS);
628: }
630: static PetscErrorCode MatSeqAIJCUSPARSEFormExplicitTranspose(Mat A)
631: {
632: Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
633: Mat_SeqAIJCUSPARSEMultStruct *matstruct, *matstructT;
634: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
635: cusparseIndexBase_t indexBase;
636: PetscDeviceContext dctx;
637: cudaStream_t stream;
639: PetscFunctionBegin;
640: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
641: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
642: matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
643: PetscCheck(matstruct, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing mat struct");
644: matstructT = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->matTranspose;
645: PetscCheck(!A->transupdated || matstructT, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing matTranspose struct");
646: if (A->transupdated) PetscFunctionReturn(PETSC_SUCCESS);
647: PetscCall(PetscLogEventBegin(MAT_CUSPARSEGenerateTranspose, A, 0, 0, 0));
648: PetscCall(PetscLogGpuTimeBegin());
649: if (cusparsestruct->format != MAT_CUSPARSE_CSR) PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_TRUE));
650: if (!cusparsestruct->matTranspose) { /* create cusparse matrix */
651: matstructT = new Mat_SeqAIJCUSPARSEMultStruct;
652: PetscCallCUSPARSE(cusparseCreateMatDescr(&matstructT->descr));
653: indexBase = cusparseGetMatIndexBase(matstruct->descr);
654: PetscCallCUSPARSE(cusparseSetMatIndexBase(matstructT->descr, indexBase));
655: PetscCallCUSPARSE(cusparseSetMatType(matstructT->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
657: /* set alpha and beta */
658: PetscCallCUDA(cudaMalloc((void **)&matstructT->alpha_one, sizeof(PetscScalar)));
659: PetscCallCUDA(cudaMalloc((void **)&matstructT->beta_zero, sizeof(PetscScalar)));
660: PetscCallCUDA(cudaMalloc((void **)&matstructT->beta_one, sizeof(PetscScalar)));
661: PetscCallCUDA(cudaMemcpyAsync(matstructT->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
662: PetscCallCUDA(cudaMemcpyAsync(matstructT->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
663: PetscCallCUDA(cudaMemcpyAsync(matstructT->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
665: if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
666: CsrMatrix *matrixT = new CsrMatrix;
667: matstructT->mat = matrixT;
668: matrixT->num_rows = A->cmap->n;
669: matrixT->num_cols = A->rmap->n;
670: matrixT->num_entries = a->nz;
671: matrixT->row_offsets = new THRUSTINTARRAY(matrixT->num_rows + 1);
672: matrixT->column_indices = new THRUSTINTARRAY(a->nz);
673: matrixT->values = new THRUSTARRAY(a->nz);
675: if (!cusparsestruct->rowoffsets_gpu) cusparsestruct->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
676: PetscCallCUDA(cudaMemcpyAsync(cusparsestruct->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
677: PetscCall(PetscDeviceContextSynchronize(dctx));
678: PetscCallCUSPARSE(cusparseCreateCsr(&matstructT->matDescr, matrixT->num_rows, matrixT->num_cols, matrixT->num_entries, matrixT->row_offsets->data().get(), matrixT->column_indices->data().get(), matrixT->values->data().get(), csrRowOffsetsType, csrColIndType, indexBase, cusparse_scalartype));
679: } else if (cusparsestruct->format == MAT_CUSPARSE_ELL || cusparsestruct->format == MAT_CUSPARSE_HYB) {
680: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
681: }
682: }
683: if (cusparsestruct->format == MAT_CUSPARSE_CSR) { /* transpose mat struct may be already present, update data */
684: CsrMatrix *matrix = (CsrMatrix *)matstruct->mat;
685: CsrMatrix *matrixT = (CsrMatrix *)matstructT->mat;
686: PetscCheck(matrix, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix");
687: PetscCheck(matrix->row_offsets, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix rows");
688: PetscCheck(matrix->column_indices, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix cols");
689: PetscCheck(matrix->values, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrix values");
690: PetscCheck(matrixT, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT");
691: PetscCheck(matrixT->row_offsets, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT rows");
692: PetscCheck(matrixT->column_indices, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT cols");
693: PetscCheck(matrixT->values, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CsrMatrixT values");
694: if (!cusparsestruct->rowoffsets_gpu) { /* this may be absent when we did not construct the transpose with csr2csc */
695: cusparsestruct->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
696: PetscCallCUDA(cudaMemcpyAsync(cusparsestruct->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
697: PetscCall(PetscDeviceContextSynchronize(dctx));
698: PetscCall(PetscLogCpuToGpu((A->rmap->n + 1) * sizeof(PetscInt)));
699: }
700: if (!cusparsestruct->csr2csc_i) { // not using cusparseCsr2cscEx2() because it requires 32-bit indices
701: THRUSTINTARRAY row_indices(matrix->num_entries);
703: // Transpose the matrix via COO, i.e., by putting the row indices in column_indices[] and the column indices in row_indices[]
704: cusparsestruct->csr2csc_i = new THRUSTINTARRAY(matrix->num_entries); // will store the matrix to matrixT permutation, i.e., entry matrixT[i] is matrix[csr2csc_i[i]]
705: PetscCallThrust(thrust::sequence(thrust::cuda::par.on(stream), cusparsestruct->csr2csc_i->begin(), cusparsestruct->csr2csc_i->end()));
706: PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(A->rmap->n), Csr2coo(cusparsestruct->rowoffsets_gpu->data().get(), matrixT->column_indices->data().get())));
707: PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), matrix->column_indices->begin(), matrix->column_indices->end(), row_indices.begin()));
708: // Sort the COO by row then column, and get the permutation csr2csc_i[]
709: PetscCallThrust(thrust::sort_by_key(thrust::cuda::par.on(stream), thrust::make_zip_iterator(thrust::make_tuple(row_indices.begin(), matrixT->column_indices->begin())),
710: thrust::make_zip_iterator(thrust::make_tuple(row_indices.end(), matrixT->column_indices->end())), cusparsestruct->csr2csc_i->begin()));
711: // Finalize matrixT's row_offsets by looking up row_indices[]
712: PetscCallThrust(thrust::lower_bound(thrust::cuda::par.on(stream), row_indices.begin(), row_indices.end(), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(A->cmap->n + 1), matrixT->row_offsets->begin()));
713: }
714: PetscCallThrust(thrust::gather(thrust::cuda::par.on(stream), cusparsestruct->csr2csc_i->begin(), cusparsestruct->csr2csc_i->end(), matrix->values->begin(), matrixT->values->begin()));
715: }
716: PetscCall(PetscLogGpuTimeEnd());
717: PetscCall(PetscLogEventEnd(MAT_CUSPARSEGenerateTranspose, A, 0, 0, 0));
718: /* the compressed row indices is not used for matTranspose */
719: matstructT->cprowIndices = NULL;
720: /* assign the pointer */
721: ((Mat_SeqAIJCUSPARSE *)A->spptr)->matTranspose = matstructT;
722: A->transupdated = PETSC_TRUE;
723: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
724: PetscFunctionReturn(PETSC_SUCCESS);
725: }
727: static PetscErrorCode MatSolve_SeqAIJCUSPARSE_LU(Mat A, Vec b, Vec x)
728: {
729: const PetscScalar *barray;
730: PetscScalar *xarray;
731: thrust::device_ptr<const PetscScalar> bGPU;
732: thrust::device_ptr<PetscScalar> xGPU;
733: Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
734: const Mat_SeqAIJ *aij = static_cast<Mat_SeqAIJ *>(A->data);
735: const cusparseOperation_t op = CUSPARSE_OPERATION_NON_TRANSPOSE;
736: const cusparseSpSVAlg_t alg = CUSPARSE_SPSV_ALG_DEFAULT;
737: PetscInt m = A->rmap->n;
738: PetscDeviceContext dctx;
739: cudaStream_t stream;
740: cusparseHandle_t handle;
742: PetscFunctionBegin;
743: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
744: {
745: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
747: PetscCall(PetscLogGpuTimeBegin());
748: PetscCall(VecCUDAGetArrayWrite(x, &xarray));
749: PetscCall(VecCUDAGetArrayRead(b, &barray));
750: xGPU = thrust::device_pointer_cast(xarray);
751: bGPU = thrust::device_pointer_cast(barray);
753: // Reorder b with the row permutation if needed, and wrap the result in fs->X
754: if (fs->rpermIndices) {
755: PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->begin()), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->end()), thrust::device_pointer_cast(fs->X)));
756: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
757: } else {
758: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
759: }
761: // Solve L Y = X
762: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
763: // Note that cusparseSpSV_solve() secretly uses the external buffer used in cusparseSpSV_analysis()!
764: PetscCallCUSPARSE(cusparseSpSV_solve(handle, op, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_L));
766: // Solve U X = Y
767: if (fs->cpermIndices) {
768: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
769: } else {
770: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
771: }
772: PetscCallCUSPARSE(cusparseSpSV_solve(handle, op, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, alg, fs->spsvDescr_U));
774: // Reorder X with the column permutation if needed, and put the result back to x
775: if (fs->cpermIndices) {
776: PetscCallThrust(
777: thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X), fs->cpermIndices->begin()), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X + m), fs->cpermIndices->end()), xGPU));
778: }
779: PetscCall(VecCUDARestoreArrayRead(b, &barray));
780: PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
781: PetscCall(PetscLogGpuTimeEnd());
782: PetscCall(PetscLogGpuFlops(2.0 * aij->nz - m));
783: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
784: }
785: PetscFunctionReturn(PETSC_SUCCESS);
786: }
788: static PetscErrorCode MatSolveTranspose_SeqAIJCUSPARSE_LU(Mat A, Vec b, Vec x)
789: {
790: Mat_SeqAIJCUSPARSETriFactors *fs = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(A->spptr);
791: Mat_SeqAIJ *aij = static_cast<Mat_SeqAIJ *>(A->data);
792: const PetscScalar *barray;
793: PetscScalar *xarray;
794: thrust::device_ptr<const PetscScalar> bGPU;
795: thrust::device_ptr<PetscScalar> xGPU;
796: const cusparseOperation_t opA = CUSPARSE_OPERATION_TRANSPOSE;
797: const cusparseSpSVAlg_t alg = CUSPARSE_SPSV_ALG_DEFAULT;
798: PetscInt m = A->rmap->n;
799: PetscDeviceContext dctx;
800: cudaStream_t stream;
801: cusparseHandle_t handle;
803: PetscFunctionBegin;
804: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
805: {
806: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
808: PetscCall(PetscLogGpuTimeBegin());
809: if (!fs->createdTransposeSpSVDescr) { // Call MatSolveTranspose() for the first time
810: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Lt));
811: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, /* The matrix is still L. We only do transpose solve with it */
812: fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Lt, &fs->spsvBufferSize_Lt));
814: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Ut));
815: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut, &fs->spsvBufferSize_Ut));
816: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Lt, fs->spsvBufferSize_Lt));
817: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Ut, fs->spsvBufferSize_Ut));
818: fs->createdTransposeSpSVDescr = PETSC_TRUE;
819: }
821: if (!fs->updatedTransposeSpSVAnalysis) {
822: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Lt, fs->spsvBuffer_Lt));
824: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut, fs->spsvBuffer_Ut));
825: fs->updatedTransposeSpSVAnalysis = PETSC_TRUE;
826: }
828: PetscCall(VecCUDAGetArrayWrite(x, &xarray));
829: PetscCall(VecCUDAGetArrayRead(b, &barray));
830: xGPU = thrust::device_pointer_cast(xarray);
831: bGPU = thrust::device_pointer_cast(barray);
833: // Reorder b with the row permutation if needed, and wrap the result in fs->X
834: if (fs->rpermIndices) {
835: PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->begin()), thrust::make_permutation_iterator(bGPU, fs->rpermIndices->end()), thrust::device_pointer_cast(fs->X)));
836: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
837: } else {
838: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
839: }
841: // Solve Ut Y = X
842: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
843: PetscCallCUSPARSE(cusparseSpSV_solve(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, alg, fs->spsvDescr_Ut));
845: // Solve Lt X = Y
846: if (fs->cpermIndices) { // if need to permute, we need to use the intermediate buffer X
847: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, fs->X));
848: } else {
849: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
850: }
851: PetscCallCUSPARSE(cusparseSpSV_solve(handle, opA, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, alg, fs->spsvDescr_Lt));
853: // Reorder X with the column permutation if needed, and put the result back to x
854: if (fs->cpermIndices) {
855: PetscCallThrust(
856: thrust::copy(thrust::cuda::par.on(stream), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X), fs->cpermIndices->begin()), thrust::make_permutation_iterator(thrust::device_pointer_cast(fs->X + m), fs->cpermIndices->end()), xGPU));
857: }
859: PetscCall(VecCUDARestoreArrayRead(b, &barray));
860: PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
861: PetscCall(PetscLogGpuTimeEnd());
862: PetscCall(PetscLogGpuFlops(2.0 * aij->nz - A->rmap->n));
863: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
864: }
865: PetscFunctionReturn(PETSC_SUCCESS);
866: }
868: static PetscErrorCode MatILUFactorNumeric_SeqAIJCUSPARSE_ILU0(Mat fact, Mat A, const MatFactorInfo *)
869: {
870: Mat_SeqAIJCUSPARSETriFactors *fs = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
871: Mat_SeqAIJ *aij = (Mat_SeqAIJ *)fact->data;
872: Mat_SeqAIJCUSPARSE *Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
873: CsrMatrix *Acsr;
874: PetscInt m, nz;
875: PetscBool flg;
876: PetscDeviceContext dctx;
877: cudaStream_t stream;
878: cusparseHandle_t handle;
880: PetscFunctionBegin;
881: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
882: {
883: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
885: if (PetscDefined(USE_DEBUG)) {
886: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
887: PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
888: }
890: /* Copy A's value to fact */
891: m = fact->rmap->n;
892: nz = aij->nz;
893: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
894: Acsr = (CsrMatrix *)Acusp->mat->mat;
895: PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Acsr->values->data().get(), sizeof(PetscScalar) * nz, cudaMemcpyDeviceToDevice, stream));
897: PetscCall(PetscLogGpuTimeBegin());
898: /* Factorize fact inplace */
899: if (m)
900: PetscCallCUSPARSE(cusparseXcsrilu02(handle, m, nz, /* cusparseXcsrilu02 errors out with empty matrices (m=0) */
901: fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ilu0Info_M, fs->policy_M, fs->factBuffer_M));
902: if (PetscDefined(USE_DEBUG)) {
903: int numerical_zero;
904: cusparseStatus_t status;
905: status = cusparseXcsrilu02_zeroPivot(handle, fs->ilu0Info_M, &numerical_zero);
906: PetscAssert(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Numerical zero pivot detected in csrilu02: A(%d,%d) is zero", numerical_zero, numerical_zero);
907: }
909: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
910: if (fs->updatedSpSVAnalysis) {
911: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_L, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
912: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_U, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
913: } else
914: #endif
915: {
916: /* cusparseSpSV_analysis() is numeric, i.e., it requires valid matrix values, therefore, we do it after cusparseXcsrilu02()
917: See discussion at https://github.com/NVIDIA/CUDALibrarySamples/issues/78
918: */
919: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, fs->spsvBuffer_L));
921: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, fs->spsvBuffer_U));
923: fs->updatedSpSVAnalysis = PETSC_TRUE;
924: /* L, U values have changed, reset the flag to indicate we need to redo cusparseSpSV_analysis() for transpose solve */
925: fs->updatedTransposeSpSVAnalysis = PETSC_FALSE;
926: }
928: fact->offloadmask = PETSC_OFFLOAD_GPU;
929: fact->ops->solve = MatSolve_SeqAIJCUSPARSE_LU; // spMatDescr_L/U uses 32-bit indices, but cusparseSpSV_solve() supports both 32 and 64. The info is encoded in cusparseSpMatDescr_t.
930: fact->ops->solvetranspose = MatSolveTranspose_SeqAIJCUSPARSE_LU;
931: fact->ops->matsolve = NULL;
932: fact->ops->matsolvetranspose = NULL;
933: PetscCall(PetscLogGpuTimeEnd());
934: PetscCall(PetscLogGpuFlops(fs->numericFactFlops));
935: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
936: }
937: PetscFunctionReturn(PETSC_SUCCESS);
938: }
940: static PetscErrorCode MatILUFactorSymbolic_SeqAIJCUSPARSE_ILU0(Mat fact, Mat A, IS, IS, const MatFactorInfo *info)
941: {
942: Mat_SeqAIJCUSPARSETriFactors *fs = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
943: Mat_SeqAIJ *aij = (Mat_SeqAIJ *)fact->data;
944: PetscInt m, nz;
945: PetscDeviceContext dctx;
946: cudaStream_t stream;
947: cusparseHandle_t handle;
949: PetscFunctionBegin;
950: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
951: {
952: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
954: if (PetscDefined(USE_DEBUG)) {
955: PetscBool flg, diagDense;
957: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
958: PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
959: PetscCheck(A->rmap->n == A->cmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Must be square matrix, rows %" PetscInt_FMT " columns %" PetscInt_FMT, A->rmap->n, A->cmap->n);
960: PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, NULL, &diagDense));
961: PetscCheck(diagDense, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Matrix is missing a diagonal entry");
962: }
964: /* Free the old stale stuff */
965: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&fs));
967: /* Copy over A's meta data to fact. Note that we also allocated fact's i,j,a on host,
968: but they will not be used. Allocate them just for easy debugging.
969: */
970: PetscCall(MatDuplicateNoCreate_SeqAIJ(fact, A, MAT_DO_NOT_COPY_VALUES, PETSC_TRUE /*malloc*/));
972: fact->offloadmask = PETSC_OFFLOAD_BOTH;
973: fact->factortype = MAT_FACTOR_ILU;
974: fact->info.factor_mallocs = 0;
975: fact->info.fill_ratio_given = info->fill;
976: fact->info.fill_ratio_needed = 1.0;
978: aij->row = NULL;
979: aij->col = NULL;
981: /* ====================================================================== */
982: /* Copy A's i, j to fact and also allocate the value array of fact. */
983: /* We'll do in-place factorization on fact */
984: /* ====================================================================== */
985: const PetscInt *Ai, *Aj;
987: m = fact->rmap->n;
988: nz = aij->nz;
990: PetscCallCUDA(cudaMalloc((void **)&fs->csrRowPtr32, sizeof(*fs->csrRowPtr32) * (m + 1)));
991: PetscCallCUDA(cudaMalloc((void **)&fs->csrColIdx32, sizeof(*fs->csrColIdx32) * nz));
992: PetscCallCUDA(cudaMalloc((void **)&fs->csrVal, sizeof(*fs->csrVal) * nz));
993: PetscCall(MatSeqAIJCUSPARSEGetIJ(A, PETSC_FALSE, &Ai, &Aj)); // Ai is uncompressed
995: PetscCheck(nz <= INT_MAX && m <= INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "nnz %" PetscInt_FMT " and rows %" PetscInt_FMT " overflow C int", nz, m);
996: PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Ai, Ai + m + 1, fs->csrRowPtr32, PetscIntToCInt()));
997: PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Aj, Aj + nz, fs->csrColIdx32, PetscIntToCInt()));
999: /* ====================================================================== */
1000: /* Create descriptors for M, L, U */
1001: /* ====================================================================== */
1002: cusparseFillMode_t fillMode;
1003: cusparseDiagType_t diagType;
1005: PetscCallCUSPARSE(cusparseCreateMatDescr(&fs->matDescr_M));
1006: PetscCallCUSPARSE(cusparseSetMatIndexBase(fs->matDescr_M, CUSPARSE_INDEX_BASE_ZERO));
1007: PetscCallCUSPARSE(cusparseSetMatType(fs->matDescr_M, CUSPARSE_MATRIX_TYPE_GENERAL));
1009: /* https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
1010: cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
1011: assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
1012: all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
1013: assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
1014: */
1015: fillMode = CUSPARSE_FILL_MODE_LOWER;
1016: diagType = CUSPARSE_DIAG_TYPE_UNIT;
1017: PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_L, m, m, nz, fs->csrRowPtr32, fs->csrColIdx32, fs->csrVal, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1018: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
1019: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));
1021: fillMode = CUSPARSE_FILL_MODE_UPPER;
1022: diagType = CUSPARSE_DIAG_TYPE_NON_UNIT;
1023: PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_U, m, m, nz, fs->csrRowPtr32, fs->csrColIdx32, fs->csrVal, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1024: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
1025: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_U, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));
1027: /* ========================================================================= */
1028: /* Query buffer sizes for csrilu0, SpSV and allocate buffers */
1029: /* ========================================================================= */
1030: PetscCallCUSPARSE(cusparseCreateCsrilu02Info(&fs->ilu0Info_M));
1031: if (m)
1032: PetscCallCUSPARSE(cusparseXcsrilu02_bufferSize(handle, m, nz, /* cusparseXcsrilu02 errors out with empty matrices (m=0) */
1033: fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ilu0Info_M, &fs->factBufferSize_M));
1035: PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(PetscScalar) * m));
1036: PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(PetscScalar) * m));
1038: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
1039: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));
1041: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_L));
1042: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, &fs->spsvBufferSize_L));
1044: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_U));
1045: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_U, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_U, &fs->spsvBufferSize_U));
1047: /* From my experiment with the example at https://github.com/NVIDIA/CUDALibrarySamples/tree/master/cuSPARSE/bicgstab,
1048: and discussion at https://github.com/NVIDIA/CUDALibrarySamples/issues/77,
1049: spsvBuffer_L/U can not be shared (i.e., the same) for our case, but factBuffer_M can share with either of spsvBuffer_L/U.
1050: To save memory, we make factBuffer_M share with the bigger of spsvBuffer_L/U.
1051: */
1052: if (fs->spsvBufferSize_L > fs->spsvBufferSize_U) {
1053: PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_L, (size_t)fs->factBufferSize_M)));
1054: fs->spsvBuffer_L = fs->factBuffer_M;
1055: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_U, fs->spsvBufferSize_U));
1056: } else {
1057: PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_U, (size_t)fs->factBufferSize_M)));
1058: fs->spsvBuffer_U = fs->factBuffer_M;
1059: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_L, fs->spsvBufferSize_L));
1060: }
1062: /* ========================================================================== */
1063: /* Perform analysis of ilu0 on M, SpSv on L and U */
1064: /* The lower(upper) triangular part of M has the same sparsity pattern as L(U)*/
1065: /* ========================================================================== */
1066: int structural_zero;
1067: cusparseStatus_t status;
1069: fs->policy_M = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
1070: if (m)
1071: PetscCallCUSPARSE(cusparseXcsrilu02_analysis(handle, m, nz, /* cusparseXcsrilu02 errors out with empty matrices (m=0) */
1072: fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ilu0Info_M, fs->policy_M, fs->factBuffer_M));
1073: if (PetscDefined(USE_DEBUG)) {
1074: /* cusparseXcsrilu02_zeroPivot() is a blocking call. It calls cudaDeviceSynchronize() to make sure all previous kernels are done. */
1075: status = cusparseXcsrilu02_zeroPivot(handle, fs->ilu0Info_M, &structural_zero);
1076: PetscCheck(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Structural zero pivot detected in csrilu02: A(%d,%d) is missing", structural_zero, structural_zero);
1077: }
1079: /* Estimate FLOPs of the numeric factorization */
1080: {
1081: Mat_SeqAIJ *Aseq = (Mat_SeqAIJ *)A->data;
1082: PetscInt *Ai, nzRow, nzLeft;
1083: const PetscInt *adiag;
1084: PetscLogDouble flops = 0.0;
1086: PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, &adiag, NULL));
1087: Ai = Aseq->i;
1088: for (PetscInt i = 0; i < m; i++) {
1089: if (Ai[i] < adiag[i] && adiag[i] < Ai[i + 1]) { /* There are nonzeros left to the diagonal of row i */
1090: nzRow = Ai[i + 1] - Ai[i];
1091: nzLeft = adiag[i] - Ai[i];
1092: /* We want to eliminate nonzeros left to the diagonal one by one. Assume each time, nonzeros right
1093: and include the eliminated one will be updated, which incurs a multiplication and an addition.
1094: */
1095: nzLeft = (nzRow - 1) / 2;
1096: flops += nzLeft * (2.0 * nzRow - nzLeft + 1);
1097: }
1098: }
1099: fs->numericFactFlops = flops;
1100: }
1101: fact->ops->lufactornumeric = MatILUFactorNumeric_SeqAIJCUSPARSE_ILU0;
1102: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1103: }
1104: PetscFunctionReturn(PETSC_SUCCESS);
1105: }
1107: static PetscErrorCode MatSolve_SeqAIJCUSPARSE_ICC0(Mat fact, Vec b, Vec x)
1108: {
1109: Mat_SeqAIJCUSPARSETriFactors *fs = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
1110: Mat_SeqAIJ *aij = (Mat_SeqAIJ *)fact->data;
1111: const PetscScalar *barray;
1112: PetscScalar *xarray;
1113: PetscDeviceContext dctx;
1114: cusparseHandle_t handle;
1116: PetscFunctionBegin;
1117: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle));
1118: {
1119: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
1121: PetscCall(VecCUDAGetArrayWrite(x, &xarray));
1122: PetscCall(VecCUDAGetArrayRead(b, &barray));
1123: PetscCall(PetscLogGpuTimeBegin());
1125: /* Solve L*y = b */
1126: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, (void *)barray));
1127: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_Y, fs->Y));
1128: PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, /* L Y = X */
1129: fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L));
1131: /* Solve Lt*x = y */
1132: PetscCallCUSPARSE(cusparseDnVecSetValues(fs->dnVecDescr_X, xarray));
1133: PetscCallCUSPARSE(cusparseSpSV_solve(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, /* Lt X = Y */
1134: fs->dnVecDescr_Y, fs->dnVecDescr_X, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Lt));
1136: PetscCall(VecCUDARestoreArrayRead(b, &barray));
1137: PetscCall(VecCUDARestoreArrayWrite(x, &xarray));
1139: PetscCall(PetscLogGpuTimeEnd());
1140: PetscCall(PetscLogGpuFlops(2.0 * aij->nz - fact->rmap->n));
1141: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1142: }
1143: PetscFunctionReturn(PETSC_SUCCESS);
1144: }
1146: static PetscErrorCode MatICCFactorNumeric_SeqAIJCUSPARSE_ICC0(Mat fact, Mat A, const MatFactorInfo *)
1147: {
1148: Mat_SeqAIJCUSPARSETriFactors *fs = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
1149: Mat_SeqAIJ *aij = (Mat_SeqAIJ *)fact->data;
1150: Mat_SeqAIJCUSPARSE *Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1151: CsrMatrix *Acsr;
1152: PetscInt m, nz;
1153: PetscBool flg;
1154: PetscDeviceContext dctx;
1155: cudaStream_t stream;
1156: cusparseHandle_t handle;
1158: PetscFunctionBegin;
1159: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
1160: {
1161: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
1163: if (PetscDefined(USE_DEBUG)) {
1164: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1165: PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
1166: }
1168: /* Copy A's value to fact */
1169: m = fact->rmap->n;
1170: nz = aij->nz;
1171: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
1172: Acsr = (CsrMatrix *)Acusp->mat->mat;
1173: PetscCallCUDA(cudaMemcpyAsync(fs->csrVal, Acsr->values->data().get(), sizeof(PetscScalar) * nz, cudaMemcpyDeviceToDevice, stream));
1175: /* Factorize fact inplace */
1176: /* https://docs.nvidia.com/cuda/cusparse/index.html#csric02_solve
1177: csric02() only takes the lower triangular part of matrix A to perform factorization.
1178: The matrix type must be CUSPARSE_MATRIX_TYPE_GENERAL, the fill mode and diagonal type are ignored,
1179: and the strictly upper triangular part is ignored and never touched. It does not matter if A is Hermitian or not.
1180: In other words, from the point of view of csric02() A is Hermitian and only the lower triangular part is provided.
1181: */
1182: if (m) PetscCallCUSPARSE(cusparseXcsric02(handle, m, nz, fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ic0Info_M, fs->policy_M, fs->factBuffer_M));
1183: if (PetscDefined(USE_DEBUG)) {
1184: int numerical_zero;
1185: cusparseStatus_t status;
1186: status = cusparseXcsric02_zeroPivot(handle, fs->ic0Info_M, &numerical_zero);
1187: PetscAssert(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Numerical zero pivot detected in csric02: A(%d,%d) is zero", numerical_zero, numerical_zero);
1188: }
1190: #if PETSC_PKG_CUDA_VERSION_GE(12, 1, 1)
1191: if (fs->updatedSpSVAnalysis) {
1192: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_L, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
1193: if (fs->csrVal) PetscCallCUSPARSE(cusparseSpSV_updateMatrix(handle, fs->spsvDescr_Lt, fs->csrVal, CUSPARSE_SPSV_UPDATE_GENERAL));
1194: } else
1195: #endif
1196: {
1197: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, fs->spsvBuffer_L));
1199: /* Note that cusparse reports this error if we use double and CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE
1200: ** On entry to cusparseSpSV_analysis(): conjugate transpose (opA) is not supported for matA data type, current -> CUDA_R_64F
1201: */
1202: PetscCallCUSPARSE(cusparseSpSV_analysis(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Lt, fs->spsvBuffer_Lt));
1203: fs->updatedSpSVAnalysis = PETSC_TRUE;
1204: }
1206: fact->offloadmask = PETSC_OFFLOAD_GPU;
1207: fact->ops->solve = MatSolve_SeqAIJCUSPARSE_ICC0;
1208: fact->ops->solvetranspose = MatSolve_SeqAIJCUSPARSE_ICC0;
1209: fact->ops->matsolve = NULL;
1210: fact->ops->matsolvetranspose = NULL;
1211: PetscCall(PetscLogGpuFlops(fs->numericFactFlops));
1212: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1213: }
1214: PetscFunctionReturn(PETSC_SUCCESS);
1215: }
1217: static PetscErrorCode MatICCFactorSymbolic_SeqAIJCUSPARSE_ICC0(Mat fact, Mat A, IS, const MatFactorInfo *info)
1218: {
1219: Mat_SeqAIJCUSPARSETriFactors *fs = (Mat_SeqAIJCUSPARSETriFactors *)fact->spptr;
1220: Mat_SeqAIJ *aij = (Mat_SeqAIJ *)fact->data;
1221: PetscInt m, nz;
1222: PetscDeviceContext dctx;
1223: cudaStream_t stream;
1224: cusparseHandle_t handle;
1226: PetscFunctionBegin;
1227: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
1228: {
1229: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
1231: if (PetscDefined(USE_DEBUG)) {
1232: PetscBool flg, diagDense;
1234: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1235: PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Expected MATSEQAIJCUSPARSE, but input is %s", ((PetscObject)A)->type_name);
1236: PetscCheck(A->rmap->n == A->cmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, "Must be square matrix, rows %" PetscInt_FMT " columns %" PetscInt_FMT, A->rmap->n, A->cmap->n);
1237: PetscCall(MatGetDiagonalMarkers_SeqAIJ(A, NULL, &diagDense));
1238: PetscCheck(diagDense, PETSC_COMM_SELF, PETSC_ERR_ARG_WRONGSTATE, "Matrix is missing diagonal entries");
1239: }
1241: /* Free the old stale stuff */
1242: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&fs));
1244: /* Copy over A's meta data to fact. Note that we also allocated fact's i,j,a on host,
1245: but they will not be used. Allocate them just for easy debugging.
1246: */
1247: PetscCall(MatDuplicateNoCreate_SeqAIJ(fact, A, MAT_DO_NOT_COPY_VALUES, PETSC_TRUE /*malloc*/));
1249: fact->offloadmask = PETSC_OFFLOAD_BOTH;
1250: fact->factortype = MAT_FACTOR_ICC;
1251: fact->info.factor_mallocs = 0;
1252: fact->info.fill_ratio_given = info->fill;
1253: fact->info.fill_ratio_needed = 1.0;
1255: aij->row = NULL;
1256: aij->col = NULL;
1258: /* ====================================================================== */
1259: /* Copy A's i, j to fact and also allocate the value array of fact. */
1260: /* We'll do in-place factorization on fact */
1261: /* ====================================================================== */
1262: const PetscInt *Ai, *Aj;
1264: m = fact->rmap->n;
1265: nz = aij->nz;
1267: PetscCallCUDA(cudaMalloc((void **)&fs->csrRowPtr32, sizeof(*fs->csrRowPtr32) * (m + 1)));
1268: PetscCallCUDA(cudaMalloc((void **)&fs->csrColIdx32, sizeof(*fs->csrColIdx32) * nz));
1269: PetscCallCUDA(cudaMalloc((void **)&fs->csrVal, sizeof(PetscScalar) * nz));
1270: PetscCall(MatSeqAIJCUSPARSEGetIJ(A, PETSC_FALSE, &Ai, &Aj)); // Ai is uncompressed
1272: PetscCheck(nz <= INT_MAX && m <= INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "nnz %" PetscInt_FMT " and rows %" PetscInt_FMT " overflow C int", nz, m);
1273: PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Ai, Ai + m + 1, fs->csrRowPtr32, PetscIntToCInt()));
1274: PetscCallThrust(thrust::transform(thrust::cuda::par.on(stream), Aj, Aj + nz, fs->csrColIdx32, PetscIntToCInt()));
1276: /* ====================================================================== */
1277: /* Create mat descriptors for M, L */
1278: /* ====================================================================== */
1279: cusparseFillMode_t fillMode;
1280: cusparseDiagType_t diagType;
1282: PetscCallCUSPARSE(cusparseCreateMatDescr(&fs->matDescr_M));
1283: PetscCallCUSPARSE(cusparseSetMatIndexBase(fs->matDescr_M, CUSPARSE_INDEX_BASE_ZERO));
1284: PetscCallCUSPARSE(cusparseSetMatType(fs->matDescr_M, CUSPARSE_MATRIX_TYPE_GENERAL));
1286: /* https://docs.nvidia.com/cuda/cusparse/index.html#cusparseDiagType_t
1287: cusparseDiagType_t: This type indicates if the matrix diagonal entries are unity. The diagonal elements are always
1288: assumed to be present, but if CUSPARSE_DIAG_TYPE_UNIT is passed to an API routine, then the routine assumes that
1289: all diagonal entries are unity and will not read or modify those entries. Note that in this case the routine
1290: assumes the diagonal entries are equal to one, regardless of what those entries are actually set to in memory.
1291: */
1292: fillMode = CUSPARSE_FILL_MODE_LOWER;
1293: diagType = CUSPARSE_DIAG_TYPE_NON_UNIT;
1294: PetscCallCUSPARSE(cusparseCreateCsr(&fs->spMatDescr_L, m, m, nz, fs->csrRowPtr32, fs->csrColIdx32, fs->csrVal, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_32I, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1295: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_FILL_MODE, &fillMode, sizeof(fillMode)));
1296: PetscCallCUSPARSE(cusparseSpMatSetAttribute(fs->spMatDescr_L, CUSPARSE_SPMAT_DIAG_TYPE, &diagType, sizeof(diagType)));
1298: /* ========================================================================= */
1299: /* Query buffer sizes for csric0, SpSV of L and Lt, and allocate buffers */
1300: /* ========================================================================= */
1301: PetscCallCUSPARSE(cusparseCreateCsric02Info(&fs->ic0Info_M));
1302: if (m) PetscCallCUSPARSE(cusparseXcsric02_bufferSize(handle, m, nz, fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ic0Info_M, &fs->factBufferSize_M));
1304: PetscCallCUDA(cudaMalloc((void **)&fs->X, sizeof(PetscScalar) * m));
1305: PetscCallCUDA(cudaMalloc((void **)&fs->Y, sizeof(PetscScalar) * m));
1307: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_X, m, fs->X, cusparse_scalartype));
1308: PetscCallCUSPARSE(cusparseCreateDnVec(&fs->dnVecDescr_Y, m, fs->Y, cusparse_scalartype));
1310: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_L));
1311: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_NON_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_L, &fs->spsvBufferSize_L));
1313: PetscCallCUSPARSE(cusparseSpSV_createDescr(&fs->spsvDescr_Lt));
1314: PetscCallCUSPARSE(cusparseSpSV_bufferSize(handle, CUSPARSE_OPERATION_TRANSPOSE, &PETSC_CUSPARSE_ONE, fs->spMatDescr_L, fs->dnVecDescr_X, fs->dnVecDescr_Y, cusparse_scalartype, CUSPARSE_SPSV_ALG_DEFAULT, fs->spsvDescr_Lt, &fs->spsvBufferSize_Lt));
1316: /* To save device memory, we make the factorization buffer share with one of the solver buffer.
1317: See also comments in MatILUFactorSymbolic_SeqAIJCUSPARSE_ILU0().
1318: */
1319: if (fs->spsvBufferSize_L > fs->spsvBufferSize_Lt) {
1320: PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_L, (size_t)fs->factBufferSize_M)));
1321: fs->spsvBuffer_L = fs->factBuffer_M;
1322: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_Lt, fs->spsvBufferSize_Lt));
1323: } else {
1324: PetscCallCUDA(cudaMalloc((void **)&fs->factBuffer_M, PetscMax(fs->spsvBufferSize_Lt, (size_t)fs->factBufferSize_M)));
1325: fs->spsvBuffer_Lt = fs->factBuffer_M;
1326: PetscCallCUDA(cudaMalloc((void **)&fs->spsvBuffer_L, fs->spsvBufferSize_L));
1327: }
1329: /* ========================================================================== */
1330: /* Perform analysis of ic0 on M */
1331: /* The lower triangular part of M has the same sparsity pattern as L */
1332: /* ========================================================================== */
1333: int structural_zero;
1334: cusparseStatus_t status;
1336: fs->policy_M = CUSPARSE_SOLVE_POLICY_USE_LEVEL;
1337: if (m) PetscCallCUSPARSE(cusparseXcsric02_analysis(handle, m, nz, fs->matDescr_M, fs->csrVal, fs->csrRowPtr32, fs->csrColIdx32, fs->ic0Info_M, fs->policy_M, fs->factBuffer_M));
1338: if (PetscDefined(USE_DEBUG)) {
1339: /* cusparseXcsric02_zeroPivot() is a blocking call. It calls cudaDeviceSynchronize() to make sure all previous kernels are done. */
1340: status = cusparseXcsric02_zeroPivot(handle, fs->ic0Info_M, &structural_zero);
1341: PetscCheck(CUSPARSE_STATUS_ZERO_PIVOT != status, PETSC_COMM_SELF, PETSC_ERR_USER_INPUT, "Structural zero pivot detected in csric02: A(%d,%d) is missing", structural_zero, structural_zero);
1342: }
1344: /* Estimate FLOPs of the numeric factorization */
1345: {
1346: Mat_SeqAIJ *Aseq = (Mat_SeqAIJ *)A->data;
1347: PetscInt *Ai, nzRow, nzLeft;
1348: PetscLogDouble flops = 0.0;
1350: Ai = Aseq->i;
1351: for (PetscInt i = 0; i < m; i++) {
1352: nzRow = Ai[i + 1] - Ai[i];
1353: if (nzRow > 1) {
1354: /* We want to eliminate nonzeros left to the diagonal one by one. Assume each time, nonzeros right
1355: and include the eliminated one will be updated, which incurs a multiplication and an addition.
1356: */
1357: nzLeft = (nzRow - 1) / 2;
1358: flops += nzLeft * (2.0 * nzRow - nzLeft + 1);
1359: }
1360: }
1361: fs->numericFactFlops = flops;
1362: }
1363: fact->ops->choleskyfactornumeric = MatICCFactorNumeric_SeqAIJCUSPARSE_ICC0;
1364: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1365: }
1366: PetscFunctionReturn(PETSC_SUCCESS);
1367: }
1369: static PetscErrorCode MatLUFactorNumeric_SeqAIJCUSPARSE(Mat B, Mat A, const MatFactorInfo *info)
1370: {
1371: // use_cpu_solve is a field in Mat_SeqAIJCUSPARSE. B, a factored matrix, uses Mat_SeqAIJCUSPARSETriFactors.
1372: Mat_SeqAIJCUSPARSE *cusparsestruct = static_cast<Mat_SeqAIJCUSPARSE *>(A->spptr);
1374: PetscFunctionBegin;
1375: PetscCall(MatSeqAIJCUSPARSECopyFromGPU(A));
1376: PetscCall(MatLUFactorNumeric_SeqAIJ(B, A, info));
1377: B->offloadmask = PETSC_OFFLOAD_CPU;
1379: if (!cusparsestruct->use_cpu_solve) {
1380: B->ops->solve = MatSolve_SeqAIJCUSPARSE_LU;
1381: B->ops->solvetranspose = MatSolveTranspose_SeqAIJCUSPARSE_LU;
1382: }
1383: B->ops->matsolve = NULL;
1384: B->ops->matsolvetranspose = NULL;
1386: /* get the triangular factors */
1387: if (!cusparsestruct->use_cpu_solve) PetscCall(MatSeqAIJCUSPARSEILUAnalysisAndCopyToGPU(B));
1388: PetscFunctionReturn(PETSC_SUCCESS);
1389: }
1391: static PetscErrorCode MatLUFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS isrow, IS iscol, const MatFactorInfo *info)
1392: {
1393: Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = static_cast<Mat_SeqAIJCUSPARSETriFactors *>(B->spptr);
1395: PetscFunctionBegin;
1396: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1397: PetscCall(MatLUFactorSymbolic_SeqAIJ(B, A, isrow, iscol, info));
1398: B->ops->lufactornumeric = MatLUFactorNumeric_SeqAIJCUSPARSE;
1399: PetscFunctionReturn(PETSC_SUCCESS);
1400: }
1402: static PetscErrorCode MatILUFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS isrow, IS iscol, const MatFactorInfo *info)
1403: {
1404: Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)B->spptr;
1406: PetscFunctionBegin;
1407: PetscBool row_identity = PETSC_FALSE, col_identity = PETSC_FALSE;
1408: if (!info->factoronhost) {
1409: PetscCall(ISIdentity(isrow, &row_identity));
1410: PetscCall(ISIdentity(iscol, &col_identity));
1411: }
1412: if (!info->levels && row_identity && col_identity) {
1413: PetscCall(MatILUFactorSymbolic_SeqAIJCUSPARSE_ILU0(B, A, isrow, iscol, info));
1414: } else {
1415: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1416: PetscCall(MatILUFactorSymbolic_SeqAIJ(B, A, isrow, iscol, info));
1417: B->ops->lufactornumeric = MatLUFactorNumeric_SeqAIJCUSPARSE;
1418: }
1419: PetscFunctionReturn(PETSC_SUCCESS);
1420: }
1422: static PetscErrorCode MatICCFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS perm, const MatFactorInfo *info)
1423: {
1424: Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)B->spptr;
1426: PetscFunctionBegin;
1427: PetscBool perm_identity = PETSC_FALSE;
1428: if (!info->factoronhost) PetscCall(ISIdentity(perm, &perm_identity));
1429: if (!info->levels && perm_identity) {
1430: PetscCall(MatICCFactorSymbolic_SeqAIJCUSPARSE_ICC0(B, A, perm, info));
1431: } else {
1432: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1433: PetscCall(MatICCFactorSymbolic_SeqAIJ(B, A, perm, info));
1434: B->ops->choleskyfactornumeric = MatCholeskyFactorNumeric_SeqAIJCUSPARSE;
1435: }
1436: PetscFunctionReturn(PETSC_SUCCESS);
1437: }
1439: static PetscErrorCode MatCholeskyFactorSymbolic_SeqAIJCUSPARSE(Mat B, Mat A, IS perm, const MatFactorInfo *info)
1440: {
1441: Mat_SeqAIJCUSPARSETriFactors *cusparseTriFactors = (Mat_SeqAIJCUSPARSETriFactors *)B->spptr;
1443: PetscFunctionBegin;
1444: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(&cusparseTriFactors));
1445: PetscCall(MatCholeskyFactorSymbolic_SeqAIJ(B, A, perm, info));
1446: B->ops->choleskyfactornumeric = MatCholeskyFactorNumeric_SeqAIJCUSPARSE;
1447: PetscFunctionReturn(PETSC_SUCCESS);
1448: }
1450: static PetscErrorCode MatFactorGetSolverType_seqaij_cusparse(Mat, MatSolverType *type)
1451: {
1452: PetscFunctionBegin;
1453: *type = MATSOLVERCUSPARSE;
1454: PetscFunctionReturn(PETSC_SUCCESS);
1455: }
1457: /*MC
1458: MATSOLVERCUSPARSE = "cusparse" - A matrix type providing triangular solvers for seq matrices
1459: on a single GPU of type, `MATSEQAIJCUSPARSE`. Currently supported
1460: algorithms are ILU(k) and ICC(k). Typically, deeper factorizations (larger k) results in poorer
1461: performance in the triangular solves. Full LU, and Cholesky decompositions can be solved through the
1462: CuSPARSE triangular solve algorithm. However, the performance can be quite poor and thus these
1463: algorithms are not recommended. This class does NOT support direct solver operations.
1465: Level: beginner
1467: .seealso: [](ch_matrices), `Mat`, `MATSEQAIJCUSPARSE`, `PCFactorSetMatSolverType()`, `MatSolverType`, `MatCreateSeqAIJCUSPARSE()`,
1468: `MATAIJCUSPARSE`, `MatCreateAIJCUSPARSE()`, `MatCUSPARSESetFormat()`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
1469: M*/
1471: PETSC_EXTERN PetscErrorCode MatGetFactor_seqaijcusparse_cusparse(Mat A, MatFactorType ftype, Mat *B)
1472: {
1473: PetscInt n = A->rmap->n;
1475: PetscFunctionBegin;
1476: PetscCall(MatCreate(PetscObjectComm((PetscObject)A), B));
1477: PetscCall(MatSetSizes(*B, n, n, n, n));
1478: (*B)->factortype = ftype; // factortype makes MatSetType() allocate spptr of type Mat_SeqAIJCUSPARSETriFactors
1479: PetscCall(MatSetType(*B, MATSEQAIJCUSPARSE));
1481: if (A->boundtocpu && A->bindingpropagates) PetscCall(MatBindToCPU(*B, PETSC_TRUE));
1482: if (ftype == MAT_FACTOR_LU || ftype == MAT_FACTOR_ILU || ftype == MAT_FACTOR_ILUDT) {
1483: PetscCall(MatSetBlockSizesFromMats(*B, A, A));
1484: if (!A->boundtocpu) {
1485: (*B)->ops->ilufactorsymbolic = MatILUFactorSymbolic_SeqAIJCUSPARSE;
1486: (*B)->ops->lufactorsymbolic = MatLUFactorSymbolic_SeqAIJCUSPARSE;
1487: } else {
1488: (*B)->ops->ilufactorsymbolic = MatILUFactorSymbolic_SeqAIJ;
1489: (*B)->ops->lufactorsymbolic = MatLUFactorSymbolic_SeqAIJ;
1490: }
1491: PetscCall(PetscStrallocpy(MATORDERINGND, (char **)&(*B)->preferredordering[MAT_FACTOR_LU]));
1492: PetscCall(PetscStrallocpy(MATORDERINGNATURAL, (char **)&(*B)->preferredordering[MAT_FACTOR_ILU]));
1493: PetscCall(PetscStrallocpy(MATORDERINGNATURAL, (char **)&(*B)->preferredordering[MAT_FACTOR_ILUDT]));
1494: } else if (ftype == MAT_FACTOR_CHOLESKY || ftype == MAT_FACTOR_ICC) {
1495: if (!A->boundtocpu) {
1496: (*B)->ops->iccfactorsymbolic = MatICCFactorSymbolic_SeqAIJCUSPARSE;
1497: (*B)->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SeqAIJCUSPARSE;
1498: } else {
1499: (*B)->ops->iccfactorsymbolic = MatICCFactorSymbolic_SeqAIJ;
1500: (*B)->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_SeqAIJ;
1501: }
1502: PetscCall(PetscStrallocpy(MATORDERINGND, (char **)&(*B)->preferredordering[MAT_FACTOR_CHOLESKY]));
1503: PetscCall(PetscStrallocpy(MATORDERINGNATURAL, (char **)&(*B)->preferredordering[MAT_FACTOR_ICC]));
1504: } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "Factor type not supported for CUSPARSE Matrix Types");
1506: PetscCall(MatSeqAIJSetPreallocation(*B, MAT_SKIP_ALLOCATION, NULL));
1507: (*B)->canuseordering = PETSC_TRUE;
1508: PetscCall(PetscObjectComposeFunction((PetscObject)*B, "MatFactorGetSolverType_C", MatFactorGetSolverType_seqaij_cusparse));
1509: PetscFunctionReturn(PETSC_SUCCESS);
1510: }
1512: static PetscErrorCode MatSeqAIJCUSPARSECopyFromGPU(Mat A)
1513: {
1514: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
1515: Mat_SeqAIJCUSPARSE *cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1516: Mat_SeqAIJCUSPARSETriFactors *fs = (Mat_SeqAIJCUSPARSETriFactors *)A->spptr;
1517: PetscDeviceContext dctx;
1518: cudaStream_t stream;
1520: PetscFunctionBegin;
1521: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
1522: if (A->offloadmask == PETSC_OFFLOAD_GPU) {
1523: PetscCall(PetscLogEventBegin(MAT_CUSPARSECopyFromGPU, A, 0, 0, 0));
1524: if (A->factortype == MAT_FACTOR_NONE) {
1525: CsrMatrix *matrix = (CsrMatrix *)cusp->mat->mat;
1526: PetscCallCUDA(cudaMemcpyAsync(a->a, matrix->values->data().get(), a->nz * sizeof(PetscScalar), cudaMemcpyDeviceToHost, stream));
1527: } else if (fs->csrVal) {
1528: /* We have a factorized matrix on device and are able to copy it to host */
1529: PetscCallCUDA(cudaMemcpyAsync(a->a, fs->csrVal, a->nz * sizeof(PetscScalar), cudaMemcpyDeviceToHost, stream));
1530: } else SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "No support for copying this type of factorized matrix from device to host");
1531: PetscCall(PetscDeviceContextSynchronize(dctx));
1532: PetscCall(PetscLogGpuToCpu(a->nz * sizeof(PetscScalar)));
1533: PetscCall(PetscLogEventEnd(MAT_CUSPARSECopyFromGPU, A, 0, 0, 0));
1534: A->offloadmask = PETSC_OFFLOAD_BOTH;
1535: }
1536: PetscFunctionReturn(PETSC_SUCCESS);
1537: }
1539: static PetscErrorCode MatSeqAIJGetArray_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1540: {
1541: return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJGetArray(A, array);
1542: }
1544: static PetscErrorCode MatSeqAIJRestoreArray_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1545: {
1546: return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJRestoreArray(A, array);
1547: }
1549: static PetscErrorCode MatSeqAIJGetArrayRead_SeqAIJCUSPARSE(Mat A, const PetscScalar *array[])
1550: {
1551: return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJGetArrayRead(A, array);
1552: }
1554: static PetscErrorCode MatSeqAIJRestoreArrayRead_SeqAIJCUSPARSE(Mat A, const PetscScalar *array[])
1555: {
1556: return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJRestoreArrayRead(A, array);
1557: }
1559: static PetscErrorCode MatSeqAIJGetArrayWrite_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1560: {
1561: return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJGetArrayWrite(A, array);
1562: }
1564: static PetscErrorCode MatSeqAIJRestoreArrayWrite_SeqAIJCUSPARSE(Mat A, PetscScalar *array[])
1565: {
1566: return MatSeqAIJCUSPARSE_CUPM_t::SeqAIJRestoreArrayWrite(A, array);
1567: }
1569: static PetscErrorCode MatSeqAIJGetCSRAndMemType_SeqAIJCUSPARSE(Mat A, const PetscInt **i, const PetscInt **j, PetscScalar **a, PetscMemType *mtype)
1570: {
1571: Mat_SeqAIJCUSPARSE *cusp;
1572: CsrMatrix *matrix;
1574: PetscFunctionBegin;
1575: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
1576: PetscCheck(A->factortype == MAT_FACTOR_NONE, PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONGSTATE, "Not for factored matrix");
1577: cusp = static_cast<Mat_SeqAIJCUSPARSE *>(A->spptr);
1578: PetscCheck(cusp != NULL, PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONGSTATE, "cusp is NULL");
1579: matrix = (CsrMatrix *)cusp->mat->mat;
1581: if (i) *i = matrix->row_offsets->data().get();
1582: if (j) *j = matrix->column_indices->data().get();
1583: if (a) *a = matrix->values->data().get();
1584: if (mtype) *mtype = PETSC_MEMTYPE_CUDA;
1585: PetscFunctionReturn(PETSC_SUCCESS);
1586: }
1588: PETSC_INTERN PetscErrorCode MatSeqAIJCUSPARSECopyToGPU(Mat A)
1589: {
1590: Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
1591: Mat_SeqAIJCUSPARSEMultStruct *matstruct = cusparsestruct->mat;
1592: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
1593: PetscInt m = A->rmap->n, *ii, *ridx, tmp;
1594: PetscBool both = PETSC_TRUE;
1595: PetscDeviceContext dctx;
1596: cudaStream_t stream;
1598: PetscFunctionBegin;
1599: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
1600: PetscCheck(!A->boundtocpu, PETSC_COMM_SELF, PETSC_ERR_GPU, "Cannot copy to GPU");
1601: if (A->offloadmask == PETSC_OFFLOAD_UNALLOCATED || A->offloadmask == PETSC_OFFLOAD_CPU) {
1602: if (A->nonzerostate == cusparsestruct->nonzerostate && cusparsestruct->format == MAT_CUSPARSE_CSR) { /* Copy values only */
1603: CsrMatrix *matrix;
1604: matrix = (CsrMatrix *)cusparsestruct->mat->mat;
1606: PetscCheck(!a->nz || a->a, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CSR values");
1607: PetscCall(PetscLogEventBegin(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1608: PetscCallCUDA(cudaMemcpyAsync(matrix->values->data().get(), a->a, a->nz * sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1609: PetscCall(PetscDeviceContextSynchronize(dctx));
1610: PetscCall(PetscLogCpuToGpu(a->nz * sizeof(PetscScalar)));
1611: PetscCall(PetscLogEventEnd(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1612: PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_FALSE));
1613: } else {
1614: PetscInt nnz;
1615: PetscCall(PetscLogEventBegin(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1616: PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusparsestruct->mat, cusparsestruct->format));
1617: PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(A, PETSC_TRUE));
1618: delete cusparsestruct->workVector;
1619: delete cusparsestruct->rowoffsets_gpu;
1620: cusparsestruct->workVector = NULL;
1621: cusparsestruct->rowoffsets_gpu = NULL;
1622: try {
1623: if (a->compressedrow.use) {
1624: m = a->compressedrow.nrows;
1625: ii = a->compressedrow.i;
1626: ridx = a->compressedrow.rindex;
1627: } else {
1628: m = A->rmap->n;
1629: ii = a->i;
1630: ridx = NULL;
1631: }
1632: PetscCheck(ii, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CSR row data");
1633: if (!a->a) {
1634: nnz = ii[m];
1635: both = PETSC_FALSE;
1636: } else nnz = a->nz;
1637: PetscCheck(!nnz || a->j, PETSC_COMM_SELF, PETSC_ERR_GPU, "Missing CSR column data");
1639: /* create cusparse matrix */
1640: cusparsestruct->nrows = m;
1641: matstruct = new Mat_SeqAIJCUSPARSEMultStruct;
1642: PetscCallCUSPARSE(cusparseCreateMatDescr(&matstruct->descr));
1643: PetscCallCUSPARSE(cusparseSetMatIndexBase(matstruct->descr, CUSPARSE_INDEX_BASE_ZERO));
1644: PetscCallCUSPARSE(cusparseSetMatType(matstruct->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
1646: PetscCallCUDA(cudaMalloc((void **)&matstruct->alpha_one, sizeof(PetscScalar)));
1647: PetscCallCUDA(cudaMalloc((void **)&matstruct->beta_zero, sizeof(PetscScalar)));
1648: PetscCallCUDA(cudaMalloc((void **)&matstruct->beta_one, sizeof(PetscScalar)));
1649: PetscCallCUDA(cudaMemcpyAsync(matstruct->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1650: PetscCallCUDA(cudaMemcpyAsync(matstruct->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1651: PetscCallCUDA(cudaMemcpyAsync(matstruct->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
1653: /* Build a hybrid/ellpack matrix if this option is chosen for the storage */
1654: if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
1655: /* set the matrix */
1656: CsrMatrix *mat = new CsrMatrix;
1657: mat->num_rows = m;
1658: mat->num_cols = A->cmap->n;
1659: mat->num_entries = nnz;
1660: PetscCallCXX(mat->row_offsets = new THRUSTINTARRAY(m + 1));
1661: PetscCallCUDA(cudaMemcpyAsync(mat->row_offsets->data().get(), ii, (m + 1) * sizeof(*ii), cudaMemcpyHostToDevice, stream));
1662: PetscCallCXX(mat->column_indices = new THRUSTINTARRAY(nnz));
1663: PetscCallCUDA(cudaMemcpyAsync(mat->column_indices->data().get(), a->j, (nnz) * sizeof(*a->j), cudaMemcpyHostToDevice, stream));
1665: PetscCallCXX(mat->values = new THRUSTARRAY(nnz));
1666: if (a->a) PetscCallCUDA(cudaMemcpyAsync(mat->values->data().get(), a->a, (nnz) * sizeof(*a->a), cudaMemcpyHostToDevice, stream));
1668: /* assign the pointer */
1669: matstruct->mat = mat;
1670: if (mat->num_rows) { /* cusparse errors on empty matrices! */
1671: PetscCallCUSPARSE(cusparseCreateCsr(&matstruct->matDescr, mat->num_rows, mat->num_cols, mat->num_entries, mat->row_offsets->data().get(), mat->column_indices->data().get(), mat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1672: }
1673: } else if (cusparsestruct->format == MAT_CUSPARSE_ELL || cusparsestruct->format == MAT_CUSPARSE_HYB) {
1674: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
1675: }
1677: /* assign the compressed row indices */
1678: if (a->compressedrow.use) {
1679: PetscCallCXX(cusparsestruct->workVector = new THRUSTARRAY(m));
1680: PetscCallCXX(matstruct->cprowIndices = new THRUSTINTARRAY(m));
1681: PetscCallCUDA(cudaMemcpyAsync(matstruct->cprowIndices->data().get(), ridx, (m) * sizeof(*ridx), cudaMemcpyHostToDevice, stream));
1682: tmp = m;
1683: } else {
1684: cusparsestruct->workVector = NULL;
1685: matstruct->cprowIndices = NULL;
1686: tmp = 0;
1687: }
1688: PetscCall(PetscLogCpuToGpu(((m + 1) + (a->nz)) * sizeof(int) + tmp * sizeof(PetscInt) + (3 + (a->nz)) * sizeof(PetscScalar)));
1690: /* assign the pointer */
1691: cusparsestruct->mat = matstruct;
1692: } catch (char *ex) {
1693: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "CUSPARSE error: %s", ex);
1694: }
1695: PetscCall(PetscDeviceContextSynchronize(dctx));
1696: PetscCall(PetscLogEventEnd(MAT_CUSPARSECopyToGPU, A, 0, 0, 0));
1697: cusparsestruct->nonzerostate = A->nonzerostate;
1698: }
1699: if (both) A->offloadmask = PETSC_OFFLOAD_BOTH;
1700: }
1701: PetscFunctionReturn(PETSC_SUCCESS);
1702: }
1704: struct VecCUDAPlusEquals {
1705: template <typename Tuple>
1706: __host__ __device__ void operator()(Tuple t)
1707: {
1708: thrust::get<1>(t) = thrust::get<1>(t) + thrust::get<0>(t);
1709: }
1710: };
1712: struct VecCUDAEquals {
1713: template <typename Tuple>
1714: __host__ __device__ void operator()(Tuple t)
1715: {
1716: thrust::get<1>(t) = thrust::get<0>(t);
1717: }
1718: };
1720: struct VecCUDAEqualsReverse {
1721: template <typename Tuple>
1722: __host__ __device__ void operator()(Tuple t)
1723: {
1724: thrust::get<0>(t) = thrust::get<1>(t);
1725: }
1726: };
1728: struct MatProductCtx_MatMatCusparse {
1729: PetscBool cisdense;
1730: PetscScalar *Bt;
1731: Mat X;
1732: PetscBool reusesym; /* Cusparse does not have split symbolic and numeric phases for sparse matmat operations */
1733: PetscLogDouble flops;
1734: CsrMatrix *Bcsr;
1736: cusparseSpMatDescr_t matSpBDescr;
1737: PetscBool initialized; /* C = alpha op(A) op(B) + beta C */
1738: cusparseDnMatDescr_t matBDescr;
1739: cusparseDnMatDescr_t matCDescr;
1740: PetscInt Blda, Clda; /* Record leading dimensions of B and C here to detect changes*/
1741: void *dBuffer4;
1742: void *dBuffer5;
1743: size_t mmBufferSize;
1744: void *mmBuffer;
1745: void *mmBuffer2; /* SpGEMM WorkEstimation buffer */
1746: cusparseSpGEMMDescr_t spgemmDesc;
1747: };
1749: static PetscErrorCode MatProductCtxDestroy_MatMatCusparse(PetscCtxRt data)
1750: {
1751: MatProductCtx_MatMatCusparse *mmdata = *(MatProductCtx_MatMatCusparse **)data;
1753: PetscFunctionBegin;
1754: PetscCallCUDA(cudaFree(mmdata->Bt));
1755: delete mmdata->Bcsr;
1756: if (mmdata->matSpBDescr) PetscCallCUSPARSE(cusparseDestroySpMat(mmdata->matSpBDescr));
1757: if (mmdata->matBDescr) PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matBDescr));
1758: if (mmdata->matCDescr) PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matCDescr));
1759: if (mmdata->spgemmDesc) PetscCallCUSPARSE(cusparseSpGEMM_destroyDescr(mmdata->spgemmDesc));
1760: PetscCallCUDA(cudaFree(mmdata->dBuffer4));
1761: PetscCallCUDA(cudaFree(mmdata->dBuffer5));
1762: PetscCallCUDA(cudaFree(mmdata->mmBuffer));
1763: PetscCallCUDA(cudaFree(mmdata->mmBuffer2));
1764: PetscCall(MatDestroy(&mmdata->X));
1765: PetscCall(PetscFree(mmdata));
1766: PetscFunctionReturn(PETSC_SUCCESS);
1767: }
1769: #include <../src/mat/impls/dense/seq/dense.h>
1771: static PetscErrorCode MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA(Mat C)
1772: {
1773: Mat_Product *product = C->product;
1774: Mat A, B;
1775: PetscInt m, n, blda, clda;
1776: PetscBool flg, biscuda, compressed;
1777: Mat_SeqAIJ *a;
1778: Mat_SeqAIJCUSPARSE *cusp;
1779: cusparseOperation_t opA;
1780: const PetscScalar *barray;
1781: PetscScalar *carray;
1782: MatProductCtx_MatMatCusparse *mmdata;
1783: Mat_SeqAIJCUSPARSEMultStruct *mat;
1784: CsrMatrix *csrmat;
1785: PetscDeviceContext dctx;
1786: cudaStream_t stream;
1787: cusparseHandle_t handle;
1789: PetscFunctionBegin;
1790: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
1791: {
1792: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};
1794: MatCheckProduct(C, 1);
1795: PetscCheck(C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data empty");
1796: mmdata = (MatProductCtx_MatMatCusparse *)product->data;
1797: A = product->A;
1798: B = product->B;
1799: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1800: PetscCheck(flg, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
1801: /* currently CopyToGpu does not copy if the matrix is bound to CPU
1802: Instead of silently accepting the wrong answer, I prefer to raise the error */
1803: PetscCheck(!A->boundtocpu, PetscObjectComm((PetscObject)A), PETSC_ERR_ARG_WRONG, "Cannot bind to CPU a CUSPARSE matrix between MatProductSymbolic and MatProductNumeric phases");
1804: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
1805: a = (Mat_SeqAIJ *)A->data;
1806: cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1807: switch (product->type) {
1808: case MATPRODUCT_AB:
1809: case MATPRODUCT_PtAP:
1810: mat = cusp->mat;
1811: opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
1812: m = A->rmap->n;
1813: n = B->cmap->n;
1814: break;
1815: case MATPRODUCT_AtB:
1816: if (!A->form_explicit_transpose) {
1817: mat = cusp->mat;
1818: opA = CUSPARSE_OPERATION_TRANSPOSE;
1819: } else {
1820: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
1821: mat = cusp->matTranspose;
1822: opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
1823: }
1824: m = A->cmap->n;
1825: n = B->cmap->n;
1826: break;
1827: case MATPRODUCT_ABt:
1828: case MATPRODUCT_RARt:
1829: mat = cusp->mat;
1830: opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
1831: m = A->rmap->n;
1832: n = B->rmap->n;
1833: break;
1834: default:
1835: SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
1836: }
1837: PetscCheck(mat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing Mat_SeqAIJCUSPARSEMultStruct");
1838: csrmat = (CsrMatrix *)mat->mat;
1839: /* when the rows of A are compressed on the device, csrmat holds only the nonempty rows of A, so the
1840: SpMM descriptor below must be built with the full row offsets instead of those of csrmat */
1841: compressed = (PetscBool)(mat->cprowIndices != NULL);
1842: /* if the user passed a CPU matrix, copy the data to the GPU */
1843: PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQDENSECUDA, &biscuda));
1844: if (!biscuda) PetscCall(MatConvert(B, MATSEQDENSECUDA, MAT_INPLACE_MATRIX, &B));
1845: PetscCall(MatDenseGetArrayReadAndMemType(B, &barray, nullptr));
1847: PetscCall(MatDenseGetLDA(B, &blda));
1848: if (product->type == MATPRODUCT_RARt || product->type == MATPRODUCT_PtAP) {
1849: PetscCall(MatDenseGetArrayWriteAndMemType(mmdata->X, &carray, nullptr));
1850: PetscCall(MatDenseGetLDA(mmdata->X, &clda));
1851: } else {
1852: PetscCall(MatDenseGetArrayWriteAndMemType(C, &carray, nullptr));
1853: PetscCall(MatDenseGetLDA(C, &clda));
1854: }
1856: PetscCall(PetscLogGpuTimeBegin());
1857: cusparseOperation_t opB = (product->type == MATPRODUCT_ABt || product->type == MATPRODUCT_RARt) ? CUSPARSE_OPERATION_TRANSPOSE : CUSPARSE_OPERATION_NON_TRANSPOSE;
1858: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0)
1859: cusparseSpMatDescr_t &matADescr = mat->matDescr_SpMM[opA];
1860: #else
1861: /* mat->matDescr is also used by the SpGEMM code, which relies on its compressed dimensions, so when A is
1862: compressed the SpMM needs a descriptor of its own */
1863: cusparseSpMatDescr_t &matADescr = compressed ? mat->matDescr_SpMM[opA] : mat->matDescr;
1864: #endif
1866: /* (re)allocate mmBuffer if not initialized or LDAs are different */
1867: if (!mmdata->initialized || mmdata->Blda != blda || mmdata->Clda != clda) {
1868: size_t mmBufferSize;
1869: if (mmdata->initialized && mmdata->Blda != blda) {
1870: PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matBDescr));
1871: mmdata->matBDescr = NULL;
1872: }
1873: if (!mmdata->matBDescr) {
1874: PetscCallCUSPARSE(cusparseCreateDnMat(&mmdata->matBDescr, B->rmap->n, B->cmap->n, blda, (void *)barray, cusparse_scalartype, CUSPARSE_ORDER_COL));
1875: mmdata->Blda = blda;
1876: }
1878: if (mmdata->initialized && mmdata->Clda != clda) {
1879: PetscCallCUSPARSE(cusparseDestroyDnMat(mmdata->matCDescr));
1880: mmdata->matCDescr = NULL;
1881: }
1882: if (!mmdata->matCDescr) { /* matCDescr is for C or mmdata->X */
1883: PetscCallCUSPARSE(cusparseCreateDnMat(&mmdata->matCDescr, m, n, clda, (void *)carray, cusparse_scalartype, CUSPARSE_ORDER_COL));
1884: mmdata->Clda = clda;
1885: }
1887: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0) // tested up to 12.6.0
1888: if (matADescr) {
1889: PetscCallCUSPARSE(cusparseDestroySpMat(matADescr)); // Because I find I could not reuse matADescr. It could be a cusparse bug
1890: matADescr = NULL;
1891: }
1892: #endif
1894: if (!matADescr) {
1895: if (compressed) {
1896: if (!cusp->rowoffsets_gpu) { /* the full row offsets may be absent when we did not construct the transpose with csr2csc */
1897: cusp->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
1898: PetscCallCUDA(cudaMemcpyAsync(cusp->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
1899: PetscCall(PetscDeviceContextSynchronize(dctx));
1900: PetscCall(PetscLogCpuToGpu((A->rmap->n + 1) * sizeof(PetscInt)));
1901: }
1902: PetscCallCUSPARSE(cusparseCreateCsr(&matADescr, A->rmap->n, csrmat->num_cols, csrmat->num_entries, cusp->rowoffsets_gpu->data().get(), csrmat->column_indices->data().get(), csrmat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1903: } else {
1904: PetscCallCUSPARSE(cusparseCreateCsr(&matADescr, csrmat->num_rows, csrmat->num_cols, csrmat->num_entries, csrmat->row_offsets->data().get(), csrmat->column_indices->data().get(), csrmat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
1905: }
1906: }
1908: PetscCallCUSPARSE(cusparseSpMM_bufferSize(handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalartype, cusp->spmmAlg, &mmBufferSize));
1910: if ((mmdata->mmBuffer && mmdata->mmBufferSize < mmBufferSize) || !mmdata->mmBuffer) {
1911: PetscCallCUDA(cudaFree(mmdata->mmBuffer));
1912: PetscCallCUDA(cudaMalloc(&mmdata->mmBuffer, mmBufferSize));
1913: mmdata->mmBufferSize = mmBufferSize;
1914: }
1916: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0) // the _preprocess was added in 11.2.1, but PETSc worked without it until 12.4.0
1917: PetscCallCUSPARSE(cusparseSpMM_preprocess(handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalartype, cusp->spmmAlg, mmdata->mmBuffer));
1918: #endif
1920: mmdata->initialized = PETSC_TRUE;
1921: } else {
1922: /* to be safe, always update pointers of the mats */
1923: PetscCallCUSPARSE(cusparseSpMatSetValues(matADescr, csrmat->values->data().get()));
1924: PetscCallCUSPARSE(cusparseDnMatSetValues(mmdata->matBDescr, (void *)barray));
1925: PetscCallCUSPARSE(cusparseDnMatSetValues(mmdata->matCDescr, (void *)carray));
1926: }
1928: /* do cusparseSpMM, which supports transpose on B */
1929: PetscCallCUSPARSE(cusparseSpMM(handle, opA, opB, mat->alpha_one, matADescr, mmdata->matBDescr, mat->beta_zero, mmdata->matCDescr, cusparse_scalartype, cusp->spmmAlg, mmdata->mmBuffer));
1931: PetscCall(PetscLogGpuTimeEnd());
1932: PetscCall(PetscLogGpuFlops(n * 2.0 * csrmat->num_entries));
1933: PetscCall(MatDenseRestoreArrayReadAndMemType(B, &barray));
1934: if (product->type == MATPRODUCT_RARt) {
1935: PetscCall(MatDenseRestoreArrayWriteAndMemType(mmdata->X, &carray));
1936: PetscCall(MatMatMultNumeric_SeqDenseCUDA_SeqDenseCUDA_Internal(B, mmdata->X, C, PETSC_FALSE, PETSC_FALSE));
1937: } else if (product->type == MATPRODUCT_PtAP) {
1938: PetscCall(MatDenseRestoreArrayWriteAndMemType(mmdata->X, &carray));
1939: PetscCall(MatMatMultNumeric_SeqDenseCUDA_SeqDenseCUDA_Internal(B, mmdata->X, C, PETSC_TRUE, PETSC_FALSE));
1940: } else {
1941: PetscCall(MatDenseRestoreArrayWriteAndMemType(C, &carray));
1942: }
1943: if (mmdata->cisdense) PetscCall(MatConvert(C, MATSEQDENSE, MAT_INPLACE_MATRIX, &C));
1944: if (!biscuda) PetscCall(MatConvert(B, MATSEQDENSE, MAT_INPLACE_MATRIX, &B));
1945: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
1946: }
1947: PetscFunctionReturn(PETSC_SUCCESS);
1948: }
1950: static PetscErrorCode MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA(Mat C)
1951: {
1952: Mat_Product *product = C->product;
1953: Mat A, B;
1954: PetscInt m, n;
1955: PetscBool cisdense, flg;
1956: MatProductCtx_MatMatCusparse *mmdata;
1957: Mat_SeqAIJCUSPARSE *cusp;
1959: PetscFunctionBegin;
1960: MatCheckProduct(C, 1);
1961: PetscCheck(!C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data not empty");
1962: A = product->A;
1963: B = product->B;
1964: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
1965: PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
1966: cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
1967: PetscCheck(cusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
1968: switch (product->type) {
1969: case MATPRODUCT_AB:
1970: m = A->rmap->n;
1971: n = B->cmap->n;
1972: PetscCall(MatSetBlockSizesFromMats(C, A, B));
1973: break;
1974: case MATPRODUCT_AtB:
1975: m = A->cmap->n;
1976: n = B->cmap->n;
1977: if (A->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, A->cmap->bs));
1978: if (B->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->cmap->bs));
1979: break;
1980: case MATPRODUCT_ABt:
1981: m = A->rmap->n;
1982: n = B->rmap->n;
1983: if (A->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, A->rmap->bs));
1984: if (B->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->rmap->bs));
1985: break;
1986: case MATPRODUCT_PtAP:
1987: m = B->cmap->n;
1988: n = B->cmap->n;
1989: if (B->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, B->cmap->bs));
1990: if (B->cmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->cmap->bs));
1991: break;
1992: case MATPRODUCT_RARt:
1993: m = B->rmap->n;
1994: n = B->rmap->n;
1995: if (B->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->rmap, B->rmap->bs));
1996: if (B->rmap->bs > 0) PetscCall(PetscLayoutSetBlockSize(C->cmap, B->rmap->bs));
1997: break;
1998: default:
1999: SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
2000: }
2001: PetscCall(MatSetSizes(C, m, n, m, n));
2002: /* if C is of type MATSEQDENSE (CPU), perform the operation on the GPU and then copy on the CPU */
2003: PetscCall(PetscObjectTypeCompare((PetscObject)C, MATSEQDENSE, &cisdense));
2004: PetscCall(MatSetType(C, MATSEQDENSECUDA));
2005: /* keep the VecType of B, e.g. VECKOKKOS from MatCreateDenseFromVecType(), but only if B is a MATSEQDENSECUDA like C, since the VecType of a MATSEQDENSE B, e.g. VECSTANDARD, is not valid for C */
2006: PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQDENSECUDA, &flg));
2007: if (flg) PetscCall(MatSetVecType(C, B->defaultvectype));
2009: /* product data */
2010: PetscCall(PetscNew(&mmdata));
2011: mmdata->cisdense = cisdense;
2012: /* for these products we need intermediate storage */
2013: if (product->type == MATPRODUCT_RARt || product->type == MATPRODUCT_PtAP) {
2014: PetscCall(MatCreate(PetscObjectComm((PetscObject)C), &mmdata->X));
2015: PetscCall(MatSetType(mmdata->X, MATSEQDENSECUDA));
2016: if (product->type == MATPRODUCT_RARt) { /* do not preallocate, since the first call to MatDenseCUDAGetArray will preallocate on the GPU for us */
2017: PetscCall(MatSetSizes(mmdata->X, A->rmap->n, B->rmap->n, A->rmap->n, B->rmap->n));
2018: } else {
2019: PetscCall(MatSetSizes(mmdata->X, A->rmap->n, B->cmap->n, A->rmap->n, B->cmap->n));
2020: }
2021: }
2022: C->product->data = mmdata;
2023: C->product->destroy = MatProductCtxDestroy_MatMatCusparse;
2025: C->ops->productnumeric = MatProductNumeric_SeqAIJCUSPARSE_SeqDENSECUDA;
2026: PetscFunctionReturn(PETSC_SUCCESS);
2027: }
2029: static PetscErrorCode MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE(Mat C)
2030: {
2031: Mat_Product *product = C->product;
2032: Mat A, B;
2033: Mat_SeqAIJCUSPARSE *Acusp, *Bcusp, *Ccusp;
2034: Mat_SeqAIJ *c = (Mat_SeqAIJ *)C->data;
2035: Mat_SeqAIJCUSPARSEMultStruct *Amat, *Bmat, *Cmat;
2036: CsrMatrix *Acsr, *Bcsr, *Ccsr;
2037: PetscBool flg;
2038: MatProductType ptype;
2039: MatProductCtx_MatMatCusparse *mmdata;
2040: cusparseSpMatDescr_t BmatSpDescr;
2041: cusparseOperation_t opA = CUSPARSE_OPERATION_NON_TRANSPOSE, opB = CUSPARSE_OPERATION_NON_TRANSPOSE; /* cuSPARSE spgemm doesn't support transpose yet */
2042: PetscDeviceContext dctx;
2043: cusparseHandle_t handle;
2045: PetscFunctionBegin;
2046: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle));
2047: {
2048: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};
2050: MatCheckProduct(C, 1);
2051: PetscCheck(C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data empty");
2052: PetscCall(PetscObjectTypeCompare((PetscObject)C, MATSEQAIJCUSPARSE, &flg));
2053: PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for C of type %s", ((PetscObject)C)->type_name);
2054: mmdata = (MatProductCtx_MatMatCusparse *)C->product->data;
2055: A = product->A;
2056: B = product->B;
2057: if (mmdata->reusesym) { /* this happens when api_user is true, meaning that the matrix values have been already computed in the MatProductSymbolic phase */
2058: mmdata->reusesym = PETSC_FALSE;
2059: Ccusp = (Mat_SeqAIJCUSPARSE *)C->spptr;
2060: PetscCheck(Ccusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2061: Cmat = Ccusp->mat;
2062: PetscCheck(Cmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C mult struct for product type %s", MatProductTypes[C->product->type]);
2063: Ccsr = (CsrMatrix *)Cmat->mat;
2064: PetscCheck(Ccsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C CSR struct");
2065: goto finalize;
2066: }
2067: if (!c->nz) goto finalize;
2068: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
2069: PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
2070: PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQAIJCUSPARSE, &flg));
2071: PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for B of type %s", ((PetscObject)B)->type_name);
2072: PetscCheck(!A->boundtocpu, PetscObjectComm((PetscObject)C), PETSC_ERR_ARG_WRONG, "Cannot bind to CPU a CUSPARSE matrix between MatProductSymbolic and MatProductNumeric phases");
2073: PetscCheck(!B->boundtocpu, PetscObjectComm((PetscObject)C), PETSC_ERR_ARG_WRONG, "Cannot bind to CPU a CUSPARSE matrix between MatProductSymbolic and MatProductNumeric phases");
2074: Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
2075: Bcusp = (Mat_SeqAIJCUSPARSE *)B->spptr;
2076: Ccusp = (Mat_SeqAIJCUSPARSE *)C->spptr;
2077: PetscCheck(Acusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2078: PetscCheck(Bcusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2079: PetscCheck(Ccusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2080: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
2081: PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
2083: ptype = product->type;
2084: if (A->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_AtB) {
2085: ptype = MATPRODUCT_AB;
2086: PetscCheck(product->symbolic_used_the_fact_A_is_symmetric, PetscObjectComm((PetscObject)C), PETSC_ERR_PLIB, "Symbolic should have been built using the fact that A is symmetric");
2087: }
2088: if (B->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_ABt) {
2089: ptype = MATPRODUCT_AB;
2090: PetscCheck(product->symbolic_used_the_fact_B_is_symmetric, PetscObjectComm((PetscObject)C), PETSC_ERR_PLIB, "Symbolic should have been built using the fact that B is symmetric");
2091: }
2092: switch (ptype) {
2093: case MATPRODUCT_AB:
2094: Amat = Acusp->mat;
2095: Bmat = Bcusp->mat;
2096: break;
2097: case MATPRODUCT_AtB:
2098: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
2099: Amat = Acusp->matTranspose;
2100: Bmat = Bcusp->mat;
2101: break;
2102: case MATPRODUCT_ABt:
2103: Amat = Acusp->mat;
2104: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(B));
2105: Bmat = Bcusp->matTranspose;
2106: break;
2107: default:
2108: SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
2109: }
2110: Cmat = Ccusp->mat;
2111: PetscCheck(Amat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A mult struct for product type %s", MatProductTypes[ptype]);
2112: PetscCheck(Bmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B mult struct for product type %s", MatProductTypes[ptype]);
2113: PetscCheck(Cmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C mult struct for product type %s", MatProductTypes[ptype]);
2114: Acsr = (CsrMatrix *)Amat->mat;
2115: Bcsr = mmdata->Bcsr ? mmdata->Bcsr : (CsrMatrix *)Bmat->mat; /* B may be in compressed row storage */
2116: Ccsr = (CsrMatrix *)Cmat->mat;
2117: PetscCheck(Acsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A CSR struct");
2118: PetscCheck(Bcsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B CSR struct");
2119: PetscCheck(Ccsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing C CSR struct");
2120: PetscCall(PetscLogGpuTimeBegin());
2121: BmatSpDescr = mmdata->Bcsr ? mmdata->matSpBDescr : Bmat->matDescr; /* B may be in compressed row storage */
2122: PetscCallCUSPARSE(cusparseSpGEMM_compute(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, mmdata->mmBuffer));
2123: PetscCallCUSPARSE(cusparseSpGEMM_copy(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc));
2124: PetscCall(PetscLogGpuFlops(mmdata->flops));
2125: PetscCall(PetscDeviceContextSynchronize(dctx));
2126: PetscCall(PetscLogGpuTimeEnd());
2127: C->offloadmask = PETSC_OFFLOAD_GPU;
2128: finalize:
2129: /* shorter version of MatAssemblyEnd_SeqAIJ */
2130: PetscCall(PetscInfo(C, "Matrix size: %" PetscInt_FMT " X %" PetscInt_FMT "; storage space: 0 unneeded, %" PetscInt_FMT " used\n", C->rmap->n, C->cmap->n, c->nz));
2131: PetscCall(PetscInfo(C, "Number of mallocs during MatSetValues() is 0\n"));
2132: PetscCall(PetscInfo(C, "Maximum nonzeros in any row is %" PetscInt_FMT "\n", c->rmax));
2133: c->reallocs = 0;
2134: C->info.mallocs += 0;
2135: C->info.nz_unneeded = 0;
2136: C->assembled = C->was_assembled = PETSC_TRUE;
2137: C->num_ass++;
2138: }
2139: PetscFunctionReturn(PETSC_SUCCESS);
2140: }
2142: static PetscErrorCode MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE(Mat C)
2143: {
2144: Mat_Product *product = C->product;
2145: Mat A, B;
2146: Mat_SeqAIJCUSPARSE *Acusp, *Bcusp, *Ccusp;
2147: Mat_SeqAIJ *a, *b, *c;
2148: Mat_SeqAIJCUSPARSEMultStruct *Amat, *Bmat, *Cmat;
2149: CsrMatrix *Acsr, *Bcsr, *Ccsr;
2150: PetscInt i, j, m, n, k;
2151: PetscBool flg;
2152: MatProductType ptype;
2153: MatProductCtx_MatMatCusparse *mmdata;
2154: PetscLogDouble flops;
2155: PetscBool biscompressed, ciscompressed;
2156: int64_t C_num_rows1, C_num_cols1, C_nnz1;
2157: cusparseSpMatDescr_t BmatSpDescr;
2158: cusparseOperation_t opA = CUSPARSE_OPERATION_NON_TRANSPOSE, opB = CUSPARSE_OPERATION_NON_TRANSPOSE; /* cuSPARSE spgemm doesn't support transpose yet */
2159: size_t bufSize2;
2160: PetscDeviceContext dctx;
2161: cudaStream_t stream;
2162: cusparseHandle_t handle;
2164: PetscFunctionBegin;
2165: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
2166: {
2167: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};
2169: MatCheckProduct(C, 1);
2170: PetscCheck(!C->product->data, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Product data not empty");
2171: A = product->A;
2172: B = product->B;
2173: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &flg));
2174: PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for type %s", ((PetscObject)A)->type_name);
2175: PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQAIJCUSPARSE, &flg));
2176: PetscCheck(flg, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Not for B of type %s", ((PetscObject)B)->type_name);
2177: a = (Mat_SeqAIJ *)A->data;
2178: b = (Mat_SeqAIJ *)B->data;
2179: /* product data */
2180: PetscCall(PetscNew(&mmdata));
2181: C->product->data = mmdata;
2182: C->product->destroy = MatProductCtxDestroy_MatMatCusparse;
2184: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
2185: PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
2186: Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr; /* Access spptr after MatSeqAIJCUSPARSECopyToGPU, not before */
2187: Bcusp = (Mat_SeqAIJCUSPARSE *)B->spptr;
2188: PetscCheck(Acusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2189: PetscCheck(Bcusp->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Only for MAT_CUSPARSE_CSR format");
2191: ptype = product->type;
2192: if (A->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_AtB) {
2193: ptype = MATPRODUCT_AB;
2194: product->symbolic_used_the_fact_A_is_symmetric = PETSC_TRUE;
2195: }
2196: if (B->symmetric == PETSC_BOOL3_TRUE && ptype == MATPRODUCT_ABt) {
2197: ptype = MATPRODUCT_AB;
2198: product->symbolic_used_the_fact_B_is_symmetric = PETSC_TRUE;
2199: }
2200: biscompressed = PETSC_FALSE;
2201: ciscompressed = PETSC_FALSE;
2202: switch (ptype) {
2203: case MATPRODUCT_AB:
2204: m = A->rmap->n;
2205: n = B->cmap->n;
2206: k = A->cmap->n;
2207: Amat = Acusp->mat;
2208: Bmat = Bcusp->mat;
2209: if (a->compressedrow.use) ciscompressed = PETSC_TRUE;
2210: if (b->compressedrow.use) biscompressed = PETSC_TRUE;
2211: break;
2212: case MATPRODUCT_AtB:
2213: m = A->cmap->n;
2214: n = B->cmap->n;
2215: k = A->rmap->n;
2216: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
2217: Amat = Acusp->matTranspose;
2218: Bmat = Bcusp->mat;
2219: if (b->compressedrow.use) biscompressed = PETSC_TRUE;
2220: break;
2221: case MATPRODUCT_ABt:
2222: m = A->rmap->n;
2223: n = B->rmap->n;
2224: k = A->cmap->n;
2225: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(B));
2226: Amat = Acusp->mat;
2227: Bmat = Bcusp->matTranspose;
2228: if (a->compressedrow.use) ciscompressed = PETSC_TRUE;
2229: break;
2230: default:
2231: SETERRQ(PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Unsupported product type %s", MatProductTypes[product->type]);
2232: }
2234: /* create cusparse matrix */
2235: PetscCall(MatSetSizes(C, m, n, m, n));
2236: PetscCall(MatSetType(C, MATSEQAIJCUSPARSE));
2237: c = (Mat_SeqAIJ *)C->data;
2238: Ccusp = (Mat_SeqAIJCUSPARSE *)C->spptr;
2239: Cmat = new Mat_SeqAIJCUSPARSEMultStruct;
2240: Ccsr = new CsrMatrix;
2242: c->compressedrow.use = ciscompressed;
2243: if (c->compressedrow.use) { /* if a is in compressed row, than c will be in compressed row format */
2244: c->compressedrow.nrows = a->compressedrow.nrows;
2245: PetscCall(PetscMalloc2(c->compressedrow.nrows + 1, &c->compressedrow.i, c->compressedrow.nrows, &c->compressedrow.rindex));
2246: PetscCall(PetscArraycpy(c->compressedrow.rindex, a->compressedrow.rindex, c->compressedrow.nrows));
2247: Ccusp->workVector = new THRUSTARRAY(c->compressedrow.nrows);
2248: Cmat->cprowIndices = new THRUSTINTARRAY(c->compressedrow.nrows);
2249: PetscCallCUDA(cudaMemcpyAsync(Cmat->cprowIndices->data().get(), c->compressedrow.rindex, (c->compressedrow.nrows) * sizeof(*c->compressedrow.rindex), cudaMemcpyHostToDevice, stream));
2250: } else {
2251: c->compressedrow.nrows = 0;
2252: c->compressedrow.i = NULL;
2253: c->compressedrow.rindex = NULL;
2254: Ccusp->workVector = NULL;
2255: Cmat->cprowIndices = NULL;
2256: }
2257: Ccusp->nrows = ciscompressed ? c->compressedrow.nrows : m;
2258: Ccusp->mat = Cmat;
2259: Ccusp->mat->mat = Ccsr;
2260: Ccsr->num_rows = Ccusp->nrows;
2261: Ccsr->num_cols = n;
2262: Ccsr->row_offsets = new THRUSTINTARRAY(Ccusp->nrows + 1);
2263: PetscCallCUSPARSE(cusparseCreateMatDescr(&Cmat->descr));
2264: PetscCallCUSPARSE(cusparseSetMatIndexBase(Cmat->descr, CUSPARSE_INDEX_BASE_ZERO));
2265: PetscCallCUSPARSE(cusparseSetMatType(Cmat->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
2266: PetscCallCUDA(cudaMalloc((void **)&Cmat->alpha_one, sizeof(PetscScalar)));
2267: PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_zero, sizeof(PetscScalar)));
2268: PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_one, sizeof(PetscScalar)));
2269: PetscCallCUDA(cudaMemcpyAsync(Cmat->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
2270: PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
2271: PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
2272: if (!Ccsr->num_rows || !Ccsr->num_cols || !a->nz || !b->nz) { /* cusparse raise errors in different calls when matrices have zero rows/columns! */
2273: PetscCallThrust(thrust::fill(thrust::cuda::par.on(stream), Ccsr->row_offsets->begin(), Ccsr->row_offsets->end(), 0));
2274: c->nz = 0;
2275: Ccsr->column_indices = new THRUSTINTARRAY(c->nz);
2276: Ccsr->values = new THRUSTARRAY(c->nz);
2277: goto finalizesym;
2278: }
2280: PetscCheck(Amat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A mult struct for product type %s", MatProductTypes[ptype]);
2281: PetscCheck(Bmat, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B mult struct for product type %s", MatProductTypes[ptype]);
2282: Acsr = (CsrMatrix *)Amat->mat;
2283: if (!biscompressed) {
2284: Bcsr = (CsrMatrix *)Bmat->mat;
2285: BmatSpDescr = Bmat->matDescr;
2286: } else { /* we need to use row offsets for the full matrix */
2287: CsrMatrix *cBcsr = (CsrMatrix *)Bmat->mat;
2288: Bcsr = new CsrMatrix;
2289: Bcsr->num_rows = B->rmap->n;
2290: Bcsr->num_cols = cBcsr->num_cols;
2291: Bcsr->num_entries = cBcsr->num_entries;
2292: Bcsr->column_indices = cBcsr->column_indices;
2293: Bcsr->values = cBcsr->values;
2294: if (!Bcusp->rowoffsets_gpu) {
2295: Bcusp->rowoffsets_gpu = new THRUSTINTARRAY(B->rmap->n + 1);
2296: PetscCallCUDA(cudaMemcpyAsync(Bcusp->rowoffsets_gpu->data().get(), b->i, (B->rmap->n + 1) * sizeof(*b->i), cudaMemcpyHostToDevice, stream));
2297: PetscCall(PetscLogCpuToGpu((B->rmap->n + 1) * sizeof(PetscInt)));
2298: }
2299: Bcsr->row_offsets = Bcusp->rowoffsets_gpu;
2300: mmdata->Bcsr = Bcsr;
2301: if (Bcsr->num_rows && Bcsr->num_cols) {
2302: PetscCallCUSPARSE(cusparseCreateCsr(&mmdata->matSpBDescr, Bcsr->num_rows, Bcsr->num_cols, Bcsr->num_entries, Bcsr->row_offsets->data().get(), Bcsr->column_indices->data().get(), Bcsr->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
2303: }
2304: BmatSpDescr = mmdata->matSpBDescr;
2305: }
2306: PetscCheck(Acsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing A CSR struct");
2307: PetscCheck(Bcsr, PetscObjectComm((PetscObject)C), PETSC_ERR_GPU, "Missing B CSR struct");
2308: /* precompute flops count */
2309: if (ptype == MATPRODUCT_AB) {
2310: for (i = 0, flops = 0; i < A->rmap->n; i++) {
2311: const PetscInt st = a->i[i];
2312: const PetscInt en = a->i[i + 1];
2313: for (j = st; j < en; j++) {
2314: const PetscInt brow = a->j[j];
2315: flops += 2. * (b->i[brow + 1] - b->i[brow]);
2316: }
2317: }
2318: } else if (ptype == MATPRODUCT_AtB) {
2319: for (i = 0, flops = 0; i < A->rmap->n; i++) {
2320: const PetscInt anzi = a->i[i + 1] - a->i[i];
2321: const PetscInt bnzi = b->i[i + 1] - b->i[i];
2322: flops += (2. * anzi) * bnzi;
2323: }
2324: } else { /* TODO */
2325: flops = 0.;
2326: }
2328: mmdata->flops = flops;
2329: PetscCall(PetscLogGpuTimeBegin());
2331: // cuda-12.2 requires non-null csrRowOffsets
2332: PetscCallCUSPARSE(cusparseCreateCsr(&Cmat->matDescr, Ccsr->num_rows, Ccsr->num_cols, 0, Ccsr->row_offsets->data().get(), NULL, NULL, csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
2333: PetscCallCUSPARSE(cusparseSpGEMM_createDescr(&mmdata->spgemmDesc));
2334: // Note that cusparseSpGEMMreuse is deprecated in CUDA 13.2.1
2336: PetscCheck(!PetscDefined(USE_64BIT_INDICES) || PETSC_PKG_CUDA_VERSION_GE(13, 0, 0), PETSC_COMM_SELF, PETSC_ERR_SUP_SYS, "cusparseSpGEMM did not support 64-bit indices before CUDA 13.0. Update your CUDA installation.");
2337: /* ask bufferSize bytes for external memory */
2338: PetscCallCUSPARSE(cusparseSpGEMM_workEstimation(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufSize2, NULL));
2339: PetscCallCUDA(cudaMalloc((void **)&mmdata->mmBuffer2, bufSize2));
2340: /* inspect the matrices A and B to understand the memory requirement for the next step */
2341: PetscCallCUSPARSE(cusparseSpGEMM_workEstimation(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &bufSize2, mmdata->mmBuffer2));
2342: /* ask bufferSize again bytes for external memory */
2343: PetscCallCUSPARSE(cusparseSpGEMM_compute(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, NULL));
2344: /* The CUSPARSE documentation is not clear, nor the API
2345: We need both buffers to perform the operations properly!
2346: mmdata->mmBuffer2 does not appear anywhere in the compute/copy API
2347: it only appears for the workEstimation stuff, but it seems it is needed in compute, so probably the address
2348: is stored in the descriptor! What a messy API... */
2349: PetscCallCUDA(cudaMalloc((void **)&mmdata->mmBuffer, mmdata->mmBufferSize));
2350: /* compute the intermediate product of A * B */
2351: PetscCallCUSPARSE(cusparseSpGEMM_compute(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc, &mmdata->mmBufferSize, mmdata->mmBuffer));
2352: /* get matrix C non-zero entries C_nnz1 */
2353: PetscCallCUSPARSE(cusparseSpMatGetSize(Cmat->matDescr, &C_num_rows1, &C_num_cols1, &C_nnz1));
2354: PetscCall(PetscIntCast(C_nnz1, &c->nz));
2355: PetscCall(PetscInfo(C, "Buffer sizes for type %s, result %" PetscInt_FMT " x %" PetscInt_FMT " (k %" PetscInt_FMT ", nzA %" PetscInt_FMT ", nzB %" PetscInt_FMT ", nzC %" PetscInt_FMT ") are: %ldKB %ldKB\n", MatProductTypes[ptype], m, n, k, a->nz, b->nz, c->nz, bufSize2 / 1024,
2356: mmdata->mmBufferSize / 1024));
2357: Ccsr->column_indices = new THRUSTINTARRAY(c->nz);
2358: PetscCallCUDA(cudaPeekAtLastError()); /* catch out of memory errors */
2359: Ccsr->values = new THRUSTARRAY(c->nz);
2360: PetscCallCUDA(cudaPeekAtLastError()); /* catch out of memory errors */
2361: if (c->nz) PetscCallCUSPARSE(cusparseCsrSetPointers(Cmat->matDescr, Ccsr->row_offsets->data().get(), Ccsr->column_indices->data().get(), Ccsr->values->data().get()));
2362: PetscCallCUSPARSE(cusparseSpGEMM_copy(handle, opA, opB, Cmat->alpha_one, Amat->matDescr, BmatSpDescr, Cmat->beta_zero, Cmat->matDescr, cusparse_scalartype, CUSPARSE_SPGEMM_DEFAULT, mmdata->spgemmDesc));
2363: PetscCall(PetscLogGpuFlops(mmdata->flops));
2364: PetscCall(PetscLogGpuTimeEnd());
2365: finalizesym:
2366: c->free_a = PETSC_TRUE;
2367: PetscCall(PetscShmgetAllocateArray(c->nz, sizeof(PetscInt), (void **)&c->j));
2368: PetscCall(PetscShmgetAllocateArray(m + 1, sizeof(PetscInt), (void **)&c->i));
2369: c->free_ij = PETSC_TRUE;
2371: PetscInt *d_i = c->i;
2372: if (ciscompressed) d_i = c->compressedrow.i;
2373: PetscCallCUDA(cudaMemcpyAsync(d_i, Ccsr->row_offsets->data().get(), Ccsr->row_offsets->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
2374: PetscCallCUDA(cudaMemcpyAsync(c->j, Ccsr->column_indices->data().get(), Ccsr->column_indices->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
2375: PetscCall(PetscDeviceContextSynchronize(dctx));
2376: if (ciscompressed) { /* need to expand host row offsets */
2377: PetscInt r = 0;
2378: c->i[0] = 0;
2379: for (k = 0; k < c->compressedrow.nrows; k++) {
2380: const PetscInt next = c->compressedrow.rindex[k];
2381: const PetscInt old = c->compressedrow.i[k];
2382: for (; r < next; r++) c->i[r + 1] = old;
2383: }
2384: for (; r < m; r++) c->i[r + 1] = c->compressedrow.i[c->compressedrow.nrows];
2385: }
2386: PetscCall(PetscLogGpuToCpu((Ccsr->column_indices->size() + Ccsr->row_offsets->size()) * sizeof(PetscInt)));
2387: PetscCall(PetscMalloc1(m, &c->ilen));
2388: PetscCall(PetscMalloc1(m, &c->imax));
2389: c->maxnz = c->nz;
2390: c->nonzerorowcnt = 0;
2391: c->rmax = 0;
2392: for (k = 0; k < m; k++) {
2393: const PetscInt nn = c->i[k + 1] - c->i[k];
2394: c->ilen[k] = c->imax[k] = nn;
2395: c->nonzerorowcnt += (PetscInt)!!nn;
2396: c->rmax = PetscMax(c->rmax, nn);
2397: }
2398: PetscCall(PetscMalloc1(c->nz, &c->a));
2399: Ccsr->num_entries = c->nz;
2401: C->nonzerostate++;
2402: PetscCall(PetscLayoutSetUp(C->rmap));
2403: PetscCall(PetscLayoutSetUp(C->cmap));
2404: Ccusp->nonzerostate = C->nonzerostate;
2405: C->offloadmask = PETSC_OFFLOAD_UNALLOCATED;
2406: C->preallocated = PETSC_TRUE;
2407: C->assembled = PETSC_FALSE;
2408: C->was_assembled = PETSC_FALSE;
2409: if (product->api_user && A->offloadmask == PETSC_OFFLOAD_BOTH && B->offloadmask == PETSC_OFFLOAD_BOTH) { /* flag the matrix C values as computed, so that the numeric phase will only call MatAssembly */
2410: mmdata->reusesym = PETSC_TRUE;
2411: C->offloadmask = PETSC_OFFLOAD_GPU;
2412: }
2413: C->ops->productnumeric = MatProductNumeric_SeqAIJCUSPARSE_SeqAIJCUSPARSE;
2414: }
2415: PetscFunctionReturn(PETSC_SUCCESS);
2416: }
2418: PETSC_INTERN PetscErrorCode MatProductSetFromOptions_SeqAIJ_SeqDense(Mat);
2420: /* handles sparse or dense B */
2421: static PetscErrorCode MatProductSetFromOptions_SeqAIJCUSPARSE(Mat mat)
2422: {
2423: Mat_Product *product = mat->product;
2424: PetscBool isdense = PETSC_FALSE, Biscusp = PETSC_FALSE, Ciscusp = PETSC_TRUE;
2426: PetscFunctionBegin;
2427: MatCheckProduct(mat, 1);
2428: PetscCall(PetscObjectBaseTypeCompare((PetscObject)product->B, MATSEQDENSE, &isdense));
2429: if (!product->A->boundtocpu && !product->B->boundtocpu) PetscCall(PetscObjectTypeCompare((PetscObject)product->B, MATSEQAIJCUSPARSE, &Biscusp));
2430: if (product->type == MATPRODUCT_ABC) {
2431: Ciscusp = PETSC_FALSE;
2432: if (!product->C->boundtocpu) PetscCall(PetscObjectTypeCompare((PetscObject)product->C, MATSEQAIJCUSPARSE, &Ciscusp));
2433: }
2434: if (Biscusp && Ciscusp) { /* we can always select the CPU backend */
2435: PetscBool usecpu = PETSC_FALSE;
2436: switch (product->type) {
2437: case MATPRODUCT_AB:
2438: if (product->api_user) {
2439: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatMatMult", "Mat");
2440: PetscCall(PetscOptionsBool("-matmatmult_backend_cpu", "Use CPU code", "MatMatMult", usecpu, &usecpu, NULL));
2441: PetscOptionsEnd();
2442: } else {
2443: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_AB", "Mat");
2444: PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatMatMult", usecpu, &usecpu, NULL));
2445: PetscOptionsEnd();
2446: }
2447: break;
2448: case MATPRODUCT_AtB:
2449: if (product->api_user) {
2450: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatTransposeMatMult", "Mat");
2451: PetscCall(PetscOptionsBool("-mattransposematmult_backend_cpu", "Use CPU code", "MatTransposeMatMult", usecpu, &usecpu, NULL));
2452: PetscOptionsEnd();
2453: } else {
2454: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_AtB", "Mat");
2455: PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatTransposeMatMult", usecpu, &usecpu, NULL));
2456: PetscOptionsEnd();
2457: }
2458: break;
2459: case MATPRODUCT_PtAP:
2460: if (product->api_user) {
2461: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatPtAP", "Mat");
2462: PetscCall(PetscOptionsBool("-matptap_backend_cpu", "Use CPU code", "MatPtAP", usecpu, &usecpu, NULL));
2463: PetscOptionsEnd();
2464: } else {
2465: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_PtAP", "Mat");
2466: PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatPtAP", usecpu, &usecpu, NULL));
2467: PetscOptionsEnd();
2468: }
2469: break;
2470: case MATPRODUCT_RARt:
2471: if (product->api_user) {
2472: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatRARt", "Mat");
2473: PetscCall(PetscOptionsBool("-matrart_backend_cpu", "Use CPU code", "MatRARt", usecpu, &usecpu, NULL));
2474: PetscOptionsEnd();
2475: } else {
2476: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_RARt", "Mat");
2477: PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatRARt", usecpu, &usecpu, NULL));
2478: PetscOptionsEnd();
2479: }
2480: break;
2481: case MATPRODUCT_ABC:
2482: if (product->api_user) {
2483: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatMatMatMult", "Mat");
2484: PetscCall(PetscOptionsBool("-matmatmatmult_backend_cpu", "Use CPU code", "MatMatMatMult", usecpu, &usecpu, NULL));
2485: PetscOptionsEnd();
2486: } else {
2487: PetscOptionsBegin(PetscObjectComm((PetscObject)mat), ((PetscObject)mat)->prefix, "MatProduct_ABC", "Mat");
2488: PetscCall(PetscOptionsBool("-mat_product_algorithm_backend_cpu", "Use CPU code", "MatMatMatMult", usecpu, &usecpu, NULL));
2489: PetscOptionsEnd();
2490: }
2491: break;
2492: default:
2493: break;
2494: }
2495: if (usecpu) Biscusp = Ciscusp = PETSC_FALSE;
2496: }
2497: /* dispatch */
2498: if (isdense) {
2499: switch (product->type) {
2500: case MATPRODUCT_AB:
2501: case MATPRODUCT_AtB:
2502: case MATPRODUCT_ABt:
2503: case MATPRODUCT_PtAP:
2504: case MATPRODUCT_RARt:
2505: if (product->A->boundtocpu) {
2506: PetscCall(MatProductSetFromOptions_SeqAIJ_SeqDense(mat));
2507: } else {
2508: mat->ops->productsymbolic = MatProductSymbolic_SeqAIJCUSPARSE_SeqDENSECUDA;
2509: }
2510: break;
2511: case MATPRODUCT_ABC:
2512: mat->ops->productsymbolic = MatProductSymbolic_ABC_Basic;
2513: break;
2514: default:
2515: break;
2516: }
2517: } else if (Biscusp && Ciscusp) {
2518: switch (product->type) {
2519: case MATPRODUCT_AB:
2520: case MATPRODUCT_AtB:
2521: case MATPRODUCT_ABt:
2522: mat->ops->productsymbolic = MatProductSymbolic_SeqAIJCUSPARSE_SeqAIJCUSPARSE;
2523: break;
2524: case MATPRODUCT_PtAP:
2525: case MATPRODUCT_RARt:
2526: case MATPRODUCT_ABC:
2527: mat->ops->productsymbolic = MatProductSymbolic_ABC_Basic;
2528: break;
2529: default:
2530: break;
2531: }
2532: } else { /* fallback for AIJ */
2533: PetscCall(MatProductSetFromOptions_SeqAIJ(mat));
2534: }
2535: PetscFunctionReturn(PETSC_SUCCESS);
2536: }
2538: static PetscErrorCode MatMult_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy)
2539: {
2540: PetscFunctionBegin;
2541: PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, NULL, yy, PETSC_FALSE, PETSC_FALSE));
2542: PetscFunctionReturn(PETSC_SUCCESS);
2543: }
2545: static PetscErrorCode MatMultAdd_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz)
2546: {
2547: PetscFunctionBegin;
2548: PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, yy, zz, PETSC_FALSE, PETSC_FALSE));
2549: PetscFunctionReturn(PETSC_SUCCESS);
2550: }
2552: static PetscErrorCode MatMultHermitianTranspose_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy)
2553: {
2554: PetscFunctionBegin;
2555: PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, NULL, yy, PETSC_TRUE, PETSC_TRUE));
2556: PetscFunctionReturn(PETSC_SUCCESS);
2557: }
2559: static PetscErrorCode MatMultHermitianTransposeAdd_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz)
2560: {
2561: PetscFunctionBegin;
2562: PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, yy, zz, PETSC_TRUE, PETSC_TRUE));
2563: PetscFunctionReturn(PETSC_SUCCESS);
2564: }
2566: static PetscErrorCode MatMultTranspose_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy)
2567: {
2568: PetscFunctionBegin;
2569: PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, NULL, yy, PETSC_TRUE, PETSC_FALSE));
2570: PetscFunctionReturn(PETSC_SUCCESS);
2571: }
2573: __global__ static void ScatterAdd(PetscInt n, PetscInt *idx, const PetscScalar *x, PetscScalar *y)
2574: {
2575: int i = blockIdx.x * blockDim.x + threadIdx.x;
2576: if (i < n) y[idx[i]] += x[i];
2577: }
2579: /* z = op(A) x + y. If trans & !herm, op = ^T; if trans & herm, op = ^H; if !trans, op = no-op */
2580: static PetscErrorCode MatMultAddKernel_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz, PetscBool trans, PetscBool herm)
2581: {
2582: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
2583: Mat_SeqAIJCUSPARSE *cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
2584: Mat_SeqAIJCUSPARSEMultStruct *matstruct;
2585: PetscScalar *xarray, *zarray, *dptr, *beta, *xptr;
2586: cusparseOperation_t opA = CUSPARSE_OPERATION_NON_TRANSPOSE;
2587: PetscBool compressed;
2588: PetscInt nx, ny;
2589: PetscDeviceContext dctx;
2590: cudaStream_t stream;
2591: cusparseHandle_t handle;
2593: PetscFunctionBegin;
2594: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
2595: {
2596: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_DEVICE};
2598: PetscCheck(!herm || trans, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "Hermitian and not transpose not supported");
2599: if (!a->nz) {
2600: if (yy) PetscCall(VecSeq_CUDA::Copy(yy, zz));
2601: else PetscCall(VecSeq_CUDA::Set(zz, 0));
2602: PetscFunctionReturn(PETSC_SUCCESS);
2603: }
2604: /* The line below is necessary due to the operations that modify the matrix on the CPU (axpy, scale, etc) */
2605: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
2606: if (!trans) {
2607: matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
2608: PetscCheck(matstruct, PetscObjectComm((PetscObject)A), PETSC_ERR_GPU, "SeqAIJCUSPARSE does not have a 'mat' (need to fix)");
2609: } else {
2610: if (herm || !A->form_explicit_transpose) {
2611: opA = herm ? CUSPARSE_OPERATION_CONJUGATE_TRANSPOSE : CUSPARSE_OPERATION_TRANSPOSE;
2612: matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
2613: } else {
2614: if (!A->transupdated) PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
2615: matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->matTranspose;
2616: }
2617: }
2618: /* Does the matrix use compressed rows (i.e., drop zero rows)? */
2619: compressed = matstruct->cprowIndices ? PETSC_TRUE : PETSC_FALSE;
2621: try {
2622: PetscCall(VecCUDAGetArrayRead(xx, (const PetscScalar **)&xarray));
2623: if (yy == zz) PetscCall(VecCUDAGetArray(zz, &zarray)); /* read & write zz, so need to get up-to-date zarray on GPU */
2624: else PetscCall(VecCUDAGetArrayWrite(zz, &zarray)); /* write zz, so no need to init zarray on GPU */
2626: PetscCall(PetscLogGpuTimeBegin());
2627: if (opA == CUSPARSE_OPERATION_NON_TRANSPOSE) {
2628: /* z = A x + beta y.
2629: If A is compressed (with less rows), then Ax is shorter than the full z, so we need a work vector to store Ax.
2630: When A is non-compressed, and z = y, we can set beta=1 to compute y = Ax + y in one call.
2631: */
2632: xptr = xarray;
2633: dptr = compressed ? cusparsestruct->workVector->data().get() : zarray;
2634: beta = (yy == zz && !compressed) ? matstruct->beta_one : matstruct->beta_zero;
2635: /* Get length of x, y for y=Ax. ny might be shorter than the work vector's allocated length, since the work vector is
2636: allocated to accommodate different uses. So we get the length info directly from mat.
2637: */
2638: if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
2639: CsrMatrix *mat = (CsrMatrix *)matstruct->mat;
2640: nx = mat->num_cols; // since y = Ax
2641: ny = mat->num_rows;
2642: }
2643: } else {
2644: /* z = A^T x + beta y
2645: If A is compressed, then we need a work vector as the shorter version of x to compute A^T x.
2646: Note A^Tx is of full length, so we set beta to 1.0 if y exists.
2647: */
2648: xptr = compressed ? cusparsestruct->workVector->data().get() : xarray;
2649: dptr = zarray;
2650: beta = yy ? matstruct->beta_one : matstruct->beta_zero;
2651: if (compressed) { /* Scatter x to work vector */
2652: thrust::device_ptr<PetscScalar> xarr = thrust::device_pointer_cast(xarray);
2654: PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::make_zip_iterator(thrust::make_tuple(cusparsestruct->workVector->begin(), thrust::make_permutation_iterator(xarr, matstruct->cprowIndices->begin()))),
2655: thrust::make_zip_iterator(thrust::make_tuple(cusparsestruct->workVector->begin(), thrust::make_permutation_iterator(xarr, matstruct->cprowIndices->begin()))) + matstruct->cprowIndices->size(), VecCUDAEqualsReverse()));
2656: }
2657: if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
2658: CsrMatrix *mat = (CsrMatrix *)matstruct->mat;
2659: nx = mat->num_rows; // since y = A^T x
2660: ny = mat->num_cols;
2661: }
2662: }
2664: /* csr_spmv does y = alpha op(A) x + beta y */
2665: if (cusparsestruct->format == MAT_CUSPARSE_CSR) {
2666: PetscCheck(opA >= 0 && opA <= 2, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuSPARSE ABI on cusparseOperation_t has changed and PETSc has not been updated accordingly");
2668: if (!matstruct->cuSpMV[opA].initialized) { /* built on demand */
2669: CsrMatrix *mat = (CsrMatrix *)matstruct->mat;
2670: PetscCallCUSPARSE(cusparseCreateCsr(&matstruct->cuSpMV[opA].matDescr, mat->num_rows, mat->num_cols, mat->num_entries, mat->row_offsets->data().get(), mat->column_indices->data().get(), mat->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
2671: PetscCallCUSPARSE(cusparseCreateDnVec(&matstruct->cuSpMV[opA].vecXDescr, nx, xptr, cusparse_scalartype));
2672: PetscCallCUSPARSE(cusparseCreateDnVec(&matstruct->cuSpMV[opA].vecYDescr, ny, dptr, cusparse_scalartype));
2673: PetscCallCUSPARSE(
2674: cusparseSpMV_bufferSize(handle, opA, matstruct->alpha_one, matstruct->cuSpMV[opA].matDescr, matstruct->cuSpMV[opA].vecXDescr, beta, matstruct->cuSpMV[opA].vecYDescr, cusparse_scalartype, cusparsestruct->spmvAlg, &matstruct->cuSpMV[opA].spmvBufferSize));
2675: PetscCallCUDA(cudaMalloc(&matstruct->cuSpMV[opA].spmvBuffer, matstruct->cuSpMV[opA].spmvBufferSize));
2676: #if PETSC_PKG_CUDA_VERSION_GE(12, 4, 0) // cusparseSpMV_preprocess is added in 12.4
2677: PetscCallCUSPARSE(
2678: cusparseSpMV_preprocess(handle, opA, matstruct->alpha_one, matstruct->cuSpMV[opA].matDescr, matstruct->cuSpMV[opA].vecXDescr, beta, matstruct->cuSpMV[opA].vecYDescr, cusparse_scalartype, cusparsestruct->spmvAlg, matstruct->cuSpMV[opA].spmvBuffer));
2679: #endif
2680: matstruct->cuSpMV[opA].initialized = PETSC_TRUE;
2681: } else {
2682: /* x, y's value pointers might change between calls, but their shape is kept, so we just update pointers */
2683: PetscCallCUSPARSE(cusparseDnVecSetValues(matstruct->cuSpMV[opA].vecXDescr, xptr));
2684: PetscCallCUSPARSE(cusparseDnVecSetValues(matstruct->cuSpMV[opA].vecYDescr, dptr));
2685: }
2687: PetscCallCUSPARSE(cusparseSpMV(handle, opA, matstruct->alpha_one, matstruct->cuSpMV[opA].matDescr, matstruct->cuSpMV[opA].vecXDescr, beta, matstruct->cuSpMV[opA].vecYDescr, cusparse_scalartype, cusparsestruct->spmvAlg, matstruct->cuSpMV[opA].spmvBuffer));
2689: } else {
2690: if (cusparsestruct->nrows) {
2691: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
2692: }
2693: }
2694: PetscCall(PetscLogGpuTimeEnd());
2696: if (opA == CUSPARSE_OPERATION_NON_TRANSPOSE) {
2697: if (yy) { /* MatMultAdd: zz = A*xx + yy */
2698: if (compressed) { /* A is compressed. We first copy yy to zz, then ScatterAdd the work vector to zz */
2699: PetscCall(VecSeq_CUDA::Copy(yy, zz)); /* zz = yy */
2700: } else if (zz != yy) { /* A is not compressed. zz already contains A*xx, and we just need to add yy */
2701: PetscCall(VecSeq_CUDA::AXPY(zz, 1.0, yy)); /* zz += yy */
2702: }
2703: } else if (compressed) { /* MatMult: zz = A*xx. A is compressed, so we zero zz first, then ScatterAdd the work vector to zz */
2704: PetscCall(VecSeq_CUDA::Set(zz, 0));
2705: }
2707: /* ScatterAdd the result from work vector into the full vector when A is compressed */
2708: if (compressed) {
2709: PetscCall(PetscLogGpuTimeBegin());
2710: PetscInt n = (PetscInt)matstruct->cprowIndices->size();
2711: ScatterAdd<<<(int)((n + 255) / 256), 256, 0, stream>>>(n, matstruct->cprowIndices->data().get(), cusparsestruct->workVector->data().get(), zarray);
2712: PetscCall(PetscLogGpuTimeEnd());
2713: }
2714: } else {
2715: if (yy && yy != zz) PetscCall(VecSeq_CUDA::AXPY(zz, 1.0, yy)); /* zz += yy */
2716: }
2717: PetscCall(VecCUDARestoreArrayRead(xx, (const PetscScalar **)&xarray));
2718: if (yy == zz) PetscCall(VecCUDARestoreArray(zz, &zarray));
2719: else PetscCall(VecCUDARestoreArrayWrite(zz, &zarray));
2720: } catch (char *ex) {
2721: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_LIB, "CUSPARSE error: %s", ex);
2722: }
2723: if (yy) PetscCall(PetscLogGpuFlops(2.0 * a->nz));
2724: else PetscCall(PetscLogGpuFlops(2.0 * a->nz - a->nonzerorowcnt));
2725: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
2726: }
2727: PetscFunctionReturn(PETSC_SUCCESS);
2728: }
2730: static PetscErrorCode MatMultTransposeAdd_SeqAIJCUSPARSE(Mat A, Vec xx, Vec yy, Vec zz)
2731: {
2732: PetscFunctionBegin;
2733: PetscCall(MatMultAddKernel_SeqAIJCUSPARSE(A, xx, yy, zz, PETSC_TRUE, PETSC_FALSE));
2734: PetscFunctionReturn(PETSC_SUCCESS);
2735: }
2737: static PetscErrorCode MatGetDiagonal_SeqAIJCUSPARSE(Mat A, Vec diag)
2738: {
2739: PetscFunctionBegin;
2740: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetDiagonal(A, diag));
2741: PetscFunctionReturn(PETSC_SUCCESS);
2742: }
2744: static PetscErrorCode MatGetRowSumAbs_SeqAIJCUSPARSE(Mat A, Vec v)
2745: {
2746: PetscFunctionBegin;
2747: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetRowSumAbs(A, v));
2748: PetscFunctionReturn(PETSC_SUCCESS);
2749: }
2751: static PetscErrorCode MatDiagonalScale_SeqAIJCUSPARSE(Mat A, Vec ll, Vec rr)
2752: {
2753: PetscFunctionBegin;
2754: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::DiagonalScale(A, ll, rr));
2755: PetscFunctionReturn(PETSC_SUCCESS);
2756: }
2758: static PetscErrorCode MatAssemblyEnd_SeqAIJCUSPARSE(Mat A, MatAssemblyType mode)
2759: {
2760: PetscFunctionBegin;
2761: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::AssemblyEnd(A, mode));
2762: PetscFunctionReturn(PETSC_SUCCESS);
2763: }
2765: /*@
2766: MatCreateSeqAIJCUSPARSE - Creates a sparse matrix in `MATAIJCUSPARSE` (compressed row) format for use on NVIDIA GPUs
2768: Collective
2770: Input Parameters:
2771: + comm - MPI communicator, set to `PETSC_COMM_SELF`
2772: . m - number of rows
2773: . n - number of columns
2774: . nz - number of nonzeros per row (same for all rows), ignored if `nnz` is provide
2775: - nnz - array containing the number of nonzeros in the various rows (possibly different for each row) or `NULL`
2777: Output Parameter:
2778: . A - the matrix
2780: Level: intermediate
2782: Notes:
2783: This matrix will ultimately pushed down to NVIDIA GPUs and use the CuSPARSE library for
2784: calculations. For good matrix assembly performance the user should preallocate the matrix
2785: storage by setting the parameter `nz` (or the array `nnz`).
2787: It is recommended that one use the `MatCreate()`, `MatSetType()` and/or `MatSetFromOptions()`,
2788: MatXXXXSetPreallocation() paradgm instead of this routine directly.
2789: [MatXXXXSetPreallocation() is, for example, `MatSeqAIJSetPreallocation()`]
2791: The AIJ format, also called
2792: compressed row storage, is fully compatible with standard Fortran
2793: storage. That is, the stored row and column indices can begin at
2794: either one (as in Fortran) or zero.
2796: Specify the preallocated storage with either nz or nnz (not both).
2797: Set `nz` = `PETSC_DEFAULT` and `nnz` = `NULL` for PETSc to control dynamic memory
2798: allocation.
2800: When working with matrices for GPUs, it is often better to use the `MatSetPreallocationCOO()` and `MatSetValuesCOO()` paradigm rather than using this routine and `MatSetValues()`
2802: .seealso: [](ch_matrices), `Mat`, `MATSEQAIJCUSPARSE`, `MatCreate()`, `MatCreateAIJ()`, `MatSetValues()`, `MatSeqAIJSetColumnIndices()`, `MatCreateSeqAIJWithArrays()`, `MATAIJCUSPARSE`,
2803: `MatSetPreallocationCOO()`, `MatSetValuesCOO()`
2804: @*/
2805: PetscErrorCode MatCreateSeqAIJCUSPARSE(MPI_Comm comm, PetscInt m, PetscInt n, PetscInt nz, const PetscInt nnz[], Mat *A)
2806: {
2807: return MatSeqAIJCUSPARSE_CUPM_t::CreateSeqAIJ(comm, m, n, nz, nnz, A);
2808: }
2810: static PetscErrorCode MatDestroy_SeqAIJCUSPARSE(Mat A)
2811: {
2812: return MatSeqAIJCUSPARSE_CUPM_t::Destroy(A);
2813: }
2815: static PetscErrorCode MatDuplicate_SeqAIJCUSPARSE(Mat A, MatDuplicateOption cpvalues, Mat *B)
2816: {
2817: PetscFunctionBegin;
2818: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::Duplicate(A, cpvalues, B));
2819: PetscFunctionReturn(PETSC_SUCCESS);
2820: }
2822: static PetscErrorCode MatAXPY_SeqAIJCUSPARSE(Mat Y, PetscScalar a, Mat X, MatStructure str)
2823: {
2824: Mat_SeqAIJ *x = (Mat_SeqAIJ *)X->data, *y = (Mat_SeqAIJ *)Y->data;
2825: Mat_SeqAIJCUSPARSE *cy;
2826: Mat_SeqAIJCUSPARSE *cx;
2827: CsrMatrix *csry, *csrx;
2828: PetscDeviceContext dctx;
2829: cudaStream_t stream;
2830: cusparseHandle_t handle;
2832: PetscFunctionBegin;
2833: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetSparseHandles_(&dctx, &handle, &stream));
2834: {
2835: const SparsePointerModeGuard pointer_mode{handle, CUSPARSE_POINTER_MODE_HOST};
2837: cy = (Mat_SeqAIJCUSPARSE *)Y->spptr;
2838: cx = (Mat_SeqAIJCUSPARSE *)X->spptr;
2839: if (X->ops->axpy != Y->ops->axpy) {
2840: PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(Y, PETSC_FALSE));
2841: PetscCall(MatAXPY_SeqAIJ(Y, a, X, str));
2842: PetscFunctionReturn(PETSC_SUCCESS);
2843: }
2844: /* if we are here, it means both matrices are bound to GPU */
2845: PetscCall(MatSeqAIJCUSPARSECopyToGPU(Y));
2846: PetscCall(MatSeqAIJCUSPARSECopyToGPU(X));
2847: PetscCheck(cy->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)Y), PETSC_ERR_GPU, "only MAT_CUSPARSE_CSR supported");
2848: PetscCheck(cx->format == MAT_CUSPARSE_CSR, PetscObjectComm((PetscObject)X), PETSC_ERR_GPU, "only MAT_CUSPARSE_CSR supported");
2849: csry = (CsrMatrix *)cy->mat->mat;
2850: csrx = (CsrMatrix *)cx->mat->mat;
2851: /* see if we can turn this into a cublas axpy */
2852: if (str != SAME_NONZERO_PATTERN && x->nz == y->nz && !x->compressedrow.use && !y->compressedrow.use) {
2853: bool eq;
2855: PetscCallThrust(eq = thrust::equal(thrust::cuda::par.on(stream), csry->row_offsets->begin(), csry->row_offsets->end(), csrx->row_offsets->begin()));
2856: if (eq) PetscCallThrust(eq = thrust::equal(thrust::cuda::par.on(stream), csry->column_indices->begin(), csry->column_indices->end(), csrx->column_indices->begin()));
2857: if (eq) str = SAME_NONZERO_PATTERN;
2858: }
2859: /* spgeam is buggy with one column */
2860: if (Y->cmap->n == 1 && str != SAME_NONZERO_PATTERN) str = DIFFERENT_NONZERO_PATTERN;
2862: #if !PetscDefined(USE_64BIT_INDICES) // cusparseScsrgeam2 etc. do not support 64bit indices
2863: if (str == SUBSET_NONZERO_PATTERN) {
2864: PetscScalar *ay, b = 1.0;
2865: const PetscScalar *ax;
2866: size_t bufferSize;
2867: void *buffer;
2869: PetscCall(MatSeqAIJCUSPARSEGetArrayRead(X, &ax));
2870: PetscCall(MatSeqAIJCUSPARSEGetArray(Y, &ay));
2871: PetscCallCUSPARSE(cusparse_csr_spgeam_bufferSize(handle, Y->rmap->n, Y->cmap->n, &a, cx->mat->descr, x->nz, ax, csrx->row_offsets->data().get(), csrx->column_indices->data().get(), &b, cy->mat->descr, y->nz, ay, csry->row_offsets->data().get(),
2872: csry->column_indices->data().get(), cy->mat->descr, ay, csry->row_offsets->data().get(), csry->column_indices->data().get(), &bufferSize));
2873: PetscCallCUDA(cudaMalloc(&buffer, bufferSize));
2874: PetscCall(PetscLogGpuTimeBegin());
2875: PetscCallCUSPARSE(cusparse_csr_spgeam(handle, Y->rmap->n, Y->cmap->n, &a, cx->mat->descr, x->nz, ax, csrx->row_offsets->data().get(), csrx->column_indices->data().get(), &b, cy->mat->descr, y->nz, ay, csry->row_offsets->data().get(),
2876: csry->column_indices->data().get(), cy->mat->descr, ay, csry->row_offsets->data().get(), csry->column_indices->data().get(), buffer));
2877: PetscCall(PetscLogGpuFlops(x->nz + y->nz));
2878: PetscCall(PetscLogGpuTimeEnd());
2879: PetscCallCUDA(cudaFree(buffer));
2881: PetscCall(MatSeqAIJCUSPARSERestoreArrayRead(X, &ax));
2882: PetscCall(MatSeqAIJCUSPARSERestoreArray(Y, &ay));
2883: } else
2884: #endif
2885: if (str == SAME_NONZERO_PATTERN) {
2886: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::AXPY_SameNZ(Y, a, X));
2887: } else {
2888: PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(Y, PETSC_FALSE));
2889: PetscCall(MatAXPY_SeqAIJ(Y, a, X, str));
2890: }
2891: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
2892: }
2893: PetscFunctionReturn(PETSC_SUCCESS);
2894: }
2896: static PetscErrorCode MatScale_SeqAIJCUSPARSE(Mat Y, PetscScalar a)
2897: {
2898: PetscFunctionBegin;
2899: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::Scale(Y, a));
2900: PetscFunctionReturn(PETSC_SUCCESS);
2901: }
2903: static PetscErrorCode MatZeroEntries_SeqAIJCUSPARSE(Mat A)
2904: {
2905: PetscFunctionBegin;
2906: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::ZeroEntries(A));
2907: PetscFunctionReturn(PETSC_SUCCESS);
2908: }
2910: static PetscErrorCode MatGetCurrentMemType_SeqAIJCUSPARSE(Mat A, PetscMemType *m)
2911: {
2912: PetscFunctionBegin;
2913: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetCurrentMemType(A, m));
2914: PetscFunctionReturn(PETSC_SUCCESS);
2915: }
2917: static PetscErrorCode MatBindToCPU_SeqAIJCUSPARSE(Mat A, PetscBool flg)
2918: {
2919: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
2921: PetscFunctionBegin;
2922: if (A->factortype != MAT_FACTOR_NONE) {
2923: A->boundtocpu = flg;
2924: PetscFunctionReturn(PETSC_SUCCESS);
2925: }
2926: if (flg) {
2927: PetscCall(MatSeqAIJCUSPARSECopyFromGPU(A));
2929: A->ops->scale = MatScale_SeqAIJ;
2930: A->ops->getdiagonal = MatGetDiagonal_SeqAIJ;
2931: A->ops->getrowsumabs = MatGetRowSumAbs_SeqAIJ;
2932: A->ops->diagonalscale = MatDiagonalScale_SeqAIJ;
2933: A->ops->axpy = MatAXPY_SeqAIJ;
2934: A->ops->zeroentries = MatZeroEntries_SeqAIJ;
2935: A->ops->mult = MatMult_SeqAIJ;
2936: A->ops->multadd = MatMultAdd_SeqAIJ;
2937: A->ops->multtranspose = MatMultTranspose_SeqAIJ;
2938: A->ops->multtransposeadd = MatMultTransposeAdd_SeqAIJ;
2939: A->ops->multhermitiantranspose = NULL;
2940: A->ops->multhermitiantransposeadd = NULL;
2941: A->ops->productsetfromoptions = MatProductSetFromOptions_SeqAIJ;
2942: A->ops->getcurrentmemtype = NULL;
2943: PetscCall(PetscMemzero(a->ops, sizeof(Mat_SeqAIJOps)));
2944: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSeqAIJCopySubArray_C", NULL));
2945: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdensecuda_C", NULL));
2946: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdense_C", NULL));
2947: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetPreallocationCOO_C", NULL));
2948: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetValuesCOO_C", NULL));
2949: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqaijcusparse_C", NULL));
2950: } else {
2951: A->ops->scale = MatScale_SeqAIJCUSPARSE;
2952: A->ops->getdiagonal = MatGetDiagonal_SeqAIJCUSPARSE;
2953: A->ops->getrowsumabs = MatGetRowSumAbs_SeqAIJCUSPARSE;
2954: A->ops->diagonalscale = MatDiagonalScale_SeqAIJCUSPARSE;
2955: A->ops->axpy = MatAXPY_SeqAIJCUSPARSE;
2956: A->ops->zeroentries = MatZeroEntries_SeqAIJCUSPARSE;
2957: A->ops->mult = MatMult_SeqAIJCUSPARSE;
2958: A->ops->multadd = MatMultAdd_SeqAIJCUSPARSE;
2959: A->ops->multtranspose = MatMultTranspose_SeqAIJCUSPARSE;
2960: A->ops->multtransposeadd = MatMultTransposeAdd_SeqAIJCUSPARSE;
2961: A->ops->multhermitiantranspose = MatMultHermitianTranspose_SeqAIJCUSPARSE;
2962: A->ops->multhermitiantransposeadd = MatMultHermitianTransposeAdd_SeqAIJCUSPARSE;
2963: A->ops->productsetfromoptions = MatProductSetFromOptions_SeqAIJCUSPARSE;
2964: A->ops->getcurrentmemtype = MatGetCurrentMemType_SeqAIJCUSPARSE;
2965: a->ops->getarray = MatSeqAIJGetArray_SeqAIJCUSPARSE;
2966: a->ops->restorearray = MatSeqAIJRestoreArray_SeqAIJCUSPARSE;
2967: a->ops->getarrayread = MatSeqAIJGetArrayRead_SeqAIJCUSPARSE;
2968: a->ops->restorearrayread = MatSeqAIJRestoreArrayRead_SeqAIJCUSPARSE;
2969: a->ops->getarraywrite = MatSeqAIJGetArrayWrite_SeqAIJCUSPARSE;
2970: a->ops->restorearraywrite = MatSeqAIJRestoreArrayWrite_SeqAIJCUSPARSE;
2971: a->ops->getcsrandmemtype = MatSeqAIJGetCSRAndMemType_SeqAIJCUSPARSE;
2973: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSeqAIJCopySubArray_C", MatSeqAIJCopySubArray_SeqAIJCUSPARSE));
2974: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdensecuda_C", MatProductSetFromOptions_SeqAIJCUSPARSE));
2975: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqdense_C", MatProductSetFromOptions_SeqAIJCUSPARSE));
2976: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetPreallocationCOO_C", MatSetPreallocationCOO_SeqAIJCUSPARSE));
2977: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatSetValuesCOO_C", MatSetValuesCOO_SeqAIJCUSPARSE));
2978: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatProductSetFromOptions_seqaijcusparse_seqaijcusparse_C", MatProductSetFromOptions_SeqAIJCUSPARSE));
2979: }
2980: A->boundtocpu = flg;
2981: a->inode.use = (flg && a->inode.size_csr) ? PETSC_TRUE : PETSC_FALSE;
2982: PetscFunctionReturn(PETSC_SUCCESS);
2983: }
2985: PETSC_INTERN PetscErrorCode MatConvert_SeqAIJ_SeqAIJCUSPARSE(Mat A, MatType, MatReuse reuse, Mat *newmat)
2986: {
2987: Mat B;
2989: PetscFunctionBegin;
2990: PetscCall(PetscDeviceInitialize(PETSC_DEVICE_CUDA)); /* first use of CUSPARSE may be via MatConvert */
2991: if (reuse == MAT_INITIAL_MATRIX) {
2992: PetscCall(MatDuplicate(A, MAT_COPY_VALUES, newmat));
2993: } else if (reuse == MAT_REUSE_MATRIX) {
2994: PetscCall(MatCopy(A, *newmat, SAME_NONZERO_PATTERN));
2995: }
2996: B = *newmat;
2998: PetscCall(PetscFree(B->defaultvectype));
2999: PetscCall(PetscStrallocpy(VECCUDA, &B->defaultvectype));
3001: if (reuse != MAT_REUSE_MATRIX && !B->spptr) {
3002: if (B->factortype == MAT_FACTOR_NONE) {
3003: Mat_SeqAIJCUSPARSE *spptr;
3004: PetscCall(PetscNew(&spptr));
3005: spptr->format = MAT_CUSPARSE_CSR;
3006: spptr->spmvAlg = CUSPARSE_SPMV_CSR_ALG1; /* default, since we only support csr */
3007: spptr->spmmAlg = CUSPARSE_SPMM_CSR_ALG1; /* default, only support column-major dense matrix B */
3008: spptr->csr2cscAlg = CUSPARSE_CSR2CSC_ALG1;
3009: B->spptr = spptr;
3010: } else {
3011: Mat_SeqAIJCUSPARSETriFactors *spptr;
3013: PetscCall(PetscNew(&spptr));
3014: B->spptr = spptr;
3015: }
3016: B->offloadmask = PETSC_OFFLOAD_UNALLOCATED;
3017: }
3018: B->ops->assemblyend = MatAssemblyEnd_SeqAIJCUSPARSE;
3019: B->ops->destroy = MatDestroy_SeqAIJCUSPARSE;
3020: B->ops->setoption = MatSetOption_SeqAIJCUSPARSE;
3021: B->ops->setfromoptions = MatSetFromOptions_SeqAIJCUSPARSE;
3022: B->ops->bindtocpu = MatBindToCPU_SeqAIJCUSPARSE;
3023: B->ops->duplicate = MatDuplicate_SeqAIJCUSPARSE;
3024: B->ops->getcurrentmemtype = MatGetCurrentMemType_SeqAIJCUSPARSE;
3026: PetscCall(MatBindToCPU_SeqAIJCUSPARSE(B, PETSC_FALSE));
3027: PetscCall(PetscObjectChangeTypeName((PetscObject)B, MATSEQAIJCUSPARSE));
3028: PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatCUSPARSESetFormat_C", MatCUSPARSESetFormat_SeqAIJCUSPARSE));
3029: #if PetscDefined(HAVE_HYPRE)
3030: PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatConvert_seqaijcusparse_hypre_C", MatConvert_AIJ_HYPRE));
3031: #endif
3032: PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatCUSPARSESetUseCPUSolve_C", MatCUSPARSESetUseCPUSolve_SeqAIJCUSPARSE));
3033: PetscFunctionReturn(PETSC_SUCCESS);
3034: }
3036: PETSC_EXTERN PetscErrorCode MatCreate_SeqAIJCUSPARSE(Mat B)
3037: {
3038: PetscFunctionBegin;
3039: PetscCall(MatCreate_SeqAIJ(B));
3040: PetscCall(MatConvert_SeqAIJ_SeqAIJCUSPARSE(B, MATSEQAIJCUSPARSE, MAT_INPLACE_MATRIX, &B));
3041: PetscFunctionReturn(PETSC_SUCCESS);
3042: }
3044: /*MC
3045: MATSEQAIJCUSPARSE - MATAIJCUSPARSE = "(seq)aijcusparse" - A matrix type to be used for sparse matrices on NVIDIA GPUs.
3047: Options Database Keys:
3048: + -mat_type aijcusparse - Sets the matrix type to `MATSEQAIJCUSPARSE` during a call to `MatSetFromOptions()`
3049: . -mat_cusparse_storage_format (csr|ell|hyb) - Sets the storage format of matrices (for `MatMult()` and factors in `MatSolve()`).
3050: . -mat_cusparse_mult_storage_format (csr|ell|hyb) - Sets the storage format of matrices (for `MatMult()`).
3051: - -mat_cusparse_use_cpu_solve (true|false) - Performs the `MatSolve()` on the CPU.
3053: Level: beginner
3055: Notes:
3056: These matrices can be in either CSR, ELL, or HYB format.
3058: All matrix calculations are performed on NVIDIA GPUs using the cuSPARSE library.
3060: Uses 32-bit integers internally. If PETSc is configured `--with-64-bit-indices`, the integer row and column indices are stored on the GPU with `int`. It is unclear what happens
3061: if some integer values passed in do not fit in `int`.
3063: .seealso: [](ch_matrices), `Mat`, `MatCreateSeqAIJCUSPARSE()`, `MatCUSPARSESetUseCPUSolve()`, `MATAIJCUSPARSE`, `MatCreateAIJCUSPARSE()`, `MatCUSPARSESetFormat()`, `MatCUSPARSEStorageFormat`, `MatCUSPARSEFormatOperation`
3064: M*/
3066: PETSC_INTERN PetscErrorCode MatSolverTypeRegister_CUSPARSE(void)
3067: {
3068: PetscFunctionBegin;
3069: PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_LU, MatGetFactor_seqaijcusparse_cusparse));
3070: PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_CHOLESKY, MatGetFactor_seqaijcusparse_cusparse));
3071: PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_ILU, MatGetFactor_seqaijcusparse_cusparse));
3072: PetscCall(MatSolverTypeRegister(MATSOLVERCUSPARSE, MATSEQAIJCUSPARSE, MAT_FACTOR_ICC, MatGetFactor_seqaijcusparse_cusparse));
3073: PetscFunctionReturn(PETSC_SUCCESS);
3074: }
3076: static PetscErrorCode MatSeqAIJCUSPARSE_Destroy(Mat mat)
3077: {
3078: Mat_SeqAIJCUSPARSE *cusp = static_cast<Mat_SeqAIJCUSPARSE *>(mat->spptr);
3080: PetscFunctionBegin;
3081: if (cusp) {
3082: PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusp->mat, cusp->format));
3083: PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusp->matTranspose, cusp->format));
3084: delete cusp->workVector;
3085: delete cusp->rowoffsets_gpu;
3086: delete cusp->csr2csc_i;
3087: delete cusp->coords;
3088: PetscCall(PetscFree(mat->spptr));
3089: }
3090: PetscFunctionReturn(PETSC_SUCCESS);
3091: }
3093: static PetscErrorCode CsrMatrix_Destroy(CsrMatrix **mat)
3094: {
3095: PetscFunctionBegin;
3096: if (*mat) {
3097: delete (*mat)->values;
3098: delete (*mat)->column_indices;
3099: delete (*mat)->row_offsets;
3100: delete *mat;
3101: *mat = 0;
3102: }
3103: PetscFunctionReturn(PETSC_SUCCESS);
3104: }
3106: static PetscErrorCode MatSeqAIJCUSPARSEMultStruct_Destroy(Mat_SeqAIJCUSPARSEMultStruct **matstruct, MatCUSPARSEStorageFormat format)
3107: {
3108: CsrMatrix *mat;
3110: PetscFunctionBegin;
3111: if (*matstruct) {
3112: if ((*matstruct)->mat) {
3113: if (format == MAT_CUSPARSE_ELL || format == MAT_CUSPARSE_HYB) {
3114: SETERRQ(PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_CUSPARSE_ELL and MAT_CUSPARSE_HYB are not supported since CUDA-11.0");
3115: } else {
3116: mat = (CsrMatrix *)(*matstruct)->mat;
3117: PetscCall(CsrMatrix_Destroy(&mat));
3118: }
3119: }
3120: if ((*matstruct)->descr) PetscCallCUSPARSE(cusparseDestroyMatDescr((*matstruct)->descr));
3121: delete (*matstruct)->cprowIndices;
3122: PetscCallCUDA(cudaFree((*matstruct)->alpha_one));
3123: PetscCallCUDA(cudaFree((*matstruct)->beta_zero));
3124: PetscCallCUDA(cudaFree((*matstruct)->beta_one));
3126: Mat_SeqAIJCUSPARSEMultStruct *mdata = *matstruct;
3127: if (mdata->matDescr) PetscCallCUSPARSE(cusparseDestroySpMat(mdata->matDescr));
3129: for (int i = 0; i < 3; i++) {
3130: if (mdata->cuSpMV[i].initialized) {
3131: PetscCallCUDA(cudaFree(mdata->cuSpMV[i].spmvBuffer));
3132: PetscCallCUSPARSE(cusparseDestroyDnVec(mdata->cuSpMV[i].vecXDescr));
3133: PetscCallCUSPARSE(cusparseDestroyDnVec(mdata->cuSpMV[i].vecYDescr));
3134: PetscCallCUSPARSE(cusparseDestroySpMat(mdata->cuSpMV[i].matDescr));
3135: }
3136: if (mdata->matDescr_SpMM[i]) PetscCallCUSPARSE(cusparseDestroySpMat(mdata->matDescr_SpMM[i]));
3137: }
3138: delete *matstruct;
3139: *matstruct = NULL;
3140: }
3141: PetscFunctionReturn(PETSC_SUCCESS);
3142: }
3144: PetscErrorCode MatSeqAIJCUSPARSETriFactors_Reset(Mat_SeqAIJCUSPARSETriFactors_p *trifactors)
3145: {
3146: Mat_SeqAIJCUSPARSETriFactors *fs = *trifactors;
3148: PetscFunctionBegin;
3149: if (fs) {
3150: delete fs->rpermIndices;
3151: delete fs->cpermIndices;
3152: fs->rpermIndices = NULL;
3153: fs->cpermIndices = NULL;
3154: fs->init_dev_prop = PETSC_FALSE;
3155: PetscCallCUDA(cudaFree(fs->csrRowPtr));
3156: PetscCallCUDA(cudaFree(fs->csrColIdx));
3157: PetscCallCUDA(cudaFree(fs->csrRowPtr32));
3158: PetscCallCUDA(cudaFree(fs->csrColIdx32));
3159: PetscCallCUDA(cudaFree(fs->csrVal));
3160: PetscCallCUDA(cudaFree(fs->diag));
3161: PetscCallCUDA(cudaFree(fs->X));
3162: PetscCallCUDA(cudaFree(fs->Y));
3163: // PetscCallCUDA(cudaFree(fs->factBuffer_M)); /* No needed since factBuffer_M shares with one of spsvBuffer_L/U */
3164: PetscCallCUDA(cudaFree(fs->spsvBuffer_L));
3165: PetscCallCUDA(cudaFree(fs->spsvBuffer_U));
3166: PetscCallCUDA(cudaFree(fs->spsvBuffer_Lt));
3167: PetscCallCUDA(cudaFree(fs->spsvBuffer_Ut));
3168: PetscCallCUSPARSE(cusparseDestroyMatDescr(fs->matDescr_M));
3169: if (fs->spMatDescr_L) PetscCallCUSPARSE(cusparseDestroySpMat(fs->spMatDescr_L));
3170: if (fs->spMatDescr_U) PetscCallCUSPARSE(cusparseDestroySpMat(fs->spMatDescr_U));
3171: PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_L));
3172: PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_Lt));
3173: PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_U));
3174: PetscCallCUSPARSE(cusparseSpSV_destroyDescr(fs->spsvDescr_Ut));
3175: if (fs->dnVecDescr_X) PetscCallCUSPARSE(cusparseDestroyDnVec(fs->dnVecDescr_X));
3176: if (fs->dnVecDescr_Y) PetscCallCUSPARSE(cusparseDestroyDnVec(fs->dnVecDescr_Y));
3177: PetscCallCUSPARSE(cusparseDestroyCsrilu02Info(fs->ilu0Info_M));
3178: PetscCallCUSPARSE(cusparseDestroyCsric02Info(fs->ic0Info_M));
3179: PetscCall(PetscFree(fs->csrRowPtr_h));
3180: PetscCall(PetscFree(fs->csrVal_h));
3181: PetscCall(PetscFree(fs->diag_h));
3182: fs->createdTransposeSpSVDescr = PETSC_FALSE;
3183: fs->updatedTransposeSpSVAnalysis = PETSC_FALSE;
3184: }
3185: PetscFunctionReturn(PETSC_SUCCESS);
3186: }
3188: static PetscErrorCode MatSeqAIJCUSPARSETriFactors_Destroy(Mat_SeqAIJCUSPARSETriFactors **trifactors)
3189: {
3190: PetscFunctionBegin;
3191: if (*trifactors) {
3192: PetscCall(MatSeqAIJCUSPARSETriFactors_Reset(trifactors));
3193: PetscCall(PetscFree(*trifactors));
3194: }
3195: PetscFunctionReturn(PETSC_SUCCESS);
3196: }
3198: static PetscErrorCode MatSeqAIJCUSPARSEInvalidateTranspose(Mat A, PetscBool destroy)
3199: {
3200: Mat_SeqAIJCUSPARSE *cusp = (Mat_SeqAIJCUSPARSE *)A->spptr;
3202: PetscFunctionBegin;
3203: PetscCheckTypeName(A, MATSEQAIJCUSPARSE);
3204: if (!cusp) PetscFunctionReturn(PETSC_SUCCESS);
3205: if (destroy) {
3206: PetscCall(MatSeqAIJCUSPARSEMultStruct_Destroy(&cusp->matTranspose, cusp->format));
3207: delete cusp->csr2csc_i;
3208: cusp->csr2csc_i = NULL;
3209: }
3210: A->transupdated = PETSC_FALSE;
3211: PetscFunctionReturn(PETSC_SUCCESS);
3212: }
3214: static PetscErrorCode MatSetPreallocationCOO_SeqAIJCUSPARSE(Mat mat, PetscCount coo_n, PetscInt coo_i[], PetscInt coo_j[])
3215: {
3216: PetscFunctionBegin;
3217: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::SetPreallocationCOO(mat, coo_n, coo_i, coo_j));
3218: PetscFunctionReturn(PETSC_SUCCESS);
3219: }
3221: static PetscErrorCode MatSetValuesCOO_SeqAIJCUSPARSE(Mat A, const PetscScalar v[], InsertMode imode)
3222: {
3223: PetscFunctionBegin;
3224: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::SetValuesCOO(A, v, imode));
3225: PetscFunctionReturn(PETSC_SUCCESS);
3226: }
3228: /*@
3229: MatSeqAIJCUSPARSEGetIJ - returns the device row storage `i` and `j` indices for `MATSEQAIJCUSPARSE` matrices.
3231: Not Collective
3233: Input Parameters:
3234: + A - the matrix
3235: - compressed - `PETSC_TRUE` or `PETSC_FALSE` indicating the matrix data structure should be always returned in compressed form
3237: Output Parameters:
3238: + i - the CSR row pointers
3239: - j - the CSR column indices
3241: Level: developer
3243: Note:
3244: When compressed is true, the CSR structure does not contain empty rows
3246: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSERestoreIJ()`, `MatSeqAIJCUSPARSEGetArrayRead()`
3247: @*/
3248: PetscErrorCode MatSeqAIJCUSPARSEGetIJ(Mat A, PetscBool compressed, const PetscInt *i[], const PetscInt *j[])
3249: {
3250: PetscFunctionBegin;
3251: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetIJ(A, compressed, i, j));
3252: PetscFunctionReturn(PETSC_SUCCESS);
3253: }
3255: /*@
3256: MatSeqAIJCUSPARSERestoreIJ - restore the device row storage `i` and `j` indices obtained with `MatSeqAIJCUSPARSEGetIJ()`
3258: Not Collective
3260: Input Parameters:
3261: + A - the matrix
3262: . compressed - `PETSC_TRUE` or `PETSC_FALSE` indicating the matrix data structure should be always returned in compressed form
3263: . i - the CSR row pointers
3264: - j - the CSR column indices
3266: Level: developer
3268: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetIJ()`
3269: @*/
3270: PetscErrorCode MatSeqAIJCUSPARSERestoreIJ(Mat A, PetscBool compressed, const PetscInt *i[], const PetscInt *j[])
3271: {
3272: PetscFunctionBegin;
3273: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::RestoreIJ(A, compressed, i, j));
3274: PetscFunctionReturn(PETSC_SUCCESS);
3275: }
3277: /*@
3278: MatSeqAIJCUSPARSEGetArrayRead - gives read-only access to the array where the device data for a `MATSEQAIJCUSPARSE` matrix nonzero entries are stored
3280: Not Collective
3282: Input Parameter:
3283: . A - a `MATSEQAIJCUSPARSE` matrix
3285: Output Parameter:
3286: . a - pointer to the device data
3288: Level: developer
3290: Note:
3291: Will trigger host-to-device copies if the most up-to-date matrix data is on the host
3293: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArray()`, `MatSeqAIJCUSPARSEGetArrayWrite()`, `MatSeqAIJCUSPARSERestoreArrayRead()`
3294: @*/
3295: PetscErrorCode MatSeqAIJCUSPARSEGetArrayRead(Mat A, const PetscScalar **a)
3296: {
3297: return MatSeqAIJCUSPARSE_CUPM_t::GetArrayRead(A, a);
3298: }
3300: /*@
3301: MatSeqAIJCUSPARSERestoreArrayRead - restore the read-only access array obtained from `MatSeqAIJCUSPARSEGetArrayRead()`
3303: Not Collective
3305: Input Parameters:
3306: + A - a `MATSEQAIJCUSPARSE` matrix
3307: - a - pointer to the device data
3309: Level: developer
3311: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArrayRead()`
3312: @*/
3313: PetscErrorCode MatSeqAIJCUSPARSERestoreArrayRead(Mat A, const PetscScalar **a)
3314: {
3315: return MatSeqAIJCUSPARSE_CUPM_t::RestoreArrayRead(A, a);
3316: }
3318: /*@
3319: MatSeqAIJCUSPARSEGetArray - gives read-write access to the array where the device data for a `MATSEQAIJCUSPARSE` matrix is stored
3321: Not Collective
3323: Input Parameter:
3324: . A - a `MATSEQAIJCUSPARSE` matrix
3326: Output Parameter:
3327: . a - pointer to the device data
3329: Level: developer
3331: Note:
3332: Will trigger host-to-device copies if the most up-to-date matrix data is on the host
3334: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArrayRead()`, `MatSeqAIJCUSPARSEGetArrayWrite()`, `MatSeqAIJCUSPARSERestoreArray()`
3335: @*/
3336: PetscErrorCode MatSeqAIJCUSPARSEGetArray(Mat A, PetscScalar **a)
3337: {
3338: return MatSeqAIJCUSPARSE_CUPM_t::GetArray(A, a);
3339: }
3340: /*@
3341: MatSeqAIJCUSPARSERestoreArray - restore the read-write access array obtained from `MatSeqAIJCUSPARSEGetArray()`
3343: Not Collective
3345: Input Parameters:
3346: + A - a `MATSEQAIJCUSPARSE` matrix
3347: - a - pointer to the device data
3349: Level: developer
3351: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArray()`
3352: @*/
3353: PetscErrorCode MatSeqAIJCUSPARSERestoreArray(Mat A, PetscScalar **a)
3354: {
3355: return MatSeqAIJCUSPARSE_CUPM_t::RestoreArray(A, a);
3356: }
3358: /*@
3359: MatSeqAIJCUSPARSEGetArrayWrite - gives write access to the array where the device data for a `MATSEQAIJCUSPARSE` matrix is stored
3361: Not Collective
3363: Input Parameter:
3364: . A - a `MATSEQAIJCUSPARSE` matrix
3366: Output Parameter:
3367: . a - pointer to the device data
3369: Level: developer
3371: Note:
3372: Does not trigger any host to device copies.
3374: It marks the data GPU valid so users must set all the values in `a` to ensure out-of-date data is not considered current
3376: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArray()`, `MatSeqAIJCUSPARSEGetArrayRead()`, `MatSeqAIJCUSPARSERestoreArrayWrite()`
3377: @*/
3378: PetscErrorCode MatSeqAIJCUSPARSEGetArrayWrite(Mat A, PetscScalar **a)
3379: {
3380: return MatSeqAIJCUSPARSE_CUPM_t::GetArrayWrite(A, a);
3381: }
3383: /*@
3384: MatSeqAIJCUSPARSERestoreArrayWrite - restore the write-only access array obtained from `MatSeqAIJCUSPARSEGetArrayWrite()`
3386: Not Collective
3388: Input Parameters:
3389: + A - a `MATSEQAIJCUSPARSE` matrix
3390: - a - pointer to the device data
3392: Level: developer
3394: .seealso: [](ch_matrices), `Mat`, `MatSeqAIJCUSPARSEGetArrayWrite()`
3395: @*/
3396: PetscErrorCode MatSeqAIJCUSPARSERestoreArrayWrite(Mat A, PetscScalar **a)
3397: {
3398: return MatSeqAIJCUSPARSE_CUPM_t::RestoreArrayWrite(A, a);
3399: }
3401: struct IJCompare4 {
3402: __host__ __device__ inline bool operator()(const thrust::tuple<PetscInt, PetscInt, PetscScalar, PetscInt> &t1, const thrust::tuple<PetscInt, PetscInt, PetscScalar, PetscInt> &t2)
3403: {
3404: if (thrust::get<0>(t1) < thrust::get<0>(t2)) return true;
3405: if (thrust::get<0>(t1) == thrust::get<0>(t2)) return thrust::get<1>(t1) < thrust::get<1>(t2);
3406: return false;
3407: }
3408: };
3410: struct Shift {
3411: PetscInt _shift;
3413: Shift(PetscInt shift) : _shift(shift) { }
3414: __host__ __device__ inline PetscInt operator()(const PetscInt &c) { return c + _shift; }
3415: };
3417: /* merges two SeqAIJCUSPARSE matrices A, B by concatenating their rows. [A';B']' operation in MATLAB notation */
3418: PetscErrorCode MatSeqAIJCUSPARSEMergeMats(Mat A, Mat B, MatReuse reuse, Mat *C)
3419: {
3420: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data, *b = (Mat_SeqAIJ *)B->data, *c;
3421: Mat_SeqAIJCUSPARSE *Acusp = (Mat_SeqAIJCUSPARSE *)A->spptr, *Bcusp = (Mat_SeqAIJCUSPARSE *)B->spptr, *Ccusp;
3422: Mat_SeqAIJCUSPARSEMultStruct *Cmat;
3423: CsrMatrix *Acsr, *Bcsr, *Ccsr;
3424: PetscInt Annz, Bnnz;
3425: PetscInt i, m, n, zero = 0;
3426: PetscDeviceContext dctx;
3427: cudaStream_t stream;
3429: PetscFunctionBegin;
3432: PetscAssertPointer(C, 4);
3433: PetscCheckTypeName(A, MATSEQAIJCUSPARSE);
3434: PetscCheckTypeName(B, MATSEQAIJCUSPARSE);
3435: PetscCheck(A->rmap->n == B->rmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Invalid number or rows %" PetscInt_FMT " != %" PetscInt_FMT, A->rmap->n, B->rmap->n);
3436: PetscCheck(reuse != MAT_INPLACE_MATRIX, PETSC_COMM_SELF, PETSC_ERR_SUP, "MAT_INPLACE_MATRIX not supported");
3437: PetscCheck(Acusp->format != MAT_CUSPARSE_ELL && Acusp->format != MAT_CUSPARSE_HYB, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
3438: PetscCheck(Bcusp->format != MAT_CUSPARSE_ELL && Bcusp->format != MAT_CUSPARSE_HYB, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
3439: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::GetHandles_(&dctx, &stream));
3440: if (reuse == MAT_INITIAL_MATRIX) {
3441: m = A->rmap->n;
3442: n = A->cmap->n + B->cmap->n;
3443: PetscCall(MatCreate(PETSC_COMM_SELF, C));
3444: PetscCall(MatSetSizes(*C, m, n, m, n));
3445: PetscCall(MatSetType(*C, MATSEQAIJCUSPARSE));
3446: c = (Mat_SeqAIJ *)(*C)->data;
3447: Ccusp = (Mat_SeqAIJCUSPARSE *)(*C)->spptr;
3448: Cmat = new Mat_SeqAIJCUSPARSEMultStruct;
3449: Ccsr = new CsrMatrix;
3450: Cmat->cprowIndices = NULL;
3451: c->compressedrow.use = PETSC_FALSE;
3452: c->compressedrow.nrows = 0;
3453: c->compressedrow.i = NULL;
3454: c->compressedrow.rindex = NULL;
3455: Ccusp->workVector = NULL;
3456: Ccusp->nrows = m;
3457: Ccusp->mat = Cmat;
3458: Ccusp->mat->mat = Ccsr;
3459: Ccsr->num_rows = m;
3460: Ccsr->num_cols = n;
3461: PetscCallCUSPARSE(cusparseCreateMatDescr(&Cmat->descr));
3462: PetscCallCUSPARSE(cusparseSetMatIndexBase(Cmat->descr, CUSPARSE_INDEX_BASE_ZERO));
3463: PetscCallCUSPARSE(cusparseSetMatType(Cmat->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
3464: PetscCallCUDA(cudaMalloc((void **)&Cmat->alpha_one, sizeof(PetscScalar)));
3465: PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_zero, sizeof(PetscScalar)));
3466: PetscCallCUDA(cudaMalloc((void **)&Cmat->beta_one, sizeof(PetscScalar)));
3467: PetscCallCUDA(cudaMemcpyAsync(Cmat->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3468: PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3469: PetscCallCUDA(cudaMemcpyAsync(Cmat->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3470: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
3471: PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
3472: PetscCheck(Acusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3473: PetscCheck(Bcusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3475: Acsr = (CsrMatrix *)Acusp->mat->mat;
3476: Bcsr = (CsrMatrix *)Bcusp->mat->mat;
3477: Annz = (PetscInt)Acsr->column_indices->size();
3478: Bnnz = (PetscInt)Bcsr->column_indices->size();
3479: c->nz = Annz + Bnnz;
3480: Ccsr->row_offsets = new THRUSTINTARRAY(m + 1);
3481: Ccsr->column_indices = new THRUSTINTARRAY(c->nz);
3482: Ccsr->values = new THRUSTARRAY(c->nz);
3483: Ccsr->num_entries = c->nz;
3484: Ccusp->coords = new THRUSTINTARRAY(c->nz);
3485: if (c->nz) {
3486: auto Acoo = new THRUSTINTARRAY(Annz); // initialized with zeros
3487: auto Bcoo = new THRUSTINTARRAY(Bnnz);
3488: auto Ccoo = new THRUSTINTARRAY(c->nz);
3489: THRUSTINTARRAY *Aroff, *Broff;
3491: if (a->compressedrow.use) { /* need full row offset */
3492: if (!Acusp->rowoffsets_gpu) {
3493: Acusp->rowoffsets_gpu = new THRUSTINTARRAY(A->rmap->n + 1);
3494: PetscCallCUDA(cudaMemcpyAsync(Acusp->rowoffsets_gpu->data().get(), a->i, (A->rmap->n + 1) * sizeof(*a->i), cudaMemcpyHostToDevice, stream));
3495: PetscCall(PetscLogCpuToGpu((A->rmap->n + 1) * sizeof(PetscInt)));
3496: }
3497: Aroff = Acusp->rowoffsets_gpu;
3498: } else Aroff = Acsr->row_offsets;
3499: if (b->compressedrow.use) { /* need full row offset */
3500: if (!Bcusp->rowoffsets_gpu) {
3501: Bcusp->rowoffsets_gpu = new THRUSTINTARRAY(B->rmap->n + 1);
3502: PetscCallCUDA(cudaMemcpyAsync(Bcusp->rowoffsets_gpu->data().get(), b->i, (B->rmap->n + 1) * sizeof(*b->i), cudaMemcpyHostToDevice, stream));
3503: PetscCall(PetscLogCpuToGpu((B->rmap->n + 1) * sizeof(PetscInt)));
3504: }
3505: Broff = Bcusp->rowoffsets_gpu;
3506: } else Broff = Bcsr->row_offsets;
3507: PetscCall(PetscLogGpuTimeBegin());
3508: // Implement cusparseXcsr2coo() with Thrust, as the former doesn't support 64-bit indices.
3509: PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(m), Csr2coo(Aroff->data().get(), Acoo->data().get())));
3510: PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(m), Csr2coo(Broff->data().get(), Bcoo->data().get())));
3512: /* Issues when using bool with large matrices on SUMMIT 10.2.89 */
3513: #if CCCL_VERSION >= 3004000
3514: auto Aperm = cuda::make_constant_iterator(1);
3515: auto Bperm = cuda::make_constant_iterator(0);
3516: #else
3517: auto Aperm = thrust::make_constant_iterator(1);
3518: auto Bperm = thrust::make_constant_iterator(0);
3519: #endif
3520: auto Bcib = thrust::make_transform_iterator(Bcsr->column_indices->begin(), Shift(A->cmap->n));
3521: auto Bcie = thrust::make_transform_iterator(Bcsr->column_indices->end(), Shift(A->cmap->n));
3522: auto wPerm = new THRUSTINTARRAY(Annz + Bnnz);
3523: auto Azb = thrust::make_zip_iterator(thrust::make_tuple(Acoo->begin(), Acsr->column_indices->begin(), Acsr->values->begin(), Aperm));
3524: auto Aze = thrust::make_zip_iterator(thrust::make_tuple(Acoo->end(), Acsr->column_indices->end(), Acsr->values->end(), Aperm));
3525: auto Bzb = thrust::make_zip_iterator(thrust::make_tuple(Bcoo->begin(), Bcib, Bcsr->values->begin(), Bperm)); // Use B column indices shifted by A->cmap->n
3526: auto Bze = thrust::make_zip_iterator(thrust::make_tuple(Bcoo->end(), Bcie, Bcsr->values->end(), Bperm));
3527: auto Czb = thrust::make_zip_iterator(thrust::make_tuple(Ccoo->begin(), Ccsr->column_indices->begin(), Ccsr->values->begin(), wPerm->begin()));
3528: auto p1 = Ccusp->coords->begin();
3529: auto p2 = Ccusp->coords->begin();
3530: #if CCCL_VERSION >= 3001000
3531: cuda::std::advance(p2, Annz);
3532: #else
3533: thrust::advance(p2, Annz);
3534: #endif
3535: PetscCallThrust(thrust::merge(thrust::cuda::par.on(stream), Azb, Aze, Bzb, Bze, Czb, IJCompare4())); // put nonzeros in A and B to C in sorted order (by row and then by column)
3536: auto cci = thrust::make_counting_iterator(zero);
3537: auto cce = thrust::make_counting_iterator(c->nz);
3538: #if THRUST_VERSION < 200800
3539: auto pred = thrust::identity<int>();
3540: #else
3541: auto pred = cuda::std::identity();
3542: #endif
3543: PetscCallThrust(thrust::copy_if(thrust::cuda::par.on(stream), cci, cce, wPerm->begin(), p1, pred));
3544: PetscCallThrust(thrust::remove_copy_if(thrust::cuda::par.on(stream), cci, cce, wPerm->begin(), p2, pred));
3545: // Implement a simplified cusparseXcoo2csr() with Thrust (assuming the row indices are already sorted), as the former doesn't support 64-bit indices.
3546: PetscCallThrust(thrust::lower_bound(thrust::cuda::par.on(stream), Ccoo->begin(), Ccoo->end(), thrust::counting_iterator<PetscInt>(0), thrust::counting_iterator<PetscInt>(m + 1), Ccsr->row_offsets->begin()));
3547: PetscCall(PetscLogGpuTimeEnd());
3548: delete wPerm;
3549: delete Acoo;
3550: delete Bcoo;
3551: delete Ccoo;
3552: PetscCallCUSPARSE(cusparseCreateCsr(&Cmat->matDescr, Ccsr->num_rows, Ccsr->num_cols, Ccsr->num_entries, Ccsr->row_offsets->data().get(), Ccsr->column_indices->data().get(), Ccsr->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
3553: if (A->form_explicit_transpose && B->form_explicit_transpose) { /* if A and B have the transpose, generate C transpose too */
3554: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(A));
3555: PetscCall(MatSeqAIJCUSPARSEFormExplicitTranspose(B));
3556: PetscBool AT = Acusp->matTranspose ? PETSC_TRUE : PETSC_FALSE, BT = Bcusp->matTranspose ? PETSC_TRUE : PETSC_FALSE;
3557: Mat_SeqAIJCUSPARSEMultStruct *CmatT = new Mat_SeqAIJCUSPARSEMultStruct;
3558: CsrMatrix *CcsrT = new CsrMatrix;
3559: CsrMatrix *AcsrT = AT ? (CsrMatrix *)Acusp->matTranspose->mat : NULL;
3560: CsrMatrix *BcsrT = BT ? (CsrMatrix *)Bcusp->matTranspose->mat : NULL;
3562: (*C)->form_explicit_transpose = PETSC_TRUE;
3563: (*C)->transupdated = PETSC_TRUE;
3564: Ccusp->rowoffsets_gpu = NULL;
3565: CmatT->cprowIndices = NULL;
3566: CmatT->mat = CcsrT;
3567: CcsrT->num_rows = n;
3568: CcsrT->num_cols = m;
3569: CcsrT->num_entries = c->nz;
3571: CcsrT->row_offsets = new THRUSTINTARRAY(n + 1);
3572: CcsrT->column_indices = new THRUSTINTARRAY(c->nz);
3573: CcsrT->values = new THRUSTARRAY(c->nz);
3575: PetscCall(PetscLogGpuTimeBegin());
3576: auto rT = CcsrT->row_offsets->begin();
3577: if (AT) {
3578: PetscCallThrust(rT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->row_offsets->begin(), AcsrT->row_offsets->end(), rT));
3579: #if CCCL_VERSION >= 3001000
3580: cuda::std::advance(rT, -1);
3581: #else
3582: thrust::advance(rT, -1);
3583: #endif
3584: }
3585: if (BT) {
3586: auto titb = thrust::make_transform_iterator(BcsrT->row_offsets->begin(), Shift(a->nz));
3587: auto tite = thrust::make_transform_iterator(BcsrT->row_offsets->end(), Shift(a->nz));
3588: PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), titb, tite, rT));
3589: }
3590: auto cT = CcsrT->column_indices->begin();
3591: if (AT) PetscCallThrust(cT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->column_indices->begin(), AcsrT->column_indices->end(), cT));
3592: if (BT) PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), BcsrT->column_indices->begin(), BcsrT->column_indices->end(), cT));
3593: auto vT = CcsrT->values->begin();
3594: if (AT) PetscCallThrust(vT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->values->begin(), AcsrT->values->end(), vT));
3595: if (BT) PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), BcsrT->values->begin(), BcsrT->values->end(), vT));
3596: PetscCall(PetscLogGpuTimeEnd());
3598: PetscCallCUSPARSE(cusparseCreateMatDescr(&CmatT->descr));
3599: PetscCallCUSPARSE(cusparseSetMatIndexBase(CmatT->descr, CUSPARSE_INDEX_BASE_ZERO));
3600: PetscCallCUSPARSE(cusparseSetMatType(CmatT->descr, CUSPARSE_MATRIX_TYPE_GENERAL));
3601: PetscCallCUDA(cudaMalloc((void **)&CmatT->alpha_one, sizeof(PetscScalar)));
3602: PetscCallCUDA(cudaMalloc((void **)&CmatT->beta_zero, sizeof(PetscScalar)));
3603: PetscCallCUDA(cudaMalloc((void **)&CmatT->beta_one, sizeof(PetscScalar)));
3604: PetscCallCUDA(cudaMemcpyAsync(CmatT->alpha_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3605: PetscCallCUDA(cudaMemcpyAsync(CmatT->beta_zero, &PETSC_CUSPARSE_ZERO, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3606: PetscCallCUDA(cudaMemcpyAsync(CmatT->beta_one, &PETSC_CUSPARSE_ONE, sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
3607: PetscCallCUSPARSE(cusparseCreateCsr(&CmatT->matDescr, CcsrT->num_rows, CcsrT->num_cols, CcsrT->num_entries, CcsrT->row_offsets->data().get(), CcsrT->column_indices->data().get(), CcsrT->values->data().get(), csrRowOffsetsType, csrColIndType, CUSPARSE_INDEX_BASE_ZERO, cusparse_scalartype));
3608: Ccusp->matTranspose = CmatT;
3609: }
3610: }
3612: c->free_a = PETSC_TRUE;
3613: PetscCall(PetscShmgetAllocateArray(c->nz, sizeof(PetscInt), (void **)&c->j));
3614: PetscCall(PetscShmgetAllocateArray(m + 1, sizeof(PetscInt), (void **)&c->i));
3615: c->free_ij = PETSC_TRUE;
3616: PetscCallCUDA(cudaMemcpyAsync(c->i, Ccsr->row_offsets->data().get(), Ccsr->row_offsets->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
3617: PetscCallCUDA(cudaMemcpyAsync(c->j, Ccsr->column_indices->data().get(), Ccsr->column_indices->size() * sizeof(PetscInt), cudaMemcpyDeviceToHost, stream));
3618: PetscCall(PetscDeviceContextSynchronize(dctx));
3619: PetscCall(PetscLogGpuToCpu((Ccsr->column_indices->size() + Ccsr->row_offsets->size()) * sizeof(PetscInt)));
3620: PetscCall(PetscMalloc1(m, &c->ilen));
3621: PetscCall(PetscMalloc1(m, &c->imax));
3622: c->maxnz = c->nz;
3623: c->nonzerorowcnt = 0;
3624: c->rmax = 0;
3625: for (i = 0; i < m; i++) {
3626: const PetscInt nn = c->i[i + 1] - c->i[i];
3627: c->ilen[i] = c->imax[i] = nn;
3628: c->nonzerorowcnt += (PetscInt)!!nn;
3629: c->rmax = PetscMax(c->rmax, nn);
3630: }
3631: PetscCall(PetscMalloc1(c->nz, &c->a));
3632: (*C)->nonzerostate++;
3633: PetscCall(PetscLayoutSetUp((*C)->rmap));
3634: PetscCall(PetscLayoutSetUp((*C)->cmap));
3635: Ccusp->nonzerostate = (*C)->nonzerostate;
3636: (*C)->preallocated = PETSC_TRUE;
3637: } else {
3638: PetscCheck((*C)->rmap->n == B->rmap->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Invalid number or rows %" PetscInt_FMT " != %" PetscInt_FMT, (*C)->rmap->n, B->rmap->n);
3639: c = (Mat_SeqAIJ *)(*C)->data;
3640: if (c->nz) {
3641: Ccusp = (Mat_SeqAIJCUSPARSE *)(*C)->spptr;
3642: PetscCheck(Ccusp->coords, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing coords");
3643: PetscCheck(Ccusp->format != MAT_CUSPARSE_ELL && Ccusp->format != MAT_CUSPARSE_HYB, PETSC_COMM_SELF, PETSC_ERR_SUP, "Not implemented");
3644: PetscCheck(Ccusp->nonzerostate == (*C)->nonzerostate, PETSC_COMM_SELF, PETSC_ERR_COR, "Wrong nonzerostate");
3645: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
3646: PetscCall(MatSeqAIJCUSPARSECopyToGPU(B));
3647: PetscCheck(Acusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3648: PetscCheck(Bcusp->mat, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing Mat_SeqAIJCUSPARSEMultStruct");
3649: Acsr = (CsrMatrix *)Acusp->mat->mat;
3650: Bcsr = (CsrMatrix *)Bcusp->mat->mat;
3651: Ccsr = (CsrMatrix *)Ccusp->mat->mat;
3652: PetscCheck(Acsr->num_entries == (PetscInt)Acsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "A nnz %" PetscInt_FMT " != %" PetscInt_FMT, Acsr->num_entries, (PetscInt)Acsr->values->size());
3653: PetscCheck(Bcsr->num_entries == (PetscInt)Bcsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "B nnz %" PetscInt_FMT " != %" PetscInt_FMT, Bcsr->num_entries, (PetscInt)Bcsr->values->size());
3654: PetscCheck(Ccsr->num_entries == (PetscInt)Ccsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "C nnz %" PetscInt_FMT " != %" PetscInt_FMT, Ccsr->num_entries, (PetscInt)Ccsr->values->size());
3655: PetscCheck(Ccsr->num_entries == Acsr->num_entries + Bcsr->num_entries, PETSC_COMM_SELF, PETSC_ERR_COR, "C nnz %" PetscInt_FMT " != %" PetscInt_FMT " + %" PetscInt_FMT, Ccsr->num_entries, Acsr->num_entries, Bcsr->num_entries);
3656: PetscCheck(Ccusp->coords->size() == Ccsr->values->size(), PETSC_COMM_SELF, PETSC_ERR_COR, "permSize %" PetscInt_FMT " != %" PetscInt_FMT, (PetscInt)Ccusp->coords->size(), (PetscInt)Ccsr->values->size());
3657: auto pmid = Ccusp->coords->begin();
3658: #if CCCL_VERSION >= 3001000
3659: cuda::std::advance(pmid, Acsr->num_entries);
3660: #else
3661: thrust::advance(pmid, Acsr->num_entries);
3662: #endif
3663: PetscCall(PetscLogGpuTimeBegin());
3664: auto zibait = thrust::make_zip_iterator(thrust::make_tuple(Acsr->values->begin(), thrust::make_permutation_iterator(Ccsr->values->begin(), Ccusp->coords->begin())));
3665: auto zieait = thrust::make_zip_iterator(thrust::make_tuple(Acsr->values->end(), thrust::make_permutation_iterator(Ccsr->values->begin(), pmid)));
3666: PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), zibait, zieait, VecCUDAEquals()));
3667: auto zibbit = thrust::make_zip_iterator(thrust::make_tuple(Bcsr->values->begin(), thrust::make_permutation_iterator(Ccsr->values->begin(), pmid)));
3668: auto ziebit = thrust::make_zip_iterator(thrust::make_tuple(Bcsr->values->end(), thrust::make_permutation_iterator(Ccsr->values->begin(), Ccusp->coords->end())));
3669: PetscCallThrust(thrust::for_each(thrust::cuda::par.on(stream), zibbit, ziebit, VecCUDAEquals()));
3670: PetscCall(MatSeqAIJCUSPARSEInvalidateTranspose(*C, PETSC_FALSE));
3671: if (A->form_explicit_transpose && B->form_explicit_transpose && (*C)->form_explicit_transpose) {
3672: PetscCheck(Ccusp->matTranspose, PETSC_COMM_SELF, PETSC_ERR_COR, "Missing transpose Mat_SeqAIJCUSPARSEMultStruct");
3673: PetscBool AT = Acusp->matTranspose ? PETSC_TRUE : PETSC_FALSE, BT = Bcusp->matTranspose ? PETSC_TRUE : PETSC_FALSE;
3674: CsrMatrix *AcsrT = AT ? (CsrMatrix *)Acusp->matTranspose->mat : NULL;
3675: CsrMatrix *BcsrT = BT ? (CsrMatrix *)Bcusp->matTranspose->mat : NULL;
3676: CsrMatrix *CcsrT = (CsrMatrix *)Ccusp->matTranspose->mat;
3677: auto vT = CcsrT->values->begin();
3678: if (AT) PetscCallThrust(vT = thrust::copy(thrust::cuda::par.on(stream), AcsrT->values->begin(), AcsrT->values->end(), vT));
3679: if (BT) PetscCallThrust(thrust::copy(thrust::cuda::par.on(stream), BcsrT->values->begin(), BcsrT->values->end(), vT));
3680: (*C)->transupdated = PETSC_TRUE;
3681: }
3682: PetscCall(PetscLogGpuTimeEnd());
3683: }
3684: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
3685: }
3686: PetscCall(PetscObjectStateIncrease((PetscObject)*C));
3687: (*C)->assembled = PETSC_TRUE;
3688: (*C)->was_assembled = PETSC_FALSE;
3689: (*C)->offloadmask = PETSC_OFFLOAD_GPU;
3690: PetscFunctionReturn(PETSC_SUCCESS);
3691: }
3693: static PetscErrorCode MatSeqAIJCopySubArray_SeqAIJCUSPARSE(Mat A, PetscInt n, const PetscInt idx[], PetscScalar v[])
3694: {
3695: PetscFunctionBegin;
3696: PetscCall(MatSeqAIJCUSPARSE_CUPM_t::CopySubArray(A, n, idx, v));
3697: PetscFunctionReturn(PETSC_SUCCESS);
3698: }