Actual source code: cudss.cu

  1: /*
  2:      This file implements a subclass of the SeqAIJ matrix class that uses
  3:      the NVIDIA cuDSS GPU-accelerated sparse direct solver library.
  4: */

  6: #include <../src/mat/impls/aij/seq/aij.h>
  7: #include <../src/mat/impls/aij/seq/seqcusparse/cusparsematimpl.h>
  8: #include <petsc/private/deviceimpl.h>
  9: #include <petsc/private/cupminterface.hpp>
 10: #include <cudss.h>
 11: #include <nvrtc.h>
 12: #include <climits>

 14: // Internal data structure for the cuDSS factored matrix
 15: typedef struct {
 16:   cudssHandle_t handle;
 17:   cudssConfig_t config;
 18:   cudssData_t   data;
 19:   cudssMatrix_t cudss_A; // sparse matrix descriptor
 20:   cudssMatrix_t cudss_b; // dense RHS descriptor
 21:   cudssMatrix_t cudss_x; // dense solution descriptor

 23:   // Device CSR arrays (owned by us when input is MATSEQAIJ)
 24:   PetscInt    *d_row_offsets;
 25:   PetscInt    *d_col_indices;
 26:   int         *d_row_i32; // narrowed 32-bit copy when PetscInt is 64-bit
 27:   int         *d_col_i32;
 28:   PetscScalar *d_values;

 30:   // User-supplied permutation (host array of 32-bit ints, owned by us)
 31:   int      *h_user_perm;
 32:   PetscBool userPermSet;

 34:   PetscInt  n; // matrix dimension
 35:   PetscInt  nnz;
 36:   PetscBool ownDeviceCSR;

 38:   cudssReorderingAlg_t reorderAlg;
 39:   cudssPivotType_t     pivotType;
 40:   double               pivotThreshold; // cuDSS expects double for CUDSS_CONFIG_PIVOT_THRESHOLD
 41:   double               pivotEpsilon;   // cuDSS expects double for CUDSS_CONFIG_PIVOT_EPSILON
 42:   int                  useMatching;    // cuDSS expects int, translated to CUDSS_CONFIG_MATCHING_ALG
 43:   int                  irNSteps;       // cuDSS expects int for CUDSS_CONFIG_IR_N_STEPS
 44:   int                  deterministic;  // cuDSS expects int for CUDSS_CONFIG_DETERMINISTIC_MODE
 45:   PetscBool            mempool;        // register a stream-ordered cudaMemPool_t as cuDSS's device workspace allocator
 46: } Mat_cuDSS;

 48: // Map PetscScalar to the cuDSS data type
 49: #if PetscDefined(USE_COMPLEX)
 50:   #if PetscDefined(USE_REAL_SINGLE)
 51:     #define CUDSS_SCALAR_TYPE CUDSS_C_32F
 52:   #else
 53:     #define CUDSS_SCALAR_TYPE CUDSS_C_64F
 54:   #endif
 55: #else
 56:   #if PetscDefined(USE_REAL_SINGLE)
 57:     #define CUDSS_SCALAR_TYPE CUDSS_R_32F
 58:   #else
 59:     #define CUDSS_SCALAR_TYPE CUDSS_R_64F
 60:   #endif
 61: #endif

 63: #define PetscCallCUDSS(...) \
 64:   do { \
 65:     cudssStatus_t cudss_status_ = __VA_ARGS__; \
 66:     PetscCheck(cudss_status_ == CUDSS_STATUS_SUCCESS, PETSC_COMM_SELF, PETSC_ERR_LIB, "cuDSS error %d in %s", (int)cudss_status_, PetscStringize(__VA_ARGS__)); \
 67:   } while (0)

 69: static PetscErrorCode MatCUDSSGetContext(PetscDeviceContext *dctx, cudaStream_t *stream)
 70: {
 71:   cudaStream_t *handle;

 73:   PetscFunctionBegin;
 74:   PetscCall(PetscDeviceContextGetCurrentContextAssertType_Internal(dctx, PETSC_DEVICE_CUDA));
 75:   PetscCall(PetscDeviceContextGetStreamHandle(*dctx, (void **)&handle));
 76:   *stream = *handle;
 77:   PetscFunctionReturn(PETSC_SUCCESS);
 78: }

 80: static int MatCUDSSPoolAlloc(void *ctx, void **ptr, size_t size, cudaStream_t stream)
 81: {
 82:   cudaError_t cerr;

 84:   (void)ctx;
 85:   *ptr = NULL;
 86:   if (size == 0) return 0;
 87:   cerr = cudaMallocAsync(ptr, size, stream);
 88:   return (cerr == cudaSuccess) ? 0 : (int)cerr;
 89: }

 91: static int MatCUDSSPoolFree(void *ctx, void *ptr, size_t size, cudaStream_t stream)
 92: {
 93:   cudaError_t cerr;

 95:   (void)ctx;
 96:   (void)size;
 97:   if (ptr == NULL) return 0;
 98:   cerr = cudaFreeAsync(ptr, stream);
 99:   return (cerr == cudaSuccess) ? 0 : (int)cerr;
100: }

102: static PetscErrorCode MatCUDSSSetMemHandler(cudssHandle_t handle, PetscBool use_mempool)
103: {
104:   PetscFunctionBegin;
105:   if (use_mempool == PETSC_TRUE) {
106:     cudssDeviceMemHandler_t handler;

108:     PetscCall(PetscMemzero(&handler, sizeof(handler)));
109:     handler.ctx          = NULL;
110:     handler.device_alloc = MatCUDSSPoolAlloc;
111:     handler.device_free  = MatCUDSSPoolFree;
112:     PetscCall(PetscStrncpy(handler.name, "petsc_cudss_pool", sizeof(handler.name)));
113:     PetscCallCUDSS(cudssSetDeviceMemHandler(handle, &handler));
114:   } else PetscCallCUDSS(cudssSetDeviceMemHandler(handle, NULL));
115:   PetscFunctionReturn(PETSC_SUCCESS);
116: }

118: /* These arrays are indexed by the cuDSS enum values directly.
119:    Mapping (cuDSS 0.8.0):
120:      cudssReorderingAlg_t: CUDSS_REORDERING_ALG_DEFAULT==0, ..._BTF_COLAMD==1, ..._COLAMD==2,
121:                            ..._AMD==3, ..._NESTED_DISSECTION==4, ..._NONE==5
122:      cudssPivotType_t: CUDSS_PIVOT_AUTO==0, ..._NONE==1, ..._GLOBAL_COL==2, ..._GLOBAL_ROW==3,
123:                        ..._DIAGONAL==4, ..._LOCAL_BLOCK==5, ..._BUNCH_KAUFMAN==6
124:    If NVIDIA renumbers these enumerators in a future release, the option parsing
125:    and view output will silently mismap. Update both arrays and this comment. */
126: static const char *const MatCUDSSReorderAlgs[] = {"default", "btf_colamd", "colamd", "amd", "nested_dissection", "none"};
127: static const char *const MatCUDSSPivotTypes[]  = {"auto", "none", "col", "row", "diagonal", "local_block", "bunch_kaufman"};

129: // Forward declarations
130: static PetscErrorCode MatFactorNumeric_cuDSS(Mat, Mat, const MatFactorInfo *);
131: static PetscErrorCode MatSetFromOptions_cuDSS(Mat);
132: static PetscErrorCode MatFactorSymbolic_cuDSS(Mat, Mat, IS, cudssMatrixType_t, cudssMatrixViewType_t);
133: static PetscErrorCode MatSetPermutation_cuDSS(Mat, IS);

135: static PetscErrorCode MatView_Info_cuDSS(Mat A, PetscViewer viewer)
136: {
137:   Mat_cuDSS  *lu          = (Mat_cuDSS *)A->data;
138:   const char *reorderName = ((int)lu->reorderAlg >= 0 && (size_t)lu->reorderAlg < PETSC_STATIC_ARRAY_LENGTH(MatCUDSSReorderAlgs)) ? MatCUDSSReorderAlgs[lu->reorderAlg] : "unknown";
139:   const char *pivotName   = ((int)lu->pivotType >= 0 && (size_t)lu->pivotType < PETSC_STATIC_ARRAY_LENGTH(MatCUDSSPivotTypes)) ? MatCUDSSPivotTypes[lu->pivotType] : "unknown";

141:   PetscFunctionBegin;
142:   PetscCall(PetscViewerASCIIPrintf(viewer, "cuDSS run parameters:\n"));
143:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Reorder algorithm: %s\n", reorderName));
144:   PetscCall(PetscViewerASCIIPrintf(viewer, "  User permutation: %s\n", lu->userPermSet == PETSC_TRUE ? "yes" : "no"));
145:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Pivot type: %s\n", pivotName));
146:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Pivot threshold: %g\n", lu->pivotThreshold));
147:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Pivot epsilon: %g\n", lu->pivotEpsilon));
148:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Use matching: %s\n", lu->useMatching ? "true" : "false"));
149:   PetscCall(PetscViewerASCIIPrintf(viewer, "  IR steps: %d\n", lu->irNSteps));
150:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Deterministic mode: %s\n", lu->deterministic ? "true" : "false"));
151:   PetscCall(PetscViewerASCIIPrintf(viewer, "  Device workspace mempool: %s\n", lu->mempool == PETSC_TRUE ? "true" : "false"));
152:   PetscFunctionReturn(PETSC_SUCCESS);
153: }

155: static PetscErrorCode MatView_cuDSS(Mat A, PetscViewer viewer)
156: {
157:   PetscBool iascii;

159:   PetscFunctionBegin;
160:   PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii));
161:   if (iascii == PETSC_TRUE) {
162:     PetscViewerFormat format;

164:     PetscCall(PetscViewerGetFormat(viewer, &format));
165:     if (format == PETSC_VIEWER_ASCII_INFO) PetscCall(MatView_Info_cuDSS(A, viewer));
166:   }
167:   PetscFunctionReturn(PETSC_SUCCESS);
168: }

170: static PetscErrorCode MatReset_cuDSS(Mat_cuDSS *lu)
171: {
172:   PetscFunctionBegin;
173:   if (lu->cudss_x != NULL) {
174:     PetscCallCUDSS(cudssMatrixDestroy(lu->cudss_x));
175:     lu->cudss_x = NULL;
176:   }
177:   if (lu->cudss_b != NULL) {
178:     PetscCallCUDSS(cudssMatrixDestroy(lu->cudss_b));
179:     lu->cudss_b = NULL;
180:   }
181:   if (lu->cudss_A != NULL) {
182:     PetscCallCUDSS(cudssMatrixDestroy(lu->cudss_A));
183:     lu->cudss_A = NULL;
184:   }
185:   if (lu->data != NULL) {
186:     PetscCallCUDSS(cudssDataDestroy(lu->handle, lu->data));
187:     lu->data = NULL;
188:   }
189:   if (lu->ownDeviceCSR == PETSC_TRUE) {
190:     PetscCallCUDA(cudaFree(lu->d_row_offsets));
191:     lu->d_row_offsets = NULL;
192:     PetscCallCUDA(cudaFree(lu->d_col_indices));
193:     lu->d_col_indices = NULL;
194:     PetscCallCUDA(cudaFree(lu->d_values));
195:     lu->d_values     = NULL;
196:     lu->ownDeviceCSR = PETSC_FALSE;
197:   }
198:   PetscCallCUDA(cudaFree(lu->d_row_i32));
199:   lu->d_row_i32 = NULL;
200:   PetscCallCUDA(cudaFree(lu->d_col_i32));
201:   lu->d_col_i32 = NULL;
202:   PetscCall(PetscFree(lu->h_user_perm));
203:   lu->userPermSet = PETSC_FALSE;
204:   PetscFunctionReturn(PETSC_SUCCESS);
205: }

207: static PetscErrorCode MatDestroy_cuDSS(Mat A)
208: {
209:   Mat_cuDSS *lu = (Mat_cuDSS *)A->data;

211:   PetscFunctionBegin;
212:   PetscCall(MatReset_cuDSS(lu));
213:   if (lu->config != NULL) PetscCallCUDSS(cudssConfigDestroy(lu->config));
214:   if (lu->handle != NULL) PetscCallCUDSS(cudssDestroy(lu->handle));
215:   PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatFactorGetSolverType_C", NULL));
216:   PetscCall(PetscFree(A->data));
217:   PetscFunctionReturn(PETSC_SUCCESS);
218: }

220: static PetscErrorCode MatSetFromOptions_cuDSS(Mat F)
221: {
222:   Mat_cuDSS *lu         = (Mat_cuDSS *)F->data;
223:   PetscInt   reorderAlg = (PetscInt)lu->reorderAlg, pivotType = (PetscInt)lu->pivotType, irNSteps = (PetscInt)lu->irNSteps;
224:   PetscReal  pivotThreshold = (PetscReal)lu->pivotThreshold, pivotEpsilon = (PetscReal)lu->pivotEpsilon;
225:   PetscBool  useMatching = (PetscBool)lu->useMatching, deterministic = (PetscBool)lu->deterministic;

227:   PetscFunctionBegin;
228:   PetscOptionsBegin(PetscObjectComm((PetscObject)F), ((PetscObject)F)->prefix, "cuDSS Options", "Mat");
229:   PetscCall(PetscOptionsEList("-mat_cudss_reorder_alg", "Reordering algorithm", "None", MatCUDSSReorderAlgs, PETSC_STATIC_ARRAY_LENGTH(MatCUDSSReorderAlgs), MatCUDSSReorderAlgs[reorderAlg], &reorderAlg, NULL));
230:   lu->reorderAlg = (cudssReorderingAlg_t)reorderAlg;
231:   PetscCall(PetscOptionsEList("-mat_cudss_pivot_type", "Pivot type", "None", MatCUDSSPivotTypes, PETSC_STATIC_ARRAY_LENGTH(MatCUDSSPivotTypes), MatCUDSSPivotTypes[pivotType], &pivotType, NULL));
232:   lu->pivotType = (cudssPivotType_t)pivotType;
233:   PetscCall(PetscOptionsReal("-mat_cudss_pivot_threshold", "Pivot threshold", "None", pivotThreshold, &pivotThreshold, NULL));
234:   lu->pivotThreshold = (double)pivotThreshold;
235:   PetscCall(PetscOptionsReal("-mat_cudss_pivot_epsilon", "Pivot epsilon", "None", pivotEpsilon, &pivotEpsilon, NULL));
236:   lu->pivotEpsilon = (double)pivotEpsilon;
237:   PetscCall(PetscOptionsBool("-mat_cudss_use_matching", "Enable matching", "None", useMatching, &useMatching, NULL));
238:   lu->useMatching = (int)useMatching;
239:   PetscCall(PetscOptionsInt("-mat_cudss_ir_n_steps", "Number of iterative refinement steps", "None", irNSteps, &irNSteps, NULL));
240:   PetscCall(PetscCIntCast(irNSteps, &lu->irNSteps));
241:   PetscCall(PetscOptionsBool("-mat_cudss_deterministic", "Enable bitwise-reproducible factorization and solve", "None", deterministic, &deterministic, NULL));
242:   lu->deterministic = (int)deterministic;
243:   PetscCall(PetscOptionsBool("-mat_cudss_mempool", "Use a stream-ordered cudaMemPool_t for cuDSS device workspace allocations", "None", lu->mempool, &lu->mempool, NULL));
244:   PetscOptionsEnd();
245:   PetscFunctionReturn(PETSC_SUCCESS);
246: }

248: static PetscErrorCode MatApplyConfig_cuDSS(Mat_cuDSS *lu)
249: {
250:   cudssMatchingAlg_t matchingAlg = lu->useMatching ? CUDSS_MATCHING_ALG_AUTO : CUDSS_MATCHING_ALG_NONE;

252:   PetscFunctionBegin;
253:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_REORDERING_ALG, &lu->reorderAlg, sizeof(lu->reorderAlg)));
254:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_PIVOT_TYPE, &lu->pivotType, sizeof(lu->pivotType)));
255:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_PIVOT_THRESHOLD, &lu->pivotThreshold, sizeof(lu->pivotThreshold)));
256:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_PIVOT_EPSILON, &lu->pivotEpsilon, sizeof(lu->pivotEpsilon)));
257:   // Matching is controlled via CUDSS_CONFIG_MATCHING_ALG (CUDSS_MATCHING_ALG_NONE disables it)
258:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_MATCHING_ALG, &matchingAlg, sizeof(matchingAlg)));
259:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_IR_N_STEPS, &lu->irNSteps, sizeof(lu->irNSteps)));
260:   PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_DETERMINISTIC_MODE, &lu->deterministic, sizeof(lu->deterministic)));
261:   PetscFunctionReturn(PETSC_SUCCESS);
262: }

264: #if PetscDefined(USE_64BIT_INDICES)
265: __global__ static void MatNarrowIndices_Kernel(PetscInt n, const PetscInt *src, int *dst)
266: {
267:   for (PetscInt tid = (PetscInt)blockIdx.x * blockDim.x + threadIdx.x; tid < n; tid += (PetscInt)gridDim.x * blockDim.x) dst[tid] = (int)src[tid];
268: }
269: #endif

271: /* Return the device CSR arrays of A in d_row_32/d_col_32 (MATSEQAIJCUSPARSE with 32-bit PetscInt) or d_row_pi/d_col_pi, and d_val.
272:    Leave the unused index pair unchanged. Reuse the arrays of a MATSEQAIJCUSPARSE matrix not bound to the CPU, else copy into buffers of lu.
273:    With valuesOnly, copy only the values because the structure is unchanged since the symbolic phase. */
274: static PetscErrorCode MatEnsureOnDevice_cuDSS(Mat A, Mat_cuDSS *lu, PetscBool valuesOnly, int **d_row_32, int **d_col_32, PetscInt **d_row_pi, PetscInt **d_col_pi, PetscScalar **d_val)
275: {
276:   PetscFunctionBegin;
277:   if (lu->ownDeviceCSR == PETSC_FALSE) {
278:     Mat_SeqAIJCUSPARSE           *cusparsestruct;
279:     Mat_SeqAIJCUSPARSEMultStruct *matstruct;
280:     CsrMatrix                    *csr;

282:     PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
283:     cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
284:     matstruct      = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
285:     csr            = (CsrMatrix *)matstruct->mat;
286: #if PetscDefined(USE_64BIT_INDICES)
287:     *d_row_pi = thrust::raw_pointer_cast(csr->row_offsets->data());
288:     *d_col_pi = thrust::raw_pointer_cast(csr->column_indices->data());
289: #else
290:     *d_row_32 = thrust::raw_pointer_cast(csr->row_offsets->data());
291:     *d_col_32 = thrust::raw_pointer_cast(csr->column_indices->data());
292: #endif
293:     *d_val = (PetscScalar *)thrust::raw_pointer_cast(csr->values->data());
294:   } else {
295:     Mat_SeqAIJ        *a = (Mat_SeqAIJ *)A->data;
296:     const PetscScalar *aa;
297:     PetscDeviceContext dctx;
298:     cudaStream_t       stream;

300:     PetscCall(MatCUDSSGetContext(&dctx, &stream));
301:     if (valuesOnly == PETSC_FALSE) {
302:       // Symbolic phase: upload structure (row offsets + column indices)
303:       PetscCallCUDA(cudaMemcpyAsync(lu->d_row_offsets, a->i, (lu->n + 1) * sizeof(PetscInt), cudaMemcpyHostToDevice, stream));
304:       PetscCallCUDA(cudaMemcpyAsync(lu->d_col_indices, a->j, lu->nnz * sizeof(PetscInt), cudaMemcpyHostToDevice, stream));
305:       PetscCall(PetscLogCpuToGpu((lu->n + 1) * sizeof(PetscInt) + lu->nnz * sizeof(PetscInt)));
306:     }
307:     // MatSeqAIJGetArrayRead(), unlike a->a, syncs device values of derived types to the host
308:     PetscCall(MatSeqAIJGetArrayRead(A, &aa));
309:     PetscCallCUDA(cudaMemcpyAsync(lu->d_values, aa, lu->nnz * sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
310:     PetscCall(PetscDeviceContextSynchronize(dctx));
311:     PetscCall(MatSeqAIJRestoreArrayRead(A, &aa));
312:     PetscCall(PetscLogCpuToGpu(lu->nnz * sizeof(PetscScalar)));
313:     *d_row_pi = lu->d_row_offsets;
314:     *d_col_pi = lu->d_col_indices;
315:     *d_val    = lu->d_values;
316:   }
317:   PetscFunctionReturn(PETSC_SUCCESS);
318: }

320: static PetscErrorCode MatFactorSymbolic_cuDSS(Mat F, Mat A, IS perm, cudssMatrixType_t mtype, cudssMatrixViewType_t mview)
321: {
322:   Mat_cuDSS         *lu = (Mat_cuDSS *)F->data;
323:   Mat_SeqAIJ        *a  = (Mat_SeqAIJ *)A->data;
324:   PetscInt           m = A->rmap->n, n = A->cmap->n, nnz = a->nz;
325:   PetscInt          *d_row_pi = NULL, *d_col_pi = NULL; // PetscInt-width path
326:   int               *d_row_32 = NULL, *d_col_32 = NULL; // MATSEQAIJCUSPARSE path with 32-bit PetscInt
327:   void              *d_row, *d_col;
328:   PetscScalar       *d_val = NULL;
329:   PetscBool          isCUSPARSE;
330:   PetscDeviceContext dctx;
331:   cudaStream_t       stream;

333:   PetscFunctionBegin;
334:   PetscCall(MatReset_cuDSS(lu));
335:   PetscCheck(m == n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "cuDSS requires a square matrix, got %" PetscInt_FMT "x%" PetscInt_FMT, m, n);
336:   if (m == 0) PetscFunctionReturn(PETSC_SUCCESS);

338:   lu->n   = n;
339:   lu->nnz = nnz;

341:   if (lu->handle == NULL) PetscCallCUDSS(cudssCreate(&lu->handle));
342:   if (lu->config == NULL) PetscCallCUDSS(cudssConfigCreate(&lu->config));
343:   PetscCall(MatSetFromOptions_cuDSS(F));
344:   PetscCall(MatCUDSSSetMemHandler(lu->handle, lu->mempool));
345:   PetscCallCUDSS(cudssDataCreate(lu->handle, &lu->data));
346:   PetscCall(MatApplyConfig_cuDSS(lu));
347:   if (perm != NULL) PetscCall(MatSetPermutation_cuDSS(F, perm));

349:   PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &isCUSPARSE));
350:   lu->ownDeviceCSR = (PetscBool)(isCUSPARSE == PETSC_FALSE || A->boundtocpu == PETSC_TRUE);
351:   if (lu->ownDeviceCSR == PETSC_TRUE) {
352:     PetscCallCUDA(cudaMalloc((void **)&lu->d_row_offsets, (m + 1) * sizeof(PetscInt)));
353:     PetscCallCUDA(cudaMalloc((void **)&lu->d_col_indices, nnz * sizeof(PetscInt)));
354:     PetscCallCUDA(cudaMalloc((void **)&lu->d_values, nnz * sizeof(PetscScalar)));
355:   }

357:   PetscCall(MatCUDSSGetContext(&dctx, &stream));
358:   PetscCallCUDSS(cudssSetStream(lu->handle, stream));

360:   PetscCall(MatEnsureOnDevice_cuDSS(A, lu, PETSC_FALSE, &d_row_32, &d_col_32, &d_row_pi, &d_col_pi, &d_val));
361: #if PetscDefined(USE_64BIT_INDICES)
362:   // Narrow the 64-bit PetscInt CSR index arrays to 32-bit on the device
363:   PetscCheck(nnz <= INT_MAX && m < INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuDSS requires 32-bit CSR indices; matrix has m=%" PetscInt_FMT ", nnz=%" PetscInt_FMT " > INT_MAX", m, nnz);
364:   PetscCallCUDA(cudaMalloc((void **)&lu->d_row_i32, (m + 1) * sizeof(int)));
365:   PetscCallCUDA(cudaMalloc((void **)&lu->d_col_i32, nnz * sizeof(int)));
366:   PetscCall(Petsc::device::cupm::impl::Interface<Petsc::device::cupm::DeviceType::CUDA>::PetscCUPMLaunchKernel1D(m + 1, 0, stream, MatNarrowIndices_Kernel, m + 1, d_row_pi, lu->d_row_i32));
367:   if (nnz > 0) PetscCall(Petsc::device::cupm::impl::Interface<Petsc::device::cupm::DeviceType::CUDA>::PetscCUPMLaunchKernel1D(nnz, 0, stream, MatNarrowIndices_Kernel, nnz, d_col_pi, lu->d_col_i32));
368:   d_row = lu->d_row_i32;
369:   d_col = lu->d_col_i32;
370: #else
371:   d_row = d_row_32 != NULL ? (void *)d_row_32 : (void *)d_row_pi;
372:   d_col = d_col_32 != NULL ? (void *)d_col_32 : (void *)d_col_pi;
373: #endif
374:   PetscCallCUDSS(cudssMatrixCreateCsr(&lu->cudss_A, m, n, nnz, d_row, NULL, d_col, d_val, CUDSS_R_32I, CUDSS_R_32I, CUDSS_SCALAR_TYPE, mtype, mview, CUDSS_BASE_ZERO));

376:   PetscCallCUDSS(cudssMatrixCreateDn(&lu->cudss_b, n, 1, n, NULL, CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
377:   PetscCallCUDSS(cudssMatrixCreateDn(&lu->cudss_x, n, 1, n, NULL, CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
378:   // cuDSS needs the user permutation before the analysis phase
379:   if (lu->userPermSet == PETSC_TRUE) {
380:     PetscCallCUDSS(cudssDataSet(lu->handle, lu->data, CUDSS_DATA_USER_PERM, lu->h_user_perm, (size_t)n * sizeof(int)));
381:     PetscCall(PetscInfo(F, "cuDSS: using user-supplied permutation of size %" PetscInt_FMT "\n", n));
382:   }
383:   // Analysis phase: reordering and symbolic factorization
384:   PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_ANALYSIS, lu->config, lu->data, lu->cudss_A, lu->cudss_x, lu->cudss_b));
385:   PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
386:   PetscFunctionReturn(PETSC_SUCCESS);
387: }

389: static PetscErrorCode MatLUFactorSymbolic_cuDSS(Mat F, Mat A, IS r, IS c, const MatFactorInfo *info)
390: {
391:   PetscFunctionBegin;
392:   if (r != NULL && c != NULL) PetscCall(PetscInfo(F, "cuDSS accepts only a single permutation; using row ordering IS, column ordering IS is ignored\n"));
393:   PetscCall(MatFactorSymbolic_cuDSS(F, A, r != NULL ? r : c, CUDSS_MTYPE_GENERAL, CUDSS_MVIEW_FULL));
394:   F->ops->lufactornumeric = MatFactorNumeric_cuDSS;
395:   PetscFunctionReturn(PETSC_SUCCESS);
396: }

398: static PetscErrorCode MatCholeskyFactorSymbolic_cuDSS(Mat F, Mat A, IS perm, const MatFactorInfo *info)
399: {
400:   cudssMatrixType_t mtype;

402:   PetscFunctionBegin;
403: #if PetscDefined(USE_COMPLEX)
404:   if (A->spd == PETSC_BOOL3_TRUE) mtype = CUDSS_MTYPE_HPD;
405:   else if (A->hermitian == PETSC_BOOL3_TRUE) mtype = CUDSS_MTYPE_HERMITIAN;
406:   else mtype = CUDSS_MTYPE_SYMMETRIC;
407: #else
408:   if (A->spd == PETSC_BOOL3_TRUE) mtype = CUDSS_MTYPE_SPD;
409:   else mtype = CUDSS_MTYPE_SYMMETRIC;
410: #endif
411:   PetscCall(MatFactorSymbolic_cuDSS(F, A, perm, mtype, CUDSS_MVIEW_UPPER));
412:   F->ops->choleskyfactornumeric = MatFactorNumeric_cuDSS;
413:   PetscFunctionReturn(PETSC_SUCCESS);
414: }

416: static PetscErrorCode MatSolve_cuDSS(Mat F, Vec b, Vec x)
417: {
418:   Mat_cuDSS         *lu = (Mat_cuDSS *)F->data;
419:   const PetscScalar *barray;
420:   PetscScalar       *xarray;
421:   PetscMemType       bmtype, xmtype;
422:   Vec                bcu = NULL, xcu = NULL;
423:   PetscDeviceContext dctx;
424:   cudaStream_t       stream;

426:   PetscFunctionBegin;
427:   if (F->rmap->n == 0) PetscFunctionReturn(PETSC_SUCCESS);
428:   /* Accept any vector whose array is device-resident (VECCUDA, VECKOKKOS on a CUDA
429:      backend, etc.) directly without a temporary Vec or device-to-device copy. */
430:   PetscCall(VecGetArrayReadAndMemType(b, &barray, &bmtype));
431:   if (!PetscMemTypeDevice(bmtype)) {
432:     PetscCall(VecRestoreArrayReadAndMemType(b, &barray));
433:     PetscCall(PetscInfo(F, "Copying b from host to temporary VECCUDA; consider using a device Vec for better performance\n"));
434:     PetscCall(VecCreate(PetscObjectComm((PetscObject)b), &bcu));
435:     PetscCall(VecSetSizes(bcu, b->map->n, b->map->N));
436:     PetscCall(VecSetType(bcu, VECCUDA));
437:     PetscCall(VecCopy(b, bcu));
438:     PetscCall(VecCUDAGetArrayRead(bcu, &barray));
439:   }
440:   PetscCall(VecGetArrayAndMemType(x, &xarray, &xmtype));
441:   if (!PetscMemTypeDevice(xmtype)) {
442:     PetscCall(VecRestoreArrayAndMemType(x, &xarray));
443:     PetscCall(VecCreate(PetscObjectComm((PetscObject)x), &xcu));
444:     PetscCall(VecSetSizes(xcu, x->map->n, x->map->N));
445:     PetscCall(VecSetType(xcu, VECCUDA));
446:     PetscCall(VecCUDAGetArrayWrite(xcu, &xarray));
447:   }
448:   PetscCallCUDSS(cudssMatrixSetValues(lu->cudss_b, (void *)barray));
449:   PetscCallCUDSS(cudssMatrixSetValues(lu->cudss_x, xarray));
450:   PetscCall(MatCUDSSGetContext(&dctx, &stream));
451:   PetscCallCUDSS(cudssSetStream(lu->handle, stream));
452:   PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_SOLVE, lu->config, lu->data, lu->cudss_A, lu->cudss_x, lu->cudss_b));
453:   PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
454:   if (bcu != NULL) {
455:     PetscCall(VecCUDARestoreArrayRead(bcu, &barray));
456:     PetscCall(VecDestroy(&bcu));
457:   } else PetscCall(VecRestoreArrayReadAndMemType(b, &barray));
458:   if (xcu != NULL) {
459:     PetscCall(VecCUDARestoreArrayWrite(xcu, &xarray));
460:     PetscCall(VecCopy(xcu, x));
461:     PetscCall(VecDestroy(&xcu));
462:   } else PetscCall(VecRestoreArrayAndMemType(x, &xarray));
463:   PetscFunctionReturn(PETSC_SUCCESS);
464: }

466: static PetscErrorCode MatMatSolve_cuDSS(Mat F, Mat B, Mat X)
467: {
468:   Mat_cuDSS         *lu = (Mat_cuDSS *)F->data;
469:   const PetscScalar *barray;
470:   PetscScalar       *xarray;
471:   PetscInt           n, nrhs, nX, nrhsX, ldb, ldx, ncol;
472:   cudssMatrix_t      cudss_B = NULL, cudss_X = NULL;
473:   Mat                Bcuda = NULL, Xcuda = NULL;
474:   PetscBool          BisCUDA = PETSC_FALSE, XisCUDA = PETSC_FALSE;
475:   PetscDeviceContext dctx;
476:   cudaStream_t       stream;

478:   PetscFunctionBegin;
479:   if (F->rmap->n == 0) PetscFunctionReturn(PETSC_SUCCESS);
480:   PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQDENSECUDA, &BisCUDA));
481:   PetscCall(PetscObjectTypeCompare((PetscObject)X, MATSEQDENSECUDA, &XisCUDA));
482:   if (BisCUDA == PETSC_FALSE) {
483:     PetscCall(PetscInfo(F, "Converting B from host to MATDENSECUDA; consider using MATSEQDENSECUDA for better performance\n"));
484:     PetscCall(MatConvert(B, MATDENSECUDA, MAT_INITIAL_MATRIX, &Bcuda));
485:   } else if (B == X) PetscCall(MatDuplicate(B, MAT_COPY_VALUES, &Bcuda)); // solve out of place from a copy of B
486:   else Bcuda = B;
487:   if (XisCUDA == PETSC_FALSE) {
488:     PetscInt m_x, n_x, M_x, N_x;

490:     PetscCall(MatGetLocalSize(X, &m_x, &n_x));
491:     PetscCall(MatGetSize(X, &M_x, &N_x));
492:     PetscCall(MatCreateDenseCUDA(PetscObjectComm((PetscObject)X), m_x, n_x, M_x, N_x, NULL, &Xcuda));
493:   } else Xcuda = X;
494:   PetscCall(MatGetSize(Bcuda, &n, &nrhs));
495:   PetscCall(MatGetSize(Xcuda, &nX, &nrhsX));
496:   PetscCheck(n == nX && nrhs == nrhsX, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Incompatible matrix dimensions: B is %" PetscInt_FMT " x %" PetscInt_FMT ", X is %" PetscInt_FMT " x %" PetscInt_FMT, n, nrhs, nX, nrhsX);
497:   PetscCheck(n == lu->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "B row count %" PetscInt_FMT " does not match factored matrix size %" PetscInt_FMT, n, lu->n);
498:   PetscCall(MatDenseGetLDA(Bcuda, &ldb));
499:   PetscCall(MatDenseGetLDA(Xcuda, &ldx));
500:   PetscCall(MatDenseCUDAGetArrayRead(Bcuda, &barray));
501:   PetscCall(MatDenseCUDAGetArrayWrite(Xcuda, &xarray));
502:   PetscCall(MatCUDSSGetContext(&dctx, &stream));
503:   PetscCallCUDSS(cudssSetStream(lu->handle, stream));
504:   // cuDSS deterministic mode requires nrhs == 1, so solve column-by-column when enabled
505:   ncol = lu->deterministic ? 1 : nrhs;
506:   for (PetscInt j = 0; j < nrhs; j += ncol) {
507:     PetscCallCUDSS(cudssMatrixCreateDn(&cudss_B, n, ncol, ldb, (void *)(barray + j * ldb), CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
508:     PetscCallCUDSS(cudssMatrixCreateDn(&cudss_X, n, ncol, ldx, xarray + j * ldx, CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
509:     PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_SOLVE, lu->config, lu->data, lu->cudss_A, cudss_X, cudss_B));
510:     PetscCallCUDSS(cudssMatrixDestroy(cudss_B));
511:     PetscCallCUDSS(cudssMatrixDestroy(cudss_X));
512:   }
513:   PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
514:   PetscCall(MatDenseCUDARestoreArrayRead(Bcuda, &barray));
515:   PetscCall(MatDenseCUDARestoreArrayWrite(Xcuda, &xarray));
516:   if (Bcuda != B) PetscCall(MatDestroy(&Bcuda));
517:   if (XisCUDA == PETSC_FALSE) {
518:     PetscCall(MatCopy(Xcuda, X, SAME_NONZERO_PATTERN));
519:     PetscCall(MatDestroy(&Xcuda));
520:   }
521:   PetscFunctionReturn(PETSC_SUCCESS);
522: }

524: static PetscErrorCode MatFactorNumeric_cuDSS(Mat F, Mat A, const MatFactorInfo *info)
525: {
526:   Mat_cuDSS         *lu       = (Mat_cuDSS *)F->data;
527:   int               *d_row_32 = NULL, *d_col_32 = NULL;
528:   int                cudss_info = 0;
529:   PetscInt          *d_row_pi = NULL, *d_col_pi = NULL;
530:   PetscScalar       *d_val = NULL;
531:   size_t             nwritten;
532:   PetscDeviceContext dctx;
533:   cudaStream_t       stream;

535:   PetscFunctionBegin;
536:   if (F->rmap->n == 0) {
537:     F->assembled = PETSC_TRUE;
538:     PetscFunctionReturn(PETSC_SUCCESS);
539:   }
540:   PetscCall(MatEnsureOnDevice_cuDSS(A, lu, PETSC_TRUE, &d_row_32, &d_col_32, &d_row_pi, &d_col_pi, &d_val));
541:   PetscCallCUDSS(cudssMatrixSetValues(lu->cudss_A, d_val));
542:   PetscCall(MatCUDSSGetContext(&dctx, &stream));
543:   PetscCallCUDSS(cudssSetStream(lu->handle, stream));
544:   PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_FACTORIZATION, lu->config, lu->data, lu->cudss_A, lu->cudss_x, lu->cudss_b));
545:   PetscCall(PetscDeviceContextSynchronize(dctx)); // CUDSS_DATA_INFO is valid only after the factorization completes
546:   PetscCallCUDSS(cudssDataGet(lu->handle, lu->data, CUDSS_DATA_INFO, &cudss_info, sizeof(cudss_info), &nwritten));
547:   if (cudss_info != 0) {
548:     PetscCall(PetscInfo(F, "cuDSS factorization failed with CUDSS_DATA_INFO %d\n", cudss_info));
549:     F->factorerrortype = MAT_FACTOR_NUMERIC_ZEROPIVOT;
550:   }
551:   F->assembled = PETSC_TRUE;
552:   PetscFunctionReturn(PETSC_SUCCESS);
553: }

555: static PetscErrorCode MatFactorGetSolverType_seqaij_cudss(Mat A, MatSolverType *type)
556: {
557:   PetscFunctionBegin;
558:   *type = MATSOLVERCUDSS;
559:   PetscFunctionReturn(PETSC_SUCCESS);
560: }

562: static PetscErrorCode MatSetPermutation_cuDSS(Mat F, IS perm)
563: {
564:   Mat_cuDSS      *lu = (Mat_cuDSS *)F->data;
565:   PetscInt        nrows, ncols, n;
566:   const PetscInt *idx;

568:   PetscFunctionBegin;
569:   PetscCall(PetscFree(lu->h_user_perm));
570:   PetscCall(MatGetLocalSize(F, &nrows, &ncols));
571:   PetscCheck(nrows <= INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuDSS requires 32-bit permutation indices; matrix has m=%" PetscInt_FMT " > INT_MAX", nrows);
572:   PetscCall(ISGetLocalSize(perm, &n));
573:   PetscCheck(n == nrows && n == ncols, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Incompatible matrix and permutation sizes");
574:   PetscCall(PetscMalloc1(n, &lu->h_user_perm));
575:   PetscCall(ISGetIndices(perm, &idx));
576:   for (PetscInt i = 0; i < n; i++) lu->h_user_perm[i] = (int)idx[i];
577:   PetscCall(ISRestoreIndices(perm, &idx));
578:   lu->userPermSet = PETSC_TRUE;
579:   PetscFunctionReturn(PETSC_SUCCESS);
580: }

582: /*MC
583:   MATSOLVERCUDSS = "cudss" - A solver package providing LU and Cholesky factorization for
584:   sequential sparse matrices via the NVIDIA cuDSS GPU-accelerated sparse direct solver library.

586:   Options Database Keys:
587: + -mat_cudss_reorder_alg (default|btf_colamd|colamd|amd|nested_dissection|none) - reordering algorithm
588: . -mat_cudss_pivot_type (auto|none|col|row|diagonal|local_block|bunch_kaufman)  - pivoting type
589: . -mat_cudss_pivot_threshold threshold                                          - Pivot threshold, default is 1.0
590: . -mat_cudss_pivot_epsilon epsilon                                              - Pivot epsilon, default is 0.0
591: . -mat_cudss_use_matching (true|false)                                          - Enable bipartite matching for heavy-diagonal scaling and permutation, default is false
592: . -mat_cudss_ir_n_steps nsteps                                                  - Number of iterative refinement steps, default is 0
593: . -mat_cudss_deterministic (true|false)                                         - Enable bitwise-reproducible factorization and solve, default is false
594: - -mat_cudss_mempool (true|false)                                               - Use a stream-ordered `cudaMemPool_t` for cuDSS device workspace allocations, default is true

596:   Level: intermediate

598:   Notes:
599:     Registered for both `MATSEQAIJ` (host) and `MATSEQAIJCUSPARSE` (device) matrix types.
600:     When the input matrix is `MATSEQAIJ`, the matrix data is transparently copied to the GPU.
601:     `MatSolveTranspose()` is not supported. The `bunch_kaufman` value of `-mat_cudss_pivot_type`
602:     applies to symmetric or Hermitian indefinite matrices.

604:     By default cuDSS performs its own internal reordering (`MATORDERINGEXTERNAL`) during the symbolic phase,
605:     selectable via `-mat_cudss_reorder_alg default`. A PETSc ordering can be selected with
606:     `-pc_factor_mat_ordering_type` or `PCFactorSetMatOrderingType()`, or by passing a non-`NULL` `IS`
607:     to `MatLUFactorSymbolic()` or `MatCholeskyFactorSymbolic()`.

609:     `MatSolve()` and `MatMatSolve()` operate on the GPU. Host right-hand-side and solution
610:     objects (`VECSEQ` vectors, `MATSEQDENSE` matrices) are accepted and transparently copied
611:     to and from the device, but this incurs extra copies; for best performance supply
612:     device vectors (e.g., `VECCUDA` or `VECKOKKOS`) and `MATSEQDENSECUDA` matrices.

614: .seealso: [](ch_matrices), `Mat`, `PCLU`, `PCCHOLESKY`, `PCFactorSetMatSolverType()`, `PCFactorSetMatOrderingType()`, `MatSolverType`, `MatLUFactorSymbolic()`, `MatCholeskyFactorSymbolic()`
615: M*/

617: static PetscErrorCode MatGetFactor_seqaij_cudss(Mat A, MatFactorType ftype, Mat *F)
618: {
619:   Mat        B;
620:   Mat_cuDSS *lu;
621:   PetscInt   m = A->rmap->n, n = A->cmap->n;
622:   int        nvrtc_major, nvrtc_minor;

624:   PetscFunctionBegin;
625:   PetscCheck(ftype == MAT_FACTOR_LU || ftype == MAT_FACTOR_CHOLESKY, PETSC_COMM_SELF, PETSC_ERR_SUP, "Factor type not supported by cuDSS");
626:   // Call nvrtcVersion() so --as-needed linkers record a direct libnvrtc dependency on libpetsc
627:   PetscCheck(nvrtcVersion(&nvrtc_major, &nvrtc_minor) == NVRTC_SUCCESS, PETSC_COMM_SELF, PETSC_ERR_LIB, "nvrtcVersion() failed");
628:   PetscCall(MatCreate(PetscObjectComm((PetscObject)A), &B));
629:   PetscCall(MatSetSizes(B, m, n, PETSC_DETERMINE, PETSC_DETERMINE));
630:   PetscCall(PetscStrallocpy(MATSOLVERCUDSS, &((PetscObject)B)->type_name));
631:   PetscCall(MatSetUp(B));
632:   B->trivialsymbolic = PETSC_FALSE;
633:   B->factortype      = ftype;
634:   B->canuseordering  = PETSC_TRUE;
635:   PetscCall(PetscStrallocpy(MATORDERINGEXTERNAL, (char **)&B->preferredordering[MAT_FACTOR_LU]));
636:   PetscCall(PetscStrallocpy(MATORDERINGEXTERNAL, (char **)&B->preferredordering[MAT_FACTOR_CHOLESKY]));
637:   B->assembled    = PETSC_TRUE;
638:   B->preallocated = PETSC_TRUE;

640:   if (ftype == MAT_FACTOR_LU) B->ops->lufactorsymbolic = MatLUFactorSymbolic_cuDSS;
641:   else B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_cuDSS;
642:   B->ops->solve    = MatSolve_cuDSS;
643:   B->ops->matsolve = MatMatSolve_cuDSS;
644:   B->ops->destroy  = MatDestroy_cuDSS;
645:   B->ops->view     = MatView_cuDSS;
646:   B->ops->getinfo  = MatGetInfo_External;

648:   PetscCall(PetscFree(B->solvertype));
649:   PetscCall(PetscStrallocpy(MATSOLVERCUDSS, &B->solvertype));
650:   PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatFactorGetSolverType_C", MatFactorGetSolverType_seqaij_cudss));

652:   PetscCall(PetscNew(&lu));
653:   lu->reorderAlg     = CUDSS_REORDERING_ALG_DEFAULT;
654:   lu->pivotType      = CUDSS_PIVOT_AUTO;
655:   lu->pivotThreshold = 1.0;
656:   lu->pivotEpsilon   = 0.0;
657:   lu->useMatching    = 0;
658:   lu->irNSteps       = 0;
659:   lu->deterministic  = 0;
660:   lu->mempool        = PETSC_TRUE;
661:   B->data            = lu;

663:   *F = B;
664:   PetscFunctionReturn(PETSC_SUCCESS);
665: }

667: PETSC_INTERN PetscErrorCode MatSolverTypeRegister_cuDSS(void)
668: {
669:   PetscFunctionBegin;
670:   PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJ, MAT_FACTOR_LU, MatGetFactor_seqaij_cudss));
671:   PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJ, MAT_FACTOR_CHOLESKY, MatGetFactor_seqaij_cudss));
672:   PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJCUSPARSE, MAT_FACTOR_LU, MatGetFactor_seqaij_cudss));
673:   PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJCUSPARSE, MAT_FACTOR_CHOLESKY, MatGetFactor_seqaij_cudss));
674:   PetscFunctionReturn(PETSC_SUCCESS);
675: }