Actual source code: cudss.cu
1: /*
2: This file implements a subclass of the SeqAIJ matrix class that uses
3: the NVIDIA cuDSS GPU-accelerated sparse direct solver library.
4: */
6: #include <../src/mat/impls/aij/seq/aij.h>
7: #include <../src/mat/impls/aij/seq/seqcusparse/cusparsematimpl.h>
8: #include <petsc/private/deviceimpl.h>
9: #include <petsc/private/cupminterface.hpp>
10: #include <cudss.h>
11: #include <nvrtc.h>
12: #include <climits>
14: // Internal data structure for the cuDSS factored matrix
15: typedef struct {
16: cudssHandle_t handle;
17: cudssConfig_t config;
18: cudssData_t data;
19: cudssMatrix_t cudss_A; // sparse matrix descriptor
20: cudssMatrix_t cudss_b; // dense RHS descriptor
21: cudssMatrix_t cudss_x; // dense solution descriptor
23: // Device CSR arrays (owned by us when input is MATSEQAIJ)
24: PetscInt *d_row_offsets;
25: PetscInt *d_col_indices;
26: int *d_row_i32; // narrowed 32-bit copy when PetscInt is 64-bit
27: int *d_col_i32;
28: PetscScalar *d_values;
30: // User-supplied permutation (host array of 32-bit ints, owned by us)
31: int *h_user_perm;
32: PetscBool userPermSet;
34: PetscInt n; // matrix dimension
35: PetscInt nnz;
36: PetscBool ownDeviceCSR;
38: cudssReorderingAlg_t reorderAlg;
39: cudssPivotType_t pivotType;
40: double pivotThreshold; // cuDSS expects double for CUDSS_CONFIG_PIVOT_THRESHOLD
41: double pivotEpsilon; // cuDSS expects double for CUDSS_CONFIG_PIVOT_EPSILON
42: int useMatching; // cuDSS expects int, translated to CUDSS_CONFIG_MATCHING_ALG
43: int irNSteps; // cuDSS expects int for CUDSS_CONFIG_IR_N_STEPS
44: int deterministic; // cuDSS expects int for CUDSS_CONFIG_DETERMINISTIC_MODE
45: PetscBool mempool; // register a stream-ordered cudaMemPool_t as cuDSS's device workspace allocator
46: } Mat_cuDSS;
48: // Map PetscScalar to the cuDSS data type
49: #if PetscDefined(USE_COMPLEX)
50: #if PetscDefined(USE_REAL_SINGLE)
51: #define CUDSS_SCALAR_TYPE CUDSS_C_32F
52: #else
53: #define CUDSS_SCALAR_TYPE CUDSS_C_64F
54: #endif
55: #else
56: #if PetscDefined(USE_REAL_SINGLE)
57: #define CUDSS_SCALAR_TYPE CUDSS_R_32F
58: #else
59: #define CUDSS_SCALAR_TYPE CUDSS_R_64F
60: #endif
61: #endif
63: #define PetscCallCUDSS(...) \
64: do { \
65: cudssStatus_t cudss_status_ = __VA_ARGS__; \
66: PetscCheck(cudss_status_ == CUDSS_STATUS_SUCCESS, PETSC_COMM_SELF, PETSC_ERR_LIB, "cuDSS error %d in %s", (int)cudss_status_, PetscStringize(__VA_ARGS__)); \
67: } while (0)
69: static PetscErrorCode MatCUDSSGetContext(PetscDeviceContext *dctx, cudaStream_t *stream)
70: {
71: cudaStream_t *handle;
73: PetscFunctionBegin;
74: PetscCall(PetscDeviceContextGetCurrentContextAssertType_Internal(dctx, PETSC_DEVICE_CUDA));
75: PetscCall(PetscDeviceContextGetStreamHandle(*dctx, (void **)&handle));
76: *stream = *handle;
77: PetscFunctionReturn(PETSC_SUCCESS);
78: }
80: static int MatCUDSSPoolAlloc(void *ctx, void **ptr, size_t size, cudaStream_t stream)
81: {
82: cudaError_t cerr;
84: (void)ctx;
85: *ptr = NULL;
86: if (size == 0) return 0;
87: cerr = cudaMallocAsync(ptr, size, stream);
88: return (cerr == cudaSuccess) ? 0 : (int)cerr;
89: }
91: static int MatCUDSSPoolFree(void *ctx, void *ptr, size_t size, cudaStream_t stream)
92: {
93: cudaError_t cerr;
95: (void)ctx;
96: (void)size;
97: if (ptr == NULL) return 0;
98: cerr = cudaFreeAsync(ptr, stream);
99: return (cerr == cudaSuccess) ? 0 : (int)cerr;
100: }
102: static PetscErrorCode MatCUDSSSetMemHandler(cudssHandle_t handle, PetscBool use_mempool)
103: {
104: PetscFunctionBegin;
105: if (use_mempool == PETSC_TRUE) {
106: cudssDeviceMemHandler_t handler;
108: PetscCall(PetscMemzero(&handler, sizeof(handler)));
109: handler.ctx = NULL;
110: handler.device_alloc = MatCUDSSPoolAlloc;
111: handler.device_free = MatCUDSSPoolFree;
112: PetscCall(PetscStrncpy(handler.name, "petsc_cudss_pool", sizeof(handler.name)));
113: PetscCallCUDSS(cudssSetDeviceMemHandler(handle, &handler));
114: } else PetscCallCUDSS(cudssSetDeviceMemHandler(handle, NULL));
115: PetscFunctionReturn(PETSC_SUCCESS);
116: }
118: /* These arrays are indexed by the cuDSS enum values directly.
119: Mapping (cuDSS 0.8.0):
120: cudssReorderingAlg_t: CUDSS_REORDERING_ALG_DEFAULT==0, ..._BTF_COLAMD==1, ..._COLAMD==2,
121: ..._AMD==3, ..._NESTED_DISSECTION==4, ..._NONE==5
122: cudssPivotType_t: CUDSS_PIVOT_AUTO==0, ..._NONE==1, ..._GLOBAL_COL==2, ..._GLOBAL_ROW==3,
123: ..._DIAGONAL==4, ..._LOCAL_BLOCK==5, ..._BUNCH_KAUFMAN==6
124: If NVIDIA renumbers these enumerators in a future release, the option parsing
125: and view output will silently mismap. Update both arrays and this comment. */
126: static const char *const MatCUDSSReorderAlgs[] = {"default", "btf_colamd", "colamd", "amd", "nested_dissection", "none"};
127: static const char *const MatCUDSSPivotTypes[] = {"auto", "none", "col", "row", "diagonal", "local_block", "bunch_kaufman"};
129: // Forward declarations
130: static PetscErrorCode MatFactorNumeric_cuDSS(Mat, Mat, const MatFactorInfo *);
131: static PetscErrorCode MatSetFromOptions_cuDSS(Mat);
132: static PetscErrorCode MatFactorSymbolic_cuDSS(Mat, Mat, IS, cudssMatrixType_t, cudssMatrixViewType_t);
133: static PetscErrorCode MatSetPermutation_cuDSS(Mat, IS);
135: static PetscErrorCode MatView_Info_cuDSS(Mat A, PetscViewer viewer)
136: {
137: Mat_cuDSS *lu = (Mat_cuDSS *)A->data;
138: const char *reorderName = ((int)lu->reorderAlg >= 0 && (size_t)lu->reorderAlg < PETSC_STATIC_ARRAY_LENGTH(MatCUDSSReorderAlgs)) ? MatCUDSSReorderAlgs[lu->reorderAlg] : "unknown";
139: const char *pivotName = ((int)lu->pivotType >= 0 && (size_t)lu->pivotType < PETSC_STATIC_ARRAY_LENGTH(MatCUDSSPivotTypes)) ? MatCUDSSPivotTypes[lu->pivotType] : "unknown";
141: PetscFunctionBegin;
142: PetscCall(PetscViewerASCIIPrintf(viewer, "cuDSS run parameters:\n"));
143: PetscCall(PetscViewerASCIIPrintf(viewer, " Reorder algorithm: %s\n", reorderName));
144: PetscCall(PetscViewerASCIIPrintf(viewer, " User permutation: %s\n", lu->userPermSet == PETSC_TRUE ? "yes" : "no"));
145: PetscCall(PetscViewerASCIIPrintf(viewer, " Pivot type: %s\n", pivotName));
146: PetscCall(PetscViewerASCIIPrintf(viewer, " Pivot threshold: %g\n", lu->pivotThreshold));
147: PetscCall(PetscViewerASCIIPrintf(viewer, " Pivot epsilon: %g\n", lu->pivotEpsilon));
148: PetscCall(PetscViewerASCIIPrintf(viewer, " Use matching: %s\n", lu->useMatching ? "true" : "false"));
149: PetscCall(PetscViewerASCIIPrintf(viewer, " IR steps: %d\n", lu->irNSteps));
150: PetscCall(PetscViewerASCIIPrintf(viewer, " Deterministic mode: %s\n", lu->deterministic ? "true" : "false"));
151: PetscCall(PetscViewerASCIIPrintf(viewer, " Device workspace mempool: %s\n", lu->mempool == PETSC_TRUE ? "true" : "false"));
152: PetscFunctionReturn(PETSC_SUCCESS);
153: }
155: static PetscErrorCode MatView_cuDSS(Mat A, PetscViewer viewer)
156: {
157: PetscBool iascii;
159: PetscFunctionBegin;
160: PetscCall(PetscObjectTypeCompare((PetscObject)viewer, PETSCVIEWERASCII, &iascii));
161: if (iascii == PETSC_TRUE) {
162: PetscViewerFormat format;
164: PetscCall(PetscViewerGetFormat(viewer, &format));
165: if (format == PETSC_VIEWER_ASCII_INFO) PetscCall(MatView_Info_cuDSS(A, viewer));
166: }
167: PetscFunctionReturn(PETSC_SUCCESS);
168: }
170: static PetscErrorCode MatReset_cuDSS(Mat_cuDSS *lu)
171: {
172: PetscFunctionBegin;
173: if (lu->cudss_x != NULL) {
174: PetscCallCUDSS(cudssMatrixDestroy(lu->cudss_x));
175: lu->cudss_x = NULL;
176: }
177: if (lu->cudss_b != NULL) {
178: PetscCallCUDSS(cudssMatrixDestroy(lu->cudss_b));
179: lu->cudss_b = NULL;
180: }
181: if (lu->cudss_A != NULL) {
182: PetscCallCUDSS(cudssMatrixDestroy(lu->cudss_A));
183: lu->cudss_A = NULL;
184: }
185: if (lu->data != NULL) {
186: PetscCallCUDSS(cudssDataDestroy(lu->handle, lu->data));
187: lu->data = NULL;
188: }
189: if (lu->ownDeviceCSR == PETSC_TRUE) {
190: PetscCallCUDA(cudaFree(lu->d_row_offsets));
191: lu->d_row_offsets = NULL;
192: PetscCallCUDA(cudaFree(lu->d_col_indices));
193: lu->d_col_indices = NULL;
194: PetscCallCUDA(cudaFree(lu->d_values));
195: lu->d_values = NULL;
196: lu->ownDeviceCSR = PETSC_FALSE;
197: }
198: PetscCallCUDA(cudaFree(lu->d_row_i32));
199: lu->d_row_i32 = NULL;
200: PetscCallCUDA(cudaFree(lu->d_col_i32));
201: lu->d_col_i32 = NULL;
202: PetscCall(PetscFree(lu->h_user_perm));
203: lu->userPermSet = PETSC_FALSE;
204: PetscFunctionReturn(PETSC_SUCCESS);
205: }
207: static PetscErrorCode MatDestroy_cuDSS(Mat A)
208: {
209: Mat_cuDSS *lu = (Mat_cuDSS *)A->data;
211: PetscFunctionBegin;
212: PetscCall(MatReset_cuDSS(lu));
213: if (lu->config != NULL) PetscCallCUDSS(cudssConfigDestroy(lu->config));
214: if (lu->handle != NULL) PetscCallCUDSS(cudssDestroy(lu->handle));
215: PetscCall(PetscObjectComposeFunction((PetscObject)A, "MatFactorGetSolverType_C", NULL));
216: PetscCall(PetscFree(A->data));
217: PetscFunctionReturn(PETSC_SUCCESS);
218: }
220: static PetscErrorCode MatSetFromOptions_cuDSS(Mat F)
221: {
222: Mat_cuDSS *lu = (Mat_cuDSS *)F->data;
223: PetscInt reorderAlg = (PetscInt)lu->reorderAlg, pivotType = (PetscInt)lu->pivotType, irNSteps = (PetscInt)lu->irNSteps;
224: PetscReal pivotThreshold = (PetscReal)lu->pivotThreshold, pivotEpsilon = (PetscReal)lu->pivotEpsilon;
225: PetscBool useMatching = (PetscBool)lu->useMatching, deterministic = (PetscBool)lu->deterministic;
227: PetscFunctionBegin;
228: PetscOptionsBegin(PetscObjectComm((PetscObject)F), ((PetscObject)F)->prefix, "cuDSS Options", "Mat");
229: PetscCall(PetscOptionsEList("-mat_cudss_reorder_alg", "Reordering algorithm", "None", MatCUDSSReorderAlgs, PETSC_STATIC_ARRAY_LENGTH(MatCUDSSReorderAlgs), MatCUDSSReorderAlgs[reorderAlg], &reorderAlg, NULL));
230: lu->reorderAlg = (cudssReorderingAlg_t)reorderAlg;
231: PetscCall(PetscOptionsEList("-mat_cudss_pivot_type", "Pivot type", "None", MatCUDSSPivotTypes, PETSC_STATIC_ARRAY_LENGTH(MatCUDSSPivotTypes), MatCUDSSPivotTypes[pivotType], &pivotType, NULL));
232: lu->pivotType = (cudssPivotType_t)pivotType;
233: PetscCall(PetscOptionsReal("-mat_cudss_pivot_threshold", "Pivot threshold", "None", pivotThreshold, &pivotThreshold, NULL));
234: lu->pivotThreshold = (double)pivotThreshold;
235: PetscCall(PetscOptionsReal("-mat_cudss_pivot_epsilon", "Pivot epsilon", "None", pivotEpsilon, &pivotEpsilon, NULL));
236: lu->pivotEpsilon = (double)pivotEpsilon;
237: PetscCall(PetscOptionsBool("-mat_cudss_use_matching", "Enable matching", "None", useMatching, &useMatching, NULL));
238: lu->useMatching = (int)useMatching;
239: PetscCall(PetscOptionsInt("-mat_cudss_ir_n_steps", "Number of iterative refinement steps", "None", irNSteps, &irNSteps, NULL));
240: PetscCall(PetscCIntCast(irNSteps, &lu->irNSteps));
241: PetscCall(PetscOptionsBool("-mat_cudss_deterministic", "Enable bitwise-reproducible factorization and solve", "None", deterministic, &deterministic, NULL));
242: lu->deterministic = (int)deterministic;
243: PetscCall(PetscOptionsBool("-mat_cudss_mempool", "Use a stream-ordered cudaMemPool_t for cuDSS device workspace allocations", "None", lu->mempool, &lu->mempool, NULL));
244: PetscOptionsEnd();
245: PetscFunctionReturn(PETSC_SUCCESS);
246: }
248: static PetscErrorCode MatApplyConfig_cuDSS(Mat_cuDSS *lu)
249: {
250: cudssMatchingAlg_t matchingAlg = lu->useMatching ? CUDSS_MATCHING_ALG_AUTO : CUDSS_MATCHING_ALG_NONE;
252: PetscFunctionBegin;
253: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_REORDERING_ALG, &lu->reorderAlg, sizeof(lu->reorderAlg)));
254: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_PIVOT_TYPE, &lu->pivotType, sizeof(lu->pivotType)));
255: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_PIVOT_THRESHOLD, &lu->pivotThreshold, sizeof(lu->pivotThreshold)));
256: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_PIVOT_EPSILON, &lu->pivotEpsilon, sizeof(lu->pivotEpsilon)));
257: // Matching is controlled via CUDSS_CONFIG_MATCHING_ALG (CUDSS_MATCHING_ALG_NONE disables it)
258: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_MATCHING_ALG, &matchingAlg, sizeof(matchingAlg)));
259: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_IR_N_STEPS, &lu->irNSteps, sizeof(lu->irNSteps)));
260: PetscCallCUDSS(cudssConfigSet(lu->config, CUDSS_CONFIG_DETERMINISTIC_MODE, &lu->deterministic, sizeof(lu->deterministic)));
261: PetscFunctionReturn(PETSC_SUCCESS);
262: }
264: #if PetscDefined(USE_64BIT_INDICES)
265: __global__ static void MatNarrowIndices_Kernel(PetscInt n, const PetscInt *src, int *dst)
266: {
267: for (PetscInt tid = (PetscInt)blockIdx.x * blockDim.x + threadIdx.x; tid < n; tid += (PetscInt)gridDim.x * blockDim.x) dst[tid] = (int)src[tid];
268: }
269: #endif
271: /* Return the device CSR arrays of A in d_row_32/d_col_32 (MATSEQAIJCUSPARSE with 32-bit PetscInt) or d_row_pi/d_col_pi, and d_val.
272: Leave the unused index pair unchanged. Reuse the arrays of a MATSEQAIJCUSPARSE matrix not bound to the CPU, else copy into buffers of lu.
273: With valuesOnly, copy only the values because the structure is unchanged since the symbolic phase. */
274: static PetscErrorCode MatEnsureOnDevice_cuDSS(Mat A, Mat_cuDSS *lu, PetscBool valuesOnly, int **d_row_32, int **d_col_32, PetscInt **d_row_pi, PetscInt **d_col_pi, PetscScalar **d_val)
275: {
276: PetscFunctionBegin;
277: if (lu->ownDeviceCSR == PETSC_FALSE) {
278: Mat_SeqAIJCUSPARSE *cusparsestruct;
279: Mat_SeqAIJCUSPARSEMultStruct *matstruct;
280: CsrMatrix *csr;
282: PetscCall(MatSeqAIJCUSPARSECopyToGPU(A));
283: cusparsestruct = (Mat_SeqAIJCUSPARSE *)A->spptr;
284: matstruct = (Mat_SeqAIJCUSPARSEMultStruct *)cusparsestruct->mat;
285: csr = (CsrMatrix *)matstruct->mat;
286: #if PetscDefined(USE_64BIT_INDICES)
287: *d_row_pi = thrust::raw_pointer_cast(csr->row_offsets->data());
288: *d_col_pi = thrust::raw_pointer_cast(csr->column_indices->data());
289: #else
290: *d_row_32 = thrust::raw_pointer_cast(csr->row_offsets->data());
291: *d_col_32 = thrust::raw_pointer_cast(csr->column_indices->data());
292: #endif
293: *d_val = (PetscScalar *)thrust::raw_pointer_cast(csr->values->data());
294: } else {
295: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
296: const PetscScalar *aa;
297: PetscDeviceContext dctx;
298: cudaStream_t stream;
300: PetscCall(MatCUDSSGetContext(&dctx, &stream));
301: if (valuesOnly == PETSC_FALSE) {
302: // Symbolic phase: upload structure (row offsets + column indices)
303: PetscCallCUDA(cudaMemcpyAsync(lu->d_row_offsets, a->i, (lu->n + 1) * sizeof(PetscInt), cudaMemcpyHostToDevice, stream));
304: PetscCallCUDA(cudaMemcpyAsync(lu->d_col_indices, a->j, lu->nnz * sizeof(PetscInt), cudaMemcpyHostToDevice, stream));
305: PetscCall(PetscLogCpuToGpu((lu->n + 1) * sizeof(PetscInt) + lu->nnz * sizeof(PetscInt)));
306: }
307: // MatSeqAIJGetArrayRead(), unlike a->a, syncs device values of derived types to the host
308: PetscCall(MatSeqAIJGetArrayRead(A, &aa));
309: PetscCallCUDA(cudaMemcpyAsync(lu->d_values, aa, lu->nnz * sizeof(PetscScalar), cudaMemcpyHostToDevice, stream));
310: PetscCall(PetscDeviceContextSynchronize(dctx));
311: PetscCall(MatSeqAIJRestoreArrayRead(A, &aa));
312: PetscCall(PetscLogCpuToGpu(lu->nnz * sizeof(PetscScalar)));
313: *d_row_pi = lu->d_row_offsets;
314: *d_col_pi = lu->d_col_indices;
315: *d_val = lu->d_values;
316: }
317: PetscFunctionReturn(PETSC_SUCCESS);
318: }
320: static PetscErrorCode MatFactorSymbolic_cuDSS(Mat F, Mat A, IS perm, cudssMatrixType_t mtype, cudssMatrixViewType_t mview)
321: {
322: Mat_cuDSS *lu = (Mat_cuDSS *)F->data;
323: Mat_SeqAIJ *a = (Mat_SeqAIJ *)A->data;
324: PetscInt m = A->rmap->n, n = A->cmap->n, nnz = a->nz;
325: PetscInt *d_row_pi = NULL, *d_col_pi = NULL; // PetscInt-width path
326: int *d_row_32 = NULL, *d_col_32 = NULL; // MATSEQAIJCUSPARSE path with 32-bit PetscInt
327: void *d_row, *d_col;
328: PetscScalar *d_val = NULL;
329: PetscBool isCUSPARSE;
330: PetscDeviceContext dctx;
331: cudaStream_t stream;
333: PetscFunctionBegin;
334: PetscCall(MatReset_cuDSS(lu));
335: PetscCheck(m == n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "cuDSS requires a square matrix, got %" PetscInt_FMT "x%" PetscInt_FMT, m, n);
336: if (m == 0) PetscFunctionReturn(PETSC_SUCCESS);
338: lu->n = n;
339: lu->nnz = nnz;
341: if (lu->handle == NULL) PetscCallCUDSS(cudssCreate(&lu->handle));
342: if (lu->config == NULL) PetscCallCUDSS(cudssConfigCreate(&lu->config));
343: PetscCall(MatSetFromOptions_cuDSS(F));
344: PetscCall(MatCUDSSSetMemHandler(lu->handle, lu->mempool));
345: PetscCallCUDSS(cudssDataCreate(lu->handle, &lu->data));
346: PetscCall(MatApplyConfig_cuDSS(lu));
347: if (perm != NULL) PetscCall(MatSetPermutation_cuDSS(F, perm));
349: PetscCall(PetscObjectTypeCompare((PetscObject)A, MATSEQAIJCUSPARSE, &isCUSPARSE));
350: lu->ownDeviceCSR = (PetscBool)(isCUSPARSE == PETSC_FALSE || A->boundtocpu == PETSC_TRUE);
351: if (lu->ownDeviceCSR == PETSC_TRUE) {
352: PetscCallCUDA(cudaMalloc((void **)&lu->d_row_offsets, (m + 1) * sizeof(PetscInt)));
353: PetscCallCUDA(cudaMalloc((void **)&lu->d_col_indices, nnz * sizeof(PetscInt)));
354: PetscCallCUDA(cudaMalloc((void **)&lu->d_values, nnz * sizeof(PetscScalar)));
355: }
357: PetscCall(MatCUDSSGetContext(&dctx, &stream));
358: PetscCallCUDSS(cudssSetStream(lu->handle, stream));
360: PetscCall(MatEnsureOnDevice_cuDSS(A, lu, PETSC_FALSE, &d_row_32, &d_col_32, &d_row_pi, &d_col_pi, &d_val));
361: #if PetscDefined(USE_64BIT_INDICES)
362: // Narrow the 64-bit PetscInt CSR index arrays to 32-bit on the device
363: PetscCheck(nnz <= INT_MAX && m < INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuDSS requires 32-bit CSR indices; matrix has m=%" PetscInt_FMT ", nnz=%" PetscInt_FMT " > INT_MAX", m, nnz);
364: PetscCallCUDA(cudaMalloc((void **)&lu->d_row_i32, (m + 1) * sizeof(int)));
365: PetscCallCUDA(cudaMalloc((void **)&lu->d_col_i32, nnz * sizeof(int)));
366: PetscCall(Petsc::device::cupm::impl::Interface<Petsc::device::cupm::DeviceType::CUDA>::PetscCUPMLaunchKernel1D(m + 1, 0, stream, MatNarrowIndices_Kernel, m + 1, d_row_pi, lu->d_row_i32));
367: if (nnz > 0) PetscCall(Petsc::device::cupm::impl::Interface<Petsc::device::cupm::DeviceType::CUDA>::PetscCUPMLaunchKernel1D(nnz, 0, stream, MatNarrowIndices_Kernel, nnz, d_col_pi, lu->d_col_i32));
368: d_row = lu->d_row_i32;
369: d_col = lu->d_col_i32;
370: #else
371: d_row = d_row_32 != NULL ? (void *)d_row_32 : (void *)d_row_pi;
372: d_col = d_col_32 != NULL ? (void *)d_col_32 : (void *)d_col_pi;
373: #endif
374: PetscCallCUDSS(cudssMatrixCreateCsr(&lu->cudss_A, m, n, nnz, d_row, NULL, d_col, d_val, CUDSS_R_32I, CUDSS_R_32I, CUDSS_SCALAR_TYPE, mtype, mview, CUDSS_BASE_ZERO));
376: PetscCallCUDSS(cudssMatrixCreateDn(&lu->cudss_b, n, 1, n, NULL, CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
377: PetscCallCUDSS(cudssMatrixCreateDn(&lu->cudss_x, n, 1, n, NULL, CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
378: // cuDSS needs the user permutation before the analysis phase
379: if (lu->userPermSet == PETSC_TRUE) {
380: PetscCallCUDSS(cudssDataSet(lu->handle, lu->data, CUDSS_DATA_USER_PERM, lu->h_user_perm, (size_t)n * sizeof(int)));
381: PetscCall(PetscInfo(F, "cuDSS: using user-supplied permutation of size %" PetscInt_FMT "\n", n));
382: }
383: // Analysis phase: reordering and symbolic factorization
384: PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_ANALYSIS, lu->config, lu->data, lu->cudss_A, lu->cudss_x, lu->cudss_b));
385: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
386: PetscFunctionReturn(PETSC_SUCCESS);
387: }
389: static PetscErrorCode MatLUFactorSymbolic_cuDSS(Mat F, Mat A, IS r, IS c, const MatFactorInfo *info)
390: {
391: PetscFunctionBegin;
392: if (r != NULL && c != NULL) PetscCall(PetscInfo(F, "cuDSS accepts only a single permutation; using row ordering IS, column ordering IS is ignored\n"));
393: PetscCall(MatFactorSymbolic_cuDSS(F, A, r != NULL ? r : c, CUDSS_MTYPE_GENERAL, CUDSS_MVIEW_FULL));
394: F->ops->lufactornumeric = MatFactorNumeric_cuDSS;
395: PetscFunctionReturn(PETSC_SUCCESS);
396: }
398: static PetscErrorCode MatCholeskyFactorSymbolic_cuDSS(Mat F, Mat A, IS perm, const MatFactorInfo *info)
399: {
400: cudssMatrixType_t mtype;
402: PetscFunctionBegin;
403: #if PetscDefined(USE_COMPLEX)
404: if (A->spd == PETSC_BOOL3_TRUE) mtype = CUDSS_MTYPE_HPD;
405: else if (A->hermitian == PETSC_BOOL3_TRUE) mtype = CUDSS_MTYPE_HERMITIAN;
406: else mtype = CUDSS_MTYPE_SYMMETRIC;
407: #else
408: if (A->spd == PETSC_BOOL3_TRUE) mtype = CUDSS_MTYPE_SPD;
409: else mtype = CUDSS_MTYPE_SYMMETRIC;
410: #endif
411: PetscCall(MatFactorSymbolic_cuDSS(F, A, perm, mtype, CUDSS_MVIEW_UPPER));
412: F->ops->choleskyfactornumeric = MatFactorNumeric_cuDSS;
413: PetscFunctionReturn(PETSC_SUCCESS);
414: }
416: static PetscErrorCode MatSolve_cuDSS(Mat F, Vec b, Vec x)
417: {
418: Mat_cuDSS *lu = (Mat_cuDSS *)F->data;
419: const PetscScalar *barray;
420: PetscScalar *xarray;
421: PetscMemType bmtype, xmtype;
422: Vec bcu = NULL, xcu = NULL;
423: PetscDeviceContext dctx;
424: cudaStream_t stream;
426: PetscFunctionBegin;
427: if (F->rmap->n == 0) PetscFunctionReturn(PETSC_SUCCESS);
428: /* Accept any vector whose array is device-resident (VECCUDA, VECKOKKOS on a CUDA
429: backend, etc.) directly without a temporary Vec or device-to-device copy. */
430: PetscCall(VecGetArrayReadAndMemType(b, &barray, &bmtype));
431: if (!PetscMemTypeDevice(bmtype)) {
432: PetscCall(VecRestoreArrayReadAndMemType(b, &barray));
433: PetscCall(PetscInfo(F, "Copying b from host to temporary VECCUDA; consider using a device Vec for better performance\n"));
434: PetscCall(VecCreate(PetscObjectComm((PetscObject)b), &bcu));
435: PetscCall(VecSetSizes(bcu, b->map->n, b->map->N));
436: PetscCall(VecSetType(bcu, VECCUDA));
437: PetscCall(VecCopy(b, bcu));
438: PetscCall(VecCUDAGetArrayRead(bcu, &barray));
439: }
440: PetscCall(VecGetArrayAndMemType(x, &xarray, &xmtype));
441: if (!PetscMemTypeDevice(xmtype)) {
442: PetscCall(VecRestoreArrayAndMemType(x, &xarray));
443: PetscCall(VecCreate(PetscObjectComm((PetscObject)x), &xcu));
444: PetscCall(VecSetSizes(xcu, x->map->n, x->map->N));
445: PetscCall(VecSetType(xcu, VECCUDA));
446: PetscCall(VecCUDAGetArrayWrite(xcu, &xarray));
447: }
448: PetscCallCUDSS(cudssMatrixSetValues(lu->cudss_b, (void *)barray));
449: PetscCallCUDSS(cudssMatrixSetValues(lu->cudss_x, xarray));
450: PetscCall(MatCUDSSGetContext(&dctx, &stream));
451: PetscCallCUDSS(cudssSetStream(lu->handle, stream));
452: PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_SOLVE, lu->config, lu->data, lu->cudss_A, lu->cudss_x, lu->cudss_b));
453: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
454: if (bcu != NULL) {
455: PetscCall(VecCUDARestoreArrayRead(bcu, &barray));
456: PetscCall(VecDestroy(&bcu));
457: } else PetscCall(VecRestoreArrayReadAndMemType(b, &barray));
458: if (xcu != NULL) {
459: PetscCall(VecCUDARestoreArrayWrite(xcu, &xarray));
460: PetscCall(VecCopy(xcu, x));
461: PetscCall(VecDestroy(&xcu));
462: } else PetscCall(VecRestoreArrayAndMemType(x, &xarray));
463: PetscFunctionReturn(PETSC_SUCCESS);
464: }
466: static PetscErrorCode MatMatSolve_cuDSS(Mat F, Mat B, Mat X)
467: {
468: Mat_cuDSS *lu = (Mat_cuDSS *)F->data;
469: const PetscScalar *barray;
470: PetscScalar *xarray;
471: PetscInt n, nrhs, nX, nrhsX, ldb, ldx, ncol;
472: cudssMatrix_t cudss_B = NULL, cudss_X = NULL;
473: Mat Bcuda = NULL, Xcuda = NULL;
474: PetscBool BisCUDA = PETSC_FALSE, XisCUDA = PETSC_FALSE;
475: PetscDeviceContext dctx;
476: cudaStream_t stream;
478: PetscFunctionBegin;
479: if (F->rmap->n == 0) PetscFunctionReturn(PETSC_SUCCESS);
480: PetscCall(PetscObjectTypeCompare((PetscObject)B, MATSEQDENSECUDA, &BisCUDA));
481: PetscCall(PetscObjectTypeCompare((PetscObject)X, MATSEQDENSECUDA, &XisCUDA));
482: if (BisCUDA == PETSC_FALSE) {
483: PetscCall(PetscInfo(F, "Converting B from host to MATDENSECUDA; consider using MATSEQDENSECUDA for better performance\n"));
484: PetscCall(MatConvert(B, MATDENSECUDA, MAT_INITIAL_MATRIX, &Bcuda));
485: } else if (B == X) PetscCall(MatDuplicate(B, MAT_COPY_VALUES, &Bcuda)); // solve out of place from a copy of B
486: else Bcuda = B;
487: if (XisCUDA == PETSC_FALSE) {
488: PetscInt m_x, n_x, M_x, N_x;
490: PetscCall(MatGetLocalSize(X, &m_x, &n_x));
491: PetscCall(MatGetSize(X, &M_x, &N_x));
492: PetscCall(MatCreateDenseCUDA(PetscObjectComm((PetscObject)X), m_x, n_x, M_x, N_x, NULL, &Xcuda));
493: } else Xcuda = X;
494: PetscCall(MatGetSize(Bcuda, &n, &nrhs));
495: PetscCall(MatGetSize(Xcuda, &nX, &nrhsX));
496: PetscCheck(n == nX && nrhs == nrhsX, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Incompatible matrix dimensions: B is %" PetscInt_FMT " x %" PetscInt_FMT ", X is %" PetscInt_FMT " x %" PetscInt_FMT, n, nrhs, nX, nrhsX);
497: PetscCheck(n == lu->n, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "B row count %" PetscInt_FMT " does not match factored matrix size %" PetscInt_FMT, n, lu->n);
498: PetscCall(MatDenseGetLDA(Bcuda, &ldb));
499: PetscCall(MatDenseGetLDA(Xcuda, &ldx));
500: PetscCall(MatDenseCUDAGetArrayRead(Bcuda, &barray));
501: PetscCall(MatDenseCUDAGetArrayWrite(Xcuda, &xarray));
502: PetscCall(MatCUDSSGetContext(&dctx, &stream));
503: PetscCallCUDSS(cudssSetStream(lu->handle, stream));
504: // cuDSS deterministic mode requires nrhs == 1, so solve column-by-column when enabled
505: ncol = lu->deterministic ? 1 : nrhs;
506: for (PetscInt j = 0; j < nrhs; j += ncol) {
507: PetscCallCUDSS(cudssMatrixCreateDn(&cudss_B, n, ncol, ldb, (void *)(barray + j * ldb), CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
508: PetscCallCUDSS(cudssMatrixCreateDn(&cudss_X, n, ncol, ldx, xarray + j * ldx, CUDSS_SCALAR_TYPE, CUDSS_LAYOUT_COL_MAJOR));
509: PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_SOLVE, lu->config, lu->data, lu->cudss_A, cudss_X, cudss_B));
510: PetscCallCUDSS(cudssMatrixDestroy(cudss_B));
511: PetscCallCUDSS(cudssMatrixDestroy(cudss_X));
512: }
513: PetscCall(PetscDeviceContextSynchronizeIfWithBarrier_Internal(dctx));
514: PetscCall(MatDenseCUDARestoreArrayRead(Bcuda, &barray));
515: PetscCall(MatDenseCUDARestoreArrayWrite(Xcuda, &xarray));
516: if (Bcuda != B) PetscCall(MatDestroy(&Bcuda));
517: if (XisCUDA == PETSC_FALSE) {
518: PetscCall(MatCopy(Xcuda, X, SAME_NONZERO_PATTERN));
519: PetscCall(MatDestroy(&Xcuda));
520: }
521: PetscFunctionReturn(PETSC_SUCCESS);
522: }
524: static PetscErrorCode MatFactorNumeric_cuDSS(Mat F, Mat A, const MatFactorInfo *info)
525: {
526: Mat_cuDSS *lu = (Mat_cuDSS *)F->data;
527: int *d_row_32 = NULL, *d_col_32 = NULL;
528: int cudss_info = 0;
529: PetscInt *d_row_pi = NULL, *d_col_pi = NULL;
530: PetscScalar *d_val = NULL;
531: size_t nwritten;
532: PetscDeviceContext dctx;
533: cudaStream_t stream;
535: PetscFunctionBegin;
536: if (F->rmap->n == 0) {
537: F->assembled = PETSC_TRUE;
538: PetscFunctionReturn(PETSC_SUCCESS);
539: }
540: PetscCall(MatEnsureOnDevice_cuDSS(A, lu, PETSC_TRUE, &d_row_32, &d_col_32, &d_row_pi, &d_col_pi, &d_val));
541: PetscCallCUDSS(cudssMatrixSetValues(lu->cudss_A, d_val));
542: PetscCall(MatCUDSSGetContext(&dctx, &stream));
543: PetscCallCUDSS(cudssSetStream(lu->handle, stream));
544: PetscCallCUDSS(cudssExecute(lu->handle, CUDSS_PHASE_FACTORIZATION, lu->config, lu->data, lu->cudss_A, lu->cudss_x, lu->cudss_b));
545: PetscCall(PetscDeviceContextSynchronize(dctx)); // CUDSS_DATA_INFO is valid only after the factorization completes
546: PetscCallCUDSS(cudssDataGet(lu->handle, lu->data, CUDSS_DATA_INFO, &cudss_info, sizeof(cudss_info), &nwritten));
547: if (cudss_info != 0) {
548: PetscCall(PetscInfo(F, "cuDSS factorization failed with CUDSS_DATA_INFO %d\n", cudss_info));
549: F->factorerrortype = MAT_FACTOR_NUMERIC_ZEROPIVOT;
550: }
551: F->assembled = PETSC_TRUE;
552: PetscFunctionReturn(PETSC_SUCCESS);
553: }
555: static PetscErrorCode MatFactorGetSolverType_seqaij_cudss(Mat A, MatSolverType *type)
556: {
557: PetscFunctionBegin;
558: *type = MATSOLVERCUDSS;
559: PetscFunctionReturn(PETSC_SUCCESS);
560: }
562: static PetscErrorCode MatSetPermutation_cuDSS(Mat F, IS perm)
563: {
564: Mat_cuDSS *lu = (Mat_cuDSS *)F->data;
565: PetscInt nrows, ncols, n;
566: const PetscInt *idx;
568: PetscFunctionBegin;
569: PetscCall(PetscFree(lu->h_user_perm));
570: PetscCall(MatGetLocalSize(F, &nrows, &ncols));
571: PetscCheck(nrows <= INT_MAX, PETSC_COMM_SELF, PETSC_ERR_SUP, "cuDSS requires 32-bit permutation indices; matrix has m=%" PetscInt_FMT " > INT_MAX", nrows);
572: PetscCall(ISGetLocalSize(perm, &n));
573: PetscCheck(n == nrows && n == ncols, PETSC_COMM_SELF, PETSC_ERR_ARG_SIZ, "Incompatible matrix and permutation sizes");
574: PetscCall(PetscMalloc1(n, &lu->h_user_perm));
575: PetscCall(ISGetIndices(perm, &idx));
576: for (PetscInt i = 0; i < n; i++) lu->h_user_perm[i] = (int)idx[i];
577: PetscCall(ISRestoreIndices(perm, &idx));
578: lu->userPermSet = PETSC_TRUE;
579: PetscFunctionReturn(PETSC_SUCCESS);
580: }
582: /*MC
583: MATSOLVERCUDSS = "cudss" - A solver package providing LU and Cholesky factorization for
584: sequential sparse matrices via the NVIDIA cuDSS GPU-accelerated sparse direct solver library.
586: Options Database Keys:
587: + -mat_cudss_reorder_alg (default|btf_colamd|colamd|amd|nested_dissection|none) - reordering algorithm
588: . -mat_cudss_pivot_type (auto|none|col|row|diagonal|local_block|bunch_kaufman) - pivoting type
589: . -mat_cudss_pivot_threshold threshold - Pivot threshold, default is 1.0
590: . -mat_cudss_pivot_epsilon epsilon - Pivot epsilon, default is 0.0
591: . -mat_cudss_use_matching (true|false) - Enable bipartite matching for heavy-diagonal scaling and permutation, default is false
592: . -mat_cudss_ir_n_steps nsteps - Number of iterative refinement steps, default is 0
593: . -mat_cudss_deterministic (true|false) - Enable bitwise-reproducible factorization and solve, default is false
594: - -mat_cudss_mempool (true|false) - Use a stream-ordered `cudaMemPool_t` for cuDSS device workspace allocations, default is true
596: Level: intermediate
598: Notes:
599: Registered for both `MATSEQAIJ` (host) and `MATSEQAIJCUSPARSE` (device) matrix types.
600: When the input matrix is `MATSEQAIJ`, the matrix data is transparently copied to the GPU.
601: `MatSolveTranspose()` is not supported. The `bunch_kaufman` value of `-mat_cudss_pivot_type`
602: applies to symmetric or Hermitian indefinite matrices.
604: By default cuDSS performs its own internal reordering (`MATORDERINGEXTERNAL`) during the symbolic phase,
605: selectable via `-mat_cudss_reorder_alg default`. A PETSc ordering can be selected with
606: `-pc_factor_mat_ordering_type` or `PCFactorSetMatOrderingType()`, or by passing a non-`NULL` `IS`
607: to `MatLUFactorSymbolic()` or `MatCholeskyFactorSymbolic()`.
609: `MatSolve()` and `MatMatSolve()` operate on the GPU. Host right-hand-side and solution
610: objects (`VECSEQ` vectors, `MATSEQDENSE` matrices) are accepted and transparently copied
611: to and from the device, but this incurs extra copies; for best performance supply
612: device vectors (e.g., `VECCUDA` or `VECKOKKOS`) and `MATSEQDENSECUDA` matrices.
614: .seealso: [](ch_matrices), `Mat`, `PCLU`, `PCCHOLESKY`, `PCFactorSetMatSolverType()`, `PCFactorSetMatOrderingType()`, `MatSolverType`, `MatLUFactorSymbolic()`, `MatCholeskyFactorSymbolic()`
615: M*/
617: static PetscErrorCode MatGetFactor_seqaij_cudss(Mat A, MatFactorType ftype, Mat *F)
618: {
619: Mat B;
620: Mat_cuDSS *lu;
621: PetscInt m = A->rmap->n, n = A->cmap->n;
622: int nvrtc_major, nvrtc_minor;
624: PetscFunctionBegin;
625: PetscCheck(ftype == MAT_FACTOR_LU || ftype == MAT_FACTOR_CHOLESKY, PETSC_COMM_SELF, PETSC_ERR_SUP, "Factor type not supported by cuDSS");
626: // Call nvrtcVersion() so --as-needed linkers record a direct libnvrtc dependency on libpetsc
627: PetscCheck(nvrtcVersion(&nvrtc_major, &nvrtc_minor) == NVRTC_SUCCESS, PETSC_COMM_SELF, PETSC_ERR_LIB, "nvrtcVersion() failed");
628: PetscCall(MatCreate(PetscObjectComm((PetscObject)A), &B));
629: PetscCall(MatSetSizes(B, m, n, PETSC_DETERMINE, PETSC_DETERMINE));
630: PetscCall(PetscStrallocpy(MATSOLVERCUDSS, &((PetscObject)B)->type_name));
631: PetscCall(MatSetUp(B));
632: B->trivialsymbolic = PETSC_FALSE;
633: B->factortype = ftype;
634: B->canuseordering = PETSC_TRUE;
635: PetscCall(PetscStrallocpy(MATORDERINGEXTERNAL, (char **)&B->preferredordering[MAT_FACTOR_LU]));
636: PetscCall(PetscStrallocpy(MATORDERINGEXTERNAL, (char **)&B->preferredordering[MAT_FACTOR_CHOLESKY]));
637: B->assembled = PETSC_TRUE;
638: B->preallocated = PETSC_TRUE;
640: if (ftype == MAT_FACTOR_LU) B->ops->lufactorsymbolic = MatLUFactorSymbolic_cuDSS;
641: else B->ops->choleskyfactorsymbolic = MatCholeskyFactorSymbolic_cuDSS;
642: B->ops->solve = MatSolve_cuDSS;
643: B->ops->matsolve = MatMatSolve_cuDSS;
644: B->ops->destroy = MatDestroy_cuDSS;
645: B->ops->view = MatView_cuDSS;
646: B->ops->getinfo = MatGetInfo_External;
648: PetscCall(PetscFree(B->solvertype));
649: PetscCall(PetscStrallocpy(MATSOLVERCUDSS, &B->solvertype));
650: PetscCall(PetscObjectComposeFunction((PetscObject)B, "MatFactorGetSolverType_C", MatFactorGetSolverType_seqaij_cudss));
652: PetscCall(PetscNew(&lu));
653: lu->reorderAlg = CUDSS_REORDERING_ALG_DEFAULT;
654: lu->pivotType = CUDSS_PIVOT_AUTO;
655: lu->pivotThreshold = 1.0;
656: lu->pivotEpsilon = 0.0;
657: lu->useMatching = 0;
658: lu->irNSteps = 0;
659: lu->deterministic = 0;
660: lu->mempool = PETSC_TRUE;
661: B->data = lu;
663: *F = B;
664: PetscFunctionReturn(PETSC_SUCCESS);
665: }
667: PETSC_INTERN PetscErrorCode MatSolverTypeRegister_cuDSS(void)
668: {
669: PetscFunctionBegin;
670: PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJ, MAT_FACTOR_LU, MatGetFactor_seqaij_cudss));
671: PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJ, MAT_FACTOR_CHOLESKY, MatGetFactor_seqaij_cudss));
672: PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJCUSPARSE, MAT_FACTOR_LU, MatGetFactor_seqaij_cudss));
673: PetscCall(MatSolverTypeRegister(MATSOLVERCUDSS, MATSEQAIJCUSPARSE, MAT_FACTOR_CHOLESKY, MatGetFactor_seqaij_cudss));
674: PetscFunctionReturn(PETSC_SUCCESS);
675: }