Actual source code: cupmblasinterface.hpp
1: #pragma once
3: #include <petsc/private/cupminterface.hpp>
5: #include <limits> // std::numeric_limits
7: namespace Petsc
8: {
10: namespace device
11: {
13: namespace cupm
14: {
16: namespace impl
17: {
19: #define PetscCallCUPMBLAS_(__abort_fn__, __comm__, ...) \
20: do { \
21: PetscStackUpdateLine; \
22: const cupmBlasError_t cberr_p_ = __VA_ARGS__; \
23: if (PetscUnlikely(cberr_p_ != CUPMBLAS_STATUS_SUCCESS)) { \
24: if ((cberr_p_ == CUPMBLAS_STATUS_NOT_INITIALIZED || cberr_p_ == CUPMBLAS_STATUS_ALLOC_FAILED) && PetscDeviceInitialized(PETSC_DEVICE_CUPM())) { \
25: __abort_fn__(__comm__, PETSC_ERR_GPU_RESOURCE, \
26: "%s error %d (%s). Reports not initialized or alloc failed; " \
27: "this indicates the GPU may have run out resources", \
28: cupmBlasName(), static_cast<PetscErrorCode>(cberr_p_), cupmBlasGetErrorName(cberr_p_)); \
29: } \
30: __abort_fn__(__comm__, PETSC_ERR_GPU, "%s error %d (%s)", cupmBlasName(), static_cast<PetscErrorCode>(cberr_p_), cupmBlasGetErrorName(cberr_p_)); \
31: } \
32: } while (0)
34: #define PetscCallCUPMBLAS(...) PetscCallCUPMBLAS_(SETERRQ, PETSC_COMM_SELF, __VA_ARGS__)
35: #define PetscCallCUPMBLASAbort(comm_, ...) PetscCallCUPMBLAS_(SETERRABORT, comm_, __VA_ARGS__)
37: // given cupmBlasaxpy() then
38: // T = PETSC_CUPBLAS_FP_TYPE
39: // given cupmBlasnrm2() then
40: // T = PETSC_CUPMBLAS_FP_INPUT_TYPE
41: // u = PETSC_CUPMBLAS_FP_RETURN_TYPE
42: #if PetscDefined(USE_COMPLEX)
43: #if PetscDefined(USE_REAL_SINGLE)
44: #define PETSC_CUPMBLAS_FP_TYPE_U C
45: #define PETSC_CUPMBLAS_FP_TYPE_L c
46: #define PETSC_CUPMBLAS_FP_INPUT_TYPE_U S
47: #define PETSC_CUPMBLAS_FP_INPUT_TYPE_L s
48: #elif PetscDefined(USE_REAL_DOUBLE)
49: #define PETSC_CUPMBLAS_FP_TYPE_U Z
50: #define PETSC_CUPMBLAS_FP_TYPE_L z
51: #define PETSC_CUPMBLAS_FP_INPUT_TYPE_U D
52: #define PETSC_CUPMBLAS_FP_INPUT_TYPE_L d
53: #endif
54: #define PETSC_CUPMBLAS_FP_RETURN_TYPE_U PETSC_CUPMBLAS_FP_TYPE_U
55: #define PETSC_CUPMBLAS_FP_RETURN_TYPE_L PETSC_CUPMBLAS_FP_TYPE_L
56: #else
57: #if PetscDefined(USE_REAL_SINGLE)
58: #define PETSC_CUPMBLAS_FP_TYPE_U S
59: #define PETSC_CUPMBLAS_FP_TYPE_L s
60: #elif PetscDefined(USE_REAL_DOUBLE)
61: #define PETSC_CUPMBLAS_FP_TYPE_U D
62: #define PETSC_CUPMBLAS_FP_TYPE_L d
63: #endif
64: #define PETSC_CUPMBLAS_FP_INPUT_TYPE_U PETSC_CUPMBLAS_FP_TYPE_U
65: #define PETSC_CUPMBLAS_FP_INPUT_TYPE_L PETSC_CUPMBLAS_FP_TYPE_L
66: #define PETSC_CUPMBLAS_FP_RETURN_TYPE_U
67: #define PETSC_CUPMBLAS_FP_RETURN_TYPE_L
68: #endif // USE_COMPLEX
70: #if !defined(PETSC_CUPMBLAS_FP_TYPE_U) && !PetscDefined(USE_REAL___FLOAT128)
71: #error "Unsupported floating-point type for CUDA/HIP BLAS"
72: #endif
74: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_MODIFIED() - Helper macro to build a "modified"
75: // blas function whose return type does not match the input type
76: //
77: // input param:
78: // func - base suffix of the blas function, e.g. nrm2
79: //
80: // notes:
81: // requires PETSC_CUPMBLAS_FP_INPUT_TYPE to be defined as the blas floating point input type
82: // letter ("S" for real/complex single, "D" for real/complex double).
83: //
84: // requires PETSC_CUPMBLAS_FP_RETURN_TYPE to be defined as the blas floating point output type
85: // letter ("c" for complex single, "z" for complex double and for real
86: // single/double).
87: //
88: // In their infinite wisdom nvidia/amd have made the upper-case vs lower-case scheme
89: // infuriatingly inconsistent...
90: //
91: // example usage:
92: // #define PETSC_CUPMBLAS_FP_INPUT_TYPE S
93: // #define PETSC_CUPMBLAS_FP_RETURN_TYPE
94: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_MODIFIED(nrm2) -> Snrm2
95: //
96: // #define PETSC_CUPMBLAS_FP_INPUT_TYPE D
97: // #define PETSC_CUPMBLAS_FP_RETURN_TYPE z
98: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_MODIFIED(nrm2) -> Dznrm2
99: #define PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_MODIFIED(func) PetscConcat(PetscConcat(PETSC_CUPMBLAS_FP_INPUT_TYPE, PETSC_CUPMBLAS_FP_RETURN_TYPE), func)
101: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_IFPTYPE() - Helper macro to build Iamax and Iamin
102: // because they are both extra special
103: //
104: // input param:
105: // func - base suffix of the blas function, either amax or amin
106: //
107: // notes:
108: // The macro name literally stands for "I" ## "floating point type" because shockingly enough,
109: // that's what it does.
110: //
111: // requires PETSC_CUPMBLAS_FP_TYPE_L to be defined as the lower-case blas floating point input type
112: // letter ("s" for complex single, "z" for complex double, "s" for real single, and "d" for
113: // real double).
114: //
115: // example usage:
116: // #define PETSC_CUPMBLAS_FP_TYPE_L s
117: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_IFPTYPE(amax) -> Isamax
118: //
119: // #define PETSC_CUPMBLAS_FP_TYPE_L z
120: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_IFPTYPE(amin) -> Izamin
121: #define PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_IFPTYPE(func) PetscConcat(I, PetscConcat(PETSC_CUPMBLAS_FP_TYPE_L, func))
123: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_STANDARD() - Helper macro to build a "standard"
124: // blas function name
125: //
126: // input param:
127: // func - base suffix of the blas function, e.g. axpy, scal
128: //
129: // notes:
130: // requires PETSC_CUPMBLAS_FP_TYPE to be defined as the blas floating-point letter ("C" for
131: // complex single, "Z" for complex double, "S" for real single, "D" for real double).
132: //
133: // example usage:
134: // #define PETSC_CUPMBLAS_FP_TYPE S
135: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_STANDARD(axpy) -> Saxpy
136: //
137: // #define PETSC_CUPMBLAS_FP_TYPE Z
138: // PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_STANDARD(axpy) -> Zaxpy
139: #define PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_STANDARD(func) PetscConcat(PETSC_CUPMBLAS_FP_TYPE, func)
141: // PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT() - In case CUDA/HIP don't agree with our suffix
142: // one can provide both here
143: //
144: // input params:
145: // MACRO_SUFFIX - suffix to one of the above blas function builder macros, e.g. STANDARD or
146: // IFPTYPE
147: // our_suffix - the suffix of the alias function
148: // their_suffix - the suffix of the function being aliased
149: //
150: // notes:
151: // requires PETSC_CUPMBLAS_PREFIX to be defined as the specific CUDA/HIP blas function
152: // prefix. requires any other specific definitions required by the specific builder macro to
153: // also be defined. See PETSC_CUPM_ALIAS_FUNCTION_EXACT() for the exact expansion of the
154: // function alias.
155: //
156: // example usage:
157: // #define PETSC_CUPMBLAS_PREFIX cublas
158: // #define PETSC_CUPMBLAS_FP_TYPE C
159: // PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD,dot,dotc) ->
160: // template
161: // static constexpr auto cupmBlasXdot(T&&... args) *noexcept and returntype detection*
162: // {
163: // return cublasCdotc(std::forward(args)...);
164: // }
165: #define PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(MACRO_SUFFIX, our_suffix, their_suffix) \
166: PETSC_CUPM_ALIAS_FUNCTION(PetscConcat(cupmBlasX, our_suffix), PetscConcat(PETSC_CUPMBLAS_PREFIX, PetscConcat(PETSC_CUPMBLAS_BUILD_BLAS_FUNCTION_ALIAS_, MACRO_SUFFIX)(their_suffix)))
168: // PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION() - Alias a CUDA/HIP blas function
169: //
170: // input params:
171: // MACRO_SUFFIX - suffix to one of the above blas function builder macros, e.g. STANDARD or
172: // IFPTYPE
173: // suffix - the common suffix between CUDA and HIP of the alias function
174: //
175: // notes:
176: // see PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(), this macro just calls that one with "suffix" as
177: // "our_prefix" and "their_prefix"
178: #define PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(MACRO_SUFFIX, suffix) PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(MACRO_SUFFIX, suffix, suffix)
180: // PETSC_CUPMBLAS_ALIAS_FUNCTION() - Alias a CUDA/HIP library function
181: //
182: // input params:
183: // suffix - the common suffix between CUDA and HIP of the alias function
184: //
185: // notes:
186: // requires PETSC_CUPMBLAS_PREFIX to be defined as the specific CUDA/HIP blas library
187: // prefix. see PETSC_CUPMM_ALIAS_FUNCTION_EXACT() for the precise expansion of this macro.
188: //
189: // example usage:
190: // #define PETSC_CUPMBLAS_PREFIX hipblas
191: // PETSC_CUPMBLAS_ALIAS_FUNCTION(Create) ->
192: // template
193: // static constexpr auto cupmBlasCreate(T&&... args) *noexcept and returntype detection*
194: // {
195: // return hipblasCreate(std::forward(args)...);
196: // }
197: #define PETSC_CUPMBLAS_ALIAS_FUNCTION(suffix) PETSC_CUPM_ALIAS_FUNCTION(PetscConcat(cupmBlas, suffix), PetscConcat(PETSC_CUPMBLAS_PREFIX, suffix))
199: template <DeviceType>
200: struct BlasInterfaceImpl;
202: // Exists because HIP (for whatever godforsaken reason) has elected to define both their
203: // hipBlasHandle_t and hipSolverHandle_t as void *. So we cannot disambiguate them for overload
204: // resolution and hence need to wrap their types int this mess.
205: template <typename T, std::size_t I>
206: class cupmBlasHandleWrapper {
207: public:
208: constexpr cupmBlasHandleWrapper() noexcept = default;
209: constexpr cupmBlasHandleWrapper(T h) noexcept : handle_{std::move(h)} { static_assert(std::is_standard_layout<cupmBlasHandleWrapper<T, I>>::value, ""); }
211: cupmBlasHandleWrapper &operator=(std::nullptr_t) noexcept
212: {
213: handle_ = nullptr;
214: return *this;
215: }
217: operator T() const { return handle_; }
219: const T *ptr_to() const { return &handle_; }
220: T *ptr_to() { return &handle_; }
222: private:
223: T handle_{};
224: };
226: #if PetscDefined(HAVE_CUDA)
227: #define PETSC_CUPMBLAS_PREFIX cublas
228: #define PETSC_CUPMBLAS_PREFIX_U CUBLAS
229: #define PETSC_CUPMBLAS_FP_TYPE PETSC_CUPMBLAS_FP_TYPE_U
230: #define PETSC_CUPMBLAS_FP_INPUT_TYPE PETSC_CUPMBLAS_FP_INPUT_TYPE_U
231: #define PETSC_CUPMBLAS_FP_RETURN_TYPE PETSC_CUPMBLAS_FP_RETURN_TYPE_L
232: template <>
233: struct PETSC_SINGLE_LIBRARY_VISIBILITY_INTERNAL BlasInterfaceImpl<DeviceType::CUDA> : Interface<DeviceType::CUDA> {
234: // typedefs
235: using cupmBlasHandle_t = cupmBlasHandleWrapper<cublasHandle_t, 0>;
236: using cupmBlasError_t = cublasStatus_t;
237: using cupmBlasInt_t = int;
238: using cupmBlasPointerMode_t = cublasPointerMode_t;
240: // values
241: static const auto CUPMBLAS_STATUS_SUCCESS = CUBLAS_STATUS_SUCCESS;
242: static const auto CUPMBLAS_STATUS_NOT_INITIALIZED = CUBLAS_STATUS_NOT_INITIALIZED;
243: static const auto CUPMBLAS_STATUS_ALLOC_FAILED = CUBLAS_STATUS_ALLOC_FAILED;
244: static const auto CUPMBLAS_POINTER_MODE_HOST = CUBLAS_POINTER_MODE_HOST;
245: static const auto CUPMBLAS_POINTER_MODE_DEVICE = CUBLAS_POINTER_MODE_DEVICE;
246: static const auto CUPMBLAS_OP_T = CUBLAS_OP_T;
247: static const auto CUPMBLAS_OP_N = CUBLAS_OP_N;
248: static const auto CUPMBLAS_OP_C = CUBLAS_OP_C;
249: static const auto CUPMBLAS_FILL_MODE_LOWER = CUBLAS_FILL_MODE_LOWER;
250: static const auto CUPMBLAS_FILL_MODE_UPPER = CUBLAS_FILL_MODE_UPPER;
251: static const auto CUPMBLAS_SIDE_LEFT = CUBLAS_SIDE_LEFT;
252: static const auto CUPMBLAS_SIDE_RIGHT = CUBLAS_SIDE_RIGHT;
253: static const auto CUPMBLAS_DIAG_NON_UNIT = CUBLAS_DIAG_NON_UNIT;
255: // utility functions
256: PETSC_CUPMBLAS_ALIAS_FUNCTION(Create)
257: PETSC_CUPMBLAS_ALIAS_FUNCTION(Destroy)
258: PETSC_CUPMBLAS_ALIAS_FUNCTION(GetStream)
259: PETSC_CUPMBLAS_ALIAS_FUNCTION(SetStream)
260: PETSC_CUPMBLAS_ALIAS_FUNCTION(GetPointerMode)
261: PETSC_CUPMBLAS_ALIAS_FUNCTION(SetPointerMode)
263: // level 1 BLAS
264: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, axpy)
265: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, copy)
266: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, scal)
267: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD, dot, PetscIfPetscDefined(USE_COMPLEX, dotc, dot))
268: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD, dotu, PetscIfPetscDefined(USE_COMPLEX, dotu, dot))
269: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, swap)
270: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(MODIFIED, nrm2)
271: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(IFPTYPE, amax)
272: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(MODIFIED, asum)
274: // level 2 BLAS
275: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, gemv)
276: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, trmv)
277: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, trsv)
278: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, gbmv)
279: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, tbmv)
280: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, tbsv)
281: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD, hemv, PetscIfPetscDefined(USE_COMPLEX, hemv, symv))
283: // level 3 BLAS
284: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, gemm)
285: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, trsm)
287: // BLAS extensions
288: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, geam)
289: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, dgmm)
291: PETSC_NODISCARD static const char *cupmBlasGetErrorName(cupmBlasError_t status) noexcept { return PetscCUBLASGetErrorName(status); }
292: };
293: #undef PETSC_CUPMBLAS_PREFIX
294: #undef PETSC_CUPMBLAS_PREFIX_U
295: #undef PETSC_CUPMBLAS_FP_TYPE
296: #undef PETSC_CUPMBLAS_FP_INPUT_TYPE
297: #undef PETSC_CUPMBLAS_FP_RETURN_TYPE
298: #endif // PetscDefined(HAVE_CUDA)
300: #if PetscDefined(HAVE_HIP)
301: #define PETSC_CUPMBLAS_PREFIX hipblas
302: #define PETSC_CUPMBLAS_PREFIX_U HIPBLAS
303: #define PETSC_CUPMBLAS_FP_TYPE PETSC_CUPMBLAS_FP_TYPE_U
304: #define PETSC_CUPMBLAS_FP_INPUT_TYPE PETSC_CUPMBLAS_FP_INPUT_TYPE_U
305: #define PETSC_CUPMBLAS_FP_RETURN_TYPE PETSC_CUPMBLAS_FP_RETURN_TYPE_L
306: template <>
307: struct PETSC_SINGLE_LIBRARY_VISIBILITY_INTERNAL BlasInterfaceImpl<DeviceType::HIP> : Interface<DeviceType::HIP> {
308: // typedefs
309: using cupmBlasHandle_t = cupmBlasHandleWrapper<hipblasHandle_t, 0>;
310: using cupmBlasError_t = hipblasStatus_t;
311: using cupmBlasInt_t = int; // rocblas will have its own
312: using cupmBlasPointerMode_t = hipblasPointerMode_t;
314: // values
315: static const auto CUPMBLAS_STATUS_SUCCESS = HIPBLAS_STATUS_SUCCESS;
316: static const auto CUPMBLAS_STATUS_NOT_INITIALIZED = HIPBLAS_STATUS_NOT_INITIALIZED;
317: static const auto CUPMBLAS_STATUS_ALLOC_FAILED = HIPBLAS_STATUS_ALLOC_FAILED;
318: static const auto CUPMBLAS_POINTER_MODE_HOST = HIPBLAS_POINTER_MODE_HOST;
319: static const auto CUPMBLAS_POINTER_MODE_DEVICE = HIPBLAS_POINTER_MODE_DEVICE;
320: static const auto CUPMBLAS_OP_T = HIPBLAS_OP_T;
321: static const auto CUPMBLAS_OP_N = HIPBLAS_OP_N;
322: static const auto CUPMBLAS_OP_C = HIPBLAS_OP_C;
323: static const auto CUPMBLAS_FILL_MODE_LOWER = HIPBLAS_FILL_MODE_LOWER;
324: static const auto CUPMBLAS_FILL_MODE_UPPER = HIPBLAS_FILL_MODE_UPPER;
325: static const auto CUPMBLAS_SIDE_LEFT = HIPBLAS_SIDE_LEFT;
326: static const auto CUPMBLAS_SIDE_RIGHT = HIPBLAS_SIDE_RIGHT;
327: static const auto CUPMBLAS_DIAG_NON_UNIT = HIPBLAS_DIAG_NON_UNIT;
329: // utility functions
330: PETSC_CUPMBLAS_ALIAS_FUNCTION(Create)
331: PETSC_CUPMBLAS_ALIAS_FUNCTION(Destroy)
332: PETSC_CUPMBLAS_ALIAS_FUNCTION(GetStream)
333: PETSC_CUPMBLAS_ALIAS_FUNCTION(SetStream)
334: PETSC_CUPMBLAS_ALIAS_FUNCTION(GetPointerMode)
335: PETSC_CUPMBLAS_ALIAS_FUNCTION(SetPointerMode)
337: // level 1 BLAS
338: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, axpy)
339: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, copy)
340: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, scal)
341: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD, dot, PetscIfPetscDefined(USE_COMPLEX, dotc, dot))
342: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD, dotu, PetscIfPetscDefined(USE_COMPLEX, dotu, dot))
343: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, swap)
344: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(MODIFIED, nrm2)
345: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(IFPTYPE, amax)
346: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(MODIFIED, asum)
348: // level 2 BLAS
349: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, gemv)
350: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, trmv)
351: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, trsv)
352: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, gbmv)
353: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, tbmv)
354: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, tbsv)
355: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION_EXACT(STANDARD, hemv, PetscIfPetscDefined(USE_COMPLEX, hemv, symv))
357: // level 3 BLAS
358: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, gemm)
359: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, trsm)
361: // BLAS extensions
362: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, geam)
363: PETSC_CUPMBLAS_ALIAS_BLAS_FUNCTION(STANDARD, dgmm)
365: PETSC_NODISCARD static const char *cupmBlasGetErrorName(cupmBlasError_t status) noexcept { return PetscHIPBLASGetErrorName(status); }
366: };
367: #undef PETSC_CUPMBLAS_PREFIX
368: #undef PETSC_CUPMBLAS_PREFIX_U
369: #undef PETSC_CUPMBLAS_FP_TYPE
370: #undef PETSC_CUPMBLAS_FP_INPUT_TYPE
371: #undef PETSC_CUPMBLAS_FP_RETURN_TYPE
372: #endif // PetscDefined(HAVE_HIP)
374: #define PETSC_CUPMBLAS_IMPL_CLASS_HEADER(T) \
375: PETSC_CUPM_INHERIT_INTERFACE_TYPEDEFS_USING(T); \
376: /* introspection */ \
377: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasGetErrorName; \
378: /* types */ \
379: using cupmBlasHandle_t = typename ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasHandle_t; \
380: using cupmBlasError_t = typename ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasError_t; \
381: using cupmBlasInt_t = typename ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasInt_t; \
382: using cupmBlasPointerMode_t = typename ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasPointerMode_t; \
383: /* values */ \
384: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_STATUS_SUCCESS; \
385: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_STATUS_NOT_INITIALIZED; \
386: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_STATUS_ALLOC_FAILED; \
387: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_POINTER_MODE_HOST; \
388: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_POINTER_MODE_DEVICE; \
389: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_OP_T; \
390: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_OP_N; \
391: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_OP_C; \
392: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_FILL_MODE_LOWER; \
393: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_FILL_MODE_UPPER; \
394: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_SIDE_LEFT; \
395: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_SIDE_RIGHT; \
396: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::CUPMBLAS_DIAG_NON_UNIT; \
397: /* utility functions */ \
398: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasCreate; \
399: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasDestroy; \
400: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasGetStream; \
401: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasSetStream; \
402: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasGetPointerMode; \
403: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasSetPointerMode; \
404: /* level 1 BLAS */ \
405: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXaxpy; \
406: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXcopy; \
407: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXscal; \
408: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXdot; \
409: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXdotu; \
410: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXswap; \
411: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXnrm2; \
412: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXamax; \
413: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXasum; \
414: /* level 2 BLAS */ \
415: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXgemv; \
416: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXtrmv; \
417: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXtrsv; \
418: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXgbmv; \
419: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXtbmv; \
420: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXtbsv; \
421: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXhemv; \
422: /* level 3 BLAS */ \
423: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXgemm; \
424: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXtrsm; \
425: /* BLAS extensions */ \
426: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXgeam; \
427: using ::Petsc::device::cupm::impl::BlasInterfaceImpl<T>::cupmBlasXdgmm
429: // The actual interface class
430: template <DeviceType T>
431: struct BlasInterface : BlasInterfaceImpl<T> {
432: PETSC_CUPMBLAS_IMPL_CLASS_HEADER(T);
434: PETSC_NODISCARD static constexpr const char *cupmBlasName() noexcept { return T == DeviceType::CUDA ? "cuBLAS" : "hipBLAS"; }
436: static PetscErrorCode PetscCUPMBlasSetPointerModeFromPointer(cupmBlasHandle_t handle, const void *ptr) noexcept
437: {
438: auto mtype = PETSC_MEMTYPE_HOST;
440: PetscFunctionBegin;
441: PetscCall(PetscCUPMGetMemType(ptr, &mtype));
442: PetscCallCUPMBLAS(cupmBlasSetPointerMode(handle, PetscMemTypeDevice(mtype) ? CUPMBLAS_POINTER_MODE_DEVICE : CUPMBLAS_POINTER_MODE_HOST));
443: PetscFunctionReturn(PETSC_SUCCESS);
444: }
446: static PetscErrorCode checkCupmBlasIntCast(PetscInt x) noexcept
447: {
448: PetscFunctionBegin;
449: PetscCheck((std::is_same<PetscInt, cupmBlasInt_t>::value) || (x <= std::numeric_limits<cupmBlasInt_t>::max()), PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "%" PetscInt_FMT " is too big for %s, which may be restricted to 32-bit integers", x, cupmBlasName());
450: PetscCheck(x >= 0, PETSC_COMM_SELF, PETSC_ERR_ARG_OUTOFRANGE, "Passing negative integer (%" PetscInt_FMT ") to %s routine", x, cupmBlasName());
451: PetscFunctionReturn(PETSC_SUCCESS);
452: }
454: static PetscErrorCode PetscCUPMBlasIntCast(PetscInt x, cupmBlasInt_t *y) noexcept
455: {
456: PetscFunctionBegin;
457: *y = static_cast<cupmBlasInt_t>(x);
458: PetscCall(checkCupmBlasIntCast(x));
459: PetscFunctionReturn(PETSC_SUCCESS);
460: }
462: class CUPMBlasPointerModeGuard {
463: public:
464: CUPMBlasPointerModeGuard(const cupmBlasHandle_t &handle, cupmBlasPointerMode_t mode) noexcept : handle_{handle}
465: {
466: PetscFunctionBegin;
467: PetscCallCUPMBLASAbort(PETSC_COMM_SELF, cupmBlasGetPointerMode(handle, &this->saved_));
468: PetscCallCUPMBLASAbort(PETSC_COMM_SELF, cupmBlasSetPointerMode(handle, mode));
469: PetscFunctionReturnVoid();
470: }
472: CUPMBlasPointerModeGuard(const cupmBlasHandle_t &handle, PetscMemType mtype) noexcept : CUPMBlasPointerModeGuard{handle, PetscMemTypeDevice(mtype) ? CUPMBLAS_POINTER_MODE_DEVICE : CUPMBLAS_POINTER_MODE_HOST} { }
474: ~CUPMBlasPointerModeGuard() noexcept
475: {
476: PetscFunctionBegin;
477: PetscCallCUPMBLASAbort(PETSC_COMM_SELF, cupmBlasSetPointerMode(this->handle_, this->saved_));
478: PetscFunctionReturnVoid();
479: }
481: private:
482: cupmBlasHandle_t handle_;
483: cupmBlasPointerMode_t saved_;
484: };
485: };
487: #define PETSC_CUPMBLAS_INHERIT_INTERFACE_TYPEDEFS_USING(T) \
488: PETSC_CUPMBLAS_IMPL_CLASS_HEADER(T); \
489: using ::Petsc::device::cupm::impl::BlasInterface<T>::cupmBlasName; \
490: using ::Petsc::device::cupm::impl::BlasInterface<T>::PetscCUPMBlasSetPointerModeFromPointer; \
491: using ::Petsc::device::cupm::impl::BlasInterface<T>::checkCupmBlasIntCast; \
492: using ::Petsc::device::cupm::impl::BlasInterface<T>::PetscCUPMBlasIntCast; \
493: using CUPMBlasPointerModeGuard = typename ::Petsc::device::cupm::impl::BlasInterface<T>::CUPMBlasPointerModeGuard
495: #if PetscDefined(HAVE_CUDA)
496: extern template struct PETSC_SINGLE_LIBRARY_VISIBILITY_INTERNAL BlasInterface<DeviceType::CUDA>;
497: #endif
499: #if PetscDefined(HAVE_HIP)
500: extern template struct PETSC_SINGLE_LIBRARY_VISIBILITY_INTERNAL BlasInterface<DeviceType::HIP>;
501: #endif
503: } // namespace impl
505: } // namespace cupm
507: } // namespace device
509: } // namespace Petsc