xref: /libCEED/backends/magma/ceed-magma-basis.c (revision 07d5b98a8feba68a643190b8ea9bcdac5c3e6570)
1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors.
2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
3 //
4 // SPDX-License-Identifier: BSD-2-Clause
5 //
6 // This file is part of CEED:  http://github.com/ceed
7 
8 #include <ceed.h>
9 #include <ceed/backend.h>
10 #include <ceed/jit-tools.h>
11 #include <string.h>
12 
13 #ifdef CEED_MAGMA_USE_HIP
14 #include "../hip/ceed-hip-common.h"
15 #include "../hip/ceed-hip-compile.h"
16 #else
17 #include "../cuda/ceed-cuda-common.h"
18 #include "../cuda/ceed-cuda-compile.h"
19 #endif
20 #include "ceed-magma-common.h"
21 #include "ceed-magma.h"
22 
23 #include "ceed-magma-gemm-nontensor.h"
24 #include "ceed-magma-gemm-selector.h"
25 
26 //------------------------------------------------------------------------------
27 // Basis apply - tensor
28 //------------------------------------------------------------------------------
29 static int CeedBasisApply_Magma(CeedBasis basis, CeedInt num_elem, CeedTransposeMode t_mode, CeedEvalMode e_mode, CeedVector u, CeedVector v) {
30   Ceed              ceed;
31   Ceed_Magma       *data;
32   CeedInt           dim, num_comp, num_nodes, P_1d, Q_1d, P, Q;
33   const CeedScalar *d_u;
34   CeedScalar       *d_v;
35   CeedBasis_Magma  *impl;
36 
37   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
38   CeedCallBackend(CeedGetData(ceed, &data));
39   CeedCallBackend(CeedBasisGetData(basis, &impl));
40   CeedCallBackend(CeedBasisGetDimension(basis, &dim));
41   CeedCallBackend(CeedBasisGetNumComponents(basis, &num_comp));
42   CeedCallBackend(CeedBasisGetNumNodes(basis, &num_nodes));
43   CeedCallBackend(CeedBasisGetNumNodes1D(basis, &P_1d));
44   CeedCallBackend(CeedBasisGetNumQuadraturePoints1D(basis, &Q_1d));
45   P = P_1d;
46   Q = Q_1d;
47   if (t_mode == CEED_TRANSPOSE) {
48     P = Q_1d;
49     Q = P_1d;
50   }
51 
52   // Read vectors
53   if (u != CEED_VECTOR_NONE) CeedCallBackend(CeedVectorGetArrayRead(u, CEED_MEM_DEVICE, &d_u));
54   else CeedCheck(e_mode == CEED_EVAL_WEIGHT, ceed, CEED_ERROR_BACKEND, "An input vector is required for this CeedEvalMode");
55   CeedCallBackend(CeedVectorGetArrayWrite(v, CEED_MEM_DEVICE, &d_v));
56 
57   // Apply basis operation
58   switch (e_mode) {
59     case CEED_EVAL_INTERP: {
60       // Define element sizes for dofs/quad
61       CeedInt elem_qpts_size = CeedIntPow(Q_1d, dim);
62       CeedInt elem_dofs_size = CeedIntPow(P_1d, dim);
63 
64       // E-vector ordering -------------- Q-vector ordering
65       //  component                        component
66       //    elem                             elem
67       //       node                            node
68 
69       // ---  Define strides for NOTRANSPOSE mode: ---
70       // Input (d_u) is E-vector, output (d_v) is Q-vector
71 
72       // Element strides
73       CeedInt u_elem_stride = elem_dofs_size;
74       CeedInt v_elem_stride = elem_qpts_size;
75       // Component strides
76       CeedInt u_comp_stride = num_elem * elem_dofs_size;
77       CeedInt v_comp_stride = num_elem * elem_qpts_size;
78       if (t_mode == CEED_TRANSPOSE) {
79         // Input (d_u) is Q-vector, output (d_v) is E-vector
80         // Element strides
81         v_elem_stride = elem_dofs_size;
82         u_elem_stride = elem_qpts_size;
83         // Component strides
84         v_comp_stride = num_elem * elem_dofs_size;
85         u_comp_stride = num_elem * elem_qpts_size;
86       }
87       CeedInt num_threads = 1;
88       CeedInt num_t_col   = 1;
89       CeedInt shared_mem  = 0;
90       CeedInt max_P_Q     = CeedIntMax(P, Q);
91 
92       switch (dim) {
93         case 1:
94           num_threads = max_P_Q;
95           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_1D);
96           shared_mem += sizeof(CeedScalar) * num_t_col * (num_comp * (1 * P + 1 * Q));
97           shared_mem += sizeof(CeedScalar) * (P * Q);
98           break;
99         case 2:
100           num_threads = max_P_Q;
101           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_2D);
102           shared_mem += P * Q * sizeof(CeedScalar);  // for sT
103           // for reforming rU we need P x P, and for the intermediate output we need P x Q
104           shared_mem += num_t_col * (P * max_P_Q * sizeof(CeedScalar));
105           break;
106         case 3:
107           num_threads = max_P_Q * max_P_Q;
108           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_3D);
109           shared_mem += sizeof(CeedScalar) * (P * Q);  // for sT
110           // rU needs P^2 x P, the intermediate output needs max(P^2 x Q, P x Q^2)
111           shared_mem += sizeof(CeedScalar) * num_t_col * (CeedIntMax(P * P * max_P_Q, P * Q * Q));
112           break;
113       }
114       CeedInt grid   = CeedDivUpInt(num_elem, num_t_col);
115       void   *args[] = {&impl->d_interp_1d, &d_u, &u_elem_stride, &u_comp_stride, &d_v, &v_elem_stride, &v_comp_stride, &num_elem};
116 
117       if (t_mode == CEED_TRANSPOSE) {
118         CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->InterpTranspose, grid, num_threads, num_t_col, 1, shared_mem, args));
119       } else {
120         CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Interp, grid, num_threads, num_t_col, 1, shared_mem, args));
121       }
122     } break;
123     case CEED_EVAL_GRAD: {
124       // Define element sizes for dofs/quad
125       CeedInt elem_qpts_size = CeedIntPow(Q_1d, dim);
126       CeedInt elem_dofs_size = CeedIntPow(P_1d, dim);
127 
128       // In CEED_NOTRANSPOSE mode:
129       // d_u is (P^dim x nc), column-major layout (nc = num_comp)
130       // d_v is (Q^dim x nc x dim), column-major layout (nc = num_comp)
131       // In CEED_TRANSPOSE mode, the sizes of d_u and d_v are switched.
132 
133       // E-vector ordering -------------- Q-vector ordering
134       //                                  dim
135       //  component                        component
136       //    elem                              elem
137       //       node                            node
138 
139       // ---  Define strides for NOTRANSPOSE mode: ---
140       // Input (d_u) is E-vector, output (d_v) is Q-vector
141 
142       // Element strides
143       CeedInt u_elem_stride = elem_dofs_size;
144       CeedInt v_elem_stride = elem_qpts_size;
145       // Component strides
146       CeedInt u_comp_stride = num_elem * elem_dofs_size;
147       CeedInt v_comp_stride = num_elem * elem_qpts_size;
148       // Dimension strides
149       CeedInt u_dim_stride = 0;
150       CeedInt v_dim_stride = num_elem * elem_qpts_size * num_comp;
151       if (t_mode == CEED_TRANSPOSE) {
152         // Input (d_u) is Q-vector, output (d_v) is E-vector
153         // Element strides
154         v_elem_stride = elem_dofs_size;
155         u_elem_stride = elem_qpts_size;
156         // Component strides
157         v_comp_stride = num_elem * elem_dofs_size;
158         u_comp_stride = num_elem * elem_qpts_size;
159         // Dimension strides
160         v_dim_stride = 0;
161         u_dim_stride = num_elem * elem_qpts_size * num_comp;
162       }
163       CeedInt num_threads = 1;
164       CeedInt num_t_col   = 1;
165       CeedInt shared_mem  = 0;
166       CeedInt max_P_Q     = CeedIntMax(P, Q);
167 
168       switch (dim) {
169         case 1:
170           num_threads = max_P_Q;
171           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_1D);
172           shared_mem += sizeof(CeedScalar) * num_t_col * (num_comp * (1 * P + 1 * Q));
173           shared_mem += sizeof(CeedScalar) * (P * Q);
174           break;
175         case 2:
176           num_threads = max_P_Q;
177           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_2D);
178           shared_mem += sizeof(CeedScalar) * 2 * P * Q;  // for sTinterp and sTgrad
179           // for reforming rU we need P x P, and for the intermediate output we need P x Q
180           shared_mem += sizeof(CeedScalar) * num_t_col * (P * max_P_Q);
181           break;
182         case 3:
183           num_threads = max_P_Q * max_P_Q;
184           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_3D);
185           shared_mem += sizeof(CeedScalar) * 2 * P * Q;  // for sTinterp and sTgrad
186           // rU needs P^2 x P, the intermediate outputs need (P^2 x Q + P x Q^2)
187           shared_mem += sizeof(CeedScalar) * num_t_col * CeedIntMax(P * P * P, (P * P * Q) + (P * Q * Q));
188           break;
189       }
190       CeedInt grid   = CeedDivUpInt(num_elem, num_t_col);
191       void   *args[] = {&impl->d_interp_1d, &impl->d_grad_1d, &d_u,          &u_elem_stride, &u_comp_stride, &u_dim_stride, &d_v,
192                         &v_elem_stride,     &v_comp_stride,   &v_dim_stride, &num_elem};
193 
194       if (t_mode == CEED_TRANSPOSE) {
195         CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->GradTranspose, grid, num_threads, num_t_col, 1, shared_mem, args));
196       } else {
197         CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Grad, grid, num_threads, num_t_col, 1, shared_mem, args));
198       }
199     } break;
200     case CEED_EVAL_WEIGHT: {
201       CeedCheck(t_mode != CEED_TRANSPOSE, ceed, CEED_ERROR_BACKEND, "CEED_EVAL_WEIGHT incompatible with CEED_TRANSPOSE");
202       CeedInt elem_dofs_size = CeedIntPow(Q, dim);
203       CeedInt num_threads    = 1;
204       CeedInt num_t_col      = 1;
205       CeedInt shared_mem     = 0;
206 
207       switch (dim) {
208         case 1:
209           num_threads = Q;
210           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_1D);
211           shared_mem += sizeof(CeedScalar) * Q;              // for d_q_weight_1d
212           shared_mem += sizeof(CeedScalar) * num_t_col * Q;  // for output
213           break;
214         case 2:
215           num_threads = Q;
216           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_2D);
217           shared_mem += sizeof(CeedScalar) * Q;  // for d_q_weight_1d
218           break;
219         case 3:
220           num_threads = Q * Q;
221           num_t_col   = MAGMA_BASIS_NTCOL(num_threads, MAGMA_MAXTHREADS_3D);
222           shared_mem += sizeof(CeedScalar) * Q;  // for d_q_weight_1d
223           break;
224       }
225       CeedInt grid   = CeedDivUpInt(num_elem, num_t_col);
226       void   *args[] = {&impl->d_q_weight_1d, &d_v, &elem_dofs_size, &num_elem};
227 
228       CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Weight, grid, num_threads, num_t_col, 1, shared_mem, args));
229     } break;
230     // LCOV_EXCL_START
231     case CEED_EVAL_DIV:
232     case CEED_EVAL_CURL:
233       return CeedError(ceed, CEED_ERROR_BACKEND, "%s not supported", CeedEvalModes[e_mode]);
234     case CEED_EVAL_NONE:
235       return CeedError(ceed, CEED_ERROR_BACKEND, "CEED_EVAL_NONE does not make sense in this context");
236       // LCOV_EXCL_STOP
237   }
238 
239   // Must sync to ensure completeness
240   ceed_magma_queue_sync(data->queue);
241 
242   // Restore vectors
243   if (e_mode != CEED_EVAL_WEIGHT) {
244     CeedCallBackend(CeedVectorRestoreArrayRead(u, &d_u));
245   }
246   CeedCallBackend(CeedVectorRestoreArray(v, &d_v));
247   return CEED_ERROR_SUCCESS;
248 }
249 
250 //------------------------------------------------------------------------------
251 // Basis apply - non-tensor
252 //------------------------------------------------------------------------------
253 static int CeedBasisApplyNonTensor_Magma(CeedBasis basis, CeedInt num_elem, CeedTransposeMode t_mode, CeedEvalMode e_mode, CeedVector u,
254                                          CeedVector v) {
255   Ceed                      ceed;
256   Ceed_Magma               *data;
257   CeedInt                   num_comp, num_nodes, num_qpts, P, Q, N;
258   const CeedScalar         *d_u;
259   CeedScalar               *d_v;
260   CeedBasisNonTensor_Magma *impl;
261 
262   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
263   CeedCallBackend(CeedGetData(ceed, &data));
264   CeedCallBackend(CeedBasisGetData(basis, &impl));
265   CeedCallBackend(CeedBasisGetNumComponents(basis, &num_comp));
266   CeedCallBackend(CeedBasisGetNumNodes(basis, &num_nodes));
267   CeedCallBackend(CeedBasisGetNumQuadraturePoints(basis, &num_qpts));
268   P = num_nodes;
269   Q = num_qpts;
270   N = num_elem * num_comp;
271 
272   // Read vectors
273   if (u != CEED_VECTOR_NONE) CeedCallBackend(CeedVectorGetArrayRead(u, CEED_MEM_DEVICE, &d_u));
274   else CeedCheck(e_mode == CEED_EVAL_WEIGHT, ceed, CEED_ERROR_BACKEND, "An input vector is required for this CeedEvalMode");
275   CeedCallBackend(CeedVectorGetArrayWrite(v, CEED_MEM_DEVICE, &d_v));
276 
277   // Compile kernels for N as needed
278   CeedInt iN = 0;
279   if (P <= MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_P && Q <= MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_Q && (e_mode != CEED_EVAL_WEIGHT || !impl->Weight)) {
280     CeedInt n_array[MAGMA_NONTENSOR_KERNEL_INSTANCES] = {MAGMA_NONTENSOR_KERNEL_N_VALUES};
281     CeedInt diff                                      = abs(n_array[iN] - N), idiff;
282 
283     for (CeedInt in = iN + 1; in < MAGMA_NONTENSOR_KERNEL_INSTANCES; in++) {
284       idiff = abs(n_array[in] - N);
285       if (idiff < diff) {
286         iN   = in;
287         diff = idiff;
288       }
289     }
290 
291     if (!impl->NB_interp[iN]) {
292       CeedFESpace fe_space;
293       CeedInt     q_comp_interp, q_comp_deriv;
294       Ceed        ceed_delegate;
295       char       *basis_kernel_source;
296       const char *basis_kernel_path, *weight_kernel_path;
297       char      **file_paths     = NULL;
298       CeedInt     num_file_paths = 0;
299       magma_int_t arch           = magma_getdevice_arch();
300 
301       // Tuning parameters for NB
302       CeedCallBackend(CeedBasisGetFESpace(basis, &fe_space));
303       CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_INTERP, &q_comp_interp));
304       switch (fe_space) {
305         case CEED_FE_SPACE_H1:
306           CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_GRAD, &q_comp_deriv));
307           break;
308         case CEED_FE_SPACE_HDIV:
309           CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_DIV, &q_comp_deriv));
310           break;
311         case CEED_FE_SPACE_HCURL:
312           CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_CURL, &q_comp_deriv));
313           break;
314       }
315       impl->NB_interp[iN]   = nontensor_rtc_get_nb(arch, 'n', q_comp_interp, P, Q, n_array[iN]);
316       impl->NB_interp_t[iN] = nontensor_rtc_get_nb(arch, 't', q_comp_interp, P, Q, n_array[iN]);
317       impl->NB_deriv[iN]    = nontensor_rtc_get_nb(arch, 'n', q_comp_deriv, P, Q, n_array[iN]);
318       impl->NB_deriv_t[iN]  = nontensor_rtc_get_nb(arch, 't', q_comp_deriv, P, Q, n_array[iN]);
319 
320       // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data
321       CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate));
322 
323       // Compile kernels
324       CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-interp-deriv-nontensor.h", &basis_kernel_path));
325       CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n");
326       CeedCallBackend(CeedLoadSourceAndInitializeBuffer(ceed, basis_kernel_path, &num_file_paths, &file_paths, &basis_kernel_source));
327       if (!impl->Weight) {
328         CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h", &weight_kernel_path));
329         CeedCallBackend(CeedLoadSourceToInitializedBuffer(ceed, weight_kernel_path, &num_file_paths, &file_paths, &basis_kernel_source));
330       }
331       CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n");
332       CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[iN], 8, "BASIS_Q_COMP_INTERP", q_comp_interp,
333                                        "BASIS_Q_COMP_DERIV", q_comp_deriv, "BASIS_P", P, "BASIS_Q", Q, "BASIS_NB_INTERP_N", impl->NB_interp[iN],
334                                        "BASIS_NB_INTERP_T", impl->NB_interp_t[iN], "BASIS_NB_DERIV_N", impl->NB_deriv[iN], "BASIS_NB_DERIV_T",
335                                        impl->NB_deriv_t[iN]));
336       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_interp_nontensor_n", &impl->Interp[iN]));
337       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_interp_nontensor_t", &impl->InterpTranspose[iN]));
338       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_deriv_nontensor_n", &impl->Deriv[iN]));
339       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_deriv_nontensor_t", &impl->DerivTranspose[iN]));
340       if (!impl->Weight) {
341         CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_weight_nontensor", &impl->Weight));
342         CeedCallBackend(CeedFree(&weight_kernel_path));
343       }
344       CeedCallBackend(CeedFree(&basis_kernel_path));
345       CeedCallBackend(CeedFree(&basis_kernel_source));
346       for (CeedInt i = 0; i < num_file_paths; i++) CeedCall(CeedFree(&file_paths[i]));
347       CeedCall(CeedFree(&file_paths));
348     }
349   }
350 
351   // Apply basis operation
352   if (e_mode != CEED_EVAL_WEIGHT) {
353     const CeedScalar *d_b = NULL;
354     CeedInt           q_comp, NB, M, K;
355     CeedMagmaFunction Kernel;
356 
357     switch (e_mode) {
358       case CEED_EVAL_INTERP:
359         d_b = impl->d_interp;
360         break;
361       case CEED_EVAL_GRAD:
362         d_b = impl->d_grad;
363         break;
364       case CEED_EVAL_DIV:
365         d_b = impl->d_div;
366         break;
367       case CEED_EVAL_CURL:
368         d_b = impl->d_curl;
369         break;
370       // LCOV_EXCL_START
371       case CEED_EVAL_WEIGHT:
372       case CEED_EVAL_NONE:
373         return CeedError(ceed, CEED_ERROR_BACKEND, "%s does not make sense in this context", CeedEvalModes[e_mode]);
374         // LCOV_EXCL_STOP
375     }
376     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, e_mode, &q_comp));
377     M = (t_mode == CEED_TRANSPOSE) ? P : Q, K = (t_mode == CEED_TRANSPOSE) ? Q : P;
378 
379     if (P <= MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_P && Q <= MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_Q) {
380       if (e_mode == CEED_EVAL_INTERP) {
381         if (t_mode == CEED_TRANSPOSE) {
382           Kernel = impl->InterpTranspose[iN];
383           NB     = impl->NB_interp_t[iN];
384         } else {
385           Kernel = impl->Interp[iN];
386           NB     = impl->NB_interp[iN];
387         }
388       } else {
389         if (t_mode == CEED_TRANSPOSE) {
390           Kernel = impl->DerivTranspose[iN];
391           NB     = impl->NB_deriv_t[iN];
392         } else {
393           Kernel = impl->Deriv[iN];
394           NB     = impl->NB_deriv[iN];
395         }
396       }
397       CeedInt num_t_col    = MAGMA_BASIS_NTCOL(M, MAGMA_MAXTHREADS_1D);
398       CeedInt grid         = CeedDivUpInt(N, num_t_col * NB);
399       CeedInt shared_mem_A = P * Q * sizeof(CeedScalar);
400       CeedInt shared_mem_B = num_t_col * K * NB * sizeof(CeedScalar);
401       CeedInt shared_mem   = (t_mode != CEED_TRANSPOSE && q_comp > 1) ? (shared_mem_A + shared_mem_B) : CeedIntMax(shared_mem_A, shared_mem_B);
402       void   *args[]       = {&N, &d_b, &d_u, &d_v};
403 
404       CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, Kernel, grid, M, num_t_col, 1, shared_mem, args));
405     } else {
406       for (CeedInt d = 0; d < q_comp; d++) {
407         if (t_mode == CEED_TRANSPOSE) {
408           const CeedScalar beta = (d > 0) ? 1.0 : 0.0;
409           magma_gemm_nontensor(MagmaNoTrans, MagmaNoTrans, P, N, Q, 1.0, d_b + d * P * Q, P, d_u + d * N * Q, Q, beta, d_v, P, data->queue);
410         } else {
411           magma_gemm_nontensor(MagmaTrans, MagmaNoTrans, Q, N, P, 1.0, d_b + d * P * Q, P, d_u, P, 0.0, d_v + d * N * Q, Q, data->queue);
412         }
413       }
414     }
415   } else {
416     CeedCheck(t_mode != CEED_TRANSPOSE, ceed, CEED_ERROR_BACKEND, "CEED_EVAL_WEIGHT incompatible with CEED_TRANSPOSE");
417     CeedInt num_t_col  = MAGMA_BASIS_NTCOL(Q, MAGMA_MAXTHREADS_1D);
418     CeedInt grid       = CeedDivUpInt(num_elem, num_t_col);
419     CeedInt shared_mem = Q * sizeof(CeedScalar) + num_t_col * Q * sizeof(CeedScalar);
420     void   *args[]     = {&num_elem, &impl->d_q_weight, &d_v};
421 
422     CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Weight, grid, Q, num_t_col, 1, shared_mem, args));
423   }
424 
425   // Must sync to ensure completeness
426   ceed_magma_queue_sync(data->queue);
427 
428   // Restore vectors
429   if (e_mode != CEED_EVAL_WEIGHT) {
430     CeedCallBackend(CeedVectorRestoreArrayRead(u, &d_u));
431   }
432   CeedCallBackend(CeedVectorRestoreArray(v, &d_v));
433   return CEED_ERROR_SUCCESS;
434 }
435 
436 //------------------------------------------------------------------------------
437 // Destroy tensor basis
438 //------------------------------------------------------------------------------
439 static int CeedBasisDestroy_Magma(CeedBasis basis) {
440   Ceed             ceed;
441   CeedBasis_Magma *impl;
442 
443   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
444   CeedCallBackend(CeedBasisGetData(basis, &impl));
445 #ifdef CEED_MAGMA_USE_HIP
446   CeedCallHip(ceed, hipModuleUnload(impl->module));
447 #else
448   CeedCallCuda(ceed, cuModuleUnload(impl->module));
449 #endif
450   CeedCallBackend(magma_free(impl->d_interp_1d));
451   CeedCallBackend(magma_free(impl->d_grad_1d));
452   CeedCallBackend(magma_free(impl->d_q_weight_1d));
453   CeedCallBackend(CeedFree(&impl));
454   return CEED_ERROR_SUCCESS;
455 }
456 
457 //------------------------------------------------------------------------------
458 // Destroy non-tensor basis
459 //------------------------------------------------------------------------------
460 static int CeedBasisDestroyNonTensor_Magma(CeedBasis basis) {
461   Ceed                      ceed;
462   CeedBasisNonTensor_Magma *impl;
463 
464   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
465   CeedCallBackend(CeedBasisGetData(basis, &impl));
466   for (CeedInt in = 0; in < MAGMA_NONTENSOR_KERNEL_INSTANCES; in++) {
467     if (impl->module[in]) {
468 #ifdef CEED_MAGMA_USE_HIP
469       CeedCallHip(ceed, hipModuleUnload(impl->module[in]));
470 #else
471       CeedCallCuda(ceed, cuModuleUnload(impl->module[in]));
472 #endif
473     }
474   }
475   CeedCallBackend(magma_free(impl->d_interp));
476   CeedCallBackend(magma_free(impl->d_grad));
477   CeedCallBackend(magma_free(impl->d_div));
478   CeedCallBackend(magma_free(impl->d_curl));
479   CeedCallBackend(magma_free(impl->d_q_weight));
480   CeedCallBackend(CeedFree(&impl));
481   return CEED_ERROR_SUCCESS;
482 }
483 
484 //------------------------------------------------------------------------------
485 // Create tensor
486 //------------------------------------------------------------------------------
487 int CeedBasisCreateTensorH1_Magma(CeedInt dim, CeedInt P_1d, CeedInt Q_1d, const CeedScalar *interp_1d, const CeedScalar *grad_1d,
488                                   const CeedScalar *q_ref_1d, const CeedScalar *q_weight_1d, CeedBasis basis) {
489   Ceed             ceed, ceed_delegate;
490   Ceed_Magma      *data;
491   char            *basis_kernel_source;
492   const char      *interp_kernel_path, *grad_kernel_path, *weight_kernel_path;
493   char           **file_paths     = NULL;
494   CeedInt          num_file_paths = 0;
495   CeedInt          num_comp;
496   CeedBasis_Magma *impl;
497 
498   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
499   CeedCallBackend(CeedGetData(ceed, &data));
500   CeedCallBackend(CeedCalloc(1, &impl));
501 
502   // Copy basis data to GPU
503   CeedCallBackend(magma_malloc((void **)&impl->d_q_weight_1d, Q_1d * sizeof(q_weight_1d[0])));
504   magma_setvector(Q_1d, sizeof(q_weight_1d[0]), q_weight_1d, 1, impl->d_q_weight_1d, 1, data->queue);
505   CeedCallBackend(magma_malloc((void **)&impl->d_interp_1d, Q_1d * P_1d * sizeof(interp_1d[0])));
506   magma_setvector(Q_1d * P_1d, sizeof(interp_1d[0]), interp_1d, 1, impl->d_interp_1d, 1, data->queue);
507   CeedCallBackend(magma_malloc((void **)&impl->d_grad_1d, Q_1d * P_1d * sizeof(grad_1d[0])));
508   magma_setvector(Q_1d * P_1d, sizeof(grad_1d[0]), grad_1d, 1, impl->d_grad_1d, 1, data->queue);
509 
510   // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data
511   CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate));
512 
513   // Compile kernels
514   CeedCallBackend(CeedBasisGetNumComponents(basis, &num_comp));
515   {
516     char   *interp_kernel_name_base = "ceed/jit-source/magma/magma-basis-interp";
517     CeedInt interp_kernel_name_len  = strlen(interp_kernel_name_base) + 6;
518     char    interp_kernel_name[interp_kernel_name_len];
519 
520     snprintf(interp_kernel_name, interp_kernel_name_len, "%s-%" CeedInt_FMT "d.h", interp_kernel_name_base, dim);
521     CeedCallBackend(CeedGetJitAbsolutePath(ceed, interp_kernel_name, &interp_kernel_path));
522   }
523   CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n");
524   CeedCallBackend(CeedLoadSourceAndInitializeBuffer(ceed, interp_kernel_path, &num_file_paths, &file_paths, &basis_kernel_source));
525   {
526     char   *grad_kernel_name_base = "ceed/jit-source/magma/magma-basis-grad";
527     CeedInt grad_kernel_name_len  = strlen(grad_kernel_name_base) + 6;
528     char    grad_kernel_name[grad_kernel_name_len];
529 
530     snprintf(grad_kernel_name, grad_kernel_name_len, "%s-%" CeedInt_FMT "d.h", grad_kernel_name_base, dim);
531     CeedCallBackend(CeedGetJitAbsolutePath(ceed, grad_kernel_name, &grad_kernel_path));
532   }
533   CeedCallBackend(CeedLoadSourceToInitializedBuffer(ceed, grad_kernel_path, &num_file_paths, &file_paths, &basis_kernel_source));
534   {
535     char   *weight_kernel_name_base = "ceed/jit-source/magma/magma-basis-weight";
536     CeedInt weight_kernel_name_len  = strlen(weight_kernel_name_base) + 6;
537     char    weight_kernel_name[weight_kernel_name_len];
538 
539     snprintf(weight_kernel_name, weight_kernel_name_len, "%s-%" CeedInt_FMT "d.h", weight_kernel_name_base, dim);
540     CeedCallBackend(CeedGetJitAbsolutePath(ceed, weight_kernel_name, &weight_kernel_path));
541   }
542   CeedCallBackend(CeedLoadSourceToInitializedBuffer(ceed, weight_kernel_path, &num_file_paths, &file_paths, &basis_kernel_source));
543   CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n");
544   CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module, 5, "BASIS_DIM", dim, "BASIS_NUM_COMP", num_comp, "BASIS_P",
545                                    P_1d, "BASIS_Q", Q_1d, "BASIS_MAX_P_Q", CeedIntMax(P_1d, Q_1d)));
546   switch (dim) {
547     case 1:
548       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpn_1d_kernel", &impl->Interp));
549       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpt_1d_kernel", &impl->InterpTranspose));
550       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradn_1d_kernel", &impl->Grad));
551       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradt_1d_kernel", &impl->GradTranspose));
552       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_weight_1d_kernel", &impl->Weight));
553       break;
554     case 2:
555       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpn_2d_kernel", &impl->Interp));
556       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpt_2d_kernel", &impl->InterpTranspose));
557       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradn_2d_kernel", &impl->Grad));
558       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradt_2d_kernel", &impl->GradTranspose));
559       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_weight_2d_kernel", &impl->Weight));
560       break;
561     case 3:
562       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpn_3d_kernel", &impl->Interp));
563       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpt_3d_kernel", &impl->InterpTranspose));
564       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradn_3d_kernel", &impl->Grad));
565       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradt_3d_kernel", &impl->GradTranspose));
566       CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_weight_3d_kernel", &impl->Weight));
567       break;
568   }
569   CeedCallBackend(CeedFree(&interp_kernel_path));
570   CeedCallBackend(CeedFree(&grad_kernel_path));
571   CeedCallBackend(CeedFree(&weight_kernel_path));
572   CeedCallBackend(CeedFree(&basis_kernel_source));
573   for (CeedInt i = 0; i < num_file_paths; i++) CeedCall(CeedFree(&file_paths[i]));
574   CeedCall(CeedFree(&file_paths));
575 
576   CeedCallBackend(CeedBasisSetData(basis, impl));
577 
578   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApply_Magma));
579   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroy_Magma));
580   return CEED_ERROR_SUCCESS;
581 }
582 
583 //------------------------------------------------------------------------------
584 // Create non-tensor H^1
585 //------------------------------------------------------------------------------
586 int CeedBasisCreateH1_Magma(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, const CeedScalar *grad,
587                             const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis) {
588   Ceed                      ceed;
589   Ceed_Magma               *data;
590   CeedBasisNonTensor_Magma *impl;
591 
592   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
593   CeedCallBackend(CeedGetData(ceed, &data));
594   CeedCallBackend(CeedCalloc(1, &impl));
595 
596   // Copy basis data to GPU
597   CeedCallBackend(magma_malloc((void **)&impl->d_q_weight, num_qpts * sizeof(q_weight[0])));
598   magma_setvector(num_qpts, sizeof(q_weight[0]), q_weight, 1, impl->d_q_weight, 1, data->queue);
599   if (interp) {
600     CeedInt q_comp_interp;
601 
602     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_INTERP, &q_comp_interp));
603     CeedCallBackend(magma_malloc((void **)&impl->d_interp, num_qpts * num_nodes * q_comp_interp * sizeof(interp[0])));
604     magma_setvector(num_qpts * num_nodes * q_comp_interp, sizeof(interp[0]), interp, 1, impl->d_interp, 1, data->queue);
605   }
606   if (grad) {
607     CeedInt q_comp_grad;
608 
609     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_GRAD, &q_comp_grad));
610     CeedCallBackend(magma_malloc((void **)&impl->d_grad, num_qpts * num_nodes * q_comp_grad * sizeof(grad[0])));
611     magma_setvector(num_qpts * num_nodes * q_comp_grad, sizeof(grad[0]), grad, 1, impl->d_grad, 1, data->queue);
612   }
613 
614   // Compile the weight kernel if it won't be compiled later on
615   if (num_nodes > MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_P || num_qpts > MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_Q) {
616     Ceed        ceed_delegate;
617     char       *basis_kernel_source;
618     const char *weight_kernel_path;
619 
620     // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data
621     CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate));
622 
623     // Compile weight kernel (the remainder of kernel compilation happens at first call to CeedBasisApply)
624     CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h", &weight_kernel_path));
625     CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n");
626     CeedCallBackend(CeedLoadSourceToBuffer(ceed, weight_kernel_path, &basis_kernel_source));
627     CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n");
628     CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[0], 1, "BASIS_Q", num_qpts));
629     CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[0], "magma_weight_nontensor", &impl->Weight));
630     CeedCallBackend(CeedFree(&weight_kernel_path));
631     CeedCallBackend(CeedFree(&basis_kernel_source));
632   }
633 
634   CeedCallBackend(CeedBasisSetData(basis, impl));
635 
636   // Register backend functions
637   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApplyNonTensor_Magma));
638   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroyNonTensor_Magma));
639   return CEED_ERROR_SUCCESS;
640 }
641 
642 //------------------------------------------------------------------------------
643 // Create non-tensor H(div)
644 //------------------------------------------------------------------------------
645 int CeedBasisCreateHdiv_Magma(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp,
646                               const CeedScalar *div, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis) {
647   Ceed                      ceed;
648   Ceed_Magma               *data;
649   CeedBasisNonTensor_Magma *impl;
650 
651   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
652   CeedCallBackend(CeedGetData(ceed, &data));
653   CeedCallBackend(CeedCalloc(1, &impl));
654 
655   // Copy basis data to GPU
656   CeedCallBackend(magma_malloc((void **)&impl->d_q_weight, num_qpts * sizeof(q_weight[0])));
657   magma_setvector(num_qpts, sizeof(q_weight[0]), q_weight, 1, impl->d_q_weight, 1, data->queue);
658   if (interp) {
659     CeedInt q_comp_interp;
660 
661     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_INTERP, &q_comp_interp));
662     CeedCallBackend(magma_malloc((void **)&impl->d_interp, num_qpts * num_nodes * q_comp_interp * sizeof(interp[0])));
663     magma_setvector(num_qpts * num_nodes * q_comp_interp, sizeof(interp[0]), interp, 1, impl->d_interp, 1, data->queue);
664   }
665   if (div) {
666     CeedInt q_comp_div;
667 
668     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_DIV, &q_comp_div));
669     CeedCallBackend(magma_malloc((void **)&impl->d_div, num_qpts * num_nodes * q_comp_div * sizeof(div[0])));
670     magma_setvector(num_qpts * num_nodes * q_comp_div, sizeof(div[0]), div, 1, impl->d_div, 1, data->queue);
671   }
672 
673   // Compile the weight kernel if it won't be compiled later on
674   if (num_nodes > MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_P || num_qpts > MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_Q) {
675     Ceed        ceed_delegate;
676     char       *basis_kernel_source;
677     const char *weight_kernel_path;
678 
679     // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data
680     CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate));
681 
682     // Compile weight kernel (the remainder of kernel compilation happens at first call to CeedBasisApply)
683     CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h", &weight_kernel_path));
684     CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n");
685     CeedCallBackend(CeedLoadSourceToBuffer(ceed, weight_kernel_path, &basis_kernel_source));
686     CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n");
687     CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[0], 1, "BASIS_Q", num_qpts));
688     CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[0], "magma_weight_nontensor", &impl->Weight));
689     CeedCallBackend(CeedFree(&weight_kernel_path));
690     CeedCallBackend(CeedFree(&basis_kernel_source));
691   }
692 
693   CeedCallBackend(CeedBasisSetData(basis, impl));
694 
695   // Register backend functions
696   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApplyNonTensor_Magma));
697   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroyNonTensor_Magma));
698   return CEED_ERROR_SUCCESS;
699 }
700 
701 //------------------------------------------------------------------------------
702 // Create non-tensor H(curl)
703 //------------------------------------------------------------------------------
704 int CeedBasisCreateHcurl_Magma(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp,
705                                const CeedScalar *curl, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis) {
706   Ceed                      ceed;
707   Ceed_Magma               *data;
708   CeedBasisNonTensor_Magma *impl;
709 
710   CeedCallBackend(CeedBasisGetCeed(basis, &ceed));
711   CeedCallBackend(CeedGetData(ceed, &data));
712   CeedCallBackend(CeedCalloc(1, &impl));
713 
714   // Copy basis data to GPU
715   CeedCallBackend(magma_malloc((void **)&impl->d_q_weight, num_qpts * sizeof(q_weight[0])));
716   magma_setvector(num_qpts, sizeof(q_weight[0]), q_weight, 1, impl->d_q_weight, 1, data->queue);
717   if (interp) {
718     CeedInt q_comp_interp;
719 
720     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_INTERP, &q_comp_interp));
721     CeedCallBackend(magma_malloc((void **)&impl->d_interp, num_qpts * num_nodes * q_comp_interp * sizeof(interp[0])));
722     magma_setvector(num_qpts * num_nodes * q_comp_interp, sizeof(interp[0]), interp, 1, impl->d_interp, 1, data->queue);
723   }
724   if (curl) {
725     CeedInt q_comp_curl;
726 
727     CeedCallBackend(CeedBasisGetNumQuadratureComponents(basis, CEED_EVAL_CURL, &q_comp_curl));
728     CeedCallBackend(magma_malloc((void **)&impl->d_curl, num_qpts * num_nodes * q_comp_curl * sizeof(curl[0])));
729     magma_setvector(num_qpts * num_nodes * q_comp_curl, sizeof(curl[0]), curl, 1, impl->d_curl, 1, data->queue);
730   }
731 
732   // Compile the weight kernel if it won't be compiled later on
733   if (num_nodes > MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_P || num_qpts > MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_Q) {
734     Ceed        ceed_delegate;
735     char       *basis_kernel_source;
736     const char *weight_kernel_path;
737 
738     // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data
739     CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate));
740 
741     // Compile weight kernel (the remainder of kernel compilation happens at first call to CeedBasisApply)
742     CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h", &weight_kernel_path));
743     CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n");
744     CeedCallBackend(CeedLoadSourceToBuffer(ceed, weight_kernel_path, &basis_kernel_source));
745     CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n");
746     CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[0], 1, "BASIS_Q", num_qpts));
747     CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[0], "magma_weight_nontensor", &impl->Weight));
748     CeedCallBackend(CeedFree(&weight_kernel_path));
749     CeedCallBackend(CeedFree(&basis_kernel_source));
750   }
751 
752   CeedCallBackend(CeedBasisSetData(basis, impl));
753 
754   // Register backend functions
755   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApplyNonTensor_Magma));
756   CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroyNonTensor_Magma));
757   return CEED_ERROR_SUCCESS;
758 }
759 
760 //------------------------------------------------------------------------------
761