13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 30d0321e0SJeremy L Thompson // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 50d0321e0SJeremy L Thompson // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 70d0321e0SJeremy L Thompson 894b7b29bSJeremy L Thompson #ifndef CEED_CUDA_REF_H 994b7b29bSJeremy L Thompson #define CEED_CUDA_REF_H 100d0321e0SJeremy L Thompson 1149aac155SJeremy L Thompson #include <ceed.h> 120d0321e0SJeremy L Thompson #include <ceed/backend.h> 1349aac155SJeremy L Thompson #include <ceed/jit-source/cuda/cuda-types.h> 1449aac155SJeremy L Thompson #include <cublas_v2.h> 150d0321e0SJeremy L Thompson #include <cuda.h> 162b730f8bSJeremy L Thompson 170d0321e0SJeremy L Thompson typedef struct { 180d0321e0SJeremy L Thompson CeedScalar *h_array; 190d0321e0SJeremy L Thompson CeedScalar *h_array_borrowed; 200d0321e0SJeremy L Thompson CeedScalar *h_array_owned; 210d0321e0SJeremy L Thompson CeedScalar *d_array; 220d0321e0SJeremy L Thompson CeedScalar *d_array_borrowed; 230d0321e0SJeremy L Thompson CeedScalar *d_array_owned; 240d0321e0SJeremy L Thompson } CeedVector_Cuda; 250d0321e0SJeremy L Thompson 260d0321e0SJeremy L Thompson typedef struct { 270d0321e0SJeremy L Thompson CUmodule module; 28437930d1SJeremy L Thompson CUfunction StridedNoTranspose; 2958549094SSebastian Grimberg CUfunction StridedTranspose; 30437930d1SJeremy L Thompson CUfunction OffsetNoTranspose; 3158549094SSebastian Grimberg CUfunction OffsetTranspose; 3258549094SSebastian Grimberg CUfunction OffsetTransposeDet; 33437930d1SJeremy L Thompson CeedInt num_nodes; 340d0321e0SJeremy L Thompson CeedInt *h_ind; 350d0321e0SJeremy L Thompson CeedInt *h_ind_allocated; 360d0321e0SJeremy L Thompson CeedInt *d_ind; 370d0321e0SJeremy L Thompson CeedInt *d_ind_allocated; 38437930d1SJeremy L Thompson CeedInt *d_t_offsets; 39437930d1SJeremy L Thompson CeedInt *d_t_indices; 40437930d1SJeremy L Thompson CeedInt *d_l_vec_indices; 410d0321e0SJeremy L Thompson } CeedElemRestriction_Cuda; 420d0321e0SJeremy L Thompson 43437930d1SJeremy L Thompson typedef struct { 44437930d1SJeremy L Thompson CUmodule module; 45437930d1SJeremy L Thompson CUfunction Interp; 46437930d1SJeremy L Thompson CUfunction Grad; 47437930d1SJeremy L Thompson CUfunction Weight; 48437930d1SJeremy L Thompson CeedScalar *d_interp_1d; 49437930d1SJeremy L Thompson CeedScalar *d_grad_1d; 50437930d1SJeremy L Thompson CeedScalar *d_q_weight_1d; 51437930d1SJeremy L Thompson } CeedBasis_Cuda; 52437930d1SJeremy L Thompson 53437930d1SJeremy L Thompson typedef struct { 54437930d1SJeremy L Thompson CUmodule module; 55437930d1SJeremy L Thompson CUfunction Interp; 56*d075f50bSSebastian Grimberg CUfunction InterpTranspose; 57*d075f50bSSebastian Grimberg CUfunction Deriv; 58*d075f50bSSebastian Grimberg CUfunction DerivTranspose; 59437930d1SJeremy L Thompson CUfunction Weight; 60437930d1SJeremy L Thompson CeedScalar *d_interp; 61437930d1SJeremy L Thompson CeedScalar *d_grad; 62*d075f50bSSebastian Grimberg CeedScalar *d_div; 63*d075f50bSSebastian Grimberg CeedScalar *d_curl; 64437930d1SJeremy L Thompson CeedScalar *d_q_weight; 65437930d1SJeremy L Thompson } CeedBasisNonTensor_Cuda; 66437930d1SJeremy L Thompson 670d0321e0SJeremy L Thompson typedef struct { 680d0321e0SJeremy L Thompson CUmodule module; 69437930d1SJeremy L Thompson char *qfunction_name; 70437930d1SJeremy L Thompson char *qfunction_source; 71437930d1SJeremy L Thompson CUfunction QFunction; 720d0321e0SJeremy L Thompson Fields_Cuda fields; 730d0321e0SJeremy L Thompson void *d_c; 740d0321e0SJeremy L Thompson } CeedQFunction_Cuda; 750d0321e0SJeremy L Thompson 760d0321e0SJeremy L Thompson typedef struct { 770d0321e0SJeremy L Thompson void *h_data; 780d0321e0SJeremy L Thompson void *h_data_borrowed; 790d0321e0SJeremy L Thompson void *h_data_owned; 800d0321e0SJeremy L Thompson void *d_data; 810d0321e0SJeremy L Thompson void *d_data_borrowed; 820d0321e0SJeremy L Thompson void *d_data_owned; 830d0321e0SJeremy L Thompson } CeedQFunctionContext_Cuda; 840d0321e0SJeremy L Thompson 850d0321e0SJeremy L Thompson typedef struct { 860d0321e0SJeremy L Thompson CUmodule module; 870d0321e0SJeremy L Thompson CUfunction linearDiagonal; 880d0321e0SJeremy L Thompson CUfunction linearPointBlock; 89ca735530SJeremy L Thompson CeedBasis basis_in, basis_out; 90506b1a0cSSebastian Grimberg CeedElemRestriction diag_rstr, point_block_diag_rstr; 91ca735530SJeremy L Thompson CeedVector elem_diag, point_block_elem_diag; 92ca735530SJeremy L Thompson CeedInt num_e_mode_in, num_e_mode_out, num_nodes; 93ca735530SJeremy L Thompson CeedEvalMode *h_e_mode_in, *h_e_mode_out; 94ca735530SJeremy L Thompson CeedEvalMode *d_e_mode_in, *d_e_mode_out; 95ca735530SJeremy L Thompson CeedScalar *d_identity, *d_interp_in, *d_interp_out, *d_grad_in, *d_grad_out; 960d0321e0SJeremy L Thompson } CeedOperatorDiag_Cuda; 970d0321e0SJeremy L Thompson 980d0321e0SJeremy L Thompson typedef struct { 99cc132f9aSnbeams CUmodule module; 100cc132f9aSnbeams CUfunction linearAssemble; 101ca735530SJeremy L Thompson CeedInt num_elem, block_size_x, block_size_y, elem_per_block; 102cc132f9aSnbeams CeedScalar *d_B_in, *d_B_out; 103cc132f9aSnbeams } CeedOperatorAssemble_Cuda; 104cc132f9aSnbeams 105cc132f9aSnbeams typedef struct { 106ca735530SJeremy L Thompson CeedVector *e_vecs; // E-vectors, inputs followed by outputs 107ca735530SJeremy L Thompson CeedVector *q_vecs_in; // Input Q-vectors needed to apply operator 108ca735530SJeremy L Thompson CeedVector *q_vecs_out; // Output Q-vectors needed to apply operator 109ca735530SJeremy L Thompson CeedInt num_inputs, num_outputs; 110ca735530SJeremy L Thompson CeedInt num_active_in, num_active_out; 111ca735530SJeremy L Thompson CeedVector *qf_active_in; 1120d0321e0SJeremy L Thompson CeedOperatorDiag_Cuda *diag; 113cc132f9aSnbeams CeedOperatorAssemble_Cuda *asmb; 1140d0321e0SJeremy L Thompson } CeedOperator_Cuda; 1150d0321e0SJeremy L Thompson 116eb7e6cafSJeremy L Thompson CEED_INTERN int CeedGetCublasHandle_Cuda(Ceed ceed, cublasHandle_t *handle); 1170d0321e0SJeremy L Thompson 1181f9221feSJeremy L Thompson CEED_INTERN int CeedVectorCreate_Cuda(CeedSize n, CeedVector vec); 1190d0321e0SJeremy L Thompson 1200305e208SSebastian Grimberg CEED_INTERN int CeedElemRestrictionCreate_Cuda(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *indices, const bool *orients, 1210c73c039SSebastian Grimberg const CeedInt8 *curl_orients, CeedElemRestriction r); 1220d0321e0SJeremy L Thompson 1232b730f8bSJeremy L Thompson CEED_INTERN int CeedBasisCreateTensorH1_Cuda(CeedInt dim, CeedInt P_1d, CeedInt Q_1d, const CeedScalar *interp_1d, const CeedScalar *grad_1d, 1246574a04fSJeremy L Thompson const CeedScalar *q_ref_1d, const CeedScalar *q_weight_1d, CeedBasis basis); 12551475c7cSJeremy L Thompson CEED_INTERN int CeedBasisCreateH1_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 12651475c7cSJeremy L Thompson const CeedScalar *grad, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 127*d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHdiv_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 128*d075f50bSSebastian Grimberg const CeedScalar *div, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 129*d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHcurl_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 130*d075f50bSSebastian Grimberg const CeedScalar *curl, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 1310d0321e0SJeremy L Thompson 1320d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionCreate_Cuda(CeedQFunction qf); 1330d0321e0SJeremy L Thompson 1340d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionContextCreate_Cuda(CeedQFunctionContext ctx); 1350d0321e0SJeremy L Thompson 1360d0321e0SJeremy L Thompson CEED_INTERN int CeedOperatorCreate_Cuda(CeedOperator op); 1370d0321e0SJeremy L Thompson 13894b7b29bSJeremy L Thompson #endif // CEED_CUDA_REF_H 139