13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 30d0321e0SJeremy L Thompson // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 50d0321e0SJeremy L Thompson // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 70d0321e0SJeremy L Thompson 894b7b29bSJeremy L Thompson #ifndef CEED_CUDA_REF_H 994b7b29bSJeremy L Thompson #define CEED_CUDA_REF_H 100d0321e0SJeremy L Thompson 1149aac155SJeremy L Thompson #include <ceed.h> 120d0321e0SJeremy L Thompson #include <ceed/backend.h> 1349aac155SJeremy L Thompson #include <ceed/jit-source/cuda/cuda-types.h> 1449aac155SJeremy L Thompson #include <cublas_v2.h> 150d0321e0SJeremy L Thompson #include <cuda.h> 162b730f8bSJeremy L Thompson 170d0321e0SJeremy L Thompson typedef struct { 180d0321e0SJeremy L Thompson CeedScalar *h_array; 190d0321e0SJeremy L Thompson CeedScalar *h_array_borrowed; 200d0321e0SJeremy L Thompson CeedScalar *h_array_owned; 210d0321e0SJeremy L Thompson CeedScalar *d_array; 220d0321e0SJeremy L Thompson CeedScalar *d_array_borrowed; 230d0321e0SJeremy L Thompson CeedScalar *d_array_owned; 240d0321e0SJeremy L Thompson } CeedVector_Cuda; 250d0321e0SJeremy L Thompson 260d0321e0SJeremy L Thompson typedef struct { 270d0321e0SJeremy L Thompson CUmodule module; 28437930d1SJeremy L Thompson CUfunction StridedNoTranspose; 2958549094SSebastian Grimberg CUfunction StridedTranspose; 30437930d1SJeremy L Thompson CUfunction OffsetNoTranspose; 3158549094SSebastian Grimberg CUfunction OffsetTranspose; 3258549094SSebastian Grimberg CUfunction OffsetTransposeDet; 33*dce49693SSebastian Grimberg CUfunction OrientedNoTranspose; 34*dce49693SSebastian Grimberg CUfunction OrientedTranspose; 35*dce49693SSebastian Grimberg CUfunction CurlOrientedNoTranspose; 36*dce49693SSebastian Grimberg CUfunction CurlOrientedTranspose; 37*dce49693SSebastian Grimberg CUfunction CurlOrientedUnsignedNoTranspose; 38*dce49693SSebastian Grimberg CUfunction CurlOrientedUnsignedTranspose; 39437930d1SJeremy L Thompson CeedInt num_nodes; 400d0321e0SJeremy L Thompson CeedInt *h_ind; 410d0321e0SJeremy L Thompson CeedInt *h_ind_allocated; 420d0321e0SJeremy L Thompson CeedInt *d_ind; 430d0321e0SJeremy L Thompson CeedInt *d_ind_allocated; 44437930d1SJeremy L Thompson CeedInt *d_t_offsets; 45437930d1SJeremy L Thompson CeedInt *d_t_indices; 46437930d1SJeremy L Thompson CeedInt *d_l_vec_indices; 47*dce49693SSebastian Grimberg bool *h_orients; 48*dce49693SSebastian Grimberg bool *h_orients_allocated; 49*dce49693SSebastian Grimberg bool *d_orients; 50*dce49693SSebastian Grimberg bool *d_orients_allocated; 51*dce49693SSebastian Grimberg CeedInt8 *h_curl_orients; 52*dce49693SSebastian Grimberg CeedInt8 *h_curl_orients_allocated; 53*dce49693SSebastian Grimberg CeedInt8 *d_curl_orients; 54*dce49693SSebastian Grimberg CeedInt8 *d_curl_orients_allocated; 550d0321e0SJeremy L Thompson } CeedElemRestriction_Cuda; 560d0321e0SJeremy L Thompson 57437930d1SJeremy L Thompson typedef struct { 58437930d1SJeremy L Thompson CUmodule module; 59437930d1SJeremy L Thompson CUfunction Interp; 60437930d1SJeremy L Thompson CUfunction Grad; 61437930d1SJeremy L Thompson CUfunction Weight; 62437930d1SJeremy L Thompson CeedScalar *d_interp_1d; 63437930d1SJeremy L Thompson CeedScalar *d_grad_1d; 64437930d1SJeremy L Thompson CeedScalar *d_q_weight_1d; 65437930d1SJeremy L Thompson } CeedBasis_Cuda; 66437930d1SJeremy L Thompson 67437930d1SJeremy L Thompson typedef struct { 68437930d1SJeremy L Thompson CUmodule module; 69437930d1SJeremy L Thompson CUfunction Interp; 70d075f50bSSebastian Grimberg CUfunction InterpTranspose; 71d075f50bSSebastian Grimberg CUfunction Deriv; 72d075f50bSSebastian Grimberg CUfunction DerivTranspose; 73437930d1SJeremy L Thompson CUfunction Weight; 74437930d1SJeremy L Thompson CeedScalar *d_interp; 75437930d1SJeremy L Thompson CeedScalar *d_grad; 76d075f50bSSebastian Grimberg CeedScalar *d_div; 77d075f50bSSebastian Grimberg CeedScalar *d_curl; 78437930d1SJeremy L Thompson CeedScalar *d_q_weight; 79437930d1SJeremy L Thompson } CeedBasisNonTensor_Cuda; 80437930d1SJeremy L Thompson 810d0321e0SJeremy L Thompson typedef struct { 820d0321e0SJeremy L Thompson CUmodule module; 83437930d1SJeremy L Thompson char *qfunction_name; 84437930d1SJeremy L Thompson char *qfunction_source; 85437930d1SJeremy L Thompson CUfunction QFunction; 860d0321e0SJeremy L Thompson Fields_Cuda fields; 870d0321e0SJeremy L Thompson void *d_c; 880d0321e0SJeremy L Thompson } CeedQFunction_Cuda; 890d0321e0SJeremy L Thompson 900d0321e0SJeremy L Thompson typedef struct { 910d0321e0SJeremy L Thompson void *h_data; 920d0321e0SJeremy L Thompson void *h_data_borrowed; 930d0321e0SJeremy L Thompson void *h_data_owned; 940d0321e0SJeremy L Thompson void *d_data; 950d0321e0SJeremy L Thompson void *d_data_borrowed; 960d0321e0SJeremy L Thompson void *d_data_owned; 970d0321e0SJeremy L Thompson } CeedQFunctionContext_Cuda; 980d0321e0SJeremy L Thompson 990d0321e0SJeremy L Thompson typedef struct { 1000d0321e0SJeremy L Thompson CUmodule module; 1010d0321e0SJeremy L Thompson CUfunction linearDiagonal; 1020d0321e0SJeremy L Thompson CUfunction linearPointBlock; 103ca735530SJeremy L Thompson CeedBasis basis_in, basis_out; 104506b1a0cSSebastian Grimberg CeedElemRestriction diag_rstr, point_block_diag_rstr; 105ca735530SJeremy L Thompson CeedVector elem_diag, point_block_elem_diag; 106ca735530SJeremy L Thompson CeedInt num_e_mode_in, num_e_mode_out, num_nodes; 107ca735530SJeremy L Thompson CeedEvalMode *h_e_mode_in, *h_e_mode_out; 108ca735530SJeremy L Thompson CeedEvalMode *d_e_mode_in, *d_e_mode_out; 109ca735530SJeremy L Thompson CeedScalar *d_identity, *d_interp_in, *d_interp_out, *d_grad_in, *d_grad_out; 1100d0321e0SJeremy L Thompson } CeedOperatorDiag_Cuda; 1110d0321e0SJeremy L Thompson 1120d0321e0SJeremy L Thompson typedef struct { 113cc132f9aSnbeams CUmodule module; 114cc132f9aSnbeams CUfunction linearAssemble; 115ca735530SJeremy L Thompson CeedInt num_elem, block_size_x, block_size_y, elem_per_block; 116cc132f9aSnbeams CeedScalar *d_B_in, *d_B_out; 117cc132f9aSnbeams } CeedOperatorAssemble_Cuda; 118cc132f9aSnbeams 119cc132f9aSnbeams typedef struct { 120ca735530SJeremy L Thompson CeedVector *e_vecs; // E-vectors, inputs followed by outputs 121ca735530SJeremy L Thompson CeedVector *q_vecs_in; // Input Q-vectors needed to apply operator 122ca735530SJeremy L Thompson CeedVector *q_vecs_out; // Output Q-vectors needed to apply operator 123ca735530SJeremy L Thompson CeedInt num_inputs, num_outputs; 124ca735530SJeremy L Thompson CeedInt num_active_in, num_active_out; 125ca735530SJeremy L Thompson CeedVector *qf_active_in; 1260d0321e0SJeremy L Thompson CeedOperatorDiag_Cuda *diag; 127cc132f9aSnbeams CeedOperatorAssemble_Cuda *asmb; 1280d0321e0SJeremy L Thompson } CeedOperator_Cuda; 1290d0321e0SJeremy L Thompson 130eb7e6cafSJeremy L Thompson CEED_INTERN int CeedGetCublasHandle_Cuda(Ceed ceed, cublasHandle_t *handle); 1310d0321e0SJeremy L Thompson 1321f9221feSJeremy L Thompson CEED_INTERN int CeedVectorCreate_Cuda(CeedSize n, CeedVector vec); 1330d0321e0SJeremy L Thompson 1340305e208SSebastian Grimberg CEED_INTERN int CeedElemRestrictionCreate_Cuda(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *indices, const bool *orients, 1350c73c039SSebastian Grimberg const CeedInt8 *curl_orients, CeedElemRestriction r); 1360d0321e0SJeremy L Thompson 1372b730f8bSJeremy L Thompson CEED_INTERN int CeedBasisCreateTensorH1_Cuda(CeedInt dim, CeedInt P_1d, CeedInt Q_1d, const CeedScalar *interp_1d, const CeedScalar *grad_1d, 1386574a04fSJeremy L Thompson const CeedScalar *q_ref_1d, const CeedScalar *q_weight_1d, CeedBasis basis); 13951475c7cSJeremy L Thompson CEED_INTERN int CeedBasisCreateH1_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 14051475c7cSJeremy L Thompson const CeedScalar *grad, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 141d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHdiv_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 142d075f50bSSebastian Grimberg const CeedScalar *div, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 143d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHcurl_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 144d075f50bSSebastian Grimberg const CeedScalar *curl, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 1450d0321e0SJeremy L Thompson 1460d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionCreate_Cuda(CeedQFunction qf); 1470d0321e0SJeremy L Thompson 1480d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionContextCreate_Cuda(CeedQFunctionContext ctx); 1490d0321e0SJeremy L Thompson 1500d0321e0SJeremy L Thompson CEED_INTERN int CeedOperatorCreate_Cuda(CeedOperator op); 1510d0321e0SJeremy L Thompson 15294b7b29bSJeremy L Thompson #endif // CEED_CUDA_REF_H 153