13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 30d0321e0SJeremy L Thompson // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 50d0321e0SJeremy L Thompson // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 70d0321e0SJeremy L Thompson 894b7b29bSJeremy L Thompson #ifndef CEED_CUDA_REF_H 994b7b29bSJeremy L Thompson #define CEED_CUDA_REF_H 100d0321e0SJeremy L Thompson 1149aac155SJeremy L Thompson #include <ceed.h> 120d0321e0SJeremy L Thompson #include <ceed/backend.h> 1349aac155SJeremy L Thompson #include <ceed/jit-source/cuda/cuda-types.h> 1449aac155SJeremy L Thompson #include <cublas_v2.h> 150d0321e0SJeremy L Thompson #include <cuda.h> 162b730f8bSJeremy L Thompson 170d0321e0SJeremy L Thompson typedef struct { 180d0321e0SJeremy L Thompson CeedScalar *h_array; 190d0321e0SJeremy L Thompson CeedScalar *h_array_borrowed; 200d0321e0SJeremy L Thompson CeedScalar *h_array_owned; 210d0321e0SJeremy L Thompson CeedScalar *d_array; 220d0321e0SJeremy L Thompson CeedScalar *d_array_borrowed; 230d0321e0SJeremy L Thompson CeedScalar *d_array_owned; 240d0321e0SJeremy L Thompson } CeedVector_Cuda; 250d0321e0SJeremy L Thompson 260d0321e0SJeremy L Thompson typedef struct { 270d0321e0SJeremy L Thompson CUmodule module; 28437930d1SJeremy L Thompson CUfunction StridedNoTranspose; 2958549094SSebastian Grimberg CUfunction StridedTranspose; 30437930d1SJeremy L Thompson CUfunction OffsetNoTranspose; 3158549094SSebastian Grimberg CUfunction OffsetTranspose; 3258549094SSebastian Grimberg CUfunction OffsetTransposeDet; 33dce49693SSebastian Grimberg CUfunction OrientedNoTranspose; 34dce49693SSebastian Grimberg CUfunction OrientedTranspose; 35dce49693SSebastian Grimberg CUfunction CurlOrientedNoTranspose; 36dce49693SSebastian Grimberg CUfunction CurlOrientedTranspose; 37dce49693SSebastian Grimberg CUfunction CurlOrientedUnsignedNoTranspose; 38dce49693SSebastian Grimberg CUfunction CurlOrientedUnsignedTranspose; 39437930d1SJeremy L Thompson CeedInt num_nodes; 400d0321e0SJeremy L Thompson CeedInt *h_ind; 410d0321e0SJeremy L Thompson CeedInt *h_ind_allocated; 420d0321e0SJeremy L Thompson CeedInt *d_ind; 430d0321e0SJeremy L Thompson CeedInt *d_ind_allocated; 44437930d1SJeremy L Thompson CeedInt *d_t_offsets; 45437930d1SJeremy L Thompson CeedInt *d_t_indices; 46437930d1SJeremy L Thompson CeedInt *d_l_vec_indices; 47dce49693SSebastian Grimberg bool *h_orients; 48dce49693SSebastian Grimberg bool *h_orients_allocated; 49dce49693SSebastian Grimberg bool *d_orients; 50dce49693SSebastian Grimberg bool *d_orients_allocated; 51dce49693SSebastian Grimberg CeedInt8 *h_curl_orients; 52dce49693SSebastian Grimberg CeedInt8 *h_curl_orients_allocated; 53dce49693SSebastian Grimberg CeedInt8 *d_curl_orients; 54dce49693SSebastian Grimberg CeedInt8 *d_curl_orients_allocated; 550d0321e0SJeremy L Thompson } CeedElemRestriction_Cuda; 560d0321e0SJeremy L Thompson 57437930d1SJeremy L Thompson typedef struct { 58437930d1SJeremy L Thompson CUmodule module; 59437930d1SJeremy L Thompson CUfunction Interp; 60437930d1SJeremy L Thompson CUfunction Grad; 61437930d1SJeremy L Thompson CUfunction Weight; 62437930d1SJeremy L Thompson CeedScalar *d_interp_1d; 63437930d1SJeremy L Thompson CeedScalar *d_grad_1d; 64437930d1SJeremy L Thompson CeedScalar *d_q_weight_1d; 65437930d1SJeremy L Thompson } CeedBasis_Cuda; 66437930d1SJeremy L Thompson 67437930d1SJeremy L Thompson typedef struct { 68437930d1SJeremy L Thompson CUmodule module; 69437930d1SJeremy L Thompson CUfunction Interp; 70d075f50bSSebastian Grimberg CUfunction InterpTranspose; 71d075f50bSSebastian Grimberg CUfunction Deriv; 72d075f50bSSebastian Grimberg CUfunction DerivTranspose; 73437930d1SJeremy L Thompson CUfunction Weight; 74437930d1SJeremy L Thompson CeedScalar *d_interp; 75437930d1SJeremy L Thompson CeedScalar *d_grad; 76d075f50bSSebastian Grimberg CeedScalar *d_div; 77d075f50bSSebastian Grimberg CeedScalar *d_curl; 78437930d1SJeremy L Thompson CeedScalar *d_q_weight; 79437930d1SJeremy L Thompson } CeedBasisNonTensor_Cuda; 80437930d1SJeremy L Thompson 810d0321e0SJeremy L Thompson typedef struct { 820d0321e0SJeremy L Thompson CUmodule module; 83437930d1SJeremy L Thompson char *qfunction_name; 84437930d1SJeremy L Thompson char *qfunction_source; 85437930d1SJeremy L Thompson CUfunction QFunction; 860d0321e0SJeremy L Thompson Fields_Cuda fields; 870d0321e0SJeremy L Thompson void *d_c; 880d0321e0SJeremy L Thompson } CeedQFunction_Cuda; 890d0321e0SJeremy L Thompson 900d0321e0SJeremy L Thompson typedef struct { 910d0321e0SJeremy L Thompson void *h_data; 920d0321e0SJeremy L Thompson void *h_data_borrowed; 930d0321e0SJeremy L Thompson void *h_data_owned; 940d0321e0SJeremy L Thompson void *d_data; 950d0321e0SJeremy L Thompson void *d_data_borrowed; 960d0321e0SJeremy L Thompson void *d_data_owned; 970d0321e0SJeremy L Thompson } CeedQFunctionContext_Cuda; 980d0321e0SJeremy L Thompson 990d0321e0SJeremy L Thompson typedef struct { 1000d0321e0SJeremy L Thompson CUmodule module; 101*004e4986SSebastian Grimberg CUfunction LinearDiagonal; 102*004e4986SSebastian Grimberg CUfunction LinearPointBlock; 103506b1a0cSSebastian Grimberg CeedElemRestriction diag_rstr, point_block_diag_rstr; 104ca735530SJeremy L Thompson CeedVector elem_diag, point_block_elem_diag; 105*004e4986SSebastian Grimberg CeedEvalMode *d_eval_modes_in, *d_eval_modes_out; 106*004e4986SSebastian Grimberg CeedScalar *d_identity, *d_interp_in, *d_grad_in, *d_div_in, *d_curl_in; 107*004e4986SSebastian Grimberg CeedScalar *d_interp_out, *d_grad_out, *d_div_out, *d_curl_out; 1080d0321e0SJeremy L Thompson } CeedOperatorDiag_Cuda; 1090d0321e0SJeremy L Thompson 1100d0321e0SJeremy L Thompson typedef struct { 111cc132f9aSnbeams CUmodule module; 112*004e4986SSebastian Grimberg CUfunction LinearAssemble; 113*004e4986SSebastian Grimberg CeedInt block_size_x, block_size_y, elems_per_block; 114cc132f9aSnbeams CeedScalar *d_B_in, *d_B_out; 115cc132f9aSnbeams } CeedOperatorAssemble_Cuda; 116cc132f9aSnbeams 117cc132f9aSnbeams typedef struct { 118ca735530SJeremy L Thompson CeedVector *e_vecs; // E-vectors, inputs followed by outputs 119ca735530SJeremy L Thompson CeedVector *q_vecs_in; // Input Q-vectors needed to apply operator 120ca735530SJeremy L Thompson CeedVector *q_vecs_out; // Output Q-vectors needed to apply operator 121ca735530SJeremy L Thompson CeedInt num_inputs, num_outputs; 122ca735530SJeremy L Thompson CeedInt num_active_in, num_active_out; 123ca735530SJeremy L Thompson CeedVector *qf_active_in; 1240d0321e0SJeremy L Thompson CeedOperatorDiag_Cuda *diag; 125cc132f9aSnbeams CeedOperatorAssemble_Cuda *asmb; 1260d0321e0SJeremy L Thompson } CeedOperator_Cuda; 1270d0321e0SJeremy L Thompson 128eb7e6cafSJeremy L Thompson CEED_INTERN int CeedGetCublasHandle_Cuda(Ceed ceed, cublasHandle_t *handle); 1290d0321e0SJeremy L Thompson 1301f9221feSJeremy L Thompson CEED_INTERN int CeedVectorCreate_Cuda(CeedSize n, CeedVector vec); 1310d0321e0SJeremy L Thompson 1320305e208SSebastian Grimberg CEED_INTERN int CeedElemRestrictionCreate_Cuda(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *indices, const bool *orients, 1330c73c039SSebastian Grimberg const CeedInt8 *curl_orients, CeedElemRestriction r); 1340d0321e0SJeremy L Thompson 1352b730f8bSJeremy L Thompson CEED_INTERN int CeedBasisCreateTensorH1_Cuda(CeedInt dim, CeedInt P_1d, CeedInt Q_1d, const CeedScalar *interp_1d, const CeedScalar *grad_1d, 1366574a04fSJeremy L Thompson const CeedScalar *q_ref_1d, const CeedScalar *q_weight_1d, CeedBasis basis); 13751475c7cSJeremy L Thompson CEED_INTERN int CeedBasisCreateH1_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 13851475c7cSJeremy L Thompson const CeedScalar *grad, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 139d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHdiv_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 140d075f50bSSebastian Grimberg const CeedScalar *div, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 141d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHcurl_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 142d075f50bSSebastian Grimberg const CeedScalar *curl, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 1430d0321e0SJeremy L Thompson 1440d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionCreate_Cuda(CeedQFunction qf); 1450d0321e0SJeremy L Thompson 1460d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionContextCreate_Cuda(CeedQFunctionContext ctx); 1470d0321e0SJeremy L Thompson 1480d0321e0SJeremy L Thompson CEED_INTERN int CeedOperatorCreate_Cuda(CeedOperator op); 1490d0321e0SJeremy L Thompson 15094b7b29bSJeremy L Thompson #endif // CEED_CUDA_REF_H 151