13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 30d0321e0SJeremy L Thompson // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 50d0321e0SJeremy L Thompson // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 70d0321e0SJeremy L Thompson 894b7b29bSJeremy L Thompson #ifndef CEED_CUDA_REF_H 994b7b29bSJeremy L Thompson #define CEED_CUDA_REF_H 100d0321e0SJeremy L Thompson 1149aac155SJeremy L Thompson #include <ceed.h> 120d0321e0SJeremy L Thompson #include <ceed/backend.h> 1349aac155SJeremy L Thompson #include <ceed/jit-source/cuda/cuda-types.h> 1449aac155SJeremy L Thompson #include <cublas_v2.h> 150d0321e0SJeremy L Thompson #include <cuda.h> 162b730f8bSJeremy L Thompson 170d0321e0SJeremy L Thompson typedef struct { 180d0321e0SJeremy L Thompson CeedScalar *h_array; 190d0321e0SJeremy L Thompson CeedScalar *h_array_borrowed; 200d0321e0SJeremy L Thompson CeedScalar *h_array_owned; 210d0321e0SJeremy L Thompson CeedScalar *d_array; 220d0321e0SJeremy L Thompson CeedScalar *d_array_borrowed; 230d0321e0SJeremy L Thompson CeedScalar *d_array_owned; 240d0321e0SJeremy L Thompson } CeedVector_Cuda; 250d0321e0SJeremy L Thompson 260d0321e0SJeremy L Thompson typedef struct { 270d0321e0SJeremy L Thompson CUmodule module; 28437930d1SJeremy L Thompson CUfunction StridedNoTranspose; 2958549094SSebastian Grimberg CUfunction StridedTranspose; 30437930d1SJeremy L Thompson CUfunction OffsetNoTranspose; 3158549094SSebastian Grimberg CUfunction OffsetTranspose; 3258549094SSebastian Grimberg CUfunction OffsetTransposeDet; 33dce49693SSebastian Grimberg CUfunction OrientedNoTranspose; 34dce49693SSebastian Grimberg CUfunction OrientedTranspose; 35*7aa91133SSebastian Grimberg CUfunction OrientedTransposeDet; 36dce49693SSebastian Grimberg CUfunction CurlOrientedNoTranspose; 37dce49693SSebastian Grimberg CUfunction CurlOrientedTranspose; 38*7aa91133SSebastian Grimberg CUfunction CurlOrientedTransposeDet; 39dce49693SSebastian Grimberg CUfunction CurlOrientedUnsignedNoTranspose; 40dce49693SSebastian Grimberg CUfunction CurlOrientedUnsignedTranspose; 41*7aa91133SSebastian Grimberg CUfunction CurlOrientedUnsignedTransposeDet; 42437930d1SJeremy L Thompson CeedInt num_nodes; 430d0321e0SJeremy L Thompson CeedInt *h_ind; 440d0321e0SJeremy L Thompson CeedInt *h_ind_allocated; 450d0321e0SJeremy L Thompson CeedInt *d_ind; 460d0321e0SJeremy L Thompson CeedInt *d_ind_allocated; 47437930d1SJeremy L Thompson CeedInt *d_t_offsets; 48437930d1SJeremy L Thompson CeedInt *d_t_indices; 49437930d1SJeremy L Thompson CeedInt *d_l_vec_indices; 50dce49693SSebastian Grimberg bool *h_orients; 51dce49693SSebastian Grimberg bool *h_orients_allocated; 52dce49693SSebastian Grimberg bool *d_orients; 53dce49693SSebastian Grimberg bool *d_orients_allocated; 54dce49693SSebastian Grimberg CeedInt8 *h_curl_orients; 55dce49693SSebastian Grimberg CeedInt8 *h_curl_orients_allocated; 56dce49693SSebastian Grimberg CeedInt8 *d_curl_orients; 57dce49693SSebastian Grimberg CeedInt8 *d_curl_orients_allocated; 580d0321e0SJeremy L Thompson } CeedElemRestriction_Cuda; 590d0321e0SJeremy L Thompson 60437930d1SJeremy L Thompson typedef struct { 61437930d1SJeremy L Thompson CUmodule module; 62437930d1SJeremy L Thompson CUfunction Interp; 63437930d1SJeremy L Thompson CUfunction Grad; 64437930d1SJeremy L Thompson CUfunction Weight; 65437930d1SJeremy L Thompson CeedScalar *d_interp_1d; 66437930d1SJeremy L Thompson CeedScalar *d_grad_1d; 67437930d1SJeremy L Thompson CeedScalar *d_q_weight_1d; 68437930d1SJeremy L Thompson } CeedBasis_Cuda; 69437930d1SJeremy L Thompson 70437930d1SJeremy L Thompson typedef struct { 71437930d1SJeremy L Thompson CUmodule module; 72437930d1SJeremy L Thompson CUfunction Interp; 73d075f50bSSebastian Grimberg CUfunction InterpTranspose; 74d075f50bSSebastian Grimberg CUfunction Deriv; 75d075f50bSSebastian Grimberg CUfunction DerivTranspose; 76437930d1SJeremy L Thompson CUfunction Weight; 77437930d1SJeremy L Thompson CeedScalar *d_interp; 78437930d1SJeremy L Thompson CeedScalar *d_grad; 79d075f50bSSebastian Grimberg CeedScalar *d_div; 80d075f50bSSebastian Grimberg CeedScalar *d_curl; 81437930d1SJeremy L Thompson CeedScalar *d_q_weight; 82437930d1SJeremy L Thompson } CeedBasisNonTensor_Cuda; 83437930d1SJeremy L Thompson 840d0321e0SJeremy L Thompson typedef struct { 850d0321e0SJeremy L Thompson CUmodule module; 86437930d1SJeremy L Thompson char *qfunction_name; 87437930d1SJeremy L Thompson char *qfunction_source; 88437930d1SJeremy L Thompson CUfunction QFunction; 890d0321e0SJeremy L Thompson Fields_Cuda fields; 900d0321e0SJeremy L Thompson void *d_c; 910d0321e0SJeremy L Thompson } CeedQFunction_Cuda; 920d0321e0SJeremy L Thompson 930d0321e0SJeremy L Thompson typedef struct { 940d0321e0SJeremy L Thompson void *h_data; 950d0321e0SJeremy L Thompson void *h_data_borrowed; 960d0321e0SJeremy L Thompson void *h_data_owned; 970d0321e0SJeremy L Thompson void *d_data; 980d0321e0SJeremy L Thompson void *d_data_borrowed; 990d0321e0SJeremy L Thompson void *d_data_owned; 1000d0321e0SJeremy L Thompson } CeedQFunctionContext_Cuda; 1010d0321e0SJeremy L Thompson 1020d0321e0SJeremy L Thompson typedef struct { 1030d0321e0SJeremy L Thompson CUmodule module; 104004e4986SSebastian Grimberg CUfunction LinearDiagonal; 105004e4986SSebastian Grimberg CUfunction LinearPointBlock; 106506b1a0cSSebastian Grimberg CeedElemRestriction diag_rstr, point_block_diag_rstr; 107ca735530SJeremy L Thompson CeedVector elem_diag, point_block_elem_diag; 108004e4986SSebastian Grimberg CeedEvalMode *d_eval_modes_in, *d_eval_modes_out; 109004e4986SSebastian Grimberg CeedScalar *d_identity, *d_interp_in, *d_grad_in, *d_div_in, *d_curl_in; 110004e4986SSebastian Grimberg CeedScalar *d_interp_out, *d_grad_out, *d_div_out, *d_curl_out; 1110d0321e0SJeremy L Thompson } CeedOperatorDiag_Cuda; 1120d0321e0SJeremy L Thompson 1130d0321e0SJeremy L Thompson typedef struct { 114cc132f9aSnbeams CUmodule module; 115004e4986SSebastian Grimberg CUfunction LinearAssemble; 116004e4986SSebastian Grimberg CeedInt block_size_x, block_size_y, elems_per_block; 117cc132f9aSnbeams CeedScalar *d_B_in, *d_B_out; 118cc132f9aSnbeams } CeedOperatorAssemble_Cuda; 119cc132f9aSnbeams 120cc132f9aSnbeams typedef struct { 121ca735530SJeremy L Thompson CeedVector *e_vecs; // E-vectors, inputs followed by outputs 122ca735530SJeremy L Thompson CeedVector *q_vecs_in; // Input Q-vectors needed to apply operator 123ca735530SJeremy L Thompson CeedVector *q_vecs_out; // Output Q-vectors needed to apply operator 124ca735530SJeremy L Thompson CeedInt num_inputs, num_outputs; 125ca735530SJeremy L Thompson CeedInt num_active_in, num_active_out; 126ca735530SJeremy L Thompson CeedVector *qf_active_in; 1270d0321e0SJeremy L Thompson CeedOperatorDiag_Cuda *diag; 128cc132f9aSnbeams CeedOperatorAssemble_Cuda *asmb; 1290d0321e0SJeremy L Thompson } CeedOperator_Cuda; 1300d0321e0SJeremy L Thompson 131eb7e6cafSJeremy L Thompson CEED_INTERN int CeedGetCublasHandle_Cuda(Ceed ceed, cublasHandle_t *handle); 1320d0321e0SJeremy L Thompson 1331f9221feSJeremy L Thompson CEED_INTERN int CeedVectorCreate_Cuda(CeedSize n, CeedVector vec); 1340d0321e0SJeremy L Thompson 1350305e208SSebastian Grimberg CEED_INTERN int CeedElemRestrictionCreate_Cuda(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *indices, const bool *orients, 1360c73c039SSebastian Grimberg const CeedInt8 *curl_orients, CeedElemRestriction r); 1370d0321e0SJeremy L Thompson 1382b730f8bSJeremy L Thompson CEED_INTERN int CeedBasisCreateTensorH1_Cuda(CeedInt dim, CeedInt P_1d, CeedInt Q_1d, const CeedScalar *interp_1d, const CeedScalar *grad_1d, 1396574a04fSJeremy L Thompson const CeedScalar *q_ref_1d, const CeedScalar *q_weight_1d, CeedBasis basis); 14051475c7cSJeremy L Thompson CEED_INTERN int CeedBasisCreateH1_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 14151475c7cSJeremy L Thompson const CeedScalar *grad, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 142d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHdiv_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 143d075f50bSSebastian Grimberg const CeedScalar *div, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 144d075f50bSSebastian Grimberg CEED_INTERN int CeedBasisCreateHcurl_Cuda(CeedElemTopology topo, CeedInt dim, CeedInt num_nodes, CeedInt num_qpts, const CeedScalar *interp, 145d075f50bSSebastian Grimberg const CeedScalar *curl, const CeedScalar *q_ref, const CeedScalar *q_weight, CeedBasis basis); 1460d0321e0SJeremy L Thompson 1470d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionCreate_Cuda(CeedQFunction qf); 1480d0321e0SJeremy L Thompson 1490d0321e0SJeremy L Thompson CEED_INTERN int CeedQFunctionContextCreate_Cuda(CeedQFunctionContext ctx); 1500d0321e0SJeremy L Thompson 1510d0321e0SJeremy L Thompson CEED_INTERN int CeedOperatorCreate_Cuda(CeedOperator op); 1520d0321e0SJeremy L Thompson 15394b7b29bSJeremy L Thompson #endif // CEED_CUDA_REF_H 154