13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 30d0321e0SJeremy L Thompson // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 50d0321e0SJeremy L Thompson // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 70d0321e0SJeremy L Thompson 80d0321e0SJeremy L Thompson #include <ceed/backend.h> 9*2b730f8bSJeremy L Thompson #include <ceed/ceed.h> 100d0321e0SJeremy L Thompson #include <cuda.h> 110d0321e0SJeremy L Thompson #include <stdio.h> 120d0321e0SJeremy L Thompson #include <string.h> 13*2b730f8bSJeremy L Thompson 140d0321e0SJeremy L Thompson #include "../cuda/ceed-cuda-compile.h" 15*2b730f8bSJeremy L Thompson #include "ceed-cuda-ref-qfunction-load.h" 16*2b730f8bSJeremy L Thompson #include "ceed-cuda-ref.h" 170d0321e0SJeremy L Thompson 180d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 190d0321e0SJeremy L Thompson // Apply QFunction 200d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 21*2b730f8bSJeremy L Thompson static int CeedQFunctionApply_Cuda(CeedQFunction qf, CeedInt Q, CeedVector *U, CeedVector *V) { 220d0321e0SJeremy L Thompson Ceed ceed; 23*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetCeed(qf, &ceed)); 240d0321e0SJeremy L Thompson 250d0321e0SJeremy L Thompson // Build and compile kernel, if not done 26*2b730f8bSJeremy L Thompson CeedCallBackend(CeedCudaBuildQFunction(qf)); 270d0321e0SJeremy L Thompson 280d0321e0SJeremy L Thompson CeedQFunction_Cuda *data; 29*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetData(qf, &data)); 300d0321e0SJeremy L Thompson Ceed_Cuda *ceed_Cuda; 31*2b730f8bSJeremy L Thompson CeedCallBackend(CeedGetData(ceed, &ceed_Cuda)); 32437930d1SJeremy L Thompson CeedInt num_input_fields, num_output_fields; 33*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetNumArgs(qf, &num_input_fields, &num_output_fields)); 340d0321e0SJeremy L Thompson 350d0321e0SJeremy L Thompson // Read vectors 36437930d1SJeremy L Thompson for (CeedInt i = 0; i < num_input_fields; i++) { 37*2b730f8bSJeremy L Thompson CeedCallBackend(CeedVectorGetArrayRead(U[i], CEED_MEM_DEVICE, &data->fields.inputs[i])); 380d0321e0SJeremy L Thompson } 39437930d1SJeremy L Thompson for (CeedInt i = 0; i < num_output_fields; i++) { 40*2b730f8bSJeremy L Thompson CeedCallBackend(CeedVectorGetArrayWrite(V[i], CEED_MEM_DEVICE, &data->fields.outputs[i])); 410d0321e0SJeremy L Thompson } 420d0321e0SJeremy L Thompson 430d0321e0SJeremy L Thompson // Get context data 44*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetInnerContextData(qf, CEED_MEM_DEVICE, &data->d_c)); 450d0321e0SJeremy L Thompson 460d0321e0SJeremy L Thompson // Run kernel 470d0321e0SJeremy L Thompson void *args[] = {&data->d_c, (void *)&Q, &data->fields}; 48*2b730f8bSJeremy L Thompson CeedCallBackend(CeedRunKernelAutoblockCuda(ceed, data->QFunction, Q, args)); 490d0321e0SJeremy L Thompson 500d0321e0SJeremy L Thompson // Restore vectors 51437930d1SJeremy L Thompson for (CeedInt i = 0; i < num_input_fields; i++) { 52*2b730f8bSJeremy L Thompson CeedCallBackend(CeedVectorRestoreArrayRead(U[i], &data->fields.inputs[i])); 530d0321e0SJeremy L Thompson } 54437930d1SJeremy L Thompson for (CeedInt i = 0; i < num_output_fields; i++) { 55*2b730f8bSJeremy L Thompson CeedCallBackend(CeedVectorRestoreArray(V[i], &data->fields.outputs[i])); 560d0321e0SJeremy L Thompson } 570d0321e0SJeremy L Thompson 580d0321e0SJeremy L Thompson // Restore context 59*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionRestoreInnerContextData(qf, &data->d_c)); 60441428dfSJeremy L Thompson 610d0321e0SJeremy L Thompson return CEED_ERROR_SUCCESS; 620d0321e0SJeremy L Thompson } 630d0321e0SJeremy L Thompson 640d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 650d0321e0SJeremy L Thompson // Destroy QFunction 660d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 670d0321e0SJeremy L Thompson static int CeedQFunctionDestroy_Cuda(CeedQFunction qf) { 680d0321e0SJeremy L Thompson CeedQFunction_Cuda *data; 69*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetData(qf, &data)); 700d0321e0SJeremy L Thompson Ceed ceed; 71*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetCeed(qf, &ceed)); 72*2b730f8bSJeremy L Thompson if (data->module) CeedCallCuda(ceed, cuModuleUnload(data->module)); 73*2b730f8bSJeremy L Thompson CeedCallBackend(CeedFree(&data)); 74437930d1SJeremy L Thompson 750d0321e0SJeremy L Thompson return CEED_ERROR_SUCCESS; 760d0321e0SJeremy L Thompson } 770d0321e0SJeremy L Thompson 780d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 790d0321e0SJeremy L Thompson // Set User QFunction 800d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 81*2b730f8bSJeremy L Thompson static int CeedQFunctionSetCUDAUserFunction_Cuda(CeedQFunction qf, CUfunction f) { 820d0321e0SJeremy L Thompson CeedQFunction_Cuda *data; 83*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetData(qf, &data)); 84437930d1SJeremy L Thompson data->QFunction = f; 850d0321e0SJeremy L Thompson return CEED_ERROR_SUCCESS; 860d0321e0SJeremy L Thompson } 870d0321e0SJeremy L Thompson 880d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 890d0321e0SJeremy L Thompson // Create QFunction 900d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 910d0321e0SJeremy L Thompson int CeedQFunctionCreate_Cuda(CeedQFunction qf) { 920d0321e0SJeremy L Thompson Ceed ceed; 930d0321e0SJeremy L Thompson CeedQFunctionGetCeed(qf, &ceed); 940d0321e0SJeremy L Thompson CeedQFunction_Cuda *data; 95*2b730f8bSJeremy L Thompson CeedCallBackend(CeedCalloc(1, &data)); 96*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionSetData(qf, data)); 970d0321e0SJeremy L Thompson 980d0321e0SJeremy L Thompson // Read QFunction source 99*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionGetKernelName(qf, &data->qfunction_name)); 10046dc0734SJeremy L Thompson CeedDebug256(ceed, 2, "----- Loading QFunction User Source -----\n"); 101*2b730f8bSJeremy L Thompson CeedCallBackend(CeedQFunctionLoadSourceToBuffer(qf, &data->qfunction_source)); 10246dc0734SJeremy L Thompson CeedDebug256(ceed, 2, "----- Loading QFunction User Source Complete! -----\n"); 1030d0321e0SJeremy L Thompson 1040d0321e0SJeremy L Thompson // Register backend functions 105*2b730f8bSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "QFunction", qf, "Apply", CeedQFunctionApply_Cuda)); 106*2b730f8bSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "QFunction", qf, "Destroy", CeedQFunctionDestroy_Cuda)); 107*2b730f8bSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "QFunction", qf, "SetCUDAUserFunction", CeedQFunctionSetCUDAUserFunction_Cuda)); 1080d0321e0SJeremy L Thompson return CEED_ERROR_SUCCESS; 1090d0321e0SJeremy L Thompson } 1100d0321e0SJeremy L Thompson //------------------------------------------------------------------------------ 111