1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <ceed/ceed.h> 18 #include <ceed/backend.h> 19 #include <cuda.h> 20 #include <stdio.h> 21 #include <string.h> 22 #include "ceed-cuda-ref.h" 23 #include "ceed-cuda-ref-qfunction-load.h" 24 #include "../cuda/ceed-cuda-compile.h" 25 26 //------------------------------------------------------------------------------ 27 // Apply QFunction 28 //------------------------------------------------------------------------------ 29 static int CeedQFunctionApply_Cuda(CeedQFunction qf, CeedInt Q, 30 CeedVector *U, CeedVector *V) { 31 int ierr; 32 Ceed ceed; 33 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChkBackend(ierr); 34 35 // Build and compile kernel, if not done 36 ierr = CeedCudaBuildQFunction(qf); CeedChkBackend(ierr); 37 38 CeedQFunction_Cuda *data; 39 ierr = CeedQFunctionGetData(qf, &data); CeedChkBackend(ierr); 40 Ceed_Cuda *ceed_Cuda; 41 ierr = CeedGetData(ceed, &ceed_Cuda); CeedChkBackend(ierr); 42 CeedInt num_input_fields, num_output_fields; 43 ierr = CeedQFunctionGetNumArgs(qf, &num_input_fields, &num_output_fields); 44 CeedChkBackend(ierr); 45 46 // Read vectors 47 for (CeedInt i = 0; i < num_input_fields; i++) { 48 ierr = CeedVectorGetArrayRead(U[i], CEED_MEM_DEVICE, &data->fields.inputs[i]); 49 CeedChkBackend(ierr); 50 } 51 for (CeedInt i = 0; i < num_output_fields; i++) { 52 ierr = CeedVectorGetArrayWrite(V[i], CEED_MEM_DEVICE, &data->fields.outputs[i]); 53 CeedChkBackend(ierr); 54 } 55 56 // Get context data 57 CeedQFunctionContext ctx; 58 ierr = CeedQFunctionGetInnerContext(qf, &ctx); CeedChkBackend(ierr); 59 if (ctx) { 60 ierr = CeedQFunctionContextGetData(ctx, CEED_MEM_DEVICE, &data->d_c); 61 CeedChkBackend(ierr); 62 } 63 64 // Run kernel 65 void *args[] = {&data->d_c, (void *) &Q, &data->fields}; 66 ierr = CeedRunKernelAutoblockCuda(ceed, data->QFunction, Q, args); 67 CeedChkBackend(ierr); 68 69 // Restore vectors 70 for (CeedInt i = 0; i < num_input_fields; i++) { 71 ierr = CeedVectorRestoreArrayRead(U[i], &data->fields.inputs[i]); 72 CeedChkBackend(ierr); 73 } 74 for (CeedInt i = 0; i < num_output_fields; i++) { 75 ierr = CeedVectorRestoreArray(V[i], &data->fields.outputs[i]); 76 CeedChkBackend(ierr); 77 } 78 79 // Restore context 80 if (ctx) { 81 ierr = CeedQFunctionContextRestoreData(ctx, &data->d_c); 82 CeedChkBackend(ierr); 83 } 84 return CEED_ERROR_SUCCESS; 85 } 86 87 //------------------------------------------------------------------------------ 88 // Destroy QFunction 89 //------------------------------------------------------------------------------ 90 static int CeedQFunctionDestroy_Cuda(CeedQFunction qf) { 91 int ierr; 92 CeedQFunction_Cuda *data; 93 ierr = CeedQFunctionGetData(qf, &data); CeedChkBackend(ierr); 94 Ceed ceed; 95 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChkBackend(ierr); 96 if (data->module) 97 CeedChk_Cu(ceed, cuModuleUnload(data->module)); 98 ierr = CeedFree(&data); CeedChkBackend(ierr); 99 100 return CEED_ERROR_SUCCESS; 101 } 102 103 //------------------------------------------------------------------------------ 104 // Set User QFunction 105 //------------------------------------------------------------------------------ 106 static int CeedQFunctionSetCUDAUserFunction_Cuda(CeedQFunction qf, 107 CUfunction f) { 108 int ierr; 109 CeedQFunction_Cuda *data; 110 ierr = CeedQFunctionGetData(qf, &data); CeedChkBackend(ierr); 111 data->QFunction = f; 112 113 return CEED_ERROR_SUCCESS; 114 } 115 116 //------------------------------------------------------------------------------ 117 // Create QFunction 118 //------------------------------------------------------------------------------ 119 int CeedQFunctionCreate_Cuda(CeedQFunction qf) { 120 int ierr; 121 Ceed ceed; 122 CeedQFunctionGetCeed(qf, &ceed); 123 CeedQFunction_Cuda *data; 124 ierr = CeedCalloc(1, &data); CeedChkBackend(ierr); 125 ierr = CeedQFunctionSetData(qf, data); CeedChkBackend(ierr); 126 127 // Read QFunction source 128 ierr = CeedQFunctionGetKernelName(qf, &data->qfunction_name); 129 CeedChkBackend(ierr); 130 CeedDebug256(ceed, 2, "----- Loading QFunction User Source -----\n"); 131 ierr = CeedQFunctionLoadSourceToBuffer(qf, &data->qfunction_source); 132 CeedChkBackend(ierr); 133 CeedDebug256(ceed, 2, "----- Loading QFunction User Source Complete! -----\n"); 134 135 // Register backend functions 136 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Apply", 137 CeedQFunctionApply_Cuda); CeedChkBackend(ierr); 138 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Destroy", 139 CeedQFunctionDestroy_Cuda); CeedChkBackend(ierr); 140 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "SetCUDAUserFunction", 141 CeedQFunctionSetCUDAUserFunction_Cuda); 142 CeedChkBackend(ierr); 143 return CEED_ERROR_SUCCESS; 144 } 145 //------------------------------------------------------------------------------ 146