1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <ceed/ceed.h> 18 #include <ceed/backend.h> 19 #include <cuda_runtime.h> 20 #include <stdio.h> 21 #include <string.h> 22 #include "ceed-cuda-gen.h" 23 #include "../cuda/ceed-cuda.h" 24 25 //------------------------------------------------------------------------------ 26 // Apply QFunction 27 //------------------------------------------------------------------------------ 28 static int CeedQFunctionApply_Cuda_gen(CeedQFunction qf, CeedInt Q, 29 CeedVector *U, CeedVector *V) { 30 int ierr; 31 Ceed ceed; 32 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChkBackend(ierr); 33 return CeedError(ceed, CEED_ERROR_BACKEND, 34 "Backend does not implement QFunctionApply"); 35 } 36 37 //------------------------------------------------------------------------------ 38 // Destroy QFunction 39 //------------------------------------------------------------------------------ 40 static int CeedQFunctionDestroy_Cuda_gen(CeedQFunction qf) { 41 int ierr; 42 CeedQFunction_Cuda_gen *data; 43 ierr = CeedQFunctionGetData(qf, &data); CeedChkBackend(ierr); 44 Ceed ceed; 45 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChkBackend(ierr); 46 ierr = cudaFree(data->d_c); CeedChk_Cu(ceed, ierr); 47 ierr = CeedFree(&data->qFunctionSource); CeedChkBackend(ierr); 48 ierr = CeedFree(&data); CeedChkBackend(ierr); 49 return CEED_ERROR_SUCCESS; 50 } 51 52 //------------------------------------------------------------------------------ 53 // Load QFunction 54 //------------------------------------------------------------------------------ 55 static int loadCudaFunction(CeedQFunction qf, char *c_src_file) { 56 int ierr; 57 Ceed ceed; 58 CeedQFunctionGetCeed(qf, &ceed); 59 CeedQFunction_Cuda_gen *data; 60 ierr = CeedQFunctionGetData(qf, &data); CeedChkBackend(ierr); 61 62 // Find source file 63 char *cuda_file; 64 ierr = CeedCalloc(CUDA_MAX_PATH, &cuda_file); CeedChkBackend(ierr); 65 memcpy(cuda_file, c_src_file, strlen(c_src_file)); 66 const char *last_dot = strrchr(cuda_file, '.'); 67 if (!last_dot) 68 return CeedError(ceed, CEED_ERROR_BACKEND, "Cannot find file's extension!"); 69 const size_t cuda_path_len = last_dot - cuda_file; 70 strncpy(&cuda_file[cuda_path_len], ".h", 3); 71 72 // Open source file 73 FILE *fp; 74 long lSize; 75 char *buffer; 76 fp = fopen (cuda_file, "rb"); 77 if (!fp) 78 // LCOV_EXCL_START 79 CeedError(ceed, CEED_ERROR_BACKEND, 80 "Couldn't open the Cuda file for the QFunction."); 81 // LCOV_EXCL_STOP 82 83 // Compute size of source file 84 fseek(fp, 0L, SEEK_END); 85 lSize = ftell(fp); 86 rewind(fp); 87 88 // Allocate memory for entire content 89 ierr = CeedCalloc(lSize+1, &buffer); CeedChkBackend(ierr); 90 91 // Copy the file into the buffer 92 if (1 != fread(buffer, lSize, 1, fp)) { 93 // LCOV_EXCL_START 94 fclose(fp); 95 ierr = CeedFree(&buffer); CeedChkBackend(ierr); 96 CeedError(ceed, CEED_ERROR_BACKEND, 97 "Couldn't read the Cuda file for the QFunction."); 98 // LCOV_EXCL_STOP 99 } 100 101 // Append typedef and save source string 102 // FIXME: the magic number 16 should be defined somewhere... 103 char *fields_string = 104 "typedef struct { const CeedScalar* inputs[16]; CeedScalar* outputs[16]; } Fields_Cuda_gen;"; 105 ierr = CeedMalloc(1 + strlen(fields_string) + strlen(buffer), 106 &data->qFunctionSource); CeedChkBackend(ierr); 107 memcpy(data->qFunctionSource, fields_string, strlen(fields_string)); 108 memcpy(data->qFunctionSource + strlen(fields_string), buffer, 109 strlen(buffer) + 1); 110 111 // Cleanup 112 ierr = CeedFree(&buffer); CeedChkBackend(ierr); 113 fclose(fp); 114 ierr = CeedFree(&cuda_file); CeedChkBackend(ierr); 115 return CEED_ERROR_SUCCESS; 116 } 117 118 //------------------------------------------------------------------------------ 119 // Create QFunction 120 //------------------------------------------------------------------------------ 121 int CeedQFunctionCreate_Cuda_gen(CeedQFunction qf) { 122 int ierr; 123 Ceed ceed; 124 CeedQFunctionGetCeed(qf, &ceed); 125 CeedQFunction_Cuda_gen *data; 126 ierr = CeedCalloc(1, &data); CeedChkBackend(ierr); 127 ierr = CeedQFunctionSetData(qf, data); CeedChkBackend(ierr); 128 129 char *source; 130 ierr = CeedQFunctionGetSourcePath(qf, &source); CeedChkBackend(ierr); 131 const char *funname = strrchr(source, ':') + 1; 132 data->qFunctionName = (char *)funname; 133 const int filenamelen = funname - source; 134 char filename[filenamelen]; 135 memcpy(filename, source, filenamelen - 1); 136 filename[filenamelen - 1] = '\0'; 137 ierr = loadCudaFunction(qf, filename); CeedChkBackend(ierr); 138 139 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Apply", 140 CeedQFunctionApply_Cuda_gen); CeedChkBackend(ierr); 141 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Destroy", 142 CeedQFunctionDestroy_Cuda_gen); CeedChkBackend(ierr); 143 return CEED_ERROR_SUCCESS; 144 } 145 //------------------------------------------------------------------------------ 146