1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <ceed.h> 18 #include <ceed-backend.h> 19 #include <cuda_runtime.h> 20 #include <stdio.h> 21 #include <string.h> 22 #include "ceed-cuda-gen.h" 23 #include "../cuda/ceed-cuda.h" 24 25 //------------------------------------------------------------------------------ 26 // Apply QFunction 27 //------------------------------------------------------------------------------ 28 static int CeedQFunctionApply_Cuda_gen(CeedQFunction qf, CeedInt Q, 29 CeedVector *U, CeedVector *V) { 30 int ierr; 31 Ceed ceed; 32 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChk(ierr); 33 return CeedError(ceed, 1, "Backend does not implement QFunctionApply"); 34 } 35 36 //------------------------------------------------------------------------------ 37 // Destroy QFunction 38 //------------------------------------------------------------------------------ 39 static int CeedQFunctionDestroy_Cuda_gen(CeedQFunction qf) { 40 int ierr; 41 CeedQFunction_Cuda_gen *data; 42 ierr = CeedQFunctionGetData(qf, &data); CeedChk(ierr); 43 Ceed ceed; 44 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChk(ierr); 45 ierr = cudaFree(data->d_c); CeedChk_Cu(ceed, ierr); 46 ierr = CeedFree(&data->qFunctionSource); CeedChk(ierr); 47 ierr = CeedFree(&data); CeedChk(ierr); 48 return 0; 49 } 50 51 //------------------------------------------------------------------------------ 52 // Load QFunction 53 //------------------------------------------------------------------------------ 54 static int loadCudaFunction(CeedQFunction qf, char *c_src_file) { 55 int ierr; 56 Ceed ceed; 57 CeedQFunctionGetCeed(qf, &ceed); 58 CeedQFunction_Cuda_gen *data; 59 ierr = CeedQFunctionGetData(qf, &data); CeedChk(ierr); 60 61 // Find source file 62 char *cuda_file; 63 ierr = CeedCalloc(CUDA_MAX_PATH, &cuda_file); CeedChk(ierr); 64 memcpy(cuda_file, c_src_file, strlen(c_src_file)); 65 const char *last_dot = strrchr(cuda_file, '.'); 66 if (!last_dot) 67 return CeedError(ceed, 1, "Cannot find file's extension!"); 68 const size_t cuda_path_len = last_dot - cuda_file; 69 strncpy(&cuda_file[cuda_path_len], ".h", 3); 70 71 // Open source file 72 FILE *fp; 73 long lSize; 74 char *buffer; 75 fp = fopen (cuda_file, "rb"); 76 if (!fp) 77 // LCOV_EXCL_START 78 CeedError(ceed, 1, "Couldn't open the Cuda file for the QFunction."); 79 // LCOV_EXCL_STOP 80 81 // Compute size of source file 82 fseek(fp, 0L, SEEK_END); 83 lSize = ftell(fp); 84 rewind(fp); 85 86 // Allocate memory for entire content 87 ierr = CeedCalloc(lSize+1, &buffer); CeedChk(ierr); 88 89 // Copy the file into the buffer 90 if (1 != fread(buffer, lSize, 1, fp)) { 91 // LCOV_EXCL_START 92 fclose(fp); 93 ierr = CeedFree(&buffer); CeedChk(ierr); 94 CeedError(ceed, 1, "Couldn't read the Cuda file for the QFunction."); 95 // LCOV_EXCL_STOP 96 } 97 98 // Append typedef and save source string 99 // FIXME: the magic number 16 should be defined somewhere... 100 char *fields_string = 101 "typedef struct { const CeedScalar* inputs[16]; CeedScalar* outputs[16]; } Fields_Cuda_gen;"; 102 ierr = CeedMalloc(1 + strlen(fields_string) + strlen(buffer), 103 &data->qFunctionSource); CeedChk(ierr); 104 memcpy(data->qFunctionSource, fields_string, strlen(fields_string)); 105 memcpy(data->qFunctionSource + strlen(fields_string), buffer, 106 strlen(buffer) + 1); 107 108 // Cleanup 109 ierr = CeedFree(&buffer); CeedChk(ierr); 110 fclose(fp); 111 ierr = CeedFree(&cuda_file); CeedChk(ierr); 112 return 0; 113 } 114 115 //------------------------------------------------------------------------------ 116 // Create QFunction 117 //------------------------------------------------------------------------------ 118 int CeedQFunctionCreate_Cuda_gen(CeedQFunction qf) { 119 int ierr; 120 Ceed ceed; 121 CeedQFunctionGetCeed(qf, &ceed); 122 CeedQFunction_Cuda_gen *data; 123 ierr = CeedCalloc(1, &data); CeedChk(ierr); 124 ierr = CeedQFunctionSetData(qf, data); CeedChk(ierr); 125 126 char *source; 127 ierr = CeedQFunctionGetSourcePath(qf, &source); CeedChk(ierr); 128 const char *funname = strrchr(source, ':') + 1; 129 data->qFunctionName = (char *)funname; 130 const int filenamelen = funname - source; 131 char filename[filenamelen]; 132 memcpy(filename, source, filenamelen - 1); 133 filename[filenamelen - 1] = '\0'; 134 ierr = loadCudaFunction(qf, filename); CeedChk(ierr); 135 136 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Apply", 137 CeedQFunctionApply_Cuda_gen); CeedChk(ierr); 138 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Destroy", 139 CeedQFunctionDestroy_Cuda_gen); CeedChk(ierr); 140 return 0; 141 } 142 //------------------------------------------------------------------------------ 143