1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <ceed-backend.h> 18 #include <string.h> 19 #include <stdio.h> 20 #include "../cuda/ceed-cuda.h" 21 #include "ceed-cuda-gen.h" 22 23 //------------------------------------------------------------------------------ 24 // Apply QFunction 25 //------------------------------------------------------------------------------ 26 static int CeedQFunctionApply_Cuda_gen(CeedQFunction qf, CeedInt Q, 27 CeedVector *U, CeedVector *V) { 28 int ierr; 29 Ceed ceed; 30 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChk(ierr); 31 return CeedError(ceed, 1, "Backend does not implement QFunctionApply"); 32 } 33 34 //------------------------------------------------------------------------------ 35 // Destroy QFunction 36 //------------------------------------------------------------------------------ 37 static int CeedQFunctionDestroy_Cuda_gen(CeedQFunction qf) { 38 int ierr; 39 CeedQFunction_Cuda_gen *data; 40 ierr = CeedQFunctionGetData(qf, (void *)&data); CeedChk(ierr); 41 Ceed ceed; 42 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChk(ierr); 43 ierr = cudaFree(data->d_c); CeedChk_Cu(ceed, ierr); 44 ierr = CeedFree(&data->qFunctionSource); CeedChk(ierr); 45 ierr = CeedFree(&data); CeedChk(ierr); 46 return 0; 47 } 48 49 //------------------------------------------------------------------------------ 50 // Load QFunction 51 //------------------------------------------------------------------------------ 52 static int loadCudaFunction(CeedQFunction qf, char *c_src_file) { 53 int ierr; 54 Ceed ceed; 55 CeedQFunctionGetCeed(qf, &ceed); 56 CeedQFunction_Cuda_gen *data; 57 ierr = CeedQFunctionGetData(qf, (void *)&data); CeedChk(ierr); 58 59 // Find source file 60 char *cuda_file; 61 ierr = CeedCalloc(CUDA_MAX_PATH, &cuda_file); CeedChk(ierr); 62 memcpy(cuda_file, c_src_file, strlen(c_src_file)); 63 const char *last_dot = strrchr(cuda_file, '.'); 64 if (!last_dot) 65 return CeedError(ceed, 1, "Cannot find file's extension!"); 66 const size_t cuda_path_len = last_dot - cuda_file; 67 strncpy(&cuda_file[cuda_path_len], ".h", 3); 68 69 // Open source file 70 FILE *fp; 71 long lSize; 72 char *buffer; 73 fp = fopen ( cuda_file, "rb" ); 74 if (!fp) 75 CeedError(ceed, 1, "Couldn't open the Cuda file for the QFunction."); 76 77 // Compute size of source file 78 fseek(fp, 0L, SEEK_END); 79 lSize = ftell(fp); 80 rewind(fp); 81 82 // Allocate memory for entire content 83 ierr = CeedCalloc(lSize+1, &buffer); CeedChk(ierr); 84 85 // Copy the file into the buffer 86 if (1 != fread(buffer, lSize, 1, fp)) { 87 fclose(fp); 88 ierr = CeedFree(&buffer); CeedChk(ierr); 89 CeedError(ceed, 1, "Couldn't read the Cuda file for the QFunction."); 90 } 91 92 // Append typedef and save source string 93 // FIXME: the magic number 16 should be defined somewhere... 94 char *fields_string = 95 "typedef struct { const CeedScalar* inputs[16]; CeedScalar* outputs[16]; } Fields_Cuda_gen;"; 96 ierr = CeedMalloc(1 + strlen(fields_string) + strlen(buffer), 97 &data->qFunctionSource); CeedChk(ierr); 98 strncpy(data->qFunctionSource, fields_string, 1 + strlen(fields_string)); 99 strncat(data->qFunctionSource, buffer, strlen(buffer)); 100 101 // Cleanup 102 ierr = CeedFree(&buffer); CeedChk(ierr); 103 fclose(fp); 104 return 0; 105 } 106 107 //------------------------------------------------------------------------------ 108 // Create QFunction 109 //------------------------------------------------------------------------------ 110 int CeedQFunctionCreate_Cuda_gen(CeedQFunction qf) { 111 int ierr; 112 Ceed ceed; 113 CeedQFunctionGetCeed(qf, &ceed); 114 CeedQFunction_Cuda_gen *data; 115 ierr = CeedCalloc(1,&data); CeedChk(ierr); 116 ierr = CeedQFunctionSetData(qf, (void *)&data); CeedChk(ierr); 117 size_t ctxsize; 118 ierr = CeedQFunctionGetContextSize(qf, &ctxsize); CeedChk(ierr); 119 ierr = cudaMalloc(&data->d_c, ctxsize); CeedChk_Cu(ceed, ierr); 120 121 char *source; 122 ierr = CeedQFunctionGetSourcePath(qf, &source); CeedChk(ierr); 123 const char *funname = strrchr(source, ':') + 1; 124 data->qFunctionName = (char *)funname; 125 const int filenamelen = funname - source; 126 char filename[filenamelen]; 127 memcpy(filename, source, filenamelen - 1); 128 filename[filenamelen - 1] = '\0'; 129 ierr = loadCudaFunction(qf, filename); CeedChk(ierr); 130 131 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Apply", 132 CeedQFunctionApply_Cuda_gen); CeedChk(ierr); 133 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Destroy", 134 CeedQFunctionDestroy_Cuda_gen); CeedChk(ierr); 135 return 0; 136 } 137 //------------------------------------------------------------------------------ 138