1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <string.h> 18 #include <stdio.h> 19 #include "ceed-cuda-gen.h" 20 21 //------------------------------------------------------------------------------ 22 // Apply QFunction 23 //------------------------------------------------------------------------------ 24 static int CeedQFunctionApply_Cuda_gen(CeedQFunction qf, CeedInt Q, 25 CeedVector *U, CeedVector *V) { 26 int ierr; 27 Ceed ceed; 28 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChk(ierr); 29 return CeedError(ceed, 1, "Backend does not implement QFunctionApply"); 30 } 31 32 //------------------------------------------------------------------------------ 33 // Destroy QFunction 34 //------------------------------------------------------------------------------ 35 static int CeedQFunctionDestroy_Cuda_gen(CeedQFunction qf) { 36 int ierr; 37 CeedQFunction_Cuda_gen *data; 38 ierr = CeedQFunctionGetData(qf, (void *)&data); CeedChk(ierr); 39 Ceed ceed; 40 ierr = CeedQFunctionGetCeed(qf, &ceed); CeedChk(ierr); 41 ierr = cudaFree(data->d_c); CeedChk_Cu(ceed, ierr); 42 ierr = CeedFree(&data->qFunctionSource); CeedChk(ierr); 43 ierr = CeedFree(&data); CeedChk(ierr); 44 return 0; 45 } 46 47 //------------------------------------------------------------------------------ 48 // Load QFunction 49 //------------------------------------------------------------------------------ 50 static int loadCudaFunction(CeedQFunction qf, char *c_src_file) { 51 int ierr; 52 Ceed ceed; 53 CeedQFunctionGetCeed(qf, &ceed); 54 CeedQFunction_Cuda_gen *data; 55 ierr = CeedQFunctionGetData(qf, (void *)&data); CeedChk(ierr); 56 57 // Find source file 58 char *cuda_file; 59 ierr = CeedCalloc(CUDA_MAX_PATH, &cuda_file); CeedChk(ierr); 60 memcpy(cuda_file, c_src_file, strlen(c_src_file)); 61 const char *last_dot = strrchr(cuda_file, '.'); 62 if (!last_dot) 63 return CeedError(ceed, 1, "Cannot find file's extension!"); 64 const size_t cuda_path_len = last_dot - cuda_file; 65 strncpy(&cuda_file[cuda_path_len], ".h", 3); 66 67 // Open source file 68 FILE *fp; 69 long lSize; 70 char *buffer; 71 fp = fopen ( cuda_file, "rb" ); 72 if (!fp) 73 CeedError(ceed, 1, "Couldn't open the Cuda file for the QFunction."); 74 75 // Compute size of source file 76 fseek(fp, 0L, SEEK_END); 77 lSize = ftell(fp); 78 rewind(fp); 79 80 // Allocate memory for entire content 81 ierr = CeedCalloc(lSize+1, &buffer); CeedChk(ierr); 82 83 // Copy the file into the buffer 84 if (1 != fread(buffer, lSize, 1, fp)) { 85 fclose(fp); 86 ierr = CeedFree(&buffer); CeedChk(ierr); 87 CeedError(ceed, 1, "Couldn't read the Cuda file for the QFunction."); 88 } 89 90 // Append typedef and save source string 91 // FIXME: the magic number 16 should be defined somewhere... 92 char *fields_string = 93 "typedef struct { const CeedScalar* inputs[16]; CeedScalar* outputs[16]; } Fields_Cuda_gen;"; 94 ierr = CeedMalloc(1 + strlen(fields_string) + strlen(buffer), 95 &data->qFunctionSource); CeedChk(ierr); 96 memcpy(data->qFunctionSource, fields_string, strlen(fields_string)); 97 memcpy(data->qFunctionSource + strlen(fields_string), buffer, 98 strlen(buffer) + 1); 99 100 // Cleanup 101 ierr = CeedFree(&buffer); CeedChk(ierr); 102 fclose(fp); 103 return 0; 104 } 105 106 //------------------------------------------------------------------------------ 107 // Create QFunction 108 //------------------------------------------------------------------------------ 109 int CeedQFunctionCreate_Cuda_gen(CeedQFunction qf) { 110 int ierr; 111 Ceed ceed; 112 CeedQFunctionGetCeed(qf, &ceed); 113 CeedQFunction_Cuda_gen *data; 114 ierr = CeedCalloc(1,&data); CeedChk(ierr); 115 ierr = CeedQFunctionSetData(qf, (void *)&data); CeedChk(ierr); 116 size_t ctxsize; 117 ierr = CeedQFunctionGetContextSize(qf, &ctxsize); CeedChk(ierr); 118 ierr = cudaMalloc(&data->d_c, ctxsize); CeedChk_Cu(ceed, ierr); 119 120 char *source; 121 ierr = CeedQFunctionGetSourcePath(qf, &source); CeedChk(ierr); 122 const char *funname = strrchr(source, ':') + 1; 123 data->qFunctionName = (char *)funname; 124 const int filenamelen = funname - source; 125 char filename[filenamelen]; 126 memcpy(filename, source, filenamelen - 1); 127 filename[filenamelen - 1] = '\0'; 128 ierr = loadCudaFunction(qf, filename); CeedChk(ierr); 129 130 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Apply", 131 CeedQFunctionApply_Cuda_gen); CeedChk(ierr); 132 ierr = CeedSetBackendFunction(ceed, "QFunction", qf, "Destroy", 133 CeedQFunctionDestroy_Cuda_gen); CeedChk(ierr); 134 return 0; 135 } 136 //------------------------------------------------------------------------------ 137