1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <ceed/ceed.h> 18 #include <ceed/backend.h> 19 #include <string.h> 20 #include "ceed-magma.h" 21 22 static int CeedDestroy_Magma(Ceed ceed) { 23 int ierr; 24 Ceed_Magma *data; 25 ierr = CeedGetData(ceed, &data); CeedChkBackend(ierr); 26 magma_queue_destroy( data->queue ); 27 ierr = CeedFree(&data); CeedChkBackend(ierr); 28 return CEED_ERROR_SUCCESS; 29 } 30 31 static int CeedInit_Magma(const char *resource, Ceed ceed) { 32 int ierr; 33 if (strcmp(resource, "/gpu/cuda/magma") && strcmp(resource, "/gpu/hip/magma")) 34 // LCOV_EXCL_START 35 return CeedError(ceed, CEED_ERROR_BACKEND, 36 "Magma backend cannot use resource: %s", resource); 37 // LCOV_EXCL_STOP 38 39 // Create reference CEED that implementation will be dispatched 40 // through unless overridden 41 Ceed ceedref; 42 #ifdef HAVE_HIP 43 CeedInit("/gpu/hip/ref", &ceedref); 44 #else 45 CeedInit("/gpu/cuda/ref", &ceedref); 46 #endif 47 ierr = CeedSetDelegate(ceed, ceedref); CeedChkBackend(ierr); 48 49 ierr = magma_init(); 50 if (ierr) 51 // LCOV_EXCL_START 52 return CeedError(ceed, CEED_ERROR_BACKEND, "error in magma_init(): %d\n", ierr); 53 // LCOV_EXCL_STOP 54 55 Ceed_Magma *data; 56 ierr = CeedCalloc(sizeof(Ceed_Magma), &data); CeedChkBackend(ierr); 57 ierr = CeedSetData(ceed, data); CeedChkBackend(ierr); 58 59 // kernel selection 60 data->basis_kernel_mode = MAGMA_KERNEL_DIM_SPECIFIC; 61 62 // kernel max threads per thread-block 63 data->maxthreads[0] = 128; // for 1D kernels 64 data->maxthreads[1] = 128; // for 2D kernels 65 data->maxthreads[2] = 64; // for 3D kernels 66 67 // create a queue that uses the null stream 68 magma_getdevice( &(data->device) ); 69 #ifdef HAVE_HIP 70 magma_queue_create_from_hip(data->device, NULL, NULL, NULL, &(data->queue)); 71 #else 72 magma_queue_create_from_cuda(data->device, NULL, NULL, NULL, &(data->queue)); 73 #endif 74 75 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "ElemRestrictionCreate", 76 CeedElemRestrictionCreate_Magma); CeedChkBackend(ierr); 77 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, 78 "ElemRestrictionCreateBlocked", 79 CeedElemRestrictionCreateBlocked_Magma); CeedChkBackend(ierr); 80 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateTensorH1", 81 CeedBasisCreateTensorH1_Magma); CeedChkBackend(ierr); 82 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateH1", 83 CeedBasisCreateH1_Magma); CeedChkBackend(ierr); 84 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "Destroy", 85 CeedDestroy_Magma); CeedChkBackend(ierr); 86 return CEED_ERROR_SUCCESS; 87 } 88 89 CEED_INTERN int CeedRegister_Magma(void) { 90 #ifdef HAVE_HIP 91 return CeedRegister("/gpu/hip/magma", CeedInit_Magma, 120); 92 #else 93 return CeedRegister("/gpu/cuda/magma", CeedInit_Magma, 120); 94 #endif 95 } 96