1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC. 2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707. 3 // All Rights reserved. See files LICENSE and NOTICE for details. 4 // 5 // This file is part of CEED, a collection of benchmarks, miniapps, software 6 // libraries and APIs for efficient high-order finite element and spectral 7 // element discretizations for exascale applications. For more information and 8 // source code availability see http://github.com/ceed. 9 // 10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11 // a collaborative effort of two U.S. Department of Energy organizations (Office 12 // of Science and the National Nuclear Security Administration) responsible for 13 // the planning and preparation of a capable exascale ecosystem, including 14 // software, applications, hardware, advanced system engineering and early 15 // testbed platforms, in support of the nation's exascale computing imperative. 16 17 #include <ceed/ceed.h> 18 #include <ceed/backend.h> 19 #include <string.h> 20 #include <stdlib.h> 21 #include "ceed-magma.h" 22 23 static int CeedDestroy_Magma(Ceed ceed) { 24 int ierr; 25 Ceed_Magma *data; 26 ierr = CeedGetData(ceed, &data); CeedChkBackend(ierr); 27 magma_queue_destroy( data->queue ); 28 ierr = CeedFree(&data); CeedChkBackend(ierr); 29 return CEED_ERROR_SUCCESS; 30 } 31 32 static int CeedInit_Magma(const char *resource, Ceed ceed) { 33 int ierr; 34 const int nrc = 14; // number of characters in resource 35 if (strncmp(resource, "/gpu/cuda/magma", nrc) 36 && strncmp(resource, "/gpu/hip/magma", nrc)) 37 // LCOV_EXCL_START 38 return CeedError(ceed, CEED_ERROR_BACKEND, 39 "Magma backend cannot use resource: %s", resource); 40 // LCOV_EXCL_STOP 41 42 ierr = magma_init(); 43 if (ierr) 44 // LCOV_EXCL_START 45 return CeedError(ceed, CEED_ERROR_BACKEND, "error in magma_init(): %d\n", ierr); 46 // LCOV_EXCL_STOP 47 48 Ceed_Magma *data; 49 ierr = CeedCalloc(sizeof(Ceed_Magma), &data); CeedChkBackend(ierr); 50 ierr = CeedSetData(ceed, data); CeedChkBackend(ierr); 51 52 // kernel selection 53 data->basis_kernel_mode = MAGMA_KERNEL_DIM_SPECIFIC; 54 55 // get/set device ID 56 const char *device_spec = strstr(resource, ":device_id="); 57 const int deviceID = (device_spec) ? atoi(device_spec+11) : -1; 58 59 int currentDeviceID; 60 magma_getdevice(¤tDeviceID); 61 if (deviceID >= 0 && currentDeviceID != deviceID) { 62 magma_setdevice(deviceID); 63 currentDeviceID = deviceID; 64 } 65 // create a queue that uses the null stream 66 data->device = currentDeviceID; 67 #ifdef HAVE_HIP 68 magma_queue_create_from_hip(data->device, NULL, NULL, NULL, &(data->queue)); 69 #else 70 magma_queue_create_from_cuda(data->device, NULL, NULL, NULL, &(data->queue)); 71 #endif 72 73 // Create reference CEED that implementation will be dispatched 74 // through unless overridden 75 Ceed ceedref; 76 #ifdef HAVE_HIP 77 CeedInit("/gpu/hip/ref", &ceedref); 78 #else 79 CeedInit("/gpu/cuda/ref", &ceedref); 80 #endif 81 ierr = CeedSetDelegate(ceed, ceedref); CeedChkBackend(ierr); 82 83 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "ElemRestrictionCreate", 84 CeedElemRestrictionCreate_Magma); CeedChkBackend(ierr); 85 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, 86 "ElemRestrictionCreateBlocked", 87 CeedElemRestrictionCreateBlocked_Magma); CeedChkBackend(ierr); 88 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateTensorH1", 89 CeedBasisCreateTensorH1_Magma); CeedChkBackend(ierr); 90 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateH1", 91 CeedBasisCreateH1_Magma); CeedChkBackend(ierr); 92 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "Destroy", 93 CeedDestroy_Magma); CeedChkBackend(ierr); 94 return CEED_ERROR_SUCCESS; 95 } 96 97 CEED_INTERN int CeedRegister_Magma(void) { 98 #ifdef HAVE_HIP 99 return CeedRegister("/gpu/hip/magma", CeedInit_Magma, 120); 100 #else 101 return CeedRegister("/gpu/cuda/magma", CeedInit_Magma, 120); 102 #endif 103 } 104