1 // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed/ceed.h> 9 #include <ceed/backend.h> 10 #include <string.h> 11 #include <stdlib.h> 12 #include "ceed-magma.h" 13 14 static int CeedDestroy_Magma(Ceed ceed) { 15 int ierr; 16 Ceed_Magma *data; 17 ierr = CeedGetData(ceed, &data); CeedChkBackend(ierr); 18 magma_queue_destroy( data->queue ); 19 ierr = CeedFree(&data); CeedChkBackend(ierr); 20 return CEED_ERROR_SUCCESS; 21 } 22 23 static int CeedInit_Magma(const char *resource, Ceed ceed) { 24 int ierr; 25 const int nrc = 14; // number of characters in resource 26 if (strncmp(resource, "/gpu/cuda/magma", nrc) 27 && strncmp(resource, "/gpu/hip/magma", nrc)) 28 // LCOV_EXCL_START 29 return CeedError(ceed, CEED_ERROR_BACKEND, 30 "Magma backend cannot use resource: %s", resource); 31 // LCOV_EXCL_STOP 32 33 ierr = magma_init(); 34 if (ierr) 35 // LCOV_EXCL_START 36 return CeedError(ceed, CEED_ERROR_BACKEND, "error in magma_init(): %d\n", ierr); 37 // LCOV_EXCL_STOP 38 39 Ceed_Magma *data; 40 ierr = CeedCalloc(sizeof(Ceed_Magma), &data); CeedChkBackend(ierr); 41 ierr = CeedSetData(ceed, data); CeedChkBackend(ierr); 42 43 // kernel selection 44 data->basis_kernel_mode = MAGMA_KERNEL_DIM_SPECIFIC; 45 46 // get/set device ID 47 const char *device_spec = strstr(resource, ":device_id="); 48 const int deviceID = (device_spec) ? atoi(device_spec+11) : -1; 49 50 int currentDeviceID; 51 magma_getdevice(¤tDeviceID); 52 if (deviceID >= 0 && currentDeviceID != deviceID) { 53 magma_setdevice(deviceID); 54 currentDeviceID = deviceID; 55 } 56 // create a queue that uses the null stream 57 data->device = currentDeviceID; 58 #ifdef HAVE_HIP 59 magma_queue_create_from_hip(data->device, NULL, NULL, NULL, &(data->queue)); 60 #else 61 magma_queue_create_from_cuda(data->device, NULL, NULL, NULL, &(data->queue)); 62 #endif 63 64 // Create reference CEED that implementation will be dispatched 65 // through unless overridden 66 Ceed ceedref; 67 #ifdef HAVE_HIP 68 CeedInit("/gpu/hip/ref", &ceedref); 69 #else 70 CeedInit("/gpu/cuda/ref", &ceedref); 71 #endif 72 ierr = CeedSetDelegate(ceed, ceedref); CeedChkBackend(ierr); 73 74 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "ElemRestrictionCreate", 75 CeedElemRestrictionCreate_Magma); CeedChkBackend(ierr); 76 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, 77 "ElemRestrictionCreateBlocked", 78 CeedElemRestrictionCreateBlocked_Magma); CeedChkBackend(ierr); 79 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateTensorH1", 80 CeedBasisCreateTensorH1_Magma); CeedChkBackend(ierr); 81 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "BasisCreateH1", 82 CeedBasisCreateH1_Magma); CeedChkBackend(ierr); 83 ierr = CeedSetBackendFunction(ceed, "Ceed", ceed, "Destroy", 84 CeedDestroy_Magma); CeedChkBackend(ierr); 85 return CEED_ERROR_SUCCESS; 86 } 87 88 CEED_INTERN int CeedRegister_Magma(void) { 89 #ifdef HAVE_HIP 90 return CeedRegister("/gpu/hip/magma", CeedInit_Magma, 120); 91 #else 92 return CeedRegister("/gpu/cuda/magma", CeedInit_Magma, 120); 93 #endif 94 } 95