1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed/backend.h> 9 #include <ceed/ceed.h> 10 #include <stddef.h> 11 12 #include "../sycl/ceed-sycl-compile.hpp" 13 #include "ceed-sycl-gen-operator-build.hpp" 14 #include "ceed-sycl-gen.hpp" 15 16 //------------------------------------------------------------------------------ 17 // Destroy operator 18 //------------------------------------------------------------------------------ 19 static int CeedOperatorDestroy_Sycl_gen(CeedOperator op) { 20 CeedOperator_Sycl_gen *impl; 21 22 CeedCallBackend(CeedOperatorGetData(op, &impl)); 23 CeedCallBackend(CeedFree(&impl)); 24 return CEED_ERROR_SUCCESS; 25 } 26 27 //------------------------------------------------------------------------------ 28 // Apply and add to output 29 //------------------------------------------------------------------------------ 30 static int CeedOperatorApplyAdd_Sycl_gen(CeedOperator op, CeedVector input_vec, CeedVector output_vec, CeedRequest *request) { 31 Ceed ceed; 32 Ceed_Sycl *ceed_Sycl; 33 CeedInt num_elem, num_input_fields, num_output_fields; 34 CeedEvalMode eval_mode; 35 CeedVector output_vecs[CEED_FIELD_MAX] = {}; 36 CeedQFunctionField *qf_input_fields, *qf_output_fields; 37 CeedQFunction_Sycl_gen *qf_impl; 38 CeedQFunction qf; 39 CeedOperatorField *op_input_fields, *op_output_fields; 40 CeedOperator_Sycl_gen *impl; 41 42 CeedCallBackend(CeedOperatorGetCeed(op, &ceed)); 43 CeedCallBackend(CeedGetData(ceed, &ceed_Sycl)); 44 CeedCallBackend(CeedOperatorGetData(op, &impl)); 45 CeedCallBackend(CeedOperatorGetQFunction(op, &qf)); 46 CeedCallBackend(CeedQFunctionGetData(qf, &qf_impl)); 47 CeedCallBackend(CeedOperatorGetNumElements(op, &num_elem)); 48 CeedCallBackend(CeedOperatorGetFields(op, &num_input_fields, &op_input_fields, &num_output_fields, &op_output_fields)); 49 CeedCallBackend(CeedQFunctionGetFields(qf, NULL, &qf_input_fields, NULL, &qf_output_fields)); 50 51 // Check for tensor-product bases 52 { 53 bool has_tensor_bases; 54 55 CeedCallBackend(CeedOperatorHasTensorBases(op, &has_tensor_bases)); 56 // -- Fallback to ref if not all bases are tensor-product 57 if (!has_tensor_bases) { 58 CeedOperator op_fallback; 59 60 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "Falling back to sycl/ref CeedOperator due to non-tensor bases"); 61 CeedCallBackend(CeedDestroy(&ceed)); 62 CeedCallBackend(CeedOperatorGetFallback(op, &op_fallback)); 63 CeedCallBackend(CeedOperatorApplyAdd(op_fallback, input_vec, output_vec, request)); 64 return CEED_ERROR_SUCCESS; 65 } 66 } 67 68 // Creation of the operator 69 CeedCallBackend(CeedOperatorBuildKernel_Sycl_gen(op)); 70 71 // Input vectors 72 for (CeedInt i = 0; i < num_input_fields; i++) { 73 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_input_fields[i], &eval_mode)); 74 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 75 impl->fields->inputs[i] = NULL; 76 } else { 77 bool is_active; 78 CeedVector vec; 79 80 // Get input vector 81 CeedCallBackend(CeedOperatorFieldGetVector(op_input_fields[i], &vec)); 82 is_active = vec == CEED_VECTOR_ACTIVE; 83 if (is_active) vec = input_vec; 84 CeedCallBackend(CeedVectorGetArrayRead(vec, CEED_MEM_DEVICE, &impl->fields->inputs[i])); 85 if (!is_active) CeedCallBackend(CeedVectorDestroy(&vec)); 86 } 87 } 88 89 // Output vectors 90 for (CeedInt i = 0; i < num_output_fields; i++) { 91 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_output_fields[i], &eval_mode)); 92 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 93 impl->fields->outputs[i] = NULL; 94 } else { 95 bool is_active; 96 CeedVector vec; 97 98 // Get output vector 99 CeedCallBackend(CeedOperatorFieldGetVector(op_output_fields[i], &vec)); 100 is_active = vec == CEED_VECTOR_ACTIVE; 101 if (is_active) vec = output_vec; 102 output_vecs[i] = vec; 103 // Check for multiple output modes 104 CeedInt index = -1; 105 for (CeedInt j = 0; j < i; j++) { 106 if (vec == output_vecs[j]) { 107 index = j; 108 break; 109 } 110 } 111 if (index == -1) { 112 CeedCallBackend(CeedVectorGetArray(vec, CEED_MEM_DEVICE, &impl->fields->outputs[i])); 113 } else { 114 impl->fields->outputs[i] = impl->fields->outputs[index]; 115 } 116 if (!is_active) CeedCallBackend(CeedVectorDestroy(&vec)); 117 } 118 } 119 120 // Get context data 121 CeedCallBackend(CeedQFunctionGetInnerContextData(qf, CEED_MEM_DEVICE, &qf_impl->d_c)); 122 123 // Apply operator 124 const CeedInt dim = impl->dim; 125 const CeedInt Q_1d = impl->Q_1d; 126 const CeedInt P_1d = impl->max_P_1d; 127 CeedInt block_sizes[3], grid = 0; 128 129 CeedCallBackend(BlockGridCalculate_Sycl_gen(dim, P_1d, Q_1d, block_sizes)); 130 if (dim == 1) { 131 grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 132 // CeedCallBackend(CeedRunKernelDimSharedSycl(ceed, impl->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 133 } else if (dim == 2) { 134 grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 135 // CeedCallBackend(CeedRunKernelDimSharedSycl(ceed, impl->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 136 } else if (dim == 3) { 137 grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 138 // CeedCallBackend(CeedRunKernelDimSharedSycl(ceed, impl->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 139 } 140 141 sycl::range<3> local_range(block_sizes[2], block_sizes[1], block_sizes[0]); 142 sycl::range<3> global_range(grid * block_sizes[2], block_sizes[1], block_sizes[0]); 143 sycl::nd_range<3> kernel_range(global_range, local_range); 144 145 //----------- 146 std::vector<sycl::event> e; 147 148 if (!ceed_Sycl->sycl_queue.is_in_order()) e = {ceed_Sycl->sycl_queue.ext_oneapi_submit_barrier()}; 149 150 CeedCallSycl(ceed, ceed_Sycl->sycl_queue.submit([&](sycl::handler &cgh) { 151 cgh.depends_on(e); 152 cgh.set_args(num_elem, qf_impl->d_c, impl->indices, impl->fields, impl->B, impl->G, impl->W); 153 cgh.parallel_for(kernel_range, *(impl->op)); 154 })); 155 CeedCallSycl(ceed, ceed_Sycl->sycl_queue.wait_and_throw()); 156 157 // Restore input arrays 158 for (CeedInt i = 0; i < num_input_fields; i++) { 159 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_input_fields[i], &eval_mode)); 160 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 161 } else { 162 bool is_active; 163 CeedVector vec; 164 165 CeedCallBackend(CeedOperatorFieldGetVector(op_input_fields[i], &vec)); 166 is_active = vec == CEED_VECTOR_ACTIVE; 167 if (is_active) vec = input_vec; 168 CeedCallBackend(CeedVectorRestoreArrayRead(vec, &impl->fields->inputs[i])); 169 if (!is_active) CeedCallBackend(CeedVectorDestroy(&vec)); 170 } 171 } 172 173 // Restore output arrays 174 for (CeedInt i = 0; i < num_output_fields; i++) { 175 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_output_fields[i], &eval_mode)); 176 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 177 } else { 178 bool is_active; 179 CeedVector vec; 180 181 CeedCallBackend(CeedOperatorFieldGetVector(op_output_fields[i], &vec)); 182 is_active = vec == CEED_VECTOR_ACTIVE; 183 if (is_active) vec = output_vec; 184 // Check for multiple output modes 185 CeedInt index = -1; 186 187 for (CeedInt j = 0; j < i; j++) { 188 if (vec == output_vecs[j]) { 189 index = j; 190 break; 191 } 192 } 193 if (index == -1) { 194 CeedCallBackend(CeedVectorRestoreArray(vec, &impl->fields->outputs[i])); 195 } 196 if (!is_active) CeedCallBackend(CeedVectorDestroy(&vec)); 197 } 198 } 199 200 // Restore context data 201 CeedCallBackend(CeedQFunctionRestoreInnerContextData(qf, &qf_impl->d_c)); 202 CeedCallBackend(CeedDestroy(&ceed)); 203 return CEED_ERROR_SUCCESS; 204 } 205 206 //------------------------------------------------------------------------------ 207 // Create operator 208 //------------------------------------------------------------------------------ 209 int CeedOperatorCreate_Sycl_gen(CeedOperator op) { 210 Ceed ceed; 211 Ceed_Sycl *sycl_data; 212 CeedOperator_Sycl_gen *impl; 213 214 CeedCallBackend(CeedOperatorGetCeed(op, &ceed)); 215 CeedCallBackend(CeedGetData(ceed, &sycl_data)); 216 217 CeedCallBackend(CeedCalloc(1, &impl)); 218 CeedCallBackend(CeedOperatorSetData(op, impl)); 219 220 impl->indices = sycl::malloc_device<FieldsInt_Sycl>(1, sycl_data->sycl_device, sycl_data->sycl_context); 221 impl->fields = sycl::malloc_host<Fields_Sycl>(1, sycl_data->sycl_context); 222 impl->B = sycl::malloc_device<Fields_Sycl>(1, sycl_data->sycl_device, sycl_data->sycl_context); 223 impl->G = sycl::malloc_device<Fields_Sycl>(1, sycl_data->sycl_device, sycl_data->sycl_context); 224 impl->W = sycl::malloc_device<CeedScalar>(1, sycl_data->sycl_device, sycl_data->sycl_context); 225 226 CeedCallBackend(CeedSetBackendFunctionCpp(ceed, "Operator", op, "ApplyAdd", CeedOperatorApplyAdd_Sycl_gen)); 227 CeedCallBackend(CeedSetBackendFunctionCpp(ceed, "Operator", op, "Destroy", CeedOperatorDestroy_Sycl_gen)); 228 CeedCallBackend(CeedDestroy(&ceed)); 229 return CEED_ERROR_SUCCESS; 230 } 231 232 //------------------------------------------------------------------------------ 233