1 // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed/backend.h> 9 #include <ceed/ceed.h> 10 #include <stddef.h> 11 12 #include "../sycl/ceed-sycl-compile.hpp" 13 #include "ceed-sycl-gen-operator-build.hpp" 14 #include "ceed-sycl-gen.hpp" 15 16 //------------------------------------------------------------------------------ 17 // Destroy operator 18 //------------------------------------------------------------------------------ 19 static int CeedOperatorDestroy_Sycl_gen(CeedOperator op) { 20 CeedOperator_Sycl_gen *impl; 21 22 CeedCallBackend(CeedOperatorGetData(op, &impl)); 23 CeedCallBackend(CeedFree(&impl)); 24 return CEED_ERROR_SUCCESS; 25 } 26 27 //------------------------------------------------------------------------------ 28 // Apply and add to output 29 //------------------------------------------------------------------------------ 30 static int CeedOperatorApplyAdd_Sycl_gen(CeedOperator op, CeedVector input_vec, CeedVector output_vec, CeedRequest *request) { 31 Ceed ceed; 32 Ceed_Sycl *ceed_Sycl; 33 CeedInt num_elem, num_input_fields, num_output_fields; 34 CeedEvalMode eval_mode; 35 CeedVector output_vecs[CEED_FIELD_MAX] = {}; 36 CeedQFunctionField *qf_input_fields, *qf_output_fields; 37 CeedQFunction_Sycl_gen *qf_impl; 38 CeedQFunction qf; 39 CeedOperatorField *op_input_fields, *op_output_fields; 40 CeedOperator_Sycl_gen *impl; 41 42 CeedCallBackend(CeedOperatorGetCeed(op, &ceed)); 43 CeedCallBackend(CeedGetData(ceed, &ceed_Sycl)); 44 CeedCallBackend(CeedOperatorGetData(op, &impl)); 45 CeedCallBackend(CeedOperatorGetQFunction(op, &qf)); 46 CeedCallBackend(CeedQFunctionGetData(qf, &qf_impl)); 47 CeedCallBackend(CeedOperatorGetNumElements(op, &num_elem)); 48 CeedCallBackend(CeedOperatorGetFields(op, &num_input_fields, &op_input_fields, &num_output_fields, &op_output_fields)); 49 CeedCallBackend(CeedQFunctionGetFields(qf, NULL, &qf_input_fields, NULL, &qf_output_fields)); 50 51 // Check for tensor-product bases 52 { 53 bool has_tensor_bases; 54 55 CeedCallBackend(CeedOperatorHasTensorBases(op, &has_tensor_bases)); 56 // -- Fallback to ref if not all bases are tensor-product 57 if (!has_tensor_bases) { 58 CeedOperator op_fallback; 59 60 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "Falling back to sycl/ref CeedOperator due to non-tensor bases"); 61 CeedCallBackend(CeedOperatorGetFallback(op, &op_fallback)); 62 CeedCallBackend(CeedOperatorApplyAdd(op_fallback, input_vec, output_vec, request)); 63 return CEED_ERROR_SUCCESS; 64 } 65 } 66 67 // Creation of the operator 68 CeedCallBackend(CeedOperatorBuildKernel_Sycl_gen(op)); 69 70 // Input vectors 71 for (CeedInt i = 0; i < num_input_fields; i++) { 72 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_input_fields[i], &eval_mode)); 73 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 74 impl->fields->inputs[i] = NULL; 75 } else { 76 CeedVector vec; 77 78 // Get input vector 79 CeedCallBackend(CeedOperatorFieldGetVector(op_input_fields[i], &vec)); 80 if (vec == CEED_VECTOR_ACTIVE) vec = input_vec; 81 CeedCallBackend(CeedVectorGetArrayRead(vec, CEED_MEM_DEVICE, &impl->fields->inputs[i])); 82 } 83 } 84 85 // Output vectors 86 for (CeedInt i = 0; i < num_output_fields; i++) { 87 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_output_fields[i], &eval_mode)); 88 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 89 impl->fields->outputs[i] = NULL; 90 } else { 91 CeedVector vec; 92 93 // Get output vector 94 CeedCallBackend(CeedOperatorFieldGetVector(op_output_fields[i], &vec)); 95 if (vec == CEED_VECTOR_ACTIVE) vec = output_vec; 96 output_vecs[i] = vec; 97 // Check for multiple output modes 98 CeedInt index = -1; 99 for (CeedInt j = 0; j < i; j++) { 100 if (vec == output_vecs[j]) { 101 index = j; 102 break; 103 } 104 } 105 if (index == -1) { 106 CeedCallBackend(CeedVectorGetArray(vec, CEED_MEM_DEVICE, &impl->fields->outputs[i])); 107 } else { 108 impl->fields->outputs[i] = impl->fields->outputs[index]; 109 } 110 } 111 } 112 113 // Get context data 114 CeedCallBackend(CeedQFunctionGetInnerContextData(qf, CEED_MEM_DEVICE, &qf_impl->d_c)); 115 116 // Apply operator 117 const CeedInt dim = impl->dim; 118 const CeedInt Q_1d = impl->Q_1d; 119 const CeedInt P_1d = impl->max_P_1d; 120 CeedInt block_sizes[3], grid = 0; 121 122 CeedCallBackend(BlockGridCalculate_Sycl_gen(dim, P_1d, Q_1d, block_sizes)); 123 if (dim == 1) { 124 grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 125 // CeedCallBackend(CeedRunKernelDimSharedSycl(ceed, impl->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 126 } else if (dim == 2) { 127 grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 128 // CeedCallBackend(CeedRunKernelDimSharedSycl(ceed, impl->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 129 } else if (dim == 3) { 130 grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 131 // CeedCallBackend(CeedRunKernelDimSharedSycl(ceed, impl->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 132 } 133 134 sycl::range<3> local_range(block_sizes[2], block_sizes[1], block_sizes[0]); 135 sycl::range<3> global_range(grid * block_sizes[2], block_sizes[1], block_sizes[0]); 136 sycl::nd_range<3> kernel_range(global_range, local_range); 137 138 //----------- 139 // Order queue 140 sycl::event e = ceed_Sycl->sycl_queue.ext_oneapi_submit_barrier(); 141 142 CeedCallSycl(ceed, ceed_Sycl->sycl_queue.submit([&](sycl::handler &cgh) { 143 cgh.depends_on(e); 144 cgh.set_args(num_elem, qf_impl->d_c, impl->indices, impl->fields, impl->B, impl->G, impl->W); 145 cgh.parallel_for(kernel_range, *(impl->op)); 146 })); 147 CeedCallSycl(ceed, ceed_Sycl->sycl_queue.wait_and_throw()); 148 149 // Restore input arrays 150 for (CeedInt i = 0; i < num_input_fields; i++) { 151 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_input_fields[i], &eval_mode)); 152 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 153 } else { 154 CeedVector vec; 155 156 CeedCallBackend(CeedOperatorFieldGetVector(op_input_fields[i], &vec)); 157 if (vec == CEED_VECTOR_ACTIVE) vec = input_vec; 158 CeedCallBackend(CeedVectorRestoreArrayRead(vec, &impl->fields->inputs[i])); 159 } 160 } 161 162 // Restore output arrays 163 for (CeedInt i = 0; i < num_output_fields; i++) { 164 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_output_fields[i], &eval_mode)); 165 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 166 } else { 167 CeedVector vec; 168 169 CeedCallBackend(CeedOperatorFieldGetVector(op_output_fields[i], &vec)); 170 if (vec == CEED_VECTOR_ACTIVE) vec = output_vec; 171 // Check for multiple output modes 172 CeedInt index = -1; 173 174 for (CeedInt j = 0; j < i; j++) { 175 if (vec == output_vecs[j]) { 176 index = j; 177 break; 178 } 179 } 180 if (index == -1) { 181 CeedCallBackend(CeedVectorRestoreArray(vec, &impl->fields->outputs[i])); 182 } 183 } 184 } 185 186 // Restore context data 187 CeedCallBackend(CeedQFunctionRestoreInnerContextData(qf, &qf_impl->d_c)); 188 return CEED_ERROR_SUCCESS; 189 } 190 191 //------------------------------------------------------------------------------ 192 // Create operator 193 //------------------------------------------------------------------------------ 194 int CeedOperatorCreate_Sycl_gen(CeedOperator op) { 195 Ceed ceed; 196 Ceed_Sycl *sycl_data; 197 CeedOperator_Sycl_gen *impl; 198 199 CeedCallBackend(CeedOperatorGetCeed(op, &ceed)); 200 CeedCallBackend(CeedGetData(ceed, &sycl_data)); 201 202 CeedCallBackend(CeedCalloc(1, &impl)); 203 CeedCallBackend(CeedOperatorSetData(op, impl)); 204 205 impl->indices = sycl::malloc_device<FieldsInt_Sycl>(1, sycl_data->sycl_device, sycl_data->sycl_context); 206 impl->fields = sycl::malloc_host<Fields_Sycl>(1, sycl_data->sycl_context); 207 impl->B = sycl::malloc_device<Fields_Sycl>(1, sycl_data->sycl_device, sycl_data->sycl_context); 208 impl->G = sycl::malloc_device<Fields_Sycl>(1, sycl_data->sycl_device, sycl_data->sycl_context); 209 impl->W = sycl::malloc_device<CeedScalar>(1, sycl_data->sycl_device, sycl_data->sycl_context); 210 211 CeedCallBackend(CeedSetBackendFunctionCpp(ceed, "Operator", op, "ApplyAdd", CeedOperatorApplyAdd_Sycl_gen)); 212 CeedCallBackend(CeedSetBackendFunctionCpp(ceed, "Operator", op, "Destroy", CeedOperatorDestroy_Sycl_gen)); 213 return CEED_ERROR_SUCCESS; 214 } 215 216 //------------------------------------------------------------------------------ 217