1 // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 #include <ceed/backend.h> 9 #include <ceed/ceed.h> 10 #include <stddef.h> 11 12 #include "../hip/ceed-hip-compile.h" 13 #include "ceed-hip-gen-operator-build.h" 14 #include "ceed-hip-gen.h" 15 16 //------------------------------------------------------------------------------ 17 // Destroy operator 18 //------------------------------------------------------------------------------ 19 static int CeedOperatorDestroy_Hip_gen(CeedOperator op) { 20 CeedOperator_Hip_gen *impl; 21 CeedCallBackend(CeedOperatorGetData(op, &impl)); 22 CeedCallBackend(CeedFree(&impl)); 23 return CEED_ERROR_SUCCESS; 24 } 25 26 //------------------------------------------------------------------------------ 27 // Apply and add to output 28 //------------------------------------------------------------------------------ 29 static int CeedOperatorApplyAdd_Hip_gen(CeedOperator op, CeedVector input_vec, CeedVector output_vec, CeedRequest *request) { 30 Ceed ceed; 31 CeedCallBackend(CeedOperatorGetCeed(op, &ceed)); 32 CeedOperator_Hip_gen *data; 33 CeedCallBackend(CeedOperatorGetData(op, &data)); 34 CeedQFunction qf; 35 CeedQFunction_Hip_gen *qf_data; 36 CeedCallBackend(CeedOperatorGetQFunction(op, &qf)); 37 CeedCallBackend(CeedQFunctionGetData(qf, &qf_data)); 38 CeedInt num_elem, num_input_fields, num_output_fields; 39 CeedCallBackend(CeedOperatorGetNumElements(op, &num_elem)); 40 CeedOperatorField *op_input_fields, *op_output_fields; 41 CeedCallBackend(CeedOperatorGetFields(op, &num_input_fields, &op_input_fields, &num_output_fields, &op_output_fields)); 42 CeedQFunctionField *qf_input_fields, *qf_output_fields; 43 CeedCallBackend(CeedQFunctionGetFields(qf, NULL, &qf_input_fields, NULL, &qf_output_fields)); 44 CeedEvalMode eval_mode; 45 CeedVector vec, output_vecs[CEED_FIELD_MAX] = {}; 46 47 // Creation of the operator 48 CeedCallBackend(CeedHipGenOperatorBuild(op)); 49 50 // Input vectors 51 for (CeedInt i = 0; i < num_input_fields; i++) { 52 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_input_fields[i], &eval_mode)); 53 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 54 data->fields.inputs[i] = NULL; 55 } else { 56 // Get input vector 57 CeedCallBackend(CeedOperatorFieldGetVector(op_input_fields[i], &vec)); 58 if (vec == CEED_VECTOR_ACTIVE) vec = input_vec; 59 CeedCallBackend(CeedVectorGetArrayRead(vec, CEED_MEM_DEVICE, &data->fields.inputs[i])); 60 } 61 } 62 63 // Output vectors 64 for (CeedInt i = 0; i < num_output_fields; i++) { 65 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_output_fields[i], &eval_mode)); 66 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 67 data->fields.outputs[i] = NULL; 68 } else { 69 // Get output vector 70 CeedCallBackend(CeedOperatorFieldGetVector(op_output_fields[i], &vec)); 71 if (vec == CEED_VECTOR_ACTIVE) vec = output_vec; 72 output_vecs[i] = vec; 73 // Check for multiple output modes 74 CeedInt index = -1; 75 for (CeedInt j = 0; j < i; j++) { 76 if (vec == output_vecs[j]) { 77 index = j; 78 break; 79 } 80 } 81 if (index == -1) { 82 CeedCallBackend(CeedVectorGetArray(vec, CEED_MEM_DEVICE, &data->fields.outputs[i])); 83 } else { 84 data->fields.outputs[i] = data->fields.outputs[index]; 85 } 86 } 87 } 88 89 // Get context data 90 CeedCallBackend(CeedQFunctionGetInnerContextData(qf, CEED_MEM_DEVICE, &qf_data->d_c)); 91 92 // Apply operator 93 void *opargs[] = {(void *)&num_elem, &qf_data->d_c, &data->indices, &data->fields, &data->B, &data->G, &data->W}; 94 const CeedInt dim = data->dim; 95 const CeedInt Q_1d = data->Q_1d; 96 const CeedInt P_1d = data->max_P_1d; 97 const CeedInt thread_1d = CeedIntMax(Q_1d, P_1d); 98 CeedInt block_sizes[3]; 99 CeedCallBackend(BlockGridCalculate_Hip_gen(dim, num_elem, P_1d, Q_1d, block_sizes)); 100 if (dim == 1) { 101 CeedInt grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 102 CeedInt sharedMem = block_sizes[2] * thread_1d * sizeof(CeedScalar); 103 CeedCallBackend(CeedRunKernelDimSharedHip(ceed, data->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 104 } else if (dim == 2) { 105 CeedInt grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 106 CeedInt sharedMem = block_sizes[2] * thread_1d * thread_1d * sizeof(CeedScalar); 107 CeedCallBackend(CeedRunKernelDimSharedHip(ceed, data->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 108 } else if (dim == 3) { 109 CeedInt grid = num_elem / block_sizes[2] + ((num_elem / block_sizes[2] * block_sizes[2] < num_elem) ? 1 : 0); 110 CeedInt sharedMem = block_sizes[2] * thread_1d * thread_1d * sizeof(CeedScalar); 111 CeedCallBackend(CeedRunKernelDimSharedHip(ceed, data->op, grid, block_sizes[0], block_sizes[1], block_sizes[2], sharedMem, opargs)); 112 } 113 114 // Restore input arrays 115 for (CeedInt i = 0; i < num_input_fields; i++) { 116 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_input_fields[i], &eval_mode)); 117 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 118 } else { 119 CeedCallBackend(CeedOperatorFieldGetVector(op_input_fields[i], &vec)); 120 if (vec == CEED_VECTOR_ACTIVE) vec = input_vec; 121 CeedCallBackend(CeedVectorRestoreArrayRead(vec, &data->fields.inputs[i])); 122 } 123 } 124 125 // Restore output arrays 126 for (CeedInt i = 0; i < num_output_fields; i++) { 127 CeedCallBackend(CeedQFunctionFieldGetEvalMode(qf_output_fields[i], &eval_mode)); 128 if (eval_mode == CEED_EVAL_WEIGHT) { // Skip 129 } else { 130 CeedCallBackend(CeedOperatorFieldGetVector(op_output_fields[i], &vec)); 131 if (vec == CEED_VECTOR_ACTIVE) vec = output_vec; 132 // Check for multiple output modes 133 CeedInt index = -1; 134 for (CeedInt j = 0; j < i; j++) { 135 if (vec == output_vecs[j]) { 136 index = j; 137 break; 138 } 139 } 140 if (index == -1) { 141 CeedCallBackend(CeedVectorRestoreArray(vec, &data->fields.outputs[i])); 142 } 143 } 144 } 145 146 // Restore context data 147 CeedCallBackend(CeedQFunctionRestoreInnerContextData(qf, &qf_data->d_c)); 148 149 return CEED_ERROR_SUCCESS; 150 } 151 152 //------------------------------------------------------------------------------ 153 // Create operator 154 //------------------------------------------------------------------------------ 155 int CeedOperatorCreate_Hip_gen(CeedOperator op) { 156 Ceed ceed; 157 CeedCallBackend(CeedOperatorGetCeed(op, &ceed)); 158 CeedOperator_Hip_gen *impl; 159 160 CeedCallBackend(CeedCalloc(1, &impl)); 161 CeedCallBackend(CeedOperatorSetData(op, impl)); 162 163 CeedCallBackend(CeedSetBackendFunction(ceed, "Operator", op, "ApplyAdd", CeedOperatorApplyAdd_Hip_gen)); 164 CeedCallBackend(CeedSetBackendFunction(ceed, "Operator", op, "Destroy", CeedOperatorDestroy_Hip_gen)); 165 return CEED_ERROR_SUCCESS; 166 } 167 //------------------------------------------------------------------------------ 168