13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 321617c04Sjeremylt // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 521617c04Sjeremylt // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 721617c04Sjeremylt 849aac155SJeremy L Thompson #include <ceed.h> 9ec3da8bcSJed Brown #include <ceed/backend.h> 103d576824SJeremy L Thompson #include <stdbool.h> 11fcbe8c06SSebastian Grimberg #include <stdlib.h> 123d576824SJeremy L Thompson #include <string.h> 132b730f8bSJeremy L Thompson 1421617c04Sjeremylt #include "ceed-ref.h" 1521617c04Sjeremylt 16f10650afSjeremylt //------------------------------------------------------------------------------ 17f10650afSjeremylt // Core ElemRestriction Apply Code 18f10650afSjeremylt //------------------------------------------------------------------------------ 191cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 2094648b7dSSebastian Grimberg CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size, 2194648b7dSSebastian Grimberg CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 2294648b7dSSebastian Grimberg // No offsets provided, identity restriction 23d1d35e2fSjeremylt bool has_backend_strides; 24ad70ee2cSJeremy L Thompson 251cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides)); 26d1d35e2fSjeremylt if (has_backend_strides) { 27d1d35e2fSjeremylt // CPU backend strides are {1, elem_size, elem_size*num_comp} 287f90ec76Sjeremylt // This if branch is left separate to allow better inlining 29ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 302b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 312b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 32ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 33ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 342b730f8bSJeremy L Thompson uu[n + k * elem_size + CeedIntMin(e + j, num_elem - 1) * elem_size * num_comp]; 352b730f8bSJeremy L Thompson } 362b730f8bSJeremy L Thompson } 372b730f8bSJeremy L Thompson } 382b730f8bSJeremy L Thompson } 397f90ec76Sjeremylt } else { 407f90ec76Sjeremylt // User provided strides 417f90ec76Sjeremylt CeedInt strides[3]; 42ad70ee2cSJeremy L Thompson 431cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetStrides(rstr, &strides)); 44ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 452b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 462b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 47ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 48ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 492b730f8bSJeremy L Thompson uu[n * strides[0] + k * strides[1] + CeedIntMin(e + j, num_elem - 1) * strides[2]]; 502b730f8bSJeremy L Thompson } 512b730f8bSJeremy L Thompson } 522b730f8bSJeremy L Thompson } 532b730f8bSJeremy L Thompson } 547509a596Sjeremylt } 5594648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 5694648b7dSSebastian Grimberg } 5794648b7dSSebastian Grimberg 581cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 5994648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 6094648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 61fcbe8c06SSebastian Grimberg // Default restriction with offsets 6294648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 63ad70ee2cSJeremy L Thompson 641cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 65ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 662b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 67ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt i = 0; i < elem_size * block_size; i++) { 68ad70ee2cSJeremy L Thompson vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] = uu[impl->offsets[i + e * elem_size] + k * comp_stride]; 69fcbe8c06SSebastian Grimberg } 70fcbe8c06SSebastian Grimberg } 71fcbe8c06SSebastian Grimberg } 7294648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 7394648b7dSSebastian Grimberg } 7494648b7dSSebastian Grimberg 751cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 7694648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 7794648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 78fcbe8c06SSebastian Grimberg // Restriction with orientations 7994648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 80ad70ee2cSJeremy L Thompson 811cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 82ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 83fcbe8c06SSebastian Grimberg CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 84ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt i = 0; i < elem_size * block_size; i++) { 85ad70ee2cSJeremy L Thompson vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] = 867c1dbaffSSebastian Grimberg uu[impl->offsets[i + e * elem_size] + k * comp_stride] * (impl->orients[i + e * elem_size] ? -1.0 : 1.0); 87fcbe8c06SSebastian Grimberg } 88fcbe8c06SSebastian Grimberg } 89fcbe8c06SSebastian Grimberg } 9094648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 9194648b7dSSebastian Grimberg } 9294648b7dSSebastian Grimberg 931cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 9494648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 9594648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, 9694648b7dSSebastian Grimberg CeedScalar *vv) { 9777d1c127SSebastian Grimberg // Restriction with tridiagonal transformation 9894648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 99ad70ee2cSJeremy L Thompson 1001cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 101ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 102fcbe8c06SSebastian Grimberg CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 1030c73c039SSebastian Grimberg CeedInt n = 0; 104ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 105ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 106ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 107ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 108ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 109ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]; 1100c73c039SSebastian Grimberg } 1110c73c039SSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 112ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 113ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 114ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 115ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] + 116ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 117ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 118ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 119ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]; 1200c73c039SSebastian Grimberg } 1210c73c039SSebastian Grimberg } 122ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 123ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 124ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 125ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] + 126ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 127ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]; 1282b730f8bSJeremy L Thompson } 1292b730f8bSJeremy L Thompson } 1302b730f8bSJeremy L Thompson } 1310c73c039SSebastian Grimberg return CEED_ERROR_SUCCESS; 132fcbe8c06SSebastian Grimberg } 1330c73c039SSebastian Grimberg 1341cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, 135ad70ee2cSJeremy L Thompson const CeedInt block_size, const CeedInt comp_stride, CeedInt start, 13694648b7dSSebastian Grimberg CeedInt stop, CeedInt num_elem, CeedInt elem_size, 13794648b7dSSebastian Grimberg CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 13894648b7dSSebastian Grimberg // Restriction with (unsigned) tridiagonal transformation 1390c73c039SSebastian Grimberg CeedElemRestriction_Ref *impl; 140ad70ee2cSJeremy L Thompson 1411cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 142ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 1437c1dbaffSSebastian Grimberg CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 1447c1dbaffSSebastian Grimberg CeedInt n = 0; 145ad70ee2cSJeremy L Thompson 146ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 147ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 148ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 149ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 150ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 151ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]); 1527c1dbaffSSebastian Grimberg } 1537c1dbaffSSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 154ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 155ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 156ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 157ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) + 158ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 159ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 160ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 161ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]); 1627c1dbaffSSebastian Grimberg } 1637c1dbaffSSebastian Grimberg } 164ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 165ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 166ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 167ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) + 168ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 169ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]); 1707c1dbaffSSebastian Grimberg } 1717c1dbaffSSebastian Grimberg } 1727c1dbaffSSebastian Grimberg } 1737c1dbaffSSebastian Grimberg return CEED_ERROR_SUCCESS; 1747c1dbaffSSebastian Grimberg } 1757c1dbaffSSebastian Grimberg 1761cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 17794648b7dSSebastian Grimberg CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size, 17894648b7dSSebastian Grimberg CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 17994648b7dSSebastian Grimberg // No offsets provided, identity restriction 180d1d35e2fSjeremylt bool has_backend_strides; 181ad70ee2cSJeremy L Thompson 1821cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides)); 183d1d35e2fSjeremylt if (has_backend_strides) { 184d1d35e2fSjeremylt // CPU backend strides are {1, elem_size, elem_size*num_comp} 1857f90ec76Sjeremylt // This if brach is left separate to allow better inlining 186ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 1872b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 1882b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 189ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < CeedIntMin(block_size, num_elem - e); j++) { 190bf9b6c6bSSebastian Grimberg vv[n + k * elem_size + (e + j) * elem_size * num_comp] += uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset]; 1912b730f8bSJeremy L Thompson } 1922b730f8bSJeremy L Thompson } 1932b730f8bSJeremy L Thompson } 1942b730f8bSJeremy L Thompson } 1957f90ec76Sjeremylt } else { 1967f90ec76Sjeremylt // User provided strides 1977f90ec76Sjeremylt CeedInt strides[3]; 198ad70ee2cSJeremy L Thompson 1991cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetStrides(rstr, &strides)); 200ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 2012b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 2022b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 203ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < CeedIntMin(block_size, num_elem - e); j++) { 204*6d5e61d4SSebastian Grimberg vv[n * strides[0] + k * strides[1] + (e + j) * strides[2]] += 205*6d5e61d4SSebastian Grimberg uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset]; 2062b730f8bSJeremy L Thompson } 2072b730f8bSJeremy L Thompson } 2082b730f8bSJeremy L Thompson } 2092b730f8bSJeremy L Thompson } 210523b8ea0Sjeremylt } 21194648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 21294648b7dSSebastian Grimberg } 21394648b7dSSebastian Grimberg 2141cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStandardTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 21594648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 21694648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 217fcbe8c06SSebastian Grimberg // Default restriction with offsets 21894648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 219ad70ee2cSJeremy L Thompson 2201cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 221ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 2222b730f8bSJeremy L Thompson for (CeedInt k = 0; k < num_comp; k++) { 223ad70ee2cSJeremy L Thompson for (CeedInt i = 0; i < elem_size * block_size; i += block_size) { 2248d94b059Sjeremylt // Iteration bound set to discard padding elements 225ad70ee2cSJeremy L Thompson for (CeedInt j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) { 22658c07c4fSSebastian Grimberg CeedScalar uu_val; 22758c07c4fSSebastian Grimberg 22858c07c4fSSebastian Grimberg uu_val = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset]; 22958c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += uu_val; 230fcbe8c06SSebastian Grimberg } 231fcbe8c06SSebastian Grimberg } 232fcbe8c06SSebastian Grimberg } 233fcbe8c06SSebastian Grimberg } 23494648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 23594648b7dSSebastian Grimberg } 23694648b7dSSebastian Grimberg 2371cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 23894648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 23994648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 240fcbe8c06SSebastian Grimberg // Restriction with orientations 24194648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 242ad70ee2cSJeremy L Thompson 2431cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 244ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 245fcbe8c06SSebastian Grimberg for (CeedInt k = 0; k < num_comp; k++) { 246ad70ee2cSJeremy L Thompson for (CeedInt i = 0; i < elem_size * block_size; i += block_size) { 247fcbe8c06SSebastian Grimberg // Iteration bound set to discard padding elements 248ad70ee2cSJeremy L Thompson for (CeedInt j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) { 24958c07c4fSSebastian Grimberg CeedScalar uu_val; 25058c07c4fSSebastian Grimberg 25158c07c4fSSebastian Grimberg uu_val = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset] * (impl->orients[j + e * elem_size] ? -1.0 : 1.0); 25258c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += uu_val; 253fcbe8c06SSebastian Grimberg } 254fcbe8c06SSebastian Grimberg } 255fcbe8c06SSebastian Grimberg } 256fcbe8c06SSebastian Grimberg } 25794648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 25894648b7dSSebastian Grimberg } 25994648b7dSSebastian Grimberg 2601cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 26194648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 26294648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 26377d1c127SSebastian Grimberg // Restriction with tridiagonal transformation 26494648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 265ad70ee2cSJeremy L Thompson 2661cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 267ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 268fcbe8c06SSebastian Grimberg for (CeedInt k = 0; k < num_comp; k++) { 269fcbe8c06SSebastian Grimberg // Iteration bound set to discard padding elements 27058c07c4fSSebastian Grimberg const CeedInt block_end = CeedIntMin(block_size, num_elem - e); 27158c07c4fSSebastian Grimberg CeedInt n = 0; 27258c07c4fSSebastian Grimberg 273ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 27458c07c4fSSebastian Grimberg CeedScalar uu_val; 27558c07c4fSSebastian Grimberg 27658c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 277ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 278ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 279ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]; 28058c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 2810c73c039SSebastian Grimberg } 2820c73c039SSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 283ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 28458c07c4fSSebastian Grimberg CeedScalar uu_val; 28558c07c4fSSebastian Grimberg 28658c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 287ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] + 288ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 289ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 290ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 291ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]; 29258c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 2930c73c039SSebastian Grimberg } 2940c73c039SSebastian Grimberg } 295ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 29658c07c4fSSebastian Grimberg CeedScalar uu_val; 29758c07c4fSSebastian Grimberg 29858c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 299ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] + 300ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 301ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]; 30258c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 30321617c04Sjeremylt } 304b435c5a6Srezgarshakeri } 3052b730f8bSJeremy L Thompson } 306e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 30721617c04Sjeremylt } 30821617c04Sjeremylt 3091cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, 310ad70ee2cSJeremy L Thompson const CeedInt block_size, const CeedInt comp_stride, CeedInt start, 31194648b7dSSebastian Grimberg CeedInt stop, CeedInt num_elem, CeedInt elem_size, CeedInt v_offset, 31294648b7dSSebastian Grimberg const CeedScalar *uu, CeedScalar *vv) { 31394648b7dSSebastian Grimberg // Restriction with (unsigned) tridiagonal transformation 3147c1dbaffSSebastian Grimberg CeedElemRestriction_Ref *impl; 315ad70ee2cSJeremy L Thompson 3161cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 317ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 3187c1dbaffSSebastian Grimberg for (CeedInt k = 0; k < num_comp; k++) { 3197c1dbaffSSebastian Grimberg // Iteration bound set to discard padding elements 320ad70ee2cSJeremy L Thompson const CeedInt block_end = CeedIntMin(block_size, num_elem - e); 32158c07c4fSSebastian Grimberg CeedInt n = 0; 322ad70ee2cSJeremy L Thompson 323ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 32458c07c4fSSebastian Grimberg CeedScalar uu_val; 32558c07c4fSSebastian Grimberg 32658c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 327ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 328ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 329ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]); 33058c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 3317c1dbaffSSebastian Grimberg } 3327c1dbaffSSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 333ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 33458c07c4fSSebastian Grimberg CeedScalar uu_val; 33558c07c4fSSebastian Grimberg 33658c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 337ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) + 338ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 339ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 340ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 341ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]); 34258c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 3437c1dbaffSSebastian Grimberg } 3447c1dbaffSSebastian Grimberg } 345ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 34658c07c4fSSebastian Grimberg CeedScalar uu_val; 34758c07c4fSSebastian Grimberg 34858c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 349ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) + 350ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 351ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]); 35258c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 3537c1dbaffSSebastian Grimberg } 3547c1dbaffSSebastian Grimberg } 3557c1dbaffSSebastian Grimberg } 3567c1dbaffSSebastian Grimberg return CEED_ERROR_SUCCESS; 3577c1dbaffSSebastian Grimberg } 3587c1dbaffSSebastian Grimberg 3591249ccc5SJeremy L Thompson static inline int CeedElemRestrictionApplyAtPointsInElement_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, CeedInt start, CeedInt stop, 36005fa913cSJeremy L Thompson CeedTransposeMode t_mode, const CeedScalar *uu, CeedScalar *vv) { 36105fa913cSJeremy L Thompson CeedInt num_points, l_vec_offset, e_vec_offset = 0; 36205fa913cSJeremy L Thompson CeedElemRestriction_Ref *impl; 36305fa913cSJeremy L Thompson 36405fa913cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 36505fa913cSJeremy L Thompson 36605fa913cSJeremy L Thompson for (CeedInt e = start; e < stop; e++) { 3670930e4e7SJeremy L Thompson l_vec_offset = impl->offsets[e]; 36805fa913cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumPointsInElement(rstr, e, &num_points)); 36905fa913cSJeremy L Thompson if (t_mode == CEED_NOTRANSPOSE) { 37005fa913cSJeremy L Thompson for (CeedInt i = 0; i < num_points; i++) { 3719c34f28eSJeremy L Thompson for (CeedInt j = 0; j < num_comp; j++) vv[j * num_points + i + e_vec_offset] = uu[impl->offsets[i + l_vec_offset] * num_comp + j]; 37205fa913cSJeremy L Thompson } 37305fa913cSJeremy L Thompson } else { 37405fa913cSJeremy L Thompson for (CeedInt i = 0; i < num_points; i++) { 3759c34f28eSJeremy L Thompson for (CeedInt j = 0; j < num_comp; j++) vv[impl->offsets[i + l_vec_offset] * num_comp + j] = uu[j * num_points + i + e_vec_offset]; 37605fa913cSJeremy L Thompson } 37705fa913cSJeremy L Thompson } 37805fa913cSJeremy L Thompson e_vec_offset += num_points * num_comp; 37905fa913cSJeremy L Thompson } 38005fa913cSJeremy L Thompson return CEED_ERROR_SUCCESS; 38105fa913cSJeremy L Thompson } 38205fa913cSJeremy L Thompson 3831cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApply_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 384ad70ee2cSJeremy L Thompson const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, 385ad70ee2cSJeremy L Thompson bool use_orients, CeedVector u, CeedVector v, CeedRequest *request) { 386ad70ee2cSJeremy L Thompson CeedInt num_elem, elem_size, v_offset; 387ad70ee2cSJeremy L Thompson CeedRestrictionType rstr_type; 3887c1dbaffSSebastian Grimberg const CeedScalar *uu; 3897c1dbaffSSebastian Grimberg CeedScalar *vv; 390ad70ee2cSJeremy L Thompson 3911cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem)); 3921cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size)); 393ad70ee2cSJeremy L Thompson v_offset = start * block_size * elem_size * num_comp; 3941cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type)); 39594648b7dSSebastian Grimberg CeedCallBackend(CeedVectorGetArrayRead(u, CEED_MEM_HOST, &uu)); 396ad70ee2cSJeremy L Thompson 39794648b7dSSebastian Grimberg if (t_mode == CEED_TRANSPOSE) { 39894648b7dSSebastian Grimberg // Sum into for transpose mode, E-vector to L-vector 39994648b7dSSebastian Grimberg CeedCallBackend(CeedVectorGetArray(v, CEED_MEM_HOST, &vv)); 40094648b7dSSebastian Grimberg } else { 40194648b7dSSebastian Grimberg // Overwrite for notranspose mode, L-vector to E-vector 40294648b7dSSebastian Grimberg CeedCallBackend(CeedVectorGetArrayWrite(v, CEED_MEM_HOST, &vv)); 40394648b7dSSebastian Grimberg } 40494648b7dSSebastian Grimberg if (t_mode == CEED_TRANSPOSE) { 4057c1dbaffSSebastian Grimberg // Restriction from E-vector to L-vector 4067c1dbaffSSebastian Grimberg // Performing v += r^T * u 4077c1dbaffSSebastian Grimberg // uu has shape [elem_size, num_comp, num_elem], row-major 4087c1dbaffSSebastian Grimberg // vv has shape [nnodes, num_comp] 4097c1dbaffSSebastian Grimberg // Sum into for transpose mode 4107c1dbaffSSebastian Grimberg switch (rstr_type) { 4117c1dbaffSSebastian Grimberg case CEED_RESTRICTION_STRIDED: 4125d10938bSJeremy L Thompson CeedCallBackend( 4131cc2c60dSJeremy L Thompson CeedElemRestrictionApplyStridedTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv)); 41494648b7dSSebastian Grimberg break; 41561a27d74SSebastian Grimberg case CEED_RESTRICTION_STANDARD: 4161cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size, 4175d10938bSJeremy L Thompson v_offset, uu, vv)); 41894648b7dSSebastian Grimberg break; 4197c1dbaffSSebastian Grimberg case CEED_RESTRICTION_ORIENTED: 42094648b7dSSebastian Grimberg if (use_signs) { 4211cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4221cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 42394648b7dSSebastian Grimberg } else { 4241cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4251cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 42694648b7dSSebastian Grimberg } 42794648b7dSSebastian Grimberg break; 42894648b7dSSebastian Grimberg case CEED_RESTRICTION_CURL_ORIENTED: 42994648b7dSSebastian Grimberg if (use_signs && use_orients) { 4301cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4315d10938bSJeremy L Thompson elem_size, v_offset, uu, vv)); 43294648b7dSSebastian Grimberg } else if (use_orients) { 4331cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, 4341cc2c60dSJeremy L Thompson num_elem, elem_size, v_offset, uu, vv)); 43594648b7dSSebastian Grimberg } else { 4361cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4371cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 43894648b7dSSebastian Grimberg } 43994648b7dSSebastian Grimberg break; 4402c7e7413SJeremy L Thompson case CEED_RESTRICTION_POINTS: 4411249ccc5SJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv)); 4422c7e7413SJeremy L Thompson break; 44394648b7dSSebastian Grimberg } 44494648b7dSSebastian Grimberg } else { 44594648b7dSSebastian Grimberg // Restriction from L-vector to E-vector 44694648b7dSSebastian Grimberg // Perform: v = r * u 44794648b7dSSebastian Grimberg // vv has shape [elem_size, num_comp, num_elem], row-major 44894648b7dSSebastian Grimberg // uu has shape [nnodes, num_comp] 44994648b7dSSebastian Grimberg // Overwrite for notranspose mode 45094648b7dSSebastian Grimberg switch (rstr_type) { 45194648b7dSSebastian Grimberg case CEED_RESTRICTION_STRIDED: 4525d10938bSJeremy L Thompson CeedCallBackend( 4531cc2c60dSJeremy L Thompson CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv)); 45494648b7dSSebastian Grimberg break; 45561a27d74SSebastian Grimberg case CEED_RESTRICTION_STANDARD: 4561cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4571cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 45894648b7dSSebastian Grimberg break; 45994648b7dSSebastian Grimberg case CEED_RESTRICTION_ORIENTED: 46094648b7dSSebastian Grimberg if (use_signs) { 4611cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4621cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 46394648b7dSSebastian Grimberg } else { 4641cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4651cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 46694648b7dSSebastian Grimberg } 46794648b7dSSebastian Grimberg break; 46894648b7dSSebastian Grimberg case CEED_RESTRICTION_CURL_ORIENTED: 46994648b7dSSebastian Grimberg if (use_signs && use_orients) { 4701cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4715d10938bSJeremy L Thompson elem_size, v_offset, uu, vv)); 47294648b7dSSebastian Grimberg } else if (use_orients) { 4731cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, 4745d10938bSJeremy L Thompson num_elem, elem_size, v_offset, uu, vv)); 47594648b7dSSebastian Grimberg } else { 4761cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4771cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 47894648b7dSSebastian Grimberg } 47994648b7dSSebastian Grimberg break; 4802c7e7413SJeremy L Thompson case CEED_RESTRICTION_POINTS: 4811249ccc5SJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv)); 4822c7e7413SJeremy L Thompson break; 48394648b7dSSebastian Grimberg } 4847c1dbaffSSebastian Grimberg } 4857c1dbaffSSebastian Grimberg CeedCallBackend(CeedVectorRestoreArrayRead(u, &uu)); 4867c1dbaffSSebastian Grimberg CeedCallBackend(CeedVectorRestoreArray(v, &vv)); 4877c1dbaffSSebastian Grimberg if (request != CEED_REQUEST_IMMEDIATE && request != CEED_REQUEST_ORDERED) *request = NULL; 4887c1dbaffSSebastian Grimberg return CEED_ERROR_SUCCESS; 4897c1dbaffSSebastian Grimberg } 4907c1dbaffSSebastian Grimberg 4917c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------ 492f10650afSjeremylt // ElemRestriction Apply - Common Sizes 493f10650afSjeremylt //------------------------------------------------------------------------------ 4941cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_110(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 4957c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 4967c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 4971cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 498d979a051Sjeremylt } 499d979a051Sjeremylt 5001cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_111(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5017c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5027c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5031cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 5044d2a38eeSjeremylt } 5054d2a38eeSjeremylt 5061cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_180(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5077c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5087c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5091cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 5109c36149bSjeremylt } 5119c36149bSjeremylt 5121cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_181(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5137c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5147c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5151cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 5169c36149bSjeremylt } 5179c36149bSjeremylt 5181cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_310(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5197c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5207c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5211cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 522d979a051Sjeremylt } 523d979a051Sjeremylt 5241cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_311(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5257c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5267c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5271cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 528d979a051Sjeremylt } 529d979a051Sjeremylt 5301cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_380(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5317c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5327c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5331cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 534d979a051Sjeremylt } 535d979a051Sjeremylt 5361cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_381(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5377c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5387c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5391cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 540d979a051Sjeremylt } 541d979a051Sjeremylt 542bf4d1581Sjeremylt // LCOV_EXCL_START 5431cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_510(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5447c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5457c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5461cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 547d979a051Sjeremylt } 548bf4d1581Sjeremylt // LCOV_EXCL_STOP 549d979a051Sjeremylt 5501cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_511(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5517c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5527c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5531cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 554d979a051Sjeremylt } 555d979a051Sjeremylt 556bf4d1581Sjeremylt // LCOV_EXCL_START 5571cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_580(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5587c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5597c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5601cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 561d979a051Sjeremylt } 562bf4d1581Sjeremylt // LCOV_EXCL_STOP 563d979a051Sjeremylt 5641cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_581(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5657c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5660c73c039SSebastian Grimberg CeedVector v, CeedRequest *request) { 5671cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 5684d2a38eeSjeremylt } 5694d2a38eeSjeremylt 570f10650afSjeremylt //------------------------------------------------------------------------------ 571f10650afSjeremylt // ElemRestriction Apply 572f10650afSjeremylt //------------------------------------------------------------------------------ 5731cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, CeedRequest *request) { 574ad70ee2cSJeremy L Thompson CeedInt num_block, block_size, num_comp, comp_stride; 575ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 576ad70ee2cSJeremy L Thompson 5771cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 5781cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 5791cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 5801cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 5811cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 5821cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, true, true, u, v, request)); 5835d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 584f30b1135SSebastian Grimberg } 585f30b1135SSebastian Grimberg 586f30b1135SSebastian Grimberg //------------------------------------------------------------------------------ 587f30b1135SSebastian Grimberg // ElemRestriction Apply Unsigned 588f30b1135SSebastian Grimberg //------------------------------------------------------------------------------ 5891cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnsigned_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 5901cc2c60dSJeremy L Thompson CeedRequest *request) { 591ad70ee2cSJeremy L Thompson CeedInt num_block, block_size, num_comp, comp_stride; 592ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 593ad70ee2cSJeremy L Thompson 5941cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 5951cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 5961cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 5971cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 5981cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 5991cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, true, u, v, request)); 6005d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 6017c1dbaffSSebastian Grimberg } 6027c1dbaffSSebastian Grimberg 6037c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------ 6047c1dbaffSSebastian Grimberg // ElemRestriction Apply Unoriented 6057c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------ 6061cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnoriented_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 6071cc2c60dSJeremy L Thompson CeedRequest *request) { 608ad70ee2cSJeremy L Thompson CeedInt num_block, block_size, num_comp, comp_stride; 609ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 610ad70ee2cSJeremy L Thompson 6111cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 6121cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 6131cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 6141cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 6151cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 6161cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, false, u, v, request)); 6175d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 6189c36149bSjeremylt } 619be9261b7Sjeremylt 620f10650afSjeremylt //------------------------------------------------------------------------------ 6212c7e7413SJeremy L Thompson // ElemRestriction Apply Points 6222c7e7413SJeremy L Thompson //------------------------------------------------------------------------------ 6231249ccc5SJeremy L Thompson static int CeedElemRestrictionApplyAtPointsInElement_Ref(CeedElemRestriction r, CeedInt elem, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 6242c7e7413SJeremy L Thompson CeedRequest *request) { 62505fa913cSJeremy L Thompson CeedInt num_comp; 6262c7e7413SJeremy L Thompson CeedElemRestriction_Ref *impl; 6272c7e7413SJeremy L Thompson 6282c7e7413SJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(r, &num_comp)); 62905fa913cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(r, &impl)); 63005fa913cSJeremy L Thompson return impl->Apply(r, num_comp, 0, 1, elem, elem + 1, t_mode, false, false, u, v, request); 6312c7e7413SJeremy L Thompson } 6322c7e7413SJeremy L Thompson 6332c7e7413SJeremy L Thompson //------------------------------------------------------------------------------ 634f10650afSjeremylt // ElemRestriction Apply Block 635f10650afSjeremylt //------------------------------------------------------------------------------ 6361cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyBlock_Ref(CeedElemRestriction rstr, CeedInt block, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 637074cb416Sjeremylt CeedRequest *request) { 638ad70ee2cSJeremy L Thompson CeedInt block_size, num_comp, comp_stride; 639ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 640ad70ee2cSJeremy L Thompson 6411cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 6421cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 6431cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 6441cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 6451cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, block, block + 1, t_mode, true, true, u, v, request)); 6465d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 6479c36149bSjeremylt } 648be9261b7Sjeremylt 649f10650afSjeremylt //------------------------------------------------------------------------------ 650bd33150aSjeremylt // ElemRestriction Get Offsets 651bd33150aSjeremylt //------------------------------------------------------------------------------ 6522b730f8bSJeremy L Thompson static int CeedElemRestrictionGetOffsets_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt **offsets) { 653bd33150aSjeremylt Ceed ceed; 654ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 655ad70ee2cSJeremy L Thompson 656ad70ee2cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 6572b730f8bSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 658bd33150aSjeremylt 6596574a04fSJeremy L Thompson CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory"); 660bd33150aSjeremylt 661bd33150aSjeremylt *offsets = impl->offsets; 662e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 663bd33150aSjeremylt } 664bd33150aSjeremylt 665bd33150aSjeremylt //------------------------------------------------------------------------------ 66677d1c127SSebastian Grimberg // ElemRestriction Get Orientations 66777d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 66877d1c127SSebastian Grimberg static int CeedElemRestrictionGetOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const bool **orients) { 66977d1c127SSebastian Grimberg Ceed ceed; 670ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 671ad70ee2cSJeremy L Thompson 672ad70ee2cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 67377d1c127SSebastian Grimberg CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 67477d1c127SSebastian Grimberg 675fcbe8c06SSebastian Grimberg CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory"); 67677d1c127SSebastian Grimberg 67777d1c127SSebastian Grimberg *orients = impl->orients; 67877d1c127SSebastian Grimberg return CEED_ERROR_SUCCESS; 67977d1c127SSebastian Grimberg } 68077d1c127SSebastian Grimberg 68177d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 68277d1c127SSebastian Grimberg // ElemRestriction Get Curl-Conforming Orientations 68377d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 6840c73c039SSebastian Grimberg static int CeedElemRestrictionGetCurlOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt8 **curl_orients) { 68577d1c127SSebastian Grimberg Ceed ceed; 686ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 687ad70ee2cSJeremy L Thompson 688ad70ee2cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 68977d1c127SSebastian Grimberg CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 69077d1c127SSebastian Grimberg 691fcbe8c06SSebastian Grimberg CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory"); 69277d1c127SSebastian Grimberg 69377d1c127SSebastian Grimberg *curl_orients = impl->curl_orients; 69477d1c127SSebastian Grimberg return CEED_ERROR_SUCCESS; 69577d1c127SSebastian Grimberg } 69677d1c127SSebastian Grimberg 69777d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 698f10650afSjeremylt // ElemRestriction Destroy 699f10650afSjeremylt //------------------------------------------------------------------------------ 7001cc2c60dSJeremy L Thompson static int CeedElemRestrictionDestroy_Ref(CeedElemRestriction rstr) { 701fe2413ffSjeremylt CeedElemRestriction_Ref *impl; 70221617c04Sjeremylt 7031cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 7042b730f8bSJeremy L Thompson CeedCallBackend(CeedFree(&impl->offsets_allocated)); 70577d1c127SSebastian Grimberg CeedCallBackend(CeedFree(&impl->orients_allocated)); 70677d1c127SSebastian Grimberg CeedCallBackend(CeedFree(&impl->curl_orients_allocated)); 7072b730f8bSJeremy L Thompson CeedCallBackend(CeedFree(&impl)); 708e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 70921617c04Sjeremylt } 71021617c04Sjeremylt 711f10650afSjeremylt //------------------------------------------------------------------------------ 712f10650afSjeremylt // ElemRestriction Create 713f10650afSjeremylt //------------------------------------------------------------------------------ 714fcbe8c06SSebastian Grimberg int CeedElemRestrictionCreate_Ref(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *offsets, const bool *orients, 7151cc2c60dSJeremy L Thompson const CeedInt8 *curl_orients, CeedElemRestriction rstr) { 716ad70ee2cSJeremy L Thompson Ceed ceed; 71707d5dec1SJeremy L Thompson CeedInt num_elem, elem_size, num_block, block_size, num_comp, comp_stride, num_points = 0, num_offsets; 718ad70ee2cSJeremy L Thompson CeedRestrictionType rstr_type; 71921617c04Sjeremylt CeedElemRestriction_Ref *impl; 720ad70ee2cSJeremy L Thompson 7211cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 7221cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem)); 7231cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size)); 7241cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 7251cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 7261cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 7271cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 728ad70ee2cSJeremy L Thompson CeedInt layout[3] = {1, elem_size, elem_size * num_comp}; 72921617c04Sjeremylt 7306574a04fSJeremy L Thompson CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Only MemType = HOST supported"); 7312b730f8bSJeremy L Thompson CeedCallBackend(CeedCalloc(1, &impl)); 7323661185eSjeremylt 73392fe105eSJeremy L Thompson // Offsets data 7341cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type)); 735fcbe8c06SSebastian Grimberg if (rstr_type != CEED_RESTRICTION_STRIDED) { 7363661185eSjeremylt const char *resource; 737ad70ee2cSJeremy L Thompson 738ad70ee2cSJeremy L Thompson // Check indices for ref or memcheck backends 73935aed383SJeremy L Thompson { 74035aed383SJeremy L Thompson Ceed current = ceed, parent = NULL; 74135aed383SJeremy L Thompson 74235aed383SJeremy L Thompson CeedCallBackend(CeedGetParent(current, &parent)); 74335aed383SJeremy L Thompson while (current != parent) { 74435aed383SJeremy L Thompson current = parent; 74535aed383SJeremy L Thompson CeedCallBackend(CeedGetParent(current, &parent)); 74635aed383SJeremy L Thompson } 74735aed383SJeremy L Thompson CeedCallBackend(CeedGetResource(parent, &resource)); 74835aed383SJeremy L Thompson } 7492b730f8bSJeremy L Thompson if (!strcmp(resource, "/cpu/self/ref/serial") || !strcmp(resource, "/cpu/self/ref/blocked") || !strcmp(resource, "/cpu/self/memcheck/serial") || 750d1d35e2fSjeremylt !strcmp(resource, "/cpu/self/memcheck/blocked")) { 751e79b91d9SJeremy L Thompson CeedSize l_size; 7523661185eSjeremylt 7531cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetLVectorSize(rstr, &l_size)); 7542b730f8bSJeremy L Thompson for (CeedInt i = 0; i < num_elem * elem_size; i++) { 7556574a04fSJeremy L Thompson CeedCheck(offsets[i] >= 0 && offsets[i] + (num_comp - 1) * comp_stride < l_size, ceed, CEED_ERROR_BACKEND, 7566574a04fSJeremy L Thompson "Restriction offset %" CeedInt_FMT " (%" CeedInt_FMT ") out of range [0, %" CeedInt_FMT "]", i, offsets[i], l_size); 7572b730f8bSJeremy L Thompson } 7582b730f8bSJeremy L Thompson } 7593661185eSjeremylt 76092fe105eSJeremy L Thompson // Copy data 76107d5dec1SJeremy L Thompson if (rstr_type == CEED_RESTRICTION_POINTS) CeedCallBackend(CeedElemRestrictionGetNumPoints(rstr, &num_points)); 76207d5dec1SJeremy L Thompson num_offsets = rstr_type == CEED_RESTRICTION_POINTS ? (num_elem + 1 + num_points) : (num_elem * elem_size); 763d1d35e2fSjeremylt switch (copy_mode) { 76421617c04Sjeremylt case CEED_COPY_VALUES: 76507d5dec1SJeremy L Thompson CeedCallBackend(CeedMalloc(num_offsets, &impl->offsets_allocated)); 76607d5dec1SJeremy L Thompson memcpy(impl->offsets_allocated, offsets, num_offsets * sizeof(offsets[0])); 767d979a051Sjeremylt impl->offsets = impl->offsets_allocated; 76821617c04Sjeremylt break; 76921617c04Sjeremylt case CEED_OWN_POINTER: 770d979a051Sjeremylt impl->offsets_allocated = (CeedInt *)offsets; 771d979a051Sjeremylt impl->offsets = impl->offsets_allocated; 77221617c04Sjeremylt break; 77321617c04Sjeremylt case CEED_USE_POINTER: 774d979a051Sjeremylt impl->offsets = offsets; 77521617c04Sjeremylt } 776fcbe8c06SSebastian Grimberg 777fcbe8c06SSebastian Grimberg // Orientation data 778fcbe8c06SSebastian Grimberg if (rstr_type == CEED_RESTRICTION_ORIENTED) { 7790305e208SSebastian Grimberg CeedCheck(orients != NULL, ceed, CEED_ERROR_BACKEND, "No orients array provided for oriented restriction"); 780fcbe8c06SSebastian Grimberg switch (copy_mode) { 781fcbe8c06SSebastian Grimberg case CEED_COPY_VALUES: 78207d5dec1SJeremy L Thompson CeedCallBackend(CeedMalloc(num_offsets, &impl->orients_allocated)); 78307d5dec1SJeremy L Thompson memcpy(impl->orients_allocated, orients, num_offsets * sizeof(orients[0])); 784fcbe8c06SSebastian Grimberg impl->orients = impl->orients_allocated; 785fcbe8c06SSebastian Grimberg break; 786fcbe8c06SSebastian Grimberg case CEED_OWN_POINTER: 787fcbe8c06SSebastian Grimberg impl->orients_allocated = (bool *)orients; 788fcbe8c06SSebastian Grimberg impl->orients = impl->orients_allocated; 789fcbe8c06SSebastian Grimberg break; 790fcbe8c06SSebastian Grimberg case CEED_USE_POINTER: 791fcbe8c06SSebastian Grimberg impl->orients = orients; 792fcbe8c06SSebastian Grimberg } 793fcbe8c06SSebastian Grimberg } else if (rstr_type == CEED_RESTRICTION_CURL_ORIENTED) { 7940305e208SSebastian Grimberg CeedCheck(curl_orients != NULL, ceed, CEED_ERROR_BACKEND, "No curl_orients array provided for oriented restriction"); 795fcbe8c06SSebastian Grimberg switch (copy_mode) { 796fcbe8c06SSebastian Grimberg case CEED_COPY_VALUES: 79707d5dec1SJeremy L Thompson CeedCallBackend(CeedMalloc(3 * num_offsets, &impl->curl_orients_allocated)); 79807d5dec1SJeremy L Thompson memcpy(impl->curl_orients_allocated, curl_orients, 3 * num_offsets * sizeof(curl_orients[0])); 799fcbe8c06SSebastian Grimberg impl->curl_orients = impl->curl_orients_allocated; 800fcbe8c06SSebastian Grimberg break; 801fcbe8c06SSebastian Grimberg case CEED_OWN_POINTER: 8020c73c039SSebastian Grimberg impl->curl_orients_allocated = (CeedInt8 *)curl_orients; 803fcbe8c06SSebastian Grimberg impl->curl_orients = impl->curl_orients_allocated; 804fcbe8c06SSebastian Grimberg break; 805fcbe8c06SSebastian Grimberg case CEED_USE_POINTER: 806fcbe8c06SSebastian Grimberg impl->curl_orients = curl_orients; 807fcbe8c06SSebastian Grimberg } 808fcbe8c06SSebastian Grimberg } 80992fe105eSJeremy L Thompson } 810fe2413ffSjeremylt 8111cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionSetData(rstr, impl)); 8121cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionSetELayout(rstr, layout)); 8131cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Apply", CeedElemRestrictionApply_Ref)); 8142c7e7413SJeremy L Thompson if (rstr_type == CEED_RESTRICTION_POINTS) { 8151249ccc5SJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyAtPointsInElement", CeedElemRestrictionApplyAtPointsInElement_Ref)); 8161249ccc5SJeremy L Thompson } 8171cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnsigned", CeedElemRestrictionApplyUnsigned_Ref)); 8181cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnoriented", CeedElemRestrictionApplyUnoriented_Ref)); 8191cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyBlock", CeedElemRestrictionApplyBlock_Ref)); 8201cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOrientations", CeedElemRestrictionGetOrientations_Ref)); 8211cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetCurlOrientations", CeedElemRestrictionGetCurlOrientations_Ref)); 8222c7e7413SJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOffsets", CeedElemRestrictionGetOffsets_Ref)); 8231cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Destroy", CeedElemRestrictionDestroy_Ref)); 824d979a051Sjeremylt 825ad70ee2cSJeremy L Thompson // Set apply function based upon num_comp, block_size, and comp_stride 826ad70ee2cSJeremy L Thompson CeedInt index = -1; 827ad70ee2cSJeremy L Thompson 828ad70ee2cSJeremy L Thompson if (block_size < 10) index = 100 * num_comp + 10 * block_size + (comp_stride == 1); 829ad70ee2cSJeremy L Thompson switch (index) { 830d979a051Sjeremylt case 110: 831d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_110; 832d979a051Sjeremylt break; 833d979a051Sjeremylt case 111: 834d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_111; 835d979a051Sjeremylt break; 836d979a051Sjeremylt case 180: 837d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_180; 838d979a051Sjeremylt break; 839d979a051Sjeremylt case 181: 840d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_181; 841d979a051Sjeremylt break; 842d979a051Sjeremylt case 310: 843d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_310; 844d979a051Sjeremylt break; 845d979a051Sjeremylt case 311: 846d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_311; 847d979a051Sjeremylt break; 848d979a051Sjeremylt case 380: 849d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_380; 850d979a051Sjeremylt break; 851d979a051Sjeremylt case 381: 852d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_381; 853d979a051Sjeremylt break; 854bf4d1581Sjeremylt // LCOV_EXCL_START 855d979a051Sjeremylt case 510: 856d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_510; 857d979a051Sjeremylt break; 858bf4d1581Sjeremylt // LCOV_EXCL_STOP 859d979a051Sjeremylt case 511: 860d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_511; 861d979a051Sjeremylt break; 862bf4d1581Sjeremylt // LCOV_EXCL_START 863d979a051Sjeremylt case 580: 864d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_580; 865d979a051Sjeremylt break; 866bf4d1581Sjeremylt // LCOV_EXCL_STOP 867d979a051Sjeremylt case 581: 868d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_581; 869d979a051Sjeremylt break; 870d979a051Sjeremylt default: 871d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_Core; 872d979a051Sjeremylt break; 873d979a051Sjeremylt } 874e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 87521617c04Sjeremylt } 876fc0567d9Srezgarshakeri 877fc0567d9Srezgarshakeri //------------------------------------------------------------------------------ 878