13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors. 23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 321617c04Sjeremylt // 43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause 521617c04Sjeremylt // 63d8e8822SJeremy L Thompson // This file is part of CEED: http://github.com/ceed 721617c04Sjeremylt 849aac155SJeremy L Thompson #include <ceed.h> 9ec3da8bcSJed Brown #include <ceed/backend.h> 103d576824SJeremy L Thompson #include <stdbool.h> 11fcbe8c06SSebastian Grimberg #include <stdlib.h> 123d576824SJeremy L Thompson #include <string.h> 132b730f8bSJeremy L Thompson 1421617c04Sjeremylt #include "ceed-ref.h" 1521617c04Sjeremylt 16f10650afSjeremylt //------------------------------------------------------------------------------ 17f10650afSjeremylt // Core ElemRestriction Apply Code 18f10650afSjeremylt //------------------------------------------------------------------------------ 191cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 2094648b7dSSebastian Grimberg CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size, 2194648b7dSSebastian Grimberg CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 2294648b7dSSebastian Grimberg // No offsets provided, identity restriction 23d1d35e2fSjeremylt bool has_backend_strides; 24ad70ee2cSJeremy L Thompson 251cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides)); 26d1d35e2fSjeremylt if (has_backend_strides) { 27d1d35e2fSjeremylt // CPU backend strides are {1, elem_size, elem_size*num_comp} 287f90ec76Sjeremylt // This if branch is left separate to allow better inlining 29ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 302b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 312b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 32ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 33ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 342b730f8bSJeremy L Thompson uu[n + k * elem_size + CeedIntMin(e + j, num_elem - 1) * elem_size * num_comp]; 352b730f8bSJeremy L Thompson } 362b730f8bSJeremy L Thompson } 372b730f8bSJeremy L Thompson } 382b730f8bSJeremy L Thompson } 397f90ec76Sjeremylt } else { 407f90ec76Sjeremylt // User provided strides 417f90ec76Sjeremylt CeedInt strides[3]; 42ad70ee2cSJeremy L Thompson 431cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetStrides(rstr, &strides)); 44ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 452b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 462b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 47ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 48ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 492b730f8bSJeremy L Thompson uu[n * strides[0] + k * strides[1] + CeedIntMin(e + j, num_elem - 1) * strides[2]]; 502b730f8bSJeremy L Thompson } 512b730f8bSJeremy L Thompson } 522b730f8bSJeremy L Thompson } 532b730f8bSJeremy L Thompson } 547509a596Sjeremylt } 5594648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 5694648b7dSSebastian Grimberg } 5794648b7dSSebastian Grimberg 581cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 5994648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 6094648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 61fcbe8c06SSebastian Grimberg // Default restriction with offsets 6294648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 63ad70ee2cSJeremy L Thompson 641cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 65ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 662b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 67ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt i = 0; i < elem_size * block_size; i++) { 68ad70ee2cSJeremy L Thompson vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] = uu[impl->offsets[i + e * elem_size] + k * comp_stride]; 69fcbe8c06SSebastian Grimberg } 70fcbe8c06SSebastian Grimberg } 71fcbe8c06SSebastian Grimberg } 7294648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 7394648b7dSSebastian Grimberg } 7494648b7dSSebastian Grimberg 751cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 7694648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 7794648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 78fcbe8c06SSebastian Grimberg // Restriction with orientations 7994648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 80ad70ee2cSJeremy L Thompson 811cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 82ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 83fcbe8c06SSebastian Grimberg CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 84ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt i = 0; i < elem_size * block_size; i++) { 85ad70ee2cSJeremy L Thompson vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] = 867c1dbaffSSebastian Grimberg uu[impl->offsets[i + e * elem_size] + k * comp_stride] * (impl->orients[i + e * elem_size] ? -1.0 : 1.0); 87fcbe8c06SSebastian Grimberg } 88fcbe8c06SSebastian Grimberg } 89fcbe8c06SSebastian Grimberg } 9094648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 9194648b7dSSebastian Grimberg } 9294648b7dSSebastian Grimberg 931cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 9494648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 9594648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, 9694648b7dSSebastian Grimberg CeedScalar *vv) { 9777d1c127SSebastian Grimberg // Restriction with tridiagonal transformation 9894648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 99ad70ee2cSJeremy L Thompson 1001cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 101ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 102fcbe8c06SSebastian Grimberg CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 1030c73c039SSebastian Grimberg CeedInt n = 0; 104ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 105ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 106ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 107ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 108ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 109ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]; 1100c73c039SSebastian Grimberg } 1110c73c039SSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 112ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 113ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 114ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 115ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] + 116ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 117ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 118ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 119ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]; 1200c73c039SSebastian Grimberg } 1210c73c039SSebastian Grimberg } 122ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 123ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 124ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 125ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] + 126ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 127ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]; 1282b730f8bSJeremy L Thompson } 1292b730f8bSJeremy L Thompson } 1302b730f8bSJeremy L Thompson } 1310c73c039SSebastian Grimberg return CEED_ERROR_SUCCESS; 132fcbe8c06SSebastian Grimberg } 1330c73c039SSebastian Grimberg 1341cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, 135ad70ee2cSJeremy L Thompson const CeedInt block_size, const CeedInt comp_stride, CeedInt start, 13694648b7dSSebastian Grimberg CeedInt stop, CeedInt num_elem, CeedInt elem_size, 13794648b7dSSebastian Grimberg CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 13894648b7dSSebastian Grimberg // Restriction with (unsigned) tridiagonal transformation 1390c73c039SSebastian Grimberg CeedElemRestriction_Ref *impl; 140ad70ee2cSJeremy L Thompson 1411cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 142ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 1437c1dbaffSSebastian Grimberg CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 1447c1dbaffSSebastian Grimberg CeedInt n = 0; 145ad70ee2cSJeremy L Thompson 146ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 147ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 148ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 149ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 150ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 151ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]); 1527c1dbaffSSebastian Grimberg } 1537c1dbaffSSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 154ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 155ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 156ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 157ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) + 158ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 159ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 160ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] * 161ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]); 1627c1dbaffSSebastian Grimberg } 1637c1dbaffSSebastian Grimberg } 164ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) { 165ad70ee2cSJeremy L Thompson vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] = 166ad70ee2cSJeremy L Thompson uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] * 167ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) + 168ad70ee2cSJeremy L Thompson uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] * 169ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]); 1707c1dbaffSSebastian Grimberg } 1717c1dbaffSSebastian Grimberg } 1727c1dbaffSSebastian Grimberg } 1737c1dbaffSSebastian Grimberg return CEED_ERROR_SUCCESS; 1747c1dbaffSSebastian Grimberg } 1757c1dbaffSSebastian Grimberg 1761cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 17794648b7dSSebastian Grimberg CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size, 17894648b7dSSebastian Grimberg CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 17994648b7dSSebastian Grimberg // No offsets provided, identity restriction 180d1d35e2fSjeremylt bool has_backend_strides; 181ad70ee2cSJeremy L Thompson 1821cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides)); 183d1d35e2fSjeremylt if (has_backend_strides) { 184d1d35e2fSjeremylt // CPU backend strides are {1, elem_size, elem_size*num_comp} 1857f90ec76Sjeremylt // This if brach is left separate to allow better inlining 186ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 1872b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 1882b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 189ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < CeedIntMin(block_size, num_elem - e); j++) { 190*bf9b6c6bSSebastian Grimberg vv[n + k * elem_size + (e + j) * elem_size * num_comp] += uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset]; 1912b730f8bSJeremy L Thompson } 1922b730f8bSJeremy L Thompson } 1932b730f8bSJeremy L Thompson } 1942b730f8bSJeremy L Thompson } 1957f90ec76Sjeremylt } else { 1967f90ec76Sjeremylt // User provided strides 1977f90ec76Sjeremylt CeedInt strides[3]; 198ad70ee2cSJeremy L Thompson 1991cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetStrides(rstr, &strides)); 200ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 2012b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) { 2022b730f8bSJeremy L Thompson CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) { 203ad70ee2cSJeremy L Thompson CeedPragmaSIMD for (CeedInt j = 0; j < CeedIntMin(block_size, num_elem - e); j++) { 204*bf9b6c6bSSebastian Grimberg vv[n * strides[0] + k * strides[1] + (e + j) * strides[2]] += uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset]; 2052b730f8bSJeremy L Thompson } 2062b730f8bSJeremy L Thompson } 2072b730f8bSJeremy L Thompson } 2082b730f8bSJeremy L Thompson } 209523b8ea0Sjeremylt } 21094648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 21194648b7dSSebastian Grimberg } 21294648b7dSSebastian Grimberg 2131cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStandardTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 21494648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 21594648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 216fcbe8c06SSebastian Grimberg // Default restriction with offsets 21794648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 218ad70ee2cSJeremy L Thompson 2191cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 220ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 2212b730f8bSJeremy L Thompson for (CeedInt k = 0; k < num_comp; k++) { 222ad70ee2cSJeremy L Thompson for (CeedInt i = 0; i < elem_size * block_size; i += block_size) { 2238d94b059Sjeremylt // Iteration bound set to discard padding elements 224ad70ee2cSJeremy L Thompson for (CeedInt j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) { 22558c07c4fSSebastian Grimberg CeedScalar uu_val; 22658c07c4fSSebastian Grimberg 22758c07c4fSSebastian Grimberg uu_val = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset]; 22858c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += uu_val; 229fcbe8c06SSebastian Grimberg } 230fcbe8c06SSebastian Grimberg } 231fcbe8c06SSebastian Grimberg } 232fcbe8c06SSebastian Grimberg } 23394648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 23494648b7dSSebastian Grimberg } 23594648b7dSSebastian Grimberg 2361cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 23794648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 23894648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 239fcbe8c06SSebastian Grimberg // Restriction with orientations 24094648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 241ad70ee2cSJeremy L Thompson 2421cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 243ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 244fcbe8c06SSebastian Grimberg for (CeedInt k = 0; k < num_comp; k++) { 245ad70ee2cSJeremy L Thompson for (CeedInt i = 0; i < elem_size * block_size; i += block_size) { 246fcbe8c06SSebastian Grimberg // Iteration bound set to discard padding elements 247ad70ee2cSJeremy L Thompson for (CeedInt j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) { 24858c07c4fSSebastian Grimberg CeedScalar uu_val; 24958c07c4fSSebastian Grimberg 25058c07c4fSSebastian Grimberg uu_val = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset] * (impl->orients[j + e * elem_size] ? -1.0 : 1.0); 25158c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += uu_val; 252fcbe8c06SSebastian Grimberg } 253fcbe8c06SSebastian Grimberg } 254fcbe8c06SSebastian Grimberg } 255fcbe8c06SSebastian Grimberg } 25694648b7dSSebastian Grimberg return CEED_ERROR_SUCCESS; 25794648b7dSSebastian Grimberg } 25894648b7dSSebastian Grimberg 2591cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 26094648b7dSSebastian Grimberg const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem, 26194648b7dSSebastian Grimberg CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) { 26277d1c127SSebastian Grimberg // Restriction with tridiagonal transformation 26394648b7dSSebastian Grimberg CeedElemRestriction_Ref *impl; 264ad70ee2cSJeremy L Thompson 2651cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 266ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 267fcbe8c06SSebastian Grimberg for (CeedInt k = 0; k < num_comp; k++) { 268fcbe8c06SSebastian Grimberg // Iteration bound set to discard padding elements 26958c07c4fSSebastian Grimberg const CeedInt block_end = CeedIntMin(block_size, num_elem - e); 27058c07c4fSSebastian Grimberg CeedInt n = 0; 27158c07c4fSSebastian Grimberg 272ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 27358c07c4fSSebastian Grimberg CeedScalar uu_val; 27458c07c4fSSebastian Grimberg 27558c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 276ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 277ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 278ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]; 27958c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 2800c73c039SSebastian Grimberg } 2810c73c039SSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 282ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 28358c07c4fSSebastian Grimberg CeedScalar uu_val; 28458c07c4fSSebastian Grimberg 28558c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 286ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] + 287ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 288ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] + 289ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 290ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]; 29158c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 2920c73c039SSebastian Grimberg } 2930c73c039SSebastian Grimberg } 294ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 29558c07c4fSSebastian Grimberg CeedScalar uu_val; 29658c07c4fSSebastian Grimberg 29758c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 298ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] + 299ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 300ad70ee2cSJeremy L Thompson impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]; 30158c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 30221617c04Sjeremylt } 303b435c5a6Srezgarshakeri } 3042b730f8bSJeremy L Thompson } 305e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 30621617c04Sjeremylt } 30721617c04Sjeremylt 3081cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, 309ad70ee2cSJeremy L Thompson const CeedInt block_size, const CeedInt comp_stride, CeedInt start, 31094648b7dSSebastian Grimberg CeedInt stop, CeedInt num_elem, CeedInt elem_size, CeedInt v_offset, 31194648b7dSSebastian Grimberg const CeedScalar *uu, CeedScalar *vv) { 31294648b7dSSebastian Grimberg // Restriction with (unsigned) tridiagonal transformation 3137c1dbaffSSebastian Grimberg CeedElemRestriction_Ref *impl; 314ad70ee2cSJeremy L Thompson 3151cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 316ad70ee2cSJeremy L Thompson for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) { 3177c1dbaffSSebastian Grimberg for (CeedInt k = 0; k < num_comp; k++) { 3187c1dbaffSSebastian Grimberg // Iteration bound set to discard padding elements 319ad70ee2cSJeremy L Thompson const CeedInt block_end = CeedIntMin(block_size, num_elem - e); 32058c07c4fSSebastian Grimberg CeedInt n = 0; 321ad70ee2cSJeremy L Thompson 322ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 32358c07c4fSSebastian Grimberg CeedScalar uu_val; 32458c07c4fSSebastian Grimberg 32558c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 326ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 327ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 328ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]); 32958c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 3307c1dbaffSSebastian Grimberg } 3317c1dbaffSSebastian Grimberg for (n = 1; n < elem_size - 1; n++) { 332ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 33358c07c4fSSebastian Grimberg CeedScalar uu_val; 33458c07c4fSSebastian Grimberg 33558c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 336ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) + 337ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 338ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) + 339ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] * 340ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]); 34158c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 3427c1dbaffSSebastian Grimberg } 3437c1dbaffSSebastian Grimberg } 344ad70ee2cSJeremy L Thompson for (CeedInt j = 0; j < block_end; j++) { 34558c07c4fSSebastian Grimberg CeedScalar uu_val; 34658c07c4fSSebastian Grimberg 34758c07c4fSSebastian Grimberg uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] * 348ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) + 349ad70ee2cSJeremy L Thompson uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] * 350ad70ee2cSJeremy L Thompson abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]); 35158c07c4fSSebastian Grimberg CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val; 3527c1dbaffSSebastian Grimberg } 3537c1dbaffSSebastian Grimberg } 3547c1dbaffSSebastian Grimberg } 3557c1dbaffSSebastian Grimberg return CEED_ERROR_SUCCESS; 3567c1dbaffSSebastian Grimberg } 3577c1dbaffSSebastian Grimberg 3581249ccc5SJeremy L Thompson static inline int CeedElemRestrictionApplyAtPointsInElement_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, CeedInt start, CeedInt stop, 35905fa913cSJeremy L Thompson CeedTransposeMode t_mode, const CeedScalar *uu, CeedScalar *vv) { 36005fa913cSJeremy L Thompson CeedInt num_points, l_vec_offset, e_vec_offset = 0; 36105fa913cSJeremy L Thompson CeedElemRestriction_Ref *impl; 36205fa913cSJeremy L Thompson 36305fa913cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 36405fa913cSJeremy L Thompson 36505fa913cSJeremy L Thompson for (CeedInt e = start; e < stop; e++) { 3660930e4e7SJeremy L Thompson l_vec_offset = impl->offsets[e]; 36705fa913cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumPointsInElement(rstr, e, &num_points)); 36805fa913cSJeremy L Thompson if (t_mode == CEED_NOTRANSPOSE) { 36905fa913cSJeremy L Thompson for (CeedInt i = 0; i < num_points; i++) { 3709c34f28eSJeremy L Thompson for (CeedInt j = 0; j < num_comp; j++) vv[j * num_points + i + e_vec_offset] = uu[impl->offsets[i + l_vec_offset] * num_comp + j]; 37105fa913cSJeremy L Thompson } 37205fa913cSJeremy L Thompson } else { 37305fa913cSJeremy L Thompson for (CeedInt i = 0; i < num_points; i++) { 3749c34f28eSJeremy L Thompson for (CeedInt j = 0; j < num_comp; j++) vv[impl->offsets[i + l_vec_offset] * num_comp + j] = uu[j * num_points + i + e_vec_offset]; 37505fa913cSJeremy L Thompson } 37605fa913cSJeremy L Thompson } 37705fa913cSJeremy L Thompson e_vec_offset += num_points * num_comp; 37805fa913cSJeremy L Thompson } 37905fa913cSJeremy L Thompson return CEED_ERROR_SUCCESS; 38005fa913cSJeremy L Thompson } 38105fa913cSJeremy L Thompson 3821cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApply_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, 383ad70ee2cSJeremy L Thompson const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, 384ad70ee2cSJeremy L Thompson bool use_orients, CeedVector u, CeedVector v, CeedRequest *request) { 385ad70ee2cSJeremy L Thompson CeedInt num_elem, elem_size, v_offset; 386ad70ee2cSJeremy L Thompson CeedRestrictionType rstr_type; 3877c1dbaffSSebastian Grimberg const CeedScalar *uu; 3887c1dbaffSSebastian Grimberg CeedScalar *vv; 389ad70ee2cSJeremy L Thompson 3901cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem)); 3911cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size)); 392ad70ee2cSJeremy L Thompson v_offset = start * block_size * elem_size * num_comp; 3931cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type)); 39494648b7dSSebastian Grimberg CeedCallBackend(CeedVectorGetArrayRead(u, CEED_MEM_HOST, &uu)); 395ad70ee2cSJeremy L Thompson 39694648b7dSSebastian Grimberg if (t_mode == CEED_TRANSPOSE) { 39794648b7dSSebastian Grimberg // Sum into for transpose mode, E-vector to L-vector 39894648b7dSSebastian Grimberg CeedCallBackend(CeedVectorGetArray(v, CEED_MEM_HOST, &vv)); 39994648b7dSSebastian Grimberg } else { 40094648b7dSSebastian Grimberg // Overwrite for notranspose mode, L-vector to E-vector 40194648b7dSSebastian Grimberg CeedCallBackend(CeedVectorGetArrayWrite(v, CEED_MEM_HOST, &vv)); 40294648b7dSSebastian Grimberg } 40394648b7dSSebastian Grimberg if (t_mode == CEED_TRANSPOSE) { 4047c1dbaffSSebastian Grimberg // Restriction from E-vector to L-vector 4057c1dbaffSSebastian Grimberg // Performing v += r^T * u 4067c1dbaffSSebastian Grimberg // uu has shape [elem_size, num_comp, num_elem], row-major 4077c1dbaffSSebastian Grimberg // vv has shape [nnodes, num_comp] 4087c1dbaffSSebastian Grimberg // Sum into for transpose mode 4097c1dbaffSSebastian Grimberg switch (rstr_type) { 4107c1dbaffSSebastian Grimberg case CEED_RESTRICTION_STRIDED: 4115d10938bSJeremy L Thompson CeedCallBackend( 4121cc2c60dSJeremy L Thompson CeedElemRestrictionApplyStridedTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv)); 41394648b7dSSebastian Grimberg break; 41461a27d74SSebastian Grimberg case CEED_RESTRICTION_STANDARD: 4151cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size, 4165d10938bSJeremy L Thompson v_offset, uu, vv)); 41794648b7dSSebastian Grimberg break; 4187c1dbaffSSebastian Grimberg case CEED_RESTRICTION_ORIENTED: 41994648b7dSSebastian Grimberg if (use_signs) { 4201cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4211cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 42294648b7dSSebastian Grimberg } else { 4231cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4241cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 42594648b7dSSebastian Grimberg } 42694648b7dSSebastian Grimberg break; 42794648b7dSSebastian Grimberg case CEED_RESTRICTION_CURL_ORIENTED: 42894648b7dSSebastian Grimberg if (use_signs && use_orients) { 4291cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4305d10938bSJeremy L Thompson elem_size, v_offset, uu, vv)); 43194648b7dSSebastian Grimberg } else if (use_orients) { 4321cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, 4331cc2c60dSJeremy L Thompson num_elem, elem_size, v_offset, uu, vv)); 43494648b7dSSebastian Grimberg } else { 4351cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4361cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 43794648b7dSSebastian Grimberg } 43894648b7dSSebastian Grimberg break; 4392c7e7413SJeremy L Thompson case CEED_RESTRICTION_POINTS: 4401249ccc5SJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv)); 4412c7e7413SJeremy L Thompson break; 44294648b7dSSebastian Grimberg } 44394648b7dSSebastian Grimberg } else { 44494648b7dSSebastian Grimberg // Restriction from L-vector to E-vector 44594648b7dSSebastian Grimberg // Perform: v = r * u 44694648b7dSSebastian Grimberg // vv has shape [elem_size, num_comp, num_elem], row-major 44794648b7dSSebastian Grimberg // uu has shape [nnodes, num_comp] 44894648b7dSSebastian Grimberg // Overwrite for notranspose mode 44994648b7dSSebastian Grimberg switch (rstr_type) { 45094648b7dSSebastian Grimberg case CEED_RESTRICTION_STRIDED: 4515d10938bSJeremy L Thompson CeedCallBackend( 4521cc2c60dSJeremy L Thompson CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv)); 45394648b7dSSebastian Grimberg break; 45461a27d74SSebastian Grimberg case CEED_RESTRICTION_STANDARD: 4551cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4561cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 45794648b7dSSebastian Grimberg break; 45894648b7dSSebastian Grimberg case CEED_RESTRICTION_ORIENTED: 45994648b7dSSebastian Grimberg if (use_signs) { 4601cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4611cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 46294648b7dSSebastian Grimberg } else { 4631cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4641cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 46594648b7dSSebastian Grimberg } 46694648b7dSSebastian Grimberg break; 46794648b7dSSebastian Grimberg case CEED_RESTRICTION_CURL_ORIENTED: 46894648b7dSSebastian Grimberg if (use_signs && use_orients) { 4691cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4705d10938bSJeremy L Thompson elem_size, v_offset, uu, vv)); 47194648b7dSSebastian Grimberg } else if (use_orients) { 4721cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, 4735d10938bSJeremy L Thompson num_elem, elem_size, v_offset, uu, vv)); 47494648b7dSSebastian Grimberg } else { 4751cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, 4761cc2c60dSJeremy L Thompson elem_size, v_offset, uu, vv)); 47794648b7dSSebastian Grimberg } 47894648b7dSSebastian Grimberg break; 4792c7e7413SJeremy L Thompson case CEED_RESTRICTION_POINTS: 4801249ccc5SJeremy L Thompson CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv)); 4812c7e7413SJeremy L Thompson break; 48294648b7dSSebastian Grimberg } 4837c1dbaffSSebastian Grimberg } 4847c1dbaffSSebastian Grimberg CeedCallBackend(CeedVectorRestoreArrayRead(u, &uu)); 4857c1dbaffSSebastian Grimberg CeedCallBackend(CeedVectorRestoreArray(v, &vv)); 4867c1dbaffSSebastian Grimberg if (request != CEED_REQUEST_IMMEDIATE && request != CEED_REQUEST_ORDERED) *request = NULL; 4877c1dbaffSSebastian Grimberg return CEED_ERROR_SUCCESS; 4887c1dbaffSSebastian Grimberg } 4897c1dbaffSSebastian Grimberg 4907c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------ 491f10650afSjeremylt // ElemRestriction Apply - Common Sizes 492f10650afSjeremylt //------------------------------------------------------------------------------ 4931cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_110(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 4947c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 4957c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 4961cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 497d979a051Sjeremylt } 498d979a051Sjeremylt 4991cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_111(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5007c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5017c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5021cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 5034d2a38eeSjeremylt } 5044d2a38eeSjeremylt 5051cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_180(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5067c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5077c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5081cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 5099c36149bSjeremylt } 5109c36149bSjeremylt 5111cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_181(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5127c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5137c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5141cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 5159c36149bSjeremylt } 5169c36149bSjeremylt 5171cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_310(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5187c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5197c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5201cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 521d979a051Sjeremylt } 522d979a051Sjeremylt 5231cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_311(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5247c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5257c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5261cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 527d979a051Sjeremylt } 528d979a051Sjeremylt 5291cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_380(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5307c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5317c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5321cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 533d979a051Sjeremylt } 534d979a051Sjeremylt 5351cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_381(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5367c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5377c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5381cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 539d979a051Sjeremylt } 540d979a051Sjeremylt 541bf4d1581Sjeremylt // LCOV_EXCL_START 5421cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_510(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5437c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5447c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5451cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 546d979a051Sjeremylt } 547bf4d1581Sjeremylt // LCOV_EXCL_STOP 548d979a051Sjeremylt 5491cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_511(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5507c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5517c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5521cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 553d979a051Sjeremylt } 554d979a051Sjeremylt 555bf4d1581Sjeremylt // LCOV_EXCL_START 5561cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_580(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5577c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5587c1dbaffSSebastian Grimberg CeedVector v, CeedRequest *request) { 5591cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request); 560d979a051Sjeremylt } 561bf4d1581Sjeremylt // LCOV_EXCL_STOP 562d979a051Sjeremylt 5631cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_581(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride, 5647c1dbaffSSebastian Grimberg CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u, 5650c73c039SSebastian Grimberg CeedVector v, CeedRequest *request) { 5661cc2c60dSJeremy L Thompson return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request); 5674d2a38eeSjeremylt } 5684d2a38eeSjeremylt 569f10650afSjeremylt //------------------------------------------------------------------------------ 570f10650afSjeremylt // ElemRestriction Apply 571f10650afSjeremylt //------------------------------------------------------------------------------ 5721cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, CeedRequest *request) { 573ad70ee2cSJeremy L Thompson CeedInt num_block, block_size, num_comp, comp_stride; 574ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 575ad70ee2cSJeremy L Thompson 5761cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 5771cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 5781cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 5791cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 5801cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 5811cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, true, true, u, v, request)); 5825d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 583f30b1135SSebastian Grimberg } 584f30b1135SSebastian Grimberg 585f30b1135SSebastian Grimberg //------------------------------------------------------------------------------ 586f30b1135SSebastian Grimberg // ElemRestriction Apply Unsigned 587f30b1135SSebastian Grimberg //------------------------------------------------------------------------------ 5881cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnsigned_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 5891cc2c60dSJeremy L Thompson CeedRequest *request) { 590ad70ee2cSJeremy L Thompson CeedInt num_block, block_size, num_comp, comp_stride; 591ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 592ad70ee2cSJeremy L Thompson 5931cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 5941cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 5951cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 5961cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 5971cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 5981cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, true, u, v, request)); 5995d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 6007c1dbaffSSebastian Grimberg } 6017c1dbaffSSebastian Grimberg 6027c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------ 6037c1dbaffSSebastian Grimberg // ElemRestriction Apply Unoriented 6047c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------ 6051cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnoriented_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 6061cc2c60dSJeremy L Thompson CeedRequest *request) { 607ad70ee2cSJeremy L Thompson CeedInt num_block, block_size, num_comp, comp_stride; 608ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 609ad70ee2cSJeremy L Thompson 6101cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 6111cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 6121cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 6131cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 6141cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 6151cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, false, u, v, request)); 6165d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 6179c36149bSjeremylt } 618be9261b7Sjeremylt 619f10650afSjeremylt //------------------------------------------------------------------------------ 6202c7e7413SJeremy L Thompson // ElemRestriction Apply Points 6212c7e7413SJeremy L Thompson //------------------------------------------------------------------------------ 6221249ccc5SJeremy L Thompson static int CeedElemRestrictionApplyAtPointsInElement_Ref(CeedElemRestriction r, CeedInt elem, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 6232c7e7413SJeremy L Thompson CeedRequest *request) { 62405fa913cSJeremy L Thompson CeedInt num_comp; 6252c7e7413SJeremy L Thompson CeedElemRestriction_Ref *impl; 6262c7e7413SJeremy L Thompson 6272c7e7413SJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(r, &num_comp)); 62805fa913cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(r, &impl)); 62905fa913cSJeremy L Thompson return impl->Apply(r, num_comp, 0, 1, elem, elem + 1, t_mode, false, false, u, v, request); 6302c7e7413SJeremy L Thompson } 6312c7e7413SJeremy L Thompson 6322c7e7413SJeremy L Thompson //------------------------------------------------------------------------------ 633f10650afSjeremylt // ElemRestriction Apply Block 634f10650afSjeremylt //------------------------------------------------------------------------------ 6351cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyBlock_Ref(CeedElemRestriction rstr, CeedInt block, CeedTransposeMode t_mode, CeedVector u, CeedVector v, 636074cb416Sjeremylt CeedRequest *request) { 637ad70ee2cSJeremy L Thompson CeedInt block_size, num_comp, comp_stride; 638ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 639ad70ee2cSJeremy L Thompson 6401cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 6411cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 6421cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 6431cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 6441cc2c60dSJeremy L Thompson CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, block, block + 1, t_mode, true, true, u, v, request)); 6455d10938bSJeremy L Thompson return CEED_ERROR_SUCCESS; 6469c36149bSjeremylt } 647be9261b7Sjeremylt 648f10650afSjeremylt //------------------------------------------------------------------------------ 649bd33150aSjeremylt // ElemRestriction Get Offsets 650bd33150aSjeremylt //------------------------------------------------------------------------------ 6512b730f8bSJeremy L Thompson static int CeedElemRestrictionGetOffsets_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt **offsets) { 652bd33150aSjeremylt Ceed ceed; 653ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 654ad70ee2cSJeremy L Thompson 655ad70ee2cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 6562b730f8bSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 657bd33150aSjeremylt 6586574a04fSJeremy L Thompson CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory"); 659bd33150aSjeremylt 660bd33150aSjeremylt *offsets = impl->offsets; 661e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 662bd33150aSjeremylt } 663bd33150aSjeremylt 664bd33150aSjeremylt //------------------------------------------------------------------------------ 66577d1c127SSebastian Grimberg // ElemRestriction Get Orientations 66677d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 66777d1c127SSebastian Grimberg static int CeedElemRestrictionGetOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const bool **orients) { 66877d1c127SSebastian Grimberg Ceed ceed; 669ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 670ad70ee2cSJeremy L Thompson 671ad70ee2cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 67277d1c127SSebastian Grimberg CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 67377d1c127SSebastian Grimberg 674fcbe8c06SSebastian Grimberg CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory"); 67577d1c127SSebastian Grimberg 67677d1c127SSebastian Grimberg *orients = impl->orients; 67777d1c127SSebastian Grimberg return CEED_ERROR_SUCCESS; 67877d1c127SSebastian Grimberg } 67977d1c127SSebastian Grimberg 68077d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 68177d1c127SSebastian Grimberg // ElemRestriction Get Curl-Conforming Orientations 68277d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 6830c73c039SSebastian Grimberg static int CeedElemRestrictionGetCurlOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt8 **curl_orients) { 68477d1c127SSebastian Grimberg Ceed ceed; 685ad70ee2cSJeremy L Thompson CeedElemRestriction_Ref *impl; 686ad70ee2cSJeremy L Thompson 687ad70ee2cSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 68877d1c127SSebastian Grimberg CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 68977d1c127SSebastian Grimberg 690fcbe8c06SSebastian Grimberg CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory"); 69177d1c127SSebastian Grimberg 69277d1c127SSebastian Grimberg *curl_orients = impl->curl_orients; 69377d1c127SSebastian Grimberg return CEED_ERROR_SUCCESS; 69477d1c127SSebastian Grimberg } 69577d1c127SSebastian Grimberg 69677d1c127SSebastian Grimberg //------------------------------------------------------------------------------ 697f10650afSjeremylt // ElemRestriction Destroy 698f10650afSjeremylt //------------------------------------------------------------------------------ 6991cc2c60dSJeremy L Thompson static int CeedElemRestrictionDestroy_Ref(CeedElemRestriction rstr) { 700fe2413ffSjeremylt CeedElemRestriction_Ref *impl; 70121617c04Sjeremylt 7021cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl)); 7032b730f8bSJeremy L Thompson CeedCallBackend(CeedFree(&impl->offsets_allocated)); 70477d1c127SSebastian Grimberg CeedCallBackend(CeedFree(&impl->orients_allocated)); 70577d1c127SSebastian Grimberg CeedCallBackend(CeedFree(&impl->curl_orients_allocated)); 7062b730f8bSJeremy L Thompson CeedCallBackend(CeedFree(&impl)); 707e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 70821617c04Sjeremylt } 70921617c04Sjeremylt 710f10650afSjeremylt //------------------------------------------------------------------------------ 711f10650afSjeremylt // ElemRestriction Create 712f10650afSjeremylt //------------------------------------------------------------------------------ 713fcbe8c06SSebastian Grimberg int CeedElemRestrictionCreate_Ref(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *offsets, const bool *orients, 7141cc2c60dSJeremy L Thompson const CeedInt8 *curl_orients, CeedElemRestriction rstr) { 715ad70ee2cSJeremy L Thompson Ceed ceed; 71607d5dec1SJeremy L Thompson CeedInt num_elem, elem_size, num_block, block_size, num_comp, comp_stride, num_points = 0, num_offsets; 717ad70ee2cSJeremy L Thompson CeedRestrictionType rstr_type; 71821617c04Sjeremylt CeedElemRestriction_Ref *impl; 719ad70ee2cSJeremy L Thompson 7201cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed)); 7211cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem)); 7221cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size)); 7231cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block)); 7241cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size)); 7251cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp)); 7261cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride)); 727ad70ee2cSJeremy L Thompson CeedInt layout[3] = {1, elem_size, elem_size * num_comp}; 72821617c04Sjeremylt 7296574a04fSJeremy L Thompson CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Only MemType = HOST supported"); 7302b730f8bSJeremy L Thompson CeedCallBackend(CeedCalloc(1, &impl)); 7313661185eSjeremylt 73292fe105eSJeremy L Thompson // Offsets data 7331cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type)); 734fcbe8c06SSebastian Grimberg if (rstr_type != CEED_RESTRICTION_STRIDED) { 7353661185eSjeremylt const char *resource; 736ad70ee2cSJeremy L Thompson 737ad70ee2cSJeremy L Thompson // Check indices for ref or memcheck backends 73835aed383SJeremy L Thompson { 73935aed383SJeremy L Thompson Ceed current = ceed, parent = NULL; 74035aed383SJeremy L Thompson 74135aed383SJeremy L Thompson CeedCallBackend(CeedGetParent(current, &parent)); 74235aed383SJeremy L Thompson while (current != parent) { 74335aed383SJeremy L Thompson current = parent; 74435aed383SJeremy L Thompson CeedCallBackend(CeedGetParent(current, &parent)); 74535aed383SJeremy L Thompson } 74635aed383SJeremy L Thompson CeedCallBackend(CeedGetResource(parent, &resource)); 74735aed383SJeremy L Thompson } 7482b730f8bSJeremy L Thompson if (!strcmp(resource, "/cpu/self/ref/serial") || !strcmp(resource, "/cpu/self/ref/blocked") || !strcmp(resource, "/cpu/self/memcheck/serial") || 749d1d35e2fSjeremylt !strcmp(resource, "/cpu/self/memcheck/blocked")) { 750e79b91d9SJeremy L Thompson CeedSize l_size; 7513661185eSjeremylt 7521cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionGetLVectorSize(rstr, &l_size)); 7532b730f8bSJeremy L Thompson for (CeedInt i = 0; i < num_elem * elem_size; i++) { 7546574a04fSJeremy L Thompson CeedCheck(offsets[i] >= 0 && offsets[i] + (num_comp - 1) * comp_stride < l_size, ceed, CEED_ERROR_BACKEND, 7556574a04fSJeremy L Thompson "Restriction offset %" CeedInt_FMT " (%" CeedInt_FMT ") out of range [0, %" CeedInt_FMT "]", i, offsets[i], l_size); 7562b730f8bSJeremy L Thompson } 7572b730f8bSJeremy L Thompson } 7583661185eSjeremylt 75992fe105eSJeremy L Thompson // Copy data 76007d5dec1SJeremy L Thompson if (rstr_type == CEED_RESTRICTION_POINTS) CeedCallBackend(CeedElemRestrictionGetNumPoints(rstr, &num_points)); 76107d5dec1SJeremy L Thompson num_offsets = rstr_type == CEED_RESTRICTION_POINTS ? (num_elem + 1 + num_points) : (num_elem * elem_size); 762d1d35e2fSjeremylt switch (copy_mode) { 76321617c04Sjeremylt case CEED_COPY_VALUES: 76407d5dec1SJeremy L Thompson CeedCallBackend(CeedMalloc(num_offsets, &impl->offsets_allocated)); 76507d5dec1SJeremy L Thompson memcpy(impl->offsets_allocated, offsets, num_offsets * sizeof(offsets[0])); 766d979a051Sjeremylt impl->offsets = impl->offsets_allocated; 76721617c04Sjeremylt break; 76821617c04Sjeremylt case CEED_OWN_POINTER: 769d979a051Sjeremylt impl->offsets_allocated = (CeedInt *)offsets; 770d979a051Sjeremylt impl->offsets = impl->offsets_allocated; 77121617c04Sjeremylt break; 77221617c04Sjeremylt case CEED_USE_POINTER: 773d979a051Sjeremylt impl->offsets = offsets; 77421617c04Sjeremylt } 775fcbe8c06SSebastian Grimberg 776fcbe8c06SSebastian Grimberg // Orientation data 777fcbe8c06SSebastian Grimberg if (rstr_type == CEED_RESTRICTION_ORIENTED) { 7780305e208SSebastian Grimberg CeedCheck(orients != NULL, ceed, CEED_ERROR_BACKEND, "No orients array provided for oriented restriction"); 779fcbe8c06SSebastian Grimberg switch (copy_mode) { 780fcbe8c06SSebastian Grimberg case CEED_COPY_VALUES: 78107d5dec1SJeremy L Thompson CeedCallBackend(CeedMalloc(num_offsets, &impl->orients_allocated)); 78207d5dec1SJeremy L Thompson memcpy(impl->orients_allocated, orients, num_offsets * sizeof(orients[0])); 783fcbe8c06SSebastian Grimberg impl->orients = impl->orients_allocated; 784fcbe8c06SSebastian Grimberg break; 785fcbe8c06SSebastian Grimberg case CEED_OWN_POINTER: 786fcbe8c06SSebastian Grimberg impl->orients_allocated = (bool *)orients; 787fcbe8c06SSebastian Grimberg impl->orients = impl->orients_allocated; 788fcbe8c06SSebastian Grimberg break; 789fcbe8c06SSebastian Grimberg case CEED_USE_POINTER: 790fcbe8c06SSebastian Grimberg impl->orients = orients; 791fcbe8c06SSebastian Grimberg } 792fcbe8c06SSebastian Grimberg } else if (rstr_type == CEED_RESTRICTION_CURL_ORIENTED) { 7930305e208SSebastian Grimberg CeedCheck(curl_orients != NULL, ceed, CEED_ERROR_BACKEND, "No curl_orients array provided for oriented restriction"); 794fcbe8c06SSebastian Grimberg switch (copy_mode) { 795fcbe8c06SSebastian Grimberg case CEED_COPY_VALUES: 79607d5dec1SJeremy L Thompson CeedCallBackend(CeedMalloc(3 * num_offsets, &impl->curl_orients_allocated)); 79707d5dec1SJeremy L Thompson memcpy(impl->curl_orients_allocated, curl_orients, 3 * num_offsets * sizeof(curl_orients[0])); 798fcbe8c06SSebastian Grimberg impl->curl_orients = impl->curl_orients_allocated; 799fcbe8c06SSebastian Grimberg break; 800fcbe8c06SSebastian Grimberg case CEED_OWN_POINTER: 8010c73c039SSebastian Grimberg impl->curl_orients_allocated = (CeedInt8 *)curl_orients; 802fcbe8c06SSebastian Grimberg impl->curl_orients = impl->curl_orients_allocated; 803fcbe8c06SSebastian Grimberg break; 804fcbe8c06SSebastian Grimberg case CEED_USE_POINTER: 805fcbe8c06SSebastian Grimberg impl->curl_orients = curl_orients; 806fcbe8c06SSebastian Grimberg } 807fcbe8c06SSebastian Grimberg } 80892fe105eSJeremy L Thompson } 809fe2413ffSjeremylt 8101cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionSetData(rstr, impl)); 8111cc2c60dSJeremy L Thompson CeedCallBackend(CeedElemRestrictionSetELayout(rstr, layout)); 8121cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Apply", CeedElemRestrictionApply_Ref)); 8132c7e7413SJeremy L Thompson if (rstr_type == CEED_RESTRICTION_POINTS) { 8141249ccc5SJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyAtPointsInElement", CeedElemRestrictionApplyAtPointsInElement_Ref)); 8151249ccc5SJeremy L Thompson } 8161cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnsigned", CeedElemRestrictionApplyUnsigned_Ref)); 8171cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnoriented", CeedElemRestrictionApplyUnoriented_Ref)); 8181cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyBlock", CeedElemRestrictionApplyBlock_Ref)); 8191cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOrientations", CeedElemRestrictionGetOrientations_Ref)); 8201cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetCurlOrientations", CeedElemRestrictionGetCurlOrientations_Ref)); 8212c7e7413SJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOffsets", CeedElemRestrictionGetOffsets_Ref)); 8221cc2c60dSJeremy L Thompson CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Destroy", CeedElemRestrictionDestroy_Ref)); 823d979a051Sjeremylt 824ad70ee2cSJeremy L Thompson // Set apply function based upon num_comp, block_size, and comp_stride 825ad70ee2cSJeremy L Thompson CeedInt index = -1; 826ad70ee2cSJeremy L Thompson 827ad70ee2cSJeremy L Thompson if (block_size < 10) index = 100 * num_comp + 10 * block_size + (comp_stride == 1); 828ad70ee2cSJeremy L Thompson switch (index) { 829d979a051Sjeremylt case 110: 830d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_110; 831d979a051Sjeremylt break; 832d979a051Sjeremylt case 111: 833d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_111; 834d979a051Sjeremylt break; 835d979a051Sjeremylt case 180: 836d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_180; 837d979a051Sjeremylt break; 838d979a051Sjeremylt case 181: 839d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_181; 840d979a051Sjeremylt break; 841d979a051Sjeremylt case 310: 842d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_310; 843d979a051Sjeremylt break; 844d979a051Sjeremylt case 311: 845d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_311; 846d979a051Sjeremylt break; 847d979a051Sjeremylt case 380: 848d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_380; 849d979a051Sjeremylt break; 850d979a051Sjeremylt case 381: 851d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_381; 852d979a051Sjeremylt break; 853bf4d1581Sjeremylt // LCOV_EXCL_START 854d979a051Sjeremylt case 510: 855d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_510; 856d979a051Sjeremylt break; 857bf4d1581Sjeremylt // LCOV_EXCL_STOP 858d979a051Sjeremylt case 511: 859d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_511; 860d979a051Sjeremylt break; 861bf4d1581Sjeremylt // LCOV_EXCL_START 862d979a051Sjeremylt case 580: 863d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_580; 864d979a051Sjeremylt break; 865bf4d1581Sjeremylt // LCOV_EXCL_STOP 866d979a051Sjeremylt case 581: 867d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_581; 868d979a051Sjeremylt break; 869d979a051Sjeremylt default: 870d979a051Sjeremylt impl->Apply = CeedElemRestrictionApply_Ref_Core; 871d979a051Sjeremylt break; 872d979a051Sjeremylt } 873e15f9bd0SJeremy L Thompson return CEED_ERROR_SUCCESS; 87421617c04Sjeremylt } 875fc0567d9Srezgarshakeri 876fc0567d9Srezgarshakeri //------------------------------------------------------------------------------ 877