xref: /libCEED/backends/ref/ceed-ref-restriction.c (revision bf9b6c6b8f0540abdebc8668a03e39060b0a48ae)
13d8e8822SJeremy L Thompson // Copyright (c) 2017-2022, Lawrence Livermore National Security, LLC and other CEED contributors.
23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
321617c04Sjeremylt //
43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause
521617c04Sjeremylt //
63d8e8822SJeremy L Thompson // This file is part of CEED:  http://github.com/ceed
721617c04Sjeremylt 
849aac155SJeremy L Thompson #include <ceed.h>
9ec3da8bcSJed Brown #include <ceed/backend.h>
103d576824SJeremy L Thompson #include <stdbool.h>
11fcbe8c06SSebastian Grimberg #include <stdlib.h>
123d576824SJeremy L Thompson #include <string.h>
132b730f8bSJeremy L Thompson 
1421617c04Sjeremylt #include "ceed-ref.h"
1521617c04Sjeremylt 
16f10650afSjeremylt //------------------------------------------------------------------------------
17f10650afSjeremylt // Core ElemRestriction Apply Code
18f10650afSjeremylt //------------------------------------------------------------------------------
191cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
2094648b7dSSebastian Grimberg                                                                       CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size,
2194648b7dSSebastian Grimberg                                                                       CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
2294648b7dSSebastian Grimberg   // No offsets provided, identity restriction
23d1d35e2fSjeremylt   bool has_backend_strides;
24ad70ee2cSJeremy L Thompson 
251cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides));
26d1d35e2fSjeremylt   if (has_backend_strides) {
27d1d35e2fSjeremylt     // CPU backend strides are {1, elem_size, elem_size*num_comp}
287f90ec76Sjeremylt     // This if branch is left separate to allow better inlining
29ad70ee2cSJeremy L Thompson     for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
302b730f8bSJeremy L Thompson       CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
312b730f8bSJeremy L Thompson         CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) {
32ad70ee2cSJeremy L Thompson           CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
33ad70ee2cSJeremy L Thompson             vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
342b730f8bSJeremy L Thompson                 uu[n + k * elem_size + CeedIntMin(e + j, num_elem - 1) * elem_size * num_comp];
352b730f8bSJeremy L Thompson           }
362b730f8bSJeremy L Thompson         }
372b730f8bSJeremy L Thompson       }
382b730f8bSJeremy L Thompson     }
397f90ec76Sjeremylt   } else {
407f90ec76Sjeremylt     // User provided strides
417f90ec76Sjeremylt     CeedInt strides[3];
42ad70ee2cSJeremy L Thompson 
431cc2c60dSJeremy L Thompson     CeedCallBackend(CeedElemRestrictionGetStrides(rstr, &strides));
44ad70ee2cSJeremy L Thompson     for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
452b730f8bSJeremy L Thompson       CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
462b730f8bSJeremy L Thompson         CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) {
47ad70ee2cSJeremy L Thompson           CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
48ad70ee2cSJeremy L Thompson             vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
492b730f8bSJeremy L Thompson                 uu[n * strides[0] + k * strides[1] + CeedIntMin(e + j, num_elem - 1) * strides[2]];
502b730f8bSJeremy L Thompson           }
512b730f8bSJeremy L Thompson         }
522b730f8bSJeremy L Thompson       }
532b730f8bSJeremy L Thompson     }
547509a596Sjeremylt   }
5594648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
5694648b7dSSebastian Grimberg }
5794648b7dSSebastian Grimberg 
581cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
5994648b7dSSebastian Grimberg                                                                        const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
6094648b7dSSebastian Grimberg                                                                        CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
61fcbe8c06SSebastian Grimberg   // Default restriction with offsets
6294648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
63ad70ee2cSJeremy L Thompson 
641cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
65ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
662b730f8bSJeremy L Thompson     CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
67ad70ee2cSJeremy L Thompson       CeedPragmaSIMD for (CeedInt i = 0; i < elem_size * block_size; i++) {
68ad70ee2cSJeremy L Thompson         vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] = uu[impl->offsets[i + e * elem_size] + k * comp_stride];
69fcbe8c06SSebastian Grimberg       }
70fcbe8c06SSebastian Grimberg     }
71fcbe8c06SSebastian Grimberg   }
7294648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
7394648b7dSSebastian Grimberg }
7494648b7dSSebastian Grimberg 
751cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
7694648b7dSSebastian Grimberg                                                                        const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
7794648b7dSSebastian Grimberg                                                                        CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
78fcbe8c06SSebastian Grimberg   // Restriction with orientations
7994648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
80ad70ee2cSJeremy L Thompson 
811cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
82ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
83fcbe8c06SSebastian Grimberg     CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
84ad70ee2cSJeremy L Thompson       CeedPragmaSIMD for (CeedInt i = 0; i < elem_size * block_size; i++) {
85ad70ee2cSJeremy L Thompson         vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] =
867c1dbaffSSebastian Grimberg             uu[impl->offsets[i + e * elem_size] + k * comp_stride] * (impl->orients[i + e * elem_size] ? -1.0 : 1.0);
87fcbe8c06SSebastian Grimberg       }
88fcbe8c06SSebastian Grimberg     }
89fcbe8c06SSebastian Grimberg   }
9094648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
9194648b7dSSebastian Grimberg }
9294648b7dSSebastian Grimberg 
931cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
9494648b7dSSebastian Grimberg                                                                            const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
9594648b7dSSebastian Grimberg                                                                            CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu,
9694648b7dSSebastian Grimberg                                                                            CeedScalar *vv) {
9777d1c127SSebastian Grimberg   // Restriction with tridiagonal transformation
9894648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
99ad70ee2cSJeremy L Thompson 
1001cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
101ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
102fcbe8c06SSebastian Grimberg     CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
1030c73c039SSebastian Grimberg       CeedInt n = 0;
104ad70ee2cSJeremy L Thompson       CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
105ad70ee2cSJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
106ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
107ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
108ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
109ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size];
1100c73c039SSebastian Grimberg       }
1110c73c039SSebastian Grimberg       for (n = 1; n < elem_size - 1; n++) {
112ad70ee2cSJeremy L Thompson         CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
113ad70ee2cSJeremy L Thompson           vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
114ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
115ad70ee2cSJeremy L Thompson                   impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] +
116ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
117ad70ee2cSJeremy L Thompson                   impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
118ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
119ad70ee2cSJeremy L Thompson                   impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size];
1200c73c039SSebastian Grimberg         }
1210c73c039SSebastian Grimberg       }
122ad70ee2cSJeremy L Thompson       CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
123ad70ee2cSJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
124ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
125ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] +
126ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
127ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size];
1282b730f8bSJeremy L Thompson       }
1292b730f8bSJeremy L Thompson     }
1302b730f8bSJeremy L Thompson   }
1310c73c039SSebastian Grimberg   return CEED_ERROR_SUCCESS;
132fcbe8c06SSebastian Grimberg }
1330c73c039SSebastian Grimberg 
1341cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp,
135ad70ee2cSJeremy L Thompson                                                                                    const CeedInt block_size, const CeedInt comp_stride, CeedInt start,
13694648b7dSSebastian Grimberg                                                                                    CeedInt stop, CeedInt num_elem, CeedInt elem_size,
13794648b7dSSebastian Grimberg                                                                                    CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
13894648b7dSSebastian Grimberg   // Restriction with (unsigned) tridiagonal transformation
1390c73c039SSebastian Grimberg   CeedElemRestriction_Ref *impl;
140ad70ee2cSJeremy L Thompson 
1411cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
142ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
1437c1dbaffSSebastian Grimberg     CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
1447c1dbaffSSebastian Grimberg       CeedInt n = 0;
145ad70ee2cSJeremy L Thompson 
146ad70ee2cSJeremy L Thompson       CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
147ad70ee2cSJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
148ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
149ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
150ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
151ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]);
1527c1dbaffSSebastian Grimberg       }
1537c1dbaffSSebastian Grimberg       for (n = 1; n < elem_size - 1; n++) {
154ad70ee2cSJeremy L Thompson         CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
155ad70ee2cSJeremy L Thompson           vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
156ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
157ad70ee2cSJeremy L Thompson                   abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) +
158ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
159ad70ee2cSJeremy L Thompson                   abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
160ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
161ad70ee2cSJeremy L Thompson                   abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]);
1627c1dbaffSSebastian Grimberg         }
1637c1dbaffSSebastian Grimberg       }
164ad70ee2cSJeremy L Thompson       CeedPragmaSIMD for (CeedInt j = 0; j < block_size; j++) {
165ad70ee2cSJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
166ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
167ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) +
168ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
169ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]);
1707c1dbaffSSebastian Grimberg       }
1717c1dbaffSSebastian Grimberg     }
1727c1dbaffSSebastian Grimberg   }
1737c1dbaffSSebastian Grimberg   return CEED_ERROR_SUCCESS;
1747c1dbaffSSebastian Grimberg }
1757c1dbaffSSebastian Grimberg 
1761cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
17794648b7dSSebastian Grimberg                                                                     CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size,
17894648b7dSSebastian Grimberg                                                                     CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
17994648b7dSSebastian Grimberg   // No offsets provided, identity restriction
180d1d35e2fSjeremylt   bool has_backend_strides;
181ad70ee2cSJeremy L Thompson 
1821cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides));
183d1d35e2fSjeremylt   if (has_backend_strides) {
184d1d35e2fSjeremylt     // CPU backend strides are {1, elem_size, elem_size*num_comp}
1857f90ec76Sjeremylt     // This if brach is left separate to allow better inlining
186ad70ee2cSJeremy L Thompson     for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
1872b730f8bSJeremy L Thompson       CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
1882b730f8bSJeremy L Thompson         CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) {
189ad70ee2cSJeremy L Thompson           CeedPragmaSIMD for (CeedInt j = 0; j < CeedIntMin(block_size, num_elem - e); j++) {
190*bf9b6c6bSSebastian Grimberg             vv[n + k * elem_size + (e + j) * elem_size * num_comp] += uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset];
1912b730f8bSJeremy L Thompson           }
1922b730f8bSJeremy L Thompson         }
1932b730f8bSJeremy L Thompson       }
1942b730f8bSJeremy L Thompson     }
1957f90ec76Sjeremylt   } else {
1967f90ec76Sjeremylt     // User provided strides
1977f90ec76Sjeremylt     CeedInt strides[3];
198ad70ee2cSJeremy L Thompson 
1991cc2c60dSJeremy L Thompson     CeedCallBackend(CeedElemRestrictionGetStrides(rstr, &strides));
200ad70ee2cSJeremy L Thompson     for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
2012b730f8bSJeremy L Thompson       CeedPragmaSIMD for (CeedInt k = 0; k < num_comp; k++) {
2022b730f8bSJeremy L Thompson         CeedPragmaSIMD for (CeedInt n = 0; n < elem_size; n++) {
203ad70ee2cSJeremy L Thompson           CeedPragmaSIMD for (CeedInt j = 0; j < CeedIntMin(block_size, num_elem - e); j++) {
204*bf9b6c6bSSebastian Grimberg             vv[n * strides[0] + k * strides[1] + (e + j) * strides[2]] += uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset];
2052b730f8bSJeremy L Thompson           }
2062b730f8bSJeremy L Thompson         }
2072b730f8bSJeremy L Thompson       }
2082b730f8bSJeremy L Thompson     }
209523b8ea0Sjeremylt   }
21094648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
21194648b7dSSebastian Grimberg }
21294648b7dSSebastian Grimberg 
2131cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStandardTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
21494648b7dSSebastian Grimberg                                                                      const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
21594648b7dSSebastian Grimberg                                                                      CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
216fcbe8c06SSebastian Grimberg   // Default restriction with offsets
21794648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
218ad70ee2cSJeremy L Thompson 
2191cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
220ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
2212b730f8bSJeremy L Thompson     for (CeedInt k = 0; k < num_comp; k++) {
222ad70ee2cSJeremy L Thompson       for (CeedInt i = 0; i < elem_size * block_size; i += block_size) {
2238d94b059Sjeremylt         // Iteration bound set to discard padding elements
224ad70ee2cSJeremy L Thompson         for (CeedInt j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) {
22558c07c4fSSebastian Grimberg           CeedScalar uu_val;
22658c07c4fSSebastian Grimberg 
22758c07c4fSSebastian Grimberg           uu_val = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset];
22858c07c4fSSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += uu_val;
229fcbe8c06SSebastian Grimberg         }
230fcbe8c06SSebastian Grimberg       }
231fcbe8c06SSebastian Grimberg     }
232fcbe8c06SSebastian Grimberg   }
23394648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
23494648b7dSSebastian Grimberg }
23594648b7dSSebastian Grimberg 
2361cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
23794648b7dSSebastian Grimberg                                                                      const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
23894648b7dSSebastian Grimberg                                                                      CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
239fcbe8c06SSebastian Grimberg   // Restriction with orientations
24094648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
241ad70ee2cSJeremy L Thompson 
2421cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
243ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
244fcbe8c06SSebastian Grimberg     for (CeedInt k = 0; k < num_comp; k++) {
245ad70ee2cSJeremy L Thompson       for (CeedInt i = 0; i < elem_size * block_size; i += block_size) {
246fcbe8c06SSebastian Grimberg         // Iteration bound set to discard padding elements
247ad70ee2cSJeremy L Thompson         for (CeedInt j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) {
24858c07c4fSSebastian Grimberg           CeedScalar uu_val;
24958c07c4fSSebastian Grimberg 
25058c07c4fSSebastian Grimberg           uu_val = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset] * (impl->orients[j + e * elem_size] ? -1.0 : 1.0);
25158c07c4fSSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += uu_val;
252fcbe8c06SSebastian Grimberg         }
253fcbe8c06SSebastian Grimberg       }
254fcbe8c06SSebastian Grimberg     }
255fcbe8c06SSebastian Grimberg   }
25694648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
25794648b7dSSebastian Grimberg }
25894648b7dSSebastian Grimberg 
2591cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
26094648b7dSSebastian Grimberg                                                                          const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
26194648b7dSSebastian Grimberg                                                                          CeedInt elem_size, CeedInt v_offset, const CeedScalar *uu, CeedScalar *vv) {
26277d1c127SSebastian Grimberg   // Restriction with tridiagonal transformation
26394648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
264ad70ee2cSJeremy L Thompson 
2651cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
266ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
267fcbe8c06SSebastian Grimberg     for (CeedInt k = 0; k < num_comp; k++) {
268fcbe8c06SSebastian Grimberg       // Iteration bound set to discard padding elements
26958c07c4fSSebastian Grimberg       const CeedInt block_end = CeedIntMin(block_size, num_elem - e);
27058c07c4fSSebastian Grimberg       CeedInt       n         = 0;
27158c07c4fSSebastian Grimberg 
272ad70ee2cSJeremy L Thompson       for (CeedInt j = 0; j < block_end; j++) {
27358c07c4fSSebastian Grimberg         CeedScalar uu_val;
27458c07c4fSSebastian Grimberg 
27558c07c4fSSebastian Grimberg         uu_val = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
276ad70ee2cSJeremy L Thompson                      impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
277ad70ee2cSJeremy L Thompson                  uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
278ad70ee2cSJeremy L Thompson                      impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size];
27958c07c4fSSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val;
2800c73c039SSebastian Grimberg       }
2810c73c039SSebastian Grimberg       for (n = 1; n < elem_size - 1; n++) {
282ad70ee2cSJeremy L Thompson         for (CeedInt j = 0; j < block_end; j++) {
28358c07c4fSSebastian Grimberg           CeedScalar uu_val;
28458c07c4fSSebastian Grimberg 
28558c07c4fSSebastian Grimberg           uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
286ad70ee2cSJeremy L Thompson                        impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] +
287ad70ee2cSJeremy L Thompson                    uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
288ad70ee2cSJeremy L Thompson                        impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
289ad70ee2cSJeremy L Thompson                    uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
290ad70ee2cSJeremy L Thompson                        impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size];
29158c07c4fSSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val;
2920c73c039SSebastian Grimberg         }
2930c73c039SSebastian Grimberg       }
294ad70ee2cSJeremy L Thompson       for (CeedInt j = 0; j < block_end; j++) {
29558c07c4fSSebastian Grimberg         CeedScalar uu_val;
29658c07c4fSSebastian Grimberg 
29758c07c4fSSebastian Grimberg         uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
298ad70ee2cSJeremy L Thompson                      impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] +
299ad70ee2cSJeremy L Thompson                  uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
300ad70ee2cSJeremy L Thompson                      impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size];
30158c07c4fSSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val;
30221617c04Sjeremylt       }
303b435c5a6Srezgarshakeri     }
3042b730f8bSJeremy L Thompson   }
305e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
30621617c04Sjeremylt }
30721617c04Sjeremylt 
3081cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp,
309ad70ee2cSJeremy L Thompson                                                                                  const CeedInt block_size, const CeedInt comp_stride, CeedInt start,
31094648b7dSSebastian Grimberg                                                                                  CeedInt stop, CeedInt num_elem, CeedInt elem_size, CeedInt v_offset,
31194648b7dSSebastian Grimberg                                                                                  const CeedScalar *uu, CeedScalar *vv) {
31294648b7dSSebastian Grimberg   // Restriction with (unsigned) tridiagonal transformation
3137c1dbaffSSebastian Grimberg   CeedElemRestriction_Ref *impl;
314ad70ee2cSJeremy L Thompson 
3151cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
316ad70ee2cSJeremy L Thompson   for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
3177c1dbaffSSebastian Grimberg     for (CeedInt k = 0; k < num_comp; k++) {
3187c1dbaffSSebastian Grimberg       // Iteration bound set to discard padding elements
319ad70ee2cSJeremy L Thompson       const CeedInt block_end = CeedIntMin(block_size, num_elem - e);
32058c07c4fSSebastian Grimberg       CeedInt       n         = 0;
321ad70ee2cSJeremy L Thompson 
322ad70ee2cSJeremy L Thompson       for (CeedInt j = 0; j < block_end; j++) {
32358c07c4fSSebastian Grimberg         CeedScalar uu_val;
32458c07c4fSSebastian Grimberg 
32558c07c4fSSebastian Grimberg         uu_val = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
326ad70ee2cSJeremy L Thompson                      abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
327ad70ee2cSJeremy L Thompson                  uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
328ad70ee2cSJeremy L Thompson                      abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]);
32958c07c4fSSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val;
3307c1dbaffSSebastian Grimberg       }
3317c1dbaffSSebastian Grimberg       for (n = 1; n < elem_size - 1; n++) {
332ad70ee2cSJeremy L Thompson         for (CeedInt j = 0; j < block_end; j++) {
33358c07c4fSSebastian Grimberg           CeedScalar uu_val;
33458c07c4fSSebastian Grimberg 
33558c07c4fSSebastian Grimberg           uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
336ad70ee2cSJeremy L Thompson                        abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) +
337ad70ee2cSJeremy L Thompson                    uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
338ad70ee2cSJeremy L Thompson                        abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
339ad70ee2cSJeremy L Thompson                    uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
340ad70ee2cSJeremy L Thompson                        abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]);
34158c07c4fSSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val;
3427c1dbaffSSebastian Grimberg         }
3437c1dbaffSSebastian Grimberg       }
344ad70ee2cSJeremy L Thompson       for (CeedInt j = 0; j < block_end; j++) {
34558c07c4fSSebastian Grimberg         CeedScalar uu_val;
34658c07c4fSSebastian Grimberg 
34758c07c4fSSebastian Grimberg         uu_val = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
348ad70ee2cSJeremy L Thompson                      abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) +
349ad70ee2cSJeremy L Thompson                  uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
350ad70ee2cSJeremy L Thompson                      abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]);
35158c07c4fSSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += uu_val;
3527c1dbaffSSebastian Grimberg       }
3537c1dbaffSSebastian Grimberg     }
3547c1dbaffSSebastian Grimberg   }
3557c1dbaffSSebastian Grimberg   return CEED_ERROR_SUCCESS;
3567c1dbaffSSebastian Grimberg }
3577c1dbaffSSebastian Grimberg 
3581249ccc5SJeremy L Thompson static inline int CeedElemRestrictionApplyAtPointsInElement_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, CeedInt start, CeedInt stop,
35905fa913cSJeremy L Thompson                                                                      CeedTransposeMode t_mode, const CeedScalar *uu, CeedScalar *vv) {
36005fa913cSJeremy L Thompson   CeedInt                  num_points, l_vec_offset, e_vec_offset = 0;
36105fa913cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
36205fa913cSJeremy L Thompson 
36305fa913cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
36405fa913cSJeremy L Thompson 
36505fa913cSJeremy L Thompson   for (CeedInt e = start; e < stop; e++) {
3660930e4e7SJeremy L Thompson     l_vec_offset = impl->offsets[e];
36705fa913cSJeremy L Thompson     CeedCallBackend(CeedElemRestrictionGetNumPointsInElement(rstr, e, &num_points));
36805fa913cSJeremy L Thompson     if (t_mode == CEED_NOTRANSPOSE) {
36905fa913cSJeremy L Thompson       for (CeedInt i = 0; i < num_points; i++) {
3709c34f28eSJeremy L Thompson         for (CeedInt j = 0; j < num_comp; j++) vv[j * num_points + i + e_vec_offset] = uu[impl->offsets[i + l_vec_offset] * num_comp + j];
37105fa913cSJeremy L Thompson       }
37205fa913cSJeremy L Thompson     } else {
37305fa913cSJeremy L Thompson       for (CeedInt i = 0; i < num_points; i++) {
3749c34f28eSJeremy L Thompson         for (CeedInt j = 0; j < num_comp; j++) vv[impl->offsets[i + l_vec_offset] * num_comp + j] = uu[j * num_points + i + e_vec_offset];
37505fa913cSJeremy L Thompson       }
37605fa913cSJeremy L Thompson     }
37705fa913cSJeremy L Thompson     e_vec_offset += num_points * num_comp;
37805fa913cSJeremy L Thompson   }
37905fa913cSJeremy L Thompson   return CEED_ERROR_SUCCESS;
38005fa913cSJeremy L Thompson }
38105fa913cSJeremy L Thompson 
3821cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApply_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
383ad70ee2cSJeremy L Thompson                                                     const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs,
384ad70ee2cSJeremy L Thompson                                                     bool use_orients, CeedVector u, CeedVector v, CeedRequest *request) {
385ad70ee2cSJeremy L Thompson   CeedInt             num_elem, elem_size, v_offset;
386ad70ee2cSJeremy L Thompson   CeedRestrictionType rstr_type;
3877c1dbaffSSebastian Grimberg   const CeedScalar   *uu;
3887c1dbaffSSebastian Grimberg   CeedScalar         *vv;
389ad70ee2cSJeremy L Thompson 
3901cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem));
3911cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size));
392ad70ee2cSJeremy L Thompson   v_offset = start * block_size * elem_size * num_comp;
3931cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type));
39494648b7dSSebastian Grimberg   CeedCallBackend(CeedVectorGetArrayRead(u, CEED_MEM_HOST, &uu));
395ad70ee2cSJeremy L Thompson 
39694648b7dSSebastian Grimberg   if (t_mode == CEED_TRANSPOSE) {
39794648b7dSSebastian Grimberg     // Sum into for transpose mode, E-vector to L-vector
39894648b7dSSebastian Grimberg     CeedCallBackend(CeedVectorGetArray(v, CEED_MEM_HOST, &vv));
39994648b7dSSebastian Grimberg   } else {
40094648b7dSSebastian Grimberg     // Overwrite for notranspose mode, L-vector to E-vector
40194648b7dSSebastian Grimberg     CeedCallBackend(CeedVectorGetArrayWrite(v, CEED_MEM_HOST, &vv));
40294648b7dSSebastian Grimberg   }
40394648b7dSSebastian Grimberg   if (t_mode == CEED_TRANSPOSE) {
4047c1dbaffSSebastian Grimberg     // Restriction from E-vector to L-vector
4057c1dbaffSSebastian Grimberg     // Performing v += r^T * u
4067c1dbaffSSebastian Grimberg     // uu has shape [elem_size, num_comp, num_elem], row-major
4077c1dbaffSSebastian Grimberg     // vv has shape [nnodes, num_comp]
4087c1dbaffSSebastian Grimberg     // Sum into for transpose mode
4097c1dbaffSSebastian Grimberg     switch (rstr_type) {
4107c1dbaffSSebastian Grimberg       case CEED_RESTRICTION_STRIDED:
4115d10938bSJeremy L Thompson         CeedCallBackend(
4121cc2c60dSJeremy L Thompson             CeedElemRestrictionApplyStridedTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv));
41394648b7dSSebastian Grimberg         break;
41461a27d74SSebastian Grimberg       case CEED_RESTRICTION_STANDARD:
4151cc2c60dSJeremy L Thompson         CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size,
4165d10938bSJeremy L Thompson                                                                            v_offset, uu, vv));
41794648b7dSSebastian Grimberg         break;
4187c1dbaffSSebastian Grimberg       case CEED_RESTRICTION_ORIENTED:
41994648b7dSSebastian Grimberg         if (use_signs) {
4201cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4211cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
42294648b7dSSebastian Grimberg         } else {
4231cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4241cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
42594648b7dSSebastian Grimberg         }
42694648b7dSSebastian Grimberg         break;
42794648b7dSSebastian Grimberg       case CEED_RESTRICTION_CURL_ORIENTED:
42894648b7dSSebastian Grimberg         if (use_signs && use_orients) {
4291cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4305d10938bSJeremy L Thompson                                                                                  elem_size, v_offset, uu, vv));
43194648b7dSSebastian Grimberg         } else if (use_orients) {
4321cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop,
4331cc2c60dSJeremy L Thompson                                                                                          num_elem, elem_size, v_offset, uu, vv));
43494648b7dSSebastian Grimberg         } else {
4351cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyStandardTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4361cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
43794648b7dSSebastian Grimberg         }
43894648b7dSSebastian Grimberg         break;
4392c7e7413SJeremy L Thompson       case CEED_RESTRICTION_POINTS:
4401249ccc5SJeremy L Thompson         CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv));
4412c7e7413SJeremy L Thompson         break;
44294648b7dSSebastian Grimberg     }
44394648b7dSSebastian Grimberg   } else {
44494648b7dSSebastian Grimberg     // Restriction from L-vector to E-vector
44594648b7dSSebastian Grimberg     // Perform: v = r * u
44694648b7dSSebastian Grimberg     // vv has shape [elem_size, num_comp, num_elem], row-major
44794648b7dSSebastian Grimberg     // uu has shape [nnodes, num_comp]
44894648b7dSSebastian Grimberg     // Overwrite for notranspose mode
44994648b7dSSebastian Grimberg     switch (rstr_type) {
45094648b7dSSebastian Grimberg       case CEED_RESTRICTION_STRIDED:
4515d10938bSJeremy L Thompson         CeedCallBackend(
4521cc2c60dSJeremy L Thompson             CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv));
45394648b7dSSebastian Grimberg         break;
45461a27d74SSebastian Grimberg       case CEED_RESTRICTION_STANDARD:
4551cc2c60dSJeremy L Thompson         CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4561cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
45794648b7dSSebastian Grimberg         break;
45894648b7dSSebastian Grimberg       case CEED_RESTRICTION_ORIENTED:
45994648b7dSSebastian Grimberg         if (use_signs) {
4601cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4611cc2c60dSJeremy L Thompson                                                                                elem_size, v_offset, uu, vv));
46294648b7dSSebastian Grimberg         } else {
4631cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4641cc2c60dSJeremy L Thompson                                                                                elem_size, v_offset, uu, vv));
46594648b7dSSebastian Grimberg         }
46694648b7dSSebastian Grimberg         break;
46794648b7dSSebastian Grimberg       case CEED_RESTRICTION_CURL_ORIENTED:
46894648b7dSSebastian Grimberg         if (use_signs && use_orients) {
4691cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4705d10938bSJeremy L Thompson                                                                                    elem_size, v_offset, uu, vv));
47194648b7dSSebastian Grimberg         } else if (use_orients) {
4721cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop,
4735d10938bSJeremy L Thompson                                                                                            num_elem, elem_size, v_offset, uu, vv));
47494648b7dSSebastian Grimberg         } else {
4751cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyStandardNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4761cc2c60dSJeremy L Thompson                                                                                elem_size, v_offset, uu, vv));
47794648b7dSSebastian Grimberg         }
47894648b7dSSebastian Grimberg         break;
4792c7e7413SJeremy L Thompson       case CEED_RESTRICTION_POINTS:
4801249ccc5SJeremy L Thompson         CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv));
4812c7e7413SJeremy L Thompson         break;
48294648b7dSSebastian Grimberg     }
4837c1dbaffSSebastian Grimberg   }
4847c1dbaffSSebastian Grimberg   CeedCallBackend(CeedVectorRestoreArrayRead(u, &uu));
4857c1dbaffSSebastian Grimberg   CeedCallBackend(CeedVectorRestoreArray(v, &vv));
4867c1dbaffSSebastian Grimberg   if (request != CEED_REQUEST_IMMEDIATE && request != CEED_REQUEST_ORDERED) *request = NULL;
4877c1dbaffSSebastian Grimberg   return CEED_ERROR_SUCCESS;
4887c1dbaffSSebastian Grimberg }
4897c1dbaffSSebastian Grimberg 
4907c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------
491f10650afSjeremylt // ElemRestriction Apply - Common Sizes
492f10650afSjeremylt //------------------------------------------------------------------------------
4931cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_110(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
4947c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
4957c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
4961cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
497d979a051Sjeremylt }
498d979a051Sjeremylt 
4991cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_111(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5007c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5017c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5021cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
5034d2a38eeSjeremylt }
5044d2a38eeSjeremylt 
5051cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_180(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5067c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5077c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5081cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
5099c36149bSjeremylt }
5109c36149bSjeremylt 
5111cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_181(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5127c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5137c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5141cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
5159c36149bSjeremylt }
5169c36149bSjeremylt 
5171cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_310(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5187c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5197c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5201cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
521d979a051Sjeremylt }
522d979a051Sjeremylt 
5231cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_311(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5247c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5257c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5261cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
527d979a051Sjeremylt }
528d979a051Sjeremylt 
5291cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_380(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5307c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5317c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5321cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
533d979a051Sjeremylt }
534d979a051Sjeremylt 
5351cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_381(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5367c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5377c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5381cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
539d979a051Sjeremylt }
540d979a051Sjeremylt 
541bf4d1581Sjeremylt // LCOV_EXCL_START
5421cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_510(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5437c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5447c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5451cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
546d979a051Sjeremylt }
547bf4d1581Sjeremylt // LCOV_EXCL_STOP
548d979a051Sjeremylt 
5491cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_511(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5507c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5517c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5521cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
553d979a051Sjeremylt }
554d979a051Sjeremylt 
555bf4d1581Sjeremylt // LCOV_EXCL_START
5561cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_580(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5577c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5587c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5591cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
560d979a051Sjeremylt }
561bf4d1581Sjeremylt // LCOV_EXCL_STOP
562d979a051Sjeremylt 
5631cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_581(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5647c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5650c73c039SSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5661cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
5674d2a38eeSjeremylt }
5684d2a38eeSjeremylt 
569f10650afSjeremylt //------------------------------------------------------------------------------
570f10650afSjeremylt // ElemRestriction Apply
571f10650afSjeremylt //------------------------------------------------------------------------------
5721cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, CeedRequest *request) {
573ad70ee2cSJeremy L Thompson   CeedInt                  num_block, block_size, num_comp, comp_stride;
574ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
575ad70ee2cSJeremy L Thompson 
5761cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
5771cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
5781cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
5791cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
5801cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
5811cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, true, true, u, v, request));
5825d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
583f30b1135SSebastian Grimberg }
584f30b1135SSebastian Grimberg 
585f30b1135SSebastian Grimberg //------------------------------------------------------------------------------
586f30b1135SSebastian Grimberg // ElemRestriction Apply Unsigned
587f30b1135SSebastian Grimberg //------------------------------------------------------------------------------
5881cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnsigned_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
5891cc2c60dSJeremy L Thompson                                                 CeedRequest *request) {
590ad70ee2cSJeremy L Thompson   CeedInt                  num_block, block_size, num_comp, comp_stride;
591ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
592ad70ee2cSJeremy L Thompson 
5931cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
5941cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
5951cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
5961cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
5971cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
5981cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, true, u, v, request));
5995d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
6007c1dbaffSSebastian Grimberg }
6017c1dbaffSSebastian Grimberg 
6027c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------
6037c1dbaffSSebastian Grimberg // ElemRestriction Apply Unoriented
6047c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------
6051cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnoriented_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
6061cc2c60dSJeremy L Thompson                                                   CeedRequest *request) {
607ad70ee2cSJeremy L Thompson   CeedInt                  num_block, block_size, num_comp, comp_stride;
608ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
609ad70ee2cSJeremy L Thompson 
6101cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
6111cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
6121cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
6131cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
6141cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6151cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, false, u, v, request));
6165d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
6179c36149bSjeremylt }
618be9261b7Sjeremylt 
619f10650afSjeremylt //------------------------------------------------------------------------------
6202c7e7413SJeremy L Thompson // ElemRestriction Apply Points
6212c7e7413SJeremy L Thompson //------------------------------------------------------------------------------
6221249ccc5SJeremy L Thompson static int CeedElemRestrictionApplyAtPointsInElement_Ref(CeedElemRestriction r, CeedInt elem, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
6232c7e7413SJeremy L Thompson                                                          CeedRequest *request) {
62405fa913cSJeremy L Thompson   CeedInt                  num_comp;
6252c7e7413SJeremy L Thompson   CeedElemRestriction_Ref *impl;
6262c7e7413SJeremy L Thompson 
6272c7e7413SJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(r, &num_comp));
62805fa913cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(r, &impl));
62905fa913cSJeremy L Thompson   return impl->Apply(r, num_comp, 0, 1, elem, elem + 1, t_mode, false, false, u, v, request);
6302c7e7413SJeremy L Thompson }
6312c7e7413SJeremy L Thompson 
6322c7e7413SJeremy L Thompson //------------------------------------------------------------------------------
633f10650afSjeremylt // ElemRestriction Apply Block
634f10650afSjeremylt //------------------------------------------------------------------------------
6351cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyBlock_Ref(CeedElemRestriction rstr, CeedInt block, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
636074cb416Sjeremylt                                              CeedRequest *request) {
637ad70ee2cSJeremy L Thompson   CeedInt                  block_size, num_comp, comp_stride;
638ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
639ad70ee2cSJeremy L Thompson 
6401cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
6411cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
6421cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
6431cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6441cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, block, block + 1, t_mode, true, true, u, v, request));
6455d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
6469c36149bSjeremylt }
647be9261b7Sjeremylt 
648f10650afSjeremylt //------------------------------------------------------------------------------
649bd33150aSjeremylt // ElemRestriction Get Offsets
650bd33150aSjeremylt //------------------------------------------------------------------------------
6512b730f8bSJeremy L Thompson static int CeedElemRestrictionGetOffsets_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt **offsets) {
652bd33150aSjeremylt   Ceed                     ceed;
653ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
654ad70ee2cSJeremy L Thompson 
655ad70ee2cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6562b730f8bSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed));
657bd33150aSjeremylt 
6586574a04fSJeremy L Thompson   CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory");
659bd33150aSjeremylt 
660bd33150aSjeremylt   *offsets = impl->offsets;
661e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
662bd33150aSjeremylt }
663bd33150aSjeremylt 
664bd33150aSjeremylt //------------------------------------------------------------------------------
66577d1c127SSebastian Grimberg // ElemRestriction Get Orientations
66677d1c127SSebastian Grimberg //------------------------------------------------------------------------------
66777d1c127SSebastian Grimberg static int CeedElemRestrictionGetOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const bool **orients) {
66877d1c127SSebastian Grimberg   Ceed                     ceed;
669ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
670ad70ee2cSJeremy L Thompson 
671ad70ee2cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
67277d1c127SSebastian Grimberg   CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed));
67377d1c127SSebastian Grimberg 
674fcbe8c06SSebastian Grimberg   CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory");
67577d1c127SSebastian Grimberg 
67677d1c127SSebastian Grimberg   *orients = impl->orients;
67777d1c127SSebastian Grimberg   return CEED_ERROR_SUCCESS;
67877d1c127SSebastian Grimberg }
67977d1c127SSebastian Grimberg 
68077d1c127SSebastian Grimberg //------------------------------------------------------------------------------
68177d1c127SSebastian Grimberg // ElemRestriction Get Curl-Conforming Orientations
68277d1c127SSebastian Grimberg //------------------------------------------------------------------------------
6830c73c039SSebastian Grimberg static int CeedElemRestrictionGetCurlOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt8 **curl_orients) {
68477d1c127SSebastian Grimberg   Ceed                     ceed;
685ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
686ad70ee2cSJeremy L Thompson 
687ad70ee2cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
68877d1c127SSebastian Grimberg   CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed));
68977d1c127SSebastian Grimberg 
690fcbe8c06SSebastian Grimberg   CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Can only provide to HOST memory");
69177d1c127SSebastian Grimberg 
69277d1c127SSebastian Grimberg   *curl_orients = impl->curl_orients;
69377d1c127SSebastian Grimberg   return CEED_ERROR_SUCCESS;
69477d1c127SSebastian Grimberg }
69577d1c127SSebastian Grimberg 
69677d1c127SSebastian Grimberg //------------------------------------------------------------------------------
697f10650afSjeremylt // ElemRestriction Destroy
698f10650afSjeremylt //------------------------------------------------------------------------------
6991cc2c60dSJeremy L Thompson static int CeedElemRestrictionDestroy_Ref(CeedElemRestriction rstr) {
700fe2413ffSjeremylt   CeedElemRestriction_Ref *impl;
70121617c04Sjeremylt 
7021cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
7032b730f8bSJeremy L Thompson   CeedCallBackend(CeedFree(&impl->offsets_allocated));
70477d1c127SSebastian Grimberg   CeedCallBackend(CeedFree(&impl->orients_allocated));
70577d1c127SSebastian Grimberg   CeedCallBackend(CeedFree(&impl->curl_orients_allocated));
7062b730f8bSJeremy L Thompson   CeedCallBackend(CeedFree(&impl));
707e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
70821617c04Sjeremylt }
70921617c04Sjeremylt 
710f10650afSjeremylt //------------------------------------------------------------------------------
711f10650afSjeremylt // ElemRestriction Create
712f10650afSjeremylt //------------------------------------------------------------------------------
713fcbe8c06SSebastian Grimberg int CeedElemRestrictionCreate_Ref(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *offsets, const bool *orients,
7141cc2c60dSJeremy L Thompson                                   const CeedInt8 *curl_orients, CeedElemRestriction rstr) {
715ad70ee2cSJeremy L Thompson   Ceed                     ceed;
71607d5dec1SJeremy L Thompson   CeedInt                  num_elem, elem_size, num_block, block_size, num_comp, comp_stride, num_points = 0, num_offsets;
717ad70ee2cSJeremy L Thompson   CeedRestrictionType      rstr_type;
71821617c04Sjeremylt   CeedElemRestriction_Ref *impl;
719ad70ee2cSJeremy L Thompson 
7201cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed));
7211cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem));
7221cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size));
7231cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
7241cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
7251cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
7261cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
727ad70ee2cSJeremy L Thompson   CeedInt layout[3] = {1, elem_size, elem_size * num_comp};
72821617c04Sjeremylt 
7296574a04fSJeremy L Thompson   CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Only MemType = HOST supported");
7302b730f8bSJeremy L Thompson   CeedCallBackend(CeedCalloc(1, &impl));
7313661185eSjeremylt 
73292fe105eSJeremy L Thompson   // Offsets data
7331cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type));
734fcbe8c06SSebastian Grimberg   if (rstr_type != CEED_RESTRICTION_STRIDED) {
7353661185eSjeremylt     const char *resource;
736ad70ee2cSJeremy L Thompson 
737ad70ee2cSJeremy L Thompson     // Check indices for ref or memcheck backends
73835aed383SJeremy L Thompson     {
73935aed383SJeremy L Thompson       Ceed current = ceed, parent = NULL;
74035aed383SJeremy L Thompson 
74135aed383SJeremy L Thompson       CeedCallBackend(CeedGetParent(current, &parent));
74235aed383SJeremy L Thompson       while (current != parent) {
74335aed383SJeremy L Thompson         current = parent;
74435aed383SJeremy L Thompson         CeedCallBackend(CeedGetParent(current, &parent));
74535aed383SJeremy L Thompson       }
74635aed383SJeremy L Thompson       CeedCallBackend(CeedGetResource(parent, &resource));
74735aed383SJeremy L Thompson     }
7482b730f8bSJeremy L Thompson     if (!strcmp(resource, "/cpu/self/ref/serial") || !strcmp(resource, "/cpu/self/ref/blocked") || !strcmp(resource, "/cpu/self/memcheck/serial") ||
749d1d35e2fSjeremylt         !strcmp(resource, "/cpu/self/memcheck/blocked")) {
750e79b91d9SJeremy L Thompson       CeedSize l_size;
7513661185eSjeremylt 
7521cc2c60dSJeremy L Thompson       CeedCallBackend(CeedElemRestrictionGetLVectorSize(rstr, &l_size));
7532b730f8bSJeremy L Thompson       for (CeedInt i = 0; i < num_elem * elem_size; i++) {
7546574a04fSJeremy L Thompson         CeedCheck(offsets[i] >= 0 && offsets[i] + (num_comp - 1) * comp_stride < l_size, ceed, CEED_ERROR_BACKEND,
7556574a04fSJeremy L Thompson                   "Restriction offset %" CeedInt_FMT " (%" CeedInt_FMT ") out of range [0, %" CeedInt_FMT "]", i, offsets[i], l_size);
7562b730f8bSJeremy L Thompson       }
7572b730f8bSJeremy L Thompson     }
7583661185eSjeremylt 
75992fe105eSJeremy L Thompson     // Copy data
76007d5dec1SJeremy L Thompson     if (rstr_type == CEED_RESTRICTION_POINTS) CeedCallBackend(CeedElemRestrictionGetNumPoints(rstr, &num_points));
76107d5dec1SJeremy L Thompson     num_offsets = rstr_type == CEED_RESTRICTION_POINTS ? (num_elem + 1 + num_points) : (num_elem * elem_size);
762d1d35e2fSjeremylt     switch (copy_mode) {
76321617c04Sjeremylt       case CEED_COPY_VALUES:
76407d5dec1SJeremy L Thompson         CeedCallBackend(CeedMalloc(num_offsets, &impl->offsets_allocated));
76507d5dec1SJeremy L Thompson         memcpy(impl->offsets_allocated, offsets, num_offsets * sizeof(offsets[0]));
766d979a051Sjeremylt         impl->offsets = impl->offsets_allocated;
76721617c04Sjeremylt         break;
76821617c04Sjeremylt       case CEED_OWN_POINTER:
769d979a051Sjeremylt         impl->offsets_allocated = (CeedInt *)offsets;
770d979a051Sjeremylt         impl->offsets           = impl->offsets_allocated;
77121617c04Sjeremylt         break;
77221617c04Sjeremylt       case CEED_USE_POINTER:
773d979a051Sjeremylt         impl->offsets = offsets;
77421617c04Sjeremylt     }
775fcbe8c06SSebastian Grimberg 
776fcbe8c06SSebastian Grimberg     // Orientation data
777fcbe8c06SSebastian Grimberg     if (rstr_type == CEED_RESTRICTION_ORIENTED) {
7780305e208SSebastian Grimberg       CeedCheck(orients != NULL, ceed, CEED_ERROR_BACKEND, "No orients array provided for oriented restriction");
779fcbe8c06SSebastian Grimberg       switch (copy_mode) {
780fcbe8c06SSebastian Grimberg         case CEED_COPY_VALUES:
78107d5dec1SJeremy L Thompson           CeedCallBackend(CeedMalloc(num_offsets, &impl->orients_allocated));
78207d5dec1SJeremy L Thompson           memcpy(impl->orients_allocated, orients, num_offsets * sizeof(orients[0]));
783fcbe8c06SSebastian Grimberg           impl->orients = impl->orients_allocated;
784fcbe8c06SSebastian Grimberg           break;
785fcbe8c06SSebastian Grimberg         case CEED_OWN_POINTER:
786fcbe8c06SSebastian Grimberg           impl->orients_allocated = (bool *)orients;
787fcbe8c06SSebastian Grimberg           impl->orients           = impl->orients_allocated;
788fcbe8c06SSebastian Grimberg           break;
789fcbe8c06SSebastian Grimberg         case CEED_USE_POINTER:
790fcbe8c06SSebastian Grimberg           impl->orients = orients;
791fcbe8c06SSebastian Grimberg       }
792fcbe8c06SSebastian Grimberg     } else if (rstr_type == CEED_RESTRICTION_CURL_ORIENTED) {
7930305e208SSebastian Grimberg       CeedCheck(curl_orients != NULL, ceed, CEED_ERROR_BACKEND, "No curl_orients array provided for oriented restriction");
794fcbe8c06SSebastian Grimberg       switch (copy_mode) {
795fcbe8c06SSebastian Grimberg         case CEED_COPY_VALUES:
79607d5dec1SJeremy L Thompson           CeedCallBackend(CeedMalloc(3 * num_offsets, &impl->curl_orients_allocated));
79707d5dec1SJeremy L Thompson           memcpy(impl->curl_orients_allocated, curl_orients, 3 * num_offsets * sizeof(curl_orients[0]));
798fcbe8c06SSebastian Grimberg           impl->curl_orients = impl->curl_orients_allocated;
799fcbe8c06SSebastian Grimberg           break;
800fcbe8c06SSebastian Grimberg         case CEED_OWN_POINTER:
8010c73c039SSebastian Grimberg           impl->curl_orients_allocated = (CeedInt8 *)curl_orients;
802fcbe8c06SSebastian Grimberg           impl->curl_orients           = impl->curl_orients_allocated;
803fcbe8c06SSebastian Grimberg           break;
804fcbe8c06SSebastian Grimberg         case CEED_USE_POINTER:
805fcbe8c06SSebastian Grimberg           impl->curl_orients = curl_orients;
806fcbe8c06SSebastian Grimberg       }
807fcbe8c06SSebastian Grimberg     }
80892fe105eSJeremy L Thompson   }
809fe2413ffSjeremylt 
8101cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionSetData(rstr, impl));
8111cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionSetELayout(rstr, layout));
8121cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Apply", CeedElemRestrictionApply_Ref));
8132c7e7413SJeremy L Thompson   if (rstr_type == CEED_RESTRICTION_POINTS) {
8141249ccc5SJeremy L Thompson     CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyAtPointsInElement", CeedElemRestrictionApplyAtPointsInElement_Ref));
8151249ccc5SJeremy L Thompson   }
8161cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnsigned", CeedElemRestrictionApplyUnsigned_Ref));
8171cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnoriented", CeedElemRestrictionApplyUnoriented_Ref));
8181cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyBlock", CeedElemRestrictionApplyBlock_Ref));
8191cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOrientations", CeedElemRestrictionGetOrientations_Ref));
8201cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetCurlOrientations", CeedElemRestrictionGetCurlOrientations_Ref));
8212c7e7413SJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOffsets", CeedElemRestrictionGetOffsets_Ref));
8221cc2c60dSJeremy L Thompson   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Destroy", CeedElemRestrictionDestroy_Ref));
823d979a051Sjeremylt 
824ad70ee2cSJeremy L Thompson   // Set apply function based upon num_comp, block_size, and comp_stride
825ad70ee2cSJeremy L Thompson   CeedInt index = -1;
826ad70ee2cSJeremy L Thompson 
827ad70ee2cSJeremy L Thompson   if (block_size < 10) index = 100 * num_comp + 10 * block_size + (comp_stride == 1);
828ad70ee2cSJeremy L Thompson   switch (index) {
829d979a051Sjeremylt     case 110:
830d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_110;
831d979a051Sjeremylt       break;
832d979a051Sjeremylt     case 111:
833d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_111;
834d979a051Sjeremylt       break;
835d979a051Sjeremylt     case 180:
836d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_180;
837d979a051Sjeremylt       break;
838d979a051Sjeremylt     case 181:
839d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_181;
840d979a051Sjeremylt       break;
841d979a051Sjeremylt     case 310:
842d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_310;
843d979a051Sjeremylt       break;
844d979a051Sjeremylt     case 311:
845d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_311;
846d979a051Sjeremylt       break;
847d979a051Sjeremylt     case 380:
848d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_380;
849d979a051Sjeremylt       break;
850d979a051Sjeremylt     case 381:
851d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_381;
852d979a051Sjeremylt       break;
853bf4d1581Sjeremylt     // LCOV_EXCL_START
854d979a051Sjeremylt     case 510:
855d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_510;
856d979a051Sjeremylt       break;
857bf4d1581Sjeremylt     // LCOV_EXCL_STOP
858d979a051Sjeremylt     case 511:
859d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_511;
860d979a051Sjeremylt       break;
861bf4d1581Sjeremylt     // LCOV_EXCL_START
862d979a051Sjeremylt     case 580:
863d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_580;
864d979a051Sjeremylt       break;
865bf4d1581Sjeremylt     // LCOV_EXCL_STOP
866d979a051Sjeremylt     case 581:
867d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_581;
868d979a051Sjeremylt       break;
869d979a051Sjeremylt     default:
870d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_Core;
871d979a051Sjeremylt       break;
872d979a051Sjeremylt   }
873e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
87421617c04Sjeremylt }
875fc0567d9Srezgarshakeri 
876fc0567d9Srezgarshakeri //------------------------------------------------------------------------------
877