xref: /libCEED/rust/libceed-sys/c-src/backends/ref/ceed-ref-restriction.c (revision d275d636ccaa61e594421fac80252590e7a77ccf)
1*d275d636SJeremy L Thompson // Copyright (c) 2017-2025, Lawrence Livermore National Security, LLC and other CEED contributors.
23d8e8822SJeremy L Thompson // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
321617c04Sjeremylt //
43d8e8822SJeremy L Thompson // SPDX-License-Identifier: BSD-2-Clause
521617c04Sjeremylt //
63d8e8822SJeremy L Thompson // This file is part of CEED:  http://github.com/ceed
721617c04Sjeremylt 
849aac155SJeremy L Thompson #include <ceed.h>
9ec3da8bcSJed Brown #include <ceed/backend.h>
103d576824SJeremy L Thompson #include <stdbool.h>
11fcbe8c06SSebastian Grimberg #include <stdlib.h>
123d576824SJeremy L Thompson #include <string.h>
132b730f8bSJeremy L Thompson 
1421617c04Sjeremylt #include "ceed-ref.h"
1521617c04Sjeremylt 
16f10650afSjeremylt //------------------------------------------------------------------------------
17f10650afSjeremylt // Core ElemRestriction Apply Code
18f10650afSjeremylt //------------------------------------------------------------------------------
191cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
2094648b7dSSebastian Grimberg                                                                       CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size,
21171d97d0SJeremy L Thompson                                                                       CeedSize v_offset, const CeedScalar *__restrict__ uu,
22eda0adbcSSebastian Grimberg                                                                       CeedScalar *__restrict__ vv) {
2394648b7dSSebastian Grimberg   // No offsets provided, identity restriction
24d1d35e2fSjeremylt   bool has_backend_strides;
25ad70ee2cSJeremy L Thompson 
261cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides));
27d1d35e2fSjeremylt   if (has_backend_strides) {
28d1d35e2fSjeremylt     // CPU backend strides are {1, elem_size, elem_size*num_comp}
297f90ec76Sjeremylt     // This if branch is left separate to allow better inlining
3033e3c889SJeremy L Thompson     for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
3133e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
3233e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize n = 0; n < elem_size; n++) {
3333e3c889SJeremy L Thompson           CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
3433e3c889SJeremy L Thompson             vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
3533e3c889SJeremy L Thompson                 uu[n + k * elem_size + CeedIntMin(e + j, num_elem - 1) * elem_size * (CeedSize)num_comp];
362b730f8bSJeremy L Thompson           }
372b730f8bSJeremy L Thompson         }
382b730f8bSJeremy L Thompson       }
392b730f8bSJeremy L Thompson     }
407f90ec76Sjeremylt   } else {
417f90ec76Sjeremylt     // User provided strides
427f90ec76Sjeremylt     CeedInt strides[3];
43ad70ee2cSJeremy L Thompson 
4456c48462SJeremy L Thompson     CeedCallBackend(CeedElemRestrictionGetStrides(rstr, strides));
4533e3c889SJeremy L Thompson     for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
4633e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
4733e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize n = 0; n < elem_size; n++) {
4833e3c889SJeremy L Thompson           CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
4933e3c889SJeremy L Thompson             vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
5033e3c889SJeremy L Thompson                 uu[n * strides[0] + k * strides[1] + CeedIntMin(e + j, num_elem - 1) * (CeedSize)strides[2]];
512b730f8bSJeremy L Thompson           }
522b730f8bSJeremy L Thompson         }
532b730f8bSJeremy L Thompson       }
542b730f8bSJeremy L Thompson     }
557509a596Sjeremylt   }
5694648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
5794648b7dSSebastian Grimberg }
5894648b7dSSebastian Grimberg 
59eda0adbcSSebastian Grimberg static inline int CeedElemRestrictionApplyOffsetNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
6094648b7dSSebastian Grimberg                                                                      const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
61171d97d0SJeremy L Thompson                                                                      CeedInt elem_size, CeedSize v_offset, const CeedScalar *__restrict__ uu,
62eda0adbcSSebastian Grimberg                                                                      CeedScalar *__restrict__ vv) {
63fcbe8c06SSebastian Grimberg   // Default restriction with offsets
6494648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
65ad70ee2cSJeremy L Thompson 
661cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6733e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
6833e3c889SJeremy L Thompson     CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
6933e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize i = 0; i < elem_size * block_size; i++) {
7033e3c889SJeremy L Thompson         vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] = uu[impl->offsets[i + e * elem_size] + k * comp_stride];
71fcbe8c06SSebastian Grimberg       }
72fcbe8c06SSebastian Grimberg     }
73fcbe8c06SSebastian Grimberg   }
7494648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
7594648b7dSSebastian Grimberg }
7694648b7dSSebastian Grimberg 
771cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
7894648b7dSSebastian Grimberg                                                                        const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
79171d97d0SJeremy L Thompson                                                                        CeedInt elem_size, CeedSize v_offset, const CeedScalar *__restrict__ uu,
80eda0adbcSSebastian Grimberg                                                                        CeedScalar *__restrict__ vv) {
81fcbe8c06SSebastian Grimberg   // Restriction with orientations
8294648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
83ad70ee2cSJeremy L Thompson 
841cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
8533e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
8633e3c889SJeremy L Thompson     CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
8733e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize i = 0; i < elem_size * block_size; i++) {
8833e3c889SJeremy L Thompson         vv[elem_size * (k * block_size + e * num_comp) + i - v_offset] =
897c1dbaffSSebastian Grimberg             uu[impl->offsets[i + e * elem_size] + k * comp_stride] * (impl->orients[i + e * elem_size] ? -1.0 : 1.0);
90fcbe8c06SSebastian Grimberg       }
91fcbe8c06SSebastian Grimberg     }
92fcbe8c06SSebastian Grimberg   }
9394648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
9494648b7dSSebastian Grimberg }
9594648b7dSSebastian Grimberg 
961cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
9794648b7dSSebastian Grimberg                                                                            const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
98171d97d0SJeremy L Thompson                                                                            CeedInt elem_size, CeedSize v_offset, const CeedScalar *__restrict__ uu,
99eda0adbcSSebastian Grimberg                                                                            CeedScalar *__restrict__ vv) {
10077d1c127SSebastian Grimberg   // Restriction with tridiagonal transformation
10194648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
102ad70ee2cSJeremy L Thompson 
1031cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
10433e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
10533e3c889SJeremy L Thompson     CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
10633e3c889SJeremy L Thompson       CeedSize n = 0;
1075c7e0f51SSebastian Grimberg 
10833e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
10933e3c889SJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
110ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
111ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
112ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
113ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size];
1140c73c039SSebastian Grimberg       }
1155c7e0f51SSebastian Grimberg       CeedPragmaSIMD for (n = 1; n < elem_size - 1; n++) {
11633e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
11733e3c889SJeremy L Thompson           vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
118ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
119ad70ee2cSJeremy L Thompson                   impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] +
120ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
121ad70ee2cSJeremy L Thompson                   impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
122ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
123ad70ee2cSJeremy L Thompson                   impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size];
1240c73c039SSebastian Grimberg         }
1250c73c039SSebastian Grimberg       }
12633e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
12733e3c889SJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
128ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
129ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size] +
130ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
131ad70ee2cSJeremy L Thompson                 impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size];
1322b730f8bSJeremy L Thompson       }
1332b730f8bSJeremy L Thompson     }
1342b730f8bSJeremy L Thompson   }
1350c73c039SSebastian Grimberg   return CEED_ERROR_SUCCESS;
136fcbe8c06SSebastian Grimberg }
1370c73c039SSebastian Grimberg 
1381cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp,
139ad70ee2cSJeremy L Thompson                                                                                    const CeedInt block_size, const CeedInt comp_stride, CeedInt start,
14094648b7dSSebastian Grimberg                                                                                    CeedInt stop, CeedInt num_elem, CeedInt elem_size,
141171d97d0SJeremy L Thompson                                                                                    CeedSize v_offset, const CeedScalar *__restrict__ uu,
142eda0adbcSSebastian Grimberg                                                                                    CeedScalar *__restrict__ vv) {
14394648b7dSSebastian Grimberg   // Restriction with (unsigned) tridiagonal transformation
1440c73c039SSebastian Grimberg   CeedElemRestriction_Ref *impl;
145ad70ee2cSJeremy L Thompson 
1461cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
14733e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
14833e3c889SJeremy L Thompson     CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
14933e3c889SJeremy L Thompson       CeedSize n = 0;
150ad70ee2cSJeremy L Thompson 
15133e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
15233e3c889SJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
153ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
154ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
155ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
156ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]);
1577c1dbaffSSebastian Grimberg       }
1585c7e0f51SSebastian Grimberg       CeedPragmaSIMD for (n = 1; n < elem_size - 1; n++) {
15933e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
16033e3c889SJeremy L Thompson           vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
161ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
162ad70ee2cSJeremy L Thompson                   abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) +
163ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
164ad70ee2cSJeremy L Thompson                   abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
165ad70ee2cSJeremy L Thompson               uu[impl->offsets[j + (n + 1) * block_size + e * elem_size] + k * comp_stride] *
166ad70ee2cSJeremy L Thompson                   abs(impl->curl_orients[j + (3 * n + 2) * block_size + e * 3 * elem_size]);
1677c1dbaffSSebastian Grimberg         }
1687c1dbaffSSebastian Grimberg       }
16933e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_size; j++) {
17033e3c889SJeremy L Thompson         vv[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] =
171ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + (n - 1) * block_size + e * elem_size] + k * comp_stride] *
172ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 0) * block_size + e * 3 * elem_size]) +
173ad70ee2cSJeremy L Thompson             uu[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] *
174ad70ee2cSJeremy L Thompson                 abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]);
1757c1dbaffSSebastian Grimberg       }
1767c1dbaffSSebastian Grimberg     }
1777c1dbaffSSebastian Grimberg   }
1787c1dbaffSSebastian Grimberg   return CEED_ERROR_SUCCESS;
1797c1dbaffSSebastian Grimberg }
1807c1dbaffSSebastian Grimberg 
1811cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyStridedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
18294648b7dSSebastian Grimberg                                                                     CeedInt start, CeedInt stop, CeedInt num_elem, CeedInt elem_size,
183171d97d0SJeremy L Thompson                                                                     CeedSize v_offset, const CeedScalar *__restrict__ uu,
184eda0adbcSSebastian Grimberg                                                                     CeedScalar *__restrict__ vv) {
18594648b7dSSebastian Grimberg   // No offsets provided, identity restriction
186d1d35e2fSjeremylt   bool has_backend_strides;
187ad70ee2cSJeremy L Thompson 
1881cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides));
189d1d35e2fSjeremylt   if (has_backend_strides) {
190d1d35e2fSjeremylt     // CPU backend strides are {1, elem_size, elem_size*num_comp}
1917f90ec76Sjeremylt     // This if brach is left separate to allow better inlining
19233e3c889SJeremy L Thompson     for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
19333e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
19433e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize n = 0; n < elem_size; n++) {
19533e3c889SJeremy L Thompson           CeedPragmaSIMD for (CeedSize j = 0; j < CeedIntMin(block_size, num_elem - e); j++) {
19633e3c889SJeremy L Thompson             vv[n + k * elem_size + (e + j) * elem_size * num_comp] += uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset];
1972b730f8bSJeremy L Thompson           }
1982b730f8bSJeremy L Thompson         }
1992b730f8bSJeremy L Thompson       }
2002b730f8bSJeremy L Thompson     }
2017f90ec76Sjeremylt   } else {
2027f90ec76Sjeremylt     // User provided strides
2037f90ec76Sjeremylt     CeedInt strides[3];
204ad70ee2cSJeremy L Thompson 
20556c48462SJeremy L Thompson     CeedCallBackend(CeedElemRestrictionGetStrides(rstr, strides));
206ad70ee2cSJeremy L Thompson     for (CeedInt e = start * block_size; e < stop * block_size; e += block_size) {
20733e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize k = 0; k < num_comp; k++) {
20833e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize n = 0; n < elem_size; n++) {
20933e3c889SJeremy L Thompson           CeedPragmaSIMD for (CeedSize j = 0; j < CeedIntMin(block_size, num_elem - e); j++) {
21033e3c889SJeremy L Thompson             vv[n * strides[0] + k * strides[1] + (e + j) * strides[2]] +=
21133e3c889SJeremy L Thompson                 uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset];
2122b730f8bSJeremy L Thompson           }
2132b730f8bSJeremy L Thompson         }
2142b730f8bSJeremy L Thompson       }
2152b730f8bSJeremy L Thompson     }
216523b8ea0Sjeremylt   }
21794648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
21894648b7dSSebastian Grimberg }
21994648b7dSSebastian Grimberg 
220eda0adbcSSebastian Grimberg static inline int CeedElemRestrictionApplyOffsetTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
22194648b7dSSebastian Grimberg                                                                    const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
222171d97d0SJeremy L Thompson                                                                    CeedInt elem_size, CeedSize v_offset, const CeedScalar *__restrict__ uu,
223eda0adbcSSebastian Grimberg                                                                    CeedScalar *__restrict__ vv) {
224fcbe8c06SSebastian Grimberg   // Default restriction with offsets
22594648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
226ad70ee2cSJeremy L Thompson 
2271cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
22833e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
22933e3c889SJeremy L Thompson     for (CeedSize k = 0; k < num_comp; k++) {
23033e3c889SJeremy L Thompson       for (CeedSize i = 0; i < elem_size * block_size; i += block_size) {
2318d94b059Sjeremylt         // Iteration bound set to discard padding elements
23233e3c889SJeremy L Thompson         for (CeedSize j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) {
2335c7e0f51SSebastian Grimberg           CeedScalar vv_loc;
23458c07c4fSSebastian Grimberg 
23533e3c889SJeremy L Thompson           vv_loc = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset];
2365c7e0f51SSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += vv_loc;
237fcbe8c06SSebastian Grimberg         }
238fcbe8c06SSebastian Grimberg       }
239fcbe8c06SSebastian Grimberg     }
240fcbe8c06SSebastian Grimberg   }
24194648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
24294648b7dSSebastian Grimberg }
24394648b7dSSebastian Grimberg 
2441cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
24594648b7dSSebastian Grimberg                                                                      const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
246171d97d0SJeremy L Thompson                                                                      CeedInt elem_size, CeedSize v_offset, const CeedScalar *__restrict__ uu,
247eda0adbcSSebastian Grimberg                                                                      CeedScalar *__restrict__ vv) {
248fcbe8c06SSebastian Grimberg   // Restriction with orientations
24994648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
250ad70ee2cSJeremy L Thompson 
2511cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
25233e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
25333e3c889SJeremy L Thompson     for (CeedSize k = 0; k < num_comp; k++) {
25433e3c889SJeremy L Thompson       for (CeedSize i = 0; i < elem_size * block_size; i += block_size) {
255fcbe8c06SSebastian Grimberg         // Iteration bound set to discard padding elements
25633e3c889SJeremy L Thompson         for (CeedSize j = i; j < i + CeedIntMin(block_size, num_elem - e); j++) {
2575c7e0f51SSebastian Grimberg           CeedScalar vv_loc;
25858c07c4fSSebastian Grimberg 
25933e3c889SJeremy L Thompson           vv_loc = uu[elem_size * (k * block_size + e * num_comp) + j - v_offset] * (impl->orients[j + e * elem_size] ? -1.0 : 1.0);
2605c7e0f51SSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + e * elem_size] + k * comp_stride] += vv_loc;
261fcbe8c06SSebastian Grimberg         }
262fcbe8c06SSebastian Grimberg       }
263fcbe8c06SSebastian Grimberg     }
264fcbe8c06SSebastian Grimberg   }
26594648b7dSSebastian Grimberg   return CEED_ERROR_SUCCESS;
26694648b7dSSebastian Grimberg }
26794648b7dSSebastian Grimberg 
2681cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
26994648b7dSSebastian Grimberg                                                                          const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedInt num_elem,
270171d97d0SJeremy L Thompson                                                                          CeedInt elem_size, CeedSize v_offset, const CeedScalar *__restrict__ uu,
271eda0adbcSSebastian Grimberg                                                                          CeedScalar *__restrict__ vv) {
27277d1c127SSebastian Grimberg   // Restriction with tridiagonal transformation
27394648b7dSSebastian Grimberg   CeedElemRestriction_Ref *impl;
2745c7e0f51SSebastian Grimberg   CeedScalar               vv_loc[block_size];
275ad70ee2cSJeremy L Thompson 
2761cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
27733e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
27833e3c889SJeremy L Thompson     for (CeedSize k = 0; k < num_comp; k++) {
279fcbe8c06SSebastian Grimberg       // Iteration bound set to discard padding elements
28033e3c889SJeremy L Thompson       const CeedSize block_end = CeedIntMin(block_size, num_elem - e);
28133e3c889SJeremy L Thompson       CeedSize       n         = 0;
28258c07c4fSSebastian Grimberg 
28333e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_end; j++) {
28433e3c889SJeremy L Thompson         vv_loc[j] = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
285ad70ee2cSJeremy L Thompson                         impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
28633e3c889SJeremy L Thompson                     uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
287ad70ee2cSJeremy L Thompson                         impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size];
2885c7e0f51SSebastian Grimberg       }
28933e3c889SJeremy L Thompson       for (CeedSize j = 0; j < block_end; j++) {
2905c7e0f51SSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += vv_loc[j];
2910c73c039SSebastian Grimberg       }
2920c73c039SSebastian Grimberg       for (n = 1; n < elem_size - 1; n++) {
2935c7e0f51SSebastian Grimberg         CeedPragmaSIMD for (CeedInt j = 0; j < block_end; j++) {
29433e3c889SJeremy L Thompson           vv_loc[j] = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
295ad70ee2cSJeremy L Thompson                           impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] +
29633e3c889SJeremy L Thompson                       uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
297ad70ee2cSJeremy L Thompson                           impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size] +
29833e3c889SJeremy L Thompson                       uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
299ad70ee2cSJeremy L Thompson                           impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size];
3000c73c039SSebastian Grimberg         }
30133e3c889SJeremy L Thompson         for (CeedSize j = 0; j < block_end; j++) {
3025c7e0f51SSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += vv_loc[j];
3035c7e0f51SSebastian Grimberg         }
3045c7e0f51SSebastian Grimberg       }
30533e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_end; j++) {
30633e3c889SJeremy L Thompson         vv_loc[j] = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
307ad70ee2cSJeremy L Thompson                         impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size] +
30833e3c889SJeremy L Thompson                     uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
309ad70ee2cSJeremy L Thompson                         impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size];
3105c7e0f51SSebastian Grimberg       }
31133e3c889SJeremy L Thompson       for (CeedSize j = 0; j < block_end; j++) {
3125c7e0f51SSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += vv_loc[j];
31321617c04Sjeremylt       }
314b435c5a6Srezgarshakeri     }
3152b730f8bSJeremy L Thompson   }
316e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
31721617c04Sjeremylt }
31821617c04Sjeremylt 
3191cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp,
320ad70ee2cSJeremy L Thompson                                                                                  const CeedInt block_size, const CeedInt comp_stride, CeedInt start,
321171d97d0SJeremy L Thompson                                                                                  CeedInt stop, CeedInt num_elem, CeedInt elem_size, CeedSize v_offset,
322eda0adbcSSebastian Grimberg                                                                                  const CeedScalar *__restrict__ uu, CeedScalar *__restrict__ vv) {
32394648b7dSSebastian Grimberg   // Restriction with (unsigned) tridiagonal transformation
3247c1dbaffSSebastian Grimberg   CeedElemRestriction_Ref *impl;
3255c7e0f51SSebastian Grimberg   CeedScalar               vv_loc[block_size];
326ad70ee2cSJeremy L Thompson 
3271cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
32833e3c889SJeremy L Thompson   for (CeedSize e = start * block_size; e < stop * block_size; e += block_size) {
32933e3c889SJeremy L Thompson     for (CeedSize k = 0; k < num_comp; k++) {
3307c1dbaffSSebastian Grimberg       // Iteration bound set to discard padding elements
33133e3c889SJeremy L Thompson       const CeedSize block_end = CeedIntMin(block_size, num_elem - e);
33233e3c889SJeremy L Thompson       CeedSize       n         = 0;
333ad70ee2cSJeremy L Thompson 
33433e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_end; j++) {
33533e3c889SJeremy L Thompson         vv_loc[j] = uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
336ad70ee2cSJeremy L Thompson                         abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
33733e3c889SJeremy L Thompson                     uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
338ad70ee2cSJeremy L Thompson                         abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]);
3395c7e0f51SSebastian Grimberg       }
34033e3c889SJeremy L Thompson       for (CeedSize j = 0; j < block_end; j++) {
3415c7e0f51SSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += vv_loc[j];
3427c1dbaffSSebastian Grimberg       }
3437c1dbaffSSebastian Grimberg       for (n = 1; n < elem_size - 1; n++) {
34433e3c889SJeremy L Thompson         CeedPragmaSIMD for (CeedSize j = 0; j < block_end; j++) {
34533e3c889SJeremy L Thompson           vv_loc[j] = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
346ad70ee2cSJeremy L Thompson                           abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) +
34733e3c889SJeremy L Thompson                       uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
348ad70ee2cSJeremy L Thompson                           abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]) +
34933e3c889SJeremy L Thompson                       uu[e * elem_size * num_comp + (k * elem_size + n + 1) * block_size + j - v_offset] *
350ad70ee2cSJeremy L Thompson                           abs(impl->curl_orients[j + (3 * n + 3) * block_size + e * 3 * elem_size]);
3517c1dbaffSSebastian Grimberg         }
35233e3c889SJeremy L Thompson         for (CeedSize j = 0; j < block_end; j++) {
3535c7e0f51SSebastian Grimberg           CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += vv_loc[j];
3545c7e0f51SSebastian Grimberg         }
3555c7e0f51SSebastian Grimberg       }
35633e3c889SJeremy L Thompson       CeedPragmaSIMD for (CeedSize j = 0; j < block_end; j++) {
35733e3c889SJeremy L Thompson         vv_loc[j] = uu[e * elem_size * num_comp + (k * elem_size + n - 1) * block_size + j - v_offset] *
358ad70ee2cSJeremy L Thompson                         abs(impl->curl_orients[j + (3 * n - 1) * block_size + e * 3 * elem_size]) +
35933e3c889SJeremy L Thompson                     uu[e * elem_size * num_comp + (k * elem_size + n) * block_size + j - v_offset] *
360ad70ee2cSJeremy L Thompson                         abs(impl->curl_orients[j + (3 * n + 1) * block_size + e * 3 * elem_size]);
3615c7e0f51SSebastian Grimberg       }
36233e3c889SJeremy L Thompson       for (CeedSize j = 0; j < block_end; j++) {
3635c7e0f51SSebastian Grimberg         CeedPragmaAtomic vv[impl->offsets[j + n * block_size + e * elem_size] + k * comp_stride] += vv_loc[j];
3647c1dbaffSSebastian Grimberg       }
3657c1dbaffSSebastian Grimberg     }
3667c1dbaffSSebastian Grimberg   }
3677c1dbaffSSebastian Grimberg   return CEED_ERROR_SUCCESS;
3687c1dbaffSSebastian Grimberg }
3697c1dbaffSSebastian Grimberg 
3701249ccc5SJeremy L Thompson static inline int CeedElemRestrictionApplyAtPointsInElement_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, CeedInt start, CeedInt stop,
371eda0adbcSSebastian Grimberg                                                                      CeedTransposeMode t_mode, const CeedScalar *__restrict__ uu,
372eda0adbcSSebastian Grimberg                                                                      CeedScalar *__restrict__ vv) {
373171d97d0SJeremy L Thompson   CeedInt                  num_points, l_vec_offset;
374171d97d0SJeremy L Thompson   CeedSize                 e_vec_offset = 0;
37505fa913cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
37605fa913cSJeremy L Thompson 
37705fa913cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
37805fa913cSJeremy L Thompson   for (CeedInt e = start; e < stop; e++) {
3790930e4e7SJeremy L Thompson     l_vec_offset = impl->offsets[e];
38005fa913cSJeremy L Thompson     CeedCallBackend(CeedElemRestrictionGetNumPointsInElement(rstr, e, &num_points));
38105fa913cSJeremy L Thompson     if (t_mode == CEED_NOTRANSPOSE) {
38233e3c889SJeremy L Thompson       for (CeedSize i = 0; i < num_points; i++) {
38333e3c889SJeremy L Thompson         for (CeedSize j = 0; j < num_comp; j++) vv[j * num_points + i + e_vec_offset] = uu[impl->offsets[i + l_vec_offset] * num_comp + j];
38405fa913cSJeremy L Thompson       }
38505fa913cSJeremy L Thompson     } else {
38633e3c889SJeremy L Thompson       for (CeedSize i = 0; i < num_points; i++) {
3870b63de31SJeremy L Thompson         for (CeedSize j = 0; j < num_comp; j++) vv[impl->offsets[i + l_vec_offset] * num_comp + j] += uu[j * num_points + i + e_vec_offset];
38805fa913cSJeremy L Thompson       }
38905fa913cSJeremy L Thompson     }
390171d97d0SJeremy L Thompson     e_vec_offset += num_points * (CeedSize)num_comp;
39105fa913cSJeremy L Thompson   }
39205fa913cSJeremy L Thompson   return CEED_ERROR_SUCCESS;
39305fa913cSJeremy L Thompson }
39405fa913cSJeremy L Thompson 
3951cc2c60dSJeremy L Thompson static inline int CeedElemRestrictionApply_Ref_Core(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size,
396ad70ee2cSJeremy L Thompson                                                     const CeedInt comp_stride, CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs,
397ad70ee2cSJeremy L Thompson                                                     bool use_orients, CeedVector u, CeedVector v, CeedRequest *request) {
398171d97d0SJeremy L Thompson   CeedInt             num_elem, elem_size;
399171d97d0SJeremy L Thompson   CeedSize            v_offset = 0;
400ad70ee2cSJeremy L Thompson   CeedRestrictionType rstr_type;
4017c1dbaffSSebastian Grimberg   const CeedScalar   *uu;
4027c1dbaffSSebastian Grimberg   CeedScalar         *vv;
403ad70ee2cSJeremy L Thompson 
4041cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem));
4051cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size));
406171d97d0SJeremy L Thompson   v_offset = start * block_size * elem_size * (CeedSize)num_comp;
4071cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type));
40894648b7dSSebastian Grimberg   CeedCallBackend(CeedVectorGetArrayRead(u, CEED_MEM_HOST, &uu));
409ad70ee2cSJeremy L Thompson 
41094648b7dSSebastian Grimberg   if (t_mode == CEED_TRANSPOSE) {
41194648b7dSSebastian Grimberg     // Sum into for transpose mode, E-vector to L-vector
41294648b7dSSebastian Grimberg     CeedCallBackend(CeedVectorGetArray(v, CEED_MEM_HOST, &vv));
41394648b7dSSebastian Grimberg   } else {
41494648b7dSSebastian Grimberg     // Overwrite for notranspose mode, L-vector to E-vector
41594648b7dSSebastian Grimberg     CeedCallBackend(CeedVectorGetArrayWrite(v, CEED_MEM_HOST, &vv));
41694648b7dSSebastian Grimberg   }
41794648b7dSSebastian Grimberg   if (t_mode == CEED_TRANSPOSE) {
4187c1dbaffSSebastian Grimberg     // Restriction from E-vector to L-vector
4197c1dbaffSSebastian Grimberg     // Performing v += r^T * u
4207c1dbaffSSebastian Grimberg     // uu has shape [elem_size, num_comp, num_elem], row-major
4217c1dbaffSSebastian Grimberg     // vv has shape [nnodes, num_comp]
4227c1dbaffSSebastian Grimberg     // Sum into for transpose mode
4237c1dbaffSSebastian Grimberg     switch (rstr_type) {
4247c1dbaffSSebastian Grimberg       case CEED_RESTRICTION_STRIDED:
4255d10938bSJeremy L Thompson         CeedCallBackend(
4261cc2c60dSJeremy L Thompson             CeedElemRestrictionApplyStridedTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv));
42794648b7dSSebastian Grimberg         break;
42861a27d74SSebastian Grimberg       case CEED_RESTRICTION_STANDARD:
429eda0adbcSSebastian Grimberg         CeedCallBackend(CeedElemRestrictionApplyOffsetTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size,
4305d10938bSJeremy L Thompson                                                                          v_offset, uu, vv));
43194648b7dSSebastian Grimberg         break;
4327c1dbaffSSebastian Grimberg       case CEED_RESTRICTION_ORIENTED:
43394648b7dSSebastian Grimberg         if (use_signs) {
4341cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4351cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
43694648b7dSSebastian Grimberg         } else {
437eda0adbcSSebastian Grimberg           CeedCallBackend(CeedElemRestrictionApplyOffsetTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size,
438eda0adbcSSebastian Grimberg                                                                            v_offset, uu, vv));
43994648b7dSSebastian Grimberg         }
44094648b7dSSebastian Grimberg         break;
44194648b7dSSebastian Grimberg       case CEED_RESTRICTION_CURL_ORIENTED:
44294648b7dSSebastian Grimberg         if (use_signs && use_orients) {
4431cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4445d10938bSJeremy L Thompson                                                                                  elem_size, v_offset, uu, vv));
44594648b7dSSebastian Grimberg         } else if (use_orients) {
4461cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop,
4471cc2c60dSJeremy L Thompson                                                                                          num_elem, elem_size, v_offset, uu, vv));
44894648b7dSSebastian Grimberg         } else {
449eda0adbcSSebastian Grimberg           CeedCallBackend(CeedElemRestrictionApplyOffsetTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size,
450eda0adbcSSebastian Grimberg                                                                            v_offset, uu, vv));
45194648b7dSSebastian Grimberg         }
45294648b7dSSebastian Grimberg         break;
4532c7e7413SJeremy L Thompson       case CEED_RESTRICTION_POINTS:
4541249ccc5SJeremy L Thompson         CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv));
4552c7e7413SJeremy L Thompson         break;
45694648b7dSSebastian Grimberg     }
45794648b7dSSebastian Grimberg   } else {
45894648b7dSSebastian Grimberg     // Restriction from L-vector to E-vector
45994648b7dSSebastian Grimberg     // Perform: v = r * u
46094648b7dSSebastian Grimberg     // vv has shape [elem_size, num_comp, num_elem], row-major
46194648b7dSSebastian Grimberg     // uu has shape [nnodes, num_comp]
46294648b7dSSebastian Grimberg     // Overwrite for notranspose mode
46394648b7dSSebastian Grimberg     switch (rstr_type) {
46494648b7dSSebastian Grimberg       case CEED_RESTRICTION_STRIDED:
4655d10938bSJeremy L Thompson         CeedCallBackend(
4661cc2c60dSJeremy L Thompson             CeedElemRestrictionApplyStridedNoTranspose_Ref_Core(rstr, num_comp, block_size, start, stop, num_elem, elem_size, v_offset, uu, vv));
46794648b7dSSebastian Grimberg         break;
46861a27d74SSebastian Grimberg       case CEED_RESTRICTION_STANDARD:
469eda0adbcSSebastian Grimberg         CeedCallBackend(CeedElemRestrictionApplyOffsetNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem, elem_size,
470eda0adbcSSebastian Grimberg                                                                            v_offset, uu, vv));
47194648b7dSSebastian Grimberg         break;
47294648b7dSSebastian Grimberg       case CEED_RESTRICTION_ORIENTED:
47394648b7dSSebastian Grimberg         if (use_signs) {
4741cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4751cc2c60dSJeremy L Thompson                                                                                elem_size, v_offset, uu, vv));
47694648b7dSSebastian Grimberg         } else {
477eda0adbcSSebastian Grimberg           CeedCallBackend(CeedElemRestrictionApplyOffsetNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4781cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
47994648b7dSSebastian Grimberg         }
48094648b7dSSebastian Grimberg         break;
48194648b7dSSebastian Grimberg       case CEED_RESTRICTION_CURL_ORIENTED:
48294648b7dSSebastian Grimberg         if (use_signs && use_orients) {
4831cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4845d10938bSJeremy L Thompson                                                                                    elem_size, v_offset, uu, vv));
48594648b7dSSebastian Grimberg         } else if (use_orients) {
4861cc2c60dSJeremy L Thompson           CeedCallBackend(CeedElemRestrictionApplyCurlOrientedUnsignedNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop,
4875d10938bSJeremy L Thompson                                                                                            num_elem, elem_size, v_offset, uu, vv));
48894648b7dSSebastian Grimberg         } else {
489eda0adbcSSebastian Grimberg           CeedCallBackend(CeedElemRestrictionApplyOffsetNoTranspose_Ref_Core(rstr, num_comp, block_size, comp_stride, start, stop, num_elem,
4901cc2c60dSJeremy L Thompson                                                                              elem_size, v_offset, uu, vv));
49194648b7dSSebastian Grimberg         }
49294648b7dSSebastian Grimberg         break;
4932c7e7413SJeremy L Thompson       case CEED_RESTRICTION_POINTS:
4941249ccc5SJeremy L Thompson         CeedCallBackend(CeedElemRestrictionApplyAtPointsInElement_Ref_Core(rstr, num_comp, start, stop, t_mode, uu, vv));
4952c7e7413SJeremy L Thompson         break;
49694648b7dSSebastian Grimberg     }
4977c1dbaffSSebastian Grimberg   }
4987c1dbaffSSebastian Grimberg   CeedCallBackend(CeedVectorRestoreArrayRead(u, &uu));
4997c1dbaffSSebastian Grimberg   CeedCallBackend(CeedVectorRestoreArray(v, &vv));
5007c1dbaffSSebastian Grimberg   if (request != CEED_REQUEST_IMMEDIATE && request != CEED_REQUEST_ORDERED) *request = NULL;
5017c1dbaffSSebastian Grimberg   return CEED_ERROR_SUCCESS;
5027c1dbaffSSebastian Grimberg }
5037c1dbaffSSebastian Grimberg 
5047c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------
505f10650afSjeremylt // ElemRestriction Apply - Common Sizes
506f10650afSjeremylt //------------------------------------------------------------------------------
5071cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_110(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5087c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5097c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5101cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
511d979a051Sjeremylt }
512d979a051Sjeremylt 
5131cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_111(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5147c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5157c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5161cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
5174d2a38eeSjeremylt }
5184d2a38eeSjeremylt 
5191cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_180(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5207c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5217c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5221cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
5239c36149bSjeremylt }
5249c36149bSjeremylt 
5251cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_181(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5267c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5277c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5281cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 1, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
5299c36149bSjeremylt }
5309c36149bSjeremylt 
5311cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_310(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5327c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5337c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5341cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
535d979a051Sjeremylt }
536d979a051Sjeremylt 
5371cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_311(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5387c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5397c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5401cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
541d979a051Sjeremylt }
542d979a051Sjeremylt 
5431cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_380(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5447c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5457c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5461cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
547d979a051Sjeremylt }
548d979a051Sjeremylt 
5491cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_381(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5507c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5517c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5521cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 3, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
553d979a051Sjeremylt }
554d979a051Sjeremylt 
555bf4d1581Sjeremylt // LCOV_EXCL_START
5561cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_510(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5577c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5587c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5591cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
560d979a051Sjeremylt }
561bf4d1581Sjeremylt // LCOV_EXCL_STOP
562d979a051Sjeremylt 
5631cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_511(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5647c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5657c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5661cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 1, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
567d979a051Sjeremylt }
568d979a051Sjeremylt 
569bf4d1581Sjeremylt // LCOV_EXCL_START
5701cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_580(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5717c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5727c1dbaffSSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5731cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, comp_stride, start, stop, t_mode, use_signs, use_orients, u, v, request);
574d979a051Sjeremylt }
575bf4d1581Sjeremylt // LCOV_EXCL_STOP
576d979a051Sjeremylt 
5771cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref_581(CeedElemRestriction rstr, const CeedInt num_comp, const CeedInt block_size, const CeedInt comp_stride,
5787c1dbaffSSebastian Grimberg                                             CeedInt start, CeedInt stop, CeedTransposeMode t_mode, bool use_signs, bool use_orients, CeedVector u,
5790c73c039SSebastian Grimberg                                             CeedVector v, CeedRequest *request) {
5801cc2c60dSJeremy L Thompson   return CeedElemRestrictionApply_Ref_Core(rstr, 5, 8, 1, start, stop, t_mode, use_signs, use_orients, u, v, request);
5814d2a38eeSjeremylt }
5824d2a38eeSjeremylt 
583f10650afSjeremylt //------------------------------------------------------------------------------
584f10650afSjeremylt // ElemRestriction Apply
585f10650afSjeremylt //------------------------------------------------------------------------------
5861cc2c60dSJeremy L Thompson static int CeedElemRestrictionApply_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v, CeedRequest *request) {
587ad70ee2cSJeremy L Thompson   CeedInt                  num_block, block_size, num_comp, comp_stride;
588ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
589ad70ee2cSJeremy L Thompson 
5901cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
5911cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
5921cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
5931cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
5941cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
5951cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, true, true, u, v, request));
5965d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
597f30b1135SSebastian Grimberg }
598f30b1135SSebastian Grimberg 
599f30b1135SSebastian Grimberg //------------------------------------------------------------------------------
600f30b1135SSebastian Grimberg // ElemRestriction Apply Unsigned
601f30b1135SSebastian Grimberg //------------------------------------------------------------------------------
6021cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnsigned_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
6031cc2c60dSJeremy L Thompson                                                 CeedRequest *request) {
604ad70ee2cSJeremy L Thompson   CeedInt                  num_block, block_size, num_comp, comp_stride;
605ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
606ad70ee2cSJeremy L Thompson 
6071cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
6081cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
6091cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
6101cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
6111cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6121cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, true, u, v, request));
6135d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
6147c1dbaffSSebastian Grimberg }
6157c1dbaffSSebastian Grimberg 
6167c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------
6177c1dbaffSSebastian Grimberg // ElemRestriction Apply Unoriented
6187c1dbaffSSebastian Grimberg //------------------------------------------------------------------------------
6191cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyUnoriented_Ref(CeedElemRestriction rstr, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
6201cc2c60dSJeremy L Thompson                                                   CeedRequest *request) {
621ad70ee2cSJeremy L Thompson   CeedInt                  num_block, block_size, num_comp, comp_stride;
622ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
623ad70ee2cSJeremy L Thompson 
6241cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
6251cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
6261cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
6271cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
6281cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6291cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, 0, num_block, t_mode, false, false, u, v, request));
6305d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
6319c36149bSjeremylt }
632be9261b7Sjeremylt 
633f10650afSjeremylt //------------------------------------------------------------------------------
6342c7e7413SJeremy L Thompson // ElemRestriction Apply Points
6352c7e7413SJeremy L Thompson //------------------------------------------------------------------------------
636eda0adbcSSebastian Grimberg static int CeedElemRestrictionApplyAtPointsInElement_Ref(CeedElemRestriction rstr, CeedInt elem, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
6372c7e7413SJeremy L Thompson                                                          CeedRequest *request) {
63805fa913cSJeremy L Thompson   CeedInt                  num_comp;
6392c7e7413SJeremy L Thompson   CeedElemRestriction_Ref *impl;
6402c7e7413SJeremy L Thompson 
641eda0adbcSSebastian Grimberg   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
642eda0adbcSSebastian Grimberg   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
643eda0adbcSSebastian Grimberg   return impl->Apply(rstr, num_comp, 0, 1, elem, elem + 1, t_mode, false, false, u, v, request);
6442c7e7413SJeremy L Thompson }
6452c7e7413SJeremy L Thompson 
6462c7e7413SJeremy L Thompson //------------------------------------------------------------------------------
647f10650afSjeremylt // ElemRestriction Apply Block
648f10650afSjeremylt //------------------------------------------------------------------------------
6491cc2c60dSJeremy L Thompson static int CeedElemRestrictionApplyBlock_Ref(CeedElemRestriction rstr, CeedInt block, CeedTransposeMode t_mode, CeedVector u, CeedVector v,
650074cb416Sjeremylt                                              CeedRequest *request) {
651ad70ee2cSJeremy L Thompson   CeedInt                  block_size, num_comp, comp_stride;
652ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
653ad70ee2cSJeremy L Thompson 
6541cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
6551cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
6561cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
6571cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
6581cc2c60dSJeremy L Thompson   CeedCallBackend(impl->Apply(rstr, num_comp, block_size, comp_stride, block, block + 1, t_mode, true, true, u, v, request));
6595d10938bSJeremy L Thompson   return CEED_ERROR_SUCCESS;
6609c36149bSjeremylt }
661be9261b7Sjeremylt 
662f10650afSjeremylt //------------------------------------------------------------------------------
663bd33150aSjeremylt // ElemRestriction Get Offsets
664bd33150aSjeremylt //------------------------------------------------------------------------------
6652b730f8bSJeremy L Thompson static int CeedElemRestrictionGetOffsets_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt **offsets) {
666ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
667ad70ee2cSJeremy L Thompson 
668ad70ee2cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
669bd33150aSjeremylt 
6706e536b99SJeremy L Thompson   CeedCheck(mem_type == CEED_MEM_HOST, CeedElemRestrictionReturnCeed(rstr), CEED_ERROR_BACKEND, "Can only provide to HOST memory");
671bd33150aSjeremylt 
672bd33150aSjeremylt   *offsets = impl->offsets;
673e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
674bd33150aSjeremylt }
675bd33150aSjeremylt 
676bd33150aSjeremylt //------------------------------------------------------------------------------
67777d1c127SSebastian Grimberg // ElemRestriction Get Orientations
67877d1c127SSebastian Grimberg //------------------------------------------------------------------------------
67977d1c127SSebastian Grimberg static int CeedElemRestrictionGetOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const bool **orients) {
680ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
681ad70ee2cSJeremy L Thompson 
682ad70ee2cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
68377d1c127SSebastian Grimberg 
6846e536b99SJeremy L Thompson   CeedCheck(mem_type == CEED_MEM_HOST, CeedElemRestrictionReturnCeed(rstr), CEED_ERROR_BACKEND, "Can only provide to HOST memory");
68577d1c127SSebastian Grimberg 
68677d1c127SSebastian Grimberg   *orients = impl->orients;
68777d1c127SSebastian Grimberg   return CEED_ERROR_SUCCESS;
68877d1c127SSebastian Grimberg }
68977d1c127SSebastian Grimberg 
69077d1c127SSebastian Grimberg //------------------------------------------------------------------------------
69177d1c127SSebastian Grimberg // ElemRestriction Get Curl-Conforming Orientations
69277d1c127SSebastian Grimberg //------------------------------------------------------------------------------
6930c73c039SSebastian Grimberg static int CeedElemRestrictionGetCurlOrientations_Ref(CeedElemRestriction rstr, CeedMemType mem_type, const CeedInt8 **curl_orients) {
694ad70ee2cSJeremy L Thompson   CeedElemRestriction_Ref *impl;
695ad70ee2cSJeremy L Thompson 
696ad70ee2cSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
69777d1c127SSebastian Grimberg 
6986e536b99SJeremy L Thompson   CeedCheck(mem_type == CEED_MEM_HOST, CeedElemRestrictionReturnCeed(rstr), CEED_ERROR_BACKEND, "Can only provide to HOST memory");
69977d1c127SSebastian Grimberg 
70077d1c127SSebastian Grimberg   *curl_orients = impl->curl_orients;
70177d1c127SSebastian Grimberg   return CEED_ERROR_SUCCESS;
70277d1c127SSebastian Grimberg }
70377d1c127SSebastian Grimberg 
70477d1c127SSebastian Grimberg //------------------------------------------------------------------------------
705f10650afSjeremylt // ElemRestriction Destroy
706f10650afSjeremylt //------------------------------------------------------------------------------
7071cc2c60dSJeremy L Thompson static int CeedElemRestrictionDestroy_Ref(CeedElemRestriction rstr) {
708fe2413ffSjeremylt   CeedElemRestriction_Ref *impl;
70921617c04Sjeremylt 
7101cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetData(rstr, &impl));
711a267acd1SJeremy L Thompson   CeedCallBackend(CeedFree(&impl->offsets_owned));
712a267acd1SJeremy L Thompson   CeedCallBackend(CeedFree(&impl->orients_owned));
713a267acd1SJeremy L Thompson   CeedCallBackend(CeedFree(&impl->curl_orients_owned));
7142b730f8bSJeremy L Thompson   CeedCallBackend(CeedFree(&impl));
715e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
71621617c04Sjeremylt }
71721617c04Sjeremylt 
718f10650afSjeremylt //------------------------------------------------------------------------------
719f10650afSjeremylt // ElemRestriction Create
720f10650afSjeremylt //------------------------------------------------------------------------------
721fcbe8c06SSebastian Grimberg int CeedElemRestrictionCreate_Ref(CeedMemType mem_type, CeedCopyMode copy_mode, const CeedInt *offsets, const bool *orients,
7221cc2c60dSJeremy L Thompson                                   const CeedInt8 *curl_orients, CeedElemRestriction rstr) {
723ad70ee2cSJeremy L Thompson   Ceed                     ceed;
72407d5dec1SJeremy L Thompson   CeedInt                  num_elem, elem_size, num_block, block_size, num_comp, comp_stride, num_points = 0, num_offsets;
725ad70ee2cSJeremy L Thompson   CeedRestrictionType      rstr_type;
72621617c04Sjeremylt   CeedElemRestriction_Ref *impl;
727ad70ee2cSJeremy L Thompson 
7281cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCeed(rstr, &ceed));
7291cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumElements(rstr, &num_elem));
7301cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetElementSize(rstr, &elem_size));
7311cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumBlocks(rstr, &num_block));
7321cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetBlockSize(rstr, &block_size));
7331cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetNumComponents(rstr, &num_comp));
7341cc2c60dSJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetCompStride(rstr, &comp_stride));
73522eb1385SJeremy L Thompson   CeedCallBackend(CeedElemRestrictionGetType(rstr, &rstr_type));
73621617c04Sjeremylt 
7376574a04fSJeremy L Thompson   CeedCheck(mem_type == CEED_MEM_HOST, ceed, CEED_ERROR_BACKEND, "Only MemType = HOST supported");
738dce49693SSebastian Grimberg 
7392b730f8bSJeremy L Thompson   CeedCallBackend(CeedCalloc(1, &impl));
740dce49693SSebastian Grimberg   CeedCallBackend(CeedElemRestrictionSetData(rstr, impl));
74122eb1385SJeremy L Thompson 
74222eb1385SJeremy L Thompson   // Set layouts
74322eb1385SJeremy L Thompson   {
74422eb1385SJeremy L Thompson     bool    has_backend_strides;
74522eb1385SJeremy L Thompson     CeedInt layout[3] = {1, elem_size, elem_size * num_comp};
74622eb1385SJeremy L Thompson 
747dce49693SSebastian Grimberg     CeedCallBackend(CeedElemRestrictionSetELayout(rstr, layout));
74822eb1385SJeremy L Thompson     if (rstr_type == CEED_RESTRICTION_STRIDED) {
74922eb1385SJeremy L Thompson       CeedCallBackend(CeedElemRestrictionHasBackendStrides(rstr, &has_backend_strides));
75022eb1385SJeremy L Thompson       if (has_backend_strides) {
75122eb1385SJeremy L Thompson         CeedCallBackend(CeedElemRestrictionSetLLayout(rstr, layout));
75222eb1385SJeremy L Thompson       }
75322eb1385SJeremy L Thompson     }
75422eb1385SJeremy L Thompson   }
7553661185eSjeremylt 
756ff1bc20eSJeremy L Thompson   // Expand E-vector size for AtPoints
757ff1bc20eSJeremy L Thompson   if (rstr_type == CEED_RESTRICTION_POINTS) {
758ff1bc20eSJeremy L Thompson     CeedSize max_points = 0, num_points_total = 0;
759ff1bc20eSJeremy L Thompson 
760ff1bc20eSJeremy L Thompson     for (CeedInt i = 0; i < num_elem; i++) {
761ff1bc20eSJeremy L Thompson       CeedInt num_points = offsets[i + 1] - offsets[i];
762ff1bc20eSJeremy L Thompson 
763ff1bc20eSJeremy L Thompson       max_points = CeedIntMax(max_points, num_points);
764ff1bc20eSJeremy L Thompson       num_points_total += num_points;
765ff1bc20eSJeremy L Thompson     }
766ff1bc20eSJeremy L Thompson     // -- Increase size for last element
767ff1bc20eSJeremy L Thompson     num_points_total += (max_points - (offsets[num_elem] - offsets[num_elem - 1]));
768ff1bc20eSJeremy L Thompson     CeedCallBackend(CeedElemRestrictionSetAtPointsEVectorSize(rstr, num_points_total * num_comp));
769ff1bc20eSJeremy L Thompson   }
770ff1bc20eSJeremy L Thompson 
77192fe105eSJeremy L Thompson   // Offsets data
772fcbe8c06SSebastian Grimberg   if (rstr_type != CEED_RESTRICTION_STRIDED) {
7733661185eSjeremylt     const char *resource;
774ad70ee2cSJeremy L Thompson 
775ad70ee2cSJeremy L Thompson     // Check indices for ref or memcheck backends
77635aed383SJeremy L Thompson     {
7779bc66399SJeremy L Thompson       Ceed current = ceed, ceed_parent = NULL;
77835aed383SJeremy L Thompson 
7799bc66399SJeremy L Thompson       CeedCallBackend(CeedGetParent(current, &ceed_parent));
7809bc66399SJeremy L Thompson       CeedCallBackend(CeedGetResource(ceed_parent, &resource));
7819bc66399SJeremy L Thompson       CeedCallBackend(CeedDestroy(&ceed_parent));
78235aed383SJeremy L Thompson     }
7831ba74105SJeremy L Thompson     if (!strcmp(resource, "/cpu/self/ref/serial") || !strcmp(resource, "/cpu/self/ref/blocked")) {
784e79b91d9SJeremy L Thompson       CeedSize l_size;
7853661185eSjeremylt 
7861cc2c60dSJeremy L Thompson       CeedCallBackend(CeedElemRestrictionGetLVectorSize(rstr, &l_size));
7872b730f8bSJeremy L Thompson       for (CeedInt i = 0; i < num_elem * elem_size; i++) {
7886574a04fSJeremy L Thompson         CeedCheck(offsets[i] >= 0 && offsets[i] + (num_comp - 1) * comp_stride < l_size, ceed, CEED_ERROR_BACKEND,
7896574a04fSJeremy L Thompson                   "Restriction offset %" CeedInt_FMT " (%" CeedInt_FMT ") out of range [0, %" CeedInt_FMT "]", i, offsets[i], l_size);
7902b730f8bSJeremy L Thompson       }
7912b730f8bSJeremy L Thompson     }
7923661185eSjeremylt 
79392fe105eSJeremy L Thompson     // Copy data
79407d5dec1SJeremy L Thompson     if (rstr_type == CEED_RESTRICTION_POINTS) CeedCallBackend(CeedElemRestrictionGetNumPoints(rstr, &num_points));
79507d5dec1SJeremy L Thompson     num_offsets = rstr_type == CEED_RESTRICTION_POINTS ? (num_elem + 1 + num_points) : (num_elem * elem_size);
796f5d1e504SJeremy L Thompson     CeedCallBackend(CeedSetHostCeedIntArray(offsets, copy_mode, num_offsets, &impl->offsets_owned, &impl->offsets_borrowed, &impl->offsets));
797fcbe8c06SSebastian Grimberg 
798fcbe8c06SSebastian Grimberg     // Orientation data
799fcbe8c06SSebastian Grimberg     if (rstr_type == CEED_RESTRICTION_ORIENTED) {
8000305e208SSebastian Grimberg       CeedCheck(orients != NULL, ceed, CEED_ERROR_BACKEND, "No orients array provided for oriented restriction");
801f5d1e504SJeremy L Thompson       CeedCallBackend(CeedSetHostBoolArray(orients, copy_mode, num_offsets, &impl->orients_owned, &impl->orients_borrowed, &impl->orients));
802fcbe8c06SSebastian Grimberg     } else if (rstr_type == CEED_RESTRICTION_CURL_ORIENTED) {
8030305e208SSebastian Grimberg       CeedCheck(curl_orients != NULL, ceed, CEED_ERROR_BACKEND, "No curl_orients array provided for oriented restriction");
804f5d1e504SJeremy L Thompson       CeedCallBackend(CeedSetHostCeedInt8Array(curl_orients, copy_mode, 3 * num_offsets, &impl->curl_orients_owned, &impl->curl_orients_borrowed,
805f5d1e504SJeremy L Thompson                                                &impl->curl_orients));
806fcbe8c06SSebastian Grimberg     }
80792fe105eSJeremy L Thompson   }
808fe2413ffSjeremylt 
809ad70ee2cSJeremy L Thompson   // Set apply function based upon num_comp, block_size, and comp_stride
810ad70ee2cSJeremy L Thompson   CeedInt index = -1;
811ad70ee2cSJeremy L Thompson 
812ad70ee2cSJeremy L Thompson   if (block_size < 10) index = 100 * num_comp + 10 * block_size + (comp_stride == 1);
813ad70ee2cSJeremy L Thompson   switch (index) {
814d979a051Sjeremylt     case 110:
815d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_110;
816d979a051Sjeremylt       break;
817d979a051Sjeremylt     case 111:
818d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_111;
819d979a051Sjeremylt       break;
820d979a051Sjeremylt     case 180:
821d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_180;
822d979a051Sjeremylt       break;
823d979a051Sjeremylt     case 181:
824d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_181;
825d979a051Sjeremylt       break;
826d979a051Sjeremylt     case 310:
827d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_310;
828d979a051Sjeremylt       break;
829d979a051Sjeremylt     case 311:
830d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_311;
831d979a051Sjeremylt       break;
832d979a051Sjeremylt     case 380:
833d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_380;
834d979a051Sjeremylt       break;
835d979a051Sjeremylt     case 381:
836d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_381;
837d979a051Sjeremylt       break;
838bf4d1581Sjeremylt     // LCOV_EXCL_START
839d979a051Sjeremylt     case 510:
840d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_510;
841d979a051Sjeremylt       break;
842bf4d1581Sjeremylt     // LCOV_EXCL_STOP
843d979a051Sjeremylt     case 511:
844d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_511;
845d979a051Sjeremylt       break;
846bf4d1581Sjeremylt     // LCOV_EXCL_START
847d979a051Sjeremylt     case 580:
848d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_580;
849d979a051Sjeremylt       break;
850bf4d1581Sjeremylt     // LCOV_EXCL_STOP
851d979a051Sjeremylt     case 581:
852d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_581;
853d979a051Sjeremylt       break;
854d979a051Sjeremylt     default:
855d979a051Sjeremylt       impl->Apply = CeedElemRestrictionApply_Ref_Core;
856d979a051Sjeremylt       break;
857d979a051Sjeremylt   }
858dce49693SSebastian Grimberg 
859dce49693SSebastian Grimberg   // Register backend functions
860dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Apply", CeedElemRestrictionApply_Ref));
861dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnsigned", CeedElemRestrictionApplyUnsigned_Ref));
862dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyUnoriented", CeedElemRestrictionApplyUnoriented_Ref));
863dce49693SSebastian Grimberg   if (rstr_type == CEED_RESTRICTION_POINTS) {
864dce49693SSebastian Grimberg     CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyAtPointsInElement", CeedElemRestrictionApplyAtPointsInElement_Ref));
865dce49693SSebastian Grimberg   }
866dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "ApplyBlock", CeedElemRestrictionApplyBlock_Ref));
867dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOffsets", CeedElemRestrictionGetOffsets_Ref));
868dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetOrientations", CeedElemRestrictionGetOrientations_Ref));
869dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "GetCurlOrientations", CeedElemRestrictionGetCurlOrientations_Ref));
870dce49693SSebastian Grimberg   CeedCallBackend(CeedSetBackendFunction(ceed, "ElemRestriction", rstr, "Destroy", CeedElemRestrictionDestroy_Ref));
8719bc66399SJeremy L Thompson   CeedCallBackend(CeedDestroy(&ceed));
872e15f9bd0SJeremy L Thompson   return CEED_ERROR_SUCCESS;
87321617c04Sjeremylt }
874fc0567d9Srezgarshakeri 
875fc0567d9Srezgarshakeri //------------------------------------------------------------------------------
876