xref: /libCEED/backends/opt/ceed-opt-operator.c (revision 69a53589b8c3c82ae8bb5be56f1100c7d3468573) 
1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC.
2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707.
3 // All Rights reserved. See files LICENSE and NOTICE for details.
4 //
5 // This file is part of CEED, a collection of benchmarks, miniapps, software
6 // libraries and APIs for efficient high-order finite element and spectral
7 // element discretizations for exascale applications. For more information and
8 // source code availability see http://github.com/ceed.
9 //
10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
11 // a collaborative effort of two U.S. Department of Energy organizations (Office
12 // of Science and the National Nuclear Security Administration) responsible for
13 // the planning and preparation of a capable exascale ecosystem, including
14 // software, applications, hardware, advanced system engineering and early
15 // testbed platforms, in support of the nation's exascale computing imperative.
16 
17 #include <string.h>
18 #include "ceed-opt.h"
19 #include "../ref/ceed-ref.h"
20 
21 //------------------------------------------------------------------------------
22 // Setup Input/Output Fields
23 //------------------------------------------------------------------------------
24 static int CeedOperatorSetupFields_Opt(CeedQFunction qf, CeedOperator op,
25                                        bool inOrOut, const CeedInt blksize,
26                                        CeedElemRestriction *blkrestr,
27                                        CeedVector *fullevecs, CeedVector *evecs,
28                                        CeedVector *qvecs, CeedInt starte,
29                                        CeedInt numfields, CeedInt Q) {
30   CeedInt dim, ierr, ncomp, size, P;
31   Ceed ceed;
32   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
33   CeedBasis basis;
34   CeedElemRestriction r;
35   CeedOperatorField *opfields;
36   CeedQFunctionField *qffields;
37   if (inOrOut) {
38     ierr = CeedOperatorGetFields(op, NULL, &opfields);
39     CeedChk(ierr);
40     ierr = CeedQFunctionGetFields(qf, NULL, &qffields);
41     CeedChk(ierr);
42   } else {
43     ierr = CeedOperatorGetFields(op, &opfields, NULL);
44     CeedChk(ierr);
45     ierr = CeedQFunctionGetFields(qf, &qffields, NULL);
46     CeedChk(ierr);
47   }
48 
49   // Loop over fields
50   for (CeedInt i=0; i<numfields; i++) {
51     CeedEvalMode emode;
52     ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode); CeedChk(ierr);
53 
54     if (emode != CEED_EVAL_WEIGHT) {
55       ierr = CeedOperatorFieldGetElemRestriction(opfields[i], &r);
56       CeedChk(ierr);
57       CeedElemRestriction_Ref *data;
58       ierr = CeedElemRestrictionGetData(r, (void *)&data); CeedChk(ierr);
59       Ceed ceed;
60       ierr = CeedElemRestrictionGetCeed(r, &ceed); CeedChk(ierr);
61       CeedInt nelem, elemsize, nnodes;
62       CeedInterlaceMode imode;
63       ierr = CeedElemRestrictionGetNumElements(r, &nelem); CeedChk(ierr);
64       ierr = CeedElemRestrictionGetElementSize(r, &elemsize); CeedChk(ierr);
65       ierr = CeedElemRestrictionGetNumNodes(r, &nnodes); CeedChk(ierr);
66       ierr = CeedElemRestrictionGetNumComponents(r, &ncomp); CeedChk(ierr);
67       if (data->indices) {
68         ierr = CeedElemRestrictionGetIMode(r, &imode); CeedChk(ierr);
69         ierr = CeedElemRestrictionCreateBlocked(ceed, imode, nelem, elemsize,
70                                                 blksize, nnodes, ncomp,
71                                                 CEED_MEM_HOST, CEED_COPY_VALUES,
72                                                 data->indices,
73                                                 &blkrestr[i+starte]);
74         CeedChk(ierr);
75       } else {
76         CeedInt strides[3];
77         ierr = CeedElemRestrictionGetStrides(r, &strides); CeedChk(ierr);
78         ierr = CeedElemRestrictionCreateBlockedStrided(ceed, nelem, elemsize,
79                blksize, nnodes, ncomp, strides, &blkrestr[i+starte]);
80         CeedChk(ierr);
81       }
82       ierr = CeedElemRestrictionCreateVector(blkrestr[i+starte], NULL,
83                                              &fullevecs[i+starte]);
84       CeedChk(ierr);
85     }
86 
87     switch(emode) {
88     case CEED_EVAL_NONE:
89       ierr = CeedQFunctionFieldGetSize(qffields[i], &size); CeedChk(ierr);
90       ierr = CeedVectorCreate(ceed, Q*size*blksize, &evecs[i]); CeedChk(ierr);
91       ierr = CeedVectorCreate(ceed, Q*size*blksize, &qvecs[i]); CeedChk(ierr);
92       break;
93     case CEED_EVAL_INTERP:
94       ierr = CeedQFunctionFieldGetSize(qffields[i], &size); CeedChk(ierr);
95       ierr = CeedElemRestrictionGetElementSize(r, &P);
96       CeedChk(ierr);
97       ierr = CeedVectorCreate(ceed, P*size*blksize, &evecs[i]); CeedChk(ierr);
98       ierr = CeedVectorCreate(ceed, Q*size*blksize, &qvecs[i]); CeedChk(ierr);
99       break;
100     case CEED_EVAL_GRAD:
101       ierr = CeedOperatorFieldGetBasis(opfields[i], &basis); CeedChk(ierr);
102       ierr = CeedQFunctionFieldGetSize(qffields[i], &size); CeedChk(ierr);
103       ierr = CeedBasisGetDimension(basis, &dim); CeedChk(ierr);
104       ierr = CeedElemRestrictionGetElementSize(r, &P);
105       CeedChk(ierr);
106       ierr = CeedVectorCreate(ceed, P*size/dim*blksize, &evecs[i]); CeedChk(ierr);
107       ierr = CeedVectorCreate(ceed, Q*size*blksize, &qvecs[i]); CeedChk(ierr);
108       break;
109     case CEED_EVAL_WEIGHT: // Only on input fields
110       ierr = CeedOperatorFieldGetBasis(opfields[i], &basis); CeedChk(ierr);
111       ierr = CeedVectorCreate(ceed, Q*blksize, &qvecs[i]); CeedChk(ierr);
112       ierr = CeedBasisApply(basis, blksize, CEED_NOTRANSPOSE,
113                             CEED_EVAL_WEIGHT, CEED_VECTOR_NONE, qvecs[i]);
114       CeedChk(ierr);
115 
116       break;
117     case CEED_EVAL_DIV:
118       break; // Not implimented
119     case CEED_EVAL_CURL:
120       break; // Not implimented
121     }
122   }
123   return 0;
124 }
125 
126 //------------------------------------------------------------------------------
127 // Setup Operator
128 //------------------------------------------------------------------------------
129 static int CeedOperatorSetup_Opt(CeedOperator op) {
130   int ierr;
131   bool setupdone;
132   ierr = CeedOperatorGetSetupStatus(op, &setupdone); CeedChk(ierr);
133   if (setupdone) return 0;
134   Ceed ceed;
135   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
136   Ceed_Opt *ceedimpl;
137   ierr = CeedGetData(ceed, (void *)&ceedimpl); CeedChk(ierr);
138   const CeedInt blksize = ceedimpl->blksize;
139   CeedOperator_Opt *impl;
140   ierr = CeedOperatorGetData(op, (void *)&impl); CeedChk(ierr);
141   CeedQFunction qf;
142   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
143   CeedInt Q, numinputfields, numoutputfields;
144   ierr = CeedOperatorGetNumQuadraturePoints(op, &Q); CeedChk(ierr);
145   ierr = CeedQFunctionGetIdentityStatus(qf, &impl->identityqf); CeedChk(ierr);
146   ierr= CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
147   CeedChk(ierr);
148   CeedOperatorField *opinputfields, *opoutputfields;
149   ierr = CeedOperatorGetFields(op, &opinputfields, &opoutputfields);
150   CeedChk(ierr);
151   CeedQFunctionField *qfinputfields, *qfoutputfields;
152   ierr = CeedQFunctionGetFields(qf, &qfinputfields, &qfoutputfields);
153   CeedChk(ierr);
154 
155   // Allocate
156   ierr = CeedCalloc(numinputfields + numoutputfields, &impl->blkrestr);
157   CeedChk(ierr);
158   ierr = CeedCalloc(numinputfields + numoutputfields, &impl->evecs);
159   CeedChk(ierr);
160   ierr = CeedCalloc(numinputfields + numoutputfields, &impl->edata);
161   CeedChk(ierr);
162 
163   ierr = CeedCalloc(16, &impl->inputstate); CeedChk(ierr);
164   ierr = CeedCalloc(16, &impl->evecsin); CeedChk(ierr);
165   ierr = CeedCalloc(16, &impl->evecsout); CeedChk(ierr);
166   ierr = CeedCalloc(16, &impl->qvecsin); CeedChk(ierr);
167   ierr = CeedCalloc(16, &impl->qvecsout); CeedChk(ierr);
168 
169   impl->numein = numinputfields; impl->numeout = numoutputfields;
170 
171   // Set up infield and outfield pointer arrays
172   // Infields
173   ierr = CeedOperatorSetupFields_Opt(qf, op, 0, blksize, impl->blkrestr,
174                                      impl->evecs, impl->evecsin,
175                                      impl->qvecsin, 0,
176                                      numinputfields, Q);
177   CeedChk(ierr);
178   // Outfields
179   ierr = CeedOperatorSetupFields_Opt(qf, op, 1, blksize, impl->blkrestr,
180                                      impl->evecs, impl->evecsout,
181                                      impl->qvecsout, numinputfields,
182                                      numoutputfields, Q);
183   CeedChk(ierr);
184 
185   // Identity QFunctions
186   if (impl->identityqf) {
187     CeedEvalMode inmode, outmode;
188     CeedQFunctionField *infields, *outfields;
189     ierr = CeedQFunctionGetFields(qf, &infields, &outfields); CeedChk(ierr);
190 
191     for (CeedInt i=0; i<numinputfields; i++) {
192       ierr = CeedQFunctionFieldGetEvalMode(infields[i], &inmode);
193       CeedChk(ierr);
194       ierr = CeedQFunctionFieldGetEvalMode(outfields[i], &outmode);
195       CeedChk(ierr);
196 
197       ierr = CeedVectorDestroy(&impl->qvecsout[i]); CeedChk(ierr);
198       impl->qvecsout[i] = impl->qvecsin[i];
199       ierr = CeedVectorAddReference(impl->qvecsin[i]); CeedChk(ierr);
200     }
201   }
202 
203   ierr = CeedOperatorSetSetupDone(op); CeedChk(ierr);
204 
205   return 0;
206 }
207 
208 //------------------------------------------------------------------------------
209 // Setup Input Fields
210 //------------------------------------------------------------------------------
211 static inline int CeedOperatorSetupInputs_Opt(CeedInt numinputfields,
212     CeedQFunctionField *qfinputfields, CeedOperatorField *opinputfields,
213     CeedVector invec, CeedOperator_Opt *impl, CeedRequest *request) {
214   CeedInt ierr;
215   CeedEvalMode emode;
216   CeedVector vec;
217   uint64_t state;
218 
219   for (CeedInt i=0; i<numinputfields; i++) {
220     ierr = CeedQFunctionFieldGetEvalMode(qfinputfields[i], &emode);
221     CeedChk(ierr);
222     if (emode == CEED_EVAL_WEIGHT) { // Skip
223     } else {
224       // Get input vector
225       ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
226       if (vec != CEED_VECTOR_ACTIVE) {
227         // Restrict
228         ierr = CeedVectorGetState(vec, &state); CeedChk(ierr);
229         if (state != impl->inputstate[i]) {
230           ierr = CeedElemRestrictionApply(impl->blkrestr[i], CEED_NOTRANSPOSE,
231                                           vec, impl->evecs[i], request);
232           CeedChk(ierr);
233           impl->inputstate[i] = state;
234         }
235       } else {
236         // Set Qvec for CEED_EVAL_NONE
237         if (emode == CEED_EVAL_NONE) {
238           ierr = CeedVectorGetArray(impl->evecsin[i], CEED_MEM_HOST,
239                                     &impl->edata[i]); CeedChk(ierr);
240           ierr = CeedVectorSetArray(impl->qvecsin[i], CEED_MEM_HOST,
241                                     CEED_USE_POINTER,
242                                     impl->edata[i]); CeedChk(ierr);
243           ierr = CeedVectorRestoreArray(impl->evecsin[i],
244                                         &impl->edata[i]); CeedChk(ierr);
245         }
246       }
247       // Get evec
248       ierr = CeedVectorGetArrayRead(impl->evecs[i], CEED_MEM_HOST,
249                                     (const CeedScalar **) &impl->edata[i]);
250       CeedChk(ierr);
251     }
252   }
253   return 0;
254 }
255 
256 //------------------------------------------------------------------------------
257 // Input Basis Action
258 //------------------------------------------------------------------------------
259 static inline int CeedOperatorInputBasis_Opt(CeedInt e, CeedInt Q,
260     CeedQFunctionField *qfinputfields, CeedOperatorField *opinputfields,
261     CeedInt numinputfields, CeedInt blksize, CeedVector invec, bool skipactive,
262     CeedOperator_Opt *impl, CeedRequest *request) {
263   CeedInt ierr;
264   CeedInt dim, elemsize, size;
265   CeedElemRestriction Erestrict;
266   CeedEvalMode emode;
267   CeedBasis basis;
268   CeedVector vec;
269 
270   for (CeedInt i=0; i<numinputfields; i++) {
271     ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
272     // Skip active input
273     if (skipactive) {
274       if (vec == CEED_VECTOR_ACTIVE)
275         continue;
276     }
277 
278     CeedInt activein = 0;
279     // Get elemsize, emode, size
280     ierr = CeedOperatorFieldGetElemRestriction(opinputfields[i], &Erestrict);
281     CeedChk(ierr);
282     ierr = CeedElemRestrictionGetElementSize(Erestrict, &elemsize);
283     CeedChk(ierr);
284     ierr = CeedQFunctionFieldGetEvalMode(qfinputfields[i], &emode);
285     CeedChk(ierr);
286     ierr = CeedQFunctionFieldGetSize(qfinputfields[i], &size); CeedChk(ierr);
287     // Restrict block active input
288     if (vec == CEED_VECTOR_ACTIVE) {
289       ierr = CeedElemRestrictionApplyBlock(impl->blkrestr[i], e/blksize,
290                                            CEED_NOTRANSPOSE, invec,
291                                            impl->evecsin[i], request);
292       CeedChk(ierr);
293       activein = 1;
294     }
295     // Basis action
296     switch(emode) {
297     case CEED_EVAL_NONE:
298       if (!activein) {
299         ierr = CeedVectorSetArray(impl->qvecsin[i], CEED_MEM_HOST,
300                                   CEED_USE_POINTER,
301                                   &impl->edata[i][e*Q*size]); CeedChk(ierr);
302       }
303       break;
304     case CEED_EVAL_INTERP:
305       ierr = CeedOperatorFieldGetBasis(opinputfields[i], &basis);
306       CeedChk(ierr);
307       if (!activein) {
308         ierr = CeedVectorSetArray(impl->evecsin[i], CEED_MEM_HOST,
309                                   CEED_USE_POINTER,
310                                   &impl->edata[i][e*elemsize*size]);
311         CeedChk(ierr);
312       }
313       ierr = CeedBasisApply(basis, blksize, CEED_NOTRANSPOSE,
314                             CEED_EVAL_INTERP, impl->evecsin[i],
315                             impl->qvecsin[i]); CeedChk(ierr);
316       break;
317     case CEED_EVAL_GRAD:
318       ierr = CeedOperatorFieldGetBasis(opinputfields[i], &basis);
319       CeedChk(ierr);
320       if (!activein) {
321         ierr = CeedBasisGetDimension(basis, &dim); CeedChk(ierr);
322         ierr = CeedVectorSetArray(impl->evecsin[i], CEED_MEM_HOST,
323                                   CEED_USE_POINTER,
324                                   &impl->edata[i][e*elemsize*size/dim]);
325         CeedChk(ierr);
326       }
327       ierr = CeedBasisApply(basis, blksize, CEED_NOTRANSPOSE,
328                             CEED_EVAL_GRAD, impl->evecsin[i],
329                             impl->qvecsin[i]); CeedChk(ierr);
330       break;
331     case CEED_EVAL_WEIGHT:
332       break;  // No action
333     case CEED_EVAL_DIV:
334     case CEED_EVAL_CURL: {
335       // LCOV_EXCL_START
336       ierr = CeedOperatorFieldGetBasis(opinputfields[i], &basis);
337       CeedChk(ierr);
338       Ceed ceed;
339       ierr = CeedBasisGetCeed(basis, &ceed); CeedChk(ierr);
340       return CeedError(ceed, 1, "Ceed evaluation mode not implemented");
341       // LCOV_EXCL_STOP
342       break; // Not implemented
343     }
344     }
345   }
346   return 0;
347 }
348 
349 //------------------------------------------------------------------------------
350 // Output Basis Action
351 //------------------------------------------------------------------------------
352 static inline int CeedOperatorOutputBasis_Opt(CeedInt e, CeedInt Q,
353     CeedQFunctionField *qfoutputfields, CeedOperatorField *opoutputfields,
354     CeedInt blksize, CeedInt numinputfields, CeedInt numoutputfields,
355     CeedOperator op, CeedVector outvec, CeedOperator_Opt *impl,
356     CeedRequest *request) {
357   CeedInt ierr;
358   CeedElemRestriction Erestrict;
359   CeedEvalMode emode;
360   CeedBasis basis;
361   CeedVector vec;
362 
363   for (CeedInt i=0; i<numoutputfields; i++) {
364     // Get elemsize, emode, size
365     ierr = CeedOperatorFieldGetElemRestriction(opoutputfields[i], &Erestrict);
366     CeedChk(ierr);
367     ierr = CeedQFunctionFieldGetEvalMode(qfoutputfields[i], &emode);
368     CeedChk(ierr);
369     // Basis action
370     switch(emode) {
371     case CEED_EVAL_NONE:
372       break; // No action
373     case CEED_EVAL_INTERP:
374       ierr = CeedOperatorFieldGetBasis(opoutputfields[i], &basis);
375       CeedChk(ierr);
376       ierr = CeedBasisApply(basis, blksize, CEED_TRANSPOSE,
377                             CEED_EVAL_INTERP, impl->qvecsout[i],
378                             impl->evecsout[i]); CeedChk(ierr);
379       break;
380     case CEED_EVAL_GRAD:
381       ierr = CeedOperatorFieldGetBasis(opoutputfields[i], &basis);
382       CeedChk(ierr);
383       ierr = CeedBasisApply(basis, blksize, CEED_TRANSPOSE,
384                             CEED_EVAL_GRAD, impl->qvecsout[i],
385                             impl->evecsout[i]); CeedChk(ierr);
386       break;
387     case CEED_EVAL_WEIGHT: {
388       // LCOV_EXCL_START
389       Ceed ceed;
390       ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
391       return CeedError(ceed, 1, "CEED_EVAL_WEIGHT cannot be an output "
392                        "evaluation mode");
393       // LCOV_EXCL_STOP
394       break; // Should not occur
395     }
396     case CEED_EVAL_DIV:
397     case CEED_EVAL_CURL: {
398       // LCOV_EXCL_START
399       Ceed ceed;
400       ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
401       return CeedError(ceed, 1, "Ceed evaluation mode not implemented");
402       // LCOV_EXCL_STOP
403       break; // Not implemented
404     }
405     }
406     // Restrict output block
407     // Get output vector
408     ierr = CeedOperatorFieldGetVector(opoutputfields[i], &vec); CeedChk(ierr);
409     if (vec == CEED_VECTOR_ACTIVE)
410       vec = outvec;
411     // Restrict
412     ierr = CeedElemRestrictionApplyBlock(impl->blkrestr[i+impl->numein],
413                                          e/blksize, CEED_TRANSPOSE,
414                                          impl->evecsout[i], vec, request);
415     CeedChk(ierr);
416   }
417   return 0;
418 }
419 
420 //------------------------------------------------------------------------------
421 // Restore Input Vectors
422 //------------------------------------------------------------------------------
423 static inline int CeedOperatorRestoreInputs_Opt(CeedInt numinputfields,
424     CeedQFunctionField *qfinputfields, CeedOperatorField *opinputfields,
425     bool skipactive, CeedOperator_Opt *impl) {
426   CeedInt ierr;
427   CeedEvalMode emode;
428 
429   for (CeedInt i=0; i<numinputfields; i++) {
430     // Skip active inputs
431     if (skipactive) {
432       CeedVector vec;
433       ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
434       if (vec == CEED_VECTOR_ACTIVE)
435         continue;
436     }
437     ierr = CeedQFunctionFieldGetEvalMode(qfinputfields[i], &emode);
438     CeedChk(ierr);
439     if (emode == CEED_EVAL_WEIGHT) { // Skip
440     } else {
441       ierr = CeedVectorRestoreArrayRead(impl->evecs[i],
442                                         (const CeedScalar **) &impl->edata[i]);
443       CeedChk(ierr);
444     }
445   }
446   return 0;
447 }
448 
449 //------------------------------------------------------------------------------
450 // Operator Apply
451 //------------------------------------------------------------------------------
452 static int CeedOperatorApply_Opt(CeedOperator op, CeedVector invec,
453                                  CeedVector outvec, CeedRequest *request) {
454   int ierr;
455   Ceed ceed;
456   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
457   Ceed_Opt *ceedimpl;
458   ierr = CeedGetData(ceed, (void *)&ceedimpl); CeedChk(ierr);
459   CeedInt blksize = ceedimpl->blksize;
460   CeedOperator_Opt *impl;
461   ierr = CeedOperatorGetData(op, (void *)&impl); CeedChk(ierr);
462   CeedInt Q, numinputfields, numoutputfields, numelements;
463   ierr = CeedOperatorGetNumElements(op, &numelements); CeedChk(ierr);
464   ierr = CeedOperatorGetNumQuadraturePoints(op, &Q); CeedChk(ierr);
465   CeedInt nblks = (numelements/blksize) + !!(numelements%blksize);
466   CeedQFunction qf;
467   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
468   ierr= CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
469   CeedChk(ierr);
470   CeedOperatorField *opinputfields, *opoutputfields;
471   ierr = CeedOperatorGetFields(op, &opinputfields, &opoutputfields);
472   CeedChk(ierr);
473   CeedQFunctionField *qfinputfields, *qfoutputfields;
474   ierr = CeedQFunctionGetFields(qf, &qfinputfields, &qfoutputfields);
475   CeedChk(ierr);
476   CeedEvalMode emode;
477 
478   // Setup
479   ierr = CeedOperatorSetup_Opt(op); CeedChk(ierr);
480 
481   // Input Evecs and Restriction
482   ierr = CeedOperatorSetupInputs_Opt(numinputfields, qfinputfields,
483                                      opinputfields, invec, impl, request);
484   CeedChk(ierr);
485 
486   // Output Lvecs, Evecs, and Qvecs
487   for (CeedInt i=0; i<numoutputfields; i++) {
488     // Set Qvec if needed
489     ierr = CeedQFunctionFieldGetEvalMode(qfoutputfields[i], &emode);
490     CeedChk(ierr);
491     if (emode == CEED_EVAL_NONE) {
492       // Set qvec to single block evec
493       ierr = CeedVectorGetArray(impl->evecsout[i], CEED_MEM_HOST,
494                                 &impl->edata[i + numinputfields]);
495       CeedChk(ierr);
496       ierr = CeedVectorSetArray(impl->qvecsout[i], CEED_MEM_HOST,
497                                 CEED_USE_POINTER,
498                                 impl->edata[i + numinputfields]); CeedChk(ierr);
499       ierr = CeedVectorRestoreArray(impl->evecsout[i],
500                                     &impl->edata[i + numinputfields]);
501       CeedChk(ierr);
502     }
503   }
504 
505   // Loop through elements
506   for (CeedInt e=0; e<nblks*blksize; e+=blksize) {
507     // Input basis apply
508     ierr = CeedOperatorInputBasis_Opt(e, Q, qfinputfields, opinputfields,
509                                       numinputfields, blksize, invec, false,
510                                       impl, request); CeedChk(ierr);
511 
512     // Q function
513     if (!impl->identityqf) {
514       ierr = CeedQFunctionApply(qf, Q*blksize, impl->qvecsin, impl->qvecsout);
515       CeedChk(ierr);
516     }
517 
518     // Output basis apply and restrict
519     ierr = CeedOperatorOutputBasis_Opt(e, Q, qfoutputfields, opoutputfields,
520                                        blksize, numinputfields, numoutputfields,
521                                        op, outvec, impl, request);
522     CeedChk(ierr);
523   }
524 
525   // Restore input arrays
526   ierr = CeedOperatorRestoreInputs_Opt(numinputfields, qfinputfields,
527                                        opinputfields, false, impl);
528   CeedChk(ierr);
529 
530   return 0;
531 }
532 
533 //------------------------------------------------------------------------------
534 // Assemble Linear QFunction
535 //------------------------------------------------------------------------------
536 static int CeedOperatorAssembleLinearQFunction_Opt(CeedOperator op,
537     CeedVector *assembled, CeedElemRestriction *rstr, CeedRequest *request) {
538   int ierr;
539   Ceed ceed;
540   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
541   Ceed_Opt *ceedimpl;
542   ierr = CeedGetData(ceed, (void *)&ceedimpl); CeedChk(ierr);
543   const CeedInt blksize = ceedimpl->blksize;
544   CeedOperator_Opt *impl;
545   ierr = CeedOperatorGetData(op, (void *)&impl); CeedChk(ierr);
546   CeedInt Q, numinputfields, numoutputfields, numelements, size;
547   ierr = CeedOperatorGetNumElements(op, &numelements); CeedChk(ierr);
548   ierr = CeedOperatorGetNumQuadraturePoints(op, &Q); CeedChk(ierr);
549   CeedInt nblks = (numelements/blksize) + !!(numelements%blksize);
550   CeedQFunction qf;
551   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
552   ierr= CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
553   CeedChk(ierr);
554   CeedOperatorField *opinputfields, *opoutputfields;
555   ierr = CeedOperatorGetFields(op, &opinputfields, &opoutputfields);
556   CeedChk(ierr);
557   CeedQFunctionField *qfinputfields, *qfoutputfields;
558   ierr = CeedQFunctionGetFields(qf, &qfinputfields, &qfoutputfields);
559   CeedChk(ierr);
560   CeedVector vec, lvec;
561   CeedInt numactivein = 0, numactiveout = 0;
562   CeedVector *activein = NULL;
563   CeedScalar *a, *tmp;
564 
565   // Setup
566   ierr = CeedOperatorSetup_Opt(op); CeedChk(ierr);
567 
568   // Check for identity
569   if (impl->identityqf)
570     // LCOV_EXCL_START
571     return CeedError(ceed, 1, "Assembling identity qfunctions not supported");
572   // LCOV_EXCL_STOP
573 
574   // Input Evecs and Restriction
575   ierr = CeedOperatorSetupInputs_Opt(numinputfields, qfinputfields,
576                                      opinputfields, NULL, impl, request);
577   CeedChk(ierr);
578 
579   // Count number of active input fields
580   for (CeedInt i=0; i<numinputfields; i++) {
581     // Get input vector
582     ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
583     // Check if active input
584     if (vec == CEED_VECTOR_ACTIVE) {
585       ierr = CeedQFunctionFieldGetSize(qfinputfields[i], &size); CeedChk(ierr);
586       ierr = CeedVectorSetValue(impl->qvecsin[i], 0.0); CeedChk(ierr);
587       ierr = CeedVectorGetArray(impl->qvecsin[i], CEED_MEM_HOST, &tmp);
588       CeedChk(ierr);
589       ierr = CeedRealloc(numactivein + size, &activein); CeedChk(ierr);
590       for (CeedInt field=0; field<size; field++) {
591         ierr = CeedVectorCreate(ceed, Q*blksize, &activein[numactivein+field]);
592         CeedChk(ierr);
593         ierr = CeedVectorSetArray(activein[numactivein+field], CEED_MEM_HOST,
594                                   CEED_USE_POINTER, &tmp[field*Q*blksize]);
595         CeedChk(ierr);
596       }
597       numactivein += size;
598       ierr = CeedVectorRestoreArray(impl->qvecsin[i], &tmp); CeedChk(ierr);
599     }
600   }
601 
602   // Count number of active output fields
603   for (CeedInt i=0; i<numoutputfields; i++) {
604     // Get output vector
605     ierr = CeedOperatorFieldGetVector(opoutputfields[i], &vec); CeedChk(ierr);
606     // Check if active output
607     if (vec == CEED_VECTOR_ACTIVE) {
608       ierr = CeedQFunctionFieldGetSize(qfoutputfields[i], &size); CeedChk(ierr);
609       numactiveout += size;
610     }
611   }
612 
613   // Check sizes
614   if (!numactivein || !numactiveout)
615     // LCOV_EXCL_START
616     return CeedError(ceed, 1, "Cannot assemble QFunction without active inputs "
617                      "and outputs");
618   // LCOV_EXCL_STOP
619 
620   // Setup lvec
621   ierr = CeedVectorCreate(ceed, nblks*blksize*Q*numactivein*numactiveout,
622                           &lvec); CeedChk(ierr);
623   ierr = CeedVectorGetArray(lvec, CEED_MEM_HOST, &a); CeedChk(ierr);
624 
625   // Create output restriction
626   CeedInt strides[3] = {1, Q, numactivein *numactiveout*Q};
627   ierr = CeedElemRestrictionCreateStrided(ceed, numelements, Q,
628                                           numelements*Q, numactivein*numactiveout, strides, rstr); CeedChk(ierr);
629   // Create assembled vector
630   ierr = CeedVectorCreate(ceed, numelements*Q*numactivein*numactiveout,
631                           assembled); CeedChk(ierr);
632 
633   // Loop through elements
634   for (CeedInt e=0; e<nblks*blksize; e+=blksize) {
635     // Input basis apply
636     ierr = CeedOperatorInputBasis_Opt(e, Q, qfinputfields, opinputfields,
637                                       numinputfields, blksize, NULL, true,
638                                       impl, request); CeedChk(ierr);
639 
640     // Assemble QFunction
641     for (CeedInt in=0; in<numactivein; in++) {
642       // Set Inputs
643       ierr = CeedVectorSetValue(activein[in], 1.0); CeedChk(ierr);
644       if (numactivein > 1) {
645         ierr = CeedVectorSetValue(activein[(in+numactivein-1)%numactivein],
646                                   0.0); CeedChk(ierr);
647       }
648       // Set Outputs
649       for (CeedInt out=0; out<numoutputfields; out++) {
650         // Get output vector
651         ierr = CeedOperatorFieldGetVector(opoutputfields[out], &vec);
652         CeedChk(ierr);
653         // Check if active output
654         if (vec == CEED_VECTOR_ACTIVE) {
655           CeedVectorSetArray(impl->qvecsout[out], CEED_MEM_HOST,
656                              CEED_USE_POINTER, a); CeedChk(ierr);
657           ierr = CeedQFunctionFieldGetSize(qfoutputfields[out], &size);
658           CeedChk(ierr);
659           a += size*Q*blksize; // Advance the pointer by the size of the output
660         }
661       }
662       // Apply QFunction
663       ierr = CeedQFunctionApply(qf, Q*blksize, impl->qvecsin, impl->qvecsout);
664       CeedChk(ierr);
665     }
666   }
667 
668   // Un-set output Qvecs to prevent accidental overwrite of Assembled
669   for (CeedInt out=0; out<numoutputfields; out++) {
670     // Get output vector
671     ierr = CeedOperatorFieldGetVector(opoutputfields[out], &vec);
672     CeedChk(ierr);
673     // Check if active output
674     if (vec == CEED_VECTOR_ACTIVE) {
675       CeedVectorSetArray(impl->qvecsout[out], CEED_MEM_HOST, CEED_COPY_VALUES,
676                          NULL); CeedChk(ierr);
677     }
678   }
679 
680   // Restore input arrays
681   ierr = CeedOperatorRestoreInputs_Opt(numinputfields, qfinputfields,
682                                        opinputfields, true, impl);
683   CeedChk(ierr);
684 
685   // Output blocked restriction
686   ierr = CeedVectorRestoreArray(lvec, &a); CeedChk(ierr);
687   ierr = CeedVectorSetValue(*assembled, 0.0); CeedChk(ierr);
688   CeedElemRestriction blkrstr;
689   ierr = CeedElemRestrictionCreateBlockedStrided(ceed, numelements, Q, blksize,
690          numelements*Q,
691          numactivein*numactiveout,
692          strides, &blkrstr);
693   CeedChk(ierr);
694   ierr = CeedElemRestrictionApply(blkrstr, CEED_TRANSPOSE, lvec, *assembled,
695                                   request); CeedChk(ierr);
696 
697   // Cleanup
698   for (CeedInt i=0; i<numactivein; i++) {
699     ierr = CeedVectorDestroy(&activein[i]); CeedChk(ierr);
700   }
701   ierr = CeedFree(&activein); CeedChk(ierr);
702   ierr = CeedVectorDestroy(&lvec); CeedChk(ierr);
703   ierr = CeedElemRestrictionDestroy(&blkrstr); CeedChk(ierr);
704 
705   return 0;
706 }
707 
708 //------------------------------------------------------------------------------
709 // Operator Destroy
710 //------------------------------------------------------------------------------
711 static int CeedOperatorDestroy_Opt(CeedOperator op) {
712   int ierr;
713   CeedOperator_Opt *impl;
714   ierr = CeedOperatorGetData(op, (void *)&impl); CeedChk(ierr);
715 
716   for (CeedInt i=0; i<impl->numein+impl->numeout; i++) {
717     ierr = CeedElemRestrictionDestroy(&impl->blkrestr[i]); CeedChk(ierr);
718     ierr = CeedVectorDestroy(&impl->evecs[i]); CeedChk(ierr);
719   }
720   ierr = CeedFree(&impl->blkrestr); CeedChk(ierr);
721   ierr = CeedFree(&impl->evecs); CeedChk(ierr);
722   ierr = CeedFree(&impl->edata); CeedChk(ierr);
723   ierr = CeedFree(&impl->inputstate); CeedChk(ierr);
724 
725   for (CeedInt i=0; i<impl->numein; i++) {
726     ierr = CeedVectorDestroy(&impl->evecsin[i]); CeedChk(ierr);
727     ierr = CeedVectorDestroy(&impl->qvecsin[i]); CeedChk(ierr);
728   }
729   ierr = CeedFree(&impl->evecsin); CeedChk(ierr);
730   ierr = CeedFree(&impl->qvecsin); CeedChk(ierr);
731 
732   for (CeedInt i=0; i<impl->numeout; i++) {
733     ierr = CeedVectorDestroy(&impl->evecsout[i]); CeedChk(ierr);
734     ierr = CeedVectorDestroy(&impl->qvecsout[i]); CeedChk(ierr);
735   }
736   ierr = CeedFree(&impl->evecsout); CeedChk(ierr);
737   ierr = CeedFree(&impl->qvecsout); CeedChk(ierr);
738 
739   ierr = CeedFree(&impl); CeedChk(ierr);
740   return 0;
741 }
742 
743 //------------------------------------------------------------------------------
744 // Operator Create
745 //------------------------------------------------------------------------------
746 int CeedOperatorCreate_Opt(CeedOperator op) {
747   int ierr;
748   Ceed ceed;
749   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
750   Ceed_Opt *ceedimpl;
751   ierr = CeedGetData(ceed, (void *)&ceedimpl); CeedChk(ierr);
752   CeedInt blksize = ceedimpl->blksize;
753   CeedOperator_Opt *impl;
754 
755   ierr = CeedCalloc(1, &impl); CeedChk(ierr);
756   ierr = CeedOperatorSetData(op, (void *)&impl); CeedChk(ierr);
757 
758   if (blksize != 1 && blksize != 8)
759     // LCOV_EXCL_START
760     return CeedError(ceed, 1, "Opt backend cannot use blocksize: %d", blksize);
761   // LCOV_EXCL_STOP
762 
763   ierr = CeedSetBackendFunction(ceed, "Operator", op, "AssembleLinearQFunction",
764                                 CeedOperatorAssembleLinearQFunction_Opt);
765   CeedChk(ierr);
766   ierr = CeedSetBackendFunction(ceed, "Operator", op, "ApplyAdd",
767                                 CeedOperatorApply_Opt); CeedChk(ierr);
768   ierr = CeedSetBackendFunction(ceed, "Operator", op, "Destroy",
769                                 CeedOperatorDestroy_Opt); CeedChk(ierr);
770   return 0;
771 }
772 //------------------------------------------------------------------------------
773