xref: /libCEED/backends/opt/ceed-opt-operator.c (revision 500adadc1e280b04511cad79cd7fe6f51a8b5aa4) 
1 // Copyright (c) 2017-2018, Lawrence Livermore National Security, LLC.
2 // Produced at the Lawrence Livermore National Laboratory. LLNL-CODE-734707.
3 // All Rights reserved. See files LICENSE and NOTICE for details.
4 //
5 // This file is part of CEED, a collection of benchmarks, miniapps, software
6 // libraries and APIs for efficient high-order finite element and spectral
7 // element discretizations for exascale applications. For more information and
8 // source code availability see http://github.com/ceed.
9 //
10 // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC,
11 // a collaborative effort of two U.S. Department of Energy organizations (Office
12 // of Science and the National Nuclear Security Administration) responsible for
13 // the planning and preparation of a capable exascale ecosystem, including
14 // software, applications, hardware, advanced system engineering and early
15 // testbed platforms, in support of the nation's exascale computing imperative.
16 
17 #include <string.h>
18 #include "ceed-opt.h"
19 
20 //------------------------------------------------------------------------------
21 // Setup Input/Output Fields
22 //------------------------------------------------------------------------------
23 static int CeedOperatorSetupFields_Opt(CeedQFunction qf, CeedOperator op,
24                                        bool inOrOut, const CeedInt blksize,
25                                        CeedElemRestriction *blkrestr,
26                                        CeedVector *fullevecs, CeedVector *evecs,
27                                        CeedVector *qvecs, CeedInt starte,
28                                        CeedInt numfields, CeedInt Q) {
29   CeedInt dim, ierr, ncomp, size, P;
30   Ceed ceed;
31   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
32   CeedBasis basis;
33   CeedElemRestriction r;
34   CeedOperatorField *opfields;
35   CeedQFunctionField *qffields;
36   if (inOrOut) {
37     ierr = CeedOperatorGetFields(op, NULL, &opfields);
38     CeedChk(ierr);
39     ierr = CeedQFunctionGetFields(qf, NULL, &qffields);
40     CeedChk(ierr);
41   } else {
42     ierr = CeedOperatorGetFields(op, &opfields, NULL);
43     CeedChk(ierr);
44     ierr = CeedQFunctionGetFields(qf, &qffields, NULL);
45     CeedChk(ierr);
46   }
47 
48   // Loop over fields
49   for (CeedInt i=0; i<numfields; i++) {
50     CeedEvalMode emode;
51     ierr = CeedQFunctionFieldGetEvalMode(qffields[i], &emode); CeedChk(ierr);
52 
53     if (emode != CEED_EVAL_WEIGHT) {
54       ierr = CeedOperatorFieldGetElemRestriction(opfields[i], &r);
55       CeedChk(ierr);
56       Ceed ceed;
57       ierr = CeedElemRestrictionGetCeed(r, &ceed); CeedChk(ierr);
58       CeedInt nelem, elemsize, lsize, compstride;
59       ierr = CeedElemRestrictionGetNumElements(r, &nelem); CeedChk(ierr);
60       ierr = CeedElemRestrictionGetElementSize(r, &elemsize); CeedChk(ierr);
61       ierr = CeedElemRestrictionGetLVectorSize(r, &lsize); CeedChk(ierr);
62       ierr = CeedElemRestrictionGetNumComponents(r, &ncomp); CeedChk(ierr);
63 
64       bool strided;
65       ierr = CeedElemRestrictionIsStrided(r, &strided); CeedChk(ierr);
66       if (strided) {
67         CeedInt strides[3];
68         ierr = CeedElemRestrictionGetStrides(r, &strides); CeedChk(ierr);
69         ierr = CeedElemRestrictionCreateBlockedStrided(ceed, nelem, elemsize,
70                blksize, ncomp, lsize, strides, &blkrestr[i+starte]);
71         CeedChk(ierr);
72       } else {
73         const CeedInt *offsets = NULL;
74         ierr = CeedElemRestrictionGetOffsets(r, CEED_MEM_HOST, &offsets);
75         CeedChk(ierr);
76         ierr = CeedElemRestrictionGetCompStride(r, &compstride); CeedChk(ierr);
77         ierr = CeedElemRestrictionCreateBlocked(ceed, nelem, elemsize,
78                                                 blksize, ncomp, compstride,
79                                                 lsize, CEED_MEM_HOST,
80                                                 CEED_COPY_VALUES, offsets,
81                                                 &blkrestr[i+starte]);
82         CeedChk(ierr);
83         ierr = CeedElemRestrictionRestoreOffsets(r, &offsets); CeedChk(ierr);
84       }
85       ierr = CeedElemRestrictionCreateVector(blkrestr[i+starte], NULL,
86                                              &fullevecs[i+starte]);
87       CeedChk(ierr);
88     }
89 
90     switch(emode) {
91     case CEED_EVAL_NONE:
92       ierr = CeedQFunctionFieldGetSize(qffields[i], &size); CeedChk(ierr);
93       ierr = CeedVectorCreate(ceed, Q*size*blksize, &evecs[i]); CeedChk(ierr);
94       ierr = CeedVectorCreate(ceed, Q*size*blksize, &qvecs[i]); CeedChk(ierr);
95       break;
96     case CEED_EVAL_INTERP:
97       ierr = CeedQFunctionFieldGetSize(qffields[i], &size); CeedChk(ierr);
98       ierr = CeedElemRestrictionGetElementSize(r, &P);
99       CeedChk(ierr);
100       ierr = CeedVectorCreate(ceed, P*size*blksize, &evecs[i]); CeedChk(ierr);
101       ierr = CeedVectorCreate(ceed, Q*size*blksize, &qvecs[i]); CeedChk(ierr);
102       break;
103     case CEED_EVAL_GRAD:
104       ierr = CeedOperatorFieldGetBasis(opfields[i], &basis); CeedChk(ierr);
105       ierr = CeedQFunctionFieldGetSize(qffields[i], &size); CeedChk(ierr);
106       ierr = CeedBasisGetDimension(basis, &dim); CeedChk(ierr);
107       ierr = CeedElemRestrictionGetElementSize(r, &P);
108       CeedChk(ierr);
109       ierr = CeedVectorCreate(ceed, P*size/dim*blksize, &evecs[i]); CeedChk(ierr);
110       ierr = CeedVectorCreate(ceed, Q*size*blksize, &qvecs[i]); CeedChk(ierr);
111       break;
112     case CEED_EVAL_WEIGHT: // Only on input fields
113       ierr = CeedOperatorFieldGetBasis(opfields[i], &basis); CeedChk(ierr);
114       ierr = CeedVectorCreate(ceed, Q*blksize, &qvecs[i]); CeedChk(ierr);
115       ierr = CeedBasisApply(basis, blksize, CEED_NOTRANSPOSE,
116                             CEED_EVAL_WEIGHT, CEED_VECTOR_NONE, qvecs[i]);
117       CeedChk(ierr);
118 
119       break;
120     case CEED_EVAL_DIV:
121       break; // Not implemented
122     case CEED_EVAL_CURL:
123       break; // Not implemented
124     }
125   }
126   return 0;
127 }
128 
129 //------------------------------------------------------------------------------
130 // Setup Operator
131 //------------------------------------------------------------------------------
132 static int CeedOperatorSetup_Opt(CeedOperator op) {
133   int ierr;
134   bool setupdone;
135   ierr = CeedOperatorIsSetupDone(op, &setupdone); CeedChk(ierr);
136   if (setupdone) return 0;
137   Ceed ceed;
138   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
139   Ceed_Opt *ceedimpl;
140   ierr = CeedGetData(ceed, &ceedimpl); CeedChk(ierr);
141   const CeedInt blksize = ceedimpl->blksize;
142   CeedOperator_Opt *impl;
143   ierr = CeedOperatorGetData(op, &impl); CeedChk(ierr);
144   CeedQFunction qf;
145   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
146   CeedInt Q, numinputfields, numoutputfields;
147   ierr = CeedOperatorGetNumQuadraturePoints(op, &Q); CeedChk(ierr);
148   ierr = CeedQFunctionIsIdentity(qf, &impl->identityqf); CeedChk(ierr);
149   ierr= CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
150   CeedChk(ierr);
151   CeedOperatorField *opinputfields, *opoutputfields;
152   ierr = CeedOperatorGetFields(op, &opinputfields, &opoutputfields);
153   CeedChk(ierr);
154   CeedQFunctionField *qfinputfields, *qfoutputfields;
155   ierr = CeedQFunctionGetFields(qf, &qfinputfields, &qfoutputfields);
156   CeedChk(ierr);
157 
158   // Allocate
159   ierr = CeedCalloc(numinputfields + numoutputfields, &impl->blkrestr);
160   CeedChk(ierr);
161   ierr = CeedCalloc(numinputfields + numoutputfields, &impl->evecs);
162   CeedChk(ierr);
163   ierr = CeedCalloc(numinputfields + numoutputfields, &impl->edata);
164   CeedChk(ierr);
165 
166   ierr = CeedCalloc(16, &impl->inputstate); CeedChk(ierr);
167   ierr = CeedCalloc(16, &impl->evecsin); CeedChk(ierr);
168   ierr = CeedCalloc(16, &impl->evecsout); CeedChk(ierr);
169   ierr = CeedCalloc(16, &impl->qvecsin); CeedChk(ierr);
170   ierr = CeedCalloc(16, &impl->qvecsout); CeedChk(ierr);
171 
172   impl->numein = numinputfields; impl->numeout = numoutputfields;
173 
174   // Set up infield and outfield pointer arrays
175   // Infields
176   ierr = CeedOperatorSetupFields_Opt(qf, op, 0, blksize, impl->blkrestr,
177                                      impl->evecs, impl->evecsin,
178                                      impl->qvecsin, 0,
179                                      numinputfields, Q);
180   CeedChk(ierr);
181   // Outfields
182   ierr = CeedOperatorSetupFields_Opt(qf, op, 1, blksize, impl->blkrestr,
183                                      impl->evecs, impl->evecsout,
184                                      impl->qvecsout, numinputfields,
185                                      numoutputfields, Q);
186   CeedChk(ierr);
187 
188   // Identity QFunctions
189   if (impl->identityqf) {
190     CeedEvalMode inmode, outmode;
191     CeedQFunctionField *infields, *outfields;
192     ierr = CeedQFunctionGetFields(qf, &infields, &outfields); CeedChk(ierr);
193 
194     for (CeedInt i=0; i<numinputfields; i++) {
195       ierr = CeedQFunctionFieldGetEvalMode(infields[i], &inmode);
196       CeedChk(ierr);
197       ierr = CeedQFunctionFieldGetEvalMode(outfields[i], &outmode);
198       CeedChk(ierr);
199 
200       ierr = CeedVectorDestroy(&impl->qvecsout[i]); CeedChk(ierr);
201       impl->qvecsout[i] = impl->qvecsin[i];
202       ierr = CeedVectorAddReference(impl->qvecsin[i]); CeedChk(ierr);
203     }
204   }
205 
206   ierr = CeedOperatorSetSetupDone(op); CeedChk(ierr);
207 
208   return 0;
209 }
210 
211 //------------------------------------------------------------------------------
212 // Setup Input Fields
213 //------------------------------------------------------------------------------
214 static inline int CeedOperatorSetupInputs_Opt(CeedInt numinputfields,
215     CeedQFunctionField *qfinputfields, CeedOperatorField *opinputfields,
216     CeedVector invec, CeedOperator_Opt *impl, CeedRequest *request) {
217   CeedInt ierr;
218   CeedEvalMode emode;
219   CeedVector vec;
220   uint64_t state;
221 
222   for (CeedInt i=0; i<numinputfields; i++) {
223     ierr = CeedQFunctionFieldGetEvalMode(qfinputfields[i], &emode);
224     CeedChk(ierr);
225     if (emode == CEED_EVAL_WEIGHT) { // Skip
226     } else {
227       // Get input vector
228       ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
229       if (vec != CEED_VECTOR_ACTIVE) {
230         // Restrict
231         ierr = CeedVectorGetState(vec, &state); CeedChk(ierr);
232         if (state != impl->inputstate[i]) {
233           ierr = CeedElemRestrictionApply(impl->blkrestr[i], CEED_NOTRANSPOSE,
234                                           vec, impl->evecs[i], request);
235           CeedChk(ierr);
236           impl->inputstate[i] = state;
237         }
238       } else {
239         // Set Qvec for CEED_EVAL_NONE
240         if (emode == CEED_EVAL_NONE) {
241           ierr = CeedVectorGetArray(impl->evecsin[i], CEED_MEM_HOST,
242                                     &impl->edata[i]); CeedChk(ierr);
243           ierr = CeedVectorSetArray(impl->qvecsin[i], CEED_MEM_HOST,
244                                     CEED_USE_POINTER,
245                                     impl->edata[i]); CeedChk(ierr);
246           ierr = CeedVectorRestoreArray(impl->evecsin[i],
247                                         &impl->edata[i]); CeedChk(ierr);
248         }
249       }
250       // Get evec
251       ierr = CeedVectorGetArrayRead(impl->evecs[i], CEED_MEM_HOST,
252                                     (const CeedScalar **) &impl->edata[i]);
253       CeedChk(ierr);
254     }
255   }
256   return 0;
257 }
258 
259 //------------------------------------------------------------------------------
260 // Input Basis Action
261 //------------------------------------------------------------------------------
262 static inline int CeedOperatorInputBasis_Opt(CeedInt e, CeedInt Q,
263     CeedQFunctionField *qfinputfields, CeedOperatorField *opinputfields,
264     CeedInt numinputfields, CeedInt blksize, CeedVector invec, bool skipactive,
265     CeedOperator_Opt *impl, CeedRequest *request) {
266   CeedInt ierr;
267   CeedInt dim, elemsize, size;
268   CeedElemRestriction Erestrict;
269   CeedEvalMode emode;
270   CeedBasis basis;
271   CeedVector vec;
272 
273   for (CeedInt i=0; i<numinputfields; i++) {
274     ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
275     // Skip active input
276     if (skipactive) {
277       if (vec == CEED_VECTOR_ACTIVE)
278         continue;
279     }
280 
281     CeedInt activein = 0;
282     // Get elemsize, emode, size
283     ierr = CeedOperatorFieldGetElemRestriction(opinputfields[i], &Erestrict);
284     CeedChk(ierr);
285     ierr = CeedElemRestrictionGetElementSize(Erestrict, &elemsize);
286     CeedChk(ierr);
287     ierr = CeedQFunctionFieldGetEvalMode(qfinputfields[i], &emode);
288     CeedChk(ierr);
289     ierr = CeedQFunctionFieldGetSize(qfinputfields[i], &size); CeedChk(ierr);
290     // Restrict block active input
291     if (vec == CEED_VECTOR_ACTIVE) {
292       ierr = CeedElemRestrictionApplyBlock(impl->blkrestr[i], e/blksize,
293                                            CEED_NOTRANSPOSE, invec,
294                                            impl->evecsin[i], request);
295       CeedChk(ierr);
296       activein = 1;
297     }
298     // Basis action
299     switch(emode) {
300     case CEED_EVAL_NONE:
301       if (!activein) {
302         ierr = CeedVectorSetArray(impl->qvecsin[i], CEED_MEM_HOST,
303                                   CEED_USE_POINTER,
304                                   &impl->edata[i][e*Q*size]); CeedChk(ierr);
305       }
306       break;
307     case CEED_EVAL_INTERP:
308       ierr = CeedOperatorFieldGetBasis(opinputfields[i], &basis);
309       CeedChk(ierr);
310       if (!activein) {
311         ierr = CeedVectorSetArray(impl->evecsin[i], CEED_MEM_HOST,
312                                   CEED_USE_POINTER,
313                                   &impl->edata[i][e*elemsize*size]);
314         CeedChk(ierr);
315       }
316       ierr = CeedBasisApply(basis, blksize, CEED_NOTRANSPOSE,
317                             CEED_EVAL_INTERP, impl->evecsin[i],
318                             impl->qvecsin[i]); CeedChk(ierr);
319       break;
320     case CEED_EVAL_GRAD:
321       ierr = CeedOperatorFieldGetBasis(opinputfields[i], &basis);
322       CeedChk(ierr);
323       if (!activein) {
324         ierr = CeedBasisGetDimension(basis, &dim); CeedChk(ierr);
325         ierr = CeedVectorSetArray(impl->evecsin[i], CEED_MEM_HOST,
326                                   CEED_USE_POINTER,
327                                   &impl->edata[i][e*elemsize*size/dim]);
328         CeedChk(ierr);
329       }
330       ierr = CeedBasisApply(basis, blksize, CEED_NOTRANSPOSE,
331                             CEED_EVAL_GRAD, impl->evecsin[i],
332                             impl->qvecsin[i]); CeedChk(ierr);
333       break;
334     case CEED_EVAL_WEIGHT:
335       break;  // No action
336     // LCOV_EXCL_START
337     case CEED_EVAL_DIV:
338     case CEED_EVAL_CURL: {
339       ierr = CeedOperatorFieldGetBasis(opinputfields[i], &basis);
340       CeedChk(ierr);
341       Ceed ceed;
342       ierr = CeedBasisGetCeed(basis, &ceed); CeedChk(ierr);
343       return CeedError(ceed, 1, "Ceed evaluation mode not implemented");
344       // LCOV_EXCL_STOP
345     }
346     }
347   }
348   return 0;
349 }
350 
351 //------------------------------------------------------------------------------
352 // Output Basis Action
353 //------------------------------------------------------------------------------
354 static inline int CeedOperatorOutputBasis_Opt(CeedInt e, CeedInt Q,
355     CeedQFunctionField *qfoutputfields, CeedOperatorField *opoutputfields,
356     CeedInt blksize, CeedInt numinputfields, CeedInt numoutputfields,
357     CeedOperator op, CeedVector outvec, CeedOperator_Opt *impl,
358     CeedRequest *request) {
359   CeedInt ierr;
360   CeedElemRestriction Erestrict;
361   CeedEvalMode emode;
362   CeedBasis basis;
363   CeedVector vec;
364 
365   for (CeedInt i=0; i<numoutputfields; i++) {
366     // Get elemsize, emode, size
367     ierr = CeedOperatorFieldGetElemRestriction(opoutputfields[i], &Erestrict);
368     CeedChk(ierr);
369     ierr = CeedQFunctionFieldGetEvalMode(qfoutputfields[i], &emode);
370     CeedChk(ierr);
371     // Basis action
372     switch(emode) {
373     case CEED_EVAL_NONE:
374       break; // No action
375     case CEED_EVAL_INTERP:
376       ierr = CeedOperatorFieldGetBasis(opoutputfields[i], &basis);
377       CeedChk(ierr);
378       ierr = CeedBasisApply(basis, blksize, CEED_TRANSPOSE,
379                             CEED_EVAL_INTERP, impl->qvecsout[i],
380                             impl->evecsout[i]); CeedChk(ierr);
381       break;
382     case CEED_EVAL_GRAD:
383       ierr = CeedOperatorFieldGetBasis(opoutputfields[i], &basis);
384       CeedChk(ierr);
385       ierr = CeedBasisApply(basis, blksize, CEED_TRANSPOSE,
386                             CEED_EVAL_GRAD, impl->qvecsout[i],
387                             impl->evecsout[i]); CeedChk(ierr);
388       break;
389     // LCOV_EXCL_START
390     case CEED_EVAL_WEIGHT: {
391       Ceed ceed;
392       ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
393       return CeedError(ceed, 1, "CEED_EVAL_WEIGHT cannot be an output "
394                        "evaluation mode");
395     }
396     case CEED_EVAL_DIV:
397     case CEED_EVAL_CURL: {
398       Ceed ceed;
399       ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
400       return CeedError(ceed, 1, "Ceed evaluation mode not implemented");
401       // LCOV_EXCL_STOP
402     }
403     }
404     // Restrict output block
405     // Get output vector
406     ierr = CeedOperatorFieldGetVector(opoutputfields[i], &vec); CeedChk(ierr);
407     if (vec == CEED_VECTOR_ACTIVE)
408       vec = outvec;
409     // Restrict
410     ierr = CeedElemRestrictionApplyBlock(impl->blkrestr[i+impl->numein],
411                                          e/blksize, CEED_TRANSPOSE,
412                                          impl->evecsout[i], vec, request);
413     CeedChk(ierr);
414   }
415   return 0;
416 }
417 
418 //------------------------------------------------------------------------------
419 // Restore Input Vectors
420 //------------------------------------------------------------------------------
421 static inline int CeedOperatorRestoreInputs_Opt(CeedInt numinputfields,
422     CeedQFunctionField *qfinputfields, CeedOperatorField *opinputfields,
423     CeedOperator_Opt *impl) {
424   CeedInt ierr;
425   CeedEvalMode emode;
426 
427   for (CeedInt i=0; i<numinputfields; i++) {
428     ierr = CeedQFunctionFieldGetEvalMode(qfinputfields[i], &emode);
429     CeedChk(ierr);
430     if (emode == CEED_EVAL_WEIGHT) { // Skip
431     } else {
432       ierr = CeedVectorRestoreArrayRead(impl->evecs[i],
433                                         (const CeedScalar **) &impl->edata[i]);
434       CeedChk(ierr);
435     }
436   }
437   return 0;
438 }
439 
440 //------------------------------------------------------------------------------
441 // Operator Apply
442 //------------------------------------------------------------------------------
443 static int CeedOperatorApplyAdd_Opt(CeedOperator op, CeedVector invec,
444                                     CeedVector outvec, CeedRequest *request) {
445   int ierr;
446   Ceed ceed;
447   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
448   Ceed_Opt *ceedimpl;
449   ierr = CeedGetData(ceed, &ceedimpl); CeedChk(ierr);
450   CeedInt blksize = ceedimpl->blksize;
451   CeedOperator_Opt *impl;
452   ierr = CeedOperatorGetData(op, &impl); CeedChk(ierr);
453   CeedInt Q, numinputfields, numoutputfields, numelements;
454   ierr = CeedOperatorGetNumElements(op, &numelements); CeedChk(ierr);
455   ierr = CeedOperatorGetNumQuadraturePoints(op, &Q); CeedChk(ierr);
456   CeedInt nblks = (numelements/blksize) + !!(numelements%blksize);
457   CeedQFunction qf;
458   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
459   ierr= CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
460   CeedChk(ierr);
461   CeedOperatorField *opinputfields, *opoutputfields;
462   ierr = CeedOperatorGetFields(op, &opinputfields, &opoutputfields);
463   CeedChk(ierr);
464   CeedQFunctionField *qfinputfields, *qfoutputfields;
465   ierr = CeedQFunctionGetFields(qf, &qfinputfields, &qfoutputfields);
466   CeedChk(ierr);
467   CeedEvalMode emode;
468 
469   // Setup
470   ierr = CeedOperatorSetup_Opt(op); CeedChk(ierr);
471 
472   // Input Evecs and Restriction
473   ierr = CeedOperatorSetupInputs_Opt(numinputfields, qfinputfields,
474                                      opinputfields, invec, impl, request);
475   CeedChk(ierr);
476 
477   // Output Lvecs, Evecs, and Qvecs
478   for (CeedInt i=0; i<numoutputfields; i++) {
479     // Set Qvec if needed
480     ierr = CeedQFunctionFieldGetEvalMode(qfoutputfields[i], &emode);
481     CeedChk(ierr);
482     if (emode == CEED_EVAL_NONE) {
483       // Set qvec to single block evec
484       ierr = CeedVectorGetArray(impl->evecsout[i], CEED_MEM_HOST,
485                                 &impl->edata[i + numinputfields]);
486       CeedChk(ierr);
487       ierr = CeedVectorSetArray(impl->qvecsout[i], CEED_MEM_HOST,
488                                 CEED_USE_POINTER,
489                                 impl->edata[i + numinputfields]); CeedChk(ierr);
490       ierr = CeedVectorRestoreArray(impl->evecsout[i],
491                                     &impl->edata[i + numinputfields]);
492       CeedChk(ierr);
493     }
494   }
495 
496   // Loop through elements
497   for (CeedInt e=0; e<nblks*blksize; e+=blksize) {
498     // Input basis apply
499     ierr = CeedOperatorInputBasis_Opt(e, Q, qfinputfields, opinputfields,
500                                       numinputfields, blksize, invec, false,
501                                       impl, request); CeedChk(ierr);
502 
503     // Q function
504     if (!impl->identityqf) {
505       ierr = CeedQFunctionApply(qf, Q*blksize, impl->qvecsin, impl->qvecsout);
506       CeedChk(ierr);
507     }
508 
509     // Output basis apply and restrict
510     ierr = CeedOperatorOutputBasis_Opt(e, Q, qfoutputfields, opoutputfields,
511                                        blksize, numinputfields, numoutputfields,
512                                        op, outvec, impl, request);
513     CeedChk(ierr);
514   }
515 
516   // Restore input arrays
517   ierr = CeedOperatorRestoreInputs_Opt(numinputfields, qfinputfields,
518                                        opinputfields, impl);
519   CeedChk(ierr);
520 
521   return 0;
522 }
523 
524 //------------------------------------------------------------------------------
525 // Assemble Linear QFunction
526 //------------------------------------------------------------------------------
527 static int CeedOperatorLinearAssembleQFunction_Opt(CeedOperator op,
528     CeedVector *assembled, CeedElemRestriction *rstr, CeedRequest *request) {
529   int ierr;
530   Ceed ceed;
531   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
532   Ceed_Opt *ceedimpl;
533   ierr = CeedGetData(ceed, &ceedimpl); CeedChk(ierr);
534   const CeedInt blksize = ceedimpl->blksize;
535   CeedOperator_Opt *impl;
536   ierr = CeedOperatorGetData(op, &impl); CeedChk(ierr);
537   CeedInt Q, numinputfields, numoutputfields, numelements, size;
538   ierr = CeedOperatorGetNumElements(op, &numelements); CeedChk(ierr);
539   ierr = CeedOperatorGetNumQuadraturePoints(op, &Q); CeedChk(ierr);
540   CeedInt nblks = (numelements/blksize) + !!(numelements%blksize);
541   CeedQFunction qf;
542   ierr = CeedOperatorGetQFunction(op, &qf); CeedChk(ierr);
543   ierr= CeedQFunctionGetNumArgs(qf, &numinputfields, &numoutputfields);
544   CeedChk(ierr);
545   CeedOperatorField *opinputfields, *opoutputfields;
546   ierr = CeedOperatorGetFields(op, &opinputfields, &opoutputfields);
547   CeedChk(ierr);
548   CeedQFunctionField *qfinputfields, *qfoutputfields;
549   ierr = CeedQFunctionGetFields(qf, &qfinputfields, &qfoutputfields);
550   CeedChk(ierr);
551   CeedVector vec, lvec;
552   CeedInt numactivein = 0, numactiveout = 0;
553   CeedVector *activein = NULL;
554   CeedScalar *a, *tmp;
555 
556   // Setup
557   ierr = CeedOperatorSetup_Opt(op); CeedChk(ierr);
558 
559   // Check for identity
560   if (impl->identityqf)
561     // LCOV_EXCL_START
562     return CeedError(ceed, 1, "Assembling identity qfunctions not supported");
563   // LCOV_EXCL_STOP
564 
565   // Input Evecs and Restriction
566   ierr = CeedOperatorSetupInputs_Opt(numinputfields, qfinputfields,
567                                      opinputfields, NULL, impl, request);
568   CeedChk(ierr);
569 
570   // Count number of active input fields
571   for (CeedInt i=0; i<numinputfields; i++) {
572     // Get input vector
573     ierr = CeedOperatorFieldGetVector(opinputfields[i], &vec); CeedChk(ierr);
574     // Check if active input
575     if (vec == CEED_VECTOR_ACTIVE) {
576       ierr = CeedQFunctionFieldGetSize(qfinputfields[i], &size); CeedChk(ierr);
577       ierr = CeedVectorSetValue(impl->qvecsin[i], 0.0); CeedChk(ierr);
578       ierr = CeedVectorGetArray(impl->qvecsin[i], CEED_MEM_HOST, &tmp);
579       CeedChk(ierr);
580       ierr = CeedRealloc(numactivein + size, &activein); CeedChk(ierr);
581       for (CeedInt field=0; field<size; field++) {
582         ierr = CeedVectorCreate(ceed, Q*blksize, &activein[numactivein+field]);
583         CeedChk(ierr);
584         ierr = CeedVectorSetArray(activein[numactivein+field], CEED_MEM_HOST,
585                                   CEED_USE_POINTER, &tmp[field*Q*blksize]);
586         CeedChk(ierr);
587       }
588       numactivein += size;
589       ierr = CeedVectorRestoreArray(impl->qvecsin[i], &tmp); CeedChk(ierr);
590     }
591   }
592 
593   // Count number of active output fields
594   for (CeedInt i=0; i<numoutputfields; i++) {
595     // Get output vector
596     ierr = CeedOperatorFieldGetVector(opoutputfields[i], &vec); CeedChk(ierr);
597     // Check if active output
598     if (vec == CEED_VECTOR_ACTIVE) {
599       ierr = CeedQFunctionFieldGetSize(qfoutputfields[i], &size); CeedChk(ierr);
600       numactiveout += size;
601     }
602   }
603 
604   // Check sizes
605   if (!numactivein || !numactiveout)
606     // LCOV_EXCL_START
607     return CeedError(ceed, 1, "Cannot assemble QFunction without active inputs "
608                      "and outputs");
609   // LCOV_EXCL_STOP
610 
611   // Setup lvec
612   ierr = CeedVectorCreate(ceed, nblks*blksize*Q*numactivein*numactiveout,
613                           &lvec); CeedChk(ierr);
614   ierr = CeedVectorGetArray(lvec, CEED_MEM_HOST, &a); CeedChk(ierr);
615 
616   // Create output restriction
617   CeedInt strides[3] = {1, Q, numactivein *numactiveout*Q};
618   ierr = CeedElemRestrictionCreateStrided(ceed, numelements, Q,
619                                           numactivein*numactiveout,
620                                           numactivein*numactiveout*numelements*Q,
621                                           strides, rstr); CeedChk(ierr);
622   // Create assembled vector
623   ierr = CeedVectorCreate(ceed, numelements*Q*numactivein*numactiveout,
624                           assembled); CeedChk(ierr);
625 
626   // Loop through elements
627   for (CeedInt e=0; e<nblks*blksize; e+=blksize) {
628     // Input basis apply
629     ierr = CeedOperatorInputBasis_Opt(e, Q, qfinputfields, opinputfields,
630                                       numinputfields, blksize, NULL, true,
631                                       impl, request); CeedChk(ierr);
632 
633     // Assemble QFunction
634     for (CeedInt in=0; in<numactivein; in++) {
635       // Set Inputs
636       ierr = CeedVectorSetValue(activein[in], 1.0); CeedChk(ierr);
637       if (numactivein > 1) {
638         ierr = CeedVectorSetValue(activein[(in+numactivein-1)%numactivein],
639                                   0.0); CeedChk(ierr);
640       }
641       // Set Outputs
642       for (CeedInt out=0; out<numoutputfields; out++) {
643         // Get output vector
644         ierr = CeedOperatorFieldGetVector(opoutputfields[out], &vec);
645         CeedChk(ierr);
646         // Check if active output
647         if (vec == CEED_VECTOR_ACTIVE) {
648           CeedVectorSetArray(impl->qvecsout[out], CEED_MEM_HOST,
649                              CEED_USE_POINTER, a); CeedChk(ierr);
650           ierr = CeedQFunctionFieldGetSize(qfoutputfields[out], &size);
651           CeedChk(ierr);
652           a += size*Q*blksize; // Advance the pointer by the size of the output
653         }
654       }
655       // Apply QFunction
656       ierr = CeedQFunctionApply(qf, Q*blksize, impl->qvecsin, impl->qvecsout);
657       CeedChk(ierr);
658     }
659   }
660 
661   // Un-set output Qvecs to prevent accidental overwrite of Assembled
662   for (CeedInt out=0; out<numoutputfields; out++) {
663     // Get output vector
664     ierr = CeedOperatorFieldGetVector(opoutputfields[out], &vec);
665     CeedChk(ierr);
666     // Check if active output
667     if (vec == CEED_VECTOR_ACTIVE) {
668       CeedVectorSetArray(impl->qvecsout[out], CEED_MEM_HOST, CEED_COPY_VALUES,
669                          NULL); CeedChk(ierr);
670     }
671   }
672 
673   // Restore input arrays
674   ierr = CeedOperatorRestoreInputs_Opt(numinputfields, qfinputfields,
675                                        opinputfields, impl);
676   CeedChk(ierr);
677 
678   // Output blocked restriction
679   ierr = CeedVectorRestoreArray(lvec, &a); CeedChk(ierr);
680   ierr = CeedVectorSetValue(*assembled, 0.0); CeedChk(ierr);
681   CeedElemRestriction blkrstr;
682   ierr = CeedElemRestrictionCreateBlockedStrided(ceed, numelements, Q, blksize,
683          numactivein*numactiveout, numactivein*numactiveout*numelements*Q,
684          strides, &blkrstr); CeedChk(ierr);
685   ierr = CeedElemRestrictionApply(blkrstr, CEED_TRANSPOSE, lvec, *assembled,
686                                   request); CeedChk(ierr);
687 
688   // Cleanup
689   for (CeedInt i=0; i<numactivein; i++) {
690     ierr = CeedVectorDestroy(&activein[i]); CeedChk(ierr);
691   }
692   ierr = CeedFree(&activein); CeedChk(ierr);
693   ierr = CeedVectorDestroy(&lvec); CeedChk(ierr);
694   ierr = CeedElemRestrictionDestroy(&blkrstr); CeedChk(ierr);
695 
696   return 0;
697 }
698 
699 //------------------------------------------------------------------------------
700 // Operator Destroy
701 //------------------------------------------------------------------------------
702 static int CeedOperatorDestroy_Opt(CeedOperator op) {
703   int ierr;
704   CeedOperator_Opt *impl;
705   ierr = CeedOperatorGetData(op, &impl); CeedChk(ierr);
706 
707   for (CeedInt i=0; i<impl->numein+impl->numeout; i++) {
708     ierr = CeedElemRestrictionDestroy(&impl->blkrestr[i]); CeedChk(ierr);
709     ierr = CeedVectorDestroy(&impl->evecs[i]); CeedChk(ierr);
710   }
711   ierr = CeedFree(&impl->blkrestr); CeedChk(ierr);
712   ierr = CeedFree(&impl->evecs); CeedChk(ierr);
713   ierr = CeedFree(&impl->edata); CeedChk(ierr);
714   ierr = CeedFree(&impl->inputstate); CeedChk(ierr);
715 
716   for (CeedInt i=0; i<impl->numein; i++) {
717     ierr = CeedVectorDestroy(&impl->evecsin[i]); CeedChk(ierr);
718     ierr = CeedVectorDestroy(&impl->qvecsin[i]); CeedChk(ierr);
719   }
720   ierr = CeedFree(&impl->evecsin); CeedChk(ierr);
721   ierr = CeedFree(&impl->qvecsin); CeedChk(ierr);
722 
723   for (CeedInt i=0; i<impl->numeout; i++) {
724     ierr = CeedVectorDestroy(&impl->evecsout[i]); CeedChk(ierr);
725     ierr = CeedVectorDestroy(&impl->qvecsout[i]); CeedChk(ierr);
726   }
727   ierr = CeedFree(&impl->evecsout); CeedChk(ierr);
728   ierr = CeedFree(&impl->qvecsout); CeedChk(ierr);
729 
730   ierr = CeedFree(&impl); CeedChk(ierr);
731   return 0;
732 }
733 
734 //------------------------------------------------------------------------------
735 // Operator Create
736 //------------------------------------------------------------------------------
737 int CeedOperatorCreate_Opt(CeedOperator op) {
738   int ierr;
739   Ceed ceed;
740   ierr = CeedOperatorGetCeed(op, &ceed); CeedChk(ierr);
741   Ceed_Opt *ceedimpl;
742   ierr = CeedGetData(ceed, &ceedimpl); CeedChk(ierr);
743   CeedInt blksize = ceedimpl->blksize;
744   CeedOperator_Opt *impl;
745 
746   ierr = CeedCalloc(1, &impl); CeedChk(ierr);
747   ierr = CeedOperatorSetData(op, impl); CeedChk(ierr);
748 
749   if (blksize != 1 && blksize != 8)
750     // LCOV_EXCL_START
751     return CeedError(ceed, 1, "Opt backend cannot use blocksize: %d", blksize);
752   // LCOV_EXCL_STOP
753 
754   ierr = CeedSetBackendFunction(ceed, "Operator", op, "LinearAssembleQFunction",
755                                 CeedOperatorLinearAssembleQFunction_Opt);
756   CeedChk(ierr);
757   ierr = CeedSetBackendFunction(ceed, "Operator", op, "ApplyAdd",
758                                 CeedOperatorApplyAdd_Opt); CeedChk(ierr);
759   ierr = CeedSetBackendFunction(ceed, "Operator", op, "Destroy",
760                                 CeedOperatorDestroy_Opt); CeedChk(ierr);
761   return 0;
762 }
763 //------------------------------------------------------------------------------
764