xref: /honee/src/differential_filter.c (revision 15c18037db158e38093c808dd65699d951853437) 
1 // Copyright (c) 2017-2023, Lawrence Livermore National Security, LLC and other CEED contributors.
2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
3 //
4 // SPDX-License-Identifier: BSD-2-Clause
5 //
6 // This file is part of CEED:  http://github.com/ceed
7 /// @file
8 /// Functions for setting up and performing differential filtering
9 
10 #include "../qfunctions/differential_filter.h"
11 
12 #include <petscdmplex.h>
13 
14 #include "../navierstokes.h"
15 
16 // @brief Create RHS and LHS operators for differential filtering
17 PetscErrorCode DifferentialFilterCreateOperators(Ceed ceed, User user, CeedData ceed_data, CeedQFunctionContext diff_filter_qfctx) {
18   DiffFilterData diff_filter = user->diff_filter;
19   DM             dm_filter   = diff_filter->dm_filter;
20   CeedInt        num_comp_q, num_comp_qd, num_comp_x;
21   PetscInt       dim;
22 
23   PetscFunctionBeginUser;
24   PetscCall(DMGetDimension(user->dm, &dim));
25   PetscCallCeed(ceed, CeedElemRestrictionGetNumComponents(ceed_data->elem_restr_x, &num_comp_x));
26   PetscCallCeed(ceed, CeedElemRestrictionGetNumComponents(ceed_data->elem_restr_q, &num_comp_q));
27   PetscCallCeed(ceed, CeedElemRestrictionGetNumComponents(ceed_data->elem_restr_qd_i, &num_comp_qd));
28 
29   {  // -- Create RHS MatopApplyContext
30     CeedQFunction qf_rhs;
31     CeedOperator  op_rhs;
32     switch (user->phys->state_var) {
33       case STATEVAR_PRIMITIVE:
34         PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_RHS_Prim, DifferentialFilter_RHS_Prim_loc, &qf_rhs));
35         break;
36       case STATEVAR_CONSERVATIVE:
37         PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_RHS_Conserv, DifferentialFilter_RHS_Conserv_loc, &qf_rhs));
38         break;
39       default:
40         SETERRQ(PetscObjectComm((PetscObject)user->dm), PETSC_ERR_SUP, "Differential filtering not available for chosen state variable");
41     }
42     if (diff_filter->do_mms_test) {
43       PetscCallCeed(ceed, CeedQFunctionDestroy(&qf_rhs));
44       PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_MMS_RHS, DifferentialFilter_MMS_RHS_loc, &qf_rhs));
45     }
46 
47     PetscCallCeed(ceed, CeedQFunctionSetContext(qf_rhs, diff_filter_qfctx));
48     PetscCallCeed(ceed, CeedQFunctionAddInput(qf_rhs, "q", num_comp_q, CEED_EVAL_INTERP));
49     PetscCallCeed(ceed, CeedQFunctionAddInput(qf_rhs, "qdata", num_comp_qd, CEED_EVAL_NONE));
50     PetscCallCeed(ceed, CeedQFunctionAddInput(qf_rhs, "x", num_comp_x, CEED_EVAL_INTERP));
51     for (PetscInt i = 0; i < diff_filter->num_filtered_fields; i++) {
52       char field_name[PETSC_MAX_PATH_LEN];
53       PetscCall(PetscSNPrintf(field_name, PETSC_MAX_PATH_LEN, "v%" PetscInt_FMT, i));
54       PetscCallCeed(ceed, CeedQFunctionAddOutput(qf_rhs, field_name, diff_filter->num_field_components[i], CEED_EVAL_INTERP));
55     }
56 
57     PetscCallCeed(ceed, CeedOperatorCreate(ceed, qf_rhs, NULL, NULL, &op_rhs));
58     PetscCallCeed(ceed, CeedOperatorSetField(op_rhs, "q", ceed_data->elem_restr_q, ceed_data->basis_q, CEED_VECTOR_ACTIVE));
59     PetscCallCeed(ceed, CeedOperatorSetField(op_rhs, "qdata", ceed_data->elem_restr_qd_i, CEED_BASIS_COLLOCATED, ceed_data->q_data));
60     PetscCallCeed(ceed, CeedOperatorSetField(op_rhs, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord));
61     for (PetscInt dm_field = 0; dm_field < diff_filter->num_filtered_fields; dm_field++) {
62       char                field_name[PETSC_MAX_PATH_LEN];
63       CeedElemRestriction elem_restr_filter;
64       CeedBasis           basis_filter;
65       DMLabel             domain_label = NULL;
66       PetscInt            label_value = 0, height = 0;
67       PetscCall(DMPlexCeedElemRestrictionCreate(ceed, dm_filter, domain_label, label_value, height, dm_field, &elem_restr_filter));
68       PetscCall(CreateBasisFromPlex(ceed, dm_filter, domain_label, label_value, height, dm_field, &basis_filter));
69 
70       PetscCall(PetscSNPrintf(field_name, PETSC_MAX_PATH_LEN, "v%" PetscInt_FMT, dm_field));
71       PetscCallCeed(ceed, CeedOperatorSetField(op_rhs, field_name, elem_restr_filter, basis_filter, CEED_VECTOR_ACTIVE));
72     }
73 
74     PetscCall(OperatorApplyContextCreate(user->dm, dm_filter, ceed, op_rhs, NULL, NULL, user->Q_loc, NULL, &diff_filter->op_rhs_ctx));
75 
76     PetscCallCeed(ceed, CeedQFunctionDestroy(&qf_rhs));
77     PetscCallCeed(ceed, CeedOperatorDestroy(&op_rhs));
78   }
79 
80   {  // Setup LHS Operator and KSP for the differential filtering solve
81     CeedOperator         op_lhs;
82     OperatorApplyContext mat_ctx;
83     Mat                  mat_lhs;
84     CeedInt              num_comp_qd;
85     PetscInt             dim, num_comp_grid_aniso;
86     CeedElemRestriction  elem_restr_grid_aniso;
87     CeedVector           grid_aniso_ceed;
88 
89     PetscCall(DMGetDimension(user->dm, &dim));
90     PetscCallCeed(ceed, CeedElemRestrictionGetNumComponents(ceed_data->elem_restr_qd_i, &num_comp_qd));
91 
92     // -- Get Grid anisotropy tensor
93     PetscCall(GridAnisotropyTensorCalculateCollocatedVector(ceed, user, ceed_data, &elem_restr_grid_aniso, &grid_aniso_ceed, &num_comp_grid_aniso));
94 
95     PetscCallCeed(ceed, CeedCompositeOperatorCreate(ceed, &op_lhs));
96     for (PetscInt i = 0; i < diff_filter->num_filtered_fields; i++) {
97       CeedQFunction       qf_lhs;
98       PetscInt            num_comp_filter = diff_filter->num_field_components[i];
99       CeedOperator        op_lhs_sub;
100       CeedElemRestriction elem_restr_filter;
101       CeedBasis           basis_filter;
102 
103       switch (num_comp_filter) {
104         case 1:
105           PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_LHS_1, DifferentialFilter_LHS_1_loc, &qf_lhs));
106           break;
107         case 5:
108           PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_LHS_5, DifferentialFilter_LHS_5_loc, &qf_lhs));
109           break;
110         case 6:
111           PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_LHS_6, DifferentialFilter_LHS_6_loc, &qf_lhs));
112           break;
113         case 11:
114           PetscCallCeed(ceed, CeedQFunctionCreateInterior(ceed, 1, DifferentialFilter_LHS_11, DifferentialFilter_LHS_11_loc, &qf_lhs));
115           break;
116         default:
117           SETERRQ(PetscObjectComm((PetscObject)user->dm), PETSC_ERR_SUP, "Differential filtering not available for (%" PetscInt_FMT ") components",
118                   num_comp_filter);
119       }
120 
121       PetscCallCeed(ceed, CeedQFunctionSetContext(qf_lhs, diff_filter_qfctx));
122       PetscCallCeed(ceed, CeedQFunctionAddInput(qf_lhs, "q", num_comp_filter, CEED_EVAL_INTERP));
123       PetscCallCeed(ceed, CeedQFunctionAddInput(qf_lhs, "Grad_q", num_comp_filter * dim, CEED_EVAL_GRAD));
124       PetscCallCeed(ceed, CeedQFunctionAddInput(qf_lhs, "anisotropy tensor", num_comp_grid_aniso, CEED_EVAL_NONE));
125       PetscCallCeed(ceed, CeedQFunctionAddInput(qf_lhs, "x", num_comp_x, CEED_EVAL_INTERP));
126       PetscCallCeed(ceed, CeedQFunctionAddInput(qf_lhs, "qdata", num_comp_qd, CEED_EVAL_NONE));
127       PetscCallCeed(ceed, CeedQFunctionAddOutput(qf_lhs, "v", num_comp_filter, CEED_EVAL_INTERP));
128       PetscCallCeed(ceed, CeedQFunctionAddOutput(qf_lhs, "Grad_v", num_comp_filter * dim, CEED_EVAL_GRAD));
129 
130       {
131         CeedOperatorField op_field;
132         char              field_name[PETSC_MAX_PATH_LEN];
133         PetscCall(PetscSNPrintf(field_name, PETSC_MAX_PATH_LEN, "v%" PetscInt_FMT, i));
134         PetscCallCeed(ceed, CeedOperatorGetFieldByName(diff_filter->op_rhs_ctx->op, field_name, &op_field));
135         PetscCallCeed(ceed, CeedOperatorFieldGetElemRestriction(op_field, &elem_restr_filter));
136         PetscCallCeed(ceed, CeedOperatorFieldGetBasis(op_field, &basis_filter));
137       }
138 
139       PetscCallCeed(ceed, CeedOperatorCreate(ceed, qf_lhs, NULL, NULL, &op_lhs_sub));
140       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "q", elem_restr_filter, basis_filter, CEED_VECTOR_ACTIVE));
141       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "Grad_q", elem_restr_filter, basis_filter, CEED_VECTOR_ACTIVE));
142       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "anisotropy tensor", elem_restr_grid_aniso, CEED_BASIS_COLLOCATED, grid_aniso_ceed));
143       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "x", ceed_data->elem_restr_x, ceed_data->basis_x, ceed_data->x_coord));
144       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "qdata", ceed_data->elem_restr_qd_i, CEED_BASIS_COLLOCATED, ceed_data->q_data));
145       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "v", elem_restr_filter, basis_filter, CEED_VECTOR_ACTIVE));
146       PetscCallCeed(ceed, CeedOperatorSetField(op_lhs_sub, "Grad_v", elem_restr_filter, basis_filter, CEED_VECTOR_ACTIVE));
147 
148       PetscCallCeed(ceed, CeedCompositeOperatorAddSub(op_lhs, op_lhs_sub));
149       PetscCallCeed(ceed, CeedQFunctionDestroy(&qf_lhs));
150       PetscCallCeed(ceed, CeedOperatorDestroy(&op_lhs_sub));
151     }
152     PetscCall(OperatorApplyContextCreate(dm_filter, dm_filter, ceed, op_lhs, NULL, NULL, NULL, NULL, &mat_ctx));
153     PetscCall(CreateMatShell_Ceed(mat_ctx, &mat_lhs));
154 
155     PetscCall(KSPCreate(PetscObjectComm((PetscObject)dm_filter), &diff_filter->ksp));
156     PetscCall(KSPSetOptionsPrefix(diff_filter->ksp, "diff_filter_"));
157     {
158       PC pc;
159       PetscCall(KSPGetPC(diff_filter->ksp, &pc));
160       PetscCall(PCSetType(pc, PCJACOBI));
161       PetscCall(PCJacobiSetType(pc, PC_JACOBI_DIAGONAL));
162       PetscCall(KSPSetType(diff_filter->ksp, KSPCG));
163       PetscCall(KSPSetNormType(diff_filter->ksp, KSP_NORM_NATURAL));
164       PetscCall(KSPSetTolerances(diff_filter->ksp, 1e-10, PETSC_DEFAULT, PETSC_DEFAULT, PETSC_DEFAULT));
165     }
166     PetscCall(KSPSetOperators(diff_filter->ksp, mat_lhs, mat_lhs));
167     PetscCall(KSPSetFromOptions(diff_filter->ksp));
168 
169     PetscCallCeed(ceed, CeedOperatorDestroy(&op_lhs));
170   }
171   PetscFunctionReturn(PETSC_SUCCESS);
172 }
173 
174 // @brief Setup DM, operators, contexts, etc. for performing differential filtering
175 PetscErrorCode DifferentialFilterSetup(Ceed ceed, User user, CeedData ceed_data, ProblemData *problem) {
176   MPI_Comm                  comm = user->comm;
177   NewtonianIdealGasContext  gas;
178   DifferentialFilterContext diff_filter_ctx;
179   CeedQFunctionContext      diff_filter_qfctx;
180 
181   PetscFunctionBeginUser;
182   PetscCall(PetscNew(&user->diff_filter));
183   DiffFilterData diff_filter = user->diff_filter;
184   PetscCall(PetscOptionsGetBool(NULL, NULL, "-diff_filter_mms", &diff_filter->do_mms_test, NULL));
185 
186   {  // Create DM for filtered quantities
187     PetscSection section;
188 
189     PetscCall(DMClone(user->dm, &diff_filter->dm_filter));
190     PetscCall(PetscObjectSetName((PetscObject)diff_filter->dm_filter, "Differential Filtering"));
191 
192     diff_filter->num_filtered_fields = diff_filter->do_mms_test ? 1 : 2;
193     PetscCall(PetscMalloc1(diff_filter->num_filtered_fields, &diff_filter->num_field_components));
194 
195     if (diff_filter->do_mms_test) {
196       diff_filter->num_field_components[0] = 1;
197       PetscCall(DMSetupByOrder_FEM(PETSC_TRUE, PETSC_TRUE, user->app_ctx->degree, 1, user->app_ctx->q_extra, diff_filter->num_filtered_fields,
198                                    diff_filter->num_field_components, diff_filter->dm_filter));
199 
200       PetscCall(DMGetLocalSection(diff_filter->dm_filter, &section));
201       PetscCall(PetscSectionSetFieldName(section, 0, ""));
202       PetscCall(PetscSectionSetComponentName(section, 0, 0, "FilteredPhi"));
203     } else {
204       diff_filter->num_field_components[0] = DIFF_FILTER_STATE_NUM;
205       diff_filter->num_field_components[1] = DIFF_FILTER_VELOCITY_SQUARED_NUM;
206       PetscCall(DMSetupByOrder_FEM(PETSC_TRUE, PETSC_TRUE, user->app_ctx->degree, 1, user->app_ctx->q_extra, diff_filter->num_filtered_fields,
207                                    diff_filter->num_field_components, diff_filter->dm_filter));
208 
209       PetscCall(DMGetLocalSection(diff_filter->dm_filter, &section));
210       PetscCall(PetscSectionSetFieldName(section, 0, "Filtered Primitive State Variables"));
211       PetscCall(PetscSectionSetComponentName(section, 0, DIFF_FILTER_PRESSURE, "FilteredPressure"));
212       PetscCall(PetscSectionSetComponentName(section, 0, DIFF_FILTER_VELOCITY_X, "FilteredVelocityX"));
213       PetscCall(PetscSectionSetComponentName(section, 0, DIFF_FILTER_VELOCITY_Y, "FilteredVelocityY"));
214       PetscCall(PetscSectionSetComponentName(section, 0, DIFF_FILTER_VELOCITY_Z, "FilteredVelocityZ"));
215       PetscCall(PetscSectionSetComponentName(section, 0, DIFF_FILTER_TEMPERATURE, "FilteredTemperature"));
216       PetscCall(PetscSectionSetFieldName(section, 1, "Filtered Velocity Products"));
217       PetscCall(PetscSectionSetComponentName(section, 1, DIFF_FILTER_VELOCITY_SQUARED_XX, "FilteredVelocitySquaredXX"));
218       PetscCall(PetscSectionSetComponentName(section, 1, DIFF_FILTER_VELOCITY_SQUARED_YY, "FilteredVelocitySquaredYY"));
219       PetscCall(PetscSectionSetComponentName(section, 1, DIFF_FILTER_VELOCITY_SQUARED_ZZ, "FilteredVelocitySquaredZZ"));
220       PetscCall(PetscSectionSetComponentName(section, 1, DIFF_FILTER_VELOCITY_SQUARED_YZ, "FilteredVelocitySquaredYZ"));
221       PetscCall(PetscSectionSetComponentName(section, 1, DIFF_FILTER_VELOCITY_SQUARED_XZ, "FilteredVelocitySquaredXZ"));
222       PetscCall(PetscSectionSetComponentName(section, 1, DIFF_FILTER_VELOCITY_SQUARED_XY, "FilteredVelocitySquaredXY"));
223     }
224   }
225 
226   PetscCall(PetscNew(&diff_filter_ctx));
227   diff_filter_ctx->grid_based_width = false;
228   for (int i = 0; i < 3; i++) diff_filter_ctx->width_scaling[i] = 1;
229   diff_filter_ctx->kernel_scaling   = 0.1;
230   diff_filter_ctx->damping_function = DIFF_FILTER_DAMP_NONE;
231   diff_filter_ctx->friction_length  = 0;
232   diff_filter_ctx->damping_constant = 25;
233 
234   PetscOptionsBegin(comm, NULL, "Differential Filtering Options", NULL);
235   PetscInt narray = 3;
236   PetscCall(PetscOptionsBool("-diff_filter_grid_based_width", "Use filter width based on the grid size", NULL, diff_filter_ctx->grid_based_width,
237                              (PetscBool *)&diff_filter_ctx->grid_based_width, NULL));
238   PetscCall(PetscOptionsRealArray("-diff_filter_width_scaling", "Anisotropic scaling of filter width tensor", NULL, diff_filter_ctx->width_scaling,
239                                   &narray, NULL));
240   PetscCall(PetscOptionsReal("-diff_filter_kernel_scaling", "Scaling to make differential kernel size \"equivalent\" to other filter kernels", NULL,
241                              diff_filter_ctx->kernel_scaling, &diff_filter_ctx->kernel_scaling, NULL));
242   PetscCall(PetscOptionsEnum("-diff_filter_wall_damping_function", "Damping function to use at the wall", NULL, DifferentialFilterDampingFunctions,
243                              (PetscEnum)(diff_filter_ctx->damping_function), (PetscEnum *)&diff_filter_ctx->damping_function, NULL));
244   PetscCall(PetscOptionsReal("-diff_filter_wall_damping_constant", "Contant for the wall-damping function", NULL, diff_filter_ctx->damping_constant,
245                              &diff_filter_ctx->damping_constant, NULL));
246   PetscCall(PetscOptionsReal("-diff_filter_friction_length", "Friction length associated with the flow, \\delta_\\nu. For wall-damping functions",
247                              NULL, diff_filter_ctx->friction_length, &diff_filter_ctx->friction_length, NULL));
248   PetscOptionsEnd();
249 
250   Units units = user->units;
251   for (int i = 0; i < 3; i++) diff_filter_ctx->width_scaling[i] *= units->meter;
252   diff_filter_ctx->kernel_scaling *= units->meter;
253   diff_filter_ctx->friction_length *= units->meter;
254 
255   // -- Create QFContext
256   PetscCallCeed(ceed, CeedQFunctionContextGetDataRead(problem->apply_vol_ifunction.qfunction_context, CEED_MEM_HOST, &gas));
257   diff_filter_ctx->gas = *gas;
258   PetscCallCeed(ceed, CeedQFunctionContextRestoreDataRead(problem->apply_vol_ifunction.qfunction_context, &gas));
259 
260   PetscCallCeed(ceed, CeedQFunctionContextCreate(ceed, &diff_filter_qfctx));
261   PetscCallCeed(ceed, CeedQFunctionContextSetData(diff_filter_qfctx, CEED_MEM_HOST, CEED_USE_POINTER, sizeof(*diff_filter_ctx), diff_filter_ctx));
262   PetscCallCeed(ceed, CeedQFunctionContextSetDataDestroy(diff_filter_qfctx, CEED_MEM_HOST, FreeContextPetsc));
263 
264   // -- Setup Operators
265   PetscCall(DifferentialFilterCreateOperators(ceed, user, ceed_data, diff_filter_qfctx));
266 
267   PetscCallCeed(ceed, CeedQFunctionContextDestroy(&diff_filter_qfctx));
268   PetscFunctionReturn(PETSC_SUCCESS);
269 }
270 
271 // @brief Apply differential filter to the solution given by Q
272 PetscErrorCode DifferentialFilterApply(User user, const PetscReal solution_time, const Vec Q, Vec Filtered_Solution) {
273   DiffFilterData diff_filter = user->diff_filter;
274 
275   PetscFunctionBeginUser;
276   PetscCall(UpdateBoundaryValues(user, diff_filter->op_rhs_ctx->X_loc, solution_time));
277   ApplyCeedOperatorGlobalToGlobal(Q, Filtered_Solution, diff_filter->op_rhs_ctx);
278   PetscCall(VecViewFromOptions(Filtered_Solution, NULL, "-diff_filter_rhs_view"));
279 
280   PetscCall(KSPSolve(diff_filter->ksp, Filtered_Solution, Filtered_Solution));
281   PetscFunctionReturn(PETSC_SUCCESS);
282 }
283 
284 // @brief TSMonitor for just applying differential filtering to the simulation
285 // This runs every time step and is primarily for testing purposes
286 PetscErrorCode TSMonitor_DifferentialFilter(TS ts, PetscInt steps, PetscReal solution_time, Vec Q, void *ctx) {
287   User           user        = (User)ctx;
288   DiffFilterData diff_filter = user->diff_filter;
289   Vec            Filtered_Field;
290 
291   PetscFunctionBeginUser;
292   PetscCall(DMGetGlobalVector(diff_filter->dm_filter, &Filtered_Field));
293 
294   PetscCall(DifferentialFilterApply(user, solution_time, Q, Filtered_Field));
295   PetscCall(VecViewFromOptions(Filtered_Field, NULL, "-diff_filter_view"));
296   if (user->app_ctx->test_type == TESTTYPE_DIFF_FILTER) PetscCall(RegressionTest(user->app_ctx, Filtered_Field));
297 
298   PetscCall(DMRestoreGlobalVector(diff_filter->dm_filter, &Filtered_Field));
299   PetscFunctionReturn(PETSC_SUCCESS);
300 }
301 
302 PetscErrorCode DifferentialFilterDataDestroy(DiffFilterData diff_filter) {
303   PetscFunctionBeginUser;
304   if (!diff_filter) PetscFunctionReturn(PETSC_SUCCESS);
305 
306   OperatorApplyContextDestroy(diff_filter->op_rhs_ctx);
307   PetscCall(DMDestroy(&diff_filter->dm_filter));
308   PetscCall(KSPDestroy(&diff_filter->ksp));
309 
310   PetscCall(PetscFree(diff_filter->num_field_components));
311   PetscCall(PetscFree(diff_filter));
312   PetscFunctionReturn(PETSC_SUCCESS);
313 }
314 
315 PetscErrorCode DifferentialFilterMmsICSetup(ProblemData *problem) {
316   PetscFunctionBeginUser;
317   problem->ics.qfunction     = DifferentialFilter_MMS_IC;
318   problem->ics.qfunction_loc = DifferentialFilter_MMS_IC_loc;
319   PetscFunctionReturn(PETSC_SUCCESS);
320 }
321