1 // Copyright (c) 2017-2024, Lawrence Livermore National Security, LLC and other CEED contributors. 2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details. 3 // 4 // SPDX-License-Identifier: BSD-2-Clause 5 // 6 // This file is part of CEED: http://github.com/ceed 7 8 /// @file 9 /// Command line option processing for Navier-Stokes example using PETSc 10 11 #include <petscdevice.h> 12 #include <petscsys.h> 13 14 #include "../navierstokes.h" 15 16 // Register problems to be available on the command line 17 PetscErrorCode RegisterProblems_NS(AppCtx app_ctx) { 18 app_ctx->problems = NULL; 19 20 PetscFunctionBeginUser; 21 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "density_current", NS_DENSITY_CURRENT)); 22 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "euler_vortex", NS_EULER_VORTEX)); 23 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "shocktube", NS_SHOCKTUBE)); 24 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "advection", NS_ADVECTION)); 25 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "blasius", NS_BLASIUS)); 26 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "channel", NS_CHANNEL)); 27 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "gaussian_wave", NS_GAUSSIAN_WAVE)); 28 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "newtonian", NS_NEWTONIAN_IG)); 29 PetscCall(PetscFunctionListAdd(&app_ctx->problems, "taylor_green", NS_TAYLOR_GREEN)); 30 PetscFunctionReturn(PETSC_SUCCESS); 31 } 32 33 // Process general command line options 34 PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx app_ctx, SimpleBC bc) { 35 PetscBool ceed_flag = PETSC_FALSE; 36 PetscBool problem_flag = PETSC_FALSE; 37 PetscBool option_set = PETSC_FALSE; 38 39 PetscFunctionBeginUser; 40 { 41 PetscInt num_options; 42 43 PetscCall(PetscOptionsLeftGet(NULL, &num_options, NULL, NULL)); 44 PetscCheck(num_options > 0, comm, PETSC_ERR_USER_INPUT, 45 "Command line options required." 46 " Please consult the documentation to see which options are required."); 47 PetscCall(PetscOptionsLeftRestore(NULL, &num_options, NULL, NULL)); 48 } 49 50 PetscOptionsBegin(comm, NULL, "Navier-Stokes in PETSc with libCEED", NULL); 51 52 PetscCall(PetscOptionsString("-ceed", "CEED resource specifier", NULL, app_ctx->ceed_resource, app_ctx->ceed_resource, 53 sizeof(app_ctx->ceed_resource), &ceed_flag)); 54 55 app_ctx->test_type = TESTTYPE_NONE; 56 PetscCall(PetscOptionsEnum("-test_type", "Type of test to run", NULL, TestTypes, (PetscEnum)(app_ctx->test_type), (PetscEnum *)&app_ctx->test_type, 57 NULL)); 58 59 app_ctx->test_tol = 1E-11; 60 PetscCall(PetscOptionsScalar("-compare_final_state_atol", "Test absolute tolerance", NULL, app_ctx->test_tol, &app_ctx->test_tol, NULL)); 61 62 PetscCall(PetscOptionsString("-compare_final_state_filename", "Test filename", NULL, app_ctx->test_file_path, app_ctx->test_file_path, 63 sizeof(app_ctx->test_file_path), NULL)); 64 65 PetscCall(PetscOptionsFList("-problem", "Problem to solve", NULL, app_ctx->problems, app_ctx->problem_name, app_ctx->problem_name, 66 sizeof(app_ctx->problem_name), &problem_flag)); 67 68 app_ctx->viz_refine = 0; 69 PetscCall(PetscOptionsInt("-viz_refine", "Regular refinement levels for visualization", NULL, app_ctx->viz_refine, &app_ctx->viz_refine, NULL)); 70 71 app_ctx->checkpoint_interval = 10; 72 app_ctx->checkpoint_vtk = PETSC_FALSE; 73 PetscCall(PetscOptionsDeprecated("-output_freq", "-checkpoint_interval", "libCEED 0.11.1", "Use -checkpoint_vtk true to include VTK output")); 74 PetscCall(PetscOptionsInt("-output_freq", "Frequency of output, in number of steps", NULL, app_ctx->checkpoint_interval, 75 &app_ctx->checkpoint_interval, &option_set)); 76 if (option_set) app_ctx->checkpoint_vtk = PETSC_TRUE; 77 PetscCall(PetscOptionsInt("-checkpoint_interval", "Frequency of output, in number of steps", NULL, app_ctx->checkpoint_interval, 78 &app_ctx->checkpoint_interval, NULL)); 79 PetscCall(PetscOptionsBool("-checkpoint_vtk", "Include VTK (*.vtu) output at each binary checkpoint", NULL, app_ctx->checkpoint_vtk, 80 &app_ctx->checkpoint_vtk, NULL)); 81 82 PetscCall(PetscOptionsBool("-output_add_stepnum2bin", "Add step number to the binary outputs", NULL, app_ctx->add_stepnum2bin, 83 &app_ctx->add_stepnum2bin, NULL)); 84 85 PetscCall(PetscStrncpy(app_ctx->output_dir, ".", 2)); 86 PetscCall(PetscOptionsString("-output_dir", "Output directory", NULL, app_ctx->output_dir, app_ctx->output_dir, sizeof(app_ctx->output_dir), NULL)); 87 88 app_ctx->cont_steps = 0; 89 PetscCall(PetscOptionsInt("-continue", "Continue from previous solution", NULL, app_ctx->cont_steps, &app_ctx->cont_steps, NULL)); 90 91 PetscCall(PetscStrcpy(app_ctx->cont_file, "[output_dir]/ns-solution.bin")); 92 PetscCall(PetscOptionsString("-continue_filename", "Filename to get initial condition from", NULL, app_ctx->cont_file, app_ctx->cont_file, 93 sizeof(app_ctx->cont_file), &option_set)); 94 if (!option_set) PetscCall(PetscSNPrintf(app_ctx->cont_file, sizeof app_ctx->cont_file, "%s/ns-solution.bin", app_ctx->output_dir)); 95 if (option_set && app_ctx->cont_steps == 0) app_ctx->cont_steps = -1; // Read time from file 96 97 PetscCall(PetscStrcpy(app_ctx->cont_time_file, "[output_dir]/ns-time.bin")); 98 PetscCall(PetscOptionsString("-continue_time_filename", "Filename to get initial condition time from", NULL, app_ctx->cont_time_file, 99 app_ctx->cont_time_file, sizeof(app_ctx->cont_time_file), &option_set)); 100 if (!option_set) PetscCall(PetscSNPrintf(app_ctx->cont_time_file, sizeof app_ctx->cont_time_file, "%s/ns-time.bin", app_ctx->output_dir)); 101 102 app_ctx->degree = 1; 103 PetscCall(PetscOptionsInt("-degree", "Polynomial degree of finite elements", NULL, app_ctx->degree, &app_ctx->degree, NULL)); 104 105 app_ctx->q_extra = 0; 106 PetscCall(PetscOptionsInt("-q_extra", "Number of extra quadrature points", NULL, app_ctx->q_extra, &app_ctx->q_extra, NULL)); 107 108 { 109 PetscBool option_set; 110 char amat_type[256] = ""; 111 PetscCall(PetscOptionsFList("-amat_type", "Set the type of Amat distinct from Pmat (-dm_mat_type)", NULL, MatList, amat_type, amat_type, 112 sizeof(amat_type), &option_set)); 113 if (option_set) PetscCall(PetscStrallocpy(amat_type, (char **)&app_ctx->amat_type)); 114 } 115 { 116 PetscBool option_set; 117 PetscCall(PetscOptionsHasName(NULL, NULL, "-pmat_pbdiagonal", &option_set)); 118 if (option_set) PetscCall(PetscPrintf(comm, "Warning! -pmat_pbdiagonal no longer used. Pmat assembly determined from -pc_type setting\n")); 119 } 120 121 // Provide default ceed resource if not specified 122 if (!ceed_flag) { 123 const char *ceed_resource = "/cpu/self"; 124 strncpy(app_ctx->ceed_resource, ceed_resource, 10); 125 } 126 // If we request a GPU, make sure PETSc has initialized its device (which is 127 // MPI-aware in case multiple devices are available) before CeedInit so that 128 // PETSc and libCEED agree about which device to use. 129 if (strncmp(app_ctx->ceed_resource, "/gpu", 4) == 0) PetscCall(PetscDeviceInitialize(PETSC_DEVICE_DEFAULT())); 130 131 // Provide default problem if not specified 132 if (!problem_flag) { 133 const char *problem_name = "density_current"; 134 strncpy(app_ctx->problem_name, problem_name, 16); 135 } 136 137 // Wall Boundary Conditions 138 bc->num_wall = 16; 139 PetscBool flg; 140 PetscCall(PetscOptionsIntArray("-bc_wall", "Face IDs to apply wall BC", NULL, bc->walls, &bc->num_wall, NULL)); 141 bc->num_comps = 5; 142 PetscCall(PetscOptionsIntArray("-wall_comps", "An array of constrained component numbers", NULL, bc->wall_comps, &bc->num_comps, &flg)); 143 144 { // Symmetry Boundary Conditions 145 const char *deprecated[3] = {"-bc_slip_x", "-bc_slip_y", "-bc_slip_z"}; 146 const char *flags[3] = {"-bc_symmetry_x", "-bc_symmetry_y", "-bc_symmetry_z"}; 147 PetscBool flg, has_symmetry = PETSC_FALSE; 148 149 for (PetscInt j = 0; j < 3; j++) { 150 bc->num_symmetry[j] = 16; 151 PetscCall(PetscOptionsDeprecated(deprecated[j], flags[j], "libCEED 0.12.0", 152 "Use -bc_symmetry_[x,y,z] for direct equivalency, or -bc_slip for weak, Riemann-based, direction-invariant " 153 "slip/no-penatration boundary conditions")); 154 PetscCall(PetscOptionsIntArray(flags[j], "Face IDs to apply symmetry BC", NULL, bc->symmetries[j], &bc->num_symmetry[j], &flg)); 155 if (bc->num_symmetry[j] > 0) has_symmetry = PETSC_TRUE; 156 } 157 158 // Error if wall and symmetry BCs are set on the same face 159 if (has_symmetry) { 160 for (PetscInt c = 0; c < 3; c++) { 161 for (PetscInt s = 0; s < bc->num_symmetry[c]; s++) { 162 for (PetscInt w = 0; w < bc->num_wall; w++) { 163 PetscCheck(bc->symmetries[c][s] != bc->walls[w], PETSC_COMM_SELF, PETSC_ERR_ARG_WRONG, 164 "Boundary condition already set on face %" PetscInt_FMT "!\n", bc->walls[w]); 165 } 166 } 167 } 168 } 169 } 170 app_ctx->wall_forces.num_wall = bc->num_wall; 171 PetscCall(PetscMalloc1(bc->num_wall, &app_ctx->wall_forces.walls)); 172 PetscCall(PetscArraycpy(app_ctx->wall_forces.walls, bc->walls, bc->num_wall)); 173 174 // Inflow BCs 175 bc->num_inflow = 16; 176 PetscCall(PetscOptionsIntArray("-bc_inflow", "Face IDs to apply inflow BC", NULL, bc->inflows, &bc->num_inflow, NULL)); 177 // Outflow BCs 178 bc->num_outflow = 16; 179 PetscCall(PetscOptionsIntArray("-bc_outflow", "Face IDs to apply outflow BC", NULL, bc->outflows, &bc->num_outflow, NULL)); 180 // Freestream BCs 181 bc->num_freestream = 16; 182 PetscCall(PetscOptionsIntArray("-bc_freestream", "Face IDs to apply freestream BC", NULL, bc->freestreams, &bc->num_freestream, NULL)); 183 184 bc->num_slip = 16; 185 PetscCall(PetscOptionsIntArray("-bc_slip", "Face IDs to apply slip BC", NULL, bc->slips, &bc->num_slip, NULL)); 186 187 // Statistics Options 188 app_ctx->turb_spanstats_collect_interval = 1; 189 PetscCall(PetscOptionsInt("-ts_monitor_turbulence_spanstats_collect_interval", "Number of timesteps between statistics collection", NULL, 190 app_ctx->turb_spanstats_collect_interval, &app_ctx->turb_spanstats_collect_interval, NULL)); 191 192 app_ctx->turb_spanstats_viewer_interval = -1; 193 PetscCall(PetscOptionsInt("-ts_monitor_turbulence_spanstats_viewer_interval", "Number of timesteps between statistics viewer writing", NULL, 194 app_ctx->turb_spanstats_viewer_interval, &app_ctx->turb_spanstats_viewer_interval, NULL)); 195 196 PetscCall(PetscOptionsViewer("-ts_monitor_turbulence_spanstats_viewer", "Viewer for the statistics", NULL, &app_ctx->turb_spanstats_viewer, 197 &app_ctx->turb_spanstats_viewer_format, &app_ctx->turb_spanstats_enable)); 198 199 PetscCall(PetscOptionsViewer("-ts_monitor_wall_force", "Viewer for force on each (no-slip) wall", NULL, &app_ctx->wall_forces.viewer, 200 &app_ctx->wall_forces.viewer_format, NULL)); 201 202 // SGS Model Options 203 app_ctx->sgs_model_type = SGS_MODEL_NONE; 204 PetscCall(PetscOptionsEnum("-sgs_model_type", "Subgrid Stress Model type", NULL, SGSModelTypes, (PetscEnum)app_ctx->sgs_model_type, 205 (PetscEnum *)&app_ctx->sgs_model_type, NULL)); 206 207 PetscCall(PetscOptionsBool("-diff_filter_monitor", "Enable differential filtering TSMonitor", NULL, app_ctx->diff_filter_monitor, 208 &app_ctx->diff_filter_monitor, NULL)); 209 210 // Mesh Transformation Options 211 app_ctx->mesh_transform_type = MESH_TRANSFORM_NONE; 212 PetscCall(PetscOptionsEnum("-mesh_transform", "Mesh transform to perform", NULL, MeshTransformTypes, (PetscEnum)app_ctx->mesh_transform_type, 213 (PetscEnum *)&app_ctx->mesh_transform_type, NULL)); 214 215 PetscCall( 216 PetscOptionsBool("-sgs_train_enable", "Enable Data-Driven SGS training", NULL, app_ctx->sgs_train_enable, &app_ctx->sgs_train_enable, NULL)); 217 218 PetscOptionsEnd(); 219 PetscFunctionReturn(PETSC_SUCCESS); 220 } 221