1*ccaff030SJeremy L Thompson // Copyright (c) 2017, Lawrence Livermore National Security, LLC. Produced at 2*ccaff030SJeremy L Thompson // the Lawrence Livermore National Laboratory. LLNL-CODE-734707. All Rights 3*ccaff030SJeremy L Thompson // reserved. See files LICENSE and NOTICE for details. 4*ccaff030SJeremy L Thompson // 5*ccaff030SJeremy L Thompson // This file is part of CEED, a collection of benchmarks, miniapps, software 6*ccaff030SJeremy L Thompson // libraries and APIs for efficient high-order finite element and spectral 7*ccaff030SJeremy L Thompson // element discretizations for exascale applications. For more information and 8*ccaff030SJeremy L Thompson // source code availability see http://github.com/ceed. 9*ccaff030SJeremy L Thompson // 10*ccaff030SJeremy L Thompson // The CEED research is supported by the Exascale Computing Project 17-SC-20-SC, 11*ccaff030SJeremy L Thompson // a collaborative effort of two U.S. Department of Energy organizations (Office 12*ccaff030SJeremy L Thompson // of Science and the National Nuclear Security Administration) responsible for 13*ccaff030SJeremy L Thompson // the planning and preparation of a capable exascale ecosystem, including 14*ccaff030SJeremy L Thompson // software, applications, hardware, advanced system engineering and early 15*ccaff030SJeremy L Thompson // testbed platforms, in support of the nation's exascale computing imperative. 16*ccaff030SJeremy L Thompson 17*ccaff030SJeremy L Thompson #ifndef setup_h 18*ccaff030SJeremy L Thompson #define setup_h 19*ccaff030SJeremy L Thompson 20*ccaff030SJeremy L Thompson #include <stdbool.h> 21*ccaff030SJeremy L Thompson #include <string.h> 22*ccaff030SJeremy L Thompson 23*ccaff030SJeremy L Thompson #include <petsc.h> 24*ccaff030SJeremy L Thompson #include <petscdmplex.h> 25*ccaff030SJeremy L Thompson #include <petscksp.h> 26*ccaff030SJeremy L Thompson #include <petscfe.h> 27*ccaff030SJeremy L Thompson 28*ccaff030SJeremy L Thompson #include <ceed.h> 29*ccaff030SJeremy L Thompson 30*ccaff030SJeremy L Thompson #ifndef PHYSICS_STRUCT 31*ccaff030SJeremy L Thompson #define PHYSICS_STRUCT 32*ccaff030SJeremy L Thompson typedef struct Physics_private *Physics; 33*ccaff030SJeremy L Thompson struct Physics_private { 34*ccaff030SJeremy L Thompson CeedScalar nu; // Poisson's ratio 35*ccaff030SJeremy L Thompson CeedScalar E; // Young's Modulus 36*ccaff030SJeremy L Thompson }; 37*ccaff030SJeremy L Thompson #endif 38*ccaff030SJeremy L Thompson 39*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 40*ccaff030SJeremy L Thompson // Command Line Options 41*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 42*ccaff030SJeremy L Thompson // Problem options 43*ccaff030SJeremy L Thompson typedef enum { 44*ccaff030SJeremy L Thompson ELAS_LIN = 0, ELAS_HYPER_SS = 1, ELAS_HYPER_FS = 2 45*ccaff030SJeremy L Thompson } problemType; 46*ccaff030SJeremy L Thompson static const char *const problemTypes[] = {"linElas", 47*ccaff030SJeremy L Thompson "hyperSS", 48*ccaff030SJeremy L Thompson "hyperFS", 49*ccaff030SJeremy L Thompson "problemType","ELAS_",0 50*ccaff030SJeremy L Thompson }; 51*ccaff030SJeremy L Thompson static const char *const problemTypesForDisp[] = {"Linear elasticity", 52*ccaff030SJeremy L Thompson "Hyper elasticity small strain", 53*ccaff030SJeremy L Thompson "Hyper elasticity finite strain" 54*ccaff030SJeremy L Thompson }; 55*ccaff030SJeremy L Thompson 56*ccaff030SJeremy L Thompson // Forcing function options 57*ccaff030SJeremy L Thompson typedef enum { 58*ccaff030SJeremy L Thompson FORCE_NONE = 0, FORCE_CONST = 1, FORCE_MMS = 2 59*ccaff030SJeremy L Thompson } forcingType; 60*ccaff030SJeremy L Thompson static const char *const forcingTypes[] = {"none", 61*ccaff030SJeremy L Thompson "constant", 62*ccaff030SJeremy L Thompson "mms", 63*ccaff030SJeremy L Thompson "forcingType","FORCE_",0 64*ccaff030SJeremy L Thompson }; 65*ccaff030SJeremy L Thompson static const char *const forcingTypesForDisp[] = {"None", 66*ccaff030SJeremy L Thompson "Constant", 67*ccaff030SJeremy L Thompson "Manufactured solution" 68*ccaff030SJeremy L Thompson }; 69*ccaff030SJeremy L Thompson 70*ccaff030SJeremy L Thompson // Multigrid options 71*ccaff030SJeremy L Thompson typedef enum { 72*ccaff030SJeremy L Thompson MULTIGRID_LOGARITHMIC = 0, MULTIGRID_UNIFORM = 1, MULTIGRID_NONE = 2 73*ccaff030SJeremy L Thompson } multigridType; 74*ccaff030SJeremy L Thompson static const char *const multigridTypes [] = {"logarithmic", 75*ccaff030SJeremy L Thompson "uniform", 76*ccaff030SJeremy L Thompson "none", 77*ccaff030SJeremy L Thompson "multigridType","MULTIGRID",0 78*ccaff030SJeremy L Thompson }; 79*ccaff030SJeremy L Thompson static const char *const multigridTypesForDisp[] = {"P-multigrid, logarithmic coarsening", 80*ccaff030SJeremy L Thompson "P-multigrind, uniform coarsening", 81*ccaff030SJeremy L Thompson "No multigrid" 82*ccaff030SJeremy L Thompson }; 83*ccaff030SJeremy L Thompson 84*ccaff030SJeremy L Thompson typedef PetscErrorCode BCFunc(PetscInt, PetscReal, const PetscReal *, PetscInt, 85*ccaff030SJeremy L Thompson PetscScalar *, void *); 86*ccaff030SJeremy L Thompson // Note: These variables should be updated if additional boundary conditions 87*ccaff030SJeremy L Thompson // are added to boundary.c. 88*ccaff030SJeremy L Thompson BCFunc BCMMS, BCZero, BCClamp; 89*ccaff030SJeremy L Thompson 90*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 91*ccaff030SJeremy L Thompson // Structs 92*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 93*ccaff030SJeremy L Thompson // Units 94*ccaff030SJeremy L Thompson typedef struct Units_private *Units; 95*ccaff030SJeremy L Thompson struct Units_private { 96*ccaff030SJeremy L Thompson // Fundamental units 97*ccaff030SJeremy L Thompson PetscScalar meter; 98*ccaff030SJeremy L Thompson PetscScalar kilogram; 99*ccaff030SJeremy L Thompson PetscScalar second; 100*ccaff030SJeremy L Thompson // Derived unit 101*ccaff030SJeremy L Thompson PetscScalar Pascal; 102*ccaff030SJeremy L Thompson }; 103*ccaff030SJeremy L Thompson 104*ccaff030SJeremy L Thompson // Application context from user command line options 105*ccaff030SJeremy L Thompson typedef struct AppCtx_private *AppCtx; 106*ccaff030SJeremy L Thompson struct AppCtx_private { 107*ccaff030SJeremy L Thompson char ceedResource[PETSC_MAX_PATH_LEN]; // libCEED backend 108*ccaff030SJeremy L Thompson char ceedResourceFine[PETSC_MAX_PATH_LEN]; // libCEED for fine grid 109*ccaff030SJeremy L Thompson char meshFile[PETSC_MAX_PATH_LEN]; // exodusII mesh file 110*ccaff030SJeremy L Thompson PetscBool testMode; 111*ccaff030SJeremy L Thompson PetscBool viewSoln; 112*ccaff030SJeremy L Thompson problemType problemChoice; 113*ccaff030SJeremy L Thompson forcingType forcingChoice; 114*ccaff030SJeremy L Thompson multigridType multigridChoice; 115*ccaff030SJeremy L Thompson PetscInt degree; 116*ccaff030SJeremy L Thompson PetscInt numLevels; 117*ccaff030SJeremy L Thompson PetscInt *levelDegrees; 118*ccaff030SJeremy L Thompson PetscInt numIncrements; // Number of steps 119*ccaff030SJeremy L Thompson PetscInt bcZeroFaces[16], bcClampFaces[16]; 120*ccaff030SJeremy L Thompson PetscInt bcZeroCount, bcClampCount; 121*ccaff030SJeremy L Thompson PetscScalar bcClampMax; 122*ccaff030SJeremy L Thompson }; 123*ccaff030SJeremy L Thompson 124*ccaff030SJeremy L Thompson // Problem specific data 125*ccaff030SJeremy L Thompson typedef struct { 126*ccaff030SJeremy L Thompson CeedInt qdatasize; 127*ccaff030SJeremy L Thompson CeedQFunctionUser setupgeo, apply, jacob; 128*ccaff030SJeremy L Thompson const char *setupgeofname, *applyfname, *jacobfname; 129*ccaff030SJeremy L Thompson CeedQuadMode qmode; 130*ccaff030SJeremy L Thompson } problemData; 131*ccaff030SJeremy L Thompson 132*ccaff030SJeremy L Thompson // Data specific to each problem option 133*ccaff030SJeremy L Thompson problemData problemOptions[3]; 134*ccaff030SJeremy L Thompson 135*ccaff030SJeremy L Thompson // Forcing function data 136*ccaff030SJeremy L Thompson typedef struct { 137*ccaff030SJeremy L Thompson CeedQFunctionUser setupforcing; 138*ccaff030SJeremy L Thompson const char *setupforcingfname; 139*ccaff030SJeremy L Thompson } forcingData; 140*ccaff030SJeremy L Thompson 141*ccaff030SJeremy L Thompson forcingData forcingOptions[3]; 142*ccaff030SJeremy L Thompson 143*ccaff030SJeremy L Thompson // Data for PETSc Matshell 144*ccaff030SJeremy L Thompson typedef struct UserMult_private *UserMult; 145*ccaff030SJeremy L Thompson struct UserMult_private { 146*ccaff030SJeremy L Thompson MPI_Comm comm; 147*ccaff030SJeremy L Thompson DM dm; 148*ccaff030SJeremy L Thompson Vec Xloc, Yloc; 149*ccaff030SJeremy L Thompson CeedVector Xceed, Yceed; 150*ccaff030SJeremy L Thompson CeedOperator op; 151*ccaff030SJeremy L Thompson Ceed ceed; 152*ccaff030SJeremy L Thompson PetscScalar loadIncrement; 153*ccaff030SJeremy L Thompson }; 154*ccaff030SJeremy L Thompson 155*ccaff030SJeremy L Thompson // Data for Jacobian setup routine 156*ccaff030SJeremy L Thompson typedef struct FormJacobCtx_private *FormJacobCtx; 157*ccaff030SJeremy L Thompson struct FormJacobCtx_private { 158*ccaff030SJeremy L Thompson UserMult *jacobCtx; 159*ccaff030SJeremy L Thompson PetscInt numLevels; 160*ccaff030SJeremy L Thompson SNES snesCoarse; 161*ccaff030SJeremy L Thompson Mat *jacobMat, jacobMatCoarse; 162*ccaff030SJeremy L Thompson Vec Ucoarse; 163*ccaff030SJeremy L Thompson }; 164*ccaff030SJeremy L Thompson 165*ccaff030SJeremy L Thompson // Data for PETSc Prolongation/Restriction Matshell 166*ccaff030SJeremy L Thompson typedef struct UserMultProlongRestr_private *UserMultProlongRestr; 167*ccaff030SJeremy L Thompson struct UserMultProlongRestr_private { 168*ccaff030SJeremy L Thompson MPI_Comm comm; 169*ccaff030SJeremy L Thompson DM dmC, dmF; 170*ccaff030SJeremy L Thompson Vec locVecC, locVecF, multVec; 171*ccaff030SJeremy L Thompson CeedVector ceedVecC, ceedVecF; 172*ccaff030SJeremy L Thompson CeedOperator opProlong, opRestrict; 173*ccaff030SJeremy L Thompson Ceed ceed; 174*ccaff030SJeremy L Thompson }; 175*ccaff030SJeremy L Thompson 176*ccaff030SJeremy L Thompson // libCEED data struct for level 177*ccaff030SJeremy L Thompson typedef struct CeedData_private *CeedData; 178*ccaff030SJeremy L Thompson struct CeedData_private { 179*ccaff030SJeremy L Thompson Ceed ceed; 180*ccaff030SJeremy L Thompson CeedBasis basisx, basisu, basisCtoF; 181*ccaff030SJeremy L Thompson CeedElemRestriction Erestrictx, Erestrictu, Erestrictqdi, ErestrictGradui; 182*ccaff030SJeremy L Thompson CeedQFunction qfApply, qfJacob; 183*ccaff030SJeremy L Thompson CeedOperator opApply, opJacob, opRestrict, opProlong; 184*ccaff030SJeremy L Thompson CeedVector qdata, gradu, xceed, yceed, truesoln; 185*ccaff030SJeremy L Thompson }; 186*ccaff030SJeremy L Thompson 187*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 188*ccaff030SJeremy L Thompson // Process command line options 189*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 190*ccaff030SJeremy L Thompson // Process general command line options 191*ccaff030SJeremy L Thompson PetscErrorCode ProcessCommandLineOptions(MPI_Comm comm, AppCtx appCtx); 192*ccaff030SJeremy L Thompson 193*ccaff030SJeremy L Thompson // Process physics options 194*ccaff030SJeremy L Thompson PetscErrorCode ProcessPhysics(MPI_Comm comm, Physics phys, Units units); 195*ccaff030SJeremy L Thompson 196*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 197*ccaff030SJeremy L Thompson // Setup DM 198*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 199*ccaff030SJeremy L Thompson PetscErrorCode CreateBCLabel(DM dm, const char name[]); 200*ccaff030SJeremy L Thompson 201*ccaff030SJeremy L Thompson // Create FE by degree 202*ccaff030SJeremy L Thompson PetscErrorCode PetscFECreateByDegree(DM dm, PetscInt dim, PetscInt Nc, 203*ccaff030SJeremy L Thompson PetscBool isSimplex, const char prefix[], 204*ccaff030SJeremy L Thompson PetscInt order, PetscFE *fem); 205*ccaff030SJeremy L Thompson 206*ccaff030SJeremy L Thompson // Read mesh and distribute DM in parallel 207*ccaff030SJeremy L Thompson PetscErrorCode CreateDistributedDM(MPI_Comm comm, AppCtx appCtx, DM *dm); 208*ccaff030SJeremy L Thompson 209*ccaff030SJeremy L Thompson // Setup DM with FE space of appropriate degree 210*ccaff030SJeremy L Thompson PetscErrorCode SetupDMByDegree(DM dm, AppCtx appCtx, PetscInt order, 211*ccaff030SJeremy L Thompson PetscInt ncompu); 212*ccaff030SJeremy L Thompson 213*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 214*ccaff030SJeremy L Thompson // libCEED Functions 215*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 216*ccaff030SJeremy L Thompson // Destroy libCEED objects 217*ccaff030SJeremy L Thompson PetscErrorCode CeedDataDestroy(CeedInt level, CeedData data); 218*ccaff030SJeremy L Thompson 219*ccaff030SJeremy L Thompson // Get libCEED restriction data from DMPlex 220*ccaff030SJeremy L Thompson PetscErrorCode CreateRestrictionPlex(Ceed ceed, CeedInterlaceMode imode, 221*ccaff030SJeremy L Thompson CeedInt P, CeedInt ncomp, 222*ccaff030SJeremy L Thompson CeedElemRestriction *Erestrict, DM dm); 223*ccaff030SJeremy L Thompson 224*ccaff030SJeremy L Thompson // Set up libCEED for a given degree 225*ccaff030SJeremy L Thompson PetscErrorCode SetupLibceedFineLevel(DM dm, Ceed ceed, AppCtx appCtx, 226*ccaff030SJeremy L Thompson Physics phys, CeedData *data, 227*ccaff030SJeremy L Thompson PetscInt fineLevel, PetscInt ncompu, 228*ccaff030SJeremy L Thompson PetscInt Ugsz, PetscInt Ulocsz, 229*ccaff030SJeremy L Thompson CeedVector forceCeed, 230*ccaff030SJeremy L Thompson CeedQFunction qfRestrict, 231*ccaff030SJeremy L Thompson CeedQFunction qfProlong); 232*ccaff030SJeremy L Thompson 233*ccaff030SJeremy L Thompson // Set up libCEED for a given degree 234*ccaff030SJeremy L Thompson PetscErrorCode SetupLibceedLevel(DM dm, Ceed ceed, AppCtx appCtx, Physics phys, 235*ccaff030SJeremy L Thompson CeedData *data, PetscInt level, 236*ccaff030SJeremy L Thompson PetscInt ncompu, PetscInt Ugsz, 237*ccaff030SJeremy L Thompson PetscInt Ulocsz, CeedVector forceCeed, 238*ccaff030SJeremy L Thompson CeedQFunction qfRestrict, 239*ccaff030SJeremy L Thompson CeedQFunction qfProlong); 240*ccaff030SJeremy L Thompson 241*ccaff030SJeremy L Thompson // Setup context data for Jacobian evaluation 242*ccaff030SJeremy L Thompson PetscErrorCode SetupJacobianCtx(MPI_Comm comm, AppCtx appCtx, DM dm, Vec V, 243*ccaff030SJeremy L Thompson Vec Vloc, CeedData ceedData, Ceed ceed, 244*ccaff030SJeremy L Thompson UserMult jacobianCtx); 245*ccaff030SJeremy L Thompson 246*ccaff030SJeremy L Thompson // Setup context data for prolongation and restriction operators 247*ccaff030SJeremy L Thompson PetscErrorCode SetupProlongRestrictCtx(MPI_Comm comm, DM dmC, DM dmF, Vec VF, 248*ccaff030SJeremy L Thompson Vec VlocC, Vec VlocF, CeedData ceedDataC, 249*ccaff030SJeremy L Thompson CeedData ceedDataF, Ceed ceed, 250*ccaff030SJeremy L Thompson UserMultProlongRestr prolongRestrCtx); 251*ccaff030SJeremy L Thompson 252*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 253*ccaff030SJeremy L Thompson // Jacobian setup 254*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 255*ccaff030SJeremy L Thompson PetscErrorCode FormJacobian(SNES snes, Vec U, Mat J, Mat Jpre, void *ctx); 256*ccaff030SJeremy L Thompson 257*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 258*ccaff030SJeremy L Thompson // SNES Monitor 259*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 260*ccaff030SJeremy L Thompson PetscErrorCode ViewSolution(MPI_Comm comm, Vec U, PetscInt increment, 261*ccaff030SJeremy L Thompson PetscScalar loadIncrement); 262*ccaff030SJeremy L Thompson 263*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 264*ccaff030SJeremy L Thompson // libCEED Operators for MatShell 265*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 266*ccaff030SJeremy L Thompson // This function uses libCEED to compute the local action of an operator 267*ccaff030SJeremy L Thompson PetscErrorCode ApplyLocalCeedOp(Vec X, Vec Y, UserMult user); 268*ccaff030SJeremy L Thompson 269*ccaff030SJeremy L Thompson // This function uses libCEED to compute the non-linear residual 270*ccaff030SJeremy L Thompson PetscErrorCode FormResidual_Ceed(SNES snes, Vec X, Vec Y, void *ctx); 271*ccaff030SJeremy L Thompson 272*ccaff030SJeremy L Thompson // This function uses libCEED to apply the Jacobian for assembly via a SNES 273*ccaff030SJeremy L Thompson PetscErrorCode ApplyJacobianCoarse_Ceed(SNES snes, Vec X, Vec Y, void *ctx); 274*ccaff030SJeremy L Thompson 275*ccaff030SJeremy L Thompson // This function uses libCEED to compute the action of the Jacobian 276*ccaff030SJeremy L Thompson PetscErrorCode ApplyJacobian_Ceed(Mat A, Vec X, Vec Y); 277*ccaff030SJeremy L Thompson 278*ccaff030SJeremy L Thompson // This function uses libCEED to compute the action of the prolongation operator 279*ccaff030SJeremy L Thompson PetscErrorCode Prolong_Ceed(Mat A, Vec X, Vec Y); 280*ccaff030SJeremy L Thompson 281*ccaff030SJeremy L Thompson // This function uses libCEED to compute the action of the restriction operator 282*ccaff030SJeremy L Thompson PetscErrorCode Restrict_Ceed(Mat A, Vec X, Vec Y); 283*ccaff030SJeremy L Thompson // This function returns the computed diagonal of the operator 284*ccaff030SJeremy L Thompson PetscErrorCode GetDiag_Ceed(Mat A, Vec D); 285*ccaff030SJeremy L Thompson 286*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 287*ccaff030SJeremy L Thompson // Boundary Functions 288*ccaff030SJeremy L Thompson // ----------------------------------------------------------------------------- 289*ccaff030SJeremy L Thompson // Note: If additional boundary conditions are added, an update is needed in 290*ccaff030SJeremy L Thompson // elasticity.h for the boundaryOptions variable. 291*ccaff030SJeremy L Thompson 292*ccaff030SJeremy L Thompson // BCMMS - boundary function 293*ccaff030SJeremy L Thompson // Values on all points of the mesh is set based on given solution below 294*ccaff030SJeremy L Thompson // for u[0], u[1], u[2] 295*ccaff030SJeremy L Thompson PetscErrorCode BCMMS(PetscInt dim, PetscReal loadIncrement, 296*ccaff030SJeremy L Thompson const PetscReal coords[], PetscInt ncompu, 297*ccaff030SJeremy L Thompson PetscScalar *u, void *ctx); 298*ccaff030SJeremy L Thompson 299*ccaff030SJeremy L Thompson // BCZero - fix boundary values at zero 300*ccaff030SJeremy L Thompson PetscErrorCode BCZero(PetscInt dim, PetscReal loadIncrement, 301*ccaff030SJeremy L Thompson const PetscReal coords[], PetscInt ncompu, 302*ccaff030SJeremy L Thompson PetscScalar *u, void *ctx); 303*ccaff030SJeremy L Thompson 304*ccaff030SJeremy L Thompson // BCClamp - fix boundary values at fraction of load increment 305*ccaff030SJeremy L Thompson PetscErrorCode BCBend1_ss(PetscInt dim, PetscReal loadIncrement, 306*ccaff030SJeremy L Thompson const PetscReal coords[], PetscInt ncompu, 307*ccaff030SJeremy L Thompson PetscScalar *u, void *ctx); 308*ccaff030SJeremy L Thompson 309*ccaff030SJeremy L Thompson #endif //setup_h 310