1 #if !defined(PETSCLANDAU_H) 2 #define PETSCLANDAU_H 3 4 #include <petscdmplex.h> /*I "petscdmplex.h" I*/ 5 #include <petscts.h> 6 7 PETSC_EXTERN PetscErrorCode LandauPrintNorms(Vec, PetscInt); 8 PETSC_EXTERN PetscErrorCode LandauCreateVelocitySpace(MPI_Comm,PetscInt,const char[],Vec*,Mat*,DM*); 9 PETSC_EXTERN PetscErrorCode LandauDestroyVelocitySpace(DM*); 10 PETSC_EXTERN PetscErrorCode LandauAddMaxwellians(DM, Vec, PetscReal, PetscReal[], PetscReal[], void *); 11 PETSC_EXTERN PetscErrorCode LandauCreateMassMatrix(DM dm, Mat *Amat); 12 PETSC_EXTERN PetscErrorCode LandauIFunction(TS, PetscReal,Vec,Vec,Vec,void *); 13 PETSC_EXTERN PetscErrorCode LandauIJacobian(TS, PetscReal,Vec,Vec,PetscReal,Mat,Mat,void *); 14 15 /* the Fokker-Planck-Landau context */ 16 #if !defined(LANDAU_DIM) 17 #define LANDAU_DIM 2 18 #endif 19 20 #if !defined(LANDAU_MAX_SPECIES) 21 #if LANDAU_DIM==2 22 #define LANDAU_MAX_SPECIES 10 23 #else 24 #define LANDAU_MAX_SPECIES 3 25 #endif 26 #endif 27 28 #if !defined(LANDAU_MAX_Q) 29 #if defined(LANDAU_MAX_NQ) 30 #error"LANDAU_MAX_NQ but not LANDAU_MAX_Q. Use -DLANDAU_MAX_Q=4 for Q3 elements" 31 #endif 32 #if LANDAU_DIM==2 33 #define LANDAU_MAX_Q 5 34 #else 35 #define LANDAU_MAX_Q 3 36 #endif 37 #else 38 #undef LANDAU_MAX_NQ 39 #endif 40 41 #if LANDAU_DIM==2 42 #define LANDAU_MAX_Q_FACE LANDAU_MAX_Q 43 #define LANDAU_MAX_NQ (LANDAU_MAX_Q*LANDAU_MAX_Q) 44 #else 45 #define LANDAU_MAX_Q_FACE (LANDAU_MAX_Q*LANDAU_MAX_Q) 46 #define LANDAU_MAX_NQ (LANDAU_MAX_Q*LANDAU_MAX_Q*LANDAU_MAX_Q) 47 #endif 48 49 typedef enum {LANDAU_CUDA, LANDAU_KOKKOS, LANDAU_CPU} LandauDeviceType; 50 51 /* typedef PetscReal LandauIPReal; */ 52 /* typedef struct { */ 53 /* LandauIPReal *coefs; */ 54 /* int dim_,ns_,nip_; */ 55 /* } LandauIPFdF; */ 56 57 typedef struct { 58 void *invJ; // nip*dim*dim 59 void *D; // nq*nb*dim 60 void *B; // nq*nb 61 void *alpha; // ns 62 void *beta; // ns 63 void *invMass; // ns 64 void *mass_w; // nip 65 void *w; // nip 66 void *x; // nip 67 void *y; // nip 68 void *z; // nip 69 void *Eq_m; // ns - dynamic 70 void *f; // nip*Nf - dynamic (IP) 71 void *dfdx; // nip*Nf - dynamic (IP) 72 void *dfdy; // nip*Nf - dynamic (IP) 73 void *dfdz; // nip*Nf - dynamic (IP) 74 void *IPf; // Ncells*Nb*Nf - dynamic (vertex in cells) 75 void *ierr; 76 int dim_,ns_,nip_,nq_,nb_; 77 } LandauGeomData; 78 79 typedef struct { 80 PetscBool interpolate; /* Generate intermediate mesh elements */ 81 PetscBool gpu_assembly; 82 PetscFE fe[LANDAU_MAX_SPECIES]; 83 /* geometry */ 84 PetscReal i_radius; 85 PetscReal e_radius; 86 PetscInt num_sections; 87 PetscReal radius; 88 PetscReal re_radius; /* radius of refinement along v_perp=0, z>0 */ 89 PetscReal vperp0_radius1; /* radius of refinement along v_perp=0 */ 90 PetscReal vperp0_radius2; /* radius of refinement along v_perp=0 after origin AMR refinement */ 91 PetscBool sphere; 92 PetscBool inflate; 93 PetscInt numRERefine; /* refinement along v_perp=0, z > 0 */ 94 PetscInt nZRefine1; /* origin refinement after v_perp=0 refinement */ 95 PetscInt nZRefine2; /* origin refinement after origin AMR refinement */ 96 PetscInt maxRefIts; /* normal AMR - refine from origin */ 97 PetscInt postAMRRefine; /* uniform refinement of AMR */ 98 /* discretization - AMR */ 99 PetscErrorCode (*errorIndicator)(PetscInt, PetscReal, PetscReal [], PetscInt, const PetscInt[], const PetscScalar[], const PetscScalar[], PetscReal *, void *); 100 PetscReal refineTol[LANDAU_MAX_SPECIES]; 101 PetscReal coarsenTol[LANDAU_MAX_SPECIES]; 102 /* physics */ 103 PetscReal thermal_temps[LANDAU_MAX_SPECIES]; 104 PetscReal masses[LANDAU_MAX_SPECIES]; /* mass of each species */ 105 PetscReal charges[LANDAU_MAX_SPECIES]; /* charge of each species */ 106 PetscReal n[LANDAU_MAX_SPECIES]; /* number density of each species */ 107 PetscReal m_0; /* reference mass */ 108 PetscReal v_0; /* reference velocity */ 109 PetscReal n_0; /* reference number density */ 110 PetscReal t_0; /* reference time */ 111 PetscReal Ez; 112 PetscReal epsilon0; 113 PetscReal k; 114 PetscReal lnLam; 115 PetscReal electronShift; /* for tests */ 116 PetscInt num_species; 117 /* cache */ 118 Mat J; 119 Mat M; 120 Vec X; 121 /* derived type */ 122 void *data; 123 PetscBool aux_bool; /* helper */ 124 /* computing */ 125 LandauDeviceType deviceType; 126 PetscInt subThreadBlockSize; 127 PetscInt numConcurrency; /* number of SMs in Cuda to use */ 128 MPI_Comm comm; /* global communicator to use for errors and diagnostics */ 129 LandauGeomData *SData_d; /* static geometric data on device, but this pointer is a host pointer */ 130 double times[1]; 131 PetscBool init; 132 PetscBool use_matrix_mass; 133 DM dmv; 134 DM plex; 135 /* diagnostics */ 136 PetscInt verbose; 137 PetscLogEvent events[20]; 138 } LandauCtx; 139 140 typedef int LandauIdx; 141 typedef struct { 142 PetscReal scale; 143 LandauIdx gid; // Lanadu matrix index (<10,000) 144 } pointInterpolationP4est; 145 typedef struct _lP4estVertexMaps { 146 LandauIdx (*gIdx)[LANDAU_MAX_SPECIES][LANDAU_MAX_NQ]; // #elems * LANDAU_MAX_NQ (spoof for max , Nb) on device, 147 LandauIdx num_elements; 148 LandauIdx num_reduced; 149 LandauIdx num_face; // (Q or Q^2 for 3D) 150 LandauDeviceType deviceType; 151 PetscInt Nf; 152 PetscInt Nq; 153 pointInterpolationP4est (*c_maps)[LANDAU_MAX_Q_FACE]; 154 struct _lP4estVertexMaps*data; 155 void *vp1,*vp2,*vp3; 156 } P4estVertexMaps; 157 158 PETSC_EXTERN PetscErrorCode LandauCreateColoring(Mat, DM, PetscContainer *); 159 #if defined(PETSC_HAVE_CUDA) 160 PETSC_EXTERN PetscErrorCode LandauCUDAJacobian(DM, const PetscInt, PetscReal[], PetscScalar[], const PetscInt, const PetscScalar[], LandauGeomData *, const PetscInt, PetscReal, const PetscLogEvent[], Mat); 161 PETSC_EXTERN PetscErrorCode LandauCUDACreateMatMaps(P4estVertexMaps *, pointInterpolationP4est (*)[LANDAU_MAX_Q_FACE], PetscInt, PetscInt); 162 PETSC_EXTERN PetscErrorCode LandauCUDADestroyMatMaps(P4estVertexMaps *); 163 PETSC_EXTERN PetscErrorCode LandauCUDAStaticDataSet(DM, const PetscInt, PetscReal [], PetscReal [], PetscReal[], PetscReal[], PetscReal[], PetscReal[], PetscReal[], PetscReal[], PetscReal[], LandauGeomData *); 164 PETSC_EXTERN PetscErrorCode LandauCUDAStaticDataClear(LandauGeomData *); 165 #endif 166 #if defined(PETSC_HAVE_KOKKOS) 167 PETSC_EXTERN PetscErrorCode LandauKokkosJacobian(DM, const PetscInt, PetscReal[], PetscScalar[], const PetscInt, const PetscScalar[], LandauGeomData *, const PetscInt, PetscReal, const PetscLogEvent[], Mat); 168 PETSC_EXTERN PetscErrorCode LandauKokkosCreateMatMaps(P4estVertexMaps *, pointInterpolationP4est (*)[LANDAU_MAX_Q_FACE], PetscInt, PetscInt); 169 PETSC_EXTERN PetscErrorCode LandauKokkosDestroyMatMaps(P4estVertexMaps *); 170 PETSC_EXTERN PetscErrorCode LandauKokkosStaticDataSet(DM, const PetscInt, PetscReal [], PetscReal [], PetscReal[], PetscReal[], PetscReal[], PetscReal[], PetscReal[], PetscReal[], PetscReal[], LandauGeomData *); 171 PETSC_EXTERN PetscErrorCode LandauKokkosStaticDataClear(LandauGeomData *); 172 #endif 173 174 #endif /* PETSCLANDAU_H */ 175