1 #if !defined(__SFPACK_H) 2 #define __SFPACK_H 3 4 #include <petsc/private/sfimpl.h> /*I "petscsf.h" I*/ 5 6 /* Optimization plans in packing(unpacking) for target processors. 7 8 Indirect accesses in packing like p[i] = u[idx[i]] are expensive and are not vectorization friendly. We 9 try to optimize them if we found cenrtain patterns among indices in idx[]. As a result, a pack might be 10 optimized into 1) a small number of contiguous memory copies; OR 2) one strided memory copy. 11 12 Each target has its own plan. n, the number of target processors, is nranks or niranks depending on the context. 13 */ 14 struct _n_PetscSFPackOpt { 15 PetscBool *optimized; /* [n] Is the packing to i-th target optimized? If yes, other fields give the opt plan */ 16 PetscInt *copy_offset; /* [n+1] We number all memory copies. Packing for target i is optimized into copies in [copy_offset[i],copy_offset[i+1]) */ 17 PetscInt *copy_start; /* [*] j-th copy starts at index copy_start[j] */ 18 PetscInt *copy_length; /* [*] with length copy_length[j] in unit of the <unit> used in for example, PetscSFReduceBegin(sf,unit,...) */ 19 PetscInt *stride_first; /* [n] If optimized[i] is TRUE but copy_offset[i] == copy_offset[i+1], then packing for remote i is strided. The first */ 20 PetscInt *stride_step; /* [n] index is stride_first[i], step is stride_step[i], */ 21 PetscInt *stride_n; /* [n] and total stride_n[i] steps */ 22 }; 23 24 typedef struct _n_PetscSFPack* PetscSFPack; 25 26 #define SFPACKHEADER \ 27 PetscErrorCode (*Pack) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,const void*,void*); \ 28 PetscErrorCode (*UnpackAndInsert)(PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 29 PetscErrorCode (*UnpackAndAdd) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 30 PetscErrorCode (*UnpackAndMin) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 31 PetscErrorCode (*UnpackAndMax) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 32 PetscErrorCode (*UnpackAndMinloc)(PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 33 PetscErrorCode (*UnpackAndMaxloc)(PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 34 PetscErrorCode (*UnpackAndMult) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 35 PetscErrorCode (*UnpackAndLAND) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 36 PetscErrorCode (*UnpackAndBAND) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 37 PetscErrorCode (*UnpackAndLOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 38 PetscErrorCode (*UnpackAndBOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 39 PetscErrorCode (*UnpackAndLXOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 40 PetscErrorCode (*UnpackAndBXOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*); \ 41 PetscErrorCode (*FetchAndInsert) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 42 PetscErrorCode (*FetchAndAdd) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 43 PetscErrorCode (*FetchAndMin) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 44 PetscErrorCode (*FetchAndMax) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 45 PetscErrorCode (*FetchAndMinloc) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 46 PetscErrorCode (*FetchAndMaxloc) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 47 PetscErrorCode (*FetchAndMult) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 48 PetscErrorCode (*FetchAndLAND) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 49 PetscErrorCode (*FetchAndBAND) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 50 PetscErrorCode (*FetchAndLOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 51 PetscErrorCode (*FetchAndBOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 52 PetscErrorCode (*FetchAndLXOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 53 PetscErrorCode (*FetchAndBXOR) (PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*); \ 54 PetscMPIInt tag; /* Each link has a tag so we can perform multiple SF ops at the same time */ \ 55 MPI_Datatype unit; \ 56 MPI_Datatype basicunit; /* unit is made of MPI builtin dataype basicunit */ \ 57 PetscBool isbuiltin; /* Is unit an MPI builtin datatype? If it is true, basicunit=unit, bs=1 */ \ 58 size_t unitbytes; /* Number of bytes in a unit */ \ 59 PetscInt bs; /* Number of basic units in a unit */ \ 60 const void *rkey,*lkey; /* rootdata and leafdata used as keys for operation */ \ 61 PetscSFPack next 62 63 /* An abstract class that defines a communication link, which includes how to 64 pack/unpack data. Subclasses may further contain fields for send/recv buffers, 65 MPI_Requests etc used in communication. 66 */ 67 struct _n_PetscSFPack { 68 SFPACKHEADER; 69 }; 70 71 PETSC_INTERN PetscErrorCode PetscSFPackGetInUse(PetscSF,MPI_Datatype,const void*,const void*,PetscCopyMode,PetscSFPack*); 72 PETSC_INTERN PetscErrorCode PetscSFPackReclaim(PetscSF,PetscSFPack*); 73 PETSC_INTERN PetscErrorCode PetscSFPackSetupType(PetscSFPack,MPI_Datatype); 74 PETSC_INTERN PetscErrorCode PetscSFPackGetUnpackAndOp(PetscSF,PetscSFPack,MPI_Op,PetscErrorCode (**UnpackAndOp)(PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,const void*)); 75 PETSC_INTERN PetscErrorCode PetscSFPackGetFetchAndOp(PetscSF,PetscSFPack,MPI_Op,PetscErrorCode (**FetchAndOp)(PetscInt,PetscInt,const PetscInt*,PetscInt,PetscSFPackOpt,void*,void*)); 76 PETSC_INTERN PetscErrorCode PetscSFPackSetupOptimization(PetscInt,const PetscInt*,const PetscInt*,PetscSFPackOpt*); 77 PETSC_INTERN PetscErrorCode PetscSFPackDestoryOptimization(PetscSFPackOpt *out); 78 PETSC_INTERN PetscErrorCode PetscSFPackSetErrorOnUnsupportedOverlap(PetscSF,MPI_Datatype,const void*,const void*); 79 80 #endif 81