1a4963045SJacob Faibussowitsch #pragma once 20c312b8eSJed Brown 3ce78bad3SBarry Smith /* MANSEC = Vec */ 4ac09b921SBarry Smith /* SUBMANSEC = PetscSF */ 5ac09b921SBarry Smith 60c312b8eSJed Brown /*S 7*2f04c522SBarry Smith PetscSF - PETSc object for managing the communication of certain entries of arrays and `Vec` between MPI processes. 80c312b8eSJed Brown 90c312b8eSJed Brown Level: intermediate 100c312b8eSJed Brown 1187497f52SBarry Smith `PetscSF` uses the concept of star forests to indicate and determine the communication patterns concisely and efficiently. 12af27ebaaSBarry Smith A star <https://en.wikipedia.org/wiki/Star_(graph_theory)> forest is simply a collection of trees of height 1. The leave nodes represent 130c312b8eSJed Brown "ghost locations" for the root nodes. 140c312b8eSJed Brown 15*2f04c522SBarry Smith The standard usage paradigm for `PetscSF` is to provide the communication pattern with `PetscSFSetGraph()` or `PetscSFSetGraphWithPattern()` and 16*2f04c522SBarry Smith then perform the communication using `PetscSFBcastBegin()` and `PetscSFBcastEnd()`, `PetscSFReduceBegin()` and `PetscSFReduceEnd()`. 17*2f04c522SBarry Smith 18*2f04c522SBarry Smith .seealso: [](sec_petscsf), `PetscSFCreate()`, `PetscSFSetGraph()`, `PetscSFSetGraphWithPattern()`, `PetscSFBcastBegin()`, `PetscSFBcastEnd()`, 19*2f04c522SBarry Smith `PetscSFReduceBegin()`, `PetscSFReduceEnd()`, `VecScatter`, `VecScatterCreate()` 200c312b8eSJed Brown S*/ 210c312b8eSJed Brown typedef struct _p_PetscSF *PetscSF; 220c312b8eSJed Brown 2397929ea7SJunchao Zhang /*J 24*2f04c522SBarry Smith PetscSFType - String with the name of a `PetscSF` type. Each `PetscSFType` uses different mechanisms to perform the communication. 2597929ea7SJunchao Zhang 2697929ea7SJunchao Zhang Level: beginner 2797929ea7SJunchao Zhang 28*2f04c522SBarry Smith Available Types: 29*2f04c522SBarry Smith + `PETSCSFBASIC` - use MPI sends and receives 30*2f04c522SBarry Smith . `PETSCSFNEIGHBOR` - use MPI_Neighbor operations 31*2f04c522SBarry Smith . `PETSCSFALLGATHERV` - use MPI_Allgatherv operations 32*2f04c522SBarry Smith . `PETSCSFALLGATHER` - use MPI_Allgather operations 33*2f04c522SBarry Smith . `PETSCSFGATHERV` - use MPI_Igatherv and MPI_Iscatterv operations 34*2f04c522SBarry Smith . `PETSCSFGATHER` - use MPI_Igather and MPI_Iscatter operations 35*2f04c522SBarry Smith . `PETSCSFALLTOALL` - use MPI_Ialltoall operations 36*2f04c522SBarry Smith - `PETSCSFWINDOW` - use MPI_Win operations 37*2f04c522SBarry Smith 38*2f04c522SBarry Smith Note: 39*2f04c522SBarry Smith Some `PetscSFType` only provide specialized code for a subset of the `PetscSF` operations and use `PETSCSFBASIC` for the others. 40*2f04c522SBarry Smith 41*2f04c522SBarry Smith .seealso: [](sec_petscsf), `PetscSFSetType()`, `PetscSF` 4297929ea7SJunchao Zhang J*/ 4397929ea7SJunchao Zhang typedef const char *PetscSFType; 4420662ed9SBarry Smith #define PETSCSFBASIC "basic" 4520662ed9SBarry Smith #define PETSCSFNEIGHBOR "neighbor" 4620662ed9SBarry Smith #define PETSCSFALLGATHERV "allgatherv" 4720662ed9SBarry Smith #define PETSCSFALLGATHER "allgather" 4820662ed9SBarry Smith #define PETSCSFGATHERV "gatherv" 4920662ed9SBarry Smith #define PETSCSFGATHER "gather" 5020662ed9SBarry Smith #define PETSCSFALLTOALL "alltoall" 5120662ed9SBarry Smith #define PETSCSFWINDOW "window" 5297929ea7SJunchao Zhang 53799f573fSMatthew G. Knepley /*S 54*2f04c522SBarry Smith PetscSFNode - specifier of MPI rank owner and local index for array or `Vec` entry locations that are to be communicated with a `PetscSF` 55799f573fSMatthew G. Knepley 56799f573fSMatthew G. Knepley Level: beginner 57799f573fSMatthew G. Knepley 5838ab3f8aSBarry Smith Sample Usage: 5916a05f60SBarry Smith .vb 6016a05f60SBarry Smith PetscSFNode *remote; 6116a05f60SBarry Smith PetscCall(PetscMalloc1(nleaves,&remote)); 6216a05f60SBarry Smith for (i=0; i<size; i++) { 6316a05f60SBarry Smith remote[i].rank = i; 6416a05f60SBarry Smith remote[i].index = rank; 6516a05f60SBarry Smith } 6616a05f60SBarry Smith .ve 6738ab3f8aSBarry Smith 6838ab3f8aSBarry Smith Sample Fortran Usage: 6916a05f60SBarry Smith .vb 7016a05f60SBarry Smith type(PetscSFNode) remote(6) 7116a05f60SBarry Smith remote(1)%rank = modulo(rank+size-1,size) 7216a05f60SBarry Smith remote(1)%index = 1 * stride 7316a05f60SBarry Smith .ve 7438ab3f8aSBarry Smith 756497c311SBarry Smith Notes: 766497c311SBarry Smith Use `MPIU_SF_NODE` when performing MPI operations on arrays of `PetscSFNode` 776497c311SBarry Smith 786497c311SBarry Smith Generally the values of `rank` should be in $[ 0,size)$ and the value of `index` greater than or equal to 0, but there are some situations that violate this. 796497c311SBarry Smith 80*2f04c522SBarry Smith .seealso: [](sec_petscsf), `PetscSF`, `PetscSFSetGraph()` 81799f573fSMatthew G. Knepley S*/ 82799f573fSMatthew G. Knepley typedef struct { 83*2f04c522SBarry Smith PetscInt rank; /* MPI rank of owner */ 84799f573fSMatthew G. Knepley PetscInt index; /* Index of node on rank */ 85799f573fSMatthew G. Knepley } PetscSFNode; 86799f573fSMatthew G. Knepley 876497c311SBarry Smith #define MPIU_SF_NODE MPIU_2INT 886497c311SBarry Smith 89ce78bad3SBarry Smith typedef enum { 90ce78bad3SBarry Smith PETSCSF_ROOT2LEAF = 0, 91ce78bad3SBarry Smith PETSCSF_LEAF2ROOT = 1 92ce78bad3SBarry Smith } PetscSFDirection; 93ce78bad3SBarry Smith typedef enum { 94ce78bad3SBarry Smith PETSCSF_BCAST = 0, 95ce78bad3SBarry Smith PETSCSF_REDUCE = 1, 96ce78bad3SBarry Smith PETSCSF_FETCH = 2 97ce78bad3SBarry Smith } PetscSFOperation; 98ce78bad3SBarry Smith /* When doing device-aware MPI, a backend refers to the SF/device interface */ 99ce78bad3SBarry Smith typedef enum { 100ce78bad3SBarry Smith PETSCSF_BACKEND_INVALID = 0, 101ce78bad3SBarry Smith PETSCSF_BACKEND_CUDA = 1, 102ce78bad3SBarry Smith PETSCSF_BACKEND_HIP = 2, 103ce78bad3SBarry Smith PETSCSF_BACKEND_KOKKOS = 3 104ce78bad3SBarry Smith } PetscSFBackend; 105ce78bad3SBarry Smith typedef struct _n_PetscSFLink *PetscSFLink; 106ce78bad3SBarry Smith 10797929ea7SJunchao Zhang /*S 10897929ea7SJunchao Zhang VecScatter - Object used to manage communication of data 1090b4b7b1cSBarry Smith between vectors in parallel or between parallel and sequential vectors. Manages both scatters and gathers 11097929ea7SJunchao Zhang 11197929ea7SJunchao Zhang Level: beginner 11297929ea7SJunchao Zhang 113ce78bad3SBarry Smith Note: 114*2f04c522SBarry Smith This is an alias for `PetscSF`. 115ce78bad3SBarry Smith 116*2f04c522SBarry Smith .seealso: [](sec_petscsf), `Vec`, `PetscSF`, `VecScatterCreate()`, `VecScatterBegin()`, `VecScatterEnd()` 11797929ea7SJunchao Zhang S*/ 11897929ea7SJunchao Zhang typedef PetscSF VecScatter; 11997929ea7SJunchao Zhang 12097929ea7SJunchao Zhang /*J 12197929ea7SJunchao Zhang VecScatterType - String with the name of a PETSc vector scatter type 12297929ea7SJunchao Zhang 12397929ea7SJunchao Zhang Level: beginner 12497929ea7SJunchao Zhang 125ce78bad3SBarry Smith Note: 126ce78bad3SBarry Smith This is an alias for `PetscSFType` 127ce78bad3SBarry Smith 128*2f04c522SBarry Smith .seealso: [](sec_petscsf), `PetscSFType`, `VecScatterSetType()`, `VecScatter`, `VecScatterCreate()`, `VecScatterDestroy()` 12997929ea7SJunchao Zhang J*/ 13097929ea7SJunchao Zhang typedef PetscSFType VecScatterType; 131