xref: /petsc/src/ksp/pc/impls/fieldsplit/fieldsplit.c (revision b94d7ded0a05f1bbd5e48daa6f92b28259c75b44)
1af0996ceSBarry Smith #include <petsc/private/pcimpl.h>     /*I "petscpc.h" I*/
2a80b646eSBarry Smith #include <petsc/private/kspimpl.h>    /*  This is needed to provide the appropriate PETSC_EXTERN for KSP_Solve_FS ....*/
31e25c274SJed Brown #include <petscdm.h>
40971522cSBarry Smith 
50a545947SLisandro Dalcin const char *const PCFieldSplitSchurPreTypes[] = {"SELF","SELFP","A11","USER","FULL","PCFieldSplitSchurPreType","PC_FIELDSPLIT_SCHUR_PRE_",NULL};
60a545947SLisandro Dalcin const char *const PCFieldSplitSchurFactTypes[] = {"DIAG","LOWER","UPPER","FULL","PCFieldSplitSchurFactType","PC_FIELDSPLIT_SCHUR_FACT_",NULL};
7c5d2311dSJed Brown 
89df09d43SBarry Smith PetscLogEvent KSP_Solve_FS_0,KSP_Solve_FS_1,KSP_Solve_FS_S,KSP_Solve_FS_U,KSP_Solve_FS_L,KSP_Solve_FS_2,KSP_Solve_FS_3,KSP_Solve_FS_4;
99df09d43SBarry Smith 
100971522cSBarry Smith typedef struct _PC_FieldSplitLink *PC_FieldSplitLink;
110971522cSBarry Smith struct _PC_FieldSplitLink {
1269a612a9SBarry Smith   KSP               ksp;
13443836d0SMatthew G Knepley   Vec               x,y,z;
14db4c96c1SJed Brown   char              *splitname;
150971522cSBarry Smith   PetscInt          nfields;
165d4c12cdSJungho Lee   PetscInt          *fields,*fields_col;
171b9fc7fcSBarry Smith   VecScatter        sctx;
18f5f0d762SBarry Smith   IS                is,is_col;
1951f519a2SBarry Smith   PC_FieldSplitLink next,previous;
209df09d43SBarry Smith   PetscLogEvent     event;
215ddf11f8SNicolas Barnafi 
225ddf11f8SNicolas Barnafi   /* Used only when setting coordinates with PCSetCoordinates */
235ddf11f8SNicolas Barnafi   PetscInt dim;
245ddf11f8SNicolas Barnafi   PetscInt ndofs;
255ddf11f8SNicolas Barnafi   PetscReal *coords;
260971522cSBarry Smith };
270971522cSBarry Smith 
280971522cSBarry Smith typedef struct {
2968dd23aaSBarry Smith   PCCompositeType type;
30ace3abfcSBarry Smith   PetscBool       defaultsplit;                    /* Flag for a system with a set of 'k' scalar fields with the same layout (and bs = k) */
31ace3abfcSBarry Smith   PetscBool       splitdefined;                    /* Flag is set after the splits have been defined, to prevent more splits from being added */
3230ad9308SMatthew Knepley   PetscInt        bs;                              /* Block size for IS and Mat structures */
3330ad9308SMatthew Knepley   PetscInt        nsplits;                         /* Number of field divisions defined */
3479416396SBarry Smith   Vec             *x,*y,w1,w2;
35519d70e2SJed Brown   Mat             *mat;                            /* The diagonal block for each split */
36519d70e2SJed Brown   Mat             *pmat;                           /* The preconditioning diagonal block for each split */
3730ad9308SMatthew Knepley   Mat             *Afield;                         /* The rows of the matrix associated with each split */
38ace3abfcSBarry Smith   PetscBool       issetup;
392fa5cd67SKarl Rupp 
4030ad9308SMatthew Knepley   /* Only used when Schur complement preconditioning is used */
4130ad9308SMatthew Knepley   Mat                       B;                     /* The (0,1) block */
4230ad9308SMatthew Knepley   Mat                       C;                     /* The (1,0) block */
43443836d0SMatthew G Knepley   Mat                       schur;                 /* The Schur complement S = A11 - A10 A00^{-1} A01, the KSP here, kspinner, is H_1 in [El08] */
44a7476a74SDmitry Karpeev   Mat                       schurp;                /* Assembled approximation to S built by MatSchurComplement to be used as a preconditioning matrix when solving with S */
45084e4875SJed Brown   Mat                       schur_user;            /* User-provided preconditioning matrix for the Schur complement */
46084e4875SJed Brown   PCFieldSplitSchurPreType  schurpre;              /* Determines which preconditioning matrix is used for the Schur complement */
47c9c6ffaaSJed Brown   PCFieldSplitSchurFactType schurfactorization;
4830ad9308SMatthew Knepley   KSP                       kspschur;              /* The solver for S */
49443836d0SMatthew G Knepley   KSP                       kspupper;              /* The solver for A in the upper diagonal part of the factorization (H_2 in [El08]) */
50c096484dSStefano Zampini   PetscScalar               schurscale;            /* Scaling factor for the Schur complement solution with DIAG factorization */
51c096484dSStefano Zampini 
52a51937d4SCarola Kruse   /* Only used when Golub-Kahan bidiagonalization preconditioning is used */
53a51937d4SCarola Kruse   Mat                       H;                     /* The modified matrix H = A00 + nu*A01*A01'              */
54a51937d4SCarola Kruse   PetscReal                 gkbtol;                /* Stopping tolerance for lower bound estimate            */
55a51937d4SCarola Kruse   PetscInt                  gkbdelay;              /* The delay window for the stopping criterion            */
56a51937d4SCarola Kruse   PetscReal                 gkbnu;                 /* Parameter for augmented Lagrangian H = A + nu*A01*A01' */
57a51937d4SCarola Kruse   PetscInt                  gkbmaxit;              /* Maximum number of iterations for outer loop            */
58de482cd7SCarola Kruse   PetscBool                 gkbmonitor;            /* Monitor for gkb iterations and the lower bound error   */
59de482cd7SCarola Kruse   PetscViewer               gkbviewer;             /* Viewer context for gkbmonitor                          */
60e071a0a4SCarola Kruse   Vec                       u,v,d,Hu;              /* Work vectors for the GKB algorithm                     */
61e071a0a4SCarola Kruse   PetscScalar               *vecz;                 /* Contains intermediate values, eg for lower bound       */
62a51937d4SCarola Kruse 
6397bbdb24SBarry Smith   PC_FieldSplitLink         head;
646dbb499eSCian Wilson   PetscBool                 isrestrict;             /* indicates PCFieldSplitRestrictIS() has been last called on this object, hack */
65c1570756SJed Brown   PetscBool                 suboptionsset;          /* Indicates that the KSPSetFromOptions() has been called on the sub-KSPs */
664ab8060aSDmitry Karpeev   PetscBool                 dm_splits;              /* Whether to use DMCreateFieldDecomposition() whenever possible */
67c84da90fSDmitry Karpeev   PetscBool                 diag_use_amat;          /* Whether to extract diagonal matrix blocks from Amat, rather than Pmat (weaker than -pc_use_amat) */
68c84da90fSDmitry Karpeev   PetscBool                 offdiag_use_amat;       /* Whether to extract off-diagonal matrix blocks from Amat, rather than Pmat (weaker than -pc_use_amat) */
697b752e3dSPatrick Sanan   PetscBool                 detect;                 /* Whether to form 2-way split by finding zero diagonal entries */
705ddf11f8SNicolas Barnafi   PetscBool                 coordinates_set;        /* Whether PCSetCoordinates has been called */
710971522cSBarry Smith } PC_FieldSplit;
720971522cSBarry Smith 
7316913363SBarry Smith /*
7495452b02SPatrick Sanan     Notes:
7595452b02SPatrick Sanan     there is no particular reason that pmat, x, and y are stored as arrays in PC_FieldSplit instead of
7616913363SBarry Smith    inside PC_FieldSplitLink, just historical. If you want to be able to add new fields after already using the
7716913363SBarry Smith    PC you could change this.
7816913363SBarry Smith */
79084e4875SJed Brown 
80e6cab6aaSJed Brown /* This helper is so that setting a user-provided preconditioning matrix is orthogonal to choosing to use it.  This way the
81084e4875SJed Brown * application-provided FormJacobian can provide this matrix without interfering with the user's (command-line) choices. */
82084e4875SJed Brown static Mat FieldSplitSchurPre(PC_FieldSplit *jac)
83084e4875SJed Brown {
84084e4875SJed Brown   switch (jac->schurpre) {
85084e4875SJed Brown   case PC_FIELDSPLIT_SCHUR_PRE_SELF: return jac->schur;
86a7476a74SDmitry Karpeev   case PC_FIELDSPLIT_SCHUR_PRE_SELFP: return jac->schurp;
87e87fab1bSBarry Smith   case PC_FIELDSPLIT_SCHUR_PRE_A11: return jac->pmat[1];
88e74569cdSMatthew G. Knepley   case PC_FIELDSPLIT_SCHUR_PRE_FULL: /* We calculate this and store it in schur_user */
89084e4875SJed Brown   case PC_FIELDSPLIT_SCHUR_PRE_USER: /* Use a user-provided matrix if it is given, otherwise diagonal block */
90084e4875SJed Brown   default:
91084e4875SJed Brown     return jac->schur_user ? jac->schur_user : jac->pmat[1];
92084e4875SJed Brown   }
93084e4875SJed Brown }
94084e4875SJed Brown 
959804daf3SBarry Smith #include <petscdraw.h>
960971522cSBarry Smith static PetscErrorCode PCView_FieldSplit(PC pc,PetscViewer viewer)
970971522cSBarry Smith {
980971522cSBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
99d9884438SBarry Smith   PetscBool         iascii,isdraw;
1000971522cSBarry Smith   PetscInt          i,j;
1015a9f2f41SSatish Balay   PC_FieldSplitLink ilink = jac->head;
1020971522cSBarry Smith 
1030971522cSBarry Smith   PetscFunctionBegin;
1049566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii));
1059566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw));
1060971522cSBarry Smith   if (iascii) {
1072c9966d7SBarry Smith     if (jac->bs > 0) {
10863a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  FieldSplit with %s composition: total splits = %" PetscInt_FMT ", blocksize = %" PetscInt_FMT "\n",PCCompositeTypes[jac->type],jac->nsplits,jac->bs));
1092c9966d7SBarry Smith     } else {
11063a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  FieldSplit with %s composition: total splits = %" PetscInt_FMT "\n",PCCompositeTypes[jac->type],jac->nsplits));
1112c9966d7SBarry Smith     }
112f5236f50SJed Brown     if (pc->useAmat) {
1139566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for blocks\n"));
114a3df900dSMatthew G Knepley     }
115c84da90fSDmitry Karpeev     if (jac->diag_use_amat) {
1169566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for diagonal blocks\n"));
117c84da90fSDmitry Karpeev     }
118c84da90fSDmitry Karpeev     if (jac->offdiag_use_amat) {
1199566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for off-diagonal blocks\n"));
120c84da90fSDmitry Karpeev     }
1219566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(viewer,"  Solver info for each split is in the following KSP objects:\n"));
1220971522cSBarry Smith     for (i=0; i<jac->nsplits; i++) {
1231ab39975SBarry Smith       if (ilink->fields) {
12463a3b9bcSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"Split number %" PetscInt_FMT " Fields ",i));
1259566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIUseTabs(viewer,PETSC_FALSE));
1265a9f2f41SSatish Balay         for (j=0; j<ilink->nfields; j++) {
12779416396SBarry Smith           if (j > 0) {
1289566063dSJacob Faibussowitsch             PetscCall(PetscViewerASCIIPrintf(viewer,","));
12979416396SBarry Smith           }
13063a3b9bcSJacob Faibussowitsch           PetscCall(PetscViewerASCIIPrintf(viewer," %" PetscInt_FMT,ilink->fields[j]));
1310971522cSBarry Smith         }
1329566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"\n"));
1339566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIUseTabs(viewer,PETSC_TRUE));
1341ab39975SBarry Smith       } else {
13563a3b9bcSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"Split number %" PetscInt_FMT " Defined by IS\n",i));
1361ab39975SBarry Smith       }
1379566063dSJacob Faibussowitsch       PetscCall(KSPView(ilink->ksp,viewer));
1385a9f2f41SSatish Balay       ilink = ilink->next;
1390971522cSBarry Smith     }
1402fa5cd67SKarl Rupp   }
1412fa5cd67SKarl Rupp 
1422fa5cd67SKarl Rupp  if (isdraw) {
143d9884438SBarry Smith     PetscDraw draw;
144d9884438SBarry Smith     PetscReal x,y,w,wd;
145d9884438SBarry Smith 
1469566063dSJacob Faibussowitsch     PetscCall(PetscViewerDrawGetDraw(viewer,0,&draw));
1479566063dSJacob Faibussowitsch     PetscCall(PetscDrawGetCurrentPoint(draw,&x,&y));
148d9884438SBarry Smith     w    = 2*PetscMin(1.0 - x,x);
149d9884438SBarry Smith     wd   = w/(jac->nsplits + 1);
150d9884438SBarry Smith     x    = x - wd*(jac->nsplits-1)/2.0;
151d9884438SBarry Smith     for (i=0; i<jac->nsplits; i++) {
1529566063dSJacob Faibussowitsch       PetscCall(PetscDrawPushCurrentPoint(draw,x,y));
1539566063dSJacob Faibussowitsch       PetscCall(KSPView(ilink->ksp,viewer));
1549566063dSJacob Faibussowitsch       PetscCall(PetscDrawPopCurrentPoint(draw));
155d9884438SBarry Smith       x    += wd;
156d9884438SBarry Smith       ilink = ilink->next;
157d9884438SBarry Smith     }
1580971522cSBarry Smith   }
1590971522cSBarry Smith   PetscFunctionReturn(0);
1600971522cSBarry Smith }
1610971522cSBarry Smith 
1623b224e63SBarry Smith static PetscErrorCode PCView_FieldSplit_Schur(PC pc,PetscViewer viewer)
1633b224e63SBarry Smith {
1643b224e63SBarry Smith   PC_FieldSplit              *jac = (PC_FieldSplit*)pc->data;
1654996c5bdSBarry Smith   PetscBool                  iascii,isdraw;
1663b224e63SBarry Smith   PetscInt                   i,j;
1673b224e63SBarry Smith   PC_FieldSplitLink          ilink = jac->head;
168a9908d51SBarry Smith   MatSchurComplementAinvType atype;
1693b224e63SBarry Smith 
1703b224e63SBarry Smith   PetscFunctionBegin;
1719566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii));
1729566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw));
1733b224e63SBarry Smith   if (iascii) {
1742c9966d7SBarry Smith     if (jac->bs > 0) {
17563a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  FieldSplit with Schur preconditioner, blocksize = %" PetscInt_FMT ", factorization %s\n",jac->bs,PCFieldSplitSchurFactTypes[jac->schurfactorization]));
1762c9966d7SBarry Smith     } else {
1779566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  FieldSplit with Schur preconditioner, factorization %s\n",PCFieldSplitSchurFactTypes[jac->schurfactorization]));
1782c9966d7SBarry Smith     }
179f5236f50SJed Brown     if (pc->useAmat) {
1809566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for blocks\n"));
181a3df900dSMatthew G Knepley     }
1823e8b8b31SMatthew G Knepley     switch (jac->schurpre) {
1833e8b8b31SMatthew G Knepley     case PC_FIELDSPLIT_SCHUR_PRE_SELF:
1849566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  Preconditioner for the Schur complement formed from S itself\n"));
185a9908d51SBarry Smith       break;
186a7476a74SDmitry Karpeev     case PC_FIELDSPLIT_SCHUR_PRE_SELFP:
1879566063dSJacob Faibussowitsch       PetscCall(MatSchurComplementGetAinvType(jac->schur,&atype));
1889566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  Preconditioner for the Schur complement formed from Sp, an assembled approximation to S, which uses A00's %sdiagonal's inverse\n",atype == MAT_SCHUR_COMPLEMENT_AINV_DIAG ? "" : (atype == MAT_SCHUR_COMPLEMENT_AINV_BLOCK_DIAG ? "block " : "lumped ")));break;
189e87fab1bSBarry Smith     case PC_FIELDSPLIT_SCHUR_PRE_A11:
1909566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  Preconditioner for the Schur complement formed from A11\n"));
191a9908d51SBarry Smith       break;
192e74569cdSMatthew G. Knepley     case PC_FIELDSPLIT_SCHUR_PRE_FULL:
1939566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  Preconditioner for the Schur complement formed from the exact Schur complement\n"));
194a9908d51SBarry Smith       break;
1953e8b8b31SMatthew G Knepley     case PC_FIELDSPLIT_SCHUR_PRE_USER:
1963e8b8b31SMatthew G Knepley       if (jac->schur_user) {
1979566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"  Preconditioner for the Schur complement formed from user provided matrix\n"));
1983e8b8b31SMatthew G Knepley       } else {
1999566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"  Preconditioner for the Schur complement formed from A11\n"));
2003e8b8b31SMatthew G Knepley       }
2013e8b8b31SMatthew G Knepley       break;
2023e8b8b31SMatthew G Knepley     default:
20398921bdaSJacob Faibussowitsch       SETERRQ(PetscObjectComm((PetscObject)pc), PETSC_ERR_ARG_OUTOFRANGE, "Invalid Schur preconditioning type: %d", jac->schurpre);
2043e8b8b31SMatthew G Knepley     }
2059566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(viewer,"  Split info:\n"));
2069566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPushTab(viewer));
2073b224e63SBarry Smith     for (i=0; i<jac->nsplits; i++) {
2083b224e63SBarry Smith       if (ilink->fields) {
20963a3b9bcSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"Split number %" PetscInt_FMT " Fields ",i));
2109566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIUseTabs(viewer,PETSC_FALSE));
2113b224e63SBarry Smith         for (j=0; j<ilink->nfields; j++) {
2123b224e63SBarry Smith           if (j > 0) {
2139566063dSJacob Faibussowitsch             PetscCall(PetscViewerASCIIPrintf(viewer,","));
2143b224e63SBarry Smith           }
21563a3b9bcSJacob Faibussowitsch           PetscCall(PetscViewerASCIIPrintf(viewer," %" PetscInt_FMT,ilink->fields[j]));
2163b224e63SBarry Smith         }
2179566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"\n"));
2189566063dSJacob Faibussowitsch         PetscCall(PetscViewerASCIIUseTabs(viewer,PETSC_TRUE));
2193b224e63SBarry Smith       } else {
22063a3b9bcSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"Split number %" PetscInt_FMT " Defined by IS\n",i));
2213b224e63SBarry Smith       }
2223b224e63SBarry Smith       ilink = ilink->next;
2233b224e63SBarry Smith     }
2249566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(viewer,"KSP solver for A00 block\n"));
2259566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPushTab(viewer));
22606de4afeSJed Brown     if (jac->head) {
2279566063dSJacob Faibussowitsch       PetscCall(KSPView(jac->head->ksp,viewer));
2289566063dSJacob Faibussowitsch     } else  PetscCall(PetscViewerASCIIPrintf(viewer,"  not yet available\n"));
2299566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPopTab(viewer));
23006de4afeSJed Brown     if (jac->head && jac->kspupper != jac->head->ksp) {
2319566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"KSP solver for upper A00 in upper triangular factor \n"));
2329566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPushTab(viewer));
2339566063dSJacob Faibussowitsch       if (jac->kspupper) PetscCall(KSPView(jac->kspupper,viewer));
2349566063dSJacob Faibussowitsch       else PetscCall(PetscViewerASCIIPrintf(viewer,"  not yet available\n"));
2359566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPopTab(viewer));
236443836d0SMatthew G Knepley     }
2379566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(viewer,"KSP solver for S = A11 - A10 inv(A00) A01 \n"));
2389566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPushTab(viewer));
23912cae6f2SJed Brown     if (jac->kspschur) {
2409566063dSJacob Faibussowitsch       PetscCall(KSPView(jac->kspschur,viewer));
24112cae6f2SJed Brown     } else {
2429566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  not yet available\n"));
24312cae6f2SJed Brown     }
2449566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPopTab(viewer));
2459566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPopTab(viewer));
24606de4afeSJed Brown   } else if (isdraw && jac->head) {
2474996c5bdSBarry Smith     PetscDraw draw;
2484996c5bdSBarry Smith     PetscReal x,y,w,wd,h;
2494996c5bdSBarry Smith     PetscInt  cnt = 2;
2504996c5bdSBarry Smith     char      str[32];
2514996c5bdSBarry Smith 
2529566063dSJacob Faibussowitsch     PetscCall(PetscViewerDrawGetDraw(viewer,0,&draw));
2539566063dSJacob Faibussowitsch     PetscCall(PetscDrawGetCurrentPoint(draw,&x,&y));
254c74581afSBarry Smith     if (jac->kspupper != jac->head->ksp) cnt++;
255c74581afSBarry Smith     w  = 2*PetscMin(1.0 - x,x);
256c74581afSBarry Smith     wd = w/(cnt + 1);
257c74581afSBarry Smith 
2589566063dSJacob Faibussowitsch     PetscCall(PetscSNPrintf(str,32,"Schur fact. %s",PCFieldSplitSchurFactTypes[jac->schurfactorization]));
2599566063dSJacob Faibussowitsch     PetscCall(PetscDrawStringBoxed(draw,x,y,PETSC_DRAW_RED,PETSC_DRAW_BLACK,str,NULL,&h));
2604996c5bdSBarry Smith     y   -= h;
2614996c5bdSBarry Smith     if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_USER &&  !jac->schur_user) {
2629566063dSJacob Faibussowitsch       PetscCall(PetscSNPrintf(str,32,"Prec. for Schur from %s",PCFieldSplitSchurPreTypes[PC_FIELDSPLIT_SCHUR_PRE_A11]));
2633b224e63SBarry Smith     } else {
2649566063dSJacob Faibussowitsch       PetscCall(PetscSNPrintf(str,32,"Prec. for Schur from %s",PCFieldSplitSchurPreTypes[jac->schurpre]));
2654996c5bdSBarry Smith     }
2669566063dSJacob Faibussowitsch     PetscCall(PetscDrawStringBoxed(draw,x+wd*(cnt-1)/2.0,y,PETSC_DRAW_RED,PETSC_DRAW_BLACK,str,NULL,&h));
2674996c5bdSBarry Smith     y   -= h;
2684996c5bdSBarry Smith     x    = x - wd*(cnt-1)/2.0;
2694996c5bdSBarry Smith 
2709566063dSJacob Faibussowitsch     PetscCall(PetscDrawPushCurrentPoint(draw,x,y));
2719566063dSJacob Faibussowitsch     PetscCall(KSPView(jac->head->ksp,viewer));
2729566063dSJacob Faibussowitsch     PetscCall(PetscDrawPopCurrentPoint(draw));
2734996c5bdSBarry Smith     if (jac->kspupper != jac->head->ksp) {
2744996c5bdSBarry Smith       x   += wd;
2759566063dSJacob Faibussowitsch       PetscCall(PetscDrawPushCurrentPoint(draw,x,y));
2769566063dSJacob Faibussowitsch       PetscCall(KSPView(jac->kspupper,viewer));
2779566063dSJacob Faibussowitsch       PetscCall(PetscDrawPopCurrentPoint(draw));
2784996c5bdSBarry Smith     }
2794996c5bdSBarry Smith     x   += wd;
2809566063dSJacob Faibussowitsch     PetscCall(PetscDrawPushCurrentPoint(draw,x,y));
2819566063dSJacob Faibussowitsch     PetscCall(KSPView(jac->kspschur,viewer));
2829566063dSJacob Faibussowitsch     PetscCall(PetscDrawPopCurrentPoint(draw));
2833b224e63SBarry Smith   }
2843b224e63SBarry Smith   PetscFunctionReturn(0);
2853b224e63SBarry Smith }
2863b224e63SBarry Smith 
287a51937d4SCarola Kruse static PetscErrorCode PCView_FieldSplit_GKB(PC pc,PetscViewer viewer)
288a51937d4SCarola Kruse {
289a51937d4SCarola Kruse   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
290a51937d4SCarola Kruse   PetscBool         iascii,isdraw;
291a51937d4SCarola Kruse   PetscInt          i,j;
292a51937d4SCarola Kruse   PC_FieldSplitLink ilink = jac->head;
293a51937d4SCarola Kruse 
294a51937d4SCarola Kruse   PetscFunctionBegin;
2959566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERASCII,&iascii));
2969566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)viewer,PETSCVIEWERDRAW,&isdraw));
297a51937d4SCarola Kruse   if (iascii) {
298a51937d4SCarola Kruse     if (jac->bs > 0) {
29963a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  FieldSplit with %s composition: total splits = %" PetscInt_FMT ", blocksize = %" PetscInt_FMT "\n",PCCompositeTypes[jac->type],jac->nsplits,jac->bs));
300a51937d4SCarola Kruse     } else {
30163a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  FieldSplit with %s composition: total splits = %" PetscInt_FMT "\n",PCCompositeTypes[jac->type],jac->nsplits));
302a51937d4SCarola Kruse     }
303a51937d4SCarola Kruse     if (pc->useAmat) {
3049566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for blocks\n"));
305a51937d4SCarola Kruse     }
306a51937d4SCarola Kruse     if (jac->diag_use_amat) {
3079566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for diagonal blocks\n"));
308a51937d4SCarola Kruse     }
309a51937d4SCarola Kruse     if (jac->offdiag_use_amat) {
3109566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"  using Amat (not Pmat) as operator for off-diagonal blocks\n"));
311a51937d4SCarola Kruse     }
312a51937d4SCarola Kruse 
31363a3b9bcSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(viewer,"  Stopping tolerance=%.1e, delay in error estimate=%" PetscInt_FMT ", maximum iterations=%" PetscInt_FMT "\n",(double)jac->gkbtol,jac->gkbdelay,jac->gkbmaxit));
3149566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(viewer,"  Solver info for H = A00 + nu*A01*A01' matrix:\n"));
3159566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPushTab(viewer));
316a51937d4SCarola Kruse 
317a51937d4SCarola Kruse     if (ilink->fields) {
31863a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"Split number 0 Fields "));
3199566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIUseTabs(viewer,PETSC_FALSE));
320a51937d4SCarola Kruse       for (j=0; j<ilink->nfields; j++) {
321a51937d4SCarola Kruse         if (j > 0) {
3229566063dSJacob Faibussowitsch           PetscCall(PetscViewerASCIIPrintf(viewer,","));
323a51937d4SCarola Kruse         }
32463a3b9bcSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer," %" PetscInt_FMT,ilink->fields[j]));
325a51937d4SCarola Kruse       }
3269566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(viewer,"\n"));
3279566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIIUseTabs(viewer,PETSC_TRUE));
328a51937d4SCarola Kruse     } else {
32963a3b9bcSJacob Faibussowitsch         PetscCall(PetscViewerASCIIPrintf(viewer,"Split number 0 Defined by IS\n"));
330a51937d4SCarola Kruse     }
3319566063dSJacob Faibussowitsch     PetscCall(KSPView(ilink->ksp,viewer));
332a51937d4SCarola Kruse 
3339566063dSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPopTab(viewer));
334a51937d4SCarola Kruse   }
335a51937d4SCarola Kruse 
336a51937d4SCarola Kruse  if (isdraw) {
337a51937d4SCarola Kruse     PetscDraw draw;
338a51937d4SCarola Kruse     PetscReal x,y,w,wd;
339a51937d4SCarola Kruse 
3409566063dSJacob Faibussowitsch     PetscCall(PetscViewerDrawGetDraw(viewer,0,&draw));
3419566063dSJacob Faibussowitsch     PetscCall(PetscDrawGetCurrentPoint(draw,&x,&y));
342a51937d4SCarola Kruse     w    = 2*PetscMin(1.0 - x,x);
343a51937d4SCarola Kruse     wd   = w/(jac->nsplits + 1);
344a51937d4SCarola Kruse     x    = x - wd*(jac->nsplits-1)/2.0;
345a51937d4SCarola Kruse     for (i=0; i<jac->nsplits; i++) {
3469566063dSJacob Faibussowitsch       PetscCall(PetscDrawPushCurrentPoint(draw,x,y));
3479566063dSJacob Faibussowitsch       PetscCall(KSPView(ilink->ksp,viewer));
3489566063dSJacob Faibussowitsch       PetscCall(PetscDrawPopCurrentPoint(draw));
349a51937d4SCarola Kruse       x    += wd;
350a51937d4SCarola Kruse       ilink = ilink->next;
351a51937d4SCarola Kruse     }
352a51937d4SCarola Kruse   }
353a51937d4SCarola Kruse   PetscFunctionReturn(0);
354a51937d4SCarola Kruse }
355a51937d4SCarola Kruse 
3566c924f48SJed Brown /* Precondition: jac->bs is set to a meaningful value */
3576c924f48SJed Brown static PetscErrorCode PCFieldSplitSetRuntimeSplits_Private(PC pc)
3586c924f48SJed Brown {
3596c924f48SJed Brown   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
3605d4c12cdSJungho Lee   PetscInt       i,nfields,*ifields,nfields_col,*ifields_col;
3615d4c12cdSJungho Lee   PetscBool      flg,flg_col;
3625d4c12cdSJungho Lee   char           optionname[128],splitname[8],optionname_col[128];
3636c924f48SJed Brown 
3646c924f48SJed Brown   PetscFunctionBegin;
3659566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(jac->bs,&ifields));
3669566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(jac->bs,&ifields_col));
3676c924f48SJed Brown   for (i=0,flg=PETSC_TRUE;; i++) {
36863a3b9bcSJacob Faibussowitsch     PetscCall(PetscSNPrintf(splitname,sizeof(splitname),"%" PetscInt_FMT,i));
36963a3b9bcSJacob Faibussowitsch     PetscCall(PetscSNPrintf(optionname,sizeof(optionname),"-pc_fieldsplit_%" PetscInt_FMT "_fields",i));
37063a3b9bcSJacob Faibussowitsch     PetscCall(PetscSNPrintf(optionname_col,sizeof(optionname_col),"-pc_fieldsplit_%" PetscInt_FMT "_fields_col",i));
3716c924f48SJed Brown     nfields     = jac->bs;
37229499fbbSJungho Lee     nfields_col = jac->bs;
3739566063dSJacob Faibussowitsch     PetscCall(PetscOptionsGetIntArray(((PetscObject)pc)->options,((PetscObject)pc)->prefix,optionname,ifields,&nfields,&flg));
3749566063dSJacob Faibussowitsch     PetscCall(PetscOptionsGetIntArray(((PetscObject)pc)->options,((PetscObject)pc)->prefix,optionname_col,ifields_col,&nfields_col,&flg_col));
3756c924f48SJed Brown     if (!flg) break;
3765d4c12cdSJungho Lee     else if (flg && !flg_col) {
37728b400f6SJacob Faibussowitsch       PetscCheck(nfields,PETSC_COMM_SELF,PETSC_ERR_USER,"Cannot list zero fields");
3789566063dSJacob Faibussowitsch       PetscCall(PCFieldSplitSetFields(pc,splitname,nfields,ifields,ifields));
3792fa5cd67SKarl Rupp     } else {
3807827d75bSBarry Smith       PetscCheck(nfields && nfields_col,PETSC_COMM_SELF,PETSC_ERR_USER,"Cannot list zero fields");
38108401ef6SPierre Jolivet       PetscCheck(nfields == nfields_col,PETSC_COMM_SELF,PETSC_ERR_USER,"Number of row and column fields must match");
3829566063dSJacob Faibussowitsch       PetscCall(PCFieldSplitSetFields(pc,splitname,nfields,ifields,ifields_col));
3835d4c12cdSJungho Lee     }
3846c924f48SJed Brown   }
3856c924f48SJed Brown   if (i > 0) {
3866c924f48SJed Brown     /* Makes command-line setting of splits take precedence over setting them in code.
3876c924f48SJed Brown        Otherwise subsequent calls to PCFieldSplitSetIS() or PCFieldSplitSetFields() would
3886c924f48SJed Brown        create new splits, which would probably not be what the user wanted. */
3896c924f48SJed Brown     jac->splitdefined = PETSC_TRUE;
3906c924f48SJed Brown   }
3919566063dSJacob Faibussowitsch   PetscCall(PetscFree(ifields));
3929566063dSJacob Faibussowitsch   PetscCall(PetscFree(ifields_col));
3936c924f48SJed Brown   PetscFunctionReturn(0);
3946c924f48SJed Brown }
3956c924f48SJed Brown 
39669a612a9SBarry Smith static PetscErrorCode PCFieldSplitSetDefaults(PC pc)
3970971522cSBarry Smith {
3980971522cSBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
3995a9f2f41SSatish Balay   PC_FieldSplitLink ilink = jac->head;
4007b752e3dSPatrick Sanan   PetscBool         fieldsplit_default = PETSC_FALSE,coupling = PETSC_FALSE;
4016c924f48SJed Brown   PetscInt          i;
4020971522cSBarry Smith 
4030971522cSBarry Smith   PetscFunctionBegin;
4047287d2a3SDmitry Karpeev   /*
405f5f0d762SBarry Smith    Kinda messy, but at least this now uses DMCreateFieldDecomposition().
4067287d2a3SDmitry Karpeev    Should probably be rewritten.
4077287d2a3SDmitry Karpeev    */
408f5f0d762SBarry Smith   if (!ilink) {
4099566063dSJacob Faibussowitsch     PetscCall(PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_fieldsplit_detect_coupling",&coupling,NULL));
4107b752e3dSPatrick Sanan     if (pc->dm && jac->dm_splits && !jac->detect && !coupling) {
411bafc1b83SMatthew G Knepley       PetscInt  numFields, f, i, j;
4120784a22cSJed Brown       char      **fieldNames;
4137b62db95SJungho Lee       IS        *fields;
414e7c4fc90SDmitry Karpeev       DM        *dms;
415bafc1b83SMatthew G Knepley       DM        subdm[128];
416bafc1b83SMatthew G Knepley       PetscBool flg;
417bafc1b83SMatthew G Knepley 
4189566063dSJacob Faibussowitsch       PetscCall(DMCreateFieldDecomposition(pc->dm, &numFields, &fieldNames, &fields, &dms));
419bafc1b83SMatthew G Knepley       /* Allow the user to prescribe the splits */
420bafc1b83SMatthew G Knepley       for (i = 0, flg = PETSC_TRUE;; i++) {
421bafc1b83SMatthew G Knepley         PetscInt ifields[128];
422bafc1b83SMatthew G Knepley         IS       compField;
423bafc1b83SMatthew G Knepley         char     optionname[128], splitname[8];
424bafc1b83SMatthew G Knepley         PetscInt nfields = numFields;
425bafc1b83SMatthew G Knepley 
42663a3b9bcSJacob Faibussowitsch         PetscCall(PetscSNPrintf(optionname, sizeof(optionname), "-pc_fieldsplit_%" PetscInt_FMT "_fields", i));
4279566063dSJacob Faibussowitsch         PetscCall(PetscOptionsGetIntArray(((PetscObject)pc)->options,((PetscObject)pc)->prefix, optionname, ifields, &nfields, &flg));
428bafc1b83SMatthew G Knepley         if (!flg) break;
42963a3b9bcSJacob Faibussowitsch         PetscCheck(numFields <= 128,PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Cannot currently support %" PetscInt_FMT " > 128 fields", numFields);
4309566063dSJacob Faibussowitsch         PetscCall(DMCreateSubDM(pc->dm, nfields, ifields, &compField, &subdm[i]));
431bafc1b83SMatthew G Knepley         if (nfields == 1) {
4329566063dSJacob Faibussowitsch           PetscCall(PCFieldSplitSetIS(pc, fieldNames[ifields[0]], compField));
433bafc1b83SMatthew G Knepley         } else {
43463a3b9bcSJacob Faibussowitsch           PetscCall(PetscSNPrintf(splitname, sizeof(splitname), "%" PetscInt_FMT, i));
4359566063dSJacob Faibussowitsch           PetscCall(PCFieldSplitSetIS(pc, splitname, compField));
4367287d2a3SDmitry Karpeev         }
4379566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&compField));
438bafc1b83SMatthew G Knepley         for (j = 0; j < nfields; ++j) {
439bafc1b83SMatthew G Knepley           f    = ifields[j];
4409566063dSJacob Faibussowitsch           PetscCall(PetscFree(fieldNames[f]));
4419566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&fields[f]));
4427b62db95SJungho Lee         }
443bafc1b83SMatthew G Knepley       }
444bafc1b83SMatthew G Knepley       if (i == 0) {
445bafc1b83SMatthew G Knepley         for (f = 0; f < numFields; ++f) {
4469566063dSJacob Faibussowitsch           PetscCall(PCFieldSplitSetIS(pc, fieldNames[f], fields[f]));
4479566063dSJacob Faibussowitsch           PetscCall(PetscFree(fieldNames[f]));
4489566063dSJacob Faibussowitsch           PetscCall(ISDestroy(&fields[f]));
449bafc1b83SMatthew G Knepley         }
450bafc1b83SMatthew G Knepley       } else {
451d724dfffSBarry Smith         for (j=0; j<numFields; j++) {
4529566063dSJacob Faibussowitsch           PetscCall(DMDestroy(dms+j));
453d724dfffSBarry Smith         }
4549566063dSJacob Faibussowitsch         PetscCall(PetscFree(dms));
4559566063dSJacob Faibussowitsch         PetscCall(PetscMalloc1(i, &dms));
4562fa5cd67SKarl Rupp         for (j = 0; j < i; ++j) dms[j] = subdm[j];
457bafc1b83SMatthew G Knepley       }
4589566063dSJacob Faibussowitsch       PetscCall(PetscFree(fieldNames));
4599566063dSJacob Faibussowitsch       PetscCall(PetscFree(fields));
460e7c4fc90SDmitry Karpeev       if (dms) {
4619566063dSJacob Faibussowitsch         PetscCall(PetscInfo(pc, "Setting up physics based fieldsplit preconditioner using the embedded DM\n"));
462bafc1b83SMatthew G Knepley         for (ilink = jac->head, i = 0; ilink; ilink = ilink->next, ++i) {
4637287d2a3SDmitry Karpeev           const char *prefix;
4649566063dSJacob Faibussowitsch           PetscCall(PetscObjectGetOptionsPrefix((PetscObject)(ilink->ksp),&prefix));
4659566063dSJacob Faibussowitsch           PetscCall(PetscObjectSetOptionsPrefix((PetscObject)(dms[i]), prefix));
4669566063dSJacob Faibussowitsch           PetscCall(KSPSetDM(ilink->ksp, dms[i]));
4679566063dSJacob Faibussowitsch           PetscCall(KSPSetDMActive(ilink->ksp, PETSC_FALSE));
46895dbaa6fSMatthew G. Knepley           {
46995dbaa6fSMatthew G. Knepley             PetscErrorCode (*func)(KSP,Mat,Mat,void*);
47095dbaa6fSMatthew G. Knepley             void            *ctx;
47195dbaa6fSMatthew G. Knepley 
4729566063dSJacob Faibussowitsch             PetscCall(DMKSPGetComputeOperators(pc->dm, &func, &ctx));
4739566063dSJacob Faibussowitsch             PetscCall(DMKSPSetComputeOperators(dms[i],  func,  ctx));
47495dbaa6fSMatthew G. Knepley           }
4759566063dSJacob Faibussowitsch           PetscCall(PetscObjectIncrementTabLevel((PetscObject)dms[i],(PetscObject)ilink->ksp,0));
4769566063dSJacob Faibussowitsch           PetscCall(DMDestroy(&dms[i]));
4772fa5ba8aSJed Brown         }
4789566063dSJacob Faibussowitsch         PetscCall(PetscFree(dms));
4798b8307b2SJed Brown       }
48066ffff09SJed Brown     } else {
481521d7252SBarry Smith       if (jac->bs <= 0) {
482704ba839SBarry Smith         if (pc->pmat) {
4839566063dSJacob Faibussowitsch           PetscCall(MatGetBlockSize(pc->pmat,&jac->bs));
4842fa5cd67SKarl Rupp         } else jac->bs = 1;
485521d7252SBarry Smith       }
486d32f9abdSBarry Smith 
4877b752e3dSPatrick Sanan       if (jac->detect) {
4886ce1633cSBarry Smith         IS       zerodiags,rest;
4896ce1633cSBarry Smith         PetscInt nmin,nmax;
4906ce1633cSBarry Smith 
4919566063dSJacob Faibussowitsch         PetscCall(MatGetOwnershipRange(pc->mat,&nmin,&nmax));
4927199da05SBarry Smith         if (jac->diag_use_amat) {
4939566063dSJacob Faibussowitsch           PetscCall(MatFindZeroDiagonals(pc->mat,&zerodiags));
4947199da05SBarry Smith         } else {
4959566063dSJacob Faibussowitsch           PetscCall(MatFindZeroDiagonals(pc->pmat,&zerodiags));
4967199da05SBarry Smith         }
4979566063dSJacob Faibussowitsch         PetscCall(ISComplement(zerodiags,nmin,nmax,&rest));
4989566063dSJacob Faibussowitsch         PetscCall(PCFieldSplitSetIS(pc,"0",rest));
4999566063dSJacob Faibussowitsch         PetscCall(PCFieldSplitSetIS(pc,"1",zerodiags));
5009566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&zerodiags));
5019566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&rest));
5023a062f41SBarry Smith       } else if (coupling) {
5033a062f41SBarry Smith         IS       coupling,rest;
5043a062f41SBarry Smith         PetscInt nmin,nmax;
5053a062f41SBarry Smith 
5069566063dSJacob Faibussowitsch         PetscCall(MatGetOwnershipRange(pc->mat,&nmin,&nmax));
5077199da05SBarry Smith         if (jac->offdiag_use_amat) {
5089566063dSJacob Faibussowitsch           PetscCall(MatFindOffBlockDiagonalEntries(pc->mat,&coupling));
5097199da05SBarry Smith         } else {
5109566063dSJacob Faibussowitsch           PetscCall(MatFindOffBlockDiagonalEntries(pc->pmat,&coupling));
5117199da05SBarry Smith         }
5129566063dSJacob Faibussowitsch         PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc->mat),nmax-nmin,nmin,1,&rest));
5139566063dSJacob Faibussowitsch         PetscCall(ISSetIdentity(rest));
5149566063dSJacob Faibussowitsch         PetscCall(PCFieldSplitSetIS(pc,"0",rest));
5159566063dSJacob Faibussowitsch         PetscCall(PCFieldSplitSetIS(pc,"1",coupling));
5169566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&coupling));
5179566063dSJacob Faibussowitsch         PetscCall(ISDestroy(&rest));
5186ce1633cSBarry Smith       } else {
5199566063dSJacob Faibussowitsch         PetscCall(PetscOptionsGetBool(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_fieldsplit_default",&fieldsplit_default,NULL));
5207287d2a3SDmitry Karpeev         if (!fieldsplit_default) {
521d32f9abdSBarry Smith           /* Allow user to set fields from command line,  if bs was known at the time of PCSetFromOptions_FieldSplit()
522d32f9abdSBarry Smith            then it is set there. This is not ideal because we should only have options set in XXSetFromOptions(). */
5239566063dSJacob Faibussowitsch           PetscCall(PCFieldSplitSetRuntimeSplits_Private(pc));
5249566063dSJacob Faibussowitsch           if (jac->splitdefined) PetscCall(PetscInfo(pc,"Splits defined using the options database\n"));
525d32f9abdSBarry Smith         }
5266dbb499eSCian Wilson         if ((fieldsplit_default || !jac->splitdefined) && !jac->isrestrict) {
5279f001fe8SStefano Zampini           Mat       M = pc->pmat;
528f3b928b9SStefano Zampini           PetscBool isnest;
529f3b928b9SStefano Zampini 
5309566063dSJacob Faibussowitsch           PetscCall(PetscInfo(pc,"Using default splitting of fields\n"));
5319566063dSJacob Faibussowitsch           PetscCall(PetscObjectTypeCompare((PetscObject)pc->pmat,MATNEST,&isnest));
532f3b928b9SStefano Zampini           if (!isnest) {
5339f001fe8SStefano Zampini             M    = pc->mat;
5349566063dSJacob Faibussowitsch             PetscCall(PetscObjectTypeCompare((PetscObject)pc->mat,MATNEST,&isnest));
535f3b928b9SStefano Zampini           }
536f3b928b9SStefano Zampini           if (isnest) {
537f3b928b9SStefano Zampini             IS       *fields;
538f3b928b9SStefano Zampini             PetscInt nf;
539f3b928b9SStefano Zampini 
5409566063dSJacob Faibussowitsch             PetscCall(MatNestGetSize(M,&nf,NULL));
5419566063dSJacob Faibussowitsch             PetscCall(PetscMalloc1(nf,&fields));
5429566063dSJacob Faibussowitsch             PetscCall(MatNestGetISs(M,fields,NULL));
543f3b928b9SStefano Zampini             for (i=0;i<nf;i++) {
5449566063dSJacob Faibussowitsch               PetscCall(PCFieldSplitSetIS(pc,NULL,fields[i]));
545f3b928b9SStefano Zampini             }
5469566063dSJacob Faibussowitsch             PetscCall(PetscFree(fields));
547f3b928b9SStefano Zampini           } else {
548db4c96c1SJed Brown             for (i=0; i<jac->bs; i++) {
5496c924f48SJed Brown               char splitname[8];
55063a3b9bcSJacob Faibussowitsch               PetscCall(PetscSNPrintf(splitname,sizeof(splitname),"%" PetscInt_FMT,i));
5519566063dSJacob Faibussowitsch               PetscCall(PCFieldSplitSetFields(pc,splitname,1,&i,&i));
55279416396SBarry Smith             }
5535d4c12cdSJungho Lee             jac->defaultsplit = PETSC_TRUE;
554521d7252SBarry Smith           }
55566ffff09SJed Brown         }
5566ce1633cSBarry Smith       }
557f3b928b9SStefano Zampini     }
558edf189efSBarry Smith   } else if (jac->nsplits == 1) {
559edf189efSBarry Smith     if (ilink->is) {
560edf189efSBarry Smith       IS       is2;
561edf189efSBarry Smith       PetscInt nmin,nmax;
562edf189efSBarry Smith 
5639566063dSJacob Faibussowitsch       PetscCall(MatGetOwnershipRange(pc->mat,&nmin,&nmax));
5649566063dSJacob Faibussowitsch       PetscCall(ISComplement(ilink->is,nmin,nmax,&is2));
5659566063dSJacob Faibussowitsch       PetscCall(PCFieldSplitSetIS(pc,"1",is2));
5669566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&is2));
56782f516ccSBarry Smith     } else SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Must provide at least two sets of fields to PCFieldSplit()");
568edf189efSBarry Smith   }
569d0af7cd3SBarry Smith 
57063a3b9bcSJacob Faibussowitsch   PetscCheck(jac->nsplits >= 2,PetscObjectComm((PetscObject)pc),PETSC_ERR_PLIB,"Unhandled case, must have at least two fields, not %" PetscInt_FMT, jac->nsplits);
57169a612a9SBarry Smith   PetscFunctionReturn(0);
57269a612a9SBarry Smith }
57369a612a9SBarry Smith 
574a51937d4SCarola Kruse static PetscErrorCode MatGolubKahanComputeExplicitOperator(Mat A,Mat B,Mat C,Mat *H,PetscReal gkbnu)
575a51937d4SCarola Kruse {
576a51937d4SCarola Kruse   Mat               BT,T;
577de482cd7SCarola Kruse   PetscReal         nrmT,nrmB;
578a51937d4SCarola Kruse 
579a51937d4SCarola Kruse   PetscFunctionBegin;
5809566063dSJacob Faibussowitsch   PetscCall(MatHermitianTranspose(C,MAT_INITIAL_MATRIX,&T));            /* Test if augmented matrix is symmetric */
5819566063dSJacob Faibussowitsch   PetscCall(MatAXPY(T,-1.0,B,DIFFERENT_NONZERO_PATTERN));
5829566063dSJacob Faibussowitsch   PetscCall(MatNorm(T,NORM_1,&nrmT));
5839566063dSJacob Faibussowitsch   PetscCall(MatNorm(B,NORM_1,&nrmB));
584de482cd7SCarola Kruse   if (nrmB > 0) {
585de482cd7SCarola Kruse     if (nrmT/nrmB >= PETSC_SMALL) {
586de482cd7SCarola Kruse       SETERRQ(PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Matrix is not symmetric/hermitian, GKB is not applicable.");
587de482cd7SCarola Kruse     }
588a51937d4SCarola Kruse   }
589a51937d4SCarola Kruse   /* Compute augmented Lagrangian matrix H = A00 + nu*A01*A01'. This corresponds to */
590a51937d4SCarola Kruse   /* setting N := 1/nu*I in [Ar13].                                                 */
5919566063dSJacob Faibussowitsch   PetscCall(MatHermitianTranspose(B,MAT_INITIAL_MATRIX,&BT));
5929566063dSJacob Faibussowitsch   PetscCall(MatMatMult(B,BT,MAT_INITIAL_MATRIX,PETSC_DEFAULT,H));       /* H = A01*A01'          */
5939566063dSJacob Faibussowitsch   PetscCall(MatAYPX(*H,gkbnu,A,DIFFERENT_NONZERO_PATTERN));             /* H = A00 + nu*A01*A01' */
594a51937d4SCarola Kruse 
5959566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&BT));
5969566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&T));
597a51937d4SCarola Kruse   PetscFunctionReturn(0);
598a51937d4SCarola Kruse }
599a51937d4SCarola Kruse 
6002d747510SLisandro Dalcin PETSC_EXTERN PetscErrorCode PetscOptionsFindPairPrefix_Private(PetscOptions,const char pre[], const char name[],const char *value[],PetscBool *flg);
601514bf10dSMatthew G Knepley 
60269a612a9SBarry Smith static PetscErrorCode PCSetUp_FieldSplit(PC pc)
60369a612a9SBarry Smith {
60469a612a9SBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
6055a9f2f41SSatish Balay   PC_FieldSplitLink ilink;
6062c9966d7SBarry Smith   PetscInt          i,nsplit;
6075f4ab4e1SJungho Lee   PetscBool         sorted, sorted_col;
60869a612a9SBarry Smith 
60969a612a9SBarry Smith   PetscFunctionBegin;
6105da88fe4STristan Konolige   pc->failedreason = PC_NOERROR;
6119566063dSJacob Faibussowitsch   PetscCall(PCFieldSplitSetDefaults(pc));
61297bbdb24SBarry Smith   nsplit = jac->nsplits;
6135a9f2f41SSatish Balay   ilink  = jac->head;
61497bbdb24SBarry Smith 
61597bbdb24SBarry Smith   /* get the matrices for each split */
616704ba839SBarry Smith   if (!jac->issetup) {
6171b9fc7fcSBarry Smith     PetscInt rstart,rend,nslots,bs;
61897bbdb24SBarry Smith 
619704ba839SBarry Smith     jac->issetup = PETSC_TRUE;
620704ba839SBarry Smith 
6215d4c12cdSJungho Lee     /* This is done here instead of in PCFieldSplitSetFields() because may not have matrix at that point */
6222c9966d7SBarry Smith     if (jac->defaultsplit || !ilink->is) {
6232c9966d7SBarry Smith       if (jac->bs <= 0) jac->bs = nsplit;
6242c9966d7SBarry Smith     }
62551f519a2SBarry Smith     bs     = jac->bs;
6269566063dSJacob Faibussowitsch     PetscCall(MatGetOwnershipRange(pc->pmat,&rstart,&rend));
6271b9fc7fcSBarry Smith     nslots = (rend - rstart)/bs;
6281b9fc7fcSBarry Smith     for (i=0; i<nsplit; i++) {
6291b9fc7fcSBarry Smith       if (jac->defaultsplit) {
6309566063dSJacob Faibussowitsch         PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc),nslots,rstart+i,nsplit,&ilink->is));
6319566063dSJacob Faibussowitsch         PetscCall(ISDuplicate(ilink->is,&ilink->is_col));
632704ba839SBarry Smith       } else if (!ilink->is) {
633ccb205f8SBarry Smith         if (ilink->nfields > 1) {
6345f4ab4e1SJungho Lee           PetscInt *ii,*jj,j,k,nfields = ilink->nfields,*fields = ilink->fields,*fields_col = ilink->fields_col;
6359566063dSJacob Faibussowitsch           PetscCall(PetscMalloc1(ilink->nfields*nslots,&ii));
6369566063dSJacob Faibussowitsch           PetscCall(PetscMalloc1(ilink->nfields*nslots,&jj));
6371b9fc7fcSBarry Smith           for (j=0; j<nslots; j++) {
6381b9fc7fcSBarry Smith             for (k=0; k<nfields; k++) {
6391b9fc7fcSBarry Smith               ii[nfields*j + k] = rstart + bs*j + fields[k];
6405f4ab4e1SJungho Lee               jj[nfields*j + k] = rstart + bs*j + fields_col[k];
64197bbdb24SBarry Smith             }
64297bbdb24SBarry Smith           }
6439566063dSJacob Faibussowitsch           PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc),nslots*nfields,ii,PETSC_OWN_POINTER,&ilink->is));
6449566063dSJacob Faibussowitsch           PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)pc),nslots*nfields,jj,PETSC_OWN_POINTER,&ilink->is_col));
6459566063dSJacob Faibussowitsch           PetscCall(ISSetBlockSize(ilink->is, nfields));
6469566063dSJacob Faibussowitsch           PetscCall(ISSetBlockSize(ilink->is_col, nfields));
647ccb205f8SBarry Smith         } else {
6489566063dSJacob Faibussowitsch           PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc),nslots,rstart+ilink->fields[0],bs,&ilink->is));
6499566063dSJacob Faibussowitsch           PetscCall(ISCreateStride(PetscObjectComm((PetscObject)pc),nslots,rstart+ilink->fields_col[0],bs,&ilink->is_col));
650ccb205f8SBarry Smith         }
6513e197d65SBarry Smith       }
6529566063dSJacob Faibussowitsch       PetscCall(ISSorted(ilink->is,&sorted));
6539566063dSJacob Faibussowitsch       if (ilink->is_col) PetscCall(ISSorted(ilink->is_col,&sorted_col));
6547827d75bSBarry Smith       PetscCheck(sorted && sorted_col,PETSC_COMM_SELF,PETSC_ERR_USER,"Fields must be sorted when creating split");
655704ba839SBarry Smith       ilink = ilink->next;
6561b9fc7fcSBarry Smith     }
6571b9fc7fcSBarry Smith   }
6581b9fc7fcSBarry Smith 
659704ba839SBarry Smith   ilink = jac->head;
66097bbdb24SBarry Smith   if (!jac->pmat) {
661bdddcaaaSMatthew G Knepley     Vec xtmp;
662bdddcaaaSMatthew G Knepley 
6639566063dSJacob Faibussowitsch     PetscCall(MatCreateVecs(pc->pmat,&xtmp,NULL));
6649566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(nsplit,&jac->pmat));
6659566063dSJacob Faibussowitsch     PetscCall(PetscMalloc2(nsplit,&jac->x,nsplit,&jac->y));
666cf502942SBarry Smith     for (i=0; i<nsplit; i++) {
667bdddcaaaSMatthew G Knepley       MatNullSpace sp;
668bdddcaaaSMatthew G Knepley 
669a3df900dSMatthew G Knepley       /* Check for preconditioning matrix attached to IS */
6709566063dSJacob Faibussowitsch       PetscCall(PetscObjectQuery((PetscObject) ilink->is, "pmat", (PetscObject*) &jac->pmat[i]));
671a3df900dSMatthew G Knepley       if (jac->pmat[i]) {
6729566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) jac->pmat[i]));
673a3df900dSMatthew G Knepley         if (jac->type == PC_COMPOSITE_SCHUR) {
674a3df900dSMatthew G Knepley           jac->schur_user = jac->pmat[i];
6752fa5cd67SKarl Rupp 
6769566063dSJacob Faibussowitsch           PetscCall(PetscObjectReference((PetscObject) jac->schur_user));
677a3df900dSMatthew G Knepley         }
678a3df900dSMatthew G Knepley       } else {
6793a062f41SBarry Smith         const char *prefix;
6809566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ilink->is_col,MAT_INITIAL_MATRIX,&jac->pmat[i]));
6819566063dSJacob Faibussowitsch         PetscCall(KSPGetOptionsPrefix(ilink->ksp,&prefix));
6829566063dSJacob Faibussowitsch         PetscCall(MatSetOptionsPrefix(jac->pmat[i],prefix));
6839566063dSJacob Faibussowitsch         PetscCall(MatViewFromOptions(jac->pmat[i],NULL,"-mat_view"));
684a3df900dSMatthew G Knepley       }
685bdddcaaaSMatthew G Knepley       /* create work vectors for each split */
6869566063dSJacob Faibussowitsch       PetscCall(MatCreateVecs(jac->pmat[i],&jac->x[i],&jac->y[i]));
6870298fd71SBarry Smith       ilink->x = jac->x[i]; ilink->y = jac->y[i]; ilink->z = NULL;
688bdddcaaaSMatthew G Knepley       /* compute scatter contexts needed by multiplicative versions and non-default splits */
6899566063dSJacob Faibussowitsch       PetscCall(VecScatterCreate(xtmp,ilink->is,jac->x[i],NULL,&ilink->sctx));
6909566063dSJacob Faibussowitsch       PetscCall(PetscObjectQuery((PetscObject) ilink->is, "nearnullspace", (PetscObject*) &sp));
691ed1f0337SMatthew G Knepley       if (sp) {
6929566063dSJacob Faibussowitsch         PetscCall(MatSetNearNullSpace(jac->pmat[i], sp));
693ed1f0337SMatthew G Knepley       }
694704ba839SBarry Smith       ilink = ilink->next;
695cf502942SBarry Smith     }
6969566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&xtmp));
69797bbdb24SBarry Smith   } else {
698ef7efd37SHong Zhang     MatReuse scall;
699ef7efd37SHong Zhang     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
700ef7efd37SHong Zhang       for (i=0; i<nsplit; i++) {
7019566063dSJacob Faibussowitsch         PetscCall(MatDestroy(&jac->pmat[i]));
702ef7efd37SHong Zhang       }
703ef7efd37SHong Zhang       scall = MAT_INITIAL_MATRIX;
704ef7efd37SHong Zhang     } else scall = MAT_REUSE_MATRIX;
705ef7efd37SHong Zhang 
706cf502942SBarry Smith     for (i=0; i<nsplit; i++) {
707a3df900dSMatthew G Knepley       Mat pmat;
708a3df900dSMatthew G Knepley 
709a3df900dSMatthew G Knepley       /* Check for preconditioning matrix attached to IS */
7109566063dSJacob Faibussowitsch       PetscCall(PetscObjectQuery((PetscObject) ilink->is, "pmat", (PetscObject*) &pmat));
711a3df900dSMatthew G Knepley       if (!pmat) {
7129566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ilink->is_col,scall,&jac->pmat[i]));
713a3df900dSMatthew G Knepley       }
714704ba839SBarry Smith       ilink = ilink->next;
715cf502942SBarry Smith     }
71697bbdb24SBarry Smith   }
7174e39094bSDmitry Karpeev   if (jac->diag_use_amat) {
718519d70e2SJed Brown     ilink = jac->head;
719519d70e2SJed Brown     if (!jac->mat) {
7209566063dSJacob Faibussowitsch       PetscCall(PetscMalloc1(nsplit,&jac->mat));
721519d70e2SJed Brown       for (i=0; i<nsplit; i++) {
7229566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ilink->is_col,MAT_INITIAL_MATRIX,&jac->mat[i]));
723519d70e2SJed Brown         ilink = ilink->next;
724519d70e2SJed Brown       }
725519d70e2SJed Brown     } else {
726ef7efd37SHong Zhang       MatReuse scall;
727ef7efd37SHong Zhang       if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
728519d70e2SJed Brown         for (i=0; i<nsplit; i++) {
7299566063dSJacob Faibussowitsch           PetscCall(MatDestroy(&jac->mat[i]));
730ef7efd37SHong Zhang         }
731ef7efd37SHong Zhang         scall = MAT_INITIAL_MATRIX;
732ef7efd37SHong Zhang       } else scall = MAT_REUSE_MATRIX;
733ef7efd37SHong Zhang 
734ef7efd37SHong Zhang       for (i=0; i<nsplit; i++) {
7359566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ilink->is_col,scall,&jac->mat[i]));
736519d70e2SJed Brown         ilink = ilink->next;
737519d70e2SJed Brown       }
738519d70e2SJed Brown     }
739519d70e2SJed Brown   } else {
740519d70e2SJed Brown     jac->mat = jac->pmat;
741519d70e2SJed Brown   }
74297bbdb24SBarry Smith 
74353935eafSBarry Smith   /* Check for null space attached to IS */
74453935eafSBarry Smith   ilink = jac->head;
74553935eafSBarry Smith   for (i=0; i<nsplit; i++) {
74653935eafSBarry Smith     MatNullSpace sp;
74753935eafSBarry Smith 
7489566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject) ilink->is, "nullspace", (PetscObject*) &sp));
74953935eafSBarry Smith     if (sp) {
7509566063dSJacob Faibussowitsch       PetscCall(MatSetNullSpace(jac->mat[i], sp));
75153935eafSBarry Smith     }
75253935eafSBarry Smith     ilink = ilink->next;
75353935eafSBarry Smith   }
75453935eafSBarry Smith 
755a51937d4SCarola Kruse   if (jac->type != PC_COMPOSITE_ADDITIVE  && jac->type != PC_COMPOSITE_SCHUR && jac->type != PC_COMPOSITE_GKB) {
75668dd23aaSBarry Smith     /* extract the rows of the matrix associated with each field: used for efficient computation of residual inside algorithm */
7574e39094bSDmitry Karpeev     /* FIXME: Can/should we reuse jac->mat whenever (jac->diag_use_amat) is true? */
75868dd23aaSBarry Smith     ilink = jac->head;
759e52d2c62SBarry Smith     if (nsplit == 2 && jac->type == PC_COMPOSITE_MULTIPLICATIVE) {
760e52d2c62SBarry Smith       /* special case need where Afield[0] is not needed and only certain columns of Afield[1] are needed since update is only on those rows of the solution */
761e52d2c62SBarry Smith       if (!jac->Afield) {
7629566063dSJacob Faibussowitsch         PetscCall(PetscCalloc1(nsplit,&jac->Afield));
76380c96bb1SFande Kong         if (jac->offdiag_use_amat) {
7649566063dSJacob Faibussowitsch           PetscCall(MatCreateSubMatrix(pc->mat,ilink->next->is,ilink->is,MAT_INITIAL_MATRIX,&jac->Afield[1]));
765e52d2c62SBarry Smith         } else {
7669566063dSJacob Faibussowitsch           PetscCall(MatCreateSubMatrix(pc->pmat,ilink->next->is,ilink->is,MAT_INITIAL_MATRIX,&jac->Afield[1]));
76780c96bb1SFande Kong         }
76880c96bb1SFande Kong       } else {
769ef7efd37SHong Zhang         MatReuse scall;
770e9422dd5SStefano Zampini 
771ef7efd37SHong Zhang         if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
7729566063dSJacob Faibussowitsch           PetscCall(MatDestroy(&jac->Afield[1]));
773ef7efd37SHong Zhang           scall = MAT_INITIAL_MATRIX;
774ef7efd37SHong Zhang         } else scall = MAT_REUSE_MATRIX;
775ef7efd37SHong Zhang 
77680c96bb1SFande Kong         if (jac->offdiag_use_amat) {
7779566063dSJacob Faibussowitsch           PetscCall(MatCreateSubMatrix(pc->mat,ilink->next->is,ilink->is,scall,&jac->Afield[1]));
77880c96bb1SFande Kong         } else {
7799566063dSJacob Faibussowitsch           PetscCall(MatCreateSubMatrix(pc->pmat,ilink->next->is,ilink->is,scall,&jac->Afield[1]));
78080c96bb1SFande Kong         }
781e52d2c62SBarry Smith       }
782e52d2c62SBarry Smith     } else {
78368dd23aaSBarry Smith       if (!jac->Afield) {
7849566063dSJacob Faibussowitsch         PetscCall(PetscMalloc1(nsplit,&jac->Afield));
78568dd23aaSBarry Smith         for (i=0; i<nsplit; i++) {
78680c96bb1SFande Kong           if (jac->offdiag_use_amat) {
7879566063dSJacob Faibussowitsch             PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,NULL,MAT_INITIAL_MATRIX,&jac->Afield[i]));
78880c96bb1SFande Kong           } else {
7899566063dSJacob Faibussowitsch             PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,NULL,MAT_INITIAL_MATRIX,&jac->Afield[i]));
79080c96bb1SFande Kong           }
79168dd23aaSBarry Smith           ilink = ilink->next;
79268dd23aaSBarry Smith         }
79368dd23aaSBarry Smith       } else {
794ef7efd37SHong Zhang         MatReuse scall;
795ef7efd37SHong Zhang         if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
796ef7efd37SHong Zhang           for (i=0; i<nsplit; i++) {
7979566063dSJacob Faibussowitsch             PetscCall(MatDestroy(&jac->Afield[i]));
798ef7efd37SHong Zhang           }
799ef7efd37SHong Zhang           scall = MAT_INITIAL_MATRIX;
800ef7efd37SHong Zhang         } else scall = MAT_REUSE_MATRIX;
801ef7efd37SHong Zhang 
80268dd23aaSBarry Smith         for (i=0; i<nsplit; i++) {
80380c96bb1SFande Kong           if (jac->offdiag_use_amat) {
8049566063dSJacob Faibussowitsch             PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,NULL,scall,&jac->Afield[i]));
80580c96bb1SFande Kong           } else {
8069566063dSJacob Faibussowitsch             PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,NULL,scall,&jac->Afield[i]));
80780c96bb1SFande Kong           }
80868dd23aaSBarry Smith           ilink = ilink->next;
80968dd23aaSBarry Smith         }
81068dd23aaSBarry Smith       }
81168dd23aaSBarry Smith     }
812e52d2c62SBarry Smith   }
81368dd23aaSBarry Smith 
8143b224e63SBarry Smith   if (jac->type == PC_COMPOSITE_SCHUR) {
8153b224e63SBarry Smith     IS          ccis;
816*b94d7dedSBarry Smith     PetscBool   isset,isspd;
8174aa3045dSJed Brown     PetscInt    rstart,rend;
818093c86ffSJed Brown     char        lscname[256];
819093c86ffSJed Brown     PetscObject LSC_L;
820ce94432eSBarry Smith 
82108401ef6SPierre Jolivet     PetscCheck(nsplit == 2,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_INCOMP,"To use Schur complement preconditioner you must have exactly 2 fields");
82268dd23aaSBarry Smith 
823c096484dSStefano Zampini     /* If pc->mat is SPD, don't scale by -1 the Schur complement */
824c096484dSStefano Zampini     if (jac->schurscale == (PetscScalar)-1.0) {
825*b94d7dedSBarry Smith       PetscCall(MatIsSPDKnown(pc->pmat,&isset,&isspd));
826*b94d7dedSBarry Smith       jac->schurscale = (isset && isspd) ? 1.0 : -1.0;
827c096484dSStefano Zampini     }
828c096484dSStefano Zampini 
829e6cab6aaSJed Brown     /* When extracting off-diagonal submatrices, we take complements from this range */
8309566063dSJacob Faibussowitsch     PetscCall(MatGetOwnershipRangeColumn(pc->mat,&rstart,&rend));
831e6cab6aaSJed Brown 
8323b224e63SBarry Smith     if (jac->schur) {
8330298fd71SBarry Smith       KSP      kspA = jac->head->ksp, kspInner = NULL, kspUpper = jac->kspupper;
834e9422dd5SStefano Zampini       MatReuse scall;
835e9422dd5SStefano Zampini 
836e9422dd5SStefano Zampini       if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
837e9422dd5SStefano Zampini         scall = MAT_INITIAL_MATRIX;
8389566063dSJacob Faibussowitsch         PetscCall(MatDestroy(&jac->B));
8399566063dSJacob Faibussowitsch         PetscCall(MatDestroy(&jac->C));
840e9422dd5SStefano Zampini       } else scall = MAT_REUSE_MATRIX;
841443836d0SMatthew G Knepley 
8429566063dSJacob Faibussowitsch       PetscCall(MatSchurComplementGetKSP(jac->schur, &kspInner));
8433b224e63SBarry Smith       ilink = jac->head;
8449566063dSJacob Faibussowitsch       PetscCall(ISComplement(ilink->is_col,rstart,rend,&ccis));
8454e39094bSDmitry Karpeev       if (jac->offdiag_use_amat) {
8469566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ccis,scall,&jac->B));
847c84da90fSDmitry Karpeev       } else {
8489566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ccis,scall,&jac->B));
849c84da90fSDmitry Karpeev       }
8509566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&ccis));
8513b224e63SBarry Smith       ilink = ilink->next;
8529566063dSJacob Faibussowitsch       PetscCall(ISComplement(ilink->is_col,rstart,rend,&ccis));
8534e39094bSDmitry Karpeev       if (jac->offdiag_use_amat) {
8549566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ccis,scall,&jac->C));
855c84da90fSDmitry Karpeev       } else {
8569566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ccis,scall,&jac->C));
857c84da90fSDmitry Karpeev       }
8589566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&ccis));
8599566063dSJacob Faibussowitsch       PetscCall(MatSchurComplementUpdateSubMatrices(jac->schur,jac->mat[0],jac->pmat[0],jac->B,jac->C,jac->mat[1]));
860a7476a74SDmitry Karpeev       if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_SELFP) {
8619566063dSJacob Faibussowitsch         PetscCall(MatDestroy(&jac->schurp));
8629566063dSJacob Faibussowitsch         PetscCall(MatSchurComplementGetPmat(jac->schur,MAT_INITIAL_MATRIX,&jac->schurp));
863a7476a74SDmitry Karpeev       }
864443836d0SMatthew G Knepley       if (kspA != kspInner) {
8659566063dSJacob Faibussowitsch         PetscCall(KSPSetOperators(kspA,jac->mat[0],jac->pmat[0]));
866443836d0SMatthew G Knepley       }
867443836d0SMatthew G Knepley       if (kspUpper != kspA) {
8689566063dSJacob Faibussowitsch         PetscCall(KSPSetOperators(kspUpper,jac->mat[0],jac->pmat[0]));
869443836d0SMatthew G Knepley       }
8709566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(jac->kspschur,jac->schur,FieldSplitSchurPre(jac)));
8713b224e63SBarry Smith     } else {
872bafc1b83SMatthew G Knepley       const char   *Dprefix;
873470b340bSDmitry Karpeev       char         schurprefix[256], schurmatprefix[256];
874514bf10dSMatthew G Knepley       char         schurtestoption[256];
875bdddcaaaSMatthew G Knepley       MatNullSpace sp;
876514bf10dSMatthew G Knepley       PetscBool    flg;
877686bed4dSStefano Zampini       KSP          kspt;
8783b224e63SBarry Smith 
879a04f6461SBarry Smith       /* extract the A01 and A10 matrices */
8803b224e63SBarry Smith       ilink = jac->head;
8819566063dSJacob Faibussowitsch       PetscCall(ISComplement(ilink->is_col,rstart,rend,&ccis));
8824e39094bSDmitry Karpeev       if (jac->offdiag_use_amat) {
8839566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->B));
884c84da90fSDmitry Karpeev       } else {
8859566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->B));
886c84da90fSDmitry Karpeev       }
8879566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&ccis));
8883b224e63SBarry Smith       ilink = ilink->next;
8899566063dSJacob Faibussowitsch       PetscCall(ISComplement(ilink->is_col,rstart,rend,&ccis));
8904e39094bSDmitry Karpeev       if (jac->offdiag_use_amat) {
8919566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->C));
892c84da90fSDmitry Karpeev       } else {
8939566063dSJacob Faibussowitsch         PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->C));
894c84da90fSDmitry Karpeev       }
8959566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&ccis));
89620252d06SBarry Smith 
897f5236f50SJed Brown       /* Use mat[0] (diagonal block of Amat) preconditioned by pmat[0] to define Schur complement */
8989566063dSJacob Faibussowitsch       PetscCall(MatCreate(((PetscObject)jac->mat[0])->comm,&jac->schur));
8999566063dSJacob Faibussowitsch       PetscCall(MatSetType(jac->schur,MATSCHURCOMPLEMENT));
9009566063dSJacob Faibussowitsch       PetscCall(MatSchurComplementSetSubMatrices(jac->schur,jac->mat[0],jac->pmat[0],jac->B,jac->C,jac->mat[1]));
9019566063dSJacob Faibussowitsch       PetscCall(PetscSNPrintf(schurmatprefix, sizeof(schurmatprefix), "%sfieldsplit_%s_", ((PetscObject)pc)->prefix ? ((PetscObject)pc)->prefix : "", ilink->splitname));
9029566063dSJacob Faibussowitsch       PetscCall(MatSetOptionsPrefix(jac->schur,schurmatprefix));
9039566063dSJacob Faibussowitsch       PetscCall(MatSchurComplementGetKSP(jac->schur,&kspt));
9049566063dSJacob Faibussowitsch       PetscCall(KSPSetOptionsPrefix(kspt,schurmatprefix));
905686bed4dSStefano Zampini 
906686bed4dSStefano Zampini       /* Note: this is not true in general */
9079566063dSJacob Faibussowitsch       PetscCall(MatGetNullSpace(jac->mat[1], &sp));
9081baa6e33SBarry Smith       if (sp) PetscCall(MatSetNullSpace(jac->schur, sp));
90920252d06SBarry Smith 
9109566063dSJacob Faibussowitsch       PetscCall(PetscSNPrintf(schurtestoption, sizeof(schurtestoption), "-fieldsplit_%s_inner_", ilink->splitname));
9119566063dSJacob Faibussowitsch       PetscCall(PetscOptionsFindPairPrefix_Private(((PetscObject)pc)->options,((PetscObject)pc)->prefix, schurtestoption, NULL, &flg));
912514bf10dSMatthew G Knepley       if (flg) {
913514bf10dSMatthew G Knepley         DM  dmInner;
91421635b76SJed Brown         KSP kspInner;
915686bed4dSStefano Zampini         PC  pcInner;
91621635b76SJed Brown 
9179566063dSJacob Faibussowitsch         PetscCall(MatSchurComplementGetKSP(jac->schur, &kspInner));
9189566063dSJacob Faibussowitsch         PetscCall(KSPReset(kspInner));
9199566063dSJacob Faibussowitsch         PetscCall(KSPSetOperators(kspInner,jac->mat[0],jac->pmat[0]));
9209566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(schurprefix, sizeof(schurprefix), "%sfieldsplit_%s_inner_", ((PetscObject)pc)->prefix ? ((PetscObject)pc)->prefix : "", ilink->splitname));
92121635b76SJed Brown         /* Indent this deeper to emphasize the "inner" nature of this solver. */
9229566063dSJacob Faibussowitsch         PetscCall(PetscObjectIncrementTabLevel((PetscObject)kspInner, (PetscObject) pc, 2));
9239566063dSJacob Faibussowitsch         PetscCall(PetscObjectIncrementTabLevel((PetscObject)kspInner->pc, (PetscObject) pc, 2));
9249566063dSJacob Faibussowitsch         PetscCall(KSPSetOptionsPrefix(kspInner, schurprefix));
925514bf10dSMatthew G Knepley 
926514bf10dSMatthew G Knepley         /* Set DM for new solver */
9279566063dSJacob Faibussowitsch         PetscCall(KSPGetDM(jac->head->ksp, &dmInner));
9289566063dSJacob Faibussowitsch         PetscCall(KSPSetDM(kspInner, dmInner));
9299566063dSJacob Faibussowitsch         PetscCall(KSPSetDMActive(kspInner, PETSC_FALSE));
930686bed4dSStefano Zampini 
931686bed4dSStefano Zampini         /* Defaults to PCKSP as preconditioner */
9329566063dSJacob Faibussowitsch         PetscCall(KSPGetPC(kspInner, &pcInner));
9339566063dSJacob Faibussowitsch         PetscCall(PCSetType(pcInner, PCKSP));
9349566063dSJacob Faibussowitsch         PetscCall(PCKSPSetKSP(pcInner, jac->head->ksp));
935514bf10dSMatthew G Knepley       } else {
93621635b76SJed Brown          /* Use the outer solver for the inner solve, but revert the KSPPREONLY from PCFieldSplitSetFields_FieldSplit or
93721635b76SJed Brown           * PCFieldSplitSetIS_FieldSplit. We don't want KSPPREONLY because it makes the Schur complement inexact,
93821635b76SJed Brown           * preventing Schur complement reduction to be an accurate solve. Usually when an iterative solver is used for
93921635b76SJed Brown           * S = D - C A_inner^{-1} B, we expect S to be defined using an accurate definition of A_inner^{-1}, so we make
94021635b76SJed Brown           * GMRES the default. Note that it is also common to use PREONLY for S, in which case S may not be used
94121635b76SJed Brown           * directly, and the user is responsible for setting an inexact method for fieldsplit's A^{-1}. */
9429566063dSJacob Faibussowitsch         PetscCall(KSPSetType(jac->head->ksp,KSPGMRES));
9439566063dSJacob Faibussowitsch         PetscCall(MatSchurComplementSetKSP(jac->schur,jac->head->ksp));
944bafc1b83SMatthew G Knepley       }
9459566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(jac->head->ksp,jac->mat[0],jac->pmat[0]));
9469566063dSJacob Faibussowitsch       PetscCall(KSPSetFromOptions(jac->head->ksp));
9479566063dSJacob Faibussowitsch       PetscCall(MatSetFromOptions(jac->schur));
9483b224e63SBarry Smith 
9499566063dSJacob Faibussowitsch       PetscCall(PetscObjectTypeCompare((PetscObject)jac->schur, MATSCHURCOMPLEMENT, &flg));
950686bed4dSStefano Zampini       if (flg) { /* Need to do this otherwise PCSetUp_KSP will overwrite the amat of jac->head->ksp */
951686bed4dSStefano Zampini         KSP kspInner;
952686bed4dSStefano Zampini         PC  pcInner;
953686bed4dSStefano Zampini 
9549566063dSJacob Faibussowitsch         PetscCall(MatSchurComplementGetKSP(jac->schur, &kspInner));
9559566063dSJacob Faibussowitsch         PetscCall(KSPGetPC(kspInner, &pcInner));
9569566063dSJacob Faibussowitsch         PetscCall(PetscObjectTypeCompare((PetscObject)pcInner, PCKSP, &flg));
957686bed4dSStefano Zampini         if (flg) {
958686bed4dSStefano Zampini           KSP ksp;
959686bed4dSStefano Zampini 
9609566063dSJacob Faibussowitsch           PetscCall(PCKSPGetKSP(pcInner, &ksp));
961686bed4dSStefano Zampini           if (ksp == jac->head->ksp) {
9629566063dSJacob Faibussowitsch             PetscCall(PCSetUseAmat(pcInner, PETSC_TRUE));
963686bed4dSStefano Zampini           }
964686bed4dSStefano Zampini         }
965686bed4dSStefano Zampini       }
9669566063dSJacob Faibussowitsch       PetscCall(PetscSNPrintf(schurtestoption, sizeof(schurtestoption), "-fieldsplit_%s_upper_", ilink->splitname));
9679566063dSJacob Faibussowitsch       PetscCall(PetscOptionsFindPairPrefix_Private(((PetscObject)pc)->options,((PetscObject)pc)->prefix, schurtestoption, NULL, &flg));
968443836d0SMatthew G Knepley       if (flg) {
969443836d0SMatthew G Knepley         DM dmInner;
970443836d0SMatthew G Knepley 
9719566063dSJacob Faibussowitsch         PetscCall(PetscSNPrintf(schurprefix, sizeof(schurprefix), "%sfieldsplit_%s_upper_", ((PetscObject)pc)->prefix ? ((PetscObject)pc)->prefix : "", ilink->splitname));
9729566063dSJacob Faibussowitsch         PetscCall(KSPCreate(PetscObjectComm((PetscObject)pc), &jac->kspupper));
9739566063dSJacob Faibussowitsch         PetscCall(KSPSetErrorIfNotConverged(jac->kspupper,pc->erroriffailure));
9749566063dSJacob Faibussowitsch         PetscCall(KSPSetOptionsPrefix(jac->kspupper, schurprefix));
9759566063dSJacob Faibussowitsch         PetscCall(PetscObjectIncrementTabLevel((PetscObject)jac->kspupper, (PetscObject) pc, 1));
9769566063dSJacob Faibussowitsch         PetscCall(PetscObjectIncrementTabLevel((PetscObject)jac->kspupper->pc, (PetscObject) pc, 1));
9779566063dSJacob Faibussowitsch         PetscCall(KSPGetDM(jac->head->ksp, &dmInner));
9789566063dSJacob Faibussowitsch         PetscCall(KSPSetDM(jac->kspupper, dmInner));
9799566063dSJacob Faibussowitsch         PetscCall(KSPSetDMActive(jac->kspupper, PETSC_FALSE));
9809566063dSJacob Faibussowitsch         PetscCall(KSPSetFromOptions(jac->kspupper));
9819566063dSJacob Faibussowitsch         PetscCall(KSPSetOperators(jac->kspupper,jac->mat[0],jac->pmat[0]));
9829566063dSJacob Faibussowitsch         PetscCall(VecDuplicate(jac->head->x, &jac->head->z));
983443836d0SMatthew G Knepley       } else {
984443836d0SMatthew G Knepley         jac->kspupper = jac->head->ksp;
9859566063dSJacob Faibussowitsch         PetscCall(PetscObjectReference((PetscObject) jac->head->ksp));
986443836d0SMatthew G Knepley       }
987443836d0SMatthew G Knepley 
988a7476a74SDmitry Karpeev       if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_SELFP) {
9899566063dSJacob Faibussowitsch         PetscCall(MatSchurComplementGetPmat(jac->schur,MAT_INITIAL_MATRIX,&jac->schurp));
990a7476a74SDmitry Karpeev       }
9919566063dSJacob Faibussowitsch       PetscCall(KSPCreate(PetscObjectComm((PetscObject)pc),&jac->kspschur));
9929566063dSJacob Faibussowitsch       PetscCall(KSPSetErrorIfNotConverged(jac->kspschur,pc->erroriffailure));
9939566063dSJacob Faibussowitsch       PetscCall(PetscLogObjectParent((PetscObject)pc,(PetscObject)jac->kspschur));
9949566063dSJacob Faibussowitsch       PetscCall(PetscObjectIncrementTabLevel((PetscObject)jac->kspschur,(PetscObject)pc,1));
995084e4875SJed Brown       if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_SELF) {
9967233a360SDmitry Karpeev         PC pcschur;
9979566063dSJacob Faibussowitsch         PetscCall(KSPGetPC(jac->kspschur,&pcschur));
9989566063dSJacob Faibussowitsch         PetscCall(PCSetType(pcschur,PCNONE));
999084e4875SJed Brown         /* Note: This is bad if there exist preconditioners for MATSCHURCOMPLEMENT */
1000e74569cdSMatthew G. Knepley       } else if (jac->schurpre == PC_FIELDSPLIT_SCHUR_PRE_FULL) {
10019566063dSJacob Faibussowitsch         PetscCall(MatSchurComplementComputeExplicitOperator(jac->schur, &jac->schur_user));
1002e69d4d44SBarry Smith       }
10039566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(jac->kspschur,jac->schur,FieldSplitSchurPre(jac)));
10049566063dSJacob Faibussowitsch       PetscCall(KSPGetOptionsPrefix(jac->head->next->ksp, &Dprefix));
10059566063dSJacob Faibussowitsch       PetscCall(KSPSetOptionsPrefix(jac->kspschur,         Dprefix));
1006c096484dSStefano Zampini       /* propagate DM */
1007b20b4189SMatthew G. Knepley       {
1008b20b4189SMatthew G. Knepley         DM sdm;
10099566063dSJacob Faibussowitsch         PetscCall(KSPGetDM(jac->head->next->ksp, &sdm));
1010b20b4189SMatthew G. Knepley         if (sdm) {
10119566063dSJacob Faibussowitsch           PetscCall(KSPSetDM(jac->kspschur, sdm));
10129566063dSJacob Faibussowitsch           PetscCall(KSPSetDMActive(jac->kspschur, PETSC_FALSE));
1013b20b4189SMatthew G. Knepley         }
1014b20b4189SMatthew G. Knepley       }
10153b224e63SBarry Smith       /* really want setfromoptions called in PCSetFromOptions_FieldSplit(), but it is not ready yet */
101620b26d62SBarry Smith       /* need to call this every time, since the jac->kspschur is freshly created, otherwise its options never get set */
10179566063dSJacob Faibussowitsch       PetscCall(KSPSetFromOptions(jac->kspschur));
10183b224e63SBarry Smith     }
10199566063dSJacob Faibussowitsch     PetscCall(MatAssemblyBegin(jac->schur,MAT_FINAL_ASSEMBLY));
10209566063dSJacob Faibussowitsch     PetscCall(MatAssemblyEnd(jac->schur,MAT_FINAL_ASSEMBLY));
1021093c86ffSJed Brown 
1022093c86ffSJed Brown     /* HACK: special support to forward L and Lp matrices that might be used by PCLSC */
10239566063dSJacob Faibussowitsch     PetscCall(PetscSNPrintf(lscname,sizeof(lscname),"%s_LSC_L",ilink->splitname));
10249566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)pc->mat,lscname,(PetscObject*)&LSC_L));
10259566063dSJacob Faibussowitsch     if (!LSC_L) PetscCall(PetscObjectQuery((PetscObject)pc->pmat,lscname,(PetscObject*)&LSC_L));
10269566063dSJacob Faibussowitsch     if (LSC_L) PetscCall(PetscObjectCompose((PetscObject)jac->schur,"LSC_L",(PetscObject)LSC_L));
10279566063dSJacob Faibussowitsch     PetscCall(PetscSNPrintf(lscname,sizeof(lscname),"%s_LSC_Lp",ilink->splitname));
10289566063dSJacob Faibussowitsch     PetscCall(PetscObjectQuery((PetscObject)pc->pmat,lscname,(PetscObject*)&LSC_L));
10299566063dSJacob Faibussowitsch     if (!LSC_L) PetscCall(PetscObjectQuery((PetscObject)pc->mat,lscname,(PetscObject*)&LSC_L));
10309566063dSJacob Faibussowitsch     if (LSC_L) PetscCall(PetscObjectCompose((PetscObject)jac->schur,"LSC_Lp",(PetscObject)LSC_L));
1031a51937d4SCarola Kruse   } else if (jac->type == PC_COMPOSITE_GKB) {
1032a51937d4SCarola Kruse     IS          ccis;
1033a51937d4SCarola Kruse     PetscInt    rstart,rend;
1034a51937d4SCarola Kruse 
103508401ef6SPierre Jolivet     PetscCheck(nsplit == 2,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_INCOMP,"To use GKB preconditioner you must have exactly 2 fields");
1036a51937d4SCarola Kruse 
1037a51937d4SCarola Kruse     ilink = jac->head;
1038a51937d4SCarola Kruse 
1039a51937d4SCarola Kruse     /* When extracting off-diagonal submatrices, we take complements from this range */
10409566063dSJacob Faibussowitsch     PetscCall(MatGetOwnershipRangeColumn(pc->mat,&rstart,&rend));
1041a51937d4SCarola Kruse 
10429566063dSJacob Faibussowitsch     PetscCall(ISComplement(ilink->is_col,rstart,rend,&ccis));
1043a51937d4SCarola Kruse     if (jac->offdiag_use_amat) {
10449566063dSJacob Faibussowitsch       PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->B));
1045a51937d4SCarola Kruse     } else {
10469566063dSJacob Faibussowitsch       PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->B));
1047a51937d4SCarola Kruse     }
10489566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&ccis));
1049e071a0a4SCarola Kruse     /* Create work vectors for GKB algorithm */
10509566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(ilink->x,&jac->u));
10519566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(ilink->x,&jac->Hu));
10529566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(ilink->x,&jac->w2));
1053a51937d4SCarola Kruse     ilink = ilink->next;
10549566063dSJacob Faibussowitsch     PetscCall(ISComplement(ilink->is_col,rstart,rend,&ccis));
1055a51937d4SCarola Kruse     if (jac->offdiag_use_amat) {
10569566063dSJacob Faibussowitsch       PetscCall(MatCreateSubMatrix(pc->mat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->C));
1057a51937d4SCarola Kruse     } else {
10589566063dSJacob Faibussowitsch       PetscCall(MatCreateSubMatrix(pc->pmat,ilink->is,ccis,MAT_INITIAL_MATRIX,&jac->C));
1059a51937d4SCarola Kruse     }
10609566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&ccis));
1061e071a0a4SCarola Kruse     /* Create work vectors for GKB algorithm */
10629566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(ilink->x,&jac->v));
10639566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(ilink->x,&jac->d));
10649566063dSJacob Faibussowitsch     PetscCall(VecDuplicate(ilink->x,&jac->w1));
10659566063dSJacob Faibussowitsch     PetscCall(MatGolubKahanComputeExplicitOperator(jac->mat[0],jac->B,jac->C,&jac->H,jac->gkbnu));
10669566063dSJacob Faibussowitsch     PetscCall(PetscCalloc1(jac->gkbdelay,&jac->vecz));
1067e071a0a4SCarola Kruse 
1068a51937d4SCarola Kruse     ilink = jac->head;
10699566063dSJacob Faibussowitsch     PetscCall(KSPSetOperators(ilink->ksp,jac->H,jac->H));
10709566063dSJacob Faibussowitsch     if (!jac->suboptionsset) PetscCall(KSPSetFromOptions(ilink->ksp));
1071e071a0a4SCarola Kruse     /* Create gkb_monitor context */
1072de482cd7SCarola Kruse     if (jac->gkbmonitor) {
1073de482cd7SCarola Kruse       PetscInt  tablevel;
10749566063dSJacob Faibussowitsch       PetscCall(PetscViewerCreate(PETSC_COMM_WORLD,&jac->gkbviewer));
10759566063dSJacob Faibussowitsch       PetscCall(PetscViewerSetType(jac->gkbviewer,PETSCVIEWERASCII));
10769566063dSJacob Faibussowitsch       PetscCall(PetscObjectGetTabLevel((PetscObject)ilink->ksp,&tablevel));
10779566063dSJacob Faibussowitsch       PetscCall(PetscViewerASCIISetTab(jac->gkbviewer,tablevel));
10789566063dSJacob Faibussowitsch       PetscCall(PetscObjectIncrementTabLevel((PetscObject)ilink->ksp,(PetscObject)ilink->ksp,1));
1079de482cd7SCarola Kruse     }
10803b224e63SBarry Smith   } else {
108168bd789dSDmitry Karpeev     /* set up the individual splits' PCs */
108297bbdb24SBarry Smith     i     = 0;
10835a9f2f41SSatish Balay     ilink = jac->head;
10845a9f2f41SSatish Balay     while (ilink) {
10859566063dSJacob Faibussowitsch       PetscCall(KSPSetOperators(ilink->ksp,jac->mat[i],jac->pmat[i]));
10863b224e63SBarry Smith       /* really want setfromoptions called in PCSetFromOptions_FieldSplit(), but it is not ready yet */
10879566063dSJacob Faibussowitsch       if (!jac->suboptionsset) PetscCall(KSPSetFromOptions(ilink->ksp));
108897bbdb24SBarry Smith       i++;
10895a9f2f41SSatish Balay       ilink = ilink->next;
10900971522cSBarry Smith     }
10913b224e63SBarry Smith   }
10923b224e63SBarry Smith 
10935ddf11f8SNicolas Barnafi   /* Set coordinates to the sub PC objects whenever these are set */
10945ddf11f8SNicolas Barnafi   if (jac->coordinates_set) {
10955ddf11f8SNicolas Barnafi     PC pc_coords;
10965ddf11f8SNicolas Barnafi     if (jac->type == PC_COMPOSITE_SCHUR) {
10975ddf11f8SNicolas Barnafi       // Head is first block.
10989566063dSJacob Faibussowitsch       PetscCall(KSPGetPC(jac->head->ksp, &pc_coords));
10999566063dSJacob Faibussowitsch       PetscCall(PCSetCoordinates(pc_coords, jac->head->dim, jac->head->ndofs, jac->head->coords));
11005ddf11f8SNicolas Barnafi       // Second one is Schur block, but its KSP object is in kspschur.
11019566063dSJacob Faibussowitsch       PetscCall(KSPGetPC(jac->kspschur, &pc_coords));
11029566063dSJacob Faibussowitsch       PetscCall(PCSetCoordinates(pc_coords, jac->head->next->dim, jac->head->next->ndofs, jac->head->next->coords));
11035ddf11f8SNicolas Barnafi     } else if (jac->type == PC_COMPOSITE_GKB) {
11049566063dSJacob Faibussowitsch       PetscCall(PetscInfo(pc, "Warning: Setting coordinates does nothing for the GKB Fieldpslit preconditioner"));
11055ddf11f8SNicolas Barnafi     } else {
11065ddf11f8SNicolas Barnafi       ilink = jac->head;
11075ddf11f8SNicolas Barnafi       while (ilink) {
11089566063dSJacob Faibussowitsch         PetscCall(KSPGetPC(ilink->ksp, &pc_coords));
11099566063dSJacob Faibussowitsch         PetscCall(PCSetCoordinates(pc_coords, ilink->dim, ilink->ndofs, ilink->coords));
11105ddf11f8SNicolas Barnafi         ilink = ilink->next;
11115ddf11f8SNicolas Barnafi       }
11125ddf11f8SNicolas Barnafi     }
11135ddf11f8SNicolas Barnafi   }
11145ddf11f8SNicolas Barnafi 
1115c1570756SJed Brown   jac->suboptionsset = PETSC_TRUE;
11160971522cSBarry Smith   PetscFunctionReturn(0);
11170971522cSBarry Smith }
11180971522cSBarry Smith 
11195a9f2f41SSatish Balay #define FieldSplitSplitSolveAdd(ilink,xx,yy) \
1120ca9f406cSSatish Balay   (VecScatterBegin(ilink->sctx,xx,ilink->x,INSERT_VALUES,SCATTER_FORWARD) || \
1121ca9f406cSSatish Balay    VecScatterEnd(ilink->sctx,xx,ilink->x,INSERT_VALUES,SCATTER_FORWARD) || \
11229df09d43SBarry Smith    PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL) ||\
11235a9f2f41SSatish Balay    KSPSolve(ilink->ksp,ilink->x,ilink->y) ||                               \
1124c0decd05SBarry Smith    KSPCheckSolve(ilink->ksp,pc,ilink->y)  || \
11259df09d43SBarry Smith    PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL) ||\
1126ca9f406cSSatish Balay    VecScatterBegin(ilink->sctx,ilink->y,yy,ADD_VALUES,SCATTER_REVERSE) ||  \
1127ca9f406cSSatish Balay    VecScatterEnd(ilink->sctx,ilink->y,yy,ADD_VALUES,SCATTER_REVERSE))
112879416396SBarry Smith 
11293b224e63SBarry Smith static PetscErrorCode PCApply_FieldSplit_Schur(PC pc,Vec x,Vec y)
11303b224e63SBarry Smith {
11313b224e63SBarry Smith   PC_FieldSplit      *jac = (PC_FieldSplit*)pc->data;
11323b224e63SBarry Smith   PC_FieldSplitLink  ilinkA = jac->head, ilinkD = ilinkA->next;
1133443836d0SMatthew G Knepley   KSP                kspA   = ilinkA->ksp, kspLower = kspA, kspUpper = jac->kspupper;
11343b224e63SBarry Smith 
11353b224e63SBarry Smith   PetscFunctionBegin;
1136c5d2311dSJed Brown   switch (jac->schurfactorization) {
1137c9c6ffaaSJed Brown   case PC_FIELDSPLIT_SCHUR_FACT_DIAG:
1138a04f6461SBarry Smith     /* [A00 0; 0 -S], positive definite, suitable for MINRES */
11399566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
11409566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,x,ilinkD->x,INSERT_VALUES,SCATTER_FORWARD));
11419566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
11429566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
11439566063dSJacob Faibussowitsch     PetscCall(KSPSolve(kspA,ilinkA->x,ilinkA->y));
11449566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(kspA,pc,ilinkA->y));
11459566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
11469566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
11479566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,x,ilinkD->x,INSERT_VALUES,SCATTER_FORWARD));
11489566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
11499566063dSJacob Faibussowitsch     PetscCall(KSPSolve(jac->kspschur,ilinkD->x,ilinkD->y));
11509566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(jac->kspschur,pc,ilinkD->y));
11519566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
11529566063dSJacob Faibussowitsch     PetscCall(VecScale(ilinkD->y,jac->schurscale));
11539566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
11549566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
11559566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
1156c5d2311dSJed Brown     break;
1157c9c6ffaaSJed Brown   case PC_FIELDSPLIT_SCHUR_FACT_LOWER:
1158a04f6461SBarry Smith     /* [A00 0; A10 S], suitable for left preconditioning */
11599566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
11609566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
11619566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
11629566063dSJacob Faibussowitsch     PetscCall(KSPSolve(kspA,ilinkA->x,ilinkA->y));
11639566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(kspA,pc,ilinkA->y));
11649566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
11659566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->C,ilinkA->y,ilinkD->x));
11669566063dSJacob Faibussowitsch     PetscCall(VecScale(ilinkD->x,-1.));
11679566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,x,ilinkD->x,ADD_VALUES,SCATTER_FORWARD));
11689566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
11699566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,x,ilinkD->x,ADD_VALUES,SCATTER_FORWARD));
11709566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
11719566063dSJacob Faibussowitsch     PetscCall(KSPSolve(jac->kspschur,ilinkD->x,ilinkD->y));
11729566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(jac->kspschur,pc,ilinkD->y));
11739566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
11749566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
11759566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
11769566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
1177c5d2311dSJed Brown     break;
1178c9c6ffaaSJed Brown   case PC_FIELDSPLIT_SCHUR_FACT_UPPER:
1179a04f6461SBarry Smith     /* [A00 A01; 0 S], suitable for right preconditioning */
11809566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,x,ilinkD->x,INSERT_VALUES,SCATTER_FORWARD));
11819566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,x,ilinkD->x,INSERT_VALUES,SCATTER_FORWARD));
11829566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
11839566063dSJacob Faibussowitsch     PetscCall(KSPSolve(jac->kspschur,ilinkD->x,ilinkD->y));
11849566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(jac->kspschur,pc,ilinkD->y));
11859566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));    PetscCall(MatMult(jac->B,ilinkD->y,ilinkA->x));
11869566063dSJacob Faibussowitsch     PetscCall(VecScale(ilinkA->x,-1.));
11879566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,x,ilinkA->x,ADD_VALUES,SCATTER_FORWARD));
11889566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
11899566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,x,ilinkA->x,ADD_VALUES,SCATTER_FORWARD));
11909566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
11919566063dSJacob Faibussowitsch     PetscCall(KSPSolve(kspA,ilinkA->x,ilinkA->y));
11929566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(kspA,pc,ilinkA->y));
11939566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
11949566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
11959566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
11969566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
1197c5d2311dSJed Brown     break;
1198c9c6ffaaSJed Brown   case PC_FIELDSPLIT_SCHUR_FACT_FULL:
1199c238f8cdSStefano Zampini     /* [1 0; A10 A00^{-1} 1] [A00 0; 0 S] [1 A00^{-1}A01; 0 1] */
12009566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
12019566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
12029566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(KSP_Solve_FS_L,kspLower,ilinkA->x,ilinkA->y,NULL));
12039566063dSJacob Faibussowitsch     PetscCall(KSPSolve(kspLower,ilinkA->x,ilinkA->y));
12049566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(kspLower,pc,ilinkA->y));
12059566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(KSP_Solve_FS_L,kspLower,ilinkA->x,ilinkA->y,NULL));
12069566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->C,ilinkA->y,ilinkD->x));
12079566063dSJacob Faibussowitsch     PetscCall(VecScale(ilinkD->x,-1.0));
12089566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,x,ilinkD->x,ADD_VALUES,SCATTER_FORWARD));
12099566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,x,ilinkD->x,ADD_VALUES,SCATTER_FORWARD));
12103b224e63SBarry Smith 
12119566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
12129566063dSJacob Faibussowitsch     PetscCall(KSPSolve(jac->kspschur,ilinkD->x,ilinkD->y));
12139566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(jac->kspschur,pc,ilinkD->y));
12149566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(KSP_Solve_FS_S,jac->kspschur,ilinkD->x,ilinkD->y,NULL));
12159566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
12163b224e63SBarry Smith 
1217443836d0SMatthew G Knepley     if (kspUpper == kspA) {
12189566063dSJacob Faibussowitsch       PetscCall(MatMult(jac->B,ilinkD->y,ilinkA->y));
12199566063dSJacob Faibussowitsch       PetscCall(VecAXPY(ilinkA->x,-1.0,ilinkA->y));
12209566063dSJacob Faibussowitsch       PetscCall(PetscLogEventBegin(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
12219566063dSJacob Faibussowitsch       PetscCall(KSPSolve(kspA,ilinkA->x,ilinkA->y));
12229566063dSJacob Faibussowitsch       PetscCall(KSPCheckSolve(kspA,pc,ilinkA->y));
12239566063dSJacob Faibussowitsch       PetscCall(PetscLogEventEnd(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
1224443836d0SMatthew G Knepley     } else {
12259566063dSJacob Faibussowitsch       PetscCall(PetscLogEventBegin(ilinkA->event,kspA,ilinkA->x,ilinkA->y,NULL));
12269566063dSJacob Faibussowitsch       PetscCall(KSPSolve(kspA,ilinkA->x,ilinkA->y));
12279566063dSJacob Faibussowitsch       PetscCall(KSPCheckSolve(kspA,pc,ilinkA->y));
12289566063dSJacob Faibussowitsch       PetscCall(MatMult(jac->B,ilinkD->y,ilinkA->x));
12299566063dSJacob Faibussowitsch       PetscCall(PetscLogEventBegin(KSP_Solve_FS_U,kspUpper,ilinkA->x,ilinkA->z,NULL));
12309566063dSJacob Faibussowitsch       PetscCall(KSPSolve(kspUpper,ilinkA->x,ilinkA->z));
12319566063dSJacob Faibussowitsch       PetscCall(KSPCheckSolve(kspUpper,pc,ilinkA->z));
12329566063dSJacob Faibussowitsch       PetscCall(PetscLogEventEnd(KSP_Solve_FS_U,kspUpper,ilinkA->x,ilinkA->z,NULL));
12339566063dSJacob Faibussowitsch       PetscCall(VecAXPY(ilinkA->y,-1.0,ilinkA->z));
1234443836d0SMatthew G Knepley     }
12359566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
12369566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
12379566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
1238c5d2311dSJed Brown   }
12393b224e63SBarry Smith   PetscFunctionReturn(0);
12403b224e63SBarry Smith }
12413b224e63SBarry Smith 
12420971522cSBarry Smith static PetscErrorCode PCApply_FieldSplit(PC pc,Vec x,Vec y)
12430971522cSBarry Smith {
12440971522cSBarry Smith   PC_FieldSplit      *jac = (PC_FieldSplit*)pc->data;
12455a9f2f41SSatish Balay   PC_FieldSplitLink  ilink = jac->head;
1246939b8a20SBarry Smith   PetscInt           cnt,bs;
12470971522cSBarry Smith 
12480971522cSBarry Smith   PetscFunctionBegin;
124979416396SBarry Smith   if (jac->type == PC_COMPOSITE_ADDITIVE) {
12501b9fc7fcSBarry Smith     if (jac->defaultsplit) {
12519566063dSJacob Faibussowitsch       PetscCall(VecGetBlockSize(x,&bs));
12522472a847SBarry Smith       PetscCheck(jac->bs <= 0 || bs == jac->bs,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Blocksize of x vector %" PetscInt_FMT " does not match fieldsplit blocksize %" PetscInt_FMT,bs,jac->bs);
12539566063dSJacob Faibussowitsch       PetscCall(VecGetBlockSize(y,&bs));
12542472a847SBarry Smith       PetscCheck(jac->bs <= 0 || bs == jac->bs,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Blocksize of y vector %" PetscInt_FMT " does not match fieldsplit blocksize %" PetscInt_FMT,bs,jac->bs);
12559566063dSJacob Faibussowitsch       PetscCall(VecStrideGatherAll(x,jac->x,INSERT_VALUES));
12565a9f2f41SSatish Balay       while (ilink) {
12579566063dSJacob Faibussowitsch         PetscCall(PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
12589566063dSJacob Faibussowitsch         PetscCall(KSPSolve(ilink->ksp,ilink->x,ilink->y));
12599566063dSJacob Faibussowitsch         PetscCall(KSPCheckSolve(ilink->ksp,pc,ilink->y));
12609566063dSJacob Faibussowitsch         PetscCall(PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
12615a9f2f41SSatish Balay         ilink = ilink->next;
12620971522cSBarry Smith       }
12639566063dSJacob Faibussowitsch       PetscCall(VecStrideScatterAll(jac->y,y,INSERT_VALUES));
12641b9fc7fcSBarry Smith     } else {
12659566063dSJacob Faibussowitsch       PetscCall(VecSet(y,0.0));
12665a9f2f41SSatish Balay       while (ilink) {
12679566063dSJacob Faibussowitsch         PetscCall(FieldSplitSplitSolveAdd(ilink,x,y));
12685a9f2f41SSatish Balay         ilink = ilink->next;
12691b9fc7fcSBarry Smith       }
12701b9fc7fcSBarry Smith     }
1271e52d2c62SBarry Smith   } else if (jac->type == PC_COMPOSITE_MULTIPLICATIVE && jac->nsplits == 2) {
12729566063dSJacob Faibussowitsch     PetscCall(VecSet(y,0.0));
1273e52d2c62SBarry Smith     /* solve on first block for first block variables */
12749566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilink->sctx,x,ilink->x,INSERT_VALUES,SCATTER_FORWARD));
12759566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilink->sctx,x,ilink->x,INSERT_VALUES,SCATTER_FORWARD));
12769566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
12779566063dSJacob Faibussowitsch     PetscCall(KSPSolve(ilink->ksp,ilink->x,ilink->y));
12789566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(ilink->ksp,pc,ilink->y));
12799566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
12809566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
12819566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
1282e52d2c62SBarry Smith 
1283e52d2c62SBarry Smith     /* compute the residual only onto second block variables using first block variables */
12849566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->Afield[1],ilink->y,ilink->next->x));
1285e52d2c62SBarry Smith     ilink = ilink->next;
12869566063dSJacob Faibussowitsch     PetscCall(VecScale(ilink->x,-1.0));
12879566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilink->sctx,x,ilink->x,ADD_VALUES,SCATTER_FORWARD));
12889566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilink->sctx,x,ilink->x,ADD_VALUES,SCATTER_FORWARD));
1289e52d2c62SBarry Smith 
1290e52d2c62SBarry Smith     /* solve on second block variables */
12919566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
12929566063dSJacob Faibussowitsch     PetscCall(KSPSolve(ilink->ksp,ilink->x,ilink->y));
12939566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(ilink->ksp,pc,ilink->y));
12949566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
12959566063dSJacob Faibussowitsch     PetscCall(VecScatterBegin(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
12969566063dSJacob Faibussowitsch     PetscCall(VecScatterEnd(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
129716913363SBarry Smith   } else if (jac->type == PC_COMPOSITE_MULTIPLICATIVE || jac->type == PC_COMPOSITE_SYMMETRIC_MULTIPLICATIVE) {
129879416396SBarry Smith     if (!jac->w1) {
12999566063dSJacob Faibussowitsch       PetscCall(VecDuplicate(x,&jac->w1));
13009566063dSJacob Faibussowitsch       PetscCall(VecDuplicate(x,&jac->w2));
130179416396SBarry Smith     }
13029566063dSJacob Faibussowitsch     PetscCall(VecSet(y,0.0));
13039566063dSJacob Faibussowitsch     PetscCall(FieldSplitSplitSolveAdd(ilink,x,y));
13043e197d65SBarry Smith     cnt  = 1;
13055a9f2f41SSatish Balay     while (ilink->next) {
13065a9f2f41SSatish Balay       ilink = ilink->next;
13073e197d65SBarry Smith       /* compute the residual only over the part of the vector needed */
13089566063dSJacob Faibussowitsch       PetscCall(MatMult(jac->Afield[cnt++],y,ilink->x));
13099566063dSJacob Faibussowitsch       PetscCall(VecScale(ilink->x,-1.0));
13109566063dSJacob Faibussowitsch       PetscCall(VecScatterBegin(ilink->sctx,x,ilink->x,ADD_VALUES,SCATTER_FORWARD));
13119566063dSJacob Faibussowitsch       PetscCall(VecScatterEnd(ilink->sctx,x,ilink->x,ADD_VALUES,SCATTER_FORWARD));
13129566063dSJacob Faibussowitsch       PetscCall(PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
13139566063dSJacob Faibussowitsch       PetscCall(KSPSolve(ilink->ksp,ilink->x,ilink->y));
13149566063dSJacob Faibussowitsch       PetscCall(KSPCheckSolve(ilink->ksp,pc,ilink->y));
13159566063dSJacob Faibussowitsch       PetscCall(PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
13169566063dSJacob Faibussowitsch       PetscCall(VecScatterBegin(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
13179566063dSJacob Faibussowitsch       PetscCall(VecScatterEnd(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
13183e197d65SBarry Smith     }
131951f519a2SBarry Smith     if (jac->type == PC_COMPOSITE_SYMMETRIC_MULTIPLICATIVE) {
132011755939SBarry Smith       cnt -= 2;
132151f519a2SBarry Smith       while (ilink->previous) {
132251f519a2SBarry Smith         ilink = ilink->previous;
132311755939SBarry Smith         /* compute the residual only over the part of the vector needed */
13249566063dSJacob Faibussowitsch         PetscCall(MatMult(jac->Afield[cnt--],y,ilink->x));
13259566063dSJacob Faibussowitsch         PetscCall(VecScale(ilink->x,-1.0));
13269566063dSJacob Faibussowitsch         PetscCall(VecScatterBegin(ilink->sctx,x,ilink->x,ADD_VALUES,SCATTER_FORWARD));
13279566063dSJacob Faibussowitsch         PetscCall(VecScatterEnd(ilink->sctx,x,ilink->x,ADD_VALUES,SCATTER_FORWARD));
13289566063dSJacob Faibussowitsch         PetscCall(PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
13299566063dSJacob Faibussowitsch         PetscCall(KSPSolve(ilink->ksp,ilink->x,ilink->y));
13309566063dSJacob Faibussowitsch         PetscCall(KSPCheckSolve(ilink->ksp,pc,ilink->y));
13319566063dSJacob Faibussowitsch         PetscCall(PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
13329566063dSJacob Faibussowitsch         PetscCall(VecScatterBegin(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
13339566063dSJacob Faibussowitsch         PetscCall(VecScatterEnd(ilink->sctx,ilink->y,y,ADD_VALUES,SCATTER_REVERSE));
133451f519a2SBarry Smith       }
133511755939SBarry Smith     }
133663a3b9bcSJacob Faibussowitsch   } else SETERRQ(PetscObjectComm((PetscObject)pc),PETSC_ERR_SUP,"Unsupported or unknown composition %d",(int) jac->type);
13370971522cSBarry Smith   PetscFunctionReturn(0);
13380971522cSBarry Smith }
13390971522cSBarry Smith 
1340a51937d4SCarola Kruse static PetscErrorCode PCApply_FieldSplit_GKB(PC pc,Vec x,Vec y)
1341a51937d4SCarola Kruse {
1342a51937d4SCarola Kruse   PC_FieldSplit      *jac = (PC_FieldSplit*)pc->data;
1343a51937d4SCarola Kruse   PC_FieldSplitLink  ilinkA = jac->head,ilinkD = ilinkA->next;
1344a51937d4SCarola Kruse   KSP                ksp = ilinkA->ksp;
1345de482cd7SCarola Kruse   Vec                u,v,Hu,d,work1,work2;
1346e071a0a4SCarola Kruse   PetscScalar        alpha,z,nrmz2,*vecz;
1347e071a0a4SCarola Kruse   PetscReal          lowbnd,nu,beta;
1348a51937d4SCarola Kruse   PetscInt           j,iterGKB;
1349a51937d4SCarola Kruse 
1350a51937d4SCarola Kruse   PetscFunctionBegin;
13519566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
13529566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(ilinkD->sctx,x,ilinkD->x,INSERT_VALUES,SCATTER_FORWARD));
13539566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(ilinkA->sctx,x,ilinkA->x,INSERT_VALUES,SCATTER_FORWARD));
13549566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(ilinkD->sctx,x,ilinkD->x,INSERT_VALUES,SCATTER_FORWARD));
1355e071a0a4SCarola Kruse 
1356e071a0a4SCarola Kruse   u     = jac->u;
1357e071a0a4SCarola Kruse   v     = jac->v;
1358e071a0a4SCarola Kruse   Hu    = jac->Hu;
1359e071a0a4SCarola Kruse   d     = jac->d;
1360e071a0a4SCarola Kruse   work1 = jac->w1;
1361e071a0a4SCarola Kruse   work2 = jac->w2;
1362e071a0a4SCarola Kruse   vecz  = jac->vecz;
1363a51937d4SCarola Kruse 
1364a51937d4SCarola Kruse   /* Change RHS to comply with matrix regularization H = A + nu*B*B' */
1365a51937d4SCarola Kruse   /* Add q = q + nu*B*b */
1366a51937d4SCarola Kruse   if (jac->gkbnu) {
1367a51937d4SCarola Kruse     nu = jac->gkbnu;
13689566063dSJacob Faibussowitsch     PetscCall(VecScale(ilinkD->x,jac->gkbnu));
13699566063dSJacob Faibussowitsch     PetscCall(MatMultAdd(jac->B,ilinkD->x,ilinkA->x,ilinkA->x));            /* q = q + nu*B*b */
1370a51937d4SCarola Kruse   } else {
1371a51937d4SCarola Kruse     /* Situation when no augmented Lagrangian is used. Then we set inner  */
1372a51937d4SCarola Kruse     /* matrix N = I in [Ar13], and thus nu = 1.                           */
1373a51937d4SCarola Kruse     nu = 1;
1374a51937d4SCarola Kruse   }
1375a51937d4SCarola Kruse 
1376a51937d4SCarola Kruse   /* Transform rhs from [q,tilde{b}] to [0,b] */
13779566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(ilinkA->event,ksp,ilinkA->x,ilinkA->y,NULL));
13789566063dSJacob Faibussowitsch   PetscCall(KSPSolve(ksp,ilinkA->x,ilinkA->y));
13799566063dSJacob Faibussowitsch   PetscCall(KSPCheckSolve(ksp,pc,ilinkA->y));
13809566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(ilinkA->event,ksp,ilinkA->x,ilinkA->y,NULL));
13819566063dSJacob Faibussowitsch   PetscCall(MatMultHermitianTranspose(jac->B,ilinkA->y,work1));
13829566063dSJacob Faibussowitsch   PetscCall(VecAXPBY(work1,1.0/nu,-1.0,ilinkD->x));            /* c = b - B'*x        */
1383a51937d4SCarola Kruse 
1384a51937d4SCarola Kruse   /* First step of algorithm */
13859566063dSJacob Faibussowitsch   PetscCall(VecNorm(work1,NORM_2,&beta));                   /* beta = sqrt(nu*c'*c)*/
1386e071a0a4SCarola Kruse   KSPCheckDot(ksp,beta);
1387addd1e01SJunchao Zhang   beta  = PetscSqrtReal(nu)*beta;
13889566063dSJacob Faibussowitsch   PetscCall(VecAXPBY(v,nu/beta,0.0,work1));                   /* v = nu/beta *c      */
13899566063dSJacob Faibussowitsch   PetscCall(MatMult(jac->B,v,work2));                       /* u = H^{-1}*B*v      */
13909566063dSJacob Faibussowitsch   PetscCall(PetscLogEventBegin(ilinkA->event,ksp,work2,u,NULL));
13919566063dSJacob Faibussowitsch   PetscCall(KSPSolve(ksp,work2,u));
13929566063dSJacob Faibussowitsch   PetscCall(KSPCheckSolve(ksp,pc,u));
13939566063dSJacob Faibussowitsch   PetscCall(PetscLogEventEnd(ilinkA->event,ksp,work2,u,NULL));
13949566063dSJacob Faibussowitsch   PetscCall(MatMult(jac->H,u,Hu));                          /* alpha = u'*H*u      */
13959566063dSJacob Faibussowitsch   PetscCall(VecDot(Hu,u,&alpha));
1396e071a0a4SCarola Kruse   KSPCheckDot(ksp,alpha);
139708401ef6SPierre Jolivet   PetscCheck(PetscRealPart(alpha) > 0.0,PETSC_COMM_SELF,PETSC_ERR_NOT_CONVERGED,"GKB preconditioner diverged, H is not positive definite");
1398addd1e01SJunchao Zhang   alpha = PetscSqrtReal(PetscAbsScalar(alpha));
13999566063dSJacob Faibussowitsch   PetscCall(VecScale(u,1.0/alpha));
14009566063dSJacob Faibussowitsch   PetscCall(VecAXPBY(d,1.0/alpha,0.0,v));                       /* v = nu/beta *c      */
1401de482cd7SCarola Kruse 
1402a51937d4SCarola Kruse   z = beta/alpha;
1403a51937d4SCarola Kruse   vecz[1] = z;
1404a51937d4SCarola Kruse 
1405de482cd7SCarola Kruse   /* Computation of first iterate x(1) and p(1) */
14069566063dSJacob Faibussowitsch   PetscCall(VecAXPY(ilinkA->y,z,u));
14079566063dSJacob Faibussowitsch   PetscCall(VecCopy(d,ilinkD->y));
14089566063dSJacob Faibussowitsch   PetscCall(VecScale(ilinkD->y,-z));
1409a51937d4SCarola Kruse 
1410a51937d4SCarola Kruse   iterGKB = 1; lowbnd = 2*jac->gkbtol;
1411de482cd7SCarola Kruse   if (jac->gkbmonitor) {
141263a3b9bcSJacob Faibussowitsch     PetscCall(PetscViewerASCIIPrintf(jac->gkbviewer,"%3" PetscInt_FMT " GKB Lower bound estimate %14.12e\n",iterGKB,(double)lowbnd));
1413de482cd7SCarola Kruse   }
1414de482cd7SCarola Kruse 
1415a51937d4SCarola Kruse   while (iterGKB < jac->gkbmaxit && lowbnd > jac->gkbtol) {
1416a51937d4SCarola Kruse     iterGKB += 1;
14179566063dSJacob Faibussowitsch     PetscCall(MatMultHermitianTranspose(jac->B,u,work1)); /* v <- nu*(B'*u-alpha/nu*v) */
14189566063dSJacob Faibussowitsch     PetscCall(VecAXPBY(v,nu,-alpha,work1));
14199566063dSJacob Faibussowitsch     PetscCall(VecNorm(v,NORM_2,&beta));                   /* beta = sqrt(nu)*v'*v      */
1420addd1e01SJunchao Zhang     beta  = beta/PetscSqrtReal(nu);
14219566063dSJacob Faibussowitsch     PetscCall(VecScale(v,1.0/beta));
14229566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->B,v,work2));                  /* u <- H^{-1}*(B*v-beta*H*u) */
14239566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->H,u,Hu));
14249566063dSJacob Faibussowitsch     PetscCall(VecAXPY(work2,-beta,Hu));
14259566063dSJacob Faibussowitsch     PetscCall(PetscLogEventBegin(ilinkA->event,ksp,work2,u,NULL));
14269566063dSJacob Faibussowitsch     PetscCall(KSPSolve(ksp,work2,u));
14279566063dSJacob Faibussowitsch     PetscCall(KSPCheckSolve(ksp,pc,u));
14289566063dSJacob Faibussowitsch     PetscCall(PetscLogEventEnd(ilinkA->event,ksp,work2,u,NULL));
14299566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->H,u,Hu));                      /* alpha = u'*H*u            */
14309566063dSJacob Faibussowitsch     PetscCall(VecDot(Hu,u,&alpha));
1431e071a0a4SCarola Kruse     KSPCheckDot(ksp,alpha);
143208401ef6SPierre Jolivet     PetscCheck(PetscRealPart(alpha) > 0.0,PETSC_COMM_SELF,PETSC_ERR_NOT_CONVERGED,"GKB preconditioner diverged, H is not positive definite");
1433addd1e01SJunchao Zhang     alpha = PetscSqrtReal(PetscAbsScalar(alpha));
14349566063dSJacob Faibussowitsch     PetscCall(VecScale(u,1.0/alpha));
1435a51937d4SCarola Kruse 
1436e071a0a4SCarola Kruse     z = -beta/alpha*z;                                            /* z <- beta/alpha*z     */
1437a51937d4SCarola Kruse     vecz[0] = z;
1438a51937d4SCarola Kruse 
1439a51937d4SCarola Kruse     /* Computation of new iterate x(i+1) and p(i+1) */
14409566063dSJacob Faibussowitsch     PetscCall(VecAXPBY(d,1.0/alpha,-beta/alpha,v));       /* d = (v-beta*d)/alpha */
14419566063dSJacob Faibussowitsch     PetscCall(VecAXPY(ilinkA->y,z,u));                  /* r = r + z*u          */
14429566063dSJacob Faibussowitsch     PetscCall(VecAXPY(ilinkD->y,-z,d));                 /* p = p - z*d          */
14439566063dSJacob Faibussowitsch     PetscCall(MatMult(jac->H,ilinkA->y,Hu));            /* ||u||_H = u'*H*u     */
14449566063dSJacob Faibussowitsch     PetscCall(VecDot(Hu,ilinkA->y,&nrmz2));
1445a51937d4SCarola Kruse 
1446a51937d4SCarola Kruse     /* Compute Lower Bound estimate */
1447a51937d4SCarola Kruse     if (iterGKB > jac->gkbdelay) {
1448a51937d4SCarola Kruse       lowbnd = 0.0;
1449a51937d4SCarola Kruse       for (j=0; j<jac->gkbdelay; j++) {
1450a51937d4SCarola Kruse         lowbnd += PetscAbsScalar(vecz[j]*vecz[j]);
1451a51937d4SCarola Kruse       }
1452addd1e01SJunchao Zhang       lowbnd = PetscSqrtReal(lowbnd/PetscAbsScalar(nrmz2));
1453a51937d4SCarola Kruse     }
1454a51937d4SCarola Kruse 
1455a51937d4SCarola Kruse     for (j=0; j<jac->gkbdelay-1; j++) {
1456a51937d4SCarola Kruse       vecz[jac->gkbdelay-j-1] = vecz[jac->gkbdelay-j-2];
1457a51937d4SCarola Kruse     }
1458de482cd7SCarola Kruse     if (jac->gkbmonitor) {
145963a3b9bcSJacob Faibussowitsch       PetscCall(PetscViewerASCIIPrintf(jac->gkbviewer,"%3" PetscInt_FMT " GKB Lower bound estimate %14.12e\n",iterGKB,(double)lowbnd));
1460de482cd7SCarola Kruse     }
1461a51937d4SCarola Kruse   }
1462a51937d4SCarola Kruse 
14639566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
14649566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(ilinkA->sctx,ilinkA->y,y,INSERT_VALUES,SCATTER_REVERSE));
14659566063dSJacob Faibussowitsch   PetscCall(VecScatterBegin(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
14669566063dSJacob Faibussowitsch   PetscCall(VecScatterEnd(ilinkD->sctx,ilinkD->y,y,INSERT_VALUES,SCATTER_REVERSE));
1467a51937d4SCarola Kruse 
1468a51937d4SCarola Kruse   PetscFunctionReturn(0);
1469a51937d4SCarola Kruse }
1470a51937d4SCarola Kruse 
1471421e10b8SBarry Smith #define FieldSplitSplitSolveAddTranspose(ilink,xx,yy) \
1472ca9f406cSSatish Balay   (VecScatterBegin(ilink->sctx,xx,ilink->y,INSERT_VALUES,SCATTER_FORWARD) || \
1473ca9f406cSSatish Balay    VecScatterEnd(ilink->sctx,xx,ilink->y,INSERT_VALUES,SCATTER_FORWARD) || \
14749df09d43SBarry Smith    PetscLogEventBegin(ilink->event,ilink->ksp,ilink->y,ilink->x,NULL) || \
1475421e10b8SBarry Smith    KSPSolveTranspose(ilink->ksp,ilink->y,ilink->x) ||                  \
1476c0decd05SBarry Smith    KSPCheckSolve(ilink->ksp,pc,ilink->x) || \
1477c0decd05SBarry Smith    PetscLogEventEnd(ilink->event,ilink->ksp,ilink->y,ilink->x,NULL) ||   \
1478ca9f406cSSatish Balay    VecScatterBegin(ilink->sctx,ilink->x,yy,ADD_VALUES,SCATTER_REVERSE) || \
1479ca9f406cSSatish Balay    VecScatterEnd(ilink->sctx,ilink->x,yy,ADD_VALUES,SCATTER_REVERSE))
1480421e10b8SBarry Smith 
1481421e10b8SBarry Smith static PetscErrorCode PCApplyTranspose_FieldSplit(PC pc,Vec x,Vec y)
1482421e10b8SBarry Smith {
1483421e10b8SBarry Smith   PC_FieldSplit      *jac = (PC_FieldSplit*)pc->data;
1484421e10b8SBarry Smith   PC_FieldSplitLink  ilink = jac->head;
1485939b8a20SBarry Smith   PetscInt           bs;
1486421e10b8SBarry Smith 
1487421e10b8SBarry Smith   PetscFunctionBegin;
1488421e10b8SBarry Smith   if (jac->type == PC_COMPOSITE_ADDITIVE) {
1489421e10b8SBarry Smith     if (jac->defaultsplit) {
14909566063dSJacob Faibussowitsch       PetscCall(VecGetBlockSize(x,&bs));
14912472a847SBarry Smith       PetscCheck(jac->bs <= 0 || bs == jac->bs,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Blocksize of x vector %" PetscInt_FMT " does not match fieldsplit blocksize %" PetscInt_FMT,bs,jac->bs);
14929566063dSJacob Faibussowitsch       PetscCall(VecGetBlockSize(y,&bs));
14932472a847SBarry Smith       PetscCheck(jac->bs <= 0 || bs == jac->bs,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Blocksize of y vector %" PetscInt_FMT " does not match fieldsplit blocksize %" PetscInt_FMT,bs,jac->bs);
14949566063dSJacob Faibussowitsch       PetscCall(VecStrideGatherAll(x,jac->x,INSERT_VALUES));
1495421e10b8SBarry Smith       while (ilink) {
14969566063dSJacob Faibussowitsch         PetscCall(PetscLogEventBegin(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
14979566063dSJacob Faibussowitsch         PetscCall(KSPSolveTranspose(ilink->ksp,ilink->x,ilink->y));
14989566063dSJacob Faibussowitsch         PetscCall(KSPCheckSolve(ilink->ksp,pc,ilink->y));
14999566063dSJacob Faibussowitsch         PetscCall(PetscLogEventEnd(ilink->event,ilink->ksp,ilink->x,ilink->y,NULL));
1500421e10b8SBarry Smith         ilink = ilink->next;
1501421e10b8SBarry Smith       }
15029566063dSJacob Faibussowitsch       PetscCall(VecStrideScatterAll(jac->y,y,INSERT_VALUES));
1503421e10b8SBarry Smith     } else {
15049566063dSJacob Faibussowitsch       PetscCall(VecSet(y,0.0));
1505421e10b8SBarry Smith       while (ilink) {
15069566063dSJacob Faibussowitsch         PetscCall(FieldSplitSplitSolveAddTranspose(ilink,x,y));
1507421e10b8SBarry Smith         ilink = ilink->next;
1508421e10b8SBarry Smith       }
1509421e10b8SBarry Smith     }
1510421e10b8SBarry Smith   } else {
1511421e10b8SBarry Smith     if (!jac->w1) {
15129566063dSJacob Faibussowitsch       PetscCall(VecDuplicate(x,&jac->w1));
15139566063dSJacob Faibussowitsch       PetscCall(VecDuplicate(x,&jac->w2));
1514421e10b8SBarry Smith     }
15159566063dSJacob Faibussowitsch     PetscCall(VecSet(y,0.0));
1516421e10b8SBarry Smith     if (jac->type == PC_COMPOSITE_SYMMETRIC_MULTIPLICATIVE) {
15179566063dSJacob Faibussowitsch       PetscCall(FieldSplitSplitSolveAddTranspose(ilink,x,y));
1518421e10b8SBarry Smith       while (ilink->next) {
1519421e10b8SBarry Smith         ilink = ilink->next;
15209566063dSJacob Faibussowitsch         PetscCall(MatMultTranspose(pc->mat,y,jac->w1));
15219566063dSJacob Faibussowitsch         PetscCall(VecWAXPY(jac->w2,-1.0,jac->w1,x));
15229566063dSJacob Faibussowitsch         PetscCall(FieldSplitSplitSolveAddTranspose(ilink,jac->w2,y));
1523421e10b8SBarry Smith       }
1524421e10b8SBarry Smith       while (ilink->previous) {
1525421e10b8SBarry Smith         ilink = ilink->previous;
15269566063dSJacob Faibussowitsch         PetscCall(MatMultTranspose(pc->mat,y,jac->w1));
15279566063dSJacob Faibussowitsch         PetscCall(VecWAXPY(jac->w2,-1.0,jac->w1,x));
15289566063dSJacob Faibussowitsch         PetscCall(FieldSplitSplitSolveAddTranspose(ilink,jac->w2,y));
1529421e10b8SBarry Smith       }
1530421e10b8SBarry Smith     } else {
1531421e10b8SBarry Smith       while (ilink->next) {   /* get to last entry in linked list */
1532421e10b8SBarry Smith         ilink = ilink->next;
1533421e10b8SBarry Smith       }
15349566063dSJacob Faibussowitsch       PetscCall(FieldSplitSplitSolveAddTranspose(ilink,x,y));
1535421e10b8SBarry Smith       while (ilink->previous) {
1536421e10b8SBarry Smith         ilink = ilink->previous;
15379566063dSJacob Faibussowitsch         PetscCall(MatMultTranspose(pc->mat,y,jac->w1));
15389566063dSJacob Faibussowitsch         PetscCall(VecWAXPY(jac->w2,-1.0,jac->w1,x));
15399566063dSJacob Faibussowitsch         PetscCall(FieldSplitSplitSolveAddTranspose(ilink,jac->w2,y));
1540421e10b8SBarry Smith       }
1541421e10b8SBarry Smith     }
1542421e10b8SBarry Smith   }
1543421e10b8SBarry Smith   PetscFunctionReturn(0);
1544421e10b8SBarry Smith }
1545421e10b8SBarry Smith 
1546574deadeSBarry Smith static PetscErrorCode PCReset_FieldSplit(PC pc)
15470971522cSBarry Smith {
15480971522cSBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
15495a9f2f41SSatish Balay   PC_FieldSplitLink ilink = jac->head,next;
15500971522cSBarry Smith 
15510971522cSBarry Smith   PetscFunctionBegin;
15525a9f2f41SSatish Balay   while (ilink) {
15539566063dSJacob Faibussowitsch     PetscCall(KSPDestroy(&ilink->ksp));
15549566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&ilink->x));
15559566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&ilink->y));
15569566063dSJacob Faibussowitsch     PetscCall(VecDestroy(&ilink->z));
15579566063dSJacob Faibussowitsch     PetscCall(VecScatterDestroy(&ilink->sctx));
15589566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&ilink->is));
15599566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&ilink->is_col));
15609566063dSJacob Faibussowitsch     PetscCall(PetscFree(ilink->splitname));
15619566063dSJacob Faibussowitsch     PetscCall(PetscFree(ilink->fields));
15629566063dSJacob Faibussowitsch     PetscCall(PetscFree(ilink->fields_col));
15635a9f2f41SSatish Balay     next  = ilink->next;
15649566063dSJacob Faibussowitsch     PetscCall(PetscFree(ilink));
15655a9f2f41SSatish Balay     ilink = next;
15660971522cSBarry Smith   }
1567f5f0d762SBarry Smith   jac->head = NULL;
15689566063dSJacob Faibussowitsch   PetscCall(PetscFree2(jac->x,jac->y));
1569574deadeSBarry Smith   if (jac->mat && jac->mat != jac->pmat) {
15709566063dSJacob Faibussowitsch     PetscCall(MatDestroyMatrices(jac->nsplits,&jac->mat));
1571574deadeSBarry Smith   } else if (jac->mat) {
15720298fd71SBarry Smith     jac->mat = NULL;
1573574deadeSBarry Smith   }
15749566063dSJacob Faibussowitsch   if (jac->pmat) PetscCall(MatDestroyMatrices(jac->nsplits,&jac->pmat));
15759566063dSJacob Faibussowitsch   if (jac->Afield) PetscCall(MatDestroyMatrices(jac->nsplits,&jac->Afield));
1576f5f0d762SBarry Smith   jac->nsplits = 0;
15779566063dSJacob Faibussowitsch   PetscCall(VecDestroy(&jac->w1));
15789566063dSJacob Faibussowitsch   PetscCall(VecDestroy(&jac->w2));
15799566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&jac->schur));
15809566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&jac->schurp));
15819566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&jac->schur_user));
15829566063dSJacob Faibussowitsch   PetscCall(KSPDestroy(&jac->kspschur));
15839566063dSJacob Faibussowitsch   PetscCall(KSPDestroy(&jac->kspupper));
15849566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&jac->B));
15859566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&jac->C));
15869566063dSJacob Faibussowitsch   PetscCall(MatDestroy(&jac->H));
15879566063dSJacob Faibussowitsch   PetscCall(VecDestroy(&jac->u));
15889566063dSJacob Faibussowitsch   PetscCall(VecDestroy(&jac->v));
15899566063dSJacob Faibussowitsch   PetscCall(VecDestroy(&jac->Hu));
15909566063dSJacob Faibussowitsch   PetscCall(VecDestroy(&jac->d));
15919566063dSJacob Faibussowitsch   PetscCall(PetscFree(jac->vecz));
15929566063dSJacob Faibussowitsch   PetscCall(PetscViewerDestroy(&jac->gkbviewer));
15936dbb499eSCian Wilson   jac->isrestrict = PETSC_FALSE;
1594574deadeSBarry Smith   PetscFunctionReturn(0);
1595574deadeSBarry Smith }
1596574deadeSBarry Smith 
1597574deadeSBarry Smith static PetscErrorCode PCDestroy_FieldSplit(PC pc)
1598574deadeSBarry Smith {
1599574deadeSBarry Smith   PetscFunctionBegin;
16009566063dSJacob Faibussowitsch   PetscCall(PCReset_FieldSplit(pc));
16019566063dSJacob Faibussowitsch   PetscCall(PetscFree(pc->data));
16022e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCSetCoordinates_C",NULL));
16039566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetFields_C",NULL));
16049566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetIS_C",NULL));
16059566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetType_C",NULL));
16069566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetBlockSize_C",NULL));
16072e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitRestrictIS_C",NULL));
16082e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSchurGetSubKSP_C",NULL));
16092e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",NULL));
16102e956fe4SStefano Zampini 
16112e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBTol_C",NULL));
16122e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBMaxit_C",NULL));
16132e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBNu_C",NULL));
16142e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBDelay_C",NULL));
16152e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",NULL));
16169566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurPre_C",NULL));
16179566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSchurPre_C",NULL));
16189566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurFactType_C",NULL));
16192e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurScale_C",NULL));
16200971522cSBarry Smith   PetscFunctionReturn(0);
16210971522cSBarry Smith }
16220971522cSBarry Smith 
16234416b707SBarry Smith static PetscErrorCode PCSetFromOptions_FieldSplit(PetscOptionItems *PetscOptionsObject,PC pc)
16240971522cSBarry Smith {
16256c924f48SJed Brown   PetscInt        bs;
16267b752e3dSPatrick Sanan   PetscBool       flg;
16279dcbbd2bSBarry Smith   PC_FieldSplit   *jac = (PC_FieldSplit*)pc->data;
16283b224e63SBarry Smith   PCCompositeType ctype;
16291b9fc7fcSBarry Smith 
16300971522cSBarry Smith   PetscFunctionBegin;
1631d0609cedSBarry Smith   PetscOptionsHeadBegin(PetscOptionsObject,"FieldSplit options");
16329566063dSJacob Faibussowitsch   PetscCall(PetscOptionsBool("-pc_fieldsplit_dm_splits","Whether to use DMCreateFieldDecomposition() for splits","PCFieldSplitSetDMSplits",jac->dm_splits,&jac->dm_splits,NULL));
16339566063dSJacob Faibussowitsch   PetscCall(PetscOptionsInt("-pc_fieldsplit_block_size","Blocksize that defines number of fields","PCFieldSplitSetBlockSize",jac->bs,&bs,&flg));
16341baa6e33SBarry Smith   if (flg) PetscCall(PCFieldSplitSetBlockSize(pc,bs));
16352686e3e9SMatthew G. Knepley   jac->diag_use_amat = pc->useAmat;
16369566063dSJacob Faibussowitsch   PetscCall(PetscOptionsBool("-pc_fieldsplit_diag_use_amat","Use Amat (not Pmat) to extract diagonal fieldsplit blocks", "PCFieldSplitSetDiagUseAmat",jac->diag_use_amat,&jac->diag_use_amat,NULL));
16372686e3e9SMatthew G. Knepley   jac->offdiag_use_amat = pc->useAmat;
16389566063dSJacob Faibussowitsch   PetscCall(PetscOptionsBool("-pc_fieldsplit_off_diag_use_amat","Use Amat (not Pmat) to extract off-diagonal fieldsplit blocks", "PCFieldSplitSetOffDiagUseAmat",jac->offdiag_use_amat,&jac->offdiag_use_amat,NULL));
16399566063dSJacob Faibussowitsch   PetscCall(PetscOptionsBool("-pc_fieldsplit_detect_saddle_point","Form 2-way split by detecting zero diagonal entries", "PCFieldSplitSetDetectSaddlePoint",jac->detect,&jac->detect,NULL));
16409566063dSJacob Faibussowitsch   PetscCall(PCFieldSplitSetDetectSaddlePoint(pc,jac->detect)); /* Sets split type and Schur PC type */
16419566063dSJacob Faibussowitsch   PetscCall(PetscOptionsEnum("-pc_fieldsplit_type","Type of composition","PCFieldSplitSetType",PCCompositeTypes,(PetscEnum)jac->type,(PetscEnum*)&ctype,&flg));
16421baa6e33SBarry Smith   if (flg) PetscCall(PCFieldSplitSetType(pc,ctype));
1643c30613efSMatthew Knepley   /* Only setup fields once */
1644c30613efSMatthew Knepley   if ((jac->bs > 0) && (jac->nsplits == 0)) {
1645d32f9abdSBarry Smith     /* only allow user to set fields from command line if bs is already known.
1646d32f9abdSBarry Smith        otherwise user can set them in PCFieldSplitSetDefaults() */
16479566063dSJacob Faibussowitsch     PetscCall(PCFieldSplitSetRuntimeSplits_Private(pc));
16489566063dSJacob Faibussowitsch     if (jac->splitdefined) PetscCall(PetscInfo(pc,"Splits defined using the options database\n"));
1649d32f9abdSBarry Smith   }
1650c5d2311dSJed Brown   if (jac->type == PC_COMPOSITE_SCHUR) {
16519566063dSJacob Faibussowitsch     PetscCall(PetscOptionsGetEnum(((PetscObject)pc)->options,((PetscObject)pc)->prefix,"-pc_fieldsplit_schur_factorization_type",PCFieldSplitSchurFactTypes,(PetscEnum*)&jac->schurfactorization,&flg));
16529566063dSJacob Faibussowitsch     if (flg) PetscCall(PetscInfo(pc,"Deprecated use of -pc_fieldsplit_schur_factorization_type\n"));
16539566063dSJacob Faibussowitsch     PetscCall(PetscOptionsEnum("-pc_fieldsplit_schur_fact_type","Which off-diagonal parts of the block factorization to use","PCFieldSplitSetSchurFactType",PCFieldSplitSchurFactTypes,(PetscEnum)jac->schurfactorization,(PetscEnum*)&jac->schurfactorization,NULL));
16549566063dSJacob Faibussowitsch     PetscCall(PetscOptionsEnum("-pc_fieldsplit_schur_precondition","How to build preconditioner for Schur complement","PCFieldSplitSetSchurPre",PCFieldSplitSchurPreTypes,(PetscEnum)jac->schurpre,(PetscEnum*)&jac->schurpre,NULL));
16559566063dSJacob Faibussowitsch     PetscCall(PetscOptionsScalar("-pc_fieldsplit_schur_scale","Scale Schur complement","PCFieldSplitSetSchurScale",jac->schurscale,&jac->schurscale,NULL));
1656a51937d4SCarola Kruse   } else if (jac->type == PC_COMPOSITE_GKB) {
16579566063dSJacob Faibussowitsch     PetscCall(PetscOptionsReal("-pc_fieldsplit_gkb_tol","The tolerance for the lower bound stopping criterion","PCFieldSplitGKBTol",jac->gkbtol,&jac->gkbtol,NULL));
16589566063dSJacob Faibussowitsch     PetscCall(PetscOptionsInt("-pc_fieldsplit_gkb_delay","The delay value for lower bound criterion","PCFieldSplitGKBDelay",jac->gkbdelay,&jac->gkbdelay,NULL));
16599566063dSJacob Faibussowitsch     PetscCall(PetscOptionsReal("-pc_fieldsplit_gkb_nu","Parameter in augmented Lagrangian approach","PCFieldSplitGKBNu",jac->gkbnu,&jac->gkbnu,NULL));
166063a3b9bcSJacob Faibussowitsch     PetscCheck(jac->gkbnu >= 0,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"nu cannot be less than 0: value %g",(double)jac->gkbnu);
16619566063dSJacob Faibussowitsch     PetscCall(PetscOptionsInt("-pc_fieldsplit_gkb_maxit","Maximum allowed number of iterations","PCFieldSplitGKBMaxit",jac->gkbmaxit,&jac->gkbmaxit,NULL));
16629566063dSJacob Faibussowitsch     PetscCall(PetscOptionsBool("-pc_fieldsplit_gkb_monitor","Prints number of GKB iterations and error","PCFieldSplitGKB",jac->gkbmonitor,&jac->gkbmonitor,NULL));
1663c5d2311dSJed Brown   }
1664d0609cedSBarry Smith   PetscOptionsHeadEnd();
16650971522cSBarry Smith   PetscFunctionReturn(0);
16660971522cSBarry Smith }
16670971522cSBarry Smith 
16680971522cSBarry Smith /*------------------------------------------------------------------------------------*/
16690971522cSBarry Smith 
16701e6b0712SBarry Smith static PetscErrorCode  PCFieldSplitSetFields_FieldSplit(PC pc,const char splitname[],PetscInt n,const PetscInt *fields,const PetscInt *fields_col)
16710971522cSBarry Smith {
167297bbdb24SBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
16735a9f2f41SSatish Balay   PC_FieldSplitLink ilink,next = jac->head;
167469a612a9SBarry Smith   char              prefix[128];
16755d4c12cdSJungho Lee   PetscInt          i;
16760971522cSBarry Smith 
16770971522cSBarry Smith   PetscFunctionBegin;
16786c924f48SJed Brown   if (jac->splitdefined) {
16799566063dSJacob Faibussowitsch     PetscCall(PetscInfo(pc,"Ignoring new split \"%s\" because the splits have already been defined\n",splitname));
16806c924f48SJed Brown     PetscFunctionReturn(0);
16816c924f48SJed Brown   }
168251f519a2SBarry Smith   for (i=0; i<n; i++) {
168363a3b9bcSJacob Faibussowitsch     PetscCheck(fields[i] < jac->bs,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Field %" PetscInt_FMT " requested but only %" PetscInt_FMT " exist",fields[i],jac->bs);
168463a3b9bcSJacob Faibussowitsch     PetscCheck(fields[i] >= 0,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Negative field %" PetscInt_FMT " requested",fields[i]);
168551f519a2SBarry Smith   }
16869566063dSJacob Faibussowitsch   PetscCall(PetscNew(&ilink));
1687a04f6461SBarry Smith   if (splitname) {
16889566063dSJacob Faibussowitsch     PetscCall(PetscStrallocpy(splitname,&ilink->splitname));
1689a04f6461SBarry Smith   } else {
16909566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(3,&ilink->splitname));
169163a3b9bcSJacob Faibussowitsch     PetscCall(PetscSNPrintf(ilink->splitname,2,"%" PetscInt_FMT,jac->nsplits));
1692a04f6461SBarry Smith   }
16939df09d43SBarry Smith   ilink->event = jac->nsplits < 5 ? KSP_Solve_FS_0 + jac->nsplits : KSP_Solve_FS_0 + 4; /* Any split great than 4 gets logged in the 4th split */
16949566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(n,&ilink->fields));
16959566063dSJacob Faibussowitsch   PetscCall(PetscArraycpy(ilink->fields,fields,n));
16969566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(n,&ilink->fields_col));
16979566063dSJacob Faibussowitsch   PetscCall(PetscArraycpy(ilink->fields_col,fields_col,n));
16982fa5cd67SKarl Rupp 
16995a9f2f41SSatish Balay   ilink->nfields = n;
17000298fd71SBarry Smith   ilink->next    = NULL;
17019566063dSJacob Faibussowitsch   PetscCall(KSPCreate(PetscObjectComm((PetscObject)pc),&ilink->ksp));
17029566063dSJacob Faibussowitsch   PetscCall(KSPSetErrorIfNotConverged(ilink->ksp,pc->erroriffailure));
17039566063dSJacob Faibussowitsch   PetscCall(PetscObjectIncrementTabLevel((PetscObject)ilink->ksp,(PetscObject)pc,1));
17049566063dSJacob Faibussowitsch   PetscCall(KSPSetType(ilink->ksp,KSPPREONLY));
17059566063dSJacob Faibussowitsch   PetscCall(PetscLogObjectParent((PetscObject)pc,(PetscObject)ilink->ksp));
170669a612a9SBarry Smith 
17079566063dSJacob Faibussowitsch   PetscCall(PetscSNPrintf(prefix,sizeof(prefix),"%sfieldsplit_%s_",((PetscObject)pc)->prefix ? ((PetscObject)pc)->prefix : "",ilink->splitname));
17089566063dSJacob Faibussowitsch   PetscCall(KSPSetOptionsPrefix(ilink->ksp,prefix));
17090971522cSBarry Smith 
17100971522cSBarry Smith   if (!next) {
17115a9f2f41SSatish Balay     jac->head       = ilink;
17120298fd71SBarry Smith     ilink->previous = NULL;
17130971522cSBarry Smith   } else {
17140971522cSBarry Smith     while (next->next) {
17150971522cSBarry Smith       next = next->next;
17160971522cSBarry Smith     }
17175a9f2f41SSatish Balay     next->next      = ilink;
171851f519a2SBarry Smith     ilink->previous = next;
17190971522cSBarry Smith   }
17200971522cSBarry Smith   jac->nsplits++;
17210971522cSBarry Smith   PetscFunctionReturn(0);
17220971522cSBarry Smith }
17230971522cSBarry Smith 
1724285fb4e2SStefano Zampini static PetscErrorCode  PCFieldSplitSchurGetSubKSP_FieldSplit(PC pc,PetscInt *n,KSP **subksp)
1725285fb4e2SStefano Zampini {
1726285fb4e2SStefano Zampini   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
1727285fb4e2SStefano Zampini 
1728285fb4e2SStefano Zampini   PetscFunctionBegin;
1729285fb4e2SStefano Zampini   *subksp = NULL;
1730285fb4e2SStefano Zampini   if (n) *n = 0;
1731285fb4e2SStefano Zampini   if (jac->type == PC_COMPOSITE_SCHUR) {
1732285fb4e2SStefano Zampini     PetscInt nn;
1733285fb4e2SStefano Zampini 
173428b400f6SJacob Faibussowitsch     PetscCheck(jac->schur,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() before calling PCFieldSplitSchurGetSubKSP()");
173563a3b9bcSJacob Faibussowitsch     PetscCheck(jac->nsplits == 2,PetscObjectComm((PetscObject)pc),PETSC_ERR_PLIB,"Unexpected number of splits %" PetscInt_FMT " != 2",jac->nsplits);
1736285fb4e2SStefano Zampini     nn   = jac->nsplits + (jac->kspupper != jac->head->ksp ? 1 : 0);
17379566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(nn,subksp));
1738285fb4e2SStefano Zampini     (*subksp)[0] = jac->head->ksp;
1739285fb4e2SStefano Zampini     (*subksp)[1] = jac->kspschur;
1740285fb4e2SStefano Zampini     if (jac->kspupper != jac->head->ksp) (*subksp)[2] = jac->kspupper;
1741285fb4e2SStefano Zampini     if (n) *n = nn;
1742285fb4e2SStefano Zampini   }
1743285fb4e2SStefano Zampini   PetscFunctionReturn(0);
1744285fb4e2SStefano Zampini }
1745285fb4e2SStefano Zampini 
17461e6b0712SBarry Smith static PetscErrorCode  PCFieldSplitGetSubKSP_FieldSplit_Schur(PC pc,PetscInt *n,KSP **subksp)
1747e69d4d44SBarry Smith {
1748e69d4d44SBarry Smith   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
1749e69d4d44SBarry Smith 
1750e69d4d44SBarry Smith   PetscFunctionBegin;
175128b400f6SJacob Faibussowitsch   PetscCheck(jac->schur,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Must call KSPSetUp() or PCSetUp() before calling PCFieldSplitGetSubKSP()");
17529566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(jac->nsplits,subksp));
17539566063dSJacob Faibussowitsch   PetscCall(MatSchurComplementGetKSP(jac->schur,*subksp));
17542fa5cd67SKarl Rupp 
1755e69d4d44SBarry Smith   (*subksp)[1] = jac->kspschur;
175613e0d083SBarry Smith   if (n) *n = jac->nsplits;
1757e69d4d44SBarry Smith   PetscFunctionReturn(0);
1758e69d4d44SBarry Smith }
17590971522cSBarry Smith 
17601e6b0712SBarry Smith static PetscErrorCode  PCFieldSplitGetSubKSP_FieldSplit(PC pc,PetscInt *n,KSP **subksp)
17610971522cSBarry Smith {
17620971522cSBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
17630971522cSBarry Smith   PetscInt          cnt   = 0;
17645a9f2f41SSatish Balay   PC_FieldSplitLink ilink = jac->head;
17650971522cSBarry Smith 
17660971522cSBarry Smith   PetscFunctionBegin;
17679566063dSJacob Faibussowitsch   PetscCall(PetscMalloc1(jac->nsplits,subksp));
17685a9f2f41SSatish Balay   while (ilink) {
17695a9f2f41SSatish Balay     (*subksp)[cnt++] = ilink->ksp;
17705a9f2f41SSatish Balay     ilink            = ilink->next;
17710971522cSBarry Smith   }
177263a3b9bcSJacob Faibussowitsch   PetscCheck(cnt == jac->nsplits,PETSC_COMM_SELF,PETSC_ERR_PLIB,"Corrupt PCFIELDSPLIT object: number of splits in linked list %" PetscInt_FMT " does not match number in object %" PetscInt_FMT,cnt,jac->nsplits);
177313e0d083SBarry Smith   if (n) *n = jac->nsplits;
17740971522cSBarry Smith   PetscFunctionReturn(0);
17750971522cSBarry Smith }
17760971522cSBarry Smith 
17776dbb499eSCian Wilson /*@C
17786dbb499eSCian Wilson     PCFieldSplitRestrictIS - Restricts the fieldsplit ISs to be within a given IS.
17796dbb499eSCian Wilson 
17806dbb499eSCian Wilson     Input Parameters:
17816dbb499eSCian Wilson +   pc  - the preconditioner context
1782a2b725a8SWilliam Gropp -   is - the index set that defines the indices to which the fieldsplit is to be restricted
17836dbb499eSCian Wilson 
17846dbb499eSCian Wilson     Level: advanced
17856dbb499eSCian Wilson 
17866dbb499eSCian Wilson @*/
17876dbb499eSCian Wilson PetscErrorCode  PCFieldSplitRestrictIS(PC pc,IS isy)
17886dbb499eSCian Wilson {
17896dbb499eSCian Wilson   PetscFunctionBegin;
17906dbb499eSCian Wilson   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
17916dbb499eSCian Wilson   PetscValidHeaderSpecific(isy,IS_CLASSID,2);
1792cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitRestrictIS_C",(PC,IS),(pc,isy));
17936dbb499eSCian Wilson   PetscFunctionReturn(0);
17946dbb499eSCian Wilson }
17956dbb499eSCian Wilson 
17966dbb499eSCian Wilson static PetscErrorCode  PCFieldSplitRestrictIS_FieldSplit(PC pc, IS isy)
17976dbb499eSCian Wilson {
17986dbb499eSCian Wilson   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
17996dbb499eSCian Wilson   PC_FieldSplitLink ilink = jac->head, next;
18006dbb499eSCian Wilson   PetscInt          localsize,size,sizez,i;
18016dbb499eSCian Wilson   const PetscInt    *ind, *indz;
18026dbb499eSCian Wilson   PetscInt          *indc, *indcz;
18036dbb499eSCian Wilson   PetscBool         flg;
18046dbb499eSCian Wilson 
18056dbb499eSCian Wilson   PetscFunctionBegin;
18069566063dSJacob Faibussowitsch   PetscCall(ISGetLocalSize(isy,&localsize));
18079566063dSJacob Faibussowitsch   PetscCallMPI(MPI_Scan(&localsize,&size,1,MPIU_INT,MPI_SUM,PetscObjectComm((PetscObject)isy)));
18086dbb499eSCian Wilson   size -= localsize;
18096dbb499eSCian Wilson   while (ilink) {
18106dbb499eSCian Wilson     IS isrl,isr;
18111c7cfba8SBarry Smith     PC subpc;
18129566063dSJacob Faibussowitsch     PetscCall(ISEmbed(ilink->is, isy, PETSC_TRUE, &isrl));
18139566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(isrl,&localsize));
18149566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(localsize,&indc));
18159566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(isrl,&ind));
18169566063dSJacob Faibussowitsch     PetscCall(PetscArraycpy(indc,ind,localsize));
18179566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(isrl,&ind));
18189566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&isrl));
18196dbb499eSCian Wilson     for (i=0; i<localsize; i++) *(indc+i) += size;
18209566063dSJacob Faibussowitsch     PetscCall(ISCreateGeneral(PetscObjectComm((PetscObject)isy),localsize,indc,PETSC_OWN_POINTER,&isr));
18219566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)isr));
18229566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&ilink->is));
18236dbb499eSCian Wilson     ilink->is     = isr;
18249566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)isr));
18259566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&ilink->is_col));
18266dbb499eSCian Wilson     ilink->is_col = isr;
18279566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&isr));
18289566063dSJacob Faibussowitsch     PetscCall(KSPGetPC(ilink->ksp, &subpc));
18299566063dSJacob Faibussowitsch     PetscCall(PetscObjectTypeCompare((PetscObject)subpc,PCFIELDSPLIT,&flg));
18306dbb499eSCian Wilson     if (flg) {
18316dbb499eSCian Wilson       IS iszl,isz;
18326dbb499eSCian Wilson       MPI_Comm comm;
18339566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(ilink->is,&localsize));
18346dbb499eSCian Wilson       comm   = PetscObjectComm((PetscObject)ilink->is);
18359566063dSJacob Faibussowitsch       PetscCall(ISEmbed(isy, ilink->is, PETSC_TRUE, &iszl));
18369566063dSJacob Faibussowitsch       PetscCallMPI(MPI_Scan(&localsize,&sizez,1,MPIU_INT,MPI_SUM,comm));
18376dbb499eSCian Wilson       sizez -= localsize;
18389566063dSJacob Faibussowitsch       PetscCall(ISGetLocalSize(iszl,&localsize));
18399566063dSJacob Faibussowitsch       PetscCall(PetscMalloc1(localsize,&indcz));
18409566063dSJacob Faibussowitsch       PetscCall(ISGetIndices(iszl,&indz));
18419566063dSJacob Faibussowitsch       PetscCall(PetscArraycpy(indcz,indz,localsize));
18429566063dSJacob Faibussowitsch       PetscCall(ISRestoreIndices(iszl,&indz));
18439566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&iszl));
18446dbb499eSCian Wilson       for (i=0; i<localsize; i++) *(indcz+i) += sizez;
18459566063dSJacob Faibussowitsch       PetscCall(ISCreateGeneral(comm,localsize,indcz,PETSC_OWN_POINTER,&isz));
18469566063dSJacob Faibussowitsch       PetscCall(PCFieldSplitRestrictIS(subpc,isz));
18479566063dSJacob Faibussowitsch       PetscCall(ISDestroy(&isz));
18486dbb499eSCian Wilson     }
18496dbb499eSCian Wilson     next = ilink->next;
18506dbb499eSCian Wilson     ilink = next;
18516dbb499eSCian Wilson   }
18526dbb499eSCian Wilson   jac->isrestrict = PETSC_TRUE;
18536dbb499eSCian Wilson   PetscFunctionReturn(0);
18546dbb499eSCian Wilson }
18556dbb499eSCian Wilson 
18561e6b0712SBarry Smith static PetscErrorCode  PCFieldSplitSetIS_FieldSplit(PC pc,const char splitname[],IS is)
1857704ba839SBarry Smith {
1858704ba839SBarry Smith   PC_FieldSplit     *jac = (PC_FieldSplit*)pc->data;
1859704ba839SBarry Smith   PC_FieldSplitLink ilink, next = jac->head;
1860704ba839SBarry Smith   char              prefix[128];
1861704ba839SBarry Smith 
1862704ba839SBarry Smith   PetscFunctionBegin;
18636c924f48SJed Brown   if (jac->splitdefined) {
18649566063dSJacob Faibussowitsch     PetscCall(PetscInfo(pc,"Ignoring new split \"%s\" because the splits have already been defined\n",splitname));
18656c924f48SJed Brown     PetscFunctionReturn(0);
18666c924f48SJed Brown   }
18679566063dSJacob Faibussowitsch   PetscCall(PetscNew(&ilink));
1868a04f6461SBarry Smith   if (splitname) {
18699566063dSJacob Faibussowitsch     PetscCall(PetscStrallocpy(splitname,&ilink->splitname));
1870a04f6461SBarry Smith   } else {
18719566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(8,&ilink->splitname));
187263a3b9bcSJacob Faibussowitsch     PetscCall(PetscSNPrintf(ilink->splitname,7,"%" PetscInt_FMT,jac->nsplits));
1873a04f6461SBarry Smith   }
18749df09d43SBarry Smith   ilink->event  = jac->nsplits < 5 ? KSP_Solve_FS_0 + jac->nsplits : KSP_Solve_FS_0 + 4; /* Any split great than 4 gets logged in the 4th split */
18759566063dSJacob Faibussowitsch   PetscCall(PetscObjectReference((PetscObject)is));
18769566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&ilink->is));
1877b5787286SJed Brown   ilink->is     = is;
18789566063dSJacob Faibussowitsch   PetscCall(PetscObjectReference((PetscObject)is));
18799566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&ilink->is_col));
1880b5787286SJed Brown   ilink->is_col = is;
18810298fd71SBarry Smith   ilink->next   = NULL;
18829566063dSJacob Faibussowitsch   PetscCall(KSPCreate(PetscObjectComm((PetscObject)pc),&ilink->ksp));
18839566063dSJacob Faibussowitsch   PetscCall(KSPSetErrorIfNotConverged(ilink->ksp,pc->erroriffailure));
18849566063dSJacob Faibussowitsch   PetscCall(PetscObjectIncrementTabLevel((PetscObject)ilink->ksp,(PetscObject)pc,1));
18859566063dSJacob Faibussowitsch   PetscCall(KSPSetType(ilink->ksp,KSPPREONLY));
18869566063dSJacob Faibussowitsch   PetscCall(PetscLogObjectParent((PetscObject)pc,(PetscObject)ilink->ksp));
1887704ba839SBarry Smith 
18889566063dSJacob Faibussowitsch   PetscCall(PetscSNPrintf(prefix,sizeof(prefix),"%sfieldsplit_%s_",((PetscObject)pc)->prefix ? ((PetscObject)pc)->prefix : "",ilink->splitname));
18899566063dSJacob Faibussowitsch   PetscCall(KSPSetOptionsPrefix(ilink->ksp,prefix));
1890704ba839SBarry Smith 
1891704ba839SBarry Smith   if (!next) {
1892704ba839SBarry Smith     jac->head       = ilink;
18930298fd71SBarry Smith     ilink->previous = NULL;
1894704ba839SBarry Smith   } else {
1895704ba839SBarry Smith     while (next->next) {
1896704ba839SBarry Smith       next = next->next;
1897704ba839SBarry Smith     }
1898704ba839SBarry Smith     next->next      = ilink;
1899704ba839SBarry Smith     ilink->previous = next;
1900704ba839SBarry Smith   }
1901704ba839SBarry Smith   jac->nsplits++;
1902704ba839SBarry Smith   PetscFunctionReturn(0);
1903704ba839SBarry Smith }
1904704ba839SBarry Smith 
190594ef8ddeSSatish Balay /*@C
19060971522cSBarry Smith     PCFieldSplitSetFields - Sets the fields for one particular split in the field split preconditioner
19070971522cSBarry Smith 
1908ad4df100SBarry Smith     Logically Collective on PC
19090971522cSBarry Smith 
19100971522cSBarry Smith     Input Parameters:
19110971522cSBarry Smith +   pc  - the preconditioner context
19120298fd71SBarry Smith .   splitname - name of this split, if NULL the number of the split is used
19130971522cSBarry Smith .   n - the number of fields in this split
1914db4c96c1SJed Brown -   fields - the fields in this split
19150971522cSBarry Smith 
19160971522cSBarry Smith     Level: intermediate
19170971522cSBarry Smith 
191895452b02SPatrick Sanan     Notes:
191995452b02SPatrick Sanan     Use PCFieldSplitSetIS() to set a completely general set of indices as a field.
1920d32f9abdSBarry Smith 
19217287d2a3SDmitry Karpeev      The PCFieldSplitSetFields() is for defining fields as strided blocks. For example, if the block
1922d32f9abdSBarry Smith      size is three then one can define a field as 0, or 1 or 2 or 0,1 or 0,2 or 1,2 which mean
1923d32f9abdSBarry Smith      0xx3xx6xx9xx12 ... x1xx4xx7xx ... xx2xx5xx8xx.. 01x34x67x... 0x1x3x5x7.. x12x45x78x....
1924d32f9abdSBarry Smith      where the numbered entries indicate what is in the field.
1925d32f9abdSBarry Smith 
1926db4c96c1SJed Brown      This function is called once per split (it creates a new split each time).  Solve options
1927db4c96c1SJed Brown      for this split will be available under the prefix -fieldsplit_SPLITNAME_.
1928db4c96c1SJed Brown 
19295d4c12cdSJungho Lee      Developer Note: This routine does not actually create the IS representing the split, that is delayed
19305d4c12cdSJungho Lee      until PCSetUp_FieldSplit(), because information about the vector/matrix layouts may not be
19315d4c12cdSJungho Lee      available when this routine is called.
19325d4c12cdSJungho Lee 
1933db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetBlockSize()`, `PCFieldSplitSetIS()`
19340971522cSBarry Smith 
19350971522cSBarry Smith @*/
19365d4c12cdSJungho Lee PetscErrorCode  PCFieldSplitSetFields(PC pc,const char splitname[],PetscInt n,const PetscInt *fields,const PetscInt *fields_col)
19370971522cSBarry Smith {
19380971522cSBarry Smith   PetscFunctionBegin;
19390700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
1940db4c96c1SJed Brown   PetscValidCharPointer(splitname,2);
194163a3b9bcSJacob Faibussowitsch   PetscCheck(n >= 1,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_OUTOFRANGE,"Provided number of fields %" PetscInt_FMT " in split \"%s\" not positive",n,splitname);
1942064a246eSJacob Faibussowitsch   PetscValidIntPointer(fields,4);
1943cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetFields_C",(PC,const char[],PetscInt,const PetscInt*,const PetscInt*),(pc,splitname,n,fields,fields_col));
19440971522cSBarry Smith   PetscFunctionReturn(0);
19450971522cSBarry Smith }
19460971522cSBarry Smith 
1947c84da90fSDmitry Karpeev /*@
1948c84da90fSDmitry Karpeev     PCFieldSplitSetDiagUseAmat - set flag indicating whether to extract diagonal blocks from Amat (rather than Pmat)
1949c84da90fSDmitry Karpeev 
1950c84da90fSDmitry Karpeev     Logically Collective on PC
1951c84da90fSDmitry Karpeev 
1952c84da90fSDmitry Karpeev     Input Parameters:
1953c84da90fSDmitry Karpeev +   pc  - the preconditioner object
1954c84da90fSDmitry Karpeev -   flg - boolean flag indicating whether or not to use Amat to extract the diagonal blocks from
1955c84da90fSDmitry Karpeev 
19569506b623SDmitry Karpeev     Options Database:
1957147403d9SBarry Smith .   -pc_fieldsplit_diag_use_amat - use the Amat to provide the diagonal blocks
1958c84da90fSDmitry Karpeev 
1959c84da90fSDmitry Karpeev     Level: intermediate
1960c84da90fSDmitry Karpeev 
1961db781477SPatrick Sanan .seealso: `PCFieldSplitGetDiagUseAmat()`, `PCFieldSplitSetOffDiagUseAmat()`, `PCFIELDSPLIT`
1962c84da90fSDmitry Karpeev 
1963c84da90fSDmitry Karpeev @*/
1964c84da90fSDmitry Karpeev PetscErrorCode  PCFieldSplitSetDiagUseAmat(PC pc,PetscBool flg)
1965c84da90fSDmitry Karpeev {
1966c84da90fSDmitry Karpeev   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
1967c84da90fSDmitry Karpeev   PetscBool      isfs;
1968c84da90fSDmitry Karpeev 
1969c84da90fSDmitry Karpeev   PetscFunctionBegin;
1970c84da90fSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
19719566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&isfs));
197228b400f6SJacob Faibussowitsch   PetscCheck(isfs,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"PC not of type %s",PCFIELDSPLIT);
1973c84da90fSDmitry Karpeev   jac->diag_use_amat = flg;
1974c84da90fSDmitry Karpeev   PetscFunctionReturn(0);
1975c84da90fSDmitry Karpeev }
1976c84da90fSDmitry Karpeev 
1977c84da90fSDmitry Karpeev /*@
1978c84da90fSDmitry Karpeev     PCFieldSplitGetDiagUseAmat - get the flag indicating whether to extract diagonal blocks from Amat (rather than Pmat)
1979c84da90fSDmitry Karpeev 
1980c84da90fSDmitry Karpeev     Logically Collective on PC
1981c84da90fSDmitry Karpeev 
1982c84da90fSDmitry Karpeev     Input Parameters:
1983c84da90fSDmitry Karpeev .   pc  - the preconditioner object
1984c84da90fSDmitry Karpeev 
1985c84da90fSDmitry Karpeev     Output Parameters:
1986c84da90fSDmitry Karpeev .   flg - boolean flag indicating whether or not to use Amat to extract the diagonal blocks from
1987c84da90fSDmitry Karpeev 
1988c84da90fSDmitry Karpeev     Level: intermediate
1989c84da90fSDmitry Karpeev 
1990db781477SPatrick Sanan .seealso: `PCFieldSplitSetDiagUseAmat()`, `PCFieldSplitGetOffDiagUseAmat()`, `PCFIELDSPLIT`
1991c84da90fSDmitry Karpeev 
1992c84da90fSDmitry Karpeev @*/
1993c84da90fSDmitry Karpeev PetscErrorCode  PCFieldSplitGetDiagUseAmat(PC pc,PetscBool *flg)
1994c84da90fSDmitry Karpeev {
1995c84da90fSDmitry Karpeev   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
1996c84da90fSDmitry Karpeev   PetscBool      isfs;
1997c84da90fSDmitry Karpeev 
1998c84da90fSDmitry Karpeev   PetscFunctionBegin;
1999c84da90fSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2000534a8f05SLisandro Dalcin   PetscValidBoolPointer(flg,2);
20019566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&isfs));
200228b400f6SJacob Faibussowitsch   PetscCheck(isfs,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"PC not of type %s",PCFIELDSPLIT);
2003c84da90fSDmitry Karpeev   *flg = jac->diag_use_amat;
2004c84da90fSDmitry Karpeev   PetscFunctionReturn(0);
2005c84da90fSDmitry Karpeev }
2006c84da90fSDmitry Karpeev 
2007c84da90fSDmitry Karpeev /*@
2008c84da90fSDmitry Karpeev     PCFieldSplitSetOffDiagUseAmat - set flag indicating whether to extract off-diagonal blocks from Amat (rather than Pmat)
2009c84da90fSDmitry Karpeev 
2010c84da90fSDmitry Karpeev     Logically Collective on PC
2011c84da90fSDmitry Karpeev 
2012c84da90fSDmitry Karpeev     Input Parameters:
2013c84da90fSDmitry Karpeev +   pc  - the preconditioner object
2014c84da90fSDmitry Karpeev -   flg - boolean flag indicating whether or not to use Amat to extract the off-diagonal blocks from
2015c84da90fSDmitry Karpeev 
20169506b623SDmitry Karpeev     Options Database:
2017147403d9SBarry Smith .     -pc_fieldsplit_off_diag_use_amat <bool> - use the Amat to extract the off-diagonal blocks
2018c84da90fSDmitry Karpeev 
2019c84da90fSDmitry Karpeev     Level: intermediate
2020c84da90fSDmitry Karpeev 
2021db781477SPatrick Sanan .seealso: `PCFieldSplitGetOffDiagUseAmat()`, `PCFieldSplitSetDiagUseAmat()`, `PCFIELDSPLIT`
2022c84da90fSDmitry Karpeev 
2023c84da90fSDmitry Karpeev @*/
2024c84da90fSDmitry Karpeev PetscErrorCode  PCFieldSplitSetOffDiagUseAmat(PC pc,PetscBool flg)
2025c84da90fSDmitry Karpeev {
2026c84da90fSDmitry Karpeev   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2027c84da90fSDmitry Karpeev   PetscBool      isfs;
2028c84da90fSDmitry Karpeev 
2029c84da90fSDmitry Karpeev   PetscFunctionBegin;
2030c84da90fSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
20319566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&isfs));
203228b400f6SJacob Faibussowitsch   PetscCheck(isfs,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"PC not of type %s",PCFIELDSPLIT);
2033c84da90fSDmitry Karpeev   jac->offdiag_use_amat = flg;
2034c84da90fSDmitry Karpeev   PetscFunctionReturn(0);
2035c84da90fSDmitry Karpeev }
2036c84da90fSDmitry Karpeev 
2037c84da90fSDmitry Karpeev /*@
2038c84da90fSDmitry Karpeev     PCFieldSplitGetOffDiagUseAmat - get the flag indicating whether to extract off-diagonal blocks from Amat (rather than Pmat)
2039c84da90fSDmitry Karpeev 
2040c84da90fSDmitry Karpeev     Logically Collective on PC
2041c84da90fSDmitry Karpeev 
2042c84da90fSDmitry Karpeev     Input Parameters:
2043c84da90fSDmitry Karpeev .   pc  - the preconditioner object
2044c84da90fSDmitry Karpeev 
2045c84da90fSDmitry Karpeev     Output Parameters:
2046c84da90fSDmitry Karpeev .   flg - boolean flag indicating whether or not to use Amat to extract the off-diagonal blocks from
2047c84da90fSDmitry Karpeev 
2048c84da90fSDmitry Karpeev     Level: intermediate
2049c84da90fSDmitry Karpeev 
2050db781477SPatrick Sanan .seealso: `PCFieldSplitSetOffDiagUseAmat()`, `PCFieldSplitGetDiagUseAmat()`, `PCFIELDSPLIT`
2051c84da90fSDmitry Karpeev 
2052c84da90fSDmitry Karpeev @*/
2053c84da90fSDmitry Karpeev PetscErrorCode  PCFieldSplitGetOffDiagUseAmat(PC pc,PetscBool *flg)
2054c84da90fSDmitry Karpeev {
2055c84da90fSDmitry Karpeev   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
2056c84da90fSDmitry Karpeev   PetscBool      isfs;
2057c84da90fSDmitry Karpeev 
2058c84da90fSDmitry Karpeev   PetscFunctionBegin;
2059c84da90fSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2060534a8f05SLisandro Dalcin   PetscValidBoolPointer(flg,2);
20619566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&isfs));
206228b400f6SJacob Faibussowitsch   PetscCheck(isfs,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"PC not of type %s",PCFIELDSPLIT);
2063c84da90fSDmitry Karpeev   *flg = jac->offdiag_use_amat;
2064c84da90fSDmitry Karpeev   PetscFunctionReturn(0);
2065c84da90fSDmitry Karpeev }
2066c84da90fSDmitry Karpeev 
2067218d4943SBarry Smith /*@C
2068704ba839SBarry Smith     PCFieldSplitSetIS - Sets the exact elements for field
2069704ba839SBarry Smith 
2070ad4df100SBarry Smith     Logically Collective on PC
2071704ba839SBarry Smith 
2072704ba839SBarry Smith     Input Parameters:
2073704ba839SBarry Smith +   pc  - the preconditioner context
20740298fd71SBarry Smith .   splitname - name of this split, if NULL the number of the split is used
2075db4c96c1SJed Brown -   is - the index set that defines the vector elements in this field
2076704ba839SBarry Smith 
2077a6ffb8dbSJed Brown     Notes:
2078a6ffb8dbSJed Brown     Use PCFieldSplitSetFields(), for fields defined by strided types.
2079a6ffb8dbSJed Brown 
2080db4c96c1SJed Brown     This function is called once per split (it creates a new split each time).  Solve options
2081db4c96c1SJed Brown     for this split will be available under the prefix -fieldsplit_SPLITNAME_.
2082d32f9abdSBarry Smith 
2083704ba839SBarry Smith     Level: intermediate
2084704ba839SBarry Smith 
2085db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetBlockSize()`
2086704ba839SBarry Smith 
2087704ba839SBarry Smith @*/
20887087cfbeSBarry Smith PetscErrorCode  PCFieldSplitSetIS(PC pc,const char splitname[],IS is)
2089704ba839SBarry Smith {
2090704ba839SBarry Smith   PetscFunctionBegin;
20910700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
20927b62db95SJungho Lee   if (splitname) PetscValidCharPointer(splitname,2);
2093db4c96c1SJed Brown   PetscValidHeaderSpecific(is,IS_CLASSID,3);
2094cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetIS_C",(PC,const char[],IS),(pc,splitname,is));
2095704ba839SBarry Smith   PetscFunctionReturn(0);
2096704ba839SBarry Smith }
2097704ba839SBarry Smith 
209894ef8ddeSSatish Balay /*@C
209957a9adfeSMatthew G Knepley     PCFieldSplitGetIS - Retrieves the elements for a field as an IS
210057a9adfeSMatthew G Knepley 
210157a9adfeSMatthew G Knepley     Logically Collective on PC
210257a9adfeSMatthew G Knepley 
210357a9adfeSMatthew G Knepley     Input Parameters:
210457a9adfeSMatthew G Knepley +   pc  - the preconditioner context
210557a9adfeSMatthew G Knepley -   splitname - name of this split
210657a9adfeSMatthew G Knepley 
210757a9adfeSMatthew G Knepley     Output Parameter:
21080298fd71SBarry Smith -   is - the index set that defines the vector elements in this field, or NULL if the field is not found
210957a9adfeSMatthew G Knepley 
211057a9adfeSMatthew G Knepley     Level: intermediate
211157a9adfeSMatthew G Knepley 
2112db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetIS()`
211357a9adfeSMatthew G Knepley 
211457a9adfeSMatthew G Knepley @*/
211557a9adfeSMatthew G Knepley PetscErrorCode PCFieldSplitGetIS(PC pc,const char splitname[],IS *is)
211657a9adfeSMatthew G Knepley {
211757a9adfeSMatthew G Knepley   PetscFunctionBegin;
211857a9adfeSMatthew G Knepley   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
211957a9adfeSMatthew G Knepley   PetscValidCharPointer(splitname,2);
212057a9adfeSMatthew G Knepley   PetscValidPointer(is,3);
212157a9adfeSMatthew G Knepley   {
212257a9adfeSMatthew G Knepley     PC_FieldSplit     *jac  = (PC_FieldSplit*) pc->data;
212357a9adfeSMatthew G Knepley     PC_FieldSplitLink ilink = jac->head;
212457a9adfeSMatthew G Knepley     PetscBool         found;
212557a9adfeSMatthew G Knepley 
21260298fd71SBarry Smith     *is = NULL;
212757a9adfeSMatthew G Knepley     while (ilink) {
21289566063dSJacob Faibussowitsch       PetscCall(PetscStrcmp(ilink->splitname, splitname, &found));
212957a9adfeSMatthew G Knepley       if (found) {
213057a9adfeSMatthew G Knepley         *is = ilink->is;
213157a9adfeSMatthew G Knepley         break;
213257a9adfeSMatthew G Knepley       }
213357a9adfeSMatthew G Knepley       ilink = ilink->next;
213457a9adfeSMatthew G Knepley     }
213557a9adfeSMatthew G Knepley   }
213657a9adfeSMatthew G Knepley   PetscFunctionReturn(0);
213757a9adfeSMatthew G Knepley }
213857a9adfeSMatthew G Knepley 
2139998f007dSPierre Jolivet /*@C
2140998f007dSPierre Jolivet     PCFieldSplitGetISByIndex - Retrieves the elements for a given index field as an IS
2141998f007dSPierre Jolivet 
2142998f007dSPierre Jolivet     Logically Collective on PC
2143998f007dSPierre Jolivet 
2144998f007dSPierre Jolivet     Input Parameters:
2145998f007dSPierre Jolivet +   pc  - the preconditioner context
2146998f007dSPierre Jolivet -   index - index of this split
2147998f007dSPierre Jolivet 
2148998f007dSPierre Jolivet     Output Parameter:
2149998f007dSPierre Jolivet -   is - the index set that defines the vector elements in this field
2150998f007dSPierre Jolivet 
2151998f007dSPierre Jolivet     Level: intermediate
2152998f007dSPierre Jolivet 
2153db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitGetIS()`, `PCFieldSplitSetIS()`
2154998f007dSPierre Jolivet 
2155998f007dSPierre Jolivet @*/
2156998f007dSPierre Jolivet PetscErrorCode PCFieldSplitGetISByIndex(PC pc,PetscInt index,IS *is)
2157998f007dSPierre Jolivet {
2158998f007dSPierre Jolivet   PetscFunctionBegin;
215963a3b9bcSJacob Faibussowitsch   PetscCheck(index >= 0,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Negative field %" PetscInt_FMT " requested",index);
2160998f007dSPierre Jolivet   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2161998f007dSPierre Jolivet   PetscValidPointer(is,3);
2162998f007dSPierre Jolivet   {
2163998f007dSPierre Jolivet     PC_FieldSplit     *jac  = (PC_FieldSplit*) pc->data;
2164998f007dSPierre Jolivet     PC_FieldSplitLink ilink = jac->head;
2165998f007dSPierre Jolivet     PetscInt          i     = 0;
216663a3b9bcSJacob Faibussowitsch     PetscCheck(index < jac->nsplits,PETSC_COMM_SELF,PETSC_ERR_ARG_OUTOFRANGE,"Field %" PetscInt_FMT " requested but only %" PetscInt_FMT " exist",index,jac->nsplits);
2167998f007dSPierre Jolivet 
2168998f007dSPierre Jolivet     while (i < index) {
2169998f007dSPierre Jolivet       ilink = ilink->next;
2170998f007dSPierre Jolivet       ++i;
2171998f007dSPierre Jolivet     }
21729566063dSJacob Faibussowitsch     PetscCall(PCFieldSplitGetIS(pc,ilink->splitname,is));
2173998f007dSPierre Jolivet   }
2174998f007dSPierre Jolivet   PetscFunctionReturn(0);
2175998f007dSPierre Jolivet }
2176998f007dSPierre Jolivet 
217751f519a2SBarry Smith /*@
217851f519a2SBarry Smith     PCFieldSplitSetBlockSize - Sets the block size for defining where fields start in the
217951f519a2SBarry Smith       fieldsplit preconditioner. If not set the matrix block size is used.
218051f519a2SBarry Smith 
2181ad4df100SBarry Smith     Logically Collective on PC
218251f519a2SBarry Smith 
218351f519a2SBarry Smith     Input Parameters:
218451f519a2SBarry Smith +   pc  - the preconditioner context
218551f519a2SBarry Smith -   bs - the block size
218651f519a2SBarry Smith 
218751f519a2SBarry Smith     Level: intermediate
218851f519a2SBarry Smith 
2189db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetFields()`
219051f519a2SBarry Smith 
219151f519a2SBarry Smith @*/
21927087cfbeSBarry Smith PetscErrorCode  PCFieldSplitSetBlockSize(PC pc,PetscInt bs)
219351f519a2SBarry Smith {
219451f519a2SBarry Smith   PetscFunctionBegin;
21950700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2196c5eb9154SBarry Smith   PetscValidLogicalCollectiveInt(pc,bs,2);
2197cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetBlockSize_C",(PC,PetscInt),(pc,bs));
219851f519a2SBarry Smith   PetscFunctionReturn(0);
219951f519a2SBarry Smith }
220051f519a2SBarry Smith 
22010971522cSBarry Smith /*@C
220269a612a9SBarry Smith    PCFieldSplitGetSubKSP - Gets the KSP contexts for all splits
22030971522cSBarry Smith 
220469a612a9SBarry Smith    Collective on KSP
22050971522cSBarry Smith 
22060971522cSBarry Smith    Input Parameter:
22070971522cSBarry Smith .  pc - the preconditioner context
22080971522cSBarry Smith 
22090971522cSBarry Smith    Output Parameters:
221013e0d083SBarry Smith +  n - the number of splits
22113111de3cSBarry Smith -  subksp - the array of KSP contexts
22120971522cSBarry Smith 
22130971522cSBarry Smith    Note:
22149a4f7e47SBarry Smith    After PCFieldSplitGetSubKSP() the array of KSPs is to be freed by the user with PetscFree()
2215d32f9abdSBarry Smith    (not the KSP just the array that contains them).
22160971522cSBarry Smith 
2217285fb4e2SStefano Zampini    You must call PCSetUp() before calling PCFieldSplitGetSubKSP().
2218285fb4e2SStefano Zampini 
2219285fb4e2SStefano Zampini    If the fieldsplit is of type PC_COMPOSITE_SCHUR, it returns the KSP object used inside the
2220285fb4e2SStefano Zampini    Schur complement and the KSP object used to iterate over the Schur complement.
2221a51937d4SCarola Kruse    To access all the KSP objects used in PC_COMPOSITE_SCHUR, use PCFieldSplitSchurGetSubKSP().
2222a51937d4SCarola Kruse 
2223a51937d4SCarola Kruse    If the fieldsplit is of type PC_COMPOSITE_GKB, it returns the KSP object used to solve the
2224a51937d4SCarola Kruse    inner linear system defined by the matrix H in each loop.
22250971522cSBarry Smith 
2226196cc216SBarry Smith    Fortran Usage: You must pass in a KSP array that is large enough to contain all the local KSPs.
22272bf68e3eSBarry Smith       You can call PCFieldSplitGetSubKSP(pc,n,PETSC_NULL_KSP,ierr) to determine how large the
2228196cc216SBarry Smith       KSP array must be.
2229196cc216SBarry Smith 
22300971522cSBarry Smith    Level: advanced
22310971522cSBarry Smith 
2232db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`
22330971522cSBarry Smith @*/
22347087cfbeSBarry Smith PetscErrorCode  PCFieldSplitGetSubKSP(PC pc,PetscInt *n,KSP *subksp[])
22350971522cSBarry Smith {
22360971522cSBarry Smith   PetscFunctionBegin;
22370700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
223813e0d083SBarry Smith   if (n) PetscValidIntPointer(n,2);
2239cac4c232SBarry Smith   PetscUseMethod(pc,"PCFieldSplitGetSubKSP_C",(PC,PetscInt*,KSP **),(pc,n,subksp));
22400971522cSBarry Smith   PetscFunctionReturn(0);
22410971522cSBarry Smith }
22420971522cSBarry Smith 
2243285fb4e2SStefano Zampini /*@C
2244285fb4e2SStefano Zampini    PCFieldSplitSchurGetSubKSP - Gets the KSP contexts used inside the Schur complement based PCFIELDSPLIT
2245285fb4e2SStefano Zampini 
2246285fb4e2SStefano Zampini    Collective on KSP
2247285fb4e2SStefano Zampini 
2248285fb4e2SStefano Zampini    Input Parameter:
2249285fb4e2SStefano Zampini .  pc - the preconditioner context
2250285fb4e2SStefano Zampini 
2251285fb4e2SStefano Zampini    Output Parameters:
2252285fb4e2SStefano Zampini +  n - the number of splits
2253285fb4e2SStefano Zampini -  subksp - the array of KSP contexts
2254285fb4e2SStefano Zampini 
2255285fb4e2SStefano Zampini    Note:
2256285fb4e2SStefano Zampini    After PCFieldSplitSchurGetSubKSP() the array of KSPs is to be freed by the user with PetscFree()
2257285fb4e2SStefano Zampini    (not the KSP just the array that contains them).
2258285fb4e2SStefano Zampini 
2259285fb4e2SStefano Zampini    You must call PCSetUp() before calling PCFieldSplitSchurGetSubKSP().
2260285fb4e2SStefano Zampini 
2261285fb4e2SStefano Zampini    If the fieldsplit type is of type PC_COMPOSITE_SCHUR, it returns (in order)
2262285fb4e2SStefano Zampini    - the KSP used for the (1,1) block
2263285fb4e2SStefano Zampini    - the KSP used for the Schur complement (not the one used for the interior Schur solver)
2264285fb4e2SStefano Zampini    - the KSP used for the (1,1) block in the upper triangular factor (if different from that of the (1,1) block).
2265285fb4e2SStefano Zampini 
2266285fb4e2SStefano Zampini    It returns a null array if the fieldsplit is not of type PC_COMPOSITE_SCHUR; in this case, you should use PCFieldSplitGetSubKSP().
2267285fb4e2SStefano Zampini 
2268285fb4e2SStefano Zampini    Fortran Usage: You must pass in a KSP array that is large enough to contain all the local KSPs.
2269285fb4e2SStefano Zampini       You can call PCFieldSplitSchurGetSubKSP(pc,n,PETSC_NULL_KSP,ierr) to determine how large the
2270285fb4e2SStefano Zampini       KSP array must be.
2271285fb4e2SStefano Zampini 
2272285fb4e2SStefano Zampini    Level: advanced
2273285fb4e2SStefano Zampini 
2274db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`
2275285fb4e2SStefano Zampini @*/
2276285fb4e2SStefano Zampini PetscErrorCode  PCFieldSplitSchurGetSubKSP(PC pc,PetscInt *n,KSP *subksp[])
2277285fb4e2SStefano Zampini {
2278285fb4e2SStefano Zampini   PetscFunctionBegin;
2279285fb4e2SStefano Zampini   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2280285fb4e2SStefano Zampini   if (n) PetscValidIntPointer(n,2);
2281cac4c232SBarry Smith   PetscUseMethod(pc,"PCFieldSplitSchurGetSubKSP_C",(PC,PetscInt*,KSP **),(pc,n,subksp));
2282285fb4e2SStefano Zampini   PetscFunctionReturn(0);
2283285fb4e2SStefano Zampini }
2284285fb4e2SStefano Zampini 
2285e69d4d44SBarry Smith /*@
2286a1e3cbf9SBarry Smith     PCFieldSplitSetSchurPre -  Indicates from what operator the preconditioner is constructucted for the Schur complement.
2287a1e3cbf9SBarry Smith       The default is the A11 matrix.
2288e69d4d44SBarry Smith 
2289e69d4d44SBarry Smith     Collective on PC
2290e69d4d44SBarry Smith 
2291e69d4d44SBarry Smith     Input Parameters:
2292e69d4d44SBarry Smith +   pc      - the preconditioner context
22936fdd48a9SBarry Smith .   ptype   - which matrix to use for preconditioning the Schur complement: PC_FIELDSPLIT_SCHUR_PRE_A11 (default), PC_FIELDSPLIT_SCHUR_PRE_SELF, PC_FIELDSPLIT_SCHUR_PRE_USER
22946fdd48a9SBarry Smith               PC_FIELDSPLIT_SCHUR_PRE_SELFP, and PC_FIELDSPLIT_SCHUR_PRE_FULL
22950298fd71SBarry Smith -   userpre - matrix to use for preconditioning, or NULL
2296084e4875SJed Brown 
2297e69d4d44SBarry Smith     Options Database:
2298a1e3cbf9SBarry Smith +    -pc_fieldsplit_schur_precondition <self,selfp,user,a11,full> - default is a11. See notes for meaning of various arguments
2299a1e3cbf9SBarry Smith -    -fieldsplit_1_pc_type <pctype> - the preconditioner algorithm that is used to construct the preconditioner from the operator
2300e69d4d44SBarry Smith 
2301fd1303e9SJungho Lee     Notes:
2302fd1303e9SJungho Lee $    If ptype is
2303a1e3cbf9SBarry Smith $        a11 - the preconditioner for the Schur complement is generated from the block diagonal part of the preconditioner
230453f2277eSBarry Smith $        matrix associated with the Schur complement (i.e. A11), not the Schur complement matrix
2305a1e3cbf9SBarry Smith $        self - the preconditioner for the Schur complement is generated from the symbolic representation of the Schur complement matrix:
2306a6a584a2SBarry Smith $             The only preconditioner that currently works with this symbolic respresentation matrix object is the PCLSC
2307fd1303e9SJungho Lee $             preconditioner
2308a1e3cbf9SBarry Smith $        user - the preconditioner for the Schur complement is generated from the user provided matrix (pre argument
23096fdd48a9SBarry Smith $             to this function).
2310a1e3cbf9SBarry Smith $        selfp - the preconditioning for the Schur complement is generated from an explicitly-assembled approximation Sp = A11 - A10 inv(diag(A00)) A01
23111d71051fSDmitry Karpeev $             This is only a good preconditioner when diag(A00) is a good preconditioner for A00. Optionally, A00 can be
23126fdd48a9SBarry Smith $             lumped before extracting the diagonal using the additional option -fieldsplit_1_mat_schur_complement_ainv_type lump
2313a1e3cbf9SBarry Smith $        full - the preconditioner for the Schur complement is generated from the exact Schur complement matrix representation computed internally by PCFIELDSPLIT (this is expensive)
23146fdd48a9SBarry Smith $             useful mostly as a test that the Schur complement approach can work for your problem
2315fd1303e9SJungho Lee 
2316e87fab1bSBarry Smith      When solving a saddle point problem, where the A11 block is identically zero, using a11 as the ptype only makes sense
2317fd1303e9SJungho Lee     with the additional option -fieldsplit_1_pc_type none. Usually for saddle point problems one would use a ptype of self and
2318a7476a74SDmitry Karpeev     -fieldsplit_1_pc_type lsc which uses the least squares commutator to compute a preconditioner for the Schur complement.
2319fd1303e9SJungho Lee 
2320e69d4d44SBarry Smith     Level: intermediate
2321e69d4d44SBarry Smith 
2322db781477SPatrick Sanan .seealso: `PCFieldSplitGetSchurPre()`, `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetFields()`, `PCFieldSplitSchurPreType`,
2323db781477SPatrick Sanan           `MatSchurComplementSetAinvType()`, `PCLSC`
2324e69d4d44SBarry Smith 
2325e69d4d44SBarry Smith @*/
232629f8a81cSJed Brown PetscErrorCode PCFieldSplitSetSchurPre(PC pc,PCFieldSplitSchurPreType ptype,Mat pre)
2327e69d4d44SBarry Smith {
2328e69d4d44SBarry Smith   PetscFunctionBegin;
23290700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2330cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetSchurPre_C",(PC,PCFieldSplitSchurPreType,Mat),(pc,ptype,pre));
2331e69d4d44SBarry Smith   PetscFunctionReturn(0);
2332e69d4d44SBarry Smith }
2333686bed4dSStefano Zampini 
233429f8a81cSJed Brown PetscErrorCode PCFieldSplitSchurPrecondition(PC pc,PCFieldSplitSchurPreType ptype,Mat pre) {return PCFieldSplitSetSchurPre(pc,ptype,pre);} /* Deprecated name */
2335e69d4d44SBarry Smith 
233637a82bf0SJed Brown /*@
233737a82bf0SJed Brown     PCFieldSplitGetSchurPre - For Schur complement fieldsplit, determine how the Schur complement will be
233829f8a81cSJed Brown     preconditioned.  See PCFieldSplitSetSchurPre() for details.
233937a82bf0SJed Brown 
234037a82bf0SJed Brown     Logically Collective on PC
234137a82bf0SJed Brown 
2342f899ff85SJose E. Roman     Input Parameter:
234337a82bf0SJed Brown .   pc      - the preconditioner context
234437a82bf0SJed Brown 
234537a82bf0SJed Brown     Output Parameters:
234637a82bf0SJed Brown +   ptype   - which matrix to use for preconditioning the Schur complement: PC_FIELDSPLIT_SCHUR_PRE_A11, PC_FIELDSPLIT_SCHUR_PRE_SELF, PC_FIELDSPLIT_PRE_USER
234737a82bf0SJed Brown -   userpre - matrix to use for preconditioning (with PC_FIELDSPLIT_PRE_USER), or NULL
234837a82bf0SJed Brown 
234937a82bf0SJed Brown     Level: intermediate
235037a82bf0SJed Brown 
2351db781477SPatrick Sanan .seealso: `PCFieldSplitSetSchurPre()`, `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetFields()`, `PCFieldSplitSchurPreType`, `PCLSC`
235237a82bf0SJed Brown 
235337a82bf0SJed Brown @*/
235437a82bf0SJed Brown PetscErrorCode PCFieldSplitGetSchurPre(PC pc,PCFieldSplitSchurPreType *ptype,Mat *pre)
235537a82bf0SJed Brown {
235637a82bf0SJed Brown   PetscFunctionBegin;
235737a82bf0SJed Brown   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2358cac4c232SBarry Smith   PetscUseMethod(pc,"PCFieldSplitGetSchurPre_C",(PC,PCFieldSplitSchurPreType*,Mat*),(pc,ptype,pre));
2359e69d4d44SBarry Smith   PetscFunctionReturn(0);
2360e69d4d44SBarry Smith }
2361e69d4d44SBarry Smith 
236245e7fc46SDmitry Karpeev /*@
2363470b340bSDmitry Karpeev     PCFieldSplitSchurGetS -  extract the MatSchurComplement object used by this PC in case it needs to be configured separately
236445e7fc46SDmitry Karpeev 
236545e7fc46SDmitry Karpeev     Not collective
236645e7fc46SDmitry Karpeev 
236745e7fc46SDmitry Karpeev     Input Parameter:
236845e7fc46SDmitry Karpeev .   pc      - the preconditioner context
236945e7fc46SDmitry Karpeev 
237045e7fc46SDmitry Karpeev     Output Parameter:
2371470b340bSDmitry Karpeev .   S       - the Schur complement matrix
237245e7fc46SDmitry Karpeev 
2373470b340bSDmitry Karpeev     Notes:
2374470b340bSDmitry Karpeev     This matrix should not be destroyed using MatDestroy(); rather, use PCFieldSplitSchurRestoreS().
237545e7fc46SDmitry Karpeev 
237645e7fc46SDmitry Karpeev     Level: advanced
237745e7fc46SDmitry Karpeev 
2378db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSchurPreType`, `PCFieldSplitSetSchurPre()`, `MatSchurComplement`, `PCFieldSplitSchurRestoreS()`
237945e7fc46SDmitry Karpeev 
238045e7fc46SDmitry Karpeev @*/
2381470b340bSDmitry Karpeev PetscErrorCode  PCFieldSplitSchurGetS(PC pc,Mat *S)
238245e7fc46SDmitry Karpeev {
238345e7fc46SDmitry Karpeev   const char*    t;
238445e7fc46SDmitry Karpeev   PetscBool      isfs;
238545e7fc46SDmitry Karpeev   PC_FieldSplit  *jac;
238645e7fc46SDmitry Karpeev 
238745e7fc46SDmitry Karpeev   PetscFunctionBegin;
238845e7fc46SDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
23899566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetType((PetscObject)pc,&t));
23909566063dSJacob Faibussowitsch   PetscCall(PetscStrcmp(t,PCFIELDSPLIT,&isfs));
239128b400f6SJacob Faibussowitsch   PetscCheck(isfs,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected PC of type PCFIELDSPLIT, got %s instead",t);
239245e7fc46SDmitry Karpeev   jac = (PC_FieldSplit*)pc->data;
239363a3b9bcSJacob Faibussowitsch   PetscCheck(jac->type == PC_COMPOSITE_SCHUR,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected PCFIELDSPLIT of type SCHUR, got %d instead",jac->type);
2394470b340bSDmitry Karpeev   if (S) *S = jac->schur;
239545e7fc46SDmitry Karpeev   PetscFunctionReturn(0);
239645e7fc46SDmitry Karpeev }
239745e7fc46SDmitry Karpeev 
2398470b340bSDmitry Karpeev /*@
2399470b340bSDmitry Karpeev     PCFieldSplitSchurRestoreS -  restores the MatSchurComplement object used by this PC
2400470b340bSDmitry Karpeev 
2401470b340bSDmitry Karpeev     Not collective
2402470b340bSDmitry Karpeev 
2403470b340bSDmitry Karpeev     Input Parameters:
2404470b340bSDmitry Karpeev +   pc      - the preconditioner context
2405a2b725a8SWilliam Gropp -   S       - the Schur complement matrix
2406470b340bSDmitry Karpeev 
2407470b340bSDmitry Karpeev     Level: advanced
2408470b340bSDmitry Karpeev 
2409db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSchurPreType`, `PCFieldSplitSetSchurPre()`, `MatSchurComplement`, `PCFieldSplitSchurGetS()`
2410470b340bSDmitry Karpeev 
2411470b340bSDmitry Karpeev @*/
2412bca69d2bSDmitry Karpeev PetscErrorCode  PCFieldSplitSchurRestoreS(PC pc,Mat *S)
2413470b340bSDmitry Karpeev {
2414470b340bSDmitry Karpeev   const char*    t;
2415470b340bSDmitry Karpeev   PetscBool      isfs;
2416470b340bSDmitry Karpeev   PC_FieldSplit  *jac;
2417470b340bSDmitry Karpeev 
2418470b340bSDmitry Karpeev   PetscFunctionBegin;
2419470b340bSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
24209566063dSJacob Faibussowitsch   PetscCall(PetscObjectGetType((PetscObject)pc,&t));
24219566063dSJacob Faibussowitsch   PetscCall(PetscStrcmp(t,PCFIELDSPLIT,&isfs));
242228b400f6SJacob Faibussowitsch   PetscCheck(isfs,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected PC of type PCFIELDSPLIT, got %s instead",t);
2423470b340bSDmitry Karpeev   jac = (PC_FieldSplit*)pc->data;
242463a3b9bcSJacob Faibussowitsch   PetscCheck(jac->type == PC_COMPOSITE_SCHUR,PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"Expected PCFIELDSPLIT of type SCHUR, got %d instead",jac->type);
24257827d75bSBarry Smith   PetscCheck(S && (*S == jac->schur),PETSC_COMM_SELF,PETSC_ERR_ARG_WRONG,"MatSchurComplement restored is not the same as gotten");
2426470b340bSDmitry Karpeev   PetscFunctionReturn(0);
2427470b340bSDmitry Karpeev }
2428470b340bSDmitry Karpeev 
242929f8a81cSJed Brown static PetscErrorCode  PCFieldSplitSetSchurPre_FieldSplit(PC pc,PCFieldSplitSchurPreType ptype,Mat pre)
2430e69d4d44SBarry Smith {
2431e69d4d44SBarry Smith   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
2432e69d4d44SBarry Smith 
2433e69d4d44SBarry Smith   PetscFunctionBegin;
2434084e4875SJed Brown   jac->schurpre = ptype;
2435a7476a74SDmitry Karpeev   if (ptype == PC_FIELDSPLIT_SCHUR_PRE_USER && pre) {
24369566063dSJacob Faibussowitsch     PetscCall(MatDestroy(&jac->schur_user));
2437084e4875SJed Brown     jac->schur_user = pre;
24389566063dSJacob Faibussowitsch     PetscCall(PetscObjectReference((PetscObject)jac->schur_user));
2439084e4875SJed Brown   }
2440e69d4d44SBarry Smith   PetscFunctionReturn(0);
2441e69d4d44SBarry Smith }
2442e69d4d44SBarry Smith 
244337a82bf0SJed Brown static PetscErrorCode  PCFieldSplitGetSchurPre_FieldSplit(PC pc,PCFieldSplitSchurPreType *ptype,Mat *pre)
244437a82bf0SJed Brown {
244537a82bf0SJed Brown   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
244637a82bf0SJed Brown 
244737a82bf0SJed Brown   PetscFunctionBegin;
244837a82bf0SJed Brown   *ptype = jac->schurpre;
244937a82bf0SJed Brown   *pre   = jac->schur_user;
245037a82bf0SJed Brown   PetscFunctionReturn(0);
245137a82bf0SJed Brown }
245237a82bf0SJed Brown 
2453ab1df9f5SJed Brown /*@
24540ffb0e17SBarry Smith     PCFieldSplitSetSchurFactType -  sets which blocks of the approximate block factorization to retain in the preconditioner
2455ab1df9f5SJed Brown 
2456ab1df9f5SJed Brown     Collective on PC
2457ab1df9f5SJed Brown 
2458ab1df9f5SJed Brown     Input Parameters:
2459ab1df9f5SJed Brown +   pc  - the preconditioner context
2460c9c6ffaaSJed Brown -   ftype - which blocks of factorization to retain, PC_FIELDSPLIT_SCHUR_FACT_FULL is default
2461ab1df9f5SJed Brown 
2462ab1df9f5SJed Brown     Options Database:
2463147403d9SBarry Smith .     -pc_fieldsplit_schur_fact_type <diag,lower,upper,full> - default is full
2464ab1df9f5SJed Brown 
2465ab1df9f5SJed Brown     Level: intermediate
2466ab1df9f5SJed Brown 
2467ab1df9f5SJed Brown     Notes:
2468ab1df9f5SJed Brown     The FULL factorization is
2469ab1df9f5SJed Brown 
2470147403d9SBarry Smith .vb
2471147403d9SBarry Smith    (A   B)  = (1       0) (A   0) (1  Ainv*B)  = L D U
2472147403d9SBarry Smith    (C   E)    (C*Ainv  1) (0   S) (0     1)
2473147403d9SBarry Smith .vb
24740ffb0e17SBarry Smith     where S = E - C*Ainv*B. In practice, the full factorization is applied via block triangular solves with the grouping L*(D*U). UPPER uses D*U, LOWER uses L*D,
2475c096484dSStefano Zampini     and DIAG is the diagonal part with the sign of S flipped (because this makes the preconditioner positive definite for many formulations, thus allowing the use of KSPMINRES). Sign flipping of S can be turned off with PCFieldSplitSetSchurScale().
2476ab1df9f5SJed Brown 
2477147403d9SBarry Smith     If A and S are solved exactly
2478147403d9SBarry Smith .vb
2479147403d9SBarry Smith       *) FULL factorization is a direct solver.
2480147403d9SBarry Smith       *) The preconditioned operator with LOWER or UPPER has all eigenvalues equal to 1 and minimal polynomial of degree 2, so KSPGMRES converges in 2 iterations.
2481147403d9SBarry Smith       *) With DIAG, the preconditioned operator has three distinct nonzero eigenvalues and minimal polynomial of degree at most 4, so KSPGMRES converges in at most 4 iterations.
2482147403d9SBarry Smith .ve
2483ab1df9f5SJed Brown 
24840ffb0e17SBarry Smith     If the iteration count is very low, consider using KSPFGMRES or KSPGCR which can use one less preconditioner
24850ffb0e17SBarry Smith     application in this case. Note that the preconditioned operator may be highly non-normal, so such fast convergence may not be observed in practice.
24860ffb0e17SBarry Smith 
24870ffb0e17SBarry Smith     For symmetric problems in which A is positive definite and S is negative definite, DIAG can be used with KSPMINRES.
24880ffb0e17SBarry Smith 
24890ffb0e17SBarry Smith     Note that a flexible method like KSPFGMRES or KSPGCR must be used if the fieldsplit preconditioner is nonlinear (e.g. a few iterations of a Krylov method is used to solve with A or S).
2490ab1df9f5SJed Brown 
2491ab1df9f5SJed Brown     References:
2492606c0280SSatish Balay +   * - Murphy, Golub, and Wathen, A note on preconditioning indefinite linear systems, SIAM J. Sci. Comput., 21 (2000).
2493606c0280SSatish Balay -   * - Ipsen, A note on preconditioning nonsymmetric matrices, SIAM J. Sci. Comput., 23 (2001).
2494ab1df9f5SJed Brown 
2495db781477SPatrick Sanan .seealso: `PCFieldSplitGetSubKSP()`, `PCFIELDSPLIT`, `PCFieldSplitSetFields()`, `PCFieldSplitSchurPreType`, `PCFieldSplitSetSchurScale()`
2496ab1df9f5SJed Brown @*/
2497c9c6ffaaSJed Brown PetscErrorCode  PCFieldSplitSetSchurFactType(PC pc,PCFieldSplitSchurFactType ftype)
2498ab1df9f5SJed Brown {
2499ab1df9f5SJed Brown   PetscFunctionBegin;
2500ab1df9f5SJed Brown   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2501cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetSchurFactType_C",(PC,PCFieldSplitSchurFactType),(pc,ftype));
2502ab1df9f5SJed Brown   PetscFunctionReturn(0);
2503ab1df9f5SJed Brown }
2504ab1df9f5SJed Brown 
25051e6b0712SBarry Smith static PetscErrorCode PCFieldSplitSetSchurFactType_FieldSplit(PC pc,PCFieldSplitSchurFactType ftype)
2506ab1df9f5SJed Brown {
2507ab1df9f5SJed Brown   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2508ab1df9f5SJed Brown 
2509ab1df9f5SJed Brown   PetscFunctionBegin;
2510ab1df9f5SJed Brown   jac->schurfactorization = ftype;
2511ab1df9f5SJed Brown   PetscFunctionReturn(0);
2512ab1df9f5SJed Brown }
2513ab1df9f5SJed Brown 
2514c096484dSStefano Zampini /*@
2515c096484dSStefano Zampini     PCFieldSplitSetSchurScale -  Controls the sign flip of S for PC_FIELDSPLIT_SCHUR_FACT_DIAG.
2516c096484dSStefano Zampini 
2517c096484dSStefano Zampini     Collective on PC
2518c096484dSStefano Zampini 
2519c096484dSStefano Zampini     Input Parameters:
2520c096484dSStefano Zampini +   pc    - the preconditioner context
2521c096484dSStefano Zampini -   scale - scaling factor for the Schur complement
2522c096484dSStefano Zampini 
2523c096484dSStefano Zampini     Options Database:
2524c096484dSStefano Zampini .     -pc_fieldsplit_schur_scale - default is -1.0
2525c096484dSStefano Zampini 
2526c096484dSStefano Zampini     Level: intermediate
2527c096484dSStefano Zampini 
2528db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetFields()`, `PCFieldSplitSchurFactType`, `PCFieldSplitSetSchurScale()`
2529c096484dSStefano Zampini @*/
2530c096484dSStefano Zampini PetscErrorCode PCFieldSplitSetSchurScale(PC pc,PetscScalar scale)
2531c096484dSStefano Zampini {
2532c096484dSStefano Zampini   PetscFunctionBegin;
2533c096484dSStefano Zampini   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2534c096484dSStefano Zampini   PetscValidLogicalCollectiveScalar(pc,scale,2);
2535cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetSchurScale_C",(PC,PetscScalar),(pc,scale));
2536c096484dSStefano Zampini   PetscFunctionReturn(0);
2537c096484dSStefano Zampini }
2538c096484dSStefano Zampini 
2539c096484dSStefano Zampini static PetscErrorCode PCFieldSplitSetSchurScale_FieldSplit(PC pc,PetscScalar scale)
2540c096484dSStefano Zampini {
2541c096484dSStefano Zampini   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2542c096484dSStefano Zampini 
2543c096484dSStefano Zampini   PetscFunctionBegin;
2544c096484dSStefano Zampini   jac->schurscale = scale;
2545c096484dSStefano Zampini   PetscFunctionReturn(0);
2546c096484dSStefano Zampini }
2547c096484dSStefano Zampini 
254830ad9308SMatthew Knepley /*@C
25498c03b21aSDmitry Karpeev    PCFieldSplitGetSchurBlocks - Gets all matrix blocks for the Schur complement
255030ad9308SMatthew Knepley 
255130ad9308SMatthew Knepley    Collective on KSP
255230ad9308SMatthew Knepley 
255330ad9308SMatthew Knepley    Input Parameter:
255430ad9308SMatthew Knepley .  pc - the preconditioner context
255530ad9308SMatthew Knepley 
255630ad9308SMatthew Knepley    Output Parameters:
2557a04f6461SBarry Smith +  A00 - the (0,0) block
2558a04f6461SBarry Smith .  A01 - the (0,1) block
2559a04f6461SBarry Smith .  A10 - the (1,0) block
2560a04f6461SBarry Smith -  A11 - the (1,1) block
256130ad9308SMatthew Knepley 
256230ad9308SMatthew Knepley    Level: advanced
256330ad9308SMatthew Knepley 
2564db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`
256530ad9308SMatthew Knepley @*/
2566a04f6461SBarry Smith PetscErrorCode  PCFieldSplitGetSchurBlocks(PC pc,Mat *A00,Mat *A01,Mat *A10, Mat *A11)
256730ad9308SMatthew Knepley {
256830ad9308SMatthew Knepley   PC_FieldSplit *jac = (PC_FieldSplit*) pc->data;
256930ad9308SMatthew Knepley 
257030ad9308SMatthew Knepley   PetscFunctionBegin;
25710700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
257208401ef6SPierre Jolivet   PetscCheck(jac->type == PC_COMPOSITE_SCHUR,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONG, "FieldSplit is not using a Schur complement approach.");
2573a04f6461SBarry Smith   if (A00) *A00 = jac->pmat[0];
2574a04f6461SBarry Smith   if (A01) *A01 = jac->B;
2575a04f6461SBarry Smith   if (A10) *A10 = jac->C;
2576a04f6461SBarry Smith   if (A11) *A11 = jac->pmat[1];
257730ad9308SMatthew Knepley   PetscFunctionReturn(0);
257830ad9308SMatthew Knepley }
257930ad9308SMatthew Knepley 
2580b09e027eSCarola Kruse /*@
2581e071a0a4SCarola Kruse     PCFieldSplitSetGKBTol -  Sets the solver tolerance for the generalized Golub-Kahan bidiagonalization preconditioner.
2582b09e027eSCarola Kruse 
2583b09e027eSCarola Kruse     Collective on PC
2584b09e027eSCarola Kruse 
2585e071a0a4SCarola Kruse     Notes:
2586e071a0a4SCarola Kruse     The generalized GKB algorithm uses a lower bound estimate of the error in energy norm as stopping criterion.
2587e071a0a4SCarola Kruse     It stops once the lower bound estimate undershoots the required solver tolerance. Although the actual error might be bigger than
2588e071a0a4SCarola Kruse     this estimate, the stopping criterion is satisfactory in practical cases [A13].
2589e071a0a4SCarola Kruse 
2590e071a0a4SCarola Kruse [Ar13] Generalized Golub-Kahan bidiagonalization and stopping criteria, SIAM J. Matrix Anal. Appl., Vol. 34, No. 2, pp. 571-592, 2013.
2591e071a0a4SCarola Kruse 
2592b09e027eSCarola Kruse     Input Parameters:
2593b09e027eSCarola Kruse +   pc        - the preconditioner context
2594b09e027eSCarola Kruse -   tolerance - the solver tolerance
2595b09e027eSCarola Kruse 
2596b09e027eSCarola Kruse     Options Database:
2597b09e027eSCarola Kruse .     -pc_fieldsplit_gkb_tol - default is 1e-5
2598b09e027eSCarola Kruse 
2599b09e027eSCarola Kruse     Level: intermediate
2600b09e027eSCarola Kruse 
2601db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetGKBDelay()`, `PCFieldSplitSetGKBNu()`, `PCFieldSplitSetGKBMaxit()`
2602b09e027eSCarola Kruse @*/
2603b09e027eSCarola Kruse PetscErrorCode PCFieldSplitSetGKBTol(PC pc,PetscReal tolerance)
2604b09e027eSCarola Kruse {
2605b09e027eSCarola Kruse   PetscFunctionBegin;
2606b09e027eSCarola Kruse   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2607de482cd7SCarola Kruse   PetscValidLogicalCollectiveReal(pc,tolerance,2);
2608cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetGKBTol_C",(PC,PetscReal),(pc,tolerance));
2609b09e027eSCarola Kruse   PetscFunctionReturn(0);
2610b09e027eSCarola Kruse }
2611b09e027eSCarola Kruse 
2612b09e027eSCarola Kruse static PetscErrorCode PCFieldSplitSetGKBTol_FieldSplit(PC pc,PetscReal tolerance)
2613b09e027eSCarola Kruse {
2614b09e027eSCarola Kruse   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2615b09e027eSCarola Kruse 
2616b09e027eSCarola Kruse   PetscFunctionBegin;
2617b09e027eSCarola Kruse   jac->gkbtol = tolerance;
2618b09e027eSCarola Kruse   PetscFunctionReturn(0);
2619b09e027eSCarola Kruse }
2620b09e027eSCarola Kruse 
2621b09e027eSCarola Kruse /*@
2622e071a0a4SCarola Kruse     PCFieldSplitSetGKBMaxit -  Sets the maximum number of iterations for the generalized Golub-Kahan bidiagonalization
2623e071a0a4SCarola Kruse     preconditioner.
2624b09e027eSCarola Kruse 
2625b09e027eSCarola Kruse     Collective on PC
2626b09e027eSCarola Kruse 
2627b09e027eSCarola Kruse     Input Parameters:
2628b09e027eSCarola Kruse +   pc     - the preconditioner context
2629b09e027eSCarola Kruse -   maxit  - the maximum number of iterations
2630b09e027eSCarola Kruse 
2631b09e027eSCarola Kruse     Options Database:
2632b09e027eSCarola Kruse .     -pc_fieldsplit_gkb_maxit - default is 100
2633b09e027eSCarola Kruse 
2634b09e027eSCarola Kruse     Level: intermediate
2635b09e027eSCarola Kruse 
2636db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetGKBDelay()`, `PCFieldSplitSetGKBTol()`, `PCFieldSplitSetGKBNu()`
2637b09e027eSCarola Kruse @*/
2638b09e027eSCarola Kruse PetscErrorCode PCFieldSplitSetGKBMaxit(PC pc,PetscInt maxit)
2639b09e027eSCarola Kruse {
2640b09e027eSCarola Kruse   PetscFunctionBegin;
2641b09e027eSCarola Kruse   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2642de482cd7SCarola Kruse   PetscValidLogicalCollectiveInt(pc,maxit,2);
2643cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetGKBMaxit_C",(PC,PetscInt),(pc,maxit));
2644b09e027eSCarola Kruse   PetscFunctionReturn(0);
2645b09e027eSCarola Kruse }
2646b09e027eSCarola Kruse 
2647b09e027eSCarola Kruse static PetscErrorCode PCFieldSplitSetGKBMaxit_FieldSplit(PC pc,PetscInt maxit)
2648b09e027eSCarola Kruse {
2649b09e027eSCarola Kruse   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2650b09e027eSCarola Kruse 
2651b09e027eSCarola Kruse   PetscFunctionBegin;
2652b09e027eSCarola Kruse   jac->gkbmaxit = maxit;
2653b09e027eSCarola Kruse   PetscFunctionReturn(0);
2654b09e027eSCarola Kruse }
2655b09e027eSCarola Kruse 
2656b09e027eSCarola Kruse /*@
2657e071a0a4SCarola Kruse     PCFieldSplitSetGKBDelay -  Sets the delay in the lower bound error estimate in the generalized Golub-Kahan bidiagonalization
2658e071a0a4SCarola Kruse     preconditioner.
2659b09e027eSCarola Kruse 
2660b09e027eSCarola Kruse     Collective on PC
2661b09e027eSCarola Kruse 
2662b09e027eSCarola Kruse     Notes:
2663e071a0a4SCarola Kruse     The algorithm uses a lower bound estimate of the error in energy norm as stopping criterion. The lower bound of the error ||u-u^k||_H
2664e071a0a4SCarola Kruse     is expressed as a truncated sum. The error at iteration k can only be measured at iteration (k + delay), and thus the algorithm needs
2665e071a0a4SCarola Kruse     at least (delay + 1) iterations to stop. For more details on the generalized Golub-Kahan bidiagonalization method and its lower bound stopping criterion, please refer to
2666b09e027eSCarola Kruse 
2667b09e027eSCarola Kruse [Ar13] Generalized Golub-Kahan bidiagonalization and stopping criteria, SIAM J. Matrix Anal. Appl., Vol. 34, No. 2, pp. 571-592, 2013.
2668b09e027eSCarola Kruse 
2669b09e027eSCarola Kruse     Input Parameters:
2670b09e027eSCarola Kruse +   pc     - the preconditioner context
2671b09e027eSCarola Kruse -   delay  - the delay window in the lower bound estimate
2672b09e027eSCarola Kruse 
2673b09e027eSCarola Kruse     Options Database:
2674b09e027eSCarola Kruse .     -pc_fieldsplit_gkb_delay - default is 5
2675b09e027eSCarola Kruse 
2676b09e027eSCarola Kruse     Level: intermediate
2677b09e027eSCarola Kruse 
2678db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetGKBNu()`, `PCFieldSplitSetGKBTol()`, `PCFieldSplitSetGKBMaxit()`
2679b09e027eSCarola Kruse @*/
2680b09e027eSCarola Kruse PetscErrorCode PCFieldSplitSetGKBDelay(PC pc,PetscInt delay)
2681b09e027eSCarola Kruse {
2682b09e027eSCarola Kruse   PetscFunctionBegin;
2683b09e027eSCarola Kruse   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2684b09e027eSCarola Kruse   PetscValidLogicalCollectiveInt(pc,delay,2);
2685cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetGKBDelay_C",(PC,PetscInt),(pc,delay));
2686b09e027eSCarola Kruse   PetscFunctionReturn(0);
2687b09e027eSCarola Kruse }
2688b09e027eSCarola Kruse 
2689b09e027eSCarola Kruse static PetscErrorCode PCFieldSplitSetGKBDelay_FieldSplit(PC pc,PetscInt delay)
2690b09e027eSCarola Kruse {
2691b09e027eSCarola Kruse   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2692b09e027eSCarola Kruse 
2693b09e027eSCarola Kruse   PetscFunctionBegin;
2694b09e027eSCarola Kruse   jac->gkbdelay = delay;
2695b09e027eSCarola Kruse   PetscFunctionReturn(0);
2696b09e027eSCarola Kruse }
2697b09e027eSCarola Kruse 
2698b09e027eSCarola Kruse /*@
2699b09e027eSCarola Kruse     PCFieldSplitSetGKBNu -  Sets the scalar value nu >= 0 in the transformation H = A00 + nu*A01*A01' of the (1,1) block in the Golub-Kahan bidiagonalization preconditioner.
2700b09e027eSCarola Kruse 
2701b09e027eSCarola Kruse     Collective on PC
2702b09e027eSCarola Kruse 
2703b09e027eSCarola Kruse     Notes:
2704e071a0a4SCarola Kruse     This shift is in general done to obtain better convergence properties for the outer loop of the algorithm. This is often achieved by chosing nu sufficiently big. However,
2705b09e027eSCarola Kruse     if nu is chosen too big, the matrix H might be badly conditioned and the solution of the linear system Hx = b in the inner loop gets difficult. It is therefore
2706b09e027eSCarola Kruse     necessary to find a good balance in between the convergence of the inner and outer loop.
2707b09e027eSCarola Kruse 
2708b09e027eSCarola Kruse     For nu = 0, no shift is done. In this case A00 has to be positive definite. The matrix N in [Ar13] is then chosen as identity.
2709b09e027eSCarola Kruse 
2710b09e027eSCarola Kruse [Ar13] Generalized Golub-Kahan bidiagonalization and stopping criteria, SIAM J. Matrix Anal. Appl., Vol. 34, No. 2, pp. 571-592, 2013.
2711b09e027eSCarola Kruse 
2712b09e027eSCarola Kruse     Input Parameters:
2713b09e027eSCarola Kruse +   pc     - the preconditioner context
2714b09e027eSCarola Kruse -   nu     - the shift parameter
2715b09e027eSCarola Kruse 
2716b09e027eSCarola Kruse     Options Database:
2717b09e027eSCarola Kruse .     -pc_fieldsplit_gkb_nu - default is 1
2718b09e027eSCarola Kruse 
2719b09e027eSCarola Kruse     Level: intermediate
2720b09e027eSCarola Kruse 
2721db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetGKBDelay()`, `PCFieldSplitSetGKBTol()`, `PCFieldSplitSetGKBMaxit()`
2722b09e027eSCarola Kruse @*/
2723b09e027eSCarola Kruse PetscErrorCode PCFieldSplitSetGKBNu(PC pc,PetscReal nu)
2724b09e027eSCarola Kruse {
2725b09e027eSCarola Kruse   PetscFunctionBegin;
2726b09e027eSCarola Kruse   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2727de482cd7SCarola Kruse   PetscValidLogicalCollectiveReal(pc,nu,2);
2728cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetGKBNu_C",(PC,PetscReal),(pc,nu));
2729b09e027eSCarola Kruse   PetscFunctionReturn(0);
2730b09e027eSCarola Kruse }
2731b09e027eSCarola Kruse 
2732b09e027eSCarola Kruse static PetscErrorCode PCFieldSplitSetGKBNu_FieldSplit(PC pc,PetscReal nu)
2733b09e027eSCarola Kruse {
2734b09e027eSCarola Kruse   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
2735b09e027eSCarola Kruse 
2736b09e027eSCarola Kruse   PetscFunctionBegin;
2737b09e027eSCarola Kruse   jac->gkbnu = nu;
2738b09e027eSCarola Kruse   PetscFunctionReturn(0);
2739b09e027eSCarola Kruse }
2740b09e027eSCarola Kruse 
27411e6b0712SBarry Smith static PetscErrorCode  PCFieldSplitSetType_FieldSplit(PC pc,PCCompositeType type)
274279416396SBarry Smith {
274379416396SBarry Smith   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
274479416396SBarry Smith 
274579416396SBarry Smith   PetscFunctionBegin;
274679416396SBarry Smith   jac->type = type;
27472e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",NULL));
27482e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurPre_C",NULL));
27492e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSchurPre_C",NULL));
27502e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurFactType_C",NULL));
27512e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurScale_C",NULL));
27522e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBTol_C",NULL));
27532e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBMaxit_C",NULL));
27542e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBNu_C",NULL));
27552e956fe4SStefano Zampini   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBDelay_C",NULL));
2756e071a0a4SCarola Kruse 
27573b224e63SBarry Smith   if (type == PC_COMPOSITE_SCHUR) {
27583b224e63SBarry Smith     pc->ops->apply = PCApply_FieldSplit_Schur;
27593b224e63SBarry Smith     pc->ops->view  = PCView_FieldSplit_Schur;
27602fa5cd67SKarl Rupp 
27619566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",PCFieldSplitGetSubKSP_FieldSplit_Schur));
27629566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurPre_C",PCFieldSplitSetSchurPre_FieldSplit));
27639566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSchurPre_C",PCFieldSplitGetSchurPre_FieldSplit));
27649566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurFactType_C",PCFieldSplitSetSchurFactType_FieldSplit));
27659566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetSchurScale_C",PCFieldSplitSetSchurScale_FieldSplit));
2766a51937d4SCarola Kruse   } else if (type == PC_COMPOSITE_GKB) {
2767a51937d4SCarola Kruse     pc->ops->apply = PCApply_FieldSplit_GKB;
2768a51937d4SCarola Kruse     pc->ops->view  = PCView_FieldSplit_GKB;
2769e69d4d44SBarry Smith 
27709566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",PCFieldSplitGetSubKSP_FieldSplit));
27719566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBTol_C",PCFieldSplitSetGKBTol_FieldSplit));
27729566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBMaxit_C",PCFieldSplitSetGKBMaxit_FieldSplit));
27739566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBNu_C",PCFieldSplitSetGKBNu_FieldSplit));
27749566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetGKBDelay_C",PCFieldSplitSetGKBDelay_FieldSplit));
27753b224e63SBarry Smith   } else {
27763b224e63SBarry Smith     pc->ops->apply = PCApply_FieldSplit;
27773b224e63SBarry Smith     pc->ops->view  = PCView_FieldSplit;
27782fa5cd67SKarl Rupp 
27799566063dSJacob Faibussowitsch     PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",PCFieldSplitGetSubKSP_FieldSplit));
27803b224e63SBarry Smith   }
278179416396SBarry Smith   PetscFunctionReturn(0);
278279416396SBarry Smith }
278379416396SBarry Smith 
27841e6b0712SBarry Smith static PetscErrorCode  PCFieldSplitSetBlockSize_FieldSplit(PC pc,PetscInt bs)
278551f519a2SBarry Smith {
278651f519a2SBarry Smith   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
278751f519a2SBarry Smith 
278851f519a2SBarry Smith   PetscFunctionBegin;
278963a3b9bcSJacob Faibussowitsch   PetscCheck(bs >= 1,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_OUTOFRANGE,"Blocksize must be positive, you gave %" PetscInt_FMT,bs);
27902472a847SBarry Smith   PetscCheck(jac->bs <= 0 || jac->bs == bs,PetscObjectComm((PetscObject)pc),PETSC_ERR_ARG_WRONGSTATE,"Cannot change fieldsplit blocksize from %" PetscInt_FMT " to %" PetscInt_FMT " after it has been set",jac->bs,bs);
279151f519a2SBarry Smith   jac->bs = bs;
279251f519a2SBarry Smith   PetscFunctionReturn(0);
279351f519a2SBarry Smith }
279451f519a2SBarry Smith 
27955ddf11f8SNicolas Barnafi static PetscErrorCode PCSetCoordinates_FieldSplit(PC pc, PetscInt dim, PetscInt nloc, PetscReal coords[])
27965ddf11f8SNicolas Barnafi {
27975ddf11f8SNicolas Barnafi   PC_FieldSplit *   jac = (PC_FieldSplit*)pc->data;
27985ddf11f8SNicolas Barnafi   PC_FieldSplitLink ilink_current = jac->head;
27995ddf11f8SNicolas Barnafi   IS                is_owned;
28005ddf11f8SNicolas Barnafi 
28015ddf11f8SNicolas Barnafi   PetscFunctionBegin;
28025ddf11f8SNicolas Barnafi   jac->coordinates_set = PETSC_TRUE; // Internal flag
28039566063dSJacob Faibussowitsch   PetscCall(MatGetOwnershipIS(pc->mat,&is_owned,PETSC_NULL));
28045ddf11f8SNicolas Barnafi 
28055ddf11f8SNicolas Barnafi   while (ilink_current) {
28065ddf11f8SNicolas Barnafi     // For each IS, embed it to get local coords indces
28075ddf11f8SNicolas Barnafi     IS        is_coords;
28085ddf11f8SNicolas Barnafi     PetscInt  ndofs_block;
28095ddf11f8SNicolas Barnafi     const PetscInt *block_dofs_enumeration; // Numbering of the dofs relevant to the current block
28105ddf11f8SNicolas Barnafi 
28115ddf11f8SNicolas Barnafi     // Setting drop to true for safety. It should make no difference.
28129566063dSJacob Faibussowitsch     PetscCall(ISEmbed(ilink_current->is, is_owned, PETSC_TRUE, &is_coords));
28139566063dSJacob Faibussowitsch     PetscCall(ISGetLocalSize(is_coords, &ndofs_block));
28149566063dSJacob Faibussowitsch     PetscCall(ISGetIndices(is_coords, &block_dofs_enumeration));
28155ddf11f8SNicolas Barnafi 
28165ddf11f8SNicolas Barnafi     // Allocate coordinates vector and set it directly
28179566063dSJacob Faibussowitsch     PetscCall(PetscMalloc1(ndofs_block * dim, &(ilink_current->coords)));
28185ddf11f8SNicolas Barnafi     for (PetscInt dof=0;dof<ndofs_block;++dof) {
28195ddf11f8SNicolas Barnafi       for (PetscInt d=0;d<dim;++d) {
28205ddf11f8SNicolas Barnafi         (ilink_current->coords)[dim*dof + d] = coords[dim * block_dofs_enumeration[dof] + d];
28215ddf11f8SNicolas Barnafi       }
28225ddf11f8SNicolas Barnafi     }
28235ddf11f8SNicolas Barnafi     ilink_current->dim = dim;
28245ddf11f8SNicolas Barnafi     ilink_current->ndofs = ndofs_block;
28259566063dSJacob Faibussowitsch     PetscCall(ISRestoreIndices(is_coords, &block_dofs_enumeration));
28269566063dSJacob Faibussowitsch     PetscCall(ISDestroy(&is_coords));
28275ddf11f8SNicolas Barnafi     ilink_current = ilink_current->next;
28285ddf11f8SNicolas Barnafi   }
28299566063dSJacob Faibussowitsch   PetscCall(ISDestroy(&is_owned));
28305ddf11f8SNicolas Barnafi   PetscFunctionReturn(0);
28315ddf11f8SNicolas Barnafi }
28325ddf11f8SNicolas Barnafi 
2833bc08b0f1SBarry Smith /*@
283479416396SBarry Smith    PCFieldSplitSetType - Sets the type of fieldsplit preconditioner.
283579416396SBarry Smith 
283679416396SBarry Smith    Collective on PC
283779416396SBarry Smith 
2838d8d19677SJose E. Roman    Input Parameters:
2839a2b725a8SWilliam Gropp +  pc - the preconditioner context
2840a2b725a8SWilliam Gropp -  type - PC_COMPOSITE_ADDITIVE, PC_COMPOSITE_MULTIPLICATIVE (default), PC_COMPOSITE_SYMMETRIC_MULTIPLICATIVE, PC_COMPOSITE_SPECIAL, PC_COMPOSITE_SCHUR
284179416396SBarry Smith 
284279416396SBarry Smith    Options Database Key:
2843a4efd8eaSMatthew Knepley .  -pc_fieldsplit_type <type: one of multiplicative, additive, symmetric_multiplicative, special, schur> - Sets fieldsplit preconditioner type
284479416396SBarry Smith 
2845b02e2d75SMatthew G Knepley    Level: Intermediate
284679416396SBarry Smith 
2847db781477SPatrick Sanan .seealso: `PCCompositeSetType()`
284879416396SBarry Smith 
284979416396SBarry Smith @*/
28507087cfbeSBarry Smith PetscErrorCode  PCFieldSplitSetType(PC pc,PCCompositeType type)
285179416396SBarry Smith {
285279416396SBarry Smith   PetscFunctionBegin;
28530700a824SBarry Smith   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2854cac4c232SBarry Smith   PetscTryMethod(pc,"PCFieldSplitSetType_C",(PC,PCCompositeType),(pc,type));
285579416396SBarry Smith   PetscFunctionReturn(0);
285679416396SBarry Smith }
285779416396SBarry Smith 
2858b02e2d75SMatthew G Knepley /*@
2859b02e2d75SMatthew G Knepley   PCFieldSplitGetType - Gets the type of fieldsplit preconditioner.
2860b02e2d75SMatthew G Knepley 
2861b02e2d75SMatthew G Knepley   Not collective
2862b02e2d75SMatthew G Knepley 
2863b02e2d75SMatthew G Knepley   Input Parameter:
2864b02e2d75SMatthew G Knepley . pc - the preconditioner context
2865b02e2d75SMatthew G Knepley 
2866b02e2d75SMatthew G Knepley   Output Parameter:
2867b02e2d75SMatthew G Knepley . type - PC_COMPOSITE_ADDITIVE, PC_COMPOSITE_MULTIPLICATIVE (default), PC_COMPOSITE_SYMMETRIC_MULTIPLICATIVE, PC_COMPOSITE_SPECIAL, PC_COMPOSITE_SCHUR
2868b02e2d75SMatthew G Knepley 
2869b02e2d75SMatthew G Knepley   Level: Intermediate
2870b02e2d75SMatthew G Knepley 
2871db781477SPatrick Sanan .seealso: `PCCompositeSetType()`
2872b02e2d75SMatthew G Knepley @*/
2873b02e2d75SMatthew G Knepley PetscErrorCode PCFieldSplitGetType(PC pc, PCCompositeType *type)
2874b02e2d75SMatthew G Knepley {
2875b02e2d75SMatthew G Knepley   PC_FieldSplit *jac = (PC_FieldSplit*) pc->data;
2876b02e2d75SMatthew G Knepley 
2877b02e2d75SMatthew G Knepley   PetscFunctionBegin;
2878b02e2d75SMatthew G Knepley   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2879b02e2d75SMatthew G Knepley   PetscValidIntPointer(type,2);
2880b02e2d75SMatthew G Knepley   *type = jac->type;
2881b02e2d75SMatthew G Knepley   PetscFunctionReturn(0);
2882b02e2d75SMatthew G Knepley }
2883b02e2d75SMatthew G Knepley 
28844ab8060aSDmitry Karpeev /*@
28854ab8060aSDmitry Karpeev    PCFieldSplitSetDMSplits - Flags whether DMCreateFieldDecomposition() should be used to define the splits, whenever possible.
28864ab8060aSDmitry Karpeev 
28874ab8060aSDmitry Karpeev    Logically Collective
28884ab8060aSDmitry Karpeev 
28894ab8060aSDmitry Karpeev    Input Parameters:
28904ab8060aSDmitry Karpeev +  pc   - the preconditioner context
28914ab8060aSDmitry Karpeev -  flg  - boolean indicating whether to use field splits defined by the DM
28924ab8060aSDmitry Karpeev 
28934ab8060aSDmitry Karpeev    Options Database Key:
289467b8a455SSatish Balay .  -pc_fieldsplit_dm_splits <bool> - use the field splits defined by the DM
28954ab8060aSDmitry Karpeev 
28964ab8060aSDmitry Karpeev    Level: Intermediate
28974ab8060aSDmitry Karpeev 
2898db781477SPatrick Sanan .seealso: `PCFieldSplitGetDMSplits()`
28994ab8060aSDmitry Karpeev 
29004ab8060aSDmitry Karpeev @*/
29014ab8060aSDmitry Karpeev PetscErrorCode  PCFieldSplitSetDMSplits(PC pc,PetscBool flg)
29024ab8060aSDmitry Karpeev {
29034ab8060aSDmitry Karpeev   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
29044ab8060aSDmitry Karpeev   PetscBool      isfs;
29054ab8060aSDmitry Karpeev 
29064ab8060aSDmitry Karpeev   PetscFunctionBegin;
29074ab8060aSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
29084ab8060aSDmitry Karpeev   PetscValidLogicalCollectiveBool(pc,flg,2);
29099566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&isfs));
29104ab8060aSDmitry Karpeev   if (isfs) {
29114ab8060aSDmitry Karpeev     jac->dm_splits = flg;
29124ab8060aSDmitry Karpeev   }
29134ab8060aSDmitry Karpeev   PetscFunctionReturn(0);
29144ab8060aSDmitry Karpeev }
29154ab8060aSDmitry Karpeev 
29164ab8060aSDmitry Karpeev /*@
29174ab8060aSDmitry Karpeev    PCFieldSplitGetDMSplits - Returns flag indicating whether DMCreateFieldDecomposition() should be used to define the splits, whenever possible.
29184ab8060aSDmitry Karpeev 
29194ab8060aSDmitry Karpeev    Logically Collective
29204ab8060aSDmitry Karpeev 
29214ab8060aSDmitry Karpeev    Input Parameter:
29224ab8060aSDmitry Karpeev .  pc   - the preconditioner context
29234ab8060aSDmitry Karpeev 
29244ab8060aSDmitry Karpeev    Output Parameter:
29254ab8060aSDmitry Karpeev .  flg  - boolean indicating whether to use field splits defined by the DM
29264ab8060aSDmitry Karpeev 
29274ab8060aSDmitry Karpeev    Level: Intermediate
29284ab8060aSDmitry Karpeev 
2929db781477SPatrick Sanan .seealso: `PCFieldSplitSetDMSplits()`
29304ab8060aSDmitry Karpeev 
29314ab8060aSDmitry Karpeev @*/
29324ab8060aSDmitry Karpeev PetscErrorCode  PCFieldSplitGetDMSplits(PC pc,PetscBool* flg)
29334ab8060aSDmitry Karpeev {
29344ab8060aSDmitry Karpeev   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
29354ab8060aSDmitry Karpeev   PetscBool      isfs;
29364ab8060aSDmitry Karpeev 
29374ab8060aSDmitry Karpeev   PetscFunctionBegin;
29384ab8060aSDmitry Karpeev   PetscValidHeaderSpecific(pc,PC_CLASSID,1);
2939534a8f05SLisandro Dalcin   PetscValidBoolPointer(flg,2);
29409566063dSJacob Faibussowitsch   PetscCall(PetscObjectTypeCompare((PetscObject)pc,PCFIELDSPLIT,&isfs));
29414ab8060aSDmitry Karpeev   if (isfs) {
29424ab8060aSDmitry Karpeev     if (flg) *flg = jac->dm_splits;
29434ab8060aSDmitry Karpeev   }
29444ab8060aSDmitry Karpeev   PetscFunctionReturn(0);
29454ab8060aSDmitry Karpeev }
29464ab8060aSDmitry Karpeev 
29477b752e3dSPatrick Sanan /*@
29487b752e3dSPatrick Sanan    PCFieldSplitGetDetectSaddlePoint - Returns flag indicating whether PCFieldSplit will attempt to automatically determine fields based on zero diagonal entries.
29497b752e3dSPatrick Sanan 
29507b752e3dSPatrick Sanan    Logically Collective
29517b752e3dSPatrick Sanan 
29527b752e3dSPatrick Sanan    Input Parameter:
29537b752e3dSPatrick Sanan .  pc   - the preconditioner context
29547b752e3dSPatrick Sanan 
29557b752e3dSPatrick Sanan    Output Parameter:
29567b752e3dSPatrick Sanan .  flg  - boolean indicating whether to detect fields or not
29577b752e3dSPatrick Sanan 
29587b752e3dSPatrick Sanan    Level: Intermediate
29597b752e3dSPatrick Sanan 
2960db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetDetectSaddlePoint()`
29617b752e3dSPatrick Sanan 
29627b752e3dSPatrick Sanan @*/
29637b752e3dSPatrick Sanan PetscErrorCode PCFieldSplitGetDetectSaddlePoint(PC pc,PetscBool *flg)
29647b752e3dSPatrick Sanan {
29657b752e3dSPatrick Sanan   PC_FieldSplit *jac = (PC_FieldSplit*)pc->data;
29667b752e3dSPatrick Sanan 
29677b752e3dSPatrick Sanan   PetscFunctionBegin;
29687b752e3dSPatrick Sanan   *flg = jac->detect;
29697b752e3dSPatrick Sanan   PetscFunctionReturn(0);
29707b752e3dSPatrick Sanan }
29717b752e3dSPatrick Sanan 
29727b752e3dSPatrick Sanan /*@
29737b752e3dSPatrick Sanan    PCFieldSplitSetDetectSaddlePoint - Sets flag indicating whether PCFieldSplit will attempt to automatically determine fields based on zero diagonal entries.
29747b752e3dSPatrick Sanan 
29757b752e3dSPatrick Sanan    Logically Collective
29767b752e3dSPatrick Sanan 
29777b752e3dSPatrick Sanan    Input Parameter:
29787b752e3dSPatrick Sanan .  pc   - the preconditioner context
29797b752e3dSPatrick Sanan 
29807b752e3dSPatrick Sanan    Output Parameter:
29817b752e3dSPatrick Sanan .  flg  - boolean indicating whether to detect fields or not
29827b752e3dSPatrick Sanan 
29837b752e3dSPatrick Sanan    Options Database Key:
2984147403d9SBarry Smith .  -pc_fieldsplit_detect_saddle_point <bool> - detect and use the saddle point
2985147403d9SBarry Smith 
2986147403d9SBarry Smith    Notes:
2987147403d9SBarry Smith    Also sets the split type to PC_COMPOSITE_SCHUR (see PCFieldSplitSetType()) and the Schur preconditioner type to PC_FIELDSPLIT_SCHUR_PRE_SELF (see PCFieldSplitSetSchurPre()).
29887b752e3dSPatrick Sanan 
29897b752e3dSPatrick Sanan    Level: Intermediate
29907b752e3dSPatrick Sanan 
2991db781477SPatrick Sanan .seealso: `PCFIELDSPLIT`, `PCFieldSplitSetDetectSaddlePoint()`, `PCFieldSplitSetType()`, `PCFieldSplitSetSchurPre()`
29927b752e3dSPatrick Sanan 
29937b752e3dSPatrick Sanan @*/
29947b752e3dSPatrick Sanan PetscErrorCode PCFieldSplitSetDetectSaddlePoint(PC pc,PetscBool flg)
29957b752e3dSPatrick Sanan {
29967b752e3dSPatrick Sanan   PC_FieldSplit  *jac = (PC_FieldSplit*)pc->data;
29977b752e3dSPatrick Sanan 
29987b752e3dSPatrick Sanan   PetscFunctionBegin;
29997b752e3dSPatrick Sanan   jac->detect = flg;
30007b752e3dSPatrick Sanan   if (jac->detect) {
30019566063dSJacob Faibussowitsch     PetscCall(PCFieldSplitSetType(pc,PC_COMPOSITE_SCHUR));
30029566063dSJacob Faibussowitsch     PetscCall(PCFieldSplitSetSchurPre(pc,PC_FIELDSPLIT_SCHUR_PRE_SELF,NULL));
30037b752e3dSPatrick Sanan   }
30047b752e3dSPatrick Sanan   PetscFunctionReturn(0);
30057b752e3dSPatrick Sanan }
30067b752e3dSPatrick Sanan 
30070971522cSBarry Smith /* -------------------------------------------------------------------------------------*/
30080971522cSBarry Smith /*MC
3009a8c7a070SBarry Smith    PCFIELDSPLIT - Preconditioner created by combining separate preconditioners for individual
3010de37d9f1SPatrick Sanan                   fields or groups of fields. See [the users manual section on "Solving Block Matrices"](sec_block_matrices) for more details.
30110971522cSBarry Smith 
3012de37d9f1SPatrick Sanan      To set options on the solvers for each block append `-fieldsplit_` to all the PC
3013de37d9f1SPatrick Sanan         options database keys. For example, `-fieldsplit_pc_type ilu -fieldsplit_pc_factor_levels 1`
30140971522cSBarry Smith 
3015de37d9f1SPatrick Sanan      To set the options on the solvers separate for each block call `PCFieldSplitGetSubKSP()`
3016de37d9f1SPatrick Sanan          and set the options directly on the resulting `KSP` object
30170971522cSBarry Smith 
30180971522cSBarry Smith    Level: intermediate
30190971522cSBarry Smith 
302079416396SBarry Smith    Options Database Keys:
3021de37d9f1SPatrick Sanan +   -pc_fieldsplit_%d_fields <a,b,..> - indicates the fields to be used in the `%d`'th split
302281540f2fSBarry Smith .   -pc_fieldsplit_default - automatically add any fields to additional splits that have not
3023de37d9f1SPatrick Sanan                               been supplied explicitly by `-pc_fieldsplit_%d_fields`
302481540f2fSBarry Smith .   -pc_fieldsplit_block_size <bs> - size of block that defines fields (i.e. there are bs fields)
3025a51937d4SCarola Kruse .   -pc_fieldsplit_type <additive,multiplicative,symmetric_multiplicative,schur,gkb> - type of relaxation or factorization splitting
3026de37d9f1SPatrick Sanan .   -pc_fieldsplit_schur_precondition <self,selfp,user,a11,full> - default is a11; see `PCFieldSplitSetSchurPre()`
3027de37d9f1SPatrick Sanan .   -pc_fieldsplit_schur_fact_type <diag,lower,upper,full> - set factorization type when using `-pc_fieldsplit_type schur`; see `PCFieldSplitSetSchurFactType()`
3028fb6809a2SPatrick Sanan -   -pc_fieldsplit_detect_saddle_point - automatically finds rows with zero diagonal and uses Schur complement with no preconditioner as the solver
302979416396SBarry Smith 
3030de37d9f1SPatrick Sanan      Options prefixes for inner solvers when using the Schur complement preconditioner are `-fieldsplit_0_` and `-fieldsplit_1_` .
3031de37d9f1SPatrick Sanan      For all other solvers they are `-fieldsplit_%d_` for the `d`th field; use `-fieldsplit_` for all fields.
3032de37d9f1SPatrick Sanan      The options prefix for the inner solver when using the Golub-Kahan biadiagonalization preconditioner is `-fieldsplit_0_`
30335d4c12cdSJungho Lee 
3034c8a0d604SMatthew G Knepley    Notes:
3035de37d9f1SPatrick Sanan     Use `PCFieldSplitSetFields()` to set fields defined by "strided" entries and `PCFieldSplitSetIS()`
3036d32f9abdSBarry Smith      to define a field by an arbitrary collection of entries.
3037d32f9abdSBarry Smith 
3038d32f9abdSBarry Smith       If no fields are set the default is used. The fields are defined by entries strided by bs,
3039de37d9f1SPatrick Sanan       beginning at 0 then 1, etc to bs-1. The block size can be set with `PCFieldSplitSetBlockSize()`,
3040d32f9abdSBarry Smith       if this is not called the block size defaults to the blocksize of the second matrix passed
3041de37d9f1SPatrick Sanan       to `KSPSetOperators()`/`PCSetOperators()`.
3042d32f9abdSBarry Smith 
3043de37d9f1SPatrick Sanan       For the Schur complement preconditioner if
30441d71051fSDmitry Karpeev 
3045de37d9f1SPatrick Sanan       ```{math}
3046de37d9f1SPatrick Sanan       J = \left[\begin{array}{cc} A_{00} & A_{01} \\ A_{10} & A_{11} \end{array}\right]
3047de37d9f1SPatrick Sanan       ```
3048e69d4d44SBarry Smith 
3049de37d9f1SPatrick Sanan       the preconditioner using `full` factorization is logically
3050de37d9f1SPatrick Sanan       ```{math}
3051de37d9f1SPatrick Sanan       \left[\begin{array}{cc} I & -\text{ksp}(A_{00}) \\ 0 & I \end{array}\right] \left[\begin{array}{cc} \text{inv}(A_{00}) & 0 \\ 0 & \text{ksp}(S) \end{array}\right] \left[\begin{array}{cc} I & 0 \\ -A_{10} \text{ksp}(A_{00}) & I \end{array}\right]
3052de37d9f1SPatrick Sanan       ```
3053de37d9f1SPatrick Sanan      where the action of $\text{inv}(A_{00})$ is applied using the KSP solver with prefix `-fieldsplit_0_`.  $S$ is the Schur complement
3054de37d9f1SPatrick Sanan      ```{math}
3055223e5b4fSPatrick Sanan      S = A_{11} - A_{10} \text{ksp}(A_{00}) A_{01}
3056de37d9f1SPatrick Sanan      ```
3057de37d9f1SPatrick Sanan      which is usually dense and not stored explicitly.  The action of $\text{ksp}(S)$ is computed using the KSP solver with prefix `-fieldsplit_splitname_` (where `splitname` was given
3058de37d9f1SPatrick Sanan      in providing the SECOND split or 1 if not given). For `PCFieldSplitGetSub\text{ksp}()` when field number is 0,
3059de37d9f1SPatrick Sanan      it returns the KSP associated with `-fieldsplit_0_` while field number 1 gives `-fieldsplit_1_` KSP. By default
3060de37d9f1SPatrick Sanan      $A_{11}$ is used to construct a preconditioner for $S$, use `PCFieldSplitSetSchurPre()` for all the possible ways to construct the preconditioner for $S$.
3061de37d9f1SPatrick Sanan 
3062de37d9f1SPatrick Sanan      The factorization type is set using `-pc_fieldsplit_schur_fact_type <diag, lower, upper, full>`. `full` is shown above,
3063de37d9f1SPatrick Sanan      `diag` gives
3064de37d9f1SPatrick Sanan       ```{math}
3065de37d9f1SPatrick Sanan       \left[\begin{array}{cc} \text{inv}(A_{00}) & 0 \\  0 & -\text{ksp}(S) \end{array}\right]
3066de37d9f1SPatrick Sanan       ```
3067de37d9f1SPatrick Sanan      Note that, slightly counter intuitively, there is a negative in front of the $\text{ksp}(S)$  so that the preconditioner is positive definite. For SPD matrices $J$, the sign flip
3068de37d9f1SPatrick Sanan      can be turned off with `PCFieldSplitSetSchurScale()` or by command line `-pc_fieldsplit_schur_scale 1.0`. The `lower` factorization is the inverse of
3069de37d9f1SPatrick Sanan       ```{math}
3070de37d9f1SPatrick Sanan       \left[\begin{array}{cc} A_{00} & 0 \\  A_{10} & S \end{array}\right]
3071de37d9f1SPatrick Sanan       ```
3072de37d9f1SPatrick Sanan      where the inverses of A_{00} and S are applied using KSPs. The upper factorization is the inverse of
3073de37d9f1SPatrick Sanan       ```{math}
3074de37d9f1SPatrick Sanan       \left[\begin{array}{cc} A_{00} & A_{01} \\  0 & S \end{array}\right]
3075de37d9f1SPatrick Sanan       ```
3076de37d9f1SPatrick Sanan      where again the inverses of $A_{00}$ and $S$ are applied using `KSP`s.
3077de37d9f1SPatrick Sanan 
3078de37d9f1SPatrick Sanan      If only one set of indices (one `IS`) is provided with `PCFieldSplitSetIS()` then the complement of that `IS`
3079edf189efSBarry Smith      is used automatically for a second block.
3080edf189efSBarry Smith 
3081ff218e97SBarry Smith      The fieldsplit preconditioner cannot currently be used with the BAIJ or SBAIJ data formats if the blocksize is larger than 1.
3082ff218e97SBarry Smith      Generally it should be used with the AIJ format.
3083ff218e97SBarry Smith 
3084ff218e97SBarry Smith      The forms of these preconditioners are closely related if not identical to forms derived as "Distributive Iterations", see,
3085de37d9f1SPatrick Sanan      for example, page 294 in "Principles of Computational Fluid Dynamics" by Pieter Wesseling {cite}`Wesseling2009`. Note that one can also use `PCFIELDSPLIT`
3086ff218e97SBarry Smith      inside a smoother resulting in "Distributive Smoothers".
30870716a85fSBarry Smith 
30882da392ccSBarry Smith      References:
30893bc631e0SBarry Smith 
3090de37d9f1SPatrick Sanan      See "A taxonomy and comparison of parallel block multi-level preconditioners for the incompressible Navier-Stokes equations" {cite}`elman2008tcp`.
3091a6a584a2SBarry Smith 
3092de37d9f1SPatrick Sanan      The Constrained Pressure Preconditioner (CPR) can be implemented using `PCCOMPOSITE` with `PCGALERKIN`. CPR first solves an $R A P$ subsystem, updates the
3093de37d9f1SPatrick Sanan      residual on all variables (`PCCompositeSetType(pc,PC_COMPOSITE_MULTIPLICATIVE)`), and then applies a simple ILU like preconditioner on all the variables.
3094a51937d4SCarola Kruse 
3095de37d9f1SPatrick Sanan      The generalized Golub-Kahan bidiagonalization preconditioner (GKB) can be applied to symmetric $2 \times 2$ block matrices of the shape
3096de37d9f1SPatrick Sanan      ```{math}
3097de37d9f1SPatrick Sanan      \left[\begin{array}{cc} A_{00} & A_{01} \\ A_{01}' & 0 \end{array}\right]
3098de37d9f1SPatrick Sanan      ```
3099de37d9f1SPatrick Sanan      with $A_{00}$ positive semi-definite. The implementation follows {cite}`Arioli2013`. Therein, we choose $N := 1/\nu * I$ and the $(1,1)$-block of the matrix is modified to $H = _{A00} + \nu*A_{01}*A_{01}'$.
3100de37d9f1SPatrick Sanan      A linear system $Hx = b$ has to be solved in each iteration of the GKB algorithm. This solver is chosen with the option prefix `-fieldsplit_0_`.
3101a51937d4SCarola Kruse 
3102de37d9f1SPatrick Sanan      ```{bibliography}
3103de37d9f1SPatrick Sanan      :filter: docname in docnames
3104de37d9f1SPatrick Sanan      ```
3105de37d9f1SPatrick Sanan 
3106db781477SPatrick Sanan .seealso: `PCCreate()`, `PCSetType()`, `PCType`, `PC`, `PCLSC`,
3107db781477SPatrick Sanan           `PCFieldSplitGetSubKSP()`, `PCFieldSplitSchurGetSubKSP()`, `PCFieldSplitSetFields()`,
3108db781477SPatrick Sanan           `PCFieldSplitSetType()`, `PCFieldSplitSetIS()`, `PCFieldSplitSetSchurPre()`, `PCFieldSplitSetSchurFactType()`,
3109db781477SPatrick Sanan           `MatSchurComplementSetAinvType()`, `PCFieldSplitSetSchurScale()`, `PCFieldSplitSetDetectSaddlePoint()`
31100971522cSBarry Smith M*/
31110971522cSBarry Smith 
31128cc058d9SJed Brown PETSC_EXTERN PetscErrorCode PCCreate_FieldSplit(PC pc)
31130971522cSBarry Smith {
31140971522cSBarry Smith   PC_FieldSplit  *jac;
31150971522cSBarry Smith 
31160971522cSBarry Smith   PetscFunctionBegin;
31179566063dSJacob Faibussowitsch   PetscCall(PetscNewLog(pc,&jac));
31182fa5cd67SKarl Rupp 
31190971522cSBarry Smith   jac->bs                 = -1;
31200971522cSBarry Smith   jac->nsplits            = 0;
31213e197d65SBarry Smith   jac->type               = PC_COMPOSITE_MULTIPLICATIVE;
3122e6cab6aaSJed Brown   jac->schurpre           = PC_FIELDSPLIT_SCHUR_PRE_USER; /* Try user preconditioner first, fall back on diagonal */
3123c9c6ffaaSJed Brown   jac->schurfactorization = PC_FIELDSPLIT_SCHUR_FACT_FULL;
3124c096484dSStefano Zampini   jac->schurscale         = -1.0;
3125fbe7908bSJed Brown   jac->dm_splits          = PETSC_TRUE;
31267b752e3dSPatrick Sanan   jac->detect             = PETSC_FALSE;
3127a51937d4SCarola Kruse   jac->gkbtol             = 1e-5;
3128a51937d4SCarola Kruse   jac->gkbdelay           = 5;
3129a51937d4SCarola Kruse   jac->gkbnu              = 1;
3130a51937d4SCarola Kruse   jac->gkbmaxit           = 100;
3131de482cd7SCarola Kruse   jac->gkbmonitor         = PETSC_FALSE;
31325ddf11f8SNicolas Barnafi   jac->coordinates_set    = PETSC_FALSE;
313351f519a2SBarry Smith 
31340971522cSBarry Smith   pc->data = (void*)jac;
31350971522cSBarry Smith 
31360971522cSBarry Smith   pc->ops->apply           = PCApply_FieldSplit;
3137421e10b8SBarry Smith   pc->ops->applytranspose  = PCApplyTranspose_FieldSplit;
31380971522cSBarry Smith   pc->ops->setup           = PCSetUp_FieldSplit;
3139574deadeSBarry Smith   pc->ops->reset           = PCReset_FieldSplit;
31400971522cSBarry Smith   pc->ops->destroy         = PCDestroy_FieldSplit;
31410971522cSBarry Smith   pc->ops->setfromoptions  = PCSetFromOptions_FieldSplit;
31420971522cSBarry Smith   pc->ops->view            = PCView_FieldSplit;
31430a545947SLisandro Dalcin   pc->ops->applyrichardson = NULL;
31440971522cSBarry Smith 
31459566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSchurGetSubKSP_C",PCFieldSplitSchurGetSubKSP_FieldSplit));
31469566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitGetSubKSP_C",PCFieldSplitGetSubKSP_FieldSplit));
31479566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetFields_C",PCFieldSplitSetFields_FieldSplit));
31489566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetIS_C",PCFieldSplitSetIS_FieldSplit));
31499566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetType_C",PCFieldSplitSetType_FieldSplit));
31509566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitSetBlockSize_C",PCFieldSplitSetBlockSize_FieldSplit));
31519566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCFieldSplitRestrictIS_C",PCFieldSplitRestrictIS_FieldSplit));
31529566063dSJacob Faibussowitsch   PetscCall(PetscObjectComposeFunction((PetscObject)pc,"PCSetCoordinates_C",PCSetCoordinates_FieldSplit));
31530971522cSBarry Smith   PetscFunctionReturn(0);
31540971522cSBarry Smith }
3155