xref: /petsc/src/mat/impls/baij/seq/baij2.c (revision 71fd2e92b2fee25f70818ffc1dca9fc57f8584d7)
1 #define PETSCMAT_DLL
2 
3 #include "../src/mat/impls/baij/seq/baij.h"
4 #include "../src/mat/blockinvert.h"
5 #include "petscbt.h"
6 
7 #undef __FUNCT__
8 #define __FUNCT__ "MatIncreaseOverlap_SeqBAIJ"
9 PetscErrorCode MatIncreaseOverlap_SeqBAIJ(Mat A,PetscInt is_max,IS is[],PetscInt ov)
10 {
11   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
12   PetscErrorCode ierr;
13   PetscInt       row,i,j,k,l,m,n,*nidx,isz,val,ival;
14   const PetscInt *idx;
15   PetscInt       start,end,*ai,*aj,bs,*nidx2;
16   PetscBT        table;
17 
18   PetscFunctionBegin;
19   m     = a->mbs;
20   ai    = a->i;
21   aj    = a->j;
22   bs    = A->rmap->bs;
23 
24   if (ov < 0)  SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Negative overlap specified");
25 
26   ierr = PetscBTCreate(m,table);CHKERRQ(ierr);
27   ierr = PetscMalloc((m+1)*sizeof(PetscInt),&nidx);CHKERRQ(ierr);
28   ierr = PetscMalloc((A->rmap->N+1)*sizeof(PetscInt),&nidx2);CHKERRQ(ierr);
29 
30   for (i=0; i<is_max; i++) {
31     /* Initialise the two local arrays */
32     isz  = 0;
33     ierr = PetscBTMemzero(m,table);CHKERRQ(ierr);
34 
35     /* Extract the indices, assume there can be duplicate entries */
36     ierr = ISGetIndices(is[i],&idx);CHKERRQ(ierr);
37     ierr = ISGetLocalSize(is[i],&n);CHKERRQ(ierr);
38 
39     /* Enter these into the temp arrays i.e mark table[row], enter row into new index */
40     for (j=0; j<n ; ++j){
41       ival = idx[j]/bs; /* convert the indices into block indices */
42       if (ival>=m) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"index greater than mat-dim");
43       if(!PetscBTLookupSet(table,ival)) { nidx[isz++] = ival;}
44     }
45     ierr = ISRestoreIndices(is[i],&idx);CHKERRQ(ierr);
46     ierr = ISDestroy(is[i]);CHKERRQ(ierr);
47 
48     k = 0;
49     for (j=0; j<ov; j++){ /* for each overlap*/
50       n = isz;
51       for (; k<n ; k++){ /* do only those rows in nidx[k], which are not done yet */
52         row   = nidx[k];
53         start = ai[row];
54         end   = ai[row+1];
55         for (l = start; l<end ; l++){
56           val = aj[l];
57           if (!PetscBTLookupSet(table,val)) {nidx[isz++] = val;}
58         }
59       }
60     }
61     /* expand the Index Set */
62     for (j=0; j<isz; j++) {
63       for (k=0; k<bs; k++)
64         nidx2[j*bs+k] = nidx[j]*bs+k;
65     }
66     ierr = ISCreateGeneral(PETSC_COMM_SELF,isz*bs,nidx2,is+i);CHKERRQ(ierr);
67   }
68   ierr = PetscBTDestroy(table);CHKERRQ(ierr);
69   ierr = PetscFree(nidx);CHKERRQ(ierr);
70   ierr = PetscFree(nidx2);CHKERRQ(ierr);
71   PetscFunctionReturn(0);
72 }
73 
74 #undef __FUNCT__
75 #define __FUNCT__ "MatGetSubMatrix_SeqBAIJ_Private"
76 PetscErrorCode MatGetSubMatrix_SeqBAIJ_Private(Mat A,IS isrow,IS iscol,MatReuse scall,Mat *B)
77 {
78   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data,*c;
79   PetscErrorCode ierr;
80   PetscInt       *smap,i,k,kstart,kend,oldcols = a->nbs,*lens;
81   PetscInt       row,mat_i,*mat_j,tcol,*mat_ilen;
82   const PetscInt *irow,*icol;
83   PetscInt       nrows,ncols,*ssmap,bs=A->rmap->bs,bs2=a->bs2;
84   PetscInt       *aj = a->j,*ai = a->i;
85   MatScalar      *mat_a;
86   Mat            C;
87   PetscTruth     flag,sorted;
88 
89   PetscFunctionBegin;
90   ierr = ISSorted(iscol,&sorted);CHKERRQ(ierr);
91   if (!sorted) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"IS is not sorted");
92 
93   ierr = ISGetIndices(isrow,&irow);CHKERRQ(ierr);
94   ierr = ISGetIndices(iscol,&icol);CHKERRQ(ierr);
95   ierr = ISGetLocalSize(isrow,&nrows);CHKERRQ(ierr);
96   ierr = ISGetLocalSize(iscol,&ncols);CHKERRQ(ierr);
97 
98   ierr = PetscMalloc((1+oldcols)*sizeof(PetscInt),&smap);CHKERRQ(ierr);
99   ssmap = smap;
100   ierr = PetscMalloc((1+nrows)*sizeof(PetscInt),&lens);CHKERRQ(ierr);
101   ierr  = PetscMemzero(smap,oldcols*sizeof(PetscInt));CHKERRQ(ierr);
102   for (i=0; i<ncols; i++) smap[icol[i]] = i+1;
103   /* determine lens of each row */
104   for (i=0; i<nrows; i++) {
105     kstart  = ai[irow[i]];
106     kend    = kstart + a->ilen[irow[i]];
107     lens[i] = 0;
108       for (k=kstart; k<kend; k++) {
109         if (ssmap[aj[k]]) {
110           lens[i]++;
111         }
112       }
113     }
114   /* Create and fill new matrix */
115   if (scall == MAT_REUSE_MATRIX) {
116     c = (Mat_SeqBAIJ *)((*B)->data);
117 
118     if (c->mbs!=nrows || c->nbs!=ncols || (*B)->rmap->bs!=bs) SETERRQ(PETSC_ERR_ARG_SIZ,"Submatrix wrong size");
119     ierr = PetscMemcmp(c->ilen,lens,c->mbs *sizeof(PetscInt),&flag);CHKERRQ(ierr);
120     if (!flag) {
121       SETERRQ(PETSC_ERR_ARG_SIZ,"Cannot reuse matrix. wrong no of nonzeros");
122     }
123     ierr = PetscMemzero(c->ilen,c->mbs*sizeof(PetscInt));CHKERRQ(ierr);
124     C = *B;
125   } else {
126     ierr = MatCreate(((PetscObject)A)->comm,&C);CHKERRQ(ierr);
127     ierr = MatSetSizes(C,nrows*bs,ncols*bs,PETSC_DETERMINE,PETSC_DETERMINE);CHKERRQ(ierr);
128     ierr = MatSetType(C,((PetscObject)A)->type_name);CHKERRQ(ierr);
129     ierr = MatSeqBAIJSetPreallocation_SeqBAIJ(C,bs,0,lens);CHKERRQ(ierr);
130   }
131   c = (Mat_SeqBAIJ *)(C->data);
132   for (i=0; i<nrows; i++) {
133     row    = irow[i];
134     kstart = ai[row];
135     kend   = kstart + a->ilen[row];
136     mat_i  = c->i[i];
137     mat_j  = c->j + mat_i;
138     mat_a  = c->a + mat_i*bs2;
139     mat_ilen = c->ilen + i;
140     for (k=kstart; k<kend; k++) {
141       if ((tcol=ssmap[a->j[k]])) {
142         *mat_j++ = tcol - 1;
143         ierr     = PetscMemcpy(mat_a,a->a+k*bs2,bs2*sizeof(MatScalar));CHKERRQ(ierr);
144         mat_a   += bs2;
145         (*mat_ilen)++;
146       }
147     }
148   }
149 
150   /* Free work space */
151   ierr = ISRestoreIndices(iscol,&icol);CHKERRQ(ierr);
152   ierr = PetscFree(smap);CHKERRQ(ierr);
153   ierr = PetscFree(lens);CHKERRQ(ierr);
154   ierr = MatAssemblyBegin(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
155   ierr = MatAssemblyEnd(C,MAT_FINAL_ASSEMBLY);CHKERRQ(ierr);
156 
157   ierr = ISRestoreIndices(isrow,&irow);CHKERRQ(ierr);
158   *B = C;
159   PetscFunctionReturn(0);
160 }
161 
162 #undef __FUNCT__
163 #define __FUNCT__ "MatGetSubMatrix_SeqBAIJ"
164 PetscErrorCode MatGetSubMatrix_SeqBAIJ(Mat A,IS isrow,IS iscol,MatReuse scall,Mat *B)
165 {
166   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
167   IS             is1,is2;
168   PetscErrorCode ierr;
169   PetscInt       *vary,*iary,nrows,ncols,i,bs=A->rmap->bs,count;
170   const PetscInt *irow,*icol;
171 
172   PetscFunctionBegin;
173   ierr = ISGetIndices(isrow,&irow);CHKERRQ(ierr);
174   ierr = ISGetIndices(iscol,&icol);CHKERRQ(ierr);
175   ierr = ISGetLocalSize(isrow,&nrows);CHKERRQ(ierr);
176   ierr = ISGetLocalSize(iscol,&ncols);CHKERRQ(ierr);
177 
178   /* Verify if the indices corespond to each element in a block
179    and form the IS with compressed IS */
180   ierr = PetscMalloc(2*(a->mbs+1)*sizeof(PetscInt),&vary);CHKERRQ(ierr);
181   iary = vary + a->mbs;
182   ierr = PetscMemzero(vary,(a->mbs)*sizeof(PetscInt));CHKERRQ(ierr);
183   for (i=0; i<nrows; i++) vary[irow[i]/bs]++;
184   count = 0;
185   for (i=0; i<a->mbs; i++) {
186     if (vary[i]!=0 && vary[i]!=bs) SETERRQ(PETSC_ERR_ARG_SIZ,"Index set does not match blocks");
187     if (vary[i]==bs) iary[count++] = i;
188   }
189   ierr = ISCreateGeneral(PETSC_COMM_SELF,count,iary,&is1);CHKERRQ(ierr);
190 
191   ierr = PetscMemzero(vary,(a->mbs)*sizeof(PetscInt));CHKERRQ(ierr);
192   for (i=0; i<ncols; i++) vary[icol[i]/bs]++;
193   count = 0;
194   for (i=0; i<a->mbs; i++) {
195     if (vary[i]!=0 && vary[i]!=bs) SETERRQ(PETSC_ERR_PLIB,"Internal error in PETSc");
196     if (vary[i]==bs) iary[count++] = i;
197   }
198   ierr = ISCreateGeneral(PETSC_COMM_SELF,count,iary,&is2);CHKERRQ(ierr);
199   ierr = ISRestoreIndices(isrow,&irow);CHKERRQ(ierr);
200   ierr = ISRestoreIndices(iscol,&icol);CHKERRQ(ierr);
201   ierr = PetscFree(vary);CHKERRQ(ierr);
202 
203   ierr = MatGetSubMatrix_SeqBAIJ_Private(A,is1,is2,scall,B);CHKERRQ(ierr);
204   ISDestroy(is1);
205   ISDestroy(is2);
206   PetscFunctionReturn(0);
207 }
208 
209 #undef __FUNCT__
210 #define __FUNCT__ "MatGetSubMatrices_SeqBAIJ"
211 PetscErrorCode MatGetSubMatrices_SeqBAIJ(Mat A,PetscInt n,const IS irow[],const IS icol[],MatReuse scall,Mat *B[])
212 {
213   PetscErrorCode ierr;
214   PetscInt       i;
215 
216   PetscFunctionBegin;
217   if (scall == MAT_INITIAL_MATRIX) {
218     ierr = PetscMalloc((n+1)*sizeof(Mat),B);CHKERRQ(ierr);
219   }
220 
221   for (i=0; i<n; i++) {
222     ierr = MatGetSubMatrix_SeqBAIJ(A,irow[i],icol[i],scall,&(*B)[i]);CHKERRQ(ierr);
223   }
224   PetscFunctionReturn(0);
225 }
226 
227 
228 /* -------------------------------------------------------*/
229 /* Should check that shapes of vectors and matrices match */
230 /* -------------------------------------------------------*/
231 #include "petscblaslapack.h"
232 
233 #undef __FUNCT__
234 #define __FUNCT__ "MatMult_SeqBAIJ_1"
235 PetscErrorCode MatMult_SeqBAIJ_1(Mat A,Vec xx,Vec zz)
236 {
237   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
238   PetscScalar       *z,sum;
239   const PetscScalar *x;
240   const MatScalar   *v;
241   PetscErrorCode    ierr;
242   PetscInt          mbs,i,n,nonzerorow=0;
243   const PetscInt    *idx,*ii,*ridx=PETSC_NULL;
244   PetscTruth        usecprow=a->compressedrow.use;
245 
246   PetscFunctionBegin;
247   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
248   ierr = VecGetArray(zz,&z);CHKERRQ(ierr);
249 
250   if (usecprow){
251     mbs  = a->compressedrow.nrows;
252     ii   = a->compressedrow.i;
253     ridx = a->compressedrow.rindex;
254     ierr = PetscMemzero(z,mbs*sizeof(PetscScalar));CHKERRQ(ierr);
255   } else {
256     mbs = a->mbs;
257     ii  = a->i;
258   }
259 
260   for (i=0; i<mbs; i++) {
261     n    = ii[1] - ii[0];
262     v    = a->a + ii[0];
263     idx  = a->j + ii[0];
264     ii++;
265     sum  = 0.0;
266     PetscSparseDensePlusDot(sum,x,v,idx,n);
267     if (usecprow){
268       z[ridx[i]] = sum;
269     } else {
270       nonzerorow += (n>0);
271       z[i] = sum;
272     }
273   }
274   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
275   ierr = VecRestoreArray(zz,&z);CHKERRQ(ierr);
276   ierr = PetscLogFlops(2.0*a->nz - nonzerorow);CHKERRQ(ierr);
277   PetscFunctionReturn(0);
278 }
279 
280 #undef __FUNCT__
281 #define __FUNCT__ "MatMult_SeqBAIJ_2"
282 PetscErrorCode MatMult_SeqBAIJ_2(Mat A,Vec xx,Vec zz)
283 {
284   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
285   PetscScalar       *z = 0,sum1,sum2,*zarray;
286   const PetscScalar *x,*xb;
287   PetscScalar       x1,x2;
288   const MatScalar   *v;
289   PetscErrorCode    ierr;
290   PetscInt          mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL,nonzerorow=0;
291   PetscTruth        usecprow=a->compressedrow.use;
292 
293   PetscFunctionBegin;
294   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
295   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
296 
297   idx = a->j;
298   v   = a->a;
299   if (usecprow){
300     mbs  = a->compressedrow.nrows;
301     ii   = a->compressedrow.i;
302     ridx = a->compressedrow.rindex;
303   } else {
304     mbs = a->mbs;
305     ii  = a->i;
306     z   = zarray;
307   }
308 
309   for (i=0; i<mbs; i++) {
310     n  = ii[1] - ii[0]; ii++;
311     sum1 = 0.0; sum2 = 0.0;
312     nonzerorow += (n>0);
313     for (j=0; j<n; j++) {
314       xb = x + 2*(*idx++); x1 = xb[0]; x2 = xb[1];
315       sum1 += v[0]*x1 + v[2]*x2;
316       sum2 += v[1]*x1 + v[3]*x2;
317       v += 4;
318     }
319     if (usecprow) z = zarray + 2*ridx[i];
320     z[0] = sum1; z[1] = sum2;
321     if (!usecprow) z += 2;
322   }
323   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
324   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
325   ierr = PetscLogFlops(8.0*a->nz - 2.0*nonzerorow);CHKERRQ(ierr);
326   PetscFunctionReturn(0);
327 }
328 
329 #undef __FUNCT__
330 #define __FUNCT__ "MatMult_SeqBAIJ_3"
331 PetscErrorCode MatMult_SeqBAIJ_3(Mat A,Vec xx,Vec zz)
332 {
333   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
334   PetscScalar       *z = 0,sum1,sum2,sum3,x1,x2,x3,*zarray;
335   const PetscScalar *x,*xb;
336   const MatScalar   *v;
337   PetscErrorCode    ierr;
338   PetscInt          mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL,nonzerorow=0;
339   PetscTruth        usecprow=a->compressedrow.use;
340 
341 
342 #if defined(PETSC_HAVE_PRAGMA_DISJOINT)
343 #pragma disjoint(*v,*z,*xb)
344 #endif
345 
346   PetscFunctionBegin;
347   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
348   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
349 
350   idx = a->j;
351   v   = a->a;
352   if (usecprow){
353     mbs  = a->compressedrow.nrows;
354     ii   = a->compressedrow.i;
355     ridx = a->compressedrow.rindex;
356   } else {
357     mbs = a->mbs;
358     ii  = a->i;
359     z   = zarray;
360   }
361 
362   for (i=0; i<mbs; i++) {
363     n  = ii[1] - ii[0]; ii++;
364     sum1 = 0.0; sum2 = 0.0; sum3 = 0.0;
365     nonzerorow += (n>0);
366     for (j=0; j<n; j++) {
367       xb = x + 3*(*idx++); x1 = xb[0]; x2 = xb[1]; x3 = xb[2];
368       sum1 += v[0]*x1 + v[3]*x2 + v[6]*x3;
369       sum2 += v[1]*x1 + v[4]*x2 + v[7]*x3;
370       sum3 += v[2]*x1 + v[5]*x2 + v[8]*x3;
371       v += 9;
372     }
373     if (usecprow) z = zarray + 3*ridx[i];
374     z[0] = sum1; z[1] = sum2; z[2] = sum3;
375     if (!usecprow) z += 3;
376   }
377   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
378   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
379   ierr = PetscLogFlops(18.0*a->nz - 3.0*nonzerorow);CHKERRQ(ierr);
380   PetscFunctionReturn(0);
381 }
382 
383 #undef __FUNCT__
384 #define __FUNCT__ "MatMult_SeqBAIJ_4"
385 PetscErrorCode MatMult_SeqBAIJ_4(Mat A,Vec xx,Vec zz)
386 {
387   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
388   PetscScalar       *z = 0,sum1,sum2,sum3,sum4,x1,x2,x3,x4,*zarray;
389   const PetscScalar *x,*xb;
390   const MatScalar   *v;
391   PetscErrorCode    ierr;
392   PetscInt          mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL,nonzerorow=0;
393   PetscTruth        usecprow=a->compressedrow.use;
394 
395   PetscFunctionBegin;
396   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
397   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
398 
399   idx = a->j;
400   v   = a->a;
401   if (usecprow){
402     mbs  = a->compressedrow.nrows;
403     ii   = a->compressedrow.i;
404     ridx = a->compressedrow.rindex;
405   } else {
406     mbs = a->mbs;
407     ii  = a->i;
408     z   = zarray;
409   }
410 
411   for (i=0; i<mbs; i++) {
412     n  = ii[1] - ii[0]; ii++;
413     sum1 = 0.0; sum2 = 0.0; sum3 = 0.0; sum4 = 0.0;
414     nonzerorow += (n>0);
415     for (j=0; j<n; j++) {
416       xb = x + 4*(*idx++);
417       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3];
418       sum1 += v[0]*x1 + v[4]*x2 + v[8]*x3   + v[12]*x4;
419       sum2 += v[1]*x1 + v[5]*x2 + v[9]*x3   + v[13]*x4;
420       sum3 += v[2]*x1 + v[6]*x2 + v[10]*x3  + v[14]*x4;
421       sum4 += v[3]*x1 + v[7]*x2 + v[11]*x3  + v[15]*x4;
422       v += 16;
423     }
424     if (usecprow) z = zarray + 4*ridx[i];
425     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4;
426     if (!usecprow) z += 4;
427   }
428   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
429   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
430   ierr = PetscLogFlops(32.0*a->nz - 4.0*nonzerorow);CHKERRQ(ierr);
431   PetscFunctionReturn(0);
432 }
433 
434 #undef __FUNCT__
435 #define __FUNCT__ "MatMult_SeqBAIJ_5"
436 PetscErrorCode MatMult_SeqBAIJ_5(Mat A,Vec xx,Vec zz)
437 {
438   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
439   PetscScalar       sum1,sum2,sum3,sum4,sum5,x1,x2,x3,x4,x5,*z = 0,*zarray;
440   const PetscScalar *xb,*x;
441   const MatScalar   *v;
442   PetscErrorCode    ierr;
443   PetscInt          mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL,nonzerorow=0;
444   PetscTruth        usecprow=a->compressedrow.use;
445 
446   PetscFunctionBegin;
447   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
448   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
449 
450   idx = a->j;
451   v   = a->a;
452   if (usecprow){
453     mbs  = a->compressedrow.nrows;
454     ii   = a->compressedrow.i;
455     ridx = a->compressedrow.rindex;
456   } else {
457     mbs = a->mbs;
458     ii  = a->i;
459     z   = zarray;
460   }
461 
462   for (i=0; i<mbs; i++) {
463     n  = ii[1] - ii[0]; ii++;
464     sum1 = 0.0; sum2 = 0.0; sum3 = 0.0; sum4 = 0.0; sum5 = 0.0;
465     nonzerorow += (n>0);
466     for (j=0; j<n; j++) {
467       xb = x + 5*(*idx++);
468       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3]; x5 = xb[4];
469       sum1 += v[0]*x1 + v[5]*x2 + v[10]*x3  + v[15]*x4 + v[20]*x5;
470       sum2 += v[1]*x1 + v[6]*x2 + v[11]*x3  + v[16]*x4 + v[21]*x5;
471       sum3 += v[2]*x1 + v[7]*x2 + v[12]*x3  + v[17]*x4 + v[22]*x5;
472       sum4 += v[3]*x1 + v[8]*x2 + v[13]*x3  + v[18]*x4 + v[23]*x5;
473       sum5 += v[4]*x1 + v[9]*x2 + v[14]*x3  + v[19]*x4 + v[24]*x5;
474       v += 25;
475     }
476     if (usecprow) z = zarray + 5*ridx[i];
477     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4; z[4] = sum5;
478     if (!usecprow) z += 5;
479   }
480   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
481   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
482   ierr = PetscLogFlops(50.0*a->nz - 5.0*nonzerorow);CHKERRQ(ierr);
483   PetscFunctionReturn(0);
484 }
485 
486 
487 #undef __FUNCT__
488 #define __FUNCT__ "MatMult_SeqBAIJ_6"
489 PetscErrorCode MatMult_SeqBAIJ_6(Mat A,Vec xx,Vec zz)
490 {
491   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
492   PetscScalar       *z = 0,sum1,sum2,sum3,sum4,sum5,sum6;
493   const PetscScalar *x,*xb;
494   PetscScalar       x1,x2,x3,x4,x5,x6,*zarray;
495   const MatScalar   *v;
496   PetscErrorCode    ierr;
497   PetscInt          mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL,nonzerorow=0;
498   PetscTruth        usecprow=a->compressedrow.use;
499 
500   PetscFunctionBegin;
501   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
502   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
503 
504   idx = a->j;
505   v   = a->a;
506   if (usecprow){
507     mbs  = a->compressedrow.nrows;
508     ii   = a->compressedrow.i;
509     ridx = a->compressedrow.rindex;
510   } else {
511     mbs = a->mbs;
512     ii  = a->i;
513     z   = zarray;
514   }
515 
516   for (i=0; i<mbs; i++) {
517     n  = ii[1] - ii[0]; ii++;
518     sum1 = 0.0; sum2 = 0.0; sum3 = 0.0; sum4 = 0.0; sum5 = 0.0; sum6 = 0.0;
519     nonzerorow += (n>0);
520     for (j=0; j<n; j++) {
521       xb = x + 6*(*idx++);
522       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3]; x5 = xb[4]; x6 = xb[5];
523       sum1 += v[0]*x1 + v[6]*x2  + v[12]*x3  + v[18]*x4 + v[24]*x5 + v[30]*x6;
524       sum2 += v[1]*x1 + v[7]*x2  + v[13]*x3  + v[19]*x4 + v[25]*x5 + v[31]*x6;
525       sum3 += v[2]*x1 + v[8]*x2  + v[14]*x3  + v[20]*x4 + v[26]*x5 + v[32]*x6;
526       sum4 += v[3]*x1 + v[9]*x2  + v[15]*x3  + v[21]*x4 + v[27]*x5 + v[33]*x6;
527       sum5 += v[4]*x1 + v[10]*x2 + v[16]*x3  + v[22]*x4 + v[28]*x5 + v[34]*x6;
528       sum6 += v[5]*x1 + v[11]*x2 + v[17]*x3  + v[23]*x4 + v[29]*x5 + v[35]*x6;
529       v += 36;
530     }
531     if (usecprow) z = zarray + 6*ridx[i];
532     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4; z[4] = sum5; z[5] = sum6;
533     if (!usecprow) z += 6;
534   }
535 
536   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
537   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
538   ierr = PetscLogFlops(72.0*a->nz - 6.0*nonzerorow);CHKERRQ(ierr);
539   PetscFunctionReturn(0);
540 }
541 #undef __FUNCT__
542 #define __FUNCT__ "MatMult_SeqBAIJ_7"
543 PetscErrorCode MatMult_SeqBAIJ_7(Mat A,Vec xx,Vec zz)
544 {
545   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
546   PetscScalar       *z = 0,sum1,sum2,sum3,sum4,sum5,sum6,sum7;
547   const PetscScalar *x,*xb;
548   PetscScalar       x1,x2,x3,x4,x5,x6,x7,*zarray;
549   const MatScalar   *v;
550   PetscErrorCode    ierr;
551   PetscInt          mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL,nonzerorow=0;
552   PetscTruth        usecprow=a->compressedrow.use;
553 
554   PetscFunctionBegin;
555   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
556   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
557 
558   idx = a->j;
559   v   = a->a;
560   if (usecprow){
561     mbs    = a->compressedrow.nrows;
562     ii     = a->compressedrow.i;
563     ridx = a->compressedrow.rindex;
564   } else {
565     mbs = a->mbs;
566     ii  = a->i;
567     z   = zarray;
568   }
569 
570   for (i=0; i<mbs; i++) {
571     n  = ii[1] - ii[0]; ii++;
572     sum1 = 0.0; sum2 = 0.0; sum3 = 0.0; sum4 = 0.0; sum5 = 0.0; sum6 = 0.0; sum7 = 0.0;
573     nonzerorow += (n>0);
574     for (j=0; j<n; j++) {
575       xb = x + 7*(*idx++);
576       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3]; x5 = xb[4]; x6 = xb[5]; x7 = xb[6];
577       sum1 += v[0]*x1 + v[7]*x2  + v[14]*x3  + v[21]*x4 + v[28]*x5 + v[35]*x6 + v[42]*x7;
578       sum2 += v[1]*x1 + v[8]*x2  + v[15]*x3  + v[22]*x4 + v[29]*x5 + v[36]*x6 + v[43]*x7;
579       sum3 += v[2]*x1 + v[9]*x2  + v[16]*x3  + v[23]*x4 + v[30]*x5 + v[37]*x6 + v[44]*x7;
580       sum4 += v[3]*x1 + v[10]*x2 + v[17]*x3  + v[24]*x4 + v[31]*x5 + v[38]*x6 + v[45]*x7;
581       sum5 += v[4]*x1 + v[11]*x2 + v[18]*x3  + v[25]*x4 + v[32]*x5 + v[39]*x6 + v[46]*x7;
582       sum6 += v[5]*x1 + v[12]*x2 + v[19]*x3  + v[26]*x4 + v[33]*x5 + v[40]*x6 + v[47]*x7;
583       sum7 += v[6]*x1 + v[13]*x2 + v[20]*x3  + v[27]*x4 + v[34]*x5 + v[41]*x6 + v[48]*x7;
584       v += 49;
585     }
586     if (usecprow) z = zarray + 7*ridx[i];
587     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4; z[4] = sum5; z[5] = sum6; z[6] = sum7;
588     if (!usecprow) z += 7;
589   }
590 
591   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
592   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
593   ierr = PetscLogFlops(98.0*a->nz - 7.0*nonzerorow);CHKERRQ(ierr);
594   PetscFunctionReturn(0);
595 }
596 
597 /*
598     This will not work with MatScalar == float because it calls the BLAS
599 */
600 #undef __FUNCT__
601 #define __FUNCT__ "MatMult_SeqBAIJ_N"
602 PetscErrorCode MatMult_SeqBAIJ_N(Mat A,Vec xx,Vec zz)
603 {
604   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
605   PetscScalar    *x,*z = 0,*xb,*work,*workt,*zarray;
606   MatScalar      *v;
607   PetscErrorCode ierr;
608   PetscInt       mbs=a->mbs,i,*idx,*ii,bs=A->rmap->bs,j,n,bs2=a->bs2;
609   PetscInt       ncols,k,*ridx=PETSC_NULL,nonzerorow=0;
610   PetscTruth     usecprow=a->compressedrow.use;
611 
612   PetscFunctionBegin;
613   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
614   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
615 
616   idx = a->j;
617   v   = a->a;
618   if (usecprow){
619     mbs  = a->compressedrow.nrows;
620     ii   = a->compressedrow.i;
621     ridx = a->compressedrow.rindex;
622   } else {
623     mbs = a->mbs;
624     ii  = a->i;
625     z   = zarray;
626   }
627 
628   if (!a->mult_work) {
629     k    = PetscMax(A->rmap->n,A->cmap->n);
630     ierr = PetscMalloc((k+1)*sizeof(PetscScalar),&a->mult_work);CHKERRQ(ierr);
631   }
632   work = a->mult_work;
633   for (i=0; i<mbs; i++) {
634     n     = ii[1] - ii[0]; ii++;
635     ncols = n*bs;
636     workt = work;
637     nonzerorow += (n>0);
638     for (j=0; j<n; j++) {
639       xb = x + bs*(*idx++);
640       for (k=0; k<bs; k++) workt[k] = xb[k];
641       workt += bs;
642     }
643     if (usecprow) z = zarray + bs*ridx[i];
644     Kernel_w_gets_Ar_times_v(bs,ncols,work,v,z);
645     /* BLASgemv_("N",&bs,&ncols,&_DOne,v,&bs,work,&_One,&_DZero,z,&_One); */
646     v += n*bs2;
647     if (!usecprow) z += bs;
648   }
649   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
650   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
651   ierr = PetscLogFlops(2.0*a->nz*bs2 - bs*nonzerorow);CHKERRQ(ierr);
652   PetscFunctionReturn(0);
653 }
654 
655 extern PetscErrorCode VecCopy_Seq(Vec,Vec);
656 #undef __FUNCT__
657 #define __FUNCT__ "MatMultAdd_SeqBAIJ_1"
658 PetscErrorCode MatMultAdd_SeqBAIJ_1(Mat A,Vec xx,Vec yy,Vec zz)
659 {
660   Mat_SeqBAIJ        *a = (Mat_SeqBAIJ*)A->data;
661   const PetscScalar  *x;
662   PetscScalar        *y,*z,sum;
663   const MatScalar    *v;
664   PetscErrorCode     ierr;
665   PetscInt           mbs=a->mbs,i,n,*ridx=PETSC_NULL,nonzerorow=0;
666   const PetscInt     *idx,*ii;
667   PetscTruth         usecprow=a->compressedrow.use;
668 
669   PetscFunctionBegin;
670   ierr = VecGetArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
671   ierr = VecGetArray(yy,&y);CHKERRQ(ierr);
672   if (zz != yy) {
673     ierr = VecGetArray(zz,&z);CHKERRQ(ierr);
674   } else {
675     z = y;
676   }
677 
678   idx = a->j;
679   v   = a->a;
680   if (usecprow){
681     if (zz != yy){
682       ierr = PetscMemcpy(z,y,mbs*sizeof(PetscScalar));CHKERRQ(ierr);
683     }
684     mbs  = a->compressedrow.nrows;
685     ii   = a->compressedrow.i;
686     ridx = a->compressedrow.rindex;
687   } else {
688     ii  = a->i;
689   }
690 
691   for (i=0; i<mbs; i++) {
692     n    = ii[1] - ii[0];
693     ii++;
694     if (!usecprow){
695       nonzerorow += (n>0);
696       sum = y[i];
697     } else {
698       sum = y[ridx[i]];
699     }
700     PetscSparseDensePlusDot(sum,x,v,idx,n);
701     v += n;
702     idx += n;
703     if (usecprow){
704       z[ridx[i]] = sum;
705     } else {
706       z[i] = sum;
707     }
708   }
709   ierr = VecRestoreArray(xx,(PetscScalar**)&x);CHKERRQ(ierr);
710   ierr = VecRestoreArray(yy,&y);CHKERRQ(ierr);
711   if (zz != yy) {
712     ierr = VecRestoreArray(zz,&z);CHKERRQ(ierr);
713   }
714   ierr = PetscLogFlops(2.0*a->nz - nonzerorow);CHKERRQ(ierr);
715   PetscFunctionReturn(0);
716 }
717 
718 #undef __FUNCT__
719 #define __FUNCT__ "MatMultAdd_SeqBAIJ_2"
720 PetscErrorCode MatMultAdd_SeqBAIJ_2(Mat A,Vec xx,Vec yy,Vec zz)
721 {
722   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
723   PetscScalar    *x,*y = 0,*z = 0,*xb,sum1,sum2;
724   PetscScalar    x1,x2,*yarray,*zarray;
725   MatScalar      *v;
726   PetscErrorCode ierr;
727   PetscInt       mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL;
728   PetscTruth     usecprow=a->compressedrow.use;
729 
730   PetscFunctionBegin;
731   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
732   ierr = VecGetArray(yy,&yarray);CHKERRQ(ierr);
733   if (zz != yy) {
734     ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
735   } else {
736     zarray = yarray;
737   }
738 
739   idx = a->j;
740   v   = a->a;
741   if (usecprow){
742     if (zz != yy){
743       ierr = PetscMemcpy(zarray,yarray,2*mbs*sizeof(PetscScalar));CHKERRQ(ierr);
744     }
745     mbs  = a->compressedrow.nrows;
746     ii   = a->compressedrow.i;
747     ridx = a->compressedrow.rindex;
748     if (zz != yy){
749       ierr = PetscMemcpy(zarray,yarray,a->mbs*sizeof(PetscScalar));CHKERRQ(ierr);
750     }
751   } else {
752     ii  = a->i;
753     y   = yarray;
754     z   = zarray;
755   }
756 
757   for (i=0; i<mbs; i++) {
758     n  = ii[1] - ii[0]; ii++;
759     if (usecprow){
760       z = zarray + 2*ridx[i];
761       y = yarray + 2*ridx[i];
762     }
763     sum1 = y[0]; sum2 = y[1];
764     for (j=0; j<n; j++) {
765       xb = x + 2*(*idx++); x1 = xb[0]; x2 = xb[1];
766       sum1 += v[0]*x1 + v[2]*x2;
767       sum2 += v[1]*x1 + v[3]*x2;
768       v += 4;
769     }
770     z[0] = sum1; z[1] = sum2;
771     if (!usecprow){
772       z += 2; y += 2;
773     }
774   }
775   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
776   ierr = VecRestoreArray(yy,&yarray);CHKERRQ(ierr);
777   if (zz != yy) {
778     ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
779   }
780   ierr = PetscLogFlops(4.0*a->nz);CHKERRQ(ierr);
781   PetscFunctionReturn(0);
782 }
783 
784 #undef __FUNCT__
785 #define __FUNCT__ "MatMultAdd_SeqBAIJ_3"
786 PetscErrorCode MatMultAdd_SeqBAIJ_3(Mat A,Vec xx,Vec yy,Vec zz)
787 {
788   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
789   PetscScalar    *x,*y = 0,*z = 0,*xb,sum1,sum2,sum3,x1,x2,x3,*yarray,*zarray;
790   MatScalar      *v;
791   PetscErrorCode ierr;
792   PetscInt       mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL;
793   PetscTruth     usecprow=a->compressedrow.use;
794 
795   PetscFunctionBegin;
796   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
797   ierr = VecGetArray(yy,&yarray);CHKERRQ(ierr);
798   if (zz != yy) {
799     ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
800   } else {
801     zarray = yarray;
802   }
803 
804   idx = a->j;
805   v   = a->a;
806   if (usecprow){
807     if (zz != yy){
808       ierr = PetscMemcpy(zarray,yarray,3*mbs*sizeof(PetscScalar));CHKERRQ(ierr);
809     }
810     mbs  = a->compressedrow.nrows;
811     ii   = a->compressedrow.i;
812     ridx = a->compressedrow.rindex;
813   } else {
814     ii  = a->i;
815     y   = yarray;
816     z   = zarray;
817   }
818 
819   for (i=0; i<mbs; i++) {
820     n  = ii[1] - ii[0]; ii++;
821     if (usecprow){
822       z = zarray + 3*ridx[i];
823       y = yarray + 3*ridx[i];
824     }
825     sum1 = y[0]; sum2 = y[1]; sum3 = y[2];
826     for (j=0; j<n; j++) {
827       xb = x + 3*(*idx++); x1 = xb[0]; x2 = xb[1]; x3 = xb[2];
828       sum1 += v[0]*x1 + v[3]*x2 + v[6]*x3;
829       sum2 += v[1]*x1 + v[4]*x2 + v[7]*x3;
830       sum3 += v[2]*x1 + v[5]*x2 + v[8]*x3;
831       v += 9;
832     }
833     z[0] = sum1; z[1] = sum2; z[2] = sum3;
834     if (!usecprow){
835       z += 3; y += 3;
836     }
837   }
838   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
839   ierr = VecRestoreArray(yy,&yarray);CHKERRQ(ierr);
840   if (zz != yy) {
841     ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
842   }
843   ierr = PetscLogFlops(18.0*a->nz);CHKERRQ(ierr);
844   PetscFunctionReturn(0);
845 }
846 
847 #undef __FUNCT__
848 #define __FUNCT__ "MatMultAdd_SeqBAIJ_4"
849 PetscErrorCode MatMultAdd_SeqBAIJ_4(Mat A,Vec xx,Vec yy,Vec zz)
850 {
851   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
852   PetscScalar    *x,*y = 0,*z = 0,*xb,sum1,sum2,sum3,sum4,x1,x2,x3,x4,*yarray,*zarray;
853   MatScalar      *v;
854   PetscErrorCode ierr;
855   PetscInt       mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL;
856   PetscTruth     usecprow=a->compressedrow.use;
857 
858   PetscFunctionBegin;
859   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
860   ierr = VecGetArray(yy,&yarray);CHKERRQ(ierr);
861   if (zz != yy) {
862     ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
863   } else {
864     zarray = yarray;
865   }
866 
867   idx   = a->j;
868   v     = a->a;
869   if (usecprow){
870     if (zz != yy){
871       ierr = PetscMemcpy(zarray,yarray,4*mbs*sizeof(PetscScalar));CHKERRQ(ierr);
872     }
873     mbs  = a->compressedrow.nrows;
874     ii   = a->compressedrow.i;
875     ridx = a->compressedrow.rindex;
876   } else {
877     ii  = a->i;
878     y   = yarray;
879     z   = zarray;
880   }
881 
882   for (i=0; i<mbs; i++) {
883     n  = ii[1] - ii[0]; ii++;
884     if (usecprow){
885       z = zarray + 4*ridx[i];
886       y = yarray + 4*ridx[i];
887     }
888     sum1 = y[0]; sum2 = y[1]; sum3 = y[2]; sum4 = y[3];
889     for (j=0; j<n; j++) {
890       xb = x + 4*(*idx++);
891       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3];
892       sum1 += v[0]*x1 + v[4]*x2 + v[8]*x3   + v[12]*x4;
893       sum2 += v[1]*x1 + v[5]*x2 + v[9]*x3   + v[13]*x4;
894       sum3 += v[2]*x1 + v[6]*x2 + v[10]*x3  + v[14]*x4;
895       sum4 += v[3]*x1 + v[7]*x2 + v[11]*x3  + v[15]*x4;
896       v += 16;
897     }
898     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4;
899     if (!usecprow){
900       z += 4; y += 4;
901     }
902   }
903   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
904   ierr = VecRestoreArray(yy,&yarray);CHKERRQ(ierr);
905   if (zz != yy) {
906     ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
907   }
908   ierr = PetscLogFlops(32.0*a->nz);CHKERRQ(ierr);
909   PetscFunctionReturn(0);
910 }
911 
912 #undef __FUNCT__
913 #define __FUNCT__ "MatMultAdd_SeqBAIJ_5"
914 PetscErrorCode MatMultAdd_SeqBAIJ_5(Mat A,Vec xx,Vec yy,Vec zz)
915 {
916   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
917   PetscScalar    *x,*y = 0,*z = 0,*xb,sum1,sum2,sum3,sum4,sum5,x1,x2,x3,x4,x5;
918   PetscScalar    *yarray,*zarray;
919   MatScalar      *v;
920   PetscErrorCode ierr;
921   PetscInt       mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL;
922   PetscTruth     usecprow=a->compressedrow.use;
923 
924   PetscFunctionBegin;
925   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
926   ierr = VecGetArray(yy,&yarray);CHKERRQ(ierr);
927   if (zz != yy) {
928     ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
929   } else {
930     zarray = yarray;
931   }
932 
933   idx = a->j;
934   v   = a->a;
935   if (usecprow){
936     if (zz != yy){
937       ierr = PetscMemcpy(zarray,yarray,5*mbs*sizeof(PetscScalar));CHKERRQ(ierr);
938     }
939     mbs  = a->compressedrow.nrows;
940     ii   = a->compressedrow.i;
941     ridx = a->compressedrow.rindex;
942   } else {
943     ii  = a->i;
944     y   = yarray;
945     z   = zarray;
946   }
947 
948   for (i=0; i<mbs; i++) {
949     n  = ii[1] - ii[0]; ii++;
950     if (usecprow){
951       z = zarray + 5*ridx[i];
952       y = yarray + 5*ridx[i];
953     }
954     sum1 = y[0]; sum2 = y[1]; sum3 = y[2]; sum4 = y[3]; sum5 = y[4];
955     for (j=0; j<n; j++) {
956       xb = x + 5*(*idx++);
957       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3]; x5 = xb[4];
958       sum1 += v[0]*x1 + v[5]*x2 + v[10]*x3  + v[15]*x4 + v[20]*x5;
959       sum2 += v[1]*x1 + v[6]*x2 + v[11]*x3  + v[16]*x4 + v[21]*x5;
960       sum3 += v[2]*x1 + v[7]*x2 + v[12]*x3  + v[17]*x4 + v[22]*x5;
961       sum4 += v[3]*x1 + v[8]*x2 + v[13]*x3  + v[18]*x4 + v[23]*x5;
962       sum5 += v[4]*x1 + v[9]*x2 + v[14]*x3  + v[19]*x4 + v[24]*x5;
963       v += 25;
964     }
965     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4; z[4] = sum5;
966     if (!usecprow){
967       z += 5; y += 5;
968     }
969   }
970   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
971   ierr = VecRestoreArray(yy,&yarray);CHKERRQ(ierr);
972   if (zz != yy) {
973     ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
974   }
975   ierr = PetscLogFlops(50.0*a->nz);CHKERRQ(ierr);
976   PetscFunctionReturn(0);
977 }
978 #undef __FUNCT__
979 #define __FUNCT__ "MatMultAdd_SeqBAIJ_6"
980 PetscErrorCode MatMultAdd_SeqBAIJ_6(Mat A,Vec xx,Vec yy,Vec zz)
981 {
982   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
983   PetscScalar    *x,*y = 0,*z = 0,*xb,sum1,sum2,sum3,sum4,sum5,sum6;
984   PetscScalar    x1,x2,x3,x4,x5,x6,*yarray,*zarray;
985   MatScalar      *v;
986   PetscErrorCode ierr;
987   PetscInt       mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL;
988   PetscTruth     usecprow=a->compressedrow.use;
989 
990   PetscFunctionBegin;
991   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
992   ierr = VecGetArray(yy,&yarray);CHKERRQ(ierr);
993   if (zz != yy) {
994     ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
995   } else {
996     zarray = yarray;
997   }
998 
999   idx = a->j;
1000   v   = a->a;
1001   if (usecprow){
1002     if (zz != yy){
1003       ierr = PetscMemcpy(zarray,yarray,6*mbs*sizeof(PetscScalar));CHKERRQ(ierr);
1004     }
1005     mbs  = a->compressedrow.nrows;
1006     ii   = a->compressedrow.i;
1007     ridx = a->compressedrow.rindex;
1008   } else {
1009     ii  = a->i;
1010     y   = yarray;
1011     z   = zarray;
1012   }
1013 
1014   for (i=0; i<mbs; i++) {
1015     n  = ii[1] - ii[0]; ii++;
1016     if (usecprow){
1017       z = zarray + 6*ridx[i];
1018       y = yarray + 6*ridx[i];
1019     }
1020     sum1 = y[0]; sum2 = y[1]; sum3 = y[2]; sum4 = y[3]; sum5 = y[4]; sum6 = y[5];
1021     for (j=0; j<n; j++) {
1022       xb = x + 6*(*idx++);
1023       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3]; x5 = xb[4]; x6 = xb[5];
1024       sum1 += v[0]*x1 + v[6]*x2  + v[12]*x3  + v[18]*x4 + v[24]*x5 + v[30]*x6;
1025       sum2 += v[1]*x1 + v[7]*x2  + v[13]*x3  + v[19]*x4 + v[25]*x5 + v[31]*x6;
1026       sum3 += v[2]*x1 + v[8]*x2  + v[14]*x3  + v[20]*x4 + v[26]*x5 + v[32]*x6;
1027       sum4 += v[3]*x1 + v[9]*x2  + v[15]*x3  + v[21]*x4 + v[27]*x5 + v[33]*x6;
1028       sum5 += v[4]*x1 + v[10]*x2 + v[16]*x3  + v[22]*x4 + v[28]*x5 + v[34]*x6;
1029       sum6 += v[5]*x1 + v[11]*x2 + v[17]*x3  + v[23]*x4 + v[29]*x5 + v[35]*x6;
1030       v += 36;
1031     }
1032     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4; z[4] = sum5; z[5] = sum6;
1033     if (!usecprow){
1034       z += 6; y += 6;
1035     }
1036   }
1037   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
1038   ierr = VecRestoreArray(yy,&yarray);CHKERRQ(ierr);
1039   if (zz != yy) {
1040     ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
1041   }
1042   ierr = PetscLogFlops(72.0*a->nz);CHKERRQ(ierr);
1043   PetscFunctionReturn(0);
1044 }
1045 
1046 #undef __FUNCT__
1047 #define __FUNCT__ "MatMultAdd_SeqBAIJ_7"
1048 PetscErrorCode MatMultAdd_SeqBAIJ_7(Mat A,Vec xx,Vec yy,Vec zz)
1049 {
1050   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
1051   PetscScalar    *x,*y = 0,*z = 0,*xb,sum1,sum2,sum3,sum4,sum5,sum6,sum7;
1052   PetscScalar    x1,x2,x3,x4,x5,x6,x7,*yarray,*zarray;
1053   MatScalar      *v;
1054   PetscErrorCode ierr;
1055   PetscInt       mbs=a->mbs,i,*idx,*ii,j,n,*ridx=PETSC_NULL;
1056   PetscTruth     usecprow=a->compressedrow.use;
1057 
1058   PetscFunctionBegin;
1059   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
1060   ierr = VecGetArray(yy,&yarray);CHKERRQ(ierr);
1061   if (zz != yy) {
1062     ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
1063   } else {
1064     zarray = yarray;
1065   }
1066 
1067   idx = a->j;
1068   v   = a->a;
1069   if (usecprow){
1070     if (zz != yy){
1071       ierr = PetscMemcpy(zarray,yarray,7*mbs*sizeof(PetscScalar));CHKERRQ(ierr);
1072     }
1073     mbs  = a->compressedrow.nrows;
1074     ii   = a->compressedrow.i;
1075     ridx = a->compressedrow.rindex;
1076   } else {
1077     ii  = a->i;
1078     y   = yarray;
1079     z   = zarray;
1080   }
1081 
1082   for (i=0; i<mbs; i++) {
1083     n  = ii[1] - ii[0]; ii++;
1084     if (usecprow){
1085       z = zarray + 7*ridx[i];
1086       y = yarray + 7*ridx[i];
1087     }
1088     sum1 = y[0]; sum2 = y[1]; sum3 = y[2]; sum4 = y[3]; sum5 = y[4]; sum6 = y[5]; sum7 = y[6];
1089     for (j=0; j<n; j++) {
1090       xb = x + 7*(*idx++);
1091       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3]; x5 = xb[4]; x6 = xb[5]; x7 = xb[6];
1092       sum1 += v[0]*x1 + v[7]*x2  + v[14]*x3  + v[21]*x4 + v[28]*x5 + v[35]*x6 + v[42]*x7;
1093       sum2 += v[1]*x1 + v[8]*x2  + v[15]*x3  + v[22]*x4 + v[29]*x5 + v[36]*x6 + v[43]*x7;
1094       sum3 += v[2]*x1 + v[9]*x2  + v[16]*x3  + v[23]*x4 + v[30]*x5 + v[37]*x6 + v[44]*x7;
1095       sum4 += v[3]*x1 + v[10]*x2 + v[17]*x3  + v[24]*x4 + v[31]*x5 + v[38]*x6 + v[45]*x7;
1096       sum5 += v[4]*x1 + v[11]*x2 + v[18]*x3  + v[25]*x4 + v[32]*x5 + v[39]*x6 + v[46]*x7;
1097       sum6 += v[5]*x1 + v[12]*x2 + v[19]*x3  + v[26]*x4 + v[33]*x5 + v[40]*x6 + v[47]*x7;
1098       sum7 += v[6]*x1 + v[13]*x2 + v[20]*x3  + v[27]*x4 + v[34]*x5 + v[41]*x6 + v[48]*x7;
1099       v += 49;
1100     }
1101     z[0] = sum1; z[1] = sum2; z[2] = sum3; z[3] = sum4; z[4] = sum5; z[5] = sum6; z[6] = sum7;
1102     if (!usecprow){
1103       z += 7; y += 7;
1104     }
1105   }
1106   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
1107   ierr = VecRestoreArray(yy,&yarray);CHKERRQ(ierr);
1108   if (zz != yy) {
1109     ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
1110   }
1111   ierr = PetscLogFlops(98.0*a->nz);CHKERRQ(ierr);
1112   PetscFunctionReturn(0);
1113 }
1114 
1115 #undef __FUNCT__
1116 #define __FUNCT__ "MatMultAdd_SeqBAIJ_N"
1117 PetscErrorCode MatMultAdd_SeqBAIJ_N(Mat A,Vec xx,Vec yy,Vec zz)
1118 {
1119   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
1120   PetscScalar    *x,*z = 0,*xb,*work,*workt,*zarray;
1121   MatScalar      *v;
1122   PetscErrorCode ierr;
1123   PetscInt       mbs,i,*idx,*ii,bs=A->rmap->bs,j,n,bs2=a->bs2;
1124   PetscInt       ncols,k,*ridx=PETSC_NULL;
1125   PetscTruth     usecprow=a->compressedrow.use;
1126 
1127   PetscFunctionBegin;
1128   ierr = VecCopy_Seq(yy,zz);CHKERRQ(ierr);
1129   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
1130   ierr = VecGetArray(zz,&zarray);CHKERRQ(ierr);
1131 
1132   idx = a->j;
1133   v   = a->a;
1134   if (usecprow){
1135     mbs    = a->compressedrow.nrows;
1136     ii     = a->compressedrow.i;
1137     ridx = a->compressedrow.rindex;
1138   } else {
1139     mbs = a->mbs;
1140     ii  = a->i;
1141     z   = zarray;
1142   }
1143 
1144   if (!a->mult_work) {
1145     k    = PetscMax(A->rmap->n,A->cmap->n);
1146     ierr = PetscMalloc((k+1)*sizeof(PetscScalar),&a->mult_work);CHKERRQ(ierr);
1147   }
1148   work = a->mult_work;
1149   for (i=0; i<mbs; i++) {
1150     n     = ii[1] - ii[0]; ii++;
1151     ncols = n*bs;
1152     workt = work;
1153     for (j=0; j<n; j++) {
1154       xb = x + bs*(*idx++);
1155       for (k=0; k<bs; k++) workt[k] = xb[k];
1156       workt += bs;
1157     }
1158     if (usecprow) z = zarray + bs*ridx[i];
1159     Kernel_w_gets_w_plus_Ar_times_v(bs,ncols,work,v,z);
1160     /* BLASgemv_("N",&bs,&ncols,&_DOne,v,&bs,work,&_One,&_DOne,z,&_One); */
1161     v += n*bs2;
1162     if (!usecprow){
1163       z += bs;
1164     }
1165   }
1166   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
1167   ierr = VecRestoreArray(zz,&zarray);CHKERRQ(ierr);
1168   ierr = PetscLogFlops(2.0*a->nz*bs2);CHKERRQ(ierr);
1169   PetscFunctionReturn(0);
1170 }
1171 
1172 #undef __FUNCT__
1173 #define __FUNCT__ "MatMultHermitianTranspose_SeqBAIJ"
1174 PetscErrorCode MatMultHermitianTranspose_SeqBAIJ(Mat A,Vec xx,Vec zz)
1175 {
1176   PetscScalar    zero = 0.0;
1177   PetscErrorCode ierr;
1178 
1179   PetscFunctionBegin;
1180   ierr = VecSet(zz,zero);CHKERRQ(ierr);
1181   ierr = MatMultHermitianTransposeAdd_SeqBAIJ(A,xx,zz,zz);CHKERRQ(ierr);
1182   PetscFunctionReturn(0);
1183 }
1184 
1185 #undef __FUNCT__
1186 #define __FUNCT__ "MatMultTranspose_SeqBAIJ"
1187 PetscErrorCode MatMultTranspose_SeqBAIJ(Mat A,Vec xx,Vec zz)
1188 {
1189   PetscScalar    zero = 0.0;
1190   PetscErrorCode ierr;
1191 
1192   PetscFunctionBegin;
1193   ierr = VecSet(zz,zero);CHKERRQ(ierr);
1194   ierr = MatMultTransposeAdd_SeqBAIJ(A,xx,zz,zz);CHKERRQ(ierr);
1195   PetscFunctionReturn(0);
1196 }
1197 
1198 #undef __FUNCT__
1199 #define __FUNCT__ "MatMultHermitianTransposeAdd_SeqBAIJ"
1200 PetscErrorCode MatMultHermitianTransposeAdd_SeqBAIJ(Mat A,Vec xx,Vec yy,Vec zz)
1201 
1202 {
1203   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
1204   PetscScalar       *zb,*x,*z,*xb = 0,x1,x2,x3,x4,x5;
1205   MatScalar         *v;
1206   PetscErrorCode    ierr;
1207   PetscInt          mbs,i,*idx,*ii,rval,bs=A->rmap->bs,j,n,bs2=a->bs2,*ib,*ridx=PETSC_NULL;
1208   Mat_CompressedRow cprow = a->compressedrow;
1209   PetscTruth        usecprow=cprow.use;
1210 
1211   PetscFunctionBegin;
1212   if (yy != zz) { ierr = VecCopy_Seq(yy,zz);CHKERRQ(ierr); }
1213   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
1214   ierr = VecGetArray(zz,&z);CHKERRQ(ierr);
1215 
1216   idx = a->j;
1217   v   = a->a;
1218   if (usecprow){
1219     mbs  = cprow.nrows;
1220     ii   = cprow.i;
1221     ridx = cprow.rindex;
1222   } else {
1223     mbs=a->mbs;
1224     ii = a->i;
1225     xb = x;
1226   }
1227 
1228   switch (bs) {
1229   case 1:
1230     for (i=0; i<mbs; i++) {
1231       if (usecprow) xb = x + ridx[i];
1232       x1 = xb[0];
1233       ib = idx + ii[0];
1234       n  = ii[1] - ii[0]; ii++;
1235       for (j=0; j<n; j++) {
1236         rval    = ib[j];
1237         z[rval] += PetscConj(*v) * x1;
1238         v++;
1239       }
1240       if (!usecprow) xb++;
1241     }
1242     break;
1243   case 2:
1244     for (i=0; i<mbs; i++) {
1245       if (usecprow) xb = x + 2*ridx[i];
1246       x1 = xb[0]; x2 = xb[1];
1247       ib = idx + ii[0];
1248       n  = ii[1] - ii[0]; ii++;
1249       for (j=0; j<n; j++) {
1250         rval      = ib[j]*2;
1251         z[rval++] += PetscConj(v[0])*x1 + PetscConj(v[1])*x2;
1252         z[rval++] += PetscConj(v[2])*x1 + PetscConj(v[3])*x2;
1253         v  += 4;
1254       }
1255       if (!usecprow) xb += 2;
1256     }
1257     break;
1258   case 3:
1259     for (i=0; i<mbs; i++) {
1260       if (usecprow) xb = x + 3*ridx[i];
1261       x1 = xb[0]; x2 = xb[1]; x3 = xb[2];
1262       ib = idx + ii[0];
1263       n  = ii[1] - ii[0]; ii++;
1264       for (j=0; j<n; j++) {
1265         rval      = ib[j]*3;
1266         z[rval++] += PetscConj(v[0])*x1 + PetscConj(v[1])*x2 + PetscConj(v[2])*x3;
1267         z[rval++] += PetscConj(v[3])*x1 + PetscConj(v[4])*x2 + PetscConj(v[5])*x3;
1268         z[rval++] += PetscConj(v[6])*x1 + PetscConj(v[7])*x2 + PetscConj(v[8])*x3;
1269         v  += 9;
1270       }
1271       if (!usecprow) xb += 3;
1272     }
1273     break;
1274   case 4:
1275     for (i=0; i<mbs; i++) {
1276       if (usecprow) xb = x + 4*ridx[i];
1277       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3];
1278       ib = idx + ii[0];
1279       n  = ii[1] - ii[0]; ii++;
1280       for (j=0; j<n; j++) {
1281         rval      = ib[j]*4;
1282         z[rval++] +=  PetscConj(v[0])*x1 + PetscConj(v[1])*x2  + PetscConj(v[2])*x3  + PetscConj(v[3])*x4;
1283         z[rval++] +=  PetscConj(v[4])*x1 + PetscConj(v[5])*x2  + PetscConj(v[6])*x3  + PetscConj(v[7])*x4;
1284         z[rval++] +=  PetscConj(v[8])*x1 + PetscConj(v[9])*x2  + PetscConj(v[10])*x3 + PetscConj(v[11])*x4;
1285         z[rval++] += PetscConj(v[12])*x1 + PetscConj(v[13])*x2 + PetscConj(v[14])*x3 + PetscConj(v[15])*x4;
1286         v  += 16;
1287       }
1288       if (!usecprow) xb += 4;
1289     }
1290     break;
1291   case 5:
1292     for (i=0; i<mbs; i++) {
1293       if (usecprow) xb = x + 5*ridx[i];
1294       x1 = xb[0]; x2 = xb[1]; x3 = xb[2];
1295       x4 = xb[3]; x5 = xb[4];
1296       ib = idx + ii[0];
1297       n  = ii[1] - ii[0]; ii++;
1298       for (j=0; j<n; j++) {
1299         rval      = ib[j]*5;
1300         z[rval++] +=  PetscConj(v[0])*x1 +  PetscConj(v[1])*x2 +  PetscConj(v[2])*x3 +  PetscConj(v[3])*x4 +  PetscConj(v[4])*x5;
1301         z[rval++] +=  PetscConj(v[5])*x1 +  PetscConj(v[6])*x2 +  PetscConj(v[7])*x3 +  PetscConj(v[8])*x4 +  PetscConj(v[9])*x5;
1302         z[rval++] += PetscConj(v[10])*x1 + PetscConj(v[11])*x2 + PetscConj(v[12])*x3 + PetscConj(v[13])*x4 + PetscConj(v[14])*x5;
1303         z[rval++] += PetscConj(v[15])*x1 + PetscConj(v[16])*x2 + PetscConj(v[17])*x3 + PetscConj(v[18])*x4 + PetscConj(v[19])*x5;
1304         z[rval++] += PetscConj(v[20])*x1 + PetscConj(v[21])*x2 + PetscConj(v[22])*x3 + PetscConj(v[23])*x4 + PetscConj(v[24])*x5;
1305         v  += 25;
1306       }
1307       if (!usecprow) xb += 5;
1308     }
1309     break;
1310   default: {      /* block sizes larger than 5 by 5 are handled by BLAS */
1311       PetscInt     ncols,k;
1312       PetscScalar  *work,*workt,*xtmp;
1313 
1314       SETERRQ(PETSC_ERR_SUP,"block size larger than 5 is not supported yet");
1315       if (!a->mult_work) {
1316         k = PetscMax(A->rmap->n,A->cmap->n);
1317         ierr = PetscMalloc((k+1)*sizeof(PetscScalar),&a->mult_work);CHKERRQ(ierr);
1318       }
1319       work = a->mult_work;
1320       xtmp = x;
1321       for (i=0; i<mbs; i++) {
1322         n     = ii[1] - ii[0]; ii++;
1323         ncols = n*bs;
1324         ierr  = PetscMemzero(work,ncols*sizeof(PetscScalar));CHKERRQ(ierr);
1325         if (usecprow) {
1326           xtmp = x + bs*ridx[i];
1327         }
1328         Kernel_w_gets_w_plus_trans_Ar_times_v(bs,ncols,xtmp,v,work);
1329         /* BLASgemv_("T",&bs,&ncols,&_DOne,v,&bs,xtmp,&_One,&_DOne,work,&_One); */
1330         v += n*bs2;
1331         if (!usecprow) xtmp += bs;
1332         workt = work;
1333         for (j=0; j<n; j++) {
1334           zb = z + bs*(*idx++);
1335           for (k=0; k<bs; k++) zb[k] += workt[k] ;
1336           workt += bs;
1337         }
1338       }
1339     }
1340   }
1341   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
1342   ierr = VecRestoreArray(zz,&z);CHKERRQ(ierr);
1343   ierr = PetscLogFlops(2.0*a->nz*a->bs2);CHKERRQ(ierr);
1344   PetscFunctionReturn(0);
1345 }
1346 
1347 #undef __FUNCT__
1348 #define __FUNCT__ "MatMultTransposeAdd_SeqBAIJ"
1349 PetscErrorCode MatMultTransposeAdd_SeqBAIJ(Mat A,Vec xx,Vec yy,Vec zz)
1350 
1351 {
1352   Mat_SeqBAIJ       *a = (Mat_SeqBAIJ*)A->data;
1353   PetscScalar       *zb,*x,*z,*xb = 0,x1,x2,x3,x4,x5;
1354   MatScalar         *v;
1355   PetscErrorCode    ierr;
1356   PetscInt          mbs,i,*idx,*ii,rval,bs=A->rmap->bs,j,n,bs2=a->bs2,*ib,*ridx=PETSC_NULL;
1357   Mat_CompressedRow cprow = a->compressedrow;
1358   PetscTruth        usecprow=cprow.use;
1359 
1360   PetscFunctionBegin;
1361   if (yy != zz) { ierr = VecCopy_Seq(yy,zz);CHKERRQ(ierr); }
1362   ierr = VecGetArray(xx,&x);CHKERRQ(ierr);
1363   ierr = VecGetArray(zz,&z);CHKERRQ(ierr);
1364 
1365   idx = a->j;
1366   v   = a->a;
1367   if (usecprow){
1368     mbs  = cprow.nrows;
1369     ii   = cprow.i;
1370     ridx = cprow.rindex;
1371   } else {
1372     mbs=a->mbs;
1373     ii = a->i;
1374     xb = x;
1375   }
1376 
1377   switch (bs) {
1378   case 1:
1379     for (i=0; i<mbs; i++) {
1380       if (usecprow) xb = x + ridx[i];
1381       x1 = xb[0];
1382       ib = idx + ii[0];
1383       n  = ii[1] - ii[0]; ii++;
1384       for (j=0; j<n; j++) {
1385         rval    = ib[j];
1386         z[rval] += *v * x1;
1387         v++;
1388       }
1389       if (!usecprow) xb++;
1390     }
1391     break;
1392   case 2:
1393     for (i=0; i<mbs; i++) {
1394       if (usecprow) xb = x + 2*ridx[i];
1395       x1 = xb[0]; x2 = xb[1];
1396       ib = idx + ii[0];
1397       n  = ii[1] - ii[0]; ii++;
1398       for (j=0; j<n; j++) {
1399         rval      = ib[j]*2;
1400         z[rval++] += v[0]*x1 + v[1]*x2;
1401         z[rval++] += v[2]*x1 + v[3]*x2;
1402         v  += 4;
1403       }
1404       if (!usecprow) xb += 2;
1405     }
1406     break;
1407   case 3:
1408     for (i=0; i<mbs; i++) {
1409       if (usecprow) xb = x + 3*ridx[i];
1410       x1 = xb[0]; x2 = xb[1]; x3 = xb[2];
1411       ib = idx + ii[0];
1412       n  = ii[1] - ii[0]; ii++;
1413       for (j=0; j<n; j++) {
1414         rval      = ib[j]*3;
1415         z[rval++] += v[0]*x1 + v[1]*x2 + v[2]*x3;
1416         z[rval++] += v[3]*x1 + v[4]*x2 + v[5]*x3;
1417         z[rval++] += v[6]*x1 + v[7]*x2 + v[8]*x3;
1418         v  += 9;
1419       }
1420       if (!usecprow) xb += 3;
1421     }
1422     break;
1423   case 4:
1424     for (i=0; i<mbs; i++) {
1425       if (usecprow) xb = x + 4*ridx[i];
1426       x1 = xb[0]; x2 = xb[1]; x3 = xb[2]; x4 = xb[3];
1427       ib = idx + ii[0];
1428       n  = ii[1] - ii[0]; ii++;
1429       for (j=0; j<n; j++) {
1430         rval      = ib[j]*4;
1431         z[rval++] +=  v[0]*x1 +  v[1]*x2 +  v[2]*x3 +  v[3]*x4;
1432         z[rval++] +=  v[4]*x1 +  v[5]*x2 +  v[6]*x3 +  v[7]*x4;
1433         z[rval++] +=  v[8]*x1 +  v[9]*x2 + v[10]*x3 + v[11]*x4;
1434         z[rval++] += v[12]*x1 + v[13]*x2 + v[14]*x3 + v[15]*x4;
1435         v  += 16;
1436       }
1437       if (!usecprow) xb += 4;
1438     }
1439     break;
1440   case 5:
1441     for (i=0; i<mbs; i++) {
1442       if (usecprow) xb = x + 5*ridx[i];
1443       x1 = xb[0]; x2 = xb[1]; x3 = xb[2];
1444       x4 = xb[3]; x5 = xb[4];
1445       ib = idx + ii[0];
1446       n  = ii[1] - ii[0]; ii++;
1447       for (j=0; j<n; j++) {
1448         rval      = ib[j]*5;
1449         z[rval++] +=  v[0]*x1 +  v[1]*x2 +  v[2]*x3 +  v[3]*x4 +  v[4]*x5;
1450         z[rval++] +=  v[5]*x1 +  v[6]*x2 +  v[7]*x3 +  v[8]*x4 +  v[9]*x5;
1451         z[rval++] += v[10]*x1 + v[11]*x2 + v[12]*x3 + v[13]*x4 + v[14]*x5;
1452         z[rval++] += v[15]*x1 + v[16]*x2 + v[17]*x3 + v[18]*x4 + v[19]*x5;
1453         z[rval++] += v[20]*x1 + v[21]*x2 + v[22]*x3 + v[23]*x4 + v[24]*x5;
1454         v  += 25;
1455       }
1456       if (!usecprow) xb += 5;
1457     }
1458     break;
1459   default: {      /* block sizes larger then 5 by 5 are handled by BLAS */
1460       PetscInt     ncols,k;
1461       PetscScalar  *work,*workt,*xtmp;
1462 
1463       if (!a->mult_work) {
1464         k = PetscMax(A->rmap->n,A->cmap->n);
1465         ierr = PetscMalloc((k+1)*sizeof(PetscScalar),&a->mult_work);CHKERRQ(ierr);
1466       }
1467       work = a->mult_work;
1468       xtmp = x;
1469       for (i=0; i<mbs; i++) {
1470         n     = ii[1] - ii[0]; ii++;
1471         ncols = n*bs;
1472         ierr  = PetscMemzero(work,ncols*sizeof(PetscScalar));CHKERRQ(ierr);
1473         if (usecprow) {
1474           xtmp = x + bs*ridx[i];
1475         }
1476         Kernel_w_gets_w_plus_trans_Ar_times_v(bs,ncols,xtmp,v,work);
1477         /* BLASgemv_("T",&bs,&ncols,&_DOne,v,&bs,xtmp,&_One,&_DOne,work,&_One); */
1478         v += n*bs2;
1479         if (!usecprow) xtmp += bs;
1480         workt = work;
1481         for (j=0; j<n; j++) {
1482           zb = z + bs*(*idx++);
1483           for (k=0; k<bs; k++) zb[k] += workt[k] ;
1484           workt += bs;
1485         }
1486       }
1487     }
1488   }
1489   ierr = VecRestoreArray(xx,&x);CHKERRQ(ierr);
1490   ierr = VecRestoreArray(zz,&z);CHKERRQ(ierr);
1491   ierr = PetscLogFlops(2.0*a->nz*a->bs2);CHKERRQ(ierr);
1492   PetscFunctionReturn(0);
1493 }
1494 
1495 #undef __FUNCT__
1496 #define __FUNCT__ "MatScale_SeqBAIJ"
1497 PetscErrorCode MatScale_SeqBAIJ(Mat inA,PetscScalar alpha)
1498 {
1499   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)inA->data;
1500   PetscInt       totalnz = a->bs2*a->nz;
1501   PetscScalar    oalpha = alpha;
1502   PetscErrorCode ierr;
1503   PetscBLASInt   one = 1,tnz = PetscBLASIntCast(totalnz);
1504 
1505   PetscFunctionBegin;
1506   BLASscal_(&tnz,&oalpha,a->a,&one);
1507   ierr = PetscLogFlops(totalnz);CHKERRQ(ierr);
1508   PetscFunctionReturn(0);
1509 }
1510 
1511 #undef __FUNCT__
1512 #define __FUNCT__ "MatNorm_SeqBAIJ"
1513 PetscErrorCode MatNorm_SeqBAIJ(Mat A,NormType type,PetscReal *norm)
1514 {
1515   PetscErrorCode ierr;
1516   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
1517   MatScalar      *v = a->a;
1518   PetscReal      sum = 0.0;
1519   PetscInt       i,j,k,bs=A->rmap->bs,nz=a->nz,bs2=a->bs2,k1;
1520 
1521   PetscFunctionBegin;
1522   if (type == NORM_FROBENIUS) {
1523     for (i=0; i< bs2*nz; i++) {
1524 #if defined(PETSC_USE_COMPLEX)
1525       sum += PetscRealPart(PetscConj(*v)*(*v)); v++;
1526 #else
1527       sum += (*v)*(*v); v++;
1528 #endif
1529     }
1530     *norm = sqrt(sum);
1531   } else if (type == NORM_1) { /* maximum column sum */
1532     PetscReal *tmp;
1533     PetscInt  *bcol = a->j;
1534     ierr = PetscMalloc((A->cmap->n+1)*sizeof(PetscReal),&tmp);CHKERRQ(ierr);
1535     ierr = PetscMemzero(tmp,A->cmap->n*sizeof(PetscReal));CHKERRQ(ierr);
1536     for (i=0; i<nz; i++){
1537       for (j=0; j<bs; j++){
1538         k1 = bs*(*bcol) + j; /* column index */
1539         for (k=0; k<bs; k++){
1540           tmp[k1] += PetscAbsScalar(*v); v++;
1541         }
1542       }
1543       bcol++;
1544     }
1545     *norm = 0.0;
1546     for (j=0; j<A->cmap->n; j++) {
1547       if (tmp[j] > *norm) *norm = tmp[j];
1548     }
1549     ierr = PetscFree(tmp);CHKERRQ(ierr);
1550   } else if (type == NORM_INFINITY) { /* maximum row sum */
1551     *norm = 0.0;
1552     for (k=0; k<bs; k++) {
1553       for (j=0; j<a->mbs; j++) {
1554         v = a->a + bs2*a->i[j] + k;
1555         sum = 0.0;
1556         for (i=0; i<a->i[j+1]-a->i[j]; i++) {
1557           for (k1=0; k1<bs; k1++){
1558             sum += PetscAbsScalar(*v);
1559             v   += bs;
1560           }
1561         }
1562         if (sum > *norm) *norm = sum;
1563       }
1564     }
1565   } else {
1566     SETERRQ(PETSC_ERR_SUP,"No support for this norm yet");
1567   }
1568   PetscFunctionReturn(0);
1569 }
1570 
1571 
1572 #undef __FUNCT__
1573 #define __FUNCT__ "MatEqual_SeqBAIJ"
1574 PetscErrorCode MatEqual_SeqBAIJ(Mat A,Mat B,PetscTruth* flg)
1575 {
1576   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ *)A->data,*b = (Mat_SeqBAIJ *)B->data;
1577   PetscErrorCode ierr;
1578 
1579   PetscFunctionBegin;
1580   /* If the  matrix/block dimensions are not equal, or no of nonzeros or shift */
1581   if ((A->rmap->N != B->rmap->N) || (A->cmap->n != B->cmap->n) || (A->rmap->bs != B->rmap->bs)|| (a->nz != b->nz)) {
1582     *flg = PETSC_FALSE;
1583     PetscFunctionReturn(0);
1584   }
1585 
1586   /* if the a->i are the same */
1587   ierr = PetscMemcmp(a->i,b->i,(a->mbs+1)*sizeof(PetscInt),flg);CHKERRQ(ierr);
1588   if (!*flg) {
1589     PetscFunctionReturn(0);
1590   }
1591 
1592   /* if a->j are the same */
1593   ierr = PetscMemcmp(a->j,b->j,(a->nz)*sizeof(PetscInt),flg);CHKERRQ(ierr);
1594   if (!*flg) {
1595     PetscFunctionReturn(0);
1596   }
1597   /* if a->a are the same */
1598   ierr = PetscMemcmp(a->a,b->a,(a->nz)*(A->rmap->bs)*(B->rmap->bs)*sizeof(PetscScalar),flg);CHKERRQ(ierr);
1599   PetscFunctionReturn(0);
1600 
1601 }
1602 
1603 #undef __FUNCT__
1604 #define __FUNCT__ "MatGetDiagonal_SeqBAIJ"
1605 PetscErrorCode MatGetDiagonal_SeqBAIJ(Mat A,Vec v)
1606 {
1607   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
1608   PetscErrorCode ierr;
1609   PetscInt       i,j,k,n,row,bs,*ai,*aj,ambs,bs2;
1610   PetscScalar    *x,zero = 0.0;
1611   MatScalar      *aa,*aa_j;
1612 
1613   PetscFunctionBegin;
1614   if (A->factor) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Not for factored matrix");
1615   bs   = A->rmap->bs;
1616   aa   = a->a;
1617   ai   = a->i;
1618   aj   = a->j;
1619   ambs = a->mbs;
1620   bs2  = a->bs2;
1621 
1622   ierr = VecSet(v,zero);CHKERRQ(ierr);
1623   ierr = VecGetArray(v,&x);CHKERRQ(ierr);
1624   ierr = VecGetLocalSize(v,&n);CHKERRQ(ierr);
1625   if (n != A->rmap->N) SETERRQ(PETSC_ERR_ARG_SIZ,"Nonconforming matrix and vector");
1626   for (i=0; i<ambs; i++) {
1627     for (j=ai[i]; j<ai[i+1]; j++) {
1628       if (aj[j] == i) {
1629         row  = i*bs;
1630         aa_j = aa+j*bs2;
1631         for (k=0; k<bs2; k+=(bs+1),row++) x[row] = aa_j[k];
1632         break;
1633       }
1634     }
1635   }
1636   ierr = VecRestoreArray(v,&x);CHKERRQ(ierr);
1637   PetscFunctionReturn(0);
1638 }
1639 
1640 #undef __FUNCT__
1641 #define __FUNCT__ "MatDiagonalScale_SeqBAIJ"
1642 PetscErrorCode MatDiagonalScale_SeqBAIJ(Mat A,Vec ll,Vec rr)
1643 {
1644   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
1645   PetscScalar    *l,*r,x,*li,*ri;
1646   MatScalar      *aa,*v;
1647   PetscErrorCode ierr;
1648   PetscInt       i,j,k,lm,rn,M,m,n,*ai,*aj,mbs,tmp,bs,bs2;
1649 
1650   PetscFunctionBegin;
1651   ai  = a->i;
1652   aj  = a->j;
1653   aa  = a->a;
1654   m   = A->rmap->n;
1655   n   = A->cmap->n;
1656   bs  = A->rmap->bs;
1657   mbs = a->mbs;
1658   bs2 = a->bs2;
1659   if (ll) {
1660     ierr = VecGetArray(ll,&l);CHKERRQ(ierr);
1661     ierr = VecGetLocalSize(ll,&lm);CHKERRQ(ierr);
1662     if (lm != m) SETERRQ(PETSC_ERR_ARG_SIZ,"Left scaling vector wrong length");
1663     for (i=0; i<mbs; i++) { /* for each block row */
1664       M  = ai[i+1] - ai[i];
1665       li = l + i*bs;
1666       v  = aa + bs2*ai[i];
1667       for (j=0; j<M; j++) { /* for each block */
1668         for (k=0; k<bs2; k++) {
1669           (*v++) *= li[k%bs];
1670         }
1671       }
1672     }
1673     ierr = VecRestoreArray(ll,&l);CHKERRQ(ierr);
1674     ierr = PetscLogFlops(a->nz);CHKERRQ(ierr);
1675   }
1676 
1677   if (rr) {
1678     ierr = VecGetArray(rr,&r);CHKERRQ(ierr);
1679     ierr = VecGetLocalSize(rr,&rn);CHKERRQ(ierr);
1680     if (rn != n) SETERRQ(PETSC_ERR_ARG_SIZ,"Right scaling vector wrong length");
1681     for (i=0; i<mbs; i++) { /* for each block row */
1682       M  = ai[i+1] - ai[i];
1683       v  = aa + bs2*ai[i];
1684       for (j=0; j<M; j++) { /* for each block */
1685         ri = r + bs*aj[ai[i]+j];
1686         for (k=0; k<bs; k++) {
1687           x = ri[k];
1688           for (tmp=0; tmp<bs; tmp++) (*v++) *= x;
1689         }
1690       }
1691     }
1692     ierr = VecRestoreArray(rr,&r);CHKERRQ(ierr);
1693     ierr = PetscLogFlops(a->nz);CHKERRQ(ierr);
1694   }
1695   PetscFunctionReturn(0);
1696 }
1697 
1698 
1699 #undef __FUNCT__
1700 #define __FUNCT__ "MatGetInfo_SeqBAIJ"
1701 PetscErrorCode MatGetInfo_SeqBAIJ(Mat A,MatInfoType flag,MatInfo *info)
1702 {
1703   Mat_SeqBAIJ *a = (Mat_SeqBAIJ*)A->data;
1704 
1705   PetscFunctionBegin;
1706   info->block_size     = a->bs2;
1707   info->nz_allocated   = a->maxnz;
1708   info->nz_used        = a->bs2*a->nz;
1709   info->nz_unneeded    = (double)(info->nz_allocated - info->nz_used);
1710   info->assemblies   = A->num_ass;
1711   info->mallocs      = a->reallocs;
1712   info->memory       = ((PetscObject)A)->mem;
1713   if (A->factor) {
1714     info->fill_ratio_given  = A->info.fill_ratio_given;
1715     info->fill_ratio_needed = A->info.fill_ratio_needed;
1716     info->factor_mallocs    = A->info.factor_mallocs;
1717   } else {
1718     info->fill_ratio_given  = 0;
1719     info->fill_ratio_needed = 0;
1720     info->factor_mallocs    = 0;
1721   }
1722   PetscFunctionReturn(0);
1723 }
1724 
1725 
1726 #undef __FUNCT__
1727 #define __FUNCT__ "MatZeroEntries_SeqBAIJ"
1728 PetscErrorCode MatZeroEntries_SeqBAIJ(Mat A)
1729 {
1730   Mat_SeqBAIJ    *a = (Mat_SeqBAIJ*)A->data;
1731   PetscErrorCode ierr;
1732 
1733   PetscFunctionBegin;
1734   ierr = PetscMemzero(a->a,a->bs2*a->i[a->mbs]*sizeof(MatScalar));CHKERRQ(ierr);
1735   PetscFunctionReturn(0);
1736 }
1737