Actual source code: mhyp.c

  1: #define PETSCMAT_DLL

  3: /*
  4:     Creates hypre ijmatrix from PETSc matrix
  5: */

 7:  #include private/matimpl.h
  8: #if defined(PETSC_HAVE_HYPRE)
 10: #include "HYPRE.h"
 11: #include "HYPRE_parcsr_ls.h"

 16: PetscErrorCode MatHYPRE_IJMatrixPreallocate(Mat A_d, Mat A_o,HYPRE_IJMatrix ij)
 17: {
 19:   PetscInt       i;
 20:   PetscInt       n_d,*ia_d,n_o,*ia_o;
 21:   PetscTruth     done_d=PETSC_FALSE,done_o=PETSC_FALSE;
 22:   PetscInt       *nnz_d=PETSC_NULL,*nnz_o=PETSC_NULL;
 23: 
 25:   if (A_d) { /* determine number of nonzero entries in local diagonal part */
 26:     MatGetRowIJ(A_d,0,PETSC_FALSE,PETSC_FALSE,&n_d,&ia_d,PETSC_NULL,&done_d);
 27:     if (done_d) {
 28:       PetscMalloc(n_d*sizeof(PetscInt),&nnz_d);
 29:       for (i=0; i<n_d; i++) {
 30:         nnz_d[i] = ia_d[i+1] - ia_d[i];
 31:       }
 32:     }
 33:     MatRestoreRowIJ(A_d,0,PETSC_FALSE,PETSC_FALSE,&n_d,&ia_d,PETSC_NULL,&done_d);
 34:   }
 35:   if (A_o) { /* determine number of nonzero entries in local off-diagonal part */
 36:     MatGetRowIJ(A_o,0,PETSC_FALSE,PETSC_FALSE,&n_o,&ia_o,PETSC_NULL,&done_o);
 37:     if (done_o) {
 38:       PetscMalloc(n_o*sizeof(PetscInt),&nnz_o);
 39:       for (i=0; i<n_o; i++) {
 40:         nnz_o[i] = ia_o[i+1] - ia_o[i];
 41:       }
 42:     }
 43:     MatRestoreRowIJ(A_o,0,PETSC_FALSE,PETSC_FALSE,&n_o,&ia_o,PETSC_NULL,&done_o);
 44:   }
 45:   if (done_d) {    /* set number of nonzeros in HYPRE IJ matrix */
 46:     if (!done_o) { /* only diagonal part */
 47:       PetscMalloc(n_d*sizeof(PetscInt),&nnz_o);
 48:       for (i=0; i<n_d; i++) {
 49:         nnz_o[i] = 0;
 50:       }
 51:     }
 52:     HYPRE_IJMatrixSetDiagOffdSizes(ij,nnz_d,nnz_o);
 53:     PetscFree(nnz_d);
 54:     PetscFree(nnz_o);
 55:   }
 56:   return(0);
 57: }


 62: PetscErrorCode MatHYPRE_IJMatrixCreate(Mat A,HYPRE_IJMatrix *ij)
 63: {
 65:   int            rstart,rend,cstart,cend;
 66: 
 71:   MatPreallocated(A);
 72:   rstart = A->rmap->rstart;
 73:   rend   = A->rmap->rend;
 74:   cstart = A->cmap->rstart;
 75:   cend   = A->cmap->rend;
 76:   HYPRE_IJMatrixCreate(((PetscObject)A)->comm,rstart,rend-1,cstart,cend-1,ij);
 77:   HYPRE_IJMatrixSetObjectType(*ij,HYPRE_PARCSR);
 78:   {
 79:     PetscTruth  same;
 80:     Mat         A_d,A_o;
 81:     PetscInt    *colmap;
 82:     PetscTypeCompare((PetscObject)A,MATMPIAIJ,&same);
 83:     if (same) {
 84:       MatMPIAIJGetSeqAIJ(A,&A_d,&A_o,&colmap);
 85:       MatHYPRE_IJMatrixPreallocate(A_d,A_o,*ij);
 86:       return(0);
 87:     }
 88:     PetscTypeCompare((PetscObject)A,MATMPIBAIJ,&same);
 89:     if (same) {
 90:       MatMPIBAIJGetSeqBAIJ(A,&A_d,&A_o,&colmap);
 91:       MatHYPRE_IJMatrixPreallocate(A_d,A_o,*ij);
 92:       return(0);
 93:     }
 94:     PetscTypeCompare((PetscObject)A,MATSEQAIJ,&same);
 95:     if (same) {
 96:       MatHYPRE_IJMatrixPreallocate(A,PETSC_NULL,*ij);
 97:       return(0);
 98:     }
 99:     PetscTypeCompare((PetscObject)A,MATSEQBAIJ,&same);
100:     if (same) {
101:       MatHYPRE_IJMatrixPreallocate(A,PETSC_NULL,*ij);
102:       return(0);
103:     }
104:   }
105:   return(0);
106: }

110: /*
111:     Copies the data over (column indices, numerical values) to hypre matrix  
112: */

116: PetscErrorCode MatHYPRE_IJMatrixCopy(Mat A,HYPRE_IJMatrix ij)
117: {
118:   PetscErrorCode    ierr;
119:   PetscInt          i,rstart,rend,ncols;
120:   const PetscScalar *values;
121:   const PetscInt    *cols;
122:   PetscTruth        flg;

125:   PetscTypeCompare((PetscObject)A,MATMPIAIJ,&flg);
126:   if (flg) {
127:     MatHYPRE_IJMatrixFastCopy_MPIAIJ(A,ij);
128:     return(0);
129:   }
130:   PetscTypeCompare((PetscObject)A,MATSEQAIJ,&flg);
131:   if (flg) {
132:     MatHYPRE_IJMatrixFastCopy_SeqAIJ(A,ij);
133:     return(0);
134:   }

136:   PetscLogEventBegin(MAT_Convert,A,0,0,0);
137:   HYPRE_IJMatrixInitialize(ij);
138:   MatGetOwnershipRange(A,&rstart,&rend);
139:   for (i=rstart; i<rend; i++) {
140:     MatGetRow(A,i,&ncols,&cols,&values);
141:     HYPRE_IJMatrixSetValues(ij,1,&ncols,&i,cols,values);
142:     MatRestoreRow(A,i,&ncols,&cols,&values);
143:   }
144:   HYPRE_IJMatrixAssemble(ij);
145:   PetscLogEventEnd(MAT_Convert,A,0,0,0);
146:   return(0);
147: }

149: /*
150:     This copies the CSR format directly from the PETSc data structure to the hypre 
151:     data structure without calls to MatGetRow() or hypre's set values.

153: */
154: #include "_hypre_IJ_mv.h"
155: #include "HYPRE_IJ_mv.h"
156:  #include ../src/mat/impls/aij/mpi/mpiaij.h

160: PetscErrorCode MatHYPRE_IJMatrixFastCopy_SeqAIJ(Mat A,HYPRE_IJMatrix ij)
161: {
162:   PetscErrorCode        ierr;
163:   Mat_SeqAIJ            *pdiag = (Mat_SeqAIJ*)A->data;;

165:   hypre_ParCSRMatrix    *par_matrix;
166:   hypre_AuxParCSRMatrix *aux_matrix;
167:   hypre_CSRMatrix       *hdiag,*hoffd;


174:   PetscLogEventBegin(MAT_Convert,A,0,0,0);
175:   HYPRE_IJMatrixInitialize(ij);
176:   par_matrix = (hypre_ParCSRMatrix*)hypre_IJMatrixObject(ij);
177:   aux_matrix = (hypre_AuxParCSRMatrix*)hypre_IJMatrixTranslator(ij);
178:   hdiag = hypre_ParCSRMatrixDiag(par_matrix);
179:   hoffd = hypre_ParCSRMatrixOffd(par_matrix);

181:   /* 
182:        this is the Hack part where we monkey directly with the hypre datastructures
183:   */

185:   PetscMemcpy(hdiag->i,pdiag->i,(A->rmap->n + 1)*sizeof(PetscInt));
186:   PetscMemcpy(hdiag->j,pdiag->j,pdiag->nz*sizeof(PetscInt));
187:   PetscMemcpy(hdiag->data,pdiag->a,pdiag->nz*sizeof(PetscScalar));

189:   hypre_AuxParCSRMatrixNeedAux(aux_matrix) = 0;
190:   HYPRE_IJMatrixAssemble(ij);
191:   PetscLogEventEnd(MAT_Convert,A,0,0,0);
192:   return(0);
193: }

197: PetscErrorCode MatHYPRE_IJMatrixFastCopy_MPIAIJ(Mat A,HYPRE_IJMatrix ij)
198: {
199:   PetscErrorCode        ierr;
200:   Mat_MPIAIJ            *pA = (Mat_MPIAIJ*)A->data;
201:   Mat_SeqAIJ            *pdiag,*poffd;
202:   PetscInt              i,*garray = pA->garray,*jj,cstart,*pjj;

204:   hypre_ParCSRMatrix    *par_matrix;
205:   hypre_AuxParCSRMatrix *aux_matrix;
206:   hypre_CSRMatrix       *hdiag,*hoffd;

212:   pdiag = (Mat_SeqAIJ*) pA->A->data;
213:   poffd = (Mat_SeqAIJ*) pA->B->data;
214:   /* cstart is only valid for square MPIAIJ layed out in the usual way */
215:   MatGetOwnershipRange(A,&cstart,PETSC_NULL);

217:   PetscLogEventBegin(MAT_Convert,A,0,0,0);

219:   HYPRE_IJMatrixInitialize(ij);
220:   par_matrix = (hypre_ParCSRMatrix*)hypre_IJMatrixObject(ij);
221:   aux_matrix = (hypre_AuxParCSRMatrix*)hypre_IJMatrixTranslator(ij);
222:   hdiag = hypre_ParCSRMatrixDiag(par_matrix);
223:   hoffd = hypre_ParCSRMatrixOffd(par_matrix);

225:   /* 
226:        this is the Hack part where we monkey directly with the hypre datastructures
227:   */

229:   PetscMemcpy(hdiag->i,pdiag->i,(pA->A->rmap->n + 1)*sizeof(PetscInt));
230:   /* need to shift the diag column indices (hdiag->j) back to global numbering since hypre is expecting this */
231:   jj  = hdiag->j;
232:   pjj = pdiag->j;
233:   for (i=0; i<pdiag->nz; i++) {
234:     jj[i] = cstart + pjj[i];
235:   }
236:   PetscMemcpy(hdiag->data,pdiag->a,pdiag->nz*sizeof(PetscScalar));

238:   PetscMemcpy(hoffd->i,poffd->i,(pA->A->rmap->n + 1)*sizeof(PetscInt));
239:   /* need to move the offd column indices (hoffd->j) back to global numbering since hypre is expecting this
240:      If we hacked a hypre a bit more we might be able to avoid this step */
241:   jj  = hoffd->j;
242:   pjj = poffd->j;
243:   for (i=0; i<poffd->nz; i++) {
244:     jj[i] = garray[pjj[i]];
245:   }
246:   PetscMemcpy(hoffd->data,poffd->a,poffd->nz*sizeof(PetscScalar));

248:   hypre_AuxParCSRMatrixNeedAux(aux_matrix) = 0;
249:   HYPRE_IJMatrixAssemble(ij);
250:   PetscLogEventEnd(MAT_Convert,A,0,0,0);
251:   return(0);
252: }

254: /*
255:     Does NOT copy the data over, instead uses DIRECTLY the pointers from the PETSc MPIAIJ format

257:     This is UNFINISHED and does NOT work! The problem is that hypre puts the diagonal entry first
258:     which will corrupt the PETSc data structure if we did this. Need a work around to this problem.
259: */
260: #include "_hypre_IJ_mv.h"
261: #include "HYPRE_IJ_mv.h"

265: PetscErrorCode MatHYPRE_IJMatrixLink(Mat A,HYPRE_IJMatrix *ij)
266: {
267:   PetscErrorCode        ierr;
268:   int                   rstart,rend,cstart,cend;
269:   PetscTruth            flg;
270:   hypre_ParCSRMatrix    *par_matrix;
271:   hypre_AuxParCSRMatrix *aux_matrix;

277:   PetscTypeCompare((PetscObject)A,MATMPIAIJ,&flg);
278:   if (!flg) SETERRQ(PETSC_ERR_SUP,"Can only use with PETSc MPIAIJ matrices");
279:   MatPreallocated(A);

281:   PetscLogEventBegin(MAT_Convert,A,0,0,0);
282:   rstart = A->rmap->rstart;
283:   rend   = A->rmap->rend;
284:   cstart = A->cmap->rstart;
285:   cend   = A->cmap->rend;
286:   HYPRE_IJMatrixCreate(((PetscObject)A)->comm,rstart,rend-1,cstart,cend-1,ij);
287:   HYPRE_IJMatrixSetObjectType(*ij,HYPRE_PARCSR);
288: 
289:   HYPRE_IJMatrixInitialize(*ij);
290:   par_matrix = (hypre_ParCSRMatrix*)hypre_IJMatrixObject(*ij);
291:   aux_matrix = (hypre_AuxParCSRMatrix*)hypre_IJMatrixTranslator(*ij);

293:   hypre_AuxParCSRMatrixNeedAux(aux_matrix) = 0;

295:   /* this is the Hack part where we monkey directly with the hypre datastructures */

297:   HYPRE_IJMatrixAssemble(*ij);
298:   PetscLogEventEnd(MAT_Convert,A,0,0,0);
299:   return(0);
300: }

302: /* -----------------------------------------------------------------------------------------------------------------*/

304: /*MC
305:    MATHYPRESTRUCT - MATHYPRESTRUCT = "hyprestruct" - A matrix type to be used for parallel sparse matrices
306:           based on the hypre HYPRE_StructMatrix.

308:    Level: intermediate

310:    Notes: Unlike the more general support for blocks in hypre this allows only one block per process and requires the block
311:           be defined by a DA.

313:           The matrix needs a DA associated with it by either a call to MatSetDA() or if the matrix is obtained from DAGetMatrix()

315: .seealso: MatCreate(), PCPFMG, MatSetDA(), DAGetMatrix()
316: M*/

318:  #include petscda.h
319:  #include mhyp.h

323: PetscErrorCode  MatSetValuesLocal_HYPREStruct_3d(Mat mat,PetscInt nrow,const PetscInt irow[],PetscInt ncol,const PetscInt icol[],const PetscScalar y[],InsertMode addv)
324: {
325:   PetscErrorCode    ierr;
326:   PetscInt          i,j,stencil,index[3],row,entries[7];
327:   const PetscScalar *values = y;
328:   Mat_HYPREStruct   *ex = (Mat_HYPREStruct*) mat->data;

331:   for (i=0; i<nrow; i++) {
332:     for (j=0; j<ncol; j++) {
333:       stencil = icol[j] - irow[i];
334:       if (!stencil) {
335:         entries[j] = 3;
336:       } else if (stencil == -1) {
337:         entries[j] = 2;
338:       } else if (stencil == 1) {
339:         entries[j] = 4;
340:       } else if (stencil == -ex->gnx) {
341:         entries[j] = 1;
342:       } else if (stencil == ex->gnx) {
343:         entries[j] = 5;
344:       } else if (stencil == -ex->gnxgny) {
345:         entries[j] = 0;
346:       } else if (stencil == ex->gnxgny) {
347:         entries[j] = 6;
348:       } else SETERRQ3(PETSC_ERR_ARG_WRONG,"Local row %D local column %D have bad stencil %D",irow[i],icol[j],stencil);
349:     }
350:     row = ex->gindices[irow[i]] - ex->rstart;
351:     index[0] = ex->xs + (row % ex->nx);
352:     index[1] = ex->ys + ((row/ex->nx) % ex->ny);
353:     index[2] = ex->zs + (row/(ex->nxny));
354:     if (addv == ADD_VALUES) {
355:       HYPRE_StructMatrixAddToValues(ex->hmat,index,ncol,entries,(PetscScalar*)values);
356:     } else {
357:       HYPRE_StructMatrixSetValues(ex->hmat,index,ncol,entries,(PetscScalar*)values);
358:     }
359:     values += ncol;
360:   }
361:   return(0);
362: }

366: PetscErrorCode  MatZeroRowsLocal_HYPREStruct_3d(Mat mat,PetscInt nrow,const PetscInt irow[],PetscScalar d)
367: {
368:   PetscErrorCode  ierr;
369:   PetscInt        i,index[3],row,entries[7] = {0,1,2,3,4,5,6};
370:   PetscScalar     values[7];
371:   Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;

374:   PetscMemzero(values,7*sizeof(PetscScalar));
375:   values[3] = d;
376:   for (i=0; i<nrow; i++) {
377:     row = ex->gindices[irow[i]] - ex->rstart;
378:     index[0] = ex->xs + (row % ex->nx);
379:     index[1] = ex->ys + ((row/ex->nx) % ex->ny);
380:     index[2] = ex->zs + (row/(ex->nxny));
381:     HYPRE_StructMatrixSetValues(ex->hmat,index,7,entries,values);
382:   }
383:   HYPRE_StructMatrixAssemble(ex->hmat);
384:   return(0);
385: }

389: PetscErrorCode MatZeroEntries_HYPREStruct_3d(Mat mat)
390: {
392:   PetscInt       indices[7] = {0,1,2,3,4,5,6};
393:   Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;

396:   /* hypre has no public interface to do this */
397:   hypre_StructMatrixClearBoxValues(ex->hmat,&ex->hbox,7,indices,0,1);
398:   HYPRE_StructMatrixAssemble(ex->hmat);
399:   return(0);
400: }

404: PetscErrorCode  MatSetDA_HYPREStruct(Mat mat,DA da)
405: {
406:   PetscErrorCode  ierr;
407:   Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;
408:   PetscInt         dim,dof,sw[3],nx,ny,nz;
409:   int              ilower[3],iupper[3],ssize,i;
410:   DAPeriodicType   p;
411:   DAStencilType    st;

414:   ex->da = da;
415:   PetscObjectReference((PetscObject)da);

417:   DAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&p,&st);
418:   DAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);
419:   iupper[0] += ilower[0] - 1;
420:   iupper[1] += ilower[1] - 1;
421:   iupper[2] += ilower[2] - 1;

423:   /* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
424:   ex->hbox.imin[0] = ilower[0];
425:   ex->hbox.imin[1] = ilower[1];
426:   ex->hbox.imin[2] = ilower[2];
427:   ex->hbox.imax[0] = iupper[0];
428:   ex->hbox.imax[1] = iupper[1];
429:   ex->hbox.imax[2] = iupper[2];

431:   /* create the hypre grid object and set its information */
432:   if (dof > 1) SETERRQ(PETSC_ERR_SUP,"Currently only support for scalar problems");
433:   if (p) SETERRQ(PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_StructGridSetPeriodic()");
434:   HYPRE_StructGridCreate(ex->hcomm,dim,&ex->hgrid);

436:   HYPRE_StructGridSetExtents(ex->hgrid,ilower,iupper);
437:   HYPRE_StructGridAssemble(ex->hgrid);
438: 
439:   sw[1] = sw[0];
440:   sw[2] = sw[1];
441:   HYPRE_StructGridSetNumGhost(ex->hgrid,sw);

443:   /* create the hypre stencil object and set its information */
444:   if (sw[0] > 1) SETERRQ(PETSC_ERR_SUP,"Ask us to add support for wider stencils");
445:   if (st == DA_STENCIL_BOX) SETERRQ(PETSC_ERR_SUP,"Ask us to add support for box stencils");
446:   if (dim == 1) {
447:     int offsets[3][1] = {{-1},{0},{1}};
448:     ssize = 3;
449:     HYPRE_StructStencilCreate(dim,ssize,&ex->hstencil);
450:     for (i=0; i<ssize; i++) {
451:       HYPRE_StructStencilSetElement(ex->hstencil,i,offsets[i]);
452:     }
453:   } else if (dim == 2) {
454:     int offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
455:     ssize = 5;
456:     HYPRE_StructStencilCreate(dim,ssize,&ex->hstencil);
457:     for (i=0; i<ssize; i++) {
458:       HYPRE_StructStencilSetElement(ex->hstencil,i,offsets[i]);
459:     }
460:   } else if (dim == 3) {
461:     int offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
462:     ssize = 7;
463:     HYPRE_StructStencilCreate(dim,ssize,&ex->hstencil);
464:     for (i=0; i<ssize; i++) {
465:       HYPRE_StructStencilSetElement(ex->hstencil,i,offsets[i]);
466:     }
467:   }
468: 
469:   /* create the HYPRE vector for rhs and solution */
470:   HYPRE_StructVectorCreate(ex->hcomm,ex->hgrid,&ex->hb);
471:   HYPRE_StructVectorCreate(ex->hcomm,ex->hgrid,&ex->hx);
472:   HYPRE_StructVectorInitialize(ex->hb);
473:   HYPRE_StructVectorInitialize(ex->hx);
474:   HYPRE_StructVectorAssemble(ex->hb);
475:   HYPRE_StructVectorAssemble(ex->hx);

477:   /* create the hypre matrix object and set its information */
478:   HYPRE_StructMatrixCreate(ex->hcomm,ex->hgrid,ex->hstencil,&ex->hmat);
479:   HYPRE_StructGridDestroy(ex->hgrid);
480:   HYPRE_StructStencilDestroy(ex->hstencil);CHKERRQ(ierr)
481:   if (ex->needsinitialization) {
482:     HYPRE_StructMatrixInitialize(ex->hmat);
483:     ex->needsinitialization = PETSC_FALSE;
484:   }

486:   /* set the global and local sizes of the matrix */
487:   DAGetCorners(da,0,0,0,&nx,&ny,&nz);
488:   MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);
489:   PetscLayoutSetBlockSize(mat->rmap,1);
490:   PetscLayoutSetBlockSize(mat->cmap,1);
491:   PetscLayoutSetUp(mat->rmap);
492:   PetscLayoutSetUp(mat->cmap);

494:   if (dim == 3) {
495:     mat->ops->setvalueslocal = MatSetValuesLocal_HYPREStruct_3d;
496:     mat->ops->zerorowslocal  = MatZeroRowsLocal_HYPREStruct_3d;
497:     mat->ops->zeroentries    = MatZeroEntries_HYPREStruct_3d;
498:     MatZeroEntries_HYPREStruct_3d(mat);
499:   } else SETERRQ(PETSC_ERR_SUP,"Only support for 3d DA currently");

501:   /* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
502:   MatGetOwnershipRange(mat,&ex->rstart,PETSC_NULL);
503:   DAGetGlobalIndices(ex->da,PETSC_NULL,&ex->gindices);
504:   DAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,0);
505:   ex->gnxgny *= ex->gnx;
506:   DAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,0);
507:   ex->nxny = ex->nx*ex->ny;
508:   return(0);
509: }

513: PetscErrorCode MatMult_HYPREStruct(Mat A,Vec x,Vec y)
514: {
515:   PetscErrorCode  ierr;
516:   PetscScalar     *xx,*yy;
517:   int             ilower[3],iupper[3];
518:   Mat_HYPREStruct *mx = (Mat_HYPREStruct *)(A->data);

521:   DAGetCorners(mx->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);
522:   iupper[0] += ilower[0] - 1;
523:   iupper[1] += ilower[1] - 1;
524:   iupper[2] += ilower[2] - 1;

526:   /* copy x values over to hypre */
527:   HYPRE_StructVectorSetConstantValues(mx->hb,0.0);
528:   VecGetArray(x,&xx);
529:   HYPRE_StructVectorSetBoxValues(mx->hb,ilower,iupper,xx);
530:   VecRestoreArray(x,&xx);
531:   HYPRE_StructVectorAssemble(mx->hb);

533:   HYPRE_StructMatrixMatvec(1.0,mx->hmat,mx->hb,0.0,mx->hx);

535:   /* copy solution values back to PETSc */
536:   VecGetArray(y,&yy);
537:   HYPRE_StructVectorGetBoxValues(mx->hx,ilower,iupper,yy);
538:   VecRestoreArray(y,&yy);
539:   return(0);
540: }

544: PetscErrorCode MatAssemblyEnd_HYPREStruct(Mat mat,MatAssemblyType mode)
545: {
546:   Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;
547:   PetscErrorCode   ierr;

550:   HYPRE_StructMatrixAssemble(ex->hmat);
551:   /* HYPRE_StructMatrixPrint("dummy",ex->hmat,0); */
552:   return(0);
553: }

557: PetscErrorCode MatZeroEntries_HYPREStruct(Mat mat)
558: {
560:   /* before the DA is set to the matrix the zero doesn't need to do anything */
561:   return(0);
562: }


567: PetscErrorCode MatDestroy_HYPREStruct(Mat mat)
568: {
569:   Mat_HYPREStruct *ex = (Mat_HYPREStruct*) mat->data;
570:   PetscErrorCode  ierr;

573:   HYPRE_StructMatrixDestroy(ex->hmat);
574:   HYPRE_StructVectorDestroy(ex->hx);
575:   HYPRE_StructVectorDestroy(ex->hb);
576:   return(0);
577: }


583: PetscErrorCode  MatCreate_HYPREStruct(Mat B)
584: {
585:   Mat_HYPREStruct *ex;
586:   PetscErrorCode  ierr;

589:   PetscNewLog(B,Mat_HYPREStruct,&ex);
590:   B->data         = (void*)ex;
591:   B->rmap->bs     = 1;
592:   B->assembled    = PETSC_FALSE;
593:   B->mapping      = 0;

595:   B->insertmode   = NOT_SET_VALUES;

597:   B->ops->assemblyend    = MatAssemblyEnd_HYPREStruct;
598:   B->ops->mult           = MatMult_HYPREStruct;
599:   B->ops->zeroentries    = MatZeroEntries_HYPREStruct;
600:   B->ops->destroy        = MatDestroy_HYPREStruct;

602:   ex->needsinitialization = PETSC_TRUE;

604:   MPI_Comm_dup(((PetscObject)B)->comm,&(ex->hcomm));
605:   PetscObjectComposeFunctionDynamic((PetscObject)B,"MatSetDA_C","MatSetDA_HYPREStruct",MatSetDA_HYPREStruct);
606:   PetscObjectChangeTypeName((PetscObject)B,MATHYPRESTRUCT);
607:   return(0);
608: }


612: /*MC
613:    MATHYPRESSTRUCT - MATHYPRESSTRUCT = "hypresstruct" - A matrix type to be used for parallel sparse matrices
614:           based on the hypre HYPRE_SStructMatrix.
615:   

617:    Level: intermediate
618:   
619:    Notes: Unlike hypre's general semi-struct object consisting of a collection of structured-grid objects and unstructured
620:           grid objects, we will restrict the semi-struct objects to consist of only structured-grid components.

622:           Unlike the more general support for parts and blocks in hypre this allows only one part, and one block per process and requires the block
623:           be defined by a DA.
624:   
625:           The matrix needs a DA associated with it by either a call to MatSetDA() or if the matrix is obtained from DAGetMatrix()
626:   
627: M*/

631: PetscErrorCode  MatSetValuesLocal_HYPRESStruct_3d(Mat mat,PetscInt nrow,const PetscInt irow[],PetscInt ncol,const PetscInt icol[],const PetscScalar y[],InsertMode addv)
632: {
633:   PetscErrorCode    ierr;
634:   PetscInt          i,j,stencil,index[3];
635:   const PetscScalar *values = y;
636:   Mat_HYPRESStruct  *ex = (Mat_HYPRESStruct*) mat->data;

638:   int               part= 0; /* Petsc sstruct interface only allows 1 part */
639:   int               ordering;
640:   int               grid_rank, to_grid_rank;
641:   int               var_type, to_var_type;
642:   int               to_var_entry = 0;

644:   int               nvars= ex->nvars;
645:   PetscInt          row,*entries;

648:   PetscMalloc(7*nvars*sizeof(PetscInt),&entries);

650:   ordering= ex-> dofs_order; /* ordering= 0   nodal ordering
651:                                           1   variable ordering */
652:   /* stencil entries are orderer by variables: var0_stencil0, var0_stencil1, ..., var0_stencil6, var1_stencil0, var1_stencil1, ...  */

654:   if (!ordering)  /* nodal ordering */
655:   {
656:     for (i=0; i<nrow; i++) {
657:       grid_rank= irow[i]/nvars;
658:       var_type = (irow[i] % nvars);

660:       for (j=0; j<ncol; j++) {
661:         to_grid_rank= icol[j]/nvars;
662:         to_var_type = (icol[j] % nvars);

664:         to_var_entry= to_var_entry*7;
665:         entries[j]= to_var_entry;

667:         stencil = to_grid_rank-grid_rank;
668:         if (!stencil) {
669:           entries[j] += 3;
670:         } else if (stencil == -1) {
671:           entries[j] += 2;
672:         } else if (stencil == 1) {
673:           entries[j] += 4;
674:         } else if (stencil == -ex->gnx) {
675:           entries[j] += 1;
676:         } else if (stencil == ex->gnx) {
677:           entries[j] += 5;
678:         } else if (stencil == -ex->gnxgny) {
679:           entries[j] += 0;
680:         } else if (stencil == ex->gnxgny) {
681:           entries[j] += 6;
682:         } else SETERRQ3(PETSC_ERR_ARG_WRONG,"Local row %D local column %D have bad stencil %D",irow[i],icol[j],stencil);
683:       }

685:       row = ex->gindices[grid_rank] - ex->rstart;
686:       index[0] = ex->xs + (row % ex->nx);
687:       index[1] = ex->ys + ((row/ex->nx) % ex->ny);
688:       index[2] = ex->zs + (row/(ex->nxny));

690:       if (addv == ADD_VALUES) {
691:         HYPRE_SStructMatrixAddToValues(ex->ss_mat,part,index,var_type,ncol,entries,(PetscScalar*)values);
692:       } else {
693:         HYPRE_SStructMatrixSetValues(ex->ss_mat,part,index,var_type,ncol,entries,(PetscScalar*)values);
694:       }
695:       values += ncol;
696:     }
697:   }

699:   else
700:   {
701:     for (i=0; i<nrow; i++) {
702:       var_type = irow[i]/(ex->gnxgnygnz);
703:       grid_rank= irow[i] - var_type*(ex->gnxgnygnz);

705:       for (j=0; j<ncol; j++) {
706:         to_var_type = icol[j]/(ex->gnxgnygnz);
707:         to_grid_rank= icol[j] - to_var_type*(ex->gnxgnygnz);

709:         to_var_entry= to_var_entry*7;
710:         entries[j]= to_var_entry;

712:         stencil = to_grid_rank-grid_rank;
713:         if (!stencil) {
714:           entries[j] += 3;
715:         } else if (stencil == -1) {
716:           entries[j] += 2;
717:         } else if (stencil == 1) {
718:           entries[j] += 4;
719:         } else if (stencil == -ex->gnx) {
720:           entries[j] += 1;
721:         } else if (stencil == ex->gnx) {
722:           entries[j] += 5;
723:         } else if (stencil == -ex->gnxgny) {
724:           entries[j] += 0;
725:         } else if (stencil == ex->gnxgny) {
726:           entries[j] += 6;
727:         } else SETERRQ3(PETSC_ERR_ARG_WRONG,"Local row %D local column %D have bad stencil %D",irow[i],icol[j],stencil);
728:       }

730:       row = ex->gindices[grid_rank] - ex->rstart;
731:       index[0] = ex->xs + (row % ex->nx);
732:       index[1] = ex->ys + ((row/ex->nx) % ex->ny);
733:       index[2] = ex->zs + (row/(ex->nxny));

735:       if (addv == ADD_VALUES) {
736:         HYPRE_SStructMatrixAddToValues(ex->ss_mat,part,index,var_type,ncol,entries,(PetscScalar*)values);
737:       } else {
738:         HYPRE_SStructMatrixSetValues(ex->ss_mat,part,index,var_type,ncol,entries,(PetscScalar*)values);
739:       }
740:       values += ncol;
741:     }

743:   }
744:   PetscFree(entries);
745:   return(0);
746: }

750: PetscErrorCode  MatZeroRowsLocal_HYPRESStruct_3d(Mat mat,PetscInt nrow,const PetscInt irow[],PetscScalar d)
751: {
752:   PetscErrorCode    ierr;
753:   PetscInt          i,index[3];
754:   PetscScalar     **values;
755:   Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;

757:   int               part= 0; /* Petsc sstruct interface only allows 1 part */
758:   int               ordering= ex->dofs_order;
759:   int               grid_rank;
760:   int               var_type;
761:   int               nvars= ex->nvars;
762:   PetscInt          row,*entries;

765:   PetscMalloc(7*nvars*sizeof(PetscInt),&entries);

767:   PetscMalloc(nvars*sizeof(PetscScalar *),&values);
768:   PetscMalloc(7*nvars*nvars*sizeof(PetscScalar),&values[0]);
769:   for (i=1; i<nvars; i++) {
770:      values[i] = values[i-1] + nvars*7;
771:   }

773:   for (i=0; i< nvars; i++) {
774:      PetscMemzero(values[i],nvars*7*sizeof(PetscScalar));
775:    *(values[i]+3)= d;
776:   }

778:   for (i= 0; i< nvars*7; i++) {
779:     entries[i]= i;
780:   }

782:   if (!ordering) {
783:     for (i=0; i<nrow; i++) {
784:        grid_rank= irow[i]/nvars;
785:        var_type = (irow[i] % nvars);

787:        row = ex->gindices[grid_rank] - ex->rstart;
788:        index[0] = ex->xs + (row % ex->nx);
789:        index[1] = ex->ys + ((row/ex->nx) % ex->ny);
790:        index[2] = ex->zs + (row/(ex->nxny));
791:        HYPRE_SStructMatrixSetValues(ex->ss_mat,part,index,var_type,7*nvars,entries,values[var_type]);
792:     }
793:   }
794: 
795:   else {
796:     for (i=0; i<nrow; i++) {
797:        var_type = irow[i]/(ex->gnxgnygnz);
798:        grid_rank= irow[i] - var_type*(ex->gnxgnygnz);

800:        row = ex->gindices[grid_rank] - ex->rstart;
801:        index[0] = ex->xs + (row % ex->nx);
802:        index[1] = ex->ys + ((row/ex->nx) % ex->ny);
803:        index[2] = ex->zs + (row/(ex->nxny));
804:        HYPRE_SStructMatrixSetValues(ex->ss_mat,part,index,var_type,7*nvars,entries,values[var_type]);
805:     }
806:   }

808:   HYPRE_SStructMatrixAssemble(ex->ss_mat);

810:   PetscFree(values[0]);
811:   PetscFree(values);

813:   PetscFree(entries);
814:   return(0);
815: }

819: PetscErrorCode MatZeroEntries_HYPRESStruct_3d(Mat mat)
820: {
821:   PetscErrorCode     ierr;
822:   Mat_HYPRESStruct  *ex = (Mat_HYPRESStruct*) mat->data;
823:   int                nvars= ex->nvars;
824:   int                size;
825:   int                part= 0; /* only one part */


829:   size= ((ex->hbox.imax[0])-(ex->hbox.imin[0])+1)*((ex->hbox.imax[1])-(ex->hbox.imin[1])+1)*((ex->hbox.imax[2])-(ex->hbox.imin[2])+1);
830:   {
831:      PetscInt          i,*entries;
832:      PetscScalar      *values;
833:      int               iupper[3], ilower[3];
834: 
835:      for (i= 0; i< 3; i++) {
836:         ilower[i]= ex->hbox.imin[i];
837:         iupper[i]= ex->hbox.imax[i];
838:      }

840:      PetscMalloc2(nvars*7,PetscInt,&entries,nvars*7*size,PetscScalar,&values);
841:      for (i= 0; i< nvars*7; i++) {
842:         entries[i]= i;
843:      }

845:      PetscMemzero(values,nvars*7*size*sizeof(PetscScalar));

847:      for (i= 0; i< nvars; i++) {
848:         HYPRE_SStructMatrixSetBoxValues(ex->ss_mat,part,ilower,iupper,i,nvars*7,entries,values);
849:      }

851:      PetscFree2(entries,values);
852:   }

854:   HYPRE_SStructMatrixAssemble(ex->ss_mat);

856:   return(0);
857: }


862: PetscErrorCode  MatSetDA_HYPRESStruct(Mat mat,DA da)
863: {
864:   PetscErrorCode    ierr;
865:   Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;
866:   PetscInt          dim,dof,sw[3],nx,ny,nz;
867:   int               ilower[3],iupper[3],ssize,i;
868:   DAPeriodicType    p;
869:   DAStencilType     st;
870:   int               nparts= 1; /* assuming only one part */
871:   int               part  = 0;

874:   ex->da = da;
875:   PetscObjectReference((PetscObject)da);

877:   DAGetInfo(ex->da,&dim,0,0,0,0,0,0,&dof,&sw[0],&p,&st);
878:   DAGetCorners(ex->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);
879:   iupper[0] += ilower[0] - 1;
880:   iupper[1] += ilower[1] - 1;
881:   iupper[2] += ilower[2] - 1;
882:   /* the hypre_Box is used to zero out the matrix entries in MatZeroValues() */
883:   ex->hbox.imin[0] = ilower[0];
884:   ex->hbox.imin[1] = ilower[1];
885:   ex->hbox.imin[2] = ilower[2];
886:   ex->hbox.imax[0] = iupper[0];
887:   ex->hbox.imax[1] = iupper[1];
888:   ex->hbox.imax[2] = iupper[2];

890:   ex->dofs_order   = 0;

892:   /* assuming that the same number of dofs on each gridpoint. Also assume all cell-centred based */
893:   ex->nvars= dof;

895:   /* create the hypre grid object and set its information */
896:   if (p) SETERRQ(PETSC_ERR_SUP,"Ask us to add periodic support by calling HYPRE_SStructGridSetPeriodic()");
897:   HYPRE_SStructGridCreate(ex->hcomm,dim,nparts,&ex->ss_grid);

899:   HYPRE_SStructGridSetExtents(ex->ss_grid,part,ex->hbox.imin,ex->hbox.imax);

901:   {
902:     HYPRE_SStructVariable *vartypes;
903:     PetscMalloc(ex->nvars*sizeof(HYPRE_SStructVariable),&vartypes);
904:     for (i= 0; i< ex->nvars; i++) {
905:       vartypes[i]= HYPRE_SSTRUCT_VARIABLE_CELL;
906:     }
907:     HYPRE_SStructGridSetVariables(ex->ss_grid, part, ex->nvars,vartypes);
908:     PetscFree(vartypes);
909:   }

911:   HYPRE_SStructGridAssemble(ex->ss_grid);

913:   sw[1] = sw[0];
914:   sw[2] = sw[1];
915: //  HYPRE_SStructGridSetNumGhost(ex->ss_grid,sw);

917:   /* create the hypre stencil object and set its information */
918:   if (sw[0] > 1) SETERRQ(PETSC_ERR_SUP,"Ask us to add support for wider stencils");
919:   if (st == DA_STENCIL_BOX) SETERRQ(PETSC_ERR_SUP,"Ask us to add support for box stencils");

921:   if (dim == 1) {
922:     int offsets[3][1] = {{-1},{0},{1}};
923:     int j, cnt;

925:     ssize = 3*(ex->nvars);
926:     HYPRE_SStructStencilCreate(dim,ssize,&ex->ss_stencil);
927:     cnt= 0;
928:     for (i= 0; i< (ex->nvars); i++) {
929:        for (j= 0; j< 3; j++) {
930:           HYPRE_SStructStencilSetEntry(ex->ss_stencil, cnt, offsets[j], i);
931:           cnt++;
932:        }
933:     }

935:   } else if (dim == 2) {
936:     int offsets[5][2] = {{0,-1},{-1,0},{0,0},{1,0},{0,1}};
937:     int j, cnt;

939:     ssize = 5*(ex->nvars);
940:     HYPRE_SStructStencilCreate(dim,ssize,&ex->ss_stencil);
941:     cnt= 0;
942:     for (i= 0; i< (ex->nvars); i++) {
943:        for (j= 0; j< 5; j++) {
944:           HYPRE_SStructStencilSetEntry(ex->ss_stencil, cnt, offsets[j], i);
945:           cnt++;
946:        }
947:     }
948:   } else if (dim == 3) {
949:     int offsets[7][3] = {{0,0,-1},{0,-1,0},{-1,0,0},{0,0,0},{1,0,0},{0,1,0},{0,0,1}};
950:     int j, cnt;

952:     ssize = 7*(ex->nvars);
953:     HYPRE_SStructStencilCreate(dim,ssize,&ex->ss_stencil);
954:     cnt= 0;
955:     for (i= 0; i< (ex->nvars); i++) {
956:        for (j= 0; j< 7; j++) {
957:           HYPRE_SStructStencilSetEntry(ex->ss_stencil, cnt, offsets[j], i);
958:           cnt++;
959:        }
960:     }
961:   }

963:   /* create the HYPRE graph */
964:   HYPRE_SStructGraphCreate(ex->hcomm, ex->ss_grid, &(ex->ss_graph));

966:   /* set the stencil graph. Note that each variable has the same graph. This means that each
967:      variable couples to all the other variable and with the same stencil pattern. */
968:   for (i= 0; i< (ex->nvars); i++) {
969:      HYPRE_SStructGraphSetStencil(ex->ss_graph,part,i,ex->ss_stencil);
970:   }
971:   ierr= HYPRE_SStructGraphAssemble(ex->ss_graph);

973:   /* create the HYPRE sstruct vectors for rhs and solution */
974:   HYPRE_SStructVectorCreate(ex->hcomm,ex->ss_grid,&ex->ss_b);
975:   HYPRE_SStructVectorCreate(ex->hcomm,ex->ss_grid,&ex->ss_x);
976:   HYPRE_SStructVectorInitialize(ex->ss_b);
977:   HYPRE_SStructVectorInitialize(ex->ss_x);
978:   HYPRE_SStructVectorAssemble(ex->ss_b);
979:   HYPRE_SStructVectorAssemble(ex->ss_x);

981:   /* create the hypre matrix object and set its information */
982:   HYPRE_SStructMatrixCreate(ex->hcomm,ex->ss_graph,&ex->ss_mat);
983:   HYPRE_SStructGridDestroy(ex->ss_grid);
984:   HYPRE_SStructStencilDestroy(ex->ss_stencil);CHKERRQ(ierr)
985:   if (ex->needsinitialization) {
986:     HYPRE_SStructMatrixInitialize(ex->ss_mat);
987:     ex->needsinitialization = PETSC_FALSE;
988:   }
989: 

991:   /* set the global and local sizes of the matrix */
992:   DAGetCorners(da,0,0,0,&nx,&ny,&nz);
993:   MatSetSizes(mat,dof*nx*ny*nz,dof*nx*ny*nz,PETSC_DECIDE,PETSC_DECIDE);
994:   PetscLayoutSetBlockSize(mat->rmap,1);
995:   PetscLayoutSetBlockSize(mat->cmap,1);
996:   PetscLayoutSetUp(mat->rmap);
997:   PetscLayoutSetUp(mat->cmap);
998: 
999:   if (dim == 3) {
1000:     mat->ops->setvalueslocal = MatSetValuesLocal_HYPRESStruct_3d;
1001:     mat->ops->zerorowslocal  = MatZeroRowsLocal_HYPRESStruct_3d;
1002:     mat->ops->zeroentries    = MatZeroEntries_HYPRESStruct_3d;
1003:     MatZeroEntries_HYPRESStruct_3d(mat);
1004:   } else SETERRQ(PETSC_ERR_SUP,"Only support for 3d DA currently");
1005: 
1006:   /* get values that will be used repeatedly in MatSetValuesLocal() and MatZeroRowsLocal() repeatedly */
1007:   MatGetOwnershipRange(mat,&ex->rstart,PETSC_NULL);
1008:   DAGetGlobalIndices(ex->da,PETSC_NULL,&ex->gindices);
1009:   DAGetGhostCorners(ex->da,0,0,0,&ex->gnx,&ex->gnxgny,&ex->gnxgnygnz);
1010:   ex->gnxgny    *= ex->gnx;
1011:   ex->gnxgnygnz *= ex->gnxgny;
1012:   DAGetCorners(ex->da,&ex->xs,&ex->ys,&ex->zs,&ex->nx,&ex->ny,&ex->nz);
1013:   ex->nxny   = ex->nx*ex->ny;
1014:   ex->nxnynz = ex->nz*ex->nxny;
1015:   return(0);
1016: }
1017: 
1020: PetscErrorCode MatMult_HYPRESStruct(Mat A,Vec x,Vec y)
1021: {
1022:   PetscErrorCode    ierr;
1023:   PetscScalar      *xx,*yy;
1024:   int               ilower[3],iupper[3];
1025:   Mat_HYPRESStruct *mx = (Mat_HYPRESStruct *)(A->data);
1026:   int               ordering= mx->dofs_order;
1027:   int               nvars= mx->nvars;
1028:   int               part= 0;
1029:   int               size;
1030:   int               i;
1031: 
1033:   DAGetCorners(mx->da,&ilower[0],&ilower[1],&ilower[2],&iupper[0],&iupper[1],&iupper[2]);
1034:   iupper[0] += ilower[0] - 1;
1035:   iupper[1] += ilower[1] - 1;
1036:   iupper[2] += ilower[2] - 1;

1038:   size= 1;
1039:   for (i= 0; i< 3; i++) {
1040:      size*= (iupper[i]-ilower[i]+1);
1041:   }

1043:   /* copy x values over to hypre for variable ordering */
1044:   if (ordering) {
1045:      HYPRE_SStructVectorSetConstantValues(mx->ss_b,0.0);
1046:      VecGetArray(x,&xx);
1047:      for (i= 0; i< nvars; i++) {
1048:         HYPRE_SStructVectorSetBoxValues(mx->ss_b,part,ilower,iupper,i,xx+(size*i));
1049:      }
1050:      VecRestoreArray(x,&xx);
1051:      HYPRE_SStructVectorAssemble(mx->ss_b);
1052: 
1053:      HYPRE_SStructMatrixMatvec(1.0,mx->ss_mat,mx->ss_b,0.0,mx->ss_x);

1055:      /* copy solution values back to PETSc */
1056:      VecGetArray(y,&yy);
1057:      for (i= 0; i< nvars; i++) {
1058:         HYPRE_SStructVectorGetBoxValues(mx->ss_x,part,ilower,iupper,i,yy+(size*i));
1059:      }
1060:      VecRestoreArray(y,&yy);
1061:   }

1063:   else {      /* nodal ordering must be mapped to variable ordering for sys_pfmg */
1064:      PetscScalar     *z;
1065:      int              j, k;

1067:      PetscMalloc(nvars*size*sizeof(PetscScalar),&z);
1068:      HYPRE_SStructVectorSetConstantValues(mx->ss_b,0.0);
1069:      VecGetArray(x,&xx);

1071:      /* transform nodal to hypre's variable ordering for sys_pfmg */
1072:      for (i= 0; i< size; i++) {
1073:         k= i*nvars;
1074:         for (j= 0; j< nvars; j++) {
1075:            z[j*size+i]= xx[k+j];
1076:         }
1077:      }
1078:      for (i= 0; i< nvars; i++) {
1079:         HYPRE_SStructVectorSetBoxValues(mx->ss_b,part,ilower,iupper,i,z+(size*i));
1080:      }
1081:      VecRestoreArray(x,&xx);

1083:      HYPRE_SStructVectorAssemble(mx->ss_b);
1084: 
1085:      HYPRE_SStructMatrixMatvec(1.0,mx->ss_mat,mx->ss_b,0.0,mx->ss_x);
1086: 
1087:      /* copy solution values back to PETSc */
1088:      VecGetArray(y,&yy);
1089:      for (i= 0; i< nvars; i++) {
1090:         HYPRE_SStructVectorGetBoxValues(mx->ss_x,part,ilower,iupper,i,z+(size*i));
1091:      }
1092:      /* transform hypre's variable ordering for sys_pfmg to nodal ordering */
1093:      for (i= 0; i< size; i++) {
1094:         k= i*nvars;
1095:         for (j= 0; j< nvars; j++) {
1096:            yy[k+j]= z[j*size+i];
1097:         }
1098:      }
1099:      VecRestoreArray(y,&yy);

1101:      PetscFree(z);
1102:   }

1104:   return(0);
1105: }

1109: PetscErrorCode MatAssemblyEnd_HYPRESStruct(Mat mat,MatAssemblyType mode)
1110: {
1111:   Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;
1112:   PetscErrorCode   ierr;

1115: printf("look 1\n");
1116:   HYPRE_SStructMatrixAssemble(ex->ss_mat);
1117: printf("look 2\n");
1118:   return(0);
1119: }

1123: PetscErrorCode MatZeroEntries_HYPRESStruct(Mat mat)
1124: {
1126:   /* before the DA is set to the matrix the zero doesn't need to do anything */
1127:   return(0);
1128: }


1133: PetscErrorCode MatDestroy_HYPRESStruct(Mat mat)
1134: {
1135:   Mat_HYPRESStruct *ex = (Mat_HYPRESStruct*) mat->data;
1136:   PetscErrorCode  ierr;

1139:   HYPRE_SStructGraphDestroy(ex->ss_graph);
1140:   HYPRE_SStructMatrixDestroy(ex->ss_mat);
1141:   HYPRE_SStructVectorDestroy(ex->ss_x);
1142:   HYPRE_SStructVectorDestroy(ex->ss_b);
1143:   return(0);
1144: }

1149: PetscErrorCode  MatCreate_HYPRESStruct(Mat B)
1150: {
1151:   Mat_HYPRESStruct *ex;
1152:   PetscErrorCode   ierr;

1155:   PetscNewLog(B,Mat_HYPRESStruct,&ex);
1156:   B->data         = (void*)ex;
1157:   B->rmap->bs     = 1;
1158:   B->assembled    = PETSC_FALSE;
1159:   B->mapping      = 0;

1161:   B->insertmode   = NOT_SET_VALUES;

1163:   B->ops->assemblyend    = MatAssemblyEnd_HYPRESStruct;
1164:   B->ops->mult           = MatMult_HYPRESStruct;
1165:   B->ops->zeroentries    = MatZeroEntries_HYPRESStruct;
1166:   B->ops->destroy        = MatDestroy_HYPRESStruct;

1168:   ex->needsinitialization = PETSC_TRUE;
1169: 
1170:   MPI_Comm_dup(((PetscObject)B)->comm,&(ex->hcomm));
1171:   PetscObjectComposeFunctionDynamic((PetscObject)B,"MatSetDA_C","MatSetDA_HYPRESStruct",MatSetDA_HYPRESStruct);
1172:   PetscObjectChangeTypeName((PetscObject)B,MATHYPRESSTRUCT);
1173:   return(0);
1174: }

1177: #endif