Actual source code: sliced.c

  1: #define PETSCDM_DLL
  2: 
 3:  #include petscda.h
 4:  #include petscmat.h
 5:  #include private/dmimpl.h


  8: typedef struct _SlicedOps *SlicedOps;
  9: struct _SlicedOps {
 10:   DMOPS(Sliced)
 11: };

 13: /* CSR storage of the nonzero structure of a bs*bs matrix */
 14: typedef struct {
 15:   PetscInt bs,nz,*i,*j;
 16: } SlicedBlockFills;

 18: struct _p_Sliced {
 19:   PETSCHEADER(struct _SlicedOps);
 20:   DMHEADER
 21:   Vec      globalvector;
 22:   PetscInt bs,n,N,Nghosts,*ghosts;
 23:   PetscInt d_nz,o_nz,*d_nnz,*o_nnz;
 24:   SlicedBlockFills *dfill,*ofill;
 25: };

 29: /*@C
 30:     SlicedGetMatrix - Creates a matrix with the correct parallel layout required for 
 31:       computing the Jacobian on a function defined using the informatin in Sliced.

 33:     Collective on Sliced

 35:     Input Parameter:
 36: +   slice - the slice object
 37: -   mtype - Supported types are MATSEQAIJ, MATMPIAIJ, MATSEQBAIJ, MATMPIBAIJ, MATSEQSBAIJ, MATMPISBAIJ,
 38:             or any type which inherits from one of these (such as MATAIJ, MATLUSOL, etc.).

 40:     Output Parameters:
 41: .   J  - matrix with the correct nonzero preallocation
 42:         (obviously without the correct Jacobian values)

 44:     Level: advanced

 46:     Notes: This properly preallocates the number of nonzeros in the sparse matrix so you 
 47:        do not need to do it yourself.

 49: .seealso ISColoringView(), ISColoringGetIS(), MatFDColoringCreate(), DASetBlockFills()

 51: @*/
 52: PetscErrorCode  SlicedGetMatrix(Sliced slice, const MatType mtype,Mat *J)
 53: {
 54:   PetscErrorCode         ierr;
 55:   PetscInt               *globals,*sd_nnz,*so_nnz,rstart,bs,i;
 56:   ISLocalToGlobalMapping lmap,blmap;
 57:   void                   (*aij)(void) = PETSC_NULL;

 60:   bs = slice->bs;
 61:   MatCreate(((PetscObject)slice)->comm,J);
 62:   MatSetSizes(*J,slice->n*bs,slice->n*bs,PETSC_DETERMINE,PETSC_DETERMINE);
 63:   MatSetType(*J,mtype);
 64:   MatSeqBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz);
 65:   MatMPIBAIJSetPreallocation(*J,bs,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);
 66:   /* In general, we have to do extra work to preallocate for scalar (AIJ) matrices so we check whether it will do any
 67:   * good before going on with it. */
 68:   PetscObjectQueryFunction((PetscObject)*J,"MatMPIAIJSetPreallocation_C",&aij);
 69:   if (!aij) {
 70:     PetscObjectQueryFunction((PetscObject)*J,"MatSeqAIJSetPreallocation_C",&aij);
 71:   }
 72:   if (aij) {
 73:     if (bs == 1) {
 74:       MatSeqAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz);
 75:       MatMPIAIJSetPreallocation(*J,slice->d_nz,slice->d_nnz,slice->o_nz,slice->o_nnz);
 76:     } else if (!slice->d_nnz) {
 77:       MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL);
 78:       MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,PETSC_NULL,slice->o_nz*bs,PETSC_NULL);
 79:     } else {
 80:       /* The user has provided preallocation per block-row, convert it to per scalar-row respecting SlicedSetBlockFills() if applicable */
 81:       PetscMalloc2(slice->n*bs,PetscInt,&sd_nnz,(!!slice->o_nnz)*slice->n*bs,PetscInt,&so_nnz);
 82:       for (i=0; i<slice->n*bs; i++) {
 83:         sd_nnz[i] = (slice->d_nnz[i/bs]-1) * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs)
 84:                                            + (slice->dfill ? slice->dfill->i[i%bs+1]-slice->dfill->i[i%bs] : bs);
 85:         if (so_nnz) {
 86:           so_nnz[i] = slice->o_nnz[i/bs] * (slice->ofill ? slice->ofill->i[i%bs+1]-slice->ofill->i[i%bs] : bs);
 87:         }
 88:       }
 89:       MatSeqAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz);
 90:       MatMPIAIJSetPreallocation(*J,slice->d_nz*bs,sd_nnz,slice->o_nz*bs,so_nnz);
 91:       PetscFree2(sd_nnz,so_nnz);
 92:     }
 93:   }

 95:   MatSetBlockSize(*J,bs);

 97:   /* Set up the local to global map.  For the scalar map, we have to translate to entry-wise indexing instead of block-wise. */
 98:   PetscMalloc((slice->n+slice->Nghosts)*bs*sizeof(PetscInt),&globals);
 99:   MatGetOwnershipRange(*J,&rstart,PETSC_NULL);
100:   for (i=0; i<slice->n*bs; i++) {
101:     globals[i] = rstart + i;
102:   }
103:   for (i=0; i<slice->Nghosts*bs; i++) {
104:     globals[slice->n*bs+i] = slice->ghosts[i/bs]*bs + i%bs;
105:   }
106:   ISLocalToGlobalMappingCreate(PETSC_COMM_SELF,(slice->n+slice->Nghosts)*bs,globals,&lmap);
107:   PetscFree(globals);
108:   ISLocalToGlobalMappingBlock(lmap,bs,&blmap);
109:   MatSetLocalToGlobalMapping(*J,lmap);
110:   MatSetLocalToGlobalMappingBlock(*J,blmap);
111:   ISLocalToGlobalMappingDestroy(lmap);
112:   ISLocalToGlobalMappingDestroy(blmap);
113:   return(0);
114: }

118: /*@C
119:     SlicedSetGhosts - Sets the global indices of other processes elements that will
120:       be ghosts on this process

122:     Not Collective

124:     Input Parameters:
125: +    slice - the Sliced object
126: .    bs - block size
127: .    nlocal - number of local (owned, non-ghost) blocks
128: .    Nghosts - number of ghost blocks on this process
129: -    ghosts - global indices of each ghost block

131:     Level: advanced

133: .seealso SlicedDestroy(), SlicedCreateGlobalVector(), SlicedGetGlobalIndices()

135: @*/
136: PetscErrorCode  SlicedSetGhosts(Sliced slice,PetscInt bs,PetscInt nlocal,PetscInt Nghosts,const PetscInt ghosts[])
137: {

142:   PetscFree(slice->ghosts);
143:   PetscMalloc(Nghosts*sizeof(PetscInt),&slice->ghosts);
144:   PetscMemcpy(slice->ghosts,ghosts,Nghosts*sizeof(PetscInt));
145:   slice->bs      = bs;
146:   slice->n       = nlocal;
147:   slice->Nghosts = Nghosts;
148:   return(0);
149: }

153: /*@C
154:     SlicedSetPreallocation - sets the matrix memory preallocation for matrices computed by Sliced

156:     Not Collective

158:     Input Parameters:
159: +    slice - the Sliced object
160: .    d_nz  - number of block nonzeros per block row in diagonal portion of local
161:            submatrix  (same for all local rows)
162: .    d_nnz - array containing the number of block nonzeros in the various block rows
163:            of the in diagonal portion of the local (possibly different for each block
164:            row) or PETSC_NULL.  You must leave room for the diagonal entry even if it is zero.
165: .    o_nz  - number of block nonzeros per block row in the off-diagonal portion of local
166:            submatrix (same for all local rows).
167: -    o_nnz - array containing the number of nonzeros in the various block rows of the
168:            off-diagonal portion of the local submatrix (possibly different for
169:            each block row) or PETSC_NULL.

171:     Notes:
172:     See MatMPIBAIJSetPreallocation() for more details on preallocation.  If a scalar matrix (AIJ) is
173:     obtained with SlicedGetMatrix(), the correct preallocation will be set, respecting SlicedSetBlockFills().

175:     Level: advanced

177: .seealso SlicedDestroy(), SlicedCreateGlobalVector(), SlicedGetGlobalIndices(), MatMPIAIJSetPreallocation(),
178:          MatMPIBAIJSetPreallocation(), SlicedGetMatrix(), SlicedSetBlockFills()

180: @*/
181: PetscErrorCode  SlicedSetPreallocation(Sliced slice,PetscInt d_nz,const PetscInt d_nnz[],PetscInt o_nz,const PetscInt o_nnz[])
182: {
185:   slice->d_nz  = d_nz;
186:   slice->d_nnz = (PetscInt*)d_nnz;
187:   slice->o_nz  = o_nz;
188:   slice->o_nnz = (PetscInt*)o_nnz;
189:   return(0);
190: }

194: static PetscErrorCode SlicedSetBlockFills_Private(PetscInt bs,const PetscInt *fill,SlicedBlockFills **inf)
195: {
196:   PetscErrorCode   ierr;
197:   PetscInt         i,j,nz,*fi,*fj;
198:   SlicedBlockFills *f;

202:   if (*inf) {PetscFree3((*inf)->i,(*inf)->j,*inf);}
203:   if (!fill) return(0);
204:   for (i=0,nz=0; i<bs*bs; i++) if (fill[i]) nz++;
205:   PetscMalloc3(1,SlicedBlockFills,&f,bs+1,PetscInt,&fi,nz,PetscInt,&fj);
206:   f->bs = bs;
207:   f->nz = nz;
208:   f->i  = fi;
209:   f->j  = fj;
210:   for (i=0,nz=0; i<bs; i++) {
211:     fi[i] = nz;
212:     for (j=0; j<bs; j++) if (fill[i*bs+j]) fj[nz++] = j;
213:   }
214:   fi[i] = nz;
215:   *inf = f;
216:   return(0);
217: }

221: /*@C
222:     SlicedSetBlockFills - Sets the fill pattern in each block for a multi-component problem
223:     of the matrix returned by SlicedGetMatrix().

225:     Collective on Sliced

227:     Input Parameter:
228: +   sliced - the Sliced object
229: .   dfill - the fill pattern in the diagonal block (may be PETSC_NULL, means use dense block)
230: -   ofill - the fill pattern in the off-diagonal blocks

232:     Notes:
233:     This only makes sense for multicomponent problems using scalar matrix formats (AIJ).
234:     See DASetBlockFills() for example usage.

236:     Level: advanced

238: .seealso SlicedGetMatrix(), DASetBlockFills()
239: @*/
240: PetscErrorCode  SlicedSetBlockFills(Sliced slice,const PetscInt *dfill,const PetscInt *ofill)
241: {

246:   SlicedSetBlockFills_Private(slice->bs,dfill,&slice->dfill);
247:   SlicedSetBlockFills_Private(slice->bs,ofill,&slice->ofill);
248:   return(0);
249: }

253: /*@C
254:     SlicedCreate - Creates a DM object, used to manage data for a unstructured problem

256:     Collective on MPI_Comm

258:     Input Parameter:
259: .   comm - the processors that will share the global vector

261:     Output Parameters:
262: .   slice - the slice object

264:     Level: advanced

266: .seealso SlicedDestroy(), SlicedCreateGlobalVector(), SlicedGetGlobalIndices()

268: @*/
269: PetscErrorCode  SlicedCreate(MPI_Comm comm,Sliced *slice)
270: {
272:   Sliced         p;

276:   *slice = PETSC_NULL;
277: #ifndef PETSC_USE_DYNAMIC_LIBRARIES
278:   DMInitializePackage(PETSC_NULL);
279: #endif

281:   PetscHeaderCreate(p,_p_Sliced,struct _SlicedOps,DM_COOKIE,0,"DM",comm,SlicedDestroy,0);
282:   PetscObjectChangeTypeName((PetscObject)p,"Sliced");
283:   p->ops->createglobalvector = SlicedCreateGlobalVector;
284:   p->ops->getmatrix          = SlicedGetMatrix;
285:   p->ops->destroy            = SlicedDestroy;
286:   *slice = p;
287:   return(0);
288: }


294: /*@C
295:     SlicedDestroy - Destroys a vector slice.

297:     Collective on Sliced

299:     Input Parameter:
300: .   slice - the slice object

302:     Level: advanced

304: .seealso SlicedCreate(), SlicedCreateGlobalVector(), SlicedGetGlobalIndices()

306: @*/
307: PetscErrorCode  SlicedDestroy(Sliced slice)
308: {
310:   PetscTruth     done;

313:   DMDestroy_Private((DM)slice,&done);
314:   if (!done) return(0);

316:   if (slice->globalvector) {VecDestroy(slice->globalvector);}
317:   PetscFree(slice->ghosts);
318:   if (slice->dfill) {PetscFree3(slice->dfill,slice->dfill->i,slice->dfill->j);}
319:   if (slice->ofill) {PetscFree3(slice->ofill,slice->ofill->i,slice->ofill->j);}
320:   PetscHeaderDestroy(slice);
321:   return(0);
322: }


327: /*@C
328:     SlicedCreateGlobalVector - Creates a vector of the correct size to be gathered into
329:         by the slice.

331:     Collective on Sliced

333:     Input Parameter:
334: .    slice - the slice object

336:     Output Parameters:
337: .   gvec - the global vector

339:     Level: advanced

341:     Notes: Once this has been created you cannot add additional arrays or vectors to be packed.

343: .seealso SlicedDestroy(), SlicedCreate(), SlicedGetGlobalIndices()

345: @*/
346: PetscErrorCode  SlicedCreateGlobalVector(Sliced slice,Vec *gvec)
347: {
348:   PetscErrorCode     ierr;
349:   PetscInt           bs,cnt,i;

354:   *gvec = 0;
355:   if (slice->globalvector) {
356:     PetscObjectGetReference((PetscObject)slice->globalvector,&cnt);
357:     if (cnt == 1) {             /* Nobody else has a reference so we can just reference it and give it away */
358:       *gvec = slice->globalvector;
359:       PetscObjectReference((PetscObject)*gvec);
360:       VecZeroEntries(*gvec);
361:     } else {                    /* Someone else has a reference so we duplicate the global vector */
362:       VecDuplicate(slice->globalvector,gvec);
363:     }
364:   } else {
365:     bs = slice->bs;
366:     /* VecCreateGhostBlock requires ghosted blocks to be given in terms of first entry, not block.  Here, we munge the
367:     * ghost array for this call, then put it back. */
368:     for (i=0; i<slice->Nghosts; i++) slice->ghosts[i] *= bs;
369:     VecCreateGhostBlock(((PetscObject)slice)->comm,bs,slice->n*bs,PETSC_DETERMINE,slice->Nghosts,slice->ghosts,&slice->globalvector);
370:     for (i=0; i<slice->Nghosts; i++) slice->ghosts[i] /= bs;
371:     *gvec = slice->globalvector;
372:     PetscObjectReference((PetscObject)*gvec);
373:   }
374:   return(0);
375: }

379: /*@C
380:     SlicedGetGlobalIndices - Gets the global indices for all the local entries

382:     Collective on Sliced

384:     Input Parameter:
385: .    slice - the slice object

387:     Output Parameters:
388: .    idx - the individual indices for each packed vector/array

390:     Level: advanced

392:     Notes:
393:        The idx parameters should be freed by the calling routine with PetscFree()

395: .seealso SlicedDestroy(), SlicedCreateGlobalVector(), SlicedCreate()

397: @*/
398: PetscErrorCode  SlicedGetGlobalIndices(Sliced slice,PetscInt *idx[])
399: {
400:   return(0);
401: }


404: /* Explanation of the missing functions for DA-style handling of the local vector:

406:    SlicedCreateLocalVector()
407:    SlicedGlobalToLocalBegin()
408:    SlicedGlobalToLocalEnd()

410:  There is no way to get the global form from a local form, so SlicedCreateLocalVector() is a memory leak without
411:  external accounting for the global vector.  Also, Sliced intends the user to work with the VecGhost interface since the
412:  ghosts are already ordered after the owned entries.  Contrast this to a DA where the local vector has a special
413:  ordering described by the structured grid, hence it cannot share memory with the global form.  For this reason, users
414:  of Sliced should work with the global vector and use

416:    VecGhostGetLocalForm(), VecGhostRestoreLocalForm()
417:    VecGhostUpdateBegin(), VecGhostUpdateEnd()

419:  rather than the missing DA-style functions.  This is conceptually simpler and offers better performance than is
420:  possible with the DA-style interface.
421: */