Actual source code: asm.c

  1: #define PETSCKSP_DLL

  3: /*
  4:   This file defines an additive Schwarz preconditioner for any Mat implementation.

  6:   Note that each processor may have any number of subdomains. But in order to 
  7:   deal easily with the VecScatter(), we treat each processor as if it has the
  8:   same number of subdomains.

 10:        n - total number of true subdomains on all processors
 11:        n_local_true - actual number of subdomains on this processor
 12:        n_local = maximum over all processors of n_local_true
 13: */
 14:  #include private/pcimpl.h

 16: typedef struct {
 17:   PetscInt   n, n_local, n_local_true;
 18:   PetscInt   overlap;             /* overlap requested by user */
 19:   KSP        *ksp;                /* linear solvers for each block */
 20:   VecScatter *restriction;        /* mapping from global to subregion */
 21:   VecScatter *localization;       /* mapping from overlapping to non-overlapping subregion */
 22:   VecScatter *prolongation;       /* mapping from subregion to global */
 23:   Vec        *x,*y,*y_local;      /* work vectors */
 24:   IS         *is;                 /* index set that defines each overlapping subdomain */
 25:   IS         *is_local;           /* index set that defines each non-overlapping subdomain, may be NULL */
 26:   Mat        *mat,*pmat;          /* mat is not currently used */
 27:   PCASMType  type;                /* use reduced interpolation, restriction or both */
 28:   PetscTruth type_set;            /* if user set this value (so won't change it for symmetric problems) */
 29:   PetscTruth same_local_solves;   /* flag indicating whether all local solvers are same */
 30:   PetscTruth sort_indices;        /* flag to sort subdomain indices */
 31: } PC_ASM;

 35: static PetscErrorCode PCView_ASM(PC pc,PetscViewer viewer)
 36: {
 37:   PC_ASM         *osm = (PC_ASM*)pc->data;
 39:   PetscMPIInt    rank;
 40:   PetscInt       i,bsz;
 41:   PetscTruth     iascii,isstring;
 42:   PetscViewer    sviewer;


 46:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 47:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_STRING,&isstring);
 48:   if (iascii) {
 49:     if (osm->n > 0) {
 50:       PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: total subdomain blocks = %D, amount of overlap = %D\n",osm->n,osm->overlap);
 51:     } else {
 52:       PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: total subdomain blocks not yet set, amount of overlap = %D\n",osm->overlap);
 53:     }
 54:     PetscViewerASCIIPrintf(viewer,"  Additive Schwarz: restriction/interpolation type - %s\n",PCASMTypes[osm->type]);
 55:     MPI_Comm_rank(((PetscObject)pc)->comm,&rank);
 56:     if (osm->same_local_solves) {
 57:       if (osm->ksp) {
 58:         PetscViewerASCIIPrintf(viewer,"  Local solve is same for all blocks, in the following KSP and PC objects:\n");
 59:         PetscViewerGetSingleton(viewer,&sviewer);
 60:         if (!rank) {
 61:           PetscViewerASCIIPushTab(viewer);
 62:           KSPView(osm->ksp[0],sviewer);
 63:           PetscViewerASCIIPopTab(viewer);
 64:         }
 65:         PetscViewerRestoreSingleton(viewer,&sviewer);
 66:       }
 67:     } else {
 68:       PetscViewerASCIISynchronizedPrintf(viewer,"  [%d] number of local blocks = %D\n",(int)rank,osm->n_local_true);
 69:       PetscViewerFlush(viewer);
 70:       PetscViewerASCIIPrintf(viewer,"  Local solve info for each block is in the following KSP and PC objects:\n");
 71:       PetscViewerASCIIPushTab(viewer);
 72:       PetscViewerASCIIPrintf(viewer,"- - - - - - - - - - - - - - - - - -\n");
 73:       for (i=0; i<osm->n_local; i++) {
 74:         PetscViewerGetSingleton(viewer,&sviewer);
 75:         if (i < osm->n_local_true) {
 76:           ISGetLocalSize(osm->is[i],&bsz);
 77:           PetscViewerASCIISynchronizedPrintf(sviewer,"[%d] local block number %D, size = %D\n",(int)rank,i,bsz);
 78:           KSPView(osm->ksp[i],sviewer);
 79:           PetscViewerASCIISynchronizedPrintf(sviewer,"- - - - - - - - - - - - - - - - - -\n");
 80:         }
 81:         PetscViewerRestoreSingleton(viewer,&sviewer);
 82:       }
 83:       PetscViewerASCIIPopTab(viewer);
 84:       PetscViewerFlush(viewer);
 85:     }
 86:   } else if (isstring) {
 87:     PetscViewerStringSPrintf(viewer," blocks=%D, overlap=%D, type=%s",osm->n,osm->overlap,PCASMTypes[osm->type]);
 88:     PetscViewerGetSingleton(viewer,&sviewer);
 89:     if (osm->ksp) {KSPView(osm->ksp[0],sviewer);}
 90:     PetscViewerRestoreSingleton(viewer,&sviewer);
 91:   } else {
 92:     SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for PCASM",((PetscObject)viewer)->type_name);
 93:   }
 94:   return(0);
 95: }

 99: static PetscErrorCode PCASMPrintSubdomains(PC pc)
100: {
101:   PC_ASM         *osm  = (PC_ASM*)pc->data;
102:   const char     *prefix;
103:   char           fname[PETSC_MAX_PATH_LEN+1];
104:   PetscViewer    viewer;
105:   PetscInt       i,j,nidx;
106:   const PetscInt *idx;

110:   PCGetOptionsPrefix(pc,&prefix);
111:   PetscOptionsGetString(prefix,"-pc_asm_print_subdomains",fname,PETSC_MAX_PATH_LEN,PETSC_NULL);
112:   if (fname[0] == 0) { PetscStrcpy(fname,"stdout"); };
113:   PetscViewerASCIIOpen(((PetscObject)pc)->comm,fname,&viewer);
114:   for (i=0;i<osm->n_local_true;i++) {
115:     ISGetLocalSize(osm->is[i],&nidx);
116:     ISGetIndices(osm->is[i],&idx);
117:     for (j=0; j<nidx; j++) {
118:       PetscViewerASCIISynchronizedPrintf(viewer,"%D ",idx[j]);
119:     }
120:     ISRestoreIndices(osm->is[i],&idx);
121:     PetscViewerASCIISynchronizedPrintf(viewer,"\n");
122:     if (osm->is_local) {
123:       ISGetLocalSize(osm->is_local[i],&nidx);
124:       ISGetIndices(osm->is_local[i],&idx);
125:       for (j=0; j<nidx; j++) {
126:         PetscViewerASCIISynchronizedPrintf(viewer,"%D ",idx[j]);
127:       }
128:       ISRestoreIndices(osm->is_local[i],&idx);
129:       PetscViewerASCIISynchronizedPrintf(viewer,"\n");
130:     }
131:   }
132:   PetscViewerFlush(viewer);
133:   PetscViewerDestroy(viewer);
134:   return(0);
135: }

139: static PetscErrorCode PCSetUp_ASM(PC pc)
140: {
141:   PC_ASM         *osm  = (PC_ASM*)pc->data;
143:   PetscTruth     symset,flg;
144:   PetscInt       i,m,m_local,firstRow,lastRow;
145:   PetscMPIInt    size;
146:   MatReuse       scall = MAT_REUSE_MATRIX;
147:   IS             isl;
148:   KSP            ksp;
149:   PC             subpc;
150:   const char     *prefix,*pprefix;
151:   Vec            vec;

154:   if (!pc->setupcalled) {

156:     if (!osm->type_set) {
157:       MatIsSymmetricKnown(pc->pmat,&symset,&flg);
158:       if (symset && flg) { osm->type = PC_ASM_BASIC; }
159:     }

161:     if (osm->n == PETSC_DECIDE && osm->n_local_true < 1) {
162:       /* no subdomains given, use one per processor */
163:       osm->n_local = osm->n_local_true = 1;
164:       MPI_Comm_size(((PetscObject)pc)->comm,&size);
165:       osm->n = size;
166:     } else if (osm->n == PETSC_DECIDE) {
167:       /* determine global number of subdomains */
168:       PetscInt inwork[2],outwork[2];
169:       inwork[0] = inwork[1] = osm->n_local_true;
170:       MPI_Allreduce(inwork,outwork,1,MPIU_2INT,PetscMaxSum_Op,((PetscObject)pc)->comm);
171:       osm->n_local = outwork[0];
172:       osm->n       = outwork[1];
173:     }

175:     if (!osm->is){ /* create the index sets */
176:       PCASMCreateSubdomains(pc->pmat,osm->n_local_true,&osm->is);
177:     }
178:     if (osm->n_local_true > 1 && !osm->is_local) {
179:       PetscMalloc(osm->n_local_true*sizeof(IS),&osm->is_local);
180:       for (i=0; i<osm->n_local_true; i++) {
181:         if (osm->overlap > 0) { /* With positive overlap, osm->is[i] will be modified */
182:           ISDuplicate(osm->is[i],&osm->is_local[i]);
183:           ISCopy(osm->is[i],osm->is_local[i]);
184:         } else {
185:           PetscObjectReference((PetscObject)osm->is[i]);
186:           osm->is_local[i] = osm->is[i];
187:         }
188:       }
189:     }
190:     PCGetOptionsPrefix(pc,&prefix);
191:     flg  = PETSC_FALSE;
192:     PetscOptionsGetTruth(prefix,"-pc_asm_print_subdomains",&flg,PETSC_NULL);
193:     if (flg) { PCASMPrintSubdomains(pc); }

195:     /*  Extend the "overlapping" regions by a number of steps  */
196:     MatIncreaseOverlap(pc->pmat,osm->n_local_true,osm->is,osm->overlap);
197:     if (osm->sort_indices) {
198:       for (i=0; i<osm->n_local_true; i++) {
199:         ISSort(osm->is[i]);
200:         if (osm->is_local) {
201:           ISSort(osm->is_local[i]);
202:         }
203:       }
204:     }

206:     /* Create the local work vectors and scatter contexts */
207:     MatGetVecs(pc->pmat,&vec,0);
208:     PetscMalloc(osm->n_local*sizeof(VecScatter *),&osm->restriction);
209:     if (osm->is_local) {PetscMalloc(osm->n_local*sizeof(VecScatter *),&osm->localization);}
210:     PetscMalloc(osm->n_local*sizeof(VecScatter *),&osm->prolongation);
211:     PetscMalloc(osm->n_local*sizeof(Vec *),&osm->x);
212:     PetscMalloc(osm->n_local*sizeof(Vec *),&osm->y);
213:     PetscMalloc(osm->n_local*sizeof(Vec *),&osm->y_local);
214:     VecGetOwnershipRange(vec, &firstRow, &lastRow);
215:     for (i=0; i<osm->n_local_true; ++i, firstRow += m_local) {
216:       ISGetLocalSize(osm->is[i],&m);
217:       VecCreateSeq(PETSC_COMM_SELF,m,&osm->x[i]);
218:       ISCreateStride(PETSC_COMM_SELF,m,0,1,&isl);
219:       VecScatterCreate(vec,osm->is[i],osm->x[i],isl,&osm->restriction[i]);
220:       ISDestroy(isl);
221:       VecDuplicate(osm->x[i],&osm->y[i]);
222:       if (osm->is_local) {
223:         ISLocalToGlobalMapping ltog;
224:         IS                     isll;
225:         const PetscInt         *idx_local;
226:         PetscInt               *idx,nout;

228:         ISLocalToGlobalMappingCreateIS(osm->is[i],&ltog);
229:         ISGetLocalSize(osm->is_local[i],&m_local);
230:         ISGetIndices(osm->is_local[i], &idx_local);
231:         PetscMalloc(m_local*sizeof(PetscInt),&idx);
232:         ISGlobalToLocalMappingApply(ltog,IS_GTOLM_DROP,m_local,idx_local,&nout,idx);
233:         ISLocalToGlobalMappingDestroy(ltog);
234:         if (nout != m_local) SETERRQ(PETSC_ERR_PLIB,"is_local not a subset of is");
235:         ISRestoreIndices(osm->is_local[i], &idx_local);
236:         ISCreateGeneral(PETSC_COMM_SELF,m_local,idx,&isll);
237:         PetscFree(idx);
238:         ISCreateStride(PETSC_COMM_SELF,m_local,0,1,&isl);
239:         VecCreateSeq(PETSC_COMM_SELF,m_local,&osm->y_local[i]);
240:         VecScatterCreate(osm->y[i],isll,osm->y_local[i],isl,&osm->localization[i]);
241:         ISDestroy(isll);

243:         VecScatterCreate(vec,osm->is_local[i],osm->y_local[i],isl,&osm->prolongation[i]);
244:         ISDestroy(isl);
245:       } else {
246:         VecGetLocalSize(vec,&m_local);
247:         osm->y_local[i] = osm->y[i];
248:         PetscObjectReference((PetscObject) osm->y[i]);
249:         osm->prolongation[i] = osm->restriction[i];
250:         PetscObjectReference((PetscObject) osm->restriction[i]);
251:       }
252:     }
253:     if (firstRow != lastRow) SETERRQ2(PETSC_ERR_PLIB, "Specified ASM subdomain sizes were invalid: %d != %d", firstRow, lastRow);
254:     for (i=osm->n_local_true; i<osm->n_local; i++) {
255:       VecCreateSeq(PETSC_COMM_SELF,0,&osm->x[i]);
256:       VecDuplicate(osm->x[i],&osm->y[i]);
257:       VecDuplicate(osm->x[i],&osm->y_local[i]);
258:       ISCreateStride(PETSC_COMM_SELF,0,0,1,&isl);
259:       VecScatterCreate(vec,isl,osm->x[i],isl,&osm->restriction[i]);
260:       if (osm->is_local) {
261:         VecScatterCreate(osm->y[i],isl,osm->y_local[i],isl,&osm->localization[i]);
262:         VecScatterCreate(vec,isl,osm->x[i],isl,&osm->prolongation[i]);
263:       } else {
264:         osm->prolongation[i] = osm->restriction[i];
265:         PetscObjectReference((PetscObject) osm->restriction[i]);
266:       }
267:       ISDestroy(isl);
268:     }
269:     VecDestroy(vec);

271:     /* Create the local solvers */
272:     PetscMalloc(osm->n_local_true*sizeof(KSP *),&osm->ksp);
273:     for (i=0; i<osm->n_local_true; i++) {
274:       KSPCreate(PETSC_COMM_SELF,&ksp);
275:       PetscLogObjectParent(pc,ksp);
276:       PetscObjectIncrementTabLevel((PetscObject)ksp,(PetscObject)pc,1);
277:       KSPSetType(ksp,KSPPREONLY);
278:       KSPGetPC(ksp,&subpc);
279:       PCGetOptionsPrefix(pc,&prefix);
280:       KSPSetOptionsPrefix(ksp,prefix);
281:       KSPAppendOptionsPrefix(ksp,"sub_");
282:       osm->ksp[i] = ksp;
283:     }
284:     scall = MAT_INITIAL_MATRIX;

286:   } else {
287:     /* 
288:        Destroy the blocks from the previous iteration
289:     */
290:     if (pc->flag == DIFFERENT_NONZERO_PATTERN) {
291:       MatDestroyMatrices(osm->n_local_true,&osm->pmat);
292:       scall = MAT_INITIAL_MATRIX;
293:     }
294:   }

296:   /* 
297:      Extract out the submatrices
298:   */
299:   MatGetSubMatrices(pc->pmat,osm->n_local_true,osm->is,osm->is,scall,&osm->pmat);
300:   if (scall == MAT_INITIAL_MATRIX) {
301:     PetscObjectGetOptionsPrefix((PetscObject)pc->pmat,&pprefix);
302:     for (i=0; i<osm->n_local_true; i++) {
303:       PetscLogObjectParent(pc,osm->pmat[i]);
304:       PetscObjectSetOptionsPrefix((PetscObject)osm->pmat[i],pprefix);
305:     }
306:   }

308:   /* Return control to the user so that the submatrices can be modified (e.g., to apply
309:      different boundary conditions for the submatrices than for the global problem) */
310:   PCModifySubMatrices(pc,osm->n_local_true,osm->is,osm->is,osm->pmat,pc->modifysubmatricesP);

312:   /* 
313:      Loop over subdomains putting them into local ksp
314:   */
315:   for (i=0; i<osm->n_local_true; i++) {
316:     KSPSetOperators(osm->ksp[i],osm->pmat[i],osm->pmat[i],pc->flag);
317:     if (!pc->setupcalled) {
318:       KSPSetFromOptions(osm->ksp[i]);
319:     }
320:   }

322:   return(0);
323: }

327: static PetscErrorCode PCSetUpOnBlocks_ASM(PC pc)
328: {
329:   PC_ASM         *osm = (PC_ASM*)pc->data;
331:   PetscInt       i;

334:   for (i=0; i<osm->n_local_true; i++) {
335:     KSPSetUp(osm->ksp[i]);
336:   }
337:   return(0);
338: }

342: static PetscErrorCode PCApply_ASM(PC pc,Vec x,Vec y)
343: {
344:   PC_ASM         *osm = (PC_ASM*)pc->data;
346:   PetscInt       i,n_local = osm->n_local,n_local_true = osm->n_local_true;
347:   ScatterMode    forward = SCATTER_FORWARD,reverse = SCATTER_REVERSE;

350:   /*
351:      Support for limiting the restriction or interpolation to only local 
352:      subdomain values (leaving the other values 0). 
353:   */
354:   if (!(osm->type & PC_ASM_RESTRICT)) {
355:     forward = SCATTER_FORWARD_LOCAL;
356:     /* have to zero the work RHS since scatter may leave some slots empty */
357:     for (i=0; i<n_local_true; i++) {
358:       VecSet(osm->x[i],0.0);
359:     }
360:   }
361:   if (!(osm->type & PC_ASM_INTERPOLATE)) {
362:     reverse = SCATTER_REVERSE_LOCAL;
363:   }

365:   for (i=0; i<n_local; i++) {
366:     VecScatterBegin(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
367:   }
368:   VecSet(y,0.0);
369:   /* do the local solves */
370:   for (i=0; i<n_local_true; i++) {
371:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
372:     KSPSolve(osm->ksp[i],osm->x[i],osm->y[i]);
373:     if (osm->localization) {
374:       VecScatterBegin(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
375:       VecScatterEnd(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
376:     }
377:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
378:   }
379:   /* handle the rest of the scatters that do not have local solves */
380:   for (i=n_local_true; i<n_local; i++) {
381:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
382:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
383:   }
384:   for (i=0; i<n_local; i++) {
385:     VecScatterEnd(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
386:   }
387:   return(0);
388: }

392: static PetscErrorCode PCApplyTranspose_ASM(PC pc,Vec x,Vec y)
393: {
394:   PC_ASM         *osm = (PC_ASM*)pc->data;
396:   PetscInt       i,n_local = osm->n_local,n_local_true = osm->n_local_true;
397:   ScatterMode    forward = SCATTER_FORWARD,reverse = SCATTER_REVERSE;

400:   /*
401:      Support for limiting the restriction or interpolation to only local 
402:      subdomain values (leaving the other values 0).

404:      Note: these are reversed from the PCApply_ASM() because we are applying the 
405:      transpose of the three terms 
406:   */
407:   if (!(osm->type & PC_ASM_INTERPOLATE)) {
408:     forward = SCATTER_FORWARD_LOCAL;
409:     /* have to zero the work RHS since scatter may leave some slots empty */
410:     for (i=0; i<n_local_true; i++) {
411:       VecSet(osm->x[i],0.0);
412:     }
413:   }
414:   if (!(osm->type & PC_ASM_RESTRICT)) {
415:     reverse = SCATTER_REVERSE_LOCAL;
416:   }

418:   for (i=0; i<n_local; i++) {
419:     VecScatterBegin(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
420:   }
421:   VecSet(y,0.0);
422:   /* do the local solves */
423:   for (i=0; i<n_local_true; i++) {
424:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
425:     KSPSolveTranspose(osm->ksp[i],osm->x[i],osm->y[i]);
426:     if (osm->localization) {
427:       VecScatterBegin(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
428:       VecScatterEnd(osm->localization[i],osm->y[i],osm->y_local[i],INSERT_VALUES,forward);
429:     }
430:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
431:   }
432:   /* handle the rest of the scatters that do not have local solves */
433:   for (i=n_local_true; i<n_local; i++) {
434:     VecScatterEnd(osm->restriction[i],x,osm->x[i],INSERT_VALUES,forward);
435:     VecScatterBegin(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
436:   }
437:   for (i=0; i<n_local; i++) {
438:     VecScatterEnd(osm->prolongation[i],osm->y_local[i],y,ADD_VALUES,reverse);
439:   }
440:   return(0);
441: }

445: static PetscErrorCode PCDestroy_ASM(PC pc)
446: {
447:   PC_ASM         *osm = (PC_ASM*)pc->data;
449:   PetscInt       i;

452:   if (osm->ksp) {
453:     for (i=0; i<osm->n_local_true; i++) {
454:       KSPDestroy(osm->ksp[i]);
455:     }
456:     PetscFree(osm->ksp);
457:   }
458:   if (osm->pmat) {
459:     if (osm->n_local_true > 0) {
460:       MatDestroyMatrices(osm->n_local_true,&osm->pmat);
461:     }
462:   }
463:   if (osm->restriction) {
464:     for (i=0; i<osm->n_local; i++) {
465:       VecScatterDestroy(osm->restriction[i]);
466:       if (osm->localization) {VecScatterDestroy(osm->localization[i]);}
467:       VecScatterDestroy(osm->prolongation[i]);
468:       VecDestroy(osm->x[i]);
469:       VecDestroy(osm->y[i]);
470:       VecDestroy(osm->y_local[i]);
471:     }
472:     PetscFree(osm->restriction);
473:     if (osm->localization) {PetscFree(osm->localization);}
474:     PetscFree(osm->prolongation);
475:     PetscFree(osm->x);
476:     PetscFree(osm->y);
477:     PetscFree(osm->y_local);
478:   }
479:   if (osm->is) {
480:     PCASMDestroySubdomains(osm->n_local_true,osm->is,osm->is_local);
481:   }
482:   PetscFree(osm);
483:   return(0);
484: }

488: static PetscErrorCode PCSetFromOptions_ASM(PC pc)
489: {
490:   PC_ASM         *osm = (PC_ASM*)pc->data;
492:   PetscInt       blocks,ovl;
493:   PetscTruth     symset,flg;
494:   PCASMType      asmtype;

497:   /* set the type to symmetric if matrix is symmetric */
498:   if (!osm->type_set && pc->pmat) {
499:     MatIsSymmetricKnown(pc->pmat,&symset,&flg);
500:     if (symset && flg) { osm->type = PC_ASM_BASIC; }
501:   }
502:   PetscOptionsHead("Additive Schwarz options");
503:     PetscOptionsInt("-pc_asm_blocks","Number of subdomains","PCASMSetTotalSubdomains",osm->n,&blocks,&flg);
504:     if (flg) {PCASMSetTotalSubdomains(pc,blocks,PETSC_NULL,PETSC_NULL); }
505:     PetscOptionsInt("-pc_asm_overlap","Number of grid points overlap","PCASMSetOverlap",osm->overlap,&ovl,&flg);
506:     if (flg) {PCASMSetOverlap(pc,ovl); }
507:     flg  = PETSC_FALSE;
508:     PetscOptionsEnum("-pc_asm_type","Type of restriction/extension","PCASMSetType",PCASMTypes,(PetscEnum)osm->type,(PetscEnum*)&asmtype,&flg);
509:     if (flg) {PCASMSetType(pc,asmtype); }
510:   PetscOptionsTail();
511:   return(0);
512: }

514: /*------------------------------------------------------------------------------------*/

519: PetscErrorCode  PCASMSetLocalSubdomains_ASM(PC pc,PetscInt n,IS is[],IS is_local[])
520: {
521:   PC_ASM         *osm = (PC_ASM*)pc->data;
523:   PetscInt       i;

526:   if (n < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Each process must have 1 or more blocks, n = %D",n);
527:   if (pc->setupcalled && (n != osm->n_local_true || is)) {
528:     SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"PCASMSetLocalSubdomains() should be called before calling PCSetUp().");
529:   }
530:   if (!pc->setupcalled) {
531:     if (is) {
532:       for (i=0; i<n; i++) {PetscObjectReference((PetscObject)is[i]);}
533:     }
534:     if (is_local) {
535:       for (i=0; i<n; i++) {PetscObjectReference((PetscObject)is_local[i]);}
536:     }
537:     if (osm->is) {
538:       PCASMDestroySubdomains(osm->n_local_true,osm->is,osm->is_local);
539:     }
540:     osm->n_local_true = n;
541:     osm->is           = 0;
542:     osm->is_local     = 0;
543:     if (is) {
544:       PetscMalloc(n*sizeof(IS *),&osm->is);
545:       for (i=0; i<n; i++) { osm->is[i] = is[i]; }
546:     }
547:     if (is_local) {
548:       PetscMalloc(n*sizeof(IS *),&osm->is_local);
549:       for (i=0; i<n; i++) { osm->is_local[i] = is_local[i]; }
550:     }
551:   }
552:   return(0);
553: }

559: PetscErrorCode  PCASMSetTotalSubdomains_ASM(PC pc,PetscInt N,IS *is)
560: {
561:   PC_ASM         *osm = (PC_ASM*)pc->data;
563:   PetscMPIInt    rank,size;
564:   PetscInt       n;

567:   if (N < 1) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"Number of total blocks must be > 0, N = %D",N);
568:   if (is) SETERRQ(PETSC_ERR_SUP,"Use PCASMSetLocalSubdomains() to set specific index sets\n\they cannot be set globally yet.");

570:   /*
571:      Split the subdomains equally among all processors
572:   */
573:   MPI_Comm_rank(((PetscObject)pc)->comm,&rank);
574:   MPI_Comm_size(((PetscObject)pc)->comm,&size);
575:   n = N/size + ((N % size) > rank);
576:   if (!n) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Process %d must have at least one block: total processors %d total blocks %D",(int)rank,(int)size,N);
577:   if (pc->setupcalled && n != osm->n_local_true) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"PCASMSetTotalSubdomains() should be called before PCSetUp().");
578:   if (!pc->setupcalled) {
579:     if (osm->is) {
580:       PCASMDestroySubdomains(osm->n_local_true,osm->is,osm->is_local);
581:     }
582:     osm->n_local_true = n;
583:     osm->is           = 0;
584:     osm->is_local     = 0;
585:   }
586:   return(0);
587: }

593: PetscErrorCode  PCASMSetOverlap_ASM(PC pc,PetscInt ovl)
594: {
595:   PC_ASM *osm = (PC_ASM*)pc->data;

598:   if (ovl < 0) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Negative overlap value requested");
599:   if (pc->setupcalled && ovl != osm->overlap) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"PCASMSetOverlap() should be called before PCSetUp().");
600:   if (!pc->setupcalled) {
601:     osm->overlap = ovl;
602:   }
603:   return(0);
604: }

610: PetscErrorCode  PCASMSetType_ASM(PC pc,PCASMType type)
611: {
612:   PC_ASM *osm = (PC_ASM*)pc->data;

615:   osm->type     = type;
616:   osm->type_set = PETSC_TRUE;
617:   return(0);
618: }

624: PetscErrorCode  PCASMSetSortIndices_ASM(PC pc,PetscTruth doSort)
625: {
626:   PC_ASM *osm = (PC_ASM*)pc->data;

629:   osm->sort_indices = doSort;
630:   return(0);
631: }

637: PetscErrorCode  PCASMGetSubKSP_ASM(PC pc,PetscInt *n_local,PetscInt *first_local,KSP **ksp)
638: {
639:   PC_ASM         *osm = (PC_ASM*)pc->data;

643:   if (osm->n_local_true < 1) {
644:     SETERRQ(PETSC_ERR_ORDER,"Need to call PCSetUP() on PC (or KSPSetUp() on the outer KSP object) before calling here");
645:   }

647:   if (n_local) {
648:     *n_local = osm->n_local_true;
649:   }
650:   if (first_local) {
651:     MPI_Scan(&osm->n_local_true,first_local,1,MPIU_INT,MPI_SUM,((PetscObject)pc)->comm);
652:     *first_local -= osm->n_local_true;
653:   }
654:   if (ksp) {
655:     /* Assume that local solves are now different; not necessarily
656:        true though!  This flag is used only for PCView_ASM() */
657:     *ksp                   = osm->ksp;
658:     osm->same_local_solves = PETSC_FALSE;
659:   }
660:   return(0);
661: }


667: /*@C
668:     PCASMSetLocalSubdomains - Sets the local subdomains (for this processor
669:     only) for the additive Schwarz preconditioner. 

671:     Collective on PC 

673:     Input Parameters:
674: +   pc - the preconditioner context
675: .   n - the number of subdomains for this processor (default value = 1)
676: .   is - the index sets that define the subdomains for this processor
677:          (or PETSC_NULL for PETSc to determine subdomains)
678: -   is_local - the index sets that define the local part of the subdomains for this processor
679:          (or PETSC_NULL to use the default of 1 subdomain per process)

681:     Notes:
682:     The IS numbering is in the parallel, global numbering of the vector.

684:     By default the ASM preconditioner uses 1 block per processor.  

686:     Use PCASMSetTotalSubdomains() to set the subdomains for all processors.

688:     Level: advanced

690: .keywords: PC, ASM, set, local, subdomains, additive Schwarz

692: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
693:           PCASMCreateSubdomains2D(), PCASMGetLocalSubdomains()
694: @*/
695: PetscErrorCode  PCASMSetLocalSubdomains(PC pc,PetscInt n,IS is[],IS is_local[])
696: {
697:   PetscErrorCode ierr,(*f)(PC,PetscInt,IS[],IS[]);

701:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetLocalSubdomains_C",(void (**)(void))&f);
702:   if (f) {
703:     (*f)(pc,n,is,is_local);
704:   }
705:   return(0);
706: }

710: /*@C
711:     PCASMSetTotalSubdomains - Sets the subdomains for all processor for the 
712:     additive Schwarz preconditioner.  Either all or no processors in the
713:     PC communicator must call this routine, with the same index sets.

715:     Collective on PC

717:     Input Parameters:
718: +   pc - the preconditioner context
719: .   n - the number of subdomains for all processors
720: .   is - the index sets that define the subdomains for all processor
721:          (or PETSC_NULL for PETSc to determine subdomains)
722: -   is_local - the index sets that define the local part of the subdomains for this processor
723:          (or PETSC_NULL to use the default of 1 subdomain per process)

725:     Options Database Key:
726:     To set the total number of subdomain blocks rather than specify the
727:     index sets, use the option
728: .    -pc_asm_blocks <blks> - Sets total blocks

730:     Notes:
731:     Currently you cannot use this to set the actual subdomains with the argument is.

733:     By default the ASM preconditioner uses 1 block per processor.  

735:     These index sets cannot be destroyed until after completion of the
736:     linear solves for which the ASM preconditioner is being used.

738:     Use PCASMSetLocalSubdomains() to set local subdomains.

740:     Level: advanced

742: .keywords: PC, ASM, set, total, global, subdomains, additive Schwarz

744: .seealso: PCASMSetLocalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
745:           PCASMCreateSubdomains2D()
746: @*/
747: PetscErrorCode  PCASMSetTotalSubdomains(PC pc,PetscInt N,IS is[],IS is_local[])
748: {
749:   PetscErrorCode ierr,(*f)(PC,PetscInt,IS[],IS[]);

753:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetTotalSubdomains_C",(void (**)(void))&f);
754:   if (f) {
755:     (*f)(pc,N,is,is_local);
756:   }
757:   return(0);
758: }

762: /*@
763:     PCASMSetOverlap - Sets the overlap between a pair of subdomains for the
764:     additive Schwarz preconditioner.  Either all or no processors in the
765:     PC communicator must call this routine. 

767:     Collective on PC

769:     Input Parameters:
770: +   pc  - the preconditioner context
771: -   ovl - the amount of overlap between subdomains (ovl >= 0, default value = 1)

773:     Options Database Key:
774: .   -pc_asm_overlap <ovl> - Sets overlap

776:     Notes:
777:     By default the ASM preconditioner uses 1 block per processor.  To use
778:     multiple blocks per perocessor, see PCASMSetTotalSubdomains() and
779:     PCASMSetLocalSubdomains() (and the option -pc_asm_blocks <blks>).

781:     The overlap defaults to 1, so if one desires that no additional
782:     overlap be computed beyond what may have been set with a call to
783:     PCASMSetTotalSubdomains() or PCASMSetLocalSubdomains(), then ovl
784:     must be set to be 0.  In particular, if one does not explicitly set
785:     the subdomains an application code, then all overlap would be computed
786:     internally by PETSc, and using an overlap of 0 would result in an ASM 
787:     variant that is equivalent to the block Jacobi preconditioner.  

789:     Note that one can define initial index sets with any overlap via
790:     PCASMSetTotalSubdomains() or PCASMSetLocalSubdomains(); the routine
791:     PCASMSetOverlap() merely allows PETSc to extend that overlap further
792:     if desired.

794:     Level: intermediate

796: .keywords: PC, ASM, set, overlap

798: .seealso: PCASMSetTotalSubdomains(), PCASMSetLocalSubdomains(), PCASMGetSubKSP(),
799:           PCASMCreateSubdomains2D(), PCASMGetLocalSubdomains()
800: @*/
801: PetscErrorCode  PCASMSetOverlap(PC pc,PetscInt ovl)
802: {
803:   PetscErrorCode ierr,(*f)(PC,PetscInt);

807:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetOverlap_C",(void (**)(void))&f);
808:   if (f) {
809:     (*f)(pc,ovl);
810:   }
811:   return(0);
812: }

816: /*@
817:     PCASMSetType - Sets the type of restriction and interpolation used
818:     for local problems in the additive Schwarz method.

820:     Collective on PC

822:     Input Parameters:
823: +   pc  - the preconditioner context
824: -   type - variant of ASM, one of
825: .vb
826:       PC_ASM_BASIC       - full interpolation and restriction
827:       PC_ASM_RESTRICT    - full restriction, local processor interpolation
828:       PC_ASM_INTERPOLATE - full interpolation, local processor restriction
829:       PC_ASM_NONE        - local processor restriction and interpolation
830: .ve

832:     Options Database Key:
833: .   -pc_asm_type [basic,restrict,interpolate,none] - Sets ASM type

835:     Level: intermediate

837: .keywords: PC, ASM, set, type

839: .seealso: PCASMSetTotalSubdomains(), PCASMSetTotalSubdomains(), PCASMGetSubKSP(),
840:           PCASMCreateSubdomains2D()
841: @*/
842: PetscErrorCode  PCASMSetType(PC pc,PCASMType type)
843: {
844:   PetscErrorCode ierr,(*f)(PC,PCASMType);

848:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetType_C",(void (**)(void))&f);
849:   if (f) {
850:     (*f)(pc,type);
851:   }
852:   return(0);
853: }

857: /*@
858:     PCASMSetSortIndices - Determines whether subdomain indices are sorted.

860:     Collective on PC

862:     Input Parameters:
863: +   pc  - the preconditioner context
864: -   doSort - sort the subdomain indices

866:     Level: intermediate

868: .keywords: PC, ASM, set, type

870: .seealso: PCASMSetLocalSubdomains(), PCASMSetTotalSubdomains(), PCASMGetSubKSP(),
871:           PCASMCreateSubdomains2D()
872: @*/
873: PetscErrorCode  PCASMSetSortIndices(PC pc,PetscTruth doSort)
874: {
875:   PetscErrorCode ierr,(*f)(PC,PetscTruth);

879:   PetscObjectQueryFunction((PetscObject)pc,"PCASMSetSortIndices_C",(void (**)(void))&f);
880:   if (f) {
881:     (*f)(pc,doSort);
882:   }
883:   return(0);
884: }

888: /*@C
889:    PCASMGetSubKSP - Gets the local KSP contexts for all blocks on
890:    this processor.
891:    
892:    Collective on PC iff first_local is requested

894:    Input Parameter:
895: .  pc - the preconditioner context

897:    Output Parameters:
898: +  n_local - the number of blocks on this processor or PETSC_NULL
899: .  first_local - the global number of the first block on this processor or PETSC_NULL,
900:                  all processors must request or all must pass PETSC_NULL
901: -  ksp - the array of KSP contexts

903:    Note:  
904:    After PCASMGetSubKSP() the array of KSPes is not to be freed

906:    Currently for some matrix implementations only 1 block per processor 
907:    is supported.
908:    
909:    You must call KSPSetUp() before calling PCASMGetSubKSP().

911:    Level: advanced

913: .keywords: PC, ASM, additive Schwarz, get, sub, KSP, context

915: .seealso: PCASMSetTotalSubdomains(), PCASMSetTotalSubdomains(), PCASMSetOverlap(),
916:           PCASMCreateSubdomains2D(),
917: @*/
918: PetscErrorCode  PCASMGetSubKSP(PC pc,PetscInt *n_local,PetscInt *first_local,KSP *ksp[])
919: {
920:   PetscErrorCode ierr,(*f)(PC,PetscInt*,PetscInt*,KSP **);

924:   PetscObjectQueryFunction((PetscObject)pc,"PCASMGetSubKSP_C",(void (**)(void))&f);
925:   if (f) {
926:     (*f)(pc,n_local,first_local,ksp);
927:   } else {
928:     SETERRQ(PETSC_ERR_ARG_WRONG,"Cannot get subksp for this type of PC");
929:   }

931:  return(0);
932: }

934: /* -------------------------------------------------------------------------------------*/
935: /*MC
936:    PCASM - Use the (restricted) additive Schwarz method, each block is (approximately) solved with 
937:            its own KSP object.

939:    Options Database Keys:
940: +  -pc_asm_truelocal - Activates PCASMSetUseTrueLocal()
941: .  -pc_asm_blocks <blks> - Sets total blocks
942: .  -pc_asm_overlap <ovl> - Sets overlap
943: -  -pc_asm_type [basic,restrict,interpolate,none] - Sets ASM type

945:      IMPORTANT: If you run with, for example, 3 blocks on 1 processor or 3 blocks on 3 processors you 
946:       will get a different convergence rate due to the default option of -pc_asm_type restrict. Use
947:       -pc_asm_type basic to use the standard ASM. 

949:    Notes: Each processor can have one or more blocks, but a block cannot be shared by more
950:      than one processor. Defaults to one block per processor.

952:      To set options on the solvers for each block append -sub_ to all the KSP, and PC
953:         options database keys. For example, -sub_pc_type ilu -sub_pc_factor_levels 1 -sub_ksp_type preonly
954:         
955:      To set the options on the solvers separate for each block call PCASMGetSubKSP()
956:          and set the options directly on the resulting KSP object (you can access its PC
957:          with KSPGetPC())


960:    Level: beginner

962:    Concepts: additive Schwarz method

964:     References:
965:     An additive variant of the Schwarz alternating method for the case of many subregions
966:     M Dryja, OB Widlund - Courant Institute, New York University Technical report

968:     Domain Decompositions: Parallel Multilevel Methods for Elliptic Partial Differential Equations, 
969:     Barry Smith, Petter Bjorstad, and William Gropp, Cambridge University Press, ISBN 0-521-49589-X.

971: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types), PC,
972:            PCBJACOBI, PCASMSetUseTrueLocal(), PCASMGetSubKSP(), PCASMSetLocalSubdomains(),
973:            PCASMSetTotalSubdomains(), PCSetModifySubmatrices(), PCASMSetOverlap(), PCASMSetType()

975: M*/

980: PetscErrorCode  PCCreate_ASM(PC pc)
981: {
983:   PC_ASM         *osm;

986:   PetscNewLog(pc,PC_ASM,&osm);
987:   osm->n                 = PETSC_DECIDE;
988:   osm->n_local           = 0;
989:   osm->n_local_true      = 0;
990:   osm->overlap           = 1;
991:   osm->ksp               = 0;
992:   osm->restriction       = 0;
993:   osm->localization      = 0;
994:   osm->prolongation      = 0;
995:   osm->x                 = 0;
996:   osm->y                 = 0;
997:   osm->y_local           = 0;
998:   osm->is                = 0;
999:   osm->is_local          = 0;
1000:   osm->mat               = 0;
1001:   osm->pmat              = 0;
1002:   osm->type              = PC_ASM_RESTRICT;
1003:   osm->same_local_solves = PETSC_TRUE;
1004:   osm->sort_indices      = PETSC_TRUE;

1006:   pc->data                   = (void*)osm;
1007:   pc->ops->apply             = PCApply_ASM;
1008:   pc->ops->applytranspose    = PCApplyTranspose_ASM;
1009:   pc->ops->setup             = PCSetUp_ASM;
1010:   pc->ops->destroy           = PCDestroy_ASM;
1011:   pc->ops->setfromoptions    = PCSetFromOptions_ASM;
1012:   pc->ops->setuponblocks     = PCSetUpOnBlocks_ASM;
1013:   pc->ops->view              = PCView_ASM;
1014:   pc->ops->applyrichardson   = 0;

1016:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetLocalSubdomains_C","PCASMSetLocalSubdomains_ASM",
1017:                     PCASMSetLocalSubdomains_ASM);
1018:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetTotalSubdomains_C","PCASMSetTotalSubdomains_ASM",
1019:                     PCASMSetTotalSubdomains_ASM);
1020:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetOverlap_C","PCASMSetOverlap_ASM",
1021:                     PCASMSetOverlap_ASM);
1022:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetType_C","PCASMSetType_ASM",
1023:                     PCASMSetType_ASM);
1024:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMSetSortIndices_C","PCASMSetSortIndices_ASM",
1025:                     PCASMSetSortIndices_ASM);
1026:   PetscObjectComposeFunctionDynamic((PetscObject)pc,"PCASMGetSubKSP_C","PCASMGetSubKSP_ASM",
1027:                     PCASMGetSubKSP_ASM);
1028:   return(0);
1029: }


1035: /*@C
1036:    PCASMCreateSubdomains - Creates the index sets for the overlapping Schwarz 
1037:    preconditioner for a any problem on a general grid.

1039:    Collective

1041:    Input Parameters:
1042: +  A - The global matrix operator
1043: -  n - the number of local blocks

1045:    Output Parameters:
1046: .  outis - the array of index sets defining the subdomains

1048:    Level: advanced

1050:    Note: this generates nonoverlapping subdomains; the PCASM will generate the overlap
1051:     from these if you use PCASMSetLocalSubdomains()

1053:     In the Fortran version you must provide the array outis[] already allocated of length n.

1055: .keywords: PC, ASM, additive Schwarz, create, subdomains, unstructured grid

1057: .seealso: PCASMSetLocalSubdomains(), PCASMDestroySubdomains()
1058: @*/
1059: PetscErrorCode  PCASMCreateSubdomains(Mat A, PetscInt n, IS* outis[])
1060: {
1061:   MatPartitioning           mpart;
1062:   const char                *prefix;
1063:   PetscErrorCode            (*f)(Mat,PetscTruth*,MatReuse,Mat*);
1064:   PetscMPIInt               size;
1065:   PetscInt                  i,j,rstart,rend,bs;
1066:   PetscTruth                iscopy = PETSC_FALSE,isbaij = PETSC_FALSE,foundpart = PETSC_FALSE;
1067:   Mat                       Ad = PETSC_NULL, adj;
1068:   IS                        ispart,isnumb,*is;
1069:   PetscErrorCode            ierr;

1074:   if (n < 1) {SETERRQ1(PETSC_ERR_ARG_WRONG,"number of local blocks must be > 0, n = %D",n);}
1075: 
1076:   /* Get prefix, row distribution, and block size */
1077:   MatGetOptionsPrefix(A,&prefix);
1078:   MatGetOwnershipRange(A,&rstart,&rend);
1079:   MatGetBlockSize(A,&bs);
1080:   if (rstart/bs*bs != rstart || rend/bs*bs != rend) {
1081:     SETERRQ3(PETSC_ERR_ARG_WRONG,"bad row distribution [%D,%D) for matrix block size %D",rstart,rend,bs);
1082:   }
1083:   /* Get diagonal block from matrix if possible */
1084:   MPI_Comm_size(((PetscObject)A)->comm,&size);
1085:   PetscObjectQueryFunction((PetscObject)A,"MatGetDiagonalBlock_C",(void (**)(void))&f);
1086:   if (f) {
1087:     (*f)(A,&iscopy,MAT_INITIAL_MATRIX,&Ad);
1088:   } else if (size == 1) {
1089:     iscopy = PETSC_FALSE; Ad = A;
1090:   } else {
1091:     iscopy = PETSC_FALSE; Ad = PETSC_NULL;
1092:   }
1093:   if (Ad) {
1094:     PetscTypeCompare((PetscObject)Ad,MATSEQBAIJ,&isbaij);
1095:     if (!isbaij) {PetscTypeCompare((PetscObject)Ad,MATSEQSBAIJ,&isbaij);}
1096:   }
1097:   if (Ad && n > 1) {
1098:     PetscTruth match,done;
1099:     /* Try to setup a good matrix partitioning if available */
1100:     MatPartitioningCreate(PETSC_COMM_SELF,&mpart);
1101:     PetscObjectSetOptionsPrefix((PetscObject)mpart,prefix);
1102:     MatPartitioningSetFromOptions(mpart);
1103:     PetscTypeCompare((PetscObject)mpart,MAT_PARTITIONING_CURRENT,&match);
1104:     if (!match) {
1105:       PetscTypeCompare((PetscObject)mpart,MAT_PARTITIONING_SQUARE,&match);
1106:     }
1107:     if (!match) { /* assume a "good" partitioner is available */
1108:       PetscInt na,*ia,*ja;
1109:       MatGetRowIJ(Ad,0,PETSC_TRUE,isbaij,&na,&ia,&ja,&done);
1110:       if (done) {
1111:         /* Build adjacency matrix by hand. Unfortunately a call to
1112:            MatConvert(Ad,MATMPIADJ,MAT_INITIAL_MATRIX,&adj) will
1113:            remove the block-aij structure and we cannot expect
1114:            MatPartitioning to split vertices as we need */
1115:         PetscInt i,j,*row,len,nnz,cnt,*iia=0,*jja=0;
1116:         nnz = 0;
1117:         for (i=0; i<na; i++) { /* count number of nonzeros */
1118:           len = ia[i+1] - ia[i];
1119:           row = ja + ia[i];
1120:           for (j=0; j<len; j++) {
1121:             if (row[j] == i) { /* don't count diagonal */
1122:               len--; break;
1123:             }
1124:           }
1125:           nnz += len;
1126:         }
1127:         PetscMalloc((na+1)*sizeof(PetscInt),&iia);
1128:         PetscMalloc((nnz)*sizeof(PetscInt),&jja);
1129:         nnz    = 0;
1130:         iia[0] = 0;
1131:         for (i=0; i<na; i++) { /* fill adjacency */
1132:           cnt = 0;
1133:           len = ia[i+1] - ia[i];
1134:           row = ja + ia[i];
1135:           for (j=0; j<len; j++) {
1136:             if (row[j] != i) { /* if not diagonal */
1137:               jja[nnz+cnt++] = row[j];
1138:             }
1139:           }
1140:           nnz += cnt;
1141:           iia[i+1] = nnz;
1142:         }
1143:         /* Partitioning of the adjacency matrix */
1144:         MatCreateMPIAdj(PETSC_COMM_SELF,na,na,iia,jja,PETSC_NULL,&adj);
1145:         MatPartitioningSetAdjacency(mpart,adj);
1146:         MatPartitioningSetNParts(mpart,n);
1147:         MatPartitioningApply(mpart,&ispart);
1148:         ISPartitioningToNumbering(ispart,&isnumb);
1149:         MatDestroy(adj);
1150:         foundpart = PETSC_TRUE;
1151:       }
1152:       MatRestoreRowIJ(Ad,0,PETSC_TRUE,isbaij,&na,&ia,&ja,&done);
1153:     }
1154:     MatPartitioningDestroy(mpart);
1155:   }
1156:   if (iscopy) {MatDestroy(Ad);}
1157: 
1158:   PetscMalloc(n*sizeof(IS),&is);
1159:   *outis = is;

1161:   if (!foundpart) {

1163:     /* Partitioning by contiguous chunks of rows */

1165:     PetscInt mbs   = (rend-rstart)/bs;
1166:     PetscInt start = rstart;
1167:     for (i=0; i<n; i++) {
1168:       PetscInt count = (mbs/n + ((mbs % n) > i)) * bs;
1169:       ISCreateStride(PETSC_COMM_SELF,count,start,1,&is[i]);
1170:       start += count;
1171:     }
1172: 
1173:   } else {

1175:     /* Partitioning by adjacency of diagonal block  */

1177:     const PetscInt *numbering;
1178:     PetscInt       *count,nidx,*indices,*newidx,start=0;
1179:     /* Get node count in each partition */
1180:     PetscMalloc(n*sizeof(PetscInt),&count);
1181:     ISPartitioningCount(ispart,n,count);
1182:     if (isbaij && bs > 1) { /* adjust for the block-aij case */
1183:       for (i=0; i<n; i++) count[i] *= bs;
1184:     }
1185:     /* Build indices from node numbering */
1186:     ISGetLocalSize(isnumb,&nidx);
1187:     PetscMalloc(nidx*sizeof(PetscInt),&indices);
1188:     for (i=0; i<nidx; i++) indices[i] = i; /* needs to be initialized */
1189:     ISGetIndices(isnumb,&numbering);
1190:     PetscSortIntWithPermutation(nidx,numbering,indices);
1191:     ISRestoreIndices(isnumb,&numbering);
1192:     if (isbaij && bs > 1) { /* adjust for the block-aij case */
1193:       PetscMalloc(nidx*bs*sizeof(PetscInt),&newidx);
1194:       for (i=0; i<nidx; i++)
1195:         for (j=0; j<bs; j++)
1196:           newidx[i*bs+j] = indices[i]*bs + j;
1197:       PetscFree(indices);
1198:       nidx   *= bs;
1199:       indices = newidx;
1200:     }
1201:     /* Shift to get global indices */
1202:     for (i=0; i<nidx; i++) indices[i] += rstart;
1203: 
1204:     /* Build the index sets for each block */
1205:     for (i=0; i<n; i++) {
1206:       ISCreateGeneral(PETSC_COMM_SELF,count[i],&indices[start],&is[i]);
1207:       ISSort(is[i]);
1208:       start += count[i];
1209:     }

1211:     PetscFree(count);
1212:     PetscFree(indices);
1213:     ISDestroy(isnumb);
1214:     ISDestroy(ispart);

1216:   }
1217: 
1218:   return(0);
1219: }

1223: /*@C
1224:    PCASMDestroySubdomains - Destroys the index sets created with
1225:    PCASMCreateSubdomains(). Should be called after setting subdomains
1226:    with PCASMSetLocalSubdomains().

1228:    Collective

1230:    Input Parameters:
1231: +  n - the number of index sets
1232: .  is - the array of index sets
1233: -  is_local - the array of local index sets, can be PETSC_NULL

1235:    Level: advanced

1237: .keywords: PC, ASM, additive Schwarz, create, subdomains, unstructured grid

1239: .seealso: PCASMCreateSubdomains(), PCASMSetLocalSubdomains()
1240: @*/
1241: PetscErrorCode  PCASMDestroySubdomains(PetscInt n, IS is[], IS is_local[])
1242: {
1243:   PetscInt       i;
1246:   if (n <= 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"n must be > 0: n = %D",n);
1248:   for (i=0; i<n; i++) { ISDestroy(is[i]); }
1249:   PetscFree(is);
1250:   if (is_local) {
1252:     for (i=0; i<n; i++) { ISDestroy(is_local[i]); }
1253:     PetscFree(is_local);
1254:   }
1255:   return(0);
1256: }

1260: /*@
1261:    PCASMCreateSubdomains2D - Creates the index sets for the overlapping Schwarz 
1262:    preconditioner for a two-dimensional problem on a regular grid.

1264:    Not Collective

1266:    Input Parameters:
1267: +  m, n - the number of mesh points in the x and y directions
1268: .  M, N - the number of subdomains in the x and y directions
1269: .  dof - degrees of freedom per node
1270: -  overlap - overlap in mesh lines

1272:    Output Parameters:
1273: +  Nsub - the number of subdomains created
1274: -  is - the array of index sets defining the subdomains

1276:    Note:
1277:    Presently PCAMSCreateSubdomains2d() is valid only for sequential
1278:    preconditioners.  More general related routines are
1279:    PCASMSetTotalSubdomains() and PCASMSetLocalSubdomains().

1281:    Level: advanced

1283: .keywords: PC, ASM, additive Schwarz, create, subdomains, 2D, regular grid

1285: .seealso: PCASMSetTotalSubdomains(), PCASMSetLocalSubdomains(), PCASMGetSubKSP(),
1286:           PCASMSetOverlap()
1287: @*/
1288: PetscErrorCode  PCASMCreateSubdomains2D(PetscInt m,PetscInt n,PetscInt M,PetscInt N,PetscInt dof,PetscInt overlap,PetscInt *Nsub,IS **is)
1289: {
1290:   PetscInt       i,j,height,width,ystart,xstart,yleft,yright,xleft,xright,loc_outter;
1292:   PetscInt       nidx,*idx,loc,ii,jj,count;

1295:   if (dof != 1) SETERRQ(PETSC_ERR_SUP," ");

1297:   *Nsub = N*M;
1298:   PetscMalloc((*Nsub)*sizeof(IS *),is);
1299:   ystart = 0;
1300:   loc_outter = 0;
1301:   for (i=0; i<N; i++) {
1302:     height = n/N + ((n % N) > i); /* height of subdomain */
1303:     if (height < 2) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Too many N subdomains for mesh dimension n");
1304:     yleft  = ystart - overlap; if (yleft < 0) yleft = 0;
1305:     yright = ystart + height + overlap; if (yright > n) yright = n;
1306:     xstart = 0;
1307:     for (j=0; j<M; j++) {
1308:       width = m/M + ((m % M) > j); /* width of subdomain */
1309:       if (width < 2) SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,"Too many M subdomains for mesh dimension m");
1310:       xleft  = xstart - overlap; if (xleft < 0) xleft = 0;
1311:       xright = xstart + width + overlap; if (xright > m) xright = m;
1312:       nidx   = (xright - xleft)*(yright - yleft);
1313:       PetscMalloc(nidx*sizeof(PetscInt),&idx);
1314:       loc    = 0;
1315:       for (ii=yleft; ii<yright; ii++) {
1316:         count = m*ii + xleft;
1317:         for (jj=xleft; jj<xright; jj++) {
1318:           idx[loc++] = count++;
1319:         }
1320:       }
1321:       ISCreateGeneral(PETSC_COMM_SELF,nidx,idx,(*is)+loc_outter++);
1322:       PetscFree(idx);
1323:       xstart += width;
1324:     }
1325:     ystart += height;
1326:   }
1327:   for (i=0; i<*Nsub; i++) { ISSort((*is)[i]); }
1328:   return(0);
1329: }

1333: /*@C
1334:     PCASMGetLocalSubdomains - Gets the local subdomains (for this processor
1335:     only) for the additive Schwarz preconditioner. 

1337:     Collective on PC 

1339:     Input Parameter:
1340: .   pc - the preconditioner context

1342:     Output Parameters:
1343: +   n - the number of subdomains for this processor (default value = 1)
1344: .   is - the index sets that define the subdomains for this processor
1345: -   is_local - the index sets that define the local part of the subdomains for this processor (can be PETSC_NULL)
1346:          

1348:     Notes:
1349:     The IS numbering is in the parallel, global numbering of the vector.

1351:     Level: advanced

1353: .keywords: PC, ASM, set, local, subdomains, additive Schwarz

1355: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
1356:           PCASMCreateSubdomains2D(), PCASMSetLocalSubdomains(), PCASMGetLocalSubmatrices()
1357: @*/
1358: PetscErrorCode  PCASMGetLocalSubdomains(PC pc,PetscInt *n,IS *is[],IS *is_local[])
1359: {
1360:   PC_ASM         *osm;
1362:   PetscTruth     match;

1368:   PetscTypeCompare((PetscObject)pc,PCASM,&match);
1369:   if (!match) {
1370:     if (n)  *n  = 0;
1371:     if (is) *is = PETSC_NULL;
1372:   } else {
1373:     osm = (PC_ASM*)pc->data;
1374:     if (n)  *n  = osm->n_local_true;
1375:     if (is) *is = osm->is;
1376:     if (is_local) *is_local = osm->is_local;
1377:   }
1378:   return(0);
1379: }

1383: /*@C
1384:     PCASMGetLocalSubmatrices - Gets the local submatrices (for this processor
1385:     only) for the additive Schwarz preconditioner. 

1387:     Collective on PC 

1389:     Input Parameter:
1390: .   pc - the preconditioner context

1392:     Output Parameters:
1393: +   n - the number of matrices for this processor (default value = 1)
1394: -   mat - the matrices
1395:          

1397:     Level: advanced

1399: .keywords: PC, ASM, set, local, subdomains, additive Schwarz, block Jacobi

1401: .seealso: PCASMSetTotalSubdomains(), PCASMSetOverlap(), PCASMGetSubKSP(),
1402:           PCASMCreateSubdomains2D(), PCASMSetLocalSubdomains(), PCASMGetLocalSubdomains()
1403: @*/
1404: PetscErrorCode  PCASMGetLocalSubmatrices(PC pc,PetscInt *n,Mat *mat[])
1405: {
1406:   PC_ASM         *osm;
1408:   PetscTruth     match;

1414:   if (!pc->setupcalled) SETERRQ(PETSC_ERR_ARG_WRONGSTATE,"Must call after KSPSetUP() or PCSetUp().");
1415:   PetscTypeCompare((PetscObject)pc,PCASM,&match);
1416:   if (!match) {
1417:     if (n)   *n   = 0;
1418:     if (mat) *mat = PETSC_NULL;
1419:   } else {
1420:     osm = (PC_ASM*)pc->data;
1421:     if (n)   *n   = osm->n_local_true;
1422:     if (mat) *mat = osm->pmat;
1423:   }
1424:   return(0);
1425: }