Actual source code: openmp.c

  1: #define PETSCKSP_DLL

 3:  #include private/pcimpl.h
 4:  #include petscksp.h

  6: typedef struct {
  7:   MatStructure flag;               /* pc->flag */
  8:   PetscInt     setupcalled;        /* pc->setupcalled */
  9:   PetscInt     n;
 10:   MPI_Comm     comm;                 /* local world used by this preconditioner */
 11:   KSP          ksp;                  /* actual solver used across local world */
 12:   Mat          mat;                  /* matrix in local world */
 13:   Mat          gmat;                 /* matrix known only to process 0 in the local world */
 14:   Vec          x,y,xdummy,ydummy;
 15:   VecScatter   scatter;
 16:   PetscTruth   nonzero_guess;
 17: } PC_OpenMP;


 22: /*
 23:     Would like to have this simply call PCView() on the inner PC. The problem is
 24:   that the outter comm does not live on the inside so cannot do this. Instead 
 25:   handle the special case when the viewer is stdout, construct a new one just
 26:   for this call.
 27: */

 29: static PetscErrorCode PCView_OpenMP_MP(MPI_Comm comm,void *ctx)
 30: {
 31:   PC_OpenMP      *red = (PC_OpenMP*)ctx;
 33:   PetscViewer    viewer;

 36:   PetscViewerASCIIGetStdout(comm,&viewer);
 37:   PetscViewerASCIIPushTab(viewer);         /* this is bogus in general */
 38:   KSPView(red->ksp,viewer);
 39:   PetscViewerASCIIPopTab(viewer);
 40:   return(0);
 41: }

 45: static PetscErrorCode PCView_OpenMP(PC pc,PetscViewer viewer)
 46: {
 47:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;
 48:   PetscMPIInt    size;
 50:   PetscTruth     iascii;


 55:   MPI_Comm_size(red->comm,&size);
 56:   PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
 57:   if (iascii) {
 58:     PetscViewerASCIIPrintf(viewer,"  Size of solver nodes %d\n",size);
 59:     PetscViewerASCIIPrintf(viewer,"  Parallel sub-solver given next\n",size);
 60:     /* should only make the next call if the viewer is associated with stdout */
 61:     PetscOpenMPRun(red->comm,PCView_OpenMP_MP,red);
 62:   }
 63:   return(0);
 64: }

 66:  #include private/matimpl.h
 67:  #include private/vecimpl.h
 68:  #include ../src/mat/impls/aij/mpi/mpiaij.h
 69:  #include ../src/mat/impls/aij/seq/aij.h



 76: static PetscErrorCode PCApply_OpenMP_1(PC pc,Vec x,Vec y)
 77: {
 78:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;

 82:   KSPSetInitialGuessNonzero(red->ksp,pc->nonzero_guess);
 83:   KSPSolve(red->ksp,x,y);
 84:   return(0);
 85: }

 89: static PetscErrorCode PCSetUp_OpenMP_MP(MPI_Comm comm,void *ctx)
 90: {
 91:   PC_OpenMP      *red = (PC_OpenMP*)ctx;
 93:   PetscInt       m;
 94:   MatReuse       scal;
 95:   PetscMPIInt    rank;

 98:   red->comm = comm;
 99:   MPI_Bcast(&red->setupcalled,1,MPIU_INT,0,comm);
100:   MPI_Bcast(&red->flag,1,MPIU_INT,0,comm);
101:   if (!red->setupcalled) {
102:     /* setup vector communication */
103:     MPI_Bcast(&red->n,1,MPIU_INT,0,comm);
104:     VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->x);
105:     VecCreateMPI(comm,PETSC_DECIDE,red->n,&red->y);
106:     VecScatterCreateToZero(red->x,&red->scatter,&red->xdummy);
107:     VecDuplicate(red->xdummy,&red->ydummy);
108:     MPI_Comm_rank(comm,&rank);
109:     if (!rank) {
110:       VecDestroy(red->xdummy);
111:       VecDestroy(red->ydummy);
112:     }
113:     scal = MAT_INITIAL_MATRIX;
114:   } else {
115:     if (red->flag == DIFFERENT_NONZERO_PATTERN) {
116:       MatDestroy(red->mat);
117:       scal = MAT_INITIAL_MATRIX;
118:       CHKMEMQ;
119:     } else {
120:       scal = MAT_REUSE_MATRIX;
121:     }
122:   }

124:   /* copy matrix out onto processes */
125:   VecGetLocalSize(red->x,&m);
126:   MatDistribute_MPIAIJ(comm,red->gmat,m,scal,&red->mat);
127:   if (!red->setupcalled) {
128:     /* create the solver */
129:     KSPCreate(comm,&red->ksp);
130:     /* would like to set proper tablevel for KSP, but do not have direct access to parent pc */
131:     KSPSetOptionsPrefix(red->ksp,"openmp_"); /* should actually append with global pc prefix */
132:     KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);
133:     KSPSetFromOptions(red->ksp);
134:   } else {
135:     KSPSetOperators(red->ksp,red->mat,red->mat,red->flag);
136:   }
137:   return(0);
138: }

142: static PetscErrorCode PCSetUp_OpenMP(PC pc)
143: {
144:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;
146:   PetscMPIInt    size;

149:   red->gmat        = pc->mat;
150:   red->flag        = pc->flag;
151:   red->setupcalled = pc->setupcalled;

153:   MPI_Comm_size(red->comm,&size);
154:   if (size == 1) {  /* special case where copy of matrix is not needed */
155:     if (!red->setupcalled) {
156:       /* create the solver */
157:       KSPCreate(((PetscObject)pc)->comm,&red->ksp);
158:       PetscObjectIncrementTabLevel((PetscObject)red->ksp,(PetscObject)pc,1);
159:       KSPSetOptionsPrefix(red->ksp,"openmp_"); /* should actually append with global pc prefix */
160:       KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);
161:       KSPSetFromOptions(red->ksp);
162:     } else {
163:       KSPSetOperators(red->ksp,red->gmat,red->gmat,red->flag);
164:     }
165:     pc->ops->apply = PCApply_OpenMP_1;
166:     return(0);
167:   } else {
168:     MatGetSize(pc->mat,&red->n,PETSC_IGNORE);
169:     PetscOpenMPRun(red->comm,PCSetUp_OpenMP_MP,red);
170:   }
171:   return(0);
172: }

176: static PetscErrorCode PCApply_OpenMP_MP(MPI_Comm comm,void *ctx)
177: {
178:   PC_OpenMP      *red = (PC_OpenMP*)ctx;

182:   VecScatterBegin(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);
183:   VecScatterEnd(red->scatter,red->xdummy,red->x,INSERT_VALUES,SCATTER_REVERSE);
184:   MPI_Bcast(&red->nonzero_guess,1,MPIU_INT,0,red->comm);
185:   if (red->nonzero_guess) {
186:     VecScatterBegin(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);
187:     VecScatterEnd(red->scatter,red->ydummy,red->y,INSERT_VALUES,SCATTER_REVERSE);
188:   }
189:   KSPSetInitialGuessNonzero(red->ksp,red->nonzero_guess);

191:   KSPSolve(red->ksp,red->x,red->y);

193:   VecScatterBegin(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);
194:   VecScatterEnd(red->scatter,red->y,red->ydummy,INSERT_VALUES,SCATTER_FORWARD);
195:   return(0);
196: }

200: static PetscErrorCode PCApply_OpenMP(PC pc,Vec x,Vec y)
201: {
202:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;

206:   red->xdummy        = x;
207:   red->ydummy        = y;
208:   red->nonzero_guess = pc->nonzero_guess;
209:   PetscOpenMPRun(red->comm,PCApply_OpenMP_MP,red);
210:   return(0);
211: }

215: static PetscErrorCode PCDestroy_OpenMP_MP(MPI_Comm comm,void *ctx)
216: {
217:   PC_OpenMP      *red = (PC_OpenMP*)ctx;
218:   PetscMPIInt    rank;

222:   if (red->scatter) {VecScatterDestroy(red->scatter);}
223:   if (red->x) {VecDestroy(red->x);}
224:   if (red->y) {VecDestroy(red->y);}
225:   if (red->ksp) {KSPDestroy(red->ksp);}
226:   if (red->mat) {MatDestroy(red->mat);}
227:   MPI_Comm_rank(comm,&rank);
228:   if (rank) {
229:     if (red->xdummy) {VecDestroy(red->xdummy);}
230:     if (red->ydummy) {VecDestroy(red->ydummy);}
231:   }
232:   return(0);
233: }

237: static PetscErrorCode PCDestroy_OpenMP(PC pc)
238: {
239:   PC_OpenMP      *red = (PC_OpenMP*)pc->data;

243:   PetscOpenMPRun(red->comm,PCDestroy_OpenMP_MP,red);
244:   PetscOpenMPFree(red->comm,red);
245:   return(0);
246: }

250: static PetscErrorCode PCSetFromOptions_OpenMP(PC pc)
251: {
253:   return(0);
254: }


257: /* -------------------------------------------------------------------------------------*/
258: /*MC
259:      PCOPENMP - Runs a preconditioner for a single process matrix across several MPI processes

261: $     This will usually be run with -pc_type openmp -ksp_type preonly
262: $     solver options are set with -openmp_ksp_... and -openmp_pc_... for example
263: $     -openmp_ksp_type cg would use cg as the Krylov method or -openmp_ksp_monitor or
264: $     -openmp_pc_type hypre -openmp_pc_hypre_type boomeramg

266:        Always run with -ksp_view (or -snes_view) to see what solver is actually being used.

268:        Currently the solver options INSIDE the OpenMP preconditioner can ONLY be set via the
269:       options database.

271:    Level: intermediate

273:    See PetscOpenMPMerge() and PetscOpenMPSpawn() for two ways to start up MPI for use with this preconditioner

275: .seealso:  PCCreate(), PCSetType(), PCType (for list of available types)

277: M*/

282: PetscErrorCode  PCCreate_OpenMP(PC pc)
283: {
285:   PC_OpenMP      *red;
286:   PetscMPIInt    size;

289:   MPI_Comm_size(((PetscObject)pc)->comm,&size);
290:   if (size > 1) SETERRQ(PETSC_ERR_ARG_SIZ,"OpenMP preconditioner only works for sequential solves");
291:   /* caste the struct length to a PetscInt for easier MPI calls */

293:   PetscOpenMPMalloc(PETSC_COMM_LOCAL_WORLD,(PetscInt)sizeof(PC_OpenMP),(void**)&red);
294:   red->comm = PETSC_COMM_LOCAL_WORLD;
295:   pc->data  = (void*) red;

297:   pc->ops->apply          = PCApply_OpenMP;
298:   pc->ops->destroy        = PCDestroy_OpenMP;
299:   pc->ops->setfromoptions = PCSetFromOptions_OpenMP;
300:   pc->ops->setup          = PCSetUp_OpenMP;
301:   pc->ops->view           = PCView_OpenMP;
302:   return(0);
303: }