Actual source code: iscoloring.c
1: #define PETSCVEC_DLL
3: #include petscis.h
5: const char *ISColoringTypes[] = {"global","ghosted","ISColoringType","IS_COLORING_",0};
9: /*@
10: ISColoringDestroy - Destroys a coloring context.
12: Collective on ISColoring
14: Input Parameter:
15: . iscoloring - the coloring context
17: Level: advanced
19: .seealso: ISColoringView(), MatGetColoring()
20: @*/
21: PetscErrorCode ISColoringDestroy(ISColoring iscoloring)
22: {
23: PetscInt i;
28: if (--iscoloring->refct > 0) return(0);
30: if (iscoloring->is) {
31: for (i=0; i<iscoloring->n; i++) {
32: ISDestroy(iscoloring->is[i]);
33: }
34: PetscFree(iscoloring->is);
35: }
36: PetscFree(iscoloring->colors);
37: PetscCommDestroy(&iscoloring->comm);
38: PetscFree(iscoloring);
39: return(0);
40: }
44: /*@C
45: ISColoringView - Views a coloring context.
47: Collective on ISColoring
49: Input Parameters:
50: + iscoloring - the coloring context
51: - viewer - the viewer
53: Level: advanced
55: .seealso: ISColoringDestroy(), ISColoringGetIS(), MatGetColoring()
56: @*/
57: PetscErrorCode ISColoringView(ISColoring iscoloring,PetscViewer viewer)
58: {
59: PetscInt i;
61: PetscTruth iascii;
62: IS *is;
66: if (!viewer) {
67: PetscViewerASCIIGetStdout(iscoloring->comm,&viewer);
68: }
71: PetscTypeCompare((PetscObject)viewer,PETSC_VIEWER_ASCII,&iascii);
72: if (iascii) {
73: MPI_Comm comm;
74: PetscMPIInt rank;
75: PetscObjectGetComm((PetscObject)viewer,&comm);
76: MPI_Comm_rank(comm,&rank);
77: PetscViewerASCIISynchronizedPrintf(viewer,"[%d] Number of colors %d\n",rank,iscoloring->n);
78: PetscViewerFlush(viewer);
79: } else {
80: SETERRQ1(PETSC_ERR_SUP,"Viewer type %s not supported for ISColoring",((PetscObject)viewer)->type_name);
81: }
83: ISColoringGetIS(iscoloring,PETSC_IGNORE,&is);
84: for (i=0; i<iscoloring->n; i++) {
85: ISView(iscoloring->is[i],viewer);
86: }
87: ISColoringRestoreIS(iscoloring,&is);
88: return(0);
89: }
93: /*@C
94: ISColoringGetIS - Extracts index sets from the coloring context
96: Collective on ISColoring
98: Input Parameter:
99: . iscoloring - the coloring context
101: Output Parameters:
102: + nn - number of index sets in the coloring context
103: - is - array of index sets
105: Level: advanced
107: .seealso: ISColoringRestoreIS(), ISColoringView()
108: @*/
109: PetscErrorCode ISColoringGetIS(ISColoring iscoloring,PetscInt *nn,IS *isis[])
110: {
116: if (nn) *nn = iscoloring->n;
117: if (isis) {
118: if (!iscoloring->is) {
119: PetscInt *mcolors,**ii,nc = iscoloring->n,i,base, n = iscoloring->N;
120: ISColoringValue *colors = iscoloring->colors;
121: IS *is;
123: #if defined(PETSC_USE_DEBUG)
124: for (i=0; i<n; i++) {
125: if (((PetscInt)colors[i]) >= nc) {
126: SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Coloring is our of range index %d value %d number colors %d",(int)i,(int)colors[i],(int)nc);
127: }
128: }
129: #endif
130:
131: /* generate the lists of nodes for each color */
132: PetscMalloc(nc*sizeof(PetscInt),&mcolors);
133: PetscMemzero(mcolors,nc*sizeof(PetscInt));
134: for (i=0; i<n; i++) {
135: mcolors[colors[i]]++;
136: }
138: PetscMalloc(nc*sizeof(PetscInt*),&ii);
139: PetscMalloc(n*sizeof(PetscInt),&ii[0]);
140: for (i=1; i<nc; i++) {
141: ii[i] = ii[i-1] + mcolors[i-1];
142: }
143: PetscMemzero(mcolors,nc*sizeof(PetscInt));
145: if (iscoloring->ctype == IS_COLORING_GLOBAL){
146: MPI_Scan(&iscoloring->N,&base,1,MPIU_INT,MPI_SUM,iscoloring->comm);
147: base -= iscoloring->N;
148: for (i=0; i<n; i++) {
149: ii[colors[i]][mcolors[colors[i]]++] = i + base; /* global idx */
150: }
151: } else if (iscoloring->ctype == IS_COLORING_GHOSTED){
152: for (i=0; i<n; i++) {
153: ii[colors[i]][mcolors[colors[i]]++] = i; /* local idx */
154: }
155: } else {
156: SETERRQ(PETSC_ERR_SUP,"Not provided for this ISColoringType type");
157: }
158:
159: PetscMalloc(nc*sizeof(IS),&is);
160: for (i=0; i<nc; i++) {
161: ISCreateGeneral(iscoloring->comm,mcolors[i],ii[i],is+i);
162: }
164: iscoloring->is = is;
165: PetscFree(ii[0]);
166: PetscFree(ii);
167: PetscFree(mcolors);
168: }
169: *isis = iscoloring->is;
170: }
171: return(0);
172: }
176: /*@C
177: ISColoringRestoreIS - Restores the index sets extracted from the coloring context
179: Collective on ISColoring
181: Input Parameter:
182: + iscoloring - the coloring context
183: - is - array of index sets
185: Level: advanced
187: .seealso: ISColoringGetIS(), ISColoringView()
188: @*/
189: PetscErrorCode ISColoringRestoreIS(ISColoring iscoloring,IS *is[])
190: {
193:
194: /* currently nothing is done here */
196: return(0);
197: }
202: /*@C
203: ISColoringCreate - Generates an ISColoring context from lists (provided
204: by each processor) of colors for each node.
206: Collective on MPI_Comm
208: Input Parameters:
209: + comm - communicator for the processors creating the coloring
210: . ncolors - max color value
211: . n - number of nodes on this processor
212: - colors - array containing the colors for this processor, color
213: numbers begin at 0. In C/C++ this array must have been obtained with PetscMalloc()
214: and should NOT be freed (The ISColoringDestroy() will free it).
216: Output Parameter:
217: . iscoloring - the resulting coloring data structure
219: Options Database Key:
220: . -is_coloring_view - Activates ISColoringView()
222: Level: advanced
223:
224: Notes: By default sets coloring type to IS_COLORING_GLOBAL
226: .seealso: MatColoringCreate(), ISColoringView(), ISColoringDestroy(), ISColoringSetType()
228: @*/
229: PetscErrorCode ISColoringCreate(MPI_Comm comm,PetscInt ncolors,PetscInt n,const ISColoringValue colors[],ISColoring *iscoloring)
230: {
232: PetscMPIInt size,rank,tag;
233: PetscInt base,top,i;
234: PetscInt nc,ncwork;
235: PetscTruth flg = PETSC_FALSE;
236: MPI_Status status;
239: if (ncolors != PETSC_DECIDE && ncolors > IS_COLORING_MAX) {
240: if (ncolors > 65535) {
241: SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds 65535 limit. This number is unrealistic. Perhaps a bug in code?\nCurrent max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
242: } else {
243: SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"Max color value exeeds limit. Perhaps reconfigure PETSc with --with-is-color-value-type=short?\n Current max: %d user rewuested: %d",IS_COLORING_MAX,ncolors);
244: }
245: }
246: PetscNew(struct _n_ISColoring,iscoloring);
247: PetscCommDuplicate(comm,&(*iscoloring)->comm,&tag);
248: comm = (*iscoloring)->comm;
250: /* compute the number of the first node on my processor */
251: MPI_Comm_size(comm,&size);
253: /* should use MPI_Scan() */
254: MPI_Comm_rank(comm,&rank);
255: if (!rank) {
256: base = 0;
257: top = n;
258: } else {
259: MPI_Recv(&base,1,MPIU_INT,rank-1,tag,comm,&status);
260: top = base+n;
261: }
262: if (rank < size-1) {
263: MPI_Send(&top,1,MPIU_INT,rank+1,tag,comm);
264: }
266: /* compute the total number of colors */
267: ncwork = 0;
268: for (i=0; i<n; i++) {
269: if (ncwork < colors[i]) ncwork = colors[i];
270: }
271: ncwork++;
272: MPI_Allreduce(&ncwork,&nc,1,MPIU_INT,MPI_MAX,comm);
273: if (nc > ncolors) SETERRQ2(PETSC_ERR_ARG_INCOMP,"Number of colors passed in %D is less then the actual number of colors in array %D",ncolors,nc);
274: (*iscoloring)->n = nc;
275: (*iscoloring)->is = 0;
276: (*iscoloring)->colors = (ISColoringValue *)colors;
277: (*iscoloring)->N = n;
278: (*iscoloring)->refct = 1;
279: (*iscoloring)->ctype = IS_COLORING_GLOBAL;
281: PetscOptionsGetTruth(PETSC_NULL,"-is_coloring_view",&flg,PETSC_NULL);
282: if (flg) {
283: PetscViewer viewer;
284: PetscViewerASCIIGetStdout((*iscoloring)->comm,&viewer);
285: ISColoringView(*iscoloring,viewer);
286: }
287: PetscInfo1(0,"Number of colors %d\n",nc);
288: return(0);
289: }
293: /*@
294: ISPartitioningToNumbering - Takes an ISPartitioning and on each processor
295: generates an IS that contains a new global node number for each index based
296: on the partitioing.
298: Collective on IS
300: Input Parameters
301: . partitioning - a partitioning as generated by MatPartitioningApply()
303: Output Parameter:
304: . is - on each processor the index set that defines the global numbers
305: (in the new numbering) for all the nodes currently (before the partitioning)
306: on that processor
308: Level: advanced
310: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningCount()
312: @*/
313: PetscErrorCode ISPartitioningToNumbering(IS part,IS *is)
314: {
315: MPI_Comm comm;
316: PetscInt i,np,npt,n,*starts = PETSC_NULL,*sums = PETSC_NULL,*lsizes = PETSC_NULL,*newi = PETSC_NULL;
317: const PetscInt *indices = PETSC_NULL;
321: PetscObjectGetComm((PetscObject)part,&comm);
323: /* count the number of partitions, i.e., virtual processors */
324: ISGetLocalSize(part,&n);
325: ISGetIndices(part,&indices);
326: np = 0;
327: for (i=0; i<n; i++) {
328: np = PetscMax(np,indices[i]);
329: }
330: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
331: np = npt+1; /* so that it looks like a MPI_Comm_size output */
333: /*
334: lsizes - number of elements of each partition on this particular processor
335: sums - total number of "previous" nodes for any particular partition
336: starts - global number of first element in each partition on this processor
337: */
338: PetscMalloc3(np,PetscInt,&lsizes,np,PetscInt,&starts,np,PetscInt,&sums);
339: PetscMemzero(lsizes,np*sizeof(PetscInt));
340: for (i=0; i<n; i++) {
341: lsizes[indices[i]]++;
342: }
343: MPI_Allreduce(lsizes,sums,np,MPIU_INT,MPI_SUM,comm);
344: MPI_Scan(lsizes,starts,np,MPIU_INT,MPI_SUM,comm);
345: for (i=0; i<np; i++) {
346: starts[i] -= lsizes[i];
347: }
348: for (i=1; i<np; i++) {
349: sums[i] += sums[i-1];
350: starts[i] += sums[i-1];
351: }
353: /*
354: For each local index give it the new global number
355: */
356: PetscMalloc(n*sizeof(PetscInt),&newi);
357: for (i=0; i<n; i++) {
358: newi[i] = starts[indices[i]]++;
359: }
360: PetscFree3(lsizes,starts,sums);
362: ISRestoreIndices(part,&indices);
363: ISCreateGeneral(comm,n,newi,is);
364: PetscFree(newi);
365: ISSetPermutation(*is);
366: return(0);
367: }
371: /*@
372: ISPartitioningCount - Takes a ISPartitioning and determines the number of
373: resulting elements on each (partition) process
375: Collective on IS
377: Input Parameters:
378: + partitioning - a partitioning as generated by MatPartitioningApply()
379: - len - length of the array count, this is the total number of partitions
381: Output Parameter:
382: . count - array of length size, to contain the number of elements assigned
383: to each partition, where size is the number of partitions generated
384: (see notes below).
386: Level: advanced
388: Notes:
389: By default the number of partitions generated (and thus the length
390: of count) is the size of the communicator associated with IS,
391: but it can be set by MatPartitioningSetNParts. The resulting array
392: of lengths can for instance serve as input of PCBJacobiSetTotalBlocks.
395: .seealso: MatPartitioningCreate(), AOCreateBasic(), ISPartitioningToNumbering(),
396: MatPartitioningSetNParts()
398: @*/
399: PetscErrorCode ISPartitioningCount(IS part,PetscInt len,PetscInt count[])
400: {
401: MPI_Comm comm;
402: PetscInt i,n,*lsizes;
403: const PetscInt *indices;
405: PetscMPIInt npp;
408: PetscObjectGetComm((PetscObject)part,&comm);
409: if (len == PETSC_DEFAULT) {
410: PetscMPIInt size;
411: MPI_Comm_size(comm,&size);
412: len = (PetscInt) size;
413: }
415: /* count the number of partitions */
416: ISGetLocalSize(part,&n);
417: ISGetIndices(part,&indices);
418: #if defined(PETSC_USE_DEBUG)
419: {
420: PetscInt np = 0,npt;
421: for (i=0; i<n; i++) {
422: np = PetscMax(np,indices[i]);
423: }
424: MPI_Allreduce(&np,&npt,1,MPIU_INT,MPI_MAX,comm);
425: np = npt+1; /* so that it looks like a MPI_Comm_size output */
426: if (np > len) SETERRQ2(PETSC_ERR_ARG_SIZ,"Length of count array %D is less than number of partitions %D",len,np);
427: }
428: #endif
430: /*
431: lsizes - number of elements of each partition on this particular processor
432: sums - total number of "previous" nodes for any particular partition
433: starts - global number of first element in each partition on this processor
434: */
435: PetscMalloc(len*sizeof(PetscInt),&lsizes);
436: PetscMemzero(lsizes,len*sizeof(PetscInt));
437: for (i=0; i<n; i++) {
438: lsizes[indices[i]]++;
439: }
440: ISRestoreIndices(part,&indices);
441: npp = PetscMPIIntCast(len);
442: MPI_Allreduce(lsizes,count,npp,MPIU_INT,MPI_SUM,comm);
443: PetscFree(lsizes);
444: return(0);
445: }
449: /*@
450: ISAllGather - Given an index set (IS) on each processor, generates a large
451: index set (same on each processor) by concatenating together each
452: processors index set.
454: Collective on IS
456: Input Parameter:
457: . is - the distributed index set
459: Output Parameter:
460: . isout - the concatenated index set (same on all processors)
462: Notes:
463: ISAllGather() is clearly not scalable for large index sets.
465: The IS created on each processor must be created with a common
466: communicator (e.g., PETSC_COMM_WORLD). If the index sets were created
467: with PETSC_COMM_SELF, this routine will not work as expected, since
468: each process will generate its own new IS that consists only of
469: itself.
471: The communicator for this new IS is PETSC_COMM_SELF
473: Level: intermediate
475: Concepts: gather^index sets
476: Concepts: index sets^gathering to all processors
477: Concepts: IS^gathering to all processors
479: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices()
480: @*/
481: PetscErrorCode ISAllGather(IS is,IS *isout)
482: {
484: PetscInt *indices,n,i,N,step,first;
485: const PetscInt *lindices;
486: MPI_Comm comm;
487: PetscMPIInt size,*sizes = PETSC_NULL,*offsets = PETSC_NULL,nn;
488: PetscTruth stride;
494: PetscObjectGetComm((PetscObject)is,&comm);
495: MPI_Comm_size(comm,&size);
496: ISGetLocalSize(is,&n);
497: ISStride(is,&stride);
498: if (size == 1 && stride) { /* should handle parallel ISStride also */
499: ISStrideGetInfo(is,&first,&step);
500: ISCreateStride(PETSC_COMM_SELF,n,first,step,isout);
501: } else {
502: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
503:
504: nn = PetscMPIIntCast(n);
505: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
506: offsets[0] = 0;
507: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
508: N = offsets[size-1] + sizes[size-1];
509:
510: PetscMalloc(N*sizeof(PetscInt),&indices);
511: ISGetIndices(is,&lindices);
512: MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
513: ISRestoreIndices(is,&lindices);
514: PetscFree2(sizes,offsets);
516: ISCreateGeneral(PETSC_COMM_SELF,N,indices,isout);
517: PetscFree(indices);
518: }
519: return(0);
520: }
524: /*@C
525: ISAllGatherIndices - Given a a set of integers on each processor, generates a large
526: set (same on each processor) by concatenating together each processors integers
528: Collective on MPI_Comm
530: Input Parameter:
531: + comm - communicator to share the indices
532: . n - local size of set
533: - lindices - local indices
535: Output Parameter:
536: + outN - total number of indices
537: - outindices - all of the integers
539: Notes:
540: ISAllGatherIndices() is clearly not scalable for large index sets.
543: Level: intermediate
545: Concepts: gather^index sets
546: Concepts: index sets^gathering to all processors
547: Concepts: IS^gathering to all processors
549: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather()
550: @*/
551: PetscErrorCode ISAllGatherIndices(MPI_Comm comm,PetscInt n,const PetscInt lindices[],PetscInt *outN,PetscInt *outindices[])
552: {
554: PetscInt *indices,i,N;
555: PetscMPIInt size,*sizes = PETSC_NULL,*offsets = PETSC_NULL,nn;
558: MPI_Comm_size(comm,&size);
559: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
560:
561: nn = PetscMPIIntCast(n);
562: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
563: offsets[0] = 0;
564: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
565: N = offsets[size-1] + sizes[size-1];
567: PetscMalloc(N*sizeof(PetscInt),&indices);
568: MPI_Allgatherv((void*)lindices,nn,MPIU_INT,indices,sizes,offsets,MPIU_INT,comm);
569: PetscFree2(sizes,offsets);
571: *outindices = indices;
572: if (outN) *outN = N;
573: return(0);
574: }
580: /*@C
581: ISAllGatherColors - Given a a set of colors on each processor, generates a large
582: set (same on each processor) by concatenating together each processors colors
584: Collective on MPI_Comm
586: Input Parameter:
587: + comm - communicator to share the indices
588: . n - local size of set
589: - lindices - local colors
591: Output Parameter:
592: + outN - total number of indices
593: - outindices - all of the colors
595: Notes:
596: ISAllGatherColors() is clearly not scalable for large index sets.
599: Level: intermediate
601: Concepts: gather^index sets
602: Concepts: index sets^gathering to all processors
603: Concepts: IS^gathering to all processors
605: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGather(), ISAllGatherIndices()
606: @*/
607: PetscErrorCode ISAllGatherColors(MPI_Comm comm,PetscInt n,ISColoringValue *lindices,PetscInt *outN,ISColoringValue *outindices[])
608: {
609: ISColoringValue *indices;
610: PetscErrorCode ierr;
611: PetscInt i,N;
612: PetscMPIInt size,*offsets = PETSC_NULL,*sizes = PETSC_NULL, nn = n;
615: MPI_Comm_size(comm,&size);
616: PetscMalloc2(size,PetscMPIInt,&sizes,size,PetscMPIInt,&offsets);
617:
618: MPI_Allgather(&nn,1,MPI_INT,sizes,1,MPI_INT,comm);
619: offsets[0] = 0;
620: for (i=1;i<size; i++) offsets[i] = offsets[i-1] + sizes[i-1];
621: N = offsets[size-1] + sizes[size-1];
622: PetscFree2(sizes,offsets);
624: PetscMalloc((N+1)*sizeof(ISColoringValue),&indices);
625: MPI_Allgatherv(lindices,(PetscMPIInt)n,MPIU_COLORING_VALUE,indices,sizes,offsets,MPIU_COLORING_VALUE,comm);
627: *outindices = indices;
628: if (outN) *outN = N;
629: return(0);
630: }
634: /*@
635: ISComplement - Given an index set (IS) generates the complement index set. That is all
636: all indices that are NOT in the given set.
638: Collective on IS
640: Input Parameter:
641: + is - the index set
642: . nmin - the first index desired in the local part of the complement
643: - nmax - the largest index desired in the local part of the complement (note that all indices in is must be greater or equal to nmin and less than nmax)
645: Output Parameter:
646: . isout - the complement
648: Notes: The communicator for this new IS is the same as for the input IS
650: For a parallel IS, this will generate the local part of the complement on each process
652: To generate the entire complement (on each process) of a parallel IS, first call ISAllGather() and then
653: call this routine.
655: Level: intermediate
657: Concepts: gather^index sets
658: Concepts: index sets^gathering to all processors
659: Concepts: IS^gathering to all processors
661: .seealso: ISCreateGeneral(), ISCreateStride(), ISCreateBlock(), ISAllGatherIndices(), ISAllGather()
662: @*/
663: PetscErrorCode ISComplement(IS is,PetscInt nmin,PetscInt nmax,IS *isout)
664: {
666: const PetscInt *indices;
667: PetscInt n,i,j,cnt,*nindices;
668: PetscTruth sorted;
673: if (nmin < 0) SETERRQ1(PETSC_ERR_ARG_OUTOFRANGE,"nmin %D cannot be negative",nmin);
674: if (nmin > nmax) SETERRQ2(PETSC_ERR_ARG_OUTOFRANGE,"nmin %D cannot be greater than nmax %D",nmin,nmax);
675: ISSorted(is,&sorted);
676: if (!sorted) SETERRQ(PETSC_ERR_ARG_WRONG,"Index set must be sorted");
678: ISGetLocalSize(is,&n);
679: ISGetIndices(is,&indices);
680: #if defined(PETSC_USE_DEBUG)
681: for (i=0; i<n; i++) {
682: if (indices[i] < nmin) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Index %D's value %D is smaller than minimum given %D",i,indices[i],nmin);
683: if (indices[i] >= nmax) SETERRQ3(PETSC_ERR_ARG_OUTOFRANGE,"Index %D's value %D is larger than maximum given %D",i,indices[i],nmax);
684: }
685: #endif
686: PetscMalloc((nmax - n)*sizeof(PetscInt),&nindices);
687: cnt = 0;
688: j = nmin;
689: for (i=0; i<n; i++) {
690: for (; j<indices[i]; j++) {
691: nindices[cnt++] = j;
692: }
693: j++;
694: }
695: for (; j<nmax; j++) {
696: nindices[cnt++] = j;
697: }
698: if (cnt != nmax-nmin - n) SETERRQ2(PETSC_ERR_PLIB,"Number entries found in complement %D does not match expected %D",cnt,nmax-n);
699: ISCreateGeneral(((PetscObject)is)->comm,nmax-nmin-n,nindices,isout);
700: ISRestoreIndices(is,&indices);
701: PetscFree(nindices);
702: return(0);
703: }