Actual source code: jostle.c
1: #define PETSCMAT_DLL
3: #include ../src/mat/impls/adj/mpi/mpiadj.h
5: #ifdef PETSC_HAVE_UNISTD_H
6: #include <unistd.h>
7: #endif
9: #ifdef PETSC_HAVE_STDLIB_H
10: #include <stdlib.h>
11: #endif
15: #include "jostle.h"
16: /* this function is not declared in 'jostle.h' */
21: typedef struct {
22: int output;
23: int coarse_seq;
24: int nbvtxcoarsed; /* number of vertices for the coarse graph */
25: char *mesg_log;
26: } MatPartitioning_Jostle;
28: #define SIZE_LOG 10000 /* size of buffer for msg_log */
32: static PetscErrorCode MatPartitioningApply_Jostle(MatPartitioning part, IS * partitioning)
33: {
35: int size, rank, i;
36: Mat mat = part->adj, matMPI;
37: Mat_MPIAdj *adj = (Mat_MPIAdj *) mat->data;
38: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
39: PetscTruth flg;
40: #ifdef PETSC_HAVE_UNISTD_H
41: int fd_stdout, fd_pipe[2], count,err;
42: #endif
46: /* check that the number of partitions is equal to the number of processors */
47: MPI_Comm_rank(((PetscObject)mat)->comm, &rank);
48: MPI_Comm_size(((PetscObject)mat)->comm, &size);
49: if (part->n != size) SETERRQ(PETSC_ERR_SUP, "Supports exactly one domain per processor");
51: /* convert adjacency matrix to MPIAdj if needed*/
52: PetscTypeCompare((PetscObject) mat, MATMPIADJ, &flg);
53: if (!flg) {
54: MatConvert(mat, MATMPIADJ, MAT_INITIAL_MATRIX, &matMPI);
55: } else {
56: matMPI = mat;
57: }
59: adj = (Mat_MPIAdj *) matMPI->data; /* adj contains adjacency graph */
60: {
61: /* definition of Jostle library arguments */
62: int nnodes = matMPI->M; /* number of vertices in full graph */
63: int offset = 0; /* 0 for C array indexing */
64: int core = matMPI->m;
65: int halo = 0; /* obsolete with contiguous format */
66: int *index_jostle; /* contribution of each processor */
67: int nparts = part->n;
68: int *part_wt = NULL;
70: int *partition; /* set number of each vtx (length n) */
71: int *degree; /* degree for each core nodes */
72: int *edges = adj->j;
73: int *node_wt = NULL; /* nodes weights */
74: int *edge_wt = NULL; /* edges weights */
75: double *coords = NULL; /* not used (cf jostle documentation) */
77: int local_nedges = adj->nz;
78: int dimension = 0; /* not used */
79: int output_level = jostle_struct->output;
80: char env_str[256];
82: /* allocate index_jostle */
83: PetscMalloc(nparts * sizeof(int), &index_jostle);
85: /* compute number of core nodes for each one */
86: for (i = 0; i < nparts - 1; i++)
87: index_jostle[i] = adj->rowners[i + 1] - adj->rowners[i];
88: index_jostle[nparts - 1] = nnodes - adj->rowners[nparts - 1];
90: /* allocate the partition vector */
91: PetscMalloc(core * sizeof(int), &partition);
93: /* build the degree vector and the local_nedges value */
94: PetscMalloc(core * sizeof(int), °ree);
95: for (i = 0; i < core; i++)
96: degree[i] = adj->i[i + 1] - adj->i[i];
98: /* library call */
99: pjostle_init(&size, &rank);
100: pjostle_comm(&((PetscObject)matMPI)->comm);
101: jostle_env("format = contiguous");
102: jostle_env("timer = off");
104: sprintf(env_str, "threshold = %d", jostle_struct->nbvtxcoarsed);
105: jostle_env(env_str);
107: if (jostle_struct->coarse_seq)
108: jostle_env("matching = local");
110: /* redirect output */
111: #ifdef PETSC_HAVE_UNISTD_H
112: fd_stdout = dup(1);
113: pipe(fd_pipe);
114: close(1);
115: dup2(fd_pipe[1], 1);
116: #endif
118: pjostle(&nnodes, &offset, &core, &halo, index_jostle, degree, node_wt,
119: partition, &local_nedges, edges, edge_wt, &nparts,
120: part_wt, &output_level, &dimension, coords);
122: printf("Jostle Partitioner statistics\ncut : %d, balance : %f, runtime : %f, mem used : %d\n",
123: jostle_cut(), jostle_bal(), jostle_tim(), jostle_mem());
125: #ifdef PETSC_HAVE_UNISTD_H
126: PetscMalloc(SIZE_LOG * sizeof(char), &(jostle_struct->mesg_log));
127: err = fflush(stdout);
128: if (err) SETERRQ(PETSC_ERR_SYS,"fflush() failed on stdout");
129: count = read(fd_pipe[0], jostle_struct->mesg_log, (SIZE_LOG - 1) * sizeof(char));
130: if (count < 0)
131: count = 0;
132: jostle_struct->mesg_log[count] = 0;
133: close(1);
134: dup2(fd_stdout, 1);
135: close(fd_stdout);
136: close(fd_pipe[0]);
137: close(fd_pipe[1]);
138: #endif
140: /* We free the memory used by jostle */
141: PetscFree(index_jostle);
142: PetscFree(degree);
144: /* Creation of the index set */
145: ISCreateGeneral(((PetscObject)part)->comm, mat->m, partition, partitioning);
147: if (matMPI != mat) {
148: MatDestroy(matMPI);
149: }
151: PetscFree(partition);
152: }
154: return(0);
155: }
160: PetscErrorCode MatPartitioningView_Jostle(MatPartitioning part, PetscViewer viewer)
161: {
162: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
163: PetscErrorCode ierr;
164: PetscTruth iascii;
167: PetscTypeCompare((PetscObject) viewer, PETSC_VIEWER_ASCII, &iascii);
168: if (iascii) {
169: if (jostle_struct->mesg_log) {
170: PetscViewerASCIIPrintf(viewer, "%s\n", jostle_struct->mesg_log);
171: }
172: } else {
173: SETERRQ1(PETSC_ERR_SUP, "Viewer type %s not supported for this Jostle partitioner",((PetscObject)viewer)->type_name);
174: }
175: return(0);
176: }
180: /*@
181: MatPartitioningJostleSetCoarseLevel - Set the coarse level
182:
183: Input Parameter:
184: . part - the partitioning context
185: . level - the coarse level in range [0.0,1.0]
187: Level: advanced
189: @*/
190: PetscErrorCode MatPartitioningJostleSetCoarseLevel(MatPartitioning part, PetscReal level)
191: {
192: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
196: if (level < 0.0 || level > 1.0) {
197: SETERRQ(PETSC_ERR_ARG_OUTOFRANGE,
198: "Jostle: level of coarsening out of range [0.0-1.0]");
199: } else
200: jostle_struct->nbvtxcoarsed = (int)(part->adj->N * level);
202: if (jostle_struct->nbvtxcoarsed < 20)
203: jostle_struct->nbvtxcoarsed = 20;
205: return(0);
206: }
210: /*@
211: MatPartitioningJostleSetCoarseSequential - Use the sequential code to
212: do the partitioning of the coarse grid.
214: Input Parameter:
215: . part - the partitioning context
217: Level: advanced
219: @*/
220: PetscErrorCode MatPartitioningJostleSetCoarseSequential(MatPartitioning part)
221: {
222: MatPartitioning_Jostle *jostle_struct =
223: (MatPartitioning_Jostle *) part->data;
225: jostle_struct->coarse_seq = 1;
226: return(0);
227: }
231: PetscErrorCode MatPartitioningSetFromOptions_Jostle(MatPartitioning part)
232: {
234: PetscTruth flag = PETSC_FALSE;
235: PetscReal level;
238: PetscOptionsHead("Set Jostle partitioning options");
240: PetscOptionsReal("-mat_partitioning_jostle_coarse_level","Coarse level", "MatPartitioningJostleSetCoarseLevel", 0, &level, &flag);
241: if (flag) {
242: MatPartitioningJostleSetCoarseLevel(part, level);
243: }
245: flag = PETSC_FALSE;
246: PetscOptionsTruth("-mat_partitioning_jostle_coarse_sequential","Use sequential coarse partitioner","MatPartitioningJostleSetCoarseSequential",flag,&flag,PETSC_NULL);
247: if (flag) {
248: MatPartitioningJostleSetCoarseSequential(part);
249: }
251: PetscOptionsTail();
252: return(0);
253: }
258: PetscErrorCode MatPartitioningDestroy_Jostle(MatPartitioning part)
259: {
260: MatPartitioning_Jostle *jostle_struct = (MatPartitioning_Jostle *) part->data;
261: PetscErrorCode ierr;
264: PetscFree(jostle_struct->mesg_log);
265: PetscFree(jostle_struct);
266: return(0);
267: }
269: /*MC
270: MAT_PARTITIONING_JOSTLE - Creates a partitioning context via the external package Jostle.
272: Collective on MPI_Comm
274: Input Parameter:
275: . part - the partitioning context
277: Options Database Keys:
278: + -mat_partitioning_jostle_coarse_level <0>: Coarse level (MatPartitioningJostleSetCoarseLevel)
279: - -mat_partitioning_jostle_coarse_sequential: Use sequential coarse partitioner (MatPartitioningJostleSetCoarseSequential)
281: Level: beginner
283: Notes: See http://www.gre.ac.uk/~c.walshaw/jostle/
285: .keywords: Partitioning, create, context
287: .seealso: MatPartitioningSetType(), MatPartitioningType
289: M*/
294: PetscErrorCode MatPartitioningCreate_Jostle(MatPartitioning part)
295: {
297: MatPartitioning_Jostle *jostle_struct;
300: PetscNewLog(part,MatPartitioning_Jostle, &jostle_struct);
301: part->data = (void*) jostle_struct;
303: jostle_struct->nbvtxcoarsed = 20;
304: jostle_struct->output = 0;
305: jostle_struct->coarse_seq = 0;
306: jostle_struct->mesg_log = NULL;
308: part->ops->apply = MatPartitioningApply_Jostle;
309: part->ops->view = MatPartitioningView_Jostle;
310: part->ops->destroy = MatPartitioningDestroy_Jostle;
311: part->ops->setfromoptions = MatPartitioningSetFromOptions_Jostle;
313: return(0);
314: }