FrontISTR  5.7.0
Large-scale structural analysis program with finit element method
hecmw_dlb_comm_util.c
Go to the documentation of this file.
1 /*****************************************************************************
2  * Copyright (c) 2019 FrontISTR Commons
3  * This software is released under the MIT License, see LICENSE.txt
4  *****************************************************************************/
5 
6 #include "hecmw_repart.h"
7 
8 /********** MOVE TO hecmw_comm.c (05/08/10 by N.Imai) ***********
9 
10 int
11 HECMW_Comm_rank( HECMW_Comm comm, int *rank )
12 {
13 #ifndef HECMW_SERIAL
14  int rtc;
15 
16  rtc = MPI_Comm_rank( comm, rank );
17  if( rtc != MPI_SUCCESS ) {
18  HECMW_set_error( HECMW_ALL_E1003, "MPI_Comm_rank" );
19  goto error;
20  }
21 
22  return 0;
23 
24 error:
25  return -1;
26 #else
27  *rank=0;
28  return 0;
29 #endif
30 }
31 
32 int
33 HECMW_Comm_size( HECMW_Comm comm, int *size )
34 {
35 #ifndef HECMW_SERIAL
36  int rtc;
37 
38  rtc = MPI_Comm_size( comm, size );
39  if( rtc != MPI_SUCCESS ) {
40  HECMW_set_error( HECMW_ALL_E1003, "MPI_Comm_size" );
41  goto error;
42  }
43 
44  return 0;
45 
46 error:
47  return -1;
48 #else
49  *size=1;
50  return 0;
51 #endif
52 }
53 
54 int
55 HECMW_Comm_dup( HECMW_Comm comm, HECMW_Comm *new_comm )
56 {
57 #ifndef HECMW_SERIAL
58  int rtc;
59 
60  rtc = MPI_Comm_dup(comm, new_comm);
61  if( rtc != MPI_SUCCESS ) {
62  HECMW_set_error( HECMW_ALL_E1003, "MPI_Comm_dup" );
63  goto error;
64  }
65 
66  return 0;
67 
68 error:
69  return -1;
70 #else
71  *new_comm=0;
72  return 0;
73 #endif
74 }
75 
76 
77 int
78 HECMW_Finalize()
79 {
80 #ifndef HECMW_SERIAL
81  int rtc;
82 
83  rtc = MPI_Finalize();
84  if( rtc != MPI_SUCCESS ) {
85  HECMW_set_error( HECMW_ALL_E1003, "MPI_Finalize" );
86  goto error;
87  }
88 
89  return 0;
90 
91 error:
92  return -1;
93 #else
94  exit(0);
95 #endif
96 }
97 
98 ***********************************************************/
99 
100 void whole_copy_array(int *recv_num, int *global_recv_num, int mynode,
101  int pesize, HECMW_Comm repart_comm) {
102  int i, j;
103  int *tmp_recv;
104  HECMW_Status stat;
105 
106  if (mynode == 0) {
107  for (j = 0; j < pesize + 1; j++) global_recv_num[j] = recv_num[j];
108  tmp_recv = (int *)calloc(pesize + 1, sizeof(int));
109  if (tmp_recv == NULL) HECMW_vis_memory_exit("tmp_recv");
110  for (i = 1; i < pesize; i++) {
111  HECMW_Recv(tmp_recv, pesize + 1, HECMW_INT, i, HECMW_ANY_TAG, repart_comm,
112  &stat);
113  for (j = 0; j < pesize + 1; j++)
114  global_recv_num[i * (pesize + 1) + j] = tmp_recv[j];
115  }
116  for (i = 1; i < pesize; i++)
117  HECMW_Send(global_recv_num, (pesize + 1) * pesize, HECMW_INT, i, 0,
118  repart_comm);
119  free(tmp_recv);
120  } else {
121  HECMW_Send(recv_num, pesize + 1, HECMW_INT, 0, 0, repart_comm);
122  HECMW_Recv(global_recv_num, (pesize + 1) * pesize, HECMW_INT, 0,
123  HECMW_ANY_TAG, repart_comm, &stat);
124  }
125  return;
126 }
127 
128 int stack_part_send_recv(int neibpetot, int *neibpe, int *stack_import,
129  int *stack_export, HECMW_Comm repart_comm,
130  int my_rank) {
131  HECMW_Status *sta1, *sta2;
132  HECMW_Request *req1, *req2;
133 
134  int nflag = 0;
135  int neib;
136  int istart, num;
137  int k;
138 
139  if (nflag == 0) {
140  sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
141  if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
142  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
143  if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
144  if ((req1 = (HECMW_Request *)calloc(neibpetot, sizeof(HECMW_Request))) ==
145  NULL)
146  HECMW_vis_memory_exit("HECMW_STATUS: req1");
147  if ((req2 = (HECMW_Request *)calloc(neibpetot, sizeof(HECMW_Request))) ==
148  NULL)
149  HECMW_vis_memory_exit("HECMW_STATUS: req2");
150  nflag = 1;
151  }
152 
153  for (neib = 0; neib < neibpetot; neib++) {
154  num = stack_import[neib];
155  HECMW_Isend(&num, 1, HECMW_INT, neibpe[neib], 0, repart_comm, &req1[neib]);
156  }
157  for (neib = 0; neib < neibpetot; neib++) {
158  HECMW_Irecv(&stack_export[neib], 1, HECMW_INT, neibpe[neib], 0, repart_comm,
159  &req2[neib]);
160  /* fprintf(stderr, "PE %d recv %d from %d\n", my_rank, stack_export[neib],
161  * neib);
162  */
163  }
164  HECMW_Barrier(repart_comm);
165  free(sta1);
166  free(sta2);
167  free(req1);
168  free(req2);
169 
170  return;
171 }
172 
173 int stack_whole_send_recv(int pesize, int *stack_export, int *stack_import,
174  HECMW_Comm repart_comm, int my_rank) {
175  HECMW_Status stat;
176  int tmp_int;
177  int i, j;
178 
179  /*
180  HECMW_Status *sta1, *sta2;
181  HECMW_Request *req1, *req2;
182 
183  int nflag = 0;
184  int neib;
185  int istart, num, num1;
186  int k;
187 
188  stack_export[0]=0;
189  stack_export[my_rank+1]=stack_import[my_rank+1]-stack_import[my_rank];
190 
191  if (nflag == 0) {
192  sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE*(pesize-1),
193  sizeof(HECMW_Status));
194  if (sta1 == NULL)
195  HECMW_vis_memory_exit("HECMW_STATUS: stat1");
196  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE*(pesize-1),
197  sizeof(HECMW_Status));
198  if (sta2 == NULL)
199  HECMW_vis_memory_exit("HECMW_STATUS: stat2");
200  if ((req1 = (HECMW_Request *)calloc(pesize-1, sizeof(HECMW_Request)))
201  == NULL)
202  HECMW_vis_memory_exit("HECMW_STATUS: req1");
203  if ((req2 = (HECMW_Request *)calloc(pesize-1, sizeof(HECMW_Request)))
204  == NULL)
205  HECMW_vis_memory_exit("HECMW_STATUS: req2");
206  nflag = 1;
207  }
208 
209  for(neib=0;neib<pesize;neib++) {
210  if(neib!=my_rank) {
211  num=stack_import[neib+1]-stack_import[neib];
212  HECMW_Isend(&num, 1, HECMW_INT,
213  neib, 0, repart_comm, &req1[neib]);
214  fprintf(stderr, "pe %d send %d to %d\n", my_rank, num, neib);
215  }
216  }
217  for (neib = 0; neib < pesize; neib++) {
218  if(neib!=my_rank) {
219  HECMW_Irecv(&stack_export[neib+1], 1, HECMW_INT,
220  neib, 0, repart_comm, &req2[neib]);
221 
222  fprintf(stderr, "pe %d recv %d from %d\n", my_rank,
223  stack_export[neib+1], neib);
224 
225 
226  }
227  }
228 
229 
230  HECMW_Barrier(repart_comm);
231 
232  free(sta1);
233  free(sta2);
234  free(req1);
235  free(req2);
236  */
237  for (i = 0; i < pesize; i++) {
238  if (i != my_rank) {
239  tmp_int = stack_export[i + 1] - stack_export[i];
240  HECMW_Send(&tmp_int, 1, HECMW_INT, i, 0, repart_comm);
241  } else if (i == my_rank) {
242  stack_import[i + 1] = stack_export[i + 1] - stack_export[i];
243  for (j = 0; j < pesize; j++) {
244  if (j != my_rank) {
245  HECMW_Recv(&stack_import[j + 1], 1, HECMW_INT, j, HECMW_ANY_TAG,
246  repart_comm, &stat);
247  }
248  }
249  stack_import[0] = 0;
250  for (j = 1; j < pesize + 1; j++)
251  stack_import[j] = stack_import[j] + stack_import[j - 1];
252  }
253  }
254 
255  HECMW_Barrier(repart_comm);
256 
257  return;
258 }
259 
260 int int_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
261  int *nod_import, int *stack_export, int *nod_export,
262  int *x, HECMW_Comm repart_comm, int my_rank) {
263  /* Important:: node ID in nod_import and nod_export are all starting from 1 */
264 
265  HECMW_Status *sta1, *sta2;
266  HECMW_Request *req1, *req2;
267 
268  int nflag = 0;
269  int neib;
270  int istart, inum;
271  int k;
272  int *ws, *wr;
273 
274  ws = (int *)calloc(n, sizeof(int));
275  wr = (int *)calloc(n, sizeof(int));
276  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
277  if (nflag == 0) {
278  sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
279  if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
280  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
281  if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
282  if ((req1 = (HECMW_Request *)calloc(neibpetot, sizeof(HECMW_Request))) ==
283  NULL)
284  HECMW_vis_memory_exit("HECMW_STATUS: req1");
285  if ((req2 = (HECMW_Request *)calloc(neibpetot, sizeof(HECMW_Request))) ==
286  NULL)
287  HECMW_vis_memory_exit("HECMW_STATUS: req2");
288  nflag = 1;
289  }
290  /* SEND */
291  for (neib = 0; neib < neibpetot; neib++) {
292  /* if (neib != 0) istart = stack_export[neib - 1];
293  else istart = 0;
294  */
295  inum = stack_export[neib + 1] - stack_export[neib];
296 
297  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
298  ws[k] = x[nod_export[k] - 1];
299  }
300  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neibpe[neib], 0,
301  repart_comm, &req1[neib]);
302  }
303 
304  /* RECEIVE */
305  for (neib = 0; neib < neibpetot; neib++) {
306  inum = stack_import[neib + 1] - stack_import[neib];
307  HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neibpe[neib], 0,
308  repart_comm, &req2[neib]);
309  }
310  HECMW_Barrier(repart_comm);
311  /*
312  HECMW_Waitall(neibpetot, req2, sta2);
313  */
314  for (neib = 0; neib < neibpetot; neib++) {
315  /* if (neib != 0) istart = stack_import[neib-1];
316  else istart = 0;
317  */
318  inum = stack_import[neib + 1] - stack_import[neib];
319  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
320  x[nod_import[k] - 1] = wr[k];
321  }
322  }
323  HECMW_Barrier(repart_comm);
324  /*
325  HECMW_Waitall(neibpetot, req1, sta1);
326  */
327  free(sta1);
328  free(sta2);
329  free(req1);
330  free(req2);
331  free(ws);
332  free(wr);
333  return 1;
334 }
335 
336 int double_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
337  int *nod_import, int *stack_export, int *nod_export,
338  double *x, HECMW_Comm repart_comm, int my_rank) {
339  HECMW_Status *sta1, *sta2;
340  HECMW_Request *req1, *req2;
341 
342  int nflag = 0;
343  int neib;
344  int istart, inum;
345  int k;
346  double *ws, *wr;
347 
348  ws = (double *)calloc(n, sizeof(double));
349  wr = (double *)calloc(n, sizeof(double));
350  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
351  if (nflag == 0) {
352  sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE * neibpetot,
353  sizeof(HECMW_Status));
354  if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
355  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE * neibpetot,
356  sizeof(HECMW_Status));
357  if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta12");
358  if ((req1 = (HECMW_Request *)calloc(neibpetot, sizeof(HECMW_Request))) ==
359  NULL)
360  HECMW_vis_memory_exit("send_recv: req1");
361  if ((req2 = (HECMW_Request *)calloc(neibpetot, sizeof(HECMW_Request))) ==
362  NULL)
363  HECMW_vis_memory_exit("send_recv: req2");
364  nflag = 1;
365  }
366  /* SEND */
367  for (neib = 0; neib < neibpetot; neib++) {
368  /* if (neib != 0) istart = stack_export[neib - 1];
369  else istart = 0;
370  */
371  inum = stack_export[neib + 1] - stack_export[neib];
372 
373  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
374  ws[k] = x[nod_export[k] - 1];
375  }
376  if (inum > 0) {
377  /* fprintf(stderr, "PE %d send %d node to PE %d\n", my_rank, inum,
378  * neibpe[neib]);
379  */
380  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
381  repart_comm, &req1[neib]);
382  }
383  }
384 
385  /* RECEIVE */
386  for (neib = 0; neib < neibpetot; neib++) {
387  inum = stack_import[neib + 1] - stack_import[neib];
388  if (inum > 0) {
389  /* fprintf(stderr, "PE %d receive %d node from PE %d\n",
390  * my_rank, inum, neibpe[neib]);
391  */
392  HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
393  repart_comm, &req2[neib]);
394  }
395  }
396  HECMW_Barrier(repart_comm);
397  /* HECMW_Waitall(neibpetot, req2, sta2);
398  */
399  for (neib = 0; neib < neibpetot; neib++) {
400  /* if (neib != 0) istart = stack_import[neib-1];
401  else istart = 0;
402  */
403  inum = stack_import[neib + 1] - stack_import[neib];
404  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
405  x[nod_import[k] - 1] = wr[k];
406  }
407  }
408  HECMW_Barrier(repart_comm);
409 
410  /* HECMW_Waitall(neibpetot, req1, sta1);
411  */
412  free(sta1);
413  free(sta2);
414  free(req1);
415  free(req2);
416  free(ws);
417  free(wr);
418  return 1;
419 }
420 
421 void int_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
422  int *nod_import, int *stack_export, int *nod_export,
423  int *x, int *y, HECMW_Comm repart_comm, int my_rank) {
424  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
425 
426  /* HECMW_Status *sta1, *sta2;
427  */
428  HECMW_Request *req1, *req2;
429 
430  int nflag = 0;
431  int neib;
432  int istart, inum;
433  int k;
434  int *ws, *wr;
435 
436  ws = (int *)calloc(n1, sizeof(int));
437  wr = (int *)calloc(n2, sizeof(int));
438  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("ws, wr");
439  if (nflag == 0) {
440  /* sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE,
441  sizeof(HECMW_Status));
442  if (sta1 == NULL) {
443  fprintf(stderr, "Not enough memory\n");
444  exit(1);
445  }
446  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
447  if (sta2 == NULL) {
448  fprintf(stderr, "Not enough memory\n");
449  exit(1);
450  }
451  */
452  if ((req1 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
453  HECMW_vis_memory_exit("send_recv: req1");
454  if ((req2 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
455  HECMW_vis_memory_exit("send_recv: req2");
456  nflag = 1;
457  }
458  /* SEND */
459  for (neib = 0; neib < pesize; neib++) {
460  /* if (neib != 0) istart = stack_export[neib - 1];
461  else istart = 0;
462  */
463  if (neib != my_rank) {
464  inum = stack_export[neib + 1] - stack_export[neib];
465 
466  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
467  ws[k] = x[nod_export[k]];
468  }
469  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neib, 0,
470  repart_comm, &req1[neib]);
471  }
472  }
473 
474  /* RECEIVE */
475  for (neib = 0; neib < pesize; neib++) {
476  /* if (neib != 0) istart = stack_import[neib-1];
477  else istart = 0;
478  */
479  inum = stack_import[neib + 1] - stack_import[neib];
480  if (neib != my_rank)
481  HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neib, 0,
482  repart_comm, &req2[neib]);
483  }
484  HECMW_Barrier(repart_comm);
485  /*
486  HECMW_Waitall(neibpetot, req2, sta2);
487  */
488  for (neib = 0; neib < pesize; neib++) {
489  /* if (neib != 0) istart = stack_import[neib-1];
490  else istart = 0;
491  */
492  inum = stack_import[neib + 1] - stack_import[neib];
493  if (neib != my_rank) {
494  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
495  y[nod_import[k]] = wr[k];
496  } else {
497  for (k = 0; k < inum; k++)
498  /* for (k = stack_import[neib]; k < stack_import[neib]+inum;
499  * k++)
500  */
501  y[nod_import[stack_import[neib] + k]] =
502  x[nod_export[stack_export[neib] + k]];
503  }
504  }
505  HECMW_Barrier(repart_comm);
506  /*
507  HECMW_Waitall(neibpetot, req1, sta1);
508 
509  free(sta1);
510  free(sta2);
511 */
512  free(req1);
513  free(req2);
514  free(ws);
515  free(wr);
516  return;
517 }
518 
519 void int2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
520  int *stack_export, int *x, int *y,
521  HECMW_Comm repart_comm, int my_rank) {
522  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
523 
524  HECMW_Status *sta1, *sta2;
525  HECMW_Request *req1, *req2;
526 
527  int nflag = 0;
528  int neib;
529  int istart, inum;
530  int k;
531  int *ws, *wr;
532 
533  if (nflag == 0) {
534  sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE * pesize,
535  sizeof(HECMW_Status));
536  if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
537  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE * pesize,
538  sizeof(HECMW_Status));
539  if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
540 
541  if ((req1 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
542  HECMW_vis_memory_exit("send_recv: req1");
543  if ((req2 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
544  HECMW_vis_memory_exit("send_recv: req2");
545  nflag = 1;
546  }
547  /* SEND */
548  for (neib = 0; neib < pesize; neib++) {
549  if (neib != my_rank) {
550  inum = stack_export[neib + 1] - stack_export[neib];
551  if (inum != 0) {
552  HECMW_Isend(&x[stack_export[neib]], inum, HECMW_INT, neib, 0,
553  repart_comm, &req1[neib]);
554  /* fprintf(stderr, "sending from PE %d to PE %d nodes
555  * %d\n", my_rank, neib, inum);
556  */
557  }
558  }
559  }
560 
561  /* RECEIVE */
562  for (neib = 0; neib < pesize; neib++) {
563  /* if (neib != 0) istart = stack_import[neib-1];
564  else istart = 0;
565  */
566  inum = stack_import[neib + 1] - stack_import[neib];
567  if ((neib != my_rank) && (inum != 0)) {
568  HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_INT, neib, 0, repart_comm,
569  &req2[neib]);
570  /* fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
571  * neib, inum);
572  */
573  }
574  }
575 
576  HECMW_Waitall(pesize, req2, sta2);
577 
578  inum = stack_import[my_rank + 1] - stack_import[my_rank];
579  for (k = 0; k < inum; k++)
580  y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
581 
582  HECMW_Waitall(pesize, req1, sta1);
583 
584  free(sta1);
585  free(sta2);
586 
587  free(req1);
588  free(req2);
589  return;
590 }
591 
592 void double2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
593  int *stack_export, double *x, double *y,
594  HECMW_Comm repart_comm, int my_rank) {
595  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
596 
597  HECMW_Status *sta1, *sta2;
598  HECMW_Request *req1, *req2;
599 
600  int nflag = 0;
601  int neib;
602  int istart, inum;
603  int k;
604 
605  if (nflag == 0) {
606  sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE * pesize,
607  sizeof(HECMW_Status));
608  if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
609  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE * pesize,
610  sizeof(HECMW_Status));
611  if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
612 
613  if ((req1 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
614  HECMW_vis_memory_exit("send_recv: req1");
615  if ((req2 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
616  HECMW_vis_memory_exit("send_recv: req2");
617  nflag = 1;
618  }
619  /* SEND */
620  for (neib = 0; neib < pesize; neib++) {
621  if (neib != my_rank) {
622  inum = stack_export[neib + 1] - stack_export[neib];
623  if (inum != 0) {
624  /* fprintf(stderr, "sending from PE %d to PE %d nodes
625  * %d\n", my_rank, neib, inum);
626  */
627  HECMW_Isend(&x[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
628  repart_comm, &req1[neib]);
629  }
630  }
631  }
632 
633  /* RECEIVE */
634  for (neib = 0; neib < pesize; neib++) {
635  /* if (neib != 0) istart = stack_import[neib-1];
636  else istart = 0;
637  */
638  inum = stack_import[neib + 1] - stack_import[neib];
639  if ((neib != my_rank) && (inum != 0)) {
640  /* fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
641  * neib, inum);
642  */
643  HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_DOUBLE, neib, 0,
644  repart_comm, &req2[neib]);
645  }
646  }
647 
648  HECMW_Waitall(pesize, req2, sta2);
649 
650  inum = stack_import[my_rank + 1] - stack_import[my_rank];
651  for (k = 0; k < inum; k++)
652  y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
653 
654  HECMW_Waitall(pesize, req1, sta1);
655 
656  free(sta1);
657  free(sta2);
658 
659  free(req1);
660  free(req2);
661  return;
662 }
663 
664 void int3_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
665  int *stack_export, int *x, int *y,
666  HECMW_Comm repart_comm, int my_rank) {
667  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
668 
669  HECMW_Status *sta1, *sta2;
670  HECMW_Request *req1, *req2;
671  HECMW_Status stat;
672 
673  int nflag = 0;
674  int neib;
675  int istart, inum;
676  int i, j, k;
677  int *ws, *wr;
678 
679  for (i = 0; i < pesize; i++) {
680  if (i != my_rank) {
681  inum = stack_export[i + 1] - stack_export[i];
682  if (inum != 0)
683  HECMW_Send(&x[stack_export[neib]], inum, HECMW_INT, i, 0, repart_comm);
684  } else if (i == my_rank) {
685  inum = stack_import[my_rank + 1] - stack_import[my_rank];
686  for (k = 0; k < inum; k++)
687  y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
688  for (j = 0; j < pesize; j++) {
689  if (j != my_rank) {
690  inum = stack_import[j + 1] - stack_import[j];
691  if (inum != 0)
692 
693  HECMW_Recv(&y[stack_import[j]], inum, HECMW_INT, j, HECMW_ANY_TAG,
694  repart_comm, &stat);
695  }
696  }
697  }
698  }
699 
700  HECMW_Barrier(repart_comm);
701 
702  return;
703 }
704 
705 void double_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
706  int *nod_import, int *stack_export, int *nod_export,
707  double *x, double *y, HECMW_Comm repart_comm,
708  int my_rank) {
709  /* HECMW_Status *sta1, *sta2;
710  */
711  HECMW_Request *req1, *req2;
712 
713  int nflag = 0;
714  int neib;
715  int istart, inum;
716  int k;
717  double *ws, *wr;
718 
719  ws = (double *)calloc(n1, sizeof(int));
720  wr = (double *)calloc(n2, sizeof(int));
721  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws,wr");
722  if (nflag == 0) {
723  /* sta1 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE,
724  sizeof(HECMW_Status));
725  if (sta1 == NULL) {
726  fprintf(stderr, "Not enough memory\n");
727  exit(1);
728  }
729  sta2 = (HECMW_Status *)calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
730  if (sta2 == NULL) {
731  fprintf(stderr, "Not enough memory\n");
732  exit(1);
733  }
734  */
735  if ((req1 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
736  HECMW_vis_memory_exit("send_recv: req1");
737  if ((req2 = (HECMW_Request *)calloc(pesize, sizeof(HECMW_Request))) == NULL)
738  HECMW_vis_memory_exit("send_recv: req2");
739  nflag = 1;
740  }
741  /* SEND */
742  for (neib = 0; neib < pesize; neib++) {
743  /* if (neib != 0) istart = stack_export[neib - 1];
744  else istart = 0;
745  */
746  if (neib != my_rank) {
747  inum = stack_export[neib + 1] - stack_export[neib];
748 
749  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
750  ws[k] = x[nod_export[k] - 1];
751  }
752  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
753  repart_comm, &req1[neib]);
754  }
755  }
756 
757  /* RECEIVE */
758  for (neib = 0; neib < pesize; neib++) {
759  /* if (neib != 0) istart = stack_import[neib-1];
760  else istart = 0;
761  */
762 
763  inum = stack_import[neib + 1] - stack_import[neib];
764  if (neib != my_rank) {
765  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
766  y[k] = wr[k];
767  } else {
768  for (k = 0; k < inum; k++)
769  /* for (k = stack_import[neib]; k < stack_import[neib]+inum;
770  * k++)
771  */
772  y[stack_import[neib] + k] = x[stack_export[neib] + k];
773  }
774  }
775 
776  HECMW_Barrier(repart_comm);
777  /*
778  HECMW_Waitall(neibpetot, req1, sta1);
779 
780  free(sta1);
781  free(sta2);
782 */
783  free(req1);
784  free(req2);
785  free(ws);
786  free(wr);
787  return;
788 }
stack_whole_send_recv
int stack_whole_send_recv(int pesize, int *stack_export, int *stack_import, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:173
HECMW_Isend
int HECMW_Isend(void *buffer, int count, HECMW_Datatype datatype, int dest, int tag, HECMW_Comm comm, HECMW_Request *request)
Definition: hecmw_comm.c:278
HECMW_DOUBLE
#define HECMW_DOUBLE
Definition: hecmw_config.h:50
int3_whole_send_recv
void int3_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *stack_export, int *x, int *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:664
HECMW_Send
int HECMW_Send(void *buffer, int count, HECMW_Datatype datatype, int dest, int tag, HECMW_Comm comm)
Definition: hecmw_comm.c:193
HECMW_Waitall
int HECMW_Waitall(int count, HECMW_Request *array_of_requests, HECMW_Status *array_of_statuses)
Definition: hecmw_comm.c:132
whole_copy_array
void whole_copy_array(int *recv_num, int *global_recv_num, int mynode, int pesize, HECMW_Comm repart_comm)
Definition: hecmw_dlb_comm_util.c:100
hecmw_repart.h
HECMW_Irecv
int HECMW_Irecv(void *buffer, int count, HECMW_Datatype datatype, int source, int tag, HECMW_Comm comm, HECMW_Request *request)
Definition: hecmw_comm.c:321
HECMW_vis_memory_exit
void HECMW_vis_memory_exit(char *var)
Definition: hecmw_vis_mem_util.c:12
HECMW_Status
MPI_Status HECMW_Status
Definition: hecmw_config.h:36
double_part_send_recv
int double_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import, int *nod_import, int *stack_export, int *nod_export, double *x, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:336
HECMW_STATUS_SIZE
#define HECMW_STATUS_SIZE
Definition: hecmw_dlb_comm_util.h:6
HECMW_ANY_TAG
int HECMW_ANY_TAG
Definition: hecmw_dlb_comm_util.h:7
int_part_send_recv
int int_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import, int *nod_import, int *stack_export, int *nod_export, int *x, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:260
HECMW_Request
MPI_Request HECMW_Request
Definition: hecmw_config.h:34
int_whole_send_recv
void int_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *nod_import, int *stack_export, int *nod_export, int *x, int *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:421
repart_comm
int repart_comm
Definition: hecmw_repart.h:70
double_whole_send_recv
void double_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *nod_import, int *stack_export, int *nod_export, double *x, double *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:705
HECMW_Comm
MPI_Comm HECMW_Comm
Definition: hecmw_config.h:30
stack_part_send_recv
int stack_part_send_recv(int neibpetot, int *neibpe, int *stack_import, int *stack_export, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:128
HECMW_INT
#define HECMW_INT
Definition: hecmw_config.h:48
HECMW_Recv
int HECMW_Recv(void *buffer, int count, HECMW_Datatype datatype, int source, int tag, HECMW_Comm comm, HECMW_Status *status)
Definition: hecmw_comm.c:235
NULL
#define NULL
Definition: hecmw_io_nastran.c:30
double2_whole_send_recv
void double2_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *stack_export, double *x, double *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:592
int2_whole_send_recv
void int2_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *stack_export, int *x, int *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_dlb_comm_util.c:519
HECMW_Barrier
int HECMW_Barrier(HECMW_Comm comm)
Definition: hecmw_comm.c:95