FrontISTR  5.7.0
Large-scale structural analysis program with finit element method
hecmw_vis_comm_util.c
Go to the documentation of this file.
1 /*****************************************************************************
2  * Copyright (c) 2019 FrontISTR Commons
3  * This software is released under the MIT License, see LICENSE.txt
4  *****************************************************************************/
5 
6 #include "hecmw_vis_comm_util.h"
7 
8 #include "hecmw_vis_mem_util.h"
9 #include "hecmw_malloc.h"
10 
11 int HECMW_ANY_TAG = -1;
12 
13 void whole_copy_array(int *recv_num, int *global_recv_num, int mynode,
14  int pesize, HECMW_Comm repart_comm) {
15  int i, j;
16  int *tmp_recv;
17  HECMW_Status stat;
18 
19  if (mynode == 0) {
20  for (j = 0; j < pesize + 1; j++) global_recv_num[j] = recv_num[j];
21  tmp_recv = (int *)HECMW_calloc(pesize + 1, sizeof(int));
22  if (tmp_recv == NULL) HECMW_vis_memory_exit("tmp_recv");
23  for (i = 1; i < pesize; i++) {
24  HECMW_Recv(tmp_recv, pesize + 1, HECMW_INT, i, HECMW_ANY_TAG, repart_comm,
25  &stat);
26  for (j = 0; j < pesize + 1; j++)
27  global_recv_num[i * (pesize + 1) + j] = tmp_recv[j];
28  }
29  for (i = 1; i < pesize; i++)
30  HECMW_Send(global_recv_num, (pesize + 1) * pesize, HECMW_INT, i, 0,
31  repart_comm);
32  HECMW_free(tmp_recv);
33  } else {
34  HECMW_Send(recv_num, pesize + 1, HECMW_INT, 0, 0, repart_comm);
35  HECMW_Recv(global_recv_num, (pesize + 1) * pesize, HECMW_INT, 0,
36  HECMW_ANY_TAG, repart_comm, &stat);
37  }
38  return;
39 }
40 
41 int stack_part_send_recv(int neibpetot, int *neibpe, int *stack_import,
42  int *stack_export, HECMW_Comm repart_comm,
43  int my_rank) {
44  HECMW_Status *sta1, *sta2;
45  HECMW_Request *req1, *req2;
46 
47  int nflag = 0;
48  int neib;
49  int num;
50 
51  if (nflag == 0) {
52  sta1 =
53  (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
54  if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
55  sta2 =
56  (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
57  if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
58  if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
59  sizeof(HECMW_Request))) == NULL)
60  HECMW_vis_memory_exit("HECMW_STATUS: req1");
61  if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
62  sizeof(HECMW_Request))) == NULL)
63  HECMW_vis_memory_exit("HECMW_STATUS: req2");
64  nflag = 1;
65  }
66 
67  for (neib = 0; neib < neibpetot; neib++) {
68  num = stack_import[neib];
69  HECMW_Isend(&num, 1, HECMW_INT, neibpe[neib], 0, repart_comm, &req1[neib]);
70  }
71  for (neib = 0; neib < neibpetot; neib++) {
72  HECMW_Irecv(&stack_export[neib], 1, HECMW_INT, neibpe[neib], 0, repart_comm,
73  &req2[neib]);
74  /* fprintf(stderr, "PE %d recv %d from %d\n", my_rank, stack_export[neib],
75  * neib);
76  */
77  }
78  HECMW_Barrier(repart_comm);
79  HECMW_free(sta1);
80  HECMW_free(sta2);
81  HECMW_free(req1);
82  HECMW_free(req2);
83 
84  return 1;
85 }
86 
87 int stack_whole_send_recv(int pesize, int *stack_export, int *stack_import,
88  HECMW_Comm repart_comm, int my_rank) {
89  HECMW_Status stat;
90  int tmp_int;
91  int i, j;
92 
93  /*
94  HECMW_Status *sta1, *sta2;
95 HECMW_Request *req1, *req2;
96 
97 int nflag = 0;
98 int neib;
99 int istart, num, num1;
100 int k;
101 
102 stack_export[0]=0;
103 stack_export[my_rank+1]=stack_import[my_rank+1]-stack_import[my_rank];
104 
105 if (nflag == 0) {
106 sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE*(pesize-1),
107 sizeof(HECMW_Status));
108 if (sta1 == NULL)
109  HECMW_vis_memory_exit("HECMW_STATUS: stat1");
110 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE*(pesize-1),
111 sizeof(HECMW_Status));
112 if (sta2 == NULL)
113  HECMW_vis_memory_exit("HECMW_STATUS: stat2");
114 if ((req1 = (HECMW_Request *)HECMW_calloc(pesize-1, sizeof(HECMW_Request)))
115  == NULL)
116  HECMW_vis_memory_exit("HECMW_STATUS: req1");
117 if ((req2 = (HECMW_Request *)HECMW_calloc(pesize-1, sizeof(HECMW_Request)))
118  == NULL)
119  HECMW_vis_memory_exit("HECMW_STATUS: req2");
120 nflag = 1;
121 }
122 
123 for(neib=0;neib<pesize;neib++) {
124  if(neib!=my_rank) {
125  num=stack_import[neib+1]-stack_import[neib];
126 HECMW_Isend(&num, 1, HECMW_INT,
127  neib, 0, repart_comm, &req1[neib]);
128  fprintf(stderr, "pe %d send %d to %d\n", my_rank, num, neib);
129 }
130 }
131 for (neib = 0; neib < pesize; neib++) {
132  if(neib!=my_rank) {
133 HECMW_Irecv(&stack_export[neib+1], 1, HECMW_INT,
134  neib, 0, repart_comm, &req2[neib]);
135 
136  fprintf(stderr, "pe %d recv %d from %d\n", my_rank, stack_export[neib+1],
137 neib);
138 
139 
140  }
141 }
142 
143 
144 HECMW_Barrier(repart_comm);
145 
146 HECMW_free(sta1);
147 HECMW_free(sta2);
148 HECMW_free(req1);
149 HECMW_free(req2);
150  */
151  for (i = 0; i < pesize; i++) {
152  if (i != my_rank) {
153  tmp_int = stack_export[i + 1] - stack_export[i];
154  HECMW_Send(&tmp_int, 1, HECMW_INT, i, 0, repart_comm);
155  } else if (i == my_rank) {
156  stack_import[i + 1] = stack_export[i + 1] - stack_export[i];
157  for (j = 0; j < pesize; j++) {
158  if (j != my_rank) {
159  HECMW_Recv(&stack_import[j + 1], 1, HECMW_INT, j, HECMW_ANY_TAG,
160  repart_comm, &stat);
161  }
162  }
163  stack_import[0] = 0;
164  for (j = 1; j < pesize + 1; j++)
165  stack_import[j] = stack_import[j] + stack_import[j - 1];
166  }
167  }
168 
169  HECMW_Barrier(repart_comm);
170 
171  return 1;
172 }
173 
174 int int_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
175  int *nod_import, int *stack_export, int *nod_export,
176  int *x, HECMW_Comm repart_comm, int my_rank) {
177  /* Important:: node ID in nod_import and nod_export are all starting from 1 */
178 
179  HECMW_Status *sta1, *sta2;
180  HECMW_Request *req1, *req2;
181 
182  int nflag = 0;
183  int neib;
184  int inum;
185  int k;
186  int *ws, *wr;
187 
188  ws = (int *)HECMW_calloc(n, sizeof(int));
189  wr = (int *)HECMW_calloc(n, sizeof(int));
190  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
191  if (nflag == 0) {
192  sta1 =
193  (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
194  if (sta1 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat1");
195  sta2 =
196  (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
197  if (sta2 == NULL) HECMW_vis_memory_exit("HECMW_STATUS: stat2");
198  if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
199  sizeof(HECMW_Request))) == NULL)
200  HECMW_vis_memory_exit("HECMW_STATUS: req1");
201  if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
202  sizeof(HECMW_Request))) == NULL)
203  HECMW_vis_memory_exit("HECMW_STATUS: req2");
204  nflag = 1;
205  }
206  /* SEND */
207  for (neib = 0; neib < neibpetot; neib++) {
208  /* if (neib != 0) istart = stack_export[neib - 1];
209 else istart = 0;
210  */
211  inum = stack_export[neib + 1] - stack_export[neib];
212 
213  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
214  ws[k] = x[nod_export[k] - 1];
215  }
216  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neibpe[neib], 0,
217  repart_comm, &req1[neib]);
218  }
219 
220  /* RECEIVE */
221  for (neib = 0; neib < neibpetot; neib++) {
222  inum = stack_import[neib + 1] - stack_import[neib];
223  HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neibpe[neib], 0,
224  repart_comm, &req2[neib]);
225  }
226  HECMW_Barrier(repart_comm);
227  /*
228 HECMW_Waitall(neibpetot, req2, sta2);
229  */
230  for (neib = 0; neib < neibpetot; neib++) {
231  /* if (neib != 0) istart = stack_import[neib-1];
232 else istart = 0;
233  */
234  inum = stack_import[neib + 1] - stack_import[neib];
235  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
236  x[nod_import[k] - 1] = wr[k];
237  }
238  }
239  HECMW_Barrier(repart_comm);
240  /*
241 HECMW_Waitall(neibpetot, req1, sta1);
242  */
243  HECMW_free(sta1);
244  HECMW_free(sta2);
245  HECMW_free(req1);
246  HECMW_free(req2);
247  HECMW_free(ws);
248  HECMW_free(wr);
249  return 1;
250 }
251 
252 int double_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import,
253  int *nod_import, int *stack_export, int *nod_export,
254  double *x, HECMW_Comm repart_comm, int my_rank) {
255  HECMW_Status *sta1, *sta2;
256  HECMW_Request *req1, *req2;
257 
258  int nflag = 0;
259  int neib;
260  int inum;
261  int k;
262  double *ws, *wr;
263 
264  ws = (double *)HECMW_calloc(n, sizeof(double));
265  wr = (double *)HECMW_calloc(n, sizeof(double));
266  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws, wr");
267  if (nflag == 0) {
268  sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * neibpetot,
269  sizeof(HECMW_Status));
270  if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
271  sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * neibpetot,
272  sizeof(HECMW_Status));
273  if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta12");
274  if ((req1 = (HECMW_Request *)HECMW_calloc(neibpetot,
275  sizeof(HECMW_Request))) == NULL)
276  HECMW_vis_memory_exit("send_recv: req1");
277  if ((req2 = (HECMW_Request *)HECMW_calloc(neibpetot,
278  sizeof(HECMW_Request))) == NULL)
279  HECMW_vis_memory_exit("send_recv: req2");
280  nflag = 1;
281  }
282  /* SEND */
283  for (neib = 0; neib < neibpetot; neib++) {
284  /* if (neib != 0) istart = stack_export[neib - 1];
285 else istart = 0;
286  */
287  inum = stack_export[neib + 1] - stack_export[neib];
288 
289  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
290  ws[k] = x[nod_export[k] - 1];
291  }
292  if (inum > 0) {
293  /* fprintf(stderr, "PE %d send %d node to PE %d\n", my_rank, inum,
294  * neibpe[neib]);
295  */
296  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
297  repart_comm, &req1[neib]);
298  }
299  }
300 
301  /* RECEIVE */
302  for (neib = 0; neib < neibpetot; neib++) {
303  inum = stack_import[neib + 1] - stack_import[neib];
304  if (inum > 0) {
305  /* fprintf(stderr, "PE %d receive %d node from PE %d\n",
306  * my_rank, inum, neibpe[neib]);
307  */
308  HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_DOUBLE, neibpe[neib], 0,
309  repart_comm, &req2[neib]);
310  }
311  }
312  HECMW_Barrier(repart_comm);
313  /* HECMW_Waitall(neibpetot, req2, sta2);
314  */
315  for (neib = 0; neib < neibpetot; neib++) {
316  /* if (neib != 0) istart = stack_import[neib-1];
317 else istart = 0;
318  */
319  inum = stack_import[neib + 1] - stack_import[neib];
320  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++) {
321  x[nod_import[k] - 1] = wr[k];
322  }
323  }
324  HECMW_Barrier(repart_comm);
325 
326  /* HECMW_Waitall(neibpetot, req1, sta1);
327  */
328  HECMW_free(sta1);
329  HECMW_free(sta2);
330  HECMW_free(req1);
331  HECMW_free(req2);
332  HECMW_free(ws);
333  HECMW_free(wr);
334  return 1;
335 }
336 
337 void int_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
338  int *nod_import, int *stack_export, int *nod_export,
339  int *x, int *y, HECMW_Comm repart_comm, int my_rank) {
340  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
341 
342  /* HECMW_Status *sta1, *sta2;
343  */
344  HECMW_Request *req1, *req2;
345 
346  int nflag = 0;
347  int neib;
348  int inum;
349  int k;
350  int *ws, *wr;
351 
352  ws = (int *)HECMW_calloc(n1, sizeof(int));
353  wr = (int *)HECMW_calloc(n2, sizeof(int));
354  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("ws, wr");
355  if (nflag == 0) {
356  /* sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE,
357 sizeof(HECMW_Status));
358 if (sta1 == NULL) {
359 fprintf(stderr, "Not enough memory\n");
360 exit(1);
361 }
362 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
363 if (sta2 == NULL) {
364 fprintf(stderr, "Not enough memory\n");
365 exit(1);
366 }
367  */
368  if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
369  NULL)
370  HECMW_vis_memory_exit("send_recv: req1");
371  if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
372  NULL)
373  HECMW_vis_memory_exit("send_recv: req2");
374  nflag = 1;
375  }
376  /* SEND */
377  for (neib = 0; neib < pesize; neib++) {
378  /* if (neib != 0) istart = stack_export[neib - 1];
379 else istart = 0;
380  */
381  if (neib != my_rank) {
382  inum = stack_export[neib + 1] - stack_export[neib];
383 
384  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
385  ws[k] = x[nod_export[k]];
386  }
387  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_INT, neib, 0,
388  repart_comm, &req1[neib]);
389  }
390  }
391 
392  /* RECEIVE */
393  for (neib = 0; neib < pesize; neib++) {
394  /* if (neib != 0) istart = stack_import[neib-1];
395 else istart = 0;
396  */
397  inum = stack_import[neib + 1] - stack_import[neib];
398  if (neib != my_rank)
399  HECMW_Irecv(&wr[stack_import[neib]], inum, HECMW_INT, neib, 0,
400  repart_comm, &req2[neib]);
401  }
402  HECMW_Barrier(repart_comm);
403  /*
404 HECMW_Waitall(neibpetot, req2, sta2);
405  */
406  for (neib = 0; neib < pesize; neib++) {
407  /* if (neib != 0) istart = stack_import[neib-1];
408 else istart = 0;
409  */
410  inum = stack_import[neib + 1] - stack_import[neib];
411  if (neib != my_rank) {
412  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
413  y[nod_import[k]] = wr[k];
414  } else {
415  for (k = 0; k < inum; k++)
416  /* for (k = stack_import[neib]; k < stack_import[neib]+inum;
417  * k++)
418  */
419  y[nod_import[stack_import[neib] + k]] =
420  x[nod_export[stack_export[neib] + k]];
421  }
422  }
423  HECMW_Barrier(repart_comm);
424  /*
425 HECMW_Waitall(neibpetot, req1, sta1);
426 
427 HECMW_free(sta1);
428 HECMW_free(sta2);
429  */
430  HECMW_free(req1);
431  HECMW_free(req2);
432  HECMW_free(ws);
433  HECMW_free(wr);
434  return;
435 }
436 
437 void int2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
438  int *stack_export, int *x, int *y,
439  HECMW_Comm repart_comm, int my_rank) {
440  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
441 
442  HECMW_Status *sta1, *sta2;
443  HECMW_Request *req1, *req2;
444 
445  int nflag = 0;
446  int neib;
447  int inum;
448  int k;
449 
450  if (nflag == 0) {
451  sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
452  sizeof(HECMW_Status));
453  if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
454  sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
455  sizeof(HECMW_Status));
456  if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
457 
458  if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
459  NULL)
460  HECMW_vis_memory_exit("send_recv: req1");
461  if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
462  NULL)
463  HECMW_vis_memory_exit("send_recv: req2");
464  nflag = 1;
465  }
466  /* SEND */
467  for (neib = 0; neib < pesize; neib++) {
468  if (neib != my_rank) {
469  inum = stack_export[neib + 1] - stack_export[neib];
470  if (inum != 0) {
471  HECMW_Isend(&x[stack_export[neib]], inum, HECMW_INT, neib, 0,
472  repart_comm, &req1[neib]);
473  /* fprintf(stderr, "sending from PE %d to PE %d nodes
474  * %d\n", my_rank, neib, inum);
475  */
476  }
477  }
478  }
479 
480  /* RECEIVE */
481  for (neib = 0; neib < pesize; neib++) {
482  /* if (neib != 0) istart = stack_import[neib-1];
483 else istart = 0;
484  */
485  inum = stack_import[neib + 1] - stack_import[neib];
486  if ((neib != my_rank) && (inum != 0)) {
487  HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_INT, neib, 0, repart_comm,
488  &req2[neib]);
489  /* fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
490  * neib, inum);
491  */
492  }
493  }
494 
495  HECMW_Waitall(pesize, req2, sta2);
496 
497  inum = stack_import[my_rank + 1] - stack_import[my_rank];
498  for (k = 0; k < inum; k++)
499  y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
500 
501  HECMW_Waitall(pesize, req1, sta1);
502 
503  HECMW_free(sta1);
504  HECMW_free(sta2);
505 
506  HECMW_free(req1);
507  HECMW_free(req2);
508  return;
509 }
510 
511 void double2_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
512  int *stack_export, double *x, double *y,
513  HECMW_Comm repart_comm, int my_rank) {
514  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
515 
516  HECMW_Status *sta1, *sta2;
517  HECMW_Request *req1, *req2;
518 
519  int nflag = 0;
520  int neib;
521  int inum;
522  int k;
523 
524  if (nflag == 0) {
525  sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
526  sizeof(HECMW_Status));
527  if (sta1 == NULL) HECMW_vis_memory_exit("send_recv: sta1");
528  sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE * pesize,
529  sizeof(HECMW_Status));
530  if (sta2 == NULL) HECMW_vis_memory_exit("send_recv: sta2");
531 
532  if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
533  NULL)
534  HECMW_vis_memory_exit("send_recv: req1");
535  if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
536  NULL)
537  HECMW_vis_memory_exit("send_recv: req2");
538  nflag = 1;
539  }
540  /* SEND */
541  for (neib = 0; neib < pesize; neib++) {
542  if (neib != my_rank) {
543  inum = stack_export[neib + 1] - stack_export[neib];
544  if (inum != 0) {
545  /* fprintf(stderr, "sending from PE %d to PE %d nodes
546  * %d\n", my_rank, neib, inum);
547  */
548  HECMW_Isend(&x[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
549  repart_comm, &req1[neib]);
550  }
551  }
552  }
553 
554  /* RECEIVE */
555  for (neib = 0; neib < pesize; neib++) {
556  /* if (neib != 0) istart = stack_import[neib-1];
557 else istart = 0;
558  */
559  inum = stack_import[neib + 1] - stack_import[neib];
560  if ((neib != my_rank) && (inum != 0)) {
561  /* fprintf(stderr, "recv: PE %d from PE %d nodes %d\n", my_rank,
562  * neib, inum);
563  */
564  HECMW_Irecv(&y[stack_import[neib]], inum, HECMW_DOUBLE, neib, 0,
565  repart_comm, &req2[neib]);
566  }
567  }
568 
569  HECMW_Waitall(pesize, req2, sta2);
570 
571  inum = stack_import[my_rank + 1] - stack_import[my_rank];
572  for (k = 0; k < inum; k++)
573  y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
574 
575  HECMW_Waitall(pesize, req1, sta1);
576 
577  HECMW_free(sta1);
578  HECMW_free(sta2);
579 
580  HECMW_free(req1);
581  HECMW_free(req2);
582  return;
583 }
584 
585 void int3_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
586  int *stack_export, int *x, int *y,
587  HECMW_Comm repart_comm, int my_rank) {
588  /* Important:: node ID in nod_import and nod_export are all starting from 0 */
589 
590  HECMW_Status stat;
591 
592  int neib;
593  int inum;
594  int i, j, k;
595 
596  for (i = 0; i < pesize; i++) {
597  if (i != my_rank) {
598  inum = stack_export[i + 1] - stack_export[i];
599  if (inum != 0)
600  HECMW_Send(&x[stack_export[neib]], inum, HECMW_INT, i, 0, repart_comm);
601  } else if (i == my_rank) {
602  inum = stack_import[my_rank + 1] - stack_import[my_rank];
603  for (k = 0; k < inum; k++)
604  y[stack_import[my_rank] + k] = x[stack_export[my_rank] + k];
605  for (j = 0; j < pesize; j++) {
606  if (j != my_rank) {
607  inum = stack_import[j + 1] - stack_import[j];
608  if (inum != 0)
609 
610  HECMW_Recv(&y[stack_import[j]], inum, HECMW_INT, j, HECMW_ANY_TAG,
611  repart_comm, &stat);
612  }
613  }
614  }
615  }
616 
617  HECMW_Barrier(repart_comm);
618 
619  return;
620 }
621 
622 void double_whole_send_recv(int n1, int n2, int pesize, int *stack_import,
623  int *nod_import, int *stack_export, int *nod_export,
624  double *x, double *y, HECMW_Comm repart_comm,
625  int my_rank) {
626  /* HECMW_Status *sta1, *sta2;
627  */
628  HECMW_Request *req1, *req2;
629 
630  int nflag = 0;
631  int neib;
632  int inum;
633  int k;
634  double *ws, *wr;
635 
636  ws = (double *)HECMW_calloc(n1, sizeof(int));
637  wr = (double *)HECMW_calloc(n2, sizeof(int));
638  if ((ws == NULL) || (wr == NULL)) HECMW_vis_memory_exit("send_recv: ws,wr");
639  if (nflag == 0) {
640  /* sta1 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE,
641 sizeof(HECMW_Status));
642 if (sta1 == NULL) {
643 fprintf(stderr, "Not enough memory\n");
644 exit(1);
645 }
646 sta2 = (HECMW_Status *)HECMW_calloc(HECMW_STATUS_SIZE, sizeof(HECMW_Status));
647 if (sta2 == NULL) {
648 fprintf(stderr, "Not enough memory\n");
649 exit(1);
650 }
651  */
652  if ((req1 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
653  NULL)
654  HECMW_vis_memory_exit("send_recv: req1");
655  if ((req2 = (HECMW_Request *)HECMW_calloc(pesize, sizeof(HECMW_Request))) ==
656  NULL)
657  HECMW_vis_memory_exit("send_recv: req2");
658  nflag = 1;
659  }
660  /* SEND */
661  for (neib = 0; neib < pesize; neib++) {
662  /* if (neib != 0) istart = stack_export[neib - 1];
663 else istart = 0;
664  */
665  if (neib != my_rank) {
666  inum = stack_export[neib + 1] - stack_export[neib];
667 
668  for (k = stack_export[neib]; k < stack_export[neib] + inum; k++) {
669  ws[k] = x[nod_export[k] - 1];
670  }
671  HECMW_Isend(&ws[stack_export[neib]], inum, HECMW_DOUBLE, neib, 0,
672  repart_comm, &req1[neib]);
673  }
674  }
675 
676  /* RECEIVE */
677  for (neib = 0; neib < pesize; neib++) {
678  /* if (neib != 0) istart = stack_import[neib-1];
679 else istart = 0;
680  */
681 
682  inum = stack_import[neib + 1] - stack_import[neib];
683  if (neib != my_rank) {
684  for (k = stack_import[neib]; k < stack_import[neib] + inum; k++)
685  y[k] = wr[k];
686  } else {
687  for (k = 0; k < inum; k++)
688  /* for (k = stack_import[neib]; k < stack_import[neib]+inum;
689  * k++)
690  */
691  y[stack_import[neib] + k] = x[stack_export[neib] + k];
692  }
693  }
694 
695  HECMW_Barrier(repart_comm);
696  /*
697 HECMW_Waitall(neibpetot, req1, sta1);
698 
699 HECMW_free(sta1);
700 HECMW_free(sta2);
701  */
702  HECMW_free(req1);
703  HECMW_free(req2);
704  HECMW_free(ws);
705  HECMW_free(wr);
706  return;
707 }
hecmw_malloc.h
HECMW_Isend
int HECMW_Isend(void *buffer, int count, HECMW_Datatype datatype, int dest, int tag, HECMW_Comm comm, HECMW_Request *request)
Definition: hecmw_comm.c:278
hecmw_vis_mem_util.h
HECMW_DOUBLE
#define HECMW_DOUBLE
Definition: hecmw_config.h:50
int_part_send_recv
int int_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import, int *nod_import, int *stack_export, int *nod_export, int *x, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:174
HECMW_Send
int HECMW_Send(void *buffer, int count, HECMW_Datatype datatype, int dest, int tag, HECMW_Comm comm)
Definition: hecmw_comm.c:193
HECMW_Waitall
int HECMW_Waitall(int count, HECMW_Request *array_of_requests, HECMW_Status *array_of_statuses)
Definition: hecmw_comm.c:132
int2_whole_send_recv
void int2_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *stack_export, int *x, int *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:437
int3_whole_send_recv
void int3_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *stack_export, int *x, int *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:585
HECMW_calloc
#define HECMW_calloc(nmemb, size)
Definition: hecmw_malloc.h:21
whole_copy_array
void whole_copy_array(int *recv_num, int *global_recv_num, int mynode, int pesize, HECMW_Comm repart_comm)
Definition: hecmw_vis_comm_util.c:13
HECMW_Irecv
int HECMW_Irecv(void *buffer, int count, HECMW_Datatype datatype, int source, int tag, HECMW_Comm comm, HECMW_Request *request)
Definition: hecmw_comm.c:321
HECMW_vis_memory_exit
void HECMW_vis_memory_exit(char *var)
Definition: hecmw_vis_mem_util.c:12
HECMW_Status
MPI_Status HECMW_Status
Definition: hecmw_config.h:36
double2_whole_send_recv
void double2_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *stack_export, double *x, double *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:511
double_whole_send_recv
void double_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *nod_import, int *stack_export, int *nod_export, double *x, double *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:622
double_part_send_recv
int double_part_send_recv(int n, int neibpetot, int *neibpe, int *stack_import, int *nod_import, int *stack_export, int *nod_export, double *x, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:252
stack_part_send_recv
int stack_part_send_recv(int neibpetot, int *neibpe, int *stack_import, int *stack_export, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:41
HECMW_STATUS_SIZE
#define HECMW_STATUS_SIZE
Definition: hecmw_dlb_comm_util.h:6
hecmw_vis_comm_util.h
HECMW_Request
MPI_Request HECMW_Request
Definition: hecmw_config.h:34
repart_comm
int repart_comm
Definition: hecmw_repart.h:70
HECMW_Comm
MPI_Comm HECMW_Comm
Definition: hecmw_config.h:30
HECMW_INT
#define HECMW_INT
Definition: hecmw_config.h:48
HECMW_Recv
int HECMW_Recv(void *buffer, int count, HECMW_Datatype datatype, int source, int tag, HECMW_Comm comm, HECMW_Status *status)
Definition: hecmw_comm.c:235
NULL
#define NULL
Definition: hecmw_io_nastran.c:30
HECMW_free
#define HECMW_free(ptr)
Definition: hecmw_malloc.h:24
HECMW_ANY_TAG
int HECMW_ANY_TAG
Definition: hecmw_vis_comm_util.c:11
HECMW_Barrier
int HECMW_Barrier(HECMW_Comm comm)
Definition: hecmw_comm.c:95
stack_whole_send_recv
int stack_whole_send_recv(int pesize, int *stack_export, int *stack_import, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:87
int_whole_send_recv
void int_whole_send_recv(int n1, int n2, int pesize, int *stack_import, int *nod_import, int *stack_export, int *nod_export, int *x, int *y, HECMW_Comm repart_comm, int my_rank)
Definition: hecmw_vis_comm_util.c:337