mapDistributeBaseTemplates.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration | Website: https://openfoam.org
5  \\ / A nd | Copyright (C) 2015-2018 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "Pstream.H"
27 #include "PstreamBuffers.H"
28 #include "PstreamCombineReduceOps.H"
29 #include "flipOp.H"
30 
31 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
32 
33 template<class T, class CombineOp, class negateOp>
34 void Foam::mapDistributeBase::flipAndCombine
35 (
36  const UList<label>& map,
37  const bool hasFlip,
38  const UList<T>& rhs,
39  const CombineOp& cop,
40  const negateOp& negOp,
41  List<T>& lhs
42 )
43 {
44  if (hasFlip)
45  {
46  forAll(map, i)
47  {
48  if (map[i] > 0)
49  {
50  label index = map[i]-1;
51  cop(lhs[index], rhs[i]);
52  }
53  else if (map[i] < 0)
54  {
55  label index = -map[i]-1;
56  cop(lhs[index], negOp(rhs[i]));
57  }
58  else
59  {
60  FatalErrorInFunction
61  << "At index " << i << " out of " << map.size()
62  << " have illegal index " << map[i]
63  << " for field " << rhs.size() << " with flipMap"
64  << exit(FatalError);
65  }
66  }
67  }
68  else
69  {
70  forAll(map, i)
71  {
72  cop(lhs[map[i]], rhs[i]);
73  }
74  }
75 }
76 
77 
78 template<class T, class negateOp>
79 T Foam::mapDistributeBase::accessAndFlip
80 (
81  const UList<T>& fld,
82  const label index,
83  const bool hasFlip,
84  const negateOp& negOp
85 )
86 {
87  T t;
88  if (hasFlip)
89  {
90  if (index > 0)
91  {
92  t = fld[index-1];
93  }
94  else if (index < 0)
95  {
96  t = negOp(fld[-index-1]);
97  }
98  else
99  {
100  FatalErrorInFunction
101  << "Illegal index " << index
102  << " into field of size " << fld.size()
103  << " with face-flipping"
104  << exit(FatalError);
105  t = fld[index];
106  }
107  }
108  else
109  {
110  t = fld[index];
111  }
112  return t;
113 }
114 
115 
116 // Distribute list.
117 template<class T, class negateOp>
118 void Foam::mapDistributeBase::distribute
119 (
120  const Pstream::commsTypes commsType,
121  const List<labelPair>& schedule,
122  const label constructSize,
123  const labelListList& subMap,
124  const bool subHasFlip,
125  const labelListList& constructMap,
126  const bool constructHasFlip,
127  List<T>& field,
128  const negateOp& negOp,
129  const int tag
130 )
131 {
132  if (!Pstream::parRun())
133  {
134  // Do only me to me.
135 
136  const labelList& mySubMap = subMap[Pstream::myProcNo()];
137 
138  List<T> subField(mySubMap.size());
139  forAll(mySubMap, i)
140  {
141  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
142  }
143 
144  // Receive sub field from myself (subField)
145  const labelList& map = constructMap[Pstream::myProcNo()];
146 
147  field.setSize(constructSize);
148 
149  flipAndCombine
150  (
151  map,
152  constructHasFlip,
153  subField,
154  eqOp<T>(),
155  negOp,
156  field
157  );
158 
159  return;
160  }
161 
162  if (commsType == Pstream::commsTypes::blocking)
163  {
164  // Since buffered sending can reuse the field to collect the
165  // received data.
166 
167  // Send sub field to neighbour
168  for (label domain = 0; domain < Pstream::nProcs(); domain++)
169  {
170  const labelList& map = subMap[domain];
171 
172  if (domain != Pstream::myProcNo() && map.size())
173  {
174  OPstream toNbr(Pstream::commsTypes::blocking, domain, 0, tag);
175 
176  List<T> subField(map.size());
177  forAll(subField, i)
178  {
179  subField[i] = accessAndFlip
180  (
181  field,
182  map[i],
183  subHasFlip,
184  negOp
185  );
186  }
187  toNbr << subField;
188  }
189  }
190 
191  // Subset myself
192  const labelList& mySubMap = subMap[Pstream::myProcNo()];
193 
194  List<T> subField(mySubMap.size());
195  forAll(mySubMap, i)
196  {
197  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
198  }
199 
200  // Receive sub field from myself (subField)
201  const labelList& map = constructMap[Pstream::myProcNo()];
202 
203  field.setSize(constructSize);
204 
205  flipAndCombine
206  (
207  map,
208  constructHasFlip,
209  subField,
210  eqOp<T>(),
211  negOp,
212  field
213  );
214 
215  // Receive sub field from neighbour
216  for (label domain = 0; domain < Pstream::nProcs(); domain++)
217  {
218  const labelList& map = constructMap[domain];
219 
220  if (domain != Pstream::myProcNo() && map.size())
221  {
222  IPstream fromNbr(Pstream::commsTypes::blocking, domain, 0, tag);
223  List<T> subField(fromNbr);
224 
225  checkReceivedSize(domain, map.size(), subField.size());
226 
227  flipAndCombine
228  (
229  map,
230  constructHasFlip,
231  subField,
232  eqOp<T>(),
233  negOp,
234  field
235  );
236  }
237  }
238  }
239  else if (commsType == Pstream::commsTypes::scheduled)
240  {
241  // Need to make sure I don't overwrite field with received data
242  // since the data might need to be sent to another processor. So
243  // allocate a new field for the results.
244  List<T> newField(constructSize);
245 
246  // Receive sub field from myself
247  {
248  const labelList& mySubMap = subMap[Pstream::myProcNo()];
249 
250  List<T> subField(mySubMap.size());
251  forAll(subField, i)
252  {
253  subField[i] = accessAndFlip
254  (
255  field,
256  mySubMap[i],
257  subHasFlip,
258  negOp
259  );
260  }
261 
262  // Receive sub field from myself (subField)
263  flipAndCombine
264  (
265  constructMap[Pstream::myProcNo()],
266  constructHasFlip,
267  subField,
268  eqOp<T>(),
269  negOp,
270  newField
271  );
272  }
273 
274  // Schedule will already have pruned 0-sized comms
275  forAll(schedule, i)
276  {
277  const labelPair& twoProcs = schedule[i];
278  // twoProcs is a swap pair of processors. The first one is the
279  // one that needs to send first and then receive.
280 
281  label sendProc = twoProcs[0];
282  label recvProc = twoProcs[1];
283 
284  if (Pstream::myProcNo() == sendProc)
285  {
286  // I am send first, receive next
287  {
288  OPstream toNbr
289  (
290  Pstream::commsTypes::scheduled,
291  recvProc,
292  0,
293  tag
294  );
295 
296  const labelList& map = subMap[recvProc];
297  List<T> subField(map.size());
298  forAll(subField, i)
299  {
300  subField[i] = accessAndFlip
301  (
302  field,
303  map[i],
304  subHasFlip,
305  negOp
306  );
307  }
308  toNbr << subField;
309  }
310  {
311  IPstream fromNbr
312  (
313  Pstream::commsTypes::scheduled,
314  recvProc,
315  0,
316  tag
317  );
318  List<T> subField(fromNbr);
319 
320  const labelList& map = constructMap[recvProc];
321 
322  checkReceivedSize(recvProc, map.size(), subField.size());
323 
324  flipAndCombine
325  (
326  map,
327  constructHasFlip,
328  subField,
329  eqOp<T>(),
330  negOp,
331  newField
332  );
333  }
334  }
335  else
336  {
337  // I am receive first, send next
338  {
339  IPstream fromNbr
340  (
341  Pstream::commsTypes::scheduled,
342  sendProc,
343  0,
344  tag
345  );
346  List<T> subField(fromNbr);
347 
348  const labelList& map = constructMap[sendProc];
349 
350  checkReceivedSize(sendProc, map.size(), subField.size());
351 
352  flipAndCombine
353  (
354  map,
355  constructHasFlip,
356  subField,
357  eqOp<T>(),
358  negOp,
359  newField
360  );
361  }
362  {
363  OPstream toNbr
364  (
365  Pstream::commsTypes::scheduled,
366  sendProc,
367  0,
368  tag
369  );
370 
371  const labelList& map = subMap[sendProc];
372  List<T> subField(map.size());
373  forAll(subField, i)
374  {
375  subField[i] = accessAndFlip
376  (
377  field,
378  map[i],
379  subHasFlip,
380  negOp
381  );
382  }
383  toNbr << subField;
384  }
385  }
386  }
387  field.transfer(newField);
388  }
389  else if (commsType == Pstream::commsTypes::nonBlocking)
390  {
391  label nOutstanding = Pstream::nRequests();
392 
393  if (!contiguous<T>())
394  {
395  PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking, tag);
396 
397  // Stream data into buffer
398  for (label domain = 0; domain < Pstream::nProcs(); domain++)
399  {
400  const labelList& map = subMap[domain];
401 
402  if (domain != Pstream::myProcNo() && map.size())
403  {
404  // Put data into send buffer
405  UOPstream toDomain(domain, pBufs);
406 
407  List<T> subField(map.size());
408  forAll(subField, i)
409  {
410  subField[i] = accessAndFlip
411  (
412  field,
413  map[i],
414  subHasFlip,
415  negOp
416  );
417  }
418  toDomain << subField;
419  }
420  }
421 
422  // Start receiving. Do not block.
423  pBufs.finishedSends(false);
424 
425  {
426  // Set up 'send' to myself
427  const labelList& mySub = subMap[Pstream::myProcNo()];
428  List<T> mySubField(mySub.size());
429  forAll(mySub, i)
430  {
431  mySubField[i] = accessAndFlip
432  (
433  field,
434  mySub[i],
435  subHasFlip,
436  negOp
437  );
438  }
439  // Combine bits. Note that can reuse field storage
440  field.setSize(constructSize);
441  // Receive sub field from myself
442  {
443  const labelList& map = constructMap[Pstream::myProcNo()];
444 
445  flipAndCombine
446  (
447  map,
448  constructHasFlip,
449  mySubField,
450  eqOp<T>(),
451  negOp,
452  field
453  );
454  }
455  }
456 
457  // Block ourselves, waiting only for the current comms
458  Pstream::waitRequests(nOutstanding);
459 
460  // Consume
461  for (label domain = 0; domain < Pstream::nProcs(); domain++)
462  {
463  const labelList& map = constructMap[domain];
464 
465  if (domain != Pstream::myProcNo() && map.size())
466  {
467  UIPstream str(domain, pBufs);
468  List<T> recvField(str);
469 
470  checkReceivedSize(domain, map.size(), recvField.size());
471 
472  flipAndCombine
473  (
474  map,
475  constructHasFlip,
476  recvField,
477  eqOp<T>(),
478  negOp,
479  field
480  );
481  }
482  }
483  }
484  else
485  {
486  // Set up sends to neighbours
487 
488  List<List<T>> sendFields(Pstream::nProcs());
489 
490  for (label domain = 0; domain < Pstream::nProcs(); domain++)
491  {
492  const labelList& map = subMap[domain];
493 
494  if (domain != Pstream::myProcNo() && map.size())
495  {
496  List<T>& subField = sendFields[domain];
497  subField.setSize(map.size());
498  forAll(map, i)
499  {
500  subField[i] = accessAndFlip
501  (
502  field,
503  map[i],
504  subHasFlip,
505  negOp
506  );
507  }
508 
509  OPstream::write
510  (
511  Pstream::commsTypes::nonBlocking,
512  domain,
513  reinterpret_cast<const char*>(subField.begin()),
514  subField.byteSize(),
515  tag
516  );
517  }
518  }
519 
520  // Set up receives from neighbours
521 
522  List<List<T>> recvFields(Pstream::nProcs());
523 
524  for (label domain = 0; domain < Pstream::nProcs(); domain++)
525  {
526  const labelList& map = constructMap[domain];
527 
528  if (domain != Pstream::myProcNo() && map.size())
529  {
530  recvFields[domain].setSize(map.size());
531  IPstream::read
532  (
533  Pstream::commsTypes::nonBlocking,
534  domain,
535  reinterpret_cast<char*>(recvFields[domain].begin()),
536  recvFields[domain].byteSize(),
537  tag
538  );
539  }
540  }
541 
542 
543  // Set up 'send' to myself
544 
545  {
546  const labelList& map = subMap[Pstream::myProcNo()];
547 
548  List<T>& subField = sendFields[Pstream::myProcNo()];
549  subField.setSize(map.size());
550  forAll(map, i)
551  {
552  subField[i] = accessAndFlip
553  (
554  field,
555  map[i],
556  subHasFlip,
557  negOp
558  );
559  }
560  }
561 
562 
563  // Combine bits. Note that can reuse field storage
564 
565  field.setSize(constructSize);
566 
567 
568  // Receive sub field from myself (sendFields[Pstream::myProcNo()])
569  {
570  const labelList& map = constructMap[Pstream::myProcNo()];
571  const List<T>& subField = sendFields[Pstream::myProcNo()];
572 
573  flipAndCombine
574  (
575  map,
576  constructHasFlip,
577  subField,
578  eqOp<T>(),
579  negOp,
580  field
581  );
582  }
583 
584 
585  // Wait for all to finish
586 
587  Pstream::waitRequests(nOutstanding);
588 
589 
590  // Collect neighbour fields
591 
592  for (label domain = 0; domain < Pstream::nProcs(); domain++)
593  {
594  const labelList& map = constructMap[domain];
595 
596  if (domain != Pstream::myProcNo() && map.size())
597  {
598  const List<T>& subField = recvFields[domain];
599 
600  checkReceivedSize(domain, map.size(), subField.size());
601 
602  flipAndCombine
603  (
604  map,
605  constructHasFlip,
606  subField,
607  eqOp<T>(),
608  negOp,
609  field
610  );
611  }
612  }
613  }
614  }
615  else
616  {
617  FatalErrorInFunction
618  << "Unknown communication schedule " << int(commsType)
619  << abort(FatalError);
620  }
621 }
622 
623 
624 // Distribute list.
625 template<class T, class CombineOp, class negateOp>
626 void Foam::mapDistributeBase::distribute
627 (
628  const Pstream::commsTypes commsType,
629  const List<labelPair>& schedule,
630  const label constructSize,
631  const labelListList& subMap,
632  const bool subHasFlip,
633  const labelListList& constructMap,
634  const bool constructHasFlip,
635  List<T>& field,
636  const CombineOp& cop,
637  const negateOp& negOp,
638  const T& nullValue,
639  const int tag
640 )
641 {
642  if (!Pstream::parRun())
643  {
644  // Do only me to me.
645 
646  const labelList& mySubMap = subMap[Pstream::myProcNo()];
647 
648  List<T> subField(mySubMap.size());
649  forAll(mySubMap, i)
650  {
651  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
652  }
653 
654  // Receive sub field from myself (subField)
655  const labelList& map = constructMap[Pstream::myProcNo()];
656 
657  field.setSize(constructSize);
658  field = nullValue;
659 
660  flipAndCombine(map, constructHasFlip, subField, cop, negOp, field);
661 
662  return;
663  }
664 
665  if (commsType == Pstream::commsTypes::blocking)
666  {
667  // Since buffered sending can reuse the field to collect the
668  // received data.
669 
670  // Send sub field to neighbour
671  for (label domain = 0; domain < Pstream::nProcs(); domain++)
672  {
673  const labelList& map = subMap[domain];
674 
675  if (domain != Pstream::myProcNo() && map.size())
676  {
677  OPstream toNbr(Pstream::commsTypes::blocking, domain, 0, tag);
678  List<T> subField(map.size());
679  forAll(subField, i)
680  {
681  subField[i] = accessAndFlip
682  (
683  field,
684  map[i],
685  subHasFlip,
686  negOp
687  );
688  }
689  toNbr << subField;
690  }
691  }
692 
693  // Subset myself
694  const labelList& mySubMap = subMap[Pstream::myProcNo()];
695 
696  List<T> subField(mySubMap.size());
697  forAll(mySubMap, i)
698  {
699  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
700  }
701 
702  // Receive sub field from myself (subField)
703  const labelList& map = constructMap[Pstream::myProcNo()];
704 
705  field.setSize(constructSize);
706  field = nullValue;
707 
708  flipAndCombine(map, constructHasFlip, subField, cop, negOp, field);
709 
710  // Receive sub field from neighbour
711  for (label domain = 0; domain < Pstream::nProcs(); domain++)
712  {
713  const labelList& map = constructMap[domain];
714 
715  if (domain != Pstream::myProcNo() && map.size())
716  {
717  IPstream fromNbr(Pstream::commsTypes::blocking, domain, 0, tag);
718  List<T> subField(fromNbr);
719 
720  checkReceivedSize(domain, map.size(), subField.size());
721 
722  flipAndCombine
723  (
724  map,
725  constructHasFlip,
726  subField,
727  cop,
728  negOp,
729  field
730  );
731  }
732  }
733  }
734  else if (commsType == Pstream::commsTypes::scheduled)
735  {
736  // Need to make sure I don't overwrite field with received data
737  // since the data might need to be sent to another processor. So
738  // allocate a new field for the results.
739  List<T> newField(constructSize, nullValue);
740 
741  {
742  const labelList& mySubMap = subMap[Pstream::myProcNo()];
743 
744  // Subset myself
745  List<T> subField(mySubMap.size());
746  forAll(subField, i)
747  {
748  subField[i] = accessAndFlip
749  (
750  field,
751  mySubMap[i],
752  subHasFlip,
753  negOp
754  );
755  }
756 
757  // Receive sub field from myself (subField)
758  const labelList& map = constructMap[Pstream::myProcNo()];
759 
760  flipAndCombine
761  (
762  map,
763  constructHasFlip,
764  subField,
765  cop,
766  negOp,
767  newField
768  );
769  }
770 
771 
772  // Schedule will already have pruned 0-sized comms
773  forAll(schedule, i)
774  {
775  const labelPair& twoProcs = schedule[i];
776  // twoProcs is a swap pair of processors. The first one is the
777  // one that needs to send first and then receive.
778 
779  label sendProc = twoProcs[0];
780  label recvProc = twoProcs[1];
781 
782  if (Pstream::myProcNo() == sendProc)
783  {
784  // I am send first, receive next
785  {
786  OPstream toNbr
787  (
788  Pstream::commsTypes::scheduled,
789  recvProc,
790  0,
791  tag
792  );
793 
794  const labelList& map = subMap[recvProc];
795 
796  List<T> subField(map.size());
797  forAll(subField, i)
798  {
799  subField[i] = accessAndFlip
800  (
801  field,
802  map[i],
803  subHasFlip,
804  negOp
805  );
806  }
807  toNbr << subField;
808  }
809  {
810  IPstream fromNbr
811  (
812  Pstream::commsTypes::scheduled,
813  recvProc,
814  0,
815  tag
816  );
817  List<T> subField(fromNbr);
818  const labelList& map = constructMap[recvProc];
819 
820  checkReceivedSize(recvProc, map.size(), subField.size());
821 
822  flipAndCombine
823  (
824  map,
825  constructHasFlip,
826  subField,
827  cop,
828  negOp,
829  newField
830  );
831  }
832  }
833  else
834  {
835  // I am receive first, send next
836  {
837  IPstream fromNbr
838  (
839  Pstream::commsTypes::scheduled,
840  sendProc,
841  0,
842  tag
843  );
844  List<T> subField(fromNbr);
845  const labelList& map = constructMap[sendProc];
846 
847  checkReceivedSize(sendProc, map.size(), subField.size());
848 
849  flipAndCombine
850  (
851  map,
852  constructHasFlip,
853  subField,
854  cop,
855  negOp,
856  newField
857  );
858  }
859  {
860  OPstream toNbr
861  (
862  Pstream::commsTypes::scheduled,
863  sendProc,
864  0,
865  tag
866  );
867 
868  const labelList& map = subMap[sendProc];
869 
870  List<T> subField(map.size());
871  forAll(subField, i)
872  {
873  subField[i] = accessAndFlip
874  (
875  field,
876  map[i],
877  subHasFlip,
878  negOp
879  );
880  }
881  toNbr << subField;
882  }
883  }
884  }
885  field.transfer(newField);
886  }
887  else if (commsType == Pstream::commsTypes::nonBlocking)
888  {
889  label nOutstanding = Pstream::nRequests();
890 
891  if (!contiguous<T>())
892  {
893  PstreamBuffers pBufs(Pstream::commsTypes::nonBlocking, tag);
894 
895  // Stream data into buffer
896  for (label domain = 0; domain < Pstream::nProcs(); domain++)
897  {
898  const labelList& map = subMap[domain];
899 
900  if (domain != Pstream::myProcNo() && map.size())
901  {
902  // Put data into send buffer
903  UOPstream toDomain(domain, pBufs);
904 
905  List<T> subField(map.size());
906  forAll(subField, i)
907  {
908  subField[i] = accessAndFlip
909  (
910  field,
911  map[i],
912  subHasFlip,
913  negOp
914  );
915  }
916  toDomain << subField;
917  }
918  }
919 
920  // Start receiving. Do not block.
921  pBufs.finishedSends(false);
922 
923  {
924  // Set up 'send' to myself
925  const labelList& myMap = subMap[Pstream::myProcNo()];
926 
927  List<T> mySubField(myMap.size());
928  forAll(myMap, i)
929  {
930  mySubField[i] = accessAndFlip
931  (
932  field,
933  myMap[i],
934  subHasFlip,
935  negOp
936  );
937  }
938 
939  // Combine bits. Note that can reuse field storage
940  field.setSize(constructSize);
941  field = nullValue;
942  // Receive sub field from myself
943  {
944  const labelList& map = constructMap[Pstream::myProcNo()];
945 
946  flipAndCombine
947  (
948  map,
949  constructHasFlip,
950  mySubField,
951  cop,
952  negOp,
953  field
954  );
955  }
956  }
957 
958  // Block ourselves, waiting only for the current comms
959  Pstream::waitRequests(nOutstanding);
960 
961  // Consume
962  for (label domain = 0; domain < Pstream::nProcs(); domain++)
963  {
964  const labelList& map = constructMap[domain];
965 
966  if (domain != Pstream::myProcNo() && map.size())
967  {
968  UIPstream str(domain, pBufs);
969  List<T> recvField(str);
970 
971  checkReceivedSize(domain, map.size(), recvField.size());
972 
973  flipAndCombine
974  (
975  map,
976  constructHasFlip,
977  recvField,
978  cop,
979  negOp,
980  field
981  );
982  }
983  }
984  }
985  else
986  {
987  // Set up sends to neighbours
988 
989  List<List<T>> sendFields(Pstream::nProcs());
990 
991  for (label domain = 0; domain < Pstream::nProcs(); domain++)
992  {
993  const labelList& map = subMap[domain];
994 
995  if (domain != Pstream::myProcNo() && map.size())
996  {
997  List<T>& subField = sendFields[domain];
998  subField.setSize(map.size());
999  forAll(map, i)
1000  {
1001  subField[i] = accessAndFlip
1002  (
1003  field,
1004  map[i],
1005  subHasFlip,
1006  negOp
1007  );
1008  }
1009 
1010  OPstream::write
1011  (
1012  Pstream::commsTypes::nonBlocking,
1013  domain,
1014  reinterpret_cast<const char*>(subField.begin()),
1015  subField.size()*sizeof(T),
1016  tag
1017  );
1018  }
1019  }
1020 
1021  // Set up receives from neighbours
1022 
1023  List<List<T>> recvFields(Pstream::nProcs());
1024 
1025  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1026  {
1027  const labelList& map = constructMap[domain];
1028 
1029  if (domain != Pstream::myProcNo() && map.size())
1030  {
1031  recvFields[domain].setSize(map.size());
1032  UIPstream::read
1033  (
1034  Pstream::commsTypes::nonBlocking,
1035  domain,
1036  reinterpret_cast<char*>(recvFields[domain].begin()),
1037  recvFields[domain].size()*sizeof(T),
1038  tag
1039  );
1040  }
1041  }
1042 
1043  // Set up 'send' to myself
1044 
1045  {
1046  const labelList& map = subMap[Pstream::myProcNo()];
1047 
1048  List<T>& subField = sendFields[Pstream::myProcNo()];
1049  subField.setSize(map.size());
1050  forAll(map, i)
1051  {
1052  subField[i] = accessAndFlip
1053  (
1054  field,
1055  map[i],
1056  subHasFlip,
1057  negOp
1058  );
1059  }
1060  }
1061 
1062 
1063  // Combine bits. Note that can reuse field storage
1064 
1065  field.setSize(constructSize);
1066  field = nullValue;
1067 
1068  // Receive sub field from myself (subField)
1069  {
1070  const labelList& map = constructMap[Pstream::myProcNo()];
1071  const List<T>& subField = sendFields[Pstream::myProcNo()];
1072 
1073  flipAndCombine
1074  (
1075  map,
1076  constructHasFlip,
1077  subField,
1078  cop,
1079  negOp,
1080  field
1081  );
1082  }
1083 
1084 
1085  // Wait for all to finish
1086 
1087  Pstream::waitRequests(nOutstanding);
1088 
1089 
1090  // Collect neighbour fields
1091 
1092  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1093  {
1094  const labelList& map = constructMap[domain];
1095 
1096  if (domain != Pstream::myProcNo() && map.size())
1097  {
1098  const List<T>& subField = recvFields[domain];
1099 
1100  checkReceivedSize(domain, map.size(), subField.size());
1101 
1102  flipAndCombine
1103  (
1104  map,
1105  constructHasFlip,
1106  subField,
1107  cop,
1108  negOp,
1109  field
1110  );
1111  }
1112  }
1113  }
1114  }
1115  else
1116  {
1117  FatalErrorInFunction
1118  << "Unknown communication schedule " << int(commsType)
1119  << abort(FatalError);
1120  }
1121 }
1122 
1123 
1124 template<class T>
1125 void Foam::mapDistributeBase::send(PstreamBuffers& pBufs, const List<T>& field)
1126 const
1127 {
1128  // Stream data into buffer
1129  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1130  {
1131  const labelList& map = subMap_[domain];
1132 
1133  if (map.size())
1134  {
1135  // Put data into send buffer
1136  UOPstream toDomain(domain, pBufs);
1137 
1138  List<T> subField(map.size());
1139  forAll(subField, i)
1140  {
1141  subField[i] = accessAndFlip
1142  (
1143  field,
1144  map[i],
1145  subHasFlip_,
1146  flipOp()
1147  );
1148  }
1149  toDomain << subField;
1150  }
1151  }
1152 
1153  // Start sending and receiving but do not block.
1154  pBufs.finishedSends(false);
1155 }
1156 
1157 
1158 template<class T>
1159 void Foam::mapDistributeBase::receive(PstreamBuffers& pBufs, List<T>& field)
1160 const
1161 {
1162  // Consume
1163  field.setSize(constructSize_);
1164 
1165  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1166  {
1167  const labelList& map = constructMap_[domain];
1168 
1169  if (map.size())
1170  {
1171  UIPstream str(domain, pBufs);
1172  List<T> recvField(str);
1173 
1174  if (recvField.size() != map.size())
1175  {
1176  FatalErrorInFunction
1177  << "Expected from processor " << domain
1178  << " " << map.size() << " but received "
1179  << recvField.size() << " elements."
1180  << abort(FatalError);
1181  }
1182 
1183  flipAndCombine
1184  (
1185  map,
1186  constructHasFlip_,
1187  recvField,
1188  eqOp<T>(),
1189  flipOp(),
1190  field
1191  );
1192  }
1193  }
1194 }
1195 
1196 
1197 //- Distribute data using default commsType.
1198 template<class T, class negateOp>
1199 void Foam::mapDistributeBase::distribute
1200 (
1201  List<T>& fld,
1202  const negateOp& negOp,
1203  const int tag
1204 ) const
1205 {
1206  if (Pstream::defaultCommsType == Pstream::commsTypes::nonBlocking)
1207  {
1208  distribute
1209  (
1210  Pstream::commsTypes::nonBlocking,
1211  List<labelPair>(),
1212  constructSize_,
1213  subMap_,
1214  subHasFlip_,
1215  constructMap_,
1216  constructHasFlip_,
1217  fld,
1218  negOp,
1219  tag
1220  );
1221  }
1222  else if (Pstream::defaultCommsType == Pstream::commsTypes::scheduled)
1223  {
1224  distribute
1225  (
1226  Pstream::commsTypes::scheduled,
1227  schedule(),
1228  constructSize_,
1229  subMap_,
1230  subHasFlip_,
1231  constructMap_,
1232  constructHasFlip_,
1233  fld,
1234  negOp,
1235  tag
1236  );
1237  }
1238  else
1239  {
1240  distribute
1241  (
1242  Pstream::commsTypes::blocking,
1243  List<labelPair>(),
1244  constructSize_,
1245  subMap_,
1246  subHasFlip_,
1247  constructMap_,
1248  constructHasFlip_,
1249  fld,
1250  negOp,
1251  tag
1252  );
1253  }
1254 }
1255 
1256 
1257 //- Distribute data using default commsType.
1258 template<class T>
1259 void Foam::mapDistributeBase::distribute
1260 (
1261  List<T>& fld,
1262  const int tag
1263 ) const
1264 {
1265  distribute(fld, flipOp(), tag);
1266 }
1267 
1268 
1269 //- Distribute data using default commsType.
1270 template<class T>
1271 void Foam::mapDistributeBase::distribute
1272 (
1273  DynamicList<T>& fld,
1274  const int tag
1275 ) const
1276 {
1277  fld.shrink();
1278 
1279  List<T>& fldList = static_cast<List<T>& >(fld);
1280 
1281  distribute(fldList, tag);
1282 
1283  fld.setCapacity(fldList.size());
1284 }
1285 
1286 
1287 //- Reverse distribute data using default commsType.
1288 template<class T>
1289 void Foam::mapDistributeBase::reverseDistribute
1290 (
1291  const label constructSize,
1292  List<T>& fld,
1293  const int tag
1294 ) const
1295 {
1296  if (Pstream::defaultCommsType == Pstream::commsTypes::nonBlocking)
1297  {
1298  distribute
1299  (
1300  Pstream::commsTypes::nonBlocking,
1301  List<labelPair>(),
1302  constructSize,
1303  constructMap_,
1304  constructHasFlip_,
1305  subMap_,
1306  subHasFlip_,
1307  fld,
1308  flipOp(),
1309  tag
1310  );
1311  }
1312  else if (Pstream::defaultCommsType == Pstream::commsTypes::scheduled)
1313  {
1314  distribute
1315  (
1316  Pstream::commsTypes::scheduled,
1317  schedule(),
1318  constructSize,
1319  constructMap_,
1320  constructHasFlip_,
1321  subMap_,
1322  subHasFlip_,
1323  fld,
1324  flipOp(),
1325  tag
1326  );
1327  }
1328  else
1329  {
1330  distribute
1331  (
1332  Pstream::commsTypes::blocking,
1333  List<labelPair>(),
1334  constructSize,
1335  constructMap_,
1336  constructHasFlip_,
1337  subMap_,
1338  subHasFlip_,
1339  fld,
1340  flipOp(),
1341  tag
1342  );
1343  }
1344 }
1345 
1346 
1347 //- Reverse distribute data using default commsType.
1348 // Since constructSize might be larger than supplied size supply
1349 // a nullValue
1350 template<class T>
1351 void Foam::mapDistributeBase::reverseDistribute
1352 (
1353  const label constructSize,
1354  const T& nullValue,
1355  List<T>& fld,
1356  const int tag
1357 ) const
1358 {
1359  if (Pstream::defaultCommsType == Pstream::commsTypes::nonBlocking)
1360  {
1361  distribute
1362  (
1363  Pstream::commsTypes::nonBlocking,
1364  List<labelPair>(),
1365  constructSize,
1366  constructMap_,
1367  constructHasFlip_,
1368  subMap_,
1369  subHasFlip_,
1370  fld,
1371  eqOp<T>(),
1372  flipOp(),
1373  nullValue,
1374  tag
1375  );
1376  }
1377  else if (Pstream::defaultCommsType == Pstream::commsTypes::scheduled)
1378  {
1379  distribute
1380  (
1381  Pstream::commsTypes::scheduled,
1382  schedule(),
1383  constructSize,
1384  constructMap_,
1385  constructHasFlip_,
1386  subMap_,
1387  subHasFlip_,
1388  fld,
1389  eqOp<T>(),
1390  flipOp(),
1391  nullValue,
1392  tag
1393  );
1394  }
1395  else
1396  {
1397  distribute
1398  (
1399  Pstream::commsTypes::blocking,
1400  List<labelPair>(),
1401  constructSize,
1402  constructMap_,
1403  constructHasFlip_,
1404  subMap_,
1405  subHasFlip_,
1406  fld,
1407  eqOp<T>(),
1408  flipOp(),
1409  nullValue,
1410  tag
1411  );
1412  }
1413 }
1414 
1415 
1416 // ************************************************************************* //
Foam::flipOp
Class containing functor to negate primitives. Dummy for all other types.
Definition: flipOp.H:50
forAll
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:434
Foam::label
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
Foam::PstreamBuffers::finishedSends
void finishedSends(const bool block=true)
Mark all sends as having been done. This will start receives.
Definition: PstreamBuffers.C:82
Foam::exit
errorManipArg< error, int > exit(error &err, const int errNo=1)
Definition: errorManip.H:124
Foam::FatalError
error FatalError
FatalErrorInFunction
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
Definition: error.H:323
Foam::UPstream::commsTypes
commsTypes
Types of communications.
Definition: UPstream.H:64
Foam::List
A 1D array of objects of type <T>, where the size of the vector is known and used for subscript bound...
Definition: HashTable.H:59
Foam::mapDistributeBase::schedule
const List< labelPair > & schedule() const
Return a schedule. Demand driven. See above.
Definition: mapDistributeBase.C:174
Foam::List::size
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:164
Foam::mapDistributeBase::subMap_
labelListList subMap_
Maps from subsetted data back to original data.
Definition: mapDistributeBase.H:110
Foam::mapDistributeBase::send
void send(PstreamBuffers &, const List< T > &) const
Do all sends using PstreamBuffers.
Definition: mapDistributeBaseTemplates.C:1125
Foam::blockMeshTools::read
void read(Istream &, label &, const dictionary &)
In-place read with dictionary lookup.
Definition: blockMeshTools.C:31
PstreamCombineReduceOps.H
Combination-Reduction operation for a parallel run. The information from all nodes is collected on th...
Foam::mapDistributeBase::constructSize_
label constructSize_
Size of reconstructed data.
Definition: mapDistributeBase.H:107
Foam::IPstream
Input inter-processor communications stream.
Definition: IPstream.H:50
Foam::mapDistributeBase::reverseDistribute
void reverseDistribute(const label constructSize, List< T > &, const int tag=UPstream::msgType()) const
Reverse distribute data using default commsType.
Definition: mapDistributeBaseTemplates.C:1290
Foam::UIPstream
Input inter-processor communications stream operating on external buffer.
Definition: UIPstream.H:53
Foam::Pair
An ordered pair of two objects of type <T> with first() and second() elements.
Definition: contiguous.H:49
fld
gmvFile<< "tracers "<< particles.size()<< nl;forAllConstIter(Cloud< passiveParticle >, particles, iter){ gmvFile<< iter().position().x()<< ' ';}gmvFile<< nl;forAllConstIter(Cloud< passiveParticle >, particles, iter){ gmvFile<< iter().position().y()<< ' ';}gmvFile<< nl;forAllConstIter(Cloud< passiveParticle >, particles, iter){ gmvFile<< iter().position().z()<< ' ';}gmvFile<< nl;forAll(lagrangianScalarNames, i){ const word &name=lagrangianScalarNames[i];IOField< scalar > fld(IOobject(name, runTime.timeName(), cloud::prefix, mesh, IOobject::MUST_READ, IOobject::NO_WRITE))
Foam::eqOp
Definition: ops.H:70
Foam::DynamicList
A 1D vector of objects of type <T> that resizes itself as necessary to accept the new objects...
Definition: DynamicList.H:56
Foam::mapDistributeBase::accessAndFlip
static T accessAndFlip(const UList< T > &fld, const label index, const bool hasFlip, const negateOp &negOp)
Definition: mapDistributeBaseTemplates.C:80
Foam::DynamicList::setCapacity
void setCapacity(const label)
Alter the size of the underlying storage.
Definition: DynamicListI.H:130
Foam::UList::begin
iterator begin()
Return an iterator to begin traversing the UList.
Definition: UListI.H:216
Foam::mapDistributeBase::distribute
static void distribute(const Pstream::commsTypes commsType, const List< labelPair > &schedule, const label constructSize, const labelListList &subMap, const bool subHasFlip, const labelListList &constructMap, const bool constructHasFlip, List< T > &, const negateOp &negOp, const int tag=UPstream::msgType())
Distribute data. Note:schedule only used for.
Definition: mapDistributeBaseTemplates.C:119
Foam::vtkWriteOps::write
void write(std::ostream &os, const bool binary, List< floatScalar > &fField)
Write floats ascii or binary.
Foam::abort
errorManip< error > abort(error &err)
Definition: errorManip.H:131
Foam::mapDistributeBase::constructMap_
labelListList constructMap_
Maps from subsetted data to new reconstructed data.
Definition: mapDistributeBase.H:113
Foam::UList
A 1D vector of objects of type <T>, where the size of the vector is known and can be used for subscri...
Definition: HashTable.H:60
Foam::UOPstream
Output inter-processor communications stream operating on external buffer.
Definition: UOPstream.H:54
Foam::DynamicList::shrink
DynamicList< T, SizeInc, SizeMult, SizeDiv > & shrink()
Shrink the allocated space to the number of elements used.
Definition: DynamicListI.H:252
Foam::OPstream
Output inter-processor communications stream.
Definition: OPstream.H:50
Foam::PstreamBuffers
Buffers for inter-processor communications streams (UOPstream, UIPstream).
Definition: PstreamBuffers.H:85
Foam::T
void T(FieldField< Field, Type > &f1, const FieldField< Field, Type > &f2)
Definition: FieldFieldFunctions.C:55
T
const volScalarField & T
Definition: createFieldRefs.H:2
Foam::mapDistributeBase::subHasFlip_
bool subHasFlip_
Whether subMap includes flip or not.
Definition: mapDistributeBase.H:116
Foam::UPstream::defaultCommsType
static commsTypes defaultCommsType
Default commsType.
Definition: UPstream.H:272
Foam::List::setSize
void setSize(const label)
Reset size of List.
Definition: List.C:281
Foam::UPstream::nProcs
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:411
Foam::mapDistributeBase::constructHasFlip_
bool constructHasFlip_
Whether constructMap includes flip or not.
Definition: mapDistributeBase.H:119
Foam::UList::byteSize
std::streamsize byteSize() const
Return the binary size in number of characters of the UList.
Definition: UList.C:100
Foam::mapDistributeBase::receive
void receive(PstreamBuffers &, List< T > &) const
Do all receives using PstreamBuffers.
Definition: mapDistributeBaseTemplates.C:1159
Foam::UList::size
label size() const
Return the number of elements in the UList.
Definition: UListI.H:311
Foam::List::transfer
void transfer(List< T > &)
Transfer the contents of the argument List into this list.
Definition: List.C:342
Foam::mapDistributeBase::flipAndCombine
static void flipAndCombine(const UList< label > &map, const bool hasFlip, const UList< T > &rhs, const CombineOp &cop, const negateOp &negOp, List< T > &lhs)
Definition: mapDistributeBaseTemplates.C:35