mapDistributeBase.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration |
5  \\ / A nd | Copyright (C) 2015-2017 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "mapDistributeBase.H"
27 #include "commSchedule.H"
28 #include "HashSet.H"
29 #include "globalIndex.H"
30 #include "ListOps.H"
31 
32 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
33 
34 namespace Foam
35 {
36  defineTypeNameAndDebug(mapDistributeBase, 0);
37 }
38 
39 
40 // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
41 
43 (
44  const labelListList& subMap,
45  const labelListList& constructMap,
46  const int tag
47 )
48 {
49  // Communications: send and receive processor
50  List<labelPair> allComms;
51 
52  {
54 
55  // Find what communication is required
56  forAll(subMap, proci)
57  {
58  if (proci != Pstream::myProcNo())
59  {
60  if (subMap[proci].size())
61  {
62  // I need to send to proci
63  commsSet.insert(labelPair(Pstream::myProcNo(), proci));
64  }
65  if (constructMap[proci].size())
66  {
67  // I need to receive from proci
68  commsSet.insert(labelPair(proci, Pstream::myProcNo()));
69  }
70  }
71  }
72  allComms = commsSet.toc();
73  }
74 
75 
76  // Reduce
77  if (Pstream::master())
78  {
79  // Receive and merge
80  for
81  (
82  int slave=Pstream::firstSlave();
83  slave<=Pstream::lastSlave();
84  slave++
85  )
86  {
87  IPstream fromSlave(Pstream::commsTypes::scheduled, slave, 0, tag);
88  List<labelPair> nbrData(fromSlave);
89 
90  forAll(nbrData, i)
91  {
92  if (findIndex(allComms, nbrData[i]) == -1)
93  {
94  label sz = allComms.size();
95  allComms.setSize(sz+1);
96  allComms[sz] = nbrData[i];
97  }
98  }
99  }
100  // Send back
101  for
102  (
103  int slave=Pstream::firstSlave();
104  slave<=Pstream::lastSlave();
105  slave++
106  )
107  {
108  OPstream toSlave(Pstream::commsTypes::scheduled, slave, 0, tag);
109  toSlave << allComms;
110  }
111  }
112  else
113  {
114  {
115  OPstream toMaster
116  (
119  0,
120  tag
121  );
122  toMaster << allComms;
123  }
124  {
125  IPstream fromMaster
126  (
129  0,
130  tag
131  );
132  fromMaster >> allComms;
133  }
134  }
135 
136 
137  // Determine my schedule.
138  labelList mySchedule
139  (
141  (
142  Pstream::nProcs(),
143  allComms
144  ).procSchedule()[Pstream::myProcNo()]
145  );
146 
147  // Processors involved in my schedule
148  return List<labelPair>(UIndirectList<labelPair>(allComms, mySchedule));
149 
150 
151  //if (debug)
152  //{
153  // Pout<< "I need to:" << endl;
154  // const List<labelPair>& comms = schedule();
155  // forAll(comms, i)
156  // {
157  // const labelPair& twoProcs = comms[i];
158  // label sendProc = twoProcs[0];
159  // label recvProc = twoProcs[1];
160  //
161  // if (recvProc == Pstream::myProcNo())
162  // {
163  // Pout<< " receive from " << sendProc << endl;
164  // }
165  // else
166  // {
167  // Pout<< " send to " << recvProc << endl;
168  // }
169  // }
170  //}
171 }
172 
173 
175 {
176  if (schedulePtr_.empty())
177  {
178  schedulePtr_.reset
179  (
180  new List<labelPair>
181  (
182  schedule(subMap_, constructMap_, Pstream::msgType())
183  )
184  );
185  }
186  return schedulePtr_();
187 }
188 
189 
191 (
192  const label proci,
193  const label expectedSize,
194  const label receivedSize
195 )
196 {
197  if (receivedSize != expectedSize)
198  {
200  << "Expected from processor " << proci
201  << " " << expectedSize << " but received "
202  << receivedSize << " elements."
203  << abort(FatalError);
204  }
205 }
206 
207 
209 {
210  // Determine offsets of remote data.
211  labelList minIndex(Pstream::nProcs(), labelMax);
212  labelList maxIndex(Pstream::nProcs(), labelMin);
213  forAll(constructMap_, proci)
214  {
215  const labelList& construct = constructMap_[proci];
216  if (constructHasFlip_)
217  {
218  forAll(construct, i)
219  {
220  label index = mag(construct[i])-1;
221  minIndex[proci] = min(minIndex[proci], index);
222  maxIndex[proci] = max(maxIndex[proci], index);
223  }
224  }
225  else
226  {
227  forAll(construct, i)
228  {
229  label index = construct[i];
230  minIndex[proci] = min(minIndex[proci], index);
231  maxIndex[proci] = max(maxIndex[proci], index);
232  }
233  }
234  }
235 
236  label localSize;
237  if (maxIndex[Pstream::myProcNo()] == labelMin)
238  {
239  localSize = 0;
240  }
241  else
242  {
243  localSize = maxIndex[Pstream::myProcNo()]+1;
244  }
245 
246  os << "Layout: (constructSize:" << constructSize_
247  << " subHasFlip:" << subHasFlip_
248  << " constructHasFlip:" << constructHasFlip_
249  << ")" << endl
250  << "local (processor " << Pstream::myProcNo() << "):" << endl
251  << " start : 0" << endl
252  << " size : " << localSize << endl;
253 
254  label offset = localSize;
255  forAll(minIndex, proci)
256  {
257  if (proci != Pstream::myProcNo())
258  {
259  if (constructMap_[proci].size() > 0)
260  {
261  if (minIndex[proci] != offset)
262  {
264  << "offset:" << offset
265  << " proci:" << proci
266  << " minIndex:" << minIndex[proci]
267  << abort(FatalError);
268  }
269 
270  label size = maxIndex[proci]-minIndex[proci]+1;
271  os << "processor " << proci << ':' << endl
272  << " start : " << offset << endl
273  << " size : " << size << endl;
274 
275  offset += size;
276  }
277  }
278  }
279 }
280 
281 
283 (
284  const globalIndex& globalNumbering,
285  const labelList& elements,
286  List<Map<label>>& compactMap
287 ) const
288 {
289  compactMap.setSize(Pstream::nProcs());
290 
291  // Count all (non-local) elements needed. Just for presizing map.
292  labelList nNonLocal(Pstream::nProcs(), 0);
293 
294  forAll(elements, i)
295  {
296  label globalIndex = elements[i];
297 
298  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
299  {
300  label proci = globalNumbering.whichProcID(globalIndex);
301  nNonLocal[proci]++;
302  }
303  }
304 
305  forAll(compactMap, proci)
306  {
307  compactMap[proci].clear();
308  if (proci != Pstream::myProcNo())
309  {
310  compactMap[proci].resize(2*nNonLocal[proci]);
311  }
312  }
313 
314 
315  // Collect all (non-local) elements needed.
316  forAll(elements, i)
317  {
318  label globalIndex = elements[i];
319 
320  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
321  {
322  label proci = globalNumbering.whichProcID(globalIndex);
323  label index = globalNumbering.toLocal(proci, globalIndex);
324  label nCompact = compactMap[proci].size();
325  compactMap[proci].insert(index, nCompact);
326  }
327  }
328 }
329 
330 
332 (
333  const globalIndex& globalNumbering,
334  const labelListList& cellCells,
335  List<Map<label>>& compactMap
336 ) const
337 {
338  compactMap.setSize(Pstream::nProcs());
339 
340  // Count all (non-local) elements needed. Just for presizing map.
341  labelList nNonLocal(Pstream::nProcs(), 0);
342 
343  forAll(cellCells, cellI)
344  {
345  const labelList& cCells = cellCells[cellI];
346 
347  forAll(cCells, i)
348  {
349  label globalIndex = cCells[i];
350 
351  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
352  {
353  label proci = globalNumbering.whichProcID(globalIndex);
354  nNonLocal[proci]++;
355  }
356  }
357  }
358 
359  forAll(compactMap, proci)
360  {
361  compactMap[proci].clear();
362  if (proci != Pstream::myProcNo())
363  {
364  compactMap[proci].resize(2*nNonLocal[proci]);
365  }
366  }
367 
368 
369  // Collect all (non-local) elements needed.
370  forAll(cellCells, cellI)
371  {
372  const labelList& cCells = cellCells[cellI];
373 
374  forAll(cCells, i)
375  {
376  label globalIndex = cCells[i];
377 
378  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
379  {
380  label proci = globalNumbering.whichProcID(globalIndex);
381  label index = globalNumbering.toLocal(proci, globalIndex);
382  label nCompact = compactMap[proci].size();
383  compactMap[proci].insert(index, nCompact);
384  }
385  }
386  }
387 }
388 
389 
391 (
392  const int tag,
393  const globalIndex& globalNumbering,
394  labelList& elements,
395  List<Map<label>>& compactMap,
396  labelList& compactStart
397 )
398 {
399  // The overall compact addressing is
400  // - myProcNo data first (uncompacted)
401  // - all other processors consecutively
402 
403  compactStart.setSize(Pstream::nProcs());
404  compactStart[Pstream::myProcNo()] = 0;
405  constructSize_ = globalNumbering.localSize();
406  forAll(compactStart, proci)
407  {
408  if (proci != Pstream::myProcNo())
409  {
410  compactStart[proci] = constructSize_;
411  constructSize_ += compactMap[proci].size();
412  }
413  }
414 
415 
416 
417  // Find out what to receive/send in compact addressing.
418 
419  // What I want to receive is what others have to send
420  labelListList wantedRemoteElements(Pstream::nProcs());
421  // Compact addressing for received data
422  constructMap_.setSize(Pstream::nProcs());
423  forAll(compactMap, proci)
424  {
425  if (proci == Pstream::myProcNo())
426  {
427  // All my own elements are used
428  label nLocal = globalNumbering.localSize();
429  wantedRemoteElements[proci] = identity(nLocal);
430  constructMap_[proci] = identity(nLocal);
431  }
432  else
433  {
434  // Remote elements wanted from processor proci
435  labelList& remoteElem = wantedRemoteElements[proci];
436  labelList& localElem = constructMap_[proci];
437  remoteElem.setSize(compactMap[proci].size());
438  localElem.setSize(compactMap[proci].size());
439  label i = 0;
440  forAllIter(Map<label>, compactMap[proci], iter)
441  {
442  const label compactI = compactStart[proci] + iter();
443  remoteElem[i] = iter.key();
444  localElem[i] = compactI;
445  iter() = compactI;
446  i++;
447  }
448  }
449  }
450 
451  subMap_.setSize(Pstream::nProcs());
452  Pstream::exchange<labelList, label>
453  (
454  wantedRemoteElements,
455  subMap_,
456  tag,
457  Pstream::worldComm //TBD
458  );
459 
460  // Renumber elements
461  forAll(elements, i)
462  {
463  elements[i] = renumber(globalNumbering, compactMap, elements[i]);
464  }
465 }
466 
467 
469 (
470  const int tag,
471  const globalIndex& globalNumbering,
472  labelListList& cellCells,
473  List<Map<label>>& compactMap,
474  labelList& compactStart
475 )
476 {
477  // The overall compact addressing is
478  // - myProcNo data first (uncompacted)
479  // - all other processors consecutively
480 
481  compactStart.setSize(Pstream::nProcs());
482  compactStart[Pstream::myProcNo()] = 0;
483  constructSize_ = globalNumbering.localSize();
484  forAll(compactStart, proci)
485  {
486  if (proci != Pstream::myProcNo())
487  {
488  compactStart[proci] = constructSize_;
489  constructSize_ += compactMap[proci].size();
490  }
491  }
492 
493 
494 
495  // Find out what to receive/send in compact addressing.
496 
497  // What I want to receive is what others have to send
498  labelListList wantedRemoteElements(Pstream::nProcs());
499  // Compact addressing for received data
500  constructMap_.setSize(Pstream::nProcs());
501  forAll(compactMap, proci)
502  {
503  if (proci == Pstream::myProcNo())
504  {
505  // All my own elements are used
506  label nLocal = globalNumbering.localSize();
507  wantedRemoteElements[proci] = identity(nLocal);
508  constructMap_[proci] = identity(nLocal);
509  }
510  else
511  {
512  // Remote elements wanted from processor proci
513  labelList& remoteElem = wantedRemoteElements[proci];
514  labelList& localElem = constructMap_[proci];
515  remoteElem.setSize(compactMap[proci].size());
516  localElem.setSize(compactMap[proci].size());
517  label i = 0;
518  forAllIter(Map<label>, compactMap[proci], iter)
519  {
520  const label compactI = compactStart[proci] + iter();
521  remoteElem[i] = iter.key();
522  localElem[i] = compactI;
523  iter() = compactI;
524  i++;
525  }
526  }
527  }
528 
529  subMap_.setSize(Pstream::nProcs());
530  Pstream::exchange<labelList, label>
531  (
532  wantedRemoteElements,
533  subMap_,
534  tag,
535  Pstream::worldComm //TBD
536  );
537 
538  // Renumber elements
539  forAll(cellCells, cellI)
540  {
541  labelList& cCells = cellCells[cellI];
542 
543  forAll(cCells, i)
544  {
545  cCells[i] = renumber(globalNumbering, compactMap, cCells[i]);
546  }
547  }
548 }
549 
550 
551 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
552 
554 :
555  constructSize_(0),
556  subHasFlip_(false),
557  constructHasFlip_(false),
558  schedulePtr_()
559 {}
560 
561 
563 (
564  const label constructSize,
565  const Xfer<labelListList>& subMap,
566  const Xfer<labelListList>& constructMap,
567  const bool subHasFlip,
568  const bool constructHasFlip
569 )
570 :
571  constructSize_(constructSize),
572  subMap_(subMap),
573  constructMap_(constructMap),
574  subHasFlip_(subHasFlip),
575  constructHasFlip_(constructHasFlip),
576  schedulePtr_()
577 {}
578 
579 
581 (
582  const labelList& sendProcs,
583  const labelList& recvProcs
584 )
585 :
586  constructSize_(0),
587  subHasFlip_(false),
588  constructHasFlip_(false),
589  schedulePtr_()
590 {
591  if (sendProcs.size() != recvProcs.size())
592  {
594  << "The send and receive data is not the same length. sendProcs:"
595  << sendProcs.size() << " recvProcs:" << recvProcs.size()
596  << abort(FatalError);
597  }
598 
599  // Per processor the number of samples we have to send/receive.
600  labelList nSend(Pstream::nProcs(), 0);
601  labelList nRecv(Pstream::nProcs(), 0);
602 
603  forAll(sendProcs, sampleI)
604  {
605  label sendProc = sendProcs[sampleI];
606  label recvProc = recvProcs[sampleI];
607 
608  // Note that also need to include local communication (both
609  // RecvProc and sendProc on local processor)
610 
611  if (Pstream::myProcNo() == sendProc)
612  {
613  // I am the sender. Count destination processor.
614  nSend[recvProc]++;
615  }
616  if (Pstream::myProcNo() == recvProc)
617  {
618  // I am the receiver.
619  nRecv[sendProc]++;
620  }
621  }
622 
625  forAll(nSend, proci)
626  {
627  subMap_[proci].setSize(nSend[proci]);
628  constructMap_[proci].setSize(nRecv[proci]);
629  }
630  nSend = 0;
631  nRecv = 0;
632 
633  forAll(sendProcs, sampleI)
634  {
635  label sendProc = sendProcs[sampleI];
636  label recvProc = recvProcs[sampleI];
637 
638  if (Pstream::myProcNo() == sendProc)
639  {
640  // I am the sender. Store index I need to send.
641  subMap_[recvProc][nSend[recvProc]++] = sampleI;
642  }
643  if (Pstream::myProcNo() == recvProc)
644  {
645  // I am the receiver.
646  constructMap_[sendProc][nRecv[sendProc]++] = sampleI;
647  // Largest entry inside constructMap
648  constructSize_ = sampleI+1;
649  }
650  }
651 }
652 
653 
655 (
656  const globalIndex& globalNumbering,
657  labelList& elements,
658  List<Map<label>>& compactMap,
659  const int tag
660 )
661 :
662  constructSize_(0),
663  subHasFlip_(false),
664  constructHasFlip_(false),
665  schedulePtr_()
666 {
667  // Construct per processor compact addressing of the global elements
668  // needed. The ones from the local processor are not included since
669  // these are always all needed.
671  (
672  globalNumbering,
673  elements,
674  compactMap
675  );
676 
678  //forAll(compactMap, proci)
679  //{
680  // if (proci != Pstream::myProcNo())
681  // {
682  // Map<label>& globalMap = compactMap[proci];
683  //
684  // SortableList<label> sorted(globalMap.toc().xfer());
685  //
686  // forAll(sorted, i)
687  // {
688  // Map<label>::iterator iter = globalMap.find(sorted[i]);
689  // iter() = i;
690  // }
691  // }
692  //}
693 
694 
695  // Exchange what I need with processor that supplies it. Renumber elements
696  // into compact numbering
697  labelList compactStart;
699  (
700  tag,
701  globalNumbering,
702  elements,
703  compactMap,
704  compactStart
705  );
706 
707  if (debug)
708  {
709  printLayout(Pout);
710  }
711 }
712 
713 
715 (
716  const globalIndex& globalNumbering,
717  labelListList& cellCells,
718  List<Map<label>>& compactMap,
719  const int tag
720 )
721 :
722  constructSize_(0),
723  subHasFlip_(false),
724  constructHasFlip_(false),
725  schedulePtr_()
726 {
727  // Construct per processor compact addressing of the global elements
728  // needed. The ones from the local processor are not included since
729  // these are always all needed.
731  (
732  globalNumbering,
733  cellCells,
734  compactMap
735  );
736 
738  //forAll(compactMap, proci)
739  //{
740  // if (proci != Pstream::myProcNo())
741  // {
742  // Map<label>& globalMap = compactMap[proci];
743  //
744  // SortableList<label> sorted(globalMap.toc().xfer());
745  //
746  // forAll(sorted, i)
747  // {
748  // Map<label>::iterator iter = globalMap.find(sorted[i]);
749  // iter() = i;
750  // }
751  // }
752  //}
753 
754 
755  // Exchange what I need with processor that supplies it. Renumber elements
756  // into compact numbering
757  labelList compactStart;
759  (
760  tag,
761  globalNumbering,
762  cellCells,
763  compactMap,
764  compactStart
765  );
766 
767  if (debug)
768  {
769  printLayout(Pout);
770  }
771 }
772 
773 
775 :
777  subMap_(map.subMap_),
781  schedulePtr_()
782 {}
783 
784 
786 :
788  subMap_(map().subMap_.xfer()),
790  subHasFlip_(map().subHasFlip_),
792  schedulePtr_()
793 {}
794 
795 
797 {
798  is >> *this;
799 }
800 
801 
802 // * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
803 
805 {
807  subMap_.transfer(rhs.subMap_);
809  subHasFlip_ = rhs.subHasFlip_;
811  schedulePtr_.clear();
812 }
813 
814 
816 {
817  return xferMove(*this);
818 }
819 
820 
822 (
823  const globalIndex& globalNumbering,
824  const List<Map<label>>& compactMap,
825  const label globalI
826 )
827 {
828  if (globalI == -1)
829  {
830  return globalI;
831  }
832  if (globalNumbering.isLocal(globalI))
833  {
834  return globalNumbering.toLocal(globalI);
835  }
836  else
837  {
838  label proci = globalNumbering.whichProcID(globalI);
839  label index = globalNumbering.toLocal(proci, globalI);
840  return compactMap[proci][index];
841  }
842 }
843 
844 
845 void Foam::mapDistributeBase::compact(const boolList& elemIsUsed, const int tag)
846 {
847  // 1. send back to sender. Have sender delete the corresponding element
848  // from the submap and do the same to the constructMap locally
849  // (and in same order).
850 
851  // Send elemIsUsed field to neighbour. Use nonblocking code from
852  // mapDistributeBase but in reverse order.
853  if (Pstream::parRun())
854  {
855  label startOfRequests = Pstream::nRequests();
856 
857  // Set up receives from neighbours
858 
859  List<boolList> recvFields(Pstream::nProcs());
860 
861  for (label domain = 0; domain < Pstream::nProcs(); domain++)
862  {
863  const labelList& map = subMap_[domain];
864 
865  if (domain != Pstream::myProcNo() && map.size())
866  {
867  recvFields[domain].setSize(map.size());
869  (
871  domain,
872  reinterpret_cast<char*>(recvFields[domain].begin()),
873  recvFields[domain].size()*sizeof(bool),
874  tag
875  );
876  }
877  }
878 
879 
880  List<boolList> sendFields(Pstream::nProcs());
881 
882  for (label domain = 0; domain < Pstream::nProcs(); domain++)
883  {
884  const labelList& map = constructMap_[domain];
885 
886  if (domain != Pstream::myProcNo() && map.size())
887  {
888  boolList& subField = sendFields[domain];
889  subField.setSize(map.size());
890  forAll(map, i)
891  {
892  subField[i] = accessAndFlip
893  (
894  elemIsUsed,
895  map[i],
897  noOp() // do not flip elemIsUsed value
898  );
899  }
900 
902  (
904  domain,
905  reinterpret_cast<const char*>(subField.begin()),
906  subField.size()*sizeof(bool),
907  tag
908  );
909  }
910  }
911 
912 
913 
914  // Set up 'send' to myself - write directly into recvFields
915 
916  {
917  const labelList& map = constructMap_[Pstream::myProcNo()];
918 
919  recvFields[Pstream::myProcNo()].setSize(map.size());
920  forAll(map, i)
921  {
922  recvFields[Pstream::myProcNo()][i] = accessAndFlip
923  (
924  elemIsUsed,
925  map[i],
927  noOp() // do not flip elemIsUsed value
928  );
929  }
930  }
931 
932 
933  // Wait for all to finish
934 
935  Pstream::waitRequests(startOfRequests);
936 
937 
938  // Compact out all submap entries that are referring to unused elements
939  for (label domain = 0; domain < Pstream::nProcs(); domain++)
940  {
941  const labelList& map = subMap_[domain];
942 
943  labelList newMap(map.size());
944  label newI = 0;
945 
946  forAll(map, i)
947  {
948  if (recvFields[domain][i])
949  {
950  // So element is used on destination side
951  newMap[newI++] = map[i];
952  }
953  }
954  if (newI < map.size())
955  {
956  newMap.setSize(newI);
957  subMap_[domain].transfer(newMap);
958  }
959  }
960  }
961 
962 
963  // 2. remove from construct map - since end-result (element in elemIsUsed)
964  // not used.
965 
966  label maxConstructIndex = -1;
967 
968  for (label domain = 0; domain < Pstream::nProcs(); domain++)
969  {
970  const labelList& map = constructMap_[domain];
971 
972  labelList newMap(map.size());
973  label newI = 0;
974 
975  forAll(map, i)
976  {
977  label destinationI = map[i];
978  if (constructHasFlip_)
979  {
980  destinationI = mag(destinationI)-1;
981  }
982 
983  // Is element is used on destination side
984  if (elemIsUsed[destinationI])
985  {
986  maxConstructIndex = max(maxConstructIndex, destinationI);
987 
988  newMap[newI++] = map[i];
989  }
990  }
991  if (newI < map.size())
992  {
993  newMap.setSize(newI);
994  constructMap_[domain].transfer(newMap);
995  }
996  }
997 
998  constructSize_ = maxConstructIndex+1;
999 
1000  // Clear the schedule (note:not necessary if nothing changed)
1001  schedulePtr_.clear();
1002 }
1003 
1004 
1007  const boolList& elemIsUsed,
1008  const label localSize, // max index for subMap
1009  labelList& oldToNewSub,
1010  labelList& oldToNewConstruct,
1011  const int tag
1012 )
1013 {
1014  // 1. send back to sender. Have sender delete the corresponding element
1015  // from the submap and do the same to the constructMap locally
1016  // (and in same order).
1017 
1018  // Send elemIsUsed field to neighbour. Use nonblocking code from
1019  // mapDistributeBase but in reverse order.
1020  if (Pstream::parRun())
1021  {
1022  label startOfRequests = Pstream::nRequests();
1023 
1024  // Set up receives from neighbours
1025 
1026  List<boolList> recvFields(Pstream::nProcs());
1027 
1028  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1029  {
1030  const labelList& map = subMap_[domain];
1031 
1032  if (domain != Pstream::myProcNo() && map.size())
1033  {
1034  recvFields[domain].setSize(map.size());
1036  (
1038  domain,
1039  reinterpret_cast<char*>(recvFields[domain].begin()),
1040  recvFields[domain].size()*sizeof(bool),
1041  tag
1042  );
1043  }
1044  }
1045 
1046 
1047  List<boolList> sendFields(Pstream::nProcs());
1048 
1049  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1050  {
1051  const labelList& map = constructMap_[domain];
1052 
1053  if (domain != Pstream::myProcNo() && map.size())
1054  {
1055  boolList& subField = sendFields[domain];
1056  subField.setSize(map.size());
1057  forAll(map, i)
1058  {
1059  label index = map[i];
1060  if (constructHasFlip_)
1061  {
1062  index = mag(index)-1;
1063  }
1064  subField[i] = elemIsUsed[index];
1065  }
1066 
1068  (
1070  domain,
1071  reinterpret_cast<const char*>(subField.begin()),
1072  subField.size()*sizeof(bool),
1073  tag
1074  );
1075  }
1076  }
1077 
1078 
1079 
1080  // Set up 'send' to myself - write directly into recvFields
1081 
1082  {
1083  const labelList& map = constructMap_[Pstream::myProcNo()];
1084 
1085  recvFields[Pstream::myProcNo()].setSize(map.size());
1086  forAll(map, i)
1087  {
1088  label index = map[i];
1089  if (constructHasFlip_)
1090  {
1091  index = mag(index)-1;
1092  }
1093  recvFields[Pstream::myProcNo()][i] = elemIsUsed[index];
1094  }
1095  }
1096 
1097 
1098  // Wait for all to finish
1099 
1100  Pstream::waitRequests(startOfRequests);
1101 
1102 
1103 
1104 
1105  // Work out which elements on the sending side are needed
1106  {
1107  oldToNewSub.setSize(localSize, -1);
1108 
1109  boolList sendElemIsUsed(localSize, false);
1110 
1111  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1112  {
1113  const labelList& map = subMap_[domain];
1114  forAll(map, i)
1115  {
1116  if (recvFields[domain][i])
1117  {
1118  label index = map[i];
1119  if (subHasFlip_)
1120  {
1121  index = mag(index)-1;
1122  }
1123  sendElemIsUsed[index] = true;
1124  }
1125  }
1126  }
1127 
1128  label newI = 0;
1129  forAll(sendElemIsUsed, i)
1130  {
1131  if (sendElemIsUsed[i])
1132  {
1133  oldToNewSub[i] = newI++;
1134  }
1135  }
1136  }
1137 
1138 
1139  // Compact out all submap entries that are referring to unused elements
1140  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1141  {
1142  const labelList& map = subMap_[domain];
1143 
1144  labelList newMap(map.size());
1145  label newI = 0;
1146 
1147  forAll(map, i)
1148  {
1149  if (recvFields[domain][i])
1150  {
1151  // So element is used on destination side
1152  label index = map[i];
1153  label sign = 1;
1154  if (subHasFlip_)
1155  {
1156  if (index < 0)
1157  {
1158  sign = -1;
1159  }
1160  index = mag(index)-1;
1161  }
1162  label newIndex = oldToNewSub[index];
1163  if (subHasFlip_)
1164  {
1165  newIndex = sign*(newIndex+1);
1166  }
1167  newMap[newI++] = newIndex;
1168  }
1169  }
1170  newMap.setSize(newI);
1171  subMap_[domain].transfer(newMap);
1172  }
1173  }
1174 
1175 
1176  // 2. remove from construct map - since end-result (element in elemIsUsed)
1177  // not used.
1178 
1179 
1180  oldToNewConstruct.setSize(elemIsUsed.size(), -1);
1181  constructSize_ = 0;
1182  forAll(elemIsUsed, i)
1183  {
1184  if (elemIsUsed[i])
1185  {
1186  oldToNewConstruct[i] = constructSize_++;
1187  }
1188  }
1189 
1190  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1191  {
1192  const labelList& map = constructMap_[domain];
1193 
1194  labelList newMap(map.size());
1195  label newI = 0;
1196 
1197  forAll(map, i)
1198  {
1199  label destinationI = map[i];
1200  label sign = 1;
1201  if (constructHasFlip_)
1202  {
1203  if (destinationI < 0)
1204  {
1205  sign = -1;
1206  }
1207  destinationI = mag(destinationI)-1;
1208  }
1209 
1210  // Is element is used on destination side
1211  if (elemIsUsed[destinationI])
1212  {
1213  label newIndex = oldToNewConstruct[destinationI];
1214  if (constructHasFlip_)
1215  {
1216  newIndex = sign*(newIndex+1);
1217  }
1218  newMap[newI++] = newIndex;
1219  }
1220  }
1221  newMap.setSize(newI);
1222  constructMap_[domain].transfer(newMap);
1223  }
1224 }
1225 
1226 
1227 // * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
1228 
1230 {
1231  // Check for assignment to self
1232  if (this == &rhs)
1233  {
1235  << "Attempted assignment to self"
1236  << abort(FatalError);
1237  }
1239  subMap_ = rhs.subMap_;
1241  subHasFlip_ = rhs.subHasFlip_;
1243  schedulePtr_.clear();
1244 }
1245 
1246 
1247 // * * * * * * * * * * * * * * Istream Operator * * * * * * * * * * * * * * //
1248 
1250 {
1251  is.fatalCheck("operator>>(Istream&, mapDistributeBase&)");
1252 
1253  is >> map.constructSize_ >> map.subMap_ >> map.constructMap_
1254  >> map.subHasFlip_ >> map.constructHasFlip_;
1255 
1256  return is;
1257 }
1258 
1259 
1260 // * * * * * * * * * * * * * * Ostream Operator * * * * * * * * * * * * * * //
1261 
1263 {
1264  os << map.constructSize_ << token::NL
1265  << map.subMap_ << token::NL
1266  << map.constructMap_ << token::NL
1267  << map.subHasFlip_ << token::SPACE << map.constructHasFlip_
1268  << token::NL;
1269 
1270  return os;
1271 }
1272 
1273 
1274 // ************************************************************************* //
ListType renumber(const labelUList &oldToNew, const ListType &)
Renumber the values (not the indices) of a list.
bool isLocal(const label i) const
Is on local processor.
Definition: globalIndexI.H:95
dimensionedScalar sign(const dimensionedScalar &ds)
A simple container for copying or transferring objects of type <T>.
Definition: Xfer.H:85
A HashTable with keys but without contents.
Definition: HashSet.H:59
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:428
void compact(const boolList &elemIsUsed, const int tag=UPstream::msgType())
Compact maps. Gets per field a bool whether it is used (locally)
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
static int masterNo()
Process index of the master.
Definition: UPstream.H:406
void calcCompactAddressing(const globalIndex &globalNumbering, const labelList &elements, List< Map< label >> &compactMap) const
Construct per processor compact addressing of the global elements.
static label renumber(const globalIndex &, const List< Map< label >> &compactMap, const label globalElement)
Helper for construct from globalIndex. Renumbers element.
error FatalError
dimensioned< Type > max(const dimensioned< Type > &, const dimensioned< Type > &)
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
Definition: error.H:319
A 1D array of objects of type <T>, where the size of the vector is known and used for subscript bound...
Definition: HashTable.H:60
static int firstSlave()
Process index of first slave.
Definition: UPstream.H:435
const List< labelPair > & schedule() const
Return a schedule. Demand driven. See above.
#define forAllIter(Container, container, iter)
Iterate across all elements in the container object of type.
Definition: UList.H:453
An Istream is an abstract base class for all input systems (streams, files, token lists etc)...
Definition: Istream.H:57
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:163
labelListList subMap_
Maps from subsetted data back to original data.
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:418
Ostream & endl(Ostream &os)
Add newline and flush stream.
Definition: Ostream.H:253
static bool master(const label communicator=0)
Am I the master process.
Definition: UPstream.H:412
labelList identity(const label len)
Create identity map (map[i] == i) of given length.
Definition: ListOps.C:104
static label nRequests()
Get number of outstanding requests.
Definition: UPstream.C:107
static int & msgType()
Message tag of standard messages.
Definition: UPstream.H:465
static label worldComm
Default communicator (all processors)
Definition: UPstream.H:275
bool insert(const Key &key)
Insert a new entry.
Definition: HashSet.H:116
Xfer< mapDistributeBase > xfer()
Transfer contents to the Xfer container.
mapDistributeBase()
Construct null.
label constructSize_
Size of reconstructed data.
Various functions to operate on Lists.
Input inter-processor communications stream.
Definition: IPstream.H:50
Calculates a unique integer (label so might not have enough room - 2G max) for processor + local inde...
Definition: globalIndex.H:63
Xfer< T > xferMove(T &)
Construct by transferring the contents of the arg.
autoPtr< List< labelPair > > schedulePtr_
Schedule.
Istream & operator>>(Istream &, directionInfo &)
void printLayout(Ostream &os) const
Debug: print layout. Can only be used on maps with sorted.
static T accessAndFlip(const UList< T > &fld, const label index, const bool hasFlip, const negateOp &negOp)
iterator begin()
Return an iterator to begin traversing the UList.
Definition: UListI.H:216
Pair< label > labelPair
Label pair.
Definition: labelPair.H:48
label toLocal(const label i) const
From global to local on current processor.
Definition: globalIndexI.H:117
static const label labelMax
Definition: label.H:62
label whichProcID(const label i) const
Which processor does global come from? Binary search.
Definition: globalIndexI.H:123
errorManip< error > abort(error &err)
Definition: errorManip.H:131
labelListList constructMap_
Maps from subsetted data to new reconstructed data.
void fatalCheck(const char *operation) const
Check IOstream status for given operation.
Definition: IOstream.C:105
void operator=(const mapDistributeBase &)
Class containing processor-to-processor mapping information.
An Ostream is an abstract base class for all output systems (streams, files, token lists...
Definition: Ostream.H:53
static label read(const commsTypes commsType, const int fromProcNo, char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Read into given buffer from given processor and return the.
Definition: UIPread.C:79
Determines the order in which a set of processors should communicate with one another.
Definition: commSchedule.H:65
defineTypeNameAndDebug(combustionModel, 0)
static bool write(const commsTypes commsType, const int toProcNo, const char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Write given buffer to given processor.
Definition: UOPwrite.C:34
Output inter-processor communications stream.
Definition: OPstream.H:50
label findIndex(const ListType &, typename ListType::const_reference, const label start=0)
Find first occurence of given element and return index,.
static void waitRequests(const label start=0)
Wait until all requests (from start onwards) have finished.
Definition: UPstream.C:117
dimensioned< Type > min(const dimensioned< Type > &, const dimensioned< Type > &)
bool subHasFlip_
Whether subMap includes flip or not.
void setSize(const label)
Reset size of List.
Definition: List.C:281
static bool & parRun()
Is this a parallel run?
Definition: UPstream.H:394
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:400
void exchangeAddressing(const int tag, const globalIndex &globalNumbering, labelList &elements, List< Map< label >> &compactMap, labelList &compactStart)
bool constructHasFlip_
Whether constructMap includes flip or not.
prefixOSstream Pout(cout, "Pout")
Definition: IOstreams.H:53
A List with indirect addressing.
Definition: fvMatrix.H:106
Ostream & operator<<(Ostream &, const ensightPart &)
static void checkReceivedSize(const label proci, const label expectedSize, const label receivedSize)
dimensioned< scalar > mag(const dimensioned< Type > &)
void transfer(mapDistributeBase &)
Transfer the contents of the argument and annul the argument.
List< Key > toc() const
Return the table of contents.
Definition: HashTable.C:202
void transfer(List< T > &)
Transfer the contents of the argument List into this list.
Definition: List.C:342
static const label labelMin
Definition: label.H:61
Namespace for OpenFOAM.
static int lastSlave(const label communicator=0)
Process index of last slave.
Definition: UPstream.H:441
label localSize() const
My local size.
Definition: globalIndexI.H:60