All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Pages
mapDistributeBase.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration | Website: https://openfoam.org
5  \\ / A nd | Copyright (C) 2015-2019 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "mapDistributeBase.H"
27 #include "commSchedule.H"
28 #include "HashSet.H"
29 #include "globalIndex.H"
30 #include "ListOps.H"
31 
32 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
33 
34 namespace Foam
35 {
36  defineTypeNameAndDebug(mapDistributeBase, 0);
37 }
38 
39 
40 // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
41 
43 (
44  const labelListList& subMap,
45  const labelListList& constructMap,
46  const int tag
47 )
48 {
49  // Communications: send and receive processor
50  List<labelPair> allComms;
51 
52  {
54 
55  // Find what communication is required
56  forAll(subMap, proci)
57  {
58  if (proci != Pstream::myProcNo())
59  {
60  if (subMap[proci].size())
61  {
62  // I need to send to proci
63  commsSet.insert(labelPair(Pstream::myProcNo(), proci));
64  }
65  if (constructMap[proci].size())
66  {
67  // I need to receive from proci
68  commsSet.insert(labelPair(proci, Pstream::myProcNo()));
69  }
70  }
71  }
72  allComms = commsSet.toc();
73  }
74 
75 
76  // Reduce
77  if (Pstream::master())
78  {
79  // Receive and merge
80  for
81  (
82  int slave=Pstream::firstSlave();
83  slave<=Pstream::lastSlave();
84  slave++
85  )
86  {
87  IPstream fromSlave(Pstream::commsTypes::scheduled, slave, 0, tag);
88  List<labelPair> nbrData(fromSlave);
89 
90  forAll(nbrData, i)
91  {
92  if (findIndex(allComms, nbrData[i]) == -1)
93  {
94  label sz = allComms.size();
95  allComms.setSize(sz+1);
96  allComms[sz] = nbrData[i];
97  }
98  }
99  }
100  // Send back
101  for
102  (
103  int slave=Pstream::firstSlave();
104  slave<=Pstream::lastSlave();
105  slave++
106  )
107  {
108  OPstream toSlave(Pstream::commsTypes::scheduled, slave, 0, tag);
109  toSlave << allComms;
110  }
111  }
112  else
113  {
114  {
115  OPstream toMaster
116  (
119  0,
120  tag
121  );
122  toMaster << allComms;
123  }
124  {
125  IPstream fromMaster
126  (
129  0,
130  tag
131  );
132  fromMaster >> allComms;
133  }
134  }
135 
136 
137  // Determine my schedule.
138  labelList mySchedule
139  (
141  (
142  Pstream::nProcs(),
143  allComms
144  ).procSchedule()[Pstream::myProcNo()]
145  );
146 
147  // Processors involved in my schedule
148  return List<labelPair>(UIndirectList<labelPair>(allComms, mySchedule));
149 
150 
151  // if (debug)
152  //{
153  // Pout<< "I need to:" << endl;
154  // const List<labelPair>& comms = schedule();
155  // forAll(comms, i)
156  // {
157  // const labelPair& twoProcs = comms[i];
158  // label sendProc = twoProcs[0];
159  // label recvProc = twoProcs[1];
160  //
161  // if (recvProc == Pstream::myProcNo())
162  // {
163  // Pout<< " receive from " << sendProc << endl;
164  // }
165  // else
166  // {
167  // Pout<< " send to " << recvProc << endl;
168  // }
169  // }
170  //}
171 }
172 
173 
175 {
176  if (schedulePtr_.empty())
177  {
178  schedulePtr_.reset
179  (
180  new List<labelPair>
181  (
182  schedule(subMap_, constructMap_, Pstream::msgType())
183  )
184  );
185  }
186  return schedulePtr_();
187 }
188 
189 
191 (
192  const label proci,
193  const label expectedSize,
194  const label receivedSize
195 )
196 {
197  if (receivedSize != expectedSize)
198  {
200  << "Expected from processor " << proci
201  << " " << expectedSize << " but received "
202  << receivedSize << " elements."
203  << abort(FatalError);
204  }
205 }
206 
207 
209 {
210  // Determine offsets of remote data.
211  labelList minIndex(Pstream::nProcs(), labelMax);
212  labelList maxIndex(Pstream::nProcs(), labelMin);
213  forAll(constructMap_, proci)
214  {
215  const labelList& construct = constructMap_[proci];
216  if (constructHasFlip_)
217  {
218  forAll(construct, i)
219  {
220  label index = mag(construct[i])-1;
221  minIndex[proci] = min(minIndex[proci], index);
222  maxIndex[proci] = max(maxIndex[proci], index);
223  }
224  }
225  else
226  {
227  forAll(construct, i)
228  {
229  label index = construct[i];
230  minIndex[proci] = min(minIndex[proci], index);
231  maxIndex[proci] = max(maxIndex[proci], index);
232  }
233  }
234  }
235 
236  label localSize;
237  if (maxIndex[Pstream::myProcNo()] == labelMin)
238  {
239  localSize = 0;
240  }
241  else
242  {
243  localSize = maxIndex[Pstream::myProcNo()]+1;
244  }
245 
246  os << "Layout: (constructSize:" << constructSize_
247  << " subHasFlip:" << subHasFlip_
248  << " constructHasFlip:" << constructHasFlip_
249  << ")" << endl
250  << "local (processor " << Pstream::myProcNo() << "):" << endl
251  << " start : 0" << endl
252  << " size : " << localSize << endl;
253 
254  label offset = localSize;
255  forAll(minIndex, proci)
256  {
257  if (proci != Pstream::myProcNo())
258  {
259  if (constructMap_[proci].size() > 0)
260  {
261  if (minIndex[proci] != offset)
262  {
264  << "offset:" << offset
265  << " proci:" << proci
266  << " minIndex:" << minIndex[proci]
267  << abort(FatalError);
268  }
269 
270  label size = maxIndex[proci]-minIndex[proci]+1;
271  os << "processor " << proci << ':' << endl
272  << " start : " << offset << endl
273  << " size : " << size << endl;
274 
275  offset += size;
276  }
277  }
278  }
279 }
280 
281 
283 (
284  const globalIndex& globalNumbering,
285  const labelList& elements,
286  List<Map<label>>& compactMap
287 ) const
288 {
289  compactMap.setSize(Pstream::nProcs());
290 
291  // Count all (non-local) elements needed. Just for presizing map.
292  labelList nNonLocal(Pstream::nProcs(), 0);
293 
294  forAll(elements, i)
295  {
296  label globalIndex = elements[i];
297 
298  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
299  {
300  label proci = globalNumbering.whichProcID(globalIndex);
301  nNonLocal[proci]++;
302  }
303  }
304 
305  forAll(compactMap, proci)
306  {
307  compactMap[proci].clear();
308  if (proci != Pstream::myProcNo())
309  {
310  compactMap[proci].resize(2*nNonLocal[proci]);
311  }
312  }
313 
314 
315  // Collect all (non-local) elements needed.
316  forAll(elements, i)
317  {
318  label globalIndex = elements[i];
319 
320  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
321  {
322  label proci = globalNumbering.whichProcID(globalIndex);
323  label index = globalNumbering.toLocal(proci, globalIndex);
324  label nCompact = compactMap[proci].size();
325  compactMap[proci].insert(index, nCompact);
326  }
327  }
328 }
329 
330 
332 (
333  const globalIndex& globalNumbering,
334  const labelListList& cellCells,
335  List<Map<label>>& compactMap
336 ) const
337 {
338  compactMap.setSize(Pstream::nProcs());
339 
340  // Count all (non-local) elements needed. Just for presizing map.
341  labelList nNonLocal(Pstream::nProcs(), 0);
342 
343  forAll(cellCells, cellI)
344  {
345  const labelList& cCells = cellCells[cellI];
346 
347  forAll(cCells, i)
348  {
349  label globalIndex = cCells[i];
350 
351  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
352  {
353  label proci = globalNumbering.whichProcID(globalIndex);
354  nNonLocal[proci]++;
355  }
356  }
357  }
358 
359  forAll(compactMap, proci)
360  {
361  compactMap[proci].clear();
362  if (proci != Pstream::myProcNo())
363  {
364  compactMap[proci].resize(2*nNonLocal[proci]);
365  }
366  }
367 
368 
369  // Collect all (non-local) elements needed.
370  forAll(cellCells, cellI)
371  {
372  const labelList& cCells = cellCells[cellI];
373 
374  forAll(cCells, i)
375  {
376  label globalIndex = cCells[i];
377 
378  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
379  {
380  label proci = globalNumbering.whichProcID(globalIndex);
381  label index = globalNumbering.toLocal(proci, globalIndex);
382  label nCompact = compactMap[proci].size();
383  compactMap[proci].insert(index, nCompact);
384  }
385  }
386  }
387 }
388 
389 
391 (
392  const int tag,
393  const globalIndex& globalNumbering,
394  labelList& elements,
395  List<Map<label>>& compactMap,
396  labelList& compactStart
397 )
398 {
399  // The overall compact addressing is
400  // - myProcNo data first (uncompacted)
401  // - all other processors consecutively
402 
403  compactStart.setSize(Pstream::nProcs());
404  compactStart[Pstream::myProcNo()] = 0;
405  constructSize_ = globalNumbering.localSize();
406  forAll(compactStart, proci)
407  {
408  if (proci != Pstream::myProcNo())
409  {
410  compactStart[proci] = constructSize_;
411  constructSize_ += compactMap[proci].size();
412  }
413  }
414 
415 
416 
417  // Find out what to receive/send in compact addressing.
418 
419  // What I want to receive is what others have to send
420  labelListList wantedRemoteElements(Pstream::nProcs());
421  // Compact addressing for received data
422  constructMap_.setSize(Pstream::nProcs());
423  forAll(compactMap, proci)
424  {
425  if (proci == Pstream::myProcNo())
426  {
427  // All my own elements are used
428  label nLocal = globalNumbering.localSize();
429  wantedRemoteElements[proci] = identity(nLocal);
430  constructMap_[proci] = identity(nLocal);
431  }
432  else
433  {
434  // Remote elements wanted from processor proci
435  labelList& remoteElem = wantedRemoteElements[proci];
436  labelList& localElem = constructMap_[proci];
437  remoteElem.setSize(compactMap[proci].size());
438  localElem.setSize(compactMap[proci].size());
439  label i = 0;
440  forAllIter(Map<label>, compactMap[proci], iter)
441  {
442  const label compactI = compactStart[proci] + iter();
443  remoteElem[i] = iter.key();
444  localElem[i] = compactI;
445  iter() = compactI;
446  i++;
447  }
448  }
449  }
450 
451  subMap_.setSize(Pstream::nProcs());
452  Pstream::exchange<labelList, label>
453  (
454  wantedRemoteElements,
455  subMap_,
456  tag,
457  Pstream::worldComm // TBD
458  );
459 
460  // Renumber elements
461  forAll(elements, i)
462  {
463  elements[i] = renumber(globalNumbering, compactMap, elements[i]);
464  }
465 }
466 
467 
469 (
470  const int tag,
471  const globalIndex& globalNumbering,
472  labelListList& cellCells,
473  List<Map<label>>& compactMap,
474  labelList& compactStart
475 )
476 {
477  // The overall compact addressing is
478  // - myProcNo data first (uncompacted)
479  // - all other processors consecutively
480 
481  compactStart.setSize(Pstream::nProcs());
482  compactStart[Pstream::myProcNo()] = 0;
483  constructSize_ = globalNumbering.localSize();
484  forAll(compactStart, proci)
485  {
486  if (proci != Pstream::myProcNo())
487  {
488  compactStart[proci] = constructSize_;
489  constructSize_ += compactMap[proci].size();
490  }
491  }
492 
493 
494 
495  // Find out what to receive/send in compact addressing.
496 
497  // What I want to receive is what others have to send
498  labelListList wantedRemoteElements(Pstream::nProcs());
499  // Compact addressing for received data
500  constructMap_.setSize(Pstream::nProcs());
501  forAll(compactMap, proci)
502  {
503  if (proci == Pstream::myProcNo())
504  {
505  // All my own elements are used
506  label nLocal = globalNumbering.localSize();
507  wantedRemoteElements[proci] = identity(nLocal);
508  constructMap_[proci] = identity(nLocal);
509  }
510  else
511  {
512  // Remote elements wanted from processor proci
513  labelList& remoteElem = wantedRemoteElements[proci];
514  labelList& localElem = constructMap_[proci];
515  remoteElem.setSize(compactMap[proci].size());
516  localElem.setSize(compactMap[proci].size());
517  label i = 0;
518  forAllIter(Map<label>, compactMap[proci], iter)
519  {
520  const label compactI = compactStart[proci] + iter();
521  remoteElem[i] = iter.key();
522  localElem[i] = compactI;
523  iter() = compactI;
524  i++;
525  }
526  }
527  }
528 
529  subMap_.setSize(Pstream::nProcs());
530  Pstream::exchange<labelList, label>
531  (
532  wantedRemoteElements,
533  subMap_,
534  tag,
535  Pstream::worldComm // TBD
536  );
537 
538  // Renumber elements
539  forAll(cellCells, cellI)
540  {
541  labelList& cCells = cellCells[cellI];
542 
543  forAll(cCells, i)
544  {
545  cCells[i] = renumber(globalNumbering, compactMap, cCells[i]);
546  }
547  }
548 }
549 
550 
551 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
552 
554 :
555  constructSize_(0),
556  subHasFlip_(false),
557  constructHasFlip_(false),
558  schedulePtr_()
559 {}
560 
561 
563 (
564  const label constructSize,
565  const labelListList&& subMap,
566  const labelListList&& constructMap,
567  const bool subHasFlip,
568  const bool constructHasFlip
569 )
570 :
571  constructSize_(constructSize),
572  subMap_(move(subMap)),
573  constructMap_(move(constructMap)),
574  subHasFlip_(subHasFlip),
575  constructHasFlip_(constructHasFlip),
576  schedulePtr_()
577 {}
578 
579 
581 (
582  const labelList& sendProcs,
583  const labelList& recvProcs
584 )
585 :
586  constructSize_(0),
587  subHasFlip_(false),
588  constructHasFlip_(false),
589  schedulePtr_()
590 {
591  if (sendProcs.size() != recvProcs.size())
592  {
594  << "The send and receive data is not the same length. sendProcs:"
595  << sendProcs.size() << " recvProcs:" << recvProcs.size()
596  << abort(FatalError);
597  }
598 
599  // Per processor the number of samples we have to send/receive.
600  labelList nSend(Pstream::nProcs(), 0);
601  labelList nRecv(Pstream::nProcs(), 0);
602 
603  forAll(sendProcs, sampleI)
604  {
605  label sendProc = sendProcs[sampleI];
606  label recvProc = recvProcs[sampleI];
607 
608  // Note that also need to include local communication (both
609  // RecvProc and sendProc on local processor)
610 
611  if (Pstream::myProcNo() == sendProc)
612  {
613  // I am the sender. Count destination processor.
614  nSend[recvProc]++;
615  }
616  if (Pstream::myProcNo() == recvProc)
617  {
618  // I am the receiver.
619  nRecv[sendProc]++;
620  }
621  }
622 
625  forAll(nSend, proci)
626  {
627  subMap_[proci].setSize(nSend[proci]);
628  constructMap_[proci].setSize(nRecv[proci]);
629  }
630  nSend = 0;
631  nRecv = 0;
632 
633  forAll(sendProcs, sampleI)
634  {
635  label sendProc = sendProcs[sampleI];
636  label recvProc = recvProcs[sampleI];
637 
638  if (Pstream::myProcNo() == sendProc)
639  {
640  // I am the sender. Store index I need to send.
641  subMap_[recvProc][nSend[recvProc]++] = sampleI;
642  }
643  if (Pstream::myProcNo() == recvProc)
644  {
645  // I am the receiver.
646  constructMap_[sendProc][nRecv[sendProc]++] = sampleI;
647  // Largest entry inside constructMap
648  constructSize_ = sampleI+1;
649  }
650  }
651 }
652 
653 
655 (
656  const globalIndex& globalNumbering,
657  labelList& elements,
658  List<Map<label>>& compactMap,
659  const int tag
660 )
661 :
662  constructSize_(0),
663  subHasFlip_(false),
664  constructHasFlip_(false),
665  schedulePtr_()
666 {
667  // Construct per processor compact addressing of the global elements
668  // needed. The ones from the local processor are not included since
669  // these are always all needed.
671  (
672  globalNumbering,
673  elements,
674  compactMap
675  );
676 
678  // forAll(compactMap, proci)
679  //{
680  // if (proci != Pstream::myProcNo())
681  // {
682  // Map<label>& globalMap = compactMap[proci];
683  //
684  // SortableList<label> sorted(move(globalMap.toc()));
685  //
686  // forAll(sorted, i)
687  // {
688  // Map<label>::iterator iter = globalMap.find(sorted[i]);
689  // iter() = i;
690  // }
691  // }
692  //}
693 
694 
695  // Exchange what I need with processor that supplies it. Renumber elements
696  // into compact numbering
697  labelList compactStart;
699  (
700  tag,
701  globalNumbering,
702  elements,
703  compactMap,
704  compactStart
705  );
706 
707  if (debug)
708  {
709  printLayout(Pout);
710  }
711 }
712 
713 
715 (
716  const globalIndex& globalNumbering,
717  labelListList& cellCells,
718  List<Map<label>>& compactMap,
719  const int tag
720 )
721 :
722  constructSize_(0),
723  subHasFlip_(false),
724  constructHasFlip_(false),
725  schedulePtr_()
726 {
727  // Construct per processor compact addressing of the global elements
728  // needed. The ones from the local processor are not included since
729  // these are always all needed.
731  (
732  globalNumbering,
733  cellCells,
734  compactMap
735  );
736 
738  // forAll(compactMap, proci)
739  //{
740  // if (proci != Pstream::myProcNo())
741  // {
742  // Map<label>& globalMap = compactMap[proci];
743  //
744  // SortableList<label> sorted(move(globalMap.toc()));
745  //
746  // forAll(sorted, i)
747  // {
748  // Map<label>::iterator iter = globalMap.find(sorted[i]);
749  // iter() = i;
750  // }
751  // }
752  //}
753 
754 
755  // Exchange what I need with processor that supplies it. Renumber elements
756  // into compact numbering
757  labelList compactStart;
759  (
760  tag,
761  globalNumbering,
762  cellCells,
763  compactMap,
764  compactStart
765  );
766 
767  if (debug)
768  {
769  printLayout(Pout);
770  }
771 }
772 
773 
775 :
777  subMap_(map.subMap_),
781  schedulePtr_()
782 {}
783 
784 
786 :
788  subMap_(move(map.subMap_)),
789  constructMap_(move(map.constructMap_)),
792  schedulePtr_()
793 {}
794 
795 
797 {
798  is >> *this;
799 }
800 
801 
802 // * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
803 
805 {
807  subMap_.transfer(rhs.subMap_);
809  subHasFlip_ = rhs.subHasFlip_;
811  schedulePtr_.clear();
812 }
813 
814 
816 (
817  const globalIndex& globalNumbering,
818  const List<Map<label>>& compactMap,
819  const label globalI
820 )
821 {
822  if (globalI == -1)
823  {
824  return globalI;
825  }
826  if (globalNumbering.isLocal(globalI))
827  {
828  return globalNumbering.toLocal(globalI);
829  }
830  else
831  {
832  label proci = globalNumbering.whichProcID(globalI);
833  label index = globalNumbering.toLocal(proci, globalI);
834  return compactMap[proci][index];
835  }
836 }
837 
838 
839 void Foam::mapDistributeBase::compact(const boolList& elemIsUsed, const int tag)
840 {
841  // 1. send back to sender. Have sender delete the corresponding element
842  // from the submap and do the same to the constructMap locally
843  // (and in same order).
844 
845  // Send elemIsUsed field to neighbour. Use nonblocking code from
846  // mapDistributeBase but in reverse order.
847  if (Pstream::parRun())
848  {
849  label startOfRequests = Pstream::nRequests();
850 
851  // Set up receives from neighbours
852 
853  List<boolList> recvFields(Pstream::nProcs());
854 
855  for (label domain = 0; domain < Pstream::nProcs(); domain++)
856  {
857  const labelList& map = subMap_[domain];
858 
859  if (domain != Pstream::myProcNo() && map.size())
860  {
861  recvFields[domain].setSize(map.size());
863  (
865  domain,
866  reinterpret_cast<char*>(recvFields[domain].begin()),
867  recvFields[domain].size()*sizeof(bool),
868  tag
869  );
870  }
871  }
872 
873 
874  List<boolList> sendFields(Pstream::nProcs());
875 
876  for (label domain = 0; domain < Pstream::nProcs(); domain++)
877  {
878  const labelList& map = constructMap_[domain];
879 
880  if (domain != Pstream::myProcNo() && map.size())
881  {
882  boolList& subField = sendFields[domain];
883  subField.setSize(map.size());
884  forAll(map, i)
885  {
886  subField[i] = accessAndFlip
887  (
888  elemIsUsed,
889  map[i],
891  noOp() // do not flip elemIsUsed value
892  );
893  }
894 
896  (
898  domain,
899  reinterpret_cast<const char*>(subField.begin()),
900  subField.size()*sizeof(bool),
901  tag
902  );
903  }
904  }
905 
906 
907 
908  // Set up 'send' to myself - write directly into recvFields
909 
910  {
911  const labelList& map = constructMap_[Pstream::myProcNo()];
912 
913  recvFields[Pstream::myProcNo()].setSize(map.size());
914  forAll(map, i)
915  {
916  recvFields[Pstream::myProcNo()][i] = accessAndFlip
917  (
918  elemIsUsed,
919  map[i],
921  noOp() // do not flip elemIsUsed value
922  );
923  }
924  }
925 
926 
927  // Wait for all to finish
928 
929  Pstream::waitRequests(startOfRequests);
930 
931 
932  // Compact out all submap entries that are referring to unused elements
933  for (label domain = 0; domain < Pstream::nProcs(); domain++)
934  {
935  const labelList& map = subMap_[domain];
936 
937  labelList newMap(map.size());
938  label newI = 0;
939 
940  forAll(map, i)
941  {
942  if (recvFields[domain][i])
943  {
944  // So element is used on destination side
945  newMap[newI++] = map[i];
946  }
947  }
948  if (newI < map.size())
949  {
950  newMap.setSize(newI);
951  subMap_[domain].transfer(newMap);
952  }
953  }
954  }
955 
956 
957  // 2. remove from construct map - since end-result (element in elemIsUsed)
958  // not used.
959 
960  label maxConstructIndex = -1;
961 
962  for (label domain = 0; domain < Pstream::nProcs(); domain++)
963  {
964  const labelList& map = constructMap_[domain];
965 
966  labelList newMap(map.size());
967  label newI = 0;
968 
969  forAll(map, i)
970  {
971  label destinationI = map[i];
972  if (constructHasFlip_)
973  {
974  destinationI = mag(destinationI)-1;
975  }
976 
977  // Is element is used on destination side
978  if (elemIsUsed[destinationI])
979  {
980  maxConstructIndex = max(maxConstructIndex, destinationI);
981 
982  newMap[newI++] = map[i];
983  }
984  }
985  if (newI < map.size())
986  {
987  newMap.setSize(newI);
988  constructMap_[domain].transfer(newMap);
989  }
990  }
991 
992  constructSize_ = maxConstructIndex+1;
993 
994  // Clear the schedule (note:not necessary if nothing changed)
995  schedulePtr_.clear();
996 }
997 
998 
1001  const boolList& elemIsUsed,
1002  const label localSize, // max index for subMap
1003  labelList& oldToNewSub,
1004  labelList& oldToNewConstruct,
1005  const int tag
1006 )
1007 {
1008  // 1. send back to sender. Have sender delete the corresponding element
1009  // from the submap and do the same to the constructMap locally
1010  // (and in same order).
1011 
1012  // Send elemIsUsed field to neighbour. Use nonblocking code from
1013  // mapDistributeBase but in reverse order.
1014  if (Pstream::parRun())
1015  {
1016  label startOfRequests = Pstream::nRequests();
1017 
1018  // Set up receives from neighbours
1019 
1020  List<boolList> recvFields(Pstream::nProcs());
1021 
1022  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1023  {
1024  const labelList& map = subMap_[domain];
1025 
1026  if (domain != Pstream::myProcNo() && map.size())
1027  {
1028  recvFields[domain].setSize(map.size());
1030  (
1032  domain,
1033  reinterpret_cast<char*>(recvFields[domain].begin()),
1034  recvFields[domain].size()*sizeof(bool),
1035  tag
1036  );
1037  }
1038  }
1039 
1040 
1041  List<boolList> sendFields(Pstream::nProcs());
1042 
1043  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1044  {
1045  const labelList& map = constructMap_[domain];
1046 
1047  if (domain != Pstream::myProcNo() && map.size())
1048  {
1049  boolList& subField = sendFields[domain];
1050  subField.setSize(map.size());
1051  forAll(map, i)
1052  {
1053  label index = map[i];
1054  if (constructHasFlip_)
1055  {
1056  index = mag(index)-1;
1057  }
1058  subField[i] = elemIsUsed[index];
1059  }
1060 
1062  (
1064  domain,
1065  reinterpret_cast<const char*>(subField.begin()),
1066  subField.size()*sizeof(bool),
1067  tag
1068  );
1069  }
1070  }
1071 
1072 
1073 
1074  // Set up 'send' to myself - write directly into recvFields
1075 
1076  {
1077  const labelList& map = constructMap_[Pstream::myProcNo()];
1078 
1079  recvFields[Pstream::myProcNo()].setSize(map.size());
1080  forAll(map, i)
1081  {
1082  label index = map[i];
1083  if (constructHasFlip_)
1084  {
1085  index = mag(index)-1;
1086  }
1087  recvFields[Pstream::myProcNo()][i] = elemIsUsed[index];
1088  }
1089  }
1090 
1091 
1092  // Wait for all to finish
1093 
1094  Pstream::waitRequests(startOfRequests);
1095 
1096 
1097 
1098 
1099  // Work out which elements on the sending side are needed
1100  {
1101  oldToNewSub.setSize(localSize, -1);
1102 
1103  boolList sendElemIsUsed(localSize, false);
1104 
1105  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1106  {
1107  const labelList& map = subMap_[domain];
1108  forAll(map, i)
1109  {
1110  if (recvFields[domain][i])
1111  {
1112  label index = map[i];
1113  if (subHasFlip_)
1114  {
1115  index = mag(index)-1;
1116  }
1117  sendElemIsUsed[index] = true;
1118  }
1119  }
1120  }
1121 
1122  label newI = 0;
1123  forAll(sendElemIsUsed, i)
1124  {
1125  if (sendElemIsUsed[i])
1126  {
1127  oldToNewSub[i] = newI++;
1128  }
1129  }
1130  }
1131 
1132 
1133  // Compact out all submap entries that are referring to unused elements
1134  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1135  {
1136  const labelList& map = subMap_[domain];
1137 
1138  labelList newMap(map.size());
1139  label newI = 0;
1140 
1141  forAll(map, i)
1142  {
1143  if (recvFields[domain][i])
1144  {
1145  // So element is used on destination side
1146  label index = map[i];
1147  label sign = 1;
1148  if (subHasFlip_)
1149  {
1150  if (index < 0)
1151  {
1152  sign = -1;
1153  }
1154  index = mag(index)-1;
1155  }
1156  label newIndex = oldToNewSub[index];
1157  if (subHasFlip_)
1158  {
1159  newIndex = sign*(newIndex+1);
1160  }
1161  newMap[newI++] = newIndex;
1162  }
1163  }
1164  newMap.setSize(newI);
1165  subMap_[domain].transfer(newMap);
1166  }
1167  }
1168 
1169 
1170  // 2. remove from construct map - since end-result (element in elemIsUsed)
1171  // not used.
1172 
1173 
1174  oldToNewConstruct.setSize(elemIsUsed.size(), -1);
1175  constructSize_ = 0;
1176  forAll(elemIsUsed, i)
1177  {
1178  if (elemIsUsed[i])
1179  {
1180  oldToNewConstruct[i] = constructSize_++;
1181  }
1182  }
1183 
1184  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1185  {
1186  const labelList& map = constructMap_[domain];
1187 
1188  labelList newMap(map.size());
1189  label newI = 0;
1190 
1191  forAll(map, i)
1192  {
1193  label destinationI = map[i];
1194  label sign = 1;
1195  if (constructHasFlip_)
1196  {
1197  if (destinationI < 0)
1198  {
1199  sign = -1;
1200  }
1201  destinationI = mag(destinationI)-1;
1202  }
1203 
1204  // Is element is used on destination side
1205  if (elemIsUsed[destinationI])
1206  {
1207  label newIndex = oldToNewConstruct[destinationI];
1208  if (constructHasFlip_)
1209  {
1210  newIndex = sign*(newIndex+1);
1211  }
1212  newMap[newI++] = newIndex;
1213  }
1214  }
1215  newMap.setSize(newI);
1216  constructMap_[domain].transfer(newMap);
1217  }
1218 }
1219 
1220 
1221 // * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
1222 
1224 {
1225  // Check for assignment to self
1226  if (this == &rhs)
1227  {
1229  << "Attempted assignment to self"
1230  << abort(FatalError);
1231  }
1233  subMap_ = rhs.subMap_;
1235  subHasFlip_ = rhs.subHasFlip_;
1237  schedulePtr_.clear();
1238 }
1239 
1240 
1241 // * * * * * * * * * * * * * * Istream Operator * * * * * * * * * * * * * * //
1242 
1244 {
1245  is.fatalCheck("operator>>(Istream&, mapDistributeBase&)");
1246 
1247  is >> map.constructSize_ >> map.subMap_ >> map.constructMap_
1248  >> map.subHasFlip_ >> map.constructHasFlip_;
1249 
1250  return is;
1251 }
1252 
1253 
1254 // * * * * * * * * * * * * * * Ostream Operator * * * * * * * * * * * * * * //
1255 
1257 {
1258  os << map.constructSize_ << token::NL
1259  << map.subMap_ << token::NL
1260  << map.constructMap_ << token::NL
1261  << map.subHasFlip_ << token::SPACE << map.constructHasFlip_
1262  << token::NL;
1263 
1264  return os;
1265 }
1266 
1267 
1268 // ************************************************************************* //
ListType renumber(const labelUList &oldToNew, const ListType &)
Renumber the values (not the indices) of a list.
bool isLocal(const label i) const
Is on local processor.
Definition: globalIndexI.H:95
dimensionedScalar sign(const dimensionedScalar &ds)
A HashTable with keys but without contents.
Definition: HashSet.H:59
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:434
void compact(const boolList &elemIsUsed, const int tag=UPstream::msgType())
Compact maps. Gets per field a bool whether it is used (locally)
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
static int masterNo()
Process index of the master.
Definition: UPstream.H:417
void calcCompactAddressing(const globalIndex &globalNumbering, const labelList &elements, List< Map< label >> &compactMap) const
Construct per processor compact addressing of the global elements.
static label renumber(const globalIndex &, const List< Map< label >> &compactMap, const label globalElement)
Helper for construct from globalIndex. Renumbers element.
error FatalError
dimensioned< Type > max(const dimensioned< Type > &, const dimensioned< Type > &)
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
Definition: error.H:319
A 1D array of objects of type <T>, where the size of the vector is known and used for subscript bound...
Definition: HashTable.H:59
static int firstSlave()
Process index of first slave.
Definition: UPstream.H:446
const List< labelPair > & schedule() const
Return a schedule. Demand driven. See above.
#define forAllIter(Container, container, iter)
Iterate across all elements in the container object of type.
Definition: UList.H:459
An Istream is an abstract base class for all input systems (streams, files, token lists etc)...
Definition: Istream.H:57
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:164
labelListList subMap_
Maps from subsetted data back to original data.
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:429
Ostream & endl(Ostream &os)
Add newline and flush stream.
Definition: Ostream.H:251
static bool master(const label communicator=0)
Am I the master process.
Definition: UPstream.H:423
labelList identity(const label len)
Create identity map (map[i] == i) of given length.
Definition: ListOps.C:104
static label nRequests()
Get number of outstanding requests.
Definition: UPstream.C:137
static int & msgType()
Message tag of standard messages.
Definition: UPstream.H:476
static label worldComm
Default communicator (all processors)
Definition: UPstream.H:278
bool insert(const Key &key)
Insert a new entry.
Definition: HashSet.H:111
mapDistributeBase()
Construct null.
label constructSize_
Size of reconstructed data.
Various functions to operate on Lists.
Input inter-processor communications stream.
Definition: IPstream.H:50
Calculates a unique integer (label so might not have enough room - 2G max) for processor + local inde...
Definition: globalIndex.H:63
autoPtr< List< labelPair > > schedulePtr_
Schedule.
Istream & operator>>(Istream &, directionInfo &)
void printLayout(Ostream &os) const
Debug: print layout. Can only be used on maps with sorted.
static T accessAndFlip(const UList< T > &fld, const label index, const bool hasFlip, const negateOp &negOp)
iterator begin()
Return an iterator to begin traversing the UList.
Definition: UListI.H:216
Pair< label > labelPair
Label pair.
Definition: labelPair.H:48
label toLocal(const label i) const
From global to local on current processor.
Definition: globalIndexI.H:117
static const label labelMax
Definition: label.H:62
label whichProcID(const label i) const
Which processor does global come from? Binary search.
Definition: globalIndexI.H:123
errorManip< error > abort(error &err)
Definition: errorManip.H:131
labelListList constructMap_
Maps from subsetted data to new reconstructed data.
void fatalCheck(const char *operation) const
Check IOstream status for given operation.
Definition: IOstream.C:105
void operator=(const mapDistributeBase &)
Class containing processor-to-processor mapping information.
An Ostream is an abstract base class for all output systems (streams, files, token lists...
Definition: Ostream.H:54
static label read(const commsTypes commsType, const int fromProcNo, char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Read into given buffer from given processor and return the.
Definition: UIPread.C:79
Determines the order in which a set of processors should communicate with one another.
Definition: commSchedule.H:65
defineTypeNameAndDebug(combustionModel, 0)
static bool write(const commsTypes commsType, const int toProcNo, const char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Write given buffer to given processor.
Definition: UOPwrite.C:34
Output inter-processor communications stream.
Definition: OPstream.H:50
label findIndex(const ListType &, typename ListType::const_reference, const label start=0)
Find first occurrence of given element and return index,.
static void waitRequests(const label start=0)
Wait until all requests (from start onwards) have finished.
Definition: UPstream.C:147
dimensioned< Type > min(const dimensioned< Type > &, const dimensioned< Type > &)
bool subHasFlip_
Whether subMap includes flip or not.
void setSize(const label)
Reset size of List.
Definition: List.C:281
static bool & parRun()
Is this a parallel run?
Definition: UPstream.H:399
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:411
void exchangeAddressing(const int tag, const globalIndex &globalNumbering, labelList &elements, List< Map< label >> &compactMap, labelList &compactStart)
bool constructHasFlip_
Whether constructMap includes flip or not.
prefixOSstream Pout(cout, "Pout")
Definition: IOstreams.H:53
A List with indirect addressing.
Definition: fvMatrix.H:106
Ostream & operator<<(Ostream &, const ensightPart &)
static void checkReceivedSize(const label proci, const label expectedSize, const label receivedSize)
dimensioned< scalar > mag(const dimensioned< Type > &)
void transfer(mapDistributeBase &)
Transfer the contents of the argument and annul the argument.
List< Key > toc() const
Return the table of contents.
Definition: HashTable.C:202
void transfer(List< T > &)
Transfer the contents of the argument List into this list.
Definition: List.C:342
static const label labelMin
Definition: label.H:61
Namespace for OpenFOAM.
static int lastSlave(const label communicator=0)
Process index of last slave.
Definition: UPstream.H:452
label localSize() const
My local size.
Definition: globalIndexI.H:60