distributionMapBase.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration | Website: https://openfoam.org
5  \\ / A nd | Copyright (C) 2015-2023 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "distributionMapBase.H"
27 #include "commSchedule.H"
28 #include "HashSet.H"
29 #include "globalIndex.H"
30 #include "ListOps.H"
31 
32 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
33 
34 namespace Foam
35 {
37 }
38 
39 
40 // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
41 
43 (
44  const labelListList& subMap,
45  const labelListList& constructMap,
46  const int tag
47 )
48 {
49  // Communications: send and receive processor
50  List<labelPair> allComms;
51 
52  {
54 
55  // Find what communication is required
56  forAll(subMap, proci)
57  {
58  if (proci != Pstream::myProcNo())
59  {
60  if (subMap[proci].size())
61  {
62  // I need to send to proci
63  commsSet.insert(labelPair(Pstream::myProcNo(), proci));
64  }
65  if (constructMap[proci].size())
66  {
67  // I need to receive from proci
68  commsSet.insert(labelPair(proci, Pstream::myProcNo()));
69  }
70  }
71  }
72  allComms = commsSet.toc();
73  }
74 
75 
76  // Reduce
77  if (Pstream::master())
78  {
79  // Receive and merge
80  for
81  (
82  int slave=Pstream::firstSlave();
83  slave<=Pstream::lastSlave();
84  slave++
85  )
86  {
87  IPstream fromSlave(Pstream::commsTypes::scheduled, slave, 0, tag);
88  List<labelPair> nbrData(fromSlave);
89 
90  forAll(nbrData, i)
91  {
92  if (findIndex(allComms, nbrData[i]) == -1)
93  {
94  label sz = allComms.size();
95  allComms.setSize(sz+1);
96  allComms[sz] = nbrData[i];
97  }
98  }
99  }
100  // Send back
101  for
102  (
103  int slave=Pstream::firstSlave();
104  slave<=Pstream::lastSlave();
105  slave++
106  )
107  {
108  OPstream toSlave(Pstream::commsTypes::scheduled, slave, 0, tag);
109  toSlave << allComms;
110  }
111  }
112  else
113  {
114  {
115  OPstream toMaster
116  (
119  0,
120  tag
121  );
122  toMaster << allComms;
123  }
124  {
125  IPstream fromMaster
126  (
129  0,
130  tag
131  );
132  fromMaster >> allComms;
133  }
134  }
135 
136 
137  // Determine my schedule.
138  labelList mySchedule
139  (
141  (
142  Pstream::nProcs(),
143  allComms
144  ).procSchedule()[Pstream::myProcNo()]
145  );
146 
147  // Processors involved in my schedule
148  return List<labelPair>(UIndirectList<labelPair>(allComms, mySchedule));
149 
150 
151  // if (debug)
152  //{
153  // Pout<< "I need to:" << endl;
154  // const List<labelPair>& comms = schedule();
155  // forAll(comms, i)
156  // {
157  // const labelPair& twoProcs = comms[i];
158  // label sendProc = twoProcs[0];
159  // label recvProc = twoProcs[1];
160  //
161  // if (recvProc == Pstream::myProcNo())
162  // {
163  // Pout<< " receive from " << sendProc << endl;
164  // }
165  // else
166  // {
167  // Pout<< " send to " << recvProc << endl;
168  // }
169  // }
170  //}
171 }
172 
173 
175 {
176  if (schedulePtr_.empty())
177  {
178  schedulePtr_.reset
179  (
180  new List<labelPair>
181  (
182  schedule(subMap_, constructMap_, Pstream::msgType())
183  )
184  );
185  }
186  return schedulePtr_();
187 }
188 
189 
191 (
192  const label proci,
193  const label expectedSize,
194  const label receivedSize
195 )
196 {
197  if (receivedSize != expectedSize)
198  {
200  << "Expected from processor " << proci
201  << " " << expectedSize << " but received "
202  << receivedSize << " elements."
203  << abort(FatalError);
204  }
205 }
206 
207 
209 {
210  // Determine offsets of remote data.
211  labelList minIndex(Pstream::nProcs(), labelMax);
212  labelList maxIndex(Pstream::nProcs(), labelMin);
213  forAll(constructMap_, proci)
214  {
215  const labelList& construct = constructMap_[proci];
216  if (constructHasFlip_)
217  {
218  forAll(construct, i)
219  {
220  label index = mag(construct[i])-1;
221  minIndex[proci] = min(minIndex[proci], index);
222  maxIndex[proci] = max(maxIndex[proci], index);
223  }
224  }
225  else
226  {
227  forAll(construct, i)
228  {
229  label index = construct[i];
230  minIndex[proci] = min(minIndex[proci], index);
231  maxIndex[proci] = max(maxIndex[proci], index);
232  }
233  }
234  }
235 
236  label localSize;
237  if (maxIndex[Pstream::myProcNo()] == labelMin)
238  {
239  localSize = 0;
240  }
241  else
242  {
243  localSize = maxIndex[Pstream::myProcNo()]+1;
244  }
245 
246  os << "Layout: (constructSize:" << constructSize_
247  << " subHasFlip:" << subHasFlip_
248  << " constructHasFlip:" << constructHasFlip_
249  << ")" << endl
250  << "local (processor " << Pstream::myProcNo() << "):" << endl
251  << " start : 0" << endl
252  << " size : " << localSize << endl;
253 
254  label offset = localSize;
255  forAll(minIndex, proci)
256  {
257  if (proci != Pstream::myProcNo())
258  {
259  if (constructMap_[proci].size() > 0)
260  {
261  if (minIndex[proci] != offset)
262  {
264  << "offset:" << offset
265  << " proci:" << proci
266  << " minIndex:" << minIndex[proci]
267  << abort(FatalError);
268  }
269 
270  label size = maxIndex[proci]-minIndex[proci]+1;
271  os << "processor " << proci << ':' << endl
272  << " start : " << offset << endl
273  << " size : " << size << endl;
274 
275  offset += size;
276  }
277  }
278  }
279 }
280 
281 
283 (
284  const globalIndex& globalNumbering,
285  const labelList& elements,
286  List<Map<label>>& compactMap
287 ) const
288 {
289  compactMap.setSize(Pstream::nProcs());
290 
291  // Count all (non-local) elements needed. Just for presizing map.
292  labelList nNonLocal(Pstream::nProcs(), 0);
293 
294  forAll(elements, i)
295  {
296  label globalIndex = elements[i];
297 
298  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
299  {
300  label proci = globalNumbering.whichProcID(globalIndex);
301  nNonLocal[proci]++;
302  }
303  }
304 
305  forAll(compactMap, proci)
306  {
307  compactMap[proci].clear();
308  if (proci != Pstream::myProcNo())
309  {
310  compactMap[proci].resize(2*nNonLocal[proci]);
311  }
312  }
313 
314 
315  // Collect all (non-local) elements needed.
316  forAll(elements, i)
317  {
318  label globalIndex = elements[i];
319 
320  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
321  {
322  label proci = globalNumbering.whichProcID(globalIndex);
323  label index = globalNumbering.toLocal(proci, globalIndex);
324  label nCompact = compactMap[proci].size();
325  compactMap[proci].insert(index, nCompact);
326  }
327  }
328 }
329 
330 
332 (
333  const globalIndex& globalNumbering,
334  const labelListList& cellCells,
335  List<Map<label>>& compactMap
336 ) const
337 {
338  compactMap.setSize(Pstream::nProcs());
339 
340  // Count all (non-local) elements needed. Just for presizing map.
341  labelList nNonLocal(Pstream::nProcs(), 0);
342 
343  forAll(cellCells, cellI)
344  {
345  const labelList& cCells = cellCells[cellI];
346 
347  forAll(cCells, i)
348  {
349  label globalIndex = cCells[i];
350 
351  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
352  {
353  label proci = globalNumbering.whichProcID(globalIndex);
354  nNonLocal[proci]++;
355  }
356  }
357  }
358 
359  forAll(compactMap, proci)
360  {
361  compactMap[proci].clear();
362  if (proci != Pstream::myProcNo())
363  {
364  compactMap[proci].resize(2*nNonLocal[proci]);
365  }
366  }
367 
368 
369  // Collect all (non-local) elements needed.
370  forAll(cellCells, cellI)
371  {
372  const labelList& cCells = cellCells[cellI];
373 
374  forAll(cCells, i)
375  {
376  label globalIndex = cCells[i];
377 
378  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
379  {
380  label proci = globalNumbering.whichProcID(globalIndex);
381  label index = globalNumbering.toLocal(proci, globalIndex);
382  label nCompact = compactMap[proci].size();
383  compactMap[proci].insert(index, nCompact);
384  }
385  }
386  }
387 }
388 
389 
391 (
392  const int tag,
393  const globalIndex& globalNumbering,
394  labelList& elements,
395  List<Map<label>>& compactMap,
396  labelList& compactStart
397 )
398 {
399  // The overall compact addressing is
400  // - myProcNo data first (uncompacted)
401  // - all other processors consecutively
402 
403  compactStart.setSize(Pstream::nProcs());
404  compactStart[Pstream::myProcNo()] = 0;
405  constructSize_ = globalNumbering.localSize();
406  forAll(compactStart, proci)
407  {
408  if (proci != Pstream::myProcNo())
409  {
410  compactStart[proci] = constructSize_;
411  constructSize_ += compactMap[proci].size();
412  }
413  }
414 
415 
416 
417  // Find out what to receive/send in compact addressing.
418 
419  // What I want to receive is what others have to send
420  labelListList wantedRemoteElements(Pstream::nProcs());
421  // Compact addressing for received data
422  constructMap_.setSize(Pstream::nProcs());
423  forAll(compactMap, proci)
424  {
425  if (proci == Pstream::myProcNo())
426  {
427  // All my own elements are used
428  label nLocal = globalNumbering.localSize();
429  wantedRemoteElements[proci] = identityMap(nLocal);
430  constructMap_[proci] = identityMap(nLocal);
431  }
432  else
433  {
434  // Remote elements wanted from processor proci
435  labelList& remoteElem = wantedRemoteElements[proci];
436  labelList& localElem = constructMap_[proci];
437  remoteElem.setSize(compactMap[proci].size());
438  localElem.setSize(compactMap[proci].size());
439  label i = 0;
440  forAllIter(Map<label>, compactMap[proci], iter)
441  {
442  const label compactI = compactStart[proci] + iter();
443  remoteElem[i] = iter.key();
444  localElem[i] = compactI;
445  iter() = compactI;
446  i++;
447  }
448  }
449  }
450 
451  subMap_.setSize(Pstream::nProcs());
452  Pstream::exchange<labelList, label>
453  (
454  wantedRemoteElements,
455  subMap_,
456  tag,
457  Pstream::worldComm // TBD
458  );
459 
460  // Renumber elements
461  forAll(elements, i)
462  {
463  elements[i] = renumber(globalNumbering, compactMap, elements[i]);
464  }
465 }
466 
467 
469 (
470  const int tag,
471  const globalIndex& globalNumbering,
472  labelListList& cellCells,
473  List<Map<label>>& compactMap,
474  labelList& compactStart
475 )
476 {
477  // The overall compact addressing is
478  // - myProcNo data first (uncompacted)
479  // - all other processors consecutively
480 
481  compactStart.setSize(Pstream::nProcs());
482  compactStart[Pstream::myProcNo()] = 0;
483  constructSize_ = globalNumbering.localSize();
484  forAll(compactStart, proci)
485  {
486  if (proci != Pstream::myProcNo())
487  {
488  compactStart[proci] = constructSize_;
489  constructSize_ += compactMap[proci].size();
490  }
491  }
492 
493 
494 
495  // Find out what to receive/send in compact addressing.
496 
497  // What I want to receive is what others have to send
498  labelListList wantedRemoteElements(Pstream::nProcs());
499  // Compact addressing for received data
500  constructMap_.setSize(Pstream::nProcs());
501  forAll(compactMap, proci)
502  {
503  if (proci == Pstream::myProcNo())
504  {
505  // All my own elements are used
506  label nLocal = globalNumbering.localSize();
507  wantedRemoteElements[proci] = identityMap(nLocal);
508  constructMap_[proci] = identityMap(nLocal);
509  }
510  else
511  {
512  // Remote elements wanted from processor proci
513  labelList& remoteElem = wantedRemoteElements[proci];
514  labelList& localElem = constructMap_[proci];
515  remoteElem.setSize(compactMap[proci].size());
516  localElem.setSize(compactMap[proci].size());
517  label i = 0;
518  forAllIter(Map<label>, compactMap[proci], iter)
519  {
520  const label compactI = compactStart[proci] + iter();
521  remoteElem[i] = iter.key();
522  localElem[i] = compactI;
523  iter() = compactI;
524  i++;
525  }
526  }
527  }
528 
529  subMap_.setSize(Pstream::nProcs());
530  Pstream::exchange<labelList, label>
531  (
532  wantedRemoteElements,
533  subMap_,
534  tag,
535  Pstream::worldComm // TBD
536  );
537 
538  // Renumber elements
539  forAll(cellCells, cellI)
540  {
541  labelList& cCells = cellCells[cellI];
542 
543  forAll(cCells, i)
544  {
545  cCells[i] = renumber(globalNumbering, compactMap, cCells[i]);
546  }
547  }
548 }
549 
550 
551 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
552 
554 :
555  constructSize_(0),
556  subHasFlip_(false),
557  constructHasFlip_(false),
558  schedulePtr_()
559 {}
560 
561 
563 (
564  const label constructSize,
565  const labelListList&& subMap,
566  const labelListList&& constructMap,
567  const bool subHasFlip,
568  const bool constructHasFlip
569 )
570 :
571  constructSize_(constructSize),
572  subMap_(move(subMap)),
573  constructMap_(move(constructMap)),
574  subHasFlip_(subHasFlip),
575  constructHasFlip_(constructHasFlip),
576  schedulePtr_()
577 {}
578 
579 
581 (
582  const labelList& sendProcs,
583  const labelList& recvProcs
584 )
585 :
586  constructSize_(0),
587  subHasFlip_(false),
588  constructHasFlip_(false),
589  schedulePtr_()
590 {
591  if (sendProcs.size() != recvProcs.size())
592  {
594  << "The send and receive data is not the same length. sendProcs:"
595  << sendProcs.size() << " recvProcs:" << recvProcs.size()
596  << abort(FatalError);
597  }
598 
599  // Per processor the number of samples we have to send/receive.
600  labelList nSend(Pstream::nProcs(), 0);
601  labelList nRecv(Pstream::nProcs(), 0);
602 
603  forAll(sendProcs, sampleI)
604  {
605  label sendProc = sendProcs[sampleI];
606  label recvProc = recvProcs[sampleI];
607 
608  // Note that also need to include local communication (both
609  // RecvProc and sendProc on local processor)
610 
611  if (Pstream::myProcNo() == sendProc)
612  {
613  // I am the sender. Count destination processor.
614  nSend[recvProc]++;
615  }
616  if (Pstream::myProcNo() == recvProc)
617  {
618  // I am the receiver.
619  nRecv[sendProc]++;
620  }
621  }
622 
625  forAll(nSend, proci)
626  {
627  subMap_[proci].setSize(nSend[proci]);
628  constructMap_[proci].setSize(nRecv[proci]);
629  }
630  nSend = 0;
631  nRecv = 0;
632 
633  forAll(sendProcs, sampleI)
634  {
635  label sendProc = sendProcs[sampleI];
636  label recvProc = recvProcs[sampleI];
637 
638  if (Pstream::myProcNo() == sendProc)
639  {
640  // I am the sender. Store index I need to send.
641  subMap_[recvProc][nSend[recvProc]++] = sampleI;
642  }
643  if (Pstream::myProcNo() == recvProc)
644  {
645  // I am the receiver.
646  constructMap_[sendProc][nRecv[sendProc]++] = sampleI;
647  // Largest entry inside constructMap
648  constructSize_ = sampleI+1;
649  }
650  }
651 }
652 
653 
655 (
656  const globalIndex& globalNumbering,
657  labelList& elements,
658  List<Map<label>>& compactMap,
659  const int tag
660 )
661 :
662  constructSize_(0),
663  subHasFlip_(false),
664  constructHasFlip_(false),
665  schedulePtr_()
666 {
667  // Construct per processor compact addressing of the global elements
668  // needed. The ones from the local processor are not included since
669  // these are always all needed.
671  (
672  globalNumbering,
673  elements,
674  compactMap
675  );
676 
678  // forAll(compactMap, proci)
679  //{
680  // if (proci != Pstream::myProcNo())
681  // {
682  // Map<label>& globalMap = compactMap[proci];
683  //
684  // SortableList<label> sorted(move(globalMap.toc()));
685  //
686  // forAll(sorted, i)
687  // {
688  // Map<label>::iterator iter = globalMap.find(sorted[i]);
689  // iter() = i;
690  // }
691  // }
692  //}
693 
694 
695  // Exchange what I need with processor that supplies it. Renumber elements
696  // into compact numbering
697  labelList compactStart;
699  (
700  tag,
701  globalNumbering,
702  elements,
703  compactMap,
704  compactStart
705  );
706 
707  if (debug)
708  {
709  printLayout(Pout);
710  }
711 }
712 
713 
715 (
716  const globalIndex& globalNumbering,
717  labelListList& cellCells,
718  List<Map<label>>& compactMap,
719  const int tag
720 )
721 :
722  constructSize_(0),
723  subHasFlip_(false),
724  constructHasFlip_(false),
725  schedulePtr_()
726 {
727  // Construct per processor compact addressing of the global elements
728  // needed. The ones from the local processor are not included since
729  // these are always all needed.
731  (
732  globalNumbering,
733  cellCells,
734  compactMap
735  );
736 
738  // forAll(compactMap, proci)
739  //{
740  // if (proci != Pstream::myProcNo())
741  // {
742  // Map<label>& globalMap = compactMap[proci];
743  //
744  // SortableList<label> sorted(move(globalMap.toc()));
745  //
746  // forAll(sorted, i)
747  // {
748  // Map<label>::iterator iter = globalMap.find(sorted[i]);
749  // iter() = i;
750  // }
751  // }
752  //}
753 
754 
755  // Exchange what I need with processor that supplies it. Renumber elements
756  // into compact numbering
757  labelList compactStart;
759  (
760  tag,
761  globalNumbering,
762  cellCells,
763  compactMap,
764  compactStart
765  );
766 
767  if (debug)
768  {
769  printLayout(Pout);
770  }
771 }
772 
773 
775 :
776  constructSize_(map.constructSize_),
777  subMap_(map.subMap_),
778  constructMap_(map.constructMap_),
779  subHasFlip_(map.subHasFlip_),
780  constructHasFlip_(map.constructHasFlip_),
781  schedulePtr_()
782 {}
783 
784 
786 :
787  constructSize_(map.constructSize_),
788  subMap_(move(map.subMap_)),
789  constructMap_(move(map.constructMap_)),
790  subHasFlip_(map.subHasFlip_),
791  constructHasFlip_(map.constructHasFlip_),
792  schedulePtr_()
793 {}
794 
795 
797 {
798  is >> *this;
799 }
800 
801 
802 // * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
803 
805 {
806  constructSize_ = rhs.constructSize_;
807  subMap_.transfer(rhs.subMap_);
808  constructMap_.transfer(rhs.constructMap_);
809  subHasFlip_ = rhs.subHasFlip_;
810  constructHasFlip_ = rhs.constructHasFlip_;
811  schedulePtr_.clear();
812 }
813 
814 
816 (
817  const globalIndex& globalNumbering,
818  const List<Map<label>>& compactMap,
819  const label globalI
820 )
821 {
822  if (globalI == -1)
823  {
824  return globalI;
825  }
826  if (globalNumbering.isLocal(globalI))
827  {
828  return globalNumbering.toLocal(globalI);
829  }
830  else
831  {
832  label proci = globalNumbering.whichProcID(globalI);
833  label index = globalNumbering.toLocal(proci, globalI);
834  return compactMap[proci][index];
835  }
836 }
837 
838 
840 (
841  const boolList& elemIsUsed,
842  const int tag
843 )
844 {
845  // 1. send back to sender. Have sender delete the corresponding element
846  // from the submap and do the same to the constructMap locally
847  // (and in same order).
848 
849  // Send elemIsUsed field to neighbours
850  List<boolList> recvFields(Pstream::nProcs());
851 
852  // "Send" to myself (i.e., write directly into recvFields)
853  {
854  const labelList& map = constructMap_[Pstream::myProcNo()];
855 
856  recvFields[Pstream::myProcNo()].setSize(map.size());
857  forAll(map, i)
858  {
859  recvFields[Pstream::myProcNo()][i] = accessAndFlip
860  (
861  elemIsUsed,
862  map[i],
863  constructHasFlip_,
864  noOp() // do not flip elemIsUsed value
865  );
866  }
867  }
868 
869  // Send to others. Use nonblocking code from distributionMapBase but in
870  // reverse order.
871  if (Pstream::parRun())
872  {
873  label startOfRequests = Pstream::nRequests();
874 
875  for (label domain = 0; domain < Pstream::nProcs(); domain++)
876  {
877  const labelList& map = subMap_[domain];
878 
879  if (domain != Pstream::myProcNo() && map.size())
880  {
881  recvFields[domain].setSize(map.size());
883  (
885  domain,
886  reinterpret_cast<char*>(recvFields[domain].begin()),
887  recvFields[domain].size()*sizeof(bool),
888  tag
889  );
890  }
891  }
892 
893  List<boolList> sendFields(Pstream::nProcs());
894 
895  for (label domain = 0; domain < Pstream::nProcs(); domain++)
896  {
897  const labelList& map = constructMap_[domain];
898 
899  if (domain != Pstream::myProcNo() && map.size())
900  {
901  boolList& subField = sendFields[domain];
902  subField.setSize(map.size());
903  forAll(map, i)
904  {
905  subField[i] = accessAndFlip
906  (
907  elemIsUsed,
908  map[i],
909  constructHasFlip_,
910  noOp() // do not flip elemIsUsed value
911  );
912  }
913 
915  (
917  domain,
918  reinterpret_cast<const char*>(subField.begin()),
919  subField.size()*sizeof(bool),
920  tag
921  );
922  }
923  }
924 
925  Pstream::waitRequests(startOfRequests);
926  }
927 
928  // Compact out all submap entries that are referring to unused elements
929  for (label domain = 0; domain < Pstream::nProcs(); domain++)
930  {
931  const labelList& map = subMap_[domain];
932 
933  labelList newMap(map.size());
934  label newI = 0;
935 
936  forAll(map, i)
937  {
938  if (recvFields[domain][i])
939  {
940  // So element is used on destination side
941  newMap[newI++] = map[i];
942  }
943  }
944  if (newI < map.size())
945  {
946  newMap.setSize(newI);
947  subMap_[domain].transfer(newMap);
948  }
949  }
950 
951  // 2. remove from construct map - since end-result (element in elemIsUsed)
952  // not used.
953  label maxConstructIndex = -1;
954  for (label domain = 0; domain < Pstream::nProcs(); domain++)
955  {
956  const labelList& map = constructMap_[domain];
957 
958  labelList newMap(map.size());
959  label newI = 0;
960 
961  forAll(map, i)
962  {
963  label destinationI = map[i];
964  if (constructHasFlip_)
965  {
966  destinationI = mag(destinationI)-1;
967  }
968 
969  // Is element is used on destination side
970  if (elemIsUsed[destinationI])
971  {
972  maxConstructIndex = max(maxConstructIndex, destinationI);
973 
974  newMap[newI++] = map[i];
975  }
976  }
977  if (newI < map.size())
978  {
979  newMap.setSize(newI);
980  constructMap_[domain].transfer(newMap);
981  }
982  }
983 
984  constructSize_ = maxConstructIndex+1;
985 
986  // Clear the schedule (note:not necessary if nothing changed)
987  schedulePtr_.clear();
988 }
989 
990 
992 (
993  const boolList& elemIsUsed,
994  const label localSize, // max index for subMap
995  labelList& oldToNewSub,
996  labelList& oldToNewConstruct,
997  const int tag
998 )
999 {
1000  // 1. send back to sender. Have sender delete the corresponding element
1001  // from the submap and do the same to the constructMap locally
1002  // (and in same order).
1003 
1004  // Send elemIsUsed field to neighbours
1005  List<boolList> recvFields(Pstream::nProcs());
1006 
1007  // "Send" to myself (i.e., write directly into recvFields)
1008  {
1009  const labelList& map = constructMap_[Pstream::myProcNo()];
1010 
1011  recvFields[Pstream::myProcNo()].setSize(map.size());
1012  forAll(map, i)
1013  {
1014  recvFields[Pstream::myProcNo()][i] = accessAndFlip
1015  (
1016  elemIsUsed,
1017  map[i],
1018  constructHasFlip_,
1019  noOp() // do not flip elemIsUsed value
1020  );
1021  }
1022  }
1023 
1024  // Send to others. Use nonblocking code from distributionMapBase but in
1025  // reverse order.
1026  if (Pstream::parRun())
1027  {
1028  label startOfRequests = Pstream::nRequests();
1029 
1030  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1031  {
1032  const labelList& map = subMap_[domain];
1033 
1034  if (domain != Pstream::myProcNo() && map.size())
1035  {
1036  recvFields[domain].setSize(map.size());
1038  (
1040  domain,
1041  reinterpret_cast<char*>(recvFields[domain].begin()),
1042  recvFields[domain].size()*sizeof(bool),
1043  tag
1044  );
1045  }
1046  }
1047 
1048  List<boolList> sendFields(Pstream::nProcs());
1049 
1050  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1051  {
1052  const labelList& map = constructMap_[domain];
1053 
1054  if (domain != Pstream::myProcNo() && map.size())
1055  {
1056  boolList& subField = sendFields[domain];
1057  subField.setSize(map.size());
1058  forAll(map, i)
1059  {
1060  subField[i] = accessAndFlip
1061  (
1062  elemIsUsed,
1063  map[i],
1064  constructHasFlip_,
1065  noOp() // do not flip elemIsUsed value
1066  );
1067  }
1068 
1070  (
1072  domain,
1073  reinterpret_cast<const char*>(subField.begin()),
1074  subField.size()*sizeof(bool),
1075  tag
1076  );
1077  }
1078  }
1079 
1080  Pstream::waitRequests(startOfRequests);
1081  }
1082 
1083  // Work out which elements on the sending side are needed
1084  {
1085  oldToNewSub.setSize(localSize, -1);
1086 
1087  boolList sendElemIsUsed(localSize, false);
1088 
1089  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1090  {
1091  const labelList& map = subMap_[domain];
1092  forAll(map, i)
1093  {
1094  if (recvFields[domain][i])
1095  {
1096  label index = map[i];
1097  if (subHasFlip_)
1098  {
1099  index = mag(index)-1;
1100  }
1101  sendElemIsUsed[index] = true;
1102  }
1103  }
1104  }
1105 
1106  label newI = 0;
1107  forAll(sendElemIsUsed, i)
1108  {
1109  if (sendElemIsUsed[i])
1110  {
1111  oldToNewSub[i] = newI++;
1112  }
1113  }
1114  }
1115 
1116  // Compact out all submap entries that are referring to unused elements
1117  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1118  {
1119  const labelList& map = subMap_[domain];
1120 
1121  labelList newMap(map.size());
1122  label newI = 0;
1123 
1124  forAll(map, i)
1125  {
1126  if (recvFields[domain][i])
1127  {
1128  // So element is used on destination side
1129  label index = map[i];
1130  label sign = 1;
1131  if (subHasFlip_)
1132  {
1133  if (index < 0)
1134  {
1135  sign = -1;
1136  }
1137  index = mag(index)-1;
1138  }
1139  label newIndex = oldToNewSub[index];
1140  if (subHasFlip_)
1141  {
1142  newIndex = sign*(newIndex+1);
1143  }
1144  newMap[newI++] = newIndex;
1145  }
1146  }
1147  newMap.setSize(newI);
1148  subMap_[domain].transfer(newMap);
1149  }
1150 
1151  // 2. remove from construct map - since end-result (element in elemIsUsed)
1152  // not used.
1153  oldToNewConstruct.setSize(elemIsUsed.size(), -1);
1154  constructSize_ = 0;
1155  forAll(elemIsUsed, i)
1156  {
1157  if (elemIsUsed[i])
1158  {
1159  oldToNewConstruct[i] = constructSize_++;
1160  }
1161  }
1162 
1163  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1164  {
1165  const labelList& map = constructMap_[domain];
1166 
1167  labelList newMap(map.size());
1168  label newI = 0;
1169 
1170  forAll(map, i)
1171  {
1172  label destinationI = map[i];
1173  label sign = 1;
1174  if (constructHasFlip_)
1175  {
1176  if (destinationI < 0)
1177  {
1178  sign = -1;
1179  }
1180  destinationI = mag(destinationI)-1;
1181  }
1182 
1183  // Is element is used on destination side
1184  if (elemIsUsed[destinationI])
1185  {
1186  label newIndex = oldToNewConstruct[destinationI];
1187  if (constructHasFlip_)
1188  {
1189  newIndex = sign*(newIndex+1);
1190  }
1191  newMap[newI++] = newIndex;
1192  }
1193  }
1194  newMap.setSize(newI);
1195  constructMap_[domain].transfer(newMap);
1196  }
1197 }
1198 
1199 
1200 // * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
1201 
1203 {
1204  // Check for assignment to self
1205  if (this == &rhs)
1206  {
1208  << "Attempted assignment to self"
1209  << abort(FatalError);
1210  }
1211  constructSize_ = rhs.constructSize_;
1212  subMap_ = rhs.subMap_;
1213  constructMap_ = rhs.constructMap_;
1214  subHasFlip_ = rhs.subHasFlip_;
1215  constructHasFlip_ = rhs.constructHasFlip_;
1216  schedulePtr_.clear();
1217 }
1218 
1219 
1220 // * * * * * * * * * * * * * * Istream Operator * * * * * * * * * * * * * * //
1221 
1223 {
1224  is.fatalCheck("operator>>(Istream&, distributionMapBase&)");
1225 
1226  is >> map.constructSize_ >> map.subMap_ >> map.constructMap_
1227  >> map.subHasFlip_ >> map.constructHasFlip_;
1228 
1229  return is;
1230 }
1231 
1232 
1233 // * * * * * * * * * * * * * * Ostream Operator * * * * * * * * * * * * * * //
1234 
1236 {
1237  os << map.constructSize_ << token::NL
1238  << map.subMap_ << token::NL
1239  << map.constructMap_ << token::NL
1240  << map.subHasFlip_ << token::SPACE << map.constructHasFlip_
1241  << token::NL;
1242 
1243  return os;
1244 }
1245 
1246 
1247 // ************************************************************************* //
Various functions to operate on Lists.
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:434
#define forAllIter(Container, container, iter)
Iterate across all elements in the container object of type.
Definition: UList.H:459
A HashTable with keys but without contents.
Definition: HashSet.H:62
bool insert(const Key &key)
Insert a new entry.
Definition: HashSet.H:109
List< Key > toc() const
Return the table of contents.
Definition: HashTable.C:227
void fatalCheck(const char *operation) const
Check IOstream status for given operation.
Definition: IOstream.C:105
Input inter-processor communications stream.
Definition: IPstream.H:54
An Istream is an abstract base class for all input systems (streams, files, token lists etc)....
Definition: Istream.H:60
A 1D array of objects of type <T>, where the size of the vector is known and used for subscript bound...
Definition: List.H:91
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:164
void setSize(const label)
Reset size of List.
Definition: List.C:281
Output inter-processor communications stream.
Definition: OPstream.H:54
An Ostream is an abstract base class for all output systems (streams, files, token lists,...
Definition: Ostream.H:57
static label read(const commsTypes commsType, const int fromProcNo, char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Read into given buffer from given processor and return the.
Definition: UIPread.C:79
A List with indirect addressing.
Definition: UIndirectList.H:60
iterator begin()
Return an iterator to begin traversing the UList.
Definition: UListI.H:216
static bool write(const commsTypes commsType, const int toProcNo, const char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Write given buffer to given processor.
Definition: UOPwrite.C:34
static int masterNo()
Process index of the master.
Definition: UPstream.H:417
static bool master(const label communicator=0)
Am I the master process.
Definition: UPstream.H:423
static int lastSlave(const label communicator=0)
Process index of last slave.
Definition: UPstream.H:452
static label worldComm
Default communicator (all processors)
Definition: UPstream.H:278
static label nRequests()
Get number of outstanding requests.
Definition: UPstream.C:137
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:411
static void waitRequests(const label start=0)
Wait until all requests (from start onwards) have finished.
Definition: UPstream.C:147
static int firstSlave()
Process index of first slave.
Definition: UPstream.H:446
static bool & parRun()
Is this a parallel run?
Definition: UPstream.H:399
static int & msgType()
Message tag of standard messages.
Definition: UPstream.H:476
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:429
Determines the order in which a set of processors should communicate with one another.
Definition: commSchedule.H:66
Class containing processor-to-processor mapping information.
bool constructHasFlip_
Whether constructMap includes flip or not.
label constructSize_
Size of reconstructed data.
const List< labelPair > & schedule() const
Return a schedule. Demand driven. See above.
const labelListList & constructMap() const
From subsetted data to new reconstructed data.
const labelListList & subMap() const
From subsetted data back to original data.
void operator=(const distributionMapBase &)
void printLayout(Ostream &os) const
Debug: print layout. Can only be used on maps with sorted.
void transfer(distributionMapBase &)
Transfer the contents of the argument and annul the argument.
bool subHasFlip_
Whether subMap includes flip or not.
static void checkReceivedSize(const label proci, const label expectedSize, const label receivedSize)
static label renumber(const globalIndex &, const List< Map< label >> &compactMap, const label globalElement)
Helper for construct from globalIndex. Renumbers element.
labelListList subMap_
Maps from subsetted data back to original data.
distributionMapBase()
Construct null.
void calcCompactAddressing(const globalIndex &globalNumbering, const labelList &elements, List< Map< label >> &compactMap) const
Construct per processor compact addressing of the global elements.
void exchangeAddressing(const int tag, const globalIndex &globalNumbering, labelList &elements, List< Map< label >> &compactMap, labelList &compactStart)
labelListList constructMap_
Maps from subsetted data to new reconstructed data.
void compact(const boolList &elemIsUsed, const int tag=UPstream::msgType())
Compact maps. Gets per field a bool whether it is used (locally)
Calculates a unique integer (label so might not have enough room - 2G max) for processor + local inde...
Definition: globalIndex.H:64
label whichProcID(const label i) const
Which processor does global come from? Binary search.
Definition: globalIndexI.H:123
label localSize() const
My local size.
Definition: globalIndexI.H:60
label toLocal(const label i) const
From global to local on current processor.
Definition: globalIndexI.H:117
bool isLocal(const label i) const
Is on local processor.
Definition: globalIndexI.H:95
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
Definition: error.H:334
Namespace for OpenFOAM.
Pair< label > labelPair
Label pair.
Definition: labelPair.H:48
dimensionedScalar sign(const dimensionedScalar &ds)
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
Ostream & endl(Ostream &os)
Add newline and flush stream.
Definition: Ostream.H:257
errorManip< error > abort(error &err)
Definition: errorManip.H:131
ListType renumber(const labelUList &oldToNew, const ListType &)
Renumber the values (not the indices) of a list.
layerAndWeight min(const layerAndWeight &a, const layerAndWeight &b)
Istream & operator>>(Istream &, pistonPointEdgeData &)
dimensioned< scalar > mag(const dimensioned< Type > &)
defineTypeNameAndDebug(combustionModel, 0)
layerAndWeight max(const layerAndWeight &a, const layerAndWeight &b)
prefixOSstream Pout(cout, "Pout")
Definition: IOstreams.H:53
label findIndex(const ListType &, typename ListType::const_reference, const label start=0)
Find first occurrence of given element and return index,.
Ostream & operator<<(Ostream &os, const fvConstraints &constraints)
error FatalError
static const label labelMax
Definition: label.H:62
labelList identityMap(const label len)
Create identity map (map[i] == i) of given length.
Definition: ListOps.C:104
void offset(label &lst, const label o)
static const label labelMin
Definition: label.H:61