mapDistribute.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration |
5  \\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "mapDistribute.H"
27 #include "commSchedule.H"
28 #include "HashSet.H"
29 #include "globalIndex.H"
31 #include "transformField.H"
32 #include "ListOps.H"
33 
34 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
35 
36 namespace Foam
37 {
38 defineTypeNameAndDebug(mapDistribute, 0);
39 }
40 
41 
42 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
43 
44 template<>
45 void Foam::mapDistribute::transform::operator()
46 (
47  const vectorTensorTransform&,
48  const bool,
50 ) const
51 {}
52 template<>
53 void Foam::mapDistribute::transform::operator()
54 (
55  const coupledPolyPatch&,
57 ) const
58 {}
59 template<>
60 void Foam::mapDistribute::transform::operator()
61 (
62  const coupledPolyPatch&,
63  Map<label>&
64 ) const
65 {}
66 template<>
67 void Foam::mapDistribute::transform::operator()
68 (
69  const coupledPolyPatch&,
71 ) const
72 {}
73 
74 
75 template<>
76 void Foam::mapDistribute::transform::operator()
77 (
78  const vectorTensorTransform&,
79  const bool,
81 ) const
82 {}
83 template<>
84 void Foam::mapDistribute::transform::operator()
85 (
86  const coupledPolyPatch&,
88 ) const
89 {}
90 template<>
91 void Foam::mapDistribute::transform::operator()
92 (
93  const coupledPolyPatch&,
95 ) const
96 {}
97 template<>
98 void Foam::mapDistribute::transform::operator()
99 (
100  const coupledPolyPatch&,
102 ) const
103 {}
104 
105 
106 template<>
107 void Foam::mapDistribute::transform::operator()
108 (
109  const vectorTensorTransform&,
110  const bool,
111  List<bool>&
112 ) const
113 {}
114 template<>
115 void Foam::mapDistribute::transform::operator()
116 (
117  const coupledPolyPatch&,
118  UList<bool>&
119 ) const
120 {}
121 template<>
122 void Foam::mapDistribute::transform::operator()
123 (
124  const coupledPolyPatch&,
125  Map<bool>&
126 ) const
127 {}
128 template<>
129 void Foam::mapDistribute::transform::operator()
130 (
131  const coupledPolyPatch&,
133 ) const
134 {}
135 
136 
138 (
139  const labelListList& subMap,
140  const labelListList& constructMap,
141  const int tag
142 )
143 {
144  // Communications: send and receive processor
145  List<labelPair> allComms;
146 
147  {
149 
150  // Find what communication is required
151  forAll(subMap, procI)
152  {
153  if (procI != Pstream::myProcNo())
154  {
155  if (subMap[procI].size())
156  {
157  // I need to send to procI
158  commsSet.insert(labelPair(Pstream::myProcNo(), procI));
159  }
160  if (constructMap[procI].size())
161  {
162  // I need to receive from procI
163  commsSet.insert(labelPair(procI, Pstream::myProcNo()));
164  }
165  }
166  }
167  allComms = commsSet.toc();
168  }
169 
170 
171  // Reduce
172  if (Pstream::master())
173  {
174  // Receive and merge
175  for
176  (
177  int slave=Pstream::firstSlave();
178  slave<=Pstream::lastSlave();
179  slave++
180  )
181  {
182  IPstream fromSlave(Pstream::scheduled, slave, 0, tag);
183  List<labelPair> nbrData(fromSlave);
184 
185  forAll(nbrData, i)
186  {
187  if (findIndex(allComms, nbrData[i]) == -1)
188  {
189  label sz = allComms.size();
190  allComms.setSize(sz+1);
191  allComms[sz] = nbrData[i];
192  }
193  }
194  }
195  // Send back
196  for
197  (
198  int slave=Pstream::firstSlave();
199  slave<=Pstream::lastSlave();
200  slave++
201  )
202  {
203  OPstream toSlave(Pstream::scheduled, slave, 0, tag);
204  toSlave << allComms;
205  }
206  }
207  else
208  {
209  {
210  OPstream toMaster(Pstream::scheduled, Pstream::masterNo(), 0, tag);
211  toMaster << allComms;
212  }
213  {
214  IPstream fromMaster
215  (
218  0,
219  tag
220  );
221  fromMaster >> allComms;
222  }
223  }
224 
225 
226  // Determine my schedule.
227  labelList mySchedule
228  (
230  (
231  Pstream::nProcs(),
232  allComms
233  ).procSchedule()[Pstream::myProcNo()]
234  );
235 
236  // Processors involved in my schedule
237  return List<labelPair>(UIndirectList<labelPair>(allComms, mySchedule));
238 
239 
240  //if (debug)
241  //{
242  // Pout<< "I need to:" << endl;
243  // const List<labelPair>& comms = schedule();
244  // forAll(comms, i)
245  // {
246  // const labelPair& twoProcs = comms[i];
247  // label sendProc = twoProcs[0];
248  // label recvProc = twoProcs[1];
249  //
250  // if (recvProc == Pstream::myProcNo())
251  // {
252  // Pout<< " receive from " << sendProc << endl;
253  // }
254  // else
255  // {
256  // Pout<< " send to " << recvProc << endl;
257  // }
258  // }
259  //}
260 }
261 
262 
264 {
265  if (schedulePtr_.empty())
266  {
267  schedulePtr_.reset
268  (
269  new List<labelPair>
270  (
271  schedule(subMap_, constructMap_, Pstream::msgType())
272  )
273  );
274  }
275  return schedulePtr_();
276 }
277 
278 
279 void Foam::mapDistribute::checkReceivedSize
280 (
281  const label procI,
282  const label expectedSize,
283  const label receivedSize
284 )
285 {
286  if (receivedSize != expectedSize)
287  {
289  (
290  "template<class T>\n"
291  "void mapDistribute::distribute\n"
292  "(\n"
293  " const Pstream::commsTypes commsType,\n"
294  " const List<labelPair>& schedule,\n"
295  " const label constructSize,\n"
296  " const labelListList& subMap,\n"
297  " const labelListList& constructMap,\n"
298  " List<T>& field\n"
299  ")\n"
300  ) << "Expected from processor " << procI
301  << " " << expectedSize << " but received "
302  << receivedSize << " elements."
303  << abort(FatalError);
304  }
305 }
306 
307 
309 {
310  // Determine offsets of remote data.
311  labelList minIndex(Pstream::nProcs(), labelMax);
312  labelList maxIndex(Pstream::nProcs(), labelMin);
313  forAll(constructMap_, procI)
314  {
315  const labelList& construct = constructMap_[procI];
316  minIndex[procI] = min(minIndex[procI], min(construct));
317  maxIndex[procI] = max(maxIndex[procI], max(construct));
318  }
319 
320  label localSize;
321  if (maxIndex[Pstream::myProcNo()] == labelMin)
322  {
323  localSize = 0;
324  }
325  else
326  {
327  localSize = maxIndex[Pstream::myProcNo()]+1;
328  }
329 
330  os << "Layout: (constructSize:" << constructSize_ << ")" << endl
331  << "local (processor " << Pstream::myProcNo() << "):" << endl
332  << " start : 0" << endl
333  << " size : " << localSize << endl;
334 
335  label offset = localSize;
336  forAll(minIndex, procI)
337  {
338  if (procI != Pstream::myProcNo())
339  {
340  if (constructMap_[procI].size() > 0)
341  {
342  if (minIndex[procI] != offset)
343  {
344  FatalErrorIn("mapDistribute::printLayout(..)")
345  << "offset:" << offset
346  << " procI:" << procI
347  << " minIndex:" << minIndex[procI]
348  << abort(FatalError);
349  }
350 
351  label size = maxIndex[procI]-minIndex[procI]+1;
352  os << "processor " << procI << ':' << endl
353  << " start : " << offset << endl
354  << " size : " << size << endl;
355 
356  offset += size;
357  }
358  }
359  }
360  forAll(transformElements_, trafoI)
361  {
362  if (transformElements_[trafoI].size() > 0)
363  {
364  os << "transform " << trafoI << ':' << endl
365  << " start : " << transformStart_[trafoI] << endl
366  << " size : " << transformElements_[trafoI].size() << endl;
367  }
368  }
369 }
370 
371 
372 // Construct per processor compact addressing of the global elements
373 // needed. The ones from the local processor are not included since
374 // these are always all needed.
375 void Foam::mapDistribute::calcCompactAddressing
376 (
377  const globalIndex& globalNumbering,
378  const labelList& elements,
379  List<Map<label> >& compactMap
380 ) const
381 {
382  compactMap.setSize(Pstream::nProcs());
383 
384  // Count all (non-local) elements needed. Just for presizing map.
385  labelList nNonLocal(Pstream::nProcs(), 0);
386 
387  forAll(elements, i)
388  {
389  label globalIndex = elements[i];
390 
391  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
392  {
393  label procI = globalNumbering.whichProcID(globalIndex);
394  nNonLocal[procI]++;
395  }
396  }
397 
398  forAll(compactMap, procI)
399  {
400  compactMap[procI].clear();
401  if (procI != Pstream::myProcNo())
402  {
403  compactMap[procI].resize(2*nNonLocal[procI]);
404  }
405  }
406 
407 
408  // Collect all (non-local) elements needed.
409  forAll(elements, i)
410  {
411  label globalIndex = elements[i];
412 
413  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
414  {
415  label procI = globalNumbering.whichProcID(globalIndex);
416  label index = globalNumbering.toLocal(procI, globalIndex);
417  label nCompact = compactMap[procI].size();
418  compactMap[procI].insert(index, nCompact);
419  }
420  }
421 }
422 
423 
424 void Foam::mapDistribute::calcCompactAddressing
425 (
426  const globalIndex& globalNumbering,
427  const labelListList& cellCells,
428  List<Map<label> >& compactMap
429 ) const
430 {
431  compactMap.setSize(Pstream::nProcs());
432 
433  // Count all (non-local) elements needed. Just for presizing map.
434  labelList nNonLocal(Pstream::nProcs(), 0);
435 
436  forAll(cellCells, cellI)
437  {
438  const labelList& cCells = cellCells[cellI];
439 
440  forAll(cCells, i)
441  {
442  label globalIndex = cCells[i];
443 
444  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
445  {
446  label procI = globalNumbering.whichProcID(globalIndex);
447  nNonLocal[procI]++;
448  }
449  }
450  }
451 
452  forAll(compactMap, procI)
453  {
454  compactMap[procI].clear();
455  if (procI != Pstream::myProcNo())
456  {
457  compactMap[procI].resize(2*nNonLocal[procI]);
458  }
459  }
460 
461 
462  // Collect all (non-local) elements needed.
463  forAll(cellCells, cellI)
464  {
465  const labelList& cCells = cellCells[cellI];
466 
467  forAll(cCells, i)
468  {
469  label globalIndex = cCells[i];
470 
471  if (globalIndex != -1 && !globalNumbering.isLocal(globalIndex))
472  {
473  label procI = globalNumbering.whichProcID(globalIndex);
474  label index = globalNumbering.toLocal(procI, globalIndex);
475  label nCompact = compactMap[procI].size();
476  compactMap[procI].insert(index, nCompact);
477  }
478  }
479  }
480 }
481 
482 
483 void Foam::mapDistribute::exchangeAddressing
484 (
485  const int tag,
486  const globalIndex& globalNumbering,
487  labelList& elements,
488  List<Map<label> >& compactMap,
489  labelList& compactStart
490 )
491 {
492  // The overall compact addressing is
493  // - myProcNo data first (uncompacted)
494  // - all other processors consecutively
495 
496  compactStart.setSize(Pstream::nProcs());
497  compactStart[Pstream::myProcNo()] = 0;
498  constructSize_ = globalNumbering.localSize();
499  forAll(compactStart, procI)
500  {
501  if (procI != Pstream::myProcNo())
502  {
503  compactStart[procI] = constructSize_;
504  constructSize_ += compactMap[procI].size();
505  }
506  }
507 
508 
509 
510  // Find out what to receive/send in compact addressing.
511 
512  // What I want to receive is what others have to send
513  labelListList wantedRemoteElements(Pstream::nProcs());
514  // Compact addressing for received data
515  constructMap_.setSize(Pstream::nProcs());
516  forAll(compactMap, procI)
517  {
518  if (procI == Pstream::myProcNo())
519  {
520  // All my own elements are used
521  label nLocal = globalNumbering.localSize();
522  wantedRemoteElements[procI] = identity(nLocal);
523  constructMap_[procI] = identity(nLocal);
524  }
525  else
526  {
527  // Remote elements wanted from processor procI
528  labelList& remoteElem = wantedRemoteElements[procI];
529  labelList& localElem = constructMap_[procI];
530  remoteElem.setSize(compactMap[procI].size());
531  localElem.setSize(compactMap[procI].size());
532  label i = 0;
533  forAllIter(Map<label>, compactMap[procI], iter)
534  {
535  const label compactI = compactStart[procI] + iter();
536  remoteElem[i] = iter.key();
537  localElem[i] = compactI;
538  iter() = compactI;
539  i++;
540  }
541  }
542  }
543 
544  subMap_.setSize(Pstream::nProcs());
545  labelListList sendSizes;
546  Pstream::exchange<labelList, label>
547  (
548  wantedRemoteElements,
549  subMap_,
550  sendSizes,
551  tag,
552  Pstream::worldComm //TBD
553  );
554 
555  // Renumber elements
556  forAll(elements, i)
557  {
558  elements[i] = renumber(globalNumbering, compactMap, elements[i]);
559  }
560 }
561 
562 
563 void Foam::mapDistribute::exchangeAddressing
564 (
565  const int tag,
566  const globalIndex& globalNumbering,
567  labelListList& cellCells,
568  List<Map<label> >& compactMap,
569  labelList& compactStart
570 )
571 {
572  // The overall compact addressing is
573  // - myProcNo data first (uncompacted)
574  // - all other processors consecutively
575 
576  compactStart.setSize(Pstream::nProcs());
577  compactStart[Pstream::myProcNo()] = 0;
578  constructSize_ = globalNumbering.localSize();
579  forAll(compactStart, procI)
580  {
581  if (procI != Pstream::myProcNo())
582  {
583  compactStart[procI] = constructSize_;
584  constructSize_ += compactMap[procI].size();
585  }
586  }
587 
588 
589 
590  // Find out what to receive/send in compact addressing.
591 
592  // What I want to receive is what others have to send
593  labelListList wantedRemoteElements(Pstream::nProcs());
594  // Compact addressing for received data
595  constructMap_.setSize(Pstream::nProcs());
596  forAll(compactMap, procI)
597  {
598  if (procI == Pstream::myProcNo())
599  {
600  // All my own elements are used
601  label nLocal = globalNumbering.localSize();
602  wantedRemoteElements[procI] = identity(nLocal);
603  constructMap_[procI] = identity(nLocal);
604  }
605  else
606  {
607  // Remote elements wanted from processor procI
608  labelList& remoteElem = wantedRemoteElements[procI];
609  labelList& localElem = constructMap_[procI];
610  remoteElem.setSize(compactMap[procI].size());
611  localElem.setSize(compactMap[procI].size());
612  label i = 0;
613  forAllIter(Map<label>, compactMap[procI], iter)
614  {
615  const label compactI = compactStart[procI] + iter();
616  remoteElem[i] = iter.key();
617  localElem[i] = compactI;
618  iter() = compactI;
619  i++;
620  }
621  }
622  }
623 
624  subMap_.setSize(Pstream::nProcs());
625  labelListList sendSizes;
626  Pstream::exchange<labelList, label>
627  (
628  wantedRemoteElements,
629  subMap_,
630  sendSizes,
631  tag,
632  Pstream::worldComm //TBD
633  );
634 
635  // Renumber elements
636  forAll(cellCells, cellI)
637  {
638  labelList& cCells = cellCells[cellI];
639 
640  forAll(cCells, i)
641  {
642  cCells[i] = renumber(globalNumbering, compactMap, cCells[i]);
643  }
644  }
645 }
646 
647 
648 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
649 
650 //- Construct null
652 :
653  constructSize_(0),
654  schedulePtr_()
655 {}
656 
657 
658 //- Construct from components
660 (
661  const label constructSize,
662  const Xfer<labelListList>& subMap,
663  const Xfer<labelListList>& constructMap
664 )
665 :
666  constructSize_(constructSize),
667  subMap_(subMap),
668  constructMap_(constructMap),
669  schedulePtr_()
670 {}
671 
672 
673 //- Construct from components
675 (
676  const label constructSize,
677  const Xfer<labelListList>& subMap,
678  const Xfer<labelListList>& constructMap,
681 )
682 :
683  constructSize_(constructSize),
684  subMap_(subMap),
685  constructMap_(constructMap),
686  transformElements_(transformElements),
687  transformStart_(transformStart),
688  schedulePtr_()
689 {}
690 
691 
693 (
694  const labelList& sendProcs,
695  const labelList& recvProcs
696 )
697 :
698  constructSize_(0),
699  schedulePtr_()
700 {
701  if (sendProcs.size() != recvProcs.size())
702  {
704  (
705  "mapDistribute::mapDistribute(const labelList&, const labelList&)"
706  ) << "The send and receive data is not the same length. sendProcs:"
707  << sendProcs.size() << " recvProcs:" << recvProcs.size()
708  << abort(FatalError);
709  }
710 
711  // Per processor the number of samples we have to send/receive.
712  labelList nSend(Pstream::nProcs(), 0);
713  labelList nRecv(Pstream::nProcs(), 0);
714 
715  forAll(sendProcs, sampleI)
716  {
717  label sendProc = sendProcs[sampleI];
718  label recvProc = recvProcs[sampleI];
719 
720  // Note that also need to include local communication (both
721  // RecvProc and sendProc on local processor)
722 
723  if (Pstream::myProcNo() == sendProc)
724  {
725  // I am the sender. Count destination processor.
726  nSend[recvProc]++;
727  }
728  if (Pstream::myProcNo() == recvProc)
729  {
730  // I am the receiver.
731  nRecv[sendProc]++;
732  }
733  }
734 
735  subMap_.setSize(Pstream::nProcs());
736  constructMap_.setSize(Pstream::nProcs());
737  forAll(nSend, procI)
738  {
739  subMap_[procI].setSize(nSend[procI]);
740  constructMap_[procI].setSize(nRecv[procI]);
741  }
742  nSend = 0;
743  nRecv = 0;
744 
745  forAll(sendProcs, sampleI)
746  {
747  label sendProc = sendProcs[sampleI];
748  label recvProc = recvProcs[sampleI];
749 
750  if (Pstream::myProcNo() == sendProc)
751  {
752  // I am the sender. Store index I need to send.
753  subMap_[recvProc][nSend[recvProc]++] = sampleI;
754  }
755  if (Pstream::myProcNo() == recvProc)
756  {
757  // I am the receiver.
758  constructMap_[sendProc][nRecv[sendProc]++] = sampleI;
759  // Largest entry inside constructMap
760  constructSize_ = sampleI+1;
761  }
762  }
763 }
764 
765 
767 (
768  const globalIndex& globalNumbering,
769  labelList& elements,
770  List<Map<label> >& compactMap,
771  const int tag
772 )
773 :
774  constructSize_(0),
775  schedulePtr_()
776 {
777  // Construct per processor compact addressing of the global elements
778  // needed. The ones from the local processor are not included since
779  // these are always all needed.
780  calcCompactAddressing
781  (
782  globalNumbering,
783  elements,
784  compactMap
785  );
786 
788  //forAll(compactMap, procI)
789  //{
790  // if (procI != Pstream::myProcNo())
791  // {
792  // Map<label>& globalMap = compactMap[procI];
793  //
794  // SortableList<label> sorted(globalMap.toc().xfer());
795  //
796  // forAll(sorted, i)
797  // {
798  // Map<label>::iterator iter = globalMap.find(sorted[i]);
799  // iter() = i;
800  // }
801  // }
802  //}
803 
804 
805  // Exchange what I need with processor that supplies it. Renumber elements
806  // into compact numbering
807  labelList compactStart;
808  exchangeAddressing
809  (
810  tag,
811  globalNumbering,
812  elements,
813  compactMap,
814  compactStart
815  );
816 
817  if (debug)
818  {
819  printLayout(Pout);
820  }
821 }
822 
823 
825 (
826  const globalIndex& globalNumbering,
827  labelListList& cellCells,
828  List<Map<label> >& compactMap,
829  const int tag
830 )
831 :
832  constructSize_(0),
833  schedulePtr_()
834 {
835  // Construct per processor compact addressing of the global elements
836  // needed. The ones from the local processor are not included since
837  // these are always all needed.
838  calcCompactAddressing
839  (
840  globalNumbering,
841  cellCells,
842  compactMap
843  );
844 
846  //forAll(compactMap, procI)
847  //{
848  // if (procI != Pstream::myProcNo())
849  // {
850  // Map<label>& globalMap = compactMap[procI];
851  //
852  // SortableList<label> sorted(globalMap.toc().xfer());
853  //
854  // forAll(sorted, i)
855  // {
856  // Map<label>::iterator iter = globalMap.find(sorted[i]);
857  // iter() = i;
858  // }
859  // }
860  //}
861 
862 
863  // Exchange what I need with processor that supplies it. Renumber elements
864  // into compact numbering
865  labelList compactStart;
866  exchangeAddressing
867  (
868  tag,
869  globalNumbering,
870  cellCells,
871  compactMap,
872  compactStart
873  );
874 
875  if (debug)
876  {
877  printLayout(Pout);
878  }
879 }
880 
881 
883 (
884  const globalIndex& globalNumbering,
885  labelList& elements,
886  const globalIndexAndTransform& globalTransforms,
887  const labelPairList& transformedElements,
888  labelList& transformedIndices,
889  List<Map<label> >& compactMap,
890  const int tag
891 )
892 :
893  constructSize_(0),
894  schedulePtr_()
895 {
896  // Construct per processor compact addressing of the global elements
897  // needed. The ones from the local processor are not included since
898  // these are always all needed.
899  calcCompactAddressing
900  (
901  globalNumbering,
902  elements,
903  compactMap
904  );
905 
906  // Add all (non-local) transformed elements needed.
907  forAll(transformedElements, i)
908  {
909  labelPair elem = transformedElements[i];
911  if (procI != Pstream::myProcNo())
912  {
914  label nCompact = compactMap[procI].size();
915  compactMap[procI].insert(index, nCompact);
916  }
917  }
918 
919 
920  // Exchange what I need with processor that supplies it. Renumber elements
921  // into compact numbering
922  labelList compactStart;
923  exchangeAddressing
924  (
925  tag,
926  globalNumbering,
927  elements,
928  compactMap,
929  compactStart
930  );
931 
932 
933  // Renumber the transformed elements
934  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
935  // Count per transformIndex
936  label nTrafo = globalTransforms.transformPermutations().size();
937  labelList nPerTransform(nTrafo, 0);
938  forAll(transformedElements, i)
939  {
940  labelPair elem = transformedElements[i];
942  nPerTransform[trafoI]++;
943  }
944  // Offset per transformIndex
945  transformStart_.setSize(nTrafo);
946  transformElements_.setSize(nTrafo);
947  forAll(transformStart_, trafoI)
948  {
949  transformStart_[trafoI] = constructSize_;
950  constructSize_ += nPerTransform[trafoI];
951  transformElements_[trafoI].setSize(nPerTransform[trafoI]);
952  }
953 
954  // Sort transformed elements into their new slot.
955  nPerTransform = 0;
956 
957  transformedIndices.setSize(transformedElements.size());
958  forAll(transformedElements, i)
959  {
960  labelPair elem = transformedElements[i];
964 
965  // Get compact index for untransformed element
966  label rawElemI =
967  (
968  procI == Pstream::myProcNo()
969  ? index
970  : compactMap[procI][index]
971  );
972 
973  label& n = nPerTransform[trafoI];
974  // index of element to transform
975  transformElements_[trafoI][n] = rawElemI;
976  // destination of transformed element
977  transformedIndices[i] = transformStart_[trafoI]+n;
978  n++;
979  }
980 
981  if (debug)
982  {
983  printLayout(Pout);
984  }
985 }
986 
987 
989 (
990  const globalIndex& globalNumbering,
991  labelListList& cellCells,
992  const globalIndexAndTransform& globalTransforms,
993  const List<labelPairList>& transformedElements,
994  labelListList& transformedIndices,
995  List<Map<label> >& compactMap,
996  const int tag
997 )
998 :
999  constructSize_(0),
1000  schedulePtr_()
1001 {
1002  // Construct per processor compact addressing of the global elements
1003  // needed. The ones from the local processor are not included since
1004  // these are always all needed.
1005  calcCompactAddressing
1006  (
1007  globalNumbering,
1008  cellCells,
1009  compactMap
1010  );
1011 
1012  // Add all (non-local) transformed elements needed.
1013  forAll(transformedElements, cellI)
1014  {
1015  const labelPairList& elems = transformedElements[cellI];
1016 
1017  forAll(elems, i)
1018  {
1019  label procI = globalIndexAndTransform::processor(elems[i]);
1020  if (procI != Pstream::myProcNo())
1021  {
1022  label index = globalIndexAndTransform::index(elems[i]);
1023  label nCompact = compactMap[procI].size();
1024  compactMap[procI].insert(index, nCompact);
1025  }
1026  }
1027  }
1028 
1029 
1030  // Exchange what I need with processor that supplies it. Renumber elements
1031  // into compact numbering
1032  labelList compactStart;
1033  exchangeAddressing
1034  (
1035  tag,
1036  globalNumbering,
1037  cellCells,
1038  compactMap,
1039  compactStart
1040  );
1041 
1042 
1043  // Renumber the transformed elements
1044  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1045  // Count per transformIndex
1046  label nTrafo = globalTransforms.transformPermutations().size();
1047  labelList nPerTransform(nTrafo, 0);
1048  forAll(transformedElements, cellI)
1049  {
1050  const labelPairList& elems = transformedElements[cellI];
1051 
1052  forAll(elems, i)
1053  {
1055  nPerTransform[trafoI]++;
1056  }
1057  }
1058  // Offset per transformIndex
1059  transformStart_.setSize(nTrafo);
1060  transformElements_.setSize(nTrafo);
1061  forAll(transformStart_, trafoI)
1062  {
1063  transformStart_[trafoI] = constructSize_;
1064  constructSize_ += nPerTransform[trafoI];
1065  transformElements_[trafoI].setSize(nPerTransform[trafoI]);
1066  }
1067 
1068  // Sort transformed elements into their new slot.
1069  nPerTransform = 0;
1070 
1071  transformedIndices.setSize(transformedElements.size());
1072  forAll(transformedElements, cellI)
1073  {
1074  const labelPairList& elems = transformedElements[cellI];
1075  transformedIndices[cellI].setSize(elems.size());
1076 
1077  forAll(elems, i)
1078  {
1079  label procI = globalIndexAndTransform::processor(elems[i]);
1080  label index = globalIndexAndTransform::index(elems[i]);
1082 
1083  // Get compact index for untransformed element
1084  label rawElemI =
1085  (
1086  procI == Pstream::myProcNo()
1087  ? index
1088  : compactMap[procI][index]
1089  );
1090 
1091  label& n = nPerTransform[trafoI];
1092  // index of element to transform
1093  transformElements_[trafoI][n] = rawElemI;
1094  // destination of transformed element
1095  transformedIndices[cellI][i] = transformStart_[trafoI]+n;
1096  n++;
1097  }
1098  }
1099 
1100  if (debug)
1101  {
1102  printLayout(Pout);
1103  }
1104 }
1105 
1106 
1108 :
1109  constructSize_(map.constructSize_),
1110  subMap_(map.subMap_),
1111  constructMap_(map.constructMap_),
1112  transformElements_(map.transformElements_),
1113  transformStart_(map.transformStart_),
1114  schedulePtr_()
1115 {}
1116 
1117 
1119 :
1120  constructSize_(map().constructSize_),
1121  subMap_(map().subMap_.xfer()),
1122  constructMap_(map().constructMap_.xfer()),
1123  transformElements_(map().transformElements_.xfer()),
1124  transformStart_(map().transformStart_.xfer()),
1125  schedulePtr_()
1126 {}
1127 
1128 
1129 // * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * * //
1130 
1132 {
1133  return findLower(transformStart_, index+1);
1134 }
1135 
1136 
1138 {
1139  constructSize_ = rhs.constructSize_;
1140  subMap_.transfer(rhs.subMap_);
1141  constructMap_.transfer(rhs.constructMap_);
1142  transformElements_.transfer(rhs.transformElements_);
1143  transformStart_.transfer(rhs.transformStart_);
1144  schedulePtr_.clear();
1145 }
1146 
1147 
1149 {
1150  return xferMove(*this);
1151 }
1152 
1153 
1156  const globalIndex& globalNumbering,
1157  const List<Map<label> >& compactMap,
1158  const label globalI
1159 )
1160 {
1161  if (globalI == -1)
1162  {
1163  return globalI;
1164  }
1165  if (globalNumbering.isLocal(globalI))
1166  {
1167  return globalNumbering.toLocal(globalI);
1168  }
1169  else
1170  {
1171  label procI = globalNumbering.whichProcID(globalI);
1172  label index = globalNumbering.toLocal(procI, globalI);
1173  return compactMap[procI][index];
1174  }
1175 }
1176 
1177 
1178 void Foam::mapDistribute::compact(const boolList& elemIsUsed, const int tag)
1179 {
1180  // 1. send back to sender. Have sender delete the corresponding element
1181  // from the submap and do the same to the constructMap locally
1182  // (and in same order).
1183 
1184  // Send elemIsUsed field to neighbour. Use nonblocking code from
1185  // mapDistribute but in reverse order.
1186  if (Pstream::parRun())
1187  {
1188  label startOfRequests = Pstream::nRequests();
1189 
1190  // Set up receives from neighbours
1191 
1192  List<boolList> recvFields(Pstream::nProcs());
1193 
1194  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1195  {
1196  const labelList& map = subMap_[domain];
1197 
1198  if (domain != Pstream::myProcNo() && map.size())
1199  {
1200  recvFields[domain].setSize(map.size());
1202  (
1204  domain,
1205  reinterpret_cast<char*>(recvFields[domain].begin()),
1206  recvFields[domain].size()*sizeof(bool),
1207  tag
1208  );
1209  }
1210  }
1211 
1212 
1213  List<boolList> sendFields(Pstream::nProcs());
1214 
1215  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1216  {
1217  const labelList& map = constructMap_[domain];
1218 
1219  if (domain != Pstream::myProcNo() && map.size())
1220  {
1221  boolList& subField = sendFields[domain];
1222  subField.setSize(map.size());
1223  forAll(map, i)
1224  {
1225  subField[i] = elemIsUsed[map[i]];
1226  }
1227 
1229  (
1231  domain,
1232  reinterpret_cast<const char*>(subField.begin()),
1233  subField.size()*sizeof(bool),
1234  tag
1235  );
1236  }
1237  }
1238 
1239 
1240 
1241  // Set up 'send' to myself - write directly into recvFields
1242 
1243  {
1244  const labelList& map = constructMap_[Pstream::myProcNo()];
1245 
1246  recvFields[Pstream::myProcNo()].setSize(map.size());
1247  forAll(map, i)
1248  {
1249  recvFields[Pstream::myProcNo()][i] = elemIsUsed[map[i]];
1250  }
1251  }
1252 
1253 
1254  // Wait for all to finish
1255 
1256  Pstream::waitRequests(startOfRequests);
1257 
1258 
1259  // Compact out all submap entries that are referring to unused elements
1260  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1261  {
1262  const labelList& map = subMap_[domain];
1263 
1264  labelList newMap(map.size());
1265  label newI = 0;
1266 
1267  forAll(map, i)
1268  {
1269  if (recvFields[domain][i])
1270  {
1271  // So element is used on destination side
1272  newMap[newI++] = map[i];
1273  }
1274  }
1275  if (newI < map.size())
1276  {
1277  newMap.setSize(newI);
1278  subMap_[domain].transfer(newMap);
1279  }
1280  }
1281  }
1282 
1283 
1284  // 2. remove from construct map - since end-result (element in elemIsUsed)
1285  // not used.
1286 
1287  label maxConstructIndex = -1;
1288 
1289  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1290  {
1291  const labelList& map = constructMap_[domain];
1292 
1293  labelList newMap(map.size());
1294  label newI = 0;
1295 
1296  forAll(map, i)
1297  {
1298  label destinationI = map[i];
1299 
1300  // Is element is used on destination side
1301  if (elemIsUsed[destinationI])
1302  {
1303  maxConstructIndex = max(maxConstructIndex, destinationI);
1304 
1305  newMap[newI++] = destinationI;
1306  }
1307  }
1308  if (newI < map.size())
1309  {
1310  newMap.setSize(newI);
1311  constructMap_[domain].transfer(newMap);
1312  }
1313  }
1314 
1315  constructSize_ = maxConstructIndex+1;
1316 
1317  // Clear the schedule (note:not necessary if nothing changed)
1318  schedulePtr_.clear();
1319 }
1320 
1321 
1322 // * * * * * * * * * * * * * * * Member Operators * * * * * * * * * * * * * //
1323 
1325 {
1326  // Check for assignment to self
1327  if (this == &rhs)
1328  {
1329  FatalErrorIn
1330  (
1331  "Foam::mapDistribute::operator=(const Foam::mapDistribute&)"
1332  ) << "Attempted assignment to self"
1333  << abort(FatalError);
1334  }
1335  constructSize_ = rhs.constructSize_;
1336  subMap_ = rhs.subMap_;
1337  constructMap_ = rhs.constructMap_;
1338  transformElements_ = rhs.transformElements_;
1339  transformStart_ = rhs.transformStart_;
1340  schedulePtr_.clear();
1341 }
1342 
1343 
1344 // * * * * * * * * * * * * * * Istream Operator * * * * * * * * * * * * * * //
1345 
1347 {
1348  is.fatalCheck("operator>>(Istream&, mapDistribute&)");
1349 
1350  is >> map.constructSize_ >> map.subMap_ >> map.constructMap_
1351  >> map.transformElements_ >> map.transformStart_;
1352 
1353  return is;
1354 }
1355 
1356 
1357 // * * * * * * * * * * * * * * Ostream Operator * * * * * * * * * * * * * * //
1358 
1360 {
1361  os << map.constructSize_ << token::NL
1362  << map.subMap_ << token::NL
1363  << map.constructMap_ << token::NL
1364  << map.transformElements_ << token::NL
1365  << map.transformStart_ << token::NL;
1366 
1367  return os;
1368 }
1369 
1370 
1371 // ************************************************************************* //
static bool & parRun()
Is this a parallel run?
Definition: UPstream.H:380
Determination and storage of the possible independent transforms introduced by coupledPolyPatches, as well as all of the possible permutations of these transforms generated by the presence of multiple coupledPolyPatches, i.e. more than one cyclic boundary.
bool isLocal(const label i) const
Is on local processor.
Definition: globalIndexI.H:95
label whichTransform(const label index) const
Find transform from transformElements.
label localSize() const
My local size.
Definition: globalIndexI.H:60
label findIndex(const ListType &, typename ListType::const_reference, const label start=0)
Find first occurence of given element and return index,.
An ordered pair of two objects of type <T> with first() and second() elements.
Definition: contiguous.H:49
void transfer(List< T > &)
Transfer the contents of the argument List into this list.
Definition: List.C:390
const labelListList & transformElements() const
For every globalIndexAndTransform::transformPermutations.
static label renumber(const globalIndex &, const List< Map< label > > &compactMap, const label globalElement)
Helper for construct from globalIndex. Renumbers element.
const List< vectorTensorTransform > & transformPermutations() const
Return access to the permuted transforms.
static bool write(const commsTypes commsType, const int toProcNo, const char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Write given buffer to given processor.
Definition: UOPwrite.C:34
Class containing processor-to-processor mapping information.
An Istream is an abstract base class for all input systems (streams, files, token lists etc)...
Definition: Istream.H:57
A simple container for copying or transferring objects of type <T>.
Definition: Xfer.H:85
#define forAllIter(Container, container, iter)
Definition: UList.H:440
Calculates a unique integer (label so might not have enough room - 2G max) for processor + local inde...
Definition: globalIndex.H:63
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
void operator=(const mapDistribute &)
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:76
Output inter-processor communications stream.
Definition: OPstream.H:50
static label read(const commsTypes commsType, const int fromProcNo, char *buf, const std::streamsize bufSize, const int tag=UPstream::msgType(), const label communicator=0)
Read into given buffer from given processor and return the.
Definition: UIPread.C:100
Various functions to operate on Lists.
Xfer< T > xferMove(T &)
void compact(const boolList &elemIsUsed, const int tag=UPstream::msgType())
Compact maps. Gets per field a bool whether it is used (locally)
Ostream & operator<<(Ostream &, const edgeMesh &)
Definition: edgeMeshIO.C:133
Namespace for OpenFOAM.
static void waitRequests(const label start=0)
Wait until all requests (from start onwards) have finished.
Definition: UPstream.C:106
mapDistribute()
Construct null.
label n
iterator begin()
Return an iterator to begin traversing the UList.
Definition: UListI.H:216
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:386
void setSize(const label)
Reset size of List.
Definition: List.C:318
Ostream & endl(Ostream &os)
Add newline and flush stream.
Definition: Ostream.H:251
A List with indirect addressing.
Definition: fvMatrix.H:106
dimensioned< Type > max(const dimensioned< Type > &, const dimensioned< Type > &)
void fatalCheck(const char *operation) const
Check IOstream status for given operation.
Definition: IOstream.C:107
#define forAll(list, i)
Definition: UList.H:421
Spatial transformation functions for primitive fields.
ListType renumber(const labelUList &oldToNew, const ListType &)
Renumber the values (not the indices) of a list.
static int & msgType()
Message tag of standard messages.
Definition: UPstream.H:451
Determines the order in which a set of processors should communicate with one another.
Definition: commSchedule.H:65
static const label labelMin
Definition: label.H:61
labelList identity(const label len)
Create identity map (map[i] == i) of given length.
Definition: ListOps.C:104
label findLower(const ListType &, typename ListType::const_reference, const label stary, const BinaryOp &bop)
Find last element < given value in sorted list and return index,.
List< Key > toc() const
Return the table of contents.
Xfer< mapDistribute > xfer()
Transfer contents to the Xfer container.
errorManip< error > abort(error &err)
Definition: errorManip.H:131
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:404
#define FatalErrorIn(functionName)
Report an error message using Foam::FatalError.
Definition: error.H:314
static label nRequests()
Get number of outstanding requests.
Definition: UPstream.C:96
Vector-tensor class used to perform translations and rotations in 3D space.
const List< labelPair > & schedule() const
Return a schedule. Demand driven. See above.
error FatalError
const labelList & transformStart() const
Destination in constructMap for transformed elements.
Input inter-processor communications stream.
Definition: IPstream.H:50
label toLocal(const label i) const
From global to local on current processor.
Definition: globalIndexI.H:117
static int masterNo()
Process index of the master.
Definition: UPstream.H:392
A 1D vector of objects of type <T>, where the size of the vector is known and can be used for subscri...
Definition: HashTable.H:60
A HashTable with keys but without contents.
Definition: HashSet.H:59
Pair< label > labelPair
Label pair.
Definition: labelPair.H:48
Istream & operator>>(Istream &, edgeMesh &)
Definition: edgeMeshIO.C:144
void transfer(mapDistribute &)
Transfer the contents of the argument and annul the argument.
An Ostream is an abstract base class for all output systems (streams, files, token lists...
Definition: Ostream.H:53
static bool master(const label communicator=0)
Am I the master process.
Definition: UPstream.H:398
static int firstSlave()
Process index of first slave.
Definition: UPstream.H:421
static int lastSlave(const label communicator=0)
Process index of last slave.
Definition: UPstream.H:427
static label index(const labelPair &globalIAndTransform)
Index carried by the object.
dimensioned< Type > min(const dimensioned< Type > &, const dimensioned< Type > &)
The coupledPolyPatch is an abstract base class for patches that couple regions of the computational d...
static label worldComm
Default communicator (all processors)
Definition: UPstream.H:261
static label transformIndex(const labelPair &globalIAndTransform)
Transform carried by the object.
static label processor(const labelPair &globalIAndTransform)
Which processor does this come from?
void printLayout(Ostream &os) const
Debug: print layout. Can only be used on maps with sorted.
label whichProcID(const label i) const
Which processor does global come from? Binary search.
Definition: globalIndexI.H:123
defineTypeNameAndDebug(combustionModel, 0)
prefixOSstream Pout(cout,"Pout")
Definition: IOstreams.H:53
label constructSize() const
Constructed data size.
static const label labelMax
Definition: label.H:62
bool insert(const Key &key)
Insert a new entry.
Definition: HashSet.H:116