ProcessorTopology.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration | Website: https://openfoam.org
5  \\ / A nd | Copyright (C) 2011-2018 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "ProcessorTopology.H"
27 #include "ListOps.H"
28 #include "Pstream.H"
29 #include "commSchedule.H"
30 #include "boolList.H"
31 
32 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
33 
34 template<class Container, class ProcPatch>
36 (
37  const label nProcs,
38  const Container& patches
39 )
40 {
41  // Determine number of processor neighbours and max neighbour id.
42 
43  label nNeighbours = 0;
44 
45  label maxNb = 0;
46 
47  boolList isNeighbourProc(nProcs, false);
48 
49  forAll(patches, patchi)
50  {
51  const typename Container::const_reference patch = patches[patchi];
52 
53  if (isA<ProcPatch>(patch))
54  {
55  const ProcPatch& procPatch =
56  refCast<const ProcPatch>(patch);
57 
58  label pNeighbProcNo = procPatch.neighbProcNo();
59 
60  if (!isNeighbourProc[pNeighbProcNo])
61  {
62  nNeighbours++;
63 
64  maxNb = max(maxNb, procPatch.neighbProcNo());
65 
66  isNeighbourProc[pNeighbProcNo] = true;
67  }
68  }
69  }
70 
71  labelList neighbours(nNeighbours, -1);
72 
73  nNeighbours = 0;
74 
75  forAll(isNeighbourProc, proci)
76  {
77  if (isNeighbourProc[proci])
78  {
79  neighbours[nNeighbours++] = proci;
80  }
81  }
82 
83  procPatchMap_.setSize(maxNb + 1);
84  procPatchMap_ = -1;
85 
86  forAll(patches, patchi)
87  {
88  const typename Container::const_reference patch = patches[patchi];
89 
90  if (isA<ProcPatch>(patch))
91  {
92  const ProcPatch& procPatch =
93  refCast<const ProcPatch>(patch);
94 
95  // Construct reverse map
96  procPatchMap_[procPatch.neighbProcNo()] = patchi;
97  }
98  }
99 
100  return neighbours;
101 }
102 
103 
104 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
105 
106 template<class Container, class ProcPatch>
108 (
109  const Container& patches,
110  const label comm
111 )
112 :
113  labelListList(Pstream::nProcs(comm)),
114  patchSchedule_(2*patches.size())
115 {
116  if (Pstream::parRun())
117  {
118  // Fill my 'slot' with my neighbours
119  operator[](Pstream::myProcNo(comm)) =
120  procNeighbours(this->size(), patches);
121 
122  // Distribute to all processors
123  Pstream::gatherList(*this, Pstream::msgType(), comm);
124  Pstream::scatterList(*this, Pstream::msgType(), comm);
125  }
126 
127  if
128  (
129  Pstream::parRun()
130  && Pstream::defaultCommsType == Pstream::commsTypes::scheduled
131  )
132  {
133  label patchEvali = 0;
134 
135  // 1. All non-processor patches
136  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
137 
138  forAll(patches, patchi)
139  {
140  if (!isA<ProcPatch>(patches[patchi]))
141  {
142  patchSchedule_[patchEvali].patch = patchi;
143  patchSchedule_[patchEvali++].init = true;
144  patchSchedule_[patchEvali].patch = patchi;
145  patchSchedule_[patchEvali++].init = false;
146  }
147  }
148 
149  // 2. All processor patches
150  // ~~~~~~~~~~~~~~~~~~~~~~~~
151 
152  // Determine the schedule for all. Insert processor pair once
153  // to determine the schedule. Each processor pair stands for both
154  // send and receive.
155  label nComms = 0;
156  forAll(*this, proci)
157  {
158  nComms += operator[](proci).size();
159  }
160  DynamicList<labelPair> comms(nComms);
161 
162  forAll(*this, proci)
163  {
164  const labelList& nbrs = operator[](proci);
165 
166  forAll(nbrs, i)
167  {
168  if (proci < nbrs[i])
169  {
170  comms.append(labelPair(proci, nbrs[i]));
171  }
172  }
173  }
174  comms.shrink();
175 
176  // Determine a schedule.
177  labelList mySchedule
178  (
180  (
181  Pstream::nProcs(comm),
182  comms
183  ).procSchedule()[Pstream::myProcNo(comm)]
184  );
185 
186  forAll(mySchedule, iter)
187  {
188  label commI = mySchedule[iter];
189 
190  // Get the other processor
191  label nb = comms[commI][0];
192  if (nb == Pstream::myProcNo(comm))
193  {
194  nb = comms[commI][1];
195  }
196  label patchi = procPatchMap_[nb];
197 
198  if (Pstream::myProcNo(comm) > nb)
199  {
200  patchSchedule_[patchEvali].patch = patchi;
201  patchSchedule_[patchEvali++].init = true;
202  patchSchedule_[patchEvali].patch = patchi;
203  patchSchedule_[patchEvali++].init = false;
204  }
205  else
206  {
207  patchSchedule_[patchEvali].patch = patchi;
208  patchSchedule_[patchEvali++].init = false;
209  patchSchedule_[patchEvali].patch = patchi;
210  patchSchedule_[patchEvali++].init = true;
211  }
212  }
213  }
214  else
215  {
216  patchSchedule_ = nonBlockingSchedule(patches);
217  }
218 }
219 
220 
221 // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
222 
223 template<class Container, class ProcPatch>
226 (
227  const Container& patches
228 )
229 {
230  lduSchedule patchSchedule(2*patches.size());
231 
232  label patchEvali = 0;
233 
234  // 1. All non-processor patches
235  // ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
236 
237  // Have evaluate directly after initEvaluate. Could have them separated
238  // as long as they're not intermingled with processor patches since
239  // then e.g. any reduce parallel traffic would interfere with the
240  // processor swaps.
241 
242  forAll(patches, patchi)
243  {
244  if (!isA<ProcPatch>(patches[patchi]))
245  {
246  patchSchedule[patchEvali].patch = patchi;
247  patchSchedule[patchEvali++].init = true;
248  patchSchedule[patchEvali].patch = patchi;
249  patchSchedule[patchEvali++].init = false;
250  }
251  }
252 
253  // 2. All processor patches
254  // ~~~~~~~~~~~~~~~~~~~~~~~~
255 
256  // 2a. initEvaluate
257  forAll(patches, patchi)
258  {
259  if (isA<ProcPatch>(patches[patchi]))
260  {
261  patchSchedule[patchEvali].patch = patchi;
262  patchSchedule[patchEvali++].init = true;
263  }
264  }
265 
266  // 2b. evaluate
267  forAll(patches, patchi)
268  {
269  if (isA<ProcPatch>(patches[patchi]))
270  {
271  patchSchedule[patchEvali].patch = patchi;
272  patchSchedule[patchEvali++].init = false;
273  }
274  }
275 
276  return patchSchedule;
277 }
278 
279 
280 // ************************************************************************* //
List< labelList > labelListList
A List of labelList.
Definition: labelList.H:57
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:434
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
dimensioned< Type > max(const dimensioned< Type > &, const dimensioned< Type > &)
Determines processor-processor connection. After instantiation contains on all processors the process...
Various functions to operate on Lists.
List< bool > boolList
Bool container classes.
Definition: boolList.H:50
ProcessorTopology(const Container &patches, const label comm)
Construct from boundaryMesh.
A 1D vector of objects of type <T> that resizes itself as necessary to accept the new objects...
Definition: DynamicList.H:56
static lduSchedule nonBlockingSchedule(const Container &patches)
Calculate non-blocking (i.e. unscheduled) schedule.
DynamicList< T, SizeInc, SizeMult, SizeDiv > & append(const T &)
Append an element at the end of the list.
Definition: DynamicListI.H:296
Pair< label > labelPair
Label pair.
Definition: labelPair.H:48
List< label > labelList
A List of labels.
Definition: labelList.H:56
DynamicList< T, SizeInc, SizeMult, SizeDiv > & shrink()
Shrink the allocated space to the number of elements used.
Definition: DynamicListI.H:252
Determines the order in which a set of processors should communicate with one another.
Definition: commSchedule.H:65
label patchi