UPstream.H
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration |
5  \\ / A nd | Copyright (C) 2011-2014 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 Class
25  Foam::UPstream
26 
27 Description
28  Inter-processor communications stream
29 
30 SourceFiles
31  UPstream.C
32  UPstreamCommsStruct.C
33  gatherScatter.C
34  combineGatherScatter.C
35  gatherScatterList.C
36 
37 \*---------------------------------------------------------------------------*/
38 
39 #ifndef UPstream_H
40 #define UPstream_H
41 
42 #include "labelList.H"
43 #include "DynamicList.H"
44 #include "HashTable.H"
45 #include "string.H"
46 #include "NamedEnum.H"
47 #include "ListOps.H"
48 #include "LIFOStack.H"
49 
50 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
51 
52 namespace Foam
53 {
54 
55 /*---------------------------------------------------------------------------*\
56  Class UPstream Declaration
57 \*---------------------------------------------------------------------------*/
58 
59 class UPstream
60 {
61 
62 public:
63 
64  //- Types of communications
65  enum commsTypes
66  {
70  };
71 
73 
74  // Public classes
75 
76  //- Structure for communicating between processors
77  class commsStruct
78  {
79  // Private data
80 
81  //- procID of above processor
82  label above_;
83 
84  //- procIDs of processors directly below me
85  labelList below_;
86 
87  //- procIDs of all processors below (so not just directly below)
88  labelList allBelow_;
89 
90  //- procIDs of all processors not below. (inverse set of
91  // allBelow_ and minus myProcNo)
92  labelList allNotBelow_;
93 
94 
95  public:
96 
97  // Constructors
98 
99  //- Construct null
100  commsStruct();
101 
102  //- Construct from components
104  (
105  const label,
106  const labelList&,
107  const labelList&,
108  const labelList&
109  );
110 
111  //- Construct from components; construct allNotBelow_
113  (
114  const label nProcs,
115  const label myProcID,
116  const label,
117  const labelList&,
118  const labelList&
119  );
120 
121 
122  // Member Functions
123 
124  // Access
126  label above() const
127  {
128  return above_;
129  }
131  const labelList& below() const
132  {
133  return below_;
134  }
136  const labelList& allBelow() const
137  {
138  return allBelow_;
139  }
141  const labelList& allNotBelow() const
142  {
143  return allNotBelow_;
144  }
145 
146 
147  // Member operators
148 
149  bool operator==(const commsStruct&) const;
150 
151  bool operator!=(const commsStruct&) const;
152 
153 
154  // Ostream Operator
155 
156  friend Ostream& operator<<(Ostream&, const commsStruct&);
157  };
158 
159 
160  //- combineReduce operator for lists. Used for counting.
161  class listEq
162  {
163 
164  public:
165 
166  template<class T>
167  void operator()(T& x, const T& y) const
168  {
169  forAll(y, i)
170  {
171  if (y[i].size())
172  {
173  x[i] = y[i];
174  }
175  }
176  }
177  };
178 
179 
180 private:
181 
182  // Private data
183 
184  static bool parRun_;
185  static int msgType_;
186 
187  // Communicator specific data
188 
189  static LIFOStack<label> freeComms_;
190  static DynamicList<int> myProcNo_;
191  static DynamicList<List<int> > procIDs_;
192  static DynamicList<label> parentCommunicator_;
193 
194  static DynamicList<List<commsStruct> > linearCommunication_;
195  static DynamicList<List<commsStruct> > treeCommunication_;
196 
197 
198  // Private Member Functions
199 
200  //- Set data for parallel running
201  static void setParRun(const label nProcs);
202 
203  //- Calculate linear communication schedule
204  static List<commsStruct> calcLinearComm(const label nProcs);
205 
206  //- Calculate tree communication schedule
207  static List<commsStruct> calcTreeComm(const label nProcs);
208 
209  //- Helper function for tree communication schedule determination
210  // Collects all processorIDs below a processor
211  static void collectReceives
212  (
213  const label procID,
214  const List<DynamicList<label> >& receives,
215  DynamicList<label>& allReceives
216  );
217 
218  //- Allocate a communicator with index
219  static void allocatePstreamCommunicator
220  (
221  const label parentIndex,
222  const label index
223  );
224 
225  //- Free a communicator
226  static void freePstreamCommunicator
227  (
228  const label index
229  );
230 
231 protected:
232 
233  // Protected data
234 
235  //- Communications type of this stream
237 
238 public:
239 
240  // Declare name of the class and its debug switch
241  ClassName("UPstream");
242 
243 
244  // Static data
245 
246  //- Should compact transfer be used in which floats replace doubles
247  // reducing the bandwidth requirement at the expense of some loss
248  // in accuracy
249  static bool floatTransfer;
250 
251  //- Number of processors at which the sum algorithm changes from linear
252  // to tree
253  static int nProcsSimpleSum;
254 
255  //- Default commsType
257 
258  //- Number of polling cycles in processor updates
259  static int nPollProcInterfaces;
260 
261  //- Default communicator (all processors)
262  static label worldComm;
263 
264  //- Debugging: warn for use of any communicator differing from warnComm
265  static label warnComm;
266 
267 
268  // Constructors
269 
270  //- Construct given optional buffer size
272  :
273  commsType_(commsType)
274  {}
275 
276 
277  // Member functions
278 
279  //- Allocate a new communicator
281  (
282  const label parent,
283  const labelList& subRanks,
284  const bool doPstream = true
285  );
286 
287  //- Free a previously allocated communicator
288  static void freeCommunicator
289  (
290  const label communicator,
291  const bool doPstream = true
292  );
293 
294  //- Free all communicators
295  static void freeCommunicators(const bool doPstream);
296 
297  //- Helper class for allocating/freeing communicators
298  class communicator
299  {
300  label comm_;
301 
302  //- Disallow copy and assignment
303  communicator(const communicator&);
304  void operator=(const communicator&);
305 
306  public:
307 
308  communicator
309  (
310  const label parent,
311  const labelList& subRanks,
312  const bool doPstream
313  )
314  :
315  comm_(allocateCommunicator(parent, subRanks, doPstream))
316  {}
318  ~communicator()
319  {
320  freeCommunicator(comm_);
321  }
323  operator label() const
324  {
325  return comm_;
326  }
327  };
328 
329  //- Return physical processor number (i.e. processor number in
330  // worldComm) given communicator and procssor
331  static int baseProcNo(const label myComm, const int procID);
332 
333  //- Return processor number in communicator (given physical processor
334  // number) (= reverse of baseProcNo)
335  static label procNo(const label comm, const int baseProcID);
336 
337  //- Return processor number in communicator (given processor number
338  // and communicator)
339  static label procNo
340  (
341  const label myComm,
342  const label currentComm,
343  const int currentProcID
344  );
345 
346  //- Add the valid option this type of communications library
347  // adds/requires on the command line
348  static void addValidParOptions(HashTable<string>& validParOptions);
349 
350  //- Initialisation function called from main
351  // Spawns slave processes and initialises inter-communication
352  static bool init(int& argc, char**& argv);
353 
354  // Non-blocking comms
355 
356  //- Get number of outstanding requests
357  static label nRequests();
358 
359  //- Truncate number of outstanding requests
360  static void resetRequests(const label sz);
361 
362  //- Wait until all requests (from start onwards) have finished.
363  static void waitRequests(const label start = 0);
364 
365  //- Wait until request i has finished.
366  static void waitRequest(const label i);
367 
368  //- Non-blocking comms: has request i finished?
369  static bool finishedRequest(const label i);
370 
371  static int allocateTag(const char*);
372 
373  static int allocateTag(const word&);
374 
375  static void freeTag(const char*, const int tag);
376 
377  static void freeTag(const word&, const int tag);
378 
379 
380  //- Is this a parallel run?
381  static bool& parRun()
382  {
383  return parRun_;
384  }
385 
386  //- Number of processes in parallel run
387  static label nProcs(const label communicator = 0)
388  {
389  return procIDs_[communicator].size();
390  }
391 
392  //- Process index of the master
393  static int masterNo()
394  {
395  return 0;
396  }
397 
398  //- Am I the master process
399  static bool master(const label communicator = 0)
400  {
401  return myProcNo_[communicator] == masterNo();
402  }
403 
404  //- Number of this process (starting from masterNo() = 0)
405  static int myProcNo(const label communicator = 0)
406  {
407  return myProcNo_[communicator];
408  }
410  static label parent(const label communicator)
411  {
412  return parentCommunicator_(communicator);
413  }
414 
415  //- Process ID of given process index
416  static List<int>& procID(label communicator)
417  {
418  return procIDs_[communicator];
419  }
420 
421  //- Process index of first slave
422  static int firstSlave()
423  {
424  return 1;
425  }
426 
427  //- Process index of last slave
428  static int lastSlave(const label communicator = 0)
429  {
430  return nProcs(communicator) - 1;
431  }
432 
433  //- Communication schedule for linear all-to-master (proc 0)
435  (
436  const label communicator = 0
437  )
438  {
439  return linearCommunication_[communicator];
440  }
441 
442  //- Communication schedule for tree all-to-master (proc 0)
444  (
445  const label communicator = 0
446  )
447  {
448  return treeCommunication_[communicator];
449  }
450 
451  //- Message tag of standard messages
452  static int& msgType()
453  {
454  return msgType_;
455  }
456 
457 
458  //- Get the communications type of the stream
459  commsTypes commsType() const
460  {
461  return commsType_;
462  }
463 
464  //- Set the communications type of the stream
466  {
467  commsTypes oldCommsType = commsType_;
468  commsType_ = ct;
469  return oldCommsType;
470  }
471 
472 
473  //- Exit program
474  static void exit(int errnum = 1);
475 
476  //- Abort program
477  static void abort();
478 
479 
480 };
481 
482 
484 
485 
486 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
487 
488 } // End namespace Foam
489 
490 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
491 
492 #endif
493 
494 // ************************************************************************* //
friend Ostream & operator<<(Ostream &, const commsStruct &)
static bool & parRun()
Is this a parallel run?
Definition: UPstream.H:380
Inter-processor communications stream.
Definition: UPstream.H:58
static int allocateTag(const char *)
Definition: UPstream.C:573
static bool finishedRequest(const label i)
Non-blocking comms: has request i finished?
Definition: UPstream.C:114
static void resetRequests(const label sz)
Truncate number of outstanding requests.
Definition: UPstream.C:102
void T(FieldField< Field, Type > &f1, const FieldField< Field, Type > &f2)
static label allocateCommunicator(const label parent, const labelList &subRanks, const bool doPstream=true)
Allocate a new communicator.
Definition: UPstream.C:229
static int baseProcNo(const label myComm, const int procID)
Return physical processor number (i.e. processor number in.
Definition: UPstream.C:337
An STL-conforming hash table.
Definition: HashTable.H:61
A class for handling words, derived from string.
Definition: word.H:59
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
Structure for communicating between processors.
Definition: UPstream.H:76
static void freeCommunicator(const label communicator, const bool doPstream=true)
Free a previously allocated communicator.
Definition: UPstream.C:298
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:76
Various functions to operate on Lists.
static void abort()
Abort program.
Definition: UPstream.C:52
static bool floatTransfer
Should compact transfer be used in which floats replace doubles.
Definition: UPstream.H:248
static label parent(const label communicator)
Definition: UPstream.H:409
Namespace for OpenFOAM.
static void exit(int errnum=1)
Exit program.
Definition: UPstream.C:46
static void waitRequests(const label start=0)
Wait until all requests (from start onwards) have finished.
Definition: UPstream.C:106
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:386
bool operator!=(const commsStruct &) const
static void freeTag(const char *, const int tag)
Definition: UPstream.C:631
static const List< commsStruct > & linearCommunication(const label communicator=0)
Communication schedule for linear all-to-master (proc 0)
Definition: UPstream.H:434
static void addValidParOptions(HashTable< string > &validParOptions)
Add the valid option this type of communications library.
Definition: UPstream.C:31
static List< int > & procID(label communicator)
Process ID of given process index.
Definition: UPstream.H:415
static label warnComm
Debugging: warn for use of any communicator differing from warnComm.
Definition: UPstream.H:264
label above() const
Definition: UPstream.H:125
commsTypes
Types of communications.
Definition: UPstream.H:64
#define forAll(list, i)
Definition: UList.H:421
static int & msgType()
Message tag of standard messages.
Definition: UPstream.H:451
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:404
const labelList & below() const
Definition: UPstream.H:130
static label nRequests()
Get number of outstanding requests.
Definition: UPstream.C:96
commsTypes commsType() const
Get the communications type of the stream.
Definition: UPstream.H:458
ClassName("UPstream")
UPstream(const commsTypes commsType)
Construct given optional buffer size.
Definition: UPstream.H:270
scalar y
static int nPollProcInterfaces
Number of polling cycles in processor updates.
Definition: UPstream.H:258
static int masterNo()
Process index of the master.
Definition: UPstream.H:392
static const NamedEnum< commsTypes, 3 > commsTypeNames
Definition: UPstream.H:71
const labelList & allBelow() const
Definition: UPstream.H:135
An Ostream is an abstract base class for all output systems (streams, files, token lists...
Definition: Ostream.H:53
static bool master(const label communicator=0)
Am I the master process.
Definition: UPstream.H:398
static void waitRequest(const label i)
Wait until request i has finished.
Definition: UPstream.C:110
static const List< commsStruct > & treeCommunication(const label communicator=0)
Communication schedule for tree all-to-master (proc 0)
Definition: UPstream.H:443
commsTypes commsType_
Communications type of this stream.
Definition: UPstream.H:235
static int firstSlave()
Process index of first slave.
Definition: UPstream.H:421
static int lastSlave(const label communicator=0)
Process index of last slave.
Definition: UPstream.H:427
static label procNo(const label comm, const int baseProcID)
Return processor number in communicator (given physical processor.
Definition: UPstream.C:353
static commsTypes defaultCommsType
Default commsType.
Definition: UPstream.H:255
combineReduce operator for lists. Used for counting.
Definition: UPstream.H:160
const labelList & allNotBelow() const
Definition: UPstream.H:140
static label worldComm
Default communicator (all processors)
Definition: UPstream.H:261
static void freeCommunicators(const bool doPstream)
Free all communicators.
Definition: UPstream.C:325
bool operator==(const commsStruct &) const
static bool init(int &argc, char **&argv)
Initialisation function called from main.
Definition: UPstream.C:35
static int nProcsSimpleSum
Number of processors at which the sum algorithm changes from linear.
Definition: UPstream.H:252