UPstream.H
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration |
5  \\ / A nd | Copyright (C) 2011-2016 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 Class
25  Foam::UPstream
26 
27 Description
28  Inter-processor communications stream
29 
30 SourceFiles
31  UPstream.C
32  UPstreamCommsStruct.C
33  gatherScatter.C
34  combineGatherScatter.C
35  gatherScatterList.C
36 
37 \*---------------------------------------------------------------------------*/
38 
39 #ifndef UPstream_H
40 #define UPstream_H
41 
42 #include "labelList.H"
43 #include "DynamicList.H"
44 #include "HashTable.H"
45 #include "string.H"
46 #include "NamedEnum.H"
47 #include "ListOps.H"
48 #include "LIFOStack.H"
49 
50 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
51 
52 namespace Foam
53 {
54 
55 /*---------------------------------------------------------------------------*\
56  Class UPstream Declaration
57 \*---------------------------------------------------------------------------*/
58 
59 class UPstream
60 {
61 
62 public:
63 
64  //- Types of communications
65  enum commsTypes
66  {
70  };
71 
73 
74  // Public classes
75 
76  //- Structure for communicating between processors
77  class commsStruct
78  {
79  // Private data
80 
81  //- procID of above processor
82  label above_;
83 
84  //- procIDs of processors directly below me
85  labelList below_;
86 
87  //- procIDs of all processors below (so not just directly below)
88  labelList allBelow_;
89 
90  //- procIDs of all processors not below. (inverse set of
91  // allBelow_ and minus myProcNo)
92  labelList allNotBelow_;
93 
94 
95  public:
96 
97  // Constructors
98 
99  //- Construct null
100  commsStruct();
101 
102  //- Construct from components
104  (
105  const label,
106  const labelList&,
107  const labelList&,
108  const labelList&
109  );
110 
111  //- Construct from components; construct allNotBelow_
113  (
114  const label nProcs,
115  const label myProcID,
116  const label,
117  const labelList&,
118  const labelList&
119  );
120 
121 
122  // Member Functions
123 
124  // Access
126  label above() const
127  {
128  return above_;
129  }
131  const labelList& below() const
132  {
133  return below_;
134  }
136  const labelList& allBelow() const
137  {
138  return allBelow_;
139  }
141  const labelList& allNotBelow() const
142  {
143  return allNotBelow_;
144  }
145 
146 
147  // Member operators
148 
149  bool operator==(const commsStruct&) const;
150 
151  bool operator!=(const commsStruct&) const;
152 
153 
154  // Ostream Operator
155 
156  friend Ostream& operator<<(Ostream&, const commsStruct&);
157  };
158 
159 
160  //- combineReduce operator for lists. Used for counting.
161  class listEq
162  {
163 
164  public:
165 
166  template<class T>
167  void operator()(T& x, const T& y) const
168  {
169  forAll(y, i)
170  {
171  if (y[i].size())
172  {
173  x[i] = y[i];
174  }
175  }
176  }
177  };
178 
179 
180 private:
181 
182  // Private data
183 
184  //- By default this is not a parallel run
185  static bool parRun_;
186 
187  //- Standard transfer message type
188  static int msgType_;
189 
190  // Communicator specific data
191 
192  //- Free communicators
193  static LIFOStack<label> freeComms_;
194 
195  //- My processor number
196  static DynamicList<int> myProcNo_;
197 
198  //- List of process IDs
199  static DynamicList<List<int>> procIDs_;
200 
201  //- Parent communicator
202  static DynamicList<label> parentCommunicator_;
203 
204  //- Linear communication schedule
205  static DynamicList<List<commsStruct>> linearCommunication_;
206 
207  //- Multi level communication schedule
208  static DynamicList<List<commsStruct>> treeCommunication_;
209 
210 
211  // Private Member Functions
212 
213  //- Set data for parallel running
214  static void setParRun(const label nProcs);
215 
216  //- Calculate linear communication schedule
217  static List<commsStruct> calcLinearComm(const label nProcs);
218 
219  //- Calculate tree communication schedule
220  static List<commsStruct> calcTreeComm(const label nProcs);
221 
222  //- Helper function for tree communication schedule determination
223  // Collects all processorIDs below a processor
224  static void collectReceives
225  (
226  const label procID,
227  const List<DynamicList<label>>& receives,
228  DynamicList<label>& allReceives
229  );
230 
231  //- Allocate a communicator with index
232  static void allocatePstreamCommunicator
233  (
234  const label parentIndex,
235  const label index
236  );
237 
238  //- Free a communicator
239  static void freePstreamCommunicator
240  (
241  const label index
242  );
243 
244 protected:
245 
246  // Protected data
247 
248  //- Communications type of this stream
250 
251 public:
252 
253  // Declare name of the class and its debug switch
254  ClassName("UPstream");
255 
256 
257  // Static data
258 
259  //- Should compact transfer be used in which floats replace doubles
260  // reducing the bandwidth requirement at the expense of some loss
261  // in accuracy
262  static bool floatTransfer;
263 
264  //- Number of processors at which the sum algorithm changes from linear
265  // to tree
266  static int nProcsSimpleSum;
267 
268  //- Default commsType
270 
271  //- Number of polling cycles in processor updates
272  static int nPollProcInterfaces;
273 
274  //- Default communicator (all processors)
275  static label worldComm;
276 
277  //- Debugging: warn for use of any communicator differing from warnComm
278  static label warnComm;
279 
280 
281  // Constructors
282 
283  //- Construct given optional buffer size
285  :
286  commsType_(commsType)
287  {}
288 
289 
290  // Member functions
291 
292  //- Allocate a new communicator
294  (
295  const label parent,
296  const labelList& subRanks,
297  const bool doPstream = true
298  );
299 
300  //- Free a previously allocated communicator
301  static void freeCommunicator
302  (
303  const label communicator,
304  const bool doPstream = true
305  );
306 
307  //- Free all communicators
308  static void freeCommunicators(const bool doPstream);
309 
310  //- Helper class for allocating/freeing communicators
311  class communicator
312  {
313  label comm_;
314 
315  //- Disallow copy and assignment
316  communicator(const communicator&);
317  void operator=(const communicator&);
318 
319  public:
320 
321  communicator
322  (
323  const label parent,
324  const labelList& subRanks,
325  const bool doPstream
326  )
327  :
328  comm_(allocateCommunicator(parent, subRanks, doPstream))
329  {}
331  ~communicator()
332  {
333  freeCommunicator(comm_);
334  }
336  operator label() const
337  {
338  return comm_;
339  }
340  };
341 
342  //- Return physical processor number (i.e. processor number in
343  // worldComm) given communicator and procssor
344  static int baseProcNo(const label myComm, const int procID);
345 
346  //- Return processor number in communicator (given physical processor
347  // number) (= reverse of baseProcNo)
348  static label procNo(const label comm, const int baseProcID);
349 
350  //- Return processor number in communicator (given processor number
351  // and communicator)
352  static label procNo
353  (
354  const label myComm,
355  const label currentComm,
356  const int currentProcID
357  );
358 
359  //- Add the valid option this type of communications library
360  // adds/requires on the command line
361  static void addValidParOptions(HashTable<string>& validParOptions);
362 
363  //- Initialisation function called from main
364  // Spawns slave processes and initialises inter-communication
365  static bool init(int& argc, char**& argv);
366 
367  // Non-blocking comms
368 
369  //- Get number of outstanding requests
370  static label nRequests();
371 
372  //- Truncate number of outstanding requests
373  static void resetRequests(const label sz);
374 
375  //- Wait until all requests (from start onwards) have finished.
376  static void waitRequests(const label start = 0);
377 
378  //- Wait until request i has finished.
379  static void waitRequest(const label i);
380 
381  //- Non-blocking comms: has request i finished?
382  static bool finishedRequest(const label i);
383 
384  static int allocateTag(const char*);
385 
386  static int allocateTag(const word&);
387 
388  static void freeTag(const char*, const int tag);
389 
390  static void freeTag(const word&, const int tag);
391 
392 
393  //- Is this a parallel run?
394  static bool& parRun()
395  {
396  return parRun_;
397  }
398 
399  //- Number of processes in parallel run
400  static label nProcs(const label communicator = 0)
401  {
402  return procIDs_[communicator].size();
403  }
404 
405  //- Process index of the master
406  static int masterNo()
407  {
408  return 0;
409  }
410 
411  //- Am I the master process
412  static bool master(const label communicator = 0)
413  {
414  return myProcNo_[communicator] == masterNo();
415  }
416 
417  //- Number of this process (starting from masterNo() = 0)
418  static int myProcNo(const label communicator = 0)
419  {
420  return myProcNo_[communicator];
421  }
423  static label parent(const label communicator)
424  {
425  return parentCommunicator_(communicator);
426  }
427 
428  //- Process ID of given process index
429  static List<int>& procID(label communicator)
430  {
431  return procIDs_[communicator];
432  }
433 
434  //- Process index of first slave
435  static int firstSlave()
436  {
437  return 1;
438  }
439 
440  //- Process index of last slave
441  static int lastSlave(const label communicator = 0)
442  {
443  return nProcs(communicator) - 1;
444  }
445 
446  //- Communication schedule for linear all-to-master (proc 0)
448  (
449  const label communicator = 0
450  )
451  {
452  return linearCommunication_[communicator];
453  }
454 
455  //- Communication schedule for tree all-to-master (proc 0)
457  (
458  const label communicator = 0
459  )
460  {
461  return treeCommunication_[communicator];
462  }
463 
464  //- Message tag of standard messages
465  static int& msgType()
466  {
467  return msgType_;
468  }
469 
470 
471  //- Get the communications type of the stream
472  commsTypes commsType() const
473  {
474  return commsType_;
475  }
476 
477  //- Set the communications type of the stream
479  {
480  commsTypes oldCommsType = commsType_;
481  commsType_ = ct;
482  return oldCommsType;
483  }
484 
485 
486  //- Exit program
487  static void exit(int errnum = 1);
488 
489  //- Abort program
490  static void abort();
491 
492  //- Exchange label with all processors (in the communicator).
493  // sendData[proci] is the label to send to proci.
494  // After return recvData contains the data from the other processors.
495  static void allToAll
496  (
497  const labelUList& sendData,
498  labelUList& recvData,
499  const label communicator = 0
500  );
501 };
502 
503 
505 
506 
507 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
508 
509 } // End namespace Foam
510 
511 // * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * //
512 
513 #endif
514 
515 // ************************************************************************* //
static bool floatTransfer
Should compact transfer be used in which floats replace doubles.
Definition: UPstream.H:261
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:428
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
static int masterNo()
Process index of the master.
Definition: UPstream.H:405
commsTypes
Types of communications.
Definition: UPstream.H:64
static int firstSlave()
Process index of first slave.
Definition: UPstream.H:434
commsTypes commsType_
Communications type of this stream.
Definition: UPstream.H:248
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:76
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:417
static int nProcsSimpleSum
Number of processors at which the sum algorithm changes from linear.
Definition: UPstream.H:265
static bool master(const label communicator=0)
Am I the master process.
Definition: UPstream.H:411
static bool finishedRequest(const label i)
Non-blocking comms: has request i finished?
Definition: UPstream.C:125
static label procNo(const label comm, const int baseProcID)
Return processor number in communicator (given physical processor.
Definition: UPstream.C:371
static label nRequests()
Get number of outstanding requests.
Definition: UPstream.C:107
static int & msgType()
Message tag of standard messages.
Definition: UPstream.H:464
static label worldComm
Default communicator (all processors)
Definition: UPstream.H:274
static const List< commsStruct > & linearCommunication(const label communicator=0)
Communication schedule for linear all-to-master (proc 0)
Definition: UPstream.H:447
ClassName("UPstream")
Various functions to operate on Lists.
static void freeCommunicators(const bool doPstream)
Free all communicators.
Definition: UPstream.C:343
scalar y
const labelList & allBelow() const
Definition: UPstream.H:135
static int nPollProcInterfaces
Number of polling cycles in processor updates.
Definition: UPstream.H:271
static label parent(const label communicator)
Definition: UPstream.H:422
static int allocateTag(const char *)
Definition: UPstream.C:597
static void freeTag(const char *, const int tag)
Definition: UPstream.C:655
bool operator==(const commsStruct &) const
A class for handling words, derived from string.
Definition: word.H:59
bool operator!=(const commsStruct &) const
static void resetRequests(const label sz)
Truncate number of outstanding requests.
Definition: UPstream.C:113
static bool init(int &argc, char **&argv)
Initialisation function called from main.
Definition: UPstream.C:35
static const List< commsStruct > & treeCommunication(const label communicator=0)
Communication schedule for tree all-to-master (proc 0)
Definition: UPstream.H:456
combineReduce operator for lists. Used for counting.
Definition: UPstream.H:160
An STL-conforming hash table.
Definition: HashTable.H:61
static label warnComm
Debugging: warn for use of any communicator differing from warnComm.
Definition: UPstream.H:277
Structure for communicating between processors.
Definition: UPstream.H:76
A 1D vector of objects of type <T>, where the size of the vector is known and can be used for subscri...
Definition: HashTable.H:60
An Ostream is an abstract base class for all output systems (streams, files, token lists...
Definition: Ostream.H:53
static void exit(int errnum=1)
Exit program.
Definition: UPstream.C:46
static int baseProcNo(const label myComm, const int procID)
Return physical processor number (i.e. processor number in.
Definition: UPstream.C:355
static const NamedEnum< commsTypes, 3 > commsTypeNames
Definition: UPstream.H:71
void T(FieldField< Field, Type > &f1, const FieldField< Field, Type > &f2)
static void waitRequests(const label start=0)
Wait until all requests (from start onwards) have finished.
Definition: UPstream.C:117
const labelList & allNotBelow() const
Definition: UPstream.H:140
static void abort()
Abort program.
Definition: UPstream.C:52
static commsTypes defaultCommsType
Default commsType.
Definition: UPstream.H:268
static bool & parRun()
Is this a parallel run?
Definition: UPstream.H:393
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:399
commsTypes commsType() const
Get the communications type of the stream.
Definition: UPstream.H:471
UPstream(const commsTypes commsType)
Construct given optional buffer size.
Definition: UPstream.H:283
const labelList & below() const
Definition: UPstream.H:130
label above() const
Definition: UPstream.H:125
static void allToAll(const labelUList &sendData, labelUList &recvData, const label communicator=0)
Exchange label with all processors (in the communicator).
Definition: UPstream.C:85
friend Ostream & operator<<(Ostream &, const commsStruct &)
static label allocateCommunicator(const label parent, const labelList &subRanks, const bool doPstream=true)
Allocate a new communicator.
Definition: UPstream.C:250
static void waitRequest(const label i)
Wait until request i has finished.
Definition: UPstream.C:121
Inter-processor communications stream.
Definition: UPstream.H:58
static List< int > & procID(label communicator)
Process ID of given process index.
Definition: UPstream.H:428
static void addValidParOptions(HashTable< string > &validParOptions)
Add the valid option this type of communications library.
Definition: UPstream.C:31
static void freeCommunicator(const label communicator, const bool doPstream=true)
Free a previously allocated communicator.
Definition: UPstream.C:316
Namespace for OpenFOAM.
static int lastSlave(const label communicator=0)
Process index of last slave.
Definition: UPstream.H:440