fvMeshDistributorsLoadBalancer.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration | Website: https://openfoam.org
5  \\ / A nd | Copyright (C) 2021-2024 OpenFOAM Foundation
6  \\/ M anipulation |
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
27 #include "decompositionMethod.H"
28 #include "cpuLoad.H"
29 #include "globalMeshData.H"
31 
32 // * * * * * * * * * * * * * * Static Data Members * * * * * * * * * * * * * //
33 
34 namespace Foam
35 {
36 namespace fvMeshDistributors
37 {
40  (
43  fvMesh
44  );
45 }
46 }
47 
48 
49 // * * * * * * * * * * * * Protected Member Functions * * * * * * * * * * * //
50 
51 void Foam::fvMeshDistributors::loadBalancer::readDict()
52 {
54 
55  const dictionary& distributorDict(dict());
56 
57  multiConstraint_ =
58  distributorDict.lookupOrDefault<Switch>("multiConstraint", true);
59 }
60 
61 
62 // * * * * * * * * * * * * * * * * Constructors * * * * * * * * * * * * * * //
63 
65 :
66  distributor(mesh)
67 {
68  readDict();
69 }
70 
71 
72 // * * * * * * * * * * * * * * * * Destructor * * * * * * * * * * * * * * * //
73 
75 {}
76 
77 
78 // * * * * * * * * * * * * * * * Member Functions * * * * * * * * * * * * * //
79 
81 {
82  const fvMesh& mesh = this->mesh();
83 
84  bool redistributed = false;
85 
86  if
87  (
88  Pstream::nProcs() > 1
89  && mesh.time().timeIndex() - mesh.time().startTimeIndex() > 1
90  && timeIndex_ != mesh.time().timeIndex()
91  )
92  {
93  timeIndex_ = mesh.time().timeIndex();
94 
95  // Get the CPU time fer this processor which includes waiting time
96  const scalar timeStepCpuTime = cpuTime_.cpuTimeIncrement();
97 
98  // CPU loads per cell
99  HashTable<cpuLoad*> cpuLoads(this->mesh().lookupClass<cpuLoad>());
100 
101  if (!cpuLoads.size())
102  {
104  << "No CPU loads have been allocated"
105  << exit(FatalError);
106  }
107 
108  if (mesh.time().timeIndex() % redistributionInterval_ == 0)
109  {
110  timeIndex_ = mesh.time().timeIndex();
111 
112  scalarList procCpuLoads(cpuLoads.size());
113 
114  label l = 0;
115  forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
116  {
117  procCpuLoads[l++] = sum(*iter());
118  }
119 
120  List<scalarList> allProcCpuLoads(Pstream::nProcs());
121  allProcCpuLoads[Pstream::myProcNo()] = procCpuLoads;
122  Pstream::gatherList(allProcCpuLoads);
123  Pstream::scatterList(allProcCpuLoads);
124 
125  scalarList sumProcCpuLoads(procCpuLoads.size(), scalar(0));
126  scalarList maxProcCpuLoads(procCpuLoads.size(), scalar(0));
127  forAll(maxProcCpuLoads, l)
128  {
129  forAll(allProcCpuLoads, proci)
130  {
131  sumProcCpuLoads[l] += allProcCpuLoads[proci][l];
132 
133  maxProcCpuLoads[l] =
134  max(maxProcCpuLoads[l], allProcCpuLoads[proci][l]);
135  }
136  }
137 
138  // Sum over loads of the maximum load CPU time per processor
139  const scalar sumMaxProcCpuLoad(sum(maxProcCpuLoads));
140 
141  // Maximum number of cells per processor
142  const label maxNcells = returnReduce(mesh.nCells(), maxOp<label>());
143 
144  // Maximum processor CPU time spent doing basic CFD
145  const scalar maxBaseCpuTime =
146  returnReduce(timeStepCpuTime, maxOp<scalar>())
147  - sumMaxProcCpuLoad;
148 
149  const scalar cellBaseCpuTime = maxBaseCpuTime/maxNcells;
150 
151  // Processor CPU time spent doing basic CFD, not waiting
152  const scalar baseCpuTime = mesh.nCells()*cellBaseCpuTime;
153 
154  // Maximum total CPU time
155  const scalar maxProcCpuTime = maxBaseCpuTime + sumMaxProcCpuLoad;
156 
157  // Total CPU time for this processor not waiting
158  const scalar procCpuTime = baseCpuTime + sum(procCpuLoads);
159 
160  // Average processor CPU time
161  const scalar averageProcessorCpuTime =
162  returnReduce(procCpuTime, sumOp<scalar>())/Pstream::nProcs();
163 
164  const scalar imbalance =
165  (maxProcCpuTime - averageProcessorCpuTime)
166  /averageProcessorCpuTime;
167 
168  Info<< nl << type() << nl;
169 
170  l = 0;
171  forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
172  {
173  Info<< " Imbalance of load " << iter()->name() << ": "
174  << (
175  maxProcCpuLoads[l]
176  - sumProcCpuLoads[l]/Pstream::nProcs()
177  )/averageProcessorCpuTime
178  << endl;
179  l++;
180  }
181 
182  Info<< " Imbalance of base load " << ": "
183  << (
184  maxBaseCpuTime
185  - mesh.globalData().nTotalCells()*cellBaseCpuTime
186  /Pstream::nProcs()
187  )/averageProcessorCpuTime
188  << endl;
189 
190  Info<< " Total imbalance " << imbalance << endl;
191 
192  if (imbalance > maxImbalance_)
193  {
194  Info<< " Redistributing mesh" << endl;
195 
196  scalarField weights;
197 
198  if (multiConstraint_)
199  {
200  const label nWeights = cpuLoads.size() + 1;
201 
202  weights.setSize(nWeights*mesh.nCells());
203 
204  for (label i=0; i<mesh.nCells(); i++)
205  {
206  weights[nWeights*i] = cellBaseCpuTime;
207  }
208 
209  label l = 1;
210  forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
211  {
212  const scalarField& cpuLoadField = *iter();
213 
214  forAll(cpuLoadField, i)
215  {
216  weights[nWeights*i + l] = cpuLoadField[i];
217  }
218 
219  iter()->checkOut();
220 
221  l++;
222  }
223  }
224  else
225  {
226  weights.setSize(mesh.nCells(), cellBaseCpuTime);
227 
228  forAllConstIter(HashTable<cpuLoad*>, cpuLoads, iter)
229  {
230  weights += *iter();
231  iter()->checkOut();
232  }
233  }
234 
235  // Create new decomposition distribution
236  const labelList distribution
237  (
238  distributor_->decompose(mesh, weights)
239  );
240 
241  distribute(distribution);
242 
243  redistributed = true;
244 
245  Info<< endl;
246  }
247  else
248  {
249  forAllIter(HashTable<cpuLoad*>, cpuLoads, iter)
250  {
251  iter()->checkOut();
252  }
253  }
254  }
255  else
256  {
257  forAllIter(HashTable<cpuLoad*>, cpuLoads, iter)
258  {
259  iter()->checkOut();
260  }
261  }
262  }
263 
264  return redistributed;
265 }
266 
267 
268 // ************************************************************************* //
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:434
#define forAllIter(Container, container, iter)
Iterate across all elements in the container object of type.
Definition: UList.H:459
#define forAllConstIter(Container, container, iter)
Iterate across all elements in the container object of type.
Definition: UList.H:477
Macros for easy insertion into run-time selection tables.
An STL-conforming hash table.
Definition: HashTable.H:127
label size() const
Return number of elements in table.
Definition: HashTableI.H:65
void size(const label)
Override size to be inconsistent with allocated storage.
Definition: ListI.H:164
void setSize(const label)
Reset size of List.
Definition: List.C:281
static void scatterList(const List< commsStruct > &comms, List< T > &Values, const int tag, const label comm)
Scatter data. Reverse of gatherList.
static void gatherList(const List< commsStruct > &comms, List< T > &Values, const int tag, const label comm)
Gather data but keep individual values separate.
label timeIndex() const
Return current time index.
Definition: TimeStateI.H:28
virtual label startTimeIndex() const
Return start time index.
Definition: Time.C:785
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:411
static int myProcNo(const label communicator=0)
Number of this process (starting from masterNo() = 0)
Definition: UPstream.H:429
Base class for statistical distributions.
Definition: distribution.H:76
Abstract base class for fvMesh movers.
const dictionary & dict() const
Return the dynamicMeshDict/distributor sub-dict.
Dynamic mesh redistribution using the distributor specified in decomposeParDict.
void readDict()
Read the projection parameters from dictionary.
Dynamic mesh redistribution using the distributor specified in decomposeParDict.
loadBalancer(fvMesh &mesh)
Construct from fvMesh.
Mesh data needed to do the Finite Volume discretisation.
Definition: fvMesh.H:99
const Time & time() const
Return the top-level database.
Definition: fvMesh.H:418
label nTotalCells() const
Return total number of cells in decomposed mesh.
const globalMeshData & globalData() const
Return parallel info.
Definition: polyMesh.C:1515
label nCells() const
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
Definition: error.H:334
addToRunTimeSelectionTable(fvMeshDistributor, none, fvMesh)
Namespace for OpenFOAM.
errorManipArg< error, int > exit(error &err, const int errNo=1)
Definition: errorManip.H:124
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
Ostream & endl(Ostream &os)
Add newline and flush stream.
Definition: Ostream.H:257
dimensioned< Type > sum(const DimensionedField< Type, GeoMesh > &df)
messageStream Info
T returnReduce(const T &Value, const BinaryOp &bop, const int tag=Pstream::msgType(), const label comm=UPstream::worldComm)
layerAndWeight max(const layerAndWeight &a, const layerAndWeight &b)
error FatalError
static const char nl
Definition: Ostream.H:266
fileType type(const fileName &, const bool checkVariants=true, const bool followLink=true)
Return the file type: directory or file.
Definition: POSIX.C:488