36 void Foam::domainDecomposition::mark
45 label pointi = zoneElems[i];
47 if (elementToZone[pointi] == -1)
50 elementToZone[pointi] = zoneI;
52 else if (elementToZone[pointi] >= 0)
55 elementToZone[pointi] = -2;
61 void Foam::domainDecomposition::addInterProcFace
64 const label ownerProc,
66 const label subPatchID,
67 List<Map<label>>& nbrToInterPatch,
68 List<DynamicList<DynamicList<label>>>& interPatchFaces,
69 List<labelListList>& subPatchIDs,
70 List<labelListList>& subPatchStarts
74 label toNbrProcPatchi = -1, toOwnerProcPatchi = -1;
75 if (!nbrToInterPatch[ownerProc].
found(nbrProc))
77 toNbrProcPatchi = nbrToInterPatch[ownerProc].size();
78 nbrToInterPatch[ownerProc].insert(nbrProc, toNbrProcPatchi);
79 interPatchFaces[ownerProc].append(DynamicList<label>());
80 subPatchIDs[ownerProc].append(
labelList(1, subPatchID));
83 if (facei != -1 && completeMesh().isInternalFace(facei))
85 toOwnerProcPatchi = nbrToInterPatch[nbrProc].size();
86 nbrToInterPatch[nbrProc].insert(ownerProc, toOwnerProcPatchi);
87 interPatchFaces[nbrProc].append(DynamicList<label>());
88 subPatchIDs[nbrProc].append(
labelList(1, subPatchID));
94 toNbrProcPatchi = nbrToInterPatch[ownerProc][nbrProc];
96 if (facei != -1 && completeMesh().isInternalFace(facei))
98 toOwnerProcPatchi = nbrToInterPatch[nbrProc][ownerProc];
103 if (subPatchIDs[ownerProc][toNbrProcPatchi].last() != subPatchID)
105 subPatchIDs[ownerProc][toNbrProcPatchi].append(subPatchID);
106 subPatchStarts[ownerProc][toNbrProcPatchi].append
108 interPatchFaces[ownerProc][toNbrProcPatchi].size()
111 if (facei != -1 && completeMesh().isInternalFace(facei))
113 subPatchIDs[nbrProc][toOwnerProcPatchi].append(subPatchID);
114 subPatchStarts[nbrProc][toOwnerProcPatchi].append
116 interPatchFaces[nbrProc][toOwnerProcPatchi].size()
124 interPatchFaces[ownerProc][toNbrProcPatchi].append(facei + 1);
126 if (completeMesh().isInternalFace(facei))
128 interPatchFaces[nbrProc][toOwnerProcPatchi].append(- facei - 1);
136 Info<<
"Calculating distribution of cells" <<
nl <<
endl;
138 cpuTime decompositionTime;
140 const dictionary decomposeParDict =
144 if (decomposeParDict.found(
"weightField"))
146 const word weightName = decomposeParDict.lookup(
"weightField");
153 completeMesh().time().
name(),
160 cellWeights = weights.primitiveField();
170 Info<<
nl <<
"Finished decomposition in "
171 << decompositionTime.elapsedCpuTime()
178 void Foam::domainDecomposition::processInterCyclics
181 const polyBoundaryMesh&
patches,
182 List<DynamicList<DynamicList<label>>>& interPatchFaces,
183 List<Map<label>>& procNbrToInterPatch,
184 List<labelListList>& subPatchIDs,
185 List<labelListList>& subPatchStarts,
195 const nonConformalCyclicPolyPatch& nccPp =
198 if (nccPp.owner() != owner)
continue;
200 const polyPatch& origPp = nccPp.origPatch();
201 const polyPatch& nbrOrigPp = nccPp.nbrPatch().origPatch();
203 const labelUList& origPatchFaceCells = origPp.faceCells();
204 const labelUList& nbrOrigPatchFaceCells = nbrOrigPp.faceCells();
208 forAll(origPatchFaceCells, origFacei)
210 const label ownerProc =
211 cellProc[origPatchFaceCells[origFacei]];
213 forAll(nbrOrigPatchFaceCells, nbrOrigFacei)
215 const label nbrProc =
216 cellProc[nbrOrigPatchFaceCells[nbrOrigFacei]];
220 (
first && ownerProc < nbrProc)
221 || (!
first && ownerProc > nbrProc)
241 const cyclicPolyPatch& cPp =
244 if (cPp.owner() != owner)
continue;
246 const labelUList& patchFaceCells = cPp.faceCells();
247 const labelUList& nbrPatchFaceCells = cPp.nbrPatch().faceCells();
250 forAll(patchFaceCells, facei)
252 const label ownerProc = cellProc[patchFaceCells[facei]];
253 const label nbrProc = cellProc[nbrPatchFaceCells[facei]];
257 (
first && ownerProc < nbrProc)
258 || (!
first && ownerProc > nbrProc)
279 void Foam::domainDecomposition::decompose()
282 cellProc_ = distributeCells();
288 Info<<
"Calculating original mesh data" <<
nl <<
endl;
291 const polyBoundaryMesh&
patches = completeMesh().boundaryMesh();
292 const faceList& fcs = completeMesh().faces();
293 const labelList& owner = completeMesh().faceOwner();
294 const labelList& neighbour = completeMesh().faceNeighbour();
299 Info<<
"Distributing cells to processors" <<
nl <<
endl;
304 Info<<
"Distributing faces to processors" <<
nl <<
endl;
310 List<DynamicList<label>> dynProcFaceAddressing(nProcs());
315 if (cellProc_[owner[facei]] == cellProc_[neighbour[facei]])
318 dynProcFaceAddressing[cellProc_[owner[facei]]].append(facei+1);
326 forAll(procPatchSize, proci)
329 procPatchStartIndex[proci].setSize(
patches.
size());
336 forAll(procPatchSize, proci)
338 procPatchSize[proci][
patchi] = 0;
339 procPatchStartIndex[proci][
patchi] =
340 dynProcFaceAddressing[proci].size();
352 forAll(patchFaceCells, facei)
354 const label curProc = cellProc_[patchFaceCells[facei]];
357 dynProcFaceAddressing[curProc].append(patchStart+facei+1);
360 procPatchSize[curProc][
patchi]++;
368 const cyclicPolyPatch& pp =
371 const labelUList& patchFaceCells = pp.faceCells();
372 const labelUList& nbrPatchFaceCells = pp.nbrPatch().faceCells();
374 forAll(patchFaceCells, facei)
376 const label curProc = cellProc_[patchFaceCells[facei]];
377 const label nbrProc = cellProc_[nbrPatchFaceCells[facei]];
379 if (curProc == nbrProc)
382 dynProcFaceAddressing[curProc].append(patchStart+facei+1);
385 procPatchSize[curProc][
patchi]++;
395 List<Map<label>> procNbrToInterPatch(nProcs());
398 List<DynamicList<DynamicList<label>>> interPatchFaces(nProcs());
401 List<labelListList> subPatchIDs(nProcs());
402 List<labelListList> subPatchStarts(nProcs());
407 const label ownerProc = cellProc_[owner[facei]];
408 const label nbrProc = cellProc_[neighbour[facei]];
410 if (ownerProc != nbrProc)
502 List<labelListList> procProcessorPatchSubPatchIDs(nProcs());
503 List<labelListList> procProcessorPatchSubPatchStarts(nProcs());
505 forAll(procNbrToInterPatch, proci)
507 label nInterfaces = procNbrToInterPatch[proci].size();
509 procNeighbourProcessors[proci].setSize(nInterfaces);
510 procProcessorPatchSize[proci].setSize(nInterfaces);
511 procProcessorPatchStartIndex[proci].setSize(nInterfaces);
512 procProcessorPatchSubPatchIDs[proci].setSize(nInterfaces);
513 procProcessorPatchSubPatchStarts[proci].setSize(nInterfaces);
516 const Map<label>& curNbrToInterPatch = procNbrToInterPatch[proci];
517 labelList nbrs = curNbrToInterPatch.toc();
521 DynamicList<DynamicList<label>>& curInterPatchFaces =
522 interPatchFaces[proci];
526 const label nbrProc = nbrs[i];
527 const label interPatch = curNbrToInterPatch[nbrProc];
529 procNeighbourProcessors[proci][i] = nbrProc;
530 procProcessorPatchSize[proci][i] =
531 curInterPatchFaces[interPatch].size();
532 procProcessorPatchStartIndex[proci][i] =
533 dynProcFaceAddressing[proci].size();
536 subPatchStarts[proci][interPatch].append
538 curInterPatchFaces[interPatch].size()
540 procProcessorPatchSubPatchIDs[proci][i].transfer
542 subPatchIDs[proci][interPatch]
544 procProcessorPatchSubPatchStarts[proci][i].transfer
546 subPatchStarts[proci][interPatch]
550 DynamicList<label>& interPatchFaces =
551 curInterPatchFaces[interPatch];
553 forAll(interPatchFaces, j)
555 dynProcFaceAddressing[proci].append(interPatchFaces[j]);
558 interPatchFaces.clearStorage();
561 curInterPatchFaces.clearStorage();
562 dynProcFaceAddressing[proci].shrink();
566 forAll(dynProcFaceAddressing, proci)
568 procFaceAddressing_[proci].transfer(dynProcFaceAddressing[proci]);
573 forAll(procPatchStartIndex, proci)
575 Info<<
"Processor:" << proci <<
endl;
576 Info<<
" total faces:" << procFaceAddressing_[proci].size()
579 const labelList& curProcPatchStartIndex =
580 procPatchStartIndex[proci];
585 <<
"\tstart:" << curProcPatchStartIndex[
patchi]
586 <<
"\tsize:" << procPatchSize[proci][
patchi]
592 forAll(procNeighbourProcessors, proci)
594 Info<<
"Processor " << proci <<
endl;
595 forAll(procNeighbourProcessors[proci], i)
597 Info<<
" nbr:" << procNeighbourProcessors[proci][i] <<
endl;
598 Info<<
" size:" << procProcessorPatchSize[proci][i] <<
endl;
599 Info<<
" start:" << procProcessorPatchStartIndex[proci][i]
608 forAll(procFaceAddressing_, proci)
610 Info<<
"Processor:" << proci <<
endl;
611 Info<<
" faces:" << procFaceAddressing_[proci] <<
endl;
615 Info<<
"Distributing points to processors" <<
nl <<
endl;
622 forAll(procPointAddressing_, proci)
627 const labelList& procFaceLabels = procFaceAddressing_[proci];
629 forAll(procFaceLabels, facei)
632 const labelList& facePoints = fcs[
mag(procFaceLabels[facei]) - 1];
634 forAll(facePoints, pointi)
642 labelList& procPointLabels = procPointAddressing_[proci];
646 label nUsedPoints = 0;
652 procPointLabels[nUsedPoints] = pointi;
659 procPointLabels.setSize(nUsedPoints);
662 Info<<
"Constructing processor meshes" <<
nl <<
endl;
674 forAll(completeMesh().pointZones(), zoneI)
676 mark(completeMesh().pointZones()[zoneI], zoneI, pointToZone);
680 labelList faceToZone(completeMesh().faces().size(), -1);
681 forAll(completeMesh().faceZones(), zoneI)
683 mark(completeMesh().faceZones()[zoneI], zoneI, faceToZone);
687 labelList cellToZone(completeMesh().nCells(), -1);
688 forAll(completeMesh().cellZones(), zoneI)
690 mark(completeMesh().cellZones()[zoneI], zoneI, cellToZone);
694 label maxProcCells = 0;
695 label totProcFaces = 0;
696 label maxProcPatches = 0;
697 label totProcPatches = 0;
698 label maxProcFaces = 0;
701 for (
label proci = 0; proci < nProcs(); proci++)
704 const labelList& curPointLabels = procPointAddressing_[proci];
705 const pointField& meshPoints = completeMesh().points();
708 forAll(curPointLabels, pointi)
710 procPoints[pointi] = meshPoints[curPointLabels[pointi]];
711 pointLookup[curPointLabels[pointi]] = pointi;
715 const labelList& curFaceLabels = procFaceAddressing_[proci];
716 const faceList& meshFaces = completeMesh().faces();
717 labelList faceLookup(completeMesh().nFaces(), -1);
718 faceList procFaces(curFaceLabels.size());
719 forAll(curFaceLabels, facei)
723 label curF =
mag(curFaceLabels[facei]) - 1;
725 faceLookup[curF] = facei;
730 if (curFaceLabels[facei] >= 0)
733 origFaceLabels = meshFaces[curF];
737 origFaceLabels = meshFaces[curF].reverseFace();
741 face& procFaceLabels = procFaces[facei];
743 procFaceLabels.
setSize(origFaceLabels.size());
745 forAll(origFaceLabels, pointi)
747 procFaceLabels[pointi] = pointLookup[origFaceLabels[pointi]];
752 const labelList& curCellLabels = procCellAddressing_[proci];
753 const cellList& meshCells = completeMesh().cells();
754 cellList procCells(curCellLabels.size());
755 forAll(curCellLabels, celli)
757 const labelList& origCellLabels = meshCells[curCellLabels[celli]];
758 cell& curCell = procCells[celli];
759 curCell.
setSize(origCellLabels.size());
760 forAll(origCellLabels, cellFacei)
762 curCell[cellFacei] = faceLookup[origCellLabels[cellFacei]];
775 completeMesh().facesInstance(),
776 runTimes_.procTimes()[proci]
783 fvMesh& procMesh = procMeshes_[proci];
784 procMesh.setPointsInstance(completeMesh().pointsInstance());
787 const labelList& curPatchSizes = procPatchSize[proci];
788 const labelList& curPatchStarts = procPatchStartIndex[proci];
789 const labelList& curNeighbourProcessors =
790 procNeighbourProcessors[proci];
791 const labelList& curProcessorPatchSizes =
792 procProcessorPatchSize[proci];
793 const labelList& curProcessorPatchStarts =
794 procProcessorPatchStartIndex[proci];
796 procProcessorPatchSubPatchIDs[proci];
798 procProcessorPatchSubPatchStarts[proci];
799 const polyPatchList& meshPatches = completeMesh().boundaryMesh();
802 label nInterProcPatches = 0;
803 forAll(curSubPatchIDs, procPatchi)
805 nInterProcPatches += curSubPatchIDs[procPatchi].size();
807 List<polyPatch*> procPatches
809 curPatchSizes.size() + nInterProcPatches,
818 const polyPatch& meshPatch = meshPatches[
patchi];
823 procMesh.boundaryMesh(),
833 forAll(curProcessorPatchSizes, procPatchi)
835 const labelList& subPatchID = curSubPatchIDs[procPatchi];
836 const labelList& subStarts = curSubStarts[procPatchi];
838 label curStart = curProcessorPatchStarts[procPatchi];
843 i < subPatchID.size() - 1
844 ? subStarts[i+1] - subStarts[i]
845 : curProcessorPatchSizes[procPatchi] - subStarts[i];
847 if (subPatchID[i] == -1)
851 new processorPolyPatch
856 procMesh.boundaryMesh(),
858 curNeighbourProcessors[procPatchi]
863 isA<nonConformalCyclicPolyPatch>
865 completeMesh().boundaryMesh()[subPatchID[i]]
871 const nonConformalCyclicPolyPatch& nccPp =
872 refCast<const nonConformalCyclicPolyPatch>
874 completeMesh().boundaryMesh()[subPatchID[i]]
878 new nonConformalProcessorCyclicPolyPatch
883 procMesh.boundaryMesh(),
885 curNeighbourProcessors[procPatchi],
887 nccPp.origPatch().name()
894 completeMesh().boundaryMesh()[subPatchID[i]]
899 const cyclicPolyPatch& cPp =
900 refCast<const cyclicPolyPatch>
902 completeMesh().boundaryMesh()[subPatchID[i]]
906 new processorCyclicPolyPatch
911 procMesh.boundaryMesh(),
913 curNeighbourProcessors[procPatchi],
920 <<
"Sub patch ID set for non-cyclic patch type"
931 procMesh.addFvPatches(procPatches);
938 const pointZoneList& pz = completeMesh().pointZones();
943 List<DynamicList<label>> zonePoints(pz.size());
948 zonePoints[zoneI].setCapacity(pz[zoneI].size()/nProcs());
953 forAll(curPointLabels, pointi)
955 label curPoint = curPointLabels[pointi];
957 label zoneI = pointToZone[curPoint];
962 zonePoints[zoneI].append(pointi);
964 else if (zoneI == -2)
969 label index = pz[zoneI].localIndex(curPoint);
973 zonePoints[zoneI].append(pointi);
979 procMesh.pointZones().setSize(zonePoints.size());
982 procMesh.pointZones().set
987 zonePoints[zoneI].shrink(),
988 procMesh.pointZones()
1002 const faceZoneList& fz = completeMesh().faceZones();
1007 List<DynamicList<label>> zoneFaces(fz.size());
1008 List<DynamicList<bool>> zoneFaceFlips(fz.size());
1013 label procSize = fz[zoneI].size()/nProcs();
1015 zoneFaces[zoneI].setCapacity(procSize);
1016 zoneFaceFlips[zoneI].setCapacity(procSize);
1022 forAll(curFaceLabels, facei)
1026 label curF =
mag(curFaceLabels[facei]) - 1;
1028 label zoneI = faceToZone[curF];
1033 zoneFaces[zoneI].append(facei);
1035 label index = fz[zoneI].localIndex(curF);
1037 bool flip = fz[zoneI].flipMap()[index];
1039 if (curFaceLabels[facei] < 0)
1044 zoneFaceFlips[zoneI].append(flip);
1046 else if (zoneI == -2)
1051 label index = fz[zoneI].localIndex(curF);
1055 zoneFaces[zoneI].append(facei);
1057 bool flip = fz[zoneI].flipMap()[index];
1059 if (curFaceLabels[facei] < 0)
1064 zoneFaceFlips[zoneI].append(flip);
1070 procMesh.faceZones().setSize(zoneFaces.size());
1073 procMesh.faceZones().set
1078 zoneFaces[zoneI].shrink(),
1079 zoneFaceFlips[zoneI].shrink(),
1080 procMesh.faceZones()
1094 const cellZoneList& cz = completeMesh().cellZones();
1099 List<DynamicList<label>> zoneCells(cz.size());
1104 zoneCells[zoneI].setCapacity(cz[zoneI].size()/nProcs());
1107 forAll(curCellLabels, celli)
1109 label curCelli = curCellLabels[celli];
1111 label zoneI = cellToZone[curCelli];
1116 zoneCells[zoneI].append(celli);
1118 else if (zoneI == -2)
1123 label index = cz[zoneI].localIndex(curCelli);
1127 zoneCells[zoneI].append(celli);
1133 procMesh.cellZones().setSize(zoneCells.size());
1136 procMesh.cellZones().set
1141 zoneCells[zoneI].shrink(),
1142 procMesh.cellZones()
1156 Info<<
"Processor " << proci <<
nl
1157 <<
" Number of cells = " << procMesh.nCells()
1160 maxProcCells =
max(maxProcCells, procMesh.nCells());
1162 label nBoundaryFaces = 0;
1163 label nProcPatches = 0;
1164 label nProcFaces = 0;
1168 if (isA<processorPolyPatch>(procMesh.boundaryMesh()[
patchi]))
1170 const processorPolyPatch& ppp =
1171 refCast<const processorPolyPatch>
1173 procMesh.boundaryMesh()[
patchi]
1176 Info<<
" Number of faces shared with processor "
1177 << ppp.neighbProcNo() <<
" = " << ppp.size() <<
endl;
1180 nProcFaces += ppp.size();
1184 nBoundaryFaces += procMesh.boundaryMesh()[
patchi].size();
1188 Info<<
" Number of processor patches = " << nProcPatches <<
nl
1189 <<
" Number of processor faces = " << nProcFaces <<
nl
1190 <<
" Number of boundary faces = " << nBoundaryFaces <<
nl
1193 totProcFaces += nProcFaces;
1194 totProcPatches += nProcPatches;
1195 maxProcPatches =
max(maxProcPatches, nProcPatches);
1196 maxProcFaces =
max(maxProcFaces, nProcFaces);
1201 scalar avgProcCells = scalar(completeMesh().nCells())/nProcs();
1202 scalar avgProcPatches = scalar(totProcPatches)/nProcs();
1203 scalar avgProcFaces = scalar(totProcFaces)/nProcs();
1206 if (totProcPatches == 0)
1210 if (totProcFaces == 0)
1215 Info<<
"Number of processor faces = " << totProcFaces/2 <<
nl
1216 <<
"Max number of cells = " << maxProcCells
1217 <<
" (" << 100.0*(maxProcCells-avgProcCells)/avgProcCells
1218 <<
"% above average " << avgProcCells <<
")" <<
nl
1219 <<
"Max number of processor patches = " << maxProcPatches
1220 <<
" (" << 100.0*(maxProcPatches-avgProcPatches)/avgProcPatches
1221 <<
"% above average " << avgProcPatches <<
")" <<
nl
1222 <<
"Max number of faces between processors = " << maxProcFaces
1223 <<
" (" << 100.0*(maxProcFaces-avgProcFaces)/avgProcFaces
1224 <<
"% above average " << avgProcFaces <<
")" <<
endl;
1227 procFaceAddressingBf_.clear();
1231 void Foam::domainDecomposition::decomposePoints()
1233 for (
label proci = 0; proci < nProcs(); proci++)
1235 fvMesh& procMesh = procMeshes_[proci];
1237 const label pointsCompare =
1240 completeMesh().pointsInstance(),
1241 procMeshes_[proci].pointsInstance()
1244 if (pointsCompare == -1)
1251 procPointAddressing_[proci]
#define forAll(list, i)
Loop across all elements in list.
void size(const label)
Override size to be inconsistent with allocated storage.
void setSize(const label)
Reset size of List.
label size() const
Return the number of elements in the UPtrList.
static IOdictionary decomposeParDict(const Time &time)
Read and return the decomposeParDict.
static autoPtr< decompositionMethod > NewDecomposer(const dictionary &decompositionDict)
Return a reference to the selected decomposition method.
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
const fvPatchList & patches
errorManipArg< error, int > exit(error &err, const int errNo=1)
List< label > labelList
A List of labels.
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
List< cell > cellList
list of cells
Ostream & endl(Ostream &os)
Add newline and flush stream.
word name(const bool)
Return a word representation of a bool.
int order(const scalar s)
vectorField pointField
pointField is a vectorField.
labelListList invertOneToMany(const label len, const labelUList &)
Invert one-to-many map. Unmapped elements will be size 0.
List< bool > boolList
Bool container classes.
Field< scalar > scalarField
Specialisation of Field<T> for scalar.
labelList first(const UList< labelPair > &p)
List< labelList > labelListList
A List of labelList.
VolField< scalar > volScalarField
dimensioned< scalar > mag(const dimensioned< Type > &)
PtrList< polyPatch > polyPatchList
container classes for polyPatch
layerAndWeight max(const layerAndWeight &a, const layerAndWeight &b)
void sortedOrder(const UList< T > &, labelList &order)
Generate the (stable) sort order for the list.
UList< label > labelUList
labelList pointLabels(nPoints, -1)