36 void Foam::domainDecomposition::mark
45 label pointi = zoneElems[i];
47 if (elementToZone[pointi] == -1)
50 elementToZone[pointi] = zoneI;
52 else if (elementToZone[pointi] >= 0)
55 elementToZone[pointi] = -2;
61 void Foam::domainDecomposition::addInterProcFace
64 const label ownerProc,
66 const label subPatchID,
67 List<Map<label>>& nbrToInterPatch,
68 List<DynamicList<DynamicList<label>>>& interPatchFaces,
69 List<labelListList>& subPatchIDs,
70 List<labelListList>& subPatchStarts
74 label toNbrProcPatchi = -1, toOwnerProcPatchi = -1;
75 if (!nbrToInterPatch[ownerProc].
found(nbrProc))
77 toNbrProcPatchi = nbrToInterPatch[ownerProc].size();
78 nbrToInterPatch[ownerProc].insert(nbrProc, toNbrProcPatchi);
79 interPatchFaces[ownerProc].append(DynamicList<label>());
80 subPatchIDs[ownerProc].append(
labelList(1, subPatchID));
83 if (facei != -1 && completeMesh().isInternalFace(facei))
85 toOwnerProcPatchi = nbrToInterPatch[nbrProc].size();
86 nbrToInterPatch[nbrProc].insert(ownerProc, toOwnerProcPatchi);
87 interPatchFaces[nbrProc].append(DynamicList<label>());
88 subPatchIDs[nbrProc].append(
labelList(1, subPatchID));
94 toNbrProcPatchi = nbrToInterPatch[ownerProc][nbrProc];
96 if (facei != -1 && completeMesh().isInternalFace(facei))
98 toOwnerProcPatchi = nbrToInterPatch[nbrProc][ownerProc];
103 if (subPatchIDs[ownerProc][toNbrProcPatchi].last() != subPatchID)
105 subPatchIDs[ownerProc][toNbrProcPatchi].append(subPatchID);
106 subPatchStarts[ownerProc][toNbrProcPatchi].append
108 interPatchFaces[ownerProc][toNbrProcPatchi].size()
111 if (facei != -1 && completeMesh().isInternalFace(facei))
113 subPatchIDs[nbrProc][toOwnerProcPatchi].append(subPatchID);
114 subPatchStarts[nbrProc][toOwnerProcPatchi].append
116 interPatchFaces[nbrProc][toOwnerProcPatchi].size()
124 interPatchFaces[ownerProc][toNbrProcPatchi].append(facei + 1);
126 if (completeMesh().isInternalFace(facei))
128 interPatchFaces[nbrProc][toOwnerProcPatchi].append(- facei - 1);
136 Info<<
"\nCalculating distribution of cells" <<
endl;
138 cpuTime decompositionTime;
140 const dictionary decomposeParDict =
144 if (decomposeParDict.found(
"weightField"))
146 const word weightName = decomposeParDict.lookup(
"weightField");
153 completeMesh().time().
name(),
160 cellWeights = weights.primitiveField();
170 Info<<
"\nFinished decomposition in "
171 << decompositionTime.elapsedCpuTime()
178 template<
class BinaryOp>
179 inline void Foam::domainDecomposition::processInterCyclics
182 const polyBoundaryMesh&
patches,
183 List<DynamicList<DynamicList<label>>>& interPatchFaces,
184 List<Map<label>>& procNbrToInterPatch,
185 List<labelListList>& subPatchIDs,
186 List<labelListList>& subPatchStarts,
196 const nonConformalCyclicPolyPatch& nccPp =
199 if (nccPp.owner() != owner)
continue;
201 const polyPatch& origPp = nccPp.origPatch();
202 const polyPatch& nbrOrigPp = nccPp.nbrPatch().origPatch();
204 const labelUList& origPatchFaceCells = origPp.faceCells();
205 const labelUList& nbrOrigPatchFaceCells = nbrOrigPp.faceCells();
209 forAll(origPatchFaceCells, origFacei)
211 const label ownerProc =
212 cellProc[origPatchFaceCells[origFacei]];
214 forAll(nbrOrigPatchFaceCells, nbrOrigFacei)
216 const label nbrProc =
217 cellProc[nbrOrigPatchFaceCells[nbrOrigFacei]];
219 if (bop(ownerProc, nbrProc))
238 const cyclicPolyPatch& cPp =
241 if (cPp.owner() != owner)
continue;
243 const labelUList& patchFaceCells = cPp.faceCells();
244 const labelUList& nbrPatchFaceCells = cPp.nbrPatch().faceCells();
247 forAll(patchFaceCells, facei)
249 const label ownerProc = cellProc[patchFaceCells[facei]];
250 const label nbrProc = cellProc[nbrPatchFaceCells[facei]];
252 if (bop(ownerProc, nbrProc))
274 void Foam::domainDecomposition::decompose()
277 cellProc_ = distributeCells();
282 Info<<
"\nCalculating original mesh data" <<
endl;
285 const polyBoundaryMesh&
patches = completeMesh().boundaryMesh();
286 const faceList& fcs = completeMesh().faces();
287 const labelList& owner = completeMesh().faceOwner();
288 const labelList& neighbour = completeMesh().faceNeighbour();
293 Info<<
"\nDistributing cells to processors" <<
endl;
298 Info<<
"\nDistributing faces to processors" <<
endl;
304 List<DynamicList<label>> dynProcFaceAddressing(nProcs());
309 if (cellProc_[owner[facei]] == cellProc_[neighbour[facei]])
312 dynProcFaceAddressing[cellProc_[owner[facei]]].append(facei+1);
320 forAll(procPatchSize, proci)
323 procPatchStartIndex[proci].setSize(
patches.
size());
330 forAll(procPatchSize, proci)
332 procPatchSize[proci][
patchi] = 0;
333 procPatchStartIndex[proci][
patchi] =
334 dynProcFaceAddressing[proci].size();
346 forAll(patchFaceCells, facei)
348 const label curProc = cellProc_[patchFaceCells[facei]];
351 dynProcFaceAddressing[curProc].append(patchStart+facei+1);
354 procPatchSize[curProc][
patchi]++;
362 const cyclicPolyPatch& pp =
365 const labelUList& patchFaceCells = pp.faceCells();
366 const labelUList& nbrPatchFaceCells = pp.nbrPatch().faceCells();
368 forAll(patchFaceCells, facei)
370 const label curProc = cellProc_[patchFaceCells[facei]];
371 const label nbrProc = cellProc_[nbrPatchFaceCells[facei]];
373 if (curProc == nbrProc)
376 dynProcFaceAddressing[curProc].append(patchStart+facei+1);
379 procPatchSize[curProc][
patchi]++;
389 List<Map<label>> procNbrToInterPatch(nProcs());
392 List<DynamicList<DynamicList<label>>> interPatchFaces(nProcs());
395 List<labelListList> subPatchIDs(nProcs());
396 List<labelListList> subPatchStarts(nProcs());
401 const label ownerProc = cellProc_[owner[facei]];
402 const label nbrProc = cellProc_[neighbour[facei]];
404 if (ownerProc != nbrProc)
496 List<labelListList> procProcessorPatchSubPatchIDs(nProcs());
497 List<labelListList> procProcessorPatchSubPatchStarts(nProcs());
499 forAll(procNbrToInterPatch, proci)
501 label nInterfaces = procNbrToInterPatch[proci].size();
503 procNeighbourProcessors[proci].setSize(nInterfaces);
504 procProcessorPatchSize[proci].setSize(nInterfaces);
505 procProcessorPatchStartIndex[proci].setSize(nInterfaces);
506 procProcessorPatchSubPatchIDs[proci].setSize(nInterfaces);
507 procProcessorPatchSubPatchStarts[proci].setSize(nInterfaces);
510 const Map<label>& curNbrToInterPatch = procNbrToInterPatch[proci];
511 labelList nbrs = curNbrToInterPatch.toc();
515 DynamicList<DynamicList<label>>& curInterPatchFaces =
516 interPatchFaces[proci];
520 const label nbrProc = nbrs[i];
521 const label interPatch = curNbrToInterPatch[nbrProc];
523 procNeighbourProcessors[proci][i] = nbrProc;
524 procProcessorPatchSize[proci][i] =
525 curInterPatchFaces[interPatch].size();
526 procProcessorPatchStartIndex[proci][i] =
527 dynProcFaceAddressing[proci].size();
530 subPatchStarts[proci][interPatch].append
532 curInterPatchFaces[interPatch].size()
534 procProcessorPatchSubPatchIDs[proci][i].transfer
536 subPatchIDs[proci][interPatch]
538 procProcessorPatchSubPatchStarts[proci][i].transfer
540 subPatchStarts[proci][interPatch]
544 DynamicList<label>& interPatchFaces =
545 curInterPatchFaces[interPatch];
547 forAll(interPatchFaces, j)
549 dynProcFaceAddressing[proci].append(interPatchFaces[j]);
552 interPatchFaces.clearStorage();
555 curInterPatchFaces.clearStorage();
556 dynProcFaceAddressing[proci].shrink();
560 forAll(dynProcFaceAddressing, proci)
562 procFaceAddressing_[proci].transfer(dynProcFaceAddressing[proci]);
567 forAll(procPatchStartIndex, proci)
569 Info<<
"Processor:" << proci <<
endl;
570 Info<<
" total faces:" << procFaceAddressing_[proci].size()
573 const labelList& curProcPatchStartIndex =
574 procPatchStartIndex[proci];
579 <<
"\tstart:" << curProcPatchStartIndex[
patchi]
580 <<
"\tsize:" << procPatchSize[proci][
patchi]
586 forAll(procNeighbourProcessors, proci)
588 Info<<
"Processor " << proci <<
endl;
589 forAll(procNeighbourProcessors[proci], i)
591 Info<<
" nbr:" << procNeighbourProcessors[proci][i] <<
endl;
592 Info<<
" size:" << procProcessorPatchSize[proci][i] <<
endl;
593 Info<<
" start:" << procProcessorPatchStartIndex[proci][i]
602 forAll(procFaceAddressing_, proci)
604 Info<<
"Processor:" << proci <<
endl;
605 Info<<
" faces:" << procFaceAddressing_[proci] <<
endl;
609 Info<<
"\nDistributing points to processors" <<
endl;
616 forAll(procPointAddressing_, proci)
621 const labelList& procFaceLabels = procFaceAddressing_[proci];
623 forAll(procFaceLabels, facei)
626 const labelList& facePoints = fcs[
mag(procFaceLabels[facei]) - 1];
628 forAll(facePoints, pointi)
636 labelList& procPointLabels = procPointAddressing_[proci];
640 label nUsedPoints = 0;
646 procPointLabels[nUsedPoints] = pointi;
653 procPointLabels.setSize(nUsedPoints);
656 Info<<
"\nConstructing processor meshes" <<
endl;
668 forAll(completeMesh().pointZones(), zoneI)
670 mark(completeMesh().pointZones()[zoneI], zoneI, pointToZone);
674 labelList faceToZone(completeMesh().faces().size(), -1);
675 forAll(completeMesh().faceZones(), zoneI)
677 mark(completeMesh().faceZones()[zoneI], zoneI, faceToZone);
681 labelList cellToZone(completeMesh().nCells(), -1);
682 forAll(completeMesh().cellZones(), zoneI)
684 mark(completeMesh().cellZones()[zoneI], zoneI, cellToZone);
688 label maxProcCells = 0;
689 label totProcFaces = 0;
690 label maxProcPatches = 0;
691 label totProcPatches = 0;
692 label maxProcFaces = 0;
695 for (
label proci = 0; proci < nProcs(); proci++)
698 const labelList& curPointLabels = procPointAddressing_[proci];
699 const pointField& meshPoints = completeMesh().points();
702 forAll(curPointLabels, pointi)
704 procPoints[pointi] = meshPoints[curPointLabels[pointi]];
705 pointLookup[curPointLabels[pointi]] = pointi;
709 const labelList& curFaceLabels = procFaceAddressing_[proci];
710 const faceList& meshFaces = completeMesh().faces();
711 labelList faceLookup(completeMesh().nFaces(), -1);
712 faceList procFaces(curFaceLabels.size());
713 forAll(curFaceLabels, facei)
717 label curF =
mag(curFaceLabels[facei]) - 1;
719 faceLookup[curF] = facei;
724 if (curFaceLabels[facei] >= 0)
727 origFaceLabels = meshFaces[curF];
731 origFaceLabels = meshFaces[curF].reverseFace();
735 face& procFaceLabels = procFaces[facei];
737 procFaceLabels.
setSize(origFaceLabels.size());
739 forAll(origFaceLabels, pointi)
741 procFaceLabels[pointi] = pointLookup[origFaceLabels[pointi]];
746 const labelList& curCellLabels = procCellAddressing_[proci];
747 const cellList& meshCells = completeMesh().cells();
748 cellList procCells(curCellLabels.size());
749 forAll(curCellLabels, celli)
751 const labelList& origCellLabels = meshCells[curCellLabels[celli]];
752 cell& curCell = procCells[celli];
753 curCell.
setSize(origCellLabels.size());
754 forAll(origCellLabels, cellFacei)
756 curCell[cellFacei] = faceLookup[origCellLabels[cellFacei]];
769 completeMesh().facesInstance(),
770 runTimes_.procTimes()[proci]
777 fvMesh& procMesh = procMeshes_[proci];
778 procMesh.setPointsInstance(completeMesh().pointsInstance());
781 const labelList& curPatchSizes = procPatchSize[proci];
782 const labelList& curPatchStarts = procPatchStartIndex[proci];
783 const labelList& curNeighbourProcessors =
784 procNeighbourProcessors[proci];
785 const labelList& curProcessorPatchSizes =
786 procProcessorPatchSize[proci];
787 const labelList& curProcessorPatchStarts =
788 procProcessorPatchStartIndex[proci];
790 procProcessorPatchSubPatchIDs[proci];
792 procProcessorPatchSubPatchStarts[proci];
793 const polyPatchList& meshPatches = completeMesh().boundaryMesh();
796 label nInterProcPatches = 0;
797 forAll(curSubPatchIDs, procPatchi)
799 nInterProcPatches += curSubPatchIDs[procPatchi].size();
801 List<polyPatch*> procPatches
803 curPatchSizes.size() + nInterProcPatches,
812 const polyPatch& meshPatch = meshPatches[
patchi];
817 procMesh.boundaryMesh(),
827 forAll(curProcessorPatchSizes, procPatchi)
829 const labelList& subPatchID = curSubPatchIDs[procPatchi];
830 const labelList& subStarts = curSubStarts[procPatchi];
832 label curStart = curProcessorPatchStarts[procPatchi];
837 i < subPatchID.size() - 1
838 ? subStarts[i+1] - subStarts[i]
839 : curProcessorPatchSizes[procPatchi] - subStarts[i];
841 if (subPatchID[i] == -1)
845 new processorPolyPatch
850 procMesh.boundaryMesh(),
852 curNeighbourProcessors[procPatchi]
857 isA<nonConformalCyclicPolyPatch>
859 completeMesh().boundaryMesh()[subPatchID[i]]
865 const nonConformalCyclicPolyPatch& nccPp =
866 refCast<const nonConformalCyclicPolyPatch>
868 completeMesh().boundaryMesh()[subPatchID[i]]
872 new nonConformalProcessorCyclicPolyPatch
877 procMesh.boundaryMesh(),
879 curNeighbourProcessors[procPatchi],
881 nccPp.origPatch().name()
888 completeMesh().boundaryMesh()[subPatchID[i]]
893 const cyclicPolyPatch& cPp =
894 refCast<const cyclicPolyPatch>
896 completeMesh().boundaryMesh()[subPatchID[i]]
900 new processorCyclicPolyPatch
905 procMesh.boundaryMesh(),
907 curNeighbourProcessors[procPatchi],
914 <<
"Sub patch ID set for non-cyclic patch type"
925 procMesh.addFvPatches(procPatches);
934 List<DynamicList<label>> zonePoints(pz.size());
939 zonePoints[zoneI].setCapacity(pz[zoneI].size()/nProcs());
944 forAll(curPointLabels, pointi)
946 label curPoint = curPointLabels[pointi];
948 label zoneI = pointToZone[curPoint];
953 zonePoints[zoneI].append(pointi);
955 else if (zoneI == -2)
960 label index = pz[zoneI].whichPoint(curPoint);
964 zonePoints[zoneI].append(pointi);
970 procMesh.pointZones().clearAddressing();
971 procMesh.pointZones().setSize(zonePoints.size());
974 procMesh.pointZones().set
979 procMesh.pointZones(),
981 zonePoints[zoneI].shrink()
1000 List<DynamicList<label>> zoneFaces(fz.size());
1001 List<DynamicList<bool>> zoneFaceFlips(fz.size());
1006 label procSize = fz[zoneI].size()/nProcs();
1008 zoneFaces[zoneI].setCapacity(procSize);
1009 zoneFaceFlips[zoneI].setCapacity(procSize);
1015 forAll(curFaceLabels, facei)
1019 label curF =
mag(curFaceLabels[facei]) - 1;
1021 label zoneI = faceToZone[curF];
1026 zoneFaces[zoneI].append(facei);
1028 label index = fz[zoneI].whichFace(curF);
1030 bool flip = fz[zoneI].flipMap()[index];
1032 if (curFaceLabels[facei] < 0)
1037 zoneFaceFlips[zoneI].append(flip);
1039 else if (zoneI == -2)
1044 label index = fz[zoneI].whichFace(curF);
1048 zoneFaces[zoneI].append(facei);
1050 bool flip = fz[zoneI].flipMap()[index];
1052 if (curFaceLabels[facei] < 0)
1057 zoneFaceFlips[zoneI].append(flip);
1063 procMesh.faceZones().clearAddressing();
1064 procMesh.faceZones().setSize(zoneFaces.size());
1067 procMesh.faceZones().set
1072 zoneFaces[zoneI].shrink(),
1073 zoneFaceFlips[zoneI].shrink(),
1075 procMesh.faceZones()
1094 List<DynamicList<label>> zoneCells(cz.size());
1099 zoneCells[zoneI].setCapacity(cz[zoneI].size()/nProcs());
1102 forAll(curCellLabels, celli)
1104 label curCelli = curCellLabels[celli];
1106 label zoneI = cellToZone[curCelli];
1111 zoneCells[zoneI].append(celli);
1113 else if (zoneI == -2)
1118 label index = cz[zoneI].whichCell(curCelli);
1122 zoneCells[zoneI].append(celli);
1128 procMesh.cellZones().clearAddressing();
1129 procMesh.cellZones().setSize(zoneCells.size());
1132 procMesh.cellZones().set
1137 zoneCells[zoneI].shrink(),
1139 procMesh.cellZones()
1154 <<
"Processor " << proci <<
nl
1155 <<
" Number of cells = " << procMesh.nCells()
1158 maxProcCells =
max(maxProcCells, procMesh.nCells());
1160 label nBoundaryFaces = 0;
1161 label nProcPatches = 0;
1162 label nProcFaces = 0;
1166 if (isA<processorPolyPatch>(procMesh.boundaryMesh()[
patchi]))
1168 const processorPolyPatch& ppp =
1169 refCast<const processorPolyPatch>
1171 procMesh.boundaryMesh()[
patchi]
1174 Info<<
" Number of faces shared with processor "
1175 << ppp.neighbProcNo() <<
" = " << ppp.size() <<
endl;
1178 nProcFaces += ppp.size();
1182 nBoundaryFaces += procMesh.boundaryMesh()[
patchi].size();
1186 Info<<
" Number of processor patches = " << nProcPatches <<
nl
1187 <<
" Number of processor faces = " << nProcFaces <<
nl
1188 <<
" Number of boundary faces = " << nBoundaryFaces <<
endl;
1190 totProcFaces += nProcFaces;
1191 totProcPatches += nProcPatches;
1192 maxProcPatches =
max(maxProcPatches, nProcPatches);
1193 maxProcFaces =
max(maxProcFaces, nProcFaces);
1198 scalar avgProcCells = scalar(completeMesh().nCells())/nProcs();
1199 scalar avgProcPatches = scalar(totProcPatches)/nProcs();
1200 scalar avgProcFaces = scalar(totProcFaces)/nProcs();
1203 if (totProcPatches == 0)
1207 if (totProcFaces == 0)
1213 <<
"Number of processor faces = " << totProcFaces/2 <<
nl
1214 <<
"Max number of cells = " << maxProcCells
1215 <<
" (" << 100.0*(maxProcCells-avgProcCells)/avgProcCells
1216 <<
"% above average " << avgProcCells <<
")" <<
nl
1217 <<
"Max number of processor patches = " << maxProcPatches
1218 <<
" (" << 100.0*(maxProcPatches-avgProcPatches)/avgProcPatches
1219 <<
"% above average " << avgProcPatches <<
")" <<
nl
1220 <<
"Max number of faces between processors = " << maxProcFaces
1221 <<
" (" << 100.0*(maxProcFaces-avgProcFaces)/avgProcFaces
1222 <<
"% above average " << avgProcFaces <<
")" <<
nl
#define forAll(list, i)
Loop across all elements in list.
void size(const label)
Override size to be inconsistent with allocated storage.
void setSize(const label)
Reset size of List.
label size() const
Return the number of elements in the UPtrList.
static autoPtr< decompositionMethod > NewDecomposer(const dictionary &decompositionDict)
Return a reference to the selected decomposition method.
static dictionary decomposeParDict(const Time &time)
Read and return the decomposeParDict.
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
const fvPatchList & patches
errorManipArg< error, int > exit(error &err, const int errNo=1)
List< label > labelList
A List of labels.
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
List< cell > cellList
list of cells
Ostream & endl(Ostream &os)
Add newline and flush stream.
MeshZones< cellZone, polyMesh > meshCellZones
A MeshZones with the type cellZone.
vectorField pointField
pointField is a vectorField.
labelListList invertOneToMany(const label len, const labelUList &)
Invert one-to-many map. Unmapped elements will be size 0.
List< bool > boolList
Bool container classes.
MeshZones< pointZone, polyMesh > meshPointZones
A MeshZones with the type pointZone.
Field< scalar > scalarField
Specialisation of Field<T> for scalar.
List< labelList > labelListList
A List of labelList.
VolField< scalar > volScalarField
dimensioned< scalar > mag(const dimensioned< Type > &)
PtrList< polyPatch > polyPatchList
container classes for polyPatch
layerAndWeight max(const layerAndWeight &a, const layerAndWeight &b)
MeshZones< faceZone, polyMesh > meshFaceZones
A MeshZones with the type faceZone.
void sortedOrder(const UList< T > &, labelList &order)
Generate the (stable) sort order for the list.
word name(const complex &)
Return a string representation of a complex.
UList< label > labelUList
labelList pointLabels(nPoints, -1)