mapDistributeBaseTemplates.C
Go to the documentation of this file.
1 /*---------------------------------------------------------------------------*\
2  ========= |
3  \\ / F ield | OpenFOAM: The Open Source CFD Toolbox
4  \\ / O peration |
5  \\ / A nd | Copyright (C) 2015 OpenFOAM Foundation
6  \\/ M anipulation | Copyright (C) 2015 OpenCFD Ltd.
7 -------------------------------------------------------------------------------
8 License
9  This file is part of OpenFOAM.
10 
11  OpenFOAM is free software: you can redistribute it and/or modify it
12  under the terms of the GNU General Public License as published by
13  the Free Software Foundation, either version 3 of the License, or
14  (at your option) any later version.
15 
16  OpenFOAM is distributed in the hope that it will be useful, but WITHOUT
17  ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
18  FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
19  for more details.
20 
21  You should have received a copy of the GNU General Public License
22  along with OpenFOAM. If not, see <http://www.gnu.org/licenses/>.
23 
24 \*---------------------------------------------------------------------------*/
25 
26 #include "Pstream.H"
27 #include "PstreamBuffers.H"
28 #include "PstreamCombineReduceOps.H"
29 #include "flipOp.H"
30 
31 // * * * * * * * * * * * * * Private Member Functions * * * * * * * * * * * //
32 
33 template<class T, class CombineOp, class negateOp>
34 void Foam::mapDistributeBase::flipAndCombine
35 (
36  const UList<label>& map,
37  const bool hasFlip,
38  const UList<T>& rhs,
39  const CombineOp& cop,
40  const negateOp& negOp,
41  List<T>& lhs
42 )
43 {
44  if (hasFlip)
45  {
46  forAll(map, i)
47  {
48  if (map[i] > 0)
49  {
50  label index = map[i]-1;
51  cop(lhs[index], rhs[i]);
52  }
53  else if (map[i] < 0)
54  {
55  label index = -map[i]-1;
56  cop(lhs[index], negOp(rhs[i]));
57  }
58  else
59  {
60  FatalErrorInFunction
61  << "At index " << i << " out of " << map.size()
62  << " have illegal index " << map[i]
63  << " for field " << rhs.size() << " with flipMap"
64  << exit(FatalError);
65  }
66  }
67  }
68  else
69  {
70  forAll(map, i)
71  {
72  cop(lhs[map[i]], rhs[i]);
73  }
74  }
75 }
76 
77 
78 template<class T, class negateOp>
79 T Foam::mapDistributeBase::accessAndFlip
80 (
81  const UList<T>& fld,
82  const label index,
83  const bool hasFlip,
84  const negateOp& negOp
85 )
86 {
87  T t;
88  if (hasFlip)
89  {
90  if (index > 0)
91  {
92  t = fld[index-1];
93  }
94  else if (index < 0)
95  {
96  t = negOp(fld[-index-1]);
97  }
98  else
99  {
100  FatalErrorInFunction
101  << "Illegal index " << index
102  << " into field of size " << fld.size()
103  << " with face-flipping"
104  << exit(FatalError);
105  t = fld[index];
106  }
107  }
108  else
109  {
110  t = fld[index];
111  }
112  return t;
113 }
114 
115 
116 // Distribute list.
117 template<class T, class negateOp>
118 void Foam::mapDistributeBase::distribute
119 (
120  const Pstream::commsTypes commsType,
121  const List<labelPair>& schedule,
122  const label constructSize,
123  const labelListList& subMap,
124  const bool subHasFlip,
125  const labelListList& constructMap,
126  const bool constructHasFlip,
127  List<T>& field,
128  const negateOp& negOp,
129  const int tag
130 )
131 {
132  if (!Pstream::parRun())
133  {
134  // Do only me to me.
135 
136  const labelList& mySubMap = subMap[Pstream::myProcNo()];
137 
138  List<T> subField(mySubMap.size());
139  forAll(mySubMap, i)
140  {
141  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
142  }
143 
144  // Receive sub field from myself (subField)
145  const labelList& map = constructMap[Pstream::myProcNo()];
146 
147  field.setSize(constructSize);
148 
149  flipAndCombine
150  (
151  map,
152  constructHasFlip,
153  subField,
154  eqOp<T>(),
155  negOp,
156  field
157  );
158 
159  return;
160  }
161 
162  if (commsType == Pstream::blocking)
163  {
164  // Since buffered sending can reuse the field to collect the
165  // received data.
166 
167  // Send sub field to neighbour
168  for (label domain = 0; domain < Pstream::nProcs(); domain++)
169  {
170  const labelList& map = subMap[domain];
171 
172  if (domain != Pstream::myProcNo() && map.size())
173  {
174  OPstream toNbr(Pstream::blocking, domain, 0, tag);
175 
176  List<T> subField(map.size());
177  forAll(subField, i)
178  {
179  subField[i] = accessAndFlip
180  (
181  field,
182  map[i],
183  subHasFlip,
184  negOp
185  );
186  }
187  toNbr << subField;
188  }
189  }
190 
191  // Subset myself
192  const labelList& mySubMap = subMap[Pstream::myProcNo()];
193 
194  List<T> subField(mySubMap.size());
195  forAll(mySubMap, i)
196  {
197  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
198  }
199 
200  // Receive sub field from myself (subField)
201  const labelList& map = constructMap[Pstream::myProcNo()];
202 
203  field.setSize(constructSize);
204 
205  flipAndCombine
206  (
207  map,
208  constructHasFlip,
209  subField,
210  eqOp<T>(),
211  negOp,
212  field
213  );
214 
215  // Receive sub field from neighbour
216  for (label domain = 0; domain < Pstream::nProcs(); domain++)
217  {
218  const labelList& map = constructMap[domain];
219 
220  if (domain != Pstream::myProcNo() && map.size())
221  {
222  IPstream fromNbr(Pstream::blocking, domain, 0, tag);
223  List<T> subField(fromNbr);
224 
225  checkReceivedSize(domain, map.size(), subField.size());
226 
227  flipAndCombine
228  (
229  map,
230  constructHasFlip,
231  subField,
232  eqOp<T>(),
233  negOp,
234  field
235  );
236  }
237  }
238  }
239  else if (commsType == Pstream::scheduled)
240  {
241  // Need to make sure I don't overwrite field with received data
242  // since the data might need to be sent to another processor. So
243  // allocate a new field for the results.
244  List<T> newField(constructSize);
245 
246  // Receive sub field from myself
247  {
248  const labelList& mySubMap = subMap[Pstream::myProcNo()];
249 
250  List<T> subField(mySubMap.size());
251  forAll(subField, i)
252  {
253  subField[i] = accessAndFlip
254  (
255  field,
256  mySubMap[i],
257  subHasFlip,
258  negOp
259  );
260  }
261 
262  // Receive sub field from myself (subField)
263  flipAndCombine
264  (
265  constructMap[Pstream::myProcNo()],
266  constructHasFlip,
267  subField,
268  eqOp<T>(),
269  negOp,
270  newField
271  );
272  }
273 
274  // Schedule will already have pruned 0-sized comms
275  forAll(schedule, i)
276  {
277  const labelPair& twoProcs = schedule[i];
278  // twoProcs is a swap pair of processors. The first one is the
279  // one that needs to send first and then receive.
280 
281  label sendProc = twoProcs[0];
282  label recvProc = twoProcs[1];
283 
284  if (Pstream::myProcNo() == sendProc)
285  {
286  // I am send first, receive next
287  {
288  OPstream toNbr(Pstream::scheduled, recvProc, 0, tag);
289 
290  const labelList& map = subMap[recvProc];
291  List<T> subField(map.size());
292  forAll(subField, i)
293  {
294  subField[i] = accessAndFlip
295  (
296  field,
297  map[i],
298  subHasFlip,
299  negOp
300  );
301  }
302  toNbr << subField;
303  }
304  {
305  IPstream fromNbr(Pstream::scheduled, recvProc, 0, tag);
306  List<T> subField(fromNbr);
307 
308  const labelList& map = constructMap[recvProc];
309 
310  checkReceivedSize(recvProc, map.size(), subField.size());
311 
312  flipAndCombine
313  (
314  map,
315  constructHasFlip,
316  subField,
317  eqOp<T>(),
318  negOp,
319  newField
320  );
321  }
322  }
323  else
324  {
325  // I am receive first, send next
326  {
327  IPstream fromNbr(Pstream::scheduled, sendProc, 0, tag);
328  List<T> subField(fromNbr);
329 
330  const labelList& map = constructMap[sendProc];
331 
332  checkReceivedSize(sendProc, map.size(), subField.size());
333 
334  flipAndCombine
335  (
336  map,
337  constructHasFlip,
338  subField,
339  eqOp<T>(),
340  negOp,
341  newField
342  );
343  }
344  {
345  OPstream toNbr(Pstream::scheduled, sendProc, 0, tag);
346 
347  const labelList& map = subMap[sendProc];
348  List<T> subField(map.size());
349  forAll(subField, i)
350  {
351  subField[i] = accessAndFlip
352  (
353  field,
354  map[i],
355  subHasFlip,
356  negOp
357  );
358  }
359  toNbr << subField;
360  }
361  }
362  }
363  field.transfer(newField);
364  }
365  else if (commsType == Pstream::nonBlocking)
366  {
367  label nOutstanding = Pstream::nRequests();
368 
369  if (!contiguous<T>())
370  {
371  PstreamBuffers pBufs(Pstream::nonBlocking, tag);
372 
373  // Stream data into buffer
374  for (label domain = 0; domain < Pstream::nProcs(); domain++)
375  {
376  const labelList& map = subMap[domain];
377 
378  if (domain != Pstream::myProcNo() && map.size())
379  {
380  // Put data into send buffer
381  UOPstream toDomain(domain, pBufs);
382 
383  List<T> subField(map.size());
384  forAll(subField, i)
385  {
386  subField[i] = accessAndFlip
387  (
388  field,
389  map[i],
390  subHasFlip,
391  negOp
392  );
393  }
394  toDomain << subField;
395  }
396  }
397 
398  // Start receiving. Do not block.
399  pBufs.finishedSends(false);
400 
401  {
402  // Set up 'send' to myself
403  const labelList& mySub = subMap[Pstream::myProcNo()];
404  List<T> mySubField(mySub.size());
405  forAll(mySub, i)
406  {
407  mySubField[i] = accessAndFlip
408  (
409  field,
410  mySub[i],
411  subHasFlip,
412  negOp
413  );
414  }
415  // Combine bits. Note that can reuse field storage
416  field.setSize(constructSize);
417  // Receive sub field from myself
418  {
419  const labelList& map = constructMap[Pstream::myProcNo()];
420 
421  flipAndCombine
422  (
423  map,
424  constructHasFlip,
425  mySubField,
426  eqOp<T>(),
427  negOp,
428  field
429  );
430  }
431  }
432 
433  // Block ourselves, waiting only for the current comms
434  Pstream::waitRequests(nOutstanding);
435 
436  // Consume
437  for (label domain = 0; domain < Pstream::nProcs(); domain++)
438  {
439  const labelList& map = constructMap[domain];
440 
441  if (domain != Pstream::myProcNo() && map.size())
442  {
443  UIPstream str(domain, pBufs);
444  List<T> recvField(str);
445 
446  checkReceivedSize(domain, map.size(), recvField.size());
447 
448  flipAndCombine
449  (
450  map,
451  constructHasFlip,
452  recvField,
453  eqOp<T>(),
454  negOp,
455  field
456  );
457  }
458  }
459  }
460  else
461  {
462  // Set up sends to neighbours
463 
464  List<List<T > > sendFields(Pstream::nProcs());
465 
466  for (label domain = 0; domain < Pstream::nProcs(); domain++)
467  {
468  const labelList& map = subMap[domain];
469 
470  if (domain != Pstream::myProcNo() && map.size())
471  {
472  List<T>& subField = sendFields[domain];
473  subField.setSize(map.size());
474  forAll(map, i)
475  {
476  subField[i] = accessAndFlip
477  (
478  field,
479  map[i],
480  subHasFlip,
481  negOp
482  );
483  }
484 
485  OPstream::write
486  (
487  Pstream::nonBlocking,
488  domain,
489  reinterpret_cast<const char*>(subField.begin()),
490  subField.byteSize(),
491  tag
492  );
493  }
494  }
495 
496  // Set up receives from neighbours
497 
498  List<List<T > > recvFields(Pstream::nProcs());
499 
500  for (label domain = 0; domain < Pstream::nProcs(); domain++)
501  {
502  const labelList& map = constructMap[domain];
503 
504  if (domain != Pstream::myProcNo() && map.size())
505  {
506  recvFields[domain].setSize(map.size());
507  IPstream::read
508  (
509  Pstream::nonBlocking,
510  domain,
511  reinterpret_cast<char*>(recvFields[domain].begin()),
512  recvFields[domain].byteSize(),
513  tag
514  );
515  }
516  }
517 
518 
519  // Set up 'send' to myself
520 
521  {
522  const labelList& map = subMap[Pstream::myProcNo()];
523 
524  List<T>& subField = sendFields[Pstream::myProcNo()];
525  subField.setSize(map.size());
526  forAll(map, i)
527  {
528  subField[i] = accessAndFlip
529  (
530  field,
531  map[i],
532  subHasFlip,
533  negOp
534  );
535  }
536  }
537 
538 
539  // Combine bits. Note that can reuse field storage
540 
541  field.setSize(constructSize);
542 
543 
544  // Receive sub field from myself (sendFields[Pstream::myProcNo()])
545  {
546  const labelList& map = constructMap[Pstream::myProcNo()];
547  const List<T>& subField = sendFields[Pstream::myProcNo()];
548 
549  flipAndCombine
550  (
551  map,
552  constructHasFlip,
553  subField,
554  eqOp<T>(),
555  negOp,
556  field
557  );
558  }
559 
560 
561  // Wait for all to finish
562 
563  Pstream::waitRequests(nOutstanding);
564 
565 
566  // Collect neighbour fields
567 
568  for (label domain = 0; domain < Pstream::nProcs(); domain++)
569  {
570  const labelList& map = constructMap[domain];
571 
572  if (domain != Pstream::myProcNo() && map.size())
573  {
574  const List<T>& subField = recvFields[domain];
575 
576  checkReceivedSize(domain, map.size(), subField.size());
577 
578  flipAndCombine
579  (
580  map,
581  constructHasFlip,
582  subField,
583  eqOp<T>(),
584  negOp,
585  field
586  );
587  }
588  }
589  }
590  }
591  else
592  {
593  FatalErrorInFunction
594  << "Unknown communication schedule " << commsType
595  << abort(FatalError);
596  }
597 }
598 
599 
600 // Distribute list.
601 template<class T, class CombineOp, class negateOp>
602 void Foam::mapDistributeBase::distribute
603 (
604  const Pstream::commsTypes commsType,
605  const List<labelPair>& schedule,
606  const label constructSize,
607  const labelListList& subMap,
608  const bool subHasFlip,
609  const labelListList& constructMap,
610  const bool constructHasFlip,
611  List<T>& field,
612  const CombineOp& cop,
613  const negateOp& negOp,
614  const T& nullValue,
615  const int tag
616 )
617 {
618  if (!Pstream::parRun())
619  {
620  // Do only me to me.
621 
622  const labelList& mySubMap = subMap[Pstream::myProcNo()];
623 
624  List<T> subField(mySubMap.size());
625  forAll(mySubMap, i)
626  {
627  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
628  }
629 
630  // Receive sub field from myself (subField)
631  const labelList& map = constructMap[Pstream::myProcNo()];
632 
633  field.setSize(constructSize);
634  field = nullValue;
635 
636  flipAndCombine(map, constructHasFlip, subField, cop, negOp, field);
637 
638  return;
639  }
640 
641  if (commsType == Pstream::blocking)
642  {
643  // Since buffered sending can reuse the field to collect the
644  // received data.
645 
646  // Send sub field to neighbour
647  for (label domain = 0; domain < Pstream::nProcs(); domain++)
648  {
649  const labelList& map = subMap[domain];
650 
651  if (domain != Pstream::myProcNo() && map.size())
652  {
653  OPstream toNbr(Pstream::blocking, domain, 0, tag);
654  List<T> subField(map.size());
655  forAll(subField, i)
656  {
657  subField[i] = accessAndFlip
658  (
659  field,
660  map[i],
661  subHasFlip,
662  negOp
663  );
664  }
665  toNbr << subField;
666  }
667  }
668 
669  // Subset myself
670  const labelList& mySubMap = subMap[Pstream::myProcNo()];
671 
672  List<T> subField(mySubMap.size());
673  forAll(mySubMap, i)
674  {
675  subField[i] = accessAndFlip(field, mySubMap[i], subHasFlip, negOp);
676  }
677 
678  // Receive sub field from myself (subField)
679  const labelList& map = constructMap[Pstream::myProcNo()];
680 
681  field.setSize(constructSize);
682  field = nullValue;
683 
684  flipAndCombine(map, constructHasFlip, subField, cop, negOp, field);
685 
686  // Receive sub field from neighbour
687  for (label domain = 0; domain < Pstream::nProcs(); domain++)
688  {
689  const labelList& map = constructMap[domain];
690 
691  if (domain != Pstream::myProcNo() && map.size())
692  {
693  IPstream fromNbr(Pstream::blocking, domain, 0, tag);
694  List<T> subField(fromNbr);
695 
696  checkReceivedSize(domain, map.size(), subField.size());
697 
698  flipAndCombine
699  (
700  map,
701  constructHasFlip,
702  subField,
703  cop,
704  negOp,
705  field
706  );
707  }
708  }
709  }
710  else if (commsType == Pstream::scheduled)
711  {
712  // Need to make sure I don't overwrite field with received data
713  // since the data might need to be sent to another processor. So
714  // allocate a new field for the results.
715  List<T> newField(constructSize, nullValue);
716 
717  {
718  const labelList& mySubMap = subMap[Pstream::myProcNo()];
719 
720  // Subset myself
721  List<T> subField(mySubMap.size());
722  forAll(subField, i)
723  {
724  subField[i] = accessAndFlip
725  (
726  field,
727  mySubMap[i],
728  subHasFlip,
729  negOp
730  );
731  }
732 
733  // Receive sub field from myself (subField)
734  const labelList& map = constructMap[Pstream::myProcNo()];
735 
736  flipAndCombine
737  (
738  map,
739  constructHasFlip,
740  subField,
741  cop,
742  negOp,
743  newField
744  );
745  }
746 
747 
748  // Schedule will already have pruned 0-sized comms
749  forAll(schedule, i)
750  {
751  const labelPair& twoProcs = schedule[i];
752  // twoProcs is a swap pair of processors. The first one is the
753  // one that needs to send first and then receive.
754 
755  label sendProc = twoProcs[0];
756  label recvProc = twoProcs[1];
757 
758  if (Pstream::myProcNo() == sendProc)
759  {
760  // I am send first, receive next
761  {
762  OPstream toNbr(Pstream::scheduled, recvProc, 0, tag);
763 
764  const labelList& map = subMap[recvProc];
765 
766  List<T> subField(map.size());
767  forAll(subField, i)
768  {
769  subField[i] = accessAndFlip
770  (
771  field,
772  map[i],
773  subHasFlip,
774  negOp
775  );
776  }
777  toNbr << subField;
778  }
779  {
780  IPstream fromNbr(Pstream::scheduled, recvProc, 0, tag);
781  List<T> subField(fromNbr);
782  const labelList& map = constructMap[recvProc];
783 
784  checkReceivedSize(recvProc, map.size(), subField.size());
785 
786  flipAndCombine
787  (
788  map,
789  constructHasFlip,
790  subField,
791  cop,
792  negOp,
793  newField
794  );
795  }
796  }
797  else
798  {
799  // I am receive first, send next
800  {
801  IPstream fromNbr(Pstream::scheduled, sendProc, 0, tag);
802  List<T> subField(fromNbr);
803  const labelList& map = constructMap[sendProc];
804 
805  checkReceivedSize(sendProc, map.size(), subField.size());
806 
807  flipAndCombine
808  (
809  map,
810  constructHasFlip,
811  subField,
812  cop,
813  negOp,
814  newField
815  );
816  }
817  {
818  OPstream toNbr(Pstream::scheduled, sendProc, 0, tag);
819 
820  const labelList& map = subMap[sendProc];
821 
822  List<T> subField(map.size());
823  forAll(subField, i)
824  {
825  subField[i] = accessAndFlip
826  (
827  field,
828  map[i],
829  subHasFlip,
830  negOp
831  );
832  }
833  toNbr << subField;
834  }
835  }
836  }
837  field.transfer(newField);
838  }
839  else if (commsType == Pstream::nonBlocking)
840  {
841  label nOutstanding = Pstream::nRequests();
842 
843  if (!contiguous<T>())
844  {
845  PstreamBuffers pBufs(Pstream::nonBlocking, tag);
846 
847  // Stream data into buffer
848  for (label domain = 0; domain < Pstream::nProcs(); domain++)
849  {
850  const labelList& map = subMap[domain];
851 
852  if (domain != Pstream::myProcNo() && map.size())
853  {
854  // Put data into send buffer
855  UOPstream toDomain(domain, pBufs);
856 
857  List<T> subField(map.size());
858  forAll(subField, i)
859  {
860  subField[i] = accessAndFlip
861  (
862  field,
863  map[i],
864  subHasFlip,
865  negOp
866  );
867  }
868  toDomain << subField;
869  }
870  }
871 
872  // Start receiving. Do not block.
873  pBufs.finishedSends(false);
874 
875  {
876  // Set up 'send' to myself
877  const labelList& myMap = subMap[Pstream::myProcNo()];
878 
879  List<T> mySubField(myMap.size());
880  forAll(myMap, i)
881  {
882  mySubField[i] = accessAndFlip
883  (
884  field,
885  myMap[i],
886  subHasFlip,
887  negOp
888  );
889  }
890 
891  // Combine bits. Note that can reuse field storage
892  field.setSize(constructSize);
893  field = nullValue;
894  // Receive sub field from myself
895  {
896  const labelList& map = constructMap[Pstream::myProcNo()];
897 
898  flipAndCombine
899  (
900  map,
901  constructHasFlip,
902  mySubField,
903  cop,
904  negOp,
905  field
906  );
907  }
908  }
909 
910  // Block ourselves, waiting only for the current comms
911  Pstream::waitRequests(nOutstanding);
912 
913  // Consume
914  for (label domain = 0; domain < Pstream::nProcs(); domain++)
915  {
916  const labelList& map = constructMap[domain];
917 
918  if (domain != Pstream::myProcNo() && map.size())
919  {
920  UIPstream str(domain, pBufs);
921  List<T> recvField(str);
922 
923  checkReceivedSize(domain, map.size(), recvField.size());
924 
925  flipAndCombine
926  (
927  map,
928  constructHasFlip,
929  recvField,
930  cop,
931  negOp,
932  field
933  );
934  }
935  }
936  }
937  else
938  {
939  // Set up sends to neighbours
940 
941  List<List<T > > sendFields(Pstream::nProcs());
942 
943  for (label domain = 0; domain < Pstream::nProcs(); domain++)
944  {
945  const labelList& map = subMap[domain];
946 
947  if (domain != Pstream::myProcNo() && map.size())
948  {
949  List<T>& subField = sendFields[domain];
950  subField.setSize(map.size());
951  forAll(map, i)
952  {
953  subField[i] = accessAndFlip
954  (
955  field,
956  map[i],
957  subHasFlip,
958  negOp
959  );
960  }
961 
962  OPstream::write
963  (
964  Pstream::nonBlocking,
965  domain,
966  reinterpret_cast<const char*>(subField.begin()),
967  subField.size()*sizeof(T),
968  tag
969  );
970  }
971  }
972 
973  // Set up receives from neighbours
974 
975  List<List<T > > recvFields(Pstream::nProcs());
976 
977  for (label domain = 0; domain < Pstream::nProcs(); domain++)
978  {
979  const labelList& map = constructMap[domain];
980 
981  if (domain != Pstream::myProcNo() && map.size())
982  {
983  recvFields[domain].setSize(map.size());
984  UIPstream::read
985  (
986  Pstream::nonBlocking,
987  domain,
988  reinterpret_cast<char*>(recvFields[domain].begin()),
989  recvFields[domain].size()*sizeof(T),
990  tag
991  );
992  }
993  }
994 
995  // Set up 'send' to myself
996 
997  {
998  const labelList& map = subMap[Pstream::myProcNo()];
999 
1000  List<T>& subField = sendFields[Pstream::myProcNo()];
1001  subField.setSize(map.size());
1002  forAll(map, i)
1003  {
1004  subField[i] = accessAndFlip
1005  (
1006  field,
1007  map[i],
1008  subHasFlip,
1009  negOp
1010  );
1011  }
1012  }
1013 
1014 
1015  // Combine bits. Note that can reuse field storage
1016 
1017  field.setSize(constructSize);
1018  field = nullValue;
1019 
1020  // Receive sub field from myself (subField)
1021  {
1022  const labelList& map = constructMap[Pstream::myProcNo()];
1023  const List<T>& subField = sendFields[Pstream::myProcNo()];
1024 
1025  flipAndCombine
1026  (
1027  map,
1028  constructHasFlip,
1029  subField,
1030  cop,
1031  negOp,
1032  field
1033  );
1034  }
1035 
1036 
1037  // Wait for all to finish
1038 
1039  Pstream::waitRequests(nOutstanding);
1040 
1041 
1042  // Collect neighbour fields
1043 
1044  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1045  {
1046  const labelList& map = constructMap[domain];
1047 
1048  if (domain != Pstream::myProcNo() && map.size())
1049  {
1050  const List<T>& subField = recvFields[domain];
1051 
1052  checkReceivedSize(domain, map.size(), subField.size());
1053 
1054  flipAndCombine
1055  (
1056  map,
1057  constructHasFlip,
1058  subField,
1059  cop,
1060  negOp,
1061  field
1062  );
1063  }
1064  }
1065  }
1066  }
1067  else
1068  {
1069  FatalErrorInFunction
1070  << "Unknown communication schedule " << commsType
1071  << abort(FatalError);
1072  }
1073 }
1074 
1075 
1076 template<class T>
1077 void Foam::mapDistributeBase::send(PstreamBuffers& pBufs, const List<T>& field)
1078 const
1079 {
1080  // Stream data into buffer
1081  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1082  {
1083  const labelList& map = subMap_[domain];
1084 
1085  if (map.size())
1086  {
1087  // Put data into send buffer
1088  UOPstream toDomain(domain, pBufs);
1089 
1090  List<T> subField(map.size());
1091  forAll(subField, i)
1092  {
1093  subField[i] = accessAndFlip
1094  (
1095  field,
1096  map[i],
1097  subHasFlip_,
1098  flipOp()
1099  );
1100  }
1101  toDomain << subField;
1102  }
1103  }
1104 
1105  // Start sending and receiving but do not block.
1106  pBufs.finishedSends(false);
1107 }
1108 
1109 
1110 template<class T>
1111 void Foam::mapDistributeBase::receive(PstreamBuffers& pBufs, List<T>& field)
1112 const
1113 {
1114  // Consume
1115  field.setSize(constructSize_);
1116 
1117  for (label domain = 0; domain < Pstream::nProcs(); domain++)
1118  {
1119  const labelList& map = constructMap_[domain];
1120 
1121  if (map.size())
1122  {
1123  UIPstream str(domain, pBufs);
1124  List<T> recvField(str);
1125 
1126  if (recvField.size() != map.size())
1127  {
1128  FatalErrorInFunction
1129  << "Expected from processor " << domain
1130  << " " << map.size() << " but received "
1131  << recvField.size() << " elements."
1132  << abort(FatalError);
1133  }
1134 
1135  flipAndCombine
1136  (
1137  map,
1138  constructHasFlip_,
1139  recvField,
1140  eqOp<T>(),
1141  flipOp(),
1142  field
1143  );
1144  }
1145  }
1146 }
1147 
1148 
1149 //- Distribute data using default commsType.
1150 template<class T, class negateOp>
1151 void Foam::mapDistributeBase::distribute
1152 (
1153  List<T>& fld,
1154  const negateOp& negOp,
1155  const int tag
1156 ) const
1157 {
1158  if (Pstream::defaultCommsType == Pstream::nonBlocking)
1159  {
1160  distribute
1161  (
1162  Pstream::nonBlocking,
1163  List<labelPair>(),
1164  constructSize_,
1165  subMap_,
1166  subHasFlip_,
1167  constructMap_,
1168  constructHasFlip_,
1169  fld,
1170  negOp,
1171  tag
1172  );
1173  }
1174  else if (Pstream::defaultCommsType == Pstream::scheduled)
1175  {
1176  distribute
1177  (
1178  Pstream::scheduled,
1179  schedule(),
1180  constructSize_,
1181  subMap_,
1182  subHasFlip_,
1183  constructMap_,
1184  constructHasFlip_,
1185  fld,
1186  negOp,
1187  tag
1188  );
1189  }
1190  else
1191  {
1192  distribute
1193  (
1194  Pstream::blocking,
1195  List<labelPair>(),
1196  constructSize_,
1197  subMap_,
1198  subHasFlip_,
1199  constructMap_,
1200  constructHasFlip_,
1201  fld,
1202  negOp,
1203  tag
1204  );
1205  }
1206 }
1207 
1208 
1209 //- Distribute data using default commsType.
1210 template<class T>
1211 void Foam::mapDistributeBase::distribute
1212 (
1213  List<T>& fld,
1214  const int tag
1215 ) const
1216 {
1217  distribute(fld, flipOp(), tag);
1218 }
1219 
1220 
1221 //- Distribute data using default commsType.
1222 template<class T>
1223 void Foam::mapDistributeBase::distribute
1224 (
1225  DynamicList<T>& fld,
1226  const int tag
1227 ) const
1228 {
1229  fld.shrink();
1230 
1231  List<T>& fldList = static_cast<List<T>& >(fld);
1232 
1233  distribute(fldList, tag);
1234 
1235  fld.setCapacity(fldList.size());
1236 }
1237 
1238 
1239 //- Reverse distribute data using default commsType.
1240 template<class T>
1241 void Foam::mapDistributeBase::reverseDistribute
1242 (
1243  const label constructSize,
1244  List<T>& fld,
1245  const int tag
1246 ) const
1247 {
1248  if (Pstream::defaultCommsType == Pstream::nonBlocking)
1249  {
1250  distribute
1251  (
1252  Pstream::nonBlocking,
1253  List<labelPair>(),
1254  constructSize,
1255  constructMap_,
1256  constructHasFlip_,
1257  subMap_,
1258  subHasFlip_,
1259  fld,
1260  flipOp(),
1261  tag
1262  );
1263  }
1264  else if (Pstream::defaultCommsType == Pstream::scheduled)
1265  {
1266  distribute
1267  (
1268  Pstream::scheduled,
1269  schedule(),
1270  constructSize,
1271  constructMap_,
1272  constructHasFlip_,
1273  subMap_,
1274  subHasFlip_,
1275  fld,
1276  flipOp(),
1277  tag
1278  );
1279  }
1280  else
1281  {
1282  distribute
1283  (
1284  Pstream::blocking,
1285  List<labelPair>(),
1286  constructSize,
1287  constructMap_,
1288  constructHasFlip_,
1289  subMap_,
1290  subHasFlip_,
1291  fld,
1292  flipOp(),
1293  tag
1294  );
1295  }
1296 }
1297 
1298 
1299 //- Reverse distribute data using default commsType.
1300 // Since constructSize might be larger than supplied size supply
1301 // a nullValue
1302 template<class T>
1303 void Foam::mapDistributeBase::reverseDistribute
1304 (
1305  const label constructSize,
1306  const T& nullValue,
1307  List<T>& fld,
1308  const int tag
1309 ) const
1310 {
1311  if (Pstream::defaultCommsType == Pstream::nonBlocking)
1312  {
1313  distribute
1314  (
1315  Pstream::nonBlocking,
1316  List<labelPair>(),
1317  constructSize,
1318  constructMap_,
1319  constructHasFlip_,
1320  subMap_,
1321  subHasFlip_,
1322  fld,
1323  eqOp<T>(),
1324  flipOp(),
1325  nullValue,
1326  tag
1327  );
1328  }
1329  else if (Pstream::defaultCommsType == Pstream::scheduled)
1330  {
1331  distribute
1332  (
1333  Pstream::scheduled,
1334  schedule(),
1335  constructSize,
1336  constructMap_,
1337  constructHasFlip_,
1338  subMap_,
1339  subHasFlip_,
1340  fld,
1341  eqOp<T>(),
1342  flipOp(),
1343  nullValue,
1344  tag
1345  );
1346  }
1347  else
1348  {
1349  distribute
1350  (
1351  Pstream::blocking,
1352  List<labelPair>(),
1353  constructSize,
1354  constructMap_,
1355  constructHasFlip_,
1356  subMap_,
1357  subHasFlip_,
1358  fld,
1359  eqOp<T>(),
1360  flipOp(),
1361  nullValue,
1362  tag
1363  );
1364  }
1365 }
1366 
1367 
1368 // ************************************************************************* //
Foam::mapDistributeBase::reverseDistribute
void reverseDistribute(const label constructSize, List< T > &, const int tag=UPstream::msgType()) const
Reverse distribute data using default commsType.
Definition: mapDistributeBaseTemplates.C:1242
Foam::mapDistributeBase::accessAndFlip
static T accessAndFlip(const UList< T > &fld, const label index, const bool hasFlip, const negateOp &negOp)
Definition: mapDistributeBaseTemplates.C:80
Foam::mapDistributeBase::subMap_
labelListList subMap_
Maps from subsetted data back to original data.
Definition: mapDistributeBase.H:101
Foam::UOPstream
Output inter-processor communications stream operating on external buffer.
Definition: UOPstream.H:54
Foam::UPstream::scheduled
@ scheduled
Definition: UPstream.H:67
forAll
#define forAll(list, i)
Loop across all elements in list.
Definition: UList.H:406
Foam::DynamicList
A 1D vector of objects of type <T> that resizes itself as necessary to accept the new objects.
Definition: DynamicList.H:56
Foam::mapDistributeBase::distribute
static void distribute(const Pstream::commsTypes commsType, const List< labelPair > &schedule, const label constructSize, const labelListList &subMap, const bool subHasFlip, const labelListList &constructMap, const bool constructHasFlip, List< T > &, const negateOp &negOp, const int tag=UPstream::msgType())
Distribute data. Note:schedule only used for Pstream::scheduled.
Definition: mapDistributeBaseTemplates.C:119
Foam::OPstream
Output inter-processor communications stream.
Definition: OPstream.H:50
Foam::mapDistributeBase::send
void send(PstreamBuffers &, const List< T > &) const
Do all sends using PstreamBuffers.
Definition: mapDistributeBaseTemplates.C:1077
Foam::read
bool read(const char *, int32_t &)
Definition: int32IO.C:87
Foam::UPstream::nProcs
static label nProcs(const label communicator=0)
Number of processes in parallel run.
Definition: UPstream.H:387
Foam::PstreamBuffers
Buffers for inter-processor communications streams (UOPstream, UIPstream).
Definition: PstreamBuffers.H:85
Foam::flipOp
Class containing functor to negate primitives. Dummy for all other types.
Definition: flipOp.H:50
Foam::List::transfer
void transfer(List< T > &)
Transfer the contents of the argument List into this list.
Foam::mapDistributeBase::receive
void receive(PstreamBuffers &, List< T > &) const
Do all receives using PstreamBuffers.
Definition: mapDistributeBaseTemplates.C:1111
Foam::UPstream::defaultCommsType
static commsTypes defaultCommsType
Default commsType.
Definition: UPstream.H:252
PstreamCombineReduceOps.H
Combination-Reduction operation for a parallel run. The information from all nodes is collected on th...
Foam::UPstream::blocking
@ blocking
Definition: UPstream.H:66
Foam::UPstream::nonBlocking
@ nonBlocking
Definition: UPstream.H:68
Foam::label
intWM_LABEL_SIZE_t label
A label is an int32_t or int64_t as specified by the pre-processor macro WM_LABEL_SIZE.
Definition: label.H:59
Foam::T
void T(FieldField< Field, Type > &f1, const FieldField< Field, Type > &f2)
Definition: FieldFieldFunctions.C:55
Foam::PstreamBuffers::finishedSends
void finishedSends(const bool block=true)
Mark all sends as having been done. This will start receives.
Definition: PstreamBuffers.C:82
Foam::FatalError
error FatalError
Foam::mapDistributeBase::subHasFlip_
bool subHasFlip_
Whether subMap includes flip or not.
Definition: mapDistributeBase.H:107
fld
gmvFile<< "tracers "<< particles.size()<< nl;forAllConstIter(Cloud< passiveParticle >, particles, iter){ gmvFile<< iter().position().x()<< ' ';}gmvFile<< nl;forAllConstIter(Cloud< passiveParticle >, particles, iter){ gmvFile<< iter().position().y()<< ' ';}gmvFile<< nl;forAllConstIter(Cloud< passiveParticle >, particles, iter){ gmvFile<< iter().position().z()<< ' ';}gmvFile<< nl;forAll(lagrangianScalarNames, i){ const word &name=lagrangianScalarNames[i];IOField< scalar > fld(IOobject(name, runTime.timeName(), cloud::prefix, mesh, IOobject::MUST_READ, IOobject::NO_WRITE))
Foam::eqOp
Definition: ops.H:70
Pstream.H
flipOp.H
Foam::abort
errorManip< error > abort(error &err)
Definition: errorManip.H:131
Foam::exit
errorManipArg< error, int > exit(error &err, const int errNo=1)
Definition: errorManip.H:124
Foam::List::setSize
void setSize(const label)
Reset size of List.
Foam::UPstream::commsTypes
commsTypes
Types of communications.
Definition: UPstream.H:64
FatalErrorInFunction
#define FatalErrorInFunction
Report an error message using Foam::FatalError.
Definition: error.H:318
T
const volScalarField & T
Definition: createFields.H:25
Foam::Pair
An ordered pair of two objects of type <T> with first() and second() elements.
Definition: contiguous.H:49
Foam::List< T >
Foam::UList
A 1D vector of objects of type <T>, where the size of the vector is known and can be used for subscri...
Definition: HashTable.H:60
Foam::mapDistributeBase::flipAndCombine
static void flipAndCombine(const UList< label > &map, const bool hasFlip, const UList< T > &rhs, const CombineOp &cop, const negateOp &negOp, List< T > &lhs)
Definition: mapDistributeBaseTemplates.C:35
PstreamBuffers.H
Foam::IPstream
Input inter-processor communications stream.
Definition: IPstream.H:50
Foam::List::size
void size(const label)
Override size to be inconsistent with allocated storage.
Foam::UIPstream
Input inter-processor communications stream operating on external buffer.
Definition: UIPstream.H:53
Foam::UList::size
label size() const
Return the number of elements in the UList.
Definition: UListI.H:299
write
Tcoeff write()

Copyright © 2011-2018  OpenFOAM | OPENFOAM® is a registered trademark of OpenCFD Ltd.

Generated by   doxygen 1.8.17