Tpetra parallel linear algebra  Version of the Day
Tpetra_Distributor.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // ************************************************************************
38 // @HEADER
39 
40 #ifndef TPETRA_DISTRIBUTOR_HPP
41 #define TPETRA_DISTRIBUTOR_HPP
42 
43 #include "Tpetra_Details_DistributorActor.hpp"
44 #include "Tpetra_Details_DistributorPlan.hpp"
45 
46 #include "Tpetra_Util.hpp"
47 #include "Teuchos_as.hpp"
48 #include "Teuchos_Describable.hpp"
49 #include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
50 #include "Teuchos_VerboseObject.hpp"
51 #include "Tpetra_Details_Behavior.hpp"
52 
53 #include "KokkosCompat_View.hpp"
54 #include "Kokkos_Core.hpp"
55 #include "Kokkos_TeuchosCommAdapters.hpp"
56 #include <memory>
57 #include <sstream>
58 #include <type_traits>
59 
60 namespace Tpetra {
61 
68  Teuchos::Array<std::string> distributorSendTypes ();
69 
137  class Distributor :
138  public Teuchos::Describable,
139  public Teuchos::ParameterListAcceptorDefaultBase {
140  public:
142 
143 
152  explicit Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm);
153 
165  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
166  const Teuchos::RCP<Teuchos::FancyOStream>& out);
167 
181  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
182  const Teuchos::RCP<Teuchos::ParameterList>& plist);
183 
200  Distributor (const Teuchos::RCP<const Teuchos::Comm<int> >& comm,
201  const Teuchos::RCP<Teuchos::FancyOStream>& out,
202  const Teuchos::RCP<Teuchos::ParameterList>& plist);
203 
205  Distributor (const Distributor& distributor);
206 
211  virtual ~Distributor () = default;
212 
218  void swap (Distributor& rhs);
219 
221 
223 
228  void setParameterList (const Teuchos::RCP<Teuchos::ParameterList>& plist);
229 
234  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters () const;
235 
237 
239 
259  size_t createFromSends (const Teuchos::ArrayView<const int>& exportProcIDs);
260 
294  template <class Ordinal>
295  void
296  createFromRecvs (const Teuchos::ArrayView<const Ordinal>& remoteIDs,
297  const Teuchos::ArrayView<const int>& remoteProcIDs,
298  Teuchos::Array<Ordinal>& exportIDs,
299  Teuchos::Array<int>& exportProcIDs);
300 
308  void
309  createFromSendsAndRecvs (const Teuchos::ArrayView<const int>& exportProcIDs,
310  const Teuchos::ArrayView<const int>& remoteProcIDs);
311 
313 
315 
319  size_t getNumReceives() const;
320 
324  size_t getNumSends() const;
325 
327  bool hasSelfMessage() const;
328 
330  size_t getMaxSendLength() const;
331 
333  size_t getTotalReceiveLength() const;
334 
339  Teuchos::ArrayView<const int> getProcsFrom() const;
340 
345  Teuchos::ArrayView<const int> getProcsTo() const;
346 
354  Teuchos::ArrayView<const size_t> getLengthsFrom() const;
355 
363  Teuchos::ArrayView<const size_t> getLengthsTo() const;
364 
369  Details::EDistributorHowInitialized howInitialized() const {
370  return plan_.howInitialized();
371  }
372 
374 
376 
387  Teuchos::RCP<Distributor> getReverse(bool create=true) const;
388 
390 
392 
413  template <class Packet>
414  void
415  doPostsAndWaits (const Teuchos::ArrayView<const Packet> &exports,
416  size_t numPackets,
417  const Teuchos::ArrayView<Packet> &imports);
418 
440  template <class Packet>
441  void
442  doPostsAndWaits (const Teuchos::ArrayView<const Packet> &exports,
443  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
444  const Teuchos::ArrayView<Packet> &imports,
445  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
446 
471  template <class Packet>
472  void
473  doPosts (const Teuchos::ArrayRCP<const Packet> &exports,
474  size_t numPackets,
475  const Teuchos::ArrayRCP<Packet> &imports);
476 
495  template <class Packet>
496  void
497  doPosts (const Teuchos::ArrayRCP<const Packet> &exports,
498  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
499  const Teuchos::ArrayRCP<Packet> &imports,
500  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
501 
508  void doWaits ();
509 
514  template <class Packet>
515  void
516  doReversePostsAndWaits (const Teuchos::ArrayView<const Packet> &exports,
517  size_t numPackets,
518  const Teuchos::ArrayView<Packet> &imports);
519 
524  template <class Packet>
525  void
526  doReversePostsAndWaits (const Teuchos::ArrayView<const Packet> &exports,
527  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
528  const Teuchos::ArrayView<Packet> &imports,
529  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
530 
535  template <class Packet>
536  void
537  doReversePosts (const Teuchos::ArrayRCP<const Packet> &exports,
538  size_t numPackets,
539  const Teuchos::ArrayRCP<Packet> &imports);
540 
545  template <class Packet>
546  void
547  doReversePosts (const Teuchos::ArrayRCP<const Packet> &exports,
548  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
549  const Teuchos::ArrayRCP<Packet> &imports,
550  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
551 
558  void doReverseWaits ();
559 
580  template <class ExpView, class ImpView>
581  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
582  doPostsAndWaits (
583  const ExpView &exports,
584  size_t numPackets,
585  const ImpView &imports);
586 
608  template <class ExpView, class ImpView>
609  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
610  doPostsAndWaits (const ExpView &exports,
611  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
612  const ImpView &imports,
613  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
614 
639  template <class ExpView, class ImpView>
640  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
641  doPosts (const ExpView &exports,
642  size_t numPackets,
643  const ImpView &imports);
644 
663  template <class ExpView, class ImpView>
664  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
665  doPosts (const ExpView &exports,
666  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
667  const ImpView &imports,
668  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
669 
674  template <class ExpView, class ImpView>
675  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
676  doReversePostsAndWaits (const ExpView &exports,
677  size_t numPackets,
678  const ImpView &imports);
679 
684  template <class ExpView, class ImpView>
685  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
686  doReversePostsAndWaits (const ExpView &exports,
687  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
688  const ImpView &imports,
689  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
690 
695  template <class ExpView, class ImpView>
696  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
697  doReversePosts (const ExpView &exports,
698  size_t numPackets,
699  const ImpView &imports);
700 
705  template <class ExpView, class ImpView>
706  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
707  doReversePosts (const ExpView &exports,
708  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
709  const ImpView &imports,
710  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID);
711 
715  void getLastDoStatistics(size_t & bytes_sent, size_t & bytes_recvd) const{
716  bytes_sent = lastRoundBytesSend_;
717  bytes_recvd = lastRoundBytesRecv_;
718  }
719 
721 
723 
725  std::string description() const;
726 
748  void
749  describe (Teuchos::FancyOStream& out,
750  const Teuchos::EVerbosityLevel verbLevel =
751  Teuchos::Describable::verbLevel_default) const;
753 
758  const Details::DistributorPlan& getPlan() const { return plan_; }
759  private:
760  Details::DistributorPlan plan_;
761  Details::DistributorActor actor_;
762 
764 
765 
767  static bool getVerbose();
768 
773  std::unique_ptr<std::string>
774  createPrefix(const char methodName[]) const;
775 
777  bool verbose_ = getVerbose();
779 
784  mutable Teuchos::RCP<Distributor> reverseDistributor_;
785 
787  size_t lastRoundBytesSend_;
788 
790  size_t lastRoundBytesRecv_;
791 
804  template <class Ordinal>
805  void computeSends (const Teuchos::ArrayView<const Ordinal> &remoteGIDs,
806  const Teuchos::ArrayView<const int> &remoteProcIDs,
807  Teuchos::Array<Ordinal> &exportGIDs,
808  Teuchos::Array<int> &exportProcIDs);
809 
811  void createReverseDistributor() const;
812 
813 
818  std::string
819  localDescribeToString (const Teuchos::EVerbosityLevel vl) const;
820  }; // class Distributor
821 
822 
823  template <class Packet>
824  void Distributor::
825  doPostsAndWaits (const Teuchos::ArrayView<const Packet>& exports,
826  size_t numPackets,
827  const Teuchos::ArrayView<Packet>& imports)
828  {
829  using Teuchos::arcp;
830  using Teuchos::ArrayRCP;
831  typedef typename ArrayRCP<const Packet>::size_type size_type;
832 
833  // doPosts() accepts the exports and imports arrays as ArrayRCPs,
834  // requiring that the memory location is persisting (as is
835  // necessary for nonblocking receives). However, it need only
836  // persist until doWaits() completes, so it is safe for us to use
837  // a nonpersisting reference in this case. The use of a
838  // nonpersisting reference is purely a performance optimization.
839 
840  //const Packet* exportsPtr = exports.getRawPtr();
841  //ArrayRCP<const Packet> exportsArcp (exportsPtr, static_cast<size_type> (0),
842  // exports.size(), false);
843  ArrayRCP<const Packet> exportsArcp (exports.getRawPtr (),
844  static_cast<size_type> (0),
845  exports.size(), false);
846 
847  // For some reason, neither of the options below (that use arcp)
848  // compile for Packet=std::complex<double> with GCC 4.5.1. The
849  // issue only arises with the exports array. This is why we
850  // construct a separate nonowning ArrayRCP.
851 
852  // doPosts (arcp<const Packet> (exports.getRawPtr(), 0, exports.size(), false),
853  // numPackets,
854  // arcp<Packet> (imports.getRawPtr(), 0, imports.size(), false));
855  // doPosts (arcp<const Packet> (exportsPtr, 0, exports.size(), false),
856  // numPackets,
857  // arcp<Packet> (imports.getRawPtr(), 0, imports.size(), false));
858  doPosts (exportsArcp,
859  numPackets,
860  arcp<Packet> (imports.getRawPtr (), 0, imports.size (), false));
861  doWaits ();
862 
863  lastRoundBytesSend_ = exports.size () * sizeof (Packet);
864  lastRoundBytesRecv_ = imports.size () * sizeof (Packet);
865  }
866 
867  template <class Packet>
868  void Distributor::
869  doPostsAndWaits (const Teuchos::ArrayView<const Packet>& exports,
870  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
871  const Teuchos::ArrayView<Packet> &imports,
872  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
873  {
874  using Teuchos::arcp;
875  using Teuchos::ArrayRCP;
876 
877  // doPosts() accepts the exports and imports arrays as ArrayRCPs,
878  // requiring that the memory location is persisting (as is
879  // necessary for nonblocking receives). However, it need only
880  // persist until doWaits() completes, so it is safe for us to use
881  // a nonpersisting reference in this case.
882 
883  // mfh 04 Apr 2012: For some reason, calling arcp<const Packet>
884  // for Packet=std::complex<T> (e.g., T=float) fails to compile
885  // with some versions of GCC. The issue only arises with the
886  // exports array. This is why we construct a separate nonowning
887  // ArrayRCP.
888  typedef typename ArrayRCP<const Packet>::size_type size_type;
889  ArrayRCP<const Packet> exportsArcp (exports.getRawPtr (),
890  static_cast<size_type> (0),
891  exports.size (), false);
892  // mfh 04 Apr 2012: This is the offending code. This statement
893  // would normally be in place of "exportsArcp" in the
894  // doPosts() call below.
895  //arcp<const Packet> (exports.getRawPtr(), 0, exports.size(), false),
896  doPosts (exportsArcp,
897  numExportPacketsPerLID,
898  arcp<Packet> (imports.getRawPtr (), 0, imports.size (), false),
899  numImportPacketsPerLID);
900  doWaits ();
901 
902  lastRoundBytesSend_ = exports.size () * sizeof (Packet);
903  lastRoundBytesRecv_ = imports.size () * sizeof (Packet);
904  }
905 
906 
907  template <class Packet>
908  void Distributor::
909  doPosts (const Teuchos::ArrayRCP<const Packet>& exports,
910  size_t numPackets,
911  const Teuchos::ArrayRCP<Packet>& imports)
912  {
913  Kokkos::View<const Packet*, Kokkos::HostSpace> exportsView(exports.get(), exports.size());
914  Kokkos::View<Packet*, Kokkos::HostSpace> importsView(imports.get(), imports.size());
915  doPosts(exportsView, numPackets, importsView);
916  }
917 
918  template <class Packet>
919  void Distributor::
920  doPosts (const Teuchos::ArrayRCP<const Packet>& exports,
921  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
922  const Teuchos::ArrayRCP<Packet>& imports,
923  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
924  {
925  Kokkos::View<const Packet*, Kokkos::HostSpace> exportsView(exports.get(), exports.size());
926  Kokkos::View<Packet*, Kokkos::HostSpace> importsView(imports.get(), imports.size());
927  doPosts(exportsView, numExportPacketsPerLID, importsView, numImportPacketsPerLID);
928  }
929 
930  template <class Packet>
931  void Distributor::
932  doReversePostsAndWaits (const Teuchos::ArrayView<const Packet>& exports,
933  size_t numPackets,
934  const Teuchos::ArrayView<Packet>& imports)
935  {
936  using Teuchos::arcp;
937  using Teuchos::ArrayRCP;
938  using Teuchos::as;
939 
940  // doReversePosts() takes exports and imports as ArrayRCPs,
941  // requiring that the memory locations are persisting. However,
942  // they need only persist within the scope of that routine, so it
943  // is safe for us to use nonpersisting references in this case.
944 
945  // mfh 04 Apr 2012: For some reason, calling arcp<const Packet>
946  // for Packet=std::complex<T> (e.g., T=float) fails to compile
947  // with some versions of GCC. The issue only arises with the
948  // exports array. This is why we construct a separate nonowning
949  // ArrayRCP.
950  typedef typename ArrayRCP<const Packet>::size_type size_type;
951  ArrayRCP<const Packet> exportsArcp (exports.getRawPtr(), as<size_type> (0),
952  exports.size(), false);
953  // mfh 04 Apr 2012: This is the offending code. This statement
954  // would normally be in place of "exportsArcp" in the
955  // doReversePosts() call below.
956  //arcp<const Packet> (exports.getRawPtr(), 0, exports.size(), false)
957  doReversePosts (exportsArcp,
958  numPackets,
959  arcp<Packet> (imports.getRawPtr (), 0, imports.size (), false));
960  doReverseWaits ();
961 
962  lastRoundBytesSend_ = exports.size() * sizeof(Packet);
963  lastRoundBytesRecv_ = imports.size() * sizeof(Packet);
964  }
965 
966  template <class Packet>
967  void Distributor::
968  doReversePostsAndWaits (const Teuchos::ArrayView<const Packet>& exports,
969  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
970  const Teuchos::ArrayView<Packet> &imports,
971  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
972  {
973  using Teuchos::as;
974  using Teuchos::arcp;
975  using Teuchos::ArrayRCP;
976 
977  // doReversePosts() accepts the exports and imports arrays as
978  // ArrayRCPs, requiring that the memory location is persisting (as
979  // is necessary for nonblocking receives). However, it need only
980  // persist until doReverseWaits() completes, so it is safe for us
981  // to use a nonpersisting reference in this case. The use of a
982  // nonpersisting reference is purely a performance optimization.
983 
984  // mfh 02 Apr 2012: For some reason, calling arcp<const Packet>
985  // for Packet=std::complex<double> fails to compile with some
986  // versions of GCC. The issue only arises with the exports array.
987  // This is why we construct a separate nonowning ArrayRCP.
988  typedef typename ArrayRCP<const Packet>::size_type size_type;
989  ArrayRCP<const Packet> exportsArcp (exports.getRawPtr (), as<size_type> (0),
990  exports.size (), false);
991  doReversePosts (exportsArcp,
992  numExportPacketsPerLID,
993  arcp<Packet> (imports.getRawPtr (), 0, imports.size (), false),
994  numImportPacketsPerLID);
995  doReverseWaits ();
996 
997  lastRoundBytesSend_ = exports.size() * sizeof(Packet);
998  lastRoundBytesRecv_ = imports.size() * sizeof(Packet);
999  }
1000 
1001  template <class Packet>
1002  void Distributor::
1003  doReversePosts (const Teuchos::ArrayRCP<const Packet>& exports,
1004  size_t numPackets,
1005  const Teuchos::ArrayRCP<Packet>& imports)
1006  {
1007  // FIXME (mfh 29 Mar 2012) WHY?
1008  TEUCHOS_TEST_FOR_EXCEPTION(
1009  ! plan_.getIndicesTo().is_null(), std::runtime_error,
1010  "Tpetra::Distributor::doReversePosts(3 args): Can only do reverse "
1011  "communication when original data are blocked by process.");
1012  if (reverseDistributor_.is_null ()) {
1013  createReverseDistributor ();
1014  }
1015  reverseDistributor_->doPosts (exports, numPackets, imports);
1016  }
1017 
1018  template <class Packet>
1019  void Distributor::
1020  doReversePosts (const Teuchos::ArrayRCP<const Packet>& exports,
1021  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1022  const Teuchos::ArrayRCP<Packet>& imports,
1023  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1024  {
1025  // FIXME (mfh 29 Mar 2012) WHY?
1026  TEUCHOS_TEST_FOR_EXCEPTION(
1027  ! plan_.getIndicesTo().is_null(), std::runtime_error,
1028  "Tpetra::Distributor::doReversePosts(3 args): Can only do reverse "
1029  "communication when original data are blocked by process.");
1030  if (reverseDistributor_.is_null ()) {
1031  createReverseDistributor ();
1032  }
1033  reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
1034  imports, numImportPacketsPerLID);
1035  }
1036 
1037  template <class ExpView, class ImpView>
1038  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1039  Distributor::
1040  doPostsAndWaits (const ExpView& exports,
1041  size_t numPackets,
1042  const ImpView& imports)
1043  {
1044  actor_.doPostsAndWaits(plan_, exports, numPackets, imports);
1045  }
1046 
1047  template <class ExpView, class ImpView>
1048  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1049  Distributor::
1050  doPostsAndWaits(const ExpView& exports,
1051  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1052  const ImpView& imports,
1053  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1054  {
1055  actor_.doPostsAndWaits(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
1056  }
1057 
1058 
1059  template <class ExpView, class ImpView>
1060  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1061  Distributor::
1062  doPosts (const ExpView &exports,
1063  size_t numPackets,
1064  const ImpView &imports)
1065  {
1066  actor_.doPosts(plan_, exports, numPackets, imports);
1067  }
1068 
1069  template <class ExpView, class ImpView>
1070  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1071  Distributor::
1072  doPosts (const ExpView &exports,
1073  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1074  const ImpView &imports,
1075  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1076  {
1077  actor_.doPosts(plan_, exports, numExportPacketsPerLID, imports, numImportPacketsPerLID);
1078  }
1079 
1080  template <class ExpView, class ImpView>
1081  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1082  Distributor::
1083  doReversePostsAndWaits (const ExpView& exports,
1084  size_t numPackets,
1085  const ImpView& imports)
1086  {
1087  doReversePosts (exports, numPackets, imports);
1088  doReverseWaits ();
1089  }
1090 
1091  template <class ExpView, class ImpView>
1092  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1093  Distributor::
1094  doReversePostsAndWaits (const ExpView& exports,
1095  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1096  const ImpView& imports,
1097  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1098  {
1099  doReversePosts (exports, numExportPacketsPerLID, imports,
1100  numImportPacketsPerLID);
1101  doReverseWaits ();
1102  }
1103 
1104  template <class ExpView, class ImpView>
1105  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1106  Distributor::
1107  doReversePosts (const ExpView &exports,
1108  size_t numPackets,
1109  const ImpView &imports)
1110  {
1111  // FIXME (mfh 29 Mar 2012) WHY?
1112  TEUCHOS_TEST_FOR_EXCEPTION(
1113  ! plan_.getIndicesTo().is_null(), std::runtime_error,
1114  "Tpetra::Distributor::doReversePosts(3 args): Can only do "
1115  "reverse communication when original data are blocked by process.");
1116  if (reverseDistributor_.is_null ()) {
1117  createReverseDistributor ();
1118  }
1119  reverseDistributor_->doPosts (exports, numPackets, imports);
1120  }
1121 
1122  template <class ExpView, class ImpView>
1123  typename std::enable_if<(Kokkos::Impl::is_view<ExpView>::value && Kokkos::Impl::is_view<ImpView>::value)>::type
1124  Distributor::
1125  doReversePosts (const ExpView &exports,
1126  const Teuchos::ArrayView<const size_t>& numExportPacketsPerLID,
1127  const ImpView &imports,
1128  const Teuchos::ArrayView<const size_t>& numImportPacketsPerLID)
1129  {
1130  // FIXME (mfh 29 Mar 2012) WHY?
1131  TEUCHOS_TEST_FOR_EXCEPTION(
1132  ! plan_.getIndicesTo().is_null(), std::runtime_error,
1133  "Tpetra::Distributor::doReversePosts(3 args): Can only do "
1134  "reverse communication when original data are blocked by process.");
1135  if (reverseDistributor_.is_null ()) {
1136  createReverseDistributor ();
1137  }
1138  reverseDistributor_->doPosts (exports, numExportPacketsPerLID,
1139  imports, numImportPacketsPerLID);
1140  }
1141 
1142  template <class OrdinalType>
1143  void Distributor::
1144  computeSends(const Teuchos::ArrayView<const OrdinalType>& importGIDs,
1145  const Teuchos::ArrayView<const int>& importProcIDs,
1146  Teuchos::Array<OrdinalType>& exportGIDs,
1147  Teuchos::Array<int>& exportProcIDs)
1148  {
1149  // NOTE (mfh 19 Apr 2012): There was a note on this code saying:
1150  // "assumes that size_t >= Ordinal". The code certainly does
1151  // assume that sizeof(size_t) >= sizeof(OrdinalType) as well as
1152  // sizeof(size_t) >= sizeof(int). This is because it casts the
1153  // OrdinalType elements of importGIDs (along with their
1154  // corresponding process IDs, as int) to size_t, and does a
1155  // doPostsAndWaits<size_t>() to send the packed data.
1156  using Teuchos::Array;
1157  using Teuchos::ArrayView;
1158  using std::endl;
1159  using size_type = typename ArrayView<const OrdinalType>::size_type;
1160  const char errPrefix[] = "Tpetra::Distributor::computeSends: ";
1161  const char suffix[] =
1162  " Please report this bug to the Tpetra developers.";
1163 
1164  const int myRank = plan_.getComm()->getRank ();
1165 
1166  TEUCHOS_TEST_FOR_EXCEPTION
1167  (importGIDs.size () != importProcIDs.size (),
1168  std::invalid_argument, errPrefix << "On Process " << myRank
1169  << ": importProcIDs.size()=" << importProcIDs.size()
1170  << " != importGIDs.size()=" << importGIDs.size() << ".");
1171 
1172  const size_type numImports = importProcIDs.size();
1173  Array<size_t> importObjs(2*numImports);
1174  // Pack pairs (importGIDs[i], my process ID) to send into importObjs.
1175  for (size_type i = 0; i < numImports; ++i) {
1176  importObjs[2*i] = static_cast<size_t>(importGIDs[i]);
1177  importObjs[2*i+1] = static_cast<size_t>(myRank);
1178  }
1179  //
1180  // Use a temporary Distributor to send the (importGIDs[i], myRank)
1181  // pairs to importProcIDs[i].
1182  //
1183  Distributor tempPlan(plan_.getComm());
1184  // mfh 20 Mar 2014: An extra-cautious cast from unsigned to
1185  // signed, in order to forestall any possible causes for Bug 6069.
1186  const size_t numExportsAsSizeT =
1187  tempPlan.createFromSends(importProcIDs);
1188  const size_type numExports =
1189  static_cast<size_type>(numExportsAsSizeT);
1190  TEUCHOS_TEST_FOR_EXCEPTION
1191  (numExports < 0, std::logic_error, errPrefix <<
1192  "tempPlan.createFromSends() returned numExports="
1193  << numExportsAsSizeT << " as a size_t, which overflows to "
1194  << numExports << " when cast to " <<
1195  Teuchos::TypeNameTraits<size_type>::name () << "." << suffix);
1196  TEUCHOS_TEST_FOR_EXCEPTION
1197  (size_type(tempPlan.getTotalReceiveLength()) != numExports,
1198  std::logic_error, errPrefix << "tempPlan.getTotalReceiveLength()="
1199  << tempPlan.getTotalReceiveLength () << " != numExports="
1200  << numExports << "." << suffix);
1201 
1202  if (numExports > 0) {
1203  exportGIDs.resize(numExports);
1204  exportProcIDs.resize(numExports);
1205  }
1206 
1207  // exportObjs: Packed receive buffer. (exportObjs[2*i],
1208  // exportObjs[2*i+1]) will give the (GID, PID) pair for export i,
1209  // after tempPlan.doPostsAndWaits(...) finishes below.
1210  //
1211  // FIXME (mfh 19 Mar 2014) This only works if OrdinalType fits in
1212  // size_t. This issue might come up, for example, on a 32-bit
1213  // machine using 64-bit global indices. I will add a check here
1214  // for that case.
1215  static_assert(sizeof(size_t) >= sizeof(OrdinalType),
1216  "Tpetra::Distributor::computeSends: "
1217  "sizeof(size_t) < sizeof(OrdinalType).");
1218 
1219  TEUCHOS_TEST_FOR_EXCEPTION
1220  (tempPlan.getTotalReceiveLength () < size_t(numExports),
1221  std::logic_error,
1222  errPrefix << "tempPlan.getTotalReceiveLength()="
1223  << tempPlan.getTotalReceiveLength() << " < numExports="
1224  << numExports << "." << suffix);
1225 
1226  Array<size_t> exportObjs (tempPlan.getTotalReceiveLength () * 2);
1227  tempPlan.doPostsAndWaits<size_t> (importObjs (), 2, exportObjs ());
1228 
1229  // Unpack received (GID, PID) pairs into exportIDs resp. exportProcIDs.
1230  for (size_type i = 0; i < numExports; ++i) {
1231  exportGIDs[i] = static_cast<OrdinalType> (exportObjs[2*i]);
1232  exportProcIDs[i] = static_cast<int> (exportObjs[2*i+1]);
1233  }
1234  }
1235 
1236  template <class OrdinalType>
1237  void Distributor::
1238  createFromRecvs (const Teuchos::ArrayView<const OrdinalType> &remoteGIDs,
1239  const Teuchos::ArrayView<const int> &remoteProcIDs,
1240  Teuchos::Array<OrdinalType> &exportGIDs,
1241  Teuchos::Array<int> &exportProcIDs)
1242  {
1243  using std::endl;
1244  const char errPrefix[] = "Tpetra::Distributor::createFromRecvs: ";
1245  const int myRank = plan_.getComm()->getRank();
1246 
1247  std::unique_ptr<std::string> prefix;
1248  if (verbose_) {
1249  prefix = createPrefix("createFromRecvs");
1250  std::ostringstream os;
1251  os << *prefix << "Start" << endl;
1252  std::cerr << os.str();
1253  }
1254 
1255  const bool debug = Details::Behavior::debug("Distributor");
1256  if (debug) {
1257  using Teuchos::outArg;
1258  using Teuchos::REDUCE_MAX;
1259  using Teuchos::reduceAll;
1260  // In debug mode, first test locally, then do an all-reduce to
1261  // make sure that all processes passed.
1262  const int errProc =
1263  (remoteGIDs.size () != remoteProcIDs.size ()) ? myRank : -1;
1264  int maxErrProc = -1;
1265  reduceAll(*plan_.getComm(), REDUCE_MAX, errProc, outArg(maxErrProc));
1266  TEUCHOS_TEST_FOR_EXCEPTION
1267  (maxErrProc != -1, std::runtime_error, errPrefix << "Lists "
1268  "of remote IDs and remote process IDs must have the same "
1269  "size on all participating processes. Maximum process ID "
1270  "with error: " << maxErrProc << ".");
1271  }
1272  else { // in non-debug mode, just test locally
1273  // NOTE (mfh 13 Feb 2020) This needs to throw std::runtime_error
1274  // in order to make an existing Distributor unit test pass.
1275  TEUCHOS_TEST_FOR_EXCEPTION
1276  (remoteGIDs.size() != remoteProcIDs.size(), std::runtime_error,
1277  errPrefix << "On Process " << myRank << ": "
1278  "remoteGIDs.size()=" << remoteGIDs.size() <<
1279  " != remoteProcIDs.size()=" << remoteProcIDs.size() << ".");
1280  }
1281 
1282  computeSends(remoteGIDs, remoteProcIDs, exportGIDs, exportProcIDs);
1283 
1284  plan_.createFromRecvs(remoteProcIDs);
1285 
1286  if (verbose_) {
1287  std::ostringstream os;
1288  os << *prefix << "Done" << endl;
1289  std::cerr << os.str();
1290  }
1291  }
1292 
1293 } // namespace Tpetra
1294 
1295 #endif // TPETRA_DISTRIBUTOR_HPP
Tpetra_Details_Behavior.hpp
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra's behavior.
Tpetra_Util.hpp
Stand-alone utility functions and macros.
Tpetra::Details::Behavior::debug
static bool debug()
Whether Tpetra is in debug mode.
Definition: Tpetra_Details_Behavior.cpp:302
Tpetra::Distributor
Sets up and executes a communication plan for a Tpetra DistObject.
Definition: Tpetra_Distributor.hpp:139
Tpetra::Distributor::doPostsAndWaits
void doPostsAndWaits(const Teuchos::ArrayView< const Packet > &exports, size_t numPackets, const Teuchos::ArrayView< Packet > &imports)
Execute the (forward) communication plan.
Definition: Tpetra_Distributor.hpp:825
Tpetra::Distributor::doReversePostsAndWaits
void doReversePostsAndWaits(const Teuchos::ArrayView< const Packet > &exports, size_t numPackets, const Teuchos::ArrayView< Packet > &imports)
Execute the reverse communication plan.
Definition: Tpetra_Distributor.hpp:932
Tpetra::Distributor::getMaxSendLength
size_t getMaxSendLength() const
Maximum number of values this process will send to another single process.
Definition: Tpetra_Distributor.cpp:254
Tpetra::Distributor::getReverse
Teuchos::RCP< Distributor > getReverse(bool create=true) const
A reverse communication plan Distributor.
Definition: Tpetra_Distributor.cpp:270
Tpetra::Distributor::createFromRecvs
void createFromRecvs(const Teuchos::ArrayView< const Ordinal > &remoteIDs, const Teuchos::ArrayView< const int > &remoteProcIDs, Teuchos::Array< Ordinal > &exportIDs, Teuchos::Array< int > &exportProcIDs)
Set up Distributor using list of process ranks from which to receive.
Tpetra::Distributor::getProcsTo
Teuchos::ArrayView< const int > getProcsTo() const
Ranks of the processes to which this process will send values.
Definition: Tpetra_Distributor.cpp:263
Tpetra::Distributor::doReversePosts
void doReversePosts(const Teuchos::ArrayRCP< const Packet > &exports, size_t numPackets, const Teuchos::ArrayRCP< Packet > &imports)
Post the data for a reverse plan, but do not execute the waits yet.
Definition: Tpetra_Distributor.hpp:1003
Tpetra::Distributor::getNumReceives
size_t getNumReceives() const
The number of processes from which we will receive data.
Definition: Tpetra_Distributor.cpp:245
Tpetra::Distributor::doPosts
void doPosts(const Teuchos::ArrayRCP< const Packet > &exports, size_t numPackets, const Teuchos::ArrayRCP< Packet > &imports)
Post the data for a forward plan, but do not execute the waits yet.
Definition: Tpetra_Distributor.hpp:909
Tpetra::Distributor::setParameterList
void setParameterList(const Teuchos::RCP< Teuchos::ParameterList > &plist)
Set Distributor parameters.
Definition: Tpetra_Distributor.cpp:163
Tpetra::Distributor::getTotalReceiveLength
size_t getTotalReceiveLength() const
Total number of values this process will receive from other processes.
Definition: Tpetra_Distributor.cpp:242
Tpetra::Distributor::~Distributor
virtual ~Distributor()=default
Destructor (virtual for memory safety).
Tpetra::Distributor::hasSelfMessage
bool hasSelfMessage() const
Whether the calling process will send or receive messages to itself.
Definition: Tpetra_Distributor.cpp:248
Tpetra::Distributor::swap
void swap(Distributor &rhs)
Swap the contents of rhs with those of *this.
Definition: Tpetra_Distributor.cpp:116
Tpetra::Distributor::getPlan
const Details::DistributorPlan & getPlan() const
Get this Distributor's DistributorPlan.
Definition: Tpetra_Distributor.hpp:758
Tpetra::Distributor::getLengthsTo
Teuchos::ArrayView< const size_t > getLengthsTo() const
Number of values this process will send to each process.
Definition: Tpetra_Distributor.cpp:266
Tpetra::Distributor::getProcsFrom
Teuchos::ArrayView< const int > getProcsFrom() const
Ranks of the processes sending values to this process.
Definition: Tpetra_Distributor.cpp:257
Tpetra::Distributor::Distributor
Distributor(const Teuchos::RCP< const Teuchos::Comm< int > > &comm)
Construct using the specified communicator and default parameters.
Definition: Tpetra_Distributor.cpp:81
Tpetra::Distributor::description
std::string description() const
Return a one-line description of this object.
Definition: Tpetra_Distributor.cpp:326
Tpetra::Distributor::createFromSends
size_t createFromSends(const Teuchos::ArrayView< const int > &exportProcIDs)
Set up Distributor using list of process ranks to which this process will send.
Definition: Tpetra_Distributor.cpp:480
Tpetra::Distributor::createFromSendsAndRecvs
void createFromSendsAndRecvs(const Teuchos::ArrayView< const int > &exportProcIDs, const Teuchos::ArrayView< const int > &remoteProcIDs)
Set up Distributor using list of process ranks to which to send, and list of process ranks from which...
Definition: Tpetra_Distributor.cpp:487
Tpetra::Distributor::getValidParameters
Teuchos::RCP< const Teuchos::ParameterList > getValidParameters() const
List of valid Distributor parameters.
Definition: Tpetra_Distributor.cpp:190
Tpetra::Distributor::getLengthsFrom
Teuchos::ArrayView< const size_t > getLengthsFrom() const
Number of values this process will receive from each process.
Definition: Tpetra_Distributor.cpp:260
Tpetra::Distributor::howInitialized
Details::EDistributorHowInitialized howInitialized() const
Return an enum indicating whether and how a Distributor was initialized.
Definition: Tpetra_Distributor.hpp:369
Tpetra::Distributor::getNumSends
size_t getNumSends() const
The number of processes to which we will send data.
Definition: Tpetra_Distributor.cpp:251
Tpetra::Distributor::describe
void describe(Teuchos::FancyOStream &out, const Teuchos::EVerbosityLevel verbLevel=Teuchos::Describable::verbLevel_default) const
Describe this object in a human-readable way to the given output stream.
Definition: Tpetra_Distributor.cpp:395
Tpetra::Distributor::getLastDoStatistics
void getLastDoStatistics(size_t &bytes_sent, size_t &bytes_recvd) const
Information on the last call to do/doReverse.
Definition: Tpetra_Distributor.hpp:715
Tpetra::Details::EDistributorHowInitialized
EDistributorHowInitialized
Enum indicating how and whether a Distributor was initialized.
Definition: Tpetra_Details_DistributorPlan.hpp:78
Tpetra
Namespace Tpetra contains the class and methods constituting the Tpetra library.
Tpetra::distributorSendTypes
Teuchos::Array< std::string > distributorSendTypes()
Valid values for Distributor's "Send type" parameter.
Definition: Tpetra_Distributor.cpp:59
Generated by doxygen 1.9.1