Ifpack2_RILUK_decl.hpp

Go to the documentation of this file.

/*@HEADER
// ***********************************************************************
//
//       Ifpack2: Tempated Object-Oriented Algebraic Preconditioner Package
//                 Copyright (2009) Sandia Corporation
//
// Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
// license for use of this work by or on behalf of the U.S. Government.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ***********************************************************************
//@HEADER
*/


#ifndef IFPACK2_CRSRILUK_DECL_HPP
#define IFPACK2_CRSRILUK_DECL_HPP

#include "Ifpack2_Preconditioner.hpp"
#include "Ifpack2_Details_CanChangeMatrix.hpp"
#include "Tpetra_CrsMatrix_decl.hpp"
#include "Ifpack2_ScalingType.hpp"
#include "Ifpack2_IlukGraph.hpp"
#include "Ifpack2_LocalSparseTriangularSolver_decl.hpp"

#include <type_traits>

//Experimental Threaded ILUK with Basker
#ifdef IFPACK2_ILUK_EXPERIMENTAL
#include <Kokkos_Core.hpp>
#include <shylubasker_decl.hpp>
# ifdef IFPACK2_HTS_EXPERIMENTAL
#  include <ShyLUHTS_config.h>
#  include <shylu_hts_decl.hpp>
# endif
#endif


namespace Teuchos {
  class ParameterList; // forward declaration
}

namespace Ifpack2 {

template<class MatrixType>
class RILUK:
    virtual public Ifpack2::Preconditioner<typename MatrixType::scalar_type,
                                           typename MatrixType::local_ordinal_type,
                                           typename MatrixType::global_ordinal_type,
                                           typename MatrixType::node_type>,
    virtual public Ifpack2::Details::CanChangeMatrix<Tpetra::RowMatrix<typename MatrixType::scalar_type,
                                                                       typename MatrixType::local_ordinal_type,
                                                                       typename MatrixType::global_ordinal_type,
                                                                       typename MatrixType::node_type> >
{
 public:
  typedef typename MatrixType::scalar_type scalar_type;

  typedef typename MatrixType::local_ordinal_type local_ordinal_type;

  typedef typename MatrixType::global_ordinal_type global_ordinal_type;

  typedef typename MatrixType::node_type node_type;

  typedef typename Teuchos::ScalarTraits<scalar_type>::magnitudeType magnitude_type;

  typedef Tpetra::RowMatrix<scalar_type,
                            local_ordinal_type,
                            global_ordinal_type,
                            node_type> row_matrix_type;


  static_assert(std::is_same<MatrixType, row_matrix_type>::value, "Ifpack2::RILUK: The template parameter MatrixType must be a Tpetra::RowMatrix specialization.  Please don't use Tpetra::CrsMatrix (a subclass of Tpetra::RowMatrix) here anymore.");

  typedef Tpetra::CrsMatrix<scalar_type,
                            local_ordinal_type,
                            global_ordinal_type,
                            node_type> crs_matrix_type;

  template <class NewMatrixType> friend class RILUK;

  RILUK (const Teuchos::RCP<const row_matrix_type>& A_in);

  RILUK (const Teuchos::RCP<const crs_matrix_type>& A_in);

 private:
  RILUK (const RILUK<MatrixType> & src);

 public:
  template <typename NewMatrixType>
  Teuchos::RCP< RILUK< NewMatrixType > >
  clone (const Teuchos::RCP<const NewMatrixType>& A_newnode) const;

  virtual ~RILUK ();

  void setParameters (const Teuchos::ParameterList& params);

  void initialize ();

  void compute ();

  bool isInitialized () const {
    return isInitialized_;
  }
  bool isComputed () const {
    return isComputed_;
  }

  int getNumInitialize () const {
    return numInitialize_;
  }
  int getNumCompute () const {
    return numCompute_;
  }
  int getNumApply () const {
    return numApply_;
  }

  double getInitializeTime () const {
    return initializeTime_;
  }
  double getComputeTime () const {
    return computeTime_;
  }
  double getApplyTime () const {
    return applyTime_;
  }



  virtual void
  setMatrix (const Teuchos::RCP<const row_matrix_type>& A);



  std::string description () const;



  Teuchos::RCP<const Tpetra::Map<local_ordinal_type,global_ordinal_type,node_type> >
  getDomainMap () const;

  Teuchos::RCP<const Tpetra::Map<local_ordinal_type,global_ordinal_type,node_type> >
  getRangeMap () const;

  void
  apply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type>& X,
         Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type>& Y,
         Teuchos::ETransp mode = Teuchos::NO_TRANS,
         scalar_type alpha = Teuchos::ScalarTraits<scalar_type>::one (),
         scalar_type beta = Teuchos::ScalarTraits<scalar_type>::zero ()) const;

private:
  void
  multiply (const Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type>& X,
            Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type>& Y,
            const Teuchos::ETransp mode = Teuchos::NO_TRANS) const;
public:

  Teuchos::RCP<const row_matrix_type> getMatrix () const;

  // Attribute access functions

  magnitude_type getRelaxValue () const { return RelaxValue_; }

  magnitude_type getAbsoluteThreshold () const { return Athresh_; }

  magnitude_type getRelativeThreshold () const {return Rthresh_;}

  int getLevelOfFill () const { return LevelOfFill_; }

  Tpetra::CombineMode getOverlapMode () {
    TEUCHOS_TEST_FOR_EXCEPTION(
      true, std::logic_error, "Ifpack2::RILUK::SetOverlapMode: "
      "RILUK no longer implements overlap on its own.  "
      "Use RILUK with AdditiveSchwarz if you want overlap.");
  }

  Tpetra::global_size_t getGlobalNumEntries () const {
    return getL ().getGlobalNumEntries () + getU ().getGlobalNumEntries ();
  }

  Teuchos::RCP<Ifpack2::IlukGraph<Tpetra::CrsGraph<local_ordinal_type,global_ordinal_type,node_type> > > getGraph () const {
    return Graph_;
  }

  const crs_matrix_type& getL () const;

  const Tpetra::Vector<scalar_type,local_ordinal_type,global_ordinal_type,node_type>&
  getD () const;

  const crs_matrix_type& getU () const;

  Teuchos::RCP<const crs_matrix_type> getCrsMatrix () const;

private:
  typedef Tpetra::MultiVector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> MV;
  typedef Teuchos::ScalarTraits<scalar_type> STS;
  typedef Teuchos::ScalarTraits<magnitude_type> STM;

  void allocateSolvers ();
  void allocate_L_and_U ();
  static void checkOrderingConsistency (const row_matrix_type& A);
  void initAllValues (const row_matrix_type& A);

  static Teuchos::RCP<const row_matrix_type>
  makeLocalFilter (const Teuchos::RCP<const row_matrix_type>& A);

protected:
  typedef Tpetra::Vector<scalar_type,local_ordinal_type,global_ordinal_type,node_type> vec_type;

  Teuchos::RCP<const row_matrix_type> A_;

  Teuchos::RCP<Ifpack2::IlukGraph<Tpetra::CrsGraph<local_ordinal_type,
                                                   global_ordinal_type,
                                                   node_type> > > Graph_;
  Teuchos::RCP<const row_matrix_type> A_local_;

  Teuchos::RCP<crs_matrix_type> L_;
  Teuchos::RCP<LocalSparseTriangularSolver<row_matrix_type> > L_solver_;
  Teuchos::RCP<crs_matrix_type> U_;
  Teuchos::RCP<LocalSparseTriangularSolver<row_matrix_type> > U_solver_;
  Teuchos::RCP<vec_type> D_;


#ifdef IFPACK2_ILUK_EXPERIMENTAL
  typedef typename node_type::device_type  kokkos_device;
  typedef typename kokkos_device::execution_space kokkos_exe;

  static_assert( std::is_same< kokkos_exe,
                 Kokkos::OpenMP>::value,
                 "Kokkos node type not supported by exepertimentalthread basker RILUK decl");

  // Basker needs the CRS matrix.
  Teuchos::RCP<const crs_matrix_type> A_local_crs_;
  // If the nonzero pattern is being reused, Basker needs A_local_ to be
  // (possibly) copied or otherwise cast to a crs_matrix_type.
  void initLocalCrs ();

  Teuchos::RCP< BaskerNS::Basker<local_ordinal_type, scalar_type, Kokkos::OpenMP> >
  myBasker;
  local_ordinal_type basker_threads;
  scalar_type        basker_user_fill;
  bool               basker_reuse;

# ifdef IFPACK2_HTS_EXPERIMENTAL
  bool use_hts_;
  int hts_nthreads_;
  typedef ::Experimental::HTS<local_ordinal_type, local_ordinal_type, scalar_type> HTST;
  struct HTSData : public HTST::Deallocator {
    local_ordinal_type n;
    local_ordinal_type* jc, * ir;
    scalar_type* v;
    HTSData();
    ~HTSData();
    virtual void free_CrsMatrix_data();
    struct Entry {
      bool operator< (const Entry& o) const { return j < o.j; }
      local_ordinal_type j; scalar_type v;
    };
    void sort();
  };
  struct HTSManager {
    typename HTST::Impl* Limpl, * Uimpl;
    HTSManager();
    ~HTSManager();
  };
  Teuchos::RCP<HTSManager> hts_mgr_;
# endif
#endif

  int LevelOfFill_;

  bool isAllocated_;
  bool isInitialized_;
  bool isComputed_;
  bool isExperimental_;

  int numInitialize_;
  int numCompute_;
  mutable int numApply_;

  double initializeTime_;
  double computeTime_;
  mutable double applyTime_;

  magnitude_type RelaxValue_;
  magnitude_type Athresh_;
  magnitude_type Rthresh_;

};

// NOTE (mfh 11 Feb 2015) This used to exist in order to deal with
// different behavior of Tpetra::Crs{Graph,Matrix} for
// KokkosClassic::ThrustGPUNode.  In particular, fillComplete on a
// CrsMatrix used to make the graph go away by default, so we had to
// pass in a parameter to keep a host copy of the graph.  With the new
// (Kokkos refactor) version of Tpetra, this problem has gone away.
namespace detail {
  template<class MatrixType, class NodeType>
  struct setLocalSolveParams{
    static Teuchos::RCP<Teuchos::ParameterList>
    setParams (const Teuchos::RCP<Teuchos::ParameterList>& param) {
      return param;
    }
  };
} // namespace detail

template <class MatrixType>
template <typename NewMatrixType>
Teuchos::RCP<RILUK<NewMatrixType> >
RILUK<MatrixType>::
clone (const Teuchos::RCP<const NewMatrixType>& A_newnode) const
{
  using Teuchos::ParameterList;
  using Teuchos::RCP;
  using Teuchos::rcp;

  typedef typename NewMatrixType::scalar_type new_scalar_type;
  typedef typename NewMatrixType::local_ordinal_type new_local_ordinal_type;
  typedef typename NewMatrixType::global_ordinal_type new_global_ordinal_type;
  typedef typename NewMatrixType::node_type new_node_type;
  typedef Tpetra::RowMatrix<new_scalar_type, new_local_ordinal_type,
    new_global_ordinal_type, new_node_type> new_row_matrix_type;
  typedef RILUK<new_row_matrix_type> new_riluk_type;

  RCP<new_riluk_type> new_riluk = rcp (new new_riluk_type (A_newnode));

  RCP<ParameterList> plClone = Teuchos::parameterList ();
  plClone = detail::setLocalSolveParams<NewMatrixType, new_node_type>::setParams (plClone);

  RCP<new_node_type> new_node = A_newnode->getNode ();
  new_riluk->L_ = L_->clone (new_node, plClone);
  new_riluk->U_ = U_->clone (new_node, plClone);
  new_riluk->D_ = D_->clone (new_node);

  new_riluk->LevelOfFill_ = LevelOfFill_;

  new_riluk->isAllocated_ = isAllocated_;
  new_riluk->isInitialized_ = isInitialized_;
  new_riluk->isComputed_ = isComputed_;

  new_riluk->numInitialize_ = numInitialize_;
  new_riluk->numCompute_ = numCompute_;
  new_riluk->numApply_ =  numApply_;

  new_riluk->RelaxValue_ = RelaxValue_;
  new_riluk->Athresh_ = Athresh_;
  new_riluk->Rthresh_ = Rthresh_;


  return new_riluk;
}

} // namespace Ifpack2

#endif /* IFPACK2_CRSRILUK_DECL_HPP */