Kokkos_HBWSpace.hpp

/*
//@HEADER
// ************************************************************************
//
//                        Kokkos v. 2.0
//              Copyright (2014) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact  H. Carter Edwards (hcedwar@sandia.gov)
//
// ************************************************************************
//@HEADER
*/

#ifndef KOKKOS_HBWSPACE_HPP
#define KOKKOS_HBWSPACE_HPP

#include <Kokkos_Macros.hpp>
#ifdef KOKKOS_ENABLE_HBWSPACE

#include <Kokkos_HostSpace.hpp>

namespace Kokkos {

namespace Experimental {

namespace Impl {

void init_lock_array_hbw_space();

bool lock_address_hbw_space( void* ptr );

void unlock_address_hbw_space( void* ptr );

} // namespace Impl

} // namespace Experimental

} // namespace Kokkos

namespace Kokkos {

namespace Experimental {

class HBWSpace {
public:
  typedef HBWSpace  memory_space;
  typedef size_t     size_type;

#if defined( KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_OPENMP )
  typedef Kokkos::OpenMP    execution_space;
#elif defined( KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_THREADS )
  typedef Kokkos::Threads   execution_space;
//#elif defined( KOKKOS_ENABLE_DEFAULT_DEVICE_TYPE_QTHREADS )
//  typedef Kokkos::Qthreads  execution_space;
#elif defined( KOKKOS_ENABLE_OPENMP )
  typedef Kokkos::OpenMP    execution_space;
#elif defined( KOKKOS_ENABLE_THREADS )
  typedef Kokkos::Threads   execution_space;
//#elif defined( KOKKOS_ENABLE_QTHREADS )
//  typedef Kokkos::Qthreads  execution_space;
#elif defined( KOKKOS_ENABLE_SERIAL )
  typedef Kokkos::Serial    execution_space;
#else
#  error "At least one of the following host execution spaces must be defined: Kokkos::OpenMP, Kokkos::Threads, Kokkos::Qhreads, or Kokkos::Serial.  You might be seeing this message if you disabled the Kokkos::Serial device explicitly using the Kokkos_ENABLE_Serial:BOOL=OFF CMake option, but did not enable any of the other host execution space devices."
#endif

  typedef Kokkos::Device< execution_space, memory_space > device_type;

  /*--------------------------------*/
  /* Functions unique to the HBWSpace */
  static int in_parallel();

  static void register_in_parallel( int (*)() );

  /*--------------------------------*/

  HBWSpace();
  HBWSpace( const HBWSpace & rhs ) = default;
  HBWSpace & operator = ( const HBWSpace & ) = default;
  ~HBWSpace() = default;

  enum AllocationMechanism { STD_MALLOC, POSIX_MEMALIGN, POSIX_MMAP, INTEL_MM_ALLOC };

  explicit
  HBWSpace( const AllocationMechanism & );

  void * allocate( const size_t arg_alloc_size ) const;

  void deallocate( void * const arg_alloc_ptr
                 , const size_t arg_alloc_size ) const;

  static constexpr const char* name();

private:

  AllocationMechanism  m_alloc_mech;
  static constexpr const char* m_name = "HBW";
  friend class Kokkos::Impl::SharedAllocationRecord< Kokkos::Experimental::HBWSpace, void >;
};

} // namespace Experimental

} // namespace Kokkos

//----------------------------------------------------------------------------

namespace Kokkos {

namespace Impl {

template<>
class SharedAllocationRecord< Kokkos::Experimental::HBWSpace, void >
  : public SharedAllocationRecord< void, void >
{
private:

  friend Kokkos::Experimental::HBWSpace;

  typedef SharedAllocationRecord< void, void >  RecordBase;

  SharedAllocationRecord( const SharedAllocationRecord & ) = delete;
  SharedAllocationRecord & operator = ( const SharedAllocationRecord & ) = delete;

  static void deallocate( RecordBase * );

  static RecordBase s_root_record;

  const Kokkos::Experimental::HBWSpace m_space;

protected:

  ~SharedAllocationRecord();
  SharedAllocationRecord() = default;

  SharedAllocationRecord( const Kokkos::Experimental::HBWSpace & arg_space
                        , const std::string                    & arg_label
                        , const size_t                           arg_alloc_size
                        , const RecordBase::function_type        arg_dealloc = & deallocate
                        );

public:

  inline
  std::string get_label() const
    {
      return std::string( RecordBase::head()->m_label );
    }

  KOKKOS_INLINE_FUNCTION static
  SharedAllocationRecord * allocate( const Kokkos::Experimental::HBWSpace & arg_space
                                   , const std::string                    & arg_label
                                   , const size_t                           arg_alloc_size
                                   )
    {
#if defined( KOKKOS_ACTIVE_EXECUTION_MEMORY_SPACE_HOST )
      return new SharedAllocationRecord( arg_space, arg_label, arg_alloc_size );
#else
      return (SharedAllocationRecord *) 0;
#endif
    }

  static
  void * allocate_tracked( const Kokkos::Experimental::HBWSpace & arg_space
                         , const std::string                    & arg_label
                         , const size_t                           arg_alloc_size );

  static
  void * reallocate_tracked( void * const arg_alloc_ptr
                           , const size_t arg_alloc_size );

  static
  void deallocate_tracked( void * const arg_alloc_ptr );

  static SharedAllocationRecord * get_record( void * arg_alloc_ptr );

  static void print_records( std::ostream &, const Kokkos::Experimental::HBWSpace &, bool detail = false );
};

} // namespace Impl

} // namespace Kokkos

//----------------------------------------------------------------------------

namespace Kokkos {

namespace Impl {

static_assert( Kokkos::Impl::MemorySpaceAccess< Kokkos::Experimental::HBWSpace, Kokkos::Experimental::HBWSpace >::assignable, "" );

template<>
struct MemorySpaceAccess< Kokkos::HostSpace, Kokkos::Experimental::HBWSpace > {
  enum { assignable = true };
  enum { accessible = true };
  enum { deepcopy   = true };
};

template<>
struct MemorySpaceAccess< Kokkos::Experimental::HBWSpace, Kokkos::HostSpace > {
  enum { assignable = false };
  enum { accessible = true };
  enum { deepcopy   = true };
};

} // namespace Impl

} // namespace Kokkos

//----------------------------------------------------------------------------

namespace Kokkos {

namespace Impl {

template< class ExecutionSpace >
struct DeepCopy< Experimental::HBWSpace, Experimental::HBWSpace, ExecutionSpace > {
  DeepCopy( void * dst, const void * src, size_t n ) {
    memcpy( dst, src, n );
  }

  DeepCopy( const ExecutionSpace& exec, void * dst, const void * src, size_t n ) {
    exec.fence();
    memcpy( dst, src, n );
  }
};

template< class ExecutionSpace >
struct DeepCopy< HostSpace, Experimental::HBWSpace, ExecutionSpace > {
  DeepCopy( void * dst, const void * src, size_t n ) {
    memcpy( dst, src, n );
  }

  DeepCopy( const ExecutionSpace& exec, void * dst, const void * src, size_t n ) {
    exec.fence();
    memcpy( dst, src, n );
  }
};

template< class ExecutionSpace >
struct DeepCopy< Experimental::HBWSpace, HostSpace, ExecutionSpace > {
  DeepCopy( void * dst, const void * src, size_t n ) {
    memcpy( dst, src, n );
  }

  DeepCopy( const ExecutionSpace& exec, void * dst, const void * src, size_t n ) {
    exec.fence();
    memcpy( dst, src, n );
  }
};

} // namespace Impl

} // namespace Kokkos

namespace Kokkos {

namespace Impl {

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::HostSpace, Kokkos::Experimental::HBWSpace >
{
  enum { value = true };
  inline static void verify( void ) { }
  inline static void verify( const void * ) { }
};

template<>
struct VerifyExecutionCanAccessMemorySpace< Kokkos::Experimental::HBWSpace, Kokkos::HostSpace >
{
  enum { value = true };
  inline static void verify( void ) { }
  inline static void verify( const void * ) { }
};

} // namespace Impl

} // namespace Kokkos

#endif
#endif // #define KOKKOS_HBWSPACE_HPP