7#ifndef HEFFFTE_COMMON_H
8#define HEFFFTE_COMMON_H
10#include "heffte_geometry.h"
11#include "heffte_trace.h"
69 template<
typename source_type,
typename destination_type>
70 static void copy_n(
void*, source_type
const source[],
size_t num_entries, destination_type destination[]){
71 std::copy_n(source, num_entries, destination);
74 template<
typename source_type,
typename destination_type>
75 static void copy_n(source_type
const source[],
size_t num_entries, destination_type destination[]){
76 std::copy_n(source, num_entries, destination);
79 template<
typename source_type,
typename destination_type>
80 static void copy_device_to_host(
void*, source_type
const source[],
size_t num_entries, destination_type destination[]){
81 std::copy_n(source, num_entries, destination);
84 template<
typename source_type,
typename destination_type>
85 static void copy_device_to_device(
void*, source_type
const source[],
size_t num_entries, destination_type destination[]){
86 std::copy_n(source, num_entries, destination);
89 template<
typename source_type,
typename destination_type>
90 static void copy_host_to_device(
void*, source_type
const source[],
size_t num_entries, destination_type destination[]){
91 std::copy_n(source, num_entries, destination);
225 template<
typename tag>
236 template<typename backend_tag, typename std::enable_if<is_enabled<backend_tag>::value,
void*>::type =
nullptr>
248 template<
typename backend_tag,
typename =
void>
255 template<
typename backend_tag>
257 typename std::enable_if<std::is_same<typename buffer_traits<backend_tag>::location, tag::gpu>::value, void>::type>
264 template<
typename backend_tag>
265 inline std::string
name(){
return "unknown"; }
318 template<>
inline std::string
name<mkl>(){
return "mkl"; }
407 template<
typename backend_tag>
445 template<
typename backend_tag,
typename input,
typename output,
typename =
void>
struct check_types : std::false_type{};
451 template<
typename backend_tag,
typename input,
typename output>
struct check_types<backend_tag, input, output,
452 typename std::enable_if<uses_fft_types<backend_tag>::value and (
453 (std::is_same<input, float>::value and is_ccomplex<output>::value)
454 or (std::is_same<input, double>::value and is_zcomplex<output>::value)
455 or (is_ccomplex<input>::value and is_ccomplex<output>::value)
456 or (is_zcomplex<input>::value and is_zcomplex<output>::value)
457 )>::type> : std::true_type{};
549 template<
typename backend_tag,
typename input,
typename output>
struct check_types<backend_tag, input, output,
550 typename std::enable_if<not uses_fft_types<backend_tag>::value and (
551 (std::is_same<input, float>::value and std::is_same<output, float>::value)
552 or (std::is_same<input, double>::value and std::is_same<output, double>::value)
553 )>::type> : std::true_type{};
568 virtual void forward(
double[],
double*)
const{}
574 virtual void forward(std::complex<float>[], std::complex<float>*)
const{}
576 virtual void forward(std::complex<double>[], std::complex<double>*)
const{}
578 virtual void backward(std::complex<float>[], std::complex<float>*)
const{}
580 virtual void backward(std::complex<double>[], std::complex<double>*)
const{}
582 virtual void forward(
float const[], std::complex<float>[], std::complex<float>*)
const{}
584 virtual void forward(
double const[], std::complex<double>[], std::complex<double>*)
const{}
586 virtual void backward(std::complex<float>[],
float[], std::complex<float>*)
const{}
588 virtual void backward(std::complex<double>[],
double[], std::complex<double>*)
const{}
601template<
typename backend_tag>
605 if (std::is_same<backend_tag, backend::cufft>::value) {
606 return (
reinterpret_cast<size_t>(p) %
sizeof(std::complex<float>) == 0) ? p : p+1;
612 static double*
pntr(
double *p) {
613 if (std::is_same<backend_tag, backend::cufft>::value) {
614 return (
reinterpret_cast<size_t>(p) %
sizeof(std::complex<double>) == 0) ? p : p+1;
620 static std::complex<float>*
pntr(std::complex<float> *p) {
621 if (std::is_same<backend_tag, backend::cufft>::value) {
622 return (
reinterpret_cast<size_t>(p) %
sizeof(std::complex<float>) == 0) ? p :
623 reinterpret_cast<std::complex<float>*
>(
reinterpret_cast<float*
>(p) + 1);
629 static std::complex<double>*
pntr(std::complex<double> *p) {
630 if (std::is_same<backend_tag, backend::cufft>::value) {
631 return (
reinterpret_cast<size_t>(p) %
sizeof(std::complex<double>) == 0) ? p :
632 reinterpret_cast<std::complex<double>*
>(
reinterpret_cast<double*
>(p) + 1);
643template<
typename scalar_type>
645 return std::vector<scalar_type>(size);
669template<
typename backend_tag,
typename index>
672 return (box.
empty()) ?
673 std::unique_ptr<typename one_dim_backend<backend_tag>::executor>() :
680template<
typename backend_tag,
typename index>
682 box3d<index>
const box,
int dir1,
int dir2){
683 return (box.empty()) ?
684 std::unique_ptr<typename one_dim_backend<backend_tag>::executor>() :
685 std::unique_ptr<typename one_dim_backend<backend_tag>::executor>(new typename one_dim_backend<backend_tag>::executor(stream, box, dir1, dir2));
691template<
typename backend_tag,
typename index>
693 box3d<index>
const box){
694 return (box.empty()) ?
695 std::unique_ptr<typename one_dim_backend<backend_tag>::executor>() :
696 std::unique_ptr<typename one_dim_backend<backend_tag>::executor>(new typename one_dim_backend<backend_tag>::executor(stream, box));
702template<
typename backend_tag,
typename index>
704 box3d<index>
const box,
int dimension){
705 return (box.empty()) ?
706 std::unique_ptr<typename one_dim_backend<backend_tag>::executor_r2c>() :
707 std::unique_ptr<typename one_dim_backend<backend_tag>::executor_r2c>(new typename one_dim_backend<backend_tag>::executor_r2c(stream, box, dimension));
714template<
typename backend_tag>
718 return not (std::is_same<backend_tag, backend::stock>::value
719 or std::is_same<backend_tag, backend::stock_cos>::value
720 or std::is_same<backend_tag, backend::mkl_cos>::value
721 or std::is_same<backend_tag, backend::cufft_cos>::value
722 or std::is_same<backend_tag, backend::rocfft_cos>::value
723 or std::is_same<backend_tag, backend::onemkl_cos>::value
724 or std::is_same<backend_tag, backend::stock_sin>::value
725 or std::is_same<backend_tag, backend::mkl_sin>::value
726 or std::is_same<backend_tag, backend::cufft_sin>::value
727 or std::is_same<backend_tag, backend::rocfft_sin>::value
728 or std::is_same<backend_tag, backend::onemkl_sin>::value
729 or std::is_same<backend_tag, backend::stock_cos1>::value
730 or std::is_same<backend_tag, backend::cufft_cos1>::value
731 or std::is_same<backend_tag, backend::rocfft_cos1>::value
738template<
typename backend_tag>
740 return not (std::is_same<backend_tag, backend::stock>::value
741 or std::is_same<backend_tag, backend::stock_cos>::value
742 or std::is_same<backend_tag, backend::mkl_cos>::value
743 or std::is_same<backend_tag, backend::cufft_cos>::value
744 or std::is_same<backend_tag, backend::rocfft_cos>::value
745 or std::is_same<backend_tag, backend::onemkl_cos>::value
746 or std::is_same<backend_tag, backend::stock_sin>::value
747 or std::is_same<backend_tag, backend::mkl_sin>::value
748 or std::is_same<backend_tag, backend::cufft_sin>::value
749 or std::is_same<backend_tag, backend::rocfft_sin>::value
750 or std::is_same<backend_tag, backend::onemkl_sin>::value
751 or std::is_same<backend_tag, backend::stock_cos1>::value
752 or std::is_same<backend_tag, backend::cufft_cos1>::value
753 or std::is_same<backend_tag, backend::rocfft_cos1>::value
Base class for all backend executors.
Definition heffte_common.h:561
virtual void backward(std::complex< double >[], double[], std::complex< double > *) const
Backward FFT real-to-complex, double precision.
Definition heffte_common.h:588
virtual int complex_size() const
Return the size of the complex-box (r2c executors).
Definition heffte_common.h:594
virtual void backward(float[], float *) const
Backward r2r, single precision.
Definition heffte_common.h:570
virtual size_t workspace_size() const
Return the workspace of the size.
Definition heffte_common.h:592
virtual void forward(std::complex< double >[], std::complex< double > *) const
Forward FFT, double precision.
Definition heffte_common.h:576
virtual void forward(float const[], std::complex< float >[], std::complex< float > *) const
Forward FFT real-to-complex, single precision.
Definition heffte_common.h:582
virtual ~executor_base()=default
Virtual destructor.
virtual void forward(float[], float *) const
Forward r2r, single precision.
Definition heffte_common.h:566
virtual void forward(double[], double *) const
Forward r2r, double precision.
Definition heffte_common.h:568
virtual void backward(std::complex< double >[], std::complex< double > *) const
Backward FFT, double precision.
Definition heffte_common.h:580
virtual void forward(std::complex< float >[], std::complex< float > *) const
Forward FFT, single precision.
Definition heffte_common.h:574
virtual void backward(std::complex< float >[], float[], std::complex< float > *) const
Backward FFT real-to-complex, single precision.
Definition heffte_common.h:586
virtual void backward(std::complex< float >[], std::complex< float > *) const
Backward FFT, single precision.
Definition heffte_common.h:578
virtual void backward(double[], double *) const
Backward r2r, double precision.
Definition heffte_common.h:572
virtual int box_size() const
Return the size of the box.
Definition heffte_common.h:590
virtual void forward(double const[], std::complex< double >[], std::complex< double > *) const
Forward FFT real-to-complex, double precision.
Definition heffte_common.h:584
std::string name< tag::gpu >()
Indicates the name of the location tag.
Definition heffte_common.h:396
std::string name()
Returns the human readable name of the backend.
Definition heffte_common.h:265
std::vector< scalar_type > make_buffer_container(void *, size_t size)
Factory method to create new buffer container for the CPU backends.
Definition heffte_common.h:644
std::string name< tag::cpu >()
Indicates the name of the location tag.
Definition heffte_common.h:391
constexpr bool has_executor3d()
Defines whether the executor has a 3D version (single rank).
Definition heffte_common.h:739
constexpr bool has_executor2d()
Defines whether the executor has a 2D version (slabs).
Definition heffte_common.h:715
direction
Indicates the direction of the FFT (internal use only).
Definition heffte_common.h:652
@ backward
Inverse DFT transform.
@ forward
Forward DFT transform.
std::string name< cufft_cos >()
Returns the human readable name of the cuFFT backend.
Definition heffte_common.h:338
std::string name< cufft_sin >()
Returns the human readable name of the cuFFT backend.
Definition heffte_common.h:343
std::string name< cufft_cos1 >()
Returns the human readable name of the cuFFT backend.
Definition heffte_common.h:348
std::string name< cufft >()
Returns the human readable name of the cuFFT backend.
Definition heffte_common.h:333
std::string name< fftw_cos >()
Returns the human readable name of the FFTW backend.
Definition heffte_common.h:276
std::string name< fftw >()
Returns the human readable name of the FFTW backend.
Definition heffte_common.h:271
std::string name< fftw_sin >()
Returns the human readable name of the FFTW backend.
Definition heffte_common.h:281
std::string name< fftw_sin1 >()
Returns the human readable name of the FFTW backend.
Definition heffte_common.h:291
std::string name< fftw_cos1 >()
Returns the human readable name of the FFTW backend.
Definition heffte_common.h:286
std::string name< mkl_sin >()
Returns the human readable name of the MKL backend.
Definition heffte_common.h:328
std::string name< mkl >()
Returns the human readable name of the MKL backend.
Definition heffte_common.h:318
std::string name< mkl_cos >()
Returns the human readable name of the MKL backend.
Definition heffte_common.h:323
std::string name< onemkl >()
Returns the human readable name of the oneMKL backend.
Definition heffte_common.h:375
std::string name< onemkl_sin >()
Returns the human readable name of the oneMKL backend.
Definition heffte_common.h:385
std::string name< onemkl_cos >()
Returns the human readable name of the oneMKL backend.
Definition heffte_common.h:380
std::string name< rocfft_cos >()
Returns the human readable name of the rocFFT backend.
Definition heffte_common.h:359
std::string name< rocfft_cos1 >()
Returns the human readable name of the rocFFT backend.
Definition heffte_common.h:369
std::string name< rocfft >()
Returns the human readable name of the rocFFT backend.
Definition heffte_common.h:354
std::string name< rocfft_sin >()
Returns the human readable name of the rocFFT backend.
Definition heffte_common.h:364
std::string name< stock_cos1 >()
Returns the human readable name of the stock backend.
Definition heffte_common.h:312
std::string name< stock_sin >()
Returns the human readable name of the stock backend.
Definition heffte_common.h:307
std::string name< stock >()
Returns the human readable name of the stock backend.
Definition heffte_common.h:297
std::string name< stock_cos >()
Returns the human readable name of the stock backend.
Definition heffte_common.h:302
Namespace containing all HeFFTe methods and classes.
Definition heffte_backend_cuda.h:38
cuFFT requires that the input and output in R2C transforms are aligned to the complex type.
Definition heffte_common.h:602
static std::complex< float > * pntr(std::complex< float > *p)
Align for complex-float.
Definition heffte_common.h:620
static float * pntr(float *p)
Align for float.
Definition heffte_common.h:604
static double * pntr(double *p)
Align for double.
Definition heffte_common.h:612
static std::complex< double > * pntr(std::complex< double > *p)
Align for complex-double.
Definition heffte_common.h:629
Defines the container for the temporary buffers.
Definition heffte_common.h:237
tag::cpu location
Tags the raw-array location tag::cpu or tag::gpu, used by the packers.
Definition heffte_common.h:239
std::vector< T > container
Defines the container template to use for the temporary buffers in heffte::fft3d.
Definition heffte_common.h:241
Set to true/false type depending whether the types are compatible with the backend transform.
Definition heffte_common.h:445
Type-tag for the Cosine Transform type 1 using the cuFFT backend.
Definition heffte_common.h:178
Type-tag for the Cosine Transform using the cuFFT backend.
Definition heffte_common.h:168
Type-tag for the Sine Transform using the cuFFT backend.
Definition heffte_common.h:173
Type-tag for the cuFFT backend.
Definition heffte_common.h:162
static void copy_n(source_type const source[], size_t num_entries, destination_type destination[])
Wrapper around std::copy_n().
Definition heffte_common.h:75
static void copy_device_to_device(void *, source_type const source[], size_t num_entries, destination_type destination[])
Wrapper around std::copy_n().
Definition heffte_common.h:85
static void copy_device_to_host(void *, source_type const source[], size_t num_entries, destination_type destination[])
Wrapper around std::copy_n().
Definition heffte_common.h:80
static void copy_n(void *, source_type const source[], size_t num_entries, destination_type destination[])
Wrapper around std::copy_n().
Definition heffte_common.h:70
void * stream_type
The stream type for the device.
Definition heffte_common.h:67
static void copy_host_to_device(void *, source_type const source[], size_t num_entries, destination_type destination[])
Wrapper around std::copy_n().
Definition heffte_common.h:90
Common data-transfer operations, must be specializes for each location (cpu/gpu).
Definition heffte_common.h:59
Defines inverse mapping from the location tag to a default backend tag.
Definition heffte_common.h:430
Holds the auxiliary variables needed by each backend.
Definition heffte_common.h:408
void synchronize_device() const
Syncs the execution with the queue, no-op in the CPU case.
Definition heffte_common.h:418
void * stream_type
The type for the internal stream, the cpu uses just a void pointer.
Definition heffte_common.h:420
device_instance(void *=nullptr)
Empty constructor.
Definition heffte_common.h:410
void * stream()
Returns the nullptr.
Definition heffte_common.h:414
void * stream() const
Returns the nullptr (const case).
Definition heffte_common.h:416
virtual ~device_instance()=default
Default destructor.
Type-tag for the Cosine Transform type 1 using the FFTW backend.
Definition heffte_common.h:114
Type-tag for the Cosine Transform using the FFTW backend.
Definition heffte_common.h:104
Type-tag for the Sine Transform type 1 using the FFTW backend.
Definition heffte_common.h:119
Type-tag for the Sine Transform using the FFTW backend.
Definition heffte_common.h:109
Type-tag for the FFTW backend.
Definition heffte_common.h:99
Allows to define whether a specific backend interface has been enabled.
Definition heffte_common.h:226
Type-tag for the Cosine Transform using the MKL FFT backend.
Definition heffte_common.h:151
Type-tag for the Sine Transform using the MKL FFT backend.
Definition heffte_common.h:156
Type-tag for the MKL backend.
Definition heffte_common.h:146
Type-tag for the Cosine Transform using the oneMKL backend.
Definition heffte_common.h:210
Type-tag for the Sine Transform using the oneMKL backend.
Definition heffte_common.h:215
Type-tag for the oneMKL backend.
Definition heffte_common.h:205
Type-tag for the Cosine Transform of type 1 using the rocFFT backend.
Definition heffte_common.h:199
Type-tag for the Cosine Transform using the rocFFT backend.
Definition heffte_common.h:189
Type-tag for the Sine Transform using the rocFFT backend.
Definition heffte_common.h:194
Type-tag for the rocFFT backend.
Definition heffte_common.h:184
Type-tag for the Cosine Transform type 1 using the stock FFT backend.
Definition heffte_common.h:140
Type-tag for the Cosine Transform using the stock FFT backend.
Definition heffte_common.h:130
Type-tag for the Sine Transform using the stock FFT backend.
Definition heffte_common.h:135
Type-tag for the stock FFT backend.
Definition heffte_common.h:125
Defines whether the backend accepts the standard FFT real-complex or complex-complex transform.
Definition heffte_common.h:439
Struct that specializes to true type if the location of the backend is on the gpu (false type otherwi...
Definition heffte_common.h:249
A generic container that describes a 3d box of indexes.
Definition heffte_geometry.h:67
bool empty() const
Returns true if the box contains no indexes.
Definition heffte_geometry.h:84
Defines a set of default plan options for a given backend.
Definition heffte_common.h:761
Indicates the structure that will be used by the fft backend.
Definition heffte_common.h:663
Indicates the use of cpu backend and that all input/output data and arrays will be bound to the cpu.
Definition heffte_common.h:38
Indicates the use of gpu backend and that all input/output data and arrays will be bound to the gpu d...
Definition heffte_common.h:45