Enable tests for externally-managed work areas by expanding hipfft_params

clients/bench/bench.cpp

@@ -31,6 +31,9 @@
 #include "../../shared/client_except.h"
 #include "../../shared/gpubuf.h"
 
+// initialize static class member of hipfft_params
+std::vector<gpubuf> hipfft_params::externally_managed_workareas = std::vector<gpubuf>();
+
 int main(int argc, char* argv[])
 {
     // This helps with mixing output of both wide and narrow characters to the screen
@@ -78,6 +81,11 @@ int main(int argc, char* argv[])
                      "Type of transform:\n0) complex forward\n1) complex inverse\n2) real "
                      "forward\n3) real inverse")
         ->default_val(fft_transform_type_complex_forward);
+    non_token
+        ->add_option("--auto_allocation",
+                     params.auto_allocate,
+                     "HipFFT's auto-allocation behavior: \"on\", \"off\", or \"default\"")
+        ->default_val("default");
     non_token
         ->add_option(
             "--precision", params.precision, "Transform precision: single (default), double, half")

clients/tests/accuracy_test_1D.cpp

@@ -81,14 +81,8 @@ static std::vector<size_t> small_1D_sizes()
     static const size_t SMALL_1D_MAX = 8192;
 
     // generate a list of sizes from 2 and up, skipping any sizes that are already covered
-    std::vector<size_t> covered_sizes;
-    std::copy(pow2_range.begin(), pow2_range.end(), std::back_inserter(covered_sizes));
-    std::copy(pow3_range.begin(), pow3_range.end(), std::back_inserter(covered_sizes));
-    std::copy(pow5_range.begin(), pow5_range.end(), std::back_inserter(covered_sizes));
-    std::copy(radX_range.begin(), radX_range.end(), std::back_inserter(covered_sizes));
-    std::copy(mix_range.begin(), mix_range.end(), std::back_inserter(covered_sizes));
-    std::copy(prime_range.begin(), prime_range.end(), std::back_inserter(covered_sizes));
-    std::sort(covered_sizes.begin(), covered_sizes.end());
+    std::vector<size_t> covered_sizes = merge_and_sort_values<size_t>(
+        {pow2_range, pow3_range, pow5_range, radX_range, mix_range, prime_range});
 
     std::vector<size_t> output;
     for(size_t i = 2; i < SMALL_1D_MAX; ++i)
@@ -319,12 +313,14 @@ INSTANTIATE_TEST_SUITE_P(DISABLED_offset_mix_1D,
 
 // small 1D sizes just need to make sure our factorization isn't
 // completely broken, so we just check simple C2C outplace interleaved
+const static std::vector<size_t> small_1D_lengths = small_1D_sizes();
+
 INSTANTIATE_TEST_SUITE_P(
     small_1D,
     accuracy_test,
     ::testing::ValuesIn(param_generator_base(test_prob,
                                              {fft_transform_type_complex_forward},
-                                             generate_lengths({small_1D_sizes()}),
+                                             generate_lengths({small_1D_lengths}),
                                              {fft_precision_single},
                                              {1},
                                              generate_types,
@@ -530,3 +526,24 @@ INSTANTIATE_TEST_SUITE_P(
                                                               ooffset_range_zero,
                                                               place_range)),
     accuracy_test::TestName);
+
+const static std::vector<size_t> lengths_for_disabled_autoalloc = merge_and_sort_values<size_t>(
+    {pow2_range, pow3_range, pow5_range, radX_range, mix_range, small_1D_lengths, prime_range},
+    128);
+
+INSTANTIATE_TEST_SUITE_P(
+    various_1D,
+    accuracy_test,
+    ::testing::ValuesIn(param_generator(test_prob,
+                                        generate_lengths({lengths_for_disabled_autoalloc}),
+                                        precision_range_sp_dp,
+                                        batch_range_1D,
+                                        stride_range,
+                                        stride_range,
+                                        ioffset_range_zero,
+                                        ooffset_range_zero,
+                                        place_range,
+                                        false,
+                                        false,
+                                        fft_auto_allocation_off)),
+    accuracy_test::TestName);

clients/tests/accuracy_test_2D.cpp

@@ -278,3 +278,24 @@ INSTANTIATE_TEST_SUITE_P(len1_swap_2D,
                              false,
                              false)),
                          accuracy_test::TestName);
+
+const static std::vector<size_t> lengths_for_disabled_autoalloc
+    = merge_and_sort_values<size_t>({pow2_range, pow3_range, prime_range, mix_range}, 12);
+
+INSTANTIATE_TEST_SUITE_P(
+    various_2D,
+    accuracy_test,
+    ::testing::ValuesIn(param_generator(test_prob,
+                                        generate_lengths({lengths_for_disabled_autoalloc,
+                                                          lengths_for_disabled_autoalloc}),
+                                        precision_range_sp_dp,
+                                        batch_range,
+                                        stride_range,
+                                        stride_range,
+                                        ioffset_range_zero,
+                                        ooffset_range_zero,
+                                        place_range,
+                                        false,
+                                        false,
+                                        fft_auto_allocation_off)),
+    accuracy_test::TestName);

clients/tests/accuracy_test_3D.cpp

@@ -284,3 +284,25 @@ INSTANTIATE_TEST_SUITE_P(
         false,
         false)),
     accuracy_test::TestName);
+
+const static std::vector<size_t> lengths_for_disabled_autoalloc = merge_and_sort_values<size_t>(
+    {pow2_range, pow3_range, pow5_range, prime_range, sbrc_range}, 5);
+
+INSTANTIATE_TEST_SUITE_P(
+    various_3D,
+    accuracy_test,
+    ::testing::ValuesIn(param_generator(test_prob,
+                                        generate_lengths({lengths_for_disabled_autoalloc,
+                                                          lengths_for_disabled_autoalloc,
+                                                          lengths_for_disabled_autoalloc}),
+                                        precision_range_sp_dp,
+                                        batch_range,
+                                        stride_range,
+                                        stride_range,
+                                        ioffset_range_zero,
+                                        ooffset_range_zero,
+                                        place_range,
+                                        false,
+                                        false,
+                                        fft_auto_allocation_off)),
+    accuracy_test::TestName);

clients/tests/gtest_main.cpp

@@ -43,6 +43,9 @@
 #include "hipfft_accuracy_test.h"
 #include "hipfft_test_params.h"
 
+// initialize static class member of hipfft_params
+std::vector<gpubuf> hipfft_params::externally_managed_workareas = std::vector<gpubuf>();
+
 // Control output verbosity:
 int verbose;
 
@@ -340,6 +343,11 @@ int main(int argc, char* argv[])
     non_token->add_flag("--callback", "Inject load/store callbacks")->each([&](const std::string&) {
         manual_params.run_callbacks = true;
     });
+    non_token
+        ->add_option("--auto_allocation",
+                     manual_params.auto_allocate,
+                     "HipFFT's auto-allocation behavior: \"on\", \"off\", or \"default\"")
+        ->default_val("default");
     non_token
         ->add_flag("--double", "Double precision transform (deprecated: use --precision double)")
         ->each([&](const std::string&) { manual_params.precision = fft_precision_double; });

clients/tests/hipfft_accuracy_test.cpp

@@ -95,6 +95,10 @@ TEST_P(accuracy_test, vs_fftw)
         {
             GTEST_SKIP() << e.msg;
         }
+        catch(const fft_params::unimplemented_exception& e)
+        {
+            GTEST_SKIP() << "Unimplemented exception: " << e.what();
+        }
         catch(ROCFFT_FAIL& e)
         {
             GTEST_FAIL() << e.msg;

clients/tests/multi_device_test.cpp

@@ -53,7 +53,9 @@ enum SplitType
     PENCIL_3D,
 };
 
-std::vector<fft_params> param_generator_multi_gpu(const std::optional<SplitType> type)
+std::vector<fft_params> param_generator_multi_gpu(const std::optional<SplitType> type,
+                                                  fft_auto_allocation            auto_alloc_setting
+                                                  = fft_auto_allocation_default)
 {
     int localDeviceCount = 0;
     (void)hipGetDeviceCount(&localDeviceCount);
@@ -80,7 +82,9 @@ std::vector<fft_params> param_generator_multi_gpu(const std::optional<SplitType>
                                                   ioffset_range_zero,
                                                   ooffset_range_zero,
                                                   place_range,
-                                                  false);
+                                                  false,
+                                                  false,
+                                                  auto_alloc_setting);
 
     auto params_real = param_generator_real(test_prob,
                                             multi_gpu_sizes,
@@ -91,7 +95,9 @@ std::vector<fft_params> param_generator_multi_gpu(const std::optional<SplitType>
                                             ioffset_range_zero,
                                             ooffset_range_zero,
                                             {fft_placement_notinplace},
-                                            false);
+                                            false,
+                                            false,
+                                            auto_alloc_setting);
 
     std::vector<fft_params> all_params;
 
@@ -183,7 +189,6 @@ std::vector<fft_params> param_generator_multi_gpu(const std::optional<SplitType>
                 // in-place transforms require identical input/output layouts
                 if(p.placement == fft_placement_inplace && input_grid != output_grid)
                     continue;
-
                 all_params.push_back(std::move(p_dist));
             }
         }
@@ -229,3 +234,11 @@ INSTANTIATE_TEST_SUITE_P(multi_gpu,
                          accuracy_test,
                          ::testing::ValuesIn(param_generator_multi_gpu({})),
                          accuracy_test::TestName);
+
+// Note: disabled for now due to implementation issues and
+// unimplemented features in hipFFT (to fix first)
+INSTANTIATE_TEST_SUITE_P(DISABLED_various_multi_gpu,
+                         accuracy_test,
+                         ::testing::ValuesIn(param_generator_multi_gpu({},
+                                                                       fft_auto_allocation_off)),
+                         accuracy_test::TestName);

library/src/amd_detail/hipfft.cpp

@@ -35,15 +35,6 @@
 #include "../../../shared/ptrdiff.h"
 #include "../../../shared/rocfft_hip.h"
 
-#define ROC_FFT_CHECK_ALLOC_FAILED(ret)   \
-    {                                     \
-        auto code = ret;                  \
-        if(code != rocfft_status_success) \
-        {                                 \
-            return HIPFFT_ALLOC_FAILED;   \
-        }                                 \
-    }
-
 #define ROC_FFT_CHECK_INVALID_VALUE(ret)  \
     {                                     \
         auto code = ret;                  \
@@ -410,31 +401,6 @@ catch(...)
     return handle_exception();
 }
 
-hipfftResult hipfftPlanMany64(hipfftHandle*  plan,
-                              int            rank,
-                              long long int* n,
-                              long long int* inembed,
-                              long long int  istride,
-                              long long int  idist,
-                              long long int* onembed,
-                              long long int  ostride,
-                              long long int  odist,
-                              hipfftType     type,
-                              long long int  batch)
-try
-{
-    hipfftHandle handle = nullptr;
-    HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&handle));
-    *plan = handle;
-
-    return hipfftMakePlanMany64(
-        *plan, rank, n, inembed, istride, idist, onembed, ostride, odist, type, batch, nullptr);
-}
-catch(...)
-{
-    return handle_exception();
-}
-
 hipfftResult hipfftMakePlan_internal(hipfftHandle               plan,
                                      size_t                     dim,
                                      size_t*                    lengths,
@@ -1262,6 +1228,7 @@ try
 
     hipfftHandle p;
     HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&p));
+    p->autoAllocate = false;
     HIP_FFT_CHECK_AND_RETURN(hipfftMakePlan1d(p, nx, type, batch, workSize));
     HIP_FFT_CHECK_AND_RETURN(hipfftDestroy(p));
 
@@ -1282,6 +1249,7 @@ try
 
     hipfftHandle p;
     HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&p));
+    p->autoAllocate = false;
     HIP_FFT_CHECK_AND_RETURN(hipfftMakePlan2d(p, nx, ny, type, workSize));
     HIP_FFT_CHECK_AND_RETURN(hipfftDestroy(p));
 
@@ -1303,6 +1271,7 @@ try
 
     hipfftHandle p;
     HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&p));
+    p->autoAllocate = false;
     HIP_FFT_CHECK_AND_RETURN(hipfftMakePlan3d(p, nx, ny, nz, type, workSize));
     HIP_FFT_CHECK_AND_RETURN(hipfftDestroy(p));
 
@@ -1327,10 +1296,13 @@ hipfftResult hipfftGetSizeMany(hipfftHandle plan,
                                size_t*      workSize)
 try
 {
-    hipfftHandle p;
-    HIP_FFT_CHECK_AND_RETURN(
-        hipfftPlanMany(&p, rank, n, inembed, istride, idist, onembed, ostride, odist, type, batch));
-    *workSize = p->workBufferSize;
+    if(workSize == nullptr)
+        return HIPFFT_INVALID_VALUE;
+    hipfftHandle p = nullptr;
+    HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&p));
+    p->autoAllocate = false;
+    HIP_FFT_CHECK_AND_RETURN(hipfftMakePlanMany(
+        p, rank, n, inembed, istride, idist, onembed, ostride, odist, type, batch, workSize));
     HIP_FFT_CHECK_AND_RETURN(hipfftDestroy(p));
 
     return HIPFFT_SUCCESS;
@@ -1354,10 +1326,13 @@ hipfftResult hipfftGetSizeMany64(hipfftHandle   plan,
                                  size_t*        workSize)
 try
 {
+    if(workSize == nullptr)
+        return HIPFFT_INVALID_VALUE;
     hipfftHandle p = nullptr;
-    HIP_FFT_CHECK_AND_RETURN(hipfftPlanMany64(
-        &p, rank, n, inembed, istride, idist, onembed, ostride, odist, type, batch));
-    *workSize = p->workBufferSize;
+    HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&p));
+    p->autoAllocate = false;
+    HIP_FFT_CHECK_AND_RETURN(hipfftMakePlanMany64(
+        p, rank, n, inembed, istride, idist, onembed, ostride, odist, type, batch, workSize));
     HIP_FFT_CHECK_AND_RETURN(hipfftDestroy(p));
 
     return HIPFFT_SUCCESS;
@@ -1831,6 +1806,7 @@ try
 
     hipfftHandle p;
     HIP_FFT_CHECK_AND_RETURN(hipfftCreate(&p));
+    p->autoAllocate = false;
 
     HIP_FFT_CHECK_AND_RETURN(hipfftMakePlanMany_internal(
         p, rank, n, inembed, istride, idist, onembed, ostride, odist, iotype, batch, workSize));

shared/accuracy_test.h

@@ -567,7 +567,7 @@ inline void run_round_trip_inverse(Tparams&              params,
     catch(fft_params::work_buffer_alloc_failure& e)
     {
         std::stringstream ss;
-        ss << "Failed to allocate work buffer (size: " << params.workbuffersize << ")";
+        ss << "Failed to allocate work buffer (size: " << e.attempted_size << ")";
         ++n_hip_failures;
         if(skip_runtime_fails)
         {
@@ -768,7 +768,7 @@ inline void fft_vs_reference_impl(Tparams& params, bool round_trip)
     {
         ++n_hip_failures;
         std::stringstream ss;
-        ss << "Work buffer allocation failed with size: " << params.workbuffersize;
+        ss << "Work buffer allocation failed with size: " << e.attempted_size;
         if(skip_runtime_fails)
         {
             throw ROCFFT_SKIP{ss.str()};