From 00a23d0eef1668b7cac741f9e2ca7c7c04c3d640 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 09:06:08 +0000 Subject: [PATCH 01/15] WIP: transforms' sizing carries row length; host forward pack behind a probe switch Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/core.hpp | 9 + native/src/dispersive.cpp | 59 +++--- native/src/general.cpp | 268 ++++++++++++++++++------- native/src/general_modal.cpp | 12 +- 4 files changed, 244 insertions(+), 104 deletions(-) diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index 67ecf730..02f467a6 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -345,6 +345,15 @@ class CpuExecutionPool { CpuSizing sizing_; }; +/** A transform batch's row length, as its CPU width and sizing record count it + (`CpuExecutionPool::get`'s `fine_samples` for the FFT and Raman classes): extent 0 + counts the rows and the remaining extents make one row, so a transverse plane is + one row; a rank-1 view is one row. */ +template [[nodiscard]] std::size_t row_length(const View& rows) noexcept { + const std::size_t count = View::rank() == 1 ? 1 : rows.extent(0); + return count == 0 ? 0 : rows.size() / count; +} + /** Whether the host FFTW's threaded loops take a scoped team (`CpuTeamScope`): the OpenMP FFTW with its loop callback (FFTW 3.3.9 and later). Without it a compact problem's transforms stay serial. The first call installs the callback, whose diff --git a/native/src/dispersive.cpp b/native/src/dispersive.cpp index 3e69fe88..1588b404 100644 --- a/native/src/dispersive.cpp +++ b/native/src/dispersive.cpp @@ -988,38 +988,38 @@ DispersiveResponse::DispersiveResponse(const Problem& problem, CpuExecutionPool& DispersiveResponse::~DispersiveResponse() { close(); } void DispersiveResponse::plan(CpuExecutionPool& transforms) { - const auto space_for = [&](const std::size_t samples) -> const ExecutionSpace& { - return transforms.get(CpuOperation::fft, samples); + const auto space_for = [&](const auto& rows) -> const ExecutionSpace& { + return transforms.get(CpuOperation::fft, rows.size(), row_length(rows)); }; constexpr int axis = -1; if (!in_place_) { if (form_ == DispersiveForm::real) inverse_real_ = std::make_unique( - space_for(filtered_real_.size()), filtered_spectrum_, filtered_real_, + space_for(filtered_real_), filtered_spectrum_, filtered_real_, KokkosFFT::Direction::backward, axis); else inverse_complex_ = std::make_unique( - space_for(filtered_complex_.size()), filtered_spectrum_, filtered_complex_, + space_for(filtered_complex_), filtered_spectrum_, filtered_complex_, KokkosFFT::Direction::backward, axis); } if (borrowed_) return; if (form_ == DispersiveForm::real) { forward_real_ = std::make_unique( - space_for(products_real_.size()), products_real_, spectra_, + space_for(products_real_), products_real_, spectra_, KokkosFFT::Direction::forward, axis); if (embedded_) coarse_real_plan_ = std::make_unique( - space_for(coarse_real_.size()), coarse_real_, spectra_, + space_for(coarse_real_), coarse_real_, spectra_, KokkosFFT::Direction::forward, axis); } else { - forward_complex_ = std::make_unique( - space_for(products_complex_.size()), products_complex_, spectra_, - KokkosFFT::Direction::forward, axis); + forward_complex_ = + std::make_unique(space_for(products_complex_), products_complex_, + spectra_, KokkosFFT::Direction::forward, axis); if (embedded_) - coarse_complex_plan_ = std::make_unique( - space_for(coarse_complex_.size()), coarse_complex_, spectra_, - KokkosFFT::Direction::forward, axis); + coarse_complex_plan_ = + std::make_unique(space_for(coarse_complex_), coarse_complex_, + spectra_, KokkosFFT::Direction::forward, axis); } } @@ -1395,32 +1395,33 @@ SeparableDispersive::SeparableDispersive(const Problem& problem, CpuExecutionPoo SeparableDispersive::~SeparableDispersive() { close(); } void SeparableDispersive::plan(CpuExecutionPool& transforms) { - const auto space_for = [&](const std::size_t samples) -> const ExecutionSpace& { - return transforms.get(CpuOperation::fft, samples); + const auto space_for = [&](const auto& rows) -> const ExecutionSpace& { + return transforms.get(CpuOperation::fft, rows.size(), row_length(rows)); }; constexpr int axis = -1; if (!in_place_) { if (form_ == DispersiveForm::real) inverse_real_ = std::make_unique( - space_for(filtered_real_.size()), filtered_spectrum_, filtered_real_, + space_for(filtered_real_), filtered_spectrum_, filtered_real_, KokkosFFT::Direction::backward, axis); else inverse_complex_ = std::make_unique( - space_for(filtered_complex_.size()), filtered_spectrum_, filtered_complex_, + space_for(filtered_complex_), filtered_spectrum_, filtered_complex_, KokkosFFT::Direction::backward, axis); } if (borrowed_) return; if (form_ == DispersiveForm::real) forward_real_ = std::make_unique( - space_for(products_real_.size()), products_real_, spectra_, + space_for(products_real_), products_real_, spectra_, KokkosFFT::Direction::forward, axis); else - forward_complex_ = std::make_unique( - space_for(products_complex_.size()), products_complex_, spectra_, - KokkosFFT::Direction::forward, axis); + forward_complex_ = + std::make_unique(space_for(products_complex_), products_complex_, + spectra_, KokkosFFT::Direction::forward, axis); if (tail_rows_ != 0) { - const auto& tail_space = space_for(tail_rows_ * n_to_); + const auto& tail_space = + transforms.get(CpuOperation::fft, tail_rows_ * n_to_, n_to_); if (!in_place_) { if (form_ == DispersiveForm::real) tail_inverse_real_.create(tail_space, filtered_spectrum_, filtered_real_, rows_, @@ -1743,18 +1744,20 @@ GeneralDispersive::~GeneralDispersive() { close(); } void GeneralDispersive::plan(CpuExecutionPool& transforms) { constexpr int axis = -1; inverse_ = std::make_unique( - transforms.get(CpuOperation::fft, filtered_.size()), filtered_spectrum_, - filtered_, KokkosFFT::Direction::backward, axis); + transforms.get(CpuOperation::fft, filtered_.size(), row_length(filtered_)), + filtered_spectrum_, filtered_, KokkosFFT::Direction::backward, axis); if (real_) forward_real_ = std::make_unique( - transforms.get(CpuOperation::fft, products_real_.size()), products_real_, - spectra_, KokkosFFT::Direction::forward, axis); + transforms.get(CpuOperation::fft, products_real_.size(), + row_length(products_real_)), + products_real_, spectra_, KokkosFFT::Direction::forward, axis); else forward_complex_ = std::make_unique( - transforms.get(CpuOperation::fft, products_complex_.size()), products_complex_, - spectra_, KokkosFFT::Direction::forward, axis); + transforms.get(CpuOperation::fft, products_complex_.size(), + row_length(products_complex_)), + products_complex_, spectra_, KokkosFFT::Direction::forward, axis); if (tail_rows_ != 0) { - const auto& space = transforms.get(CpuOperation::fft, tail_rows_ * n_to_); + const auto& space = transforms.get(CpuOperation::fft, tail_rows_ * n_to_, n_to_); tail_inverse_.create(space, filtered_spectrum_, filtered_, rows_, tail_rows_, KokkosFFT::Direction::backward); if (real_) diff --git a/native/src/general.cpp b/native/src/general.cpp index 619035b2..0f7cdc72 100644 --- a/native/src/general.cpp +++ b/native/src/general.cpp @@ -2607,17 +2607,27 @@ class GeneralWorkspace { // microseconds on 16 of its 84 multiprocessors, whatever the rows. Batched, the // stage's rows share them. A device never defers the coarse quadrature // (`deferred_quadrature_eligible`), so the coarse rows are projected at every stage. - // A host keeps its separate plans. + // A host packs them too unless SOLPROP_NATIVE_FORWARD_PACK=0 (a probe switch, + // solprop issue #584): one threaded transform of every row in place of one a + // transform, each over only the modes' rows. A host keeps the coarse rows after the + // fine ones so that a deferred quadrature's stages transform the fine block alone. bool packs_forward() const noexcept { -#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + if (!host_forward_pack()) + return false; +#endif return problem_.family == FieldFamily::field_real && !problem_.separable_geometry() && !problem_.general_modal && !problem_.dispersive && !problem_.envelope_plasma && !held_radial() && (problem_.in_stage_absorber || held_plasma() || !problem_.projection_coarse_matrix.empty()); -#else - return false; -#endif + } + static bool host_forward_pack() noexcept { + static const bool enabled = [] { + const char* text = std::getenv("SOLPROP_NATIVE_FORWARD_PACK"); + return text == nullptr || std::string(text) != "0"; + }(); + return enabled; } void transform_forward_pack(); void held_response(std::size_t stage, std::size_t first_node, std::size_t node_count, @@ -2683,7 +2693,14 @@ class GeneralWorkspace { bool use_chunked_blas_for_synthesis() const noexcept; bool folds_single_node() const noexcept; bool use_chunked_blas_for_projection() const noexcept; - [[nodiscard]] const ExecutionSpace& fft_space_for(std::size_t samples); + [[nodiscard]] const ExecutionSpace& fft_space_for(std::size_t samples, + std::size_t length); + /** A batch's FFT instance, sized by its rows: extent 0 counts the rows, the + remaining extents make one row (a transverse plane counts as one row). */ + template + [[nodiscard]] const ExecutionSpace& fft_space_for_rows(const View& rows) { + return fft_space_for(rows.size(), row_length(rows)); + } Problem problem_; // A compact problem's pools (`compact_problem`) and the team its FFTW loops take. @@ -2936,6 +2953,14 @@ class GeneralWorkspace { ComplexView2 forward_pack_spectrum_, forward_pack_tail_spectrum_; std::unique_ptr forward_pack_plan_, forward_pack_tail_plan_; std::size_t forward_pack_absorber_rows_ = 0; + // A host's deferred quadrature (`deferred_quadrature_eligible`): the pack's fine + // block (the absorber's, the projection's and the held currents' fine rows) with and + // without the absorber's, for the stages, and its coarse block for the finish. + RealView2 forward_pack_fine_, forward_pack_fine_tail_, forward_pack_coarse_; + ComplexView2 forward_pack_fine_spectrum_, forward_pack_fine_tail_spectrum_, + forward_pack_coarse_spectrum_; + std::unique_ptr forward_pack_fine_plan_, + forward_pack_fine_tail_plan_, forward_pack_coarse_plan_; RealView2 held_current_, held_projection_; ComplexView2 held_nodal_spectrum_, held_projected_spectrum_; RealView held_derivative_, held_inverse_; @@ -3974,6 +3999,17 @@ void GeneralWorkspace::allocate() { const auto pack_slice = [&](const std::size_t first, const std::size_t count) { return std::pair(first, first + count); }; +#if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) + constexpr bool host_pack_layout = false; +#else + constexpr bool host_pack_layout = true; +#endif + // A host's pack: [absorber | fine | held fine x2 | coarse | held coarse x2]; a + // device's: [absorber | fine | coarse | (held fine, held coarse) x2]. + const std::size_t held_fine_rows = held_plasma() ? 2 * temporal_columns : 0; + const std::size_t fine_end = absorber_rows + temporal_columns + held_fine_rows; + const std::size_t coarse_first = + host_pack_layout ? fine_end : absorber_rows + temporal_columns; forward_pack_absorber_rows_ = absorber_rows; if (pack) { forward_pack_ = @@ -3988,6 +4024,27 @@ void GeneralWorkspace::allocate() { forward_pack_spectrum_, pack_slice(absorber_rows, pack_rows - absorber_rows), Kokkos::ALL); } + if (host_pack_layout && coarse_rows != 0 && deferred_quadrature_eligible()) { + forward_pack_fine_ = + Kokkos::subview(forward_pack_, pack_slice(0, fine_end), Kokkos::ALL); + forward_pack_fine_spectrum_ = + Kokkos::subview(forward_pack_spectrum_, pack_slice(0, fine_end), Kokkos::ALL); + if (absorber_rows != 0 && fine_end - absorber_rows > temporal_columns) { + forward_pack_fine_tail_ = Kokkos::subview( + forward_pack_, pack_slice(absorber_rows, fine_end - absorber_rows), + Kokkos::ALL); + forward_pack_fine_tail_spectrum_ = Kokkos::subview( + forward_pack_spectrum_, pack_slice(absorber_rows, fine_end - absorber_rows), + Kokkos::ALL); + } + if (held_plasma()) { + forward_pack_coarse_ = Kokkos::subview( + forward_pack_, pack_slice(fine_end, pack_rows - fine_end), Kokkos::ALL); + forward_pack_coarse_spectrum_ = + Kokkos::subview(forward_pack_spectrum_, + pack_slice(fine_end, pack_rows - fine_end), Kokkos::ALL); + } + } } projected_spectrum_ = pack ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, @@ -3998,10 +4055,9 @@ void GeneralWorkspace::allocate() { if (!problem_.projection_coarse_matrix.empty()) projected_coarse_spectrum_ = coarse_rows != 0 - ? ComplexView2(Kokkos::subview( - forward_pack_spectrum_, - pack_slice(absorber_rows + temporal_columns, coarse_rows), - Kokkos::ALL)) + ? ComplexView2(Kokkos::subview(forward_pack_spectrum_, + pack_slice(coarse_first, coarse_rows), + Kokkos::ALL)) : ComplexView2("solprop_native.fft.projected_coarse_spectrum", columns, projected_spectrum_size()); absorber_spectrum_ = @@ -4060,9 +4116,7 @@ void GeneralWorkspace::allocate() { projected_coarse_real_ = coarse_rows != 0 ? RealView2(Kokkos::subview( - forward_pack_, - pack_slice(absorber_rows + temporal_columns, coarse_rows), - Kokkos::ALL)) + forward_pack_, pack_slice(coarse_first, coarse_rows), Kokkos::ALL)) : RealView2("solprop_native.response.projected_coarse_real", temporal_columns, problem_.n_to); absorber_real_ = @@ -4091,6 +4145,8 @@ void GeneralWorkspace::allocate() { const bool packed = false; const bool radial_packed = false; #endif + // A host's pack holds the streams' rows itself, each fine block then each coarse. + const bool host_held = host_pack_layout && pack && !radial; if (packed && pack) { // The stage's forward pack ends with the held pack's rows. const auto held_rows = @@ -4114,6 +4170,19 @@ void GeneralWorkspace::allocate() { std::string("solprop_native.held.stream") + std::to_string(index) + "."; if (index < 2) stream.node = RealView2(label + "node", chunk_rows, problem_.n_to); + if (host_held && index < 2) { + const auto fine = pack_slice(absorber_rows + rows + index * rows, rows); + stream.projected = Kokkos::subview(forward_pack_, fine, Kokkos::ALL); + stream.spectrum = Kokkos::subview(forward_pack_spectrum_, fine, Kokkos::ALL); + if (coarse) { + const auto held_coarse = pack_slice(fine_end + rows + index * rows, rows); + stream.coarse = Kokkos::subview(forward_pack_, held_coarse, Kokkos::ALL); + stream.coarse_spectrum = + Kokkos::subview(forward_pack_spectrum_, held_coarse, Kokkos::ALL); + stream.stacked = RealView2(label + "stacked", 2 * rows, problem_.n_to); + } + continue; + } if (packed) { // Each of the two streams holds half of the pack. const std::size_t first = index * (held_pack_.extent(0) / 2); @@ -4513,8 +4582,9 @@ void GeneralWorkspace::upload() { // thresholds in core.hpp. Explicit FFT-pool requests bypass automatic sizing. // FFTW's factorisation can change with the width; comparisons use solprop's SEP 25 // transform/propagation rounding budgets rather than bitwise agreement. -const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples) { - return fft_pool_.get(CpuOperation::fft, samples); +const ExecutionSpace& GeneralWorkspace::fft_space_for(const std::size_t samples, + const std::size_t length) { + return fft_pool_.get(CpuOperation::fft, samples, length); } #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) @@ -4527,7 +4597,7 @@ void GeneralWorkspace::seed_free_wisdom() { const auto n_y = static_cast(transverse_.extent(1)); const auto n_x = static_cast(transverse_.extent(2)); for (const int sign : {FFTW_FORWARD, FFTW_BACKWARD}) - seed_fftw_wisdom(fft_space_for(transverse_.size()), transverse_.data(), + seed_fftw_wisdom(fft_space_for_rows(transverse_), transverse_.data(), transverse_.data(), {n_y, n_x}, planes, n_y * n_x, n_y * n_x, sign); graph::deep_copy(execution_space_, transverse_, Complex{}); @@ -4544,20 +4614,20 @@ void GeneralWorkspace::seed_free_wisdom() { // twice the measured cost). The plan policy (`fftw_seed_effort`) decides the effort. void GeneralWorkspace::seed_modal_wisdom() { if (problem_.family == FieldFamily::field_real) { - seed_row_fft_wisdom(fft_space_for(modal_real_.size()), fine_spectrum_, modal_real_, + seed_row_fft_wisdom(fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, KokkosFFT::Direction::backward); graph::deep_copy(execution_space_, modal_real_, 0.0); } else { - seed_row_fft_wisdom(fft_space_for(modal_complex_.size()), fine_spectrum_, + seed_row_fft_wisdom(fft_space_for_rows(modal_complex_), fine_spectrum_, modal_complex_, KokkosFFT::Direction::backward); graph::deep_copy(execution_space_, modal_complex_, Complex{}); } if (problem_.family == FieldFamily::envelope) { - seed_row_fft_wisdom(fft_space_for(projected_complex_.size()), projected_complex_, + seed_row_fft_wisdom(fft_space_for_rows(projected_complex_), projected_complex_, projected_spectrum_, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, projected_complex_, Complex{}); } else { - seed_row_fft_wisdom(fft_space_for(projected_real_.size()), projected_real_, + seed_row_fft_wisdom(fft_space_for_rows(projected_real_), projected_real_, projected_spectrum_, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, projected_real_, 0.0); } @@ -4569,30 +4639,46 @@ void GeneralWorkspace::seed_modal_wisdom() { // projected pair seeded above, whose wisdom FFTW then finds at once; the radial held // streams' chunk rows do not. void GeneralWorkspace::seed_stream_wisdom() { + // The forward pack's plans (`packs_forward`); seeding overwrites their rows. + for (const auto& [rows, spectrum] : + {std::pair{forward_pack_, forward_pack_spectrum_}, + std::pair{forward_pack_tail_, forward_pack_tail_spectrum_}, + std::pair{forward_pack_fine_, forward_pack_fine_spectrum_}, + std::pair{forward_pack_fine_tail_, forward_pack_fine_tail_spectrum_}, + std::pair{forward_pack_coarse_, forward_pack_coarse_spectrum_}}) { + if (rows.empty()) + continue; + seed_row_fft_wisdom(fft_space_for_rows(rows), rows, spectrum, + KokkosFFT::Direction::forward); + } + if (!forward_pack_.empty()) { + graph::deep_copy(execution_space_, forward_pack_, 0.0); + graph::deep_copy(execution_space_, forward_pack_spectrum_, Complex{}); + } if (held_plasma()) for (auto& stream : held_streams_) { if (stream.projected.empty()) continue; - seed_row_fft_wisdom(fft_space_for(stream.projected.size()), stream.projected, + seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, stream.projected, 0.0); graph::deep_copy(execution_space_, stream.spectrum, Complex{}); if (stream.coarse.empty()) continue; - seed_row_fft_wisdom(fft_space_for(stream.coarse.size()), stream.coarse, + seed_row_fft_wisdom(fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, stream.coarse, 0.0); graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); } if (problem_.envelope_plasma) for (auto& stream : envelope_streams_) { - seed_row_fft_wisdom(fft_space_for(stream.projected.size()), stream.projected, + seed_row_fft_wisdom(fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, stream.projected, Complex{}); graph::deep_copy(execution_space_, stream.spectrum, Complex{}); if (stream.coarse.empty()) continue; - seed_row_fft_wisdom(fft_space_for(stream.coarse.size()), stream.coarse, + seed_row_fft_wisdom(fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, KokkosFFT::Direction::forward); graph::deep_copy(execution_space_, stream.coarse, Complex{}); graph::deep_copy(execution_space_, stream.coarse_spectrum, Complex{}); @@ -4610,16 +4696,18 @@ void GeneralWorkspace::plan() { seed_modal_wisdom(); seed_stream_wisdom(); #endif - fft_execution_space_ = - fft_space_for(std::max(modal_complex_.size(), modal_real_.size())); + fft_execution_space_ = modal_complex_.size() > modal_real_.size() + ? fft_space_for_rows(modal_complex_) + : fft_space_for_rows(modal_real_); if (problem_.raman) { - raman_execution_space_ = - raman_pool_.get(CpuOperation::raman, node_raman_drive_.size()); + raman_execution_space_ = raman_pool_.get( + CpuOperation::raman, node_raman_drive_.size(), row_length(node_raman_drive_)); const auto raman_tail = problem_.n_nodes % problem_.node_chunk; raman_tail_execution_space_ = raman_tail_pool_.get( - CpuOperation::raman, raman_tail * problem_.raman_drives() * 2 * problem_.n_to); - raman_kernel_execution_space_ = - raman_pool_.get(CpuOperation::raman, raman_kernel_time_.size()); + CpuOperation::raman, raman_tail * problem_.raman_drives() * 2 * problem_.n_to, + 2 * problem_.n_to); + raman_kernel_execution_space_ = raman_pool_.get( + CpuOperation::raman, raman_kernel_time_.size(), row_length(raman_kernel_time_)); raman_multiply_space_ = kernel_pool_.get(CpuOperation::stage, node_raman_spectrum_.size()); raman_tail_multiply_space_ = @@ -4632,7 +4720,7 @@ void GeneralWorkspace::plan() { raman_products_ = raman_modal_products(); if (raman_products_ != 0) { raman_product_execution_space_ = raman_tail_pool_.get( - CpuOperation::raman, raman_products_ * 2 * problem_.n_to); + CpuOperation::raman, raman_products_ * 2 * problem_.n_to, 2 * problem_.n_to); raman_product_space_ = kernel_pool_.get(CpuOperation::stage, raman_products_ * 2 * problem_.n_to); } @@ -4647,7 +4735,7 @@ void GeneralWorkspace::plan() { // The y transforms are many short ones; below the threaded-plan threshold they // run on one FFTW thread, as every other small plan (`fft_space_for`). constexpr int y_axis = -1; - const auto& space = fft_space_for(transverse_.size()); + const auto& space = fft_space_for_rows(transverse_); slab_forward_ = std::make_unique(space, transverse_, transverse_, KokkosFFT::Direction::forward, y_axis); slab_inverse_ = std::make_unique(space, transverse_, transverse_, @@ -4659,10 +4747,10 @@ void GeneralWorkspace::plan() { const KokkosFFT::axis_type<2> axes{-2, -1}; #endif transverse_forward_ = std::make_unique( - fft_space_for(transverse_.size()), transverse_, transverse_, + fft_space_for_rows(transverse_), transverse_, transverse_, KokkosFFT::Direction::forward, axes); transverse_inverse_ = std::make_unique( - fft_space_for(transverse_.size()), transverse_, transverse_, + fft_space_for_rows(transverse_), transverse_, transverse_, KokkosFFT::Direction::backward, axes); #if defined(KOKKOS_ENABLE_CUDA) || defined(KOKKOS_ENABLE_HIP) bool identity_crop = problem_.crop_index.size() == problem_.n_omega; @@ -4718,38 +4806,38 @@ void GeneralWorkspace::plan() { if (stream.projected.empty()) continue; stream.forward = std::make_unique( - fft_space_for(stream.projected.size()), stream.projected, stream.spectrum, + fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, KokkosFFT::Direction::forward, axis); if (!stream.coarse.empty()) stream.coarse_forward = std::make_unique( - fft_space_for(stream.coarse.size()), stream.coarse, stream.coarse_spectrum, + fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, KokkosFFT::Direction::forward, axis); } if (!held_pack_.empty() && forward_pack_.empty()) held_pack_forward_ = std::make_unique( - fft_space_for(held_pack_.size()), held_pack_, held_pack_spectrum_, + fft_space_for_rows(held_pack_), held_pack_, held_pack_spectrum_, KokkosFFT::Direction::forward, axis); } if (problem_.envelope_plasma) for (auto& stream : envelope_streams_) { stream.forward = std::make_unique( - fft_space_for(stream.projected.size()), stream.projected, stream.spectrum, + fft_space_for_rows(stream.projected), stream.projected, stream.spectrum, KokkosFFT::Direction::forward, axis); if (!stream.coarse.empty()) stream.coarse_forward = std::make_unique( - fft_space_for(stream.coarse.size()), stream.coarse, stream.coarse_spectrum, + fft_space_for_rows(stream.coarse), stream.coarse, stream.coarse_spectrum, KokkosFFT::Direction::forward, axis); } statistics_inverse_plan_ = std::make_unique( - fft_space_for(statistics_time_.size()), statistics_spectrum_, statistics_time_, + fft_space_for_rows(statistics_time_), statistics_spectrum_, statistics_time_, KokkosFFT::Direction::backward, axis); if (problem_.family == FieldFamily::field_real) { inverse_real_plan_ = std::make_unique( - fft_space_for(modal_real_.size()), fine_spectrum_, modal_real_, + fft_space_for_rows(modal_real_), fine_spectrum_, modal_real_, KokkosFFT::Direction::backward, axis); } else { inverse_complex_plan_ = std::make_unique( - fft_space_for(modal_complex_.size()), fine_spectrum_, modal_complex_, + fft_space_for_rows(modal_complex_), fine_spectrum_, modal_complex_, KokkosFFT::Direction::backward, axis); } if (retained_synthesis_) { @@ -4757,7 +4845,7 @@ void GeneralWorkspace::plan() { const auto rows = std::make_pair(std::size_t{0}, count); const ComplexView2 padded = Kokkos::subview(modal_complex_, rows, Kokkos::ALL()); const ComplexView2 node = Kokkos::subview(node_complex_, rows, Kokkos::ALL()); - const auto& space = fft_space_for(node.size()); + const auto& space = fft_space_for_rows(node); #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) seed_row_fft_wisdom(space, padded, node, KokkosFFT::Direction::backward); #endif @@ -4775,33 +4863,45 @@ void GeneralWorkspace::plan() { } if (problem_.family == FieldFamily::envelope) { forward_complex_plan_ = std::make_unique( - fft_space_for(projected_complex_.size()), projected_complex_, - projected_spectrum_, KokkosFFT::Direction::forward, axis); + fft_space_for_rows(projected_complex_), projected_complex_, projected_spectrum_, + KokkosFFT::Direction::forward, axis); absorber_complex_plan_ = std::make_unique( - fft_space_for(absorber_complex_.size()), absorber_complex_, absorber_spectrum_, + fft_space_for_rows(absorber_complex_), absorber_complex_, absorber_spectrum_, KokkosFFT::Direction::forward, axis); if (!problem_.projection_coarse_matrix.empty()) forward_coarse_complex_plan_ = std::make_unique( - fft_space_for(projected_coarse_complex_.size()), projected_coarse_complex_, + fft_space_for_rows(projected_coarse_complex_), projected_coarse_complex_, projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); } else { forward_real_plan_ = std::make_unique( - fft_space_for(projected_real_.size()), projected_real_, projected_spectrum_, + fft_space_for_rows(projected_real_), projected_real_, projected_spectrum_, KokkosFFT::Direction::forward, axis); absorber_real_plan_ = std::make_unique( - fft_space_for(absorber_real_.size()), absorber_real_, absorber_spectrum_, + fft_space_for_rows(absorber_real_), absorber_real_, absorber_spectrum_, KokkosFFT::Direction::forward, axis); if (!forward_pack_.empty()) forward_pack_plan_ = std::make_unique( - fft_space_for(forward_pack_.size()), forward_pack_, forward_pack_spectrum_, + fft_space_for_rows(forward_pack_), forward_pack_, forward_pack_spectrum_, KokkosFFT::Direction::forward, axis); if (!forward_pack_tail_.empty()) forward_pack_tail_plan_ = std::make_unique( - fft_space_for(forward_pack_tail_.size()), forward_pack_tail_, + fft_space_for_rows(forward_pack_tail_), forward_pack_tail_, forward_pack_tail_spectrum_, KokkosFFT::Direction::forward, axis); + if (!forward_pack_fine_.empty()) + forward_pack_fine_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_fine_), forward_pack_fine_, + forward_pack_fine_spectrum_, KokkosFFT::Direction::forward, axis); + if (!forward_pack_fine_tail_.empty()) + forward_pack_fine_tail_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_fine_tail_), forward_pack_fine_tail_, + forward_pack_fine_tail_spectrum_, KokkosFFT::Direction::forward, axis); + if (!forward_pack_coarse_.empty()) + forward_pack_coarse_plan_ = std::make_unique( + fft_space_for_rows(forward_pack_coarse_), forward_pack_coarse_, + forward_pack_coarse_spectrum_, KokkosFFT::Direction::forward, axis); if (!problem_.projection_coarse_matrix.empty()) forward_coarse_real_plan_ = std::make_unique( - fft_space_for(projected_coarse_real_.size()), projected_coarse_real_, + fft_space_for_rows(projected_coarse_real_), projected_coarse_real_, projected_coarse_spectrum_, KokkosFFT::Direction::forward, axis); } if constexpr (cpu_size_policy_enabled) { @@ -4812,7 +4912,8 @@ void GeneralWorkspace::plan() { problem_.slab_geometry() ? problem_.n_p : problem_.n_pol_state; temporal_tail_rows_ = tail_units * rows_per_unit; if (temporal_tail_rows_ != 0) { - const auto& space = fft_space_for(temporal_tail_rows_ * problem_.n_to); + const auto& space = + fft_space_for(temporal_tail_rows_ * problem_.n_to, problem_.n_to); const auto rows = fine_spectrum_.extent(0); if (problem_.family == FieldFamily::field_real) temporal_tail_inverse_real_.create(space, fine_spectrum_, modal_real_, rows, @@ -4897,7 +4998,7 @@ void GeneralWorkspace::plan() { const KokkosFFT::Direction direction) { const ComplexView2 rows = Kokkos::subview(state, std::make_pair(std::size_t{0}, count), Kokkos::ALL()); - const auto& space = fft_space_for(rows.size()); + const auto& space = fft_space_for_rows(rows); #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) // Seeding may overwrite the rows; nothing is loaded yet. seed_row_fft_wisdom(space, rows, rows, direction); @@ -4915,21 +5016,21 @@ void GeneralWorkspace::plan() { } if (problem_.family == FieldFamily::envelope) { coarse_inverse_complex_plan_ = std::make_unique( - fft_space_for(coarse_time_complex_.size()), coarse_spectrum_, + fft_space_for_rows(coarse_time_complex_), coarse_spectrum_, coarse_time_complex_, KokkosFFT::Direction::backward, axis); coarse_forward_complex_plan_ = std::make_unique( - fft_space_for(coarse_time_complex_.size()), coarse_time_complex_, + fft_space_for_rows(coarse_time_complex_), coarse_time_complex_, coarse_spectrum_, KokkosFFT::Direction::forward, axis); } else { coarse_inverse_real_plan_ = std::make_unique( - fft_space_for(coarse_time_real_.size()), coarse_spectrum_, coarse_time_real_, + fft_space_for_rows(coarse_time_real_), coarse_spectrum_, coarse_time_real_, KokkosFFT::Direction::backward, axis); coarse_forward_real_plan_ = std::make_unique( - fft_space_for(coarse_time_real_.size()), coarse_time_real_, coarse_spectrum_, + fft_space_for_rows(coarse_time_real_), coarse_time_real_, coarse_spectrum_, KokkosFFT::Direction::forward, axis); } if (temporal_tail_rows_ != 0) { - const auto& space = fft_space_for(temporal_tail_rows_ * problem_.n_t); + const auto& space = fft_space_for(temporal_tail_rows_ * problem_.n_t, problem_.n_t); const auto rows = coarse_spectrum_.extent(0); if (problem_.family == FieldFamily::envelope) { coarse_tail_inverse_complex_.create(space, coarse_spectrum_, coarse_time_complex_, @@ -12633,18 +12734,31 @@ void GeneralWorkspace::held_response(const std::size_t stage, } } -// A device's stage forward rows in one transform: all of them when the absorber's are -// due, else those after the absorber's (or the projection's alone). +// A stage's forward rows in one transform: all of them when the absorber's are due, +// else those after the absorber's (or the projection's alone). A host deferring its +// quadrature transforms the fine block only; `finish_quadrature` takes the coarse. void GeneralWorkspace::transform_forward_pack() { - if (absorbing() || forward_pack_absorber_rows_ == 0) - graph::execute(*forward_pack_plan_, forward_pack_, forward_pack_spectrum_, - KokkosFFT::Normalization::none); - else if (forward_pack_tail_plan_) - graph::execute(*forward_pack_tail_plan_, forward_pack_tail_, - forward_pack_tail_spectrum_, KokkosFFT::Normalization::none); - else + const Kokkos::Profiling::ScopedRegion region("solprop_native.fft.forward_pack"); + const bool fine = defer_quadrature_ && forward_pack_fine_plan_; + if (absorbing() || forward_pack_absorber_rows_ == 0) { + if (fine) + graph::execute(*forward_pack_fine_plan_, forward_pack_fine_, + forward_pack_fine_spectrum_, KokkosFFT::Normalization::none); + else + graph::execute(*forward_pack_plan_, forward_pack_, forward_pack_spectrum_, + KokkosFFT::Normalization::none); + } else if (fine ? static_cast(forward_pack_fine_tail_plan_) + : static_cast(forward_pack_tail_plan_)) { + if (fine) + graph::execute(*forward_pack_fine_tail_plan_, forward_pack_fine_tail_, + forward_pack_fine_tail_spectrum_, KokkosFFT::Normalization::none); + else + graph::execute(*forward_pack_tail_plan_, forward_pack_tail_, + forward_pack_tail_spectrum_, KokkosFFT::Normalization::none); + } else { graph::execute(*forward_real_plan_, projected_real_, projected_spectrum_, KokkosFFT::Normalization::none); + } } void GeneralWorkspace::held_transform(const bool coarse_only, const bool transformed) { @@ -14945,11 +15059,14 @@ void GeneralWorkspace::finish_quadrature() { if (problem_.family == FieldFamily::envelope) graph::execute(*forward_coarse_complex_plan_, projected_coarse_complex_, projected_coarse_spectrum_, KokkosFFT::Normalization::none); + else if (forward_pack_coarse_plan_) + graph::execute(*forward_pack_coarse_plan_, forward_pack_coarse_, + forward_pack_coarse_spectrum_, KokkosFFT::Normalization::none); else graph::execute(*forward_coarse_real_plan_, projected_coarse_real_, projected_coarse_spectrum_, KokkosFFT::Normalization::none); if (held_plasma()) - held_transform(true); + held_transform(true, static_cast(forward_pack_coarse_plan_)); coarse_pending_ = false; } @@ -15650,7 +15767,7 @@ void GeneralWorkspace::bind_edge_lines(const EdgeLines& lines) { const auto heap_before = heap_bytes(); const auto device_before = device_free_bytes(); constexpr int along = -1; - batch.plan = std::make_unique(fft_space_for(batch.buffer.size()), + batch.plan = std::make_unique(fft_space_for_rows(batch.buffer), batch.buffer, batch.buffer, KokkosFFT::Direction::forward, along); execution_space_.fence("solprop_native edge strip plan"); @@ -16533,6 +16650,15 @@ void GeneralWorkspace::close() noexcept { held_pack_spectrum_ = {}; forward_pack_plan_.reset(); forward_pack_tail_plan_.reset(); + forward_pack_fine_plan_.reset(); + forward_pack_fine_tail_plan_.reset(); + forward_pack_coarse_plan_.reset(); + forward_pack_fine_ = {}; + forward_pack_fine_tail_ = {}; + forward_pack_coarse_ = {}; + forward_pack_fine_spectrum_ = {}; + forward_pack_fine_tail_spectrum_ = {}; + forward_pack_coarse_spectrum_ = {}; forward_pack_ = {}; forward_pack_tail_ = {}; forward_pack_spectrum_ = {}; diff --git a/native/src/general_modal.cpp b/native/src/general_modal.cpp index ab7ffafc..e9342573 100644 --- a/native/src/general_modal.cpp +++ b/native/src/general_modal.cpp @@ -365,8 +365,10 @@ GeneralModalSource::GeneralModalSource(const Problem& problem) GeneralModalSource::~GeneralModalSource() { close(); } void GeneralModalSource::plan() { - transforms_ = fft_pool_.get(CpuOperation::fft, node_field_.size()); - delayed_transforms_ = raman_pool_.get(CpuOperation::raman, node_drive_.size()); + transforms_ = + fft_pool_.get(CpuOperation::fft, node_field_.size(), row_length(node_field_)); + delayed_transforms_ = + raman_pool_.get(CpuOperation::raman, node_drive_.size(), row_length(node_drive_)); constexpr int axis = -1; #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) if (real_signal_) @@ -390,7 +392,7 @@ void GeneralModalSource::plan() { const auto tail = nodes_ % chunk_; if (tail != 0) { const auto& tail_transforms = - fft_pool_.get(CpuOperation::fft, tail * components_ * n_to_); + fft_pool_.get(CpuOperation::fft, tail * components_ * n_to_, n_to_); const auto rows = std::make_pair(std::size_t{0}, tail * components_); tail_spectrum_ = Kokkos::subview(node_spectrum_, rows, Kokkos::ALL()); tail_field_ = Kokkos::subview(node_field_, rows, Kokkos::ALL()); @@ -439,8 +441,8 @@ void GeneralModalSource::plan() { tail_histories_ = Kokkos::subview(node_histories_, rows, Kokkos::ALL()); tail_drive_spectrum_ = Kokkos::subview(node_drive_spectrum_, rows, Kokkos::ALL()); tail_product_ = Kokkos::subview(node_product_, rows, Kokkos::ALL()); - const auto& tail_delayed = - raman_tail_pool_.get(CpuOperation::raman, tail_drive_.size()); + const auto& tail_delayed = raman_tail_pool_.get( + CpuOperation::raman, tail_drive_.size(), row_length(tail_drive_)); tail_drive_plan_ = std::make_unique( tail_delayed, tail_drive_, tail_drive_spectrum_, KokkosFFT::Direction::forward, axis); From f95f219f30fa0a351341250b8bbc5d76f0fe842a Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 12:22:28 +0000 Subject: [PATCH 02/15] WIP: transforms thread by rows and row length (#587); the FFT grain is timed again Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/core.hpp | 9 ++- native/src/bindings.cpp | 12 +++- native/src/core.cpp | 20 ++++++- native/src/solprop_native/_execution.py | 34 +++++++++-- src/solprop/calibration/grain.py | 79 +++++++++++++++++++++---- 5 files changed, 132 insertions(+), 22 deletions(-) diff --git a/native/include/solprop_native/core.hpp b/native/include/solprop_native/core.hpp index 02f467a6..98d7c3fc 100644 --- a/native/include/solprop_native/core.hpp +++ b/native/include/solprop_native/core.hpp @@ -257,13 +257,20 @@ struct OperationGrain { `compact_max_fine_samples` fine samples) runs its kernels on a team of at most `compact_threads` and its threaded transforms on that whole team. - `stable_kernel_team`: once a kernel pool holds a full-width instance, a - medium-width kernel takes it too (`CpuExecutionPool::get`). */ + medium-width kernel takes it too (`CpuExecutionPool::get`). + - A batch of transforms whose row length is known threads by its shape (solprop + issue #587): from the FFT class's threshold, one thread per + `fft_thread_elements` samples and at most one a row -- or `medium_threads`, if + more, when its rows are at least `fft_long_row` samples long -- and its pool's + capacity from the class's full size. FFTW's threaded plans divide the rows + among the threads, and split a row only where it is long. */ struct CpuExecution { std::array grain{}; std::size_t response_full_samples{}, response_full_columns{}; int medium_threads{}, compact_threads{}; std::size_t compact_max_fine_samples{}; bool stable_kernel_team{}; + std::size_t fft_thread_elements{}, fft_long_row{}; bool operator==(const CpuExecution&) const = default; [[nodiscard]] const OperationGrain& operator[](CpuOperation operation) const noexcept { diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index 2484cb21..00f72b8d 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -179,9 +179,11 @@ native::TableKind table_kind(const std::string& name) { // size, then the response bounds, widths and rules, every key required. constexpr std::array cpu_operation_names{ "stage", "response", "dense", "fft", "raman", "copy", "flow", "modal", "adams"}; -constexpr std::array cpu_execution_scalars{ - "response_full_samples", "response_full_columns", "medium_threads", - "compact_threads", "compact_max_fine_samples", "stable_kernel_team"}; +constexpr std::array cpu_execution_scalars{ + "response_full_samples", "response_full_columns", + "medium_threads", "compact_threads", + "compact_max_fine_samples", "stable_kernel_team", + "fft_thread_elements", "fft_long_row"}; native::CpuExecution cpu_execution_from(const nb::dict& values) { native::CpuExecution result; @@ -211,6 +213,8 @@ native::CpuExecution cpu_execution_from(const nb::dict& values) { result.compact_max_fine_samples = nb::cast(field("compact_max_fine_samples")); result.stable_kernel_team = nb::cast(field("stable_kernel_team")); + result.fft_thread_elements = nb::cast(field("fft_thread_elements")); + result.fft_long_row = nb::cast(field("fft_long_row")); } catch (const nb::cast_error&) { throw std::invalid_argument( "a CPU execution record holds nonnegative integers and one bool"); @@ -303,6 +307,8 @@ nb::dict cpu_execution_dict(const native::CpuExecution& execution) { result["compact_threads"] = execution.compact_threads; result["compact_max_fine_samples"] = execution.compact_max_fine_samples; result["stable_kernel_team"] = execution.stable_kernel_team; + result["fft_thread_elements"] = execution.fft_thread_elements; + result["fft_long_row"] = execution.fft_long_row; return result; } diff --git a/native/src/core.cpp b/native/src/core.cpp index dbeddd34..37a68506 100644 --- a/native/src/core.cpp +++ b/native/src/core.cpp @@ -73,10 +73,11 @@ void require_cpu_execution(const CpuExecution& execution) { "as large for every operation (solprop_native's _execution table)"); if (execution.response_full_samples == 0 || execution.response_full_columns == 0 || execution.medium_threads < 1 || execution.compact_threads < 1 || - execution.compact_max_fine_samples == 0) + execution.compact_max_fine_samples == 0 || execution.fft_thread_elements == 0 || + execution.fft_long_row == 0) throw std::invalid_argument( - "a CPU execution record needs positive response bounds, widths and compact " - "bound (solprop_native's _execution table)"); + "a CPU execution record needs positive response bounds, widths, compact " + "bound and transform shape bounds (solprop_native's _execution table)"); } int operation_threads(const ThreadPolicy& policy, const CpuExecution& execution, @@ -93,6 +94,19 @@ int operation_threads(const ThreadPolicy& policy, const CpuExecution& execution, const OperationGrain& grain = execution[operation]; if (elements < grain.threshold) return 1; + // A transform batch of known shape (`fine_samples` is its row length): a thread per + // `fft_thread_elements` samples, at most a thread a row, or the medium width when + // the rows are long enough to split. + if (operation == CpuOperation::fft && fine_samples != 0 && elements < grain.full) { + const std::size_t rows = elements / fine_samples; + const std::size_t split = + fine_samples < execution.fft_long_row + ? rows + : std::max(rows, static_cast(execution.medium_threads)); + const std::size_t width = std::min(elements / execution.fft_thread_elements, split); + return static_cast( + std::clamp(width, 1, static_cast(capacity))); + } if (operation == CpuOperation::response && (fine_samples >= execution.response_full_samples || (fine_samples != 0 && diff --git a/native/src/solprop_native/_execution.py b/native/src/solprop_native/_execution.py index 18ea37d3..9bb0fdf6 100644 --- a/native/src/solprop_native/_execution.py +++ b/native/src/solprop_native/_execution.py @@ -100,6 +100,11 @@ class ExecutionRecord: stable_kernel_team : bool Whether a medium-width kernel takes the full width once its pool holds a full-width instance. + fft_thread_elements, fft_long_row : int + A transform batch from ``fft_threshold`` up to ``fft_full`` takes a thread per + ``fft_thread_elements`` samples and at most one a row, or ``medium_threads`` + if more when its rows are at least ``fft_long_row`` samples long (solprop + issue #587); from ``fft_full`` its pool's capacity. """ stage_threshold: int @@ -126,6 +131,8 @@ class ExecutionRecord: compact_threads: int compact_max_fine_samples: int stable_kernel_team: bool + fft_thread_elements: int + fft_long_row: int def __post_init__(self) -> None: """Refuse a record the native core would refuse.""" @@ -144,6 +151,8 @@ def __post_init__(self) -> None: "medium_threads", "compact_threads", "compact_max_fine_samples", + "fft_thread_elements", + "fft_long_row", ): if not _count(getattr(self, name)): raise ValueError( @@ -278,10 +287,20 @@ def builtin( stage = 8192 if apple else 4096 # The FFT start. On Linux at 32768 the four-mode RDW's 32000-sample coarse batch # (n_t 4000) ran serially, its transforms 2.5-3.2 times slower on an 18-core - # Cascade Lake socket than the 32768-sample batch's four-thread ones; 16384 also - # gains on Zen 3. On an Apple arm64 laptop the serial transforms are the faster - # ones at that size, so macOS keeps 32768. - fft = 32768 if apple else 16384 + # Cascade Lake socket than the 32768-sample batch's four-thread ones. Above the + # start a batch threads by its shape (solprop issue #587): FFTW divides its rows + # among the threads and splits a row only where it is long. Timed alone, batched + # real transforms of 1-7 rows ran fastest at about a thread a row on an Intel + # Xeon Gold 6240, 16 cores of an AMD EPYC 7763 and an Apple M5 Max, the full + # width 1.1-1.6 times slower; many rows of a few thousand samples (128 x 576, + # 16 x 6144) wanted 8-18 threads, where four threads were 2-3 times slower; rows + # gained from being split, at about four threads, from 16384 samples on Linux + # (4 x 3072 also gained 1.3-2.9 times from three threads, so the start is 8192). + # On the M5 Max's spinning top tier the start, thread grain and long row are a + # sixteenth of the values below: rows split from 4608 samples, at four threads. + fft = 32768 if apple else 8192 + fft_thread_elements = 32768 if apple else 4096 + fft_long_row = 65536 if apple else 16384 record = ExecutionRecord( stage_threshold=stage, stage_full=stage, @@ -300,7 +319,8 @@ def builtin( dense_threshold=1048576, dense_full=1048576, fft_threshold=fft, - fft_full=131072, + # Every batch this large takes the full pool, whatever its shape. + fft_full=4194304, raman_threshold=65536, raman_full=262144, # An 8192-element temporal copy must stay serial: threading it changed @@ -344,6 +364,8 @@ def builtin( # kinds, the promoted passes took argon_blueshift from 3.0 to 3.2 s at 18 # threads. stable_kernel_team=not apple, + fft_thread_elements=fft_thread_elements, + fft_long_row=fft_long_row, ) if apple and spin_milliseconds > 0 and width <= SPIN_MAX_TEAM: record = _divided(record, SPIN_DIVISOR) @@ -365,4 +387,6 @@ def _divided(record: ExecutionRecord, divisor: int) -> ExecutionRecord: values[f"{name}_full"] = max( threshold, getattr(record, f"{name}_full") // divisor ) + for name in ("fft_thread_elements", "fft_long_row"): + values[name] = max(1, getattr(record, name) // divisor) return dataclasses.replace(record, **values) diff --git a/src/solprop/calibration/grain.py b/src/solprop/calibration/grain.py index 724b0cc7..dd641212 100644 --- a/src/solprop/calibration/grain.py +++ b/src/solprop/calibration/grain.py @@ -29,6 +29,7 @@ from solprop.calibration.selection import Verdict __all__ = [ + "SHAPE_MOVES", "TIMED", "Candidate", "Operation", @@ -43,18 +44,21 @@ type Record = Mapping[str, int | bool] """A native execution record (``ExecutionRecord.native()``).""" -TIMED = ("stage", "response", "raman", "copy", "dense", "modal") +TIMED = ("stage", "response", "fft", "raman", "copy", "dense", "modal") """The classes whose thresholds the grain moves, in the order their candidates go. The exponential Adams trial's passes and a coupled flow's matrix functions run in -no calibration window: their thresholds keep the built-in table's. Nor are the -transforms' moved: their thresholds count elements, and transforms of the same count -thread oppositely by shape. On AMD EPYC 7763 and 9B45 cores, a full width from 98304 -elements won 0.56 on a line's many short rows and ran ``grin_nonlinear``'s few long -rows 8-10 % slower (SEP 46 Part II), so they keep the table until threading keys on -rows and row length. +no calibration window: their thresholds keep the built-in table's. The transforms +thread by shape (rows and row length, solprop issue #587), so beside their threshold +the grain moves their thread grain and long row (:data:`SHAPE_MOVES`) rather than +their full size: by element count alone, a full width from 98304 elements won 0.56 +on a line's many short rows and ran ``grin_nonlinear``'s few long rows 8-10 % slower +on AMD EPYC cores (SEP 46 Part II). """ +SHAPE_MOVES = ("fft_thread_elements", "fft_long_row") +"""The transforms' shape settings the grain halves and doubles.""" + @dataclass(frozen=True) class Operation: @@ -70,7 +74,8 @@ class Operation: elements : int Its work. samples : int - A response's fine samples per column; zero otherwise. + A response's fine samples per column, a transform batch's row length; zero + otherwise. """ pool: str @@ -113,7 +118,10 @@ def width(record: Record, operation: Operation, capacity: int) -> int: The native rule (``operation_threads``), without the compact cap, the stable team or an explicit pool: one below the threshold, the full pool for a long or wide - response, the medium width below the full size, the full pool from it. + response, a thread per ``fft_thread_elements`` samples (at most one a row, or the + medium width for rows ``fft_long_row`` long) for a transform batch of known row + length, + the medium width below the full size, the full pool from it. Parameters ---------- @@ -139,6 +147,20 @@ def width(record: Record, operation: Operation, capacity: int) -> int: full = int(record[f"{operation.kind}_full"]) if operation.elements < threshold: return 1 + if ( + operation.kind == "fft" + and operation.samples + and operation.elements < full + and "fft_thread_elements" in record + ): + rows = operation.elements // operation.samples + split = ( + rows + if operation.samples < int(record["fft_long_row"]) + else max(rows, int(record["medium_threads"])) + ) + threads = min(operation.elements // int(record["fft_thread_elements"]), split) + return max(1, min(capacity, threads)) if ( operation.kind == "response" and operation.samples @@ -332,10 +354,47 @@ def proposals( ) for kind in TIMED: if kind in sizes and f"{kind}_threshold" in record: - candidates += _class_moves(kind, sorted(sizes[kind]), record, fulls=fulls) + # A transform's full size is a backstop; its shape settings move instead. + candidates += _class_moves( + kind, sorted(sizes[kind]), record, fulls=fulls and kind != "fft" + ) + if kind == "fft": + candidates += [ + candidate + for candidate in _shape_moves(record) + if any( + affected(candidate.record(record), record, found, capacity) + for found in problems.values() + ) + ] return candidates if limit is None else candidates[:limit] +def _shape_moves(record: Record) -> list[Candidate]: + """Return the transforms' shape settings halved and doubled. + + Parameters + ---------- + record : Mapping[str, int or bool] + The base record. + + Returns + ------- + list of Candidate + Up to four; none when the record has no shape settings. + """ + moves: list[Candidate] = [] + for name in SHAPE_MOVES: + if name not in record: + continue + value = int(record[name]) + for moved in (value // 2, value * 2): + if moved >= 1 and moved != value: + short = name.removeprefix("fft_").replace("_", " ") + moves.append(Candidate("fft", f"fft {short} {moved}", {name: moved})) + return moves + + def best( candidates: Iterable[Candidate], verdicts: Mapping[str, Verdict] ) -> list[Candidate]: From d99c750a70daae6b153950aaeb45149a54db130d Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 12:38:24 +0000 Subject: [PATCH 03/15] WIP: compact problems keep their separate transforms; tests for the shape rule Co-Authored-By: Claude Opus 5.5 --- native/src/general.cpp | 14 ++++++++---- native/tests/test_execution.py | 22 ++++++++++++------ tests/calibration/test_grain.py | 40 ++++++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 16 deletions(-) diff --git a/native/src/general.cpp b/native/src/general.cpp index 0f7cdc72..a247d78a 100644 --- a/native/src/general.cpp +++ b/native/src/general.cpp @@ -2607,13 +2607,19 @@ class GeneralWorkspace { // microseconds on 16 of its 84 multiprocessors, whatever the rows. Batched, the // stage's rows share them. A device never defers the coarse quadrature // (`deferred_quadrature_eligible`), so the coarse rows are projected at every stage. - // A host packs them too unless SOLPROP_NATIVE_FORWARD_PACK=0 (a probe switch, - // solprop issue #584): one threaded transform of every row in place of one a - // transform, each over only the modes' rows. A host keeps the coarse rows after the + // A host packs them too, unless the problem is compact or SOLPROP_NATIVE_FORWARD_PACK + // is 0 (solprop issue #584): one transform of every row, threaded by its shape, in + // place of one a transform over only the modes' rows, which left most of the + // threads idle: on 18 cores of an Intel Xeon Gold 6240 the pack ran rdw 0.90, + // argon_blueshift 0.88 and ctc 0.92 of their windows. A host keeps the coarse rows after the // fine ones so that a deferred quadrature's stages transform the fine block alone. bool packs_forward() const noexcept { #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) - if (!host_forward_pack()) + // A compact problem keeps its separate transforms: its one row a transform runs + // serially below the FFT start, and the pack's few rows on the compact team ran + // averaged helium 1.35 times slower on 18 cores of an Intel Xeon Gold 6240 and + // 1.11 on 16 of an AMD EPYC 7763 (solprop issue #584). + if (!host_forward_pack() || compact_) return false; #endif return problem_.family == FieldFamily::field_real && diff --git a/native/tests/test_execution.py b/native/tests/test_execution.py index ad7cd444..3a6c10f4 100644 --- a/native/tests/test_execution.py +++ b/native/tests/test_execution.py @@ -28,8 +28,8 @@ ) # The constants of core.hpp before SEP 46 moved them here (solprop PR #576), Linux and -# Windows: every value of the table, frozen so that a machine without a profile runs -# exactly as before. +# Windows, with the transforms' shape rule of solprop issue #587: every value of the +# table, frozen so that a change to one is deliberate. LINUX = { "stage_threshold": 4096, "stage_full": 4096, @@ -37,8 +37,8 @@ "response_full": 196608, "dense_threshold": 1048576, "dense_full": 1048576, - "fft_threshold": 16384, - "fft_full": 131072, + "fft_threshold": 8192, + "fft_full": 4194304, "raman_threshold": 65536, "raman_full": 262144, "copy_threshold": 65536, @@ -55,6 +55,8 @@ "compact_threads": 8, "compact_max_fine_samples": 65536, "stable_kernel_team": True, + "fft_thread_elements": 4096, + "fft_long_row": 16384, } # macOS: its own stage and FFT starts and no stable kernel team. APPLE = LINUX | { @@ -62,6 +64,8 @@ "stage_full": 8192, "fft_threshold": 32768, "stable_kernel_team": False, + "fft_thread_elements": 32768, + "fft_long_row": 65536, } # macOS on a spinning team of at most eight: every threshold and full size divided by # 16 (ADR-073), each full size kept at least its threshold. @@ -73,7 +77,7 @@ "dense_threshold": 65536, "dense_full": 65536, "fft_threshold": 2048, - "fft_full": 8192, + "fft_full": 262144, "raman_threshold": 4096, "raman_full": 16384, "copy_threshold": 4096, @@ -84,6 +88,8 @@ "modal_full": 4096, "adams_threshold": 512, "adams_full": 4096, + "fft_thread_elements": 2048, + "fft_long_row": 4096, } @@ -112,10 +118,10 @@ def test_a_users_fft_start_replaces_the_tables_undivided() -> None: # SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES: the start as given, never divided, its # full size the table's undivided one or the start where that is larger. spinning = builtin("Darwin", spin_milliseconds=1, width=6, fft_threshold=1000) - assert (spinning.fft_threshold, spinning.fft_full) == (1000, 131072) + assert (spinning.fft_threshold, spinning.fft_full) == (1000, 4194304) assert spinning.stage_threshold == APPLE_SPINNING["stage_threshold"] large = builtin("Linux", fft_threshold=1 << 20) - assert (large.fft_threshold, large.fft_full) == (1 << 20, 1 << 20) + assert (large.fft_threshold, large.fft_full) == (1 << 20, 1 << 22) def test_the_process_record_is_its_platforms_table() -> None: @@ -141,6 +147,8 @@ def test_the_process_record_is_its_platforms_table() -> None: ("compact_threads", True), ("response_full_columns", 1.5), ("stable_kernel_team", 1), + ("fft_thread_elements", 0), + ("fft_long_row", -4), ], ) def test_a_record_refuses_a_value_out_of_range(field: str, value: object) -> None: diff --git a/tests/calibration/test_grain.py b/tests/calibration/test_grain.py index cb9b484b..d12e5abc 100644 --- a/tests/calibration/test_grain.py +++ b/tests/calibration/test_grain.py @@ -98,13 +98,43 @@ def test_the_medium_width_and_the_compact_team_have_short_ladders() -> None: adams = {"a": [Operation("kernels", "adams", 4096)]} assert proposals(TABLE, adams, 18) == [] assert "adams" not in TIMED - # Nor are the transforms' thresholds moved: counted in elements, transforms of one - # count thread oppositely by shape (SEP 46 Part II). + assert "fft" in TIMED + + +def test_transforms_thread_by_rows_and_row_length() -> None: + # Linux: from 8192 samples a thread per 4096, at most one a row; rows of 16384 + # samples or more may be split up to the medium width; the full pool from fft_full. + def threads(rows: int, length: int, capacity: int = 18) -> int: + return width(TABLE, Operation("fft", "fft", rows * length, length), capacity) + + assert threads(1, 4096) == 1 # below the start + assert threads(4, 3072) == 3 # 12288 samples, three threads' worth + assert threads(4, 10240) == 4 # a thread a row + assert threads(1, 10240) == 1 # one short row is not split + assert threads(1, 32768) == 4 # a long row, split up to the medium width + assert threads(3, 20480) == 4 + assert threads(12, 10240) == 12 + assert threads(128, 1152) == 18 # many short rows take the pool + assert threads(289, 128) == 9 # 36992 samples + assert threads(128, 1152, capacity=8) == 8 + # A batch of unknown row length keeps the element rule. + assert width(TABLE, Operation("fft", "fft", 40960), 18) == 4 + assert width(TABLE, Operation("fft", "fft", 1 << 22, 1024), 36) == 36 + + +def test_the_transforms_shape_settings_are_moved_where_they_change_a_width() -> None: transforms = { - "a": [Operation("fft", "fft", 98304, 768), Operation("fft", "fft", 131072, 512)] + "a": [Operation("fft", "fft", 40960, 10240), Operation("fft", "fft", 36992, 128)] } - assert [c for c in proposals(TABLE, transforms, 18) if "fft" in c.name] == [] - assert "fft" not in TIMED + names = [c.name for c in proposals(TABLE, transforms, 18) if c.setting == "fft"] + # The full size is a backstop and is not moved, and neither long row changes a + # width here: four rows take four threads split or not, and 128 samples is short + # either way. + assert names == [ + "fft threshold 36993", + "fft thread elements 2048", + "fft thread elements 8192", + ] def test_the_best_admitted_candidate_of_each_setting_is_kept() -> None: From 3b63a68ceb044181d84f6ddb35fd9e516042df44 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 12:43:56 +0000 Subject: [PATCH 04/15] WIP: calibration records each captured plan's execution alone (#595) Co-Authored-By: Claude Opus 5.5 --- native/include/solprop_native/fft_plan.hpp | 124 +++++++++++++++++++++ native/src/bindings.cpp | 18 +++ native/src/solprop_native/_core.pyi | 1 + src/solprop/calibration/child.py | 19 +++- src/solprop/calibration/controller.py | 67 +++++++++++ 5 files changed, 227 insertions(+), 2 deletions(-) diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/fft_plan.hpp index 988442f4..2f0081c4 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/fft_plan.hpp @@ -10,14 +10,20 @@ #include "solprop_native/core.hpp" #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) +#include +#include +#include #include #include +#include #include +#include #include #include #include #include #include +#include namespace solprop_native { @@ -241,6 +247,124 @@ void seed_row_fft_wisdom(const Execution& execution, const Input& input, fftw_destroy_plan(seeded); } +/** A plan's execution time alone (solprop issue #595): the seconds of one execution + of the transform a plan key names (`fft_plan_key`), planned from `wisdom` at + `effort` with `FFTW_WISDOM_ONLY` (`plan`; NaN when the wisdom no longer builds + it), and of the same transform at FFTW_ESTIMATE (`estimate`), each the median of + five batches after a warm execution, on the key's threads and on arrays of its + alignment. FFTW's wisdom is forgotten before each plan and after. */ +struct PlanAlone { + double plan{}, estimate{}; +}; + +namespace detail { +inline std::vector split_key(const std::string& text, const char by) { + std::vector parts; + std::size_t start = 0; + for (std::size_t at = text.find(by); at != std::string::npos; + start = at + 1, at = text.find(by, start)) + parts.push_back(text.substr(start, at - start)); + parts.push_back(text.substr(start)); + return parts; +} + +inline double median_execution(const fftw_plan plan) { + using Clock = std::chrono::steady_clock; + fftw_execute(plan); + const auto first = Clock::now(); + fftw_execute(plan); + const double once = std::chrono::duration(Clock::now() - first).count(); + const int repeats = std::max(2, static_cast(0.005 / std::max(once, 1e-7))); + std::array batches{}; + for (double& batch : batches) { + const auto start = Clock::now(); + for (int repeat = 0; repeat < repeats; ++repeat) + fftw_execute(plan); + batch = std::chrono::duration(Clock::now() - start).count() / repeats; + } + std::sort(batches.begin(), batches.end()); + return batches[2]; +} +} // namespace detail + +inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom, + const std::string& effort) { + const auto parts = detail::split_key(key, ':'); + if (parts.size() != 9 || parts[8].empty() || parts[8][0] != 't') + throw std::invalid_argument("not an FFT plan key: " + key); + const std::string& kind = parts[0]; + std::vector extents; + for (const auto& extent : detail::split_key(parts[1], 'x')) + extents.push_back(std::stoi(extent)); + const int howmany = std::stoi(parts[2]); + const int input_distance = std::stoi(parts[3]); + const int output_distance = std::stoi(parts[4]); + const bool in_place = parts[5] == "in"; + const auto input_offset = static_cast(std::stoi(parts[6])); + const auto output_offset = static_cast(std::stoi(parts[7])); + const int threads = std::stoi(parts[8].substr(1)); + const bool r2c = kind == "r2c", c2r = kind == "c2r"; + const bool planes = kind.starts_with("planes-"); + const int sign = kind.ends_with("backward") ? FFTW_BACKWARD : FFTW_FORWARD; + if (!r2c && !c2r && !planes && !kind.starts_with("c2c-")) + throw std::invalid_argument("not an FFT plan key: " + key); + const auto rank = static_cast(extents.size()); + // Real input or output counts doubles, complex counts double pairs. + const std::size_t input_bytes = static_cast(howmany) * + static_cast(input_distance) * + (r2c ? sizeof(double) : sizeof(fftw_complex)); + const std::size_t output_bytes = static_cast(howmany) * + static_cast(output_distance) * + (c2r ? sizeof(double) : sizeof(fftw_complex)); + fftw_init_threads(); + const auto release = [](void* pointer) { fftw_free(pointer); }; + const std::unique_ptr input_memory( + fftw_malloc(std::max(input_bytes, output_bytes) + 64), release); + const std::unique_ptr output_memory( + in_place ? nullptr : fftw_malloc(output_bytes + 64), release); + char* input_base = static_cast(input_memory.get()) + input_offset; + char* output_base = + in_place ? input_base : static_cast(output_memory.get()) + output_offset; + const auto make = [&](const unsigned flags) -> fftw_plan { + fftw_plan_with_nthreads(threads); + if (r2c) + return fftw_plan_many_dft_r2c( + rank, extents.data(), howmany, reinterpret_cast(input_base), nullptr, + 1, input_distance, reinterpret_cast(output_base), nullptr, 1, + output_distance, flags); + if (c2r) + return fftw_plan_many_dft_c2r( + rank, extents.data(), howmany, reinterpret_cast(input_base), + nullptr, 1, input_distance, reinterpret_cast(output_base), nullptr, + 1, output_distance, flags); + return fftw_plan_many_dft( + rank, extents.data(), howmany, reinterpret_cast(input_base), + nullptr, 1, input_distance, reinterpret_cast(output_base), + nullptr, 1, output_distance, sign, flags); + }; + const auto timed = [&](const unsigned flags) { + fftw_forget_wisdom(); + if ((flags & FFTW_WISDOM_ONLY) != 0U && + fftw_import_wisdom_from_string(wisdom.c_str()) == 0) + return std::numeric_limits::quiet_NaN(); + const fftw_plan plan = make(flags); + if (plan == nullptr) { + fftw_forget_wisdom(); + return std::numeric_limits::quiet_NaN(); + } + // Real input is overwritten by complex-to-real plans and harmless otherwise. + std::memset(input_base, 0, input_bytes); + const double seconds = detail::median_execution(plan); + fftw_destroy_plan(plan); + fftw_forget_wisdom(); + return seconds; + }; + PlanAlone result; + result.plan = timed(fftw_effort_flag(effort) | FFTW_WISDOM_ONLY); + result.estimate = timed(FFTW_ESTIMATE); + return result; +} + // FFTW's process-wide wisdom as text, for solprop's opt-in SOLPROP_FFTW_WISDOM: the // Python adapter owns the file, its key and its lock, and calls these with the GIL // held, so they never race the planner. Empty when the provider implements FFTW's API diff --git a/native/src/bindings.cpp b/native/src/bindings.cpp index 00f72b8d..e35fae94 100644 --- a/native/src/bindings.cpp +++ b/native/src/bindings.cpp @@ -2718,6 +2718,24 @@ NB_MODULE(_core, module_) { return nb::none(); #endif }); + // solprop issue #595: a captured plan's execution alone, against estimate's. + module_.def( + "time_fft_plan", + [](const std::string& key, const std::string& wisdom, const std::string& effort) { + nb::dict result; +#if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) + // With the GIL held, as every planning: it serialises FFTW's planner. + const auto timed = native::time_fft_plan(key, wisdom, effort); + result["plan"] = timed.plan; + result["estimate"] = timed.estimate; +#else + static_cast(key); + static_cast(wisdom); + static_cast(effort); +#endif + return result; + }, + nb::arg("key"), nb::arg("wisdom"), nb::arg("effort")); // solprop SEP 28 section 4.2's reference parts, for its benchmarks only. module_.def( "reference_fft3_pairs", diff --git a/native/src/solprop_native/_core.pyi b/native/src/solprop_native/_core.pyi index f14b5d6b..bc21c1dd 100644 --- a/native/src/solprop_native/_core.pyi +++ b/native/src/solprop_native/_core.pyi @@ -427,6 +427,7 @@ def reference_fft3_pairs( values: npt.NDArray[np.complex128], warm: int, pairs: int ) -> list[float]: ... def reference_stream_copies(elements: int, warm: int, repeats: int) -> list[float]: ... +def time_fft_plan(key: str, wisdom: str, effort: str) -> dict[str, float]: ... def forget_fftw_wisdom() -> None: ... def scoped_openmp_default() -> int: ... def configure_native_spin(milliseconds: int) -> None: ... diff --git a/src/solprop/calibration/child.py b/src/solprop/calibration/child.py index b0a124ac..6f2c1908 100644 --- a/src/solprop/calibration/child.py +++ b/src/solprop/calibration/child.py @@ -272,8 +272,13 @@ def capture( dict ``wisdom`` by plan key, each key's ``groups`` and the problems containing it (``problems``), the problems skipped for want of time - (``skipped``), and the seconds spent (``seconds``). + (``skipped``), the seconds spent (``seconds``), and each plan's execution + alone (``alone``: the seconds of the captured plan and of the same + transform at estimate, ``_core.time_fft_plan``; solprop issue #595). """ + from solprop_native import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] + _core, # pyright: ignore[reportUnknownVariableType] + ) from solprop_native._execution import ( # noqa: PLC0415 # pyright: ignore[reportMissingImports] ExecutionRecord, ) @@ -284,6 +289,7 @@ def capture( wisdom: dict[str, str] = {} groups: dict[str, str] = {} contains: dict[str, list[str]] = {} + alone: dict[str, dict[str, float]] = {} skipped: list[str] = [] begun = time.perf_counter() names = list(self.problems) @@ -315,16 +321,25 @@ def capture( found = backend._native_state.fft_planning # pyright: ignore[reportPrivateUsage] finally: backend.close() - wisdom |= cast(dict[str, str], found["captured"]) + captured = cast(dict[str, str], found["captured"]) + wisdom |= captured for key, group in cast(dict[str, str], found["groups"]).items(): groups[key] = group contains.setdefault(name, []).append(key) + # Each plan alone, as its capture chose it and at estimate, to tell a plan + # that loses in the step from one that is slower alone (issue #595). + for key, text in captured.items(): + if key not in alone and text: + alone[key] = cast( + dict[str, float], _core.time_fft_plan(key, text, effort) + ) return { "wisdom": wisdom, "groups": groups, "problems": contains, "skipped": skipped, "seconds": time.perf_counter() - begun, + "alone": alone, } def sizing(self, record: Mapping[str, object] | None) -> dict[str, object]: diff --git a/src/solprop/calibration/controller.py b/src/solprop/calibration/controller.py index 65c9d038..a0d3ae34 100644 --- a/src/solprop/calibration/controller.py +++ b/src/solprop/calibration/controller.py @@ -1405,6 +1405,72 @@ def planning( return found +def _alone_ratios( + candidates: Sequence[tuple[str, str, Mapping[str, object]]], + groups: Mapping[str, str], + measured: Mapping[str, object], + patient: Mapping[str, object] | None, +) -> dict[str, dict[str, float]]: + """Return each candidate's plans' execution alone over the policy's (issue #595). + + Parameters + ---------- + candidates : sequence of tuple + Each candidate's name, group and planning (:func:`_planning_candidates`). + groups : Mapping[str, str] + Each plan key's group. + measured, patient : Mapping[str, object] or None + The captures, whose ``alone`` holds each plan's seconds as captured + (``plan``) and at estimate (``estimate``). + + Returns + ------- + dict + By candidate, by plan key of its group: the candidate effort's seconds alone + over the policy effort's; a key timed at neither is left out. Set beside the + candidate's verdict in the step, it tells a plan slower alone from one that + loses only in the step. + """ + timed = { + "measure": cast(dict[str, dict[str, float]], measured.get("alone", {})), + "patient": cast(dict[str, dict[str, float]], (patient or {}).get("alone", {})), + } + + def seconds(key: str, effort: str) -> float | None: + """Return a plan's seconds alone at `effort`, or ``None``. + + Parameters + ---------- + key : str + The plan key. + effort : str + ``"estimate"``, ``"measure"`` or ``"patient"``. + + Returns + ------- + float or None + The seconds, when timed and finite. + """ + if effort == "estimate": + value = timed["measure"].get(key, {}).get("estimate") + else: + value = timed[effort].get(key, {}).get("plan") + return value if value is not None and math.isfinite(value) else None + + ratios: dict[str, dict[str, float]] = {} + for name, group, planning in candidates: + effort = cast(dict[str, str], planning["efforts"])[group] + found: dict[str, float] = {} + for key, of in groups.items(): + if of != group: + continue + mine, base = seconds(key, effort), seconds(key, POLICY[group]) + if mine is not None and base: + found[key] = mine / base + ratios[name] = found + return ratios + + _CAPTURE_SHARE = 0.5 """The share of the FFT stage's time left that each capture may take.""" @@ -1501,6 +1567,7 @@ def capture(effort: str, share: float) -> dict[str, object]: "skipped": list(cast(list[str], measured.get("skipped", []))), "planning_seconds": measured["seconds"], "candidates": [name for name, _, _ in candidates], + "alone": _alone_ratios(candidates, groups, measured, patient), } timed = [ ( From 0e54aca68bfc8610602750f3af91e09d6577ab57 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 12:50:39 +0000 Subject: [PATCH 05/15] WIP: guide and sizing table describe the transforms' shape rule Co-Authored-By: Claude Opus 5.5 --- docs/dev/benchmarks.md | 2 +- docs/guide/performance/calibration.md | 11 ++++++----- native/src/solprop_native/_execution.py | 4 +++- tests/calibration/test_grain.py | 5 ++++- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index 37be043a..85c5d608 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -2967,7 +2967,7 @@ partial chunk. | General-modal synthesis/projection/maps | 65,536 | 65,536 | none | | Coupled matrix functions | 32,768 | 32,768 | none | | Exponential Adams trial passes | 8,192 | 65,536 | up to four threads | -| Modal/spatial FFT | 32,768 | 131,072 | up to four threads | +| Modal/spatial FFT | 8,192 (macOS 32,768) | 4,194,304 | by shape: a thread per 4,096 samples (macOS 32,768), at most one a row; rows of 16,384 samples (macOS 65,536) or more split up to four threads ([transforms by shape](#transforms-by-shape-and-one-forward-pack)) | | Raman FFT | 65,536 | 262,144 | up to four threads | A response column is one active node and one component. Every width is capped by the diff --git a/docs/guide/performance/calibration.md b/docs/guide/performance/calibration.md index 65d95a06..29a9a1cd 100644 --- a/docs/guide/performance/calibration.md +++ b/docs/guide/performance/calibration.md @@ -38,11 +38,12 @@ never runs your own simulation; for that, see production-sized problems can show what a narrower team costs). Each candidate is a child process with those settings. 2. **Operation grain.** Each kind of operation (stage passes, responses, products, - Raman histories, copies) threads above a size threshold; the transforms keep the - built-in thresholds, which count elements, since transforms of one size thread best - in opposite ways by their shape. The - calibration moves each threshold past the work sizes the problems actually run, - and tries other medium widths and compact teams for one-column problems. + Raman histories, copies, transforms) threads above a size threshold. A batch of + transforms threads by its shape: about one thread per row, more only where the rows + are long, so the calibration also halves and doubles the samples per thread and the + row length from which a row is split. The calibration moves each threshold past the + work sizes the problems actually run, and tries other medium widths and compact + teams for one-column problems. 3. **FFT planning** (standard and thorough). Every transform the problems create is planned alone at FFTW's `measure` effort, and serial ones at `patient` too at the thorough level, within a time limit; each group of plans (serial rows, threaded rows, diff --git a/native/src/solprop_native/_execution.py b/native/src/solprop_native/_execution.py index 9bb0fdf6..3335000f 100644 --- a/native/src/solprop_native/_execution.py +++ b/native/src/solprop_native/_execution.py @@ -18,7 +18,9 @@ modes^3 for matrix functions, elements * history rows for the exponential Adams trial's passes, and actual real-side batch samples for FFT and Raman plans (tails included). Below an operation's threshold it runs on one thread, below its full size -on the medium width, and from it on its pool's capacity. solprop's +on the medium width, and from it on its pool's capacity; a batch of transforms whose +row length is known threads instead by its rows (``ExecutionRecord.fft_thread_elements`` +and ``fft_long_row``, solprop issue #587). solprop's ``docs/dev/benchmarks.md``, section "Kokkos CPU operation-size threading", records the windows and sweeps behind each built-in value; ``docs/dev/cpu-performance.md`` gives the table's provenance. diff --git a/tests/calibration/test_grain.py b/tests/calibration/test_grain.py index d12e5abc..ad944525 100644 --- a/tests/calibration/test_grain.py +++ b/tests/calibration/test_grain.py @@ -124,7 +124,10 @@ def threads(rows: int, length: int, capacity: int = 18) -> int: def test_the_transforms_shape_settings_are_moved_where_they_change_a_width() -> None: transforms = { - "a": [Operation("fft", "fft", 40960, 10240), Operation("fft", "fft", 36992, 128)] + "a": [ + Operation("fft", "fft", 40960, 10240), + Operation("fft", "fft", 36992, 128), + ] } names = [c.name for c in proposals(TABLE, transforms, 18) if c.setting == "fft"] # The full size is a backstop and is not moved, and neither long row changes a From 6f47fb3b461570f64ceef1d8f90eb7b1caf1743b Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 14:04:44 +0000 Subject: [PATCH 06/15] WIP: the forward pack's test, the shape ledger and the guide's tables Co-Authored-By: Claude Opus 5.5 --- docs/dev/backends.md | 13 +++--- docs/dev/benchmarks.md | 75 ++++++++++++++++++++++++++++++++ docs/guide/performance/tuning.md | 12 ++++- native/tests/test_band_plasma.py | 34 +++++++++++++++ native/tests/test_execution.py | 2 +- 5 files changed, 128 insertions(+), 8 deletions(-) diff --git a/docs/dev/backends.md b/docs/dev/backends.md index 6ec77648..cb4e0282 100644 --- a/docs/dev/backends.md +++ b/docs/dev/backends.md @@ -1586,11 +1586,14 @@ at most: lengths below, at and past one block, partial tiles, both `beta` and row subviews, so the kernel's blocking and order are free. Vendor-BLAS and device builds use `KokkosBlas::gemm`. -- **Small transforms on one FFTW thread.** A modal or spatial plan whose real side holds - fewer than **16 384** samples per execution (32 768 on macOS; `default_threaded_fft_min_samples`; - `SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES` overrides it) is made on a one-thread - execution instance, because below that a threaded FFTW plan only costs time: the - threshold is a speed threshold. FFTW's planner may choose a transform's factorisation by +- **Small transforms on one FFTW thread; larger ones by shape.** A modal or spatial plan + whose real side holds fewer than **8 192** samples per execution (32 768 on macOS; the + execution record's `fft_threshold`; `SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES` + overrides it) is made on a one-thread execution instance, because below that a + threaded FFTW plan only costs time: the threshold is a speed threshold. Above it a + batch takes a thread per `fft_thread_elements` samples, at most one a row unless its + rows are `fft_long_row` long, when it may take the medium width + ([transforms by shape](benchmarks.md#transforms-by-shape-and-one-forward-pack)). FFTW's planner may choose a transform's factorisation by the thread count (a two-column 256-point inverse took radix 4 with 64-point leaves at three and four threads and radix 8 with 32-point leaves at others), which moves ten realistic steps by at most 8 u: within the transform's rounding bound, so it constrains diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index 85c5d608..14d90c31 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -2942,6 +2942,81 @@ time does not change with the thread count, so its two columns are one result. N machine has a case that reaches the 10 % at the default width that [#337](https://github.com/LupoLab/solprop/issues/337) asks of run-time dispatch. +### Transforms by shape and one forward pack + +Issues [#587](https://github.com/LupoLab/solprop/issues/587) and +[#584](https://github.com/LupoLab/solprop/issues/584), November 2026. The transforms' +width was one element count (serial below 16384, four threads below 131072, the full +pool above), and a modal stage ran each forward transform as its own threaded call over +the modes' rows only. + +**Width by shape, timed alone.** A batched real-to-complex forward transform of each +shape the tracking set runs, at FFTW's estimate and measure, on 1 to 36 threads, with +native's OpenMP loop callback and spinning waits (an Intel Xeon Gold 6240 node, 16 +cores of an AMD EPYC 7763 on one NUMA domain, an Apple M5 Max). Microseconds a +transform, measured plans; the table's width in brackets: + +| rows × length | Xeon 1 | 4 | 8 | 18 | EPYC 1 | 4 | 8 | 16 | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| 4 × 3072 | [20.5] | 15.5 | 8.5 | 16.0 | [16.7] | 5.8 | 7.8 | 9.3 | +| 4 × 10240 | 73.5 | [21.8] | 24.5 | 29.1 | 66.5 | [17.7] | 19.9 | 20.9 | +| 1 × 32768 | 70.2 | [47.6] | 50.5 | 62.7 | 71.1 | [34.2] | 29.0 | 38.8 | +| 7 × 6400 | 77.7 | [38.5] | 16.9 | 18.8 | 67.0 | [20.6] | 13.6 | 14.2 | +| 289 × 128 | 46.5 | [16.2] | 22.7 | 9.8 | 40.4 | [12.2] | 7.5 | 7.7 | +| 128 × 576 | 119.5 | [30.4] | 16.4 | 28.4 | 89.7 | [23.5] | 13.0 | 9.5 | +| 16 × 6144 | 198.9 | [46.9] | 29.5 | 45.1 | 149.8 | [38.9] | 19.8 | 13.3 | +| 12 × 10240 | 279.1 | [61.7] | 44.4 | 31.5 | 205.0 | [52.5] | 37.6 | 23.9 | + +A few rows ran best at about a thread a row, more threads splitting rows at a loss; +many rows of a few thousand samples wanted 8 to 18 threads, where four were two to three +times slower. One row gained from being split only from about 16384 samples on these +two Linux machines (from 4608 on the M5 Max's top tier, at four threads). Hence the rule +(`operation_threads`): from `fft_threshold`, a thread per `fft_thread_elements` +samples, at most one a row, or up to the medium width for rows of `fft_long_row` +samples; Linux 8192, 4096 and 16384, macOS 32768, 32768 and 65536 (a sixteenth on a +spinning top tier). Estimate plans of a prime length (4096 × 193) at more than four +threads ran 15 to 47 times slower than at four; measured ones did not. + +**One forward pack.** A modal stage's forward rows -- the projection's fine and coarse, +the in-stage absorber's, the two held plasma currents' fine and coarse -- now go +through one plan, as on a device: `rdw` transforms 28 rows of 10240 at once where it +transformed seven batches of four. The host keeps the coarse rows after the fine ones, +so that a deferred quadrature transforms the fine block in its stages and the coarse +block once at its finish. Pack on against off, one build (5 % windows, three rounds, +fields within 1e-14): + +| case | Xeon Gold 6240, 18 | 36 | EPYC 7763, 16 | M5 Max, 6 (contended) | +|---|---:|---:|---:|---:| +| `argon_blueshift` | 0.88 | 0.90 | 0.88 | 0.92 | +| `rdw` | 0.90 | 0.94 | 0.95 | 0.90 | +| `ctc` | 0.92 | 0.94 | 0.94 | 0.97 | +| `he_rdw_averaged` (one column) | 1.35 | 1.32 | 1.11 | 0.97 | + +On the one-column case the pack's four rows, threaded on the compact team, took 152 µs +a stage against 90 µs for its three serial transforms (Kokkos section timer, Xeon, 18 +threads), though the same transform alone took 25 µs on eight threads; measured plans +did not change it (1.34). One-column problems keep their separate transforms. + +**Against the previous build** (both together; one-column problems unpacked; 5 % +windows, three rounds, fields within 1e-12): + +| case | Xeon 18 | Xeon 36 | EPYC 16 | +|---|---:|---:|---:| +| `argon_blueshift` | 0.87 | 0.80 | 0.86 | +| `ctc` | 0.89 | 0.91 | 0.92 | +| `rdw` | 0.91 | 0.95 | 0.95 | +| `line` | 0.93 | 0.87 | 0.88 | +| `radial_filament_256` | 0.97 | 1.01 | 0.99 | +| `gnlse_raman_shock` | 0.94 | 0.99 | 0.98 | +| `grin_nonlinear` | 1.01 | 0.93 | 1.01 | +| `kerr_beam_192` | 1.02 | 1.04 | 1.01 | +| geometric mean (with one-thread rows, 11 cases) | 0.977 | | 0.981 | + +The shape rule alone (pack off) gave `line` 0.91 and 0.90 (Xeon 18 and 36) and 0.89 +(EPYC), and `grin_nonlinear` 0.95 on the EPYC; the one-thread rows stay within +0.98-1.02. `he_rdw_averaged` and `h2_raman_averaged`, whose code is unchanged, ranged +0.92-1.12 on the Xeon between runs of either arm: the per-process modes of #81. + ## Reference tables These tables back the indicative results above. diff --git a/docs/guide/performance/tuning.md b/docs/guide/performance/tuning.md index ee352cd0..a509d905 100644 --- a/docs/guide/performance/tuning.md +++ b/docs/guide/performance/tuning.md @@ -163,7 +163,8 @@ imported: | `SOLPROP_NATIVE_FFT_THREADS` | FFTW threads for the modal and spatial transforms | sized per plan, up to the cap | | `SOLPROP_NATIVE_RAMAN_FFT_THREADS` | threads for the Raman convolution transforms of complete node chunks | sized per plan, up to the cap | | `SOLPROP_NATIVE_RAMAN_TAIL_FFT_THREADS` | threads for the Raman transforms of the last, partial node chunk | sized from active rows, up to the cap | -| `SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES` | the real-side samples per execution from which a modal or spatial FFT plan is threaded | 16,384 (32,768 on macOS) | +| `SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES` | the real-side samples per execution from which a modal or spatial FFT plan is threaded | 8,192 (32,768 on macOS) | +| `SOLPROP_NATIVE_FORWARD_PACK` | `0` transforms a modal stage's forward rows (projection, absorber, plasma currents) one transform at a time instead of in one batch | unset (one batch, except for one-column problems) | | `SOLPROP_NATIVE_BAND_TILES` | the field plasma column's band form: unset for the build's own (tiled on a GPU, a team per node on a CPU), `0` for the team column, a positive `N` for the tiled column with host teams of `N` threads | unset | The four pool variables are optional positive integers, clamped to the cap; an invalid @@ -192,6 +193,13 @@ one and starts them again for the next wide one, which cost a four-thread kernel full-width ones about half a millisecond each time. FFT plans keep their four threads, since FFTW's threaded loops run on the runtime's whole team. +Modal and spatial FFT plans thread by their shape instead: from the start below, a +thread per 4,096 samples (32,768 on macOS, a sixteenth of that on a spinning top tier) +and at most one per row, since FFTW divides a batch's rows among its threads; rows of at +least 16,384 samples (65,536 on macOS) are split, up to `min(4, C)` threads. A batch of +a few long rows therefore takes about one thread a row, many short rows the full pool. +The calibration moves both settings. + | operation | work used for sizing | threading starts | full pool starts | |---|---|---:|---:| | stage, update and reduction | elements in the field or fused batch | 4,096 (8,192 on macOS) | as threading | @@ -201,7 +209,7 @@ since FFTW's threaded loops run on the runtime's whole team. | general modal synthesis/projection | active nodes × components × spectral samples × modes | 65,536 | 65,536 | | coupled matrix functions | active spectral bins × modes³ | 32,768 | 32,768 | | exponential Adams trial | elements in the field × history rows | 8,192 | 65,536 | -| modal/spatial FFT | real-side samples in the actual batch | 16,384, 32,768 on macOS (`SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES`) | 131,072 | +| modal/spatial FFT | real-side samples in the actual batch, by rows and row length (above) | 8,192, 32,768 on macOS (`SOLPROP_NATIVE_THREADED_FFT_MIN_SAMPLES`) | 4,194,304 | | Raman FFT | real-side samples in the actual batch | 65,536 | 262,144 | A response column is one active node and one component; its column-length and diff --git a/native/tests/test_band_plasma.py b/native/tests/test_band_plasma.py index 85ec8329..62375c02 100644 --- a/native/tests/test_band_plasma.py +++ b/native/tests/test_band_plasma.py @@ -485,6 +485,40 @@ def test_every_stage_of_a_trial_matches_the_reference( assert_regression_guard(got, want, f"{kind} {integration} stage {stage}") +@pytest.mark.skipif( + not solprop_native.build_info().cpu_size_policy_enabled + or os.environ.get("SOLPROP_NATIVE_FORWARD_PACK") == "0", + reason="the host's forward pack and its sizing record (solprop issue #584)", +) +def test_a_modal_stage_transforms_its_forward_rows_as_one_batch() -> None: + """Pack the projection's and both held currents' rows, fine then coarse. + + Two columns with held currents and a coarse projection: one batch of twelve rows + (fine, both currents' fine, coarse, both currents' coarse) and its six-row fine + block for a deferred quadrature's stages; the one-column problem is compact and + keeps a transform a row (the quadrature test above checks both against the + reference). + """ + + def rows(kind: str) -> set[int]: + grid, problem = modal(kind) + backend = NativeBackend(problem, grid=grid) + try: + sizing = cast( + dict[str, dict[str, int]], backend.metadata()["execution_sizing"] + ) + finally: + backend.close() + return { + entry["columns"] + for key, entry in sizing.items() + if key.startswith("fft.fft.") and "columns" in entry + } + + assert {12, 6} <= rows("multimode") + assert max(rows("analytic")) == 1 + + @pytest.mark.parametrize("kind", ["multimode", "analytic", "mixture"]) @pytest.mark.parametrize("chunk", [1, 4]) def test_quadrature_keeps_accepted_scores_through_rejection_and_reset( diff --git a/native/tests/test_execution.py b/native/tests/test_execution.py index 3a6c10f4..c283e709 100644 --- a/native/tests/test_execution.py +++ b/native/tests/test_execution.py @@ -215,7 +215,7 @@ def test_a_state_reads_back_the_record_it_was_built_with( ("change", "message"), [ ({"cpu_execution": None}, "must be a dict"), - ({"cpu_execution": {"stage_threshold": 1}}, "has 24 fields, got 1"), + ({"cpu_execution": {"stage_threshold": 1}}, "has 26 fields, got 1"), ("missing", "needs its CPU execution record"), ], ) From 8723da3a9042f89f1a084b3b893f1acc719df884 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 14:17:47 +0000 Subject: [PATCH 07/15] WIP: the thorough calibration plans nothing at patient (#595) Co-Authored-By: Claude Opus 5.5 --- docs/guide/performance/calibration.md | 8 ++-- src/solprop/calibration/controller.py | 64 ++++++++------------------- tests/calibration/test_controller.py | 28 +++++------- 3 files changed, 34 insertions(+), 66 deletions(-) diff --git a/docs/guide/performance/calibration.md b/docs/guide/performance/calibration.md index 29a9a1cd..8018d777 100644 --- a/docs/guide/performance/calibration.md +++ b/docs/guide/performance/calibration.md @@ -45,9 +45,9 @@ never runs your own simulation; for that, see work sizes the problems actually run, and tries other medium widths and compact teams for one-column problems. 3. **FFT planning** (standard and thorough). Every transform the problems create is - planned alone at FFTW's `measure` effort, and serial ones at `patient` too at the - thorough level, within a time limit; each group of plans (serial rows, threaded rows, - transverse planes) keeps the effort that runs fastest *inside the step*, against + planned alone at FFTW's `measure` effort within a time limit; each group of plans + (serial rows, threaded rows, transverse planes) keeps the effort that runs fastest + *inside the step*, against plans made as a run without a profile makes them. A more exhaustive plan is not always a faster one, and a plan chosen alone not always the best in the step. Only a winning effort's wisdom is kept, so later runs build those plans at once instead @@ -65,7 +65,7 @@ the calibration says so and publishes nothing. |---|---|---|---|---|---| | `--quick` | about 20 s | the wait only | medium width and compact team | plan policy | 3 | | standard | about 90 s | yes, two narrower teams | thresholds of the main operations | `measure`, 0.25 s a plan | 5 | -| `--thorough` | about 5 min | yes | thresholds and full sizes of every operation | `measure` and `patient`, 20 s a plan | 7 | +| `--thorough` | about 5 min | yes | thresholds and full sizes of every operation | `measure`, 20 s a plan | 7 | The standard and thorough levels also calibrate the smaller allocations they can pin children to inside the job, such as one socket, one NUMA node or one L3 cache group, diff --git a/src/solprop/calibration/controller.py b/src/solprop/calibration/controller.py index a0d3ae34..40386531 100644 --- a/src/solprop/calibration/controller.py +++ b/src/solprop/calibration/controller.py @@ -1321,17 +1321,19 @@ def _validate_setups( def _planning_candidates( - level: Level, measured: Mapping[str, object], patient: Mapping[str, object] | None + measured: Mapping[str, object], ) -> list[tuple[str, str, dict[str, object]]]: """Return each group's candidate plannings. + No group tries ``patient``: on an Intel Xeon Gold 6240, 16 cores of an AMD EPYC + 7763 and an Apple M5 Max its capture never finished within the thorough level's + share of the stage, so it was never timed and only took the time the timing in + the step needed (solprop issue #595). + Parameters ---------- - level : Level - The level: patient plans only at thorough, and only for serial rows. - measured, patient : Mapping[str, object] - Captures at measure and at patient effort (``patient`` ``None`` below - thorough): ``wisdom`` and ``groups`` by plan key. + measured : Mapping[str, object] + The capture at measure: ``wisdom`` and ``groups`` by plan key. Returns ------- @@ -1375,17 +1377,6 @@ def planning( found.append( ("rows estimate", "rows", planning(POLICY | {"rows": "estimate"}, wisdom)) ) - if patient is not None and level.fulls: - found.append( - ( - "rows patient", - "rows", - planning( - POLICY | {"rows": "patient"}, - wisdom | cast(dict[str, str], patient["wisdom"]), - ), - ) - ) if "threaded_rows" in present: found.append( ( @@ -1409,7 +1400,6 @@ def _alone_ratios( candidates: Sequence[tuple[str, str, Mapping[str, object]]], groups: Mapping[str, str], measured: Mapping[str, object], - patient: Mapping[str, object] | None, ) -> dict[str, dict[str, float]]: """Return each candidate's plans' execution alone over the policy's (issue #595). @@ -1419,9 +1409,9 @@ def _alone_ratios( Each candidate's name, group and planning (:func:`_planning_candidates`). groups : Mapping[str, str] Each plan key's group. - measured, patient : Mapping[str, object] or None - The captures, whose ``alone`` holds each plan's seconds as captured - (``plan``) and at estimate (``estimate``). + measured : Mapping[str, object] + The capture at measure, whose ``alone`` holds each plan's seconds as + captured (``plan``) and at estimate (``estimate``). Returns ------- @@ -1431,10 +1421,7 @@ def _alone_ratios( candidate's verdict in the step, it tells a plan slower alone from one that loses only in the step. """ - timed = { - "measure": cast(dict[str, dict[str, float]], measured.get("alone", {})), - "patient": cast(dict[str, dict[str, float]], (patient or {}).get("alone", {})), - } + timed = cast(dict[str, dict[str, float]], measured.get("alone", {})) def seconds(key: str, effort: str) -> float | None: """Return a plan's seconds alone at `effort`, or ``None``. @@ -1444,17 +1431,14 @@ def seconds(key: str, effort: str) -> float | None: key : str The plan key. effort : str - ``"estimate"``, ``"measure"`` or ``"patient"``. + ``"estimate"`` or ``"measure"``. Returns ------- float or None The seconds, when timed and finite. """ - if effort == "estimate": - value = timed["measure"].get(key, {}).get("estimate") - else: - value = timed[effort].get(key, {}).get("plan") + value = timed.get(key, {}).get("estimate" if effort == "estimate" else "plan") return value if value is not None and math.isfinite(value) else None ratios: dict[str, dict[str, float]] = {} @@ -1484,8 +1468,8 @@ def _planning( ) -> tuple[dict[str, object], dict[str, str], dict[str, object]]: """Choose each FFT plan group's effort in the chosen setup (SEP 46 phase 3). - The child plans every transform alone at measure (and at patient, thorough), - from no wisdom within the level's time limit; the policy's efforts with that + The child plans every transform alone at measure, from no wisdom within the + level's time limit; the policy's efforts with that wisdom are the base, and each group's other effort a candidate, timed in the step on the problems holding that group's transforms and admitted by §6.5's rule; the winners are validated together. Quick leaves the plan policy. @@ -1551,15 +1535,9 @@ def capture(effort: str, share: float) -> dict[str, object]: # timed, and plan as a run without a profile does. if not measured["groups"]: return {}, {}, {"reason": "no transform planned within the stage's share"} - patient = None - if plan.level.fulls: - plan.say("planning the serial transforms at patient") - patient = capture("patient", _CAPTURE_SHARE) - if patient.get("skipped"): - patient = None # incomplete: patient is not a candidate except _ChildError as error: return {}, {}, {"reason": str(error)} - candidates = _planning_candidates(plan.level, measured, patient) + candidates = _planning_candidates(measured) groups = cast(dict[str, str], measured["groups"]) contains = cast(dict[str, list[str]], measured["problems"]) evidence: dict[str, object] = { @@ -1567,7 +1545,7 @@ def capture(effort: str, share: float) -> dict[str, object]: "skipped": list(cast(list[str], measured.get("skipped", []))), "planning_seconds": measured["seconds"], "candidates": [name for name, _, _ in candidates], - "alone": _alone_ratios(candidates, groups, measured, patient), + "alone": _alone_ratios(candidates, groups, measured), } timed = [ ( @@ -1616,12 +1594,8 @@ def capture(effort: str, share: float) -> dict[str, object]: "efforts": chosen, "wisdom": { key: text - for key, text in ( - cast(dict[str, str], measured["wisdom"]) - | (cast(dict[str, str], patient["wisdom"]) if patient else {}) - ).items() + for key, text in cast(dict[str, str], measured["wisdom"]).items() if chosen[groups.get(key, "rows")] != "estimate" - and (chosen[groups.get(key, "rows")] != "patient" or patient) }, } validation = _validate_setups( diff --git a/tests/calibration/test_controller.py b/tests/calibration/test_controller.py index 88f99e9b..3e9e12fb 100644 --- a/tests/calibration/test_controller.py +++ b/tests/calibration/test_controller.py @@ -827,7 +827,7 @@ def test_an_admitted_effort_is_validated_and_its_wisdom_kept( assert set(generation["wisdom"]) == set(script.plans) -def test_patient_serial_plans_are_tried_at_thorough( +def test_thorough_tries_no_patient_plans_and_records_plans_alone( scripted: tuple[type[_Scripted], dict[str, str]], ) -> None: script, environ = scripted @@ -837,10 +837,12 @@ def test_patient_serial_plans_are_tried_at_thorough( ) ) report = calibrate("thorough", environ=environ, cpus=range(8)) - assert "rows patient" in report.reason - generation = json.loads(cast(Path, report.path).read_text(encoding="utf-8")) - key = "r2c:4096:1:4096:2049:out:0:0:t1" - assert generation["wisdom"][key] == f"(patient {key})" + assert "patient" not in report.reason + (row,) = _rows(environ) + evidence = cast(dict[str, Any], row["evidence"])["planning"] + assert all("patient" not in name for name in evidence["candidates"]) + # Each candidate's plans timed alone against the policy's (issue #595). + assert set(evidence["alone"]) == set(evidence["candidates"]) def test_quick_leaves_the_plan_policy_and_a_failed_capture_keeps_it( @@ -943,7 +945,7 @@ def wrapper(plan: Any, *args: Any) -> Any: assert controller.LEVELS["quick"].planning == 0.0 -def test_captures_take_a_share_of_the_stage_and_an_incomplete_one_is_dropped( +def test_a_capture_takes_a_share_of_the_stage_and_an_incomplete_one_is_dropped( scripted: tuple[type[_Scripted], dict[str, str]], monkeypatch: pytest.MonkeyPatch ) -> None: # On 16 cores of an AMD EPYC 9B45 the thorough captures outran the deadline. @@ -957,21 +959,13 @@ def receive(self: _Scripted, timeout: float) -> object: budgets.append(float(cast(float, self.pending["budget"]))) assert timeout <= budgets[-1] + 30.0 assert isinstance(reply, dict) - if self.pending["effort"] == "patient": - reply["skipped"] = ["radial"] return reply monkeypatch.setattr(script, "receive", receive) - script.by_planning = staticmethod( - lambda planning, call: ( - 0.8 if _efforts(planning).get("rows") == "patient" else 1.0 - ) - ) - report = calibrate("thorough", environ=environ, cpus=range(8)) - assert len(budgets) == 2 + calibrate("thorough", environ=environ, cpus=range(8)) + # One capture, at measure: the thorough level plans nothing at patient. + assert len(budgets) == 1 assert 0.0 < budgets[0] < controller.LEVELS["thorough"].budget - # The incomplete patient capture is no candidate, however fast it would be. - assert "rows patient" not in report.reason # An incomplete measure capture keeps the plan policy. budgets.clear() From c00f1f86bb1322d35cbe6c22a302027e877eecd8 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 14:19:28 +0000 Subject: [PATCH 08/15] WIP: ADR-075, the CPU performance record, calibration version 2 Co-Authored-By: Claude Opus 5.5 --- docs/dev/cpu-performance.md | 38 ++++++++++++++++++++++- docs/dev/decisions.md | 38 +++++++++++++++++++++++ native/src/solprop_native/_fingerprint.py | 7 +++-- 3 files changed, 80 insertions(+), 3 deletions(-) diff --git a/docs/dev/cpu-performance.md b/docs/dev/cpu-performance.md index accd90f9..8f303fe9 100644 --- a/docs/dev/cpu-performance.md +++ b/docs/dev/cpu-performance.md @@ -160,6 +160,22 @@ measured while it existed. pool's cores for the whole run: at 16 threads the four-mode capillary became 2.2 times faster on a Xeon Gold 6240 (1.3 times on an AMD EPYC 7763), the radial filament 1.4 (1.6) times and averaged helium 1.4 (2.9) times; one thread is unchanged. +- **Transforms thread by shape** ([measurements](benchmarks.md#transforms-by-shape-and-one-forward-pack), + issue [#587](https://github.com/LupoLab/solprop/issues/587)). A batch of transforms took + its width from its element count, though FFTW divides a batch's rows among its + threads: a few rows ran fastest at about a thread a row and full width was 1.1–1.6 + times slower, while many short rows wanted 8–18 threads where four were two to three + times slower. Now a batch takes a thread per `fft_thread_elements` samples, at most + one a row, or up to the medium width when its rows are `fft_long_row` long, and the + calibration moves both. `line` takes 0.87–0.93 of its time at 16–36 threads on an + Intel Xeon Gold 6240 and an AMD EPYC 7763. +- **A modal stage's forward transforms as one batch** (issue + [#584](https://github.com/LupoLab/solprop/issues/584)). The device's pack of a stage's + forward rows (projection, absorber, held currents) now runs on the host too, so four + modes no longer leave most threads idle: `argon_blueshift` 0.80–0.87, `ctc` + 0.89–0.92 and `rdw` 0.91–0.95 at 16–36 threads on those two machines. One-column + problems keep their separate transforms: there the pack's few rows on the compact + team ran averaged helium 1.1–1.35 times slower. - **Measured FFTW wisdom only for serial transforms by default** ([measurements](benchmarks.md#threaded-fft-plan-selection)). FFTW's construction-time trials sometimes chose a slower *threaded* plan: on a 32-core AVX-512 server at 32 @@ -180,9 +196,29 @@ measured while it existed. (`fftw_set_timelimit` of 0.01–0.2 s) gave plans 10–30 % slower to execute, and estimate plans ran one-thread windows up to 1.6 times slower on an AMD EPYC 7763. +- **Measured threaded plans through the calibration, not by default** (issue + [#595](https://github.com/LupoLab/solprop/issues/595)). In the step, measured plans for + the threaded rows (`SOLPROP_FFTW_PLAN=measure`) ran the tracking cases 0.94 on an + Intel Xeon Gold 6240 (`kerr_beam_192` 0.54–0.79, `ctc` 0.90–0.94), 0.96 on 16 cores + of an AMD EPYC 7763 (`grin_nonlinear` 0.83, `ctc` 0.90) and 0.97 on an Apple M5 Max, + but planning them took up to 12 s more construction (`ctc` on the EPYC 7763: 13.8 s + against 1.6 s), more than a whole run of it gains. So the default stays at estimate, + and a calibration that admits `threaded_rows measure` stores the wisdom once. + - Why the calibration's measured and patient plans "rarely won": they were rarely + timed. On all three machines the thorough level's patient capture never finished + within its share of the stage, and the timing in the step then ran out of rounds + before a verdict (no noise estimate), so every group fell back to the policy. + Timed alone, the measured threaded plans ran 0.53–1.05 of estimate's time + (medians 0.74 on the EPYC 7763, 0.91 on the Xeon, about 0.95 on the M5 Max), and + estimate's serial plans 1.06–2.8 of measure's. + - The thorough level no longer plans at patient, which leaves the time to its + timing in the step; each candidate's plans timed alone are kept beside its verdict + (`evidence.planning.alone`). + **Tried and not adopted** - **`FFTW_PATIENT` planning.** It adds seconds of construction and gave no measured - throughput gain. + throughput gain. SEP 46's thorough calibration tried it for serial rows; its capture + never finished in time on three machines, and it was dropped (#595). ## Kernels diff --git a/docs/dev/decisions.md b/docs/dev/decisions.md index 890b508a..cbd85b04 100644 --- a/docs/dev/decisions.md +++ b/docs/dev/decisions.md @@ -2122,3 +2122,41 @@ Withdrawn (development-process record, not a design decision). Every run records the profile and the settings it applied (`calibration`). The test suites and the documentation build run with `SOLPROP_CALIBRATION=off`; benchmark series calibrate each build thorough first and record the profile per row. + +## ADR-075 — Transforms thread by shape; a host packs a stage's forward rows + +- **Status:** proposed with PR #598 (issues #584, #587, #595), for the maintainer. +- **Context:** a batch of native CPU transforms took its width from its element count + (serial below 16384, four threads below 131072, the full pool above), but FFTW divides a + batch's rows among its threads, so transforms of one count want opposite widths by + shape: SEP 46's grain saw a full width from 98304 elements win 0.56 on many short rows + and lose 8–10 % on few long ones, and froze the transforms' grain. Timed alone on an + Intel Xeon Gold 6240, 16 cores of an AMD EPYC 7763 and an Apple M5 Max, batches of 1–7 + rows ran fastest at about a thread a row and many short rows at 8–18 threads. A modal + stage also ran each forward transform as its own call over the modes' rows: four modes + left most of 16 threads idle, as Julia's batched form had shown (#560). And SEP 46's + thorough calibration never reached a verdict on FFT planning: its patient capture never + finished in time, and the timing in the step then ran out of rounds. +- **Decision:** + - *Width by shape.* From `fft_threshold` a batch takes a thread per + `fft_thread_elements` samples, at most one a row, or up to the medium width when its + rows are at least `fft_long_row` long; `fft_full` is a backstop. Linux and Windows + 8192, 4096 and 16384; macOS 32768, 32768 and 65536, a sixteenth on a spinning top tier + (ADR-073). The calibration's grain moves the threshold and halves and doubles the two + shape settings. + - *One forward pack.* On a host, as on a device, a modal stage's forward rows (the + projection's fine and coarse, the in-stage absorber's, the held currents') are one + batch and one plan, the coarse rows after the fine so that a deferred quadrature's + stages transform the fine block alone. Compact problems keep their separate transforms. + `SOLPROP_NATIVE_FORWARD_PACK=0` turns the pack off. + - *Planning.* Threaded rows stay at estimate by default; the thorough calibration no + longer plans at patient, and records each candidate's plans timed alone beside its + verdict in the step. +- **Consequences:** against the previous build, the tracking set ran 0.977 (Xeon Gold + 6240) and 0.981 (EPYC 7763) on geometric mean: `argon_blueshift` 0.80–0.87, `ctc` + 0.89–0.92, `rdw` 0.91–0.95, `line` 0.87–0.93, the rest within their spread; fields agree + to rounding (plan choice and width move results by rounding only, as before). Measured + threaded plans win in the step (0.94–0.97) but cost up to 12 s of construction, so they + reach a run through a calibration's stored wisdom, not the default. Stored profiles keyed + on the old widths miss and replan (each plan key holds its threads); the calibration's + version is bumped with this change. diff --git a/native/src/solprop_native/_fingerprint.py b/native/src/solprop_native/_fingerprint.py index 4bed724e..603950c3 100644 --- a/native/src/solprop_native/_fingerprint.py +++ b/native/src/solprop_native/_fingerprint.py @@ -29,8 +29,11 @@ __all__ = ["CALIBRATION_VERSION", "Fingerprint", "build_identity", "fingerprint"] -CALIBRATION_VERSION = 1 -"""The calibration's own version, raised when its probes, problems or rules change.""" +CALIBRATION_VERSION = 2 +"""The calibration's own version, raised when its probes, problems or rules change. + +2: the transforms' grain moves by shape and no plan is captured at patient (solprop +ADR-075).""" @dataclass(frozen=True) From 557021d7950cf882f72747c9b0eae62c97186775 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 14:35:32 +0000 Subject: [PATCH 09/15] WIP: the FFT grain moves its thread grain only, after the other classes Co-Authored-By: Claude Opus 5.5 --- docs/dev/benchmarks.md | 22 +++++++++++----------- docs/dev/decisions.md | 10 +++++++--- docs/guide/performance/calibration.md | 3 +-- docs/guide/performance/tuning.md | 2 +- src/solprop/calibration/grain.py | 19 ++++++++++++------- tests/calibration/test_grain.py | 2 ++ 6 files changed, 34 insertions(+), 24 deletions(-) diff --git a/docs/dev/benchmarks.md b/docs/dev/benchmarks.md index 14d90c31..8e5c439f 100644 --- a/docs/dev/benchmarks.md +++ b/docs/dev/benchmarks.md @@ -3000,17 +3000,17 @@ did not change it (1.34). One-column problems keep their separate transforms. **Against the previous build** (both together; one-column problems unpacked; 5 % windows, three rounds, fields within 1e-12): -| case | Xeon 18 | Xeon 36 | EPYC 16 | -|---|---:|---:|---:| -| `argon_blueshift` | 0.87 | 0.80 | 0.86 | -| `ctc` | 0.89 | 0.91 | 0.92 | -| `rdw` | 0.91 | 0.95 | 0.95 | -| `line` | 0.93 | 0.87 | 0.88 | -| `radial_filament_256` | 0.97 | 1.01 | 0.99 | -| `gnlse_raman_shock` | 0.94 | 0.99 | 0.98 | -| `grin_nonlinear` | 1.01 | 0.93 | 1.01 | -| `kerr_beam_192` | 1.02 | 1.04 | 1.01 | -| geometric mean (with one-thread rows, 11 cases) | 0.977 | | 0.981 | +| case | Xeon 18 | Xeon 36 | EPYC 16 | M5 Max 6 (contended) | +|---|---:|---:|---:|---:| +| `argon_blueshift` | 0.87 | 0.80 | 0.86 | 0.92 | +| `ctc` | 0.89 | 0.91 | 0.92 | 0.98 | +| `rdw` | 0.91 | 0.95 | 0.95 | 0.95 | +| `line` | 0.93 | 0.87 | 0.88 | 1.00 | +| `radial_filament_256` | 0.97 | 1.01 | 0.99 | 1.00 | +| `gnlse_raman_shock` | 0.94 | 0.99 | 0.98 | 1.00 | +| `grin_nonlinear` | 1.01 | 0.93 | 1.01 | 1.00 | +| `kerr_beam_192` | 1.02 | 1.04 | 1.01 | 0.99 | +| geometric mean, with the one-thread rows | 0.977 | | 0.981 | 0.993 | The shape rule alone (pack off) gave `line` 0.91 and 0.90 (Xeon 18 and 36) and 0.89 (EPYC), and `grin_nonlinear` 0.95 on the EPYC; the one-thread rows stay within diff --git a/docs/dev/decisions.md b/docs/dev/decisions.md index cbd85b04..20c0a011 100644 --- a/docs/dev/decisions.md +++ b/docs/dev/decisions.md @@ -2142,8 +2142,10 @@ Withdrawn (development-process record, not a design decision). `fft_thread_elements` samples, at most one a row, or up to the medium width when its rows are at least `fft_long_row` long; `fft_full` is a backstop. Linux and Windows 8192, 4096 and 16384; macOS 32768, 32768 and 65536, a sixteenth on a spinning top tier - (ADR-073). The calibration's grain moves the threshold and halves and doubles the two - shape settings. + (ADR-073). The calibration's grain moves the threshold and halves and doubles + `fft_thread_elements`, after every other class (each such move changes nearly every + problem's width, and with the long row's moves too a thorough calibration ran out of + rounds). - *One forward pack.* On a host, as on a device, a modal stage's forward rows (the projection's fine and coarse, the in-stage absorber's, the held currents') are one batch and one plan, the coarse rows after the fine so that a deferred quadrature's @@ -2151,7 +2153,9 @@ Withdrawn (development-process record, not a design decision). `SOLPROP_NATIVE_FORWARD_PACK=0` turns the pack off. - *Planning.* Threaded rows stay at estimate by default; the thorough calibration no longer plans at patient, and records each candidate's plans timed alone beside its - verdict in the step. + verdict in the step. The FFT stage's timing in the step still runs out of rounds at + the standard and thorough levels on the machines measured, as it did before this + change; giving it time is the maintainer's call on the levels' budgets. - **Consequences:** against the previous build, the tracking set ran 0.977 (Xeon Gold 6240) and 0.981 (EPYC 7763) on geometric mean: `argon_blueshift` 0.80–0.87, `ctc` 0.89–0.92, `rdw` 0.91–0.95, `line` 0.87–0.93, the rest within their spread; fields agree diff --git a/docs/guide/performance/calibration.md b/docs/guide/performance/calibration.md index 8018d777..ce2afe40 100644 --- a/docs/guide/performance/calibration.md +++ b/docs/guide/performance/calibration.md @@ -40,8 +40,7 @@ never runs your own simulation; for that, see 2. **Operation grain.** Each kind of operation (stage passes, responses, products, Raman histories, copies, transforms) threads above a size threshold. A batch of transforms threads by its shape: about one thread per row, more only where the rows - are long, so the calibration also halves and doubles the samples per thread and the - row length from which a row is split. The calibration moves each threshold past the + are long, so the calibration also halves and doubles the samples per thread. The calibration moves each threshold past the work sizes the problems actually run, and tries other medium widths and compact teams for one-column problems. 3. **FFT planning** (standard and thorough). Every transform the problems create is diff --git a/docs/guide/performance/tuning.md b/docs/guide/performance/tuning.md index a509d905..20e0d0c6 100644 --- a/docs/guide/performance/tuning.md +++ b/docs/guide/performance/tuning.md @@ -198,7 +198,7 @@ thread per 4,096 samples (32,768 on macOS, a sixteenth of that on a spinning top and at most one per row, since FFTW divides a batch's rows among its threads; rows of at least 16,384 samples (65,536 on macOS) are split, up to `min(4, C)` threads. A batch of a few long rows therefore takes about one thread a row, many short rows the full pool. -The calibration moves both settings. +The calibration moves the samples per thread. | operation | work used for sizing | threading starts | full pool starts | |---|---|---:|---:| diff --git a/src/solprop/calibration/grain.py b/src/solprop/calibration/grain.py index dd641212..a5042920 100644 --- a/src/solprop/calibration/grain.py +++ b/src/solprop/calibration/grain.py @@ -44,20 +44,25 @@ type Record = Mapping[str, int | bool] """A native execution record (``ExecutionRecord.native()``).""" -TIMED = ("stage", "response", "fft", "raman", "copy", "dense", "modal") +TIMED = ("stage", "response", "raman", "copy", "dense", "modal", "fft") """The classes whose thresholds the grain moves, in the order their candidates go. The exponential Adams trial's passes and a coupled flow's matrix functions run in no calibration window: their thresholds keep the built-in table's. The transforms thread by shape (rows and row length, solprop issue #587), so beside their threshold -the grain moves their thread grain and long row (:data:`SHAPE_MOVES`) rather than -their full size: by element count alone, a full width from 98304 elements won 0.56 -on a line's many short rows and ran ``grin_nonlinear``'s few long rows 8-10 % slower -on AMD EPYC cores (SEP 46 Part II). +the grain moves their thread grain (:data:`SHAPE_MOVES`) rather than their full size: +by element count alone, a full width from 98304 elements won 0.56 on a line's many +short rows and ran ``grin_nonlinear``'s few long rows 8-10 % slower on AMD EPYC cores +(SEP 46 Part II). They go last: a move of theirs changes nearly every problem's width, +so each costs a round's worth of windows, and a level's limit drops them first. """ -SHAPE_MOVES = ("fft_thread_elements", "fft_long_row") -"""The transforms' shape settings the grain halves and doubles.""" +SHAPE_MOVES = ("fft_thread_elements",) +"""The transforms' shape settings the grain halves and doubles. + +Not the long row: with its moves too, a thorough calibration of 16 cores of an AMD +EPYC 7763 timed 21 grain candidates and ran out of rounds before any verdict, where 17 +had concluded.""" @dataclass(frozen=True) diff --git a/tests/calibration/test_grain.py b/tests/calibration/test_grain.py index ad944525..024d3df2 100644 --- a/tests/calibration/test_grain.py +++ b/tests/calibration/test_grain.py @@ -138,6 +138,8 @@ def test_the_transforms_shape_settings_are_moved_where_they_change_a_width() -> "fft thread elements 2048", "fft thread elements 8192", ] + # The transforms go last, after every other class's moves. + assert TIMED[-1] == "fft" def test_the_best_admitted_candidate_of_each_setting_is_kept() -> None: From cabb103d8426a005bb1342c4c0b9ad65e773786a Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 14:56:05 +0000 Subject: [PATCH 10/15] WIP: C++ tests take the shape rule; clang-format Co-Authored-By: Claude Opus 5.5 --- docs/dev/cpu-performance.md | 5 +++- native/src/general.cpp | 5 ++-- native/tests/native/execution.hpp | 10 +++++-- native/tests/native/test_core.cpp | 43 ++++++++++++++++++++++++++++--- 4 files changed, 54 insertions(+), 9 deletions(-) diff --git a/docs/dev/cpu-performance.md b/docs/dev/cpu-performance.md index 8f303fe9..df4e3006 100644 --- a/docs/dev/cpu-performance.md +++ b/docs/dev/cpu-performance.md @@ -207,7 +207,10 @@ measured while it existed. - Why the calibration's measured and patient plans "rarely won": they were rarely timed. On all three machines the thorough level's patient capture never finished within its share of the stage, and the timing in the step then ran out of rounds - before a verdict (no noise estimate), so every group fell back to the policy. + before a verdict (no noise estimate), so every group fell back to the policy. The + previous build did the same on the EPYC 7763 (thorough: no verdict; standard: no + transform planned within the stage's share), and so still does this one without + patient plans: the stage needs more of the levels' time than it gets. Timed alone, the measured threaded plans ran 0.53–1.05 of estimate's time (medians 0.74 on the EPYC 7763, 0.91 on the Xeon, about 0.95 on the M5 Max), and estimate's serial plans 1.06–2.8 of measure's. diff --git a/native/src/general.cpp b/native/src/general.cpp index a247d78a..b320a795 100644 --- a/native/src/general.cpp +++ b/native/src/general.cpp @@ -2611,8 +2611,9 @@ class GeneralWorkspace { // is 0 (solprop issue #584): one transform of every row, threaded by its shape, in // place of one a transform over only the modes' rows, which left most of the // threads idle: on 18 cores of an Intel Xeon Gold 6240 the pack ran rdw 0.90, - // argon_blueshift 0.88 and ctc 0.92 of their windows. A host keeps the coarse rows after the - // fine ones so that a deferred quadrature's stages transform the fine block alone. + // argon_blueshift 0.88 and ctc 0.92 of their windows. A host keeps the coarse rows + // after the fine ones so that a deferred quadrature's stages transform the fine block + // alone. bool packs_forward() const noexcept { #if !defined(KOKKOS_ENABLE_CUDA) && !defined(KOKKOS_ENABLE_HIP) // A compact problem keeps its separate transforms: its one row a transform runs diff --git a/native/tests/native/execution.hpp b/native/tests/native/execution.hpp index bf751b1c..e61a0224 100644 --- a/native/tests/native/execution.hpp +++ b/native/tests/native/execution.hpp @@ -18,10 +18,14 @@ inline CpuExecution make_table() { #if defined(__APPLE__) constexpr std::size_t stage = 8192; constexpr std::size_t fft = 32768; + constexpr std::size_t fft_thread = 32768; + constexpr std::size_t fft_long_row = 65536; constexpr bool stable = false; #else constexpr std::size_t stage = 4096; - constexpr std::size_t fft = 16384; + constexpr std::size_t fft = 8192; + constexpr std::size_t fft_thread = 4096; + constexpr std::size_t fft_long_row = 16384; constexpr bool stable = true; #endif const auto set = [&](CpuOperation operation, std::size_t threshold, @@ -31,7 +35,7 @@ inline CpuExecution make_table() { set(CpuOperation::stage, stage, stage); set(CpuOperation::response, 32768, 196608); set(CpuOperation::dense, 1048576, 1048576); - set(CpuOperation::fft, fft, 131072); + set(CpuOperation::fft, fft, 4194304); set(CpuOperation::raman, 65536, 262144); set(CpuOperation::copy, 65536, 65536); set(CpuOperation::flow, 32768, 32768); @@ -43,6 +47,8 @@ inline CpuExecution make_table() { table.compact_threads = 8; table.compact_max_fine_samples = 65536; table.stable_kernel_team = stable; + table.fft_thread_elements = fft_thread; + table.fft_long_row = fft_long_row; return table; } diff --git a/native/tests/native/test_core.cpp b/native/tests/native/test_core.cpp index 3040ff9f..2e2446c3 100644 --- a/native/tests/native/test_core.cpp +++ b/native/tests/native/test_core.cpp @@ -2989,11 +2989,42 @@ TEST(CpuThreadPolicy, AKernelPoolAtFullWidthKeepsItsTeam) { // The FFT pools keep their medium widths: FFTW's threaded loops run on the // runtime's whole team and only distribute fewer iterations. native::CpuExecutionPool transforms(native::CpuPool::fft, native::test::table()); - EXPECT_EQ(transforms.get(native::CpuOperation::fft, 1U << 20).concurrency(), cap); + EXPECT_EQ(transforms + .get(native::CpuOperation::fft, + native::test::table()[native::CpuOperation::fft].full) + .concurrency(), + cap); EXPECT_EQ(transforms.get(native::CpuOperation::fft, 32768).concurrency(), native::test::table().medium_threads); } +TEST(CpuThreadPolicy, TransformBatchesThreadByRowsAndRowLength) { + // solprop issue #587: from the start, a thread per fft_thread_elements samples, at + // most one a row, or up to the medium width for rows of fft_long_row samples. + if (!native::cpu_size_policy_enabled) + GTEST_SKIP() << "needs automatic per-operation sizing"; + native::CpuExecution execution = native::test::table(); + execution.grain[static_cast(native::CpuOperation::fft)] = {8192, + 1U << 22}; + execution.fft_thread_elements = 4096; + execution.fft_long_row = 16384; + execution.medium_threads = 4; + const native::ThreadPolicy policy{18, 18, 18, 18}; + const auto threads = [&](const std::size_t rows, const std::size_t length) { + return native::operation_threads(policy, execution, native::CpuPool::fft, + native::CpuOperation::fft, rows * length, length); + }; + EXPECT_EQ(threads(1, 4096), 1); // below the start + EXPECT_EQ(threads(4, 3072), 3); // three threads' worth of samples + EXPECT_EQ(threads(4, 10240), 4); // a thread a row + EXPECT_EQ(threads(1, 10240), 1); // a short row is not split + EXPECT_EQ(threads(1, 32768), 4); // a long row, split to the medium width + EXPECT_EQ(threads(12, 10240), 12); // one a row + EXPECT_EQ(threads(128, 1152), 18); // many short rows take the pool + EXPECT_EQ(threads(289, 128), 9); // 36992 samples + EXPECT_EQ(threads(1, 1U << 22), 18); // the full size is a backstop +} + TEST(CpuThreadPolicy, ACompactPoolKeepsAFewThreads) { // A compact problem's kernels share a team of at most cpu_compact_threads, medium // widths promoted to it except on macOS; its transforms take that team only where @@ -3105,7 +3136,8 @@ TEST(CpuThreadPolicy, MatrixPackingDoesNotInheritTheDenseWorkWidth) { TEST(CpuThreadPolicy, WorkspacesKeepIndependentChoicesAndReleaseTheirPools) { auto problem = general_problem(native::FieldFamily::envelope); native::GeneralState small(problem); - problem.n_to = native::test::table()[native::CpuOperation::fft].full; + // One long row, past the compact bound: its transforms thread by shape. + problem.n_to = 65536; problem.time_window.assign(problem.n_to, 1.0); problem.time_window_derivative.assign(problem.n_to, 0.0); problem.carrier.assign(problem.n_to, Complex(1.0, 0.0)); @@ -3113,10 +3145,13 @@ TEST(CpuThreadPolicy, WorkspacesKeepIndependentChoicesAndReleaseTheirPools) { native::GeneralState another_small(general_problem(native::FieldFamily::envelope)); const auto small_sizing = small.execution_sizing(); EXPECT_EQ(small_sizing, another_small.execution_sizing()); - const auto key = "fft.fft." + std::to_string(problem.n_to); + const auto key = "fft.fft." + std::to_string(problem.n_to) + ".samples." + + std::to_string(problem.n_to); EXPECT_EQ(large.execution_sizing().at(key).at("elements"), problem.n_to); EXPECT_EQ(large.execution_sizing().at(key).at("threads"), - static_cast(native::ExecutionSpace().concurrency())); + static_cast(native::operation_threads( + native::thread_policy(), native::test::table(), native::CpuPool::fft, + native::CpuOperation::fft, problem.n_to, problem.n_to))); large.close(); EXPECT_EQ(small.execution_sizing(), small_sizing); small.close(); From e27d9b5a5e670de83078a622bd873e63cffd4b8d Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 16:01:08 +0000 Subject: [PATCH 11/15] Calibration: prune planning candidates no faster alone; levels of 30 s, 2 min and 7 min (#595) Co-Authored-By: Claude Opus 5.5 --- docs/guide/performance/calibration.md | 14 +++++----- src/solprop/calibration/__init__.py | 4 +-- src/solprop/calibration/command.py | 4 +-- src/solprop/calibration/controller.py | 36 ++++++++++++++++++++----- src/solprop/calibration/first_use.py | 2 +- src/solprop/exceptions.py | 2 +- tests/calibration/test_controller.py | 39 +++++++++++++++++++++++++++ 7 files changed, 81 insertions(+), 20 deletions(-) diff --git a/docs/guide/performance/calibration.md b/docs/guide/performance/calibration.md index ce2afe40..108e2bd1 100644 --- a/docs/guide/performance/calibration.md +++ b/docs/guide/performance/calibration.md @@ -10,9 +10,9 @@ changes a result beyond rounding: it chooses among exact algorithms and thread c as the built-in defaults do. ```sh -solprop calibrate # standard: about a minute and a half -solprop calibrate --quick # about 20 s -solprop calibrate --thorough # about five minutes +solprop calibrate # standard: about two minutes +solprop calibrate --quick # about 30 s +solprop calibrate --thorough # about seven minutes solprop calibrate --show # the profile this job would use, and its evidence ``` @@ -62,9 +62,9 @@ the calibration says so and publishes nothing. | level | time | waiting and team | grain | FFT planning | rounds | |---|---|---|---|---|---| -| `--quick` | about 20 s | the wait only | medium width and compact team | plan policy | 3 | -| standard | about 90 s | yes, two narrower teams | thresholds of the main operations | `measure`, 0.25 s a plan | 5 | -| `--thorough` | about 5 min | yes | thresholds and full sizes of every operation | `measure`, 20 s a plan | 7 | +| `--quick` | about 30 s | the wait only | medium width and compact team | plan policy | 3 | +| standard | about 2 min | yes, two narrower teams | thresholds of the main operations | `measure`, 0.25 s a plan | 5 | +| `--thorough` | about 7 min | yes | thresholds and full sizes of every operation | `measure`, 20 s a plan | 7 | The standard and thorough levels also calibrate the smaller allocations they can pin children to inside the job, such as one socket, one NUMA node or one L3 cache group, @@ -100,7 +100,7 @@ run prints a notice like ```text CalibrationNotice: solprop is calibrating the native backend to this machine (quick, -about 20 s on 18 cores; once per machine, allocation and release). The profile is kept +about 30 s on 18 cores; once per machine, allocation and release). The profile is kept in /home/me/.cache/solprop. Run `solprop calibrate` for a fuller calibration, or set SOLPROP_CALIBRATION=use to never measure, or =off to ignore profiles. ``` diff --git a/src/solprop/calibration/__init__.py b/src/solprop/calibration/__init__.py index 2d786d83..c84d35dd 100644 --- a/src/solprop/calibration/__init__.py +++ b/src/solprop/calibration/__init__.py @@ -102,8 +102,8 @@ def calibrate( Parameters ---------- level : str, optional - ``"quick"`` (about 20 s), ``"standard"`` (about 90 s) or ``"thorough"`` - (about five minutes). + ``"quick"`` (about 30 s), ``"standard"`` (about two minutes) or + ``"thorough"`` (about seven minutes). budget : float or None, optional The deadline in seconds, the level's when ``None``. diff --git a/src/solprop/calibration/command.py b/src/solprop/calibration/command.py index 0aa01379..636480f5 100644 --- a/src/solprop/calibration/command.py +++ b/src/solprop/calibration/command.py @@ -38,14 +38,14 @@ def add_arguments(parser: argparse.ArgumentParser) -> None: dest="level", action="store_const", const="quick", - help="about 20 s: the waiting policy and the team at reduced depth", + help="about 30 s: the waiting policy and the team at reduced depth", ) action.add_argument( "--thorough", dest="level", action="store_const", const="thorough", - help="about five minutes: more rounds and longer windows", + help="about seven minutes: more rounds and longer windows", ) action.add_argument( "--recipe", diff --git a/src/solprop/calibration/controller.py b/src/solprop/calibration/controller.py index 40386531..4a1b6b8a 100644 --- a/src/solprop/calibration/controller.py +++ b/src/solprop/calibration/controller.py @@ -27,6 +27,7 @@ import os import queue import random +import statistics import subprocess import sys import threading @@ -124,12 +125,18 @@ class Level: # Quick: the wait, the team and the coarse grain (medium width, compact team). # Quick has no production-sized problem, so it tries no narrower team. "quick": Level( - "quick", 20.0, 3, 0.08, ("gnlse", "averaged", "line"), 0, 4, planning=0.0 + "quick", 30.0, 3, 0.08, ("gnlse", "averaged", "line"), 0, 4, planning=0.0 ), - "standard": Level("standard", 90.0, 5, 0.08, _ALL + _LARGE, 2, 10, reserve=0.15), - "thorough": Level("thorough", 300.0, 7, 0.3, _ALL + _LARGE, 2, 40, fulls=True), + "standard": Level("standard", 120.0, 5, 0.08, _ALL + _LARGE, 2, 10, reserve=0.15), + "thorough": Level("thorough", 420.0, 7, 0.3, _ALL + _LARGE, 2, 40, fulls=True), } -"""The levels: about 20 s, a minute and a half and five minutes (SEP 46 section 6.9).""" +"""The levels: about 30 s, two minutes and seven minutes. + +SEP 46 section 6.9 set 20 s, 90 s and 5 min. With those, the FFT stage's timing in the +step ran out of rounds at the standard and thorough levels on an Intel Xeon Gold 6240, +16 cores of an AMD EPYC 7763 and an Apple M5 Max, and the standard level's capture ran +out of its share (solprop issue #595); the levels now give it the time. +""" ORDER: dict[str, int] = {"quick": 0, "standard": 1, "thorough": 2} """Levels by depth: a higher level replaces a lower one for the same allocation.""" @@ -1455,6 +1462,10 @@ def seconds(key: str, effort: str) -> float | None: return ratios +_ALONE_BAR = 0.98 +"""The median ratio, alone, of a candidate's plans to the policy's at or above which +the candidate is not timed in the step.""" + _CAPTURE_SHARE = 0.5 """The share of the FFT stage's time left that each capture may take.""" @@ -1540,12 +1551,23 @@ def capture(effort: str, share: float) -> dict[str, object]: candidates = _planning_candidates(measured) groups = cast(dict[str, str], measured["groups"]) contains = cast(dict[str, list[str]], measured["problems"]) + alone = _alone_ratios(candidates, groups, measured) + # A candidate whose plans are no faster alone is not timed in the step: the step's + # timing is what the stage runs short of (issue #595), and estimate's serial rows + # and transverse planes were 1.06-3.8 times slower alone on every machine timed. + pruned = { + name: statistics.median(ratios.values()) + for name, ratios in alone.items() + if ratios and statistics.median(ratios.values()) >= _ALONE_BAR + } + candidates = [item for item in candidates if item[0] not in pruned] evidence: dict[str, object] = { "plans": len(groups), "skipped": list(cast(list[str], measured.get("skipped", []))), "planning_seconds": measured["seconds"], "candidates": [name for name, _, _ in candidates], - "alone": _alone_ratios(candidates, groups, measured), + "alone": alone, + "pruned": pruned, } timed = [ ( @@ -1811,8 +1833,8 @@ def calibrate( Parameters ---------- level : str, optional - ``"quick"`` (about 20 s), ``"standard"`` (about 90 s) or ``"thorough"`` - (about five minutes). + ``"quick"`` (about 30 s), ``"standard"`` (about two minutes) or + ``"thorough"`` (about seven minutes). budget : float or None, optional The deadline in seconds, the level's when ``None``. environ : Mapping[str, str] or None, optional diff --git a/src/solprop/calibration/first_use.py b/src/solprop/calibration/first_use.py index c8ea3ca4..e01a9fb7 100644 --- a/src/solprop/calibration/first_use.py +++ b/src/solprop/calibration/first_use.py @@ -6,7 +6,7 @@ Under ``SOLPROP_CALIBRATION=auto``, the default, the first native workspace built in an allocation with no row in its machine's profile first runs the quick calibration, -about 20 s, after a :class:`~solprop.exceptions.CalibrationNotice`. A process that +about 30 s, after a :class:`~solprop.exceptions.CalibrationNotice`. A process that had not loaded the native extension yet then takes the whole new profile when it does; one that had applies the new workspace settings that hold under the process settings it started with, and the rest from its next process. Importing the native backend diff --git a/src/solprop/exceptions.py b/src/solprop/exceptions.py index 2352522d..fde38aa6 100644 --- a/src/solprop/exceptions.py +++ b/src/solprop/exceptions.py @@ -393,7 +393,7 @@ class CalibrationNotice(SolpropWarning): """The native backend is calibrating itself to this machine (SEP 46 section 6.10). Issued once, before a first native run with no profile for its machine, build - and allocation runs the quick calibration (about 20 s), saying where the result + and allocation runs the quick calibration (about 30 s), saying where the result goes and how to choose another level or turn it off. Silence it with the standard warnings filters, or with ``SOLPROP_CALIBRATION=use``. """ diff --git a/tests/calibration/test_controller.py b/tests/calibration/test_controller.py index 3e9e12fb..47d4ffe0 100644 --- a/tests/calibration/test_controller.py +++ b/tests/calibration/test_controller.py @@ -845,6 +845,45 @@ def test_thorough_tries_no_patient_plans_and_records_plans_alone( assert set(evidence["alone"]) == set(evidence["candidates"]) +def test_a_candidate_no_faster_alone_is_not_timed_in_the_step( + scripted: tuple[type[_Scripted], dict[str, str]], monkeypatch: pytest.MonkeyPatch +) -> None: + # Issue #595: estimate's serial rows slower alone than measure's are pruned, and + # the step times the threaded rows' measured plans alone. + script, environ = scripted + original = script.receive + timed: list[dict[str, str]] = [] + + def receive(self: _Scripted, timeout: float) -> object: + reply = original(self, timeout) + if self.pending.get("op") == "capture": + assert isinstance(reply, dict) + reply["alone"] = { + "r2c:4096:1:4096:2049:out:0:0:t1": {"plan": 1.0, "estimate": 2.0}, + "c2c-forward:4096:4:4096:4096:out:0:0:t8": { + "plan": 0.8, + "estimate": 1.0, + }, + } + elif self.pending.get("op") == "time" and self.pending.get("planning"): + timed.append( + dict(_efforts(cast(Mapping[str, object], self.pending["planning"]))) + ) + return reply + + monkeypatch.setattr(script, "receive", receive) + calibrate("standard", environ=environ, cpus=range(8)) + (row,) = _rows(environ) + evidence = cast(dict[str, Any], row["evidence"])["planning"] + assert evidence["candidates"] == ["threaded_rows measure"] + assert evidence["pruned"] == {"rows estimate": 2.0} + assert evidence["alone"]["threaded_rows measure"] == { + "c2c-forward:4096:4:4096:4096:out:0:0:t8": 0.8 + } + assert timed + assert all(efforts.get("rows") != "estimate" for efforts in timed) + + def test_quick_leaves_the_plan_policy_and_a_failed_capture_keeps_it( scripted: tuple[type[_Scripted], dict[str, str]], ) -> None: From 7fed8b5ed13258d365494ff5c34761af86f2bb9c Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 16:15:06 +0000 Subject: [PATCH 12/15] Tests read the transforms' sizing keys with their row lengths; the #595 record Co-Authored-By: Claude Opus 5.5 --- docs/dev/cpu-performance.md | 10 +++++++--- docs/dev/decisions.md | 7 ++++--- native/tests/test_raman_tail.py | 12 ++++++++---- tests/backends/test_native_forced_threads.py | 7 +++++-- 4 files changed, 24 insertions(+), 12 deletions(-) diff --git a/docs/dev/cpu-performance.md b/docs/dev/cpu-performance.md index df4e3006..b19536a9 100644 --- a/docs/dev/cpu-performance.md +++ b/docs/dev/cpu-performance.md @@ -214,9 +214,13 @@ measured while it existed. Timed alone, the measured threaded plans ran 0.53–1.05 of estimate's time (medians 0.74 on the EPYC 7763, 0.91 on the Xeon, about 0.95 on the M5 Max), and estimate's serial plans 1.06–2.8 of measure's. - - The thorough level no longer plans at patient, which leaves the time to its - timing in the step; each candidate's plans timed alone are kept beside its verdict - (`evidence.planning.alone`). + - The thorough level no longer plans at patient; a candidate whose plans are no + faster alone (median ratio 0.98 or more) is not timed in the step; and the levels + take 30 s, 2 min and 7 min (from 20 s, 90 s and 5 min). On the EPYC 7763 the stage + then reached a verdict: estimate's serial rows (2.11 alone) and planes (1.26) were + pruned, and `threaded_rows measure` scored 0.99 on the calibration's problems but + tied a round, so the policy held. Each candidate's plans timed alone are kept beside + its verdict (`evidence.planning.alone`, `pruned`). **Tried and not adopted** - **`FFTW_PATIENT` planning.** It adds seconds of construction and gave no measured diff --git a/docs/dev/decisions.md b/docs/dev/decisions.md index 20c0a011..2f401b39 100644 --- a/docs/dev/decisions.md +++ b/docs/dev/decisions.md @@ -2153,9 +2153,10 @@ Withdrawn (development-process record, not a design decision). `SOLPROP_NATIVE_FORWARD_PACK=0` turns the pack off. - *Planning.* Threaded rows stay at estimate by default; the thorough calibration no longer plans at patient, and records each candidate's plans timed alone beside its - verdict in the step. The FFT stage's timing in the step still runs out of rounds at - the standard and thorough levels on the machines measured, as it did before this - change; giving it time is the maintainer's call on the levels' budgets. + verdict in the step, and times in the step only the candidates faster alone. The + levels take 30 s, 2 min and 7 min (maintainer, 2026-11-09): with 20 s, 90 s and + 5 min the FFT stage ran out of rounds at the standard and thorough levels on every + machine measured, before this change too. - **Consequences:** against the previous build, the tracking set ran 0.977 (Xeon Gold 6240) and 0.981 (EPYC 7763) on geometric mean: `argon_blueshift` 0.80–0.87, `ctc` 0.89–0.92, `rdw` 0.91–0.95, `line` 0.87–0.93, the rest within their spread; fields agree diff --git a/native/tests/test_raman_tail.py b/native/tests/test_raman_tail.py index dd2cc143..9da66e12 100644 --- a/native/tests/test_raman_tail.py +++ b/native/tests/test_raman_tail.py @@ -115,12 +115,16 @@ def test_raman_partial_chunk_matches_reference_and_repeats( full = info.raman_fft_threads tail = info.raman_tail_fft_threads kernel = full + # Each Raman batch's key names its row length, a doubled time grid. + row = f".samples.{2 * grid.n_to}" if info.cpu_size_policy_enabled: - full = sizing[f"raman_fft.raman.{chunk * 2 * grid.n_to}"]["threads"] - kernel = sizing[f"raman_fft.raman.{2 * grid.n_to}"]["threads"] + full = sizing[f"raman_fft.raman.{chunk * 2 * grid.n_to}{row}"][ + "threads" + ] + kernel = sizing[f"raman_fft.raman.{2 * grid.n_to}{row}"]["threads"] if 7 % chunk: tail = sizing[ - f"raman_tail_fft.raman.{(7 % chunk) * 2 * grid.n_to}" + f"raman_tail_fft.raman.{(7 % chunk) * 2 * grid.n_to}{row}" ]["threads"] accelerated = _accelerated(2 * grid.n_to, full) tail_accelerated = _accelerated(2 * grid.n_to, tail) @@ -134,7 +138,7 @@ def test_raman_partial_chunk_matches_reference_and_repeats( if info.device_backend == "none": products = info.raman_tail_fft_threads if info.cpu_size_policy_enabled: - products = sizing[f"raman_tail_fft.raman.{2 * grid.n_to}"][ + products = sizing[f"raman_tail_fft.raman.{2 * grid.n_to}{row}"][ "threads" ] if _accelerated(2 * grid.n_to, products): diff --git a/tests/backends/test_native_forced_threads.py b/tests/backends/test_native_forced_threads.py index c7248ad8..df421406 100644 --- a/tests/backends/test_native_forced_threads.py +++ b/tests/backends/test_native_forced_threads.py @@ -385,7 +385,8 @@ def test_the_transverse_pool_threads_its_transform_and_agrees(tmp_path, threads) """ problem, _ = wide_transverse_problem() samples = math.prod(problem.shape) - key = f"fft.fft.{samples}" + # The transverse planes' batch: each plane of n_y x n_x is one row of the sizing. + key = f"fft.fft.{samples}.samples.{problem.shape[-1] * problem.shape[-2]}" bound = parity_bound(problem, transverse=problem.transform.n_nodes) one = forced_run("wide_transverse", 1, tmp_path, pools={}) several = forced_run("wide_transverse", threads, tmp_path, pools={}) @@ -404,7 +405,9 @@ def test_the_transverse_pool_varied_alone_agrees(tmp_path, march): #322 recommends beside it, the default one, on its own rounding budget. """ problem, _ = wide_transverse_problem(march) - key = f"fft.fft.{math.prod(problem.shape)}" + # The transverse planes' batch: each plane of n_y x n_x is one row of the sizing. + plane = problem.shape[-1] * problem.shape[-2] + key = f"fft.fft.{math.prod(problem.shape)}.samples.{plane}" bound = ( parity_bound(problem, transverse=problem.transform.n_nodes) if march == "march" From 5adbd5039e88018b6bb8e60bd13750169fc2a951 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 16:57:24 +0000 Subject: [PATCH 13/15] Plans timed alone inside the native wait scope; release note; no unreachable branches Co-Authored-By: Claude Opus 5.5 --- changelog.d/598.changed.md | 1 + .../sep-0046-machine-calibration.added.md | 2 +- docs/dev/cpu-performance.md | 5 ++++- native/include/solprop_native/fft_plan.hpp | 7 ++++++- src/solprop/calibration/child.py | 17 +++++++---------- src/solprop/calibration/grain.py | 19 ++++++++----------- 6 files changed, 27 insertions(+), 24 deletions(-) create mode 100644 changelog.d/598.changed.md diff --git a/changelog.d/598.changed.md b/changelog.d/598.changed.md new file mode 100644 index 00000000..3f4cd47c --- /dev/null +++ b/changelog.d/598.changed.md @@ -0,0 +1 @@ +The native backend's CPU transforms are faster on many threads ([#598](https://github.com/LupoLab/solprop/pull/598); [ADR-075](docs/dev/decisions.md)): a batch of transforms now threads by its shape, about one thread per row and the whole team only for many rows, instead of by its element count, and a multi-mode stage transforms its forward rows (projection, absorber, plasma currents) as one batch, as on a GPU (`SOLPROP_NATIVE_FORWARD_PACK=0` turns that off). The tracking cases ran 0.98 of their time on geometric mean on 18–36 cores of an Intel Xeon Gold 6240 and 16 of an AMD EPYC 7763, the few-mode capillaries and the one-dimensional free-space beam 0.80–0.95. `solprop calibrate` now also tunes the transforms' threading, reaches a decision on FFT plan efforts (it no longer tries `patient` plans, and skips efforts whose plans are slower alone), and takes about 30 s, two minutes or seven minutes (from 20 s, 90 s and five minutes). Existing calibration profiles no longer apply: the next native run on each machine calibrates quickly again by itself, or run `solprop calibrate`. diff --git a/changelog.d/sep-0046-machine-calibration.added.md b/changelog.d/sep-0046-machine-calibration.added.md index 5dc99fab..8ef82868 100644 --- a/changelog.d/sep-0046-machine-calibration.added.md +++ b/changelog.d/sep-0046-machine-calibration.added.md @@ -1 +1 @@ -`solprop calibrate` fits the native backend to the machine it runs on ([SEP 46](docs/dev/seps/archive/sep-0046-machine-calibration.md); [Calibrating solprop to your machine](docs/guide/performance/calibration.md)): in about 20 s (`--quick`), a minute and a half, or five minutes (`--thorough`), child processes pinned to the job's CPUs measure how idle threads should wait, how wide the team should be, where each kind of operation should start to thread and which FFTW plan effort runs fastest inside the step, and a setting replaces the built-in value only if it wins every round by more than the measured noise, slows none of the problems (production-sized ones among them) and validates with the other winners. The profile is kept per user in the platform's cache directory (`SOLPROP_CACHE_DIR` moves it; `SOLPROP_CALIBRATION_PATH` adds shared read-only directories, such as a cluster's per node type), keyed by machine, build and allocation, and every later run applies it and records the settings it applied in its metadata (`calibration`). The first native run with no profile for its machine, build and allocation calibrates quickly by itself after a `CalibrationNotice`, and a scan calibrates its workers' allocation first (unless they run one thread each, which has nothing to choose); `SOLPROP_CALIBRATION=use` never measures and `off` ignores profiles. `Simulation.calibrate()` and `solprop calibrate --recipe` refine the settings for one simulation, reused by every run and scan point of the same execution. `--show`, `--export`, `--import` and `--clear` manage profiles. Benchmark series calibrate each build first and record the profile per row. Without a profile, every run uses exactly the former settings, now a built-in table keyed by operating system; `BuildInfo`'s threshold fields report that table for the process. +`solprop calibrate` fits the native backend to the machine it runs on ([SEP 46](docs/dev/seps/archive/sep-0046-machine-calibration.md); [Calibrating solprop to your machine](docs/guide/performance/calibration.md)): in about 30 s (`--quick`), two minutes, or seven minutes (`--thorough`), child processes pinned to the job's CPUs measure how idle threads should wait, how wide the team should be, where each kind of operation should start to thread and which FFTW plan effort runs fastest inside the step, and a setting replaces the built-in value only if it wins every round by more than the measured noise, slows none of the problems (production-sized ones among them) and validates with the other winners. The profile is kept per user in the platform's cache directory (`SOLPROP_CACHE_DIR` moves it; `SOLPROP_CALIBRATION_PATH` adds shared read-only directories, such as a cluster's per node type), keyed by machine, build and allocation, and every later run applies it and records the settings it applied in its metadata (`calibration`). The first native run with no profile for its machine, build and allocation calibrates quickly by itself after a `CalibrationNotice`, and a scan calibrates its workers' allocation first (unless they run one thread each, which has nothing to choose); `SOLPROP_CALIBRATION=use` never measures and `off` ignores profiles. `Simulation.calibrate()` and `solprop calibrate --recipe` refine the settings for one simulation, reused by every run and scan point of the same execution. `--show`, `--export`, `--import` and `--clear` manage profiles. Benchmark series calibrate each build first and record the profile per row. Without a profile, every run uses exactly the former settings, now a built-in table keyed by operating system; `BuildInfo`'s threshold fields report that table for the process. diff --git a/docs/dev/cpu-performance.md b/docs/dev/cpu-performance.md index b19536a9..b127df9c 100644 --- a/docs/dev/cpu-performance.md +++ b/docs/dev/cpu-performance.md @@ -213,7 +213,10 @@ measured while it existed. patient plans: the stage needs more of the levels' time than it gets. Timed alone, the measured threaded plans ran 0.53–1.05 of estimate's time (medians 0.74 on the EPYC 7763, 0.91 on the Xeon, about 0.95 on the M5 Max), and - estimate's serial plans 1.06–2.8 of measure's. + estimate's serial plans 1.06–2.8 of measure's. Those timings ran outside the native + wait scope, so LLVM's runtime woke its idle threads at each loop; the timing alone + now runs inside it, as a step's transforms do (on the M5 Max a 4 × 10240 plan took + 11 µs there against 22 µs outside). - The thorough level no longer plans at patient; a candidate whose plans are no faster alone (median ratio 0.98 or more) is not timed in the step; and the levels take 30 s, 2 min and 7 min (from 20 s, 90 s and 5 min). On the EPYC 7763 the stage diff --git a/native/include/solprop_native/fft_plan.hpp b/native/include/solprop_native/fft_plan.hpp index 2f0081c4..263e5a10 100644 --- a/native/include/solprop_native/fft_plan.hpp +++ b/native/include/solprop_native/fft_plan.hpp @@ -252,7 +252,9 @@ void seed_row_fft_wisdom(const Execution& execution, const Input& input, `effort` with `FFTW_WISDOM_ONLY` (`plan`; NaN when the wisdom no longer builds it), and of the same transform at FFTW_ESTIMATE (`estimate`), each the median of five batches after a warm execution, on the key's threads and on arrays of its - alignment. FFTW's wisdom is forgotten before each plan and after. */ + alignment, inside a native scope (`CpuTeamScope`) as a step's transforms run, so + that idle threads wait as they do there. FFTW's wisdom is forgotten before each + plan and after. */ struct PlanAlone { double plan{}, estimate{}; }; @@ -303,6 +305,9 @@ inline PlanAlone time_fft_plan(const std::string& key, const std::string& wisdom const auto input_offset = static_cast(std::stoi(parts[6])); const auto output_offset = static_cast(std::stoi(parts[7])); const int threads = std::stoi(parts[8].substr(1)); + // The step's waiting policy and FFTW team: without the scope LLVM's runtime sleeps + // its idle threads at once (`KMP_BLOCKTIME=0`) and each loop pays their wake-up. + const CpuTeamScope scope(threads); const bool r2c = kind == "r2c", c2r = kind == "c2r"; const bool planes = kind.starts_with("planes-"); const int sign = kind.ends_with("backward") ? FFTW_BACKWARD : FFTW_FORWARD; diff --git a/src/solprop/calibration/child.py b/src/solprop/calibration/child.py index 6f2c1908..8b0e849c 100644 --- a/src/solprop/calibration/child.py +++ b/src/solprop/calibration/child.py @@ -289,7 +289,6 @@ def capture( wisdom: dict[str, str] = {} groups: dict[str, str] = {} contains: dict[str, list[str]] = {} - alone: dict[str, dict[str, float]] = {} skipped: list[str] = [] begun = time.perf_counter() names = list(self.problems) @@ -321,18 +320,16 @@ def capture( found = backend._native_state.fft_planning # pyright: ignore[reportPrivateUsage] finally: backend.close() - captured = cast(dict[str, str], found["captured"]) - wisdom |= captured + wisdom |= cast(dict[str, str], found["captured"]) for key, group in cast(dict[str, str], found["groups"]).items(): groups[key] = group contains.setdefault(name, []).append(key) - # Each plan alone, as its capture chose it and at estimate, to tell a plan - # that loses in the step from one that is slower alone (issue #595). - for key, text in captured.items(): - if key not in alone and text: - alone[key] = cast( - dict[str, float], _core.time_fft_plan(key, text, effort) - ) + # Each plan alone, as its capture chose it and at estimate, to tell a plan that + # loses in the step from one that is slower alone (issue #595). + alone = { + key: cast(dict[str, float], _core.time_fft_plan(key, text, effort)) + for key, text in wisdom.items() + } return { "wisdom": wisdom, "groups": groups, diff --git a/src/solprop/calibration/grain.py b/src/solprop/calibration/grain.py index a5042920..4f44b9bf 100644 --- a/src/solprop/calibration/grain.py +++ b/src/solprop/calibration/grain.py @@ -386,18 +386,15 @@ def _shape_moves(record: Record) -> list[Candidate]: Returns ------- list of Candidate - Up to four; none when the record has no shape settings. + Two at most, smallest first: a grain of one thread per sample is not halved. """ - moves: list[Candidate] = [] - for name in SHAPE_MOVES: - if name not in record: - continue - value = int(record[name]) - for moved in (value // 2, value * 2): - if moved >= 1 and moved != value: - short = name.removeprefix("fft_").replace("_", " ") - moves.append(Candidate("fft", f"fft {short} {moved}", {name: moved})) - return moves + (name,) = SHAPE_MOVES + value = int(record[name]) + short = name.removeprefix("fft_").replace("_", " ") + return [ + Candidate("fft", f"fft {short} {moved}", {name: moved}) + for moved in sorted({max(1, value // 2), 2 * value} - {value}) + ] def best( From 308ddc003ef4dd4702842634988ef98deae991fa Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 17:11:57 +0000 Subject: [PATCH 14/15] The #595 record: plan timings alone inside the native scope Co-Authored-By: Claude Opus 5.5 --- docs/dev/cpu-performance.md | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/dev/cpu-performance.md b/docs/dev/cpu-performance.md index b127df9c..165beb04 100644 --- a/docs/dev/cpu-performance.md +++ b/docs/dev/cpu-performance.md @@ -216,7 +216,11 @@ measured while it existed. estimate's serial plans 1.06–2.8 of measure's. Those timings ran outside the native wait scope, so LLVM's runtime woke its idle threads at each loop; the timing alone now runs inside it, as a step's transforms do (on the M5 Max a 4 × 10240 plan took - 11 µs there against 22 µs outside). + 11 µs there against 22 µs outside). Inside it, on the EPYC 7763: measured threaded + plans 0.72–0.92 of estimate's time (medians by level and allocation), estimate's + serial rows 2.1–3.0 of measure's and its planes 1.05–1.34, so both are pruned. In + the step `threaded_rows measure` won every round on an 8-core allocation (0.915) + and lost one on 16 cores (1.039). - The thorough level no longer plans at patient; a candidate whose plans are no faster alone (median ratio 0.98 or more) is not timed in the step; and the levels take 30 s, 2 min and 7 min (from 20 s, 90 s and 5 min). On the EPYC 7763 the stage From ef7ee5a41b67c80dff6084b9782f7ffb4ef3c3e2 Mon Sep 17 00:00:00 2001 From: John Travers Date: Mon, 9 Nov 2026 18:07:51 +0000 Subject: [PATCH 15/15] Cover a grain class whose sizes all run serially Co-Authored-By: Claude Opus 5.5 --- tests/calibration/test_grain.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/calibration/test_grain.py b/tests/calibration/test_grain.py index 024d3df2..8a84e773 100644 --- a/tests/calibration/test_grain.py +++ b/tests/calibration/test_grain.py @@ -83,6 +83,11 @@ def test_each_class_moves_one_step_past_its_nearest_sizes() -> None: "stage threshold 5001", ] assert len(proposals(record, problems, 18, limit=3)) == 3 + # A class whose sizes all run serially can only thread one more of them. + serial = {"a": [Operation("kernels", "stage", 500)]} + assert [c.name for c in proposals(record, serial, 18) if c.setting == "stage"] == [ + "stage threshold 500" + ] def test_the_medium_width_and_the_compact_team_have_short_ladders() -> None: