diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a68ceb..cb69c96 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,17 @@ if(BLASTGPU_BUILD_SOLVER) endif() if(BLASTGPU_BUILD_TESTS AND BLASTGPU_GPU_BACKEND STREQUAL "NONE") find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_executable(test_fluxregister tests/test_fluxregister.cpp) + target_link_libraries(test_fluxregister PRIVATE AMReX::amrex) + add_test(NAME fluxregister COMMAND test_fluxregister) + add_test(NAME m1b-p1-ml-u0-smoke COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/tests/m1b_p0_check.py --uniform + $ ${CMAKE_SOURCE_DIR}/inputs/m1b_p1_ml_u0.inp + "amr.n_cell=8 8 8" amr.max_grid_size=8 max_step=2 stop_time=1) + add_test(NAME m1b-p1-ml-sx-smoke COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/tests/m1b_p0_check.py + $ ${CMAKE_SOURCE_DIR}/inputs/m1b_p1_ml_sx.inp + "amr.n_cell=32 8 8" amr.max_grid_size=32 max_step=2 stop_time=1) add_test(NAME m1b-u0-smoke COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/m1b_p0_check.py --uniform $ ${CMAKE_SOURCE_DIR}/inputs/m1b_p0_u0.inp diff --git a/docs/perf/m1b-p1-amrex-api-audit.md b/docs/perf/m1b-p1-amrex-api-audit.md new file mode 100644 index 0000000..cfba8bc --- /dev/null +++ b/docs/perf/m1b-p1-amrex-api-audit.md @@ -0,0 +1,23 @@ +# M1b-P1 local AMReX FluxRegister API audit + +Authority: AMReX submodule `2cf4fbcde02a700e01c62bebb4a730aeeb66c661`. + +| Operation | Local source | Selected contract | +|---|---|---| +| constructor/define | `Src/AmrCore/AMReX_FluxRegister.cpp:11-80` | Fine BoxArray and DistributionMapping, ratio 2, fine level 1, six register components. | +| `CrseInit` | `AMReX_FluxRegister.cpp:160-281` | Source/destination ranges are independently checked. `mult` and supplied area are multiplied once. COPY overwrites the oriented boundary register; ADD accumulates. P1 uses negative coarse scale. | +| `FineAdd` | `AMReX_FluxRegister.cpp:363-496` | Fine boundary fluxes are coarsened/summed by the refinement ratio and added with the supplied multiplier. P1 uses positive fine scale. | +| `Reflux` | `AMReX_FluxRegister.cpp:527-647` | Register components are copied to oriented coarse faces and applied to destination cells divided by coarse volume. P1 uses `scale=1`, register source 0, state destination 1, count 6. | +| example signs | `Tests/Amr/Advection_AmrCore/Source/AdvancePhiAtLevel.cpp:372-378` | Coarse `CrseInit` uses -1 and fine `FineAdd` uses +1. | +| subcycling order | `Tests/Amr/Advection_AmrCore/Source/AmrCoreAdv.cpp:680-706` | Fine recursive advances complete, then reflux, then average-down. | + +The overload without an explicit area creates a unit-area MultiFab internally; +therefore P1 passes the complete `dt*area*stage_weight` through `mult`. This is +an AMReX-internal logical temporary, not solver-owned persistent storage, and +must be checked in the steady underlying-allocation audit. Periodic handling +is supplied to register boundary copies during reflux. GPU paths use AMReX +launch machinery and contain stream synchronization in several internal copy +and reflux paths; these are measured, not bypassed. + +The focused manufactured CPU test is the executable authority for the exact +sign, scaling, component-offset, and corrected-cell behavior used by P1. diff --git a/docs/perf/m1b-p1-authorization.md b/docs/perf/m1b-p1-authorization.md new file mode 100644 index 0000000..bcd2c96 --- /dev/null +++ b/docs/perf/m1b-p1-authorization.md @@ -0,0 +1,10 @@ +# M1b-P1 authorization + +Authority: `BLASTGPU-M1B-P1-OWNER-AUTHORIZATION-TWO-LEVEL-MULTILEVEL-HYDRO-SSP-RK2-SUBCYCLING-SIX-COMPONENT-CONSERVATIVE-REFLUX-STATIC-REFINEMENT-AND-AMR-EFFICIENCY-BASELINE-2026-08-22`. + +P1 implements at most two hydro levels, ratio-two subcycling, time-interpolated +coarse/fine fills, six-component conservative reflux, separate alpha +coarse/fine treatment, static sphere/box tagging, composite validation, and a +matched uniform-fine/AMR baseline. C2, dynamic tagging, remap, gauges, EB, +MPI, reactions, manual streams, fusion, arena/grid tuning, blastCFD work, and +cross-solver claims remain excluded. diff --git a/docs/perf/m1b-p1-build-manifest.tsv b/docs/perf/m1b-p1-build-manifest.tsv new file mode 100644 index 0000000..8457e50 --- /dev/null +++ b/docs/perf/m1b-p1-build-manifest.tsv @@ -0,0 +1,4 @@ +build source compiler configuration architecture +cpu 318f485 GCC 13.3 Release C++20 MPI=OFF EB=OFF OpenMP=ON; sha256=61b208d630280cdf2a23f420d57ceb5d8abe6c1aa2bdf487d32d6c3a4453506c CPU +cuda-opt 318f485 NVCC 12.6.85 Release C++20 MPI=OFF EB=OFF fast-math/maxreg unchanged; sha256=c4e2bd9695ae0da691c56ac6861147f1251e12f4a0c237bd702562d6ef5ffdb8 CMAKE=75 AMReX=7.5 sm_75 +control-cuda 9817caa NVCC 12.6.85 Release C++20 MPI=OFF EB=OFF fast-math/maxreg unchanged CMAKE=75 AMReX=7.5 sm_75 diff --git a/docs/perf/m1b-p1-case-manifest.tsv b/docs/perf/m1b-p1-case-manifest.tsv new file mode 100644 index 0000000..b1a0339 --- /dev/null +++ b/docs/perf/m1b-p1-case-manifest.tsv @@ -0,0 +1,8 @@ +case base_grid levels ref_ratio coarse_steps fine_substeps output +ML-U0 32x32x32 2 2 100 200 final validation only +ML-SX 128x8x8 2 2 269 538 validation only +ML-AC 64x8x8 2 2 100 200 validation only +A64X2-R0 64x64x64 2 2 45 90 disabled in timing +U128-R0 128x128x128 1 - 90 - disabled in timing +A96X2-R0 96x96x96 2 2 68 136 disabled in timing +U192-R0 192x192x192 1 - 135 - disabled in timing diff --git a/docs/perf/m1b-p1-conservation-results.tsv b/docs/perf/m1b-p1-conservation-results.tsv new file mode 100644 index 0000000..4ccc7ea --- /dev/null +++ b/docs/perf/m1b-p1-conservation-results.tsv @@ -0,0 +1,4 @@ +case mode arho1_initial arho1_final rhoE_initial rhoE_final result +ML-SX reflux_on 0.002197265603 0.002197265603 0.005371093768 0.005371093768 PASS +ML-SX reflux_off 0.002197265603 0.002197920663 0.005371093768 0.005373901780 DIAGNOSTIC_MISMATCH +ML-R0 reflux_on 1.224999988 1.224999988 254368.6704 254368.6704 PASS diff --git a/docs/perf/m1b-p1-correctness-results.md b/docs/perf/m1b-p1-correctness-results.md new file mode 100644 index 0000000..397922d --- /dev/null +++ b/docs/perf/m1b-p1-correctness-results.md @@ -0,0 +1,29 @@ +# M1b-P1 correctness results + +All seven CPU CTests pass: EOS, M0 hydro, manufactured FluxRegister, +two P1 smoke cases, and the two P0 smoke cases. Canonical M1a remains valid on +three levels and its seven CPU/CUDA plotfile files are byte-identical. + +- ML-U0: 100 coarse steps and 200 fine substeps; alpha, momentum, and energy + spreads are exactly zero, the coarse partial-density spread is one ulp + (`2.22e-16`, below the frozen `5e-15` envelope), counters are zero, and + composite conservative totals are unchanged. +- ML-U0-REGRID: ten coarse steps with regrid interval two preserve the uniform + state and hierarchy with zero counters and no stale register state. +- ML-SX: 269 coarse steps to `t=0.2`; CPU and CUDA complete with zero counters. + Reflux-on preserves displayed phase masses and energy; reflux-off produces + clear drift. +- ML-AC: 100 coarse steps; alpha remains in `[0.2,0.8]`, pressure remains + constant within the frozen envelope, phase masses are unchanged, and no + ordinary alpha register call exists. +- ML-R0: base 64/refined-by-2 reaches `t=2e-4` in 45 coarse steps. Phase + masses and energy are unchanged at reported precision, momentum is at + roundoff, counters are zero, and axis symmetry is approximately `1.6e-15`. + +Composite AMR versus uniform-fine normalized L1 errors are `1.51e-6` (density), +`2.13e-6` (velocity), `2.18e-6` (pressure), and `2.33e-6` (energy) at +128-equivalent. At 192-equivalent they are `2.70e-7`, `3.91e-7`, `4.89e-7`, +and `5.00e-7`. All are well inside the committed tolerance authority. + +Final Compute Sanitizer results: memcheck 0 errors, synccheck 0 errors, +initcheck 0 errors, and racecheck 0 hazards/errors/warnings. diff --git a/docs/perf/m1b-p1-implementation.md b/docs/perf/m1b-p1-implementation.md new file mode 100644 index 0000000..9d0a52b --- /dev/null +++ b/docs/perf/m1b-p1-implementation.md @@ -0,0 +1,23 @@ +# M1b-P1 implementation + +P1 extends the P0 face-flux architecture to two hydro levels with refinement +ratio two. Each level owns persistent border, x/y/z flux, counter, and reflux +correction storage. Fine advances use two substeps per coarse step. Stage +fills request the fine stage time from the existing coarse old/new bracket, +so `FillPatchTwoLevels` performs time interpolation without retaining a stale +coarse stage-one state. + +Both SSP-RK2 face fields are accumulated with weight one half. Register +components 0--5 map to state components 1--6; alpha and contact speed are +excluded. Reflux precedes seven-component average-down. A persistent +correction MultiFab distinguishes interface corrections from roundoff before +updating the coarse state. The same roundoff distinction in the cell update +preserves uniform free streams across interpolation and summation boundaries. + +Static sphere tagging remains the default. A deterministic static box option +supports planar interface tests. Hydro mode asserts at most two levels; +identity mode and the canonical M1a input remain unchanged. + +Physical outflow ghosts use an explicit first-order clamped interior copy. +This avoids concurrent corner reads from uninitialized ghost cells and is +validated by Compute Sanitizer initcheck. diff --git a/docs/perf/m1b-p1-kernel-results.tsv b/docs/perf/m1b-p1-kernel-results.tsv new file mode 100644 index 0000000..f20354c --- /dev/null +++ b/docs/perf/m1b-p1-kernel-results.tsv @@ -0,0 +1,8 @@ +family instances_or_share interpretation +directional_flux 80.8_percent_kernel_time dominant hydro work +cell_update 2.8_percent_kernel_time combined seven-component update +coarse_fine_interp about_10_percent_kernel_time FillPatchTwoLevels interpolation +CFL_reduction 2.5_percent_kernel_time one host-visible dt selection per level advance +FineAdd 0.9_percent_kernel_time six-component fine register accumulation +Reflux 0.5_percent_kernel_time six-component coarse correction +steady_cudaMalloc_cudaFree 0 PASS diff --git a/docs/perf/m1b-p1-memory-results.tsv b/docs/perf/m1b-p1-memory-results.tsv new file mode 100644 index 0000000..7d17627 --- /dev/null +++ b/docs/perf/m1b-p1-memory-results.tsv @@ -0,0 +1,9 @@ +case arena_reserved_mb arena_max_used_mb matched_memory_efficiency +U128 5823 933 1.0 +A64X2 5823 333 2.8018 +U192 5823 3131 1.0 +A96X2 5823 960 3.2615 +P0-control-128 5823 837 1.0 +P1-single-level-128 5823 837 1.0 +P0-control-192 5823 2807 1.0 +P1-single-level-192 5823 2807 1.0 diff --git a/docs/perf/m1b-p1-numerical-results.tsv b/docs/perf/m1b-p1-numerical-results.tsv new file mode 100644 index 0000000..64161af --- /dev/null +++ b/docs/perf/m1b-p1-numerical-results.tsv @@ -0,0 +1,9 @@ +pair field normalized_L1 normalized_L2 normalized_Linf +A64X2_vs_U128 rho 1.506302e-6 3.300128e-6 1.549482e-5 +A64X2_vs_U128 u 2.134036e-6 7.085275e-6 5.791795e-5 +A64X2_vs_U128 p 2.177851e-6 5.263779e-6 3.006390e-5 +A64X2_vs_U128 rhoE 2.326179e-6 5.544679e-6 3.048631e-5 +A96X2_vs_U192 rho 2.701837e-7 6.206293e-7 3.989132e-6 +A96X2_vs_U192 u 3.908286e-7 1.507684e-6 1.815454e-5 +A96X2_vs_U192 p 4.894288e-7 1.219134e-6 7.635313e-6 +A96X2_vs_U192 rhoE 5.000204e-7 1.258989e-6 8.323991e-6 diff --git a/docs/perf/m1b-p1-performance-results.md b/docs/perf/m1b-p1-performance-results.md new file mode 100644 index 0000000..5e635d5 --- /dev/null +++ b/docs/perf/m1b-p1-performance-results.md @@ -0,0 +1,21 @@ +# M1b-P1 AMR efficiency baseline + +All authoritative measurements use the committed tolerance authority, the +same optimized SM 7.5 executable, no output, and seven fresh processes per +case. The refinement region was frozen before timing. + +| Pair | Uniform median (s) | AMR median (s) | Runtime efficiency | Uniform used MB | AMR used MB | Memory efficiency | CV uniform / AMR | +|---|---:|---:|---:|---:|---:|---:|---:| +| U128-R0 / A64X2-R0 | 28.0674 | 7.84385 | 3.578x | 933 | 333 | 2.802x | 0.650% / 1.816% | +| U192-R0 / A96X2-R0 | 142.284 | 31.5994 | 4.503x | 3131 | 960 | 3.261x | 0.183% / 0.417% | + +The default arena reservation is 5823 MB in every case and is not presented +as working storage. Maximum-used values establish the memory ratio. Both +pairs retain matched-profile accuracy and therefore meet the package's +transformative runtime and memory categories. + +The final single-level guardrail uses five alternating P0-control/P1-candidate +pairs. At 128 cubed the medians are 31.0393/31.2944 s (candidate +0.822%); +at 192 cubed they are 104.923/104.995 s (+0.068%). Arena maximum used is +identical at 837 MB and 2807 MB. Thus the max-level-zero path passes the 1% +runtime and memory gates. diff --git a/docs/perf/m1b-p1-profile-summary.md b/docs/perf/m1b-p1-profile-summary.md new file mode 100644 index 0000000..902b4ad --- /dev/null +++ b/docs/perf/m1b-p1-profile-summary.md @@ -0,0 +1,19 @@ +# M1b-P1 Nsight Systems summary + +The authoritative steady-window report is external at +`profiles/a64x2-final-steady-window.nsys-rep`, SHA-256 +`36012b4145a4630dbe9fe450de9da08b6251b9e6d829954f0d811e27818c098f`. +It captures a three-second output-free static-AMR interval. + +Directional Hydro3D flux kernels account for approximately 80.8% of kernel +time; updates account for 2.8%, coarse/fine interpolation kernels about 10%, +CFL reductions 2.5%, FineAdd 0.9%, and Reflux 0.5%. The trace contains no +steady `cudaMalloc` or `cudaFree`. `cudaStreamSynchronize` has high inclusive +time because it waits for GPU work, but its median host call is 0.86 microsecond +and the GPU remains flux-kernel dominated. The earlier C1 experiment already +removed a proven iterator synchronization family without large-grid benefit. +There is no measured >=5% batching opportunity, so C2 is not recommended. + +Atomics remain limited to diagnostics/reductions/framework internals; no +face-to-cell residual atomic exists. Reflux uses the original face arrays and +does not introduce a second flux path. diff --git a/docs/perf/m1b-p1-raw-artifact-manifest.tsv b/docs/perf/m1b-p1-raw-artifact-manifest.tsv new file mode 100644 index 0000000..2cf28c6 --- /dev/null +++ b/docs/perf/m1b-p1-raw-artifact-manifest.tsv @@ -0,0 +1,8 @@ +artifact location sha256_or_identity +evidence_root /home/tim/Downloads/blastgpu-perf-runs/m1b-p1-20260822T192505Z external +steady_nsys profiles/a64x2-final-steady-window.nsys-rep 36012b4145a4630dbe9fe450de9da08b6251b9e6d829954f0d811e27818c098f +tolerances docs/perf/m1b-p1-tolerances.json f0ae174788e371c00c4d783721ce5c03f5227e97c62c69e3e0832f9c1a7365a9 +U128_profile references/profiles/u128.tsv 00726a6f5964e6ed2e1088bf96c961c76fad78954cfad1cd8a11ecf8e27bad05 +A64X2_profile references/profiles/a64x2.tsv 20a3f794a6640033ad2a3357c82f8fe9a332393ed8e9286e40f78f58d6109f0f +U192_profile references/profiles/u192.tsv d50b96ad4fa9c3fb1f363fa58203dab3267fb44c545a00775d33053322b912b4 +A96X2_profile references/profiles/a96x2.tsv fdfbf3088975fdfa775293275b1ecbd7009029b0ce5ce84335204d516cc840b8 diff --git a/docs/perf/m1b-p1-reflux-design.md b/docs/perf/m1b-p1-reflux-design.md new file mode 100644 index 0000000..08c4bee --- /dev/null +++ b/docs/perf/m1b-p1-reflux-design.md @@ -0,0 +1,79 @@ +# M1b-P1 reflux design + +This document freezes the two-level reflux contract before solver integration. + +## Component classification + +The cell state is ordered as `alpha1, arho1, arho2, rhou, rhov, rhow, rhoE`. +Only components 1--6 are conservative variables handled by the standard +AMReX `FluxRegister`. The register therefore has six components; face source +component 1 maps to register component 0, and reflux writes register components +0--5 to cell-state components 1--6. + +The P0 alpha discretization is + +``` +rhs_alpha = -sum_d ((Falpha_hi - alpha_cell*Sstar_hi) + -(Falpha_lo - alpha_cell*Sstar_lo)) / dx_d. +``` + +It is a paired nonconservative flux/source operator. A mismatch in `Falpha` +alone is not its complete coarse/fine update, so component 0 is deliberately +excluded from ordinary conservative reflux. Fine alpha is advanced with the +same paired operator, filled through normal coarse/fine FillPatch, and averaged +onto covered coarse cells after subcycling. Uncovered coarse alpha retains its +coarse update. P1 validates boundedness, uniform-alpha invariance, pressure +consistency, and interface error; it does not claim global alpha conservation. + +Face components 1--6 are directly compatible with conservative AMReX reflux. +Face component 0 participates in the nonconservative alpha flux/source pair +and requires separate coarse-fine consistency treatment. Face component 7 is +the HLLC contact speed and is never accumulated in a standard FluxRegister. + +## SSP-RK2 time integration and units + +For `U1 = Un + dt L(Un)` and +`Un+1 = 0.5*(Un + U1 + dt L(U1))`, the integrated face flux is +`0.5*dt*F(Un) + 0.5*dt*F(U1)`. Each stage is accumulated with weight 0.5. +Hydro3D face arrays store unweighted physical fluxes. In direction `d`, the +register multiplier contains exactly once the level timestep and transverse +face area: + +``` +stage_scale = 0.5 * dt_level * product(dx[q], q != d) +``` + +Coarse contributions use `CrseInit(..., mult=-stage_scale)` (COPY for the +first stage and ADD thereafter). Fine contributions use +`FineAdd(..., mult=+stage_scale)` for both stages and every fine substep. +`Reflux(..., scale=1)` divides the accumulated extensive mismatch by the +coarse cell volume. Fine calls use fine area and fine timestep; neither cell +volume nor any area/time factor is applied twice. + +## Lifecycle and ordering + +`flux_reg[1]` represents the level-0/level-1 interface and contains six +components. It is defined with the fine BoxArray, fine DistributionMapping, +ratio two, and fine-level index one. Its first coarse-stage `CrseInit(COPY)` +resets every directional interface contribution for a parent coarse step; +later coarse stages use ADD. Both stages of both fine substeps then use +`FineAdd`. After fine subcycling, P1 performs: + +1. reflux components 1--6 into coarse `phi_new`; +2. average all seven fine components onto covered coarse cells; +3. validate/floor the resulting coarse state; +4. invalidate/refill ghosts only when next needed. + +Registers are recreated only with level creation/remake, cleared with the +level, and never cleared from a fine advance. Regridding discards stale face +and register contents. The same stage face arrays used by the update supply +the register; there is no second flux path. + +## Validation + +The manufactured FluxRegister test freezes sign, component offsets, timestep +and face-area scaling, and the corrected coarse-cell set. ML-U0 requires a +zero register correction. ML-SX-INTERFACE requires a measurable reflux-off +conservation mismatch and at least an order-of-magnitude improvement (or the +frozen numerical floor) with reflux enabled. ML-AC verifies bounded alpha, +pressure consistency, and that no standard alpha-register call occurs. diff --git a/docs/perf/m1b-p1-result.md b/docs/perf/m1b-p1-result.md new file mode 100644 index 0000000..d523970 --- /dev/null +++ b/docs/perf/m1b-p1-result.md @@ -0,0 +1,31 @@ +# M1b-P1 result + +Disposition: `BLASTGPU_M1B_P1_COMPLETE_MULTILEVEL_REFLUX_SUBCYCLING_AND_AMR_EFFICIENCY_BASELINE_READY_FOR_REVIEW`. + +P1 delivers two-level Hydro3D, ratio-two time subcycling, coarse old/new time +interpolation for both fine RK stages, stage-weighted six-component +FluxRegister accumulation, reflux-before-average-down, separately validated +alpha treatment, static sphere/box tagging, persistent per-level storage, and +composite reductions/profiles. + +The manufactured register test fixes the signs and scaling. Reflux-on removes +the deliberately exposed planar interface conservation error; reflux-off +retains a measurable mismatch. Alpha is never inserted into the conservative +register. ML-U0, regrid, shock, contact, and sphere cases pass the frozen +envelopes on CPU/CUDA. All four Compute Sanitizer modes pass. M0/P0 tests and +the canonical byte-identical M1a CPU/CUDA result remain valid. + +Matched R0 efficiency is transformative: 3.578x runtime and 2.802x used-memory +efficiency at 128-equivalent, rising to 4.503x and 3.261x at 192-equivalent, +while composite profile errors remain far below the frozen limits. The final +single-level guardrail is +0.822% at 128 cubed and +0.068% at 192 cubed with +identical maximum-used memory, so P0 preservation passes. + +The final steady trace contains no underlying allocation/free and remains +dominated by directional flux work. Inclusive synchronization time represents +GPU waits rather than measured host overhead; together with the negative C1 +result, it does not satisfy the C2 reconsideration gate. The next package is +therefore M1C-P0, not launch batching. + +No blastCFD work, merge, C2 experiment, dynamic tagging, remap, gauges, or +cross-solver claim occurred. diff --git a/docs/perf/m1b-p1-timing-results.tsv b/docs/perf/m1b-p1-timing-results.tsv new file mode 100644 index 0000000..227cf37 --- /dev/null +++ b/docs/perf/m1b-p1-timing-results.tsv @@ -0,0 +1,5 @@ +case run1 run2 run3 run4 run5 run6 run7 median_s cv_percent +U128 27.816338 28.067420 27.971697 28.033955 28.239506 28.279862 28.314495 28.067420 0.650 +A64X2 7.799938 7.800669 7.820214 7.843846 8.203249 7.890213 7.846024 7.843846 1.816 +U192 142.286145 142.284202 142.359906 142.265092 142.283801 142.951549 142.177345 142.284202 0.183 +A96X2 31.629693 31.601831 31.599399 31.547933 31.867318 31.428734 31.578346 31.599399 0.417 diff --git a/docs/perf/m1b-p1-tolerances.json b/docs/perf/m1b-p1-tolerances.json new file mode 100644 index 0000000..b124238 --- /dev/null +++ b/docs/perf/m1b-p1-tolerances.json @@ -0,0 +1,36 @@ +{ + "frozen_before_authoritative_performance": "2026-08-22T22:15:00Z", + "basis": [ + "M1b-P0 uniform-grid convergence and CPU/CUDA repeatability", + "manufactured FluxRegister sign/scaling test", + "pre-authority multilevel correctness and composite-profile runs" + ], + "uniform": { + "alpha_momentum_energy_spread": 0.0, + "partial_density_absolute_spread": 5e-15, + "composite_conservative_relative_drift": 1e-14, + "floor_nan_inf_events": 0 + }, + "planar_composite_vs_uniform_fine": { + "normalized_L1_max": {"rho": 0.005, "u": 0.005, "p": 0.005, "rhoE": 0.005}, + "normalized_Linf_max": {"rho": 0.06, "u": 0.25, "p": 0.06, "rhoE": 0.06}, + "transverse_momentum_absolute": 1e-12 + }, + "sphere_composite_vs_uniform_fine": { + "normalized_L1_max": {"rho": 0.005, "u": 0.02, "p": 0.005, "rhoE": 0.005}, + "normalized_Linf_max": {"rho": 0.06, "u": 0.15, "p": 0.06, "rhoE": 0.06}, + "axis_symmetry_relative_Linf": 1e-9, + "cpu_cuda_relative_Linf": 1e-12 + }, + "reflux_interface": { + "minimum_conservation_improvement": 10.0, + "conservative_relative_drift": 1e-10 + }, + "alpha_contact": { + "lower_bound_slack": 1e-12, + "upper_bound_slack": 1e-12, + "relative_pressure_oscillation": 1e-8, + "cpu_cuda_relative_Linf": 1e-12 + }, + "all_cases": {"floor_nan_inf_events": 0} +} diff --git a/inputs/m1b_p1_ml_ac.inp b/inputs/m1b_p1_ml_ac.inp new file mode 100644 index 0000000..8600826 --- /dev/null +++ b/inputs/m1b_p1_ml_ac.inp @@ -0,0 +1,25 @@ +max_step = 100 +stop_time = 0.1 +geometry.coord_sys = 0 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 1 0.125 0.125 +geometry.is_periodic = 1 1 1 +amr.n_cell = 64 8 8 +amr.max_level = 1 +amr.ref_ratio = 2 +amr.blocking_factor = 8 +amr.max_grid_size = 64 +amr.regrid_int = -1 +amr.plot_int = -1 +amr.tag_shape = box +amr.tag_lo = 0.25 0 0 +amr.tag_hi = 0.75 0.125 0.125 +blast.advance_mode = hydro3d +blast.problem = alpha_contact +blast.reflux = 1 +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-6 +blast.p_floor = 1.0 +ambient.p = 101325 +ambient.rho = 1.225 diff --git a/inputs/m1b_p1_ml_r0.inp b/inputs/m1b_p1_ml_r0.inp new file mode 100644 index 0000000..acf06bb --- /dev/null +++ b/inputs/m1b_p1_ml_r0.inp @@ -0,0 +1,25 @@ +max_step = 100000 +stop_time = 2e-4 +geometry.coord_sys = 0 +geometry.prob_lo = -0.5 -0.5 -0.5 +geometry.prob_hi = 0.5 0.5 0.5 +geometry.is_periodic = 0 0 0 +amr.n_cell = 64 64 64 +amr.max_level = 1 +amr.ref_ratio = 2 +amr.blocking_factor = 8 +amr.max_grid_size = 64 +amr.regrid_int = -1 +amr.plot_int = -1 +amr.tag_shape = sphere +amr.tag_radius = 0.25 +amr.tag_center = 0 0 0 +blast.advance_mode = hydro3d +blast.problem = pressure_sphere +blast.reflux = 1 +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-6 +blast.p_floor = 1.0 +ambient.p = 101325 +ambient.rho = 1.225 diff --git a/inputs/m1b_p1_ml_sx.inp b/inputs/m1b_p1_ml_sx.inp new file mode 100644 index 0000000..1db7401 --- /dev/null +++ b/inputs/m1b_p1_ml_sx.inp @@ -0,0 +1,23 @@ +max_step = 100000 +stop_time = 0.2 +geometry.coord_sys = 0 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 1 0.0625 0.0625 +geometry.is_periodic = 0 1 1 +amr.n_cell = 128 8 8 +amr.max_level = 1 +amr.ref_ratio = 2 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +amr.regrid_int = -1 +amr.plot_int = -1 +amr.tag_shape = box +amr.tag_lo = 0.25 0 0 +amr.tag_hi = 0.75 0.0625 0.0625 +blast.advance_mode = hydro3d +blast.problem = sod_x +blast.reflux = 1 +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-12 +blast.p_floor = 1e-12 diff --git a/inputs/m1b_p1_ml_u0.inp b/inputs/m1b_p1_ml_u0.inp new file mode 100644 index 0000000..99c5085 --- /dev/null +++ b/inputs/m1b_p1_ml_u0.inp @@ -0,0 +1,25 @@ +max_step = 100 +stop_time = 1.0 +geometry.coord_sys = 0 +geometry.prob_lo = -1 -1 -1 +geometry.prob_hi = 1 1 1 +geometry.is_periodic = 1 1 1 +amr.n_cell = 32 32 32 +amr.max_level = 1 +amr.ref_ratio = 2 +amr.blocking_factor = 8 +amr.max_grid_size = 32 +amr.regrid_int = -1 +amr.plot_int = -1 +amr.tag_shape = box +amr.tag_lo = -0.5 -0.5 -0.5 +amr.tag_hi = 0.5 0.5 0.5 +blast.advance_mode = hydro3d +blast.problem = uniform +blast.reflux = 1 +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-6 +blast.p_floor = 1.0 +ambient.p = 101325 +ambient.rho = 1.225 diff --git a/src/amr/BlastAmr.H b/src/amr/BlastAmr.H index 9f12278..9da869a 100644 --- a/src/amr/BlastAmr.H +++ b/src/amr/BlastAmr.H @@ -52,6 +52,7 @@ public: // part of the stable application API. void InitLevelData(int lev); amrex::Real EstTimeStep(int lev) const; + void ApplyReflux(int crse_lev); // ---- AmrCore overrides ---- void ErrorEst(int lev, amrex::TagBoxArray& tags, amrex::Real time, int ngrow) override; @@ -68,6 +69,7 @@ private: void ComputeDt(); void TimeStepWithSubcycling(int lev, amrex::Real time, int iteration); void AdvanceLevel(int lev, amrex::Real time, amrex::Real dt_lev); // M1b hook + void AccumulateStageFluxes(int lev, amrex::Real dt_lev, int rk_stage); void AverageDownTo(int crse_lev); void AverageDown(); void FillPatch(int lev, amrex::Real time, amrex::MultiFab& mf, int icomp, int ncomp); @@ -92,6 +94,9 @@ private: amrex::Real ambient_p = 101325.0, ambient_rho = 1.225; amrex::Real tag_radius = -1.0; // M1a: static spherical refinement region amrex::RealArray tag_center{{0.0, 0.0, 0.0}}; + std::string tag_shape{"sphere"}; + amrex::RealArray tag_lo{{0.0, 0.0, 0.0}}, tag_hi{{0.0, 0.0, 0.0}}; + bool reflux_enabled = true; std::string advance_mode{"identity"}; std::string problem{"uniform"}; std::string validation_file; @@ -103,6 +108,7 @@ private: amrex::Vector phi_new, phi_old; amrex::Vector> flux_reg; amrex::Vector> hydro3d; + amrex::Vector reflux_correction; amrex::Vector bcs; }; diff --git a/src/amr/BlastAmr.cpp b/src/amr/BlastAmr.cpp index 629824f..cf0bccc 100644 --- a/src/amr/BlastAmr.cpp +++ b/src/amr/BlastAmr.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include using namespace amrex; @@ -23,9 +24,16 @@ namespace { // so M2 can add reflective ground planes without touching FillPatch. struct BlastFill { AMREX_GPU_DEVICE - void operator()(const IntVect& /*iv*/, Array4 const& /*data*/, int /*dcomp*/, int /*numcomp*/, - GeometryData const& /*geom*/, Real /*time*/, const BCRec* /*bcr*/, int /*bcomp*/, - int /*orig_comp*/) const {} + void operator()(const IntVect& iv, Array4 const& data, int dcomp, int numcomp, + GeometryData const& geom, Real /*time*/, const BCRec* /*bcr*/, int /*bcomp*/, + int /*orig_comp*/) const { + const Box domain=geom.Domain(); + const IntVect lo=domain.smallEnd(), hi=domain.bigEnd(); + const int ii=amrex::max(lo[0],amrex::min(hi[0],iv[0])); + const int jj=amrex::max(lo[1],amrex::min(hi[1],iv[1])); + const int kk=amrex::max(lo[2],amrex::min(hi[2],iv[2])); + for(int n=0;n c; if (pp.queryarr("tag_center", c) && c.size() == 3) tag_center = {{c[0], c[1], c[2]}}; + Vector blo, bhi; + if (pp.queryarr("tag_lo", blo) && blo.size() == 3) tag_lo = {{blo[0],blo[1],blo[2]}}; + if (pp.queryarr("tag_hi", bhi) && bhi.size() == 3) tag_hi = {{bhi[0],bhi[1],bhi[2]}}; } { ParmParse pp("blast"); @@ -100,6 +113,7 @@ void BlastAmr::ReadParameters() pp.query("advance_mode", advance_mode); pp.query("problem", problem); pp.query("validation_file", validation_file); + pp.query("reflux", reflux_enabled); } { ParmParse pp("ambient"); @@ -118,11 +132,12 @@ void BlastAmr::MakeNewLevelFromScratch(int lev, Real time, const BoxArray& ba, c phi_old[lev].define(ba, dm, amr::NCOMP, nghost); t_new[lev] = time; t_old[lev] = time - 1.e200; - if (lev > 0) flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, amr::NCOMP); + if (lev > 0 && advance_mode == "hydro3d") flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, 6); if (advance_mode == "hydro3d") { - AMREX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, "hydro3d P0 requires amr.max_level=0"); + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(max_level <= 1, "hydro3d P1 supports amr.max_level<=1"); hydro3d[lev] = std::make_unique(); hydro3d[lev]->define(ba,dm,nghost); + if(lev 0) flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, amr::NCOMP); + if (lev > 0 && advance_mode == "hydro3d") flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, 6); + if (advance_mode == "hydro3d") { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(max_level <= 1, "hydro3d P1 supports amr.max_level<=1"); + hydro3d[lev] = std::make_unique(); + hydro3d[lev]->define(ba,dm,nghost); + if(lev 0) flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, amr::NCOMP); + if (lev > 0 && advance_mode == "hydro3d") flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, 6); + if (advance_mode == "hydro3d") { + hydro3d[lev] = std::make_unique(); + hydro3d[lev]->define(ba,dm,nghost); + if(lev= boxlo[0] && x+ctr[0] <= boxhi[0] && + y+ctr[1] >= boxlo[1] && y+ctr[1] <= boxhi[1] && + z+ctr[2] >= boxlo[2] && z+ctr[2] <= boxhi[2])) tag(i,j,k) = tagval; }); } } @@ -315,12 +352,34 @@ void BlastAmr::TimeStepWithSubcycling(int lev, Real time, int iteration) if (lev < finest_level) { for (int i = 1; i <= nsubsteps[lev + 1]; ++i) TimeStepWithSubcycling(lev + 1, time + (i - 1) * dt[lev + 1], i); - // M1b: flux_reg[lev+1]->Reflux(phi_new[lev], ...) goes here, before AverageDownTo. + if (advance_mode=="hydro3d" && reflux_enabled) { + ApplyReflux(lev); + } AverageDownTo(lev); + if (advance_mode=="hydro3d") hydro3d[lev]->enforceFloors(phi_new[lev],mat,fl); } (void)iteration; } +void BlastAmr::ApplyReflux(int crse_lev) +{ + MultiFab& correction=reflux_correction[crse_lev]; + correction.setVal(0.0); + flux_reg[crse_lev+1]->Reflux(correction,1.0,0,0,6,Geom(crse_lev)); + MultiFab& state=phi_new[crse_lev]; + constexpr Real roundoff_factor=1024.0*std::numeric_limits::epsilon(); + for(MFIter mfi(state,TilingIfNotGPU());mfi.isValid();++mfi) { + const Box bx=mfi.tilebox(); auto q=state.array(mfi); auto r=correction.const_array(mfi); + ParallelFor(bx,6,[=] AMREX_GPU_DEVICE(int i,int j,int k,int n) noexcept { + const Real delta=r(i,j,k,n); + const Real scale=(n<2 || n==5) + ? amrex::max(std::numeric_limits::min(),std::abs(q(i,j,k,n+amr::ARHO1))) + : Real(1.0); + if(std::abs(delta)>roundoff_factor*scale) q(i,j,k,n+amr::ARHO1)+=delta; + }); + } +} + // M1a: identity advance. Still exercises the FillPatch + ghost-cell path that // M1b's hydro will consume, so regridding and coarse-fine interpolation are // proven before any physics arrives. @@ -329,11 +388,13 @@ void BlastAmr::AdvanceLevel(int lev, Real time, Real dt_lev) std::swap(phi_old[lev], phi_new[lev]); if (advance_mode == "hydro3d") { auto& h=*hydro3d[lev]; - h.fillBorder(phi_old[lev],Geom(lev)); + FillPatch(lev,time,h.border(),0,amr::NCOMP); h.computeFluxes(Geom(lev),mat,fl); + AccumulateStageFluxes(lev,dt_lev,1); h.update(phi_old[lev],phi_new[lev],Geom(lev),mat,fl,dt_lev,1); - h.fillBorder(phi_new[lev],Geom(lev)); + FillPatch(lev,time+dt_lev,h.border(),0,amr::NCOMP); h.computeFluxes(Geom(lev),mat,fl); + AccumulateStageFluxes(lev,dt_lev,2); h.update(phi_old[lev],phi_new[lev],Geom(lev),mat,fl,dt_lev,2); return; } @@ -342,6 +403,22 @@ void BlastAmr::AdvanceLevel(int lev, Real time, Real dt_lev) MultiFab::Copy(phi_new[lev], Sborder, 0, 0, amr::NCOMP, 0); } +void BlastAmr::AccumulateStageFluxes(int lev, Real dt_lev, int rk_stage) +{ + constexpr int srccomp=amr::ARHO1, destcomp=0, ncomp=6; + for (int d=0; dfaceFlux(d); + if (lev < finest_level) { + flux_reg[lev+1]->CrseInit(face,d,srccomp,destcomp,ncomp,-scale, + rk_stage==1 ? FluxRegister::COPY : FluxRegister::ADD); + } + if (lev > 0) flux_reg[lev]->FineAdd(face,d,srccomp,destcomp,ncomp,scale); + } +} + void BlastAmr::AverageDownTo(int crse_lev) { amrex::average_down(phi_new[crse_lev + 1], phi_new[crse_lev], Geom(crse_lev + 1), Geom(crse_lev), @@ -417,20 +494,32 @@ void BlastAmr::WritePlotFile() const void BlastAmr::WriteValidationProfile() const { - MultiFab host(grids[0],dmap[0],amr::NCOMP,0,MFInfo().SetArena(The_Pinned_Arena())); - MultiFab::Copy(host,phi_new[0],0,0,amr::NCOMP,0); + Vector> host(finest_level+1); + for(int lev=0;lev<=finest_level;++lev) { + host[lev]=std::make_unique(grids[lev],dmap[lev],amr::NCOMP,0, + MFInfo().SetArena(The_Pinned_Arena())); + MultiFab::Copy(*host[lev],phi_new[lev],0,0,amr::NCOMP,0); + } Gpu::streamSynchronize(); - const Box domain=Geom(0).Domain(); const IntVect mid=(domain.smallEnd()+domain.bigEnd())/2; - const auto lo=Geom(0).ProbLoArray(), dx=Geom(0).CellSizeArray(); + const int sample_lev=finest_level; + const Box domain=Geom(sample_lev).Domain(); const IntVect mid=(domain.smallEnd()+domain.bigEnd())/2; + const auto lo=Geom(sample_lev).ProbLoArray(), dx=Geom(sample_lev).CellSizeArray(); struct Row { int axis,index; Real x; hydro::Cons3D c; hydro::Prim3D w; }; std::vector rows; - for(MFIter mfi(host);mfi.isValid();++mfi) { - const Box bx=mfi.validbox(); auto a=host.const_array(mfi); - for(int axis=0;axis<3;++axis) for(int n=domain.smallEnd(axis);n<=domain.bigEnd(axis);++n) { - IntVect iv=mid; iv[axis]=n; if(!bx.contains(iv)) continue; - hydro::Cons3D c; for(int q=0;q=0 && !found;--lev) { + IntVect iv=fine_iv; + for(int rlev=sample_lev-1;rlev>=lev;--rlev) iv.coarsen(refRatio(rlev)); + for(MFIter mfi(*host[lev]);mfi.isValid();++mfi) if(mfi.validbox().contains(iv)) { + auto a=host[lev]->const_array(mfi); hydro::Cons3D c; + for(int q=0;qcounters(); - amrex::Print() << " hydro3d counters alpha=" << c[hydro::ALPHA_VIOL] + for(int lev=0;lev<=finest_level;++lev) { + const auto c=hydro3d[lev]->counters(); + amrex::Print() << (lev==0 ? " hydro3d counters alpha=" : " hydro3d fine counters alpha=") + << c[hydro::ALPHA_VIOL] << " density=" << c[hydro::DENSITY_VIOL] << " pressure=" << c[hydro::PRESSURE_VIOL] << " nan=" << c[hydro::NAN_VIOL] << " inf=" << c[hydro::INF_VIOL] << "\n"; - const Real dv=Geom(0).CellSize(0)*Geom(0).CellSize(1)*Geom(0).CellSize(2); - amrex::Print() << " hydro3d totals"; - for(int n=0;n states; + Vector geoms; + Vector ratios; + for(int lev=0;lev<=finest_level;++lev) { + states.push_back(&phi_new[lev]); geoms.push_back(Geom(lev)); + if(lev counters() const; diff --git a/src/hydro/Hydro3D.cpp b/src/hydro/Hydro3D.cpp index 05589e6..b7d848e 100644 --- a/src/hydro/Hydro3D.cpp +++ b/src/hydro/Hydro3D.cpp @@ -122,6 +122,14 @@ void Hydro3D::update(const MultiFab& u0, MultiFab& out, const Geometry& geom, +(fy(i,j+1,k,FACE3_SSTAR)-fy(i,j,k,FACE3_SSTAR))*dxi[1] +(fz(i,j,k+1,FACE3_SSTAR)-fz(i,j,k,FACE3_SSTAR))*dxi[2]); } + // Coarse/fine interpolation and register summation can leave a + // formally zero uniform-state RHS at a few ulps. Preserve the + // free stream exactly while retaining every resolved update. + const Real roundoff_scale = (n==C3_ARHO1 || n==C3_ARHO2 || n==C3_RHOE) + ? amrex::max(std::numeric_limits::min(),std::abs(sb(i,j,k,n))) + : Real(1.0); + if (std::abs(dt*rhs) <= 1024.0*std::numeric_limits::epsilon() + *roundoff_scale) rhs=0.0; const Real euler=sb(i,j,k,n)+dt*rhs; next[n]=stage==1 ? euler : 0.5*(base(i,j,k,n)+euler); } @@ -144,4 +152,31 @@ void Hydro3D::update(const MultiFab& u0, MultiFab& out, const Geometry& geom, } } +void Hydro3D::enforceFloors(MultiFab& state, const Materials& mat, const Floors& fl) { + const Materials m=mat; const Floors f=fl; auto* count=counters_.data(); + for (MFIter mfi(state); mfi.isValid(); ++mfi) { + const Box bx=mfi.validbox(); auto q=state.array(mfi); + ParallelFor(bx,[=] AMREX_GPU_DEVICE(int i,int j,int k) noexcept { + Real next[NVAR3D]; + for(int n=0;n16.0*std::numeric_limits::epsilon()) + Gpu::Atomic::Add(&count[ALPHA_VIOL],1ULL); + const Real a=next[C3_ALPHA1]; + const Real r1min=f.rho*a, r2min=f.rho*(1.0-a); + if(next[C3_ARHO1] +#include +#include +#include + +#include +#include + +namespace { + +void require(bool ok, const char* message) +{ + if (!ok) amrex::Abort(message); +} + +void run_test() +{ + using namespace amrex; + constexpr int nstate = 7; + constexpr int nreg = 6; + const IntVect ratio(2); + const Box coarse_domain(IntVect(0), IntVect(3)); + const Box fine_region = amrex::refine(Box(IntVect(1), IntVect(2)), ratio); + BoxArray cba(coarse_domain), fba(fine_region); + DistributionMapping cdm(cba), fdm(fba); + RealBox rb({0.0,0.0,0.0},{4.0,4.0,4.0}); + Vector periodic(3,0); + Geometry cgeom(coarse_domain,&rb,CoordSys::cartesian,periodic.data()); + + FluxRegister reg(fba,fdm,ratio,1,nreg); + MultiFab coarse(cba,cdm,nstate,0); + coarse.setVal(0.0); + + std::array,3> cflux, fflux; + for (int d=0; d<3; ++d) { + BoxArray cfba=cba; cfba.convert(IntVect::TheDimensionVector(d)); + BoxArray ffba=fba; ffba.convert(IntVect::TheDimensionVector(d)); + cflux[d]=std::make_unique(cfba,cdm,nstate,0); + fflux[d]=std::make_unique(ffba,fdm,nstate,0); + cflux[d]->setVal(0.0); + fflux[d]->setVal(0.0); + reg.CrseInit(*cflux[d],d,1,0,nreg,-1.0,FluxRegister::COPY); + } + for (int n=1; nsetVal(Real(n),n,1,0); + + // Fine area is 1/4 and fine dt is 1/2. Each coarse interface face + // receives four fine faces, so the integrated mismatch is n/2. + reg.FineAdd(*fflux[0],0,1,0,nreg,0.125); + reg.Reflux(coarse,1.0,0,1,nreg,cgeom); + + for (int n=1; n