diff --git a/CMakeLists.txt b/CMakeLists.txt index d7db98a..3a68ceb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,9 @@ if(BLASTGPU_BUILD_TESTS) add_executable(test_hydro tests/test_hydro.cpp) target_include_directories(test_hydro PRIVATE src/eos src/hydro) add_test(NAME hydro COMMAND test_hydro) + + add_executable(m1b_p0_reference tools/m1b_p0_reference.cpp) + target_include_directories(m1b_p0_reference PRIVATE src/eos src/hydro) endif() # ---- M0 1D driver (no AMReX) --------------------------------------------- @@ -45,6 +48,11 @@ if(BLASTGPU_BUILD_SOLVER) else() set(AMReX_GPU_BACKEND ${BLASTGPU_GPU_BACKEND} CACHE STRING "" FORCE) if(BLASTGPU_GPU_BACKEND STREQUAL "CUDA") + if(NOT CMAKE_CUDA_ARCHITECTURES) + message(FATAL_ERROR + "CUDA builds require -DCMAKE_CUDA_ARCHITECTURES=, " + "for example 75 for an RTX 2070 SUPER") + endif() enable_language(CUDA) endif() endif() @@ -58,6 +66,20 @@ if(BLASTGPU_BUILD_SOLVER) target_link_libraries(blastgpu PRIVATE AMReX::amrex) if(BLASTGPU_GPU_BACKEND STREQUAL "CUDA") set_source_files_properties(${BLASTGPU_SRC} PROPERTIES LANGUAGE CUDA) + setup_target_for_cuda_compilation(blastgpu) + set_property(TARGET blastgpu PROPERTY CUDA_ARCHITECTURES + "${CMAKE_CUDA_ARCHITECTURES}") + endif() + if(BLASTGPU_BUILD_TESTS AND BLASTGPU_GPU_BACKEND STREQUAL "NONE") + find_package(Python3 REQUIRED COMPONENTS Interpreter) + add_test(NAME m1b-u0-smoke COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/tests/m1b_p0_check.py --uniform + $ ${CMAKE_SOURCE_DIR}/inputs/m1b_p0_u0.inp + "amr.n_cell=8 8 8" max_step=2 stop_time=1) + add_test(NAME m1b-sod-x-smoke COMMAND ${Python3_EXECUTABLE} + ${CMAKE_SOURCE_DIR}/tests/m1b_p0_check.py + $ ${CMAKE_SOURCE_DIR}/inputs/m1b_p0_sx.inp + "amr.n_cell=32 8 8" max_step=2 stop_time=1) endif() endif() endif() diff --git a/docs/perf/m1b-p0-authorization.md b/docs/perf/m1b-p0-authorization.md new file mode 100644 index 0000000..d4f3533 --- /dev/null +++ b/docs/perf/m1b-p0-authorization.md @@ -0,0 +1,5 @@ +# M1b-P0 authorization + +Authority: `BLASTGPU-M1B-P0-OWNER-AUTHORIZATION-SINGLE-LEVEL-THREE-DIMENSIONAL-MUSCL-HLLC-SSP-RK2-HYDRO-PERFORMANCE-SLICE-FROM-983CE171C0865789EACA6D878B0DD68D9A6C6022-2026-08-22`. + +This branch implements only the authorized single-level Cartesian hydro slice. It does not execute multilevel hydro, reflux, regridding, remap, EB, MPI, reactive physics, tuning, or blastCFD comparison. blastCFD was not accessed. diff --git a/docs/perf/m1b-p0-build-manifest.tsv b/docs/perf/m1b-p0-build-manifest.tsv new file mode 100644 index 0000000..0889fff --- /dev/null +++ b/docs/perf/m1b-p0-build-manifest.tsv @@ -0,0 +1,4 @@ +build source_commit compiler flags architecture executable_sha256 build_id +cpu abcec70687e86382510fd182b493d829ee03d424 GCC-13.3 -O3 -DNDEBUG -fopenmp CPU 89f04c83ae9809f14027904b9faf57cf3f22ddd57602ad462bfb12290095fcbd d0d0f5eaabbba741b40c8a2e988b2fedbb22e091 +cuda-opt abcec70687e86382510fd182b493d829ee03d424 NVCC-12.6.85 -O3 -DNDEBUG --use_fast_math -maxrregcount=255 --generate-line-info sm_75 2662fde779aa47a6cb1faf61a4696f0ae3326d8a386f464f5afa3242eb54d2d6 d6c0ec1533a6cfe89470b147bac7ab6613764f56 +cuda-prof abcec70687e86382510fd182b493d829ee03d424 NVCC-12.6.85 -O3 -DNDEBUG --use_fast_math -maxrregcount=255 --generate-line-info sm_75 160142147f2e2280865cdc695d15f3e9ef82551017e7eee5690f8393a7847d8a 94e603c05d0a721f9228ab3786697a22e434128a diff --git a/docs/perf/m1b-p0-c1-authorization.md b/docs/perf/m1b-p0-c1-authorization.md new file mode 100644 index 0000000..9f70802 --- /dev/null +++ b/docs/perf/m1b-p0-c1-authorization.md @@ -0,0 +1,5 @@ +# M1b-P0-C1 authorization + +Authorization: `BLASTGPU-M1B-P0-C1-OWNER-AUTHORIZATION-CONSOLIDATE-PER-STAGE-DIRECTIONAL-FLUX-AND-CELL-UPDATE-MFITER-REGIONS-WITHOUT-CHANGING-NUMERICS-FACE-STORAGE-KERNELS-LAUNCH-DIMENSIONS-OR-REFLUX-CONTRACT-2026-08-22`. + +The experiment was limited to replacing four hydro-owned `MFIter` lifetimes per RK stage with one lifetime that enqueued x flux, y flux, z flux, and the combined update for each box. No numerical formula, device kernel body, launch dimension, storage layout, stream configuration, input, tolerance, or reflux contract changed. No blastCFD work was performed. diff --git a/docs/perf/m1b-p0-c1-paired-timing.tsv b/docs/perf/m1b-p0-c1-paired-timing.tsv new file mode 100644 index 0000000..49e0ceb --- /dev/null +++ b/docs/perf/m1b-p0-c1-paired-timing.tsv @@ -0,0 +1,28 @@ +case pair control_hydro_s candidate_hydro_s speedup_percent +P64 1 3.868392944 3.861083600 0.189308 +P64 2 3.915733783 3.839798750 1.977578 +P64 3 3.893767722 3.881358292 0.319719 +P64 4 3.887339670 3.880021630 0.188608 +P64 5 3.886211541 3.879493847 0.173159 +P64 6 3.893417659 3.886099986 0.188304 +P64 7 3.892887575 3.904904430 -0.307737 +P128 1 30.963360350 30.808829210 0.501581 +P128 2 30.883421200 30.783218620 0.325510 +P128 3 30.972175890 30.861715390 0.357921 +P128 4 30.944392030 30.760813300 0.596794 +P128 5 30.945798280 30.913120160 0.105710 +P128 6 31.232841610 31.121937100 0.356355 +P128 7 31.167226470 31.091800770 0.242590 +P128 8 31.148733100 31.043899820 0.337694 +P128 9 31.153316650 31.149445820 0.012427 +P128 10 31.291841880 31.038931050 0.814818 +P192 1 104.986156700 104.942506300 0.041595 +P192 2 104.966131400 105.125216900 -0.151330 +P192 3 104.901928100 104.961307400 -0.056573 +P192 4 104.821090700 104.895476700 -0.070914 +P192 5 104.848120900 104.880481900 -0.030855 +P192 6 105.107488100 104.890759800 0.206623 +P192 7 105.060223200 104.894155600 0.158319 +P192 8 104.882113400 105.009893800 -0.121684 +P192 9 104.894422800 104.944744100 -0.047950 +P192 10 104.998942800 105.084878000 -0.081777 diff --git a/docs/perf/m1b-p0-c1-raw-artifact-manifest.tsv b/docs/perf/m1b-p0-c1-raw-artifact-manifest.tsv new file mode 100644 index 0000000..90c6ca8 --- /dev/null +++ b/docs/perf/m1b-p0-c1-raw-artifact-manifest.tsv @@ -0,0 +1,8 @@ +artifact external_relative_path sha256 +control-nsys profiles/P128-control.nsys-rep e932891b96e6949a20d586cfda1ddb91b50051ecf9d8b464764063cce862932d +candidate-nsys profiles/P128-candidate.nsys-rep 6cc9449b82f3c7df2c01e53937793b1b576eb9c11669626d33f3d3e70edc4b54 +control-flux-sass builds/control-flux-body.sass ab1c528bd6054c5c4d203a14f0bf6c4d5d1d210a41a85a305deee2ea2d2b6faa +candidate-flux-sass builds/candidate-flux-body.sass ab1c528bd6054c5c4d203a14f0bf6c4d5d1d210a41a85a305deee2ea2d2b6faa +control-update-sass builds/control-update-body.sass c6e82c9c654f5145c49131b8811662dc4d720d3397fe040a9d0a69938d25b324 +candidate-update-sass builds/candidate-update-body.sass c6e82c9c654f5145c49131b8811662dc4d720d3397fe040a9d0a69938d25b324 +paired-summary timing/paired-summary.txt c207f293706b4944cf4625e7907fd963d1aa1affe366049518b2b145c8dc16a2 diff --git a/docs/perf/m1b-p0-c1-result.md b/docs/perf/m1b-p0-c1-result.md new file mode 100644 index 0000000..55addc8 --- /dev/null +++ b/docs/perf/m1b-p0-c1-result.md @@ -0,0 +1,35 @@ +# M1b-P0-C1 result + +Disposition: `BLASTGPU_M1B_P0_C1_REVERT_NO_ELIGIBLE_SYNCHRONIZATION_OR_PERFORMANCE_BENEFIT`. + +The candidate was numerically and architecturally sound but did not meet the frozen performance gates. It was preserved as commit `a0a77baf64b992f69f8316d412a37e6a2e4bb8c9` and explicitly reverted by `13ddca2392c995e9b113758adfbf3bdbc5a317c0`. + +## Correctness and execution invariants + +- CPU CTest: 4/4 passed. +- Canonical M1a: CPU and CUDA passed; all seven plotfile files were byte-identical. +- U0, SX, SY, SZ, R0-64, and R0-128 passed unchanged tolerances with zero violations. +- Candidate CUDA SX/SY/SZ/R0 TSVs were identical to the authoritative P0 TSVs. +- Compute Sanitizer memcheck, synccheck, initcheck, and racecheck passed. +- Flux and update normalized SASS were byte-identical between control and candidate. +- Kernel count, launch dimensions, storage, memory/copy topology, and reflux-compatible face components were unchanged. + +## Timing adjudication + +Balanced order was alternated by pair; bootstrap seed was `20260822` with 100,000 resamples. + +| Case | Control median (s) | Candidate median (s) | Paired median | 95% bootstrap interval | Candidate wins | Decision | +|---|---:|---:|---:|---:|---:|---| +| P64 | 3.892888 | 3.880022 | +0.189% | +0.173% to +0.320% | 6/7 | no material regression | +| P128 | 31.060454 | 30.976026 | +0.347% | +0.216% to +0.502% | 10/10 | FAIL: below 1% | +| P192 | 104.934030 | 104.943625 | -0.052% | -0.096% to +0.064% | 3/10 | FAIL: below 1%, interval includes zero | + +Candidate timing CV was 0.530% (P64), 0.480% (P128), and 0.081% (P192). Fresh control drift from the original P0 medians was +4.04%, +2.39%, and +1.65%; paired results are authoritative. + +## Nsight Systems delta + +Matched four-step P128 traces contained 466 kernels each, including 192 directional flux kernels and 64 combined updates. Control made 307 `cudaStreamSynchronize` calls; candidate made 187: the exact predicted reduction of 120, with no compensating synchronization family. Core launch count stayed 32 per stage for eight boxes. Candidate streams showed `flux, flux, flux, update` pipelines. Kernel time remained materially unchanged and flux work remained about 91% of kernel time, explaining why removal of host convergence delivered only a sub-percent P128 gain and no P192 benefit. + +Arena maximum-used memory was unchanged: 111 MiB (P64), 837 MiB (P128), and 2807 MiB (P192). No steady `cudaMalloc`/`cudaFree`, new copy, or face-to-cell atomic appeared. Average candidate power differed by less than 1%, so no >2% energy regression was established. + +The original four-region topology is retained. If recurrent synchronization is pursued again, the next bounded question is whole-level or batched multi-box launch feasibility; C1 shows that iterator consolidation alone is insufficient. diff --git a/docs/perf/m1b-p0-c1-source-audit.md b/docs/perf/m1b-p0-c1-source-audit.md new file mode 100644 index 0000000..839da6f --- /dev/null +++ b/docs/perf/m1b-p0-c1-source-audit.md @@ -0,0 +1,23 @@ +# M1b-P0-C1 source and AMReX stream audit + +## Control topology + +At `43dd468`, `Hydro3D::computeFluxes` used three sequential direction loops, each with its own `MFIter(border_)`; `Hydro3D::update` used a fourth `MFIter(out)`. `BlastAmr::AdvanceLevel` invoked fill, flux, update twice per step. + +AMReX commit `2cf4fbcde02a700e01c62bebb4a730aeeb66c661` establishes the relevant behavior: + +- `Src/Base/AMReX_MFIter.cpp:196-221` constructs the iterator with up to the configured stream count. +- `AMReX_MFIter.cpp:371-382` performs the multi-box pre-sync and assigns `currentIndex % streams`. +- `AMReX_MFIter.cpp:526-542` advances boxes and selects the next stream deterministically. +- `AMReX_MFIter.cpp:225-255` synchronizes each active stream at iterator destruction and resets stream zero. +- GPU `ParallelFor` launches use the current `Gpu::gpuStream()`; no hydro helper changes it. + +For eight boxes and four streams, each control region produces one pre-sync plus four post-sync calls. One combined region therefore predicted 15 fewer `cudaStreamSynchronize` calls per stage, or 120 over the profiled eight RK stages. + +## Dependency proof + +The cell state, border, and three direction-nodal face `MultiFab`s share the same DistributionMapping and one FAB entry per cell BoxArray entry. `array(mfi)` resolves the same FAB index across them. Each box writes its own face-FAB allocation; geometrically duplicated nodal boundary faces are separate storage populated from consistent border ghosts. The update reads the three face FABs at that same index and writes only its cell-valid box. There is no cross-box face dependency or residual atomic; diagnostic counters are the only shared atomic target. CPU execution remains serial within each MFIter body. + +The candidate commit `a0a77baf64b992f69f8316d412a37e6a2e4bb8c9` introduced `Hydro3D::advanceStage`, one `MFIter(out)` per stage, and enqueued x/y/z/update on the stream assigned to each box. It changed 13 lines and removed 20 across `src/hydro/Hydro3D.H`, `src/hydro/Hydro3D.cpp`, and `src/amr/BlastAmr.cpp`. It introduced no helper, allocation, manual stream, event, or explicit synchronization. + +Static comparison proved identical flux and update SASS. The candidate trace proved repeated per-stream `flux, flux, flux, update` ordering with unchanged grid sizes (1040 flux blocks, 1024 update blocks), 256-thread blocks, and zero dynamic shared memory. diff --git a/docs/perf/m1b-p0-case-manifest.tsv b/docs/perf/m1b-p0-case-manifest.tsv new file mode 100644 index 0000000..3337f09 --- /dev/null +++ b/docs/perf/m1b-p0-case-manifest.tsv @@ -0,0 +1,5 @@ +case grid boxes valid_cells steps stages max_grid_size blocking_factor output +P64 64x64x64 1 262144 100 200 64 8 disabled +P128 128x128x128 8 2097152 100 200 64 8 disabled +P192 192x192x192 27 7077888 100 200 64 8 disabled +P256A 256x256x256 64 16777216 0 0 64 8 allocation-only diff --git a/docs/perf/m1b-p0-correctness-results.md b/docs/perf/m1b-p0-correctness-results.md new file mode 100644 index 0000000..3be9fc9 --- /dev/null +++ b/docs/perf/m1b-p0-correctness-results.md @@ -0,0 +1,9 @@ +# M1b-P0 correctness results + +- Existing EOS and M0 hydro tests pass without changed M0 numerical interfaces. +- Canonical M1a passes on CPU and CUDA; all seven plotfile files remain byte-identical. +- U0 preserves all seven components with exactly zero spread for 100 steps on CPU and CUDA. Counters are all zero. +- SX/SY/SZ complete 537 steps to t=0.2. CPU/CUDA profile differences are at most `8.9e-15`; rotated conserved/thermodynamic differences are at most `4.5e-15`; tangential momentum is exactly zero. +- Against M0 N=256, normalized L1 errors are `8.60e-5` density, `2.22e-4` velocity, `9.04e-5` pressure, and `9.52e-5` energy. +- R0 completes 44 steps at 64 cubed and 90 at 128 cubed. Axis differences are at floating-point roundoff; CPU/CUDA relative differences are below `6.7e-15`; all counters are zero. CPU 256 cubed was not practical and was not run. +- Compute Sanitizer memcheck, synccheck, initcheck, and racecheck all report zero errors/hazards. diff --git a/docs/perf/m1b-p0-implementation.md b/docs/perf/m1b-p0-implementation.md new file mode 100644 index 0000000..52ccad7 --- /dev/null +++ b/docs/perf/m1b-p0-implementation.md @@ -0,0 +1,11 @@ +# M1b-P0 implementation + +The additive `Cons3D` layout is `(alpha1, arho1, arho2, rhou, rhov, rhow, rhoE)`; `Prim3D` is `(alpha1, rho1, rho2, u, v, w, p)`. M0 types and overloads are unchanged. Direction-local HLLC reconstructs every primitive on demand, treats one velocity as normal and two as tangential, adds pressure only to normal momentum, and stores seven global-order fluxes plus contact speed. + +`Hydro3D` persistently owns one three-ghost border MultiFab, three nodal eight-component face MultiFabs, and five device counters. `phi_old` is `U0`; `phi_new` is stage one and then the in-place final RK2 state. The paired alpha update is `-sum[(Falpha_hi-a*Sstar_hi)-(Falpha_lo-a*Sstar_lo)]/dx`. CFL is the maximum of the sum of three directional spectral radii. + +Face components 0–6 are unweighted physical fluxes for future `FluxRegister` `srccomp=0`, `destcomp=0`, `numcomp=7`; component 7 is excluded. Area, timestep, and sign scaling must occur exactly once in future `CrseInit/FineAdd` calls. + +CUDA builds require an explicit `CMAKE_CUDA_ARCHITECTURES`; AMReX's setup helper and the application target property propagate the same architecture. Identity mode remains the default and allocates no Hydro3D storage. + +Measured limitation: the first valid implementation uses separate MFIter regions for the three flux directions and update. P128 therefore issues 32 hydro-core launches per stage and performs iterator-boundary synchronization. This was recorded without a second tuning pass. diff --git a/docs/perf/m1b-p0-kernel-results.tsv b/docs/perf/m1b-p0-kernel-results.tsv new file mode 100644 index 0000000..5d2ac27 --- /dev/null +++ b/docs/perf/m1b-p0-kernel-results.tsv @@ -0,0 +1,2 @@ +interval steps stages boxes kernels flux update cfl_box boundary_fill copy streams stream_sync steady_cuda_malloc_free +P128-hydro-span 4 8 8 353 192 64 24 56 14 4 226 0 diff --git a/docs/perf/m1b-p0-memory-results.tsv b/docs/perf/m1b-p0-memory-results.tsv new file mode 100644 index 0000000..6f44eba --- /dev/null +++ b/docs/perf/m1b-p0-memory-results.tsv @@ -0,0 +1,5 @@ +case raw_baseline_mib raw_peak_mib raw_net_mib arena_reserved_mib arena_max_used_mib arena_used_bytes_per_cell modeled_mib modeled_bytes_per_cell +P64 648 6577 5929 5823 111 444 103.705 414.818 +P128 648 6577 5929 5823 837 418.5 829.636 414.818 +P192 641 6571 5930 5823 2807 416.1 2800.022 414.818 +P256A 641 7487 6846 6652 6644 415.3 6637.090 414.818 diff --git a/docs/perf/m1b-p0-numerical-results.tsv b/docs/perf/m1b-p0-numerical-results.tsv new file mode 100644 index 0000000..1b51aec --- /dev/null +++ b/docs/perf/m1b-p0-numerical-results.tsv @@ -0,0 +1,10 @@ +case backend grid steps final_time primary_metric value floor_nan_inf result +U0 CPU 64x64x64 100 0.001377486203 max_component_spread 0 0/0/0 PASS +U0 CUDA 64x64x64 100 0.001377486203 max_component_spread 0 0/0/0 PASS +SX CPU+CUDA 256x16x16 537 0.2 max_cpu_cuda_profile_abs 4.06e-15 0/0/0 PASS +SY CPU+CUDA 16x256x16 537 0.2 max_cpu_cuda_profile_abs 4.06e-15 0/0/0 PASS +SZ CPU+CUDA 16x16x256 537 0.2 max_cpu_cuda_profile_abs 8.89e-15 0/0/0 PASS +SX/SY/SZ CPU+CUDA rotated 537 0.2 max_rotated_abs 4.45e-15 0/0/0 PASS +Sod-M0 CUDA 256-normal 537 0.2 max_normalized_L1 2.22e-4 0/0/0 PASS +R0 CPU+CUDA 64x64x64 44 0.0002 max_cpu_cuda_relative 1.59e-15 0/0/0 PASS +R0 CPU+CUDA 128x128x128 90 0.0002 max_cpu_cuda_relative 6.61e-15 0/0/0 PASS diff --git a/docs/perf/m1b-p0-performance-results.md b/docs/perf/m1b-p0-performance-results.md new file mode 100644 index 0000000..119694c --- /dev/null +++ b/docs/perf/m1b-p0-performance-results.md @@ -0,0 +1,13 @@ +# M1b-P0 CUDA baseline + +This is the untuned first valid hydro implementation on the RTX 2070 SUPER. It is not a blastCFD comparison. + +| Case | Boxes | Median hydro / 100 steps | CV | Cell-stage/s | Face eval/s | Arena max used | +|---|---:|---:|---:|---:|---:|---:| +| P64 | 1 | 3.7418 s | 0.198% | 14.01 M | 42.69 M | 111 MiB | +| P128 | 8 | 30.3362 s | 0.346% | 13.83 M | 41.80 M | 837 MiB | +| P192 | 27 | 103.2347 s | 0.513% | 13.71 M | 41.35 M | 2807 MiB | + +Every measured run completed 100 steps/200 stages with identical case-specific numerical checksums and zero floor/finite-value violations. Raw baseline-subtracted device footprint remained about 5929 MiB because the default AMReX arena reserved approximately 5823 MiB. Actual maximum-used storage scales near the frozen 415 B/valid-cell model. + +P256A allocated and initialized successfully: arena maximum used 6644 MiB and only 277 MiB remained free. It was therefore not timed and does not satisfy 20% headroom. diff --git a/docs/perf/m1b-p0-profile-summary.md b/docs/perf/m1b-p0-profile-summary.md new file mode 100644 index 0000000..4b8d8ed --- /dev/null +++ b/docs/perf/m1b-p0-profile-summary.md @@ -0,0 +1,7 @@ +# M1b-P0 Nsight Systems summary + +The P128 profiling build captured four complete physical steps (eight stages), including initialization and final reports. The steady hydro span contains 353 kernels across four streams: 192 flux, 64 update, 24 included CFL box reductions, 56 physical-boundary fill, seven copy, seven FillBoundary copy, and three reduction-finalization kernels. + +The hydro core is exactly three directional flux operations plus one update per box and stage: 32 physical core launches/stage for eight boxes. Flux kernels consume 1189.45 ms (90.8% of all traced GPU-kernel time), updates 51.54 ms, and included CFL box reductions 36.79 ms. No face-to-cell atomic exists; atomics are limited to diagnostic counters and AMReX reductions. + +The hydro span contains 226 `cudaStreamSynchronize` calls with 1228.75 ms inclusive time. There is no steady `cudaMalloc` or `cudaFree`; the only `cudaMalloc`, managed allocation, and host allocation occur during initialization, and frees occur at teardown. This synchronization topology is the material P0 limitation to correct before adding multilevel work. diff --git a/docs/perf/m1b-p0-raw-artifact-manifest.tsv b/docs/perf/m1b-p0-raw-artifact-manifest.tsv new file mode 100644 index 0000000..9717ef4 --- /dev/null +++ b/docs/perf/m1b-p0-raw-artifact-manifest.tsv @@ -0,0 +1,10 @@ +artifact location sha256 +external-manifest /home/tim/Downloads/blastgpu-perf-runs/m1b-p0-20260822T142031Z/manifests/manifest.sha256 0f6ebd27d1d5af10801cd0ca075eeb1f46b2d78d49b9190d3b67708e5ec07019 +nsight-P128 profiles/P128-steady.nsys-rep d96e1d3bf4be09adaa10356fb52069166b1ac66e524b500a150170eae5d7d210 +m0-sod-N256 references/sod-m0-n256.tsv 1bcc73742e2f94c326e1930bed0b29490143da0b711d1a1ac31e943b82bbcc68 +m0-sod-N512 references/sod-m0-n512.tsv a75a9edab9f67343f20482078447ca7a3df733fb1575fe6fea6b79b76119e85a +m0-sod-N1024 references/sod-m0-n1024.tsv 349c2cd47498ed18e70434d28b837ef33e2101e9b1fb39bfdc6024d7bcb6e91b +cuda-U0-log correctness/authoritative-cuda/u0.log 919659daac4ac31a1c41565c30619d765b8080666b60b85954ea77940e058ffa +cuda-SX-log correctness/authoritative-cuda/sx.log 8ade98882efcb55155db8fade66e775eb0c8b5ba9dc56e7f26cb8fd7ac9982a8 +cuda-SY-log correctness/authoritative-cuda/sy.log f47c6482ad030b4cb0890654463b93493f6f960bcdb58b09032fbd5e2102dd80 +cuda-SZ-log correctness/authoritative-cuda/sz.log fcf3b84b93e8011edaadb80b42b8eac49f2bc8ee3972d1cd164d9286d809dc53 diff --git a/docs/perf/m1b-p0-reference-generation.md b/docs/perf/m1b-p0-reference-generation.md new file mode 100644 index 0000000..39e64d7 --- /dev/null +++ b/docs/perf/m1b-p0-reference-generation.md @@ -0,0 +1,5 @@ +# M1b-P0 reference generation + +`m1b_p0_reference` runs the frozen M0 planar Sod case at N=256, 512, and 1024 using the exact primitive states, floors, van Leer MUSCL, HLLC, SSP-RK2, CFL 0.45, outflow boundaries, and final time 0.2. All three references reported zero violations. + +The N=256-to-512 normalized L1 differences were 0.00104 density, 0.00213 velocity, 0.000764 pressure, and 0.000830 energy. The N=512-to-1024 values decreased to 0.000566, 0.00106, 0.000377, and 0.000416. Tolerances in `m1b-p0-tolerances.json` were committed before performance measurements. diff --git a/docs/perf/m1b-p0-timing-results.tsv b/docs/perf/m1b-p0-timing-results.tsv new file mode 100644 index 0000000..1c795e5 --- /dev/null +++ b/docs/perf/m1b-p0-timing-results.tsv @@ -0,0 +1,4 @@ +case hydro_runs_s median_s mean_s min_s max_s cv_percent step_ms cell_stage_per_s face_eval_per_s +P64 3.733731498,3.748369142,3.741828144,3.749401300,3.731769491,3.733964577,3.749514113 3.741828144 3.741225466 3.731769491 3.749514113 0.1983 37.4183 14011547 42691432 +P128 30.147818350,30.393191330,30.336207410,30.255957270,30.277352760,30.445533850,30.464592920 30.336207410 30.331521984 30.147818350 30.464592920 0.3456 303.3621 13826066 41802246 +P192 103.314296300,103.567200900,103.784572600,103.234737200,102.828022400,102.880156900,102.059773500 103.234737200 103.095537114 102.059773500 103.784572600 0.5126 1032.3474 13712222 41350918 diff --git a/docs/perf/m1b-p0-tolerances.json b/docs/perf/m1b-p0-tolerances.json new file mode 100644 index 0000000..4298878 --- /dev/null +++ b/docs/perf/m1b-p0-tolerances.json @@ -0,0 +1,23 @@ +{ + "frozen_before_performance_measurement": "2026-08-22T15:34:00Z", + "sod_reference": { + "method": "M0 N=256/512/1024 restriction convergence", + "normalized_L1_max": {"rho": 0.003, "u": 0.003, "p": 0.003, "rhoE": 0.003}, + "normalized_Linf_max": {"rho": 0.05, "u": 0.25, "p": 0.05, "rhoE": 0.05}, + "rotated_absolute_Linf": 1e-12, + "zero_tangential_momentum_absolute": 1e-12 + }, + "uniform": { + "component_spread": 0.0, + "floor_violations": 0, + "nan_or_inf": 0 + }, + "sphere": { + "basis": "CPU 64^3 to 128^3 centerline convergence; 256^3 CPU was not practical", + "normalized_L1_64_to_128_max": {"rho": 0.02, "u": 0.06, "p": 0.02, "rhoE": 0.02}, + "axis_rotation_absolute_Linf": 1e-9, + "cpu_cuda_relative_Linf": 1e-12, + "floor_violations": 0, + "nan_or_inf": 0 + } +} diff --git a/inputs/m1b_p0_r0.inp b/inputs/m1b_p0_r0.inp new file mode 100644 index 0000000..1f0cf2d --- /dev/null +++ b/inputs/m1b_p0_r0.inp @@ -0,0 +1,20 @@ +max_step = 100000 +stop_time = 2e-4 +geometry.coord_sys = 0 +geometry.prob_lo = -0.5 -0.5 -0.5 +geometry.prob_hi = 0.5 0.5 0.5 +geometry.is_periodic = 0 0 0 +amr.n_cell = 128 128 128 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 64 +amr.regrid_int = -1 +amr.plot_int = -1 +blast.advance_mode = hydro3d +blast.problem = pressure_sphere +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-6 +blast.p_floor = 1.0 +ambient.p = 101325 +ambient.rho = 1.225 diff --git a/inputs/m1b_p0_sx.inp b/inputs/m1b_p0_sx.inp new file mode 100644 index 0000000..6f82b3c --- /dev/null +++ b/inputs/m1b_p0_sx.inp @@ -0,0 +1,18 @@ +max_step = 100000 +stop_time = 0.2 +geometry.coord_sys = 0 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 1 0.0625 0.0625 +geometry.is_periodic = 0 1 1 +amr.n_cell = 256 16 16 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 +amr.regrid_int = -1 +amr.plot_int = -1 +blast.advance_mode = hydro3d +blast.problem = sod_x +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-12 +blast.p_floor = 1e-12 diff --git a/inputs/m1b_p0_sy.inp b/inputs/m1b_p0_sy.inp new file mode 100644 index 0000000..6e8990b --- /dev/null +++ b/inputs/m1b_p0_sy.inp @@ -0,0 +1,18 @@ +max_step = 100000 +stop_time = 0.2 +geometry.coord_sys = 0 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 0.0625 1 0.0625 +geometry.is_periodic = 1 0 1 +amr.n_cell = 16 256 16 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 +amr.regrid_int = -1 +amr.plot_int = -1 +blast.advance_mode = hydro3d +blast.problem = sod_y +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-12 +blast.p_floor = 1e-12 diff --git a/inputs/m1b_p0_sz.inp b/inputs/m1b_p0_sz.inp new file mode 100644 index 0000000..52c7305 --- /dev/null +++ b/inputs/m1b_p0_sz.inp @@ -0,0 +1,18 @@ +max_step = 100000 +stop_time = 0.2 +geometry.coord_sys = 0 +geometry.prob_lo = 0 0 0 +geometry.prob_hi = 0.0625 0.0625 1 +geometry.is_periodic = 1 1 0 +amr.n_cell = 16 16 256 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 256 +amr.regrid_int = -1 +amr.plot_int = -1 +blast.advance_mode = hydro3d +blast.problem = sod_z +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-12 +blast.p_floor = 1e-12 diff --git a/inputs/m1b_p0_u0.inp b/inputs/m1b_p0_u0.inp new file mode 100644 index 0000000..f58b9cd --- /dev/null +++ b/inputs/m1b_p0_u0.inp @@ -0,0 +1,21 @@ +max_step = 100 +stop_time = 1.0 +geometry.coord_sys = 0 +geometry.prob_lo = -1 -1 -1 +geometry.prob_hi = 1 1 1 +geometry.is_periodic = 1 1 1 +amr.n_cell = 64 64 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 64 +amr.regrid_int = -1 +amr.plot_int = -1 +amr.v = 0 +blast.advance_mode = hydro3d +blast.problem = uniform +blast.cfl = 0.45 +blast.alpha_floor = 1e-8 +blast.rho_floor = 1e-6 +blast.p_floor = 1.0 +ambient.p = 101325 +ambient.rho = 1.225 diff --git a/src/amr/BlastAmr.H b/src/amr/BlastAmr.H index 2c8ac5d..9f12278 100644 --- a/src/amr/BlastAmr.H +++ b/src/amr/BlastAmr.H @@ -14,6 +14,7 @@ #include "MieGruneisen.H" #include "State.H" +#include "Hydro3D.H" #include #include @@ -27,6 +28,12 @@ namespace amr { enum : int { ALPHA1 = 0, ARHO1 = 1, ARHO2 = 2, RHOU = 3, RHOV = 4, RHOW = 5, RHOE = 6 }; inline const char* const kVarNames[NCOMP] = { "alpha1", "arho1", "arho2", "rhou", "rhov", "rhow", "rhoE" }; } +static_assert(amr::NCOMP == hydro::NVAR3D); +static_assert(amr::ALPHA1 == hydro::C3_ALPHA1 && amr::ARHO1 == hydro::C3_ARHO1 && + amr::ARHO2 == hydro::C3_ARHO2 && amr::RHOU == hydro::C3_RHOU && + amr::RHOV == hydro::C3_RHOV && amr::RHOW == hydro::C3_RHOW && + amr::RHOE == hydro::C3_RHOE); +static_assert(hydro::FACE3_ALPHA1 == hydro::C3_ALPHA1 && hydro::FACE3_RHOE == hydro::C3_RHOE); class BlastAmr : public amrex::AmrCore { @@ -68,6 +75,7 @@ private: void GetData(int lev, amrex::Real time, amrex::Vector& data, amrex::Vector& datatime); void WritePlotFile() const; + void WriteValidationProfile() const; std::string PlotFileName(int step) const; // ---- configuration ---- @@ -84,12 +92,17 @@ private: amrex::Real ambient_p = 101325.0, ambient_rho = 1.225; amrex::Real tag_radius = -1.0; // M1a: static spherical refinement region amrex::RealArray tag_center{{0.0, 0.0, 0.0}}; + std::string advance_mode{"identity"}; + std::string problem{"uniform"}; + std::string validation_file; + amrex::Real hydro_wall_time = 0.0; // ---- per-level data ---- amrex::Vector istep, nsubsteps; amrex::Vector t_new, t_old, dt; amrex::Vector phi_new, phi_old; amrex::Vector> flux_reg; + amrex::Vector> hydro3d; amrex::Vector bcs; }; diff --git a/src/amr/BlastAmr.cpp b/src/amr/BlastAmr.cpp index 9aa090c..629824f 100644 --- a/src/amr/BlastAmr.cpp +++ b/src/amr/BlastAmr.cpp @@ -8,6 +8,9 @@ #include #include #include +#include +#include +#include using namespace amrex; @@ -61,6 +64,7 @@ BlastAmr::BlastAmr() phi_new.resize(nlevs_max); phi_old.resize(nlevs_max); flux_reg.resize(nlevs_max + 1); + hydro3d.resize(nlevs_max); bcs.resize(amr::NCOMP); for (int n = 0; n < amr::NCOMP; ++n) @@ -93,6 +97,9 @@ void BlastAmr::ReadParameters() pp.query("alpha_floor", fl.alpha); pp.query("rho_floor", fl.rho); pp.query("p_floor", fl.p); + pp.query("advance_mode", advance_mode); + pp.query("problem", problem); + pp.query("validation_file", validation_file); } { ParmParse pp("ambient"); @@ -112,6 +119,11 @@ void BlastAmr::MakeNewLevelFromScratch(int lev, Real time, const BoxArray& ba, c t_new[lev] = time; t_old[lev] = time - 1.e200; if (lev > 0) flux_reg[lev] = std::make_unique(ba, dm, refRatio(lev - 1), lev, amr::NCOMP); + if (advance_mode == "hydro3d") { + AMREX_ALWAYS_ASSERT_WITH_MESSAGE(max_level == 0, "hydro3d P0 requires amr.max_level=0"); + hydro3d[lev] = std::make_unique(); + hydro3d[lev]->define(ba,dm,nghost); + } InitLevelData(lev); } @@ -142,28 +154,41 @@ void BlastAmr::ClearLevel(int lev) phi_new[lev].clear(); phi_old[lev].clear(); flux_reg[lev].reset(nullptr); + hydro3d[lev].reset(nullptr); } // M1a initial condition: uniform ambient air at rest, trace products at rho0/10 // (same convention as Solver1D). M1c replaces this with the remapped 1D profile. void BlastAmr::InitLevelData(int lev) { - hydro::Prim w; - w.a1 = 1.0 - fl.alpha; w.rho1 = ambient_rho; w.rho2 = mat.m2.rho0 / 10.0; w.u = 0.0; w.p = ambient_p; - const hydro::Cons c = hydro::prim_to_cons(mat, w); + const hydro::Prim3D ambient{1.0-fl.alpha,ambient_rho,mat.m2.rho0/10.0,0.0,0.0,0.0,ambient_p}; + const hydro::Prim3D sod_l{1.0-fl.alpha,1.0,1.0,0.0,0.0,0.0,1.0}; + const hydro::Prim3D sod_r{1.0-fl.alpha,0.125,1.0,0.0,0.0,0.0,0.1}; + const hydro::Cons3D ca=hydro::prim_to_cons3d(mat,ambient); + const hydro::Cons3D cl=hydro::prim_to_cons3d(mat,sod_l); + const hydro::Cons3D cr=hydro::prim_to_cons3d(mat,sod_r); + const auto problo=Geom(lev).ProbLoArray(), dx=Geom(lev).CellSizeArray(); + const hydro::Materials m=mat; + const Real pambient=ambient_p; + const int prob = problem=="sod_x" ? 1 : problem=="sod_y" ? 2 : + problem=="sod_z" ? 3 : problem=="pressure_sphere" ? 4 : 0; MultiFab& state = phi_new[lev]; for (MFIter mfi(state, TilingIfNotGPU()); mfi.isValid(); ++mfi) { const Box& bx = mfi.tilebox(); Array4 const& s = state.array(mfi); ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - s(i,j,k,amr::ALPHA1) = c.q[hydro::ALPHA1]; - s(i,j,k,amr::ARHO1) = c.q[hydro::ARHO1]; - s(i,j,k,amr::ARHO2) = c.q[hydro::ARHO2]; - s(i,j,k,amr::RHOU) = 0.0; - s(i,j,k,amr::RHOV) = 0.0; - s(i,j,k,amr::RHOW) = 0.0; - s(i,j,k,amr::RHOE) = c.q[hydro::RHOE]; + const Real x=problo[0]+(i+0.5)*dx[0], y=problo[1]+(j+0.5)*dx[1], z=problo[2]+(k+0.5)*dx[2]; + hydro::Cons3D c=ca; + if (prob==1) c=x<0.5?cl:cr; + else if (prob==2) c=y<0.5?cl:cr; + else if (prob==3) c=z<0.5?cl:cr; + else if (prob==4) { + hydro::Prim3D w=ambient; + if(x*x+y*y+z*z<0.01) w.p=2.0*pambient; + c=hydro::prim_to_cons3d(m,w); + } + for(int n=0;n reduce_op; @@ -214,12 +240,17 @@ Real BlastAmr::EstTimeStep(int lev) const const Box& bx = mfi.tilebox(); Array4 const& s = phi_new[lev].const_array(mfi); reduce_op.eval(bx, reduce_data, [=] AMREX_GPU_DEVICE (int i, int j, int k) -> ReduceTuple { - return { max_wave_speed(s, i, j, k, m, f) }; + if (!hydro_mode) return { max_wave_speed(s, i, j, k, m, f) }; + hydro::Cons3D c3; + for(int n=0;n(reduce_data.value(reduce_op)); ParallelDescriptor::ReduceRealMax(smax); - return cfl * dxmin / smax; + return hydro_mode ? cfl/smax : cfl*dxmin/smax; } void BlastAmr::ComputeDt() @@ -245,6 +276,8 @@ void BlastAmr::Evolve() { Real cur_time = t_new[0]; int last_plot_file_step = 0; + if (advance_mode=="hydro3d") Gpu::streamSynchronize(); + const Real wall_start=amrex::second(); for (int step = istep[0]; step < max_step && cur_time < stop_time; ++step) { ComputeDt(); TimeStepWithSubcycling(0, cur_time, 1); @@ -254,7 +287,12 @@ void BlastAmr::Evolve() if (plot_int > 0 && (step + 1) % plot_int == 0) { last_plot_file_step = step + 1; WritePlotFile(); } if (cur_time >= stop_time - 1.e-6 * dt[0]) break; } + if (advance_mode=="hydro3d") Gpu::streamSynchronize(); + hydro_wall_time=amrex::second()-wall_start; + amrex::Print() << "evolution steps=" << istep[0] << " final_time=" << cur_time + << " hydro_wall=" << hydro_wall_time << "\n"; if (plot_int > 0 && istep[0] > last_plot_file_step) WritePlotFile(); + if (advance_mode=="hydro3d" && !validation_file.empty()) WriteValidationProfile(); } void BlastAmr::TimeStepWithSubcycling(int lev, Real time, int iteration) @@ -286,9 +324,19 @@ void BlastAmr::TimeStepWithSubcycling(int lev, Real time, int iteration) // M1a: identity advance. Still exercises the FillPatch + ghost-cell path that // M1b's hydro will consume, so regridding and coarse-fine interpolation are // proven before any physics arrives. -void BlastAmr::AdvanceLevel(int lev, Real time, Real /*dt_lev*/) +void BlastAmr::AdvanceLevel(int lev, Real time, Real dt_lev) { std::swap(phi_old[lev], phi_new[lev]); + if (advance_mode == "hydro3d") { + auto& h=*hydro3d[lev]; + h.fillBorder(phi_old[lev],Geom(lev)); + h.computeFluxes(Geom(lev),mat,fl); + h.update(phi_old[lev],phi_new[lev],Geom(lev),mat,fl,dt_lev,1); + h.fillBorder(phi_new[lev],Geom(lev)); + h.computeFluxes(Geom(lev),mat,fl); + h.update(phi_old[lev],phi_new[lev],Geom(lev),mat,fl,dt_lev,2); + return; + } MultiFab Sborder(grids[lev], dmap[lev], amr::NCOMP, nghost); FillPatch(lev, time, Sborder, 0, amr::NCOMP); MultiFab::Copy(phi_new[lev], Sborder, 0, 0, amr::NCOMP, 0); @@ -367,6 +415,32 @@ void BlastAmr::WritePlotFile() const amrex::WriteMultiLevelPlotfile(name, finest_level + 1, mf, names, Geom(), t_new[0], istep, refRatio()); } +void BlastAmr::WriteValidationProfile() const +{ + MultiFab host(grids[0],dmap[0],amr::NCOMP,0,MFInfo().SetArena(The_Pinned_Arena())); + MultiFab::Copy(host,phi_new[0],0,0,amr::NCOMP,0); + Gpu::streamSynchronize(); + const Box domain=Geom(0).Domain(); const IntVect mid=(domain.smallEnd()+domain.bigEnd())/2; + const auto lo=Geom(0).ProbLoArray(), dx=Geom(0).CellSizeArray(); + struct Row { int axis,index; Real x; hydro::Cons3D c; hydro::Prim3D w; }; + std::vector rows; + for(MFIter mfi(host);mfi.isValid();++mfi) { + const Box bx=mfi.validbox(); auto a=host.const_array(mfi); + for(int axis=0;axis<3;++axis) for(int n=domain.smallEnd(axis);n<=domain.bigEnd(axis);++n) { + IntVect iv=mid; iv[axis]=n; if(!bx.contains(iv)) continue; + hydro::Cons3D c; for(int q=0;qcounters(); + amrex::Print() << " hydro3d counters alpha=" << c[hydro::ALPHA_VIOL] + << " density=" << c[hydro::DENSITY_VIOL] + << " pressure=" << c[hydro::PRESSURE_VIOL] + << " nan=" << c[hydro::NAN_VIOL] + << " inf=" << c[hydro::INF_VIOL] << "\n"; + const Real dv=Geom(0).CellSize(0)*Geom(0).CellSize(1)*Geom(0).CellSize(2); + amrex::Print() << " hydro3d totals"; + for(int n=0;n +#include +#include +#include +#include "Reconstruct.H" +#include "Riemann.H" + +namespace blastgpu::hydro { + +enum Counter3D : int { ALPHA_VIOL=0, DENSITY_VIOL=1, PRESSURE_VIOL=2, + NAN_VIOL=3, INF_VIOL=4, NCOUNTER3D=5 }; + +class Hydro3D { +public: + Hydro3D() = default; + void define(const amrex::BoxArray&, const amrex::DistributionMapping&, int ngrow); + void clear(); + bool isDefined() const noexcept { return border_.isDefined(); } + + // Public CUDA-launch hooks: NVCC 12.6 requires public access for methods + // enclosing extended device lambdas. These are internal, not stable API. + void fillBorder(const amrex::MultiFab&, const amrex::Geometry&); + void computeFluxes(const amrex::Geometry&, const Materials&, const Floors&); + void update(const amrex::MultiFab& u0, amrex::MultiFab& out, + const amrex::Geometry&, const Materials&, const Floors&, + amrex::Real dt, int rk_stage); + void resetCounters(); + std::array counters() const; + + const amrex::MultiFab& faceFlux(int d) const { return flux_[d]; } + amrex::MultiFab& border() { return border_; } + +private: + amrex::MultiFab border_; + std::array flux_; + amrex::Gpu::DeviceVector counters_; +}; + +} // namespace blastgpu::hydro +#endif diff --git a/src/hydro/Hydro3D.cpp b/src/hydro/Hydro3D.cpp new file mode 100644 index 0000000..05589e6 --- /dev/null +++ b/src/hydro/Hydro3D.cpp @@ -0,0 +1,147 @@ +#include "Hydro3D.H" +#include +#include +#include +#include +#include +#include + +using namespace amrex; + +namespace blastgpu::hydro { +namespace { + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +Cons3D load3(Array4 const& a, int i, int j, int k) { + Cons3D c; + for (int n=0; n const& a, int i, int j, int k, + const Materials& mat, const Floors& fl) { + return cons_to_prim3d(mat,load3(a,i,j,k),fl); +} + +AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE +void shift(int d, int n, int& i, int& j, int& k) { + if (d==0) i+=n; else if (d==1) j+=n; else k+=n; +} + +} // namespace + +void Hydro3D::define(const BoxArray& ba, const DistributionMapping& dm, int ngrow) { + border_.define(ba,dm,NVAR3D,ngrow); + for (int d=0; d<3; ++d) { + BoxArray fba=ba; fba.convert(IntVect::TheDimensionVector(d)); + flux_[d].define(fba,dm,NFACE3D,0); + } + counters_.resize(NCOUNTER3D); + resetCounters(); +} + +void Hydro3D::clear() { + border_.clear(); + for (auto& f: flux_) f.clear(); + counters_.clear(); +} + +void Hydro3D::resetCounters() { + Gpu::fillAsync(counters_.begin(),counters_.end(), + [] AMREX_GPU_DEVICE (unsigned long long& value, Long) noexcept { value=0ULL; }); +} + +std::array Hydro3D::counters() const { + std::array h{}; + Gpu::copy(Gpu::deviceToHost,counters_.begin(),counters_.end(),h.begin()); + return h; +} + +void Hydro3D::fillBorder(const MultiFab& src, const Geometry& geom) { + MultiFab::Copy(border_,src,0,0,NVAR3D,0); + border_.FillBoundary(geom.periodicity()); + const Box domain=geom.Domain(); + const auto dlo=domain.smallEnd(), dhi=domain.bigEnd(); + const auto periodic=geom.isPeriodicArray(); + for (MFIter mfi(border_); mfi.isValid(); ++mfi) { + const Box bx=mfi.fabbox(); + auto a=border_.array(mfi); + ParallelFor(bx,NVAR3D,[=] AMREX_GPU_DEVICE(int i,int j,int k,int n) noexcept { + int ii=i,jj=j,kk=k; + if (!periodic[0]) ii=amrex::max(dlo[0],amrex::min(dhi[0],ii)); + if (!periodic[1]) jj=amrex::max(dlo[1],amrex::min(dhi[1],jj)); + if (!periodic[2]) kk=amrex::max(dlo[2],amrex::min(dhi[2],kk)); + if (ii!=i || jj!=j || kk!=k) a(i,j,k,n)=a(ii,jj,kk,n); + }); + } +} + +void Hydro3D::computeFluxes(const Geometry&, const Materials& mat, const Floors& fl) { + const Materials m=mat; const Floors f=fl; + for (int d=0; d<3; ++d) { + for (MFIter mfi(border_); mfi.isValid(); ++mfi) { + const Box bx=surroundingNodes(mfi.validbox(),d); + auto s=border_.const_array(mfi); auto ff=flux_[d].array(mfi); + ParallelFor(bx,[=] AMREX_GPU_DEVICE(int i,int j,int k) noexcept { + int lm_i=i,lm_j=j,lm_k=k; shift(d,-2,lm_i,lm_j,lm_k); + int l_i=i,l_j=j,l_k=k; shift(d,-1,l_i,l_j,l_k); + int r_i=i,r_j=j,r_k=k; + int rp_i=i,rp_j=j,rp_k=k; shift(d,1,rp_i,rp_j,rp_k); + const Prim3D wm=primitive_at(s,lm_i,lm_j,lm_k,m,f); + const Prim3D wl=primitive_at(s,l_i,l_j,l_k,m,f); + const Prim3D wr=primitive_at(s,r_i,r_j,r_k,m,f); + const Prim3D wp=primitive_at(s,rp_i,rp_j,rp_k,m,f); + const Prim3D left=extrapolate3d(wl,limited_slope3d(wm,wl,wr),1.0,f); + const Prim3D right=extrapolate3d(wr,limited_slope3d(wl,wr,wp),-1.0,f); + const RiemannResult3D rr=hllc3d(m,left,right,d); + for (int n=0;n16.0*std::numeric_limits::epsilon()) Gpu::Atomic::Add(&count[ALPHA_VIOL],1ULL); + const Real a=next[C3_ALPHA1]; + const Real r1min=f.rho*a, r2min=f.rho*(1.0-a); + if(next[C3_ARHO1]= 0) { for(int n=0;n= 0 ? wL : wR; + const Cons3D& c = ss >= 0 ? cL : cR; + const Real* f = ss >= 0 ? FL : FR; + const Real rho = ss >= 0 ? rhoL : rhoR; + const Real S = ss >= 0 ? SL : SR; + const Real un = ss >= 0 ? unL : unR; + const Real fac=(S-un)/(S-ss); + Cons3D us=c; + us.q[C3_ALPHA1]=c.q[C3_ALPHA1]*fac; + us.q[C3_ARHO1]=c.q[C3_ARHO1]*fac; us.q[C3_ARHO2]=c.q[C3_ARHO2]*fac; + us.q[C3_RHOU]=c.q[C3_RHOU]*fac; us.q[C3_RHOV]=c.q[C3_RHOV]*fac; us.q[C3_RHOW]=c.q[C3_RHOW]*fac; + us.q[C3_RHOU+d]=rho*fac*ss; + us.q[C3_RHOE]=rho*fac*(c.q[C3_RHOE]/rho+(ss-un)*(ss+w.p/(rho*(S-un)))); + for(int n=0;n= 0 ? wL.a1 : wR.a1)*r.sstar; + return r; +} + BLASTGPU_HD RiemannResult hllc(const Materials& mat, const Prim& wL, const Prim& wR) { const Cons cL = prim_to_cons(mat, wL); const Cons cR = prim_to_cons(mat, wR); diff --git a/src/hydro/State.H b/src/hydro/State.H index 1119580..3aecb1d 100644 --- a/src/hydro/State.H +++ b/src/hydro/State.H @@ -21,6 +21,16 @@ enum : int { ALPHA1 = 0, ARHO1 = 1, ARHO2 = 2, RHOU = 3, RHOE = 4 }; struct Cons { Real q[NVAR]; }; struct Prim { Real a1, rho1, rho2, u, p; }; +// Explicit M1b three-dimensional state. The M0 types above intentionally +// retain their frozen one-dimensional meaning. +constexpr int NVAR3D = 7; +enum : int { + C3_ALPHA1 = 0, C3_ARHO1 = 1, C3_ARHO2 = 2, + C3_RHOU = 3, C3_RHOV = 4, C3_RHOW = 5, C3_RHOE = 6 +}; +struct Cons3D { Real q[NVAR3D]; }; +struct Prim3D { Real a1, rho1, rho2, u, v, w, p; }; + struct Materials { MGMaterial m1; // air MGMaterial m2; // products @@ -74,6 +84,36 @@ BLASTGPU_HD Real sound_speed(const Materials& mat, const Prim& w) { return c2 > 0.0 ? std::sqrt(c2) : 0.0; } +BLASTGPU_HD Prim3D cons_to_prim3d(const Materials& mat, const Cons3D& c, const Floors& fl) { + const Real rho = c.q[C3_ARHO1] + c.q[C3_ARHO2]; + const Real u = c.q[C3_RHOU] / rho; + const Real v = c.q[C3_RHOV] / rho; + const Real wv = c.q[C3_RHOW] / rho; + const Real rhoe = c.q[C3_RHOE] - 0.5 * rho * (u*u + v*v + wv*wv); + const Real a1 = clamp_alpha(c.q[C3_ALPHA1], fl.alpha); + const MixtureState s = mixture_pressure(mat.m1, mat.m2, a1, + c.q[C3_ARHO1], c.q[C3_ARHO2], rhoe, fl.alpha); + return {a1, s.rho1, s.rho2, u, v, wv, s.p}; +} + +BLASTGPU_HD Cons3D prim_to_cons3d(const Materials& mat, const Prim3D& w) { + const Real a2 = 1.0 - w.a1; + const Real arho1 = w.a1 * w.rho1; + const Real arho2 = a2 * w.rho2; + const Real rho = arho1 + arho2; + const Real rhoe = arho1 * internal_energy(mat.m1, w.rho1, w.p) + + arho2 * internal_energy(mat.m2, w.rho2, w.p); + Cons3D c; + c.q[C3_ALPHA1] = w.a1; c.q[C3_ARHO1] = arho1; c.q[C3_ARHO2] = arho2; + c.q[C3_RHOU] = rho*w.u; c.q[C3_RHOV] = rho*w.v; c.q[C3_RHOW] = rho*w.w; + c.q[C3_RHOE] = rhoe + 0.5*rho*(w.u*w.u + w.v*w.v + w.w*w.w); + return c; +} + +BLASTGPU_HD Real sound_speed3d(const Materials& mat, const Prim3D& w) { + return sound_speed(mat, Prim{w.a1, w.rho1, w.rho2, w.u, w.p}); +} + // Physical flux in the x (or r) direction. alpha1 is carried in flux form // (alpha1*u) with the compensating alpha1*div(u) source applied by the solver, // which is what makes HLLC usable for the non-conservative equation. diff --git a/tests/m1b_p0_check.py b/tests/m1b_p0_check.py new file mode 100644 index 0000000..e867f05 --- /dev/null +++ b/tests/m1b_p0_check.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import argparse,re,subprocess,sys +p=argparse.ArgumentParser(); p.add_argument("--uniform",action="store_true"); p.add_argument("executable"); p.add_argument("input"); p.add_argument("overrides",nargs="*"); a=p.parse_args() +r=subprocess.run([a.executable,a.input,*a.overrides],text=True,stdout=subprocess.PIPE,stderr=subprocess.STDOUT) +print(r.stdout,end="") +if r.returncode: raise SystemExit(r.returncode) +m=re.findall(r"hydro3d counters alpha=(\d+) density=(\d+) pressure=(\d+) nan=(\d+) inf=(\d+)",r.stdout) +if not m or any(int(x) for x in m[-1]): raise SystemExit("nonzero or missing hydro3d counters") +if "evolution steps=" not in r.stdout: raise SystemExit("missing evolution report") +if a.uniform: + final=r.stdout.split("final: finest_level",1)[-1] + spreads=[float(x) for x in re.findall(r"spread = ([^\s]+)",final)] + if len(spreads)<7 or any(x!=0 for x in spreads[:7]): raise SystemExit("uniform state changed") +print("ALL PASSED") diff --git a/tests/m1b_p0_compare_axes.py b/tests/m1b_p0_compare_axes.py new file mode 100644 index 0000000..5d357cb --- /dev/null +++ b/tests/m1b_p0_compare_axes.py @@ -0,0 +1,14 @@ +#!/usr/bin/env python3 +import argparse,csv,math + +def axis(path,d): + with open(path,newline="") as f: return [{k:float(v) for k,v in r.items()} for r in csv.DictReader(f,delimiter="\t") if int(r["axis"])==d] +p=argparse.ArgumentParser(); p.add_argument("sx"); p.add_argument("sy"); p.add_argument("sz"); p.add_argument("--tolerance",type=float,default=1e-13); a=p.parse_args() +x,y,z=axis(a.sx,0),axis(a.sy,1),axis(a.sz,2); failed=False +for name in ("alpha1","arho1","arho2","rho","p","rhoE"): + err=max(abs(q[name]-r[name]) for q,r in zip(x,y)) + err=max(err,max(abs(q[name]-r[name]) for q,r in zip(x,z))) + print(f"{name} rotated Linf={err:.17g}"); failed |= err>a.tolerance +for data,normal in ((x,"u"),(y,"v"),(z,"w")): + print(f"normal {normal} max={max(abs(r[normal]) for r in data):.9g}") +print("ALL PASSED" if not failed else "FAILED"); raise SystemExit(failed) diff --git a/tests/m1b_p0_compare_reference.py b/tests/m1b_p0_compare_reference.py new file mode 100644 index 0000000..a517259 --- /dev/null +++ b/tests/m1b_p0_compare_reference.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python3 +import argparse,csv,math + +def rows(path): + with open(path,newline="") as f: return [{k:float(v) for k,v in r.items()} for r in csv.DictReader(f,delimiter="\t")] + +p=argparse.ArgumentParser(); p.add_argument("profile"); p.add_argument("reference"); p.add_argument("--axis",type=int,default=0); p.add_argument("--tolerance",type=float,default=0.08); a=p.parse_args() +q=[r for r in rows(a.profile) if int(r["axis"])==a.axis]; ref=rows(a.reference) +if len(q)!=len(ref): raise SystemExit(f"size mismatch {len(q)} != {len(ref)}") +failed=False +for name in ("rho","u","p","rhoE"): + d=[x[name]-y[name] for x,y in zip(q,ref)]; scale=max(max(abs(y[name]) for y in ref),1e-30) + l1=sum(abs(x) for x in d)/len(d)/scale; l2=math.sqrt(sum(x*x for x in d)/len(d))/scale; li=max(abs(x) for x in d)/scale + print(f"{name} L1={l1:.9g} L2={l2:.9g} Linf={li:.9g}") + failed |= l1>a.tolerance +print("ALL PASSED" if not failed else "FAILED") +raise SystemExit(failed) diff --git a/tests/test_hydro.cpp b/tests/test_hydro.cpp index 8db0786..3469290 100644 --- a/tests/test_hydro.cpp +++ b/tests/test_hydro.cpp @@ -19,6 +19,28 @@ static void run_for(Solver1D& s, double t_end) { } int main() { + // M1b-P0 additive 3D algebra: M0 interfaces above remain unchanged. + { + Materials mat{materials::AIR, materials::TNT}; Floors fl; fl.rho=1e-12; fl.p=1e-12; + Prim3D w{0.73,1.2,100.0,2.0,-3.0,4.0,2.5e5}; + Cons3D c=prim_to_cons3d(mat,w); Prim3D q=cons_to_prim3d(mat,c,fl); + CHECK_CLOSE(q.a1,w.a1,1e-13,"3D state: alpha roundtrip"); + CHECK_CLOSE(q.u,w.u,1e-13,"3D state: u roundtrip"); + CHECK_CLOSE(q.v,w.v,1e-13,"3D state: v roundtrip"); + CHECK_CLOSE(q.w,w.w,1e-13,"3D state: w roundtrip"); + CHECK_CLOSE(q.p,w.p,1e-12,"3D state: pressure roundtrip with three-component kinetic energy"); + Prim3D a{0.9,1.0,1.0,0.3,0.0,0.0,1.0}; + auto rx=hllc3d(mat,a,a,0), ry=hllc3d(mat,Prim3D{a.a1,a.rho1,a.rho2,0,a.u,0,a.p},Prim3D{a.a1,a.rho1,a.rho2,0,a.u,0,a.p},1); + CHECK_CLOSE(rx.F[C3_RHOU],ry.F[C3_RHOV],1e-13,"3D HLLC: axis-permuted normal momentum"); + CHECK_CLOSE(rx.F[C3_RHOE],ry.F[C3_RHOE],1e-13,"3D HLLC: axis-permuted energy"); + CHECK_CLOSE(rx.F[C3_RHOV],0.0,0.0,"3D HLLC: zero tangential momentum flux"); + Prim3D t=a; t.v=7.0; auto rt=hllc3d(mat,t,t,0); auto ct=prim_to_cons3d(mat,t); + CHECK_CLOSE(rt.F[C3_RHOV],ct.q[C3_RHOV]*t.u,1e-13,"3D HLLC: tangential momentum advection"); + CHECK_CLOSE(rx.F[C3_ALPHA1],a.a1*rx.sstar,1e-13,"3D HLLC: alpha/contact-speed pairing"); + const double al=0.37, slo=-2.0, shi=3.0; + const double paired=-((al*shi-al*shi)-(al*slo-al*slo)); + CHECK(paired==0.0,"3D alpha source: manufactured face-by-face cancellation"); + } // 1. Uniform ambient air at rest in spherical geometry stays uniform: the // r^2-weighted fluxes and the 2p/r source must cancel exactly. { diff --git a/tools/m1b_p0_reference.cpp b/tools/m1b_p0_reference.cpp new file mode 100644 index 0000000..5f2d1e1 --- /dev/null +++ b/tools/m1b_p0_reference.cpp @@ -0,0 +1,23 @@ +#include "Solver1D.H" +#include +#include + +using namespace blastgpu::hydro; + +int main(int argc,char** argv) { + if(argc!=3) { std::fprintf(stderr,"usage: m1b_p0_reference N output.tsv\n"); return 2; } + Solver1D s; s.N=std::atoi(argv[1]); s.r_max=1.0; s.geom=Geometry::Planar; + s.bc_lo=s.bc_hi=BC::Outflow; s.cfl=0.45; s.fl.alpha=1e-8; s.fl.rho=1e-12; s.fl.p=1e-12; + const Prim l{1.0-s.fl.alpha,1.0,1.0,0.0,1.0}; + const Prim r{1.0-s.fl.alpha,0.125,1.0,0.0,0.1}; + s.init_two_state(l,r,0.5); + while(s.t<0.2) { auto dt=s.compute_dt(); if(s.t+dt>0.2)dt=0.2-s.t; s.advance(dt); } + FILE* f=std::fopen(argv[2],"w"); if(!f)return 3; + std::fprintf(f,"x\talpha1\trho1\trho2\trho\tu\tp\trhoE\n"); + for(int i=0;i