From 8cbaf8744909e9bfc5d873317b7cf6d5f8be3e39 Mon Sep 17 00:00:00 2001 From: okuznetsov Date: Thu, 25 Jun 2026 15:28:58 +0200 Subject: [PATCH 1/2] Isolate render passes during replay (#3016) Add a --isolate-render-passes replay option backed by a new VulkanCommandSplitter, which splits a command buffer at each render pass boundary and submits the segments separately, chained through a semaphore. It is hooked into OverrideCmdBegin/EndRenderPass, OverrideQueueSubmit, OverrideBeginCommandBuffer, and OverrideResetCommandBuffer. Also adds the capture_and_replay() test helper, which captures a test app and replays the resulting gfxr headless against the mock ICD, plus an isolate-render-passes test app and test case exercising it. Cherry-pick notes: this fork already declares VulkanCommandPoolInfo as a struct with a `flags` member (alongside `queue_family_index`, used by the frame loop consumer), so the upstream `create_flags` member was dropped and its two uses now read the pre-existing field. (cherry picked from commit 9a83f8ab7d399da3a6bbdf18a8c3c920e0f9f5d4) Change-Id: I8ade75b52d52d66bdd6ccfaf34d5f7fd6a6a6964 --- USAGE_android.md | 3 + USAGE_desktop_Vulkan.md | 3 + android/scripts/gfxrecon.py | 4 + android/test/test_apps/common/CMakeLists.txt | 1 + .../test/test_apps/launcher/CMakeLists.txt | 1 + framework/decode/CMakeLists.txt | 2 + framework/decode/vulkan_command_splitter.cpp | 372 ++++++++++++++++++ framework/decode/vulkan_command_splitter.h | 153 +++++++ .../decode/vulkan_replay_consumer_base.cpp | 121 +++++- .../decode/vulkan_replay_consumer_base.h | 10 + framework/decode/vulkan_replay_options.h | 3 + framework/decode/vulkan_submit_info_helper.h | 9 +- .../generated_vulkan_replay_consumer.cpp | 12 +- .../vulkan_generators/replay_overrides.json | 1 + test/CMakeLists.txt | 4 +- test/run-tests_macos.sh | 1 + test/test_apps/CMakeLists.txt | 1 + .../isolate-render-passes/CMakeLists.txt | 39 ++ test/test_apps/isolate-render-passes/app.cpp | 366 +++++++++++++++++ .../isolate_render_passes_app.h | 71 ++++ test/test_apps/launcher/CMakeLists.txt | 1 + test/test_apps/launcher/test_launcher.cpp | 8 +- test/test_cases/isolate-render-passes.cpp | 14 + test/verify-gfxr.cpp | 40 ++ test/verify-gfxr.h | 27 +- tools/replay/replay_settings.h | 5 +- tools/tool_settings.h | 2 + 27 files changed, 1254 insertions(+), 20 deletions(-) create mode 100644 framework/decode/vulkan_command_splitter.cpp create mode 100644 framework/decode/vulkan_command_splitter.h create mode 100644 test/test_apps/isolate-render-passes/CMakeLists.txt create mode 100644 test/test_apps/isolate-render-passes/app.cpp create mode 100644 test/test_apps/isolate-render-passes/isolate_render_passes_app.h create mode 100644 test/test_cases/isolate-render-passes.cpp diff --git a/USAGE_android.md b/USAGE_android.md index 9b2996141a..654f429ef6 100644 --- a/USAGE_android.md +++ b/USAGE_android.md @@ -800,6 +800,7 @@ usage: gfxrecon.py replay [-h] [-p LOCAL_FILE] [--version] [--log-level LEVEL] [--serialize-queue-submissions] [--replay-event-plugin-path PATH] [--replay-event-plugin-params PARAMS] + [--isolate-render-passes] [file] Launch the replay tool. @@ -1043,6 +1044,8 @@ options: Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool) + --isolate-render-passes + Isolate render passes by splitting the command buffer into multiple submits. ``` The command will force-stop an active replay process before starting the replay diff --git a/USAGE_desktop_Vulkan.md b/USAGE_desktop_Vulkan.md index 98d6e4cf1c..203c133f43 100644 --- a/USAGE_desktop_Vulkan.md +++ b/USAGE_desktop_Vulkan.md @@ -639,6 +639,7 @@ gfxrecon-replay [-h | --help] [--version] [--cpu-mask ] [-- [--wait-before-frame MILLISECONDS] [--serialize-queue-submissions] [--replay-event-plugin-path ] [--replay-event-plugin-params ] + [--isolate-render-passes] Required arguments: @@ -904,6 +905,8 @@ Optional arguments: Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool) + --isolate-render-passes + Isolate render passes by splitting the command buffer into multiple submits. ``` ### Frame Warm-Up diff --git a/android/scripts/gfxrecon.py b/android/scripts/gfxrecon.py index 8cb5a91459..17a83eb952 100644 --- a/android/scripts/gfxrecon.py +++ b/android/scripts/gfxrecon.py @@ -155,6 +155,7 @@ def CreateReplayParser(): parser.add_argument('--serialize-queue-submissions', action='store_true', default=False, help='Serialize submit entries within one vkQueueSubmit or vkQueueSubmit2 call by adding semaphores between consecutive submits during replay. (forwarded to replay tool)') parser.add_argument('--replay-event-plugin-path', metavar='PATH', help='Path to a replay event plugin library. If specified, the plugin will be loaded and used to process replay events. (forwarded to replay tool)') parser.add_argument('--replay-event-plugin-params', metavar='PARAMS', help='Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool)') + parser.add_argument('--isolate-render-passes', action='store_true', default=False, help='Isolate render passes by splitting the command buffer into multiple submits. (forwarded to replay tool)') return parser @@ -375,6 +376,9 @@ def MakeExtrasString(args): arg_list.append('--replay-event-plugin-params') arg_list.append('{}'.format(args.replay_event_plugin_params)) + if args.isolate_render_passes: + arg_list.append('--isolate-render-passes') + if args.file: arg_list.append(args.file) elif not args.version: diff --git a/android/test/test_apps/common/CMakeLists.txt b/android/test/test_apps/common/CMakeLists.txt index 75aac63915..dc8922140e 100644 --- a/android/test/test_apps/common/CMakeLists.txt +++ b/android/test/test_apps/common/CMakeLists.txt @@ -61,6 +61,7 @@ add_test_app(debug-utils) add_test_app(external-memory-fd-export) add_test_app(external-memory-fd-import) add_test_app(host-image-copy) +add_test_app(isolate-render-passes) add_test_app(multisample-depth) add_test_app(pipeline-binaries) add_test_app(triangle) diff --git a/android/test/test_apps/launcher/CMakeLists.txt b/android/test/test_apps/launcher/CMakeLists.txt index 7c6b2ff08f..a3a52e7fab 100644 --- a/android/test/test_apps/launcher/CMakeLists.txt +++ b/android/test/test_apps/launcher/CMakeLists.txt @@ -52,6 +52,7 @@ set(GFXRECON_TEST_LAUNCHER_LINK_LIBRARIES gfxrecon-testapp-acquired-image gfxrecon-testapp-debug-utils gfxrecon-testapp-host-image-copy + gfxrecon-testapp-isolate-render-passes gfxrecon-testapp-multisample-depth gfxrecon-testapp-pipeline-binaries gfxrecon-testapp-shader-objects diff --git a/framework/decode/CMakeLists.txt b/framework/decode/CMakeLists.txt index b141923b98..c88e075d74 100644 --- a/framework/decode/CMakeLists.txt +++ b/framework/decode/CMakeLists.txt @@ -238,6 +238,8 @@ target_sources(gfxrecon_decode ${CMAKE_CURRENT_LIST_DIR}/vulkan_referenced_block_consumer_base.cpp ${CMAKE_CURRENT_LIST_DIR}/vulkan_remap_allocator.h ${CMAKE_CURRENT_LIST_DIR}/vulkan_remap_allocator.cpp + ${CMAKE_CURRENT_LIST_DIR}/vulkan_command_splitter.h + ${CMAKE_CURRENT_LIST_DIR}/vulkan_command_splitter.cpp ${CMAKE_CURRENT_LIST_DIR}/vulkan_replay_consumer_base.h ${CMAKE_CURRENT_LIST_DIR}/vulkan_replay_consumer_base.cpp ${CMAKE_CURRENT_LIST_DIR}/vulkan_replay_options.h diff --git a/framework/decode/vulkan_command_splitter.cpp b/framework/decode/vulkan_command_splitter.cpp new file mode 100644 index 0000000000..81b0571d8a --- /dev/null +++ b/framework/decode/vulkan_command_splitter.cpp @@ -0,0 +1,372 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ +#include "decode/vulkan_command_splitter.h" +#include "decode/vulkan_submit_info_helper.h" +#include "generated/generated_vulkan_enum_to_string.h" +#include "util/callbacks.h" +#include "util/logging.h" + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(decode) + +VulkanCommandBufferSplitInfo::VulkanCommandBufferSplitInfo(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table, + format::HandleId command_buffer_id) : + device_info_(device_info), + device_table_(device_table), + object_table_(object_table), + split_semaphore_(device_info, device_table) +{ + GFXRECON_ASSERT(device_info != nullptr); + GFXRECON_ASSERT(device_table != nullptr); + GFXRECON_ASSERT(object_table != nullptr); + + // Store the original command buffer handle and map it to the command buffer ID for future reference. + VulkanCommandBufferInfo* command_buffer_info = object_table_->GetVkCommandBufferInfo(command_buffer_id); + GFXRECON_ASSERT(command_buffer_info != nullptr); + original_handle_ = command_buffer_info->handle; +} + +VkCommandBuffer VulkanCommandBufferSplitInfo::GetNextHandle(const VulkanCommandBufferInfo* command_buffer_info) +{ + VkCommandBuffer next_handle = VK_NULL_HANDLE; + + // Create a new handle if there are none to reuse. + if (next_associated_index_ >= associated_handles_.size()) + { + VulkanCommandPoolInfo* pool_info = object_table_->GetVkCommandPoolInfo(command_buffer_info->pool_id); + + const VulkanDeviceInfo* device_info = object_table_->GetVkDeviceInfo(command_buffer_info->parent_id); + GFXRECON_ASSERT(device_info == device_info_); + + constexpr size_t NEW_COMMAND_BUFFER_COUNT = 8; + const size_t previous_size = associated_handles_.size(); + associated_handles_.resize(previous_size + NEW_COMMAND_BUFFER_COUNT); + VkCommandBuffer* new_handles_ptr = associated_handles_.data() + previous_size; + + // Create a bunch of new handles for the split. + VkCommandBufferAllocateInfo alloc_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; + alloc_info.commandPool = pool_info->handle; + alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + alloc_info.commandBufferCount = NEW_COMMAND_BUFFER_COUNT; + + VkResult result = device_table_->AllocateCommandBuffers(device_info_->handle, &alloc_info, new_handles_ptr); + GFXRECON_ASSERT(result == VK_SUCCESS); + } + + next_handle = associated_handles_[next_associated_index_]; + ++next_associated_index_; + + // Store current handle in split vector. + split_handles_.push_back(command_buffer_info->handle); + + return next_handle; +} + +VulkanCommandSplitter::VulkanCommandSplitter(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table) : + device_info_(device_info), device_table_(device_table), object_table_(object_table) +{ + GFXRECON_ASSERT(device_info != nullptr); + GFXRECON_ASSERT(device_table != nullptr); + GFXRECON_ASSERT(object_table != nullptr); +} + +VulkanCommandBufferSplitInfo& VulkanCommandSplitter::GetOrCreateSplitInfo(format::HandleId command_buffer_id) +{ + if (auto it = split_infos_.find(command_buffer_id); it != split_infos_.end()) + { + return it->second; + } + + auto [new_it, success] = split_infos_.insert( + { command_buffer_id, + VulkanCommandBufferSplitInfo(device_info_, device_table_, object_table_, command_buffer_id) }); + GFXRECON_ASSERT(success); + return new_it->second; +} + +VulkanCommandBufferSplitInfo* VulkanCommandSplitter::GetSplitInfo(format::HandleId command_buffer_id) +{ + if (auto it = split_infos_.find(command_buffer_id); it != split_infos_.end()) + { + return &it->second; + } + return nullptr; +} + +VkCommandBuffer VulkanCommandBufferSplitInfo::ResetSplitHandles() +{ + GFXRECON_ASSERT(!split_handles_.empty()); + + // Reset all the handles of the split. + for (VkCommandBuffer handle : split_handles_) + { + VkResult result = device_table_->ResetCommandBuffer(handle, 0); + GFXRECON_ASSERT(result == VK_SUCCESS); + } + + // Clear the split handles vector and reset the index. + split_handles_.clear(); + next_associated_index_ = 0; + + return original_handle_; +} + +void VulkanCommandBufferSplitInfo::FreeCommandBuffers(VkCommandPool pool) +{ + GFXRECON_ASSERT(!associated_handles_.empty()); + + device_table_->FreeCommandBuffers( + device_info_->handle, pool, static_cast(associated_handles_.size()), associated_handles_.data()); +} + +void VulkanCommandSplitter::SplitCommandBuffer(VulkanCommandBufferInfo* command_buffer_info) +{ + // Make sure current handle is mapped to the original command buffer ID. + original_command_buffer_id_[command_buffer_info->handle] = command_buffer_info->capture_id; + + device_table_->EndCommandBuffer(command_buffer_info->handle); + + // Update the command buffer info to use the new handle for subsequent calls. + command_buffer_info->handle = + GetOrCreateSplitInfo(command_buffer_info->capture_id).GetNextHandle(command_buffer_info); + original_command_buffer_id_[command_buffer_info->handle] = command_buffer_info->capture_id; + + VkCommandBufferBeginInfo begin_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + device_table_->BeginCommandBuffer(command_buffer_info->handle, &begin_info); +} + +graphics::VulkanSemaphore VulkanCommandSplitter::SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores) +{ + auto submits_command_buffers = GetCommandBuffersFromSubmitInfos(current_submits_span); + return SubmitPreviouslySplitCommandBuffers(queue_info, submits_command_buffers, wait_semaphores); +} + +graphics::VulkanSemaphore VulkanCommandSplitter::SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores) +{ + auto submits_command_buffers = GetCommandBuffersFromSubmitInfos(current_submits_span); + return SubmitPreviouslySplitCommandBuffers(queue_info, submits_command_buffers, wait_semaphores); +} + +graphics::VulkanSemaphore VulkanCommandSplitter::SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span> current_submits_cmdbufs, + const std::span wait_semaphores) +{ + util::MarkInjectedCommandsHelper mark_injected_commands_helper; + graphics::VulkanSemaphore signal_semaphore{ VK_NULL_HANDLE, 0 }; + + // A QueueSubmit might have multiple submit infos (current_submits_span). + // Each submit info might have multiple command buffers. + // We check those command buffers, to see if any of them were split from the original command buffer. + + std::vector prev_submits; + std::vector> prev_submits_command_buffers; + + for (size_t span_index = 0; span_index < current_submits_cmdbufs.size(); ++span_index) + { + const auto& command_buffers = current_submits_cmdbufs[span_index]; + + for (uint32_t cmdbuf_index = 0; cmdbuf_index < command_buffers.size(); ++cmdbuf_index) + { + VkCommandBuffer command_buffer = command_buffers[cmdbuf_index]; + if (original_command_buffer_id_.contains(command_buffer)) + { + // The command buffer in this submit was split, + // so we need to submit the previous splits before submitting this one. + auto command_buffer_id = original_command_buffer_id_[command_buffer]; + + // Each command buffer handle will go in its own submit. + const VulkanCommandBufferSplitInfo* split_info = GetSplitInfo(command_buffer_id); + auto& split_handles = split_info->GetSplitHandles(); + GFXRECON_ASSERT(!split_handles.empty()); + + // Make enough space for the new submits. + size_t submit_count = std::max(prev_submits.size(), split_handles.size()); + prev_submits.resize(submit_count); + prev_submits_command_buffers.resize(submit_count); + + // Store each handle in the corresponding submit. + for (size_t split_index = 0; split_index < split_handles.size(); ++split_index) + { + auto& prev_command_buffers = prev_submits_command_buffers[split_index]; + // Prev submits command buffers correspond to the previous handles in order. + prev_command_buffers.push_back(split_handles[split_index]); + } + } + } + } + + // Each submit will go in its own QueueSubmit. + for (size_t i = 0; i < prev_submits.size(); ++i) + { + auto& prev_command_buffers = prev_submits_command_buffers[i]; + + if (!prev_command_buffers.empty()) + { + format::HandleId command_buffer_id = original_command_buffer_id_[prev_command_buffers[0]]; + VulkanCommandBufferSplitInfo* split_info = GetSplitInfo(command_buffer_id); + + VkSubmitInfo submit_info = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + VulkanSubmitInfoHelper submit_helper(submit_info); + + // External waits should gate only the first segment, the split semaphore will order the rest. + if (i == 0) + { + for (auto& wait_semaphore : wait_semaphores) + { + submit_helper.AddWaitSemaphore(wait_semaphore); + } + } + + submit_info.commandBufferCount = static_cast(prev_command_buffers.size()); + submit_info.pCommandBuffers = prev_command_buffers.data(); + + VulkanInjectedSemaphore& split_semaphore = split_info->GetSplitSemaphore(); + submit_helper.InjectSemaphore(split_semaphore); + if (i == prev_submits.size() - 1) + { + signal_semaphore = split_semaphore.GetSemaphore(); + } + + prev_submits[i] = submit_info; + + VkResult result = device_table_->QueueSubmit(queue_info->handle, 1, &prev_submits[i], VK_NULL_HANDLE); + GFXRECON_ASSERT(result == VK_SUCCESS); + } + } + + return signal_semaphore; +} + +std::vector> VulkanCommandSplitter::GetCommandBuffersFromSubmitInfos( + const std::span submits_span) +{ + std::vector> submits_command_buffers(submits_span.size()); + + for (size_t span_index = 0; span_index < submits_span.size(); ++span_index) + { + VkSubmitInfo& submit_info = submits_span[span_index]; + + auto& command_buffers = submits_command_buffers[span_index]; + command_buffers = std::vector(submit_info.pCommandBuffers, + submit_info.pCommandBuffers + submit_info.commandBufferCount); + } + + return submits_command_buffers; +} + +std::vector> VulkanCommandSplitter::GetCommandBuffersFromSubmitInfos( + const std::span submits_span) +{ + std::vector> submits_command_buffers(submits_span.size()); + + for (size_t span_index = 0; span_index < submits_span.size(); ++span_index) + { + VkSubmitInfo2& submit_info = submits_span[span_index]; + + for (size_t cmdbuf_info_index = 0; cmdbuf_info_index < submit_info.commandBufferInfoCount; ++cmdbuf_info_index) + { + const VkCommandBufferSubmitInfo& info = submit_info.pCommandBufferInfos[cmdbuf_info_index]; + VkCommandBuffer command_buffer = info.commandBuffer; + submits_command_buffers[span_index].push_back(command_buffer); + } + } + + return submits_command_buffers; +} + +void VulkanCommandSplitter::FreeCommandBuffers(VkCommandPool command_pool, + const std::span command_buffers) +{ + // Check whether any of the command buffers being freed are split command buffers. + for (VkCommandBuffer command_buffer : command_buffers) + { + if (original_command_buffer_id_.contains(command_buffer)) + { + // This command buffer is a split command buffer, so we need to free all the splits together. + format::HandleId command_buffer_id = original_command_buffer_id_[command_buffer]; + auto* split_info = GetSplitInfo(command_buffer_id); + split_info->FreeCommandBuffers(command_pool); + + // Remove the associated handles from the tracking maps. + for (VkCommandBuffer handle : split_info->GetAssociatedHandles()) + { + original_command_buffer_id_.erase(handle); + } + + // Update the command buffer info to use the original handle for subsequent calls. + VulkanCommandBufferInfo* command_buffer_info = object_table_->GetVkCommandBufferInfo(command_buffer_id); + command_buffer_info->handle = split_info->ResetSplitHandles(); + + split_infos_.erase(command_buffer_id); + } + } +} + +void VulkanCommandSplitter::ResetCommandBuffer(VulkanCommandBufferInfo* command_buffer_info) +{ + // This command buffer is expected to be already reset at this point. + VkCommandBuffer command_buffer = command_buffer_info->handle; + + // Check whether this command buffer is result of a split. + // If so, we need to reset all the split command buffers together and restore the command buffer info handle. + if (original_command_buffer_id_.contains(command_buffer)) + { + format::HandleId command_buffer_id = original_command_buffer_id_[command_buffer]; + VulkanCommandBufferSplitInfo* split_info = GetSplitInfo(command_buffer_id); + + // Update the command buffer info to use the original handle for subsequent calls. + command_buffer_info->handle = split_info->ResetSplitHandles(); + } +} + +void VulkanCommandSplitter::BeginCommandBuffer(VulkanCommandBufferInfo* command_buffer_info) +{ + VulkanCommandPoolInfo* command_pool_info = object_table_->GetVkCommandPoolInfo(command_buffer_info->pool_id); + GFXRECON_ASSERT(command_pool_info != nullptr); + + if (command_pool_info->flags & VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT) + { + // If the command pool was created with the RESET_COMMAND_BUFFER_BIT flag, then the command buffer is implicitly + // reset when vkBeginCommandBuffer is called. In this case, we need to reset all the split command buffers + // together and restore the command buffer info handle. + if (original_command_buffer_id_.contains(command_buffer_info->handle)) + { + // Reset current handle and let `ResetCommandBuffer` handle the rest of the split handles. + device_table_->ResetCommandBuffer(command_buffer_info->handle, 0); + ResetCommandBuffer(command_buffer_info); + } + } +} + +GFXRECON_END_NAMESPACE(decode) +GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_command_splitter.h b/framework/decode/vulkan_command_splitter.h new file mode 100644 index 0000000000..f2d3820d05 --- /dev/null +++ b/framework/decode/vulkan_command_splitter.h @@ -0,0 +1,153 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ +#ifndef GFXRECON_DECODE_VULKAN_COMMAND_SPLITTER_H +#define GFXRECON_DECODE_VULKAN_COMMAND_SPLITTER_H + +#include "decode/common_object_info_table.h" +#include "decode/vulkan_object_info.h" +#include "decode/vulkan_submit_job.h" +#include "util/defines.h" + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(decode) + +/// This helper class manages command buffer handles addressing cases where a single command buffer is reset and +/// recorded multiple times. In such cases, after splitting the command buffer for the first time, a reset is expected +/// before the next recording, and we want to avoid splitting again and instead reuse the handles of the first split. +class VulkanCommandBufferSplitInfo +{ + private: + const VulkanDeviceInfo* device_info_ = nullptr; + const graphics::VulkanDeviceTable* device_table_ = nullptr; + CommonObjectInfoTable* object_table_ = nullptr; + + VulkanInjectedSemaphore split_semaphore_; + + VkCommandBuffer original_handle_ = VK_NULL_HANDLE; + + /// Associated command buffer handles, namely command buffers created at split-time that are associated with the + /// original command buffer. This does not include the original handle itself. + std::vector associated_handles_; + + /// The index of the next associated handle to use. This is used to retrieve the next handle at each split point. + /// When a command buffer is split for the first time, a new command buffer is created for the split and stored in + /// the associated handles vector. In case of a subsequent split, the index resets to 0 to reuse any associated + /// handles that were created at the first split, before creating new ones if needed. + size_t next_associated_index_ = 0; + + /// The command buffer handles of the current split. + /// First handle in the vector is the original command buffer. The rest are other handles in + /// order for each split. The very last handle, which is the most recent split, is not stored in this vector, + /// but is instead stored in the command buffer info's handle field. + std::vector split_handles_; + + public: + VulkanCommandBufferSplitInfo(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table, + format::HandleId command_buffer_id); + + VulkanCommandBufferSplitInfo(const VulkanCommandBufferSplitInfo&) = delete; + VulkanCommandBufferSplitInfo& operator=(const VulkanCommandBufferSplitInfo&) = delete; + VulkanCommandBufferSplitInfo(VulkanCommandBufferSplitInfo&&) = default; + VulkanCommandBufferSplitInfo& operator=(VulkanCommandBufferSplitInfo&&) = default; + ~VulkanCommandBufferSplitInfo() = default; + + [[nodiscard]] VkCommandBuffer GetNextHandle(const VulkanCommandBufferInfo* command_buffer_info); + + [[nodiscard]] const std::vector& GetAssociatedHandles() const { return associated_handles_; } + [[nodiscard]] const std::vector& GetSplitHandles() const { return split_handles_; } + + [[nodiscard]] VkCommandBuffer ResetSplitHandles(); + + [[nodiscard]] VulkanInjectedSemaphore& GetSplitSemaphore() { return split_semaphore_; } + + void FreeCommandBuffers(VkCommandPool pool); +}; + +class VulkanCommandSplitter +{ + public: + VulkanCommandSplitter(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table); + + ~VulkanCommandSplitter() = default; + + VulkanCommandSplitter(const VulkanCommandSplitter&) = delete; + VulkanCommandSplitter& operator=(const VulkanCommandSplitter&) = delete; + VulkanCommandSplitter(VulkanCommandSplitter&&) = default; + VulkanCommandSplitter& operator=(VulkanCommandSplitter&&) = default; + + void SplitCommandBuffer(VulkanCommandBufferInfo* command_buffer_info); + + graphics::VulkanSemaphore SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores = {}); + + graphics::VulkanSemaphore SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores = {}); + + void FreeCommandBuffers(VkCommandPool command_pool, const std::span command_buffers); + + /// @brief To be called after resetting the current command buffer. + /// @param command_buffer_info The command buffer info structure to reset. + void ResetCommandBuffer(VulkanCommandBufferInfo* command_buffer_info); + + void BeginCommandBuffer(VulkanCommandBufferInfo* command_buffer_info); + + private: + graphics::VulkanSemaphore SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span> current_submits_cmdbufs, + const std::span wait_semaphores = {}); + + std::vector> GetCommandBuffersFromSubmitInfos( + const std::span submits_span); + std::vector> GetCommandBuffersFromSubmitInfos( + const std::span submits_span); + + VulkanCommandBufferSplitInfo& GetOrCreateSplitInfo(format::HandleId command_buffer_id); + VulkanCommandBufferSplitInfo* GetSplitInfo(format::HandleId command_buffer_id); + + const VulkanDeviceInfo* device_info_ = nullptr; + const graphics::VulkanDeviceTable* device_table_ = nullptr; + CommonObjectInfoTable* object_table_ = nullptr; + + /// Map from the command buffer ID to the structure representing the split. + std::unordered_map split_infos_; + + /// Map from the command buffer handles to the original ID. + std::unordered_map original_command_buffer_id_; +}; + +using VulkanPerDeviceCommandSplitters = std::unordered_map; + +GFXRECON_END_NAMESPACE(decode) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // GFXRECON_DECODE_VULKAN_COMMAND_SPLITTER_H \ No newline at end of file diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index d60b24359e..7f197dc965 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -318,6 +318,7 @@ VulkanReplayConsumerBase::~VulkanReplayConsumerBase() // free replacer internal vulkan-resources device_address_replacers_.clear(); + device_command_splitters_.clear(); // free frame warm up resources device_frame_warmups_.clear(); @@ -1475,7 +1476,7 @@ void VulkanReplayConsumerBase::RaiseFatalError(const char* message) const void VulkanReplayConsumerBase::InitializeLoader() { - loader_handle_ = graphics::InitializeLoader(); + loader_handle_ = graphics::InitializeLoader(getenv("GFXRECON_VULKAN_LIBRARY_PATH")); // Only get get_instance_proc_addr_ from the loader if it wasn't already set via SetupForRecapture() if ((loader_handle_ != nullptr) && (get_instance_proc_addr_ == nullptr)) @@ -3766,6 +3767,8 @@ void VulkanReplayConsumerBase::OverrideDestroyDevice( // free replacer internal vulkan-resources for the device device_address_replacers_.erase(device_info); + device_command_splitters_.erase(device_info); + // free potential swapchain-resources for the device GFXRECON_ASSERT(swapchain_) swapchain_->CleanDeviceResources(device_info->handle, device_table); @@ -4377,6 +4380,17 @@ VkResult VulkanReplayConsumerBase::OverrideQueueSubmit(PFN_vkQueueSubmit VulkanSubmitJobPlan plan; + if (options_.isolate_render_passes) + { + auto& command_splitter = GetDeviceCommandSplitter(device_info); + plan.Push(0, + [&command_splitter, ¤t_submits_span, queue_info]( + const std::span wait_semaphores) { + return command_splitter.SubmitPreviouslySplitCommandBuffers( + queue_info, current_submits_span, wait_semaphores); + }); + } + if (options_.frame_warm_up_load != 0 && !fps_info_->IsFirstSubmitDone()) { auto& frame_warm_up = GetDeviceFrameWarmUp(device_info); @@ -4656,6 +4670,17 @@ VkResult VulkanReplayConsumerBase::OverrideQueueSubmit2(PFN_vkQueueSubmit2 VulkanSubmitJobPlan plan; + if (options_.isolate_render_passes) + { + auto& command_splitter = GetDeviceCommandSplitter(device_info); + plan.Push(0, + [&command_splitter, ¤t_submits_span, queue_info]( + const std::span wait_semaphores) { + return command_splitter.SubmitPreviouslySplitCommandBuffers( + queue_info, current_submits_span, wait_semaphores); + }); + } + if (options_.frame_warm_up_load != 0 && !fps_info_->IsFirstSubmitDone()) { auto& frame_warm_up = GetDeviceFrameWarmUp(device_info); @@ -5703,6 +5728,13 @@ void VulkanReplayConsumerBase::OverrideFreeCommandBuffers(PFN_vkFreeCommandBuffe } } } + + if (options_.isolate_render_passes) + { + auto command_buffers = std::span(pCommandBuffers->GetHandlePointer(), command_buffer_count); + GetDeviceCommandSplitter(device_info).FreeCommandBuffers(command_pool_info->handle, command_buffers); + } + const VkCommandBuffer* in_pCommandBuffers = pCommandBuffers->GetHandlePointer(); func(device_info->handle, command_pool_info->handle, command_buffer_count, in_pCommandBuffers); } @@ -10410,6 +10442,13 @@ VkResult VulkanReplayConsumerBase::OverrideBeginCommandBuffer( VulkanCommandBufferInfo* command_buffer_info, StructPointerDecoder* begin_info_decoder) { + if (options_.isolate_render_passes) + { + auto* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GFXRECON_ASSERT(device_info != nullptr); + GetDeviceCommandSplitter(device_info).BeginCommandBuffer(command_buffer_info); + } + ClearCommandBufferInfo(command_buffer_info); VkCommandBuffer command_buffer = command_buffer_info->handle; @@ -10458,7 +10497,36 @@ VkResult VulkanReplayConsumerBase::OverrideResetCommandBuffer(PFN_vkResetCommand resource_dumper_->ResetCommandBuffer((command_buffer)); } - return func(command_buffer, flags); + VkResult result = func(command_buffer, flags); + + if (options_.isolate_render_passes) + { + auto* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).ResetCommandBuffer(command_buffer_info); + } + + return result; +} + +VkResult VulkanReplayConsumerBase::OverrideCreateCommandPool( + PFN_vkCreateCommandPool func, + VkResult original_result, + const VulkanDeviceInfo* device_info, + const StructPointerDecoder* pCreateInfo, + const StructPointerDecoder* pAllocator, + HandlePointerDecoder* pCommandPool) +{ + const VkCommandPoolCreateInfo* create_info = pCreateInfo->GetPointer(); + if (create_info != nullptr) + { + auto* command_pool_info = reinterpret_cast(pCommandPool->GetConsumerData(0)); + command_pool_info->flags = create_info->flags; + } + + return func(device_info->handle, + pCreateInfo->GetPointer(), + GetAllocationCallbacks(pAllocator), + pCommandPool->GetHandlePointer()); } VkResult VulkanReplayConsumerBase::OverrideResetCommandPool(PFN_vkResetCommandPool func, @@ -10709,6 +10777,12 @@ void VulkanReplayConsumerBase::OverrideCmdBeginRenderPass( StructPointerDecoder* render_pass_begin_info_decoder, VkSubpassContents contents) { + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } + MaybeInjectExecutionBarrier(command_buffer_info); const auto render_pass_info_meta = render_pass_begin_info_decoder->GetMetaStructPointer(); @@ -10739,6 +10813,12 @@ void VulkanReplayConsumerBase::OverrideCmdBeginRenderPass2( StructPointerDecoder* render_pass_begin_info_decoder, StructPointerDecoder* subpass_begin_info_decode) { + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } + MaybeInjectExecutionBarrier(command_buffer_info); const auto render_pass_info_meta = render_pass_begin_info_decoder->GetMetaStructPointer(); @@ -10810,6 +10890,12 @@ void VulkanReplayConsumerBase::OverrideCmdEndRenderPass(PFN_vkCmdEndRenderPass command_buffer_info->active_framebuffer_id = format::kNullHandleId; command_buffer_info->active_render_pass_attachment_image_view_ids.clear(); command_buffer_info->in_rendering_scope = false; + + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } } void VulkanReplayConsumerBase::OverrideCmdEndRenderPass2( @@ -10824,6 +10910,12 @@ void VulkanReplayConsumerBase::OverrideCmdEndRenderPass2( command_buffer_info->active_framebuffer_id = format::kNullHandleId; command_buffer_info->active_render_pass_attachment_image_view_ids.clear(); command_buffer_info->in_rendering_scope = false; + + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } } void VulkanReplayConsumerBase::OverrideCmdBeginRendering( @@ -10833,6 +10925,12 @@ void VulkanReplayConsumerBase::OverrideCmdBeginRendering( { GFXRECON_ASSERT(command_buffer_info != nullptr); + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } + MaybeInjectExecutionBarrier(command_buffer_info); command_buffer_info->in_rendering_scope = true; @@ -10885,6 +10983,12 @@ void VulkanReplayConsumerBase::OverrideCmdEndRendering(PFN_vkCmdEndRendering GFXRECON_ASSERT(command_buffer_info != nullptr); command_buffer_info->in_rendering_scope = false; func(command_buffer_info->handle); + + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } } void VulkanReplayConsumerBase::OverrideCmdTraceRaysKHR( @@ -11919,6 +12023,19 @@ VulkanFrameWarmUp& VulkanReplayConsumerBase::GetDeviceFrameWarmUp(const VulkanDe return new_it->second; } +VulkanCommandSplitter& VulkanReplayConsumerBase::GetDeviceCommandSplitter(const VulkanDeviceInfo* device_info) +{ + if (auto it = device_command_splitters_.find(device_info); it != device_command_splitters_.end()) + { + return it->second; + } + + auto [new_it, success] = device_command_splitters_.insert( + { device_info, VulkanCommandSplitter(device_info, GetDeviceTable(device_info->handle), object_info_table_) }); + GFXRECON_ASSERT(success); + return new_it->second; +} + VulkanSubmitJobExecutor& VulkanReplayConsumerBase::GetDeviceSubmitJobExecutor(const VulkanDeviceInfo* device_info) { if (auto it = device_submit_job_executors_.find(device_info); it != device_submit_job_executors_.end()) diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h index 70b3c79823..b6bdc0a657 100644 --- a/framework/decode/vulkan_replay_consumer_base.h +++ b/framework/decode/vulkan_replay_consumer_base.h @@ -36,6 +36,7 @@ #include "decode/vulkan_object_info.h" #include "decode/common_object_info_table.h" #include "decode/vulkan_replay_options.h" +#include "decode/vulkan_command_splitter.h" #include "decode/vulkan_resource_allocator.h" #include "decode/vulkan_submit_job.h" #include "decode/vulkan_swapchain.h" @@ -1359,6 +1360,13 @@ class VulkanReplayConsumerBase : public VulkanConsumer VulkanCommandBufferInfo* command_buffer_info, VkCommandBufferResetFlags flags); + VkResult OverrideCreateCommandPool(PFN_vkCreateCommandPool func, + VkResult original_result, + const VulkanDeviceInfo* device_info, + const StructPointerDecoder* pCreateInfo, + const StructPointerDecoder* pAllocator, + HandlePointerDecoder* pCommandPool); + VkResult OverrideResetCommandPool(PFN_vkResetCommandPool func, VkResult original_result, const VulkanDeviceInfo* device_info, @@ -1861,6 +1869,7 @@ class VulkanReplayConsumerBase : public VulkanConsumer decode::VulkanDeviceAddressTracker& GetDeviceAddressTracker(const decode::VulkanDeviceInfo* device_info); decode::VulkanAddressReplacer& GetDeviceAddressReplacer(const decode::VulkanDeviceInfo* device_info); VulkanFrameWarmUp& GetDeviceFrameWarmUp(const VulkanDeviceInfo* device_info); + VulkanCommandSplitter& GetDeviceCommandSplitter(const VulkanDeviceInfo* device_info); VulkanSubmitJobExecutor& GetDeviceSubmitJobExecutor(const VulkanDeviceInfo* device_info); /** @@ -1998,6 +2007,7 @@ class VulkanReplayConsumerBase : public VulkanConsumer VulkanPerDeviceAddressTrackers device_address_trackers_; VulkanPerDeviceAddressReplacers device_address_replacers_; VulkanPerDeviceFrameWarmUp device_frame_warmups_; + VulkanPerDeviceCommandSplitters device_command_splitters_; VulkanPerDeviceSubmitJobExecutors device_submit_job_executors_; util::ThreadPool main_thread_queue_; diff --git a/framework/decode/vulkan_replay_options.h b/framework/decode/vulkan_replay_options.h index 772544e325..9c76cc2bbe 100644 --- a/framework/decode/vulkan_replay_options.h +++ b/framework/decode/vulkan_replay_options.h @@ -264,6 +264,9 @@ struct VulkanReplayOptions : public ReplayOptions /// Parameters to pass to the replay event plugin. std::string replay_event_plugin_params; + /// Isolate render passes by splitting the command buffer into multiple submits. + bool isolate_render_passes{ false }; + void MaybeWaitBeforeFirstSubmit() const; void MaybeWaitBeforeFrame() const; }; diff --git a/framework/decode/vulkan_submit_info_helper.h b/framework/decode/vulkan_submit_info_helper.h index 685862d1bc..d5f9ad9a0c 100644 --- a/framework/decode/vulkan_submit_info_helper.h +++ b/framework/decode/vulkan_submit_info_helper.h @@ -47,10 +47,11 @@ class VulkanInjectedSemaphore const graphics::VulkanDeviceTable* device_table_; public: - bool HasReachedTargetValue() const; - VkSemaphore GetHandle() const { return semaphore_.semaphore; } - uint64_t GetTargetValue() const { return semaphore_.timeline_value; } - void IncreaseTargetValue() { semaphore_.timeline_value++; } + bool HasReachedTargetValue() const; + graphics::VulkanSemaphore GetSemaphore() const { return semaphore_; } + VkSemaphore GetHandle() const { return semaphore_.semaphore; } + uint64_t GetTargetValue() const { return semaphore_.timeline_value; } + void IncreaseTargetValue() { semaphore_.timeline_value++; } VulkanInjectedSemaphore(const VulkanDeviceInfo* device_info, const graphics::VulkanDeviceTable* table); ~VulkanInjectedSemaphore(); diff --git a/framework/generated/generated_vulkan_replay_consumer.cpp b/framework/generated/generated_vulkan_replay_consumer.cpp index 18f1b2cdd7..1a0e4dd05d 100644 --- a/framework/generated/generated_vulkan_replay_consumer.cpp +++ b/framework/generated/generated_vulkan_replay_consumer.cpp @@ -792,19 +792,19 @@ void VulkanReplayConsumer::Process_vkCreateCommandPool( StructPointerDecoder* pAllocator, HandlePointerDecoder* pCommandPool) { - VkDevice in_device = MapHandle(device, &CommonObjectInfoTable::GetVkDeviceInfo); - const VkCommandPoolCreateInfo* in_pCreateInfo = pCreateInfo->GetPointer(); + auto in_device = GetObjectInfoTable().GetVkDeviceInfo(device); + MapStructHandles(pCreateInfo->GetMetaStructPointer(), GetObjectInfoTable()); - const VkAllocationCallbacks* in_pAllocator = GetAllocationCallbacks(pAllocator); if (!pCommandPool->IsNull()) { pCommandPool->SetHandleLength(1); } - VkCommandPool* out_pCommandPool = pCommandPool->GetHandlePointer(); + VulkanCommandPoolInfo handle_info; + pCommandPool->SetConsumerData(0, &handle_info); PushRecaptureHandleId(pCommandPool->GetPointer()); - VkResult replay_result = GetDeviceTable(in_device)->CreateCommandPool(in_device, in_pCreateInfo, in_pAllocator, out_pCommandPool); + VkResult replay_result = OverrideCreateCommandPool(GetDeviceTable(in_device->handle)->CreateCommandPool, returnValue, in_device, pCreateInfo, pAllocator, pCommandPool); CheckResult("vkCreateCommandPool", returnValue, replay_result, call_info); ClearRecaptureHandleIds(); - AddHandle(device, pCommandPool->GetPointer(), out_pCommandPool, &CommonObjectInfoTable::AddVkCommandPoolInfo); + AddHandle(device, pCommandPool->GetPointer(), pCommandPool->GetHandlePointer(), std::move(handle_info), &CommonObjectInfoTable::AddVkCommandPoolInfo); } void VulkanReplayConsumer::Process_vkDestroyCommandPool( diff --git a/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json b/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json index 74061e4ed4..baaafd2e7b 100644 --- a/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json +++ b/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json @@ -108,6 +108,7 @@ "vkGetPhysicalDeviceToolProperties": "OverrideGetPhysicalDeviceToolProperties", "vkGetPhysicalDeviceToolPropertiesEXT": "OverrideGetPhysicalDeviceToolProperties", "vkBeginCommandBuffer": "OverrideBeginCommandBuffer", + "vkCreateCommandPool": "OverrideCreateCommandPool", "vkResetCommandPool": "OverrideResetCommandPool", "vkResetCommandBuffer": "OverrideResetCommandBuffer", "vkCmdWaitEvents": "OverrideCmdWaitEvents", diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3703689ace..3c206dd04a 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,6 +45,7 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) set(GFXRECON_TESTCASES test_cases/ahb.cpp test_cases/debug-utils.cpp + test_cases/isolate-render-passes.cpp test_cases/triangle.cpp test_cases/multisample-depth.cpp test_cases/shader-objects.cpp @@ -62,7 +63,7 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) add_executable(gfxrecon-testapp-runner ${GFXRECON_TESTCASES}) - add_dependencies(gfxrecon-testapp-runner gfxrecon-test-launcher VkICD_mock_icd gfxrecon-convert VkLayer_gfxreconstruct) + add_dependencies(gfxrecon-testapp-runner gfxrecon-test-launcher VkICD_mock_icd gfxrecon-convert gfxrecon-replay VkLayer_gfxreconstruct) target_include_directories(gfxrecon-testapp-runner PRIVATE ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(gfxrecon-testapp-runner gfxrecon_util @@ -72,6 +73,7 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) install(TARGETS gfxrecon-testapp-runner RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}) install(DIRECTORY known_good DESTINATION ${GFXRECON_INSTALL_TESTDIR}) install(TARGETS gfxrecon-convert RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}) + install(TARGETS gfxrecon-replay RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}) if (APPLE) install(PROGRAMS run-tests_macos.sh DESTINATION ${GFXRECON_INSTALL_TESTDIR}) install(FILES $/staging-json/VkLayer_gfxreconstruct.json DESTINATION ${GFXRECON_INSTALL_LAYER_INSTALL_DIR}) diff --git a/test/run-tests_macos.sh b/test/run-tests_macos.sh index d7204bda18..1447f71d1a 100644 --- a/test/run-tests_macos.sh +++ b/test/run-tests_macos.sh @@ -11,6 +11,7 @@ export VK_LOADER_LAYERS_ENABLE=VK_LAYER_LUNARG_gfxreconstruct export DISPLAY=:0 export GFXRECON_TESTAPP_HEADLESS=true export GFXRECON_TESTAPP_MOCK_ICD="$PWD/test_apps/libVkICD_mock_icd.dylib" +export GFXRECON_VULKAN_LIBRARY_PATH="$VULKAN_LIBRARY_PATH" #export GFXRECON_LOG_LEVEL="info" #export VK_LOADER_DEBUG="layer" diff --git a/test/test_apps/CMakeLists.txt b/test/test_apps/CMakeLists.txt index 7da5924465..d3fd68f83a 100644 --- a/test/test_apps/CMakeLists.txt +++ b/test/test_apps/CMakeLists.txt @@ -33,6 +33,7 @@ endif() add_subdirectory(common) add_subdirectory(debug-utils) +add_subdirectory(isolate-render-passes) add_subdirectory(triangle) add_subdirectory(triangle-extra-device) add_subdirectory(multisample-depth) diff --git a/test/test_apps/isolate-render-passes/CMakeLists.txt b/test/test_apps/isolate-render-passes/CMakeLists.txt new file mode 100644 index 0000000000..40b224576e --- /dev/null +++ b/test/test_apps/isolate-render-passes/CMakeLists.txt @@ -0,0 +1,39 @@ +############################################################################### +# Copyright (c) 2026 LunarG, Inc. +# All rights reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# Author: LunarG Team +# Description: CMake script for isolate-render-passes test app +############################################################################### + +add_library(gfxrecon-testapp-isolate-render-passes app.cpp) + +target_include_directories(gfxrecon-testapp-isolate-render-passes + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(gfxrecon-testapp-isolate-render-passes gfxrecon-testapp-base) + +common_build_directives(gfxrecon-testapp-isolate-render-passes) + +if(NOT CMAKE_SYSTEM_NAME MATCHES "Android") + install(TARGETS gfxrecon-testapp-isolate-render-passes RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTAPPDIR}/isolate-render-passes) +endif() diff --git a/test/test_apps/isolate-render-passes/app.cpp b/test/test_apps/isolate-render-passes/app.cpp new file mode 100644 index 0000000000..e205680394 --- /dev/null +++ b/test/test_apps/isolate-render-passes/app.cpp @@ -0,0 +1,366 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(isolate_render_passes) + +void App::configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig* test_config) +{ + if (test_config) + { + test_config->device_api_version_override = VK_MAKE_API_VERSION(0, 1, 3, 296); + } + + TestAppBase::configure_instance_builder(instance_builder, test_config); +} + +void App::create_render_pass() +{ + // A single colour attachment that is cleared on load. The image is transitioned to + // VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL before the first render pass, so the same render pass object can be + // reused for the two passes recorded each frame. + VkAttachmentDescription color_attachment = {}; + color_attachment.format = init.swapchain.image_format; + color_attachment.samples = VK_SAMPLE_COUNT_1_BIT; + color_attachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + color_attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + color_attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + color_attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + color_attachment.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + color_attachment.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkAttachmentReference color_attachment_ref = {}; + color_attachment_ref.attachment = 0; + color_attachment_ref.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkSubpassDescription subpass = {}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &color_attachment_ref; + + VkRenderPassCreateInfo render_pass_info = {}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; + render_pass_info.attachmentCount = 1; + render_pass_info.pAttachments = &color_attachment; + render_pass_info.subpassCount = 1; + render_pass_info.pSubpasses = &subpass; + render_pass_info.dependencyCount = 0; + + auto result = init.disp.createRenderPass(&render_pass_info, nullptr, &render_pass_); + VERIFY_VK_RESULT("failed to create render pass", result); +} + +void App::create_framebuffers() +{ + framebuffers_.resize(init.swapchain_image_views.size()); + + for (size_t i = 0; i < init.swapchain_image_views.size(); i++) + { + VkImageView attachments[] = { init.swapchain_image_views[i] }; + + VkFramebufferCreateInfo framebuffer_info = {}; + framebuffer_info.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO; + framebuffer_info.renderPass = render_pass_; + framebuffer_info.attachmentCount = 1; + framebuffer_info.pAttachments = attachments; + framebuffer_info.width = init.swapchain.extent.width; + framebuffer_info.height = init.swapchain.extent.height; + framebuffer_info.layers = 1; + + auto result = init.disp.createFramebuffer(&framebuffer_info, nullptr, &framebuffers_[i]); + VERIFY_VK_RESULT("failed to create framebuffer", result); + } +} + +void App::recreate_swapchain() +{ + init.disp.deviceWaitIdle(); + + for (auto framebuffer : framebuffers_) + { + init.disp.destroyFramebuffer(framebuffer, nullptr); + } + + TestAppBase::recreate_swapchain(false); + + create_framebuffers(); +} + +const int NUM_FRAMES = 10; +#define IS_RUNNING(frame_num) frame_num < NUM_FRAMES; + +/// Records a single render pass that clears the swapchain image to the provided colour. +static void record_clear_render_pass(vkb::DispatchTable const& disp, + VkCommandBuffer command_buffer, + VkRenderPass render_pass, + VkFramebuffer framebuffer, + VkExtent2D extent, + VkClearValue clear_color) +{ + VkRenderPassBeginInfo render_pass_info = {}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; + render_pass_info.renderPass = render_pass; + render_pass_info.framebuffer = framebuffer; + render_pass_info.renderArea.offset = { 0, 0 }; + render_pass_info.renderArea.extent = extent; + render_pass_info.clearValueCount = 1; + render_pass_info.pClearValues = &clear_color; + + disp.cmdBeginRenderPass(command_buffer, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE); + disp.cmdEndRenderPass(command_buffer); +} + +bool App::frame(const int frame_num) +{ + init.disp.waitForFences(1, &sync_.in_flight_fences[current_frame_], VK_TRUE, UINT64_MAX); + + uint32_t image_index = 0; + VkResult result = init.disp.acquireNextImageKHR( + init.swapchain, UINT64_MAX, sync_.available_semaphores[current_frame_], VK_NULL_HANDLE, &image_index); + + if (result == VK_ERROR_OUT_OF_DATE_KHR) + { + recreate_swapchain(); + return IS_RUNNING(frame_num); + } + else if (result != VK_SUCCESS && result != VK_SUBOPTIMAL_KHR) + { + throw test::vulkan_exception("failed to acquire next image", result); + } + + if (sync_.image_in_flight[image_index] != VK_NULL_HANDLE) + { + init.disp.waitForFences(1, &sync_.image_in_flight[image_index], VK_TRUE, UINT64_MAX); + } + sync_.image_in_flight[image_index] = sync_.in_flight_fences[current_frame_]; + + init.disp.resetCommandPool(command_pools_[current_frame_], 0); + VkCommandBufferAllocateInfo allocate_info = {}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandBufferCount = 1; + allocate_info.commandPool = command_pools_[current_frame_]; + VkCommandBuffer command_buffer; + result = init.disp.allocateCommandBuffers(&allocate_info, &command_buffer); + VERIFY_VK_RESULT("failed to allocate command buffer", result); + + { + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + result = init.disp.beginCommandBuffer(command_buffer, &begin_info); + VERIFY_VK_RESULT("failed to begin command buffer", result); + + { + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + image_barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_NONE; + image_barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + // First render pass: clear to red. + // With --isolate-render-passes this segment is split out and submitted on + // its own, chained to the next segment through an injected semaphore. + VkClearValue first_clear{ { { 1.0f, 0.0f, 0.0f, 1.0f } } }; + record_clear_render_pass( + init.disp, command_buffer, render_pass_, framebuffers_[image_index], init.swapchain.extent, first_clear); + + { + // Order the second render pass after the first (write-after-write on the same attachment). + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + image_barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + // Second render pass: clear to green. + // Two render passes in one command buffer exercise the multi-segment + // semaphore chaining performed by the command splitter. + VkClearValue second_clear{ { { 0.0f, 1.0f, 0.0f, 1.0f } } }; + record_clear_render_pass( + init.disp, command_buffer, render_pass_, framebuffers_[image_index], init.swapchain.extent, second_clear); + + { + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + image_barrier.dstAccessMask = VK_ACCESS_NONE; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + result = init.disp.endCommandBuffer(command_buffer); + VERIFY_VK_RESULT("failed to end command buffer", result); + } + + VkSubmitInfo submitInfo = {}; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + + // The submit waits on the binary swapchain-acquire semaphore. + // This is the wait-semaphore that the splitter will hand off to the first split segment. + VkSemaphore wait_semaphores[] = { sync_.available_semaphores[current_frame_] }; + VkPipelineStageFlags wait_stages[] = { VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT }; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = wait_semaphores; + submitInfo.pWaitDstStageMask = wait_stages; + + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &command_buffer; + + VkSemaphore signal_semaphores[] = { sync_.finished_semaphore[image_index] }; + submitInfo.signalSemaphoreCount = 1; + submitInfo.pSignalSemaphores = signal_semaphores; + + init.disp.resetFences(1, &sync_.in_flight_fences[current_frame_]); + + result = init.disp.queueSubmit(graphics_queue_, 1, &submitInfo, sync_.in_flight_fences[current_frame_]); + VERIFY_VK_RESULT("failed to submit queue", result); + + VkPresentInfoKHR present_info = {}; + present_info.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; + + present_info.waitSemaphoreCount = 1; + present_info.pWaitSemaphores = signal_semaphores; + + VkSwapchainKHR swapChains[] = { init.swapchain }; + present_info.swapchainCount = 1; + present_info.pSwapchains = swapChains; + + present_info.pImageIndices = &image_index; + + result = init.disp.queuePresentKHR(present_queue_, &present_info); + if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR) + { + recreate_swapchain(); + return frame_num >= NUM_FRAMES; + } + VERIFY_VK_RESULT("failed to present queue", result); + + current_frame_ = (current_frame_ + 1) % MAX_FRAMES_IN_FLIGHT; + + return IS_RUNNING(frame_num); +} + +void App::cleanup() +{ + for (size_t i = 0; i < sync_.finished_semaphore.size(); ++i) + { + init.disp.destroySemaphore(sync_.finished_semaphore[i], nullptr); + } + for (size_t i = 0; i < MAX_FRAMES_IN_FLIGHT; i++) + { + init.disp.destroySemaphore(sync_.available_semaphores[i], nullptr); + init.disp.destroyFence(sync_.in_flight_fences[i], nullptr); + } + + for (auto command_pool : command_pools_) + { + init.disp.destroyCommandPool(command_pool, nullptr); + } + + for (auto framebuffer : framebuffers_) + { + init.disp.destroyFramebuffer(framebuffer, nullptr); + } + + init.disp.destroyRenderPass(render_pass_, nullptr); +} + +void App::setup() +{ + auto graphics_queue = init.device.get_queue(test::QueueType::graphics); + if (!graphics_queue.has_value()) + throw std::runtime_error("could not get graphics queue"); + graphics_queue_ = *graphics_queue; + + auto present_queue = init.device.get_queue(test::QueueType::present); + if (!present_queue.has_value()) + throw std::runtime_error("could not get present queue"); + present_queue_ = *present_queue; + + create_render_pass(); + + create_framebuffers(); + + auto queue_family_index = init.device.get_queue_index(test::QueueType::graphics); + if (!queue_family_index) + throw std::runtime_error("could not find graphics queue"); + for (auto& command_pool : command_pools_) + { + command_pool = test::create_command_pool(init.disp, *queue_family_index); + } + + sync_ = test::create_sync_objects(init.swapchain, init.disp, MAX_FRAMES_IN_FLIGHT); +} + +GFXRECON_END_NAMESPACE(isolate_render_passes) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/test/test_apps/isolate-render-passes/isolate_render_passes_app.h b/test/test_apps/isolate-render-passes/isolate_render_passes_app.h new file mode 100644 index 0000000000..1e76bf2f4e --- /dev/null +++ b/test/test_apps/isolate-render-passes/isolate_render_passes_app.h @@ -0,0 +1,71 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#ifndef GFXRECON_TESTAPP_ISOLATE_RENDER_PASSES_H +#define GFXRECON_TESTAPP_ISOLATE_RENDER_PASSES_H + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(isolate_render_passes) + +const size_t MAX_FRAMES_IN_FLIGHT = 2; + +/** + * Minimal app that records two legacy (vkCmdBeginRenderPass/vkCmdEndRenderPass) render passes into a single command + * buffer each frame and submits it while waiting on the binary swapchain-acquire semaphore. It deliberately uses no + * graphics pipeline (each render pass only clears the swapchain image via loadOp), keeping the focus on the + * render-pass boundaries that gfxrecon-replay --isolate-render-passes splits and chains. + */ +class App : public test::TestAppBase +{ + VkQueue graphics_queue_; + VkQueue present_queue_; + + std::vector framebuffers_; + + VkRenderPass render_pass_; + + VkCommandPool command_pools_[MAX_FRAMES_IN_FLIGHT]; + + size_t current_frame_ = 0; + + test::Sync sync_; + + void create_render_pass(); + void create_framebuffers(); + void recreate_swapchain(); + void cleanup() override; + bool frame(const int frame_num) override; + void setup() override; + + void configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig*) override; +}; + +GFXRECON_END_NAMESPACE(isolate_render_passes) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // GFXRECON_TESTAPP_ISOLATE_RENDER_PASSES_H diff --git a/test/test_apps/launcher/CMakeLists.txt b/test/test_apps/launcher/CMakeLists.txt index 7b7664ecaa..0c2e655b05 100644 --- a/test/test_apps/launcher/CMakeLists.txt +++ b/test/test_apps/launcher/CMakeLists.txt @@ -42,6 +42,7 @@ set(GFXRECON_TEST_LAUNCHER_LINK_LIBRARIES gfxrecon-testapp-ahb gfxrecon-testapp-debug-utils gfxrecon-testapp-host-image-copy + gfxrecon-testapp-isolate-render-passes gfxrecon-testapp-multisample-depth gfxrecon-testapp-pipeline-binaries gfxrecon-testapp-set-environment diff --git a/test/test_apps/launcher/test_launcher.cpp b/test/test_apps/launcher/test_launcher.cpp index b38cfc1ca8..c927bfb657 100644 --- a/test/test_apps/launcher/test_launcher.cpp +++ b/test/test_apps/launcher/test_launcher.cpp @@ -1,5 +1,5 @@ /* -** Copyright (c) 2025 LunarG, Inc. +** Copyright (c) 2025-2026 LunarG, Inc. ** ** Permission is hereby granted, free of charge, to any person obtaining a ** copy of this software and associated documentation files (the "Software"), @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -65,6 +66,7 @@ static const char* kAppNames[] = { "acquired-image", "debug-utils", "host-image-copy", + "isolate-render-passes", "multisample-depth", "pipeline-binaries", #ifndef __ANDROID__ @@ -157,6 +159,10 @@ CreateTestApp(std::unique_ptr application, { app = std::make_unique(); } + else if (app_name == "isolate-render-passes") + { + app = std::make_unique(); + } else if (app_name == "multisample-depth") { app = std::make_unique(); diff --git a/test/test_cases/isolate-render-passes.cpp b/test/test_cases/isolate-render-passes.cpp new file mode 100644 index 0000000000..fe596e3901 --- /dev/null +++ b/test/test_cases/isolate-render-passes.cpp @@ -0,0 +1,14 @@ +#include + +#include "verify-gfxr.h" + +/** + * Capture the legacy-render-pass app and replay it with --isolate-render-passes. + * The replay tool splits the command buffer at each vkCmdBeginRenderPass/vkCmdEndRenderPass boundary + * and submits the segments separately. + * This test exercises that code path against the mock ICD and asserts the replay completes successfully. + */ +TEST(IsolateRenderPasses, ReplaySplitsLegacyRenderPasses) +{ + capture_and_replay("isolate-render-passes", { "--isolate-render-passes" }); +} diff --git a/test/verify-gfxr.cpp b/test/verify-gfxr.cpp index 55a0942ddd..a08cd5761d 100644 --- a/test/verify-gfxr.cpp +++ b/test/verify-gfxr.cpp @@ -69,8 +69,10 @@ bool clean_gfxr_json(int depth, nlohmann::json::parse_event_t event, nlohmann::j #if defined(__linux__) || defined(__APPLE__) static char const* CONVERT_FILENAME = "gfxrecon-convert"; +static char const* REPLAY_FILENAME = "gfxrecon-replay"; #elif defined(_WIN32) static char const* CONVERT_FILENAME = "gfxrecon-convert.exe"; +static char const* REPLAY_FILENAME = "gfxrecon-replay.exe"; #endif struct Paths @@ -80,6 +82,7 @@ struct Paths std::filesystem::path full_app_directory{ base_path }; std::filesystem::path full_executable_path; std::filesystem::path convert_path{ base_path }; + std::filesystem::path replay_path{ base_path }; std::filesystem::path capture_path{ base_path }; std::filesystem::path known_good_path{ base_path }; std::filesystem::path app_json_path; @@ -161,6 +164,7 @@ struct Paths #endif convert_path.append(CONVERT_FILENAME); + replay_path.append(REPLAY_FILENAME); std::string gfxr_file_name = test_name + std::string(".gfxr"); capture_path.append(gfxr_file_name); @@ -345,3 +349,39 @@ void verify_gfxr(const char* test_name, char const* trimming_frames, bool trigge run_trimming_app(paths, test_name, trimming_frames, trigger_trimming); } } + +void capture_and_replay(const char* test_name, std::vector extra_replay_args) +{ + EnvironmentVariables env_vars; + + Paths paths{ test_name, nullptr, false }; + int result; + + bool working_directory_exists = std::filesystem::exists(paths.working_directory); + ASSERT_TRUE(working_directory_exists) << "working directory does not exist: " << paths.working_directory; + + // Run the app with capture enabled to produce the gfxr to replay. + env_vars.SetEnv("GFXRECON_CAPTURE_FILE", paths.capture_path.string().c_str()); + result = run_command(paths.working_directory, paths.full_executable_path, { test_name }); + ASSERT_EQ(result, 0) << "capture command failed " << paths.full_executable_path << " " << test_name << " in path " + << paths.working_directory; + + ASSERT_TRUE(std::filesystem::exists(paths.capture_path)) << "capture file was not produced: " << paths.capture_path; + + // The gfxreconstruct capture layer is still enabled in the environment, so point GFXRECON_CAPTURE_FILE at a + // throwaway path for the replay step. This keeps the layer (if it loads during replay) from re-capturing over the + // input gfxr we are about to read. + std::filesystem::path replay_capture_path{ paths.base_path }; + replay_capture_path.append(test_name + std::string("_replay.gfxr")); + env_vars.SetEnv("GFXRECON_CAPTURE_FILE", replay_capture_path.string().c_str()); + + // Replay the capture headless (offscreen swapchain) against the mock ICD, forwarding any extra arguments. + // Asserts the replay tool exits successfully (no crash, assertion, or replay error). + std::vector replay_args = { "--swapchain", "offscreen" }; + replay_args.insert(replay_args.end(), extra_replay_args.begin(), extra_replay_args.end()); + replay_args.push_back(paths.capture_path.string()); + + result = run_command(paths.base_path, paths.replay_path, replay_args); + ASSERT_EQ(result, 0) << "replay command failed " << paths.replay_path << " for capture " << paths.capture_path + << " in path " << paths.base_path; +} diff --git a/test/verify-gfxr.h b/test/verify-gfxr.h index 24623c0c42..7f3417e2b0 100644 --- a/test/verify-gfxr.h +++ b/test/verify-gfxr.h @@ -1,23 +1,40 @@ #ifndef GFXRECONSTRUCT_VERIFY_GFXR_H #define GFXRECONSTRUCT_VERIFY_GFXR_H +#include +#include + void run_in_background(const char* test_name); -/* +/** * Run an application with capture enabled, and compare the resulting gfxr file to a known good gfxr * - * test_name - the name of the test to launch - * trimming_frames - It's not nullptr if it needs to run trimming app. frame info ex: "10" or "10-100" + * @param test_name - the name of the test to launch + * @param trimming_frames - It's not nullptr if it needs to run trimming app. frame info ex: "10" or "10-100" * - * expects the following environment variables to be set + * @note expects the following environment variables to be set * VK_LAYER_PATH - path to VkLayer_gfxreconstruct.dll * VK_DRIVER_FILES - path to VkICD_mock_icd.json * - * expects the following environment variables to be set with the following values + * @note expects the following environment variables to be set with the following values * VK_INSTANCE_LAYERS=VK_LAYER_LUNARG_gfxreconstruct * GFXRECON_CAPTURE_FILE_TIMESTAMP=false * GFXRECON_CAPTURE_FILE=actual.gfxr */ void verify_gfxr(const char* test_name, char const* trimming_frames = nullptr, bool trigger_trimming = false); +/** + * Run an application with capture enabled, then replay the resulting gfxr with gfxrecon-replay, asserting that the + * replay process exits successfully. + * + * Replay is forced offscreen (--swapchain offscreen) so it runs headless against the mock ICD, and any additional + * arguments are forwarded to the replay tool. + * + * @param test_name - the name of the test app to launch and capture + * @param extra_replay_args - additional arguments forwarded verbatim to gfxrecon-replay + * + * @note expects the same environment variables as verify_gfxr(). + */ +void capture_and_replay(const char* test_name, std::vector extra_replay_args = {}); + #endif // GFXRECONSTRUCT_VERIFY_GFXR_H diff --git a/tools/replay/replay_settings.h b/tools/replay/replay_settings.h index 7aa93107ae..a8f5eb0973 100644 --- a/tools/replay/replay_settings.h +++ b/tools/replay/replay_settings.h @@ -36,7 +36,7 @@ const char kOptions[] = "--dump-resources-modifiable-state-only,--pbi-all,--preload-measurement-range,--add-new-pipeline-caches,--" "screenshot-ignore-FrameBoundaryANDROID,--screenshot-apply-prerotation,--deduplicate-device,--log-timestamps,--" "capture,--idle-before-submit,--" - "serialize-render-passes,--serialize-queue-submissions,--async-processing"; + "serialize-render-passes,--serialize-queue-submissions,--async-processing,--isolate-render-passes"; const char kArguments[] = "--log-level,--log-file,--cpu-mask,--gpu,--gpu-group,--pause-frame,--wsi,--surface-index,-m|--memory-translation," "--replace-shaders,--screenshots,--screenshot-interval,--denied-messages,--allowed-messages,--screenshot-format,--" @@ -405,6 +405,9 @@ static void PrintUsage(const char* exe_name) GFXRECON_WRITE_CONSOLE(" \t\tParameters to forward to the replay event plugin. The format"); GFXRECON_WRITE_CONSOLE(" \t\tof the parameters is determined by the plugin and is not"); GFXRECON_WRITE_CONSOLE(" \t\tinterpreted by the replay tool. (forwarded to replay tool)"); + GFXRECON_WRITE_CONSOLE(" --isolate-render-passes"); + GFXRECON_WRITE_CONSOLE( + " \t\tIsolate render passes by splitting the command buffer into multiple submits."); #if defined(WIN32) GFXRECON_WRITE_CONSOLE("") diff --git a/tools/tool_settings.h b/tools/tool_settings.h index 18b21d6f2e..ec8b4eb452 100644 --- a/tools/tool_settings.h +++ b/tools/tool_settings.h @@ -175,6 +175,7 @@ const char kFrameWarmUpLoad[] = "--frame-warm-up-load"; const char kSerializeQueueSubmissions[] = "--serialize-queue-submissions"; const char kReplayEventPluginPath[] = "--replay-event-plugin-path"; const char kReplayEventPluginParams[] = "--replay-event-plugin-params"; +const char kIsolateRenderPasses[] = "--isolate-render-passes"; enum class WsiPlatform { @@ -1402,6 +1403,7 @@ GetVulkanReplayOptions(const gfxrecon::util::ArgumentParser& arg_parse replay_options.replay_event_plugin_path = arg_parser.GetArgumentValue(kReplayEventPluginPath); replay_options.replay_event_plugin_params = arg_parser.GetArgumentValue(kReplayEventPluginParams); + replay_options.isolate_render_passes = arg_parser.IsOptionSet(kIsolateRenderPasses); return replay_options; } From 999ad797a322e85f15aad88c40ab66ef7dd157d4 Mon Sep 17 00:00:00 2001 From: okuznetsov Date: Wed, 1 Jul 2026 11:12:50 +0200 Subject: [PATCH 2/2] replay: Serialize compute and transfer operations (#3058) Add a --serialize-compute-and-transfer option that injects a pipeline barrier before and after each compute dispatch, preventing compute and transfer/copy work from overlapping on the GPU. - Add OverrideCmdDispatch, OverrideCmdDispatchIndirect, and OverrideCmdDispatchBase replay overrides that wrap every compute dispatch with MaybeInjectComputeTransferBarrier. - Plumb the new option through gfxrecon.py, replay_settings.h and tool_settings.h, and document it in USAGE_android.md and USAGE_desktop_Vulkan.md. - Add a serialize-compute-and-transfer test app and smoke test case, which uses the capture_and_replay() helper introduced by #3016. (cherry picked from commit a545b2a9a77e6da6e8941ecf4c2d4d15370d5242) Change-Id: If66ad74d38c8d3a97fb4bd02bf6761a26a6a6964 --- USAGE_android.md | 4 + USAGE_desktop_Vulkan.md | 4 + android/scripts/gfxrecon.py | 4 + android/test/test_apps/common/CMakeLists.txt | 1 + .../test/test_apps/launcher/CMakeLists.txt | 1 + .../decode/vulkan_replay_consumer_base.cpp | 71 ++++ .../decode/vulkan_replay_consumer_base.h | 26 ++ framework/decode/vulkan_replay_options.h | 4 + .../generated_vulkan_replay_consumer.cpp | 26 +- .../vulkan_generators/replay_overrides.json | 6 +- test/CMakeLists.txt | 1 + test/test_apps/CMakeLists.txt | 1 + test/test_apps/launcher/CMakeLists.txt | 1 + test/test_apps/launcher/test_launcher.cpp | 6 + .../CMakeLists.txt | 39 +++ .../serialize-compute-and-transfer/app.cpp | 321 ++++++++++++++++++ .../serialize_compute_and_transfer_app.h | 81 +++++ .../serialize-compute-and-transfer.cpp | 14 + tools/replay/replay_settings.h | 3 +- tools/tool_settings.h | 20 +- 20 files changed, 610 insertions(+), 24 deletions(-) create mode 100644 test/test_apps/serialize-compute-and-transfer/CMakeLists.txt create mode 100644 test/test_apps/serialize-compute-and-transfer/app.cpp create mode 100644 test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h create mode 100644 test/test_cases/serialize-compute-and-transfer.cpp diff --git a/USAGE_android.md b/USAGE_android.md index 654f429ef6..904d401043 100644 --- a/USAGE_android.md +++ b/USAGE_android.md @@ -801,6 +801,7 @@ usage: gfxrecon.py replay [-h] [-p LOCAL_FILE] [--version] [--log-level LEVEL] [--replay-event-plugin-path PATH] [--replay-event-plugin-params PARAMS] [--isolate-render-passes] + [--serialize-compute-and-transfer] [file] Launch the replay tool. @@ -1046,6 +1047,9 @@ options: interpreted by the replay tool. (forwarded to replay tool) --isolate-render-passes Isolate render passes by splitting the command buffer into multiple submits. + --serialize-compute-and-transfer + Prevent compute dispatches from overlapping adjacent transfer work by injecting a barrier before + and after each dispatch. (forwarded to replay tool) ``` The command will force-stop an active replay process before starting the replay diff --git a/USAGE_desktop_Vulkan.md b/USAGE_desktop_Vulkan.md index 203c133f43..cbe926c7d1 100644 --- a/USAGE_desktop_Vulkan.md +++ b/USAGE_desktop_Vulkan.md @@ -640,6 +640,7 @@ gfxrecon-replay [-h | --help] [--version] [--cpu-mask ] [-- [--serialize-queue-submissions] [--replay-event-plugin-path ] [--replay-event-plugin-params ] [--isolate-render-passes] + [--serialize-compute-and-transfer] Required arguments: @@ -907,6 +908,9 @@ Optional arguments: interpreted by the replay tool. (forwarded to replay tool) --isolate-render-passes Isolate render passes by splitting the command buffer into multiple submits. + --serialize-compute-and-transfer + Prevent compute dispatches from overlapping adjacent transfer work by injecting a barrier before and after + each dispatch. ``` ### Frame Warm-Up diff --git a/android/scripts/gfxrecon.py b/android/scripts/gfxrecon.py index 17a83eb952..81573a81d1 100644 --- a/android/scripts/gfxrecon.py +++ b/android/scripts/gfxrecon.py @@ -156,6 +156,7 @@ def CreateReplayParser(): parser.add_argument('--replay-event-plugin-path', metavar='PATH', help='Path to a replay event plugin library. If specified, the plugin will be loaded and used to process replay events. (forwarded to replay tool)') parser.add_argument('--replay-event-plugin-params', metavar='PARAMS', help='Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool)') parser.add_argument('--isolate-render-passes', action='store_true', default=False, help='Isolate render passes by splitting the command buffer into multiple submits. (forwarded to replay tool)') + parser.add_argument('--serialize-compute-and-transfer', action='store_true', default=False, help='Prevent compute dispatches from overlapping adjacent transfer work by injecting a barrier before and after each dispatch. (forwarded to replay tool)') return parser @@ -379,6 +380,9 @@ def MakeExtrasString(args): if args.isolate_render_passes: arg_list.append('--isolate-render-passes') + if args.serialize_compute_and_transfer: + arg_list.append('--serialize-compute-and-transfer') + if args.file: arg_list.append(args.file) elif not args.version: diff --git a/android/test/test_apps/common/CMakeLists.txt b/android/test/test_apps/common/CMakeLists.txt index dc8922140e..e6a208f8d9 100644 --- a/android/test/test_apps/common/CMakeLists.txt +++ b/android/test/test_apps/common/CMakeLists.txt @@ -67,6 +67,7 @@ add_test_app(pipeline-binaries) add_test_app(triangle) add_test_app(triangle-extra-device) add_test_app(shader-objects) +add_test_app(serialize-compute-and-transfer) add_test_app(sparse-resources) add_test_app(wait-for-present) add_test_app(trigger-trimming) diff --git a/android/test/test_apps/launcher/CMakeLists.txt b/android/test/test_apps/launcher/CMakeLists.txt index a3a52e7fab..0f00bd02b8 100644 --- a/android/test/test_apps/launcher/CMakeLists.txt +++ b/android/test/test_apps/launcher/CMakeLists.txt @@ -56,6 +56,7 @@ set(GFXRECON_TEST_LAUNCHER_LINK_LIBRARIES gfxrecon-testapp-multisample-depth gfxrecon-testapp-pipeline-binaries gfxrecon-testapp-shader-objects + gfxrecon-testapp-serialize-compute-and-transfer gfxrecon-testapp-sparse-resources gfxrecon-testapp-triangle gfxrecon-testapp-triangle-extra-device diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index 7f197dc965..667dde9f91 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -14113,5 +14113,76 @@ void VulkanReplayConsumerBase::OverrideCmdExecuteGeneratedCommandsEXT( func(command_buffer_info->handle, isPreprocessed, in_pGeneratedCommandsInfo); } +void VulkanReplayConsumerBase::MaybeInjectComputeTransferBarrier( + const VulkanCommandBufferInfo* command_buffer_info) const +{ + if (!options_.serialize_compute_and_transfer) + { + return; + } + + util::BeginInjectedCommands(); + + GFXRECON_ASSERT(command_buffer_info != nullptr); + const VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GFXRECON_ASSERT(device_info != nullptr); + const graphics::VulkanDeviceTable* device_table = GetDeviceTable(device_info->handle); + GFXRECON_ASSERT(device_table != nullptr); + + VkMemoryBarrier memory_barrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER }; + memory_barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + memory_barrier.dstAccessMask = + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT; + + const VkPipelineStageFlags stages = + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT; + + // Make sure compute and transfer write operations before this barrier are completed + // before doing other compute and transfer read/write operations. + device_table->CmdPipelineBarrier( + command_buffer_info->handle, stages, stages, 0, 1, &memory_barrier, 0, nullptr, 0, nullptr); + + util::EndInjectedCommands(); +} + +void VulkanReplayConsumerBase::OverrideCmdDispatch(PFN_vkCmdDispatch func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ) +{ + GFXRECON_ASSERT(command_buffer_info != nullptr); + MaybeInjectComputeTransferBarrier(command_buffer_info); + func(command_buffer_info->handle, groupCountX, groupCountY, groupCountZ); + MaybeInjectComputeTransferBarrier(command_buffer_info); +} + +void VulkanReplayConsumerBase::OverrideCmdDispatchIndirect(PFN_vkCmdDispatchIndirect func, + const VulkanCommandBufferInfo* command_buffer_info, + const VulkanBufferInfo* buffer_info, + VkDeviceSize offset) +{ + GFXRECON_ASSERT(command_buffer_info != nullptr); + GFXRECON_ASSERT(buffer_info != nullptr); + MaybeInjectComputeTransferBarrier(command_buffer_info); + func(command_buffer_info->handle, buffer_info->handle, offset); + MaybeInjectComputeTransferBarrier(command_buffer_info); +} + +void VulkanReplayConsumerBase::OverrideCmdDispatchBase(PFN_vkCmdDispatchBase func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t baseGroupX, + uint32_t baseGroupY, + uint32_t baseGroupZ, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ) +{ + GFXRECON_ASSERT(command_buffer_info != nullptr); + MaybeInjectComputeTransferBarrier(command_buffer_info); + func(command_buffer_info->handle, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + MaybeInjectComputeTransferBarrier(command_buffer_info); +} + GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h index b6bdc0a657..efe58d87aa 100644 --- a/framework/decode/vulkan_replay_consumer_base.h +++ b/framework/decode/vulkan_replay_consumer_base.h @@ -1682,6 +1682,26 @@ class VulkanReplayConsumerBase : public VulkanConsumer VkBool32 isPreprocessed, StructPointerDecoder* pGeneratedCommandsInfo); + void OverrideCmdDispatch(PFN_vkCmdDispatch func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ); + + void OverrideCmdDispatchIndirect(PFN_vkCmdDispatchIndirect func, + const VulkanCommandBufferInfo* command_buffer_info, + const VulkanBufferInfo* buffer_info, + VkDeviceSize offset); + + void OverrideCmdDispatchBase(PFN_vkCmdDispatchBase func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t baseGroupX, + uint32_t baseGroupY, + uint32_t baseGroupZ, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ); + std::function()> AsyncCreateGraphicsPipelines(PFN_vkCreateGraphicsPipelines func, VkResult returnValue, @@ -1931,6 +1951,12 @@ class VulkanReplayConsumerBase : public VulkanConsumer */ void MaybeInjectExecutionBarrier(const VulkanCommandBufferInfo* command_buffer_info) const; + /** + * @brief If the option to serialize compute and transfer operations is enabled, inject a memory barrier + * before and after each compute dispatch to ensure compute and transfer operations do not overlap. + */ + void MaybeInjectComputeTransferBarrier(const VulkanCommandBufferInfo* command_buffer_info) const; + private: struct HardwareBufferInfo { diff --git a/framework/decode/vulkan_replay_options.h b/framework/decode/vulkan_replay_options.h index 9c76cc2bbe..64e8d89bbc 100644 --- a/framework/decode/vulkan_replay_options.h +++ b/framework/decode/vulkan_replay_options.h @@ -267,6 +267,10 @@ struct VulkanReplayOptions : public ReplayOptions /// Isolate render passes by splitting the command buffer into multiple submits. bool isolate_render_passes{ false }; + /// Prevent compute dispatches from overlapping adjacent transfer work by injecting + /// a barrier before and after each dispatch. + bool serialize_compute_and_transfer{ false }; + void MaybeWaitBeforeFirstSubmit() const; void MaybeWaitBeforeFrame() const; }; diff --git a/framework/generated/generated_vulkan_replay_consumer.cpp b/framework/generated/generated_vulkan_replay_consumer.cpp index 1a0e4dd05d..34d8edacc6 100644 --- a/framework/generated/generated_vulkan_replay_consumer.cpp +++ b/framework/generated/generated_vulkan_replay_consumer.cpp @@ -1755,13 +1755,13 @@ void VulkanReplayConsumer::Process_vkCmdDispatch( uint32_t groupCountY, uint32_t groupCountZ) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); - GetDeviceTable(in_commandBuffer)->CmdDispatch(in_commandBuffer, groupCountX, groupCountY, groupCountZ); + OverrideCmdDispatch(GetDeviceTable(in_commandBuffer->handle)->CmdDispatch, in_commandBuffer, groupCountX, groupCountY, groupCountZ); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatch(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatch, in_commandBuffer, groupCountX, groupCountY, groupCountZ); + resource_dumper_->Process_vkCmdDispatch(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatch, in_commandBuffer->handle, groupCountX, groupCountY, groupCountZ); } } @@ -1771,14 +1771,14 @@ void VulkanReplayConsumer::Process_vkCmdDispatchIndirect( format::HandleId buffer, VkDeviceSize offset) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); - VkBuffer in_buffer = MapHandle(buffer, &CommonObjectInfoTable::GetVkBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); + auto in_buffer = GetObjectInfoTable().GetVkBufferInfo(buffer); - GetDeviceTable(in_commandBuffer)->CmdDispatchIndirect(in_commandBuffer, in_buffer, offset); + OverrideCmdDispatchIndirect(GetDeviceTable(in_commandBuffer->handle)->CmdDispatchIndirect, in_commandBuffer, in_buffer, offset); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatchIndirect(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatchIndirect, in_commandBuffer, GetObjectInfoTable().GetVkBufferInfo(buffer), offset); + resource_dumper_->Process_vkCmdDispatchIndirect(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatchIndirect, in_commandBuffer->handle, in_buffer, offset); } } @@ -2705,13 +2705,13 @@ void VulkanReplayConsumer::Process_vkCmdDispatchBase( uint32_t groupCountY, uint32_t groupCountZ) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); - GetDeviceTable(in_commandBuffer)->CmdDispatchBase(in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + OverrideCmdDispatchBase(GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBase, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatchBase(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatchBase, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + resource_dumper_->Process_vkCmdDispatchBase(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBase, in_commandBuffer->handle, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); } } @@ -4963,13 +4963,13 @@ void VulkanReplayConsumer::Process_vkCmdDispatchBaseKHR( uint32_t groupCountY, uint32_t groupCountZ) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); - GetDeviceTable(in_commandBuffer)->CmdDispatchBaseKHR(in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + OverrideCmdDispatchBase(GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBaseKHR, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatchBaseKHR(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatchBaseKHR, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + resource_dumper_->Process_vkCmdDispatchBaseKHR(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBaseKHR, in_commandBuffer->handle, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); } } diff --git a/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json b/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json index baaafd2e7b..b334c7138e 100644 --- a/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json +++ b/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json @@ -187,6 +187,10 @@ "vkCmdEndRenderingKHR": "OverrideCmdEndRendering", "vkCreateIndirectExecutionSetEXT": "OverrideCreateIndirectExecutionSetEXT", "vkCmdPreprocessGeneratedCommandsEXT": "OverrideCmdPreprocessGeneratedCommandsEXT", - "vkCmdExecuteGeneratedCommandsEXT": "OverrideCmdExecuteGeneratedCommandsEXT" + "vkCmdExecuteGeneratedCommandsEXT": "OverrideCmdExecuteGeneratedCommandsEXT", + "vkCmdDispatch": "OverrideCmdDispatch", + "vkCmdDispatchIndirect": "OverrideCmdDispatchIndirect", + "vkCmdDispatchBase": "OverrideCmdDispatchBase", + "vkCmdDispatchBaseKHR": "OverrideCmdDispatchBase" } } \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3c206dd04a..0a4e8441c7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -46,6 +46,7 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) test_cases/ahb.cpp test_cases/debug-utils.cpp test_cases/isolate-render-passes.cpp + test_cases/serialize-compute-and-transfer.cpp test_cases/triangle.cpp test_cases/multisample-depth.cpp test_cases/shader-objects.cpp diff --git a/test/test_apps/CMakeLists.txt b/test/test_apps/CMakeLists.txt index d3fd68f83a..b883ccd91b 100644 --- a/test/test_apps/CMakeLists.txt +++ b/test/test_apps/CMakeLists.txt @@ -34,6 +34,7 @@ endif() add_subdirectory(common) add_subdirectory(debug-utils) add_subdirectory(isolate-render-passes) +add_subdirectory(serialize-compute-and-transfer) add_subdirectory(triangle) add_subdirectory(triangle-extra-device) add_subdirectory(multisample-depth) diff --git a/test/test_apps/launcher/CMakeLists.txt b/test/test_apps/launcher/CMakeLists.txt index 0c2e655b05..08d8809dd7 100644 --- a/test/test_apps/launcher/CMakeLists.txt +++ b/test/test_apps/launcher/CMakeLists.txt @@ -43,6 +43,7 @@ set(GFXRECON_TEST_LAUNCHER_LINK_LIBRARIES gfxrecon-testapp-debug-utils gfxrecon-testapp-host-image-copy gfxrecon-testapp-isolate-render-passes + gfxrecon-testapp-serialize-compute-and-transfer gfxrecon-testapp-multisample-depth gfxrecon-testapp-pipeline-binaries gfxrecon-testapp-set-environment diff --git a/test/test_apps/launcher/test_launcher.cpp b/test/test_apps/launcher/test_launcher.cpp index c927bfb657..8e4c805632 100644 --- a/test/test_apps/launcher/test_launcher.cpp +++ b/test/test_apps/launcher/test_launcher.cpp @@ -32,6 +32,7 @@ #include #include #include +#include #include #include #include @@ -67,6 +68,7 @@ static const char* kAppNames[] = { "debug-utils", "host-image-copy", "isolate-render-passes", + "serialize-compute-and-transfer", "multisample-depth", "pipeline-binaries", #ifndef __ANDROID__ @@ -163,6 +165,10 @@ CreateTestApp(std::unique_ptr application, { app = std::make_unique(); } + else if (app_name == "serialize-compute-and-transfer") + { + app = std::make_unique(); + } else if (app_name == "multisample-depth") { app = std::make_unique(); diff --git a/test/test_apps/serialize-compute-and-transfer/CMakeLists.txt b/test/test_apps/serialize-compute-and-transfer/CMakeLists.txt new file mode 100644 index 0000000000..1648b189d3 --- /dev/null +++ b/test/test_apps/serialize-compute-and-transfer/CMakeLists.txt @@ -0,0 +1,39 @@ +############################################################################### +# Copyright (c) 2026 LunarG, Inc. +# All rights reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# Author: LunarG Team +# Description: CMake script for serialize-compute-and-transfer test app +############################################################################### + +add_library(gfxrecon-testapp-serialize-compute-and-transfer app.cpp) + +target_include_directories(gfxrecon-testapp-serialize-compute-and-transfer + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(gfxrecon-testapp-serialize-compute-and-transfer gfxrecon-testapp-base) + +common_build_directives(gfxrecon-testapp-serialize-compute-and-transfer) + +if(NOT CMAKE_SYSTEM_NAME MATCHES "Android") + install(TARGETS gfxrecon-testapp-serialize-compute-and-transfer RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTAPPDIR}/serialize-compute-and-transfer) +endif() diff --git a/test/test_apps/serialize-compute-and-transfer/app.cpp b/test/test_apps/serialize-compute-and-transfer/app.cpp new file mode 100644 index 0000000000..78bb04d595 --- /dev/null +++ b/test/test_apps/serialize-compute-and-transfer/app.cpp @@ -0,0 +1,321 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(serialize_compute_and_transfer) + +// SPIR-V for a trivial compute shader with no descriptors: +// #version 450 +// layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in; +// void main() {} +static const uint32_t kComputeShaderSpirv[] = { + 0x07230203, 0x00010000, 0x000d000b, 0x0000000a, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, 0x00000001, + 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, 0x0005000f, 0x00000005, + 0x00000004, 0x6e69616d, 0x00000000, 0x00060010, 0x00000004, 0x00000011, 0x00000001, 0x00000001, 0x00000001, + 0x00030003, 0x00000002, 0x000001c2, 0x000a0004, 0x475f4c47, 0x4c474f4f, 0x70635f45, 0x74735f70, 0x5f656c79, + 0x656e696c, 0x7269645f, 0x69746365, 0x00006576, 0x00080004, 0x475f4c47, 0x4c474f4f, 0x6e695f45, 0x64756c63, + 0x69645f65, 0x74636572, 0x00657669, 0x00040005, 0x00000004, 0x6e69616d, 0x00000000, 0x00040047, 0x00000009, + 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, + 0x00000020, 0x00000000, 0x00040017, 0x00000007, 0x00000006, 0x00000003, 0x0004002b, 0x00000006, 0x00000008, + 0x00000001, 0x0006002c, 0x00000007, 0x00000009, 0x00000008, 0x00000008, 0x00000008, 0x00050036, 0x00000002, + 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x000100fd, 0x00010038 +}; + +const VkDeviceSize BUFFER_SIZE = 256; + +const int NUM_FRAMES = 3; +#define IS_RUNNING(frame_num) frame_num < NUM_FRAMES; + +void App::configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig* test_config) +{ + if (test_config) + { + test_config->device_api_version_override = VK_MAKE_API_VERSION(0, 1, 3, 296); + } + + TestAppBase::configure_instance_builder(instance_builder, test_config); +} + +void App::create_buffer(VkDeviceSize size, VkBufferUsageFlags usage, VkBuffer* buffer, VkDeviceMemory* memory) +{ + VkBufferCreateInfo buffer_info = {}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = size; + buffer_info.usage = usage; + buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VERIFY_VK_RESULT("failed to create buffer", init.disp.createBuffer(&buffer_info, nullptr, buffer)); + + VkMemoryRequirements memory_requirements; + init.disp.getBufferMemoryRequirements(*buffer, &memory_requirements); + + VkMemoryAllocateInfo allocate_info = {}; + allocate_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocate_info.allocationSize = memory_requirements.size; + allocate_info.memoryTypeIndex = find_memory_type( + memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + VERIFY_VK_RESULT("failed to allocate buffer memory", init.disp.allocateMemory(&allocate_info, nullptr, memory)); + + VERIFY_VK_RESULT("failed to bind buffer memory", init.disp.bindBufferMemory(*buffer, *memory, 0)); +} + +void App::create_buffers() +{ + // Two buffers copied back and forth as the transfer work. + create_buffer(BUFFER_SIZE, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + &buffer_a_, + &buffer_a_memory_); + create_buffer(BUFFER_SIZE, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + &buffer_b_, + &buffer_b_memory_); + + // Indirect-args buffer for vkCmdDispatchIndirect, filled with a single workgroup. + create_buffer(sizeof(VkDispatchIndirectCommand), + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, + &indirect_buffer_, + &indirect_buffer_memory_); + + void* data = nullptr; + VERIFY_VK_RESULT("failed to map indirect buffer", + init.disp.mapMemory(indirect_buffer_memory_, 0, sizeof(VkDispatchIndirectCommand), 0, &data)); + VkDispatchIndirectCommand indirect_command{ 1, 1, 1 }; + memcpy(data, &indirect_command, sizeof(indirect_command)); + init.disp.unmapMemory(indirect_buffer_memory_); +} + +void App::create_compute_pipeline() +{ + VkShaderModuleCreateInfo shader_module_info = {}; + shader_module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + shader_module_info.codeSize = sizeof(kComputeShaderSpirv); + shader_module_info.pCode = kComputeShaderSpirv; + VERIFY_VK_RESULT("failed to create compute shader module", + init.disp.createShaderModule(&shader_module_info, nullptr, &compute_shader_)); + + // No descriptor sets and no push constants: the shader only needs to make the dispatches valid. + VkPipelineLayoutCreateInfo pipeline_layout_info = {}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + VERIFY_VK_RESULT("failed to create pipeline layout", + init.disp.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline_layout_)); + + VkComputePipelineCreateInfo pipeline_info = {}; + pipeline_info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + pipeline_info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pipeline_info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipeline_info.stage.module = compute_shader_; + pipeline_info.stage.pName = "main"; + pipeline_info.layout = pipeline_layout_; + VERIFY_VK_RESULT("failed to create compute pipeline", + init.disp.createComputePipelines(VK_NULL_HANDLE, 1, &pipeline_info, nullptr, &compute_pipeline_)); +} + +bool App::frame(const int frame_num) +{ + init.disp.waitForFences(1, &sync_.in_flight_fences[current_frame_], VK_TRUE, UINT64_MAX); + + uint32_t image_index = 0; + VkResult result = init.disp.acquireNextImageKHR( + init.swapchain, UINT64_MAX, sync_.available_semaphores[current_frame_], VK_NULL_HANDLE, &image_index); + + if (result == VK_ERROR_OUT_OF_DATE_KHR) + { + recreate_swapchain(true); + return IS_RUNNING(frame_num); + } + else if (result != VK_SUCCESS && result != VK_SUBOPTIMAL_KHR) + { + throw test::vulkan_exception("failed to acquire next image", result); + } + + if (sync_.image_in_flight[image_index] != VK_NULL_HANDLE) + { + init.disp.waitForFences(1, &sync_.image_in_flight[image_index], VK_TRUE, UINT64_MAX); + } + sync_.image_in_flight[image_index] = sync_.in_flight_fences[current_frame_]; + + init.disp.resetCommandPool(command_pools_[current_frame_], 0); + VkCommandBufferAllocateInfo allocate_info = {}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandBufferCount = 1; + allocate_info.commandPool = command_pools_[current_frame_]; + VkCommandBuffer command_buffer; + result = init.disp.allocateCommandBuffers(&allocate_info, &command_buffer); + VERIFY_VK_RESULT("failed to allocate command buffer", result); + + { + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + result = init.disp.beginCommandBuffer(command_buffer, &begin_info); + VERIFY_VK_RESULT("failed to begin command buffer", result); + + // The test does not render: transition the acquired image straight to PRESENT_SRC so it can be presented. + { + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + image_barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_NONE; + image_barrier.dstAccessMask = VK_ACCESS_NONE; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + // Transfer, then a run of compute dispatches, then transfer again. With --serialize-compute-and-transfer the + // replay tool injects a pipeline barrier before and after each of the three dispatches below. + VkBufferCopy copy_region = {}; + copy_region.size = BUFFER_SIZE; + + init.disp.cmdCopyBuffer(command_buffer, buffer_a_, buffer_b_, 1, ©_region); + + init.disp.cmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute_pipeline_); + init.disp.cmdDispatch(command_buffer, 1, 1, 1); + init.disp.cmdDispatchBase(command_buffer, 0, 0, 0, 1, 1, 1); + init.disp.cmdDispatchIndirect(command_buffer, indirect_buffer_, 0); + + init.disp.cmdCopyBuffer(command_buffer, buffer_b_, buffer_a_, 1, ©_region); + + result = init.disp.endCommandBuffer(command_buffer); + VERIFY_VK_RESULT("failed to end command buffer", result); + } + + VkSubmitInfo submitInfo = {}; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + + VkSemaphore wait_semaphores[] = { sync_.available_semaphores[current_frame_] }; + VkPipelineStageFlags wait_stages[] = { VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT }; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = wait_semaphores; + submitInfo.pWaitDstStageMask = wait_stages; + + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &command_buffer; + + VkSemaphore signal_semaphores[] = { sync_.finished_semaphore[image_index] }; + submitInfo.signalSemaphoreCount = 1; + submitInfo.pSignalSemaphores = signal_semaphores; + + init.disp.resetFences(1, &sync_.in_flight_fences[current_frame_]); + + result = init.disp.queueSubmit(graphics_queue_, 1, &submitInfo, sync_.in_flight_fences[current_frame_]); + VERIFY_VK_RESULT("failed to submit queue", result); + + VkPresentInfoKHR present_info = {}; + present_info.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; + + present_info.waitSemaphoreCount = 1; + present_info.pWaitSemaphores = signal_semaphores; + + VkSwapchainKHR swapChains[] = { init.swapchain }; + present_info.swapchainCount = 1; + present_info.pSwapchains = swapChains; + + present_info.pImageIndices = &image_index; + + result = init.disp.queuePresentKHR(present_queue_, &present_info); + if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR) + { + recreate_swapchain(true); + return frame_num >= NUM_FRAMES; + } + VERIFY_VK_RESULT("failed to present queue", result); + + current_frame_ = (current_frame_ + 1) % MAX_FRAMES_IN_FLIGHT; + + return IS_RUNNING(frame_num); +} + +void App::cleanup() +{ + init.disp.destroyPipeline(compute_pipeline_, nullptr); + init.disp.destroyPipelineLayout(pipeline_layout_, nullptr); + init.disp.destroyShaderModule(compute_shader_, nullptr); + + init.disp.destroyBuffer(buffer_a_, nullptr); + init.disp.freeMemory(buffer_a_memory_, nullptr); + init.disp.destroyBuffer(buffer_b_, nullptr); + init.disp.freeMemory(buffer_b_memory_, nullptr); + init.disp.destroyBuffer(indirect_buffer_, nullptr); + init.disp.freeMemory(indirect_buffer_memory_, nullptr); + + for (size_t i = 0; i < sync_.finished_semaphore.size(); ++i) + { + init.disp.destroySemaphore(sync_.finished_semaphore[i], nullptr); + } + for (size_t i = 0; i < MAX_FRAMES_IN_FLIGHT; i++) + { + init.disp.destroySemaphore(sync_.available_semaphores[i], nullptr); + init.disp.destroyFence(sync_.in_flight_fences[i], nullptr); + } + + for (auto command_pool : command_pools_) + { + init.disp.destroyCommandPool(command_pool, nullptr); + } +} + +void App::setup() +{ + auto graphics_queue = init.device.get_queue(test::QueueType::graphics); + if (!graphics_queue.has_value()) + throw std::runtime_error("could not get graphics queue"); + graphics_queue_ = *graphics_queue; + + auto present_queue = init.device.get_queue(test::QueueType::present); + if (!present_queue.has_value()) + throw std::runtime_error("could not get present queue"); + present_queue_ = *present_queue; + + auto queue_family_index = init.device.get_queue_index(test::QueueType::graphics); + if (!queue_family_index) + throw std::runtime_error("could not find graphics queue"); + for (auto& command_pool : command_pools_) + { + command_pool = test::create_command_pool(init.disp, *queue_family_index); + } + + sync_ = test::create_sync_objects(init.swapchain, init.disp, MAX_FRAMES_IN_FLIGHT); + + create_compute_pipeline(); + create_buffers(); +} + +GFXRECON_END_NAMESPACE(serialize_compute_and_transfer) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h b/test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h new file mode 100644 index 0000000000..3471fc6879 --- /dev/null +++ b/test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h @@ -0,0 +1,81 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#ifndef GFXRECON_TESTAPP_SERIALIZE_COMPUTE_AND_TRANSFER_H +#define GFXRECON_TESTAPP_SERIALIZE_COMPUTE_AND_TRANSFER_H + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(serialize_compute_and_transfer) + +const size_t MAX_FRAMES_IN_FLIGHT = 2; + +/** + * Minimal app that, each frame, records transfer (vkCmdCopyBuffer) and compute (vkCmdDispatch, + * vkCmdDispatchBase, vkCmdDispatchIndirect) commands into a single command buffer interleaved with each other, then + * submits it while waiting on the binary swapchain-acquire semaphore. It deliberately uses a trivial empty compute + * shader with no descriptors, keeping the focus on the dispatch commands that gfxrecon-replay + * --serialize-compute-and-transfer wraps with injected pipeline barriers. + */ +class App : public test::TestAppBase +{ + VkQueue graphics_queue_; + VkQueue present_queue_; + + VkCommandPool command_pools_[MAX_FRAMES_IN_FLIGHT]; + + size_t current_frame_ = 0; + + test::Sync sync_; + + // Trivial compute pipeline (empty shader, no descriptors) used to make the dispatches valid. + VkShaderModule compute_shader_ = VK_NULL_HANDLE; + VkPipelineLayout pipeline_layout_ = VK_NULL_HANDLE; + VkPipeline compute_pipeline_ = VK_NULL_HANDLE; + + // Two buffers copied back and forth (the transfer work), and an indirect-args buffer for vkCmdDispatchIndirect. + VkBuffer buffer_a_ = VK_NULL_HANDLE; + VkDeviceMemory buffer_a_memory_ = VK_NULL_HANDLE; + VkBuffer buffer_b_ = VK_NULL_HANDLE; + VkDeviceMemory buffer_b_memory_ = VK_NULL_HANDLE; + VkBuffer indirect_buffer_ = VK_NULL_HANDLE; + VkDeviceMemory indirect_buffer_memory_ = VK_NULL_HANDLE; + + void create_buffer(VkDeviceSize size, VkBufferUsageFlags usage, VkBuffer* buffer, VkDeviceMemory* memory); + void create_buffers(); + void create_compute_pipeline(); + void cleanup() override; + bool frame(const int frame_num) override; + void setup() override; + + void configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig*) override; +}; + +GFXRECON_END_NAMESPACE(serialize_compute_and_transfer) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // GFXRECON_TESTAPP_SERIALIZE_COMPUTE_AND_TRANSFER_H diff --git a/test/test_cases/serialize-compute-and-transfer.cpp b/test/test_cases/serialize-compute-and-transfer.cpp new file mode 100644 index 0000000000..021fcc7a98 --- /dev/null +++ b/test/test_cases/serialize-compute-and-transfer.cpp @@ -0,0 +1,14 @@ +#include + +#include "verify-gfxr.h" + +/** + * Capture the serialize-compute-and-transfer app and replay it with --serialize-compute-and-transfer. + * The replay tool wraps every compute dispatch with an injected pipeline barrier before and after, + * so compute and transfer work cannot overlap on the GPU. + * This test exercises that code path against the mock ICD and asserts the replay completes successfully. + */ +TEST(SerializeComputeAndTransfer, ReplayInjectsBarriersAroundDispatches) +{ + capture_and_replay("serialize-compute-and-transfer", { "--serialize-compute-and-transfer" }); +} diff --git a/tools/replay/replay_settings.h b/tools/replay/replay_settings.h index a8f5eb0973..9a993f8093 100644 --- a/tools/replay/replay_settings.h +++ b/tools/replay/replay_settings.h @@ -36,7 +36,8 @@ const char kOptions[] = "--dump-resources-modifiable-state-only,--pbi-all,--preload-measurement-range,--add-new-pipeline-caches,--" "screenshot-ignore-FrameBoundaryANDROID,--screenshot-apply-prerotation,--deduplicate-device,--log-timestamps,--" "capture,--idle-before-submit,--" - "serialize-render-passes,--serialize-queue-submissions,--async-processing,--isolate-render-passes"; + "serialize-render-passes,--serialize-queue-submissions,--async-processing,--isolate-render-passes,--serialize-" + "compute-and-transfer"; const char kArguments[] = "--log-level,--log-file,--cpu-mask,--gpu,--gpu-group,--pause-frame,--wsi,--surface-index,-m|--memory-translation," "--replace-shaders,--screenshots,--screenshot-interval,--denied-messages,--allowed-messages,--screenshot-format,--" diff --git a/tools/tool_settings.h b/tools/tool_settings.h index ec8b4eb452..70ea4fdf97 100644 --- a/tools/tool_settings.h +++ b/tools/tool_settings.h @@ -168,14 +168,15 @@ const char kDumpResourcesModifiableStateOnly[] = "--dump-resources-modifiable-st const char kDumpResourcesBeforeDrawOption[] = "--dump-resources-before-draw"; #endif -const char kDumpResourcesArgument[] = "--dump-resources"; -const char kDumpResourcesDirArgument[] = "--dump-resources-dir"; -const char kFrameWarmUpSpirv[] = "--frame-warm-up-spirv"; -const char kFrameWarmUpLoad[] = "--frame-warm-up-load"; -const char kSerializeQueueSubmissions[] = "--serialize-queue-submissions"; -const char kReplayEventPluginPath[] = "--replay-event-plugin-path"; -const char kReplayEventPluginParams[] = "--replay-event-plugin-params"; -const char kIsolateRenderPasses[] = "--isolate-render-passes"; +const char kDumpResourcesArgument[] = "--dump-resources"; +const char kDumpResourcesDirArgument[] = "--dump-resources-dir"; +const char kFrameWarmUpSpirv[] = "--frame-warm-up-spirv"; +const char kFrameWarmUpLoad[] = "--frame-warm-up-load"; +const char kSerializeQueueSubmissions[] = "--serialize-queue-submissions"; +const char kReplayEventPluginPath[] = "--replay-event-plugin-path"; +const char kReplayEventPluginParams[] = "--replay-event-plugin-params"; +const char kIsolateRenderPasses[] = "--isolate-render-passes"; +const char kSerializeComputeAndTransfer[] = "--serialize-compute-and-transfer"; enum class WsiPlatform { @@ -1403,7 +1404,8 @@ GetVulkanReplayOptions(const gfxrecon::util::ArgumentParser& arg_parse replay_options.replay_event_plugin_path = arg_parser.GetArgumentValue(kReplayEventPluginPath); replay_options.replay_event_plugin_params = arg_parser.GetArgumentValue(kReplayEventPluginParams); - replay_options.isolate_render_passes = arg_parser.IsOptionSet(kIsolateRenderPasses); + replay_options.isolate_render_passes = arg_parser.IsOptionSet(kIsolateRenderPasses); + replay_options.serialize_compute_and_transfer = arg_parser.IsOptionSet(kSerializeComputeAndTransfer); return replay_options; }