diff --git a/USAGE_android.md b/USAGE_android.md index 9b2996141a..904d401043 100644 --- a/USAGE_android.md +++ b/USAGE_android.md @@ -800,6 +800,8 @@ usage: gfxrecon.py replay [-h] [-p LOCAL_FILE] [--version] [--log-level LEVEL] [--serialize-queue-submissions] [--replay-event-plugin-path PATH] [--replay-event-plugin-params PARAMS] + [--isolate-render-passes] + [--serialize-compute-and-transfer] [file] Launch the replay tool. @@ -1043,6 +1045,11 @@ options: Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool) + --isolate-render-passes + Isolate render passes by splitting the command buffer into multiple submits. + --serialize-compute-and-transfer + Prevent compute dispatches from overlapping adjacent transfer work by injecting a barrier before + and after each dispatch. (forwarded to replay tool) ``` The command will force-stop an active replay process before starting the replay diff --git a/USAGE_desktop_Vulkan.md b/USAGE_desktop_Vulkan.md index 98d6e4cf1c..cbe926c7d1 100644 --- a/USAGE_desktop_Vulkan.md +++ b/USAGE_desktop_Vulkan.md @@ -639,6 +639,8 @@ gfxrecon-replay [-h | --help] [--version] [--cpu-mask ] [-- [--wait-before-frame MILLISECONDS] [--serialize-queue-submissions] [--replay-event-plugin-path ] [--replay-event-plugin-params ] + [--isolate-render-passes] + [--serialize-compute-and-transfer] Required arguments: @@ -904,6 +906,11 @@ Optional arguments: Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool) + --isolate-render-passes + Isolate render passes by splitting the command buffer into multiple submits. + --serialize-compute-and-transfer + Prevent compute dispatches from overlapping adjacent transfer work by injecting a barrier before and after + each dispatch. ``` ### Frame Warm-Up diff --git a/android/scripts/gfxrecon.py b/android/scripts/gfxrecon.py index 8cb5a91459..81573a81d1 100644 --- a/android/scripts/gfxrecon.py +++ b/android/scripts/gfxrecon.py @@ -155,6 +155,8 @@ def CreateReplayParser(): parser.add_argument('--serialize-queue-submissions', action='store_true', default=False, help='Serialize submit entries within one vkQueueSubmit or vkQueueSubmit2 call by adding semaphores between consecutive submits during replay. (forwarded to replay tool)') parser.add_argument('--replay-event-plugin-path', metavar='PATH', help='Path to a replay event plugin library. If specified, the plugin will be loaded and used to process replay events. (forwarded to replay tool)') parser.add_argument('--replay-event-plugin-params', metavar='PARAMS', help='Parameters to forward to the replay event plugin. The format of the parameters is determined by the plugin and is not interpreted by the replay tool. (forwarded to replay tool)') + parser.add_argument('--isolate-render-passes', action='store_true', default=False, help='Isolate render passes by splitting the command buffer into multiple submits. (forwarded to replay tool)') + parser.add_argument('--serialize-compute-and-transfer', action='store_true', default=False, help='Prevent compute dispatches from overlapping adjacent transfer work by injecting a barrier before and after each dispatch. (forwarded to replay tool)') return parser @@ -375,6 +377,12 @@ def MakeExtrasString(args): arg_list.append('--replay-event-plugin-params') arg_list.append('{}'.format(args.replay_event_plugin_params)) + if args.isolate_render_passes: + arg_list.append('--isolate-render-passes') + + if args.serialize_compute_and_transfer: + arg_list.append('--serialize-compute-and-transfer') + if args.file: arg_list.append(args.file) elif not args.version: diff --git a/android/test/test_apps/common/CMakeLists.txt b/android/test/test_apps/common/CMakeLists.txt index 75aac63915..e6a208f8d9 100644 --- a/android/test/test_apps/common/CMakeLists.txt +++ b/android/test/test_apps/common/CMakeLists.txt @@ -61,11 +61,13 @@ add_test_app(debug-utils) add_test_app(external-memory-fd-export) add_test_app(external-memory-fd-import) add_test_app(host-image-copy) +add_test_app(isolate-render-passes) add_test_app(multisample-depth) add_test_app(pipeline-binaries) add_test_app(triangle) add_test_app(triangle-extra-device) add_test_app(shader-objects) +add_test_app(serialize-compute-and-transfer) add_test_app(sparse-resources) add_test_app(wait-for-present) add_test_app(trigger-trimming) diff --git a/android/test/test_apps/launcher/CMakeLists.txt b/android/test/test_apps/launcher/CMakeLists.txt index 7c6b2ff08f..0f00bd02b8 100644 --- a/android/test/test_apps/launcher/CMakeLists.txt +++ b/android/test/test_apps/launcher/CMakeLists.txt @@ -52,9 +52,11 @@ set(GFXRECON_TEST_LAUNCHER_LINK_LIBRARIES gfxrecon-testapp-acquired-image gfxrecon-testapp-debug-utils gfxrecon-testapp-host-image-copy + gfxrecon-testapp-isolate-render-passes gfxrecon-testapp-multisample-depth gfxrecon-testapp-pipeline-binaries gfxrecon-testapp-shader-objects + gfxrecon-testapp-serialize-compute-and-transfer gfxrecon-testapp-sparse-resources gfxrecon-testapp-triangle gfxrecon-testapp-triangle-extra-device diff --git a/framework/decode/CMakeLists.txt b/framework/decode/CMakeLists.txt index b141923b98..c88e075d74 100644 --- a/framework/decode/CMakeLists.txt +++ b/framework/decode/CMakeLists.txt @@ -238,6 +238,8 @@ target_sources(gfxrecon_decode ${CMAKE_CURRENT_LIST_DIR}/vulkan_referenced_block_consumer_base.cpp ${CMAKE_CURRENT_LIST_DIR}/vulkan_remap_allocator.h ${CMAKE_CURRENT_LIST_DIR}/vulkan_remap_allocator.cpp + ${CMAKE_CURRENT_LIST_DIR}/vulkan_command_splitter.h + ${CMAKE_CURRENT_LIST_DIR}/vulkan_command_splitter.cpp ${CMAKE_CURRENT_LIST_DIR}/vulkan_replay_consumer_base.h ${CMAKE_CURRENT_LIST_DIR}/vulkan_replay_consumer_base.cpp ${CMAKE_CURRENT_LIST_DIR}/vulkan_replay_options.h diff --git a/framework/decode/vulkan_command_splitter.cpp b/framework/decode/vulkan_command_splitter.cpp new file mode 100644 index 0000000000..81b0571d8a --- /dev/null +++ b/framework/decode/vulkan_command_splitter.cpp @@ -0,0 +1,372 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ +#include "decode/vulkan_command_splitter.h" +#include "decode/vulkan_submit_info_helper.h" +#include "generated/generated_vulkan_enum_to_string.h" +#include "util/callbacks.h" +#include "util/logging.h" + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(decode) + +VulkanCommandBufferSplitInfo::VulkanCommandBufferSplitInfo(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table, + format::HandleId command_buffer_id) : + device_info_(device_info), + device_table_(device_table), + object_table_(object_table), + split_semaphore_(device_info, device_table) +{ + GFXRECON_ASSERT(device_info != nullptr); + GFXRECON_ASSERT(device_table != nullptr); + GFXRECON_ASSERT(object_table != nullptr); + + // Store the original command buffer handle and map it to the command buffer ID for future reference. + VulkanCommandBufferInfo* command_buffer_info = object_table_->GetVkCommandBufferInfo(command_buffer_id); + GFXRECON_ASSERT(command_buffer_info != nullptr); + original_handle_ = command_buffer_info->handle; +} + +VkCommandBuffer VulkanCommandBufferSplitInfo::GetNextHandle(const VulkanCommandBufferInfo* command_buffer_info) +{ + VkCommandBuffer next_handle = VK_NULL_HANDLE; + + // Create a new handle if there are none to reuse. + if (next_associated_index_ >= associated_handles_.size()) + { + VulkanCommandPoolInfo* pool_info = object_table_->GetVkCommandPoolInfo(command_buffer_info->pool_id); + + const VulkanDeviceInfo* device_info = object_table_->GetVkDeviceInfo(command_buffer_info->parent_id); + GFXRECON_ASSERT(device_info == device_info_); + + constexpr size_t NEW_COMMAND_BUFFER_COUNT = 8; + const size_t previous_size = associated_handles_.size(); + associated_handles_.resize(previous_size + NEW_COMMAND_BUFFER_COUNT); + VkCommandBuffer* new_handles_ptr = associated_handles_.data() + previous_size; + + // Create a bunch of new handles for the split. + VkCommandBufferAllocateInfo alloc_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO }; + alloc_info.commandPool = pool_info->handle; + alloc_info.level = VK_COMMAND_BUFFER_LEVEL_PRIMARY; + alloc_info.commandBufferCount = NEW_COMMAND_BUFFER_COUNT; + + VkResult result = device_table_->AllocateCommandBuffers(device_info_->handle, &alloc_info, new_handles_ptr); + GFXRECON_ASSERT(result == VK_SUCCESS); + } + + next_handle = associated_handles_[next_associated_index_]; + ++next_associated_index_; + + // Store current handle in split vector. + split_handles_.push_back(command_buffer_info->handle); + + return next_handle; +} + +VulkanCommandSplitter::VulkanCommandSplitter(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table) : + device_info_(device_info), device_table_(device_table), object_table_(object_table) +{ + GFXRECON_ASSERT(device_info != nullptr); + GFXRECON_ASSERT(device_table != nullptr); + GFXRECON_ASSERT(object_table != nullptr); +} + +VulkanCommandBufferSplitInfo& VulkanCommandSplitter::GetOrCreateSplitInfo(format::HandleId command_buffer_id) +{ + if (auto it = split_infos_.find(command_buffer_id); it != split_infos_.end()) + { + return it->second; + } + + auto [new_it, success] = split_infos_.insert( + { command_buffer_id, + VulkanCommandBufferSplitInfo(device_info_, device_table_, object_table_, command_buffer_id) }); + GFXRECON_ASSERT(success); + return new_it->second; +} + +VulkanCommandBufferSplitInfo* VulkanCommandSplitter::GetSplitInfo(format::HandleId command_buffer_id) +{ + if (auto it = split_infos_.find(command_buffer_id); it != split_infos_.end()) + { + return &it->second; + } + return nullptr; +} + +VkCommandBuffer VulkanCommandBufferSplitInfo::ResetSplitHandles() +{ + GFXRECON_ASSERT(!split_handles_.empty()); + + // Reset all the handles of the split. + for (VkCommandBuffer handle : split_handles_) + { + VkResult result = device_table_->ResetCommandBuffer(handle, 0); + GFXRECON_ASSERT(result == VK_SUCCESS); + } + + // Clear the split handles vector and reset the index. + split_handles_.clear(); + next_associated_index_ = 0; + + return original_handle_; +} + +void VulkanCommandBufferSplitInfo::FreeCommandBuffers(VkCommandPool pool) +{ + GFXRECON_ASSERT(!associated_handles_.empty()); + + device_table_->FreeCommandBuffers( + device_info_->handle, pool, static_cast(associated_handles_.size()), associated_handles_.data()); +} + +void VulkanCommandSplitter::SplitCommandBuffer(VulkanCommandBufferInfo* command_buffer_info) +{ + // Make sure current handle is mapped to the original command buffer ID. + original_command_buffer_id_[command_buffer_info->handle] = command_buffer_info->capture_id; + + device_table_->EndCommandBuffer(command_buffer_info->handle); + + // Update the command buffer info to use the new handle for subsequent calls. + command_buffer_info->handle = + GetOrCreateSplitInfo(command_buffer_info->capture_id).GetNextHandle(command_buffer_info); + original_command_buffer_id_[command_buffer_info->handle] = command_buffer_info->capture_id; + + VkCommandBufferBeginInfo begin_info = { VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO }; + device_table_->BeginCommandBuffer(command_buffer_info->handle, &begin_info); +} + +graphics::VulkanSemaphore VulkanCommandSplitter::SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores) +{ + auto submits_command_buffers = GetCommandBuffersFromSubmitInfos(current_submits_span); + return SubmitPreviouslySplitCommandBuffers(queue_info, submits_command_buffers, wait_semaphores); +} + +graphics::VulkanSemaphore VulkanCommandSplitter::SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores) +{ + auto submits_command_buffers = GetCommandBuffersFromSubmitInfos(current_submits_span); + return SubmitPreviouslySplitCommandBuffers(queue_info, submits_command_buffers, wait_semaphores); +} + +graphics::VulkanSemaphore VulkanCommandSplitter::SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span> current_submits_cmdbufs, + const std::span wait_semaphores) +{ + util::MarkInjectedCommandsHelper mark_injected_commands_helper; + graphics::VulkanSemaphore signal_semaphore{ VK_NULL_HANDLE, 0 }; + + // A QueueSubmit might have multiple submit infos (current_submits_span). + // Each submit info might have multiple command buffers. + // We check those command buffers, to see if any of them were split from the original command buffer. + + std::vector prev_submits; + std::vector> prev_submits_command_buffers; + + for (size_t span_index = 0; span_index < current_submits_cmdbufs.size(); ++span_index) + { + const auto& command_buffers = current_submits_cmdbufs[span_index]; + + for (uint32_t cmdbuf_index = 0; cmdbuf_index < command_buffers.size(); ++cmdbuf_index) + { + VkCommandBuffer command_buffer = command_buffers[cmdbuf_index]; + if (original_command_buffer_id_.contains(command_buffer)) + { + // The command buffer in this submit was split, + // so we need to submit the previous splits before submitting this one. + auto command_buffer_id = original_command_buffer_id_[command_buffer]; + + // Each command buffer handle will go in its own submit. + const VulkanCommandBufferSplitInfo* split_info = GetSplitInfo(command_buffer_id); + auto& split_handles = split_info->GetSplitHandles(); + GFXRECON_ASSERT(!split_handles.empty()); + + // Make enough space for the new submits. + size_t submit_count = std::max(prev_submits.size(), split_handles.size()); + prev_submits.resize(submit_count); + prev_submits_command_buffers.resize(submit_count); + + // Store each handle in the corresponding submit. + for (size_t split_index = 0; split_index < split_handles.size(); ++split_index) + { + auto& prev_command_buffers = prev_submits_command_buffers[split_index]; + // Prev submits command buffers correspond to the previous handles in order. + prev_command_buffers.push_back(split_handles[split_index]); + } + } + } + } + + // Each submit will go in its own QueueSubmit. + for (size_t i = 0; i < prev_submits.size(); ++i) + { + auto& prev_command_buffers = prev_submits_command_buffers[i]; + + if (!prev_command_buffers.empty()) + { + format::HandleId command_buffer_id = original_command_buffer_id_[prev_command_buffers[0]]; + VulkanCommandBufferSplitInfo* split_info = GetSplitInfo(command_buffer_id); + + VkSubmitInfo submit_info = { VK_STRUCTURE_TYPE_SUBMIT_INFO }; + VulkanSubmitInfoHelper submit_helper(submit_info); + + // External waits should gate only the first segment, the split semaphore will order the rest. + if (i == 0) + { + for (auto& wait_semaphore : wait_semaphores) + { + submit_helper.AddWaitSemaphore(wait_semaphore); + } + } + + submit_info.commandBufferCount = static_cast(prev_command_buffers.size()); + submit_info.pCommandBuffers = prev_command_buffers.data(); + + VulkanInjectedSemaphore& split_semaphore = split_info->GetSplitSemaphore(); + submit_helper.InjectSemaphore(split_semaphore); + if (i == prev_submits.size() - 1) + { + signal_semaphore = split_semaphore.GetSemaphore(); + } + + prev_submits[i] = submit_info; + + VkResult result = device_table_->QueueSubmit(queue_info->handle, 1, &prev_submits[i], VK_NULL_HANDLE); + GFXRECON_ASSERT(result == VK_SUCCESS); + } + } + + return signal_semaphore; +} + +std::vector> VulkanCommandSplitter::GetCommandBuffersFromSubmitInfos( + const std::span submits_span) +{ + std::vector> submits_command_buffers(submits_span.size()); + + for (size_t span_index = 0; span_index < submits_span.size(); ++span_index) + { + VkSubmitInfo& submit_info = submits_span[span_index]; + + auto& command_buffers = submits_command_buffers[span_index]; + command_buffers = std::vector(submit_info.pCommandBuffers, + submit_info.pCommandBuffers + submit_info.commandBufferCount); + } + + return submits_command_buffers; +} + +std::vector> VulkanCommandSplitter::GetCommandBuffersFromSubmitInfos( + const std::span submits_span) +{ + std::vector> submits_command_buffers(submits_span.size()); + + for (size_t span_index = 0; span_index < submits_span.size(); ++span_index) + { + VkSubmitInfo2& submit_info = submits_span[span_index]; + + for (size_t cmdbuf_info_index = 0; cmdbuf_info_index < submit_info.commandBufferInfoCount; ++cmdbuf_info_index) + { + const VkCommandBufferSubmitInfo& info = submit_info.pCommandBufferInfos[cmdbuf_info_index]; + VkCommandBuffer command_buffer = info.commandBuffer; + submits_command_buffers[span_index].push_back(command_buffer); + } + } + + return submits_command_buffers; +} + +void VulkanCommandSplitter::FreeCommandBuffers(VkCommandPool command_pool, + const std::span command_buffers) +{ + // Check whether any of the command buffers being freed are split command buffers. + for (VkCommandBuffer command_buffer : command_buffers) + { + if (original_command_buffer_id_.contains(command_buffer)) + { + // This command buffer is a split command buffer, so we need to free all the splits together. + format::HandleId command_buffer_id = original_command_buffer_id_[command_buffer]; + auto* split_info = GetSplitInfo(command_buffer_id); + split_info->FreeCommandBuffers(command_pool); + + // Remove the associated handles from the tracking maps. + for (VkCommandBuffer handle : split_info->GetAssociatedHandles()) + { + original_command_buffer_id_.erase(handle); + } + + // Update the command buffer info to use the original handle for subsequent calls. + VulkanCommandBufferInfo* command_buffer_info = object_table_->GetVkCommandBufferInfo(command_buffer_id); + command_buffer_info->handle = split_info->ResetSplitHandles(); + + split_infos_.erase(command_buffer_id); + } + } +} + +void VulkanCommandSplitter::ResetCommandBuffer(VulkanCommandBufferInfo* command_buffer_info) +{ + // This command buffer is expected to be already reset at this point. + VkCommandBuffer command_buffer = command_buffer_info->handle; + + // Check whether this command buffer is result of a split. + // If so, we need to reset all the split command buffers together and restore the command buffer info handle. + if (original_command_buffer_id_.contains(command_buffer)) + { + format::HandleId command_buffer_id = original_command_buffer_id_[command_buffer]; + VulkanCommandBufferSplitInfo* split_info = GetSplitInfo(command_buffer_id); + + // Update the command buffer info to use the original handle for subsequent calls. + command_buffer_info->handle = split_info->ResetSplitHandles(); + } +} + +void VulkanCommandSplitter::BeginCommandBuffer(VulkanCommandBufferInfo* command_buffer_info) +{ + VulkanCommandPoolInfo* command_pool_info = object_table_->GetVkCommandPoolInfo(command_buffer_info->pool_id); + GFXRECON_ASSERT(command_pool_info != nullptr); + + if (command_pool_info->flags & VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT) + { + // If the command pool was created with the RESET_COMMAND_BUFFER_BIT flag, then the command buffer is implicitly + // reset when vkBeginCommandBuffer is called. In this case, we need to reset all the split command buffers + // together and restore the command buffer info handle. + if (original_command_buffer_id_.contains(command_buffer_info->handle)) + { + // Reset current handle and let `ResetCommandBuffer` handle the rest of the split handles. + device_table_->ResetCommandBuffer(command_buffer_info->handle, 0); + ResetCommandBuffer(command_buffer_info); + } + } +} + +GFXRECON_END_NAMESPACE(decode) +GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_command_splitter.h b/framework/decode/vulkan_command_splitter.h new file mode 100644 index 0000000000..f2d3820d05 --- /dev/null +++ b/framework/decode/vulkan_command_splitter.h @@ -0,0 +1,153 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ +#ifndef GFXRECON_DECODE_VULKAN_COMMAND_SPLITTER_H +#define GFXRECON_DECODE_VULKAN_COMMAND_SPLITTER_H + +#include "decode/common_object_info_table.h" +#include "decode/vulkan_object_info.h" +#include "decode/vulkan_submit_job.h" +#include "util/defines.h" + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(decode) + +/// This helper class manages command buffer handles addressing cases where a single command buffer is reset and +/// recorded multiple times. In such cases, after splitting the command buffer for the first time, a reset is expected +/// before the next recording, and we want to avoid splitting again and instead reuse the handles of the first split. +class VulkanCommandBufferSplitInfo +{ + private: + const VulkanDeviceInfo* device_info_ = nullptr; + const graphics::VulkanDeviceTable* device_table_ = nullptr; + CommonObjectInfoTable* object_table_ = nullptr; + + VulkanInjectedSemaphore split_semaphore_; + + VkCommandBuffer original_handle_ = VK_NULL_HANDLE; + + /// Associated command buffer handles, namely command buffers created at split-time that are associated with the + /// original command buffer. This does not include the original handle itself. + std::vector associated_handles_; + + /// The index of the next associated handle to use. This is used to retrieve the next handle at each split point. + /// When a command buffer is split for the first time, a new command buffer is created for the split and stored in + /// the associated handles vector. In case of a subsequent split, the index resets to 0 to reuse any associated + /// handles that were created at the first split, before creating new ones if needed. + size_t next_associated_index_ = 0; + + /// The command buffer handles of the current split. + /// First handle in the vector is the original command buffer. The rest are other handles in + /// order for each split. The very last handle, which is the most recent split, is not stored in this vector, + /// but is instead stored in the command buffer info's handle field. + std::vector split_handles_; + + public: + VulkanCommandBufferSplitInfo(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table, + format::HandleId command_buffer_id); + + VulkanCommandBufferSplitInfo(const VulkanCommandBufferSplitInfo&) = delete; + VulkanCommandBufferSplitInfo& operator=(const VulkanCommandBufferSplitInfo&) = delete; + VulkanCommandBufferSplitInfo(VulkanCommandBufferSplitInfo&&) = default; + VulkanCommandBufferSplitInfo& operator=(VulkanCommandBufferSplitInfo&&) = default; + ~VulkanCommandBufferSplitInfo() = default; + + [[nodiscard]] VkCommandBuffer GetNextHandle(const VulkanCommandBufferInfo* command_buffer_info); + + [[nodiscard]] const std::vector& GetAssociatedHandles() const { return associated_handles_; } + [[nodiscard]] const std::vector& GetSplitHandles() const { return split_handles_; } + + [[nodiscard]] VkCommandBuffer ResetSplitHandles(); + + [[nodiscard]] VulkanInjectedSemaphore& GetSplitSemaphore() { return split_semaphore_; } + + void FreeCommandBuffers(VkCommandPool pool); +}; + +class VulkanCommandSplitter +{ + public: + VulkanCommandSplitter(const VulkanDeviceInfo* device_info, + const graphics::VulkanDeviceTable* device_table, + CommonObjectInfoTable* object_table); + + ~VulkanCommandSplitter() = default; + + VulkanCommandSplitter(const VulkanCommandSplitter&) = delete; + VulkanCommandSplitter& operator=(const VulkanCommandSplitter&) = delete; + VulkanCommandSplitter(VulkanCommandSplitter&&) = default; + VulkanCommandSplitter& operator=(VulkanCommandSplitter&&) = default; + + void SplitCommandBuffer(VulkanCommandBufferInfo* command_buffer_info); + + graphics::VulkanSemaphore SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores = {}); + + graphics::VulkanSemaphore SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span current_submits_span, + const std::span wait_semaphores = {}); + + void FreeCommandBuffers(VkCommandPool command_pool, const std::span command_buffers); + + /// @brief To be called after resetting the current command buffer. + /// @param command_buffer_info The command buffer info structure to reset. + void ResetCommandBuffer(VulkanCommandBufferInfo* command_buffer_info); + + void BeginCommandBuffer(VulkanCommandBufferInfo* command_buffer_info); + + private: + graphics::VulkanSemaphore SubmitPreviouslySplitCommandBuffers( + const VulkanQueueInfo* queue_info, + const std::span> current_submits_cmdbufs, + const std::span wait_semaphores = {}); + + std::vector> GetCommandBuffersFromSubmitInfos( + const std::span submits_span); + std::vector> GetCommandBuffersFromSubmitInfos( + const std::span submits_span); + + VulkanCommandBufferSplitInfo& GetOrCreateSplitInfo(format::HandleId command_buffer_id); + VulkanCommandBufferSplitInfo* GetSplitInfo(format::HandleId command_buffer_id); + + const VulkanDeviceInfo* device_info_ = nullptr; + const graphics::VulkanDeviceTable* device_table_ = nullptr; + CommonObjectInfoTable* object_table_ = nullptr; + + /// Map from the command buffer ID to the structure representing the split. + std::unordered_map split_infos_; + + /// Map from the command buffer handles to the original ID. + std::unordered_map original_command_buffer_id_; +}; + +using VulkanPerDeviceCommandSplitters = std::unordered_map; + +GFXRECON_END_NAMESPACE(decode) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // GFXRECON_DECODE_VULKAN_COMMAND_SPLITTER_H \ No newline at end of file diff --git a/framework/decode/vulkan_replay_consumer_base.cpp b/framework/decode/vulkan_replay_consumer_base.cpp index d60b24359e..667dde9f91 100644 --- a/framework/decode/vulkan_replay_consumer_base.cpp +++ b/framework/decode/vulkan_replay_consumer_base.cpp @@ -318,6 +318,7 @@ VulkanReplayConsumerBase::~VulkanReplayConsumerBase() // free replacer internal vulkan-resources device_address_replacers_.clear(); + device_command_splitters_.clear(); // free frame warm up resources device_frame_warmups_.clear(); @@ -1475,7 +1476,7 @@ void VulkanReplayConsumerBase::RaiseFatalError(const char* message) const void VulkanReplayConsumerBase::InitializeLoader() { - loader_handle_ = graphics::InitializeLoader(); + loader_handle_ = graphics::InitializeLoader(getenv("GFXRECON_VULKAN_LIBRARY_PATH")); // Only get get_instance_proc_addr_ from the loader if it wasn't already set via SetupForRecapture() if ((loader_handle_ != nullptr) && (get_instance_proc_addr_ == nullptr)) @@ -3766,6 +3767,8 @@ void VulkanReplayConsumerBase::OverrideDestroyDevice( // free replacer internal vulkan-resources for the device device_address_replacers_.erase(device_info); + device_command_splitters_.erase(device_info); + // free potential swapchain-resources for the device GFXRECON_ASSERT(swapchain_) swapchain_->CleanDeviceResources(device_info->handle, device_table); @@ -4377,6 +4380,17 @@ VkResult VulkanReplayConsumerBase::OverrideQueueSubmit(PFN_vkQueueSubmit VulkanSubmitJobPlan plan; + if (options_.isolate_render_passes) + { + auto& command_splitter = GetDeviceCommandSplitter(device_info); + plan.Push(0, + [&command_splitter, ¤t_submits_span, queue_info]( + const std::span wait_semaphores) { + return command_splitter.SubmitPreviouslySplitCommandBuffers( + queue_info, current_submits_span, wait_semaphores); + }); + } + if (options_.frame_warm_up_load != 0 && !fps_info_->IsFirstSubmitDone()) { auto& frame_warm_up = GetDeviceFrameWarmUp(device_info); @@ -4656,6 +4670,17 @@ VkResult VulkanReplayConsumerBase::OverrideQueueSubmit2(PFN_vkQueueSubmit2 VulkanSubmitJobPlan plan; + if (options_.isolate_render_passes) + { + auto& command_splitter = GetDeviceCommandSplitter(device_info); + plan.Push(0, + [&command_splitter, ¤t_submits_span, queue_info]( + const std::span wait_semaphores) { + return command_splitter.SubmitPreviouslySplitCommandBuffers( + queue_info, current_submits_span, wait_semaphores); + }); + } + if (options_.frame_warm_up_load != 0 && !fps_info_->IsFirstSubmitDone()) { auto& frame_warm_up = GetDeviceFrameWarmUp(device_info); @@ -5703,6 +5728,13 @@ void VulkanReplayConsumerBase::OverrideFreeCommandBuffers(PFN_vkFreeCommandBuffe } } } + + if (options_.isolate_render_passes) + { + auto command_buffers = std::span(pCommandBuffers->GetHandlePointer(), command_buffer_count); + GetDeviceCommandSplitter(device_info).FreeCommandBuffers(command_pool_info->handle, command_buffers); + } + const VkCommandBuffer* in_pCommandBuffers = pCommandBuffers->GetHandlePointer(); func(device_info->handle, command_pool_info->handle, command_buffer_count, in_pCommandBuffers); } @@ -10410,6 +10442,13 @@ VkResult VulkanReplayConsumerBase::OverrideBeginCommandBuffer( VulkanCommandBufferInfo* command_buffer_info, StructPointerDecoder* begin_info_decoder) { + if (options_.isolate_render_passes) + { + auto* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GFXRECON_ASSERT(device_info != nullptr); + GetDeviceCommandSplitter(device_info).BeginCommandBuffer(command_buffer_info); + } + ClearCommandBufferInfo(command_buffer_info); VkCommandBuffer command_buffer = command_buffer_info->handle; @@ -10458,7 +10497,36 @@ VkResult VulkanReplayConsumerBase::OverrideResetCommandBuffer(PFN_vkResetCommand resource_dumper_->ResetCommandBuffer((command_buffer)); } - return func(command_buffer, flags); + VkResult result = func(command_buffer, flags); + + if (options_.isolate_render_passes) + { + auto* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).ResetCommandBuffer(command_buffer_info); + } + + return result; +} + +VkResult VulkanReplayConsumerBase::OverrideCreateCommandPool( + PFN_vkCreateCommandPool func, + VkResult original_result, + const VulkanDeviceInfo* device_info, + const StructPointerDecoder* pCreateInfo, + const StructPointerDecoder* pAllocator, + HandlePointerDecoder* pCommandPool) +{ + const VkCommandPoolCreateInfo* create_info = pCreateInfo->GetPointer(); + if (create_info != nullptr) + { + auto* command_pool_info = reinterpret_cast(pCommandPool->GetConsumerData(0)); + command_pool_info->flags = create_info->flags; + } + + return func(device_info->handle, + pCreateInfo->GetPointer(), + GetAllocationCallbacks(pAllocator), + pCommandPool->GetHandlePointer()); } VkResult VulkanReplayConsumerBase::OverrideResetCommandPool(PFN_vkResetCommandPool func, @@ -10709,6 +10777,12 @@ void VulkanReplayConsumerBase::OverrideCmdBeginRenderPass( StructPointerDecoder* render_pass_begin_info_decoder, VkSubpassContents contents) { + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } + MaybeInjectExecutionBarrier(command_buffer_info); const auto render_pass_info_meta = render_pass_begin_info_decoder->GetMetaStructPointer(); @@ -10739,6 +10813,12 @@ void VulkanReplayConsumerBase::OverrideCmdBeginRenderPass2( StructPointerDecoder* render_pass_begin_info_decoder, StructPointerDecoder* subpass_begin_info_decode) { + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } + MaybeInjectExecutionBarrier(command_buffer_info); const auto render_pass_info_meta = render_pass_begin_info_decoder->GetMetaStructPointer(); @@ -10810,6 +10890,12 @@ void VulkanReplayConsumerBase::OverrideCmdEndRenderPass(PFN_vkCmdEndRenderPass command_buffer_info->active_framebuffer_id = format::kNullHandleId; command_buffer_info->active_render_pass_attachment_image_view_ids.clear(); command_buffer_info->in_rendering_scope = false; + + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } } void VulkanReplayConsumerBase::OverrideCmdEndRenderPass2( @@ -10824,6 +10910,12 @@ void VulkanReplayConsumerBase::OverrideCmdEndRenderPass2( command_buffer_info->active_framebuffer_id = format::kNullHandleId; command_buffer_info->active_render_pass_attachment_image_view_ids.clear(); command_buffer_info->in_rendering_scope = false; + + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } } void VulkanReplayConsumerBase::OverrideCmdBeginRendering( @@ -10833,6 +10925,12 @@ void VulkanReplayConsumerBase::OverrideCmdBeginRendering( { GFXRECON_ASSERT(command_buffer_info != nullptr); + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } + MaybeInjectExecutionBarrier(command_buffer_info); command_buffer_info->in_rendering_scope = true; @@ -10885,6 +10983,12 @@ void VulkanReplayConsumerBase::OverrideCmdEndRendering(PFN_vkCmdEndRendering GFXRECON_ASSERT(command_buffer_info != nullptr); command_buffer_info->in_rendering_scope = false; func(command_buffer_info->handle); + + if (options_.isolate_render_passes) + { + VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GetDeviceCommandSplitter(device_info).SplitCommandBuffer(command_buffer_info); + } } void VulkanReplayConsumerBase::OverrideCmdTraceRaysKHR( @@ -11919,6 +12023,19 @@ VulkanFrameWarmUp& VulkanReplayConsumerBase::GetDeviceFrameWarmUp(const VulkanDe return new_it->second; } +VulkanCommandSplitter& VulkanReplayConsumerBase::GetDeviceCommandSplitter(const VulkanDeviceInfo* device_info) +{ + if (auto it = device_command_splitters_.find(device_info); it != device_command_splitters_.end()) + { + return it->second; + } + + auto [new_it, success] = device_command_splitters_.insert( + { device_info, VulkanCommandSplitter(device_info, GetDeviceTable(device_info->handle), object_info_table_) }); + GFXRECON_ASSERT(success); + return new_it->second; +} + VulkanSubmitJobExecutor& VulkanReplayConsumerBase::GetDeviceSubmitJobExecutor(const VulkanDeviceInfo* device_info) { if (auto it = device_submit_job_executors_.find(device_info); it != device_submit_job_executors_.end()) @@ -13996,5 +14113,76 @@ void VulkanReplayConsumerBase::OverrideCmdExecuteGeneratedCommandsEXT( func(command_buffer_info->handle, isPreprocessed, in_pGeneratedCommandsInfo); } +void VulkanReplayConsumerBase::MaybeInjectComputeTransferBarrier( + const VulkanCommandBufferInfo* command_buffer_info) const +{ + if (!options_.serialize_compute_and_transfer) + { + return; + } + + util::BeginInjectedCommands(); + + GFXRECON_ASSERT(command_buffer_info != nullptr); + const VulkanDeviceInfo* device_info = GetObjectInfoTable().GetVkDeviceInfo(command_buffer_info->parent_id); + GFXRECON_ASSERT(device_info != nullptr); + const graphics::VulkanDeviceTable* device_table = GetDeviceTable(device_info->handle); + GFXRECON_ASSERT(device_table != nullptr); + + VkMemoryBarrier memory_barrier = { VK_STRUCTURE_TYPE_MEMORY_BARRIER }; + memory_barrier.srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT; + memory_barrier.dstAccessMask = + VK_ACCESS_MEMORY_READ_BIT | VK_ACCESS_MEMORY_WRITE_BIT | VK_ACCESS_INDIRECT_COMMAND_READ_BIT; + + const VkPipelineStageFlags stages = + VK_PIPELINE_STAGE_TRANSFER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT | VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT; + + // Make sure compute and transfer write operations before this barrier are completed + // before doing other compute and transfer read/write operations. + device_table->CmdPipelineBarrier( + command_buffer_info->handle, stages, stages, 0, 1, &memory_barrier, 0, nullptr, 0, nullptr); + + util::EndInjectedCommands(); +} + +void VulkanReplayConsumerBase::OverrideCmdDispatch(PFN_vkCmdDispatch func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ) +{ + GFXRECON_ASSERT(command_buffer_info != nullptr); + MaybeInjectComputeTransferBarrier(command_buffer_info); + func(command_buffer_info->handle, groupCountX, groupCountY, groupCountZ); + MaybeInjectComputeTransferBarrier(command_buffer_info); +} + +void VulkanReplayConsumerBase::OverrideCmdDispatchIndirect(PFN_vkCmdDispatchIndirect func, + const VulkanCommandBufferInfo* command_buffer_info, + const VulkanBufferInfo* buffer_info, + VkDeviceSize offset) +{ + GFXRECON_ASSERT(command_buffer_info != nullptr); + GFXRECON_ASSERT(buffer_info != nullptr); + MaybeInjectComputeTransferBarrier(command_buffer_info); + func(command_buffer_info->handle, buffer_info->handle, offset); + MaybeInjectComputeTransferBarrier(command_buffer_info); +} + +void VulkanReplayConsumerBase::OverrideCmdDispatchBase(PFN_vkCmdDispatchBase func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t baseGroupX, + uint32_t baseGroupY, + uint32_t baseGroupZ, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ) +{ + GFXRECON_ASSERT(command_buffer_info != nullptr); + MaybeInjectComputeTransferBarrier(command_buffer_info); + func(command_buffer_info->handle, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + MaybeInjectComputeTransferBarrier(command_buffer_info); +} + GFXRECON_END_NAMESPACE(decode) GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/framework/decode/vulkan_replay_consumer_base.h b/framework/decode/vulkan_replay_consumer_base.h index 70b3c79823..efe58d87aa 100644 --- a/framework/decode/vulkan_replay_consumer_base.h +++ b/framework/decode/vulkan_replay_consumer_base.h @@ -36,6 +36,7 @@ #include "decode/vulkan_object_info.h" #include "decode/common_object_info_table.h" #include "decode/vulkan_replay_options.h" +#include "decode/vulkan_command_splitter.h" #include "decode/vulkan_resource_allocator.h" #include "decode/vulkan_submit_job.h" #include "decode/vulkan_swapchain.h" @@ -1359,6 +1360,13 @@ class VulkanReplayConsumerBase : public VulkanConsumer VulkanCommandBufferInfo* command_buffer_info, VkCommandBufferResetFlags flags); + VkResult OverrideCreateCommandPool(PFN_vkCreateCommandPool func, + VkResult original_result, + const VulkanDeviceInfo* device_info, + const StructPointerDecoder* pCreateInfo, + const StructPointerDecoder* pAllocator, + HandlePointerDecoder* pCommandPool); + VkResult OverrideResetCommandPool(PFN_vkResetCommandPool func, VkResult original_result, const VulkanDeviceInfo* device_info, @@ -1674,6 +1682,26 @@ class VulkanReplayConsumerBase : public VulkanConsumer VkBool32 isPreprocessed, StructPointerDecoder* pGeneratedCommandsInfo); + void OverrideCmdDispatch(PFN_vkCmdDispatch func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ); + + void OverrideCmdDispatchIndirect(PFN_vkCmdDispatchIndirect func, + const VulkanCommandBufferInfo* command_buffer_info, + const VulkanBufferInfo* buffer_info, + VkDeviceSize offset); + + void OverrideCmdDispatchBase(PFN_vkCmdDispatchBase func, + const VulkanCommandBufferInfo* command_buffer_info, + uint32_t baseGroupX, + uint32_t baseGroupY, + uint32_t baseGroupZ, + uint32_t groupCountX, + uint32_t groupCountY, + uint32_t groupCountZ); + std::function()> AsyncCreateGraphicsPipelines(PFN_vkCreateGraphicsPipelines func, VkResult returnValue, @@ -1861,6 +1889,7 @@ class VulkanReplayConsumerBase : public VulkanConsumer decode::VulkanDeviceAddressTracker& GetDeviceAddressTracker(const decode::VulkanDeviceInfo* device_info); decode::VulkanAddressReplacer& GetDeviceAddressReplacer(const decode::VulkanDeviceInfo* device_info); VulkanFrameWarmUp& GetDeviceFrameWarmUp(const VulkanDeviceInfo* device_info); + VulkanCommandSplitter& GetDeviceCommandSplitter(const VulkanDeviceInfo* device_info); VulkanSubmitJobExecutor& GetDeviceSubmitJobExecutor(const VulkanDeviceInfo* device_info); /** @@ -1922,6 +1951,12 @@ class VulkanReplayConsumerBase : public VulkanConsumer */ void MaybeInjectExecutionBarrier(const VulkanCommandBufferInfo* command_buffer_info) const; + /** + * @brief If the option to serialize compute and transfer operations is enabled, inject a memory barrier + * before and after each compute dispatch to ensure compute and transfer operations do not overlap. + */ + void MaybeInjectComputeTransferBarrier(const VulkanCommandBufferInfo* command_buffer_info) const; + private: struct HardwareBufferInfo { @@ -1998,6 +2033,7 @@ class VulkanReplayConsumerBase : public VulkanConsumer VulkanPerDeviceAddressTrackers device_address_trackers_; VulkanPerDeviceAddressReplacers device_address_replacers_; VulkanPerDeviceFrameWarmUp device_frame_warmups_; + VulkanPerDeviceCommandSplitters device_command_splitters_; VulkanPerDeviceSubmitJobExecutors device_submit_job_executors_; util::ThreadPool main_thread_queue_; diff --git a/framework/decode/vulkan_replay_options.h b/framework/decode/vulkan_replay_options.h index 772544e325..64e8d89bbc 100644 --- a/framework/decode/vulkan_replay_options.h +++ b/framework/decode/vulkan_replay_options.h @@ -264,6 +264,13 @@ struct VulkanReplayOptions : public ReplayOptions /// Parameters to pass to the replay event plugin. std::string replay_event_plugin_params; + /// Isolate render passes by splitting the command buffer into multiple submits. + bool isolate_render_passes{ false }; + + /// Prevent compute dispatches from overlapping adjacent transfer work by injecting + /// a barrier before and after each dispatch. + bool serialize_compute_and_transfer{ false }; + void MaybeWaitBeforeFirstSubmit() const; void MaybeWaitBeforeFrame() const; }; diff --git a/framework/decode/vulkan_submit_info_helper.h b/framework/decode/vulkan_submit_info_helper.h index 685862d1bc..d5f9ad9a0c 100644 --- a/framework/decode/vulkan_submit_info_helper.h +++ b/framework/decode/vulkan_submit_info_helper.h @@ -47,10 +47,11 @@ class VulkanInjectedSemaphore const graphics::VulkanDeviceTable* device_table_; public: - bool HasReachedTargetValue() const; - VkSemaphore GetHandle() const { return semaphore_.semaphore; } - uint64_t GetTargetValue() const { return semaphore_.timeline_value; } - void IncreaseTargetValue() { semaphore_.timeline_value++; } + bool HasReachedTargetValue() const; + graphics::VulkanSemaphore GetSemaphore() const { return semaphore_; } + VkSemaphore GetHandle() const { return semaphore_.semaphore; } + uint64_t GetTargetValue() const { return semaphore_.timeline_value; } + void IncreaseTargetValue() { semaphore_.timeline_value++; } VulkanInjectedSemaphore(const VulkanDeviceInfo* device_info, const graphics::VulkanDeviceTable* table); ~VulkanInjectedSemaphore(); diff --git a/framework/generated/generated_vulkan_replay_consumer.cpp b/framework/generated/generated_vulkan_replay_consumer.cpp index 18f1b2cdd7..34d8edacc6 100644 --- a/framework/generated/generated_vulkan_replay_consumer.cpp +++ b/framework/generated/generated_vulkan_replay_consumer.cpp @@ -792,19 +792,19 @@ void VulkanReplayConsumer::Process_vkCreateCommandPool( StructPointerDecoder* pAllocator, HandlePointerDecoder* pCommandPool) { - VkDevice in_device = MapHandle(device, &CommonObjectInfoTable::GetVkDeviceInfo); - const VkCommandPoolCreateInfo* in_pCreateInfo = pCreateInfo->GetPointer(); + auto in_device = GetObjectInfoTable().GetVkDeviceInfo(device); + MapStructHandles(pCreateInfo->GetMetaStructPointer(), GetObjectInfoTable()); - const VkAllocationCallbacks* in_pAllocator = GetAllocationCallbacks(pAllocator); if (!pCommandPool->IsNull()) { pCommandPool->SetHandleLength(1); } - VkCommandPool* out_pCommandPool = pCommandPool->GetHandlePointer(); + VulkanCommandPoolInfo handle_info; + pCommandPool->SetConsumerData(0, &handle_info); PushRecaptureHandleId(pCommandPool->GetPointer()); - VkResult replay_result = GetDeviceTable(in_device)->CreateCommandPool(in_device, in_pCreateInfo, in_pAllocator, out_pCommandPool); + VkResult replay_result = OverrideCreateCommandPool(GetDeviceTable(in_device->handle)->CreateCommandPool, returnValue, in_device, pCreateInfo, pAllocator, pCommandPool); CheckResult("vkCreateCommandPool", returnValue, replay_result, call_info); ClearRecaptureHandleIds(); - AddHandle(device, pCommandPool->GetPointer(), out_pCommandPool, &CommonObjectInfoTable::AddVkCommandPoolInfo); + AddHandle(device, pCommandPool->GetPointer(), pCommandPool->GetHandlePointer(), std::move(handle_info), &CommonObjectInfoTable::AddVkCommandPoolInfo); } void VulkanReplayConsumer::Process_vkDestroyCommandPool( @@ -1755,13 +1755,13 @@ void VulkanReplayConsumer::Process_vkCmdDispatch( uint32_t groupCountY, uint32_t groupCountZ) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); - GetDeviceTable(in_commandBuffer)->CmdDispatch(in_commandBuffer, groupCountX, groupCountY, groupCountZ); + OverrideCmdDispatch(GetDeviceTable(in_commandBuffer->handle)->CmdDispatch, in_commandBuffer, groupCountX, groupCountY, groupCountZ); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatch(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatch, in_commandBuffer, groupCountX, groupCountY, groupCountZ); + resource_dumper_->Process_vkCmdDispatch(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatch, in_commandBuffer->handle, groupCountX, groupCountY, groupCountZ); } } @@ -1771,14 +1771,14 @@ void VulkanReplayConsumer::Process_vkCmdDispatchIndirect( format::HandleId buffer, VkDeviceSize offset) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); - VkBuffer in_buffer = MapHandle(buffer, &CommonObjectInfoTable::GetVkBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); + auto in_buffer = GetObjectInfoTable().GetVkBufferInfo(buffer); - GetDeviceTable(in_commandBuffer)->CmdDispatchIndirect(in_commandBuffer, in_buffer, offset); + OverrideCmdDispatchIndirect(GetDeviceTable(in_commandBuffer->handle)->CmdDispatchIndirect, in_commandBuffer, in_buffer, offset); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatchIndirect(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatchIndirect, in_commandBuffer, GetObjectInfoTable().GetVkBufferInfo(buffer), offset); + resource_dumper_->Process_vkCmdDispatchIndirect(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatchIndirect, in_commandBuffer->handle, in_buffer, offset); } } @@ -2705,13 +2705,13 @@ void VulkanReplayConsumer::Process_vkCmdDispatchBase( uint32_t groupCountY, uint32_t groupCountZ) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); - GetDeviceTable(in_commandBuffer)->CmdDispatchBase(in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + OverrideCmdDispatchBase(GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBase, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatchBase(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatchBase, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + resource_dumper_->Process_vkCmdDispatchBase(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBase, in_commandBuffer->handle, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); } } @@ -4963,13 +4963,13 @@ void VulkanReplayConsumer::Process_vkCmdDispatchBaseKHR( uint32_t groupCountY, uint32_t groupCountZ) { - VkCommandBuffer in_commandBuffer = MapHandle(commandBuffer, &CommonObjectInfoTable::GetVkCommandBufferInfo); + auto in_commandBuffer = GetObjectInfoTable().GetVkCommandBufferInfo(commandBuffer); - GetDeviceTable(in_commandBuffer)->CmdDispatchBaseKHR(in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + OverrideCmdDispatchBase(GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBaseKHR, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); if (options_.dumping_resources) { - resource_dumper_->Process_vkCmdDispatchBaseKHR(call_info, GetDeviceTable(in_commandBuffer)->CmdDispatchBaseKHR, in_commandBuffer, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); + resource_dumper_->Process_vkCmdDispatchBaseKHR(call_info, GetDeviceTable(in_commandBuffer->handle)->CmdDispatchBaseKHR, in_commandBuffer->handle, baseGroupX, baseGroupY, baseGroupZ, groupCountX, groupCountY, groupCountZ); } } diff --git a/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json b/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json index 74061e4ed4..b334c7138e 100644 --- a/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json +++ b/framework/generated/khronos_generators/vulkan_generators/replay_overrides.json @@ -108,6 +108,7 @@ "vkGetPhysicalDeviceToolProperties": "OverrideGetPhysicalDeviceToolProperties", "vkGetPhysicalDeviceToolPropertiesEXT": "OverrideGetPhysicalDeviceToolProperties", "vkBeginCommandBuffer": "OverrideBeginCommandBuffer", + "vkCreateCommandPool": "OverrideCreateCommandPool", "vkResetCommandPool": "OverrideResetCommandPool", "vkResetCommandBuffer": "OverrideResetCommandBuffer", "vkCmdWaitEvents": "OverrideCmdWaitEvents", @@ -186,6 +187,10 @@ "vkCmdEndRenderingKHR": "OverrideCmdEndRendering", "vkCreateIndirectExecutionSetEXT": "OverrideCreateIndirectExecutionSetEXT", "vkCmdPreprocessGeneratedCommandsEXT": "OverrideCmdPreprocessGeneratedCommandsEXT", - "vkCmdExecuteGeneratedCommandsEXT": "OverrideCmdExecuteGeneratedCommandsEXT" + "vkCmdExecuteGeneratedCommandsEXT": "OverrideCmdExecuteGeneratedCommandsEXT", + "vkCmdDispatch": "OverrideCmdDispatch", + "vkCmdDispatchIndirect": "OverrideCmdDispatchIndirect", + "vkCmdDispatchBase": "OverrideCmdDispatchBase", + "vkCmdDispatchBaseKHR": "OverrideCmdDispatchBase" } } \ No newline at end of file diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 3703689ace..0a4e8441c7 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -45,6 +45,8 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) set(GFXRECON_TESTCASES test_cases/ahb.cpp test_cases/debug-utils.cpp + test_cases/isolate-render-passes.cpp + test_cases/serialize-compute-and-transfer.cpp test_cases/triangle.cpp test_cases/multisample-depth.cpp test_cases/shader-objects.cpp @@ -62,7 +64,7 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) add_executable(gfxrecon-testapp-runner ${GFXRECON_TESTCASES}) - add_dependencies(gfxrecon-testapp-runner gfxrecon-test-launcher VkICD_mock_icd gfxrecon-convert VkLayer_gfxreconstruct) + add_dependencies(gfxrecon-testapp-runner gfxrecon-test-launcher VkICD_mock_icd gfxrecon-convert gfxrecon-replay VkLayer_gfxreconstruct) target_include_directories(gfxrecon-testapp-runner PRIVATE ${CMAKE_CURRENT_LIST_DIR}) target_link_libraries(gfxrecon-testapp-runner gfxrecon_util @@ -72,6 +74,7 @@ if (NOT DEFINED ENV{GFXRECON_NO_TEST_APPS}) install(TARGETS gfxrecon-testapp-runner RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}) install(DIRECTORY known_good DESTINATION ${GFXRECON_INSTALL_TESTDIR}) install(TARGETS gfxrecon-convert RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}) + install(TARGETS gfxrecon-replay RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTDIR}) if (APPLE) install(PROGRAMS run-tests_macos.sh DESTINATION ${GFXRECON_INSTALL_TESTDIR}) install(FILES $/staging-json/VkLayer_gfxreconstruct.json DESTINATION ${GFXRECON_INSTALL_LAYER_INSTALL_DIR}) diff --git a/test/run-tests_macos.sh b/test/run-tests_macos.sh index d7204bda18..1447f71d1a 100644 --- a/test/run-tests_macos.sh +++ b/test/run-tests_macos.sh @@ -11,6 +11,7 @@ export VK_LOADER_LAYERS_ENABLE=VK_LAYER_LUNARG_gfxreconstruct export DISPLAY=:0 export GFXRECON_TESTAPP_HEADLESS=true export GFXRECON_TESTAPP_MOCK_ICD="$PWD/test_apps/libVkICD_mock_icd.dylib" +export GFXRECON_VULKAN_LIBRARY_PATH="$VULKAN_LIBRARY_PATH" #export GFXRECON_LOG_LEVEL="info" #export VK_LOADER_DEBUG="layer" diff --git a/test/test_apps/CMakeLists.txt b/test/test_apps/CMakeLists.txt index 7da5924465..b883ccd91b 100644 --- a/test/test_apps/CMakeLists.txt +++ b/test/test_apps/CMakeLists.txt @@ -33,6 +33,8 @@ endif() add_subdirectory(common) add_subdirectory(debug-utils) +add_subdirectory(isolate-render-passes) +add_subdirectory(serialize-compute-and-transfer) add_subdirectory(triangle) add_subdirectory(triangle-extra-device) add_subdirectory(multisample-depth) diff --git a/test/test_apps/isolate-render-passes/CMakeLists.txt b/test/test_apps/isolate-render-passes/CMakeLists.txt new file mode 100644 index 0000000000..40b224576e --- /dev/null +++ b/test/test_apps/isolate-render-passes/CMakeLists.txt @@ -0,0 +1,39 @@ +############################################################################### +# Copyright (c) 2026 LunarG, Inc. +# All rights reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# Author: LunarG Team +# Description: CMake script for isolate-render-passes test app +############################################################################### + +add_library(gfxrecon-testapp-isolate-render-passes app.cpp) + +target_include_directories(gfxrecon-testapp-isolate-render-passes + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(gfxrecon-testapp-isolate-render-passes gfxrecon-testapp-base) + +common_build_directives(gfxrecon-testapp-isolate-render-passes) + +if(NOT CMAKE_SYSTEM_NAME MATCHES "Android") + install(TARGETS gfxrecon-testapp-isolate-render-passes RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTAPPDIR}/isolate-render-passes) +endif() diff --git a/test/test_apps/isolate-render-passes/app.cpp b/test/test_apps/isolate-render-passes/app.cpp new file mode 100644 index 0000000000..e205680394 --- /dev/null +++ b/test/test_apps/isolate-render-passes/app.cpp @@ -0,0 +1,366 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(isolate_render_passes) + +void App::configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig* test_config) +{ + if (test_config) + { + test_config->device_api_version_override = VK_MAKE_API_VERSION(0, 1, 3, 296); + } + + TestAppBase::configure_instance_builder(instance_builder, test_config); +} + +void App::create_render_pass() +{ + // A single colour attachment that is cleared on load. The image is transitioned to + // VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL before the first render pass, so the same render pass object can be + // reused for the two passes recorded each frame. + VkAttachmentDescription color_attachment = {}; + color_attachment.format = init.swapchain.image_format; + color_attachment.samples = VK_SAMPLE_COUNT_1_BIT; + color_attachment.loadOp = VK_ATTACHMENT_LOAD_OP_CLEAR; + color_attachment.storeOp = VK_ATTACHMENT_STORE_OP_STORE; + color_attachment.stencilLoadOp = VK_ATTACHMENT_LOAD_OP_DONT_CARE; + color_attachment.stencilStoreOp = VK_ATTACHMENT_STORE_OP_DONT_CARE; + color_attachment.initialLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + color_attachment.finalLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkAttachmentReference color_attachment_ref = {}; + color_attachment_ref.attachment = 0; + color_attachment_ref.layout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + + VkSubpassDescription subpass = {}; + subpass.pipelineBindPoint = VK_PIPELINE_BIND_POINT_GRAPHICS; + subpass.colorAttachmentCount = 1; + subpass.pColorAttachments = &color_attachment_ref; + + VkRenderPassCreateInfo render_pass_info = {}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_CREATE_INFO; + render_pass_info.attachmentCount = 1; + render_pass_info.pAttachments = &color_attachment; + render_pass_info.subpassCount = 1; + render_pass_info.pSubpasses = &subpass; + render_pass_info.dependencyCount = 0; + + auto result = init.disp.createRenderPass(&render_pass_info, nullptr, &render_pass_); + VERIFY_VK_RESULT("failed to create render pass", result); +} + +void App::create_framebuffers() +{ + framebuffers_.resize(init.swapchain_image_views.size()); + + for (size_t i = 0; i < init.swapchain_image_views.size(); i++) + { + VkImageView attachments[] = { init.swapchain_image_views[i] }; + + VkFramebufferCreateInfo framebuffer_info = {}; + framebuffer_info.sType = VK_STRUCTURE_TYPE_FRAMEBUFFER_CREATE_INFO; + framebuffer_info.renderPass = render_pass_; + framebuffer_info.attachmentCount = 1; + framebuffer_info.pAttachments = attachments; + framebuffer_info.width = init.swapchain.extent.width; + framebuffer_info.height = init.swapchain.extent.height; + framebuffer_info.layers = 1; + + auto result = init.disp.createFramebuffer(&framebuffer_info, nullptr, &framebuffers_[i]); + VERIFY_VK_RESULT("failed to create framebuffer", result); + } +} + +void App::recreate_swapchain() +{ + init.disp.deviceWaitIdle(); + + for (auto framebuffer : framebuffers_) + { + init.disp.destroyFramebuffer(framebuffer, nullptr); + } + + TestAppBase::recreate_swapchain(false); + + create_framebuffers(); +} + +const int NUM_FRAMES = 10; +#define IS_RUNNING(frame_num) frame_num < NUM_FRAMES; + +/// Records a single render pass that clears the swapchain image to the provided colour. +static void record_clear_render_pass(vkb::DispatchTable const& disp, + VkCommandBuffer command_buffer, + VkRenderPass render_pass, + VkFramebuffer framebuffer, + VkExtent2D extent, + VkClearValue clear_color) +{ + VkRenderPassBeginInfo render_pass_info = {}; + render_pass_info.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO; + render_pass_info.renderPass = render_pass; + render_pass_info.framebuffer = framebuffer; + render_pass_info.renderArea.offset = { 0, 0 }; + render_pass_info.renderArea.extent = extent; + render_pass_info.clearValueCount = 1; + render_pass_info.pClearValues = &clear_color; + + disp.cmdBeginRenderPass(command_buffer, &render_pass_info, VK_SUBPASS_CONTENTS_INLINE); + disp.cmdEndRenderPass(command_buffer); +} + +bool App::frame(const int frame_num) +{ + init.disp.waitForFences(1, &sync_.in_flight_fences[current_frame_], VK_TRUE, UINT64_MAX); + + uint32_t image_index = 0; + VkResult result = init.disp.acquireNextImageKHR( + init.swapchain, UINT64_MAX, sync_.available_semaphores[current_frame_], VK_NULL_HANDLE, &image_index); + + if (result == VK_ERROR_OUT_OF_DATE_KHR) + { + recreate_swapchain(); + return IS_RUNNING(frame_num); + } + else if (result != VK_SUCCESS && result != VK_SUBOPTIMAL_KHR) + { + throw test::vulkan_exception("failed to acquire next image", result); + } + + if (sync_.image_in_flight[image_index] != VK_NULL_HANDLE) + { + init.disp.waitForFences(1, &sync_.image_in_flight[image_index], VK_TRUE, UINT64_MAX); + } + sync_.image_in_flight[image_index] = sync_.in_flight_fences[current_frame_]; + + init.disp.resetCommandPool(command_pools_[current_frame_], 0); + VkCommandBufferAllocateInfo allocate_info = {}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandBufferCount = 1; + allocate_info.commandPool = command_pools_[current_frame_]; + VkCommandBuffer command_buffer; + result = init.disp.allocateCommandBuffers(&allocate_info, &command_buffer); + VERIFY_VK_RESULT("failed to allocate command buffer", result); + + { + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + result = init.disp.beginCommandBuffer(command_buffer, &begin_info); + VERIFY_VK_RESULT("failed to begin command buffer", result); + + { + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + image_barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_NONE; + image_barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + // First render pass: clear to red. + // With --isolate-render-passes this segment is split out and submitted on + // its own, chained to the next segment through an injected semaphore. + VkClearValue first_clear{ { { 1.0f, 0.0f, 0.0f, 1.0f } } }; + record_clear_render_pass( + init.disp, command_buffer, render_pass_, framebuffers_[image_index], init.swapchain.extent, first_clear); + + { + // Order the second render pass after the first (write-after-write on the same attachment). + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + image_barrier.dstAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + // Second render pass: clear to green. + // Two render passes in one command buffer exercise the multi-segment + // semaphore chaining performed by the command splitter. + VkClearValue second_clear{ { { 0.0f, 1.0f, 0.0f, 1.0f } } }; + record_clear_render_pass( + init.disp, command_buffer, render_pass_, framebuffers_[image_index], init.swapchain.extent, second_clear); + + { + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL; + image_barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT; + image_barrier.dstAccessMask = VK_ACCESS_NONE; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + result = init.disp.endCommandBuffer(command_buffer); + VERIFY_VK_RESULT("failed to end command buffer", result); + } + + VkSubmitInfo submitInfo = {}; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + + // The submit waits on the binary swapchain-acquire semaphore. + // This is the wait-semaphore that the splitter will hand off to the first split segment. + VkSemaphore wait_semaphores[] = { sync_.available_semaphores[current_frame_] }; + VkPipelineStageFlags wait_stages[] = { VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT }; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = wait_semaphores; + submitInfo.pWaitDstStageMask = wait_stages; + + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &command_buffer; + + VkSemaphore signal_semaphores[] = { sync_.finished_semaphore[image_index] }; + submitInfo.signalSemaphoreCount = 1; + submitInfo.pSignalSemaphores = signal_semaphores; + + init.disp.resetFences(1, &sync_.in_flight_fences[current_frame_]); + + result = init.disp.queueSubmit(graphics_queue_, 1, &submitInfo, sync_.in_flight_fences[current_frame_]); + VERIFY_VK_RESULT("failed to submit queue", result); + + VkPresentInfoKHR present_info = {}; + present_info.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; + + present_info.waitSemaphoreCount = 1; + present_info.pWaitSemaphores = signal_semaphores; + + VkSwapchainKHR swapChains[] = { init.swapchain }; + present_info.swapchainCount = 1; + present_info.pSwapchains = swapChains; + + present_info.pImageIndices = &image_index; + + result = init.disp.queuePresentKHR(present_queue_, &present_info); + if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR) + { + recreate_swapchain(); + return frame_num >= NUM_FRAMES; + } + VERIFY_VK_RESULT("failed to present queue", result); + + current_frame_ = (current_frame_ + 1) % MAX_FRAMES_IN_FLIGHT; + + return IS_RUNNING(frame_num); +} + +void App::cleanup() +{ + for (size_t i = 0; i < sync_.finished_semaphore.size(); ++i) + { + init.disp.destroySemaphore(sync_.finished_semaphore[i], nullptr); + } + for (size_t i = 0; i < MAX_FRAMES_IN_FLIGHT; i++) + { + init.disp.destroySemaphore(sync_.available_semaphores[i], nullptr); + init.disp.destroyFence(sync_.in_flight_fences[i], nullptr); + } + + for (auto command_pool : command_pools_) + { + init.disp.destroyCommandPool(command_pool, nullptr); + } + + for (auto framebuffer : framebuffers_) + { + init.disp.destroyFramebuffer(framebuffer, nullptr); + } + + init.disp.destroyRenderPass(render_pass_, nullptr); +} + +void App::setup() +{ + auto graphics_queue = init.device.get_queue(test::QueueType::graphics); + if (!graphics_queue.has_value()) + throw std::runtime_error("could not get graphics queue"); + graphics_queue_ = *graphics_queue; + + auto present_queue = init.device.get_queue(test::QueueType::present); + if (!present_queue.has_value()) + throw std::runtime_error("could not get present queue"); + present_queue_ = *present_queue; + + create_render_pass(); + + create_framebuffers(); + + auto queue_family_index = init.device.get_queue_index(test::QueueType::graphics); + if (!queue_family_index) + throw std::runtime_error("could not find graphics queue"); + for (auto& command_pool : command_pools_) + { + command_pool = test::create_command_pool(init.disp, *queue_family_index); + } + + sync_ = test::create_sync_objects(init.swapchain, init.disp, MAX_FRAMES_IN_FLIGHT); +} + +GFXRECON_END_NAMESPACE(isolate_render_passes) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/test/test_apps/isolate-render-passes/isolate_render_passes_app.h b/test/test_apps/isolate-render-passes/isolate_render_passes_app.h new file mode 100644 index 0000000000..1e76bf2f4e --- /dev/null +++ b/test/test_apps/isolate-render-passes/isolate_render_passes_app.h @@ -0,0 +1,71 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#ifndef GFXRECON_TESTAPP_ISOLATE_RENDER_PASSES_H +#define GFXRECON_TESTAPP_ISOLATE_RENDER_PASSES_H + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(isolate_render_passes) + +const size_t MAX_FRAMES_IN_FLIGHT = 2; + +/** + * Minimal app that records two legacy (vkCmdBeginRenderPass/vkCmdEndRenderPass) render passes into a single command + * buffer each frame and submits it while waiting on the binary swapchain-acquire semaphore. It deliberately uses no + * graphics pipeline (each render pass only clears the swapchain image via loadOp), keeping the focus on the + * render-pass boundaries that gfxrecon-replay --isolate-render-passes splits and chains. + */ +class App : public test::TestAppBase +{ + VkQueue graphics_queue_; + VkQueue present_queue_; + + std::vector framebuffers_; + + VkRenderPass render_pass_; + + VkCommandPool command_pools_[MAX_FRAMES_IN_FLIGHT]; + + size_t current_frame_ = 0; + + test::Sync sync_; + + void create_render_pass(); + void create_framebuffers(); + void recreate_swapchain(); + void cleanup() override; + bool frame(const int frame_num) override; + void setup() override; + + void configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig*) override; +}; + +GFXRECON_END_NAMESPACE(isolate_render_passes) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // GFXRECON_TESTAPP_ISOLATE_RENDER_PASSES_H diff --git a/test/test_apps/launcher/CMakeLists.txt b/test/test_apps/launcher/CMakeLists.txt index 7b7664ecaa..08d8809dd7 100644 --- a/test/test_apps/launcher/CMakeLists.txt +++ b/test/test_apps/launcher/CMakeLists.txt @@ -42,6 +42,8 @@ set(GFXRECON_TEST_LAUNCHER_LINK_LIBRARIES gfxrecon-testapp-ahb gfxrecon-testapp-debug-utils gfxrecon-testapp-host-image-copy + gfxrecon-testapp-isolate-render-passes + gfxrecon-testapp-serialize-compute-and-transfer gfxrecon-testapp-multisample-depth gfxrecon-testapp-pipeline-binaries gfxrecon-testapp-set-environment diff --git a/test/test_apps/launcher/test_launcher.cpp b/test/test_apps/launcher/test_launcher.cpp index b38cfc1ca8..8e4c805632 100644 --- a/test/test_apps/launcher/test_launcher.cpp +++ b/test/test_apps/launcher/test_launcher.cpp @@ -1,5 +1,5 @@ /* -** Copyright (c) 2025 LunarG, Inc. +** Copyright (c) 2025-2026 LunarG, Inc. ** ** Permission is hereby granted, free of charge, to any person obtaining a ** copy of this software and associated documentation files (the "Software"), @@ -31,6 +31,8 @@ #include #include #include +#include +#include #include #include #include @@ -65,6 +67,8 @@ static const char* kAppNames[] = { "acquired-image", "debug-utils", "host-image-copy", + "isolate-render-passes", + "serialize-compute-and-transfer", "multisample-depth", "pipeline-binaries", #ifndef __ANDROID__ @@ -157,6 +161,14 @@ CreateTestApp(std::unique_ptr application, { app = std::make_unique(); } + else if (app_name == "isolate-render-passes") + { + app = std::make_unique(); + } + else if (app_name == "serialize-compute-and-transfer") + { + app = std::make_unique(); + } else if (app_name == "multisample-depth") { app = std::make_unique(); diff --git a/test/test_apps/serialize-compute-and-transfer/CMakeLists.txt b/test/test_apps/serialize-compute-and-transfer/CMakeLists.txt new file mode 100644 index 0000000000..1648b189d3 --- /dev/null +++ b/test/test_apps/serialize-compute-and-transfer/CMakeLists.txt @@ -0,0 +1,39 @@ +############################################################################### +# Copyright (c) 2026 LunarG, Inc. +# All rights reserved +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to +# deal in the Software without restriction, including without limitation the +# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +# sell copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +# FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. +# +# Author: LunarG Team +# Description: CMake script for serialize-compute-and-transfer test app +############################################################################### + +add_library(gfxrecon-testapp-serialize-compute-and-transfer app.cpp) + +target_include_directories(gfxrecon-testapp-serialize-compute-and-transfer + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}) + +target_link_libraries(gfxrecon-testapp-serialize-compute-and-transfer gfxrecon-testapp-base) + +common_build_directives(gfxrecon-testapp-serialize-compute-and-transfer) + +if(NOT CMAKE_SYSTEM_NAME MATCHES "Android") + install(TARGETS gfxrecon-testapp-serialize-compute-and-transfer RUNTIME DESTINATION ${GFXRECON_INSTALL_TESTAPPDIR}/serialize-compute-and-transfer) +endif() diff --git a/test/test_apps/serialize-compute-and-transfer/app.cpp b/test/test_apps/serialize-compute-and-transfer/app.cpp new file mode 100644 index 0000000000..78bb04d595 --- /dev/null +++ b/test/test_apps/serialize-compute-and-transfer/app.cpp @@ -0,0 +1,321 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(serialize_compute_and_transfer) + +// SPIR-V for a trivial compute shader with no descriptors: +// #version 450 +// layout(local_size_x = 1, local_size_y = 1, local_size_z = 1) in; +// void main() {} +static const uint32_t kComputeShaderSpirv[] = { + 0x07230203, 0x00010000, 0x000d000b, 0x0000000a, 0x00000000, 0x00020011, 0x00000001, 0x0006000b, 0x00000001, + 0x4c534c47, 0x6474732e, 0x3035342e, 0x00000000, 0x0003000e, 0x00000000, 0x00000001, 0x0005000f, 0x00000005, + 0x00000004, 0x6e69616d, 0x00000000, 0x00060010, 0x00000004, 0x00000011, 0x00000001, 0x00000001, 0x00000001, + 0x00030003, 0x00000002, 0x000001c2, 0x000a0004, 0x475f4c47, 0x4c474f4f, 0x70635f45, 0x74735f70, 0x5f656c79, + 0x656e696c, 0x7269645f, 0x69746365, 0x00006576, 0x00080004, 0x475f4c47, 0x4c474f4f, 0x6e695f45, 0x64756c63, + 0x69645f65, 0x74636572, 0x00657669, 0x00040005, 0x00000004, 0x6e69616d, 0x00000000, 0x00040047, 0x00000009, + 0x0000000b, 0x00000019, 0x00020013, 0x00000002, 0x00030021, 0x00000003, 0x00000002, 0x00040015, 0x00000006, + 0x00000020, 0x00000000, 0x00040017, 0x00000007, 0x00000006, 0x00000003, 0x0004002b, 0x00000006, 0x00000008, + 0x00000001, 0x0006002c, 0x00000007, 0x00000009, 0x00000008, 0x00000008, 0x00000008, 0x00050036, 0x00000002, + 0x00000004, 0x00000000, 0x00000003, 0x000200f8, 0x00000005, 0x000100fd, 0x00010038 +}; + +const VkDeviceSize BUFFER_SIZE = 256; + +const int NUM_FRAMES = 3; +#define IS_RUNNING(frame_num) frame_num < NUM_FRAMES; + +void App::configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig* test_config) +{ + if (test_config) + { + test_config->device_api_version_override = VK_MAKE_API_VERSION(0, 1, 3, 296); + } + + TestAppBase::configure_instance_builder(instance_builder, test_config); +} + +void App::create_buffer(VkDeviceSize size, VkBufferUsageFlags usage, VkBuffer* buffer, VkDeviceMemory* memory) +{ + VkBufferCreateInfo buffer_info = {}; + buffer_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO; + buffer_info.size = size; + buffer_info.usage = usage; + buffer_info.sharingMode = VK_SHARING_MODE_EXCLUSIVE; + VERIFY_VK_RESULT("failed to create buffer", init.disp.createBuffer(&buffer_info, nullptr, buffer)); + + VkMemoryRequirements memory_requirements; + init.disp.getBufferMemoryRequirements(*buffer, &memory_requirements); + + VkMemoryAllocateInfo allocate_info = {}; + allocate_info.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO; + allocate_info.allocationSize = memory_requirements.size; + allocate_info.memoryTypeIndex = find_memory_type( + memory_requirements.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT); + VERIFY_VK_RESULT("failed to allocate buffer memory", init.disp.allocateMemory(&allocate_info, nullptr, memory)); + + VERIFY_VK_RESULT("failed to bind buffer memory", init.disp.bindBufferMemory(*buffer, *memory, 0)); +} + +void App::create_buffers() +{ + // Two buffers copied back and forth as the transfer work. + create_buffer(BUFFER_SIZE, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + &buffer_a_, + &buffer_a_memory_); + create_buffer(BUFFER_SIZE, + VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT, + &buffer_b_, + &buffer_b_memory_); + + // Indirect-args buffer for vkCmdDispatchIndirect, filled with a single workgroup. + create_buffer(sizeof(VkDispatchIndirectCommand), + VK_BUFFER_USAGE_INDIRECT_BUFFER_BIT, + &indirect_buffer_, + &indirect_buffer_memory_); + + void* data = nullptr; + VERIFY_VK_RESULT("failed to map indirect buffer", + init.disp.mapMemory(indirect_buffer_memory_, 0, sizeof(VkDispatchIndirectCommand), 0, &data)); + VkDispatchIndirectCommand indirect_command{ 1, 1, 1 }; + memcpy(data, &indirect_command, sizeof(indirect_command)); + init.disp.unmapMemory(indirect_buffer_memory_); +} + +void App::create_compute_pipeline() +{ + VkShaderModuleCreateInfo shader_module_info = {}; + shader_module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO; + shader_module_info.codeSize = sizeof(kComputeShaderSpirv); + shader_module_info.pCode = kComputeShaderSpirv; + VERIFY_VK_RESULT("failed to create compute shader module", + init.disp.createShaderModule(&shader_module_info, nullptr, &compute_shader_)); + + // No descriptor sets and no push constants: the shader only needs to make the dispatches valid. + VkPipelineLayoutCreateInfo pipeline_layout_info = {}; + pipeline_layout_info.sType = VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO; + VERIFY_VK_RESULT("failed to create pipeline layout", + init.disp.createPipelineLayout(&pipeline_layout_info, nullptr, &pipeline_layout_)); + + VkComputePipelineCreateInfo pipeline_info = {}; + pipeline_info.sType = VK_STRUCTURE_TYPE_COMPUTE_PIPELINE_CREATE_INFO; + pipeline_info.stage.sType = VK_STRUCTURE_TYPE_PIPELINE_SHADER_STAGE_CREATE_INFO; + pipeline_info.stage.stage = VK_SHADER_STAGE_COMPUTE_BIT; + pipeline_info.stage.module = compute_shader_; + pipeline_info.stage.pName = "main"; + pipeline_info.layout = pipeline_layout_; + VERIFY_VK_RESULT("failed to create compute pipeline", + init.disp.createComputePipelines(VK_NULL_HANDLE, 1, &pipeline_info, nullptr, &compute_pipeline_)); +} + +bool App::frame(const int frame_num) +{ + init.disp.waitForFences(1, &sync_.in_flight_fences[current_frame_], VK_TRUE, UINT64_MAX); + + uint32_t image_index = 0; + VkResult result = init.disp.acquireNextImageKHR( + init.swapchain, UINT64_MAX, sync_.available_semaphores[current_frame_], VK_NULL_HANDLE, &image_index); + + if (result == VK_ERROR_OUT_OF_DATE_KHR) + { + recreate_swapchain(true); + return IS_RUNNING(frame_num); + } + else if (result != VK_SUCCESS && result != VK_SUBOPTIMAL_KHR) + { + throw test::vulkan_exception("failed to acquire next image", result); + } + + if (sync_.image_in_flight[image_index] != VK_NULL_HANDLE) + { + init.disp.waitForFences(1, &sync_.image_in_flight[image_index], VK_TRUE, UINT64_MAX); + } + sync_.image_in_flight[image_index] = sync_.in_flight_fences[current_frame_]; + + init.disp.resetCommandPool(command_pools_[current_frame_], 0); + VkCommandBufferAllocateInfo allocate_info = {}; + allocate_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO; + allocate_info.commandBufferCount = 1; + allocate_info.commandPool = command_pools_[current_frame_]; + VkCommandBuffer command_buffer; + result = init.disp.allocateCommandBuffers(&allocate_info, &command_buffer); + VERIFY_VK_RESULT("failed to allocate command buffer", result); + + { + VkCommandBufferBeginInfo begin_info = {}; + begin_info.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO; + result = init.disp.beginCommandBuffer(command_buffer, &begin_info); + VERIFY_VK_RESULT("failed to begin command buffer", result); + + // The test does not render: transition the acquired image straight to PRESENT_SRC so it can be presented. + { + VkImageMemoryBarrier image_barrier = {}; + image_barrier.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER; + image_barrier.image = init.swapchain_images[image_index]; + image_barrier.oldLayout = VK_IMAGE_LAYOUT_UNDEFINED; + image_barrier.newLayout = VK_IMAGE_LAYOUT_PRESENT_SRC_KHR; + image_barrier.subresourceRange.aspectMask = VK_IMAGE_ASPECT_COLOR_BIT; + image_barrier.subresourceRange.layerCount = VK_REMAINING_ARRAY_LAYERS; + image_barrier.subresourceRange.levelCount = VK_REMAINING_MIP_LEVELS; + image_barrier.srcAccessMask = VK_ACCESS_NONE; + image_barrier.dstAccessMask = VK_ACCESS_NONE; + init.disp.cmdPipelineBarrier(command_buffer, + VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, + VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT, + 0, + 0, + nullptr, + 0, + nullptr, + 1, + &image_barrier); + } + + // Transfer, then a run of compute dispatches, then transfer again. With --serialize-compute-and-transfer the + // replay tool injects a pipeline barrier before and after each of the three dispatches below. + VkBufferCopy copy_region = {}; + copy_region.size = BUFFER_SIZE; + + init.disp.cmdCopyBuffer(command_buffer, buffer_a_, buffer_b_, 1, ©_region); + + init.disp.cmdBindPipeline(command_buffer, VK_PIPELINE_BIND_POINT_COMPUTE, compute_pipeline_); + init.disp.cmdDispatch(command_buffer, 1, 1, 1); + init.disp.cmdDispatchBase(command_buffer, 0, 0, 0, 1, 1, 1); + init.disp.cmdDispatchIndirect(command_buffer, indirect_buffer_, 0); + + init.disp.cmdCopyBuffer(command_buffer, buffer_b_, buffer_a_, 1, ©_region); + + result = init.disp.endCommandBuffer(command_buffer); + VERIFY_VK_RESULT("failed to end command buffer", result); + } + + VkSubmitInfo submitInfo = {}; + submitInfo.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO; + + VkSemaphore wait_semaphores[] = { sync_.available_semaphores[current_frame_] }; + VkPipelineStageFlags wait_stages[] = { VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT }; + submitInfo.waitSemaphoreCount = 1; + submitInfo.pWaitSemaphores = wait_semaphores; + submitInfo.pWaitDstStageMask = wait_stages; + + submitInfo.commandBufferCount = 1; + submitInfo.pCommandBuffers = &command_buffer; + + VkSemaphore signal_semaphores[] = { sync_.finished_semaphore[image_index] }; + submitInfo.signalSemaphoreCount = 1; + submitInfo.pSignalSemaphores = signal_semaphores; + + init.disp.resetFences(1, &sync_.in_flight_fences[current_frame_]); + + result = init.disp.queueSubmit(graphics_queue_, 1, &submitInfo, sync_.in_flight_fences[current_frame_]); + VERIFY_VK_RESULT("failed to submit queue", result); + + VkPresentInfoKHR present_info = {}; + present_info.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR; + + present_info.waitSemaphoreCount = 1; + present_info.pWaitSemaphores = signal_semaphores; + + VkSwapchainKHR swapChains[] = { init.swapchain }; + present_info.swapchainCount = 1; + present_info.pSwapchains = swapChains; + + present_info.pImageIndices = &image_index; + + result = init.disp.queuePresentKHR(present_queue_, &present_info); + if (result == VK_ERROR_OUT_OF_DATE_KHR || result == VK_SUBOPTIMAL_KHR) + { + recreate_swapchain(true); + return frame_num >= NUM_FRAMES; + } + VERIFY_VK_RESULT("failed to present queue", result); + + current_frame_ = (current_frame_ + 1) % MAX_FRAMES_IN_FLIGHT; + + return IS_RUNNING(frame_num); +} + +void App::cleanup() +{ + init.disp.destroyPipeline(compute_pipeline_, nullptr); + init.disp.destroyPipelineLayout(pipeline_layout_, nullptr); + init.disp.destroyShaderModule(compute_shader_, nullptr); + + init.disp.destroyBuffer(buffer_a_, nullptr); + init.disp.freeMemory(buffer_a_memory_, nullptr); + init.disp.destroyBuffer(buffer_b_, nullptr); + init.disp.freeMemory(buffer_b_memory_, nullptr); + init.disp.destroyBuffer(indirect_buffer_, nullptr); + init.disp.freeMemory(indirect_buffer_memory_, nullptr); + + for (size_t i = 0; i < sync_.finished_semaphore.size(); ++i) + { + init.disp.destroySemaphore(sync_.finished_semaphore[i], nullptr); + } + for (size_t i = 0; i < MAX_FRAMES_IN_FLIGHT; i++) + { + init.disp.destroySemaphore(sync_.available_semaphores[i], nullptr); + init.disp.destroyFence(sync_.in_flight_fences[i], nullptr); + } + + for (auto command_pool : command_pools_) + { + init.disp.destroyCommandPool(command_pool, nullptr); + } +} + +void App::setup() +{ + auto graphics_queue = init.device.get_queue(test::QueueType::graphics); + if (!graphics_queue.has_value()) + throw std::runtime_error("could not get graphics queue"); + graphics_queue_ = *graphics_queue; + + auto present_queue = init.device.get_queue(test::QueueType::present); + if (!present_queue.has_value()) + throw std::runtime_error("could not get present queue"); + present_queue_ = *present_queue; + + auto queue_family_index = init.device.get_queue_index(test::QueueType::graphics); + if (!queue_family_index) + throw std::runtime_error("could not find graphics queue"); + for (auto& command_pool : command_pools_) + { + command_pool = test::create_command_pool(init.disp, *queue_family_index); + } + + sync_ = test::create_sync_objects(init.swapchain, init.disp, MAX_FRAMES_IN_FLIGHT); + + create_compute_pipeline(); + create_buffers(); +} + +GFXRECON_END_NAMESPACE(serialize_compute_and_transfer) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) diff --git a/test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h b/test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h new file mode 100644 index 0000000000..3471fc6879 --- /dev/null +++ b/test/test_apps/serialize-compute-and-transfer/serialize_compute_and_transfer_app.h @@ -0,0 +1,81 @@ +/* +** Copyright (c) 2026 LunarG, Inc. +** +** Permission is hereby granted, free of charge, to any person obtaining a +** copy of this software and associated documentation files (the "Software"), +** to deal in the Software without restriction, including without limitation +** the rights to use, copy, modify, merge, publish, distribute, sublicense, +** and/or sell copies of the Software, and to permit persons to whom the +** Software is furnished to do so, subject to the following conditions: +** +** The above copyright notice and this permission notice shall be included in +** all copies or substantial portions of the Software. +** +** THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +** IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +** FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +** AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +** LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +** FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +** DEALINGS IN THE SOFTWARE. +*/ + +#ifndef GFXRECON_TESTAPP_SERIALIZE_COMPUTE_AND_TRANSFER_H +#define GFXRECON_TESTAPP_SERIALIZE_COMPUTE_AND_TRANSFER_H + +#include + +#include + +GFXRECON_BEGIN_NAMESPACE(gfxrecon) +GFXRECON_BEGIN_NAMESPACE(test_app) +GFXRECON_BEGIN_NAMESPACE(serialize_compute_and_transfer) + +const size_t MAX_FRAMES_IN_FLIGHT = 2; + +/** + * Minimal app that, each frame, records transfer (vkCmdCopyBuffer) and compute (vkCmdDispatch, + * vkCmdDispatchBase, vkCmdDispatchIndirect) commands into a single command buffer interleaved with each other, then + * submits it while waiting on the binary swapchain-acquire semaphore. It deliberately uses a trivial empty compute + * shader with no descriptors, keeping the focus on the dispatch commands that gfxrecon-replay + * --serialize-compute-and-transfer wraps with injected pipeline barriers. + */ +class App : public test::TestAppBase +{ + VkQueue graphics_queue_; + VkQueue present_queue_; + + VkCommandPool command_pools_[MAX_FRAMES_IN_FLIGHT]; + + size_t current_frame_ = 0; + + test::Sync sync_; + + // Trivial compute pipeline (empty shader, no descriptors) used to make the dispatches valid. + VkShaderModule compute_shader_ = VK_NULL_HANDLE; + VkPipelineLayout pipeline_layout_ = VK_NULL_HANDLE; + VkPipeline compute_pipeline_ = VK_NULL_HANDLE; + + // Two buffers copied back and forth (the transfer work), and an indirect-args buffer for vkCmdDispatchIndirect. + VkBuffer buffer_a_ = VK_NULL_HANDLE; + VkDeviceMemory buffer_a_memory_ = VK_NULL_HANDLE; + VkBuffer buffer_b_ = VK_NULL_HANDLE; + VkDeviceMemory buffer_b_memory_ = VK_NULL_HANDLE; + VkBuffer indirect_buffer_ = VK_NULL_HANDLE; + VkDeviceMemory indirect_buffer_memory_ = VK_NULL_HANDLE; + + void create_buffer(VkDeviceSize size, VkBufferUsageFlags usage, VkBuffer* buffer, VkDeviceMemory* memory); + void create_buffers(); + void create_compute_pipeline(); + void cleanup() override; + bool frame(const int frame_num) override; + void setup() override; + + void configure_instance_builder(test::InstanceBuilder& instance_builder, vkmock::TestConfig*) override; +}; + +GFXRECON_END_NAMESPACE(serialize_compute_and_transfer) +GFXRECON_END_NAMESPACE(test_app) +GFXRECON_END_NAMESPACE(gfxrecon) + +#endif // GFXRECON_TESTAPP_SERIALIZE_COMPUTE_AND_TRANSFER_H diff --git a/test/test_cases/isolate-render-passes.cpp b/test/test_cases/isolate-render-passes.cpp new file mode 100644 index 0000000000..fe596e3901 --- /dev/null +++ b/test/test_cases/isolate-render-passes.cpp @@ -0,0 +1,14 @@ +#include + +#include "verify-gfxr.h" + +/** + * Capture the legacy-render-pass app and replay it with --isolate-render-passes. + * The replay tool splits the command buffer at each vkCmdBeginRenderPass/vkCmdEndRenderPass boundary + * and submits the segments separately. + * This test exercises that code path against the mock ICD and asserts the replay completes successfully. + */ +TEST(IsolateRenderPasses, ReplaySplitsLegacyRenderPasses) +{ + capture_and_replay("isolate-render-passes", { "--isolate-render-passes" }); +} diff --git a/test/test_cases/serialize-compute-and-transfer.cpp b/test/test_cases/serialize-compute-and-transfer.cpp new file mode 100644 index 0000000000..021fcc7a98 --- /dev/null +++ b/test/test_cases/serialize-compute-and-transfer.cpp @@ -0,0 +1,14 @@ +#include + +#include "verify-gfxr.h" + +/** + * Capture the serialize-compute-and-transfer app and replay it with --serialize-compute-and-transfer. + * The replay tool wraps every compute dispatch with an injected pipeline barrier before and after, + * so compute and transfer work cannot overlap on the GPU. + * This test exercises that code path against the mock ICD and asserts the replay completes successfully. + */ +TEST(SerializeComputeAndTransfer, ReplayInjectsBarriersAroundDispatches) +{ + capture_and_replay("serialize-compute-and-transfer", { "--serialize-compute-and-transfer" }); +} diff --git a/test/verify-gfxr.cpp b/test/verify-gfxr.cpp index 55a0942ddd..a08cd5761d 100644 --- a/test/verify-gfxr.cpp +++ b/test/verify-gfxr.cpp @@ -69,8 +69,10 @@ bool clean_gfxr_json(int depth, nlohmann::json::parse_event_t event, nlohmann::j #if defined(__linux__) || defined(__APPLE__) static char const* CONVERT_FILENAME = "gfxrecon-convert"; +static char const* REPLAY_FILENAME = "gfxrecon-replay"; #elif defined(_WIN32) static char const* CONVERT_FILENAME = "gfxrecon-convert.exe"; +static char const* REPLAY_FILENAME = "gfxrecon-replay.exe"; #endif struct Paths @@ -80,6 +82,7 @@ struct Paths std::filesystem::path full_app_directory{ base_path }; std::filesystem::path full_executable_path; std::filesystem::path convert_path{ base_path }; + std::filesystem::path replay_path{ base_path }; std::filesystem::path capture_path{ base_path }; std::filesystem::path known_good_path{ base_path }; std::filesystem::path app_json_path; @@ -161,6 +164,7 @@ struct Paths #endif convert_path.append(CONVERT_FILENAME); + replay_path.append(REPLAY_FILENAME); std::string gfxr_file_name = test_name + std::string(".gfxr"); capture_path.append(gfxr_file_name); @@ -345,3 +349,39 @@ void verify_gfxr(const char* test_name, char const* trimming_frames, bool trigge run_trimming_app(paths, test_name, trimming_frames, trigger_trimming); } } + +void capture_and_replay(const char* test_name, std::vector extra_replay_args) +{ + EnvironmentVariables env_vars; + + Paths paths{ test_name, nullptr, false }; + int result; + + bool working_directory_exists = std::filesystem::exists(paths.working_directory); + ASSERT_TRUE(working_directory_exists) << "working directory does not exist: " << paths.working_directory; + + // Run the app with capture enabled to produce the gfxr to replay. + env_vars.SetEnv("GFXRECON_CAPTURE_FILE", paths.capture_path.string().c_str()); + result = run_command(paths.working_directory, paths.full_executable_path, { test_name }); + ASSERT_EQ(result, 0) << "capture command failed " << paths.full_executable_path << " " << test_name << " in path " + << paths.working_directory; + + ASSERT_TRUE(std::filesystem::exists(paths.capture_path)) << "capture file was not produced: " << paths.capture_path; + + // The gfxreconstruct capture layer is still enabled in the environment, so point GFXRECON_CAPTURE_FILE at a + // throwaway path for the replay step. This keeps the layer (if it loads during replay) from re-capturing over the + // input gfxr we are about to read. + std::filesystem::path replay_capture_path{ paths.base_path }; + replay_capture_path.append(test_name + std::string("_replay.gfxr")); + env_vars.SetEnv("GFXRECON_CAPTURE_FILE", replay_capture_path.string().c_str()); + + // Replay the capture headless (offscreen swapchain) against the mock ICD, forwarding any extra arguments. + // Asserts the replay tool exits successfully (no crash, assertion, or replay error). + std::vector replay_args = { "--swapchain", "offscreen" }; + replay_args.insert(replay_args.end(), extra_replay_args.begin(), extra_replay_args.end()); + replay_args.push_back(paths.capture_path.string()); + + result = run_command(paths.base_path, paths.replay_path, replay_args); + ASSERT_EQ(result, 0) << "replay command failed " << paths.replay_path << " for capture " << paths.capture_path + << " in path " << paths.base_path; +} diff --git a/test/verify-gfxr.h b/test/verify-gfxr.h index 24623c0c42..7f3417e2b0 100644 --- a/test/verify-gfxr.h +++ b/test/verify-gfxr.h @@ -1,23 +1,40 @@ #ifndef GFXRECONSTRUCT_VERIFY_GFXR_H #define GFXRECONSTRUCT_VERIFY_GFXR_H +#include +#include + void run_in_background(const char* test_name); -/* +/** * Run an application with capture enabled, and compare the resulting gfxr file to a known good gfxr * - * test_name - the name of the test to launch - * trimming_frames - It's not nullptr if it needs to run trimming app. frame info ex: "10" or "10-100" + * @param test_name - the name of the test to launch + * @param trimming_frames - It's not nullptr if it needs to run trimming app. frame info ex: "10" or "10-100" * - * expects the following environment variables to be set + * @note expects the following environment variables to be set * VK_LAYER_PATH - path to VkLayer_gfxreconstruct.dll * VK_DRIVER_FILES - path to VkICD_mock_icd.json * - * expects the following environment variables to be set with the following values + * @note expects the following environment variables to be set with the following values * VK_INSTANCE_LAYERS=VK_LAYER_LUNARG_gfxreconstruct * GFXRECON_CAPTURE_FILE_TIMESTAMP=false * GFXRECON_CAPTURE_FILE=actual.gfxr */ void verify_gfxr(const char* test_name, char const* trimming_frames = nullptr, bool trigger_trimming = false); +/** + * Run an application with capture enabled, then replay the resulting gfxr with gfxrecon-replay, asserting that the + * replay process exits successfully. + * + * Replay is forced offscreen (--swapchain offscreen) so it runs headless against the mock ICD, and any additional + * arguments are forwarded to the replay tool. + * + * @param test_name - the name of the test app to launch and capture + * @param extra_replay_args - additional arguments forwarded verbatim to gfxrecon-replay + * + * @note expects the same environment variables as verify_gfxr(). + */ +void capture_and_replay(const char* test_name, std::vector extra_replay_args = {}); + #endif // GFXRECONSTRUCT_VERIFY_GFXR_H diff --git a/tools/replay/replay_settings.h b/tools/replay/replay_settings.h index 7aa93107ae..9a993f8093 100644 --- a/tools/replay/replay_settings.h +++ b/tools/replay/replay_settings.h @@ -36,7 +36,8 @@ const char kOptions[] = "--dump-resources-modifiable-state-only,--pbi-all,--preload-measurement-range,--add-new-pipeline-caches,--" "screenshot-ignore-FrameBoundaryANDROID,--screenshot-apply-prerotation,--deduplicate-device,--log-timestamps,--" "capture,--idle-before-submit,--" - "serialize-render-passes,--serialize-queue-submissions,--async-processing"; + "serialize-render-passes,--serialize-queue-submissions,--async-processing,--isolate-render-passes,--serialize-" + "compute-and-transfer"; const char kArguments[] = "--log-level,--log-file,--cpu-mask,--gpu,--gpu-group,--pause-frame,--wsi,--surface-index,-m|--memory-translation," "--replace-shaders,--screenshots,--screenshot-interval,--denied-messages,--allowed-messages,--screenshot-format,--" @@ -405,6 +406,9 @@ static void PrintUsage(const char* exe_name) GFXRECON_WRITE_CONSOLE(" \t\tParameters to forward to the replay event plugin. The format"); GFXRECON_WRITE_CONSOLE(" \t\tof the parameters is determined by the plugin and is not"); GFXRECON_WRITE_CONSOLE(" \t\tinterpreted by the replay tool. (forwarded to replay tool)"); + GFXRECON_WRITE_CONSOLE(" --isolate-render-passes"); + GFXRECON_WRITE_CONSOLE( + " \t\tIsolate render passes by splitting the command buffer into multiple submits."); #if defined(WIN32) GFXRECON_WRITE_CONSOLE("") diff --git a/tools/tool_settings.h b/tools/tool_settings.h index 18b21d6f2e..70ea4fdf97 100644 --- a/tools/tool_settings.h +++ b/tools/tool_settings.h @@ -168,13 +168,15 @@ const char kDumpResourcesModifiableStateOnly[] = "--dump-resources-modifiable-st const char kDumpResourcesBeforeDrawOption[] = "--dump-resources-before-draw"; #endif -const char kDumpResourcesArgument[] = "--dump-resources"; -const char kDumpResourcesDirArgument[] = "--dump-resources-dir"; -const char kFrameWarmUpSpirv[] = "--frame-warm-up-spirv"; -const char kFrameWarmUpLoad[] = "--frame-warm-up-load"; -const char kSerializeQueueSubmissions[] = "--serialize-queue-submissions"; -const char kReplayEventPluginPath[] = "--replay-event-plugin-path"; -const char kReplayEventPluginParams[] = "--replay-event-plugin-params"; +const char kDumpResourcesArgument[] = "--dump-resources"; +const char kDumpResourcesDirArgument[] = "--dump-resources-dir"; +const char kFrameWarmUpSpirv[] = "--frame-warm-up-spirv"; +const char kFrameWarmUpLoad[] = "--frame-warm-up-load"; +const char kSerializeQueueSubmissions[] = "--serialize-queue-submissions"; +const char kReplayEventPluginPath[] = "--replay-event-plugin-path"; +const char kReplayEventPluginParams[] = "--replay-event-plugin-params"; +const char kIsolateRenderPasses[] = "--isolate-render-passes"; +const char kSerializeComputeAndTransfer[] = "--serialize-compute-and-transfer"; enum class WsiPlatform { @@ -1402,6 +1404,8 @@ GetVulkanReplayOptions(const gfxrecon::util::ArgumentParser& arg_parse replay_options.replay_event_plugin_path = arg_parser.GetArgumentValue(kReplayEventPluginPath); replay_options.replay_event_plugin_params = arg_parser.GetArgumentValue(kReplayEventPluginParams); + replay_options.isolate_render_passes = arg_parser.IsOptionSet(kIsolateRenderPasses); + replay_options.serialize_compute_and_transfer = arg_parser.IsOptionSet(kSerializeComputeAndTransfer); return replay_options; }