Skip to content

Frame Warm Up #2713

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
antonio-lunarg wants to merge 1 commit into
base: dev
Choose a base branch
from
+698 −57
Open

Frame Warm Up #2713

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions USAGE_android.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1007,6 +1007,14 @@ options:
Wait for the specified amount of milliseconds before
processing the first submit. (forwarded to replay tool)
--idle-before-submit Wait for the GPU to become idle before each submit.
--frame-warm-up-spirv DEVICE_FILE
Specify a SPIR-V file for the compute warm-up pass.
Warm-up runs only when this option and a non-zero
--frame-warm-up-load are both provided.
(forwarded to replay tool)
--frame-warm-up-load LOAD
Specify workload scale factor for a compute dispatch warm-up pass
run before each frame replay. Default is 0 (disabled).
(forwarded to replay tool)
```

Expand Down
7 changes: 7 additions & 0 deletions USAGE_desktop_Vulkan.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -876,6 +876,13 @@ Optional arguments:
Wait for the specified amount of milliseconds before processing the first submit.
--idle-before-submit
Wait for the GPU to become idle before each submit.
--frame-warm-up-spirv <spirv-file>
Specify a SPIR-V file for the compute warm-up pass.
Warm-up runs only when this option and a non-zero
--frame-warm-up-load are both provided.
--frame-warm-up-load <load>
Specify workload scale factor for a compute dispatch warm-up pass
run before each frame replay. Default is 0 (disabled).
```

### Key Controls
Expand Down
2 changes: 2 additions & 0 deletions android/framework/decode/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@ target_sources(gfxrecon_decode
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_captured_swapchain.h
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_captured_swapchain.cpp
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_enum_util.h
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_frame_warm_up.h
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_frame_warm_up.cpp
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_handle_mapping_util.h
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_handle_mapping_util.cpp
${GFXRECON_SOURCE_DIR}/framework/decode/vulkan_object_cleanup_util.h
Expand Down
10 changes: 10 additions & 0 deletions android/scripts/gfxrecon.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,8 @@ def CreateReplayParser():
parser.add_argument('--screenshot-ignore-FrameBoundaryANDROID', action='store_true', default=False, help='If set, frames switced with vkFrameBoundANDROID will be ignored from the screenshot handler.')
parser.add_argument('--wait-before-first-submit', metavar='MILLISECONDS', help='Wait for the specified amount of milliseconds before processing the first submit. (forwarded to replay tool)')
parser.add_argument('--idle-before-submit', action='store_true', default=False, help='Wait for the GPU to become idle before each submit. (forwarded to replay tool)')
parser.add_argument('--frame-warm-up-spirv', metavar='DEVICE_FILE', help='Specify a SPIR-V file for the compute warm-up pass. Warm-up runs only when this option and a non-zero --frame-warm-up-load are both provided. (forwarded to replay tool)')
parser.add_argument('--frame-warm-up-load', metavar='LOAD', default=0, help='Specify workload scale factor for a compute dispatch warm-up pass run before each frame replay. Default is 0 (disabled). (forwarded to replay tool)')

return parser

Expand Down Expand Up @@ -330,6 +332,14 @@ def MakeExtrasString(args):
if args.idle_before_submit:
arg_list.append('--idle-before-submit')

if args.frame_warm_up_spirv:
arg_list.append('--frame-warm-up-spirv')
arg_list.append('{}'.format(args.frame_warm_up_spirv))

if args.frame_warm_up_load:
arg_list.append('--frame-warm-up-load')
arg_list.append('{}'.format(args.frame_warm_up_load))

if args.file:
arg_list.append(args.file)
elif not args.version:
Expand Down
2 changes: 2 additions & 0 deletions framework/decode/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,8 @@ target_sources(gfxrecon_decode
${CMAKE_CURRENT_LIST_DIR}/vulkan_device_address_tracker.cpp
${CMAKE_CURRENT_LIST_DIR}/vulkan_captured_swapchain.h
${CMAKE_CURRENT_LIST_DIR}/vulkan_captured_swapchain.cpp
${CMAKE_CURRENT_LIST_DIR}/vulkan_frame_warm_up.h
${CMAKE_CURRENT_LIST_DIR}/vulkan_frame_warm_up.cpp
${CMAKE_CURRENT_LIST_DIR}/vulkan_json_consumer_base.h
${CMAKE_CURRENT_LIST_DIR}/vulkan_json_consumer_base.cpp
${CMAKE_CURRENT_LIST_DIR}/marker_json_consumer.h
Expand Down
86 changes: 30 additions & 56 deletions framework/decode/vulkan_address_replacer.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,32 +34,6 @@
GFXRECON_BEGIN_NAMESPACE(gfxrecon)
GFXRECON_BEGIN_NAMESPACE(decode)

//! RAII helper to mark injected commands in scope
struct MarkInjectedCommandsHelper
{
// allow nested usage without hitting an assertion
static thread_local std::atomic<uint32_t> semaphore;

MarkInjectedCommandsHelper()
{
// mark injected commands
if (semaphore++ == 0)
{
util::BeginInjectedCommands();
};
}

~MarkInjectedCommandsHelper()
{
// mark end of injected commands
if (--semaphore == 0)
{
util::EndInjectedCommands();
}
}
};
thread_local std::atomic<uint32_t> MarkInjectedCommandsHelper::semaphore = 0;

//! RAII helper submit a command-buffer to a queue and synchronize via fence
struct QueueSubmitHelper
{
Expand All @@ -86,7 +60,7 @@ struct QueueSubmitHelper
device_table(device_table_),
device(device_), command_buffer(command_buffer_), fence(fence_), queue(queue_)
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;

device_table->ResetFences(device, 1, &fence);

Expand All @@ -102,7 +76,7 @@ struct QueueSubmitHelper
{
if (device_table != nullptr)
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;

device_table->EndCommandBuffer(command_buffer);

Expand Down Expand Up @@ -330,7 +304,7 @@ void VulkanAddressReplacer::SetRaytracingProperties(const decode::VulkanPhysical

VulkanAddressReplacer::~VulkanAddressReplacer()
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;

// explicitly free resources here, in order to mark destruction API-calls as injected
pipeline_context_map_.clear();
Expand Down Expand Up @@ -435,10 +409,10 @@ VkSemaphore VulkanAddressReplacer::UpdateBufferAddresses(const VulkanCommandBuff
VkTimelineSemaphoreSubmitInfo timeline_info = {};
timeline_info.sType = VK_STRUCTURE_TYPE_TIMELINE_SEMAPHORE_SUBMIT_INFO;
GFXRECON_NARROWING_ASSIGN(timeline_info.waitSemaphoreValueCount, wait_semaphores.size());
timeline_info.pWaitSemaphoreValues = wait_semaphores.empty() ? nullptr : semaphore_values.data();
timeline_info.pWaitSemaphoreValues = wait_semaphores.empty() ? nullptr : semaphore_values.data();

VkSubmitInfo submit_info = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
submit_info.pNext = &timeline_info;
VkSubmitInfo submit_info = { VK_STRUCTURE_TYPE_SUBMIT_INFO };
submit_info.pNext = &timeline_info;
GFXRECON_NARROWING_ASSIGN(submit_info.waitSemaphoreCount, wait_semaphores.size());
submit_info.pWaitSemaphores = wait_semaphores.empty() ? nullptr : semaphore_handles.data();
submit_info.pWaitDstStageMask = wait_semaphores.empty() ? nullptr : wait_dst_stages.data();
Expand Down Expand Up @@ -573,8 +547,8 @@ VulkanAddressReplacer::ResolveBufferAddresses(std::vector<VulkanCommandBufferInf
std::vector<VulkanCommandBufferInfo*>
VulkanAddressReplacer::GetCommandBufferInfosFromSubmitInfo(Decoded_VkSubmitInfo& submit_info)
{
size_t num_command_buffers = submit_info.pCommandBuffers.GetLength();
auto* cmd_buf_handles = submit_info.pCommandBuffers.GetPointer();
size_t num_command_buffers = submit_info.pCommandBuffers.GetLength();
auto* cmd_buf_handles = submit_info.pCommandBuffers.GetPointer();

std::vector<VulkanCommandBufferInfo*> command_buffer_infos(num_command_buffers);

Expand All @@ -590,8 +564,8 @@ VulkanAddressReplacer::GetCommandBufferInfosFromSubmitInfo(Decoded_VkSubmitInfo&
std::vector<VulkanCommandBufferInfo*>
VulkanAddressReplacer::GetCommandBufferInfosFromSubmitInfo(Decoded_VkSubmitInfo2& submit_info2)
{
size_t num_command_buffers = submit_info2.pCommandBufferInfos->GetLength();
auto* cmd_buf_info_metas = submit_info2.pCommandBufferInfos->GetMetaStructPointer();
size_t num_command_buffers = submit_info2.pCommandBufferInfos->GetLength();
auto* cmd_buf_info_metas = submit_info2.pCommandBufferInfos->GetMetaStructPointer();

std::vector<VulkanCommandBufferInfo*> command_buffer_infos(num_command_buffers);

Expand Down Expand Up @@ -648,7 +622,7 @@ void VulkanAddressReplacer::ProcessCmdPushConstants(const VulkanCommandBufferInf
GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdPushConstants(): Replay is adjusting "
"buffer-device-addresses in push-constants");
VkDeviceAddress address_offset = *address - buffer_info->capture_address;
*address = buffer_info->replay_address + address_offset;
*address = buffer_info->replay_address + address_offset;
}
}
}
Expand Down Expand Up @@ -786,18 +760,18 @@ void VulkanAddressReplacer::ProcessCmdBindDescriptorSets(VulkanCommandBufferInfo
GFXRECON_LOG_INFO_ONCE("VulkanAddressReplacer::ProcessCmdBindDescriptorSets(): Replay is adjusting "
"buffer-device-addresses in inline-uniform-block");
VkDeviceAddress address_offset = *address - buffer_info->capture_address;
*address = buffer_info->replay_address + address_offset;
*address = buffer_info->replay_address + address_offset;

// TODO: come back for this. we don't treat arrays in push-constants and here, but probably should
GFXRECON_ASSERT(buffer_ref_info.array_stride == 0);

// batch descriptor-updates
auto& write_inline_uniform_block =
sets_requiring_update[{ descriptor_set_info->handle, buffer_ref_info.binding }];
write_inline_uniform_block.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK;
write_inline_uniform_block.sType = VK_STRUCTURE_TYPE_WRITE_DESCRIPTOR_SET_INLINE_UNIFORM_BLOCK;
GFXRECON_NARROWING_ASSIGN(write_inline_uniform_block.dataSize,
descriptor_set_binding_info.inline_uniform_block.size());
write_inline_uniform_block.pData = descriptor_set_binding_info.inline_uniform_block.data();
write_inline_uniform_block.pData = descriptor_set_binding_info.inline_uniform_block.data();
}
}

Expand All @@ -824,7 +798,7 @@ void VulkanAddressReplacer::ProcessCmdBindDescriptorSets(VulkanCommandBufferInfo
if (!descriptor_updates.empty())
{
// mark injected commands
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
device_table_->UpdateDescriptorSets(device_,
GFXRECON_NARROWING_CAST(uint32_t, descriptor_updates.size()),
descriptor_updates.data(),
Expand Down Expand Up @@ -917,7 +891,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays(
if (!valid_sbt_alignment_ || !valid_group_handles)
{
// mark injected commands
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;

if (pipeline_sbt_ == VK_NULL_HANDLE)
{
Expand Down Expand Up @@ -964,7 +938,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays(
auto input_addresses = reinterpret_cast<VkDeviceAddress*>(pipeline_context_sbt.input_handle_buffer.mapped_data);

std::unordered_map<const VkStridedDeviceAddressRegionKHR*, VkDeviceSize> num_addresses_map;
uint32_t num_addresses = 0;
uint32_t num_addresses = 0;

{
const auto handle_size_aligned = static_cast<uint32_t>(util::aligned_value(
Expand Down Expand Up @@ -1044,7 +1018,7 @@ void VulkanAddressReplacer::ProcessCmdTraceRays(
if (region != nullptr && region->size != 0 && region->stride != 0)
{
VkDeviceSize num_handles = num_addresses_map[region];
auto group_size = static_cast<uint32_t>(util::aligned_value(
auto group_size = static_cast<uint32_t>(util::aligned_value(
num_handles * handle_size_aligned, replay_ray_properties_->shaderGroupBaseAlignment));

// increase group-size/stride, if required
Expand Down Expand Up @@ -1210,7 +1184,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR(
primitive_counts[j] = range_infos[j].primitiveCount;
}

MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
device_table_->GetAccelerationStructureBuildSizesKHR(device_,
VK_ACCELERATION_STRUCTURE_BUILD_TYPE_DEVICE_KHR,
&build_geometry_info,
Expand Down Expand Up @@ -1280,7 +1254,7 @@ void VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR(

if (!as_buffer_usable || !scratch_buffer_usable)
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
GFXRECON_LOG_INFO_ONCE(
"VulkanAddressReplacer::ProcessCmdBuildAccelerationStructuresKHR: Replay is adjusting "
"acceleration-structures using shadow-structures and -buffers");
Expand Down Expand Up @@ -1606,7 +1580,7 @@ void VulkanAddressReplacer::ProcessBuildVulkanAccelerationStructuresMetaCommand(
&command_buffer_info, info_count, geometry_infos, range_infos, address_tracker);

// issue build-command
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
device_table_->CmdBuildAccelerationStructuresKHR(
submit_asset_.command_buffer, info_count, geometry_infos, range_infos);
}
Expand All @@ -1632,7 +1606,7 @@ void VulkanAddressReplacer::ProcessCopyVulkanAccelerationStructuresMetaCommand(
ProcessCmdCopyAccelerationStructuresKHR(copy_info, address_tracker);

// issue copy command
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
device_table_->CmdCopyAccelerationStructureKHR(submit_asset_.command_buffer, copy_info);
}
else
Expand All @@ -1658,7 +1632,7 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe
1, &acceleration_structure, query_type, query_pool_, 0, address_tracker);

// issue vkCmdResetQueryPool and vkCmdWriteAccelerationStructuresPropertiesKHR
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
device_table_->CmdResetQueryPool(submit_asset_.command_buffer, query_pool_, 0, 1);
device_table_->CmdWriteAccelerationStructuresPropertiesKHR(
submit_asset_.command_buffer, 1, &acceleration_structure, query_type, query_pool_, 0);
Expand All @@ -1667,7 +1641,7 @@ void VulkanAddressReplacer::ProcessVulkanAccelerationStructuresWritePropertiesMe
VkDeviceSize compact_size = 0;

// the above command-buffer is already synced here, retrieve result using vkGetQueryPoolResults
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
device_table_->GetQueryPoolResults(device_,
query_pool_,
0,
Expand Down Expand Up @@ -1858,7 +1832,7 @@ bool VulkanAddressReplacer::create_buffer(VulkanAddressReplacer::buffer_context_
buffer_context = {};
buffer_context.resource_allocator = resource_allocator_;
GFXRECON_NARROWING_ASSIGN(buffer_context.num_bytes, num_bytes);
buffer_context.name = name;
buffer_context.name = name;

VkBufferCreateInfo buffer_create_info = {};
buffer_create_info.sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO;
Expand Down Expand Up @@ -2019,7 +1993,7 @@ void VulkanAddressReplacer::DestroyShadowResources(const VulkanBufferInfo*
auto remove_shadow_as_it = shadow_as_map_.find(replay_address_it->first);
if (remove_shadow_as_it != shadow_as_map_.end())
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
shadow_as_map_.erase(remove_shadow_as_it);
}
}
Expand All @@ -2034,21 +2008,21 @@ void VulkanAddressReplacer::DestroyShadowResources(VkCommandBuffer handle)
auto shadow_sbt_it = shadow_sbt_map_.find(handle);
if (shadow_sbt_it != shadow_sbt_map_.end())
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
shadow_sbt_map_.erase(shadow_sbt_it);
}

auto pipeline_sbt_it = pipeline_context_map_.find(handle);
if (pipeline_sbt_it != pipeline_context_map_.end())
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
pipeline_context_map_.erase(pipeline_sbt_it);
}

auto submit_asset_it = submit_asset_map_.find(handle);
if (submit_asset_it != submit_asset_map_.end())
{
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;
submit_asset_map_.erase(submit_asset_it);
}
}
Expand Down Expand Up @@ -2184,7 +2158,7 @@ void VulkanAddressReplacer::run_compute_replace(const VulkanCommandBufferInfo*
};

// mark injected commands
MarkInjectedCommandsHelper mark_injected_commands_helper;
util::MarkInjectedCommandsHelper mark_injected_commands_helper;

if (pipeline_bda_ == VK_NULL_HANDLE && !init_pipeline())
{
Expand Down
Loading
Loading