xref: /aosp_15_r20/external/mesa3d/src/virtio/vulkan/vn_feedback.c (revision 6104692788411f58d303aa86923a9ff6ecaded22)

/*
 * Copyright 2022 Google LLC
 * SPDX-License-Identifier: MIT
 */

#include "vn_feedback.h"

#include "vn_command_buffer.h"
#include "vn_device.h"
#include "vn_physical_device.h"
#include "vn_query_pool.h"
#include "vn_queue.h"

static uint32_t
vn_get_memory_type_index(const VkPhysicalDeviceMemoryProperties *mem_props,
                         uint32_t mem_type_bits,
                         VkMemoryPropertyFlags required_mem_flags)
{
   u_foreach_bit(mem_type_index, mem_type_bits)
   {
      assert(mem_type_index < mem_props->memoryTypeCount);
      if ((mem_props->memoryTypes[mem_type_index].propertyFlags &
           required_mem_flags) == required_mem_flags)
         return mem_type_index;
   }

   return UINT32_MAX;
}

VkResult
vn_feedback_buffer_create(struct vn_device *dev,
                          uint32_t size,
                          const VkAllocationCallbacks *alloc,
                          struct vn_feedback_buffer **out_fb_buf)
{
   const bool exclusive = dev->queue_family_count == 1;
   const VkPhysicalDeviceMemoryProperties *mem_props =
      &dev->physical_device->memory_properties;
   VkDevice dev_handle = vn_device_to_handle(dev);
   VkResult result;

   struct vn_feedback_buffer *fb_buf =
      vk_zalloc(alloc, sizeof(*fb_buf), VN_DEFAULT_ALIGN,
                VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
   if (!fb_buf)
      return VK_ERROR_OUT_OF_HOST_MEMORY;

   /* use concurrent to avoid explicit queue family ownership transfer for
    * device created with queues from multiple queue families
    */
   const VkBufferCreateInfo buf_create_info = {
      .sType = VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO,
      .size = size,
      /* Feedback for fences and timeline semaphores will write to this buffer
       * as a DST when signalling. Timeline semaphore feedback will also read
       * from this buffer as a SRC to retrieve the counter value to signal.
       */
      .usage =
         VK_BUFFER_USAGE_TRANSFER_SRC_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT,
      .sharingMode =
         exclusive ? VK_SHARING_MODE_EXCLUSIVE : VK_SHARING_MODE_CONCURRENT,
      /* below favors the current venus protocol */
      .queueFamilyIndexCount = exclusive ? 0 : dev->queue_family_count,
      .pQueueFamilyIndices = exclusive ? NULL : dev->queue_families,
   };
   result = vn_CreateBuffer(dev_handle, &buf_create_info, alloc,
                            &fb_buf->buf_handle);
   if (result != VK_SUCCESS)
      goto out_free_feedback_buffer;

   struct vn_buffer *buf = vn_buffer_from_handle(fb_buf->buf_handle);
   const VkMemoryRequirements *mem_req =
      &buf->requirements.memory.memoryRequirements;
   const uint32_t mem_type_index =
      vn_get_memory_type_index(mem_props, mem_req->memoryTypeBits,
                               VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
   if (mem_type_index >= mem_props->memoryTypeCount) {
      result = VK_ERROR_INITIALIZATION_FAILED;
      goto out_destroy_buffer;
   }

   const VkMemoryAllocateInfo mem_alloc_info = {
      .sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO,
      .allocationSize = mem_req->size,
      .memoryTypeIndex = mem_type_index,
   };
   result = vn_AllocateMemory(dev_handle, &mem_alloc_info, alloc,
                              &fb_buf->mem_handle);
   if (result != VK_SUCCESS)
      goto out_destroy_buffer;

   const VkBindBufferMemoryInfo bind_info = {
      .sType = VK_STRUCTURE_TYPE_BIND_BUFFER_MEMORY_INFO,
      .buffer = fb_buf->buf_handle,
      .memory = fb_buf->mem_handle,
      .memoryOffset = 0,
   };
   result = vn_BindBufferMemory2(dev_handle, 1, &bind_info);
   if (result != VK_SUCCESS)
      goto out_free_memory;

   result = vn_MapMemory(dev_handle, fb_buf->mem_handle, 0, VK_WHOLE_SIZE, 0,
                         &fb_buf->data);
   if (result != VK_SUCCESS)
      goto out_free_memory;

   *out_fb_buf = fb_buf;

   return VK_SUCCESS;

out_free_memory:
   vn_FreeMemory(dev_handle, fb_buf->mem_handle, alloc);

out_destroy_buffer:
   vn_DestroyBuffer(dev_handle, fb_buf->buf_handle, alloc);

out_free_feedback_buffer:
   vk_free(alloc, fb_buf);

   return result;
}

void
vn_feedback_buffer_destroy(struct vn_device *dev,
                           struct vn_feedback_buffer *fb_buf,
                           const VkAllocationCallbacks *alloc)
{
   VkDevice dev_handle = vn_device_to_handle(dev);

   vn_UnmapMemory(dev_handle, fb_buf->mem_handle);
   vn_FreeMemory(dev_handle, fb_buf->mem_handle, alloc);
   vn_DestroyBuffer(dev_handle, fb_buf->buf_handle, alloc);
   vk_free(alloc, fb_buf);
}

static inline uint32_t
vn_get_feedback_buffer_alignment(struct vn_device *dev,
                                 struct vn_feedback_buffer *fb_buf)
{
   struct vn_buffer *buf = vn_buffer_from_handle(fb_buf->buf_handle);
   return align(buf->requirements.memory.memoryRequirements.alignment,
                dev->physical_device->wa_min_fb_align);
}

static VkResult
vn_feedback_pool_grow_locked(struct vn_feedback_pool *pool)
{
   VN_TRACE_FUNC();
   struct vn_feedback_buffer *fb_buf = NULL;
   VkResult result;

   result =
      vn_feedback_buffer_create(pool->dev, pool->size, pool->alloc, &fb_buf);
   if (result != VK_SUCCESS)
      return result;

   pool->used = 0;
   pool->alignment = vn_get_feedback_buffer_alignment(pool->dev, fb_buf);

   list_add(&fb_buf->head, &pool->fb_bufs);

   return VK_SUCCESS;
}

VkResult
vn_feedback_pool_init(struct vn_device *dev,
                      struct vn_feedback_pool *pool,
                      uint32_t size,
                      const VkAllocationCallbacks *alloc)
{
   simple_mtx_init(&pool->mutex, mtx_plain);

   pool->dev = dev;
   pool->alloc = alloc;
   pool->size = size;
   pool->used = size;
   pool->alignment = 1;
   list_inithead(&pool->fb_bufs);
   list_inithead(&pool->free_slots);

   return VK_SUCCESS;
}

void
vn_feedback_pool_fini(struct vn_feedback_pool *pool)
{
   list_for_each_entry_safe(struct vn_feedback_slot, slot, &pool->free_slots,
                            head)
      vk_free(pool->alloc, slot);

   list_for_each_entry_safe(struct vn_feedback_buffer, fb_buf, &pool->fb_bufs,
                            head)
      vn_feedback_buffer_destroy(pool->dev, fb_buf, pool->alloc);

   simple_mtx_destroy(&pool->mutex);
}

static struct vn_feedback_buffer *
vn_feedback_pool_alloc_locked(struct vn_feedback_pool *pool,
                              uint32_t size,
                              uint32_t *out_offset)
{
   /* Default values of pool->used and pool->alignment are used to trigger the
    * initial pool grow, and will be properly initialized after that.
    */
   if (unlikely(align(size, pool->alignment) > pool->size - pool->used)) {
      VkResult result = vn_feedback_pool_grow_locked(pool);
      if (result != VK_SUCCESS)
         return NULL;

      assert(align(size, pool->alignment) <= pool->size - pool->used);
   }

   *out_offset = pool->used;
   pool->used += align(size, pool->alignment);

   return list_first_entry(&pool->fb_bufs, struct vn_feedback_buffer, head);
}

struct vn_feedback_slot *
vn_feedback_pool_alloc(struct vn_feedback_pool *pool,
                       enum vn_feedback_type type)
{
   static const uint32_t slot_size = 8;
   struct vn_feedback_buffer *fb_buf;
   uint32_t offset;
   struct vn_feedback_slot *slot;

   simple_mtx_lock(&pool->mutex);
   if (!list_is_empty(&pool->free_slots)) {
      slot =
         list_first_entry(&pool->free_slots, struct vn_feedback_slot, head);
      list_del(&slot->head);
      simple_mtx_unlock(&pool->mutex);

      slot->type = type;
      return slot;
   }

   slot = vk_alloc(pool->alloc, sizeof(*slot), VN_DEFAULT_ALIGN,
                   VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
   if (!slot) {
      simple_mtx_unlock(&pool->mutex);
      return NULL;
   }

   fb_buf = vn_feedback_pool_alloc_locked(pool, slot_size, &offset);
   simple_mtx_unlock(&pool->mutex);

   if (!fb_buf) {
      vk_free(pool->alloc, slot);
      return NULL;
   }

   slot->type = type;
   slot->offset = offset;
   slot->buf_handle = fb_buf->buf_handle;
   slot->data = fb_buf->data + offset;

   return slot;
}

void
vn_feedback_pool_free(struct vn_feedback_pool *pool,
                      struct vn_feedback_slot *slot)
{
   simple_mtx_lock(&pool->mutex);
   list_add(&slot->head, &pool->free_slots);
   simple_mtx_unlock(&pool->mutex);
}

static inline bool
mask_is_32bit(uint64_t x)
{
   return (x & 0xffffffff00000000) == 0;
}

static void
vn_build_buffer_memory_barrier(const VkDependencyInfo *dep_info,
                               VkBufferMemoryBarrier *barrier1,
                               VkPipelineStageFlags *src_stage_mask,
                               VkPipelineStageFlags *dst_stage_mask)
{

   assert(dep_info->pNext == NULL);
   assert(dep_info->memoryBarrierCount == 0);
   assert(dep_info->bufferMemoryBarrierCount == 1);
   assert(dep_info->imageMemoryBarrierCount == 0);

   const VkBufferMemoryBarrier2 *barrier2 =
      &dep_info->pBufferMemoryBarriers[0];
   assert(barrier2->pNext == NULL);
   assert(mask_is_32bit(barrier2->srcStageMask));
   assert(mask_is_32bit(barrier2->srcAccessMask));
   assert(mask_is_32bit(barrier2->dstStageMask));
   assert(mask_is_32bit(barrier2->dstAccessMask));

   *barrier1 = (VkBufferMemoryBarrier){
      .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
      .pNext = NULL,
      .srcAccessMask = barrier2->srcAccessMask,
      .dstAccessMask = barrier2->dstAccessMask,
      .srcQueueFamilyIndex = barrier2->srcQueueFamilyIndex,
      .dstQueueFamilyIndex = barrier2->dstQueueFamilyIndex,
      .buffer = barrier2->buffer,
      .offset = barrier2->offset,
      .size = barrier2->size,
   };

   *src_stage_mask = barrier2->srcStageMask;
   *dst_stage_mask = barrier2->dstStageMask;
}

static void
vn_cmd_buffer_memory_barrier(VkCommandBuffer cmd_handle,
                             const VkDependencyInfo *dep_info,
                             bool sync2)
{
   if (sync2)
      vn_CmdPipelineBarrier2(cmd_handle, dep_info);
   else {
      VkBufferMemoryBarrier barrier1;
      VkPipelineStageFlags src_stage_mask;
      VkPipelineStageFlags dst_stage_mask;

      vn_build_buffer_memory_barrier(dep_info, &barrier1, &src_stage_mask,
                                     &dst_stage_mask);
      vn_CmdPipelineBarrier(cmd_handle, src_stage_mask, dst_stage_mask,
                            dep_info->dependencyFlags, 0, NULL, 1, &barrier1,
                            0, NULL);
   }
}

void
vn_event_feedback_cmd_record(VkCommandBuffer cmd_handle,
                             VkEvent ev_handle,
                             VkPipelineStageFlags2 src_stage_mask,
                             VkResult status,
                             bool sync2)
{
   /* For vkCmdSetEvent and vkCmdResetEvent feedback interception.
    *
    * The injection point is after the event call to avoid introducing
    * unexpected src stage waiting for VK_PIPELINE_STAGE_HOST_BIT and
    * VK_PIPELINE_STAGE_TRANSFER_BIT if they are not already being waited by
    * vkCmdSetEvent or vkCmdResetEvent. On the other hand, the delay in the
    * feedback signal is acceptable for the nature of VkEvent, and the event
    * feedback cmds lifecycle is guarded by the intercepted command buffer.
    */
   struct vn_event *ev = vn_event_from_handle(ev_handle);
   struct vn_feedback_slot *slot = ev->feedback_slot;

   if (!slot)
      return;

   STATIC_ASSERT(sizeof(*slot->status) == 4);

   const VkDependencyInfo dep_before = {
      .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
      .dependencyFlags = 0,
      .bufferMemoryBarrierCount = 1,
      .pBufferMemoryBarriers =
         (VkBufferMemoryBarrier2[]){
            {
               .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2,
               .srcStageMask = src_stage_mask | VK_PIPELINE_STAGE_HOST_BIT |
                               VK_PIPELINE_STAGE_TRANSFER_BIT,
               .srcAccessMask =
                  VK_ACCESS_HOST_WRITE_BIT | VK_ACCESS_TRANSFER_WRITE_BIT,
               .dstStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT,
               .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
               .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
               .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
               .buffer = slot->buf_handle,
               .offset = slot->offset,
               .size = 4,
            },
         },
   };
   vn_cmd_buffer_memory_barrier(cmd_handle, &dep_before, sync2);

   vn_CmdFillBuffer(cmd_handle, slot->buf_handle, slot->offset, 4, status);

   const VkDependencyInfo dep_after = {
      .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO,
      .dependencyFlags = 0,
      .bufferMemoryBarrierCount = 1,
      .pBufferMemoryBarriers =
         (VkBufferMemoryBarrier2[]){
            {
               .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2,
               .srcStageMask = VK_PIPELINE_STAGE_TRANSFER_BIT,
               .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
               .dstStageMask = VK_PIPELINE_STAGE_HOST_BIT,
               .dstAccessMask =
                  VK_ACCESS_HOST_READ_BIT | VK_ACCESS_HOST_WRITE_BIT,
               .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
               .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
               .buffer = slot->buf_handle,
               .offset = slot->offset,
               .size = 4,
            },
         },
   };
   vn_cmd_buffer_memory_barrier(cmd_handle, &dep_after, sync2);
}

static inline void
vn_feedback_cmd_record_flush_barrier(VkCommandBuffer cmd_handle,
                                     VkBuffer buffer,
                                     VkDeviceSize offset,
                                     VkDeviceSize size)
{
   const VkBufferMemoryBarrier buf_flush_barrier = {
      .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
      .pNext = NULL,
      .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
      .dstAccessMask = VK_ACCESS_HOST_READ_BIT | VK_ACCESS_HOST_WRITE_BIT,
      .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
      .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
      .buffer = buffer,
      .offset = offset,
      .size = size,
   };
   vn_CmdPipelineBarrier(cmd_handle, VK_PIPELINE_STAGE_TRANSFER_BIT,
                         VK_PIPELINE_STAGE_HOST_BIT, 0, 0, NULL, 1,
                         &buf_flush_barrier, 0, NULL);
}

static VkResult
vn_feedback_cmd_record(VkCommandBuffer cmd_handle,
                       struct vn_feedback_slot *dst_slot,
                       struct vn_feedback_slot *src_slot)
{
   STATIC_ASSERT(sizeof(*dst_slot->status) == 4);
   STATIC_ASSERT(sizeof(*dst_slot->counter) == 8);
   STATIC_ASSERT(sizeof(*src_slot->counter) == 8);

   /* slot size is 8 bytes for timeline semaphore and 4 bytes fence.
    * src slot is non-null for timeline semaphore.
    */
   const VkDeviceSize buf_size = src_slot ? 8 : 4;

   static const VkCommandBufferBeginInfo begin_info = {
      .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
      .pNext = NULL,
      .flags = 0,
      .pInheritanceInfo = NULL,
   };
   VkResult result = vn_BeginCommandBuffer(cmd_handle, &begin_info);
   if (result != VK_SUCCESS)
      return result;

   static const VkMemoryBarrier mem_barrier_before = {
      .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER,
      .pNext = NULL,
      /* make pending writes available to stay close to signal op */
      .srcAccessMask = VK_ACCESS_MEMORY_WRITE_BIT,
      /* no need to make all memory visible for feedback update */
      .dstAccessMask = 0,
   };

   const VkBufferMemoryBarrier buf_barrier_before = {
      .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
      .pNext = NULL,
      /* slot memory has been made available via mem_barrier_before */
      .srcAccessMask = 0,
      .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
      .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
      .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
      .buffer = dst_slot->buf_handle,
      .offset = dst_slot->offset,
      .size = buf_size,
   };

   /* host writes for src_slots should implicitly be made visible upon
    * QueueSubmit call */
   vn_CmdPipelineBarrier(cmd_handle, VK_PIPELINE_STAGE_ALL_COMMANDS_BIT,
                         VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 1,
                         &mem_barrier_before, 1, &buf_barrier_before, 0,
                         NULL);

   /* If passed a src_slot, timeline semaphore feedback records a
    * cmd to copy the counter value from the src slot to the dst slot.
    * If src_slot is NULL, then fence feedback records a cmd to fill
    * the dst slot with VK_SUCCESS.
    */
   if (src_slot) {
      assert(src_slot->type == VN_FEEDBACK_TYPE_SEMAPHORE);
      assert(dst_slot->type == VN_FEEDBACK_TYPE_SEMAPHORE);

      const VkBufferCopy buffer_copy = {
         .srcOffset = src_slot->offset,
         .dstOffset = dst_slot->offset,
         .size = buf_size,
      };
      vn_CmdCopyBuffer(cmd_handle, src_slot->buf_handle, dst_slot->buf_handle,
                       1, &buffer_copy);
   } else {
      assert(dst_slot->type == VN_FEEDBACK_TYPE_FENCE);

      vn_CmdFillBuffer(cmd_handle, dst_slot->buf_handle, dst_slot->offset,
                       buf_size, VK_SUCCESS);
   }

   vn_feedback_cmd_record_flush_barrier(cmd_handle, dst_slot->buf_handle,
                                        dst_slot->offset, buf_size);

   return vn_EndCommandBuffer(cmd_handle);
}

struct vn_semaphore_feedback_cmd *
vn_semaphore_feedback_cmd_alloc(struct vn_device *dev,
                                struct vn_feedback_slot *dst_slot)
{
   const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
   struct vn_semaphore_feedback_cmd *sfb_cmd;
   VkCommandBuffer *cmd_handles;

   VK_MULTIALLOC(ma);
   vk_multialloc_add(&ma, &sfb_cmd, __typeof__(*sfb_cmd), 1);
   vk_multialloc_add(&ma, &cmd_handles, __typeof__(*cmd_handles),
                     dev->queue_family_count);
   if (!vk_multialloc_zalloc(&ma, alloc, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT))
      return NULL;

   struct vn_feedback_slot *src_slot =
      vn_feedback_pool_alloc(&dev->feedback_pool, VN_FEEDBACK_TYPE_SEMAPHORE);
   if (!src_slot) {
      vk_free(alloc, sfb_cmd);
      return NULL;
   }

   for (uint32_t i = 0; i < dev->queue_family_count; i++) {
      VkDevice dev_handle = vn_device_to_handle(dev);
      VkResult result =
         vn_feedback_cmd_alloc(dev_handle, &dev->fb_cmd_pools[i], dst_slot,
                               src_slot, &cmd_handles[i]);
      if (result != VK_SUCCESS) {
         for (uint32_t j = 0; j < i; j++) {
            vn_feedback_cmd_free(dev_handle, &dev->fb_cmd_pools[j],
                                 cmd_handles[j]);
         }

         vn_feedback_pool_free(&dev->feedback_pool, src_slot);
         vk_free(alloc, sfb_cmd);
         return NULL;
      }
   }

   sfb_cmd->cmd_handles = cmd_handles;
   sfb_cmd->src_slot = src_slot;
   return sfb_cmd;
}

void
vn_semaphore_feedback_cmd_free(struct vn_device *dev,
                               struct vn_semaphore_feedback_cmd *sfb_cmd)
{
   const VkAllocationCallbacks *alloc = &dev->base.base.alloc;

   for (uint32_t i = 0; i < dev->queue_family_count; i++) {
      vn_feedback_cmd_free(vn_device_to_handle(dev), &dev->fb_cmd_pools[i],
                           sfb_cmd->cmd_handles[i]);
   }

   vn_feedback_pool_free(&dev->feedback_pool, sfb_cmd->src_slot);
   vk_free(alloc, sfb_cmd);
}

static void
vn_query_feedback_cmd_record_internal(VkCommandBuffer cmd_handle,
                                      VkQueryPool pool_handle,
                                      uint32_t query,
                                      uint32_t count,
                                      bool copy)
{
   struct vn_query_pool *pool = vn_query_pool_from_handle(pool_handle);
   assert(pool->fb_buf);

   /* Results are always 64 bit and include availability bit (also 64 bit) */
   const VkDeviceSize slot_size = (pool->result_array_size * 8) + 8;
   const VkDeviceSize offset = slot_size * query;
   const VkDeviceSize buf_size = slot_size * count;

   /* The first synchronization scope of vkCmdCopyQueryPoolResults does not
    * include the query feedback buffer. Insert a barrier to ensure ordering
    * against feedback buffer fill cmd injected in vkCmdResetQueryPool.
    *
    * The second synchronization scope of vkCmdResetQueryPool does not include
    * the query feedback buffer. Insert a barrer to ensure ordering against
    * prior cmds referencing the queries.
    *
    * For srcAccessMask, VK_ACCESS_TRANSFER_WRITE_BIT is sufficient since the
    * gpu cache invalidation for feedback buffer fill in vkResetQueryPool is
    * done implicitly via queue submission.
    */
   const VkPipelineStageFlags src_stage_mask =
      copy ? VK_PIPELINE_STAGE_TRANSFER_BIT
           : VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;

   const VkBufferMemoryBarrier buf_barrier_before = {
      .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER,
      .pNext = NULL,
      .srcAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
      .dstAccessMask = VK_ACCESS_TRANSFER_WRITE_BIT,
      .srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
      .dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED,
      .buffer = pool->fb_buf->buf_handle,
      .offset = offset,
      .size = buf_size,
   };
   vn_CmdPipelineBarrier(cmd_handle, src_stage_mask,
                         VK_PIPELINE_STAGE_TRANSFER_BIT, 0, 0, NULL, 1,
                         &buf_barrier_before, 0, NULL);

   if (copy) {
      /* Per spec: "The first synchronization scope includes all commands
       * which reference the queries in queryPool indicated by query that
       * occur earlier in submission order. If flags does not include
       * VK_QUERY_RESULT_WAIT_BIT, vkCmdEndQueryIndexedEXT,
       * vkCmdWriteTimestamp2, vkCmdEndQuery, and vkCmdWriteTimestamp are
       * excluded from this scope."
       *
       * Set VK_QUERY_RESULT_WAIT_BIT to ensure ordering after
       * vkCmdEndQuery or vkCmdWriteTimestamp makes the query available.
       *
       * Set VK_QUERY_RESULT_64_BIT as we can convert it to 32 bit if app
       * requested that.
       *
       * Per spec: "vkCmdCopyQueryPoolResults is considered to be a transfer
       * operation, and its writes to buffer memory must be synchronized using
       * VK_PIPELINE_STAGE_TRANSFER_BIT and VK_ACCESS_TRANSFER_WRITE_BIT
       * before using the results."
       *
       * So we can reuse the flush barrier after this copy cmd.
       */
      vn_CmdCopyQueryPoolResults(cmd_handle, pool_handle, query, count,
                                 pool->fb_buf->buf_handle, offset, slot_size,
                                 VK_QUERY_RESULT_WITH_AVAILABILITY_BIT |
                                    VK_QUERY_RESULT_64_BIT |
                                    VK_QUERY_RESULT_WAIT_BIT);
   } else {
      vn_CmdFillBuffer(cmd_handle, pool->fb_buf->buf_handle, offset, buf_size,
                       0);
   }

   vn_feedback_cmd_record_flush_barrier(cmd_handle, pool->fb_buf->buf_handle,
                                        offset, buf_size);
}

static VkResult
vn_query_feedback_cmd_record(VkDevice dev_handle,
                             struct list_head *query_records,
                             struct vn_query_feedback_cmd *qfb_cmd)
{
   assert(!list_is_empty(query_records));

   static const VkCommandBufferBeginInfo begin_info = {
      .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO,
   };
   VkResult result = vn_BeginCommandBuffer(qfb_cmd->cmd_handle, &begin_info);
   if (result != VK_SUCCESS)
      return result;

   list_for_each_entry_safe(struct vn_cmd_query_record, record, query_records,
                            head) {
      vn_query_feedback_cmd_record_internal(
         qfb_cmd->cmd_handle, vn_query_pool_to_handle(record->query_pool),
         record->query, record->query_count, record->copy);
   }

   return vn_EndCommandBuffer(qfb_cmd->cmd_handle);
}

VkResult
vn_query_feedback_cmd_alloc(VkDevice dev_handle,
                            struct vn_feedback_cmd_pool *fb_cmd_pool,
                            struct list_head *query_records,
                            struct vn_query_feedback_cmd **out_qfb_cmd)
{
   struct vn_query_feedback_cmd *qfb_cmd;
   VkResult result;

   simple_mtx_lock(&fb_cmd_pool->mutex);

   if (list_is_empty(&fb_cmd_pool->free_qfb_cmds)) {
      struct vn_command_pool *cmd_pool =
         vn_command_pool_from_handle(fb_cmd_pool->pool_handle);

      qfb_cmd = vk_alloc(&cmd_pool->allocator, sizeof(*qfb_cmd),
                         VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
      if (!qfb_cmd) {
         result = VK_ERROR_OUT_OF_HOST_MEMORY;
         goto out_unlock;
      }

      const VkCommandBufferAllocateInfo info = {
         .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
         .commandPool = fb_cmd_pool->pool_handle,
         .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
         .commandBufferCount = 1,
      };
      VkCommandBuffer qfb_cmd_handle;
      result = vn_AllocateCommandBuffers(dev_handle, &info, &qfb_cmd_handle);
      if (result != VK_SUCCESS) {
         vk_free(&cmd_pool->allocator, qfb_cmd);
         goto out_unlock;
      }

      qfb_cmd->fb_cmd_pool = fb_cmd_pool;
      qfb_cmd->cmd_handle = qfb_cmd_handle;
   } else {
      qfb_cmd = list_first_entry(&fb_cmd_pool->free_qfb_cmds,
                                 struct vn_query_feedback_cmd, head);
      list_del(&qfb_cmd->head);
      vn_ResetCommandBuffer(qfb_cmd->cmd_handle, 0);
   }

   result = vn_query_feedback_cmd_record(dev_handle, query_records, qfb_cmd);
   if (result != VK_SUCCESS) {
      list_add(&qfb_cmd->head, &fb_cmd_pool->free_qfb_cmds);
      goto out_unlock;
   }

   *out_qfb_cmd = qfb_cmd;

out_unlock:
   simple_mtx_unlock(&fb_cmd_pool->mutex);

   return result;
}

void
vn_query_feedback_cmd_free(struct vn_query_feedback_cmd *qfb_cmd)
{
   simple_mtx_lock(&qfb_cmd->fb_cmd_pool->mutex);
   list_add(&qfb_cmd->head, &qfb_cmd->fb_cmd_pool->free_qfb_cmds);
   simple_mtx_unlock(&qfb_cmd->fb_cmd_pool->mutex);
}

VkResult
vn_feedback_cmd_alloc(VkDevice dev_handle,
                      struct vn_feedback_cmd_pool *fb_cmd_pool,
                      struct vn_feedback_slot *dst_slot,
                      struct vn_feedback_slot *src_slot,
                      VkCommandBuffer *out_cmd_handle)
{
   VkCommandPool cmd_pool_handle = fb_cmd_pool->pool_handle;
   const VkCommandBufferAllocateInfo info = {
      .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
      .pNext = NULL,
      .commandPool = cmd_pool_handle,
      .level = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
      .commandBufferCount = 1,
   };
   VkCommandBuffer cmd_handle;
   VkResult result;

   simple_mtx_lock(&fb_cmd_pool->mutex);
   result = vn_AllocateCommandBuffers(dev_handle, &info, &cmd_handle);
   if (result != VK_SUCCESS)
      goto out_unlock;

   result = vn_feedback_cmd_record(cmd_handle, dst_slot, src_slot);
   if (result != VK_SUCCESS) {
      vn_FreeCommandBuffers(dev_handle, cmd_pool_handle, 1, &cmd_handle);
      goto out_unlock;
   }

   *out_cmd_handle = cmd_handle;

out_unlock:
   simple_mtx_unlock(&fb_cmd_pool->mutex);

   return result;
}

void
vn_feedback_cmd_free(VkDevice dev_handle,
                     struct vn_feedback_cmd_pool *fb_cmd_pool,
                     VkCommandBuffer cmd_handle)
{
   simple_mtx_lock(&fb_cmd_pool->mutex);
   vn_FreeCommandBuffers(dev_handle, fb_cmd_pool->pool_handle, 1,
                         &cmd_handle);
   simple_mtx_unlock(&fb_cmd_pool->mutex);
}

VkResult
vn_feedback_cmd_pools_init(struct vn_device *dev)
{
   const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
   VkDevice dev_handle = vn_device_to_handle(dev);
   struct vn_feedback_cmd_pool *fb_cmd_pools;
   VkCommandPoolCreateInfo info = {
      .sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO,
      .pNext = NULL,
      .flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT,
   };

   if (VN_PERF(NO_FENCE_FEEDBACK) && VN_PERF(NO_SEMAPHORE_FEEDBACK) &&
       VN_PERF(NO_QUERY_FEEDBACK))
      return VK_SUCCESS;

   assert(dev->queue_family_count);

   fb_cmd_pools =
      vk_zalloc(alloc, sizeof(*fb_cmd_pools) * dev->queue_family_count,
                VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_DEVICE);
   if (!fb_cmd_pools)
      return VK_ERROR_OUT_OF_HOST_MEMORY;

   for (uint32_t i = 0; i < dev->queue_family_count; i++) {
      VkResult result;

      info.queueFamilyIndex = dev->queue_families[i];
      result = vn_CreateCommandPool(dev_handle, &info, alloc,
                                    &fb_cmd_pools[i].pool_handle);
      if (result != VK_SUCCESS) {
         for (uint32_t j = 0; j < i; j++) {
            vn_DestroyCommandPool(dev_handle, fb_cmd_pools[j].pool_handle,
                                  alloc);
            simple_mtx_destroy(&fb_cmd_pools[j].mutex);
         }

         vk_free(alloc, fb_cmd_pools);
         return result;
      }

      simple_mtx_init(&fb_cmd_pools[i].mutex, mtx_plain);
      list_inithead(&fb_cmd_pools[i].free_qfb_cmds);
   }

   dev->fb_cmd_pools = fb_cmd_pools;

   return VK_SUCCESS;
}

void
vn_feedback_cmd_pools_fini(struct vn_device *dev)
{
   const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
   VkDevice dev_handle = vn_device_to_handle(dev);

   if (!dev->fb_cmd_pools)
      return;

   for (uint32_t i = 0; i < dev->queue_family_count; i++) {
      list_for_each_entry_safe(struct vn_query_feedback_cmd, feedback_cmd,
                               &dev->fb_cmd_pools[i].free_qfb_cmds, head)
         vk_free(alloc, feedback_cmd);

      vn_DestroyCommandPool(dev_handle, dev->fb_cmd_pools[i].pool_handle,
                            alloc);
      simple_mtx_destroy(&dev->fb_cmd_pools[i].mutex);
   }

   vk_free(alloc, dev->fb_cmd_pools);
}