intel/vulkan/anv_rmv.c

/*
 * Copyright © 2023 Intel Corporation
 *
 * Permission is hereby granted, free of charge, to any person obtaining a
 * copy of this software and associated documentation files (the "Software"),
 * to deal in the Software without restriction, including without limitation
 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
 * and/or sell copies of the Software, and to permit persons to whom the
 * Software is furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice (including the next
 * paragraph) shall be included in all copies or substantial portions of the
 * Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 * IN THE SOFTWARE.
 */

#include <errno.h>
#include <fcntl.h>
#include <stdio.h>
#include <stdlib.h>
#include "anv_private.h"

static VkResult
capture_trace(VkQueue _queue)
{
   ANV_FROM_HANDLE(anv_queue, queue, _queue);

   simple_mtx_lock(&queue->device->vk.memory_trace_data.token_mtx);
   vk_dump_rmv_capture(&queue->device->vk.memory_trace_data);
   simple_mtx_unlock(&queue->device->vk.memory_trace_data.token_mtx);

   return VK_SUCCESS;
}

void
anv_memory_trace_init(struct anv_device *device)
{
   struct vk_rmv_device_info info;
   memset(&info, 0, sizeof(info));
   anv_rmv_fill_device_info(device->physical, &info);
   vk_memory_trace_init(&device->vk, &info);

   if (!device->vk.memory_trace_data.is_enabled)
      return;

   device->vk.capture_trace = capture_trace;
}

static void
fill_memory_info(const struct anv_physical_device *device,
                 struct vk_rmv_memory_info *out_info,
                 int32_t index)
{
   switch (index) {
   case VK_RMV_MEMORY_LOCATION_DEVICE:
      out_info->physical_base_address = 0;
      out_info->size = device->memory.heaps[0].size;
      break;
   case VK_RMV_MEMORY_LOCATION_DEVICE_INVISIBLE:
      out_info->physical_base_address = device->memory.heaps[0].size;
      out_info->size = device->vram_non_mappable.size;
      break;
   case VK_RMV_MEMORY_LOCATION_HOST:
      out_info->physical_base_address = 0;
      out_info->size = device->memory.heaps[1].size;
      break;
   default:
      unreachable("invalid memory index");
   }
}

void
anv_rmv_fill_device_info(const struct anv_physical_device *device,
                         struct vk_rmv_device_info *info)
{
   for (int32_t i = 0; i < VK_RMV_MEMORY_LOCATION_COUNT; ++i)
      fill_memory_info(device, &info->memory_infos[i], i);

   strncpy(info->device_name, device->info.name, sizeof(info->device_name) - 1);
   info->pcie_revision_id = device->info.pci_revision_id;
   info->pcie_device_id = device->info.pci_device_id;
   /* TODO: */
   info->pcie_family_id = 0;
   info->minimum_shader_clock = 0;
   info->maximum_shader_clock = 1 * 1024 * 1024 * 1024;
   info->vram_type = VK_RMV_MEMORY_TYPE_DDR4;
   info->vram_bus_width = 256;
   info->vram_operations_per_clock = 1;
   info->minimum_memory_clock = 0;
   info->maximum_memory_clock = 1;
   info->vram_bandwidth = 256;
}

void
anv_memory_trace_finish(struct anv_device *device)
{
}

static uint32_t
resource_id_locked(struct anv_device *device, const void *obj)
{
   return vk_rmv_get_resource_id_locked(&device->vk, (uint64_t)(uintptr_t)obj);
}

static void
resource_destroy_locked(struct anv_device *device, const void *obj)
{
   vk_rmv_destroy_resource_id_locked(&device->vk, (uint64_t)(uintptr_t)obj);
}

/* The token lock must be held when entering _locked functions */
static void
log_resource_bind_locked(struct anv_device *device, uint64_t resource_id,
                         struct anv_bo *bo, uint64_t offset,
                         uint64_t size)
{
   struct vk_rmv_resource_bind_token token = {
      .resource_id      = resource_id,
      .is_system_memory = bo ? (bo->alloc_flags & ANV_BO_ALLOC_NO_LOCAL_MEM) : 0,
      .address          = (bo ? bo->offset : 0) + offset,
      .size             = size,
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_BIND, &token);
}

static void
log_state_pool_bind_locked(struct anv_device *device, uint64_t resource_id,
                           struct anv_state_pool *pool, struct anv_state *state)
{
   struct vk_rmv_resource_bind_token token = {
      .resource_id      = resource_id,
      .is_system_memory = (pool->block_pool.bo_alloc_flags &
                           ANV_BO_ALLOC_NO_LOCAL_MEM) != 0,
      .address          = anv_address_physical(
         anv_state_pool_state_address(pool, *state)),
      .size             = state->alloc_size,
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_BIND, &token);
}

static enum vk_rmv_memory_location
anv_heap_index_to_memory_location(struct anv_device *device,
                                  unsigned heap_index)
{
   if (heap_index == 0)
      return device->physical->vram_non_mappable.size != 0 ?
             VK_RMV_MEMORY_LOCATION_DEVICE_INVISIBLE :
             VK_RMV_MEMORY_LOCATION_DEVICE;
   else if (heap_index == 1)
      return VK_RMV_MEMORY_LOCATION_HOST;
   else
      return VK_RMV_MEMORY_LOCATION_DEVICE;
}

static void
anv_rmv_log_bo_gtt_unmap_locked(struct anv_device *device,
                                struct anv_bo *bo)
{
   if (!bo->gtt_mapped)
      return;

   struct vk_rmv_token token = {
      .type      = VK_RMV_TOKEN_TYPE_PAGE_TABLE_UPDATE,
      .timestamp = (uint64_t)os_time_get_nano(),
      .data      = {
         .page_table_update = {
            .type             = VK_RMV_PAGE_TABLE_UPDATE_TYPE_UPDATE,
            .page_size        = device->info->mem_alignment,
            .page_count       = DIV_ROUND_UP(bo->size,
                                             device->info->mem_alignment),
            .pid              = getpid(),
            .virtual_address  = bo->offset,
            .physical_address = bo->offset,
            .is_unmap         = true,
         },
      },
   };
   util_dynarray_append(&device->vk.memory_trace_data.tokens,
                        struct vk_rmv_token, token);

   bo->gtt_mapped = false;
}

void
anv_rmv_log_bo_gtt_unmap(struct anv_device *device,
                         struct anv_bo *bo)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   anv_rmv_log_bo_gtt_unmap_locked(device, bo);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_bo_gtt_map(struct anv_device *device,
                       struct anv_bo *bo)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_token token = {
      .type      = VK_RMV_TOKEN_TYPE_PAGE_TABLE_UPDATE,
      .timestamp = (uint64_t)os_time_get_nano(),
      .data      = {
         .page_table_update = {
            .type             = VK_RMV_PAGE_TABLE_UPDATE_TYPE_UPDATE,
            .page_size        = device->info->mem_alignment,
            .page_count       = DIV_ROUND_UP(bo->size,
                                                device->info->mem_alignment),
            .pid              = getpid(),
            .virtual_address  = bo->offset,
            .physical_address = bo->offset,
            .is_unmap         = false,
            },
      },
   };
   util_dynarray_append(&device->vk.memory_trace_data.tokens,
                        struct vk_rmv_token, token);

   bo->gtt_mapped = true;

   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_bos_gtt_map(struct anv_device *device,
                        struct anv_bo **bos,
                        uint32_t bo_count)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   for (uint32_t i = 0; i < bo_count; i++) {
      struct anv_bo *bo = bos[i];

      if (bo->gtt_mapped)
         continue;

      struct vk_rmv_token token = {
         .type      = VK_RMV_TOKEN_TYPE_PAGE_TABLE_UPDATE,
         .timestamp = (uint64_t)os_time_get_nano(),
         .data      = {
            .page_table_update = {
               .type             = VK_RMV_PAGE_TABLE_UPDATE_TYPE_UPDATE,
               .page_size        = device->info->mem_alignment,
               .page_count       = DIV_ROUND_UP(bo->size,
                                                device->info->mem_alignment),
               .pid              = getpid(),
               .virtual_address  = bo->offset,
               .physical_address = bo->offset,
               .is_unmap         = false,
            },
         },
      };
      util_dynarray_append(&device->vk.memory_trace_data.tokens,
                           struct vk_rmv_token, token);

      bo->gtt_mapped = true;
   }
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_vm_binds(struct anv_device *device,
                     struct anv_vm_bind *binds,
                     uint32_t bind_count)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   for (uint32_t i = 0; i < bind_count; i++) {

      struct vk_rmv_token token = {
         .type      = VK_RMV_TOKEN_TYPE_PAGE_TABLE_UPDATE,
         .timestamp = (uint64_t)os_time_get_nano(),
         .data      = {
            .page_table_update = {
               .type             = VK_RMV_PAGE_TABLE_UPDATE_TYPE_UPDATE,
               .page_size        = device->info->mem_alignment,
               .page_count       = DIV_ROUND_UP(binds[i].size,
                                                device->info->mem_alignment),
               .pid              = getpid(),
               .virtual_address  = binds[i].address,
               .physical_address = binds[i].bo_offset,
               .is_unmap         = binds[i].op == ANV_VM_UNBIND,
            },
         },
      };
      util_dynarray_append(&device->vk.memory_trace_data.tokens,
                           struct vk_rmv_token, token);
   }
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_heap_create(struct anv_device *device,
                        struct anv_device_memory *memory,
                        bool is_internal,
                        VkMemoryAllocateFlags alloc_flags)
{
   /* Do not log zero-sized device memory objects. */
   if (!memory->vk.size)
      return;

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);

   struct vk_rmv_resource_create_token token = {
      .type               = VK_RMV_RESOURCE_TYPE_HEAP,
      .resource_id        = resource_id_locked(device, memory),
      .is_driver_internal = is_internal,
      .heap = {
         .alignment   = device->info->mem_alignment,
         .size        = memory->vk.size,
         .heap_index  = anv_heap_index_to_memory_location(device,
                                                          memory->type->heapIndex),
         .alloc_flags = alloc_flags,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &token);
   log_resource_bind_locked(device, token.resource_id, memory->bo, 0, memory->vk.size);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

static void
anv_rmv_log_vma_locked(struct anv_device *device, uint64_t address, uint64_t size,
                       bool internal, bool vram, bool in_invisible_vram)
{
   struct vk_rmv_virtual_allocate_token token = {
      .address              = address,
      /* If all VRAM is visible, no bo will be in invisible memory. */
      .is_in_invisible_vram = in_invisible_vram,
      .preferred_domains    = (vram ?
                               VK_RMV_KERNEL_MEMORY_DOMAIN_VRAM :
                               VK_RMV_KERNEL_MEMORY_DOMAIN_GTT),
      .is_driver_internal   = internal,
      .page_count           = DIV_ROUND_UP(size, 4096),
   };


   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_VIRTUAL_ALLOCATE, &token);
}

void
anv_rmv_log_bo_allocate(struct anv_device *device,
                        struct anv_bo *bo)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   anv_rmv_log_vma_locked(device, bo->offset, bo->size,
                          bo->alloc_flags & ANV_BO_ALLOC_INTERNAL,
                          (bo->alloc_flags & ANV_BO_ALLOC_NO_LOCAL_MEM) == 0,
                          device->physical->vram_non_mappable.size != 0 &&
                          (bo->alloc_flags & (ANV_BO_ALLOC_MAPPED |
                                              ANV_BO_ALLOC_HOST_CACHED_COHERENT |
                                              ANV_BO_ALLOC_LOCAL_MEM_CPU_VISIBLE |
                                              ANV_BO_ALLOC_NO_LOCAL_MEM)) == 0);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);

   if (bo->alloc_flags & ANV_BO_ALLOC_MAPPED)
      vk_rmv_log_cpu_map(&device->vk, bo->offset, false);
}

void
anv_rmv_log_bo_destroy(struct anv_device *device, struct anv_bo *bo)
{
   struct vk_rmv_virtual_free_token token = {
      .address = bo->offset,
   };

   if (bo->alloc_flags & ANV_BO_ALLOC_MAPPED)
      vk_rmv_log_cpu_map(&device->vk, bo->offset, true);
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   anv_rmv_log_bo_gtt_unmap_locked(device, bo);
   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_VIRTUAL_FREE, &token);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_buffer_create(struct anv_device *device,
                          bool is_internal,
                          struct anv_buffer *buffer)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token token = {
      .type               = VK_RMV_RESOURCE_TYPE_BUFFER,
      .is_driver_internal = is_internal,
      .resource_id        = resource_id_locked(device, buffer),
      .buffer             = {
         .create_flags = buffer->vk.create_flags,
         .size         = buffer->vk.size,
         .usage_flags  = buffer->vk.usage,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &token);
   if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
      assert(buffer->sparse_data.size != 0);
      anv_rmv_log_vma_locked(device,
                             buffer->sparse_data.address,
                             buffer->sparse_data.size,
                             false /* internal */, true /* TODO: vram */,
                             true /* in_invisible_vram */);
      log_resource_bind_locked(device,
                               resource_id_locked(device, buffer),
                               NULL,
                               buffer->sparse_data.address,
                               buffer->sparse_data.size);
   }
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);

}

void
anv_rmv_log_buffer_destroy(struct anv_device *device,
                           struct anv_buffer *buffer)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   if (buffer->vk.create_flags & VK_BUFFER_CREATE_SPARSE_BINDING_BIT) {
      struct vk_rmv_virtual_free_token token = {
         .address = buffer->sparse_data.address,
      };
      vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_VIRTUAL_FREE, &token);
   }
   resource_destroy_locked(device, buffer);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);

}

void
anv_rmv_log_buffer_bind(struct anv_device *device, struct anv_buffer *buffer)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   log_resource_bind_locked(device,
                            resource_id_locked(device, buffer),
                            buffer->address.bo,
                            buffer->address.offset, buffer->vk.size);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_image_create(struct anv_device *device,
                         bool is_internal,
                         struct anv_image *image)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token token = {
      .type               = VK_RMV_RESOURCE_TYPE_IMAGE,
      .resource_id        = resource_id_locked(device, image),
      .is_driver_internal = is_internal,
      .image              = {
         .create_flags            = image->vk.create_flags,
         .usage_flags             = image->vk.usage,
         .type                    = image->vk.image_type,
         .extent                  = image->vk.extent,
         .format                  = image->vk.format,
         .num_mips                = image->vk.mip_levels,
         .num_slices              = image->vk.array_layers,
         .tiling                  = image->vk.tiling,
         .alignment_log2          = util_logbase2(
            image->bindings[ANV_IMAGE_MEMORY_BINDING_MAIN].memory_range.alignment),
         .log2_samples            = util_logbase2(image->vk.samples),
         .metadata_alignment_log2 = util_logbase2(
            image->planes[0].aux_surface.isl.alignment_B),
         .image_alignment_log2    = util_logbase2(
            image->planes[0].primary_surface.isl.alignment_B),
         .size                    = image->planes[0].primary_surface.memory_range.size,
         .metadata_size           = image->planes[0].aux_surface.memory_range.size,
         .metadata_header_size    = 0,
         .metadata_offset         = image->planes[0].aux_surface.memory_range.offset,
         .metadata_header_offset  = image->planes[0].aux_surface.memory_range.offset,
         .presentable             = (image->planes[0].primary_surface.isl.usage &
                                     ISL_SURF_USAGE_DISPLAY_BIT) != 0,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &token);
   if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) {
      for (uint32_t b = 0; b < ARRAY_SIZE(image->bindings); b++) {
         if (image->bindings[b].sparse_data.size != 0) {
            anv_rmv_log_vma_locked(device,
                                   image->bindings[b].sparse_data.address,
                                   image->bindings[b].sparse_data.size,
                                   false /* internal */, true /* TODO: vram */,
                                   true /* in_invisible_vram */);
            log_resource_bind_locked(device,
                                     resource_id_locked(device, image),
                                     NULL,
                                     image->bindings[b].sparse_data.address,
                                     image->bindings[b].sparse_data.size);
         }
      }
   }
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_image_destroy(struct anv_device *device,
                          struct anv_image *image)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   if (image->vk.create_flags & VK_IMAGE_CREATE_SPARSE_BINDING_BIT) {
      for (uint32_t b = 0; b < ARRAY_SIZE(image->bindings); b++) {
         if (image->bindings[b].sparse_data.size != 0) {
            struct vk_rmv_virtual_free_token token = {
               .address = image->bindings[b].sparse_data.address,
            };

            vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_VIRTUAL_FREE, &token);
         }
      }
   }
   resource_destroy_locked(device, image);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_image_bind(struct anv_device *device,
                       struct anv_image *image,
                       enum anv_image_memory_binding binding)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   log_resource_bind_locked(device,
                            resource_id_locked(device, image),
                            image->bindings[binding].address.bo,
                            image->bindings[binding].address.offset,
                            image->bindings[binding].memory_range.size);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_query_pool_create(struct anv_device *device,
                              struct anv_query_pool *pool,
                              bool is_internal)
{
   if (pool->vk.query_type != VK_QUERY_TYPE_OCCLUSION &&
       pool->vk.query_type != VK_QUERY_TYPE_PIPELINE_STATISTICS &&
       pool->vk.query_type != VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT)
      return;

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token create_token = {
      .type               = VK_RMV_RESOURCE_TYPE_QUERY_HEAP,
      .resource_id        = resource_id_locked(device, pool),
      .is_driver_internal = is_internal,
      .query_pool = {
         .type           = pool->vk.query_type,
         .has_cpu_access = true,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data,
                     VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &create_token);
   log_resource_bind_locked(device, create_token.resource_id,
                            pool->bo, 0, pool->bo->size);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

static void
bind_cmd_buffer_state_stream_locked(struct anv_device *device,
                                    uint64_t resource_id,
                                    struct anv_state_stream *stream)
{
   util_dynarray_foreach(&stream->all_blocks, struct anv_state, block)
      log_state_pool_bind_locked(device, resource_id, stream->state_pool, block);
}

void
anv_rmv_log_cmd_buffer_create(struct anv_device *device,
                              struct anv_cmd_buffer *cmd_buffer)
{
   uint64_t data_size =
      cmd_buffer->surface_state_stream.total_size +
      cmd_buffer->dynamic_state_stream.total_size +
      cmd_buffer->general_state_stream.total_size +
      cmd_buffer->indirect_push_descriptor_stream.total_size;

   uint64_t executable_size = 0;
   list_for_each_entry(struct anv_batch_bo, bbo, &cmd_buffer->batch_bos, link)
      executable_size += bbo->length;

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token create_token = {
      .type               = VK_RMV_RESOURCE_TYPE_COMMAND_ALLOCATOR,
      .resource_id        = resource_id_locked(device, cmd_buffer),
      .is_driver_internal = true,
      .command_buffer     = {
         .preferred_domain                 = VK_RMV_KERNEL_MEMORY_DOMAIN_GTT /* TODO */,
         .executable_size                  = executable_size,
         .app_available_executable_size    = executable_size,
         .embedded_data_size               = data_size,
         .app_available_embedded_data_size = data_size,
         .scratch_size                     = 0,
         .app_available_scratch_size       = 0,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data,
                     VK_RMV_TOKEN_TYPE_RESOURCE_CREATE,
                     &create_token);
   list_for_each_entry(struct anv_batch_bo, bbo, &cmd_buffer->batch_bos, link) {
      log_resource_bind_locked(device, create_token.resource_id,
                               bbo->bo, 0, bbo->length);
   }
   bind_cmd_buffer_state_stream_locked(device, create_token.resource_id,
                                       &cmd_buffer->surface_state_stream);
   bind_cmd_buffer_state_stream_locked(device, create_token.resource_id,
                                       &cmd_buffer->dynamic_state_stream);
   bind_cmd_buffer_state_stream_locked(device, create_token.resource_id,
                                       &cmd_buffer->general_state_stream);
   bind_cmd_buffer_state_stream_locked(device, create_token.resource_id,
                                       &cmd_buffer->indirect_push_descriptor_stream);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_cmd_buffer_destroy(struct anv_device *device,
                               struct anv_cmd_buffer *cmd_buffer)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_destroy_token destroy_token = {
      .resource_id = resource_id_locked(device, cmd_buffer),
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data,
                     VK_RMV_TOKEN_TYPE_RESOURCE_DESTROY, &destroy_token);
   resource_destroy_locked(device, cmd_buffer);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_sparse_add_residency(struct anv_device *device,
                                 struct anv_bo *src_bo,
                                 uint64_t offset)
{
   struct vk_rmv_resource_reference_token token = {
      .virtual_address   = src_bo->offset + offset,
      .residency_removed = false,
   };

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   vk_rmv_emit_token(&device->vk.memory_trace_data,
                     VK_RMV_TOKEN_TYPE_RESOURCE_REFERENCE, &token);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_sparse_remove_residency(struct anv_device *device,
                                    struct anv_bo *src_bo,
                                    uint64_t offset)
{
   struct vk_rmv_resource_reference_token token = {
      .virtual_address   = src_bo->offset + offset,
      .residency_removed = true,
   };

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   vk_rmv_emit_token(&device->vk.memory_trace_data,
                     VK_RMV_TOKEN_TYPE_RESOURCE_REFERENCE, &token);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_descriptor_pool_create(struct anv_device *device,
                                   const VkDescriptorPoolCreateInfo *create_info,
                                   struct anv_descriptor_pool *pool,
                                   bool is_internal)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token create_token = {
      .type               = VK_RMV_RESOURCE_TYPE_DESCRIPTOR_POOL,
      .resource_id        = resource_id_locked(device, pool),
      .is_driver_internal = false,
      .descriptor_pool    = {
         .max_sets        = create_info->maxSets,
         .pool_size_count = create_info->poolSizeCount,
         /* Using vk_rmv_token_pool_alloc frees the allocation automatically
          * when the trace is done. */
         .pool_sizes      = malloc(create_info->poolSizeCount *
                                   sizeof(VkDescriptorPoolSize)),
      },
   };

   if (!create_token.descriptor_pool.pool_sizes) {
      simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
      return;
   }

   memcpy(create_token.descriptor_pool.pool_sizes, create_info->pPoolSizes,
          create_info->poolSizeCount * sizeof(VkDescriptorPoolSize));

   vk_rmv_emit_token(&device->vk.memory_trace_data,
                     VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &create_token);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);

   if (pool->surfaces.bo) {
      struct vk_rmv_resource_bind_token bind_token = {
         .resource_id      = create_token.resource_id,
         .is_system_memory = false,
         .address          = pool->surfaces.bo->offset,
         .size             = pool->surfaces.bo->size,
      };

      simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
      vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_BIND, &bind_token);
      simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
   }
   if (pool->samplers.bo) {
      struct vk_rmv_resource_bind_token bind_token = {
         .resource_id      = create_token.resource_id,
         .is_system_memory = false,
         .address          = pool->samplers.bo->offset,
         .size             = pool->samplers.bo->size,
      };

      simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
      vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_BIND, &bind_token);
      simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
   }
}

void
anv_rmv_log_graphics_pipeline_create(struct anv_device *device,
                                     struct anv_graphics_pipeline *pipeline,
                                     bool is_internal)
{
   struct vk_rmv_resource_create_token create_token = {
      .type               = VK_RMV_RESOURCE_TYPE_PIPELINE,
      .resource_id        = resource_id_locked(device, pipeline),
      .is_driver_internal = is_internal,
      .pipeline           = {
         .is_internal   = is_internal,
         .hash_lo       = 0,/* TODO pipeline->pipeline_hash; */
         .shader_stages = pipeline->base.base.active_stages,
      },
   };

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &create_token);
   for (unsigned s = 0; s < ARRAY_SIZE(pipeline->base.shaders); s++) {
      struct anv_shader_bin *shader = pipeline->base.shaders[s];

      if (!shader)
         continue;

      log_state_pool_bind_locked(device, create_token.resource_id,
                                 &device->instruction_state_pool,
                                 &shader->kernel);
   }
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_compute_pipeline_create(struct anv_device *device,
                                    struct anv_compute_pipeline *pipeline,
                                    bool is_internal)
{
   VkShaderStageFlagBits active_stages =
      pipeline->base.type == ANV_PIPELINE_COMPUTE ?
      VK_SHADER_STAGE_COMPUTE_BIT : VK_SHADER_STAGE_RAYGEN_BIT_KHR;

   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token create_token = {
      .type               = VK_RMV_RESOURCE_TYPE_PIPELINE,
      .resource_id        = resource_id_locked(device, pipeline),
      .is_driver_internal = is_internal,
      .pipeline           = {
         .is_internal   = is_internal,
         .hash_lo       = 0,/* TODO pipeline->pipeline_hash; */
         .shader_stages = active_stages,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &create_token);
   struct anv_shader_bin *shader = pipeline->cs;
   log_state_pool_bind_locked(device, create_token.resource_id,
                              &device->instruction_state_pool,
                              &shader->kernel);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_rt_pipeline_create(struct anv_device *device,
                               struct anv_ray_tracing_pipeline *pipeline,
                               bool is_internal)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);

   struct vk_rmv_resource_create_token create_token = {
      .resource_id        = resource_id_locked(device, pipeline),
      .type               = VK_RMV_RESOURCE_TYPE_PIPELINE,
      .is_driver_internal = is_internal,
      .pipeline           = {
         .is_internal   = is_internal,
         .hash_lo       = 0, /* TODO */
         .shader_stages = pipeline->base.active_stages,
      },
   };
   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &create_token);

   struct anv_state_pool *state_pool = &device->instruction_state_pool;
   for (uint32_t i = 0; i < pipeline->group_count; i++) {
      struct anv_rt_shader_group *group = &pipeline->groups[i];

      if (group->imported)
         continue;

      if (group->general) {
         log_state_pool_bind_locked(device, create_token.resource_id, state_pool,
                                    &group->general->kernel);
      }
      if (group->closest_hit) {
         log_state_pool_bind_locked(device, create_token.resource_id, state_pool,
                                    &group->closest_hit->kernel);
      }
      if (group->any_hit) {
         log_state_pool_bind_locked(device, create_token.resource_id, state_pool,
                                    &group->any_hit->kernel);
      }
      if (group->intersection) {
         log_state_pool_bind_locked(device, create_token.resource_id, state_pool,
                                    &group->intersection->kernel);
      }
   }

   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_event_create(struct anv_device *device,
                         struct anv_event *event,
                         VkEventCreateFlags flags,
                         bool is_internal)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_create_token create_token = {
      .type               = VK_RMV_RESOURCE_TYPE_GPU_EVENT,
      .resource_id        = resource_id_locked(device, event),
      .is_driver_internal = is_internal,
      .event              = {
         .flags = flags,
      },
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_CREATE, &create_token);
   log_state_pool_bind_locked(device, create_token.resource_id,
                              &device->dynamic_state_pool,
                              &event->state);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}

void
anv_rmv_log_resource_destroy(struct anv_device *device, const void *obj)
{
   simple_mtx_lock(&device->vk.memory_trace_data.token_mtx);
   struct vk_rmv_resource_destroy_token token = {
      .resource_id = resource_id_locked(device, obj),
   };

   vk_rmv_emit_token(&device->vk.memory_trace_data, VK_RMV_TOKEN_TYPE_RESOURCE_DESTROY, &token);
   resource_destroy_locked(device, obj);
   simple_mtx_unlock(&device->vk.memory_trace_data.token_mtx);
}