/* * Copyright 2021 Valve Corporation * * SPDX-License-Identifier: MIT */ #include "ac_spm.h" #include "util/bitscan.h" #include "util/u_memory.h" #include "ac_perfcounter.h" /* SPM counters definition. */ /* GFX10+ */ static struct ac_spm_counter_descr gfx10_num_l2_hits = {TCP, 0x9}; static struct ac_spm_counter_descr gfx10_num_l2_misses = {TCP, 0x12}; static struct ac_spm_counter_descr gfx10_num_scache_hits = {SQ, 0x14f}; static struct ac_spm_counter_descr gfx10_num_scache_misses = {SQ, 0x150}; static struct ac_spm_counter_descr gfx10_num_scache_misses_dup = {SQ, 0x151}; static struct ac_spm_counter_descr gfx10_num_icache_hits = {SQ, 0x12c}; static struct ac_spm_counter_descr gfx10_num_icache_misses = {SQ, 0x12d}; static struct ac_spm_counter_descr gfx10_num_icache_misses_dup = {SQ, 0x12e}; static struct ac_spm_counter_descr gfx10_num_gl1c_hits = {GL1C, 0xe}; static struct ac_spm_counter_descr gfx10_num_gl1c_misses = {GL1C, 0x12}; static struct ac_spm_counter_descr gfx10_num_gl2c_hits = {GL2C, 0x3}; static struct ac_spm_counter_descr gfx10_num_gl2c_misses = {GL2C, 0x23}; static struct ac_spm_counter_create_info gfx10_spm_counters[] = { {&gfx10_num_l2_hits}, {&gfx10_num_l2_misses}, {&gfx10_num_scache_hits}, {&gfx10_num_scache_misses}, {&gfx10_num_scache_misses_dup}, {&gfx10_num_icache_hits}, {&gfx10_num_icache_misses}, {&gfx10_num_icache_misses_dup}, {&gfx10_num_gl1c_hits}, {&gfx10_num_gl1c_misses}, {&gfx10_num_gl2c_hits}, {&gfx10_num_gl2c_misses}, }; /* GFX10.3+ */ static struct ac_spm_counter_descr gfx103_num_gl2c_misses = {GL2C, 0x2b}; static struct ac_spm_counter_create_info gfx103_spm_counters[] = { {&gfx10_num_l2_hits}, {&gfx10_num_l2_misses}, {&gfx10_num_scache_hits}, {&gfx10_num_scache_misses}, {&gfx10_num_scache_misses_dup}, {&gfx10_num_icache_hits}, {&gfx10_num_icache_misses}, {&gfx10_num_icache_misses_dup}, {&gfx10_num_gl1c_hits}, {&gfx10_num_gl1c_misses}, {&gfx10_num_gl2c_hits}, {&gfx103_num_gl2c_misses}, }; /* GFX11+ */ static struct ac_spm_counter_descr gfx11_num_l2_misses = {TCP, 0x11}; static struct ac_spm_counter_descr gfx11_num_scache_hits = {SQ_WGP, 0x126}; static struct ac_spm_counter_descr gfx11_num_scache_misses = {SQ_WGP, 0x127}; static struct ac_spm_counter_descr gfx11_num_scache_misses_dup = {SQ_WGP, 0x128}; static struct ac_spm_counter_descr gfx11_num_icache_hits = {SQ_WGP, 0x10e}; static struct ac_spm_counter_descr gfx11_num_icache_misses = {SQ_WGP, 0x10f}; static struct ac_spm_counter_descr gfx11_num_icache_misses_dup = {SQ_WGP, 0x110}; static struct ac_spm_counter_create_info gfx11_spm_counters[] = { {&gfx10_num_l2_hits}, {&gfx11_num_l2_misses}, {&gfx11_num_scache_hits}, {&gfx11_num_scache_misses}, {&gfx11_num_scache_misses_dup}, {&gfx11_num_icache_hits}, {&gfx11_num_icache_misses}, {&gfx11_num_icache_misses_dup}, {&gfx10_num_gl1c_hits}, {&gfx10_num_gl1c_misses}, {&gfx10_num_gl2c_hits}, {&gfx103_num_gl2c_misses}, }; static struct ac_spm_block_select * ac_spm_get_block_select(struct ac_spm *spm, const struct ac_pc_block *block) { struct ac_spm_block_select *block_sel, *new_block_sel; uint32_t num_block_sel; for (uint32_t i = 0; i < spm->num_block_sel; i++) { if (spm->block_sel[i].b->b->b->gpu_block == block->b->b->gpu_block) return &spm->block_sel[i]; } /* Allocate a new select block if it doesn't already exist. */ num_block_sel = spm->num_block_sel + 1; block_sel = realloc(spm->block_sel, num_block_sel * sizeof(*block_sel)); if (!block_sel) return NULL; spm->num_block_sel = num_block_sel; spm->block_sel = block_sel; /* Initialize the new select block. */ new_block_sel = &spm->block_sel[spm->num_block_sel - 1]; memset(new_block_sel, 0, sizeof(*new_block_sel)); new_block_sel->b = block; new_block_sel->instances = calloc(block->num_global_instances, sizeof(*new_block_sel->instances)); if (!new_block_sel->instances) return NULL; new_block_sel->num_instances = block->num_global_instances; for (unsigned i = 0; i < new_block_sel->num_instances; i++) new_block_sel->instances[i].num_counters = block->b->b->num_spm_counters; return new_block_sel; } struct ac_spm_instance_mapping { uint32_t se_index; /* SE index or 0 if global */ uint32_t sa_index; /* SA index or 0 if global or per-SE */ uint32_t instance_index; }; static bool ac_spm_init_instance_mapping(const struct radeon_info *info, const struct ac_pc_block *block, const struct ac_spm_counter_info *counter, struct ac_spm_instance_mapping *mapping) { uint32_t instance_index = 0, se_index = 0, sa_index = 0; if (block->b->b->flags & AC_PC_BLOCK_SE) { if (block->b->b->gpu_block == SQ) { /* Per-SE blocks. */ se_index = counter->instance / block->num_instances; instance_index = counter->instance % block->num_instances; } else { /* Per-SA blocks. */ assert(block->b->b->gpu_block == GL1C || block->b->b->gpu_block == TCP || block->b->b->gpu_block == SQ_WGP); se_index = (counter->instance / block->num_instances) / info->max_sa_per_se; sa_index = (counter->instance / block->num_instances) % info->max_sa_per_se; instance_index = counter->instance % block->num_instances; } } else { /* Global blocks. */ assert(block->b->b->gpu_block == GL2C); instance_index = counter->instance; } if (se_index >= info->num_se || sa_index >= info->max_sa_per_se || instance_index >= block->num_instances) return false; mapping->se_index = se_index; mapping->sa_index = sa_index; mapping->instance_index = instance_index; return true; } static void ac_spm_init_muxsel(const struct radeon_info *info, const struct ac_pc_block *block, const struct ac_spm_instance_mapping *mapping, struct ac_spm_counter_info *counter, uint32_t spm_wire) { const uint16_t counter_idx = 2 * spm_wire + (counter->is_even ? 0 : 1); union ac_spm_muxsel *muxsel = &counter->muxsel; if (info->gfx_level >= GFX11) { muxsel->gfx11.counter = counter_idx; muxsel->gfx11.block = block->b->b->spm_block_select; muxsel->gfx11.shader_array = mapping->sa_index; muxsel->gfx11.instance = mapping->instance_index; } else { muxsel->gfx10.counter = counter_idx; muxsel->gfx10.block = block->b->b->spm_block_select; muxsel->gfx10.shader_array = mapping->sa_index; muxsel->gfx10.instance = mapping->instance_index; } } static uint32_t ac_spm_init_grbm_gfx_index(const struct ac_pc_block *block, const struct ac_spm_instance_mapping *mapping) { uint32_t instance = mapping->instance_index; uint32_t grbm_gfx_index = 0; grbm_gfx_index |= S_030800_SE_INDEX(mapping->se_index) | S_030800_SH_INDEX(mapping->sa_index); switch (block->b->b->gpu_block) { case GL2C: /* Global blocks. */ grbm_gfx_index |= S_030800_SE_BROADCAST_WRITES(1); break; case SQ: /* Per-SE blocks. */ grbm_gfx_index |= S_030800_SH_BROADCAST_WRITES(1); break; default: /* Other blocks shouldn't broadcast. */ break; } if (block->b->b->gpu_block == SQ_WGP) { union { struct { uint32_t block_index : 2; /* Block index withing WGP */ uint32_t wgp_index : 3; uint32_t is_below_spi : 1; /* 0: lower WGP numbers, 1: higher WGP numbers */ uint32_t reserved : 26; }; uint32_t value; } instance_index = {0}; const uint32_t num_wgp_above_spi = 4; const bool is_below_spi = mapping->instance_index >= num_wgp_above_spi; instance_index.wgp_index = is_below_spi ? (mapping->instance_index - num_wgp_above_spi) : mapping->instance_index; instance_index.is_below_spi = is_below_spi; instance = instance_index.value; } grbm_gfx_index |= S_030800_INSTANCE_INDEX(instance); return grbm_gfx_index; } static bool ac_spm_map_counter(struct ac_spm *spm, struct ac_spm_block_select *block_sel, struct ac_spm_counter_info *counter, const struct ac_spm_instance_mapping *mapping, uint32_t *spm_wire) { uint32_t instance = counter->instance; if (block_sel->b->b->b->gpu_block == SQ_WGP) { if (!spm->sq_wgp[instance].grbm_gfx_index) { spm->sq_wgp[instance].grbm_gfx_index = ac_spm_init_grbm_gfx_index(block_sel->b, mapping); } for (unsigned i = 0; i < ARRAY_SIZE(spm->sq_wgp[instance].counters); i++) { struct ac_spm_counter_select *cntr_sel = &spm->sq_wgp[instance].counters[i]; if (i < spm->sq_wgp[instance].num_counters) continue; cntr_sel->sel0 |= S_036700_PERF_SEL(counter->event_id) | S_036700_SPM_MODE(1) | /* 16-bit clamp */ S_036700_PERF_MODE(0); /* Each SQ_WQP modules (GFX11+) share one 32-bit accumulator/wire * per pair of selects. */ cntr_sel->active |= 1 << (i % 2); *spm_wire = i / 2; if (cntr_sel->active & 0x1) counter->is_even = true; spm->sq_wgp[instance].num_counters++; return true; } } else if (block_sel->b->b->b->gpu_block == SQ) { for (unsigned i = 0; i < ARRAY_SIZE(spm->sqg[instance].counters); i++) { struct ac_spm_counter_select *cntr_sel = &spm->sqg[instance].counters[i]; if (i < spm->sqg[instance].num_counters) continue; /* SQ doesn't support 16-bit counters. */ cntr_sel->sel0 |= S_036700_PERF_SEL(counter->event_id) | S_036700_SPM_MODE(3) | /* 32-bit clamp */ S_036700_PERF_MODE(0); cntr_sel->active |= 0x3; /* 32-bits counter are always even. */ counter->is_even = true; /* One wire per SQ module. */ *spm_wire = i; spm->sqg[instance].num_counters++; return true; } } else { /* Generic blocks. */ struct ac_spm_block_instance *block_instance = &block_sel->instances[instance]; if (!block_instance->grbm_gfx_index) { block_instance->grbm_gfx_index = ac_spm_init_grbm_gfx_index(block_sel->b, mapping); } for (unsigned i = 0; i < block_instance->num_counters; i++) { struct ac_spm_counter_select *cntr_sel = &block_instance->counters[i]; int index = ffs(~cntr_sel->active) - 1; switch (index) { case 0: /* use S_037004_PERF_SEL */ cntr_sel->sel0 |= S_037004_PERF_SEL(counter->event_id) | S_037004_CNTR_MODE(1) | /* 16-bit clamp */ S_037004_PERF_MODE(0); /* accum */ break; case 1: /* use S_037004_PERF_SEL1 */ cntr_sel->sel0 |= S_037004_PERF_SEL1(counter->event_id) | S_037004_PERF_MODE1(0); break; case 2: /* use S_037004_PERF_SEL2 */ cntr_sel->sel1 |= S_037008_PERF_SEL2(counter->event_id) | S_037008_PERF_MODE2(0); break; case 3: /* use S_037004_PERF_SEL3 */ cntr_sel->sel1 |= S_037008_PERF_SEL3(counter->event_id) | S_037008_PERF_MODE3(0); break; default: return false; } /* Mark this 16-bit counter as used. */ cntr_sel->active |= 1 << index; /* Determine if the counter is even or odd. */ counter->is_even = !(index % 2); /* Determine the SPM wire (one wire holds two 16-bit counters). */ *spm_wire = !!(index >= 2); return true; } } return false; } static bool ac_spm_add_counter(const struct radeon_info *info, const struct ac_perfcounters *pc, struct ac_spm *spm, const struct ac_spm_counter_create_info *counter_info) { struct ac_spm_instance_mapping instance_mapping = {0}; struct ac_spm_counter_info *counter; struct ac_spm_block_select *block_sel; struct ac_pc_block *block; uint32_t spm_wire; /* Check if the GPU block is valid. */ block = ac_pc_get_block(pc, counter_info->b->gpu_block); if (!block) { fprintf(stderr, "ac/spm: Invalid GPU block.\n"); return false; } /* Check if the number of instances is valid. */ if (counter_info->instance > block->num_global_instances - 1) { fprintf(stderr, "ac/spm: Invalid instance ID.\n"); return false; } /* Check if the event ID is valid. */ if (counter_info->b->event_id > block->b->selectors) { fprintf(stderr, "ac/spm: Invalid event ID.\n"); return false; } counter = &spm->counters[spm->num_counters]; spm->num_counters++; counter->gpu_block = counter_info->b->gpu_block; counter->event_id = counter_info->b->event_id; counter->instance = counter_info->instance; /* Get the select block used to configure the counter. */ block_sel = ac_spm_get_block_select(spm, block); if (!block_sel) return false; /* Initialize instance mapping for the counter. */ if (!ac_spm_init_instance_mapping(info, block, counter, &instance_mapping)) { fprintf(stderr, "ac/spm: Failed to initialize instance mapping.\n"); return false; } /* Map the counter to the select block. */ if (!ac_spm_map_counter(spm, block_sel, counter, &instance_mapping, &spm_wire)) { fprintf(stderr, "ac/spm: No free slots available!\n"); return false; } /* Determine the counter segment type. */ if (block->b->b->flags & AC_PC_BLOCK_SE) { counter->segment_type = instance_mapping.se_index; } else { counter->segment_type = AC_SPM_SEGMENT_TYPE_GLOBAL; } /* Configure the muxsel for SPM. */ ac_spm_init_muxsel(info, block, &instance_mapping, counter, spm_wire); return true; } static void ac_spm_fill_muxsel_ram(const struct radeon_info *info, struct ac_spm *spm, enum ac_spm_segment_type segment_type, uint32_t offset) { struct ac_spm_muxsel_line *mappings = spm->muxsel_lines[segment_type]; uint32_t even_counter_idx = 0, even_line_idx = 0; uint32_t odd_counter_idx = 0, odd_line_idx = 1; /* Add the global timestamps first. */ if (segment_type == AC_SPM_SEGMENT_TYPE_GLOBAL) { if (info->gfx_level >= GFX11) { mappings[even_line_idx].muxsel[even_counter_idx++].value = 0xf840; mappings[even_line_idx].muxsel[even_counter_idx++].value = 0xf841; mappings[even_line_idx].muxsel[even_counter_idx++].value = 0xf842; mappings[even_line_idx].muxsel[even_counter_idx++].value = 0xf843; } else { for (unsigned i = 0; i < 4; i++) { mappings[even_line_idx].muxsel[even_counter_idx++].value = 0xf0f0; } } } for (unsigned i = 0; i < spm->num_counters; i++) { struct ac_spm_counter_info *counter = &spm->counters[i]; if (counter->segment_type != segment_type) continue; if (counter->is_even) { counter->offset = (offset + even_line_idx) * AC_SPM_NUM_COUNTER_PER_MUXSEL + even_counter_idx; mappings[even_line_idx].muxsel[even_counter_idx] = spm->counters[i].muxsel; if (++even_counter_idx == AC_SPM_NUM_COUNTER_PER_MUXSEL) { even_counter_idx = 0; even_line_idx += 2; } } else { counter->offset = (offset + odd_line_idx) * AC_SPM_NUM_COUNTER_PER_MUXSEL + odd_counter_idx; mappings[odd_line_idx].muxsel[odd_counter_idx] = spm->counters[i].muxsel; if (++odd_counter_idx == AC_SPM_NUM_COUNTER_PER_MUXSEL) { odd_counter_idx = 0; odd_line_idx += 2; } } } } bool ac_init_spm(const struct radeon_info *info, const struct ac_perfcounters *pc, struct ac_spm *spm) { const struct ac_spm_counter_create_info *create_info; unsigned create_info_count; unsigned num_counters = 0; switch (info->gfx_level) { case GFX10: create_info_count = ARRAY_SIZE(gfx10_spm_counters); create_info = gfx10_spm_counters; break; case GFX10_3: create_info_count = ARRAY_SIZE(gfx103_spm_counters); create_info = gfx103_spm_counters; break; case GFX11: case GFX11_5: create_info_count = ARRAY_SIZE(gfx11_spm_counters); create_info = gfx11_spm_counters; break; default: return false; /* not implemented */ } /* Count the total number of counters. */ for (unsigned i = 0; i < create_info_count; i++) { const struct ac_pc_block *block = ac_pc_get_block(pc, create_info[i].b->gpu_block); if (!block) return false; num_counters += block->num_global_instances; } spm->counters = CALLOC(num_counters, sizeof(*spm->counters)); if (!spm->counters) return false; for (unsigned i = 0; i < create_info_count; i++) { const struct ac_pc_block *block = ac_pc_get_block(pc, create_info[i].b->gpu_block); struct ac_spm_counter_create_info counter = create_info[i]; for (unsigned j = 0; j < block->num_global_instances; j++) { counter.instance = j; if (!ac_spm_add_counter(info, pc, spm, &counter)) { fprintf(stderr, "ac/spm: Failed to add SPM counter (%d).\n", i); return false; } } } /* Determine the segment size and create a muxsel ram for every segment. */ for (unsigned s = 0; s < AC_SPM_SEGMENT_TYPE_COUNT; s++) { unsigned num_even_counters = 0, num_odd_counters = 0; if (s == AC_SPM_SEGMENT_TYPE_GLOBAL) { /* The global segment always start with a 64-bit timestamp. */ num_even_counters += AC_SPM_GLOBAL_TIMESTAMP_COUNTERS; } /* Count the number of even/odd counters for this segment. */ for (unsigned c = 0; c < spm->num_counters; c++) { struct ac_spm_counter_info *counter = &spm->counters[c]; if (counter->segment_type != s) continue; if (counter->is_even) { num_even_counters++; } else { num_odd_counters++; } } /* Compute the number of lines. */ unsigned even_lines = DIV_ROUND_UP(num_even_counters, AC_SPM_NUM_COUNTER_PER_MUXSEL); unsigned odd_lines = DIV_ROUND_UP(num_odd_counters, AC_SPM_NUM_COUNTER_PER_MUXSEL); unsigned num_lines = (even_lines > odd_lines) ? (2 * even_lines - 1) : (2 * odd_lines); spm->muxsel_lines[s] = CALLOC(num_lines, sizeof(*spm->muxsel_lines[s])); if (!spm->muxsel_lines[s]) return false; spm->num_muxsel_lines[s] = num_lines; } /* Compute the maximum number of muxsel lines among all SEs. On GFX11, * there is only one SE segment size value and the highest value is used. */ for (unsigned s = 0; s < AC_SPM_SEGMENT_TYPE_GLOBAL; s++) { spm->max_se_muxsel_lines = MAX2(spm->num_muxsel_lines[s], spm->max_se_muxsel_lines); } /* RLC uses the following order: Global, SE0, SE1, SE2, SE3, SE4, SE5. */ ac_spm_fill_muxsel_ram(info, spm, AC_SPM_SEGMENT_TYPE_GLOBAL, 0); const uint32_t num_global_lines = spm->num_muxsel_lines[AC_SPM_SEGMENT_TYPE_GLOBAL]; if (info->gfx_level >= GFX11) { /* On GFX11, RLC uses one segment size for every single SE. */ for (unsigned i = 0; i < info->num_se; i++) { assert(i < AC_SPM_SEGMENT_TYPE_GLOBAL); uint32_t offset = num_global_lines + i * spm->max_se_muxsel_lines; ac_spm_fill_muxsel_ram(info, spm, i, offset); } } else { uint32_t offset = num_global_lines; for (unsigned i = 0; i < info->num_se; i++) { assert(i < AC_SPM_SEGMENT_TYPE_GLOBAL); ac_spm_fill_muxsel_ram(info, spm, i, offset); offset += spm->num_muxsel_lines[i]; } } /* On GFX11, the data size written by the hw is in units of segment. */ spm->ptr_granularity = info->gfx_level >= GFX11 ? 32 : 1; return true; } void ac_destroy_spm(struct ac_spm *spm) { for (unsigned s = 0; s < AC_SPM_SEGMENT_TYPE_COUNT; s++) { FREE(spm->muxsel_lines[s]); } for (unsigned i = 0; i < spm->num_block_sel; i++) { FREE(spm->block_sel[i].instances); } FREE(spm->block_sel); FREE(spm->counters); } static uint32_t ac_spm_get_sample_size(const struct ac_spm *spm) { uint32_t sample_size = 0; /* in bytes */ for (unsigned s = 0; s < AC_SPM_SEGMENT_TYPE_COUNT; s++) { sample_size += spm->num_muxsel_lines[s] * AC_SPM_MUXSEL_LINE_SIZE * 4; } return sample_size; } static uint32_t ac_spm_get_num_samples(const struct ac_spm *spm) { uint32_t sample_size = ac_spm_get_sample_size(spm); uint32_t *ptr = (uint32_t *)spm->ptr; uint32_t data_size, num_lines_written; uint32_t num_samples = 0; /* Get the data size (in bytes) written by the hw to the ring buffer. */ data_size = ptr[0] * spm->ptr_granularity; /* Compute the number of 256 bits (16 * 16-bits counters) lines written. */ num_lines_written = data_size / (2 * AC_SPM_NUM_COUNTER_PER_MUXSEL); /* Check for overflow. */ if (num_lines_written % (sample_size / 32)) { abort(); } else { num_samples = num_lines_written / (sample_size / 32); } return num_samples; } void ac_spm_get_trace(const struct ac_spm *spm, struct ac_spm_trace *trace) { memset(trace, 0, sizeof(*trace)); trace->ptr = spm->ptr; trace->sample_interval = spm->sample_interval; trace->num_counters = spm->num_counters; trace->counters = spm->counters; trace->sample_size_in_bytes = ac_spm_get_sample_size(spm); trace->num_samples = ac_spm_get_num_samples(spm); }