1 /*
2 * Copyright © 2009 Corbin Simpson
3 * Copyright © 2011 Marek Olšák <[email protected]>
4 *
5 * SPDX-License-Identifier: MIT
6 */
7
8 #include "radeon_drm_bo.h"
9 #include "radeon_drm_cs.h"
10
11 #include "util/os_file.h"
12 #include "util/simple_mtx.h"
13 #include "util/thread_sched.h"
14 #include "util/u_cpu_detect.h"
15 #include "util/u_memory.h"
16 #include "util/u_hash_table.h"
17 #include "util/u_pointer.h"
18
19 #include <xf86drm.h>
20 #include <stdio.h>
21 #include <sys/types.h>
22 #include <sys/stat.h>
23 #include <unistd.h>
24 #include <fcntl.h>
25 #include <radeon_surface.h>
26
27 static struct hash_table *fd_tab = NULL;
28 static simple_mtx_t fd_tab_mutex = SIMPLE_MTX_INITIALIZER;
29
30 /* Enable/disable feature access for one command stream.
31 * If enable == true, return true on success.
32 * Otherwise, return false.
33 *
34 * We basically do the same thing kernel does, because we have to deal
35 * with multiple contexts (here command streams) backed by one winsys. */
radeon_set_fd_access(struct radeon_drm_cs * applier,struct radeon_drm_cs ** owner,mtx_t * mutex,unsigned request,const char * request_name,bool enable)36 static bool radeon_set_fd_access(struct radeon_drm_cs *applier,
37 struct radeon_drm_cs **owner,
38 mtx_t *mutex,
39 unsigned request, const char *request_name,
40 bool enable)
41 {
42 struct drm_radeon_info info;
43 unsigned value = enable ? 1 : 0;
44
45 memset(&info, 0, sizeof(info));
46
47 mtx_lock(&*mutex);
48
49 /* Early exit if we are sure the request will fail. */
50 if (enable) {
51 if (*owner) {
52 mtx_unlock(&*mutex);
53 return false;
54 }
55 } else {
56 if (*owner != applier) {
57 mtx_unlock(&*mutex);
58 return false;
59 }
60 }
61
62 /* Pass through the request to the kernel. */
63 info.value = (unsigned long)&value;
64 info.request = request;
65 if (drmCommandWriteRead(applier->ws->fd, DRM_RADEON_INFO,
66 &info, sizeof(info)) != 0) {
67 mtx_unlock(&*mutex);
68 return false;
69 }
70
71 /* Update the rights in the winsys. */
72 if (enable) {
73 if (value) {
74 *owner = applier;
75 mtx_unlock(&*mutex);
76 return true;
77 }
78 } else {
79 *owner = NULL;
80 }
81
82 mtx_unlock(&*mutex);
83 return false;
84 }
85
radeon_get_drm_value(int fd,unsigned request,const char * errname,uint32_t * out)86 static bool radeon_get_drm_value(int fd, unsigned request,
87 const char *errname, uint32_t *out)
88 {
89 struct drm_radeon_info info;
90 int retval;
91
92 memset(&info, 0, sizeof(info));
93
94 info.value = (unsigned long)out;
95 info.request = request;
96
97 retval = drmCommandWriteRead(fd, DRM_RADEON_INFO, &info, sizeof(info));
98 if (retval) {
99 if (errname) {
100 fprintf(stderr, "radeon: Failed to get %s, error number %d\n",
101 errname, retval);
102 }
103 return false;
104 }
105 return true;
106 }
107
108 /* Helper function to do the ioctls needed for setup and init. */
do_winsys_init(struct radeon_drm_winsys * ws)109 static bool do_winsys_init(struct radeon_drm_winsys *ws)
110 {
111 struct drm_radeon_gem_info gem_info;
112 int retval;
113 drmVersionPtr version;
114
115 memset(&gem_info, 0, sizeof(gem_info));
116
117 /* We do things in a specific order here.
118 *
119 * DRM version first. We need to be sure we're running on a KMS chipset.
120 * This is also for some features.
121 *
122 * Then, the PCI ID. This is essential and should return usable numbers
123 * for all Radeons. If this fails, we probably got handed an FD for some
124 * non-Radeon card.
125 *
126 * The GEM info is actually bogus on the kernel side, as well as our side
127 * (see radeon_gem_info_ioctl in radeon_gem.c) but that's alright because
128 * we don't actually use the info for anything yet.
129 *
130 * The GB and Z pipe requests should always succeed, but they might not
131 * return sensical values for all chipsets, but that's alright because
132 * the pipe drivers already know that.
133 */
134
135 /* Get DRM version. */
136 version = drmGetVersion(ws->fd);
137 if (!version)
138 return false;
139
140 if (version->version_major != 2 ||
141 version->version_minor < 50) {
142 fprintf(stderr, "%s: DRM version is %d.%d.%d but this driver is "
143 "only compatible with 2.50.0 (kernel 4.12) or later.\n",
144 __func__,
145 version->version_major,
146 version->version_minor,
147 version->version_patchlevel);
148 drmFreeVersion(version);
149 return false;
150 }
151
152 ws->info.drm_major = version->version_major;
153 ws->info.drm_minor = version->version_minor;
154 ws->info.drm_patchlevel = version->version_patchlevel;
155 ws->info.is_amdgpu = false;
156 drmFreeVersion(version);
157
158 /* Get PCI ID. */
159 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_DEVICE_ID, "PCI ID",
160 &ws->info.pci_id))
161 return false;
162
163 /* Check PCI ID. */
164 switch (ws->info.pci_id) {
165 #define CHIPSET(pci_id, name, cfamily) case pci_id: ws->info.family = CHIP_##cfamily; ws->gen = DRV_R300; break;
166 #include "pci_ids/r300_pci_ids.h"
167 #undef CHIPSET
168
169 #define CHIPSET(pci_id, name, cfamily) case pci_id: ws->info.family = CHIP_##cfamily; ws->gen = DRV_R600; break;
170 #include "pci_ids/r600_pci_ids.h"
171 #undef CHIPSET
172
173 #define CHIPSET(pci_id, cfamily) \
174 case pci_id: \
175 ws->info.family = CHIP_##cfamily; \
176 ws->info.name = #cfamily; \
177 ws->gen = DRV_SI; \
178 break;
179 #include "pci_ids/radeonsi_pci_ids.h"
180 #undef CHIPSET
181
182 default:
183 fprintf(stderr, "radeon: Invalid PCI ID.\n");
184 return false;
185 }
186
187 switch (ws->info.family) {
188 default:
189 case CHIP_UNKNOWN:
190 fprintf(stderr, "radeon: Unknown family.\n");
191 return false;
192 case CHIP_R300:
193 case CHIP_R350:
194 case CHIP_RV350:
195 case CHIP_RV370:
196 case CHIP_RV380:
197 case CHIP_RS400:
198 case CHIP_RC410:
199 case CHIP_RS480:
200 ws->info.gfx_level = R300;
201 break;
202 case CHIP_R420: /* R4xx-based cores. */
203 case CHIP_R423:
204 case CHIP_R430:
205 case CHIP_R480:
206 case CHIP_R481:
207 case CHIP_RV410:
208 case CHIP_RS600:
209 case CHIP_RS690:
210 case CHIP_RS740:
211 ws->info.gfx_level = R400;
212 break;
213 case CHIP_RV515: /* R5xx-based cores. */
214 case CHIP_R520:
215 case CHIP_RV530:
216 case CHIP_R580:
217 case CHIP_RV560:
218 case CHIP_RV570:
219 ws->info.gfx_level = R500;
220 break;
221 case CHIP_R600:
222 case CHIP_RV610:
223 case CHIP_RV630:
224 case CHIP_RV670:
225 case CHIP_RV620:
226 case CHIP_RV635:
227 case CHIP_RS780:
228 case CHIP_RS880:
229 ws->info.gfx_level = R600;
230 break;
231 case CHIP_RV770:
232 case CHIP_RV730:
233 case CHIP_RV710:
234 case CHIP_RV740:
235 ws->info.gfx_level = R700;
236 break;
237 case CHIP_CEDAR:
238 case CHIP_REDWOOD:
239 case CHIP_JUNIPER:
240 case CHIP_CYPRESS:
241 case CHIP_HEMLOCK:
242 case CHIP_PALM:
243 case CHIP_SUMO:
244 case CHIP_SUMO2:
245 case CHIP_BARTS:
246 case CHIP_TURKS:
247 case CHIP_CAICOS:
248 ws->info.gfx_level = EVERGREEN;
249 break;
250 case CHIP_CAYMAN:
251 case CHIP_ARUBA:
252 ws->info.gfx_level = CAYMAN;
253 break;
254 case CHIP_TAHITI:
255 case CHIP_PITCAIRN:
256 case CHIP_VERDE:
257 case CHIP_OLAND:
258 case CHIP_HAINAN:
259 ws->info.gfx_level = GFX6;
260 break;
261 case CHIP_BONAIRE:
262 case CHIP_KAVERI:
263 case CHIP_KABINI:
264 case CHIP_HAWAII:
265 ws->info.gfx_level = GFX7;
266 break;
267 }
268
269 /* Set which chips don't have dedicated VRAM. */
270 switch (ws->info.family) {
271 case CHIP_RS400:
272 case CHIP_RC410:
273 case CHIP_RS480:
274 case CHIP_RS600:
275 case CHIP_RS690:
276 case CHIP_RS740:
277 case CHIP_RS780:
278 case CHIP_RS880:
279 case CHIP_PALM:
280 case CHIP_SUMO:
281 case CHIP_SUMO2:
282 case CHIP_ARUBA:
283 case CHIP_KAVERI:
284 case CHIP_KABINI:
285 ws->info.has_dedicated_vram = false;
286 break;
287
288 default:
289 ws->info.has_dedicated_vram = true;
290 }
291
292 ws->info.ip[AMD_IP_GFX].num_queues = 1;
293 /* Check for dma */
294 ws->info.ip[AMD_IP_SDMA].num_queues = 0;
295 /* DMA is disabled on R700. There is IB corruption and hangs. */
296 if (ws->info.gfx_level >= EVERGREEN) {
297 ws->info.ip[AMD_IP_SDMA].num_queues = 1;
298 }
299
300 /* Check for UVD and VCE */
301 ws->info.vce_fw_version = 0x00000000;
302
303 uint32_t value = RADEON_CS_RING_UVD;
304 if (radeon_get_drm_value(ws->fd, RADEON_INFO_RING_WORKING,
305 "UVD Ring working", &value)) {
306 ws->info.ip[AMD_IP_UVD].num_queues = 1;
307 }
308
309 value = RADEON_CS_RING_VCE;
310 if (radeon_get_drm_value(ws->fd, RADEON_INFO_RING_WORKING,
311 NULL, &value) && value) {
312
313 if (radeon_get_drm_value(ws->fd, RADEON_INFO_VCE_FW_VERSION,
314 "VCE FW version", &value)) {
315 ws->info.vce_fw_version = value;
316 ws->info.ip[AMD_IP_VCE].num_queues = 1;
317 }
318 }
319
320 /* Check for userptr support. */
321 {
322 struct drm_radeon_gem_userptr args = {0};
323
324 /* If the ioctl doesn't exist, -EINVAL is returned.
325 *
326 * If the ioctl exists, it should return -EACCES
327 * if RADEON_GEM_USERPTR_READONLY or RADEON_GEM_USERPTR_REGISTER
328 * aren't set.
329 */
330 ws->info.has_userptr =
331 drmCommandWriteRead(ws->fd, DRM_RADEON_GEM_USERPTR,
332 &args, sizeof(args)) == -EACCES;
333 }
334
335 /* Get GEM info. */
336 retval = drmCommandWriteRead(ws->fd, DRM_RADEON_GEM_INFO,
337 &gem_info, sizeof(gem_info));
338 if (retval) {
339 fprintf(stderr, "radeon: Failed to get MM info, error number %d\n",
340 retval);
341 return false;
342 }
343 ws->info.gart_size_kb = DIV_ROUND_UP(gem_info.gart_size, 1024);
344 ws->info.vram_size_kb = DIV_ROUND_UP(gem_info.vram_size, 1024);
345 ws->info.vram_vis_size_kb = DIV_ROUND_UP(gem_info.vram_visible, 1024);
346
347 /* Radeon allocates all buffers contiguously, which makes large allocations
348 * unlikely to succeed. */
349 if (ws->info.has_dedicated_vram)
350 ws->info.max_heap_size_kb = ws->info.vram_size_kb;
351 else
352 ws->info.max_heap_size_kb = ws->info.gart_size_kb;
353
354 /* Both 32-bit and 64-bit address spaces only have 4GB.
355 * This is a limitation of the VM allocator in the winsys.
356 */
357 ws->info.max_heap_size_kb = MIN2(ws->info.max_heap_size_kb, 4 * 1024 * 1024); /* 4 GB */
358
359 /* Get max clock frequency info and convert it to MHz */
360 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_SCLK, NULL,
361 &ws->info.max_gpu_freq_mhz);
362 ws->info.max_gpu_freq_mhz /= 1000;
363
364 ws->num_cpus = sysconf(_SC_NPROCESSORS_ONLN);
365
366 /* Generation-specific queries. */
367 if (ws->gen == DRV_R300) {
368 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_GB_PIPES,
369 "GB pipe count",
370 &ws->info.r300_num_gb_pipes))
371 return false;
372
373 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_Z_PIPES,
374 "Z pipe count",
375 &ws->info.r300_num_z_pipes))
376 return false;
377 }
378 else if (ws->gen >= DRV_R600) {
379 uint32_t tiling_config = 0;
380
381 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_BACKENDS,
382 "num backends",
383 &ws->info.max_render_backends))
384 return false;
385
386 /* get the GPU counter frequency, failure is not fatal */
387 radeon_get_drm_value(ws->fd, RADEON_INFO_CLOCK_CRYSTAL_FREQ, NULL,
388 &ws->info.clock_crystal_freq);
389
390 radeon_get_drm_value(ws->fd, RADEON_INFO_TILING_CONFIG, NULL,
391 &tiling_config);
392
393 ws->info.r600_num_banks =
394 ws->info.gfx_level >= EVERGREEN ?
395 4 << ((tiling_config & 0xf0) >> 4) :
396 4 << ((tiling_config & 0x30) >> 4);
397
398 ws->info.pipe_interleave_bytes =
399 ws->info.gfx_level >= EVERGREEN ?
400 256 << ((tiling_config & 0xf00) >> 8) :
401 256 << ((tiling_config & 0xc0) >> 6);
402
403 if (!ws->info.pipe_interleave_bytes)
404 ws->info.pipe_interleave_bytes =
405 ws->info.gfx_level >= EVERGREEN ? 512 : 256;
406
407 radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_TILE_PIPES, NULL,
408 &ws->info.num_tile_pipes);
409
410 /* "num_tiles_pipes" must be equal to the number of pipes (Px) in the
411 * pipe config field of the GB_TILE_MODE array. Only one card (Tahiti)
412 * reports a different value (12). Fix it by setting what's in the
413 * GB_TILE_MODE array (8).
414 */
415 if (ws->gen == DRV_SI && ws->info.num_tile_pipes == 12)
416 ws->info.num_tile_pipes = 8;
417
418 if (radeon_get_drm_value(ws->fd, RADEON_INFO_BACKEND_MAP, NULL,
419 &ws->info.r600_gb_backend_map))
420 ws->info.r600_gb_backend_map_valid = true;
421
422 /* Default value. */
423 ws->info.enabled_rb_mask = u_bit_consecutive(0, ws->info.max_render_backends);
424 /*
425 * This fails (silently) on non-GCN or older kernels, overwriting the
426 * default enabled_rb_mask with the result of the last query.
427 */
428 if (ws->gen >= DRV_SI) {
429 uint32_t mask;
430
431 radeon_get_drm_value(ws->fd, RADEON_INFO_SI_BACKEND_ENABLED_MASK, NULL, &mask);
432 ws->info.enabled_rb_mask = mask;
433 }
434
435 ws->info.r600_has_virtual_memory = false;
436
437 uint32_t ib_vm_max_size;
438
439 ws->info.r600_has_virtual_memory = true;
440 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_VA_START, NULL,
441 &ws->va_start))
442 ws->info.r600_has_virtual_memory = false;
443 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_IB_VM_MAX_SIZE, NULL,
444 &ib_vm_max_size))
445 ws->info.r600_has_virtual_memory = false;
446 radeon_get_drm_value(ws->fd, RADEON_INFO_VA_UNMAP_WORKING, NULL,
447 &ws->va_unmap_working);
448
449 if (ws->gen == DRV_R600 && !debug_get_bool_option("RADEON_VA", false))
450 ws->info.r600_has_virtual_memory = false;
451 }
452
453 /* Get max pipes, this is only needed for compute shaders. All evergreen+
454 * chips have at least 2 pipes, so we use 2 as a default. */
455 ws->info.r600_max_quad_pipes = 2;
456 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_PIPES, NULL,
457 &ws->info.r600_max_quad_pipes);
458
459 /* All GPUs have at least one compute unit */
460 ws->info.num_cu = 1;
461 radeon_get_drm_value(ws->fd, RADEON_INFO_ACTIVE_CU_COUNT, NULL,
462 &ws->info.num_cu);
463
464 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_SE, NULL,
465 &ws->info.max_se);
466
467 switch (ws->info.family) {
468 case CHIP_HAINAN:
469 case CHIP_KABINI:
470 ws->info.max_tcc_blocks = 2;
471 break;
472 case CHIP_VERDE:
473 case CHIP_OLAND:
474 case CHIP_BONAIRE:
475 case CHIP_KAVERI:
476 ws->info.max_tcc_blocks = 4;
477 break;
478 case CHIP_PITCAIRN:
479 ws->info.max_tcc_blocks = 8;
480 break;
481 case CHIP_TAHITI:
482 ws->info.max_tcc_blocks = 12;
483 break;
484 case CHIP_HAWAII:
485 ws->info.max_tcc_blocks = 16;
486 break;
487 default:
488 ws->info.max_tcc_blocks = 0;
489 break;
490 }
491
492 if (!ws->info.max_se) {
493 switch (ws->info.family) {
494 default:
495 ws->info.max_se = 1;
496 break;
497 case CHIP_CYPRESS:
498 case CHIP_HEMLOCK:
499 case CHIP_BARTS:
500 case CHIP_CAYMAN:
501 case CHIP_TAHITI:
502 case CHIP_PITCAIRN:
503 case CHIP_BONAIRE:
504 ws->info.max_se = 2;
505 break;
506 case CHIP_HAWAII:
507 ws->info.max_se = 4;
508 break;
509 }
510 }
511
512 ws->info.num_se = ws->info.max_se;
513
514 radeon_get_drm_value(ws->fd, RADEON_INFO_MAX_SH_PER_SE, NULL,
515 &ws->info.max_sa_per_se);
516 if (ws->gen == DRV_SI) {
517 ws->info.max_good_cu_per_sa =
518 ws->info.min_good_cu_per_sa = ws->info.num_cu /
519 (ws->info.max_se * ws->info.max_sa_per_se);
520 }
521
522 radeon_get_drm_value(ws->fd, RADEON_INFO_ACCEL_WORKING2, NULL,
523 &ws->accel_working2);
524 if (ws->info.family == CHIP_HAWAII && ws->accel_working2 < 2) {
525 fprintf(stderr, "radeon: GPU acceleration for Hawaii disabled, "
526 "returned accel_working2 value %u is smaller than 2. "
527 "Please install a newer kernel.\n",
528 ws->accel_working2);
529 return false;
530 }
531
532 if (ws->info.gfx_level == GFX7) {
533 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_CIK_MACROTILE_MODE_ARRAY, NULL,
534 ws->info.cik_macrotile_mode_array)) {
535 fprintf(stderr, "radeon: Kernel 3.13 is required for Sea Islands support.\n");
536 return false;
537 }
538 }
539
540 if (ws->info.gfx_level >= GFX6) {
541 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_SI_TILE_MODE_ARRAY, NULL,
542 ws->info.si_tile_mode_array)) {
543 fprintf(stderr, "radeon: Kernel 3.10 is required for Southern Islands support.\n");
544 return false;
545 }
546 }
547
548 for (unsigned ip_type = 0; ip_type < AMD_NUM_IP_TYPES; ip_type++)
549 ws->info.ip[ip_type].ib_alignment = 4096;
550
551 /* Hawaii with old firmware needs type2 nop packet.
552 * accel_working2 with value 3 indicates the new firmware.
553 */
554 ws->info.gfx_ib_pad_with_type2 = ws->info.gfx_level <= GFX6 ||
555 (ws->info.family == CHIP_HAWAII &&
556 ws->accel_working2 < 3);
557 ws->info.has_cp_dma = true;
558 ws->info.tcc_cache_line_size = 64; /* TC L2 line size on GCN */
559 ws->info.has_bo_metadata = false;
560 ws->info.has_eqaa_surface_allocator = false;
561 ws->info.has_sparse_vm_mappings = false;
562 ws->info.max_alignment = 1024*1024;
563 ws->info.has_graphics = true;
564 ws->info.cpdma_prefetch_writes_memory = true;
565 ws->info.max_waves_per_simd = 10;
566 ws->info.num_physical_sgprs_per_simd = 512;
567 ws->info.num_physical_wave64_vgprs_per_simd = 256;
568 ws->info.has_3d_cube_border_color_mipmap = true;
569 ws->info.has_image_opcodes = true;
570 ws->info.spi_cu_en_has_effect = false;
571 ws->info.spi_cu_en = 0xffff;
572 ws->info.never_stop_sq_perf_counters = false;
573 ws->info.num_rb = util_bitcount64(ws->info.enabled_rb_mask);
574 ws->info.max_gflops = 128 * ws->info.num_cu * ws->info.max_gpu_freq_mhz / 1000;
575 ws->info.num_tcc_blocks = ws->info.max_tcc_blocks;
576 ws->info.tcp_cache_size = 16 * 1024;
577 ws->info.num_simd_per_compute_unit = 4;
578 ws->info.min_sgpr_alloc = 8;
579 ws->info.max_sgpr_alloc = 104;
580 ws->info.sgpr_alloc_granularity = 8;
581 ws->info.min_wave64_vgpr_alloc = 4;
582 ws->info.max_vgpr_alloc = 256;
583 ws->info.wave64_vgpr_alloc_granularity = 4;
584 ws->info.lds_size_per_workgroup = ws->info.gfx_level == GFX7 ? 64 * 1024 : 32 * 1024;
585 ws->info.lds_encode_granularity = ws->info.gfx_level == GFX7 ? 128 * 4 : 64 * 4;
586 ws->info.lds_alloc_granularity = ws->info.lds_encode_granularity;
587
588 for (unsigned se = 0; se < ws->info.max_se; se++) {
589 for (unsigned sa = 0; sa < ws->info.max_sa_per_se; sa++)
590 ws->info.cu_mask[se][sa] = BITFIELD_MASK(ws->info.max_good_cu_per_sa);
591 }
592
593 /* The maximum number of scratch waves. The number is only a function of the number of CUs.
594 * It should be large enough to hold at least 1 threadgroup. Use the minimum per-SA CU count.
595 */
596 const unsigned max_waves_per_tg = 1024 / 64; /* LLVM only supports 1024 threads per block */
597 ws->info.max_scratch_waves = MAX2(32 * ws->info.min_good_cu_per_sa * ws->info.max_sa_per_se *
598 ws->info.num_se, max_waves_per_tg);
599
600 switch (ws->info.family) {
601 case CHIP_TAHITI:
602 case CHIP_PITCAIRN:
603 case CHIP_OLAND:
604 case CHIP_HAWAII:
605 case CHIP_KABINI:
606 ws->info.l2_cache_size = ws->info.num_tcc_blocks * 64 * 1024;
607 break;
608 case CHIP_VERDE:
609 case CHIP_HAINAN:
610 case CHIP_BONAIRE:
611 case CHIP_KAVERI:
612 ws->info.l2_cache_size = ws->info.num_tcc_blocks * 128 * 1024;
613 break;
614 default:;
615 }
616
617 ws->info.ip[AMD_IP_GFX].num_queues = 1;
618
619 switch (ws->info.gfx_level) {
620 case R300:
621 case R400:
622 case R500:
623 ws->info.ip[AMD_IP_GFX].ver_major = 2;
624 break;
625 case R600:
626 case R700:
627 ws->info.ip[AMD_IP_GFX].ver_major = 3;
628 break;
629 case EVERGREEN:
630 ws->info.ip[AMD_IP_GFX].ver_major = 4;
631 break;
632 case CAYMAN:
633 ws->info.ip[AMD_IP_GFX].ver_major = 5;
634 break;
635 case GFX6:
636 ws->info.ip[AMD_IP_GFX].ver_major = 6;
637 break;
638 case GFX7:
639 ws->info.ip[AMD_IP_GFX].ver_major = 7;
640 break;
641 default:;
642 }
643
644 ws->check_vm = strstr(debug_get_option("R600_DEBUG", ""), "check_vm") != NULL ||
645 strstr(debug_get_option("AMD_DEBUG", ""), "check_vm") != NULL;
646 ws->noop_cs = debug_get_bool_option("RADEON_NOOP", false);
647
648 return true;
649 }
650
radeon_winsys_destroy(struct radeon_winsys * rws)651 static void radeon_winsys_destroy(struct radeon_winsys *rws)
652 {
653 struct radeon_drm_winsys *ws = (struct radeon_drm_winsys*)rws;
654
655 if (util_queue_is_initialized(&ws->cs_queue))
656 util_queue_destroy(&ws->cs_queue);
657
658 mtx_destroy(&ws->hyperz_owner_mutex);
659 mtx_destroy(&ws->cmask_owner_mutex);
660
661 if (ws->info.r600_has_virtual_memory)
662 pb_slabs_deinit(&ws->bo_slabs);
663 pb_cache_deinit(&ws->bo_cache);
664
665 if (ws->gen >= DRV_R600) {
666 radeon_surface_manager_free(ws->surf_man);
667 }
668
669 _mesa_hash_table_destroy(ws->bo_names, NULL);
670 _mesa_hash_table_destroy(ws->bo_handles, NULL);
671 _mesa_hash_table_u64_destroy(ws->bo_vas);
672 mtx_destroy(&ws->bo_handles_mutex);
673 mtx_destroy(&ws->vm32.mutex);
674 mtx_destroy(&ws->vm64.mutex);
675 mtx_destroy(&ws->bo_fence_lock);
676
677 if (ws->fd >= 0)
678 close(ws->fd);
679
680 FREE(rws);
681 }
682
radeon_query_info(struct radeon_winsys * rws,struct radeon_info * info)683 static void radeon_query_info(struct radeon_winsys *rws, struct radeon_info *info)
684 {
685 *info = ((struct radeon_drm_winsys *)rws)->info;
686 }
687
radeon_cs_request_feature(struct radeon_cmdbuf * rcs,enum radeon_feature_id fid,bool enable)688 static bool radeon_cs_request_feature(struct radeon_cmdbuf *rcs,
689 enum radeon_feature_id fid,
690 bool enable)
691 {
692 struct radeon_drm_cs *cs = radeon_drm_cs(rcs);
693
694 switch (fid) {
695 case RADEON_FID_R300_HYPERZ_ACCESS:
696 return radeon_set_fd_access(cs, &cs->ws->hyperz_owner,
697 &cs->ws->hyperz_owner_mutex,
698 RADEON_INFO_WANT_HYPERZ, "Hyper-Z",
699 enable);
700
701 case RADEON_FID_R300_CMASK_ACCESS:
702 return radeon_set_fd_access(cs, &cs->ws->cmask_owner,
703 &cs->ws->cmask_owner_mutex,
704 RADEON_INFO_WANT_CMASK, "AA optimizations",
705 enable);
706 }
707 return false;
708 }
709
radeon_drm_get_gpu_reset_counter(struct radeon_drm_winsys * ws)710 uint32_t radeon_drm_get_gpu_reset_counter(struct radeon_drm_winsys *ws)
711 {
712 uint64_t retval = 0;
713
714 radeon_get_drm_value(ws->fd, RADEON_INFO_GPU_RESET_COUNTER,
715 "gpu-reset-counter", (uint32_t*)&retval);
716 return retval;
717 }
718
radeon_query_value(struct radeon_winsys * rws,enum radeon_value_id value)719 static uint64_t radeon_query_value(struct radeon_winsys *rws,
720 enum radeon_value_id value)
721 {
722 struct radeon_drm_winsys *ws = (struct radeon_drm_winsys*)rws;
723 uint64_t retval = 0;
724
725 switch (value) {
726 case RADEON_REQUESTED_VRAM_MEMORY:
727 return ws->allocated_vram;
728 case RADEON_REQUESTED_GTT_MEMORY:
729 return ws->allocated_gtt;
730 case RADEON_MAPPED_VRAM:
731 return ws->mapped_vram;
732 case RADEON_MAPPED_GTT:
733 return ws->mapped_gtt;
734 case RADEON_BUFFER_WAIT_TIME_NS:
735 return ws->buffer_wait_time;
736 case RADEON_NUM_MAPPED_BUFFERS:
737 return ws->num_mapped_buffers;
738 case RADEON_TIMESTAMP:
739 if (ws->gen < DRV_R600) {
740 assert(0);
741 return 0;
742 }
743
744 radeon_get_drm_value(ws->fd, RADEON_INFO_TIMESTAMP, "timestamp",
745 (uint32_t*)&retval);
746 return retval;
747 case RADEON_NUM_GFX_IBS:
748 return ws->num_gfx_IBs;
749 case RADEON_NUM_SDMA_IBS:
750 return ws->num_sdma_IBs;
751 case RADEON_NUM_BYTES_MOVED:
752 radeon_get_drm_value(ws->fd, RADEON_INFO_NUM_BYTES_MOVED,
753 "num-bytes-moved", (uint32_t*)&retval);
754 return retval;
755 case RADEON_NUM_EVICTIONS:
756 case RADEON_NUM_VRAM_CPU_PAGE_FAULTS:
757 case RADEON_VRAM_VIS_USAGE:
758 case RADEON_GFX_BO_LIST_COUNTER:
759 case RADEON_GFX_IB_SIZE_COUNTER:
760 case RADEON_SLAB_WASTED_VRAM:
761 case RADEON_SLAB_WASTED_GTT:
762 return 0; /* unimplemented */
763 case RADEON_VRAM_USAGE:
764 radeon_get_drm_value(ws->fd, RADEON_INFO_VRAM_USAGE,
765 "vram-usage", (uint32_t*)&retval);
766 return retval;
767 case RADEON_GTT_USAGE:
768 radeon_get_drm_value(ws->fd, RADEON_INFO_GTT_USAGE,
769 "gtt-usage", (uint32_t*)&retval);
770 return retval;
771 case RADEON_GPU_TEMPERATURE:
772 radeon_get_drm_value(ws->fd, RADEON_INFO_CURRENT_GPU_TEMP,
773 "gpu-temp", (uint32_t*)&retval);
774 return retval;
775 case RADEON_CURRENT_SCLK:
776 radeon_get_drm_value(ws->fd, RADEON_INFO_CURRENT_GPU_SCLK,
777 "current-gpu-sclk", (uint32_t*)&retval);
778 return retval;
779 case RADEON_CURRENT_MCLK:
780 radeon_get_drm_value(ws->fd, RADEON_INFO_CURRENT_GPU_MCLK,
781 "current-gpu-mclk", (uint32_t*)&retval);
782 return retval;
783 case RADEON_CS_THREAD_TIME:
784 return util_queue_get_thread_time_nano(&ws->cs_queue, 0);
785 }
786 return 0;
787 }
788
radeon_read_registers(struct radeon_winsys * rws,unsigned reg_offset,unsigned num_registers,uint32_t * out)789 static bool radeon_read_registers(struct radeon_winsys *rws,
790 unsigned reg_offset,
791 unsigned num_registers, uint32_t *out)
792 {
793 struct radeon_drm_winsys *ws = (struct radeon_drm_winsys*)rws;
794 unsigned i;
795
796 for (i = 0; i < num_registers; i++) {
797 uint32_t reg = reg_offset + i*4;
798
799 if (!radeon_get_drm_value(ws->fd, RADEON_INFO_READ_REG, NULL, ®))
800 return false;
801 out[i] = reg;
802 }
803 return true;
804 }
805
806 DEBUG_GET_ONCE_BOOL_OPTION(thread, "RADEON_THREAD", true)
807
radeon_winsys_unref(struct radeon_winsys * ws)808 static bool radeon_winsys_unref(struct radeon_winsys *ws)
809 {
810 struct radeon_drm_winsys *rws = (struct radeon_drm_winsys*)ws;
811 bool destroy;
812
813 /* When the reference counter drops to zero, remove the fd from the table.
814 * This must happen while the mutex is locked, so that
815 * radeon_drm_winsys_create in another thread doesn't get the winsys
816 * from the table when the counter drops to 0. */
817 simple_mtx_lock(&fd_tab_mutex);
818
819 destroy = pipe_reference(&rws->reference, NULL);
820 if (destroy && fd_tab) {
821 _mesa_hash_table_remove_key(fd_tab, intptr_to_pointer(rws->fd));
822 if (_mesa_hash_table_num_entries(fd_tab) == 0) {
823 _mesa_hash_table_destroy(fd_tab, NULL);
824 fd_tab = NULL;
825 }
826 }
827
828 simple_mtx_unlock(&fd_tab_mutex);
829 return destroy;
830 }
831
radeon_pin_threads_to_L3_cache(struct radeon_winsys * ws,unsigned cpu)832 static void radeon_pin_threads_to_L3_cache(struct radeon_winsys *ws,
833 unsigned cpu)
834 {
835 struct radeon_drm_winsys *rws = (struct radeon_drm_winsys*)ws;
836
837 if (util_queue_is_initialized(&rws->cs_queue)) {
838 util_thread_sched_apply_policy(rws->cs_queue.threads[0],
839 UTIL_THREAD_DRIVER_SUBMIT, cpu, NULL);
840 }
841 }
842
radeon_cs_is_secure(struct radeon_cmdbuf * cs)843 static bool radeon_cs_is_secure(struct radeon_cmdbuf* cs)
844 {
845 return false;
846 }
847
radeon_cs_set_pstate(struct radeon_cmdbuf * cs,enum radeon_ctx_pstate state)848 static bool radeon_cs_set_pstate(struct radeon_cmdbuf* cs, enum radeon_ctx_pstate state)
849 {
850 return false;
851 }
852
853 static int
radeon_drm_winsys_get_fd(struct radeon_winsys * ws)854 radeon_drm_winsys_get_fd(struct radeon_winsys *ws)
855 {
856 struct radeon_drm_winsys *rws = (struct radeon_drm_winsys*)ws;
857
858 return rws->fd;
859 }
860
861 PUBLIC struct radeon_winsys *
radeon_drm_winsys_create(int fd,const struct pipe_screen_config * config,radeon_screen_create_t screen_create)862 radeon_drm_winsys_create(int fd, const struct pipe_screen_config *config,
863 radeon_screen_create_t screen_create)
864 {
865 struct radeon_drm_winsys *ws;
866
867 simple_mtx_lock(&fd_tab_mutex);
868 if (!fd_tab) {
869 fd_tab = util_hash_table_create_fd_keys();
870 }
871
872 ws = util_hash_table_get(fd_tab, intptr_to_pointer(fd));
873 if (ws) {
874 pipe_reference(NULL, &ws->reference);
875 simple_mtx_unlock(&fd_tab_mutex);
876 return &ws->base;
877 }
878
879 ws = CALLOC_STRUCT(radeon_drm_winsys);
880 if (!ws) {
881 simple_mtx_unlock(&fd_tab_mutex);
882 return NULL;
883 }
884
885 ws->fd = os_dupfd_cloexec(fd);
886
887 if (!do_winsys_init(ws))
888 goto fail1;
889
890 pb_cache_init(&ws->bo_cache, RADEON_NUM_HEAPS,
891 500000, ws->check_vm ? 1.0f : 2.0f, 0,
892 (uint64_t)MIN2(ws->info.vram_size_kb, ws->info.gart_size_kb) * 1024,
893 offsetof(struct radeon_bo, u.real.cache_entry),
894 NULL,
895 radeon_bo_destroy,
896 radeon_bo_can_reclaim);
897
898 if (ws->info.r600_has_virtual_memory) {
899 /* There is no fundamental obstacle to using slab buffer allocation
900 * without GPUVM, but enabling it requires making sure that the drivers
901 * honor the address offset.
902 */
903 if (!pb_slabs_init(&ws->bo_slabs,
904 RADEON_SLAB_MIN_SIZE_LOG2, RADEON_SLAB_MAX_SIZE_LOG2,
905 RADEON_NUM_HEAPS, false,
906 ws,
907 radeon_bo_can_reclaim_slab,
908 radeon_bo_slab_alloc,
909 radeon_bo_slab_free))
910 goto fail_cache;
911
912 ws->info.min_alloc_size = 1 << RADEON_SLAB_MIN_SIZE_LOG2;
913 } else {
914 ws->info.min_alloc_size = ws->info.gart_page_size;
915 }
916
917 if (ws->gen >= DRV_R600) {
918 ws->surf_man = radeon_surface_manager_new(ws->fd);
919 if (!ws->surf_man)
920 goto fail_slab;
921 }
922
923 /* init reference */
924 pipe_reference_init(&ws->reference, 1);
925
926 /* Set functions. */
927 ws->base.unref = radeon_winsys_unref;
928 ws->base.destroy = radeon_winsys_destroy;
929 ws->base.get_fd = radeon_drm_winsys_get_fd;
930 ws->base.query_info = radeon_query_info;
931 ws->base.pin_threads_to_L3_cache = radeon_pin_threads_to_L3_cache;
932 ws->base.cs_request_feature = radeon_cs_request_feature;
933 ws->base.query_value = radeon_query_value;
934 ws->base.read_registers = radeon_read_registers;
935 ws->base.cs_is_secure = radeon_cs_is_secure;
936 ws->base.cs_set_pstate = radeon_cs_set_pstate;
937
938 radeon_drm_bo_init_functions(ws);
939 radeon_drm_cs_init_functions(ws);
940 radeon_surface_init_functions(ws);
941
942 (void) mtx_init(&ws->hyperz_owner_mutex, mtx_plain);
943 (void) mtx_init(&ws->cmask_owner_mutex, mtx_plain);
944
945 ws->bo_names = util_hash_table_create_ptr_keys();
946 ws->bo_handles = util_hash_table_create_ptr_keys();
947 ws->bo_vas = _mesa_hash_table_u64_create(NULL);
948 (void) mtx_init(&ws->bo_handles_mutex, mtx_plain);
949 (void) mtx_init(&ws->vm32.mutex, mtx_plain);
950 (void) mtx_init(&ws->vm64.mutex, mtx_plain);
951 (void) mtx_init(&ws->bo_fence_lock, mtx_plain);
952 list_inithead(&ws->vm32.holes);
953 list_inithead(&ws->vm64.holes);
954
955 /* The kernel currently returns 8MB. Make sure this doesn't change. */
956 if (ws->va_start > 8 * 1024 * 1024) {
957 /* Not enough 32-bit address space. */
958 radeon_winsys_destroy(&ws->base);
959 simple_mtx_unlock(&fd_tab_mutex);
960 return NULL;
961 }
962
963 ws->vm32.start = ws->va_start;
964 ws->vm32.end = 1ull << 32;
965
966 ws->vm64.start = 1ull << 32;
967 ws->vm64.end = 1ull << 33;
968
969 /* TTM aligns the BO size to the CPU page size */
970 ws->info.gart_page_size = sysconf(_SC_PAGESIZE);
971 ws->info.pte_fragment_size = 64 * 1024; /* GPUVM page size */
972
973 if (ws->num_cpus > 1 && debug_get_option_thread())
974 util_queue_init(&ws->cs_queue, "rcs", 8, 1, 0, NULL);
975
976 /* Create the screen at the end. The winsys must be initialized
977 * completely.
978 *
979 * Alternatively, we could create the screen based on "ws->gen"
980 * and link all drivers into one binary blob. */
981 ws->base.screen = screen_create(&ws->base, config);
982 if (!ws->base.screen) {
983 radeon_winsys_destroy(&ws->base);
984 simple_mtx_unlock(&fd_tab_mutex);
985 return NULL;
986 }
987
988 _mesa_hash_table_insert(fd_tab, intptr_to_pointer(ws->fd), ws);
989
990 /* We must unlock the mutex once the winsys is fully initialized, so that
991 * other threads attempting to create the winsys from the same fd will
992 * get a fully initialized winsys and not just half-way initialized. */
993 simple_mtx_unlock(&fd_tab_mutex);
994
995 return &ws->base;
996
997 fail_slab:
998 if (ws->info.r600_has_virtual_memory)
999 pb_slabs_deinit(&ws->bo_slabs);
1000 fail_cache:
1001 pb_cache_deinit(&ws->bo_cache);
1002 fail1:
1003 simple_mtx_unlock(&fd_tab_mutex);
1004 if (ws->surf_man)
1005 radeon_surface_manager_free(ws->surf_man);
1006 if (ws->fd >= 0)
1007 close(ws->fd);
1008
1009 FREE(ws);
1010 return NULL;
1011 }
1012