1 #include <assert.h>
2 #include <errno.h>
3 #include <fcntl.h>
4 #include <inttypes.h>
5 #include <stdio.h>
6 #include <stdlib.h>
7 #include <xf86drm.h>
8
9 #include "drm-uapi/nouveau_drm.h"
10 #include "nouveau.h"
11 #include "nvif/class.h"
12 #include "nvif/cl0080.h"
13 #include "nvif/ioctl.h"
14
15 #include "nv_push.h"
16
17 #include "util/bitscan.h"
18 #include "util/list.h"
19 #include "util/os_mman.h"
20 #include "util/simple_mtx.h"
21 #include "util/u_atomic.h"
22 #include "util/u_memory.h"
23
24 static FILE *nouveau_out = NULL;
25 static uint32_t nouveau_debug = 0;
26
27 #define dbg_on(lvl) (nouveau_debug & (1 << lvl))
28 #define dbg(lvl, fmt, args...) do { \
29 if (dbg_on((lvl))) \
30 fprintf(nouveau_out, "nouveau: "fmt, ##args); \
31 } while(0)
32 #define err(fmt, args...) fprintf(nouveau_out, "nouveau: "fmt, ##args)
33
34 static void
debug_init(void)35 debug_init(void)
36 {
37 static bool once = false;
38 char *debug, *out;
39
40 if (once)
41 return;
42 once = true;
43
44 debug = getenv("NOUVEAU_LIBDRM_DEBUG");
45 if (debug) {
46 int n = strtol(debug, NULL, 0);
47 if (n >= 0)
48 nouveau_debug = n;
49 }
50
51 nouveau_out = stderr;
52 out = getenv("NOUVEAU_LIBDRM_OUT");
53 if (out) {
54 FILE *fout = fopen(out, "w");
55 if (fout)
56 nouveau_out = fout;
57 }
58 }
59
60 int
nouveau_drm_new(int fd,struct nouveau_drm ** pdrm)61 nouveau_drm_new(int fd, struct nouveau_drm **pdrm)
62 {
63 debug_init();
64
65 struct nouveau_drm *drm = calloc(1, sizeof(*drm));
66 if (!drm)
67 return -ENOMEM;
68 drm->fd = fd;
69 *pdrm = drm;
70
71 drmVersionPtr ver = drmGetVersion(fd);
72 if (!ver)
73 goto out_err;
74
75 drm->version = (ver->version_major << 24) |
76 (ver->version_minor << 8) |
77 ver->version_patchlevel;
78 if (drm->version < 0x01000301)
79 goto out_err;
80
81 drmFreeVersion(ver);
82 return 0;
83
84 out_err:
85 nouveau_drm_del(pdrm);
86 return -EINVAL;
87 }
88
89 void
nouveau_drm_del(struct nouveau_drm ** pdrm)90 nouveau_drm_del(struct nouveau_drm **pdrm)
91 {
92 free(*pdrm);
93 *pdrm = NULL;
94 }
95
96 static int
nouveau_object_channel_new(struct nouveau_object * parent,uint64_t handle,uint32_t oclass,struct nouveau_object * obj)97 nouveau_object_channel_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass,
98 struct nouveau_object *obj)
99 {
100 // TODO nv04?
101 struct nouveau_drm *drm = nouveau_drm(parent);
102 struct nouveau_device *dev = (void*)obj->parent;
103 struct nouveau_fifo *fifo = obj->data;
104 struct drm_nouveau_channel_alloc req = { };
105
106 /* nvc0 doesn't need any special handling here */
107 if (dev->chipset < 0xc0) {
108 struct nv04_fifo *nv04 = obj->data;
109 req.fb_ctxdma_handle = nv04->vram;
110 req.tt_ctxdma_handle = nv04->gart;
111 } else if (dev->chipset >= 0xe0) {
112 struct nve0_fifo *nve0 = obj->data;
113 req.fb_ctxdma_handle = 0xffffffff;
114 req.tt_ctxdma_handle = nve0->engine;
115 }
116
117 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_CHANNEL_ALLOC, &req, sizeof(req));
118 if (ret)
119 return ret;
120
121 fifo->pushbuf = req.pushbuf_domains;
122 fifo->notify = req.notifier_handle;
123 obj->handle = req.channel;
124 return 0;
125 }
126
127 static int
nouveau_object_notifier_new(struct nouveau_object * parent,uint64_t handle,struct nouveau_object * obj)128 nouveau_object_notifier_new(struct nouveau_object *parent, uint64_t handle,
129 struct nouveau_object *obj)
130 {
131 struct nouveau_drm *drm = nouveau_drm(obj);
132 struct nv04_notify *ntfy = obj->data;
133 struct drm_nouveau_notifierobj_alloc req = {
134 .channel = parent->handle,
135 .handle = handle,
136 .size = ntfy->length,
137 };
138
139 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NOTIFIEROBJ_ALLOC, &req, sizeof(req));
140 if (ret)
141 return ret;
142
143 ntfy->offset = req.offset;
144 return 0;
145 }
146
147 static int
nouveau_object_subchan_new(struct nouveau_object * parent,uint64_t handle,uint32_t oclass,struct nouveau_object * obj)148 nouveau_object_subchan_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass,
149 struct nouveau_object *obj)
150 {
151 struct nouveau_drm *drm = nouveau_drm(parent);
152 struct {
153 struct nvif_ioctl_v0 ioctl;
154 struct nvif_ioctl_new_v0 new;
155 } args = {
156 .ioctl = {
157 .route = 0xff,
158 .token = parent->handle,
159 .type = NVIF_IOCTL_V0_NEW,
160 .version = 0,
161 },
162 .new = {
163 .handle = handle,
164 .object = (uintptr_t)obj,
165 .oclass = oclass,
166 .route = NVIF_IOCTL_V0_ROUTE_NVIF,
167 .token = (uintptr_t)obj,
168 .version = 0,
169 },
170 };
171
172 return drmCommandWrite(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
173 }
174
175 /* TODO: split this interfaces up so we can verify the right parent object type gets passed in */
176 int
nouveau_object_new(struct nouveau_object * parent,uint64_t handle,uint32_t oclass,void * data,uint32_t length,struct nouveau_object ** pobj)177 nouveau_object_new(struct nouveau_object *parent, uint64_t handle, uint32_t oclass, void *data,
178 uint32_t length, struct nouveau_object **pobj)
179 {
180 struct nouveau_object *obj = calloc(1, sizeof(*obj));
181 if (!obj)
182 return -ENOMEM;
183
184 obj->parent = parent;
185 obj->handle = handle;
186 obj->oclass = oclass;
187 if (length) {
188 obj->data = malloc(length);
189 memcpy(obj->data, data, length);
190 }
191
192 int ret;
193 switch (oclass) {
194 case NOUVEAU_FIFO_CHANNEL_CLASS:
195 ret = nouveau_object_channel_new(parent, handle, oclass, obj);
196 break;
197 case NOUVEAU_NOTIFIER_CLASS:
198 ret = nouveau_object_notifier_new(parent, handle, obj);
199 break;
200 default:
201 ret = nouveau_object_subchan_new(parent, handle, oclass, obj);
202 break;
203 }
204
205 if (ret) {
206 free(obj->data);
207 free(obj);
208 return ret;
209 }
210
211 *pobj = obj;
212 return 0;
213 }
214
215 static void
nouveau_object_channel_del(struct nouveau_object * obj)216 nouveau_object_channel_del(struct nouveau_object *obj)
217 {
218 struct nouveau_drm *drm = nouveau_drm(obj->parent);
219 struct drm_nouveau_channel_free req = {
220 .channel = obj->handle,
221 };
222
223 int ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_CHANNEL_FREE, &req, sizeof(req));
224 assert(!ret);
225 }
226
227 static void
nouveau_object_subchan_del(struct nouveau_object * obj)228 nouveau_object_subchan_del(struct nouveau_object *obj)
229 {
230 struct {
231 struct nvif_ioctl_v0 ioctl;
232 struct nvif_ioctl_del del;
233 } args = {
234 .ioctl = {
235 .object = (uintptr_t)obj,
236 .owner = NVIF_IOCTL_V0_OWNER_ANY,
237 .route = 0x00,
238 .type = NVIF_IOCTL_V0_DEL,
239 .version = 0,
240 },
241 };
242
243 drmCommandWrite(obj->parent->handle, DRM_NOUVEAU_NVIF, &args, sizeof(args));
244 }
245
246 static void
nouveau_object_gpuobj_del(struct nouveau_object * obj)247 nouveau_object_gpuobj_del(struct nouveau_object *obj)
248 {
249 struct nouveau_drm *drm = nouveau_drm(obj->parent);
250 struct drm_nouveau_gpuobj_free req = {
251 .channel = obj->parent->handle,
252 .handle = obj->handle,
253 };
254
255 drmCommandWrite(drm->fd, DRM_NOUVEAU_GPUOBJ_FREE, &req, sizeof(req));
256 }
257
258 void
nouveau_object_del(struct nouveau_object ** pobj)259 nouveau_object_del(struct nouveau_object **pobj)
260 {
261 if (!*pobj)
262 return;
263
264 struct nouveau_object *obj = *pobj;
265 switch (obj->oclass) {
266 case NOUVEAU_FIFO_CHANNEL_CLASS:
267 nouveau_object_channel_del(obj);
268 break;
269 case NOUVEAU_NOTIFIER_CLASS:
270 nouveau_object_gpuobj_del(obj);
271 break;
272 default:
273 nouveau_object_subchan_del(obj);
274 break;
275 }
276 free(obj->data);
277 free(obj);
278 *pobj = NULL;
279 }
280
281 #define NOUVEAU_OBJECT_MAX_CLASSES 16
282 int
nouveau_object_mclass(struct nouveau_object * obj,const struct nouveau_mclass * mclass)283 nouveau_object_mclass(struct nouveau_object *obj, const struct nouveau_mclass *mclass)
284 {
285 struct nouveau_drm *drm = nouveau_drm(obj->parent);
286 struct {
287 struct nvif_ioctl_v0 ioctl;
288 struct nvif_ioctl_sclass_v0 sclass;
289 struct nvif_ioctl_sclass_oclass_v0 list[NOUVEAU_OBJECT_MAX_CLASSES];
290 } args = {
291 .ioctl = {
292 .route = 0xff,
293 .token = obj->handle,
294 .type = NVIF_IOCTL_V0_SCLASS,
295 .version = 0,
296 },
297 .sclass = {
298 .count = NOUVEAU_OBJECT_MAX_CLASSES,
299 .version = 0,
300 },
301 };
302
303 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
304 if (ret)
305 return ret;
306
307 int i;
308 for (i = 0; mclass[i].oclass; i++) {
309 for (int j = 0; j < args.sclass.count; j++) {
310 if (args.list[j].oclass == mclass[i].oclass)
311 return i;
312 }
313 }
314
315 return -ENODEV;
316 }
317
318 struct nouveau_device_priv {
319 struct nouveau_device base;
320 simple_mtx_t lock;
321 struct list_head bo_list;
322 int gart_limit_percent;
323 int vram_limit_percent;
324 };
325
326 static inline struct nouveau_device_priv *
nouveau_device(struct nouveau_device * dev)327 nouveau_device(struct nouveau_device *dev)
328 {
329 return (struct nouveau_device_priv *)dev;
330 }
331
332 int
nouveau_device_info(struct nouveau_device * dev,struct nv_device_info_v0 * info)333 nouveau_device_info(struct nouveau_device *dev, struct nv_device_info_v0 *info)
334 {
335 struct nouveau_drm *drm = nouveau_drm(dev->object.parent);
336 struct {
337 struct nvif_ioctl_v0 ioctl;
338 struct nvif_ioctl_mthd_v0 mthd;
339 struct nv_device_info_v0 info;
340 } args = {
341 .ioctl = {
342 .object = (uintptr_t)dev,
343 .owner = NVIF_IOCTL_V0_OWNER_ANY,
344 .route = 0x00,
345 .type = NVIF_IOCTL_V0_MTHD,
346 .version = 0,
347 },
348 .mthd = {
349 .method = NV_DEVICE_V0_INFO,
350 .version = 0,
351 },
352 .info = {
353 .version = 0,
354 },
355 };
356
357 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
358 if (ret)
359 return ret;
360
361 *info = args.info;
362 return 0;
363 }
364
365 int
nouveau_device_new(struct nouveau_object * parent,struct nouveau_device ** pdev)366 nouveau_device_new(struct nouveau_object *parent, struct nouveau_device **pdev)
367 {
368 struct nouveau_drm *drm = nouveau_drm(parent);
369 struct nouveau_device *dev;
370 uint64_t v;
371 char *tmp;
372
373 struct nouveau_device_priv *nvdev = calloc(1, sizeof(*nvdev));
374 if (!nvdev)
375 return -ENOMEM;
376 dev = *pdev = &nvdev->base;
377 dev->object.parent = parent;
378
379 struct {
380 struct nvif_ioctl_v0 ioctl;
381 struct nvif_ioctl_new_v0 new;
382 struct nv_device_v0 dev;
383 } args = {
384 .ioctl = {
385 .object = 0,
386 .owner = NVIF_IOCTL_V0_OWNER_ANY,
387 .route = 0x00,
388 .type = NVIF_IOCTL_V0_NEW,
389 .version = 0,
390 },
391 .new = {
392 .handle = 0,
393 .object = (uintptr_t)&nvdev->base.object,
394 .oclass = NV_DEVICE,
395 .route = NVIF_IOCTL_V0_ROUTE_NVIF,
396 .token = (uintptr_t)&nvdev->base.object,
397 .version = 0,
398 },
399 .dev = {
400 .device = ~0ULL,
401 },
402 };
403
404 int ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_NVIF, &args, sizeof(args));
405 if (ret)
406 goto done;
407
408 struct nv_device_info_v0 info;
409 ret = nouveau_device_info(dev, &info);
410 if (ret)
411 goto done;
412
413 nvdev->base.chipset = info.chipset;
414 nvdev->base.info.chipset = info.chipset;
415 switch (info.platform) {
416 case NV_DEVICE_INFO_V0_PCI:
417 case NV_DEVICE_INFO_V0_AGP:
418 case NV_DEVICE_INFO_V0_PCIE:
419 nvdev->base.info.type = NV_DEVICE_TYPE_DIS;
420 break;
421 case NV_DEVICE_INFO_V0_IGP:
422 nvdev->base.info.type = NV_DEVICE_TYPE_IGP;
423 break;
424 case NV_DEVICE_INFO_V0_SOC:
425 nvdev->base.info.type = NV_DEVICE_TYPE_SOC;
426 break;
427 default:
428 unreachable("unhandled nvidia device type");
429 break;
430 }
431
432 drmDevicePtr drm_device;
433 ret = drmGetDevice2(drm->fd, 0, &drm_device);
434 if (ret)
435 goto done;
436
437 if (drm_device->bustype == DRM_BUS_PCI) {
438 nvdev->base.info.pci.domain = drm_device->businfo.pci->domain;
439 nvdev->base.info.pci.bus = drm_device->businfo.pci->bus;
440 nvdev->base.info.pci.dev = drm_device->businfo.pci->dev;
441 nvdev->base.info.pci.func = drm_device->businfo.pci->func;
442 nvdev->base.info.pci.revision_id = drm_device->deviceinfo.pci->revision_id;
443 nvdev->base.info.device_id = drm_device->deviceinfo.pci->device_id;
444 }
445
446 drmFreeDevice(&drm_device);
447
448 ret = nouveau_getparam(dev, NOUVEAU_GETPARAM_FB_SIZE, &v);
449 if (ret)
450 goto done;
451 nvdev->base.vram_size = v;
452
453 ret = nouveau_getparam(dev, NOUVEAU_GETPARAM_AGP_SIZE, &v);
454 if (ret)
455 goto done;
456 nvdev->base.gart_size = v;
457
458 tmp = getenv("NOUVEAU_LIBDRM_VRAM_LIMIT_PERCENT");
459 if (tmp)
460 nvdev->vram_limit_percent = atoi(tmp);
461 else
462 nvdev->vram_limit_percent = 80;
463 nvdev->base.vram_limit = (nvdev->base.vram_size * nvdev->vram_limit_percent) / 100;
464
465 tmp = getenv("NOUVEAU_LIBDRM_GART_LIMIT_PERCENT");
466 if (tmp)
467 nvdev->gart_limit_percent = atoi(tmp);
468 else
469 nvdev->gart_limit_percent = 80;
470 nvdev->base.gart_limit = (nvdev->base.gart_size * nvdev->gart_limit_percent) / 100;
471
472 simple_mtx_init(&nvdev->lock, mtx_plain);
473 list_inithead(&nvdev->bo_list);
474 done:
475 if (ret)
476 nouveau_device_del(pdev);
477 return ret;
478 }
479
480 void
nouveau_device_set_classes_for_debug(struct nouveau_device * dev,uint32_t cls_eng3d,uint32_t cls_compute,uint32_t cls_m2mf,uint32_t cls_copy)481 nouveau_device_set_classes_for_debug(struct nouveau_device *dev,
482 uint32_t cls_eng3d,
483 uint32_t cls_compute,
484 uint32_t cls_m2mf,
485 uint32_t cls_copy)
486 {
487 dev->info.cls_eng3d = cls_eng3d;
488 dev->info.cls_compute = cls_compute;
489 dev->info.cls_m2mf = cls_m2mf;
490 dev->info.cls_copy = cls_copy;
491 }
492
493 void
nouveau_device_del(struct nouveau_device ** pdev)494 nouveau_device_del(struct nouveau_device **pdev)
495 {
496 struct nouveau_device_priv *nvdev = nouveau_device(*pdev);
497 if (!nvdev)
498 return;
499
500 simple_mtx_destroy(&nvdev->lock);
501 free(nvdev);
502 *pdev = NULL;
503 }
504
505 int
nouveau_getparam(struct nouveau_device * dev,uint64_t param,uint64_t * value)506 nouveau_getparam(struct nouveau_device *dev, uint64_t param, uint64_t *value)
507 {
508 struct nouveau_drm *drm = nouveau_drm(&dev->object);
509 struct drm_nouveau_getparam r = { .param = param };
510 int ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GETPARAM, &r, sizeof(r));
511 *value = r.value;
512 return ret;
513 }
514
515 struct nouveau_client_kref {
516 struct drm_nouveau_gem_pushbuf_bo *kref;
517 struct nouveau_pushbuf *push;
518 };
519
520 struct nouveau_client_priv {
521 struct nouveau_client base;
522 struct nouveau_client_kref *kref;
523 unsigned kref_nr;
524 };
525
526 static inline struct nouveau_client_priv *
nouveau_client(struct nouveau_client * client)527 nouveau_client(struct nouveau_client *client)
528 {
529 return (struct nouveau_client_priv *)client;
530 }
531
532 int
nouveau_client_new(struct nouveau_device * dev,struct nouveau_client ** pclient)533 nouveau_client_new(struct nouveau_device *dev, struct nouveau_client **pclient)
534 {
535 struct nouveau_client_priv *pcli = calloc(1, sizeof(*pcli));
536 if (!pcli)
537 return -ENOMEM;
538
539 pcli->base.device = dev;
540 *pclient = &pcli->base;
541
542 return 0;
543 }
544
545 void
nouveau_client_del(struct nouveau_client ** pclient)546 nouveau_client_del(struct nouveau_client **pclient)
547 {
548 struct nouveau_client_priv *pcli = nouveau_client(*pclient);
549 if (pcli) {
550 free(pcli->kref);
551 free(pcli);
552 }
553 }
554
555 struct nouveau_bo_priv {
556 struct nouveau_bo base;
557 struct list_head head;
558 uint32_t refcnt;
559 uint64_t map_handle;
560 uint32_t name;
561 uint32_t access;
562 };
563
564 static inline struct nouveau_bo_priv *
nouveau_bo(struct nouveau_bo * bo)565 nouveau_bo(struct nouveau_bo *bo)
566 {
567 return (struct nouveau_bo_priv *)bo;
568 }
569
570 static void
bo_info(struct nouveau_bo * bo,struct drm_nouveau_gem_info * info)571 bo_info(struct nouveau_bo *bo, struct drm_nouveau_gem_info *info)
572 {
573 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
574
575 nvbo->map_handle = info->map_handle;
576 bo->handle = info->handle;
577 bo->size = info->size;
578 bo->offset = info->offset;
579
580 bo->flags = 0;
581 if (info->domain & NOUVEAU_GEM_DOMAIN_VRAM)
582 bo->flags |= NOUVEAU_BO_VRAM;
583 if (info->domain & NOUVEAU_GEM_DOMAIN_GART)
584 bo->flags |= NOUVEAU_BO_GART;
585 if (!(info->tile_flags & NOUVEAU_GEM_TILE_NONCONTIG))
586 bo->flags |= NOUVEAU_BO_CONTIG;
587 if (nvbo->map_handle)
588 bo->flags |= NOUVEAU_BO_MAP;
589
590 if (bo->device->chipset >= 0xc0) {
591 bo->config.nvc0.memtype = (info->tile_flags & 0xff00) >> 8;
592 bo->config.nvc0.tile_mode = info->tile_mode;
593 } else
594 if (bo->device->chipset >= 0x80 || bo->device->chipset == 0x50) {
595 bo->config.nv50.memtype = (info->tile_flags & 0x07f00) >> 8 |
596 (info->tile_flags & 0x30000) >> 9;
597 bo->config.nv50.tile_mode = info->tile_mode << 4;
598 } else {
599 //bo->config.nv04.surf_flags = info->tile_flags & 7;
600 //bo->config.nv04.surf_pitch = info->tile_mode;
601 }
602 }
603
604 static void
nouveau_bo_make_global(struct nouveau_bo_priv * nvbo)605 nouveau_bo_make_global(struct nouveau_bo_priv *nvbo)
606 {
607 if (!nvbo->head.next) {
608 struct nouveau_device_priv *nvdev = nouveau_device(nvbo->base.device);
609 simple_mtx_lock(&nvdev->lock);
610 if (!nvbo->head.next)
611 list_add(&nvbo->head, &nvdev->bo_list);
612 simple_mtx_unlock(&nvdev->lock);
613 }
614 }
615
616 static int
nouveau_bo_wrap_locked(struct nouveau_device * dev,uint32_t handle,struct nouveau_bo ** pbo,int name)617 nouveau_bo_wrap_locked(struct nouveau_device *dev, uint32_t handle,
618 struct nouveau_bo **pbo, int name)
619 {
620 struct nouveau_drm *drm = nouveau_drm(&dev->object);
621 struct nouveau_device_priv *nvdev = nouveau_device(dev);
622 struct drm_nouveau_gem_info req = { .handle = handle };
623 struct nouveau_bo_priv *nvbo;
624 int ret;
625
626 list_for_each_entry(struct nouveau_bo_priv, nvbo, &nvdev->bo_list, head) {
627 if (nvbo->base.handle == handle) {
628 if (p_atomic_inc_return(&nvbo->refcnt) == 1) {
629 /*
630 * Uh oh, this bo is dead and someone else
631 * will free it, but because refcnt is
632 * now non-zero fortunately they won't
633 * call the ioctl to close the bo.
634 *
635 * Remove this bo from the list so other
636 * calls to nouveau_bo_wrap_locked will
637 * see our replacement nvbo.
638 */
639 list_del(&nvbo->head);
640 if (!name)
641 name = nvbo->name;
642 break;
643 }
644
645 *pbo = &nvbo->base;
646 return 0;
647 }
648 }
649
650 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_INFO, &req, sizeof(req));
651 if (ret)
652 return ret;
653
654 nvbo = calloc(1, sizeof(*nvbo));
655 if (nvbo) {
656 p_atomic_set(&nvbo->refcnt, 1);
657 nvbo->base.device = dev;
658 bo_info(&nvbo->base, &req);
659 nvbo->name = name;
660 list_add(&nvbo->head, &nvdev->bo_list);
661 *pbo = &nvbo->base;
662 return 0;
663 }
664
665 return -ENOMEM;
666 }
667
668 int
nouveau_bo_new(struct nouveau_device * dev,uint32_t flags,uint32_t align,uint64_t size,union nouveau_bo_config * config,struct nouveau_bo ** pbo)669 nouveau_bo_new(struct nouveau_device *dev, uint32_t flags, uint32_t align, uint64_t size,
670 union nouveau_bo_config *config, struct nouveau_bo **pbo)
671 {
672 struct nouveau_drm *drm = nouveau_drm(&dev->object);
673 struct drm_nouveau_gem_new req = {};
674 struct drm_nouveau_gem_info *info = &req.info;
675 int ret;
676
677 struct nouveau_bo_priv *nvbo = calloc(1, sizeof(*nvbo));
678 if (!nvbo)
679 return -ENOMEM;
680
681 struct nouveau_bo *bo = &nvbo->base;
682 p_atomic_set(&nvbo->refcnt, 1);
683 bo->device = dev;
684 bo->flags = flags;
685 bo->size = size;
686
687 if (bo->flags & NOUVEAU_BO_VRAM)
688 info->domain |= NOUVEAU_GEM_DOMAIN_VRAM;
689 if (bo->flags & NOUVEAU_BO_GART)
690 info->domain |= NOUVEAU_GEM_DOMAIN_GART;
691 if (!info->domain)
692 info->domain |= NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART;
693
694 if (bo->flags & NOUVEAU_BO_MAP)
695 info->domain |= NOUVEAU_GEM_DOMAIN_MAPPABLE;
696
697 if (bo->flags & NOUVEAU_BO_COHERENT)
698 info->domain |= NOUVEAU_GEM_DOMAIN_COHERENT;
699
700 if (!(bo->flags & NOUVEAU_BO_CONTIG))
701 info->tile_flags = NOUVEAU_GEM_TILE_NONCONTIG;
702
703 info->size = bo->size;
704 req.align = align;
705
706 if (config) {
707 if (dev->chipset >= 0xc0) {
708 info->tile_flags = (config->nvc0.memtype & 0xff) << 8;
709 info->tile_mode = config->nvc0.tile_mode;
710 } else
711 if (dev->chipset >= 0x80 || dev->chipset == 0x50) {
712 info->tile_flags = (config->nv50.memtype & 0x07f) << 8 |
713 (config->nv50.memtype & 0x180) << 9;
714 info->tile_mode = config->nv50.tile_mode >> 4;
715 }
716 }
717
718 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_NEW, &req, sizeof(req));
719 if (ret) {
720 free(nvbo);
721 return ret;
722 }
723
724 bo_info(bo, info);
725
726 *pbo = bo;
727 return 0;
728 }
729
730 static void
nouveau_bo_del(struct nouveau_bo * bo)731 nouveau_bo_del(struct nouveau_bo *bo)
732 {
733 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
734 struct nouveau_device_priv *nvdev = nouveau_device(bo->device);
735 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
736
737 if (nvbo->head.next) {
738 simple_mtx_lock(&nvdev->lock);
739 if (p_atomic_read(&nvbo->refcnt) == 0) {
740 list_del(&nvbo->head);
741 drmCloseBufferHandle(drm->fd, bo->handle);
742 }
743 simple_mtx_unlock(&nvdev->lock);
744 } else {
745 drmCloseBufferHandle(drm->fd, bo->handle);
746 }
747 if (bo->map)
748 os_munmap(bo->map, bo->size);
749 free(nvbo);
750 }
751
752 int
nouveau_bo_map(struct nouveau_bo * bo,uint32_t access,struct nouveau_client * client)753 nouveau_bo_map(struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client)
754 {
755 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
756 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
757 if (bo->map == NULL) {
758 bo->map = os_mmap(0, bo->size, PROT_READ | PROT_WRITE, MAP_SHARED, drm->fd, nvbo->map_handle);
759 if (bo->map == MAP_FAILED) {
760 bo->map = NULL;
761 return -errno;
762 }
763 }
764 return nouveau_bo_wait(bo, access, client);
765 }
766
767 int
nouveau_bo_name_get(struct nouveau_bo * bo,uint32_t * name)768 nouveau_bo_name_get(struct nouveau_bo *bo, uint32_t *name)
769 {
770 struct drm_gem_flink req = { .handle = bo->handle };
771 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
772 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
773
774 *name = nvbo->name;
775 if (!*name) {
776 int ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_FLINK, &req);
777
778 if (ret) {
779 *name = 0;
780 return ret;
781 }
782 nvbo->name = *name = req.name;
783
784 nouveau_bo_make_global(nvbo);
785 }
786 return 0;
787 }
788
789 int
nouveau_bo_name_ref(struct nouveau_device * dev,uint32_t name,struct nouveau_bo ** pbo)790 nouveau_bo_name_ref(struct nouveau_device *dev, uint32_t name, struct nouveau_bo **pbo)
791 {
792 struct nouveau_drm *drm = nouveau_drm(&dev->object);
793 struct nouveau_device_priv *nvdev = nouveau_device(dev);
794 struct drm_gem_open req = { .name = name };
795 int ret;
796
797 simple_mtx_lock(&nvdev->lock);
798 list_for_each_entry(struct nouveau_bo_priv, nvbo, &nvdev->bo_list, head) {
799 if (nvbo->name == name) {
800 ret = nouveau_bo_wrap_locked(dev, nvbo->base.handle, pbo, name);
801 simple_mtx_unlock(&nvdev->lock);
802 return ret;
803 }
804 }
805
806 ret = drmIoctl(drm->fd, DRM_IOCTL_GEM_OPEN, &req);
807 if (ret == 0) {
808 ret = nouveau_bo_wrap_locked(dev, req.handle, pbo, name);
809 }
810
811 simple_mtx_unlock(&nvdev->lock);
812 return ret;
813 }
814
815 int
nouveau_bo_prime_handle_ref(struct nouveau_device * dev,int prime_fd,struct nouveau_bo ** bo)816 nouveau_bo_prime_handle_ref(struct nouveau_device *dev, int prime_fd, struct nouveau_bo **bo)
817 {
818 struct nouveau_drm *drm = nouveau_drm(&dev->object);
819 struct nouveau_device_priv *nvdev = nouveau_device(dev);
820 int ret;
821 unsigned int handle;
822
823 nouveau_bo_ref(NULL, bo);
824
825 simple_mtx_lock(&nvdev->lock);
826 ret = drmPrimeFDToHandle(drm->fd, prime_fd, &handle);
827 if (ret == 0) {
828 ret = nouveau_bo_wrap_locked(dev, handle, bo, 0);
829 }
830 simple_mtx_unlock(&nvdev->lock);
831 return ret;
832 }
833
834 void
nouveau_bo_ref(struct nouveau_bo * bo,struct nouveau_bo ** pref)835 nouveau_bo_ref(struct nouveau_bo *bo, struct nouveau_bo **pref)
836 {
837 struct nouveau_bo *ref = *pref;
838 if (bo) {
839 p_atomic_inc(&nouveau_bo(bo)->refcnt);
840 }
841 if (ref) {
842 if (p_atomic_dec_zero(&nouveau_bo(ref)->refcnt))
843 nouveau_bo_del(ref);
844 }
845 *pref = bo;
846 }
847
848 int
nouveau_bo_set_prime(struct nouveau_bo * bo,int * prime_fd)849 nouveau_bo_set_prime(struct nouveau_bo *bo, int *prime_fd)
850 {
851 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
852 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
853 int ret;
854
855 ret = drmPrimeHandleToFD(drm->fd, nvbo->base.handle, O_CLOEXEC, prime_fd);
856 if (ret)
857 return ret;
858
859 nouveau_bo_make_global(nvbo);
860 return 0;
861 }
862
863 static inline struct nouveau_pushbuf *
cli_push_get(struct nouveau_client * client,struct nouveau_bo * bo)864 cli_push_get(struct nouveau_client *client, struct nouveau_bo *bo)
865 {
866 struct nouveau_client_priv *pcli = nouveau_client(client);
867 struct nouveau_pushbuf *push = NULL;
868 if (pcli->kref_nr > bo->handle)
869 push = pcli->kref[bo->handle].push;
870 return push;
871 }
872
873 static inline struct drm_nouveau_gem_pushbuf_bo *
cli_kref_get(struct nouveau_client * client,struct nouveau_bo * bo)874 cli_kref_get(struct nouveau_client *client, struct nouveau_bo *bo)
875 {
876 struct nouveau_client_priv *pcli = nouveau_client(client);
877 struct drm_nouveau_gem_pushbuf_bo *kref = NULL;
878 if (pcli->kref_nr > bo->handle)
879 kref = pcli->kref[bo->handle].kref;
880 return kref;
881 }
882
883 static inline int
cli_kref_set(struct nouveau_client * client,struct nouveau_bo * bo,struct drm_nouveau_gem_pushbuf_bo * kref,struct nouveau_pushbuf * push)884 cli_kref_set(struct nouveau_client *client, struct nouveau_bo *bo,
885 struct drm_nouveau_gem_pushbuf_bo *kref, struct nouveau_pushbuf *push)
886 {
887 struct nouveau_client_priv *pcli = nouveau_client(client);
888 if (pcli->kref_nr <= bo->handle) {
889 void *new_ptr = realloc(pcli->kref, sizeof(*pcli->kref) * bo->handle * 2);
890 if (!new_ptr) {
891 err("Failed to realloc memory, expect faulty rendering.\n");
892 return -ENOMEM;
893 }
894 pcli->kref = new_ptr;
895 while (pcli->kref_nr < bo->handle * 2) {
896 pcli->kref[pcli->kref_nr].kref = NULL;
897 pcli->kref[pcli->kref_nr].push = NULL;
898 pcli->kref_nr++;
899 }
900 }
901 pcli->kref[bo->handle].kref = kref;
902 pcli->kref[bo->handle].push = push;
903 return 0;
904 }
905
906 int
nouveau_bo_wait(struct nouveau_bo * bo,uint32_t access,struct nouveau_client * client)907 nouveau_bo_wait(struct nouveau_bo *bo, uint32_t access, struct nouveau_client *client)
908 {
909 struct nouveau_drm *drm = nouveau_drm(&bo->device->object);
910 struct nouveau_bo_priv *nvbo = nouveau_bo(bo);
911 struct drm_nouveau_gem_cpu_prep req;
912 struct nouveau_pushbuf *push;
913 int ret = 0;
914
915 if (!(access & NOUVEAU_BO_RDWR))
916 return 0;
917
918 push = cli_push_get(client, bo);
919 if (push)
920 nouveau_pushbuf_kick(push);
921
922 if (!nvbo->head.next && !(nvbo->access & NOUVEAU_BO_WR) && !(access & NOUVEAU_BO_WR))
923 return 0;
924
925 req.handle = bo->handle;
926 req.flags = 0;
927 if (access & NOUVEAU_BO_WR)
928 req.flags |= NOUVEAU_GEM_CPU_PREP_WRITE;
929 if (access & NOUVEAU_BO_NOBLOCK)
930 req.flags |= NOUVEAU_GEM_CPU_PREP_NOWAIT;
931
932 ret = drmCommandWrite(drm->fd, DRM_NOUVEAU_GEM_CPU_PREP, &req, sizeof(req));
933 if (ret == 0)
934 nvbo->access = 0;
935 return ret;
936 }
937
938 int
nouveau_bo_wrap(struct nouveau_device * dev,uint32_t handle,struct nouveau_bo ** pbo)939 nouveau_bo_wrap(struct nouveau_device *dev, uint32_t handle, struct nouveau_bo **pbo)
940 {
941 struct nouveau_device_priv *nvdev = nouveau_device(dev);
942 int ret;
943 simple_mtx_lock(&nvdev->lock);
944 ret = nouveau_bo_wrap_locked(dev, handle, pbo, 0);
945 simple_mtx_unlock(&nvdev->lock);
946 return ret;
947 }
948
949 struct nouveau_bufref_priv {
950 struct nouveau_bufref base;
951 struct nouveau_bufref_priv *next;
952 struct nouveau_bufctx *bufctx;
953 };
954
955 struct nouveau_bufbin_priv {
956 struct nouveau_bufref_priv *list;
957 int relocs;
958 };
959
960 struct nouveau_bufctx_priv {
961 struct nouveau_bufctx base;
962 struct nouveau_bufref_priv *free;
963 int nr_bins;
964 struct nouveau_bufbin_priv bins[];
965 };
966
967 static inline struct nouveau_bufctx_priv *
nouveau_bufctx(struct nouveau_bufctx * bctx)968 nouveau_bufctx(struct nouveau_bufctx *bctx)
969 {
970 return (struct nouveau_bufctx_priv *)bctx;
971 }
972
973 int
nouveau_bufctx_new(struct nouveau_client * client,int bins,struct nouveau_bufctx ** pbctx)974 nouveau_bufctx_new(struct nouveau_client *client, int bins, struct nouveau_bufctx **pbctx)
975 {
976 struct nouveau_bufctx_priv *priv;
977
978 priv = CALLOC_VARIANT_LENGTH_STRUCT(nouveau_bufctx_priv, sizeof(priv->bins[0]) * bins);
979 if (priv) {
980 list_inithead(&priv->base.head);
981 list_inithead(&priv->base.pending);
982 list_inithead(&priv->base.current);
983 priv->base.client = client;
984 priv->nr_bins = bins;
985 *pbctx = &priv->base;
986 return 0;
987 }
988
989 return -ENOMEM;
990 }
991
992 void
nouveau_bufctx_del(struct nouveau_bufctx ** pbctx)993 nouveau_bufctx_del(struct nouveau_bufctx **pbctx)
994 {
995 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(*pbctx);
996 struct nouveau_bufref_priv *pref;
997 if (pctx) {
998 while (pctx->nr_bins--)
999 nouveau_bufctx_reset(&pctx->base, pctx->nr_bins);
1000 while ((pref = pctx->free)) {
1001 pctx->free = pref->next;
1002 free(pref);
1003 }
1004 free(pctx);
1005 *pbctx = NULL;
1006 }
1007 }
1008
1009 struct nouveau_bufref *
nouveau_bufctx_mthd(struct nouveau_bufctx * bctx,int bin,uint32_t packet,struct nouveau_bo * bo,uint64_t data,uint32_t flags,uint32_t vor,uint32_t tor)1010 nouveau_bufctx_mthd(struct nouveau_bufctx *bctx, int bin, uint32_t packet, struct nouveau_bo *bo,
1011 uint64_t data, uint32_t flags, uint32_t vor, uint32_t tor)
1012 {
1013 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx);
1014 struct nouveau_bufbin_priv *pbin = &pctx->bins[bin];
1015 struct nouveau_bufref *bref = nouveau_bufctx_refn(bctx, bin, bo, flags);
1016 if (bref) {
1017 bref->packet = packet;
1018 bref->data = data;
1019 bref->vor = vor;
1020 bref->tor = tor;
1021 pbin->relocs++;
1022 bctx->relocs++;
1023 }
1024 return bref;
1025 }
1026
1027 struct nouveau_bufref *
nouveau_bufctx_refn(struct nouveau_bufctx * bctx,int bin,struct nouveau_bo * bo,uint32_t flags)1028 nouveau_bufctx_refn(struct nouveau_bufctx *bctx, int bin,
1029 struct nouveau_bo *bo, uint32_t flags)
1030 {
1031 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx);
1032 struct nouveau_bufbin_priv *pbin = &pctx->bins[bin];
1033 struct nouveau_bufref_priv *pref = pctx->free;
1034
1035 if (!pref)
1036 pref = malloc(sizeof(*pref));
1037 else
1038 pctx->free = pref->next;
1039
1040 if (!pref)
1041 return NULL;
1042
1043 pref->base.bo = bo;
1044 pref->base.flags = flags;
1045 pref->base.packet = 0;
1046
1047 list_addtail(&pref->base.thead, &bctx->pending);
1048 pref->bufctx = bctx;
1049 pref->next = pbin->list;
1050 pbin->list = pref;
1051
1052 return &pref->base;
1053 }
1054
1055 struct nouveau_pushbuf_krec {
1056 struct nouveau_pushbuf_krec *next;
1057 struct drm_nouveau_gem_pushbuf_bo buffer[NOUVEAU_GEM_MAX_BUFFERS];
1058 struct drm_nouveau_gem_pushbuf_reloc reloc[NOUVEAU_GEM_MAX_RELOCS];
1059 struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH];
1060 int nr_buffer;
1061 int nr_reloc;
1062 int nr_push;
1063 uint64_t vram_used;
1064 uint64_t gart_used;
1065 };
1066
1067 struct nouveau_pushbuf_priv {
1068 struct nouveau_pushbuf base;
1069 struct nouveau_pushbuf_krec *list;
1070 struct nouveau_pushbuf_krec *krec;
1071 struct list_head bctx_list;
1072 struct nouveau_bo *bo;
1073 uint32_t type;
1074 uint32_t suffix0;
1075 uint32_t suffix1;
1076 uint32_t *ptr;
1077 uint32_t *bgn;
1078 int bo_next;
1079 int bo_nr;
1080 struct nouveau_bo *bos[];
1081 };
1082
1083 static inline struct nouveau_pushbuf_priv *
nouveau_pushbuf(struct nouveau_pushbuf * push)1084 nouveau_pushbuf(struct nouveau_pushbuf *push)
1085 {
1086 return (struct nouveau_pushbuf_priv *)push;
1087 }
1088
1089 static void
pushbuf_dump(struct nouveau_device * dev,struct nouveau_pushbuf_krec * krec,int krec_id,int chid)1090 pushbuf_dump(struct nouveau_device *dev,
1091 struct nouveau_pushbuf_krec *krec, int krec_id, int chid)
1092 {
1093 struct drm_nouveau_gem_pushbuf_reloc *krel;
1094 struct drm_nouveau_gem_pushbuf_push *kpsh;
1095 struct drm_nouveau_gem_pushbuf_bo *kref;
1096 struct nouveau_bo *bo;
1097 uint32_t *bgn, *end;
1098 int i;
1099
1100 err("ch%d: krec %d pushes %d bufs %d relocs %d\n",
1101 chid, krec_id, krec->nr_push, krec->nr_buffer, krec->nr_reloc);
1102
1103 kref = krec->buffer;
1104 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1105 bo = (void *)(uintptr_t)kref->user_priv;
1106 err("ch%d: buf %08x %08x %08x %08x %08x %p 0x%"PRIx64" 0x%"PRIx64"\n",
1107 chid, i, kref->handle, kref->valid_domains,
1108 kref->read_domains, kref->write_domains, bo->map, bo->offset, bo->size);
1109 }
1110
1111 krel = krec->reloc;
1112 for (i = 0; i < krec->nr_reloc; i++, krel++) {
1113 err("ch%d: rel %08x %08x %08x %08x %08x %08x %08x\n",
1114 chid, krel->reloc_bo_index, krel->reloc_bo_offset,
1115 krel->bo_index, krel->flags, krel->data,
1116 krel->vor, krel->tor);
1117 }
1118
1119 kpsh = krec->push;
1120 for (i = 0; i < krec->nr_push; i++, kpsh++) {
1121 kref = krec->buffer + kpsh->bo_index;
1122 bo = (void *)(unsigned long)kref->user_priv;
1123 bgn = (uint32_t *)((char *)bo->map + kpsh->offset);
1124 end = bgn + ((kpsh->length & 0x7fffff) /4);
1125
1126 err("ch%d: psh %s%08x %010llx %010llx\n", chid,
1127 bo->map ? "" : "(unmapped) ", kpsh->bo_index,
1128 (unsigned long long)kpsh->offset,
1129 (unsigned long long)(kpsh->offset + kpsh->length));
1130 if (!bo->map)
1131 continue;
1132
1133 if (dev->info.cls_eng3d) {
1134 struct nv_push push = {
1135 .start = bgn,
1136 .end = end
1137 };
1138 vk_push_print(nouveau_out, &push, &dev->info);
1139 } else {
1140 while (bgn < end)
1141 err("\t0x%08x\n", *bgn++);
1142 }
1143 }
1144 }
1145
1146 static int
pushbuf_submit(struct nouveau_pushbuf * push,struct nouveau_object * chan)1147 pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan)
1148 {
1149 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1150 struct nouveau_pushbuf_krec *krec = nvpb->list;
1151 struct nouveau_device *dev = push->client->device;
1152 struct nouveau_drm *drm = nouveau_drm(&dev->object);
1153 struct drm_nouveau_gem_pushbuf_bo_presumed *info;
1154 struct drm_nouveau_gem_pushbuf_bo *kref;
1155 struct drm_nouveau_gem_pushbuf req;
1156 int channel = chan->handle;
1157 struct nouveau_bo *bo;
1158 int krec_id = 0;
1159 int ret = 0, i;
1160
1161 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS)
1162 return -EINVAL;
1163
1164 if (push->kick_notify)
1165 push->kick_notify(push);
1166
1167 nouveau_pushbuf_data(push, NULL, 0, 0);
1168
1169 while (krec && krec->nr_push) {
1170 req.channel = channel;
1171 req.nr_buffers = krec->nr_buffer;
1172 req.buffers = (uint64_t)(unsigned long)krec->buffer;
1173 req.nr_relocs = krec->nr_reloc;
1174 req.nr_push = krec->nr_push;
1175 req.relocs = (uint64_t)(unsigned long)krec->reloc;
1176 req.push = (uint64_t)(unsigned long)krec->push;
1177 req.suffix0 = nvpb->suffix0;
1178 req.suffix1 = nvpb->suffix1;
1179 req.vram_available = 0;
1180 if (dbg_on(1))
1181 req.vram_available |= NOUVEAU_GEM_PUSHBUF_SYNC;
1182 req.gart_available = 0;
1183
1184 if (dbg_on(0))
1185 pushbuf_dump(dev, krec, krec_id++, channel);
1186
1187 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, &req, sizeof(req));
1188 nvpb->suffix0 = req.suffix0;
1189 nvpb->suffix1 = req.suffix1;
1190 dev->vram_limit = (req.vram_available * nouveau_device(dev)->vram_limit_percent) / 100;
1191 dev->gart_limit = (req.gart_available * nouveau_device(dev)->gart_limit_percent) / 100;
1192
1193 if (ret) {
1194 err("kernel rejected pushbuf: %s\n", strerror(-ret));
1195 pushbuf_dump(dev, krec, krec_id++, channel);
1196 break;
1197 }
1198
1199 kref = krec->buffer;
1200 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1201 bo = (void *)(unsigned long)kref->user_priv;
1202
1203 info = &kref->presumed;
1204 if (!info->valid) {
1205 bo->flags &= ~NOUVEAU_BO_APER;
1206 if (info->domain == NOUVEAU_GEM_DOMAIN_VRAM)
1207 bo->flags |= NOUVEAU_BO_VRAM;
1208 else
1209 bo->flags |= NOUVEAU_BO_GART;
1210 bo->offset = info->offset;
1211 }
1212
1213 if (kref->write_domains)
1214 nouveau_bo(bo)->access |= NOUVEAU_BO_WR;
1215 if (kref->read_domains)
1216 nouveau_bo(bo)->access |= NOUVEAU_BO_RD;
1217 }
1218
1219 krec = krec->next;
1220 }
1221
1222 return ret;
1223 }
1224
1225 static int
pushbuf_flush(struct nouveau_pushbuf * push)1226 pushbuf_flush(struct nouveau_pushbuf *push)
1227 {
1228 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1229 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1230 struct drm_nouveau_gem_pushbuf_bo *kref;
1231 int i;
1232
1233 int ret = pushbuf_submit(push, push->channel);
1234
1235 kref = krec->buffer;
1236 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1237 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv;
1238 ret = cli_kref_set(push->client, bo, NULL, NULL);
1239 if (ret)
1240 return ret;
1241 nouveau_bo_ref(NULL, &bo);
1242 }
1243
1244 krec = nvpb->krec;
1245 krec->vram_used = 0;
1246 krec->gart_used = 0;
1247 krec->nr_buffer = 0;
1248 krec->nr_reloc = 0;
1249 krec->nr_push = 0;
1250
1251 list_for_each_entry_safe(struct nouveau_bufctx, bctx, &nvpb->bctx_list, head) {
1252 list_splice(&bctx->current, &bctx->pending);
1253 list_inithead(&bctx->current);
1254 list_delinit(&bctx->head);
1255 }
1256
1257 return ret;
1258 }
1259
1260 static bool
pushbuf_kref_fits(struct nouveau_pushbuf * push,struct nouveau_bo * bo,uint32_t * domains)1261 pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t *domains)
1262 {
1263 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1264 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1265 struct nouveau_device *dev = push->client->device;
1266 struct nouveau_bo *kbo;
1267 struct drm_nouveau_gem_pushbuf_bo *kref;
1268 int i;
1269
1270 /* VRAM is the only valid domain. GART and VRAM|GART buffers
1271 * are all accounted to GART, so if this doesn't fit in VRAM
1272 * straight up, a flush is needed.
1273 */
1274 if (*domains == NOUVEAU_GEM_DOMAIN_VRAM) {
1275 if (krec->vram_used + bo->size > dev->vram_limit)
1276 return false;
1277 krec->vram_used += bo->size;
1278 return true;
1279 }
1280
1281 /* GART or VRAM|GART buffer. Account both of these buffer types
1282 * to GART only for the moment, which simplifies things. If the
1283 * buffer can fit already, we're done here.
1284 */
1285 if (krec->gart_used + bo->size <= dev->gart_limit) {
1286 krec->gart_used += bo->size;
1287 return true;
1288 }
1289
1290 /* Ran out of GART space, if it's a VRAM|GART buffer and it'll
1291 * fit into available VRAM, turn it into a VRAM buffer
1292 */
1293 if ((*domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
1294 krec->vram_used + bo->size <= dev->vram_limit) {
1295 *domains &= NOUVEAU_GEM_DOMAIN_VRAM;
1296 krec->vram_used += bo->size;
1297 return true;
1298 }
1299
1300 /* Still couldn't fit the buffer in anywhere, so as a last resort;
1301 * scan the buffer list for VRAM|GART buffers and turn them into
1302 * VRAM buffers until we have enough space in GART for this one
1303 */
1304 kref = krec->buffer;
1305 for (i = 0; i < krec->nr_buffer; i++, kref++) {
1306 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
1307 continue;
1308
1309 kbo = (void *)(unsigned long)kref->user_priv;
1310 if (!(kref->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) ||
1311 krec->vram_used + kbo->size > dev->vram_limit)
1312 continue;
1313
1314 kref->valid_domains &= NOUVEAU_GEM_DOMAIN_VRAM;
1315 krec->gart_used -= kbo->size;
1316 krec->vram_used += kbo->size;
1317 if (krec->gart_used + bo->size <= dev->gart_limit) {
1318 krec->gart_used += bo->size;
1319 return true;
1320 }
1321 }
1322
1323 /* Couldn't resolve a placement, need to force a flush */
1324 return false;
1325 }
1326
1327 static struct drm_nouveau_gem_pushbuf_bo *
pushbuf_kref(struct nouveau_pushbuf * push,struct nouveau_bo * bo,uint32_t flags)1328 pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t flags)
1329 {
1330 struct nouveau_device *dev = push->client->device;
1331 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1332 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1333 struct nouveau_pushbuf *fpush;
1334 struct drm_nouveau_gem_pushbuf_bo *kref;
1335 uint32_t domains, domains_wr, domains_rd;
1336 int ret;
1337
1338 domains = 0;
1339 if (flags & NOUVEAU_BO_VRAM)
1340 domains |= NOUVEAU_GEM_DOMAIN_VRAM;
1341 if (flags & NOUVEAU_BO_GART)
1342 domains |= NOUVEAU_GEM_DOMAIN_GART;
1343 domains_wr = domains * !!(flags & NOUVEAU_BO_WR);
1344 domains_rd = domains * !!(flags & NOUVEAU_BO_RD);
1345
1346 /* if buffer is referenced on another pushbuf that is owned by the
1347 * same client, we need to flush the other pushbuf first to ensure
1348 * the correct ordering of commands
1349 */
1350 fpush = cli_push_get(push->client, bo);
1351 if (fpush && fpush != push)
1352 pushbuf_flush(fpush);
1353
1354 kref = cli_kref_get(push->client, bo);
1355 if (kref) {
1356 /* possible conflict in memory types - flush and retry */
1357 if (!(kref->valid_domains & domains))
1358 return NULL;
1359
1360 /* VRAM|GART buffer turning into a VRAM buffer. Make sure
1361 * it'll fit in VRAM and force a flush if not.
1362 */
1363 if ((kref->valid_domains & NOUVEAU_GEM_DOMAIN_GART) &&
1364 ( domains == NOUVEAU_GEM_DOMAIN_VRAM)) {
1365 if (krec->vram_used + bo->size > dev->vram_limit)
1366 return NULL;
1367 krec->vram_used += bo->size;
1368 krec->gart_used -= bo->size;
1369 }
1370
1371 kref->valid_domains &= domains;
1372 kref->write_domains |= domains_wr;
1373 kref->read_domains |= domains_rd;
1374 } else {
1375 if (krec->nr_buffer == NOUVEAU_GEM_MAX_BUFFERS ||
1376 !pushbuf_kref_fits(push, bo, &domains))
1377 return NULL;
1378
1379 kref = &krec->buffer[krec->nr_buffer++];
1380 kref->user_priv = (unsigned long)bo;
1381 kref->handle = bo->handle;
1382 kref->valid_domains = domains;
1383 kref->write_domains = domains_wr;
1384 kref->read_domains = domains_rd;
1385 kref->presumed.valid = 1;
1386 kref->presumed.offset = bo->offset;
1387 if (bo->flags & NOUVEAU_BO_VRAM)
1388 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
1389 else
1390 kref->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
1391
1392 ret = cli_kref_set(push->client, bo, kref, push);
1393 if (ret)
1394 return NULL;
1395 p_atomic_inc(&nouveau_bo(bo)->refcnt);
1396 }
1397
1398 return kref;
1399 }
1400
1401 static uint32_t
pushbuf_krel(struct nouveau_pushbuf * push,struct nouveau_bo * bo,uint32_t data,uint32_t flags,uint32_t vor,uint32_t tor)1402 pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo,
1403 uint32_t data, uint32_t flags, uint32_t vor, uint32_t tor)
1404 {
1405 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1406 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1407 struct drm_nouveau_gem_pushbuf_reloc *krel;
1408 struct drm_nouveau_gem_pushbuf_bo *pkref;
1409 struct drm_nouveau_gem_pushbuf_bo *bkref;
1410 uint32_t reloc = data;
1411
1412 pkref = cli_kref_get(push->client, nvpb->bo);
1413 bkref = cli_kref_get(push->client, bo);
1414 krel = &krec->reloc[krec->nr_reloc++];
1415
1416 assert(pkref);
1417 assert(bkref);
1418 krel->reloc_bo_index = pkref - krec->buffer;
1419 krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4;
1420 krel->bo_index = bkref - krec->buffer;
1421 krel->flags = 0;
1422 krel->data = data;
1423 krel->vor = vor;
1424 krel->tor = tor;
1425
1426 if (flags & NOUVEAU_BO_LOW) {
1427 reloc = (bkref->presumed.offset + data);
1428 krel->flags |= NOUVEAU_GEM_RELOC_LOW;
1429 }
1430 if (flags & NOUVEAU_BO_OR) {
1431 if (bkref->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM)
1432 reloc |= vor;
1433 else
1434 reloc |= tor;
1435 krel->flags |= NOUVEAU_GEM_RELOC_OR;
1436 }
1437
1438 return reloc;
1439 }
1440
1441 static int
pushbuf_refn_fail(struct nouveau_pushbuf * push,int sref,int srel)1442 pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel)
1443 {
1444 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1445 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1446 struct drm_nouveau_gem_pushbuf_bo *kref;
1447 int ret;
1448
1449 kref = krec->buffer + sref;
1450 while (krec->nr_buffer-- > sref) {
1451 struct nouveau_bo *bo = (void *)(unsigned long)kref->user_priv;
1452 ret = cli_kref_set(push->client, bo, NULL, NULL);
1453 if (ret)
1454 return ret;
1455 nouveau_bo_ref(NULL, &bo);
1456 kref++;
1457 }
1458 krec->nr_buffer = sref;
1459 krec->nr_reloc = srel;
1460 return 0;
1461 }
1462
1463 static int
pushbuf_refn(struct nouveau_pushbuf * push,bool retry,struct nouveau_pushbuf_refn * refs,int nr)1464 pushbuf_refn(struct nouveau_pushbuf *push, bool retry,
1465 struct nouveau_pushbuf_refn *refs, int nr)
1466 {
1467 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1468 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1469 struct drm_nouveau_gem_pushbuf_bo *kref;
1470 int sref = krec->nr_buffer;
1471 int ret = 0, i;
1472
1473 for (i = 0; i < nr; i++) {
1474 kref = pushbuf_kref(push, refs[i].bo, refs[i].flags);
1475 if (!kref) {
1476 ret = -ENOSPC;
1477 break;
1478 }
1479 }
1480
1481 if (ret) {
1482 ret = pushbuf_refn_fail(push, sref, krec->nr_reloc);
1483 if (ret)
1484 return ret;
1485 if (retry) {
1486 pushbuf_flush(push);
1487 nouveau_pushbuf_space(push, 0, 0, 0);
1488 return pushbuf_refn(push, false, refs, nr);
1489 }
1490 }
1491
1492 return ret;
1493 }
1494
1495 static int
pushbuf_validate(struct nouveau_pushbuf * push,bool retry)1496 pushbuf_validate(struct nouveau_pushbuf *push, bool retry)
1497 {
1498 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1499 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1500 struct drm_nouveau_gem_pushbuf_bo *kref;
1501 struct nouveau_bufctx *bctx = push->bufctx;
1502 int relocs = bctx ? bctx->relocs * 2: 0;
1503 int sref, srel, ret;
1504
1505 ret = nouveau_pushbuf_space(push, relocs, relocs, 0);
1506 if (ret || bctx == NULL)
1507 return ret;
1508
1509 sref = krec->nr_buffer;
1510 srel = krec->nr_reloc;
1511
1512 list_del(&bctx->head);
1513 list_add(&bctx->head, &nvpb->bctx_list);
1514
1515 list_for_each_entry(struct nouveau_bufref, bref, &bctx->pending, thead) {
1516 kref = pushbuf_kref(push, bref->bo, bref->flags);
1517 if (!kref) {
1518 ret = -ENOSPC;
1519 break;
1520 }
1521
1522 if (bref->packet) {
1523 pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0);
1524 *push->cur++ = 0;
1525 pushbuf_krel(push, bref->bo, bref->data, bref->flags,
1526 bref->vor, bref->tor);
1527 *push->cur++ = 0;
1528 }
1529 }
1530
1531 list_splice(&bctx->pending, &bctx->current);
1532 list_inithead(&bctx->pending);
1533
1534 if (ret) {
1535 ret = pushbuf_refn_fail(push, sref, srel);
1536 if (ret)
1537 return ret;
1538 if (retry) {
1539 pushbuf_flush(push);
1540 return pushbuf_validate(push, false);
1541 }
1542 }
1543
1544 return ret;
1545 }
1546
1547 int
nouveau_pushbuf_new(struct nouveau_client * client,struct nouveau_object * chan,int nr,uint32_t size,struct nouveau_pushbuf ** ppush)1548 nouveau_pushbuf_new(struct nouveau_client *client, struct nouveau_object *chan, int nr,
1549 uint32_t size, struct nouveau_pushbuf **ppush)
1550 {
1551 struct nouveau_drm *drm = nouveau_drm(&client->device->object);
1552 struct nouveau_fifo *fifo = chan->data;
1553 struct nouveau_pushbuf_priv *nvpb;
1554 struct nouveau_pushbuf *push;
1555 struct drm_nouveau_gem_pushbuf req = {};
1556 int ret;
1557
1558 if (chan->oclass != NOUVEAU_FIFO_CHANNEL_CLASS)
1559 return -EINVAL;
1560
1561 /* nop pushbuf call, to get the current "return to main" sequence
1562 * we need to append to the pushbuf on early chipsets
1563 */
1564 req.channel = chan->handle;
1565 req.nr_push = 0;
1566 ret = drmCommandWriteRead(drm->fd, DRM_NOUVEAU_GEM_PUSHBUF, &req, sizeof(req));
1567 if (ret)
1568 return ret;
1569
1570 nvpb = calloc(1, sizeof(*nvpb) + nr * sizeof(*nvpb->bos));
1571 if (!nvpb)
1572 return -ENOMEM;
1573
1574 nvpb->suffix0 = req.suffix0;
1575 nvpb->suffix1 = req.suffix1;
1576 nvpb->krec = calloc(1, sizeof(*nvpb->krec));
1577 nvpb->list = nvpb->krec;
1578 if (!nvpb->krec) {
1579 free(nvpb);
1580 return -ENOMEM;
1581 }
1582
1583 push = &nvpb->base;
1584 push->client = client;
1585 push->channel = chan;
1586 push->flags = NOUVEAU_BO_RD;
1587 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_GART) {
1588 push->flags |= NOUVEAU_BO_GART;
1589 nvpb->type = NOUVEAU_BO_GART;
1590 } else
1591 if (fifo->pushbuf & NOUVEAU_GEM_DOMAIN_VRAM) {
1592 push->flags |= NOUVEAU_BO_VRAM;
1593 nvpb->type = NOUVEAU_BO_VRAM;
1594 }
1595 nvpb->type |= NOUVEAU_BO_MAP;
1596
1597 for (nvpb->bo_nr = 0; nvpb->bo_nr < nr; nvpb->bo_nr++) {
1598 ret = nouveau_bo_new(client->device, nvpb->type, 0, size, NULL, &nvpb->bos[nvpb->bo_nr]);
1599 if (ret) {
1600 nouveau_pushbuf_del(&push);
1601 return ret;
1602 }
1603 }
1604
1605 list_inithead(&nvpb->bctx_list);
1606 *ppush = push;
1607 return 0;
1608 }
1609
1610 void
nouveau_pushbuf_del(struct nouveau_pushbuf ** ppush)1611 nouveau_pushbuf_del(struct nouveau_pushbuf **ppush)
1612 {
1613 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(*ppush);
1614 if (nvpb) {
1615 struct drm_nouveau_gem_pushbuf_bo *kref;
1616 struct nouveau_pushbuf_krec *krec;
1617 while ((krec = nvpb->list)) {
1618 kref = krec->buffer;
1619 while (krec->nr_buffer--) {
1620 unsigned long priv = kref++->user_priv;
1621 struct nouveau_bo *bo = (void *)priv;
1622 cli_kref_set(nvpb->base.client, bo, NULL, NULL);
1623 nouveau_bo_ref(NULL, &bo);
1624 }
1625 nvpb->list = krec->next;
1626 free(krec);
1627 }
1628 while (nvpb->bo_nr--)
1629 nouveau_bo_ref(NULL, &nvpb->bos[nvpb->bo_nr]);
1630 nouveau_bo_ref(NULL, &nvpb->bo);
1631 free(nvpb);
1632 }
1633 *ppush = NULL;
1634 }
1635
1636 void
nouveau_bufctx_reset(struct nouveau_bufctx * bctx,int bin)1637 nouveau_bufctx_reset(struct nouveau_bufctx *bctx, int bin)
1638 {
1639 struct nouveau_bufctx_priv *pctx = nouveau_bufctx(bctx);
1640 struct nouveau_bufbin_priv *pbin = &pctx->bins[bin];
1641 struct nouveau_bufref_priv *pref;
1642
1643 while ((pref = pbin->list)) {
1644 list_delinit(&pref->base.thead);
1645 pbin->list = pref->next;
1646 pref->next = pctx->free;
1647 pctx->free = pref;
1648 }
1649
1650 bctx->relocs -= pbin->relocs;
1651 pbin->relocs = 0;
1652 }
1653
1654 struct nouveau_bufctx *
nouveau_pushbuf_bufctx(struct nouveau_pushbuf * push,struct nouveau_bufctx * ctx)1655 nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx)
1656 {
1657 struct nouveau_bufctx *prev = push->bufctx;
1658 push->bufctx = ctx;
1659 return prev;
1660 }
1661
1662 void
nouveau_pushbuf_data(struct nouveau_pushbuf * push,struct nouveau_bo * bo,uint64_t offset,uint64_t length)1663 nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint64_t offset,
1664 uint64_t length)
1665 {
1666 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1667 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1668 struct drm_nouveau_gem_pushbuf_push *kpsh;
1669 struct drm_nouveau_gem_pushbuf_bo *kref;
1670
1671 if (bo != nvpb->bo && nvpb->bgn != push->cur) {
1672 if (nvpb->suffix0 || nvpb->suffix1) {
1673 *push->cur++ = nvpb->suffix0;
1674 *push->cur++ = nvpb->suffix1;
1675 }
1676
1677 nouveau_pushbuf_data(push, nvpb->bo, (nvpb->bgn - nvpb->ptr) * 4, (push->cur - nvpb->bgn) * 4);
1678 nvpb->bgn = push->cur;
1679 }
1680
1681 if (bo) {
1682 kref = cli_kref_get(push->client, bo);
1683 assert(kref);
1684 kpsh = &krec->push[krec->nr_push++];
1685 kpsh->bo_index = kref - krec->buffer;
1686 kpsh->offset = offset;
1687 kpsh->length = length;
1688 }
1689 }
1690
1691 int
nouveau_pushbuf_kick(struct nouveau_pushbuf * push)1692 nouveau_pushbuf_kick(struct nouveau_pushbuf *push)
1693 {
1694 pushbuf_flush(push);
1695 return pushbuf_validate(push, false);
1696 }
1697
1698 int
nouveau_pushbuf_refn(struct nouveau_pushbuf * push,struct nouveau_pushbuf_refn * refs,int nr)1699 nouveau_pushbuf_refn(struct nouveau_pushbuf *push, struct nouveau_pushbuf_refn *refs, int nr)
1700 {
1701 return pushbuf_refn(push, true, refs, nr);
1702 }
1703
1704 void
nouveau_pushbuf_reloc(struct nouveau_pushbuf * push,struct nouveau_bo * bo,uint32_t data,uint32_t flags,uint32_t vor,uint32_t tor)1705 nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo, uint32_t data,
1706 uint32_t flags, uint32_t vor, uint32_t tor)
1707 {
1708 *push->cur = pushbuf_krel(push, bo, data, flags, vor, tor);
1709 push->cur++;
1710 }
1711
1712 int
nouveau_pushbuf_space(struct nouveau_pushbuf * push,uint32_t dwords,uint32_t relocs,uint32_t pushes)1713 nouveau_pushbuf_space(struct nouveau_pushbuf *push, uint32_t dwords, uint32_t relocs,
1714 uint32_t pushes)
1715 {
1716 struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);
1717 struct nouveau_pushbuf_krec *krec = nvpb->krec;
1718 struct nouveau_client *client = push->client;
1719 struct nouveau_bo *bo = NULL;
1720 bool flushed = false;
1721 int ret = 0;
1722
1723 /* switch to next buffer if insufficient space in the current one */
1724 if (push->cur + dwords >= push->end) {
1725 if (nvpb->bo_next < nvpb->bo_nr) {
1726 nouveau_bo_ref(nvpb->bos[nvpb->bo_next++], &bo);
1727 if (nvpb->bo_next == nvpb->bo_nr)
1728 nvpb->bo_next = 0;
1729 } else {
1730 ret = nouveau_bo_new(client->device, nvpb->type, 0, nvpb->bos[0]->size, NULL, &bo);
1731 if (ret)
1732 return ret;
1733 }
1734 }
1735
1736 /* make sure there's always enough space to queue up the pending
1737 * data in the pushbuf proper
1738 */
1739 pushes++;
1740
1741 /* need to flush if we've run out of space on an immediate pushbuf,
1742 * if the new buffer won't fit, or if the kernel push/reloc limits
1743 * have been hit
1744 */
1745 if (bo ||
1746 krec->nr_reloc + relocs >= NOUVEAU_GEM_MAX_RELOCS ||
1747 krec->nr_push + pushes >= NOUVEAU_GEM_MAX_PUSH) {
1748 if (nvpb->bo && krec->nr_buffer)
1749 pushbuf_flush(push);
1750 flushed = true;
1751 }
1752
1753 /* if necessary, switch to new buffer */
1754 if (bo) {
1755 ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client);
1756 if (ret)
1757 return ret;
1758
1759 nouveau_pushbuf_data(push, NULL, 0, 0);
1760 nouveau_bo_ref(bo, &nvpb->bo);
1761 nouveau_bo_ref(NULL, &bo);
1762
1763 nvpb->bgn = nvpb->bo->map;
1764 nvpb->ptr = nvpb->bgn;
1765 push->cur = nvpb->bgn;
1766 push->end = push->cur + (nvpb->bo->size / 4);
1767 push->end -= 2 + push->rsvd_kick; /* space for suffix */
1768 }
1769
1770 pushbuf_kref(push, nvpb->bo, push->flags);
1771 return flushed ? pushbuf_validate(push, false) : 0;
1772 }
1773
1774 int
nouveau_pushbuf_validate(struct nouveau_pushbuf * push)1775 nouveau_pushbuf_validate(struct nouveau_pushbuf *push)
1776 {
1777 return pushbuf_validate(push, true);
1778 }
1779