46 	struct drm_nouveau_gem_pushbuf_push push[NOUVEAU_GEM_MAX_PUSH];  member
71 nouveau_pushbuf(struct nouveau_pushbuf *push)  in nouveau_pushbuf()  argument
73 	return (struct nouveau_pushbuf_priv *)push;  in nouveau_pushbuf()
80 pushbuf_kref_fits(struct nouveau_pushbuf *push, struct nouveau_bo *bo,  in pushbuf_kref_fits()  argument
83 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_kref_fits()
85 	struct nouveau_device *dev = push->client->device;  in pushbuf_kref_fits()
148 pushbuf_kref(struct nouveau_pushbuf *push, struct nouveau_bo *bo,  in pushbuf_kref()  argument
151 	struct nouveau_device *dev = push->client->device;  in pushbuf_kref()
152 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_kref()
170 	fpush = cli_push_get(push->client, bo);  in pushbuf_kref()
171 	if (fpush && fpush != push)  in pushbuf_kref()
174 	kref = cli_kref_get(push->client, bo);  in pushbuf_kref()
196 		    !pushbuf_kref_fits(push, bo, &domains))  in pushbuf_kref()
212 		cli_kref_set(push->client, bo, kref, push);  in pushbuf_kref()
220 pushbuf_krel(struct nouveau_pushbuf *push, struct nouveau_bo *bo,  in pushbuf_krel()  argument
223 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_krel()
230 	pkref = cli_kref_get(push->client, nvpb->bo);  in pushbuf_krel()
231 	bkref = cli_kref_get(push->client, bo);  in pushbuf_krel()
237 	krel->reloc_bo_offset = (push->cur - nvpb->ptr) * 4;  in pushbuf_krel()
292 	kpsh = krec->push;  in pushbuf_dump()
311 pushbuf_submit(struct nouveau_pushbuf *push, struct nouveau_object *chan)  in pushbuf_submit()  argument
313 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_submit()
315 	struct nouveau_device *dev = push->client->device;  in pushbuf_submit()
328 	if (push->kick_notify)  in pushbuf_submit()
329 		push->kick_notify(push);  in pushbuf_submit()
331 	nouveau_pushbuf_data(push, NULL, 0, 0);  in pushbuf_submit()
340 		req.push = (uint64_t)(unsigned long)krec->push;  in pushbuf_submit()
398 pushbuf_flush(struct nouveau_pushbuf *push)  in pushbuf_flush()  argument
400 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_flush()
407 	if (push->channel) {  in pushbuf_flush()
408 		ret = pushbuf_submit(push, push->channel);  in pushbuf_flush()
410 		nouveau_pushbuf_data(push, NULL, 0, 0);  in pushbuf_flush()
418 		cli_kref_set(push->client, bo, NULL, NULL);  in pushbuf_flush()
419 		if (push->channel)  in pushbuf_flush()
440 pushbuf_refn_fail(struct nouveau_pushbuf *push, int sref, int srel)  in pushbuf_refn_fail()  argument
442 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_refn_fail()
449 		cli_kref_set(push->client, bo, NULL, NULL);  in pushbuf_refn_fail()
458 pushbuf_refn(struct nouveau_pushbuf *push, bool retry,  in pushbuf_refn()  argument
461 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_refn()
468 		kref = pushbuf_kref(push, refs[i].bo, refs[i].flags);  in pushbuf_refn()
476 		pushbuf_refn_fail(push, sref, krec->nr_reloc);  in pushbuf_refn()
478 			pushbuf_flush(push);  in pushbuf_refn()
479 			nouveau_pushbuf_space(push, 0, 0, 0);  in pushbuf_refn()
480 			return pushbuf_refn(push, false, refs, nr);  in pushbuf_refn()
488 pushbuf_validate(struct nouveau_pushbuf *push, bool retry)  in pushbuf_validate()  argument
490 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in pushbuf_validate()
493 	struct nouveau_bufctx *bctx = push->bufctx;  in pushbuf_validate()
498 	ret = nouveau_pushbuf_space(push, relocs, relocs, 0);  in pushbuf_validate()
509 		kref = pushbuf_kref(push, bref->bo, bref->flags);  in pushbuf_validate()
516 			pushbuf_krel(push, bref->bo, bref->packet, 0, 0, 0);  in pushbuf_validate()
517 			*push->cur++ = 0;  in pushbuf_validate()
518 			pushbuf_krel(push, bref->bo, bref->data, bref->flags,  in pushbuf_validate()
520 			*push->cur++ = 0;  in pushbuf_validate()
528 		pushbuf_refn_fail(push, sref, srel);  in pushbuf_validate()
530 			pushbuf_flush(push);  in pushbuf_validate()
531 			return pushbuf_validate(push, false);  in pushbuf_validate()
546 	struct nouveau_pushbuf *push;  in nouveau_pushbuf_new()  local
581 	push = &nvpb->base;  in nouveau_pushbuf_new()
582 	push->client = client;  in nouveau_pushbuf_new()
583 	push->channel = immediate ? chan : NULL;  in nouveau_pushbuf_new()
584 	push->flags = NOUVEAU_BO_RD;  in nouveau_pushbuf_new()
586 		push->flags |= NOUVEAU_BO_GART;  in nouveau_pushbuf_new()
590 		push->flags |= NOUVEAU_BO_VRAM;  in nouveau_pushbuf_new()
599 			nouveau_pushbuf_del(&push);  in nouveau_pushbuf_new()
605 	*ppush = push;  in nouveau_pushbuf_new()
636 nouveau_pushbuf_bufctx(struct nouveau_pushbuf *push, struct nouveau_bufctx *ctx)  in nouveau_pushbuf_bufctx()  argument
638 	struct nouveau_bufctx *prev = push->bufctx;  in nouveau_pushbuf_bufctx()
639 	push->bufctx = ctx;  in nouveau_pushbuf_bufctx()
644 nouveau_pushbuf_space(struct nouveau_pushbuf *push,  in nouveau_pushbuf_space()  argument
647 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in nouveau_pushbuf_space()
649 	struct nouveau_client *client = push->client;  in nouveau_pushbuf_space()
655 	if (push->cur + dwords >= push->end) {  in nouveau_pushbuf_space()
658 			if (nvpb->bo_next == nvpb->bo_nr && push->channel)  in nouveau_pushbuf_space()
674 	 * if the new buffer won't fit, or if the kernel push/reloc limits  in nouveau_pushbuf_space()
677 	if ((bo && ( push->channel ||  in nouveau_pushbuf_space()
678 		    !pushbuf_kref(push, bo, push->flags))) ||  in nouveau_pushbuf_space()
682 			pushbuf_flush(push);  in nouveau_pushbuf_space()
688 		ret = nouveau_bo_map(bo, NOUVEAU_BO_WR, push->client);  in nouveau_pushbuf_space()
692 		nouveau_pushbuf_data(push, NULL, 0, 0);  in nouveau_pushbuf_space()
698 		push->cur = nvpb->bgn;  in nouveau_pushbuf_space()
699 		push->end = push->cur + (nvpb->bo->size / 4);  in nouveau_pushbuf_space()
700 		push->end -= 2 + push->rsvd_kick; /* space for suffix */  in nouveau_pushbuf_space()
703 	pushbuf_kref(push, nvpb->bo, push->flags);  in nouveau_pushbuf_space()
704 	return flushed ? pushbuf_validate(push, false) : 0;  in nouveau_pushbuf_space()
708 nouveau_pushbuf_data(struct nouveau_pushbuf *push, struct nouveau_bo *bo,  in nouveau_pushbuf_data()  argument
711 	struct nouveau_pushbuf_priv *nvpb = nouveau_pushbuf(push);  in nouveau_pushbuf_data()
716 	if (bo != nvpb->bo && nvpb->bgn != push->cur) {  in nouveau_pushbuf_data()
718 			*push->cur++ = nvpb->suffix0;  in nouveau_pushbuf_data()
719 			*push->cur++ = nvpb->suffix1;  in nouveau_pushbuf_data()
722 		nouveau_pushbuf_data(push, nvpb->bo,  in nouveau_pushbuf_data()
724 				     (push->cur - nvpb->bgn) * 4);  in nouveau_pushbuf_data()
725 		nvpb->bgn = push->cur;  in nouveau_pushbuf_data()
729 		kref = cli_kref_get(push->client, bo);  in nouveau_pushbuf_data()
731 		kpsh = &krec->push[krec->nr_push++];  in nouveau_pushbuf_data()
739 nouveau_pushbuf_refn(struct nouveau_pushbuf *push,  in nouveau_pushbuf_refn()  argument
742 	return pushbuf_refn(push, true, refs, nr);  in nouveau_pushbuf_refn()
746 nouveau_pushbuf_reloc(struct nouveau_pushbuf *push, struct nouveau_bo *bo,  in nouveau_pushbuf_reloc()  argument
749 	*push->cur = pushbuf_krel(push, bo, data, flags, vor, tor);  in nouveau_pushbuf_reloc()
750 	push->cur++;  in nouveau_pushbuf_reloc()
754 nouveau_pushbuf_validate(struct nouveau_pushbuf *push)  in nouveau_pushbuf_validate()  argument
756 	return pushbuf_validate(push, true);  in nouveau_pushbuf_validate()
760 nouveau_pushbuf_refd(struct nouveau_pushbuf *push, struct nouveau_bo *bo)  in nouveau_pushbuf_refd()  argument
765 	if (cli_push_get(push->client, bo) == push) {  in nouveau_pushbuf_refd()
766 		kref = cli_kref_get(push->client, bo);  in nouveau_pushbuf_refd()
778 nouveau_pushbuf_kick(struct nouveau_pushbuf *push, struct nouveau_object *chan)  in nouveau_pushbuf_kick()  argument
780 	if (!push->channel)  in nouveau_pushbuf_kick()
781 		return pushbuf_submit(push, chan);  in nouveau_pushbuf_kick()
782 	pushbuf_flush(push);  in nouveau_pushbuf_kick()
783 	return pushbuf_validate(push, false);  in nouveau_pushbuf_kick()