Lines Matching +full:access +full:- +full:granularity
1 // SPDX-License-Identifier: MIT
55 u8 granularity; member
69 return BIT(tile->id) & vma->tile_present && in vma_is_valid()
70 !(BIT(tile->id) & vma->tile_invalidated); in vma_is_valid()
75 if (page_addr > xe_vma_end(vma) - 1 || in vma_matches()
76 page_addr + SZ_4K - 1 < xe_vma_start(vma)) in vma_matches()
86 if (vm->usm.last_fault_vma) { /* Fast lookup */ in lookup_vma()
87 if (vma_matches(vm->usm.last_fault_vma, page_addr)) in lookup_vma()
88 vma = vm->usm.last_fault_vma; in lookup_vma()
107 if (atomic && IS_DGFX(vm->xe)) { in xe_pf_begin()
109 err = -EACCES; in xe_pf_begin()
138 atomic = access_is_atomic(pf->access_type); in handle_vma_pagefault()
154 /* Lock VM and BOs dma-resv */ in handle_vma_pagefault()
157 err = xe_pf_begin(&exec, vma, atomic, tile->id); in handle_vma_pagefault()
160 err = -EAGAIN; in handle_vma_pagefault()
166 fence = xe_vma_rebind(vm, vma, BIT(tile->id)); in handle_vma_pagefault()
170 err = -EAGAIN; in handle_vma_pagefault()
177 vma->tile_invalidated &= ~BIT(tile->id); in handle_vma_pagefault()
181 if (err == -EAGAIN) in handle_vma_pagefault()
191 down_read(&xe->usm.lock); in asid_to_vm()
192 vm = xa_load(&xe->usm.asid_to_vm, asid); in asid_to_vm()
196 vm = ERR_PTR(-EINVAL); in asid_to_vm()
197 up_read(&xe->usm.lock); in asid_to_vm()
211 if (pf->trva_fault) in handle_pagefault()
212 return -EFAULT; in handle_pagefault()
214 vm = asid_to_vm(xe, pf->asid); in handle_pagefault()
221 down_write(&vm->lock); in handle_pagefault()
224 err = -ENOENT; in handle_pagefault()
228 vma = lookup_vma(vm, pf->page_addr); in handle_pagefault()
230 err = -EINVAL; in handle_pagefault()
238 vm->usm.last_fault_vma = vma; in handle_pagefault()
239 up_write(&vm->lock); in handle_pagefault()
250 reply->dw0, in send_pagefault_reply()
251 reply->dw1, in send_pagefault_reply()
254 return xe_guc_ct_send(&guc->ct, action, ARRAY_SIZE(action), 0, 0); in send_pagefault_reply()
259 drm_dbg(&xe->drm, "\n\tASID: %d\n" in print_pagefault()
268 pf->asid, pf->vfid, pf->pdata, upper_32_bits(pf->page_addr), in print_pagefault()
269 lower_32_bits(pf->page_addr), in print_pagefault()
270 pf->fault_type, pf->access_type, pf->fault_level, in print_pagefault()
271 pf->engine_class, pf->engine_instance); in print_pagefault()
281 spin_lock_irq(&pf_queue->lock); in get_pagefault()
282 if (pf_queue->tail != pf_queue->head) { in get_pagefault()
284 (pf_queue->data + pf_queue->tail); in get_pagefault()
286 pf->fault_level = FIELD_GET(PFD_FAULT_LEVEL, desc->dw0); in get_pagefault()
287 pf->trva_fault = FIELD_GET(XE2_PFD_TRVA_FAULT, desc->dw0); in get_pagefault()
288 pf->engine_class = FIELD_GET(PFD_ENG_CLASS, desc->dw0); in get_pagefault()
289 pf->engine_instance = FIELD_GET(PFD_ENG_INSTANCE, desc->dw0); in get_pagefault()
290 pf->pdata = FIELD_GET(PFD_PDATA_HI, desc->dw1) << in get_pagefault()
292 pf->pdata |= FIELD_GET(PFD_PDATA_LO, desc->dw0); in get_pagefault()
293 pf->asid = FIELD_GET(PFD_ASID, desc->dw1); in get_pagefault()
294 pf->vfid = FIELD_GET(PFD_VFID, desc->dw2); in get_pagefault()
295 pf->access_type = FIELD_GET(PFD_ACCESS_TYPE, desc->dw2); in get_pagefault()
296 pf->fault_type = FIELD_GET(PFD_FAULT_TYPE, desc->dw2); in get_pagefault()
297 pf->page_addr = (u64)(FIELD_GET(PFD_VIRTUAL_ADDR_HI, desc->dw3)) << in get_pagefault()
299 pf->page_addr |= FIELD_GET(PFD_VIRTUAL_ADDR_LO, desc->dw2) << in get_pagefault()
302 pf_queue->tail = (pf_queue->tail + PF_MSG_LEN_DW) % in get_pagefault()
303 pf_queue->num_dw; in get_pagefault()
306 spin_unlock_irq(&pf_queue->lock); in get_pagefault()
313 lockdep_assert_held(&pf_queue->lock); in pf_queue_full()
315 return CIRC_SPACE(pf_queue->head, pf_queue->tail, in pf_queue_full()
316 pf_queue->num_dw) <= in pf_queue_full()
330 return -EPROTO; in xe_guc_pagefault_handler()
333 pf_queue = gt->usm.pf_queue + (asid % NUM_PF_QUEUE); in xe_guc_pagefault_handler()
338 xe_gt_assert(gt, !(pf_queue->num_dw % PF_MSG_LEN_DW)); in xe_guc_pagefault_handler()
340 spin_lock_irqsave(&pf_queue->lock, flags); in xe_guc_pagefault_handler()
343 memcpy(pf_queue->data + pf_queue->head, msg, len * sizeof(u32)); in xe_guc_pagefault_handler()
344 pf_queue->head = (pf_queue->head + len) % in xe_guc_pagefault_handler()
345 pf_queue->num_dw; in xe_guc_pagefault_handler()
346 queue_work(gt->usm.pf_wq, &pf_queue->worker); in xe_guc_pagefault_handler()
348 drm_warn(&xe->drm, "PF Queue full, shouldn't be possible"); in xe_guc_pagefault_handler()
350 spin_unlock_irqrestore(&pf_queue->lock, flags); in xe_guc_pagefault_handler()
352 return full ? -ENOSPC : 0; in xe_guc_pagefault_handler()
360 struct xe_gt *gt = pf_queue->gt; in pf_queue_work_func()
374 drm_dbg(&xe->drm, "Fault response: Unsuccessful %d\n", ret); in pf_queue_work_func()
388 send_pagefault_reply(>->uc.guc, &reply); in pf_queue_work_func()
391 pf_queue->tail != pf_queue->head) { in pf_queue_work_func()
392 queue_work(gt->usm.pf_wq, w); in pf_queue_work_func()
405 if (!xe->info.has_usm) in pagefault_fini()
408 destroy_workqueue(gt->usm.acc_wq); in pagefault_fini()
409 destroy_workqueue(gt->usm.pf_wq); in pagefault_fini()
418 bitmap_or(all_dss, gt->fuse_topo.g_dss_mask, gt->fuse_topo.c_dss_mask, in xe_alloc_pf_queue()
422 num_eus = bitmap_weight(gt->fuse_topo.eu_mask_per_dss, in xe_alloc_pf_queue()
426 pf_queue->num_dw = in xe_alloc_pf_queue()
429 pf_queue->gt = gt; in xe_alloc_pf_queue()
430 pf_queue->data = devm_kcalloc(xe->drm.dev, pf_queue->num_dw, in xe_alloc_pf_queue()
432 if (!pf_queue->data) in xe_alloc_pf_queue()
433 return -ENOMEM; in xe_alloc_pf_queue()
435 spin_lock_init(&pf_queue->lock); in xe_alloc_pf_queue()
436 INIT_WORK(&pf_queue->worker, pf_queue_work_func); in xe_alloc_pf_queue()
446 if (!xe->info.has_usm) in xe_gt_pagefault_init()
450 ret = xe_alloc_pf_queue(gt, >->usm.pf_queue[i]); in xe_gt_pagefault_init()
455 gt->usm.acc_queue[i].gt = gt; in xe_gt_pagefault_init()
456 spin_lock_init(>->usm.acc_queue[i].lock); in xe_gt_pagefault_init()
457 INIT_WORK(>->usm.acc_queue[i].worker, acc_queue_work_func); in xe_gt_pagefault_init()
460 gt->usm.pf_wq = alloc_workqueue("xe_gt_page_fault_work_queue", in xe_gt_pagefault_init()
462 if (!gt->usm.pf_wq) in xe_gt_pagefault_init()
463 return -ENOMEM; in xe_gt_pagefault_init()
465 gt->usm.acc_wq = alloc_workqueue("xe_gt_access_counter_work_queue", in xe_gt_pagefault_init()
468 if (!gt->usm.acc_wq) { in xe_gt_pagefault_init()
469 destroy_workqueue(gt->usm.pf_wq); in xe_gt_pagefault_init()
470 return -ENOMEM; in xe_gt_pagefault_init()
473 return devm_add_action_or_reset(xe->drm.dev, pagefault_fini, gt); in xe_gt_pagefault_init()
481 if (!xe->info.has_usm) in xe_gt_pagefault_reset()
485 spin_lock_irq(>->usm.pf_queue[i].lock); in xe_gt_pagefault_reset()
486 gt->usm.pf_queue[i].head = 0; in xe_gt_pagefault_reset()
487 gt->usm.pf_queue[i].tail = 0; in xe_gt_pagefault_reset()
488 spin_unlock_irq(>->usm.pf_queue[i].lock); in xe_gt_pagefault_reset()
492 spin_lock(>->usm.acc_queue[i].lock); in xe_gt_pagefault_reset()
493 gt->usm.acc_queue[i].head = 0; in xe_gt_pagefault_reset()
494 gt->usm.acc_queue[i].tail = 0; in xe_gt_pagefault_reset()
495 spin_unlock(>->usm.acc_queue[i].lock); in xe_gt_pagefault_reset()
522 drm_warn(&xe->drm, "Access counter request:\n" in print_acc()
527 "\tGranularity: 0x%x KB Region/ %d KB sub-granularity\n" in print_acc()
530 acc->access_type ? "AC_NTFY_VAL" : "AC_TRIG_VAL", in print_acc()
531 acc->asid, acc->vfid, acc->engine_class, acc->engine_instance, in print_acc()
532 granularity_in_byte(acc->granularity) / SZ_1K, in print_acc()
533 sub_granularity_in_byte(acc->granularity) / SZ_1K, in print_acc()
534 acc->sub_granularity, acc->va_range_base); in print_acc()
539 u64 page_va = acc->va_range_base + (ffs(acc->sub_granularity) - 1) * in get_acc_vma()
540 sub_granularity_in_byte(acc->granularity); in get_acc_vma()
555 if (acc->access_type != ACC_TRIGGER) in handle_acc()
556 return -EINVAL; in handle_acc()
558 vm = asid_to_vm(xe, acc->asid); in handle_acc()
562 down_read(&vm->lock); in handle_acc()
567 ret = -EINVAL; in handle_acc()
577 /* Lock VM and BOs dma-resv */ in handle_acc()
580 ret = xe_pf_begin(&exec, vma, true, tile->id); in handle_acc()
588 up_read(&vm->lock); in handle_acc()
603 spin_lock(&acc_queue->lock); in get_acc()
604 if (acc_queue->tail != acc_queue->head) { in get_acc()
606 (acc_queue->data + acc_queue->tail); in get_acc()
608 acc->granularity = FIELD_GET(ACC_GRANULARITY, desc->dw2); in get_acc()
609 acc->sub_granularity = FIELD_GET(ACC_SUBG_HI, desc->dw1) << 31 | in get_acc()
610 FIELD_GET(ACC_SUBG_LO, desc->dw0); in get_acc()
611 acc->engine_class = FIELD_GET(ACC_ENG_CLASS, desc->dw1); in get_acc()
612 acc->engine_instance = FIELD_GET(ACC_ENG_INSTANCE, desc->dw1); in get_acc()
613 acc->asid = FIELD_GET(ACC_ASID, desc->dw1); in get_acc()
614 acc->vfid = FIELD_GET(ACC_VFID, desc->dw2); in get_acc()
615 acc->access_type = FIELD_GET(ACC_TYPE, desc->dw0); in get_acc()
616 acc->va_range_base = make_u64(desc->dw3 & ACC_VIRTUAL_ADDR_RANGE_HI, in get_acc()
617 desc->dw2 & ACC_VIRTUAL_ADDR_RANGE_LO); in get_acc()
619 acc_queue->tail = (acc_queue->tail + ACC_MSG_LEN_DW) % in get_acc()
623 spin_unlock(&acc_queue->lock); in get_acc()
631 struct xe_gt *gt = acc_queue->gt; in acc_queue_work_func()
643 drm_warn(&xe->drm, "ACC: Unsuccessful %d\n", ret); in acc_queue_work_func()
647 acc_queue->tail != acc_queue->head) { in acc_queue_work_func()
648 queue_work(gt->usm.acc_wq, w); in acc_queue_work_func()
656 lockdep_assert_held(&acc_queue->lock); in acc_queue_full()
658 return CIRC_SPACE(acc_queue->head, acc_queue->tail, ACC_QUEUE_NUM_DW) <= in acc_queue_full()
675 return -EPROTO; in xe_guc_access_counter_notify_handler()
678 acc_queue = >->usm.acc_queue[asid % NUM_ACC_QUEUE]; in xe_guc_access_counter_notify_handler()
680 spin_lock(&acc_queue->lock); in xe_guc_access_counter_notify_handler()
683 memcpy(acc_queue->data + acc_queue->head, msg, in xe_guc_access_counter_notify_handler()
685 acc_queue->head = (acc_queue->head + len) % ACC_QUEUE_NUM_DW; in xe_guc_access_counter_notify_handler()
686 queue_work(gt->usm.acc_wq, &acc_queue->worker); in xe_guc_access_counter_notify_handler()
688 drm_warn(>_to_xe(gt)->drm, "ACC Queue full, dropping ACC"); in xe_guc_access_counter_notify_handler()
690 spin_unlock(&acc_queue->lock); in xe_guc_access_counter_notify_handler()
692 return full ? -ENOSPC : 0; in xe_guc_access_counter_notify_handler()