Lines Matching +full:gfx +full:- +full:mem
46 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_kernel_doorbell_get()
49 offset = adev->doorbell_index.sdma_engine[0]; in amdgpu_mes_kernel_doorbell_get()
53 found = find_next_zero_bit(mes->doorbell_bitmap, mes->num_mes_dbs, offset); in amdgpu_mes_kernel_doorbell_get()
54 if (found >= mes->num_mes_dbs) { in amdgpu_mes_kernel_doorbell_get()
56 return -ENOSPC; in amdgpu_mes_kernel_doorbell_get()
59 set_bit(found, mes->doorbell_bitmap); in amdgpu_mes_kernel_doorbell_get()
62 *doorbell_index = mes->db_start_dw_offset + found * 2; in amdgpu_mes_kernel_doorbell_get()
70 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_kernel_doorbell_free()
73 rel_index = (doorbell_index - mes->db_start_dw_offset) / 2; in amdgpu_mes_kernel_doorbell_free()
74 old = test_and_clear_bit(rel_index, mes->doorbell_bitmap); in amdgpu_mes_kernel_doorbell_free()
81 struct amdgpu_mes *mes = &adev->mes; in amdgpu_mes_doorbell_init()
84 mes->doorbell_bitmap = bitmap_zalloc(PAGE_SIZE / sizeof(u32), GFP_KERNEL); in amdgpu_mes_doorbell_init()
85 if (!mes->doorbell_bitmap) { in amdgpu_mes_doorbell_init()
87 return -ENOMEM; in amdgpu_mes_doorbell_init()
90 mes->num_mes_dbs = PAGE_SIZE / AMDGPU_ONE_DOORBELL_SIZE; in amdgpu_mes_doorbell_init()
92 adev->mes.aggregated_doorbells[i] = mes->db_start_dw_offset + i * 2; in amdgpu_mes_doorbell_init()
93 set_bit(i, mes->doorbell_bitmap); in amdgpu_mes_doorbell_init()
106 r = amdgpu_bo_create_kernel(adev, adev->mes.event_log_size, PAGE_SIZE, in amdgpu_mes_event_log_init()
108 &adev->mes.event_log_gpu_obj, in amdgpu_mes_event_log_init()
109 &adev->mes.event_log_gpu_addr, in amdgpu_mes_event_log_init()
110 &adev->mes.event_log_cpu_addr); in amdgpu_mes_event_log_init()
112 dev_warn(adev->dev, "failed to create MES event log buffer (%d)", r); in amdgpu_mes_event_log_init()
116 memset(adev->mes.event_log_cpu_addr, 0, adev->mes.event_log_size); in amdgpu_mes_event_log_init()
124 bitmap_free(adev->mes.doorbell_bitmap); in amdgpu_mes_doorbell_free()
131 adev->mes.adev = adev; in amdgpu_mes_init()
133 idr_init(&adev->mes.pasid_idr); in amdgpu_mes_init()
134 idr_init(&adev->mes.gang_id_idr); in amdgpu_mes_init()
135 idr_init(&adev->mes.queue_id_idr); in amdgpu_mes_init()
136 ida_init(&adev->mes.doorbell_ida); in amdgpu_mes_init()
137 spin_lock_init(&adev->mes.queue_id_lock); in amdgpu_mes_init()
138 mutex_init(&adev->mes.mutex_hidden); in amdgpu_mes_init()
141 spin_lock_init(&adev->mes.ring_lock[i]); in amdgpu_mes_init()
143 adev->mes.total_max_queue = AMDGPU_FENCE_MES_QUEUE_ID_MASK; in amdgpu_mes_init()
144 adev->mes.vmid_mask_mmhub = 0xffffff00; in amdgpu_mes_init()
145 adev->mes.vmid_mask_gfxhub = 0xffffff00; in amdgpu_mes_init()
148 if (i >= (adev->gfx.mec.num_pipe_per_mec * adev->gfx.mec.num_mec)) in amdgpu_mes_init()
150 adev->mes.compute_hqd_mask[i] = 0xc; in amdgpu_mes_init()
154 adev->mes.gfx_hqd_mask[i] = i ? 0 : 0xfffffffe; in amdgpu_mes_init()
159 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0x3fc; in amdgpu_mes_init()
160 /* zero sdma_hqd_mask for non-existent engine */ in amdgpu_mes_init()
161 else if (adev->sdma.num_instances == 1) in amdgpu_mes_init()
162 adev->mes.sdma_hqd_mask[i] = i ? 0 : 0xfc; in amdgpu_mes_init()
164 adev->mes.sdma_hqd_mask[i] = 0xfc; in amdgpu_mes_init()
168 r = amdgpu_device_wb_get(adev, &adev->mes.sch_ctx_offs[i]); in amdgpu_mes_init()
170 dev_err(adev->dev, in amdgpu_mes_init()
175 adev->mes.sch_ctx_gpu_addr[i] = in amdgpu_mes_init()
176 adev->wb.gpu_addr + (adev->mes.sch_ctx_offs[i] * 4); in amdgpu_mes_init()
177 adev->mes.sch_ctx_ptr[i] = in amdgpu_mes_init()
178 (uint64_t *)&adev->wb.wb[adev->mes.sch_ctx_offs[i]]; in amdgpu_mes_init()
181 &adev->mes.query_status_fence_offs[i]); in amdgpu_mes_init()
183 dev_err(adev->dev, in amdgpu_mes_init()
188 adev->mes.query_status_fence_gpu_addr[i] = adev->wb.gpu_addr + in amdgpu_mes_init()
189 (adev->mes.query_status_fence_offs[i] * 4); in amdgpu_mes_init()
190 adev->mes.query_status_fence_ptr[i] = in amdgpu_mes_init()
191 (uint64_t *)&adev->wb.wb[adev->mes.query_status_fence_offs[i]]; in amdgpu_mes_init()
208 if (adev->mes.sch_ctx_ptr[i]) in amdgpu_mes_init()
209 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); in amdgpu_mes_init()
210 if (adev->mes.query_status_fence_ptr[i]) in amdgpu_mes_init()
212 adev->mes.query_status_fence_offs[i]); in amdgpu_mes_init()
215 idr_destroy(&adev->mes.pasid_idr); in amdgpu_mes_init()
216 idr_destroy(&adev->mes.gang_id_idr); in amdgpu_mes_init()
217 idr_destroy(&adev->mes.queue_id_idr); in amdgpu_mes_init()
218 ida_destroy(&adev->mes.doorbell_ida); in amdgpu_mes_init()
219 mutex_destroy(&adev->mes.mutex_hidden); in amdgpu_mes_init()
227 amdgpu_bo_free_kernel(&adev->mes.event_log_gpu_obj, in amdgpu_mes_fini()
228 &adev->mes.event_log_gpu_addr, in amdgpu_mes_fini()
229 &adev->mes.event_log_cpu_addr); in amdgpu_mes_fini()
232 if (adev->mes.sch_ctx_ptr[i]) in amdgpu_mes_fini()
233 amdgpu_device_wb_free(adev, adev->mes.sch_ctx_offs[i]); in amdgpu_mes_fini()
234 if (adev->mes.query_status_fence_ptr[i]) in amdgpu_mes_fini()
236 adev->mes.query_status_fence_offs[i]); in amdgpu_mes_fini()
241 idr_destroy(&adev->mes.pasid_idr); in amdgpu_mes_fini()
242 idr_destroy(&adev->mes.gang_id_idr); in amdgpu_mes_fini()
243 idr_destroy(&adev->mes.queue_id_idr); in amdgpu_mes_fini()
244 ida_destroy(&adev->mes.doorbell_ida); in amdgpu_mes_fini()
245 mutex_destroy(&adev->mes.mutex_hidden); in amdgpu_mes_fini()
250 amdgpu_bo_free_kernel(&q->mqd_obj, in amdgpu_mes_queue_free_mqd()
251 &q->mqd_gpu_addr, in amdgpu_mes_queue_free_mqd()
252 &q->mqd_cpu_ptr); in amdgpu_mes_queue_free_mqd()
265 return -ENOMEM; in amdgpu_mes_create_process()
271 &process->proc_ctx_bo, in amdgpu_mes_create_process()
272 &process->proc_ctx_gpu_addr, in amdgpu_mes_create_process()
273 &process->proc_ctx_cpu_ptr); in amdgpu_mes_create_process()
278 memset(process->proc_ctx_cpu_ptr, 0, AMDGPU_MES_PROC_CTX_SIZE); in amdgpu_mes_create_process()
284 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_create_process()
287 r = idr_alloc(&adev->mes.pasid_idr, process, pasid, pasid + 1, in amdgpu_mes_create_process()
294 INIT_LIST_HEAD(&process->gang_list); in amdgpu_mes_create_process()
295 process->vm = vm; in amdgpu_mes_create_process()
296 process->pasid = pasid; in amdgpu_mes_create_process()
297 process->process_quantum = adev->mes.default_process_quantum; in amdgpu_mes_create_process()
298 process->pd_gpu_addr = amdgpu_bo_gpu_offset(vm->root.bo); in amdgpu_mes_create_process()
300 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_create_process()
304 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_create_process()
305 amdgpu_bo_free_kernel(&process->proc_ctx_bo, in amdgpu_mes_create_process()
306 &process->proc_ctx_gpu_addr, in amdgpu_mes_create_process()
307 &process->proc_ctx_cpu_ptr); in amdgpu_mes_create_process()
326 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_destroy_process()
328 process = idr_find(&adev->mes.pasid_idr, pasid); in amdgpu_mes_destroy_process()
331 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_destroy_process()
336 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { in amdgpu_mes_destroy_process()
337 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { in amdgpu_mes_destroy_process()
338 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_destroy_process()
339 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); in amdgpu_mes_destroy_process()
340 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_destroy_process()
342 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_destroy_process()
343 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_destroy_process()
345 r = adev->mes.funcs->remove_hw_queue(&adev->mes, in amdgpu_mes_destroy_process()
351 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); in amdgpu_mes_destroy_process()
354 idr_remove(&adev->mes.pasid_idr, pasid); in amdgpu_mes_destroy_process()
355 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_destroy_process()
358 list_for_each_entry_safe(gang, tmp1, &process->gang_list, list) { in amdgpu_mes_destroy_process()
360 list_for_each_entry_safe(queue, tmp2, &gang->queue_list, list) { in amdgpu_mes_destroy_process()
362 list_del(&queue->list); in amdgpu_mes_destroy_process()
365 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_destroy_process()
366 &gang->gang_ctx_gpu_addr, in amdgpu_mes_destroy_process()
367 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_destroy_process()
368 list_del(&gang->list); in amdgpu_mes_destroy_process()
372 amdgpu_bo_free_kernel(&process->proc_ctx_bo, in amdgpu_mes_destroy_process()
373 &process->proc_ctx_gpu_addr, in amdgpu_mes_destroy_process()
374 &process->proc_ctx_cpu_ptr); in amdgpu_mes_destroy_process()
389 return -ENOMEM; in amdgpu_mes_add_gang()
395 &gang->gang_ctx_bo, in amdgpu_mes_add_gang()
396 &gang->gang_ctx_gpu_addr, in amdgpu_mes_add_gang()
397 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_add_gang()
402 memset(gang->gang_ctx_cpu_ptr, 0, AMDGPU_MES_GANG_CTX_SIZE); in amdgpu_mes_add_gang()
408 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_gang()
410 process = idr_find(&adev->mes.pasid_idr, pasid); in amdgpu_mes_add_gang()
413 r = -EINVAL; in amdgpu_mes_add_gang()
418 r = idr_alloc(&adev->mes.gang_id_idr, gang, 1, 0, in amdgpu_mes_add_gang()
425 gang->gang_id = r; in amdgpu_mes_add_gang()
428 INIT_LIST_HEAD(&gang->queue_list); in amdgpu_mes_add_gang()
429 gang->process = process; in amdgpu_mes_add_gang()
430 gang->priority = gprops->priority; in amdgpu_mes_add_gang()
431 gang->gang_quantum = gprops->gang_quantum ? in amdgpu_mes_add_gang()
432 gprops->gang_quantum : adev->mes.default_gang_quantum; in amdgpu_mes_add_gang()
433 gang->global_priority_level = gprops->global_priority_level; in amdgpu_mes_add_gang()
434 gang->inprocess_gang_priority = gprops->inprocess_gang_priority; in amdgpu_mes_add_gang()
435 list_add_tail(&gang->list, &process->gang_list); in amdgpu_mes_add_gang()
437 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_gang()
441 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_gang()
442 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_add_gang()
443 &gang->gang_ctx_gpu_addr, in amdgpu_mes_add_gang()
444 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_add_gang()
458 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_remove_gang()
460 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_remove_gang()
463 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
464 return -EINVAL; in amdgpu_mes_remove_gang()
467 if (!list_empty(&gang->queue_list)) { in amdgpu_mes_remove_gang()
469 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
470 return -EBUSY; in amdgpu_mes_remove_gang()
473 idr_remove(&adev->mes.gang_id_idr, gang->gang_id); in amdgpu_mes_remove_gang()
474 list_del(&gang->list); in amdgpu_mes_remove_gang()
475 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_gang()
477 amdgpu_bo_free_kernel(&gang->gang_ctx_bo, in amdgpu_mes_remove_gang()
478 &gang->gang_ctx_gpu_addr, in amdgpu_mes_remove_gang()
479 &gang->gang_ctx_cpu_ptr); in amdgpu_mes_remove_gang()
501 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_suspend()
502 r = adev->mes.funcs->suspend_gang(&adev->mes, &input); in amdgpu_mes_suspend()
503 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_suspend()
525 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_resume()
526 r = adev->mes.funcs->resume_gang(&adev->mes, &input); in amdgpu_mes_resume()
527 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_resume()
538 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; in amdgpu_mes_queue_alloc_mqd()
539 u32 mqd_size = mqd_mgr->mqd_size; in amdgpu_mes_queue_alloc_mqd()
544 &q->mqd_obj, in amdgpu_mes_queue_alloc_mqd()
545 &q->mqd_gpu_addr, &q->mqd_cpu_ptr); in amdgpu_mes_queue_alloc_mqd()
547 dev_warn(adev->dev, "failed to create queue mqd bo (%d)", r); in amdgpu_mes_queue_alloc_mqd()
550 memset(q->mqd_cpu_ptr, 0, mqd_size); in amdgpu_mes_queue_alloc_mqd()
552 r = amdgpu_bo_reserve(q->mqd_obj, false); in amdgpu_mes_queue_alloc_mqd()
559 amdgpu_bo_free_kernel(&q->mqd_obj, in amdgpu_mes_queue_alloc_mqd()
560 &q->mqd_gpu_addr, in amdgpu_mes_queue_alloc_mqd()
561 &q->mqd_cpu_ptr); in amdgpu_mes_queue_alloc_mqd()
569 struct amdgpu_mqd *mqd_mgr = &adev->mqds[p->queue_type]; in amdgpu_mes_queue_init_mqd()
572 mqd_prop.mqd_gpu_addr = q->mqd_gpu_addr; in amdgpu_mes_queue_init_mqd()
573 mqd_prop.hqd_base_gpu_addr = p->hqd_base_gpu_addr; in amdgpu_mes_queue_init_mqd()
574 mqd_prop.rptr_gpu_addr = p->rptr_gpu_addr; in amdgpu_mes_queue_init_mqd()
575 mqd_prop.wptr_gpu_addr = p->wptr_gpu_addr; in amdgpu_mes_queue_init_mqd()
576 mqd_prop.queue_size = p->queue_size; in amdgpu_mes_queue_init_mqd()
578 mqd_prop.doorbell_index = p->doorbell_off; in amdgpu_mes_queue_init_mqd()
579 mqd_prop.eop_gpu_addr = p->eop_gpu_addr; in amdgpu_mes_queue_init_mqd()
580 mqd_prop.hqd_pipe_priority = p->hqd_pipe_priority; in amdgpu_mes_queue_init_mqd()
581 mqd_prop.hqd_queue_priority = p->hqd_queue_priority; in amdgpu_mes_queue_init_mqd()
584 if (p->queue_type == AMDGPU_RING_TYPE_GFX || in amdgpu_mes_queue_init_mqd()
585 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { in amdgpu_mes_queue_init_mqd()
586 mutex_lock(&adev->srbm_mutex); in amdgpu_mes_queue_init_mqd()
587 amdgpu_gfx_select_me_pipe_q(adev, p->ring->me, p->ring->pipe, 0, 0, 0); in amdgpu_mes_queue_init_mqd()
590 mqd_mgr->init_mqd(adev, q->mqd_cpu_ptr, &mqd_prop); in amdgpu_mes_queue_init_mqd()
592 if (p->queue_type == AMDGPU_RING_TYPE_GFX || in amdgpu_mes_queue_init_mqd()
593 p->queue_type == AMDGPU_RING_TYPE_COMPUTE) { in amdgpu_mes_queue_init_mqd()
595 mutex_unlock(&adev->srbm_mutex); in amdgpu_mes_queue_init_mqd()
598 amdgpu_bo_unreserve(q->mqd_obj); in amdgpu_mes_queue_init_mqd()
617 return -ENOMEM; in amdgpu_mes_add_hw_queue()
629 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_hw_queue()
631 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_add_hw_queue()
634 r = -EINVAL; in amdgpu_mes_add_hw_queue()
639 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
640 r = idr_alloc(&adev->mes.queue_id_idr, queue, 1, 0, in amdgpu_mes_add_hw_queue()
643 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
646 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
647 *queue_id = queue->queue_id = r; in amdgpu_mes_add_hw_queue()
651 qprops->queue_type, in amdgpu_mes_add_hw_queue()
652 &qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
660 queue_input.process_id = gang->process->pasid; in amdgpu_mes_add_hw_queue()
663 adev->vm_manager.vram_base_offset + gang->process->pd_gpu_addr - in amdgpu_mes_add_hw_queue()
664 adev->gmc.vram_start; in amdgpu_mes_add_hw_queue()
668 (adev->vm_manager.max_pfn - 1) << AMDGPU_GPU_PAGE_SHIFT; in amdgpu_mes_add_hw_queue()
669 queue_input.process_quantum = gang->process->process_quantum; in amdgpu_mes_add_hw_queue()
670 queue_input.process_context_addr = gang->process->proc_ctx_gpu_addr; in amdgpu_mes_add_hw_queue()
671 queue_input.gang_quantum = gang->gang_quantum; in amdgpu_mes_add_hw_queue()
672 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_add_hw_queue()
673 queue_input.inprocess_gang_priority = gang->inprocess_gang_priority; in amdgpu_mes_add_hw_queue()
674 queue_input.gang_global_priority_level = gang->global_priority_level; in amdgpu_mes_add_hw_queue()
675 queue_input.doorbell_offset = qprops->doorbell_off; in amdgpu_mes_add_hw_queue()
676 queue_input.mqd_addr = queue->mqd_gpu_addr; in amdgpu_mes_add_hw_queue()
677 queue_input.wptr_addr = qprops->wptr_gpu_addr; in amdgpu_mes_add_hw_queue()
678 queue_input.wptr_mc_addr = qprops->wptr_mc_addr; in amdgpu_mes_add_hw_queue()
679 queue_input.queue_type = qprops->queue_type; in amdgpu_mes_add_hw_queue()
680 queue_input.paging = qprops->paging; in amdgpu_mes_add_hw_queue()
683 r = adev->mes.funcs->add_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_add_hw_queue()
686 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
692 gang->process->pasid, gang_id, qprops->queue_type, in amdgpu_mes_add_hw_queue()
693 qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
695 queue->ring = qprops->ring; in amdgpu_mes_add_hw_queue()
696 queue->doorbell_off = qprops->doorbell_off; in amdgpu_mes_add_hw_queue()
697 queue->wptr_gpu_addr = qprops->wptr_gpu_addr; in amdgpu_mes_add_hw_queue()
698 queue->queue_type = qprops->queue_type; in amdgpu_mes_add_hw_queue()
699 queue->paging = qprops->paging; in amdgpu_mes_add_hw_queue()
700 queue->gang = gang; in amdgpu_mes_add_hw_queue()
701 queue->ring->mqd_ptr = queue->mqd_cpu_ptr; in amdgpu_mes_add_hw_queue()
702 list_add_tail(&queue->list, &gang->queue_list); in amdgpu_mes_add_hw_queue()
704 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_hw_queue()
708 amdgpu_mes_kernel_doorbell_free(adev, qprops->doorbell_off); in amdgpu_mes_add_hw_queue()
710 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
711 idr_remove(&adev->mes.queue_id_idr, queue->queue_id); in amdgpu_mes_add_hw_queue()
712 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_add_hw_queue()
714 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_hw_queue()
733 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_remove_hw_queue()
736 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
738 queue = idr_find(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_remove_hw_queue()
740 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
741 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_hw_queue()
743 return -EINVAL; in amdgpu_mes_remove_hw_queue()
746 idr_remove(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_remove_hw_queue()
747 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_remove_hw_queue()
750 queue->doorbell_off); in amdgpu_mes_remove_hw_queue()
752 gang = queue->gang; in amdgpu_mes_remove_hw_queue()
753 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_remove_hw_queue()
754 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_remove_hw_queue()
756 r = adev->mes.funcs->remove_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_remove_hw_queue()
761 list_del(&queue->list); in amdgpu_mes_remove_hw_queue()
762 amdgpu_mes_kernel_doorbell_free(adev, queue->doorbell_off); in amdgpu_mes_remove_hw_queue()
763 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_remove_hw_queue()
782 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_reset_hw_queue()
785 spin_lock_irqsave(&adev->mes.queue_id_lock, flags); in amdgpu_mes_reset_hw_queue()
787 queue = idr_find(&adev->mes.queue_id_idr, queue_id); in amdgpu_mes_reset_hw_queue()
789 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_reset_hw_queue()
790 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_reset_hw_queue()
792 return -EINVAL; in amdgpu_mes_reset_hw_queue()
794 spin_unlock_irqrestore(&adev->mes.queue_id_lock, flags); in amdgpu_mes_reset_hw_queue()
797 queue->doorbell_off); in amdgpu_mes_reset_hw_queue()
799 gang = queue->gang; in amdgpu_mes_reset_hw_queue()
800 queue_input.doorbell_offset = queue->doorbell_off; in amdgpu_mes_reset_hw_queue()
801 queue_input.gang_context_addr = gang->gang_ctx_gpu_addr; in amdgpu_mes_reset_hw_queue()
803 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_reset_hw_queue()
808 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_reset_hw_queue()
825 r = adev->mes.funcs->reset_hw_queue(&adev->mes, &queue_input); in amdgpu_mes_reset_hw_queue_mmio()
840 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_map_legacy_queue()
841 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_map_legacy_queue()
842 queue_input.pipe_id = ring->pipe; in amdgpu_mes_map_legacy_queue()
843 queue_input.queue_id = ring->queue; in amdgpu_mes_map_legacy_queue()
844 queue_input.mqd_addr = amdgpu_bo_gpu_offset(ring->mqd_obj); in amdgpu_mes_map_legacy_queue()
845 queue_input.wptr_addr = ring->wptr_gpu_addr; in amdgpu_mes_map_legacy_queue()
847 r = adev->mes.funcs->map_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_map_legacy_queue()
863 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_unmap_legacy_queue()
864 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_unmap_legacy_queue()
865 queue_input.pipe_id = ring->pipe; in amdgpu_mes_unmap_legacy_queue()
866 queue_input.queue_id = ring->queue; in amdgpu_mes_unmap_legacy_queue()
870 r = adev->mes.funcs->unmap_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_unmap_legacy_queue()
887 queue_input.queue_type = ring->funcs->type; in amdgpu_mes_reset_legacy_queue()
888 queue_input.doorbell_offset = ring->doorbell_index; in amdgpu_mes_reset_legacy_queue()
889 queue_input.me_id = ring->me; in amdgpu_mes_reset_legacy_queue()
890 queue_input.pipe_id = ring->pipe; in amdgpu_mes_reset_legacy_queue()
891 queue_input.queue_id = ring->queue; in amdgpu_mes_reset_legacy_queue()
892 queue_input.mqd_addr = ring->mqd_obj ? amdgpu_bo_gpu_offset(ring->mqd_obj) : 0; in amdgpu_mes_reset_legacy_queue()
893 queue_input.wptr_addr = ring->wptr_gpu_addr; in amdgpu_mes_reset_legacy_queue()
897 r = adev->mes.funcs->reset_legacy_queue(&adev->mes, &queue_input); in amdgpu_mes_reset_legacy_queue()
916 read_val_gpu_addr = adev->wb.gpu_addr + (addr_offset * 4); in amdgpu_mes_rreg()
917 read_val_ptr = (uint32_t *)&adev->wb.wb[addr_offset]; in amdgpu_mes_rreg()
922 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_rreg()
927 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_rreg()
949 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_wreg()
951 r = -EINVAL; in amdgpu_mes_wreg()
955 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_wreg()
976 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_reg_write_reg_wait()
978 r = -EINVAL; in amdgpu_mes_reg_write_reg_wait()
982 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_reg_write_reg_wait()
1001 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_reg_wait()
1003 r = -EINVAL; in amdgpu_mes_reg_wait()
1007 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_reg_wait()
1025 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_set_shader_debugger()
1027 return -EINVAL; in amdgpu_mes_set_shader_debugger()
1036 return -EINVAL; in amdgpu_mes_set_shader_debugger()
1042 if (((adev->mes.sched_version & AMDGPU_MES_API_VERSION_MASK) >> in amdgpu_mes_set_shader_debugger()
1046 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_set_shader_debugger()
1048 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_set_shader_debugger()
1052 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_set_shader_debugger()
1063 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_flush_shader_debugger()
1065 return -EINVAL; in amdgpu_mes_flush_shader_debugger()
1072 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_flush_shader_debugger()
1074 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_flush_shader_debugger()
1078 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_flush_shader_debugger()
1088 props->queue_type = ring->funcs->type; in amdgpu_mes_ring_to_queue_props()
1089 props->hqd_base_gpu_addr = ring->gpu_addr; in amdgpu_mes_ring_to_queue_props()
1090 props->rptr_gpu_addr = ring->rptr_gpu_addr; in amdgpu_mes_ring_to_queue_props()
1091 props->wptr_gpu_addr = ring->wptr_gpu_addr; in amdgpu_mes_ring_to_queue_props()
1092 props->wptr_mc_addr = in amdgpu_mes_ring_to_queue_props()
1093 ring->mes_ctx->meta_data_mc_addr + ring->wptr_offs; in amdgpu_mes_ring_to_queue_props()
1094 props->queue_size = ring->ring_size; in amdgpu_mes_ring_to_queue_props()
1095 props->eop_gpu_addr = ring->eop_gpu_addr; in amdgpu_mes_ring_to_queue_props()
1096 props->hqd_pipe_priority = AMDGPU_GFX_PIPE_PRIO_NORMAL; in amdgpu_mes_ring_to_queue_props()
1097 props->hqd_queue_priority = AMDGPU_GFX_QUEUE_PRIORITY_MINIMUM; in amdgpu_mes_ring_to_queue_props()
1098 props->paging = false; in amdgpu_mes_ring_to_queue_props()
1099 props->ring = ring; in amdgpu_mes_ring_to_queue_props()
1106 _eng[ring->idx].slots[id_offs]); \
1109 _eng[ring->idx].ring); \
1112 _eng[ring->idx].ib); \
1115 _eng[ring->idx].padding); \
1120 switch (ring->funcs->type) { in amdgpu_mes_ctx_get_offs()
1122 DEFINE_AMDGPU_MES_CTX_GET_OFFS_ENG(gfx); in amdgpu_mes_ctx_get_offs()
1135 return -EINVAL; in amdgpu_mes_ctx_get_offs()
1152 amdgpu_mes_lock(&adev->mes); in amdgpu_mes_add_ring()
1153 gang = idr_find(&adev->mes.gang_id_idr, gang_id); in amdgpu_mes_add_ring()
1156 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1157 return -EINVAL; in amdgpu_mes_add_ring()
1159 pasid = gang->process->pasid; in amdgpu_mes_add_ring()
1163 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1164 return -ENOMEM; in amdgpu_mes_add_ring()
1167 ring->ring_obj = NULL; in amdgpu_mes_add_ring()
1168 ring->use_doorbell = true; in amdgpu_mes_add_ring()
1169 ring->is_mes_queue = true; in amdgpu_mes_add_ring()
1170 ring->mes_ctx = ctx_data; in amdgpu_mes_add_ring()
1171 ring->idx = idx; in amdgpu_mes_add_ring()
1172 ring->no_scheduler = true; in amdgpu_mes_add_ring()
1176 compute[ring->idx].mec_hpd); in amdgpu_mes_add_ring()
1177 ring->eop_gpu_addr = in amdgpu_mes_add_ring()
1183 ring->funcs = adev->gfx.gfx_ring[0].funcs; in amdgpu_mes_add_ring()
1184 ring->me = adev->gfx.gfx_ring[0].me; in amdgpu_mes_add_ring()
1185 ring->pipe = adev->gfx.gfx_ring[0].pipe; in amdgpu_mes_add_ring()
1188 ring->funcs = adev->gfx.compute_ring[0].funcs; in amdgpu_mes_add_ring()
1189 ring->me = adev->gfx.compute_ring[0].me; in amdgpu_mes_add_ring()
1190 ring->pipe = adev->gfx.compute_ring[0].pipe; in amdgpu_mes_add_ring()
1193 ring->funcs = adev->sdma.instance[0].ring.funcs; in amdgpu_mes_add_ring()
1202 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1208 dma_fence_wait(gang->process->vm->last_update, false); in amdgpu_mes_add_ring()
1209 dma_fence_wait(ctx_data->meta_data_va->last_pt_update, false); in amdgpu_mes_add_ring()
1210 amdgpu_mes_unlock(&adev->mes); in amdgpu_mes_add_ring()
1216 ring->hw_queue_id = queue_id; in amdgpu_mes_add_ring()
1217 ring->doorbell_index = qprops.doorbell_off; in amdgpu_mes_add_ring()
1220 sprintf(ring->name, "gfx_%d.%d.%d", pasid, gang_id, queue_id); in amdgpu_mes_add_ring()
1222 sprintf(ring->name, "compute_%d.%d.%d", pasid, gang_id, in amdgpu_mes_add_ring()
1225 sprintf(ring->name, "sdma_%d.%d.%d", pasid, gang_id, in amdgpu_mes_add_ring()
1246 amdgpu_mes_remove_hw_queue(adev, ring->hw_queue_id); in amdgpu_mes_remove_ring()
1247 del_timer_sync(&ring->fence_drv.fallback_timer); in amdgpu_mes_remove_ring()
1255 return adev->mes.aggregated_doorbells[prio]; in amdgpu_mes_get_aggregated_doorbell_index()
1266 &ctx_data->meta_data_obj, in amdgpu_mes_ctx_alloc_meta_data()
1267 &ctx_data->meta_data_mc_addr, in amdgpu_mes_ctx_alloc_meta_data()
1268 &ctx_data->meta_data_ptr); in amdgpu_mes_ctx_alloc_meta_data()
1270 dev_warn(adev->dev, "(%d) create CTX bo failed\n", r); in amdgpu_mes_ctx_alloc_meta_data()
1274 if (!ctx_data->meta_data_obj) in amdgpu_mes_ctx_alloc_meta_data()
1275 return -ENOMEM; in amdgpu_mes_ctx_alloc_meta_data()
1277 memset(ctx_data->meta_data_ptr, 0, in amdgpu_mes_ctx_alloc_meta_data()
1285 if (ctx_data->meta_data_obj) in amdgpu_mes_ctx_free_meta_data()
1286 amdgpu_bo_free_kernel(&ctx_data->meta_data_obj, in amdgpu_mes_ctx_free_meta_data()
1287 &ctx_data->meta_data_mc_addr, in amdgpu_mes_ctx_free_meta_data()
1288 &ctx_data->meta_data_ptr); in amdgpu_mes_ctx_free_meta_data()
1305 &ctx_data->meta_data_obj->tbo.base); in amdgpu_mes_ctx_map_meta_data()
1316 bo_va = amdgpu_vm_bo_add(adev, vm, ctx_data->meta_data_obj); in amdgpu_mes_ctx_map_meta_data()
1319 r = -ENOMEM; in amdgpu_mes_ctx_map_meta_data()
1323 r = amdgpu_vm_bo_map(adev, bo_va, ctx_data->meta_data_gpu_addr, 0, in amdgpu_mes_ctx_map_meta_data()
1338 amdgpu_sync_fence(&sync, bo_va->last_pt_update); in amdgpu_mes_ctx_map_meta_data()
1345 amdgpu_sync_fence(&sync, vm->last_update); in amdgpu_mes_ctx_map_meta_data()
1351 ctx_data->meta_data_va = bo_va; in amdgpu_mes_ctx_map_meta_data()
1366 struct amdgpu_bo_va *bo_va = ctx_data->meta_data_va; in amdgpu_mes_ctx_unmap_meta_data()
1367 struct amdgpu_bo *bo = ctx_data->meta_data_obj; in amdgpu_mes_ctx_unmap_meta_data()
1368 struct amdgpu_vm *vm = bo_va->base.vm; in amdgpu_mes_ctx_unmap_meta_data()
1376 &ctx_data->meta_data_obj->tbo.base); in amdgpu_mes_ctx_unmap_meta_data()
1391 r = dma_resv_get_singleton(bo->tbo.base.resv, DMA_RESV_USAGE_BOOKKEEP, in amdgpu_mes_ctx_unmap_meta_data()
1410 dev_err(adev->dev, "failed to clear page tables (%ld)\n", r); in amdgpu_mes_ctx_unmap_meta_data()
1428 gprops.gang_quantum = adev->mes.default_gang_quantum; in amdgpu_mes_test_create_gang_and_queues()
1448 DRM_INFO("ring %s was added\n", ring->name); in amdgpu_mes_test_create_gang_and_queues()
1471 DRM_DEV_ERROR(ring->adev->dev, in amdgpu_mes_test_queues()
1473 ring->name, r); in amdgpu_mes_test_queues()
1476 DRM_INFO("ring %s ib test pass\n", ring->name); in amdgpu_mes_test_queues()
1495 dev_warn(adev->dev, "No more PASIDs available!"); in amdgpu_mes_self_test()
1501 r = -ENOMEM; in amdgpu_mes_self_test()
1505 r = amdgpu_vm_init(adev, vm, -1); in amdgpu_mes_self_test()
1531 /* On GFX v10.3, fw hasn't supported to map sdma queue. */ in amdgpu_mes_self_test()
1597 if (adev->enable_uni_mes) { in amdgpu_mes_init_microcode()
1612 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], AMDGPU_UCODE_REQUIRED, in amdgpu_mes_init_microcode()
1615 dev_info(adev->dev, "try to fall back to %s_mes.bin\n", ucode_prefix); in amdgpu_mes_init_microcode()
1616 r = amdgpu_ucode_request(adev, &adev->mes.fw[pipe], in amdgpu_mes_init_microcode()
1625 adev->mes.fw[pipe]->data; in amdgpu_mes_init_microcode()
1626 adev->mes.uc_start_addr[pipe] = in amdgpu_mes_init_microcode()
1627 le32_to_cpu(mes_hdr->mes_uc_start_addr_lo) | in amdgpu_mes_init_microcode()
1628 ((uint64_t)(le32_to_cpu(mes_hdr->mes_uc_start_addr_hi)) << 32); in amdgpu_mes_init_microcode()
1629 adev->mes.data_start_addr[pipe] = in amdgpu_mes_init_microcode()
1630 le32_to_cpu(mes_hdr->mes_data_start_addr_lo) | in amdgpu_mes_init_microcode()
1631 ((uint64_t)(le32_to_cpu(mes_hdr->mes_data_start_addr_hi)) << 32); in amdgpu_mes_init_microcode()
1632 ucode_ptr = (u32 *)(adev->mes.fw[pipe]->data + in amdgpu_mes_init_microcode()
1634 adev->mes.fw_version[pipe] = in amdgpu_mes_init_microcode()
1637 if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { in amdgpu_mes_init_microcode()
1648 info = &adev->firmware.ucode[ucode]; in amdgpu_mes_init_microcode()
1649 info->ucode_id = ucode; in amdgpu_mes_init_microcode()
1650 info->fw = adev->mes.fw[pipe]; in amdgpu_mes_init_microcode()
1651 adev->firmware.fw_size += in amdgpu_mes_init_microcode()
1652 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_size_bytes), in amdgpu_mes_init_microcode()
1655 info = &adev->firmware.ucode[ucode_data]; in amdgpu_mes_init_microcode()
1656 info->ucode_id = ucode_data; in amdgpu_mes_init_microcode()
1657 info->fw = adev->mes.fw[pipe]; in amdgpu_mes_init_microcode()
1658 adev->firmware.fw_size += in amdgpu_mes_init_microcode()
1659 ALIGN(le32_to_cpu(mes_hdr->mes_ucode_data_size_bytes), in amdgpu_mes_init_microcode()
1665 amdgpu_ucode_release(&adev->mes.fw[pipe]); in amdgpu_mes_init_microcode()
1671 uint32_t mes_rev = adev->mes.sched_version & AMDGPU_MES_VERSION_MASK; in amdgpu_mes_suspend_resume_all_supported()
1682 /* Fix me -- node_id is used to identify the correct MES instances in the future */
1692 if (!adev->mes.funcs->misc_op) { in amdgpu_mes_set_enforce_isolation()
1693 dev_err(adev->dev, "mes change config is not supported!\n"); in amdgpu_mes_set_enforce_isolation()
1694 r = -EINVAL; in amdgpu_mes_set_enforce_isolation()
1698 r = adev->mes.funcs->misc_op(&adev->mes, &op_input); in amdgpu_mes_set_enforce_isolation()
1700 dev_err(adev->dev, "failed to change_config.\n"); in amdgpu_mes_set_enforce_isolation()
1710 if (adev->enable_mes && adev->gfx.enable_cleaner_shader) { in amdgpu_mes_update_enforce_isolation()
1711 mutex_lock(&adev->enforce_isolation_mutex); in amdgpu_mes_update_enforce_isolation()
1712 for (i = 0; i < (adev->xcp_mgr ? adev->xcp_mgr->num_xcps : 1); i++) { in amdgpu_mes_update_enforce_isolation()
1713 if (adev->enforce_isolation[i]) in amdgpu_mes_update_enforce_isolation()
1718 mutex_unlock(&adev->enforce_isolation_mutex); in amdgpu_mes_update_enforce_isolation()
1727 struct amdgpu_device *adev = m->private; in amdgpu_debugfs_mes_event_log_show()
1728 uint32_t *mem = (uint32_t *)(adev->mes.event_log_cpu_addr); in amdgpu_debugfs_mes_event_log_show() local
1731 mem, adev->mes.event_log_size, false); in amdgpu_debugfs_mes_event_log_show()
1744 struct drm_minor *minor = adev_to_drm(adev)->primary; in amdgpu_debugfs_mes_event_log_init()
1745 struct dentry *root = minor->debugfs_root; in amdgpu_debugfs_mes_event_log_init()
1746 if (adev->enable_mes && amdgpu_mes_log_enable) in amdgpu_debugfs_mes_event_log_init()