Lines Matching full:group
42 * The scheduling happens at the scheduling group level, each group
51 * rotating the groups passed to the firmware so every group gets
67 * queue ring-buffer, and the group is scheduled for execution if it
70 * Kernel-side group scheduling is timeslice-based. When we have less
73 * groups than slots, we let each group a chance to execute stuff for
75 * to schedule. The group selection algorithm is based on
80 * group/queue state that would be based on information we wouldn't have
82 * reason we don't do 'cooperative' scheduling (encoding FW group slot
84 * drm_gpu_scheduler::prepare_job() hook, and treating group rotation as
103 * struct panthor_csg_slot - Command stream group slot
105 * This represents a FW slot for a scheduling group.
108 /** @group: Scheduling group bound to this slot. */
109 struct panthor_group *group; member
111 /** @priority: Group priority. */
115 * @idle: True if the group bound to this slot is idle.
117 * A group is idle when it has nothing waiting for execution on
125 * enum panthor_csg_priority - Group priority
128 /** @PANTHOR_CSG_PRIORITY_LOW: Low priority group. */
131 /** @PANTHOR_CSG_PRIORITY_MEDIUM: Medium priority group. */
134 /** @PANTHOR_CSG_PRIORITY_HIGH: High priority group. */
138 * @PANTHOR_CSG_PRIORITY_RT: Real-time priority group.
141 * non-real-time groups. When such a group becomes executable,
142 * it will evict the group with the lowest non-rt priority if
143 * there's no free group slot available.
162 * Used for the scheduler tick, group update or other kind of FW
234 * @runnable: Runnable group lists.
236 * When a group has queues that want to execute something,
244 * @idle: Idle group lists.
246 * When all queues of a group are idle (either because they
258 * Insert panthor_group::wait_node here when a group is waiting
267 * @csg_slots: FW command stream group slots.
271 /** @csg_slot_count: Number of command stream group slots exposed by the FW. */
274 /** @cs_slot_count: Number of command stream slot per group slot exposed by the FW. */
280 /** @used_csg_slot_count: Number of command stream group slot currently used. */
287 * @might_have_idle_groups: True if an active group might have become idle.
382 * is assigned to group_slot + 1 when the group is assigned a slot. But
384 * point, so don't have to wake up all queues in a group every time one
390 * @priority: Priority of the queue inside the group.
496 * enum panthor_group_state - Scheduling group state.
499 /** @PANTHOR_CS_GROUP_CREATED: Group was created, but not scheduled yet. */
502 /** @PANTHOR_CS_GROUP_ACTIVE: Group is currently scheduled. */
506 * @PANTHOR_CS_GROUP_SUSPENDED: Group was scheduled at least once, but is
512 * @PANTHOR_CS_GROUP_TERMINATED: Group was terminated.
519 * @PANTHOR_CS_GROUP_UNKNOWN_STATE: Group is an unknown state.
521 * The FW returned an inconsistent state. The group is flagged unusable
532 * struct panthor_group - Scheduling group object
541 /** @vm: VM bound to the group. */
562 /** @priority: Group priority (check panthor_csg_priority). */
580 /** @queue_count: Number of queues in this group. */
583 /** @queues: Queues owned by this group. */
587 * @csg_id: ID of the FW group slot.
589 * -1 when the group is not scheduled/active.
594 * @destroyed: True when the group has been destroyed.
596 * If a group is destroyed it becomes useless: no further jobs can be submitted
598 * release the group object.
604 * this group.
607 * and the group can't be suspended, this also leads to a timeout. In any case,
608 * any timeout situation is unrecoverable, and the group becomes useless. We
609 * simply wait for all references to be dropped so we can release the group
615 * @innocent: True when the group becomes unusable because the group suspension
618 * Sometimes the FW was put in a bad state by other groups, causing the group
620 * group innocent.
634 /** @data: Total sampled values for jobs in queues from this group. */
643 /** @fdinfo.kbo_sizes: Aggregate size of private kernel BO's held by the group. */
647 /** @state: Group state. */
653 * Stores the state of the group and its queues when a group is suspended.
654 * Used at resume time to restore the group in its previous state.
663 * Stores the state of the group and its queues when a group that's in
666 * Used at resume time to restore the group in its previous state.
676 /** @tiler_oom_work: Work used to process tiler OOM events happening on this group. */
679 /** @term_work: Work used to finish the group termination procedure. */
683 * @release_work: Work used to release group resources.
685 * We need to postpone the group release to avoid a deadlock when
691 * @run_node: Node used to insert the group in the
698 * @wait_node: Node used to insert the group in the
717 * group_queue_work() - Queue a group work
718 * @group: Group to queue the work for.
726 #define group_queue_work(group, wname) \ argument
728 group_get(group); \
729 if (!queue_work((group)->ptdev->scheduler->wq, &(group)->wname ## _work)) \
730 group_put(group); \
770 * struct panthor_group_pool - Group pool
772 * Each file get assigned a group pool.
775 /** @xa: Xarray used to manage group handles. */
789 /** @group: Group of the queue this job will be pushed to. */
790 struct panthor_group *group; member
792 /** @queue_idx: Index of the queue inside @group. */
855 panthor_queue_get_syncwait_obj(struct panthor_group *group, struct panthor_queue *queue) in panthor_queue_get_syncwait_obj() argument
857 struct panthor_device *ptdev = group->ptdev; in panthor_queue_get_syncwait_obj()
865 bo = panthor_vm_get_bo_for_va(group->vm, in panthor_queue_get_syncwait_obj()
887 static void group_free_queue(struct panthor_group *group, struct panthor_queue *queue) in group_free_queue() argument
912 struct panthor_group *group = container_of(work, in group_release_work() local
917 for (i = 0; i < group->queue_count; i++) in group_release_work()
918 group_free_queue(group, group->queues[i]); in group_release_work()
920 panthor_kernel_bo_destroy(group->suspend_buf); in group_release_work()
921 panthor_kernel_bo_destroy(group->protm_suspend_buf); in group_release_work()
922 panthor_kernel_bo_destroy(group->syncobjs); in group_release_work()
924 panthor_vm_put(group->vm); in group_release_work()
925 kfree(group); in group_release_work()
930 struct panthor_group *group = container_of(kref, in group_release() local
933 struct panthor_device *ptdev = group->ptdev; in group_release()
935 drm_WARN_ON(&ptdev->base, group->csg_id >= 0); in group_release()
936 drm_WARN_ON(&ptdev->base, !list_empty(&group->run_node)); in group_release()
937 drm_WARN_ON(&ptdev->base, !list_empty(&group->wait_node)); in group_release()
939 queue_work(panthor_cleanup_wq, &group->release_work); in group_release()
942 static void group_put(struct panthor_group *group) in group_put() argument
944 if (group) in group_put()
945 kref_put(&group->refcount, group_release); in group_put()
949 group_get(struct panthor_group *group) in group_get() argument
951 if (group) in group_get()
952 kref_get(&group->refcount); in group_get()
954 return group; in group_get()
958 * group_bind_locked() - Bind a group to a group slot
959 * @group: Group.
965 group_bind_locked(struct panthor_group *group, u32 csg_id) in group_bind_locked() argument
967 struct panthor_device *ptdev = group->ptdev; in group_bind_locked()
973 if (drm_WARN_ON(&ptdev->base, group->csg_id != -1 || csg_id >= MAX_CSGS || in group_bind_locked()
974 ptdev->scheduler->csg_slots[csg_id].group)) in group_bind_locked()
977 ret = panthor_vm_active(group->vm); in group_bind_locked()
982 group_get(group); in group_bind_locked()
983 group->csg_id = csg_id; in group_bind_locked()
985 /* Dummy doorbell allocation: doorbell is assigned to the group and in group_bind_locked()
990 * on queues belonging to the same group that are rarely updated. in group_bind_locked()
992 for (u32 i = 0; i < group->queue_count; i++) in group_bind_locked()
993 group->queues[i]->doorbell_id = csg_id + 1; in group_bind_locked()
995 csg_slot->group = group; in group_bind_locked()
1001 * group_unbind_locked() - Unbind a group from a slot.
1002 * @group: Group to unbind.
1007 group_unbind_locked(struct panthor_group *group) in group_unbind_locked() argument
1009 struct panthor_device *ptdev = group->ptdev; in group_unbind_locked()
1014 if (drm_WARN_ON(&ptdev->base, group->csg_id < 0 || group->csg_id >= MAX_CSGS)) in group_unbind_locked()
1017 if (drm_WARN_ON(&ptdev->base, group->state == PANTHOR_CS_GROUP_ACTIVE)) in group_unbind_locked()
1020 slot = &ptdev->scheduler->csg_slots[group->csg_id]; in group_unbind_locked()
1021 panthor_vm_idle(group->vm); in group_unbind_locked()
1022 group->csg_id = -1; in group_unbind_locked()
1024 /* Tiler OOM events will be re-issued next time the group is scheduled. */ in group_unbind_locked()
1025 atomic_set(&group->tiler_oom, 0); in group_unbind_locked()
1026 cancel_work(&group->tiler_oom_work); in group_unbind_locked()
1028 for (u32 i = 0; i < group->queue_count; i++) in group_unbind_locked()
1029 group->queues[i]->doorbell_id = -1; in group_unbind_locked()
1031 slot->group = NULL; in group_unbind_locked()
1033 group_put(group); in group_unbind_locked()
1040 * @csg_id: Group slot ID.
1046 * The group slot must have a group bound to it already (group_bind_locked()).
1051 struct panthor_queue *queue = ptdev->scheduler->csg_slots[csg_id].group->queues[cs_id]; in cs_slot_prog_locked()
1084 * @csg_id: Group slot.
1090 * The group slot must have a group bound to it (group_bind_locked()).
1096 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; in cs_slot_reset_locked() local
1097 struct panthor_queue *queue = group->queues[cs_id]; in cs_slot_reset_locked()
1106 * we can detect unbounded waits and kill the group when that happens. in cs_slot_reset_locked()
1108 if (!(group->blocked_queues & BIT(cs_id)) && !queue->timeout_suspended) { in cs_slot_reset_locked()
1118 * csg_slot_sync_priority_locked() - Synchronize the group slot priority
1120 * @csg_id: Group slot ID.
1122 * Group slot priority update happens asynchronously. When we receive a
1141 * @csg_id: Group slot.
1144 * Queue state is updated on group suspend or STATUS_UPDATE event.
1149 struct panthor_group *group = ptdev->scheduler->csg_slots[csg_id].group; in cs_slot_sync_queue_state_locked() local
1150 struct panthor_queue *queue = group->queues[cs_id]; in cs_slot_sync_queue_state_locked()
1152 panthor_fw_get_cs_iface(group->ptdev, csg_id, cs_id); in cs_slot_sync_queue_state_locked()
1160 group->idle_queues |= BIT(cs_id); in cs_slot_sync_queue_state_locked()
1164 if (list_empty(&group->wait_node)) { in cs_slot_sync_queue_state_locked()
1165 list_move_tail(&group->wait_node, in cs_slot_sync_queue_state_locked()
1166 &group->ptdev->scheduler->groups.waiting); in cs_slot_sync_queue_state_locked()
1173 group->blocked_queues |= BIT(cs_id); in cs_slot_sync_queue_state_locked()
1201 struct panthor_group *group = csg_slot->group; in csg_slot_sync_queues_state_locked() local
1206 group->idle_queues = 0; in csg_slot_sync_queues_state_locked()
1207 group->blocked_queues = 0; in csg_slot_sync_queues_state_locked()
1209 for (i = 0; i < group->queue_count; i++) { in csg_slot_sync_queues_state_locked()
1210 if (group->queues[i]) in csg_slot_sync_queues_state_locked()
1220 struct panthor_group *group; in csg_slot_sync_state_locked() local
1227 group = csg_slot->group; in csg_slot_sync_state_locked()
1229 if (!group) in csg_slot_sync_state_locked()
1232 old_state = group->state; in csg_slot_sync_state_locked()
1247 * which means the group metadata can't be trusted anymore, and in csg_slot_sync_state_locked()
1249 * suspend buffers. Flag the group state as unknown to make in csg_slot_sync_state_locked()
1274 * state when starting/resuming a new group on this in csg_slot_sync_state_locked()
1279 for (i = 0; i < group->queue_count; i++) { in csg_slot_sync_state_locked()
1280 if (group->queues[i]) in csg_slot_sync_state_locked()
1285 group->state = new_state; in csg_slot_sync_state_locked()
1293 struct panthor_group *group; in csg_slot_prog_locked() local
1305 group = csg_slot->group; in csg_slot_prog_locked()
1306 if (!group || group->state == PANTHOR_CS_GROUP_ACTIVE) in csg_slot_prog_locked()
1309 csg_iface = panthor_fw_get_csg_iface(group->ptdev, csg_id); in csg_slot_prog_locked()
1311 for (i = 0; i < group->queue_count; i++) { in csg_slot_prog_locked()
1312 if (group->queues[i]) { in csg_slot_prog_locked()
1318 csg_iface->input->allow_compute = group->compute_core_mask; in csg_slot_prog_locked()
1319 csg_iface->input->allow_fragment = group->fragment_core_mask; in csg_slot_prog_locked()
1320 csg_iface->input->allow_other = group->tiler_core_mask; in csg_slot_prog_locked()
1321 csg_iface->input->endpoint_req = CSG_EP_REQ_COMPUTE(group->max_compute_cores) | in csg_slot_prog_locked()
1322 CSG_EP_REQ_FRAGMENT(group->max_fragment_cores) | in csg_slot_prog_locked()
1323 CSG_EP_REQ_TILER(group->max_tiler_cores) | in csg_slot_prog_locked()
1325 csg_iface->input->config = panthor_vm_as(group->vm); in csg_slot_prog_locked()
1327 if (group->suspend_buf) in csg_slot_prog_locked()
1328 csg_iface->input->suspend_buf = panthor_kernel_bo_gpuva(group->suspend_buf); in csg_slot_prog_locked()
1332 if (group->protm_suspend_buf) { in csg_slot_prog_locked()
1334 panthor_kernel_bo_gpuva(group->protm_suspend_buf); in csg_slot_prog_locked()
1350 struct panthor_group *group = csg_slot->group; in cs_slot_process_fatal_event_locked() local
1361 if (group) in cs_slot_process_fatal_event_locked()
1362 group->fatal_queues |= BIT(cs_id); in cs_slot_process_fatal_event_locked()
1392 struct panthor_group *group = csg_slot->group; in cs_slot_process_fault_event_locked() local
1393 struct panthor_queue *queue = group && cs_id < group->queue_count ? in cs_slot_process_fault_event_locked()
1394 group->queues[cs_id] : NULL; in cs_slot_process_fault_event_locked()
1434 static int group_process_tiler_oom(struct panthor_group *group, u32 cs_id) in group_process_tiler_oom() argument
1436 struct panthor_device *ptdev = group->ptdev; in group_process_tiler_oom()
1445 csg_id = group->csg_id; in group_process_tiler_oom()
1450 heaps = panthor_vm_get_heap_pool(group->vm, false); in group_process_tiler_oom()
1460 /* The group got scheduled out, we stop here. We will get a new tiler OOM event in group_process_tiler_oom()
1484 group->fatal_queues |= BIT(cs_id); in group_process_tiler_oom()
1490 csg_id = group->csg_id; in group_process_tiler_oom()
1507 * context because the group was scheduled out while we were in group_process_tiler_oom()
1522 struct panthor_group *group = in group_tiler_oom_work() local
1524 u32 tiler_oom = atomic_xchg(&group->tiler_oom, 0); in group_tiler_oom_work()
1529 group_process_tiler_oom(group, cs_id); in group_tiler_oom_work()
1533 group_put(group); in group_tiler_oom_work()
1542 struct panthor_group *group = csg_slot->group; in cs_slot_process_tiler_oom_event_locked() local
1546 if (drm_WARN_ON(&ptdev->base, !group)) in cs_slot_process_tiler_oom_event_locked()
1549 atomic_or(BIT(cs_id), &group->tiler_oom); in cs_slot_process_tiler_oom_event_locked()
1554 group_get(group); in cs_slot_process_tiler_oom_event_locked()
1555 if (!queue_work(sched->heap_alloc_wq, &group->tiler_oom_work)) in cs_slot_process_tiler_oom_event_locked()
1556 group_put(group); in cs_slot_process_tiler_oom_event_locked()
1619 struct panthor_group *group = csg_slot->group; in csg_slot_sync_update_locked() local
1623 if (group) in csg_slot_sync_update_locked()
1624 group_queue_work(group, sync_upd); in csg_slot_sync_update_locked()
1634 struct panthor_group *group = csg_slot->group; in csg_slot_process_progress_timer_event_locked() local
1640 group = csg_slot->group; in csg_slot_process_progress_timer_event_locked()
1641 if (!drm_WARN_ON(&ptdev->base, !group)) in csg_slot_process_progress_timer_event_locked()
1642 group->timedout = true; in csg_slot_process_progress_timer_event_locked()
1895 group_is_idle(struct panthor_group *group) in group_is_idle() argument
1897 struct panthor_device *ptdev = group->ptdev; in group_is_idle()
1900 if (group->csg_id >= 0) in group_is_idle()
1901 return ptdev->scheduler->csg_slots[group->csg_id].idle; in group_is_idle()
1903 inactive_queues = group->idle_queues | group->blocked_queues; in group_is_idle()
1904 return hweight32(inactive_queues) == group->queue_count; in group_is_idle()
1908 group_can_run(struct panthor_group *group) in group_can_run() argument
1910 return group->state != PANTHOR_CS_GROUP_TERMINATED && in group_can_run()
1911 group->state != PANTHOR_CS_GROUP_UNKNOWN_STATE && in group_can_run()
1912 !group->destroyed && group->fatal_queues == 0 && in group_can_run()
1913 !group->timedout; in group_can_run()
1923 struct panthor_group *group, *tmp; in tick_ctx_pick_groups_from_list() local
1928 list_for_each_entry_safe(group, tmp, queue, run_node) { in tick_ctx_pick_groups_from_list()
1931 if (!group_can_run(group)) in tick_ctx_pick_groups_from_list()
1934 if (skip_idle_groups && group_is_idle(group)) in tick_ctx_pick_groups_from_list()
1938 if (ctx->vms[i] == group->vm) in tick_ctx_pick_groups_from_list()
1946 group_get(group); in tick_ctx_pick_groups_from_list()
1948 list_move_tail(&group->run_node, &ctx->groups[group->priority]); in tick_ctx_pick_groups_from_list()
1950 if (group_is_idle(group)) in tick_ctx_pick_groups_from_list()
1954 ctx->vms[ctx->as_count++] = group->vm; in tick_ctx_pick_groups_from_list()
1956 if (ctx->min_priority > group->priority) in tick_ctx_pick_groups_from_list()
1957 ctx->min_priority = group->priority; in tick_ctx_pick_groups_from_list()
1967 struct panthor_group *group, in tick_ctx_insert_old_group() argument
1970 struct panthor_csg_slot *csg_slot = &sched->csg_slots[group->csg_id]; in tick_ctx_insert_old_group()
1974 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); in tick_ctx_insert_old_group()
1982 * important to make sure we don't let one group starve in tick_ctx_insert_old_group()
1983 * all other groups with the same group priority. in tick_ctx_insert_old_group()
1986 &ctx->old_groups[csg_slot->group->priority], in tick_ctx_insert_old_group()
1991 list_add_tail(&csg_slot->group->run_node, &other_group->run_node); in tick_ctx_insert_old_group()
1996 list_add_tail(&group->run_node, &ctx->old_groups[group->priority]); in tick_ctx_insert_old_group()
2020 struct panthor_group *group = csg_slot->group; in tick_ctx_init() local
2023 if (!group) in tick_ctx_init()
2027 group_get(group); in tick_ctx_init()
2032 if (panthor_vm_has_unhandled_faults(group->vm)) { in tick_ctx_init()
2036 if (!group->fatal_queues) in tick_ctx_init()
2037 group->fatal_queues |= GENMASK(group->queue_count - 1, 0); in tick_ctx_init()
2040 tick_ctx_insert_old_group(sched, ctx, group, full_tick); in tick_ctx_init()
2054 group_term_post_processing(struct panthor_group *group) in group_term_post_processing() argument
2061 if (drm_WARN_ON(&group->ptdev->base, group_can_run(group))) in group_term_post_processing()
2065 for (i = 0; i < group->queue_count; i++) { in group_term_post_processing()
2066 struct panthor_queue *queue = group->queues[i]; in group_term_post_processing()
2070 if (group->fatal_queues & BIT(i)) in group_term_post_processing()
2072 else if (group->timedout) in group_term_post_processing()
2089 syncobj = group->syncobjs->kmap + (i * sizeof(*syncobj)); in group_term_post_processing()
2092 sched_queue_work(group->ptdev->scheduler, sync_upd); in group_term_post_processing()
2104 struct panthor_group *group = in group_term_work() local
2107 group_term_post_processing(group); in group_term_work()
2108 group_put(group); in group_term_work()
2116 struct panthor_group *group, *tmp; in tick_ctx_cleanup() local
2120 list_for_each_entry_safe(group, tmp, &ctx->old_groups[i], run_node) { in tick_ctx_cleanup()
2125 group_can_run(group)); in tick_ctx_cleanup()
2127 if (!group_can_run(group)) { in tick_ctx_cleanup()
2128 list_del_init(&group->run_node); in tick_ctx_cleanup()
2129 list_del_init(&group->wait_node); in tick_ctx_cleanup()
2130 group_queue_work(group, term); in tick_ctx_cleanup()
2131 } else if (group->csg_id >= 0) { in tick_ctx_cleanup()
2132 list_del_init(&group->run_node); in tick_ctx_cleanup()
2134 list_move(&group->run_node, in tick_ctx_cleanup()
2135 group_is_idle(group) ? in tick_ctx_cleanup()
2136 &sched->groups.idle[group->priority] : in tick_ctx_cleanup()
2137 &sched->groups.runnable[group->priority]); in tick_ctx_cleanup()
2139 group_put(group); in tick_ctx_cleanup()
2150 list_for_each_entry_safe(group, tmp, &ctx->groups[i], run_node) { in tick_ctx_cleanup()
2151 if (group->csg_id >= 0) { in tick_ctx_cleanup()
2152 list_del_init(&group->run_node); in tick_ctx_cleanup()
2154 list_move(&group->run_node, in tick_ctx_cleanup()
2155 group_is_idle(group) ? in tick_ctx_cleanup()
2156 &sched->groups.idle[group->priority] : in tick_ctx_cleanup()
2157 &sched->groups.runnable[group->priority]); in tick_ctx_cleanup()
2159 group_put(group); in tick_ctx_cleanup()
2167 struct panthor_group *group, *tmp; in tick_ctx_apply() local
2179 list_for_each_entry(group, &ctx->old_groups[prio], run_node) { in tick_ctx_apply()
2180 bool term = !group_can_run(group); in tick_ctx_apply()
2181 int csg_id = group->csg_id; in tick_ctx_apply()
2193 list_for_each_entry(group, &ctx->groups[prio], run_node) { in tick_ctx_apply()
2195 int csg_id = group->csg_id; in tick_ctx_apply()
2228 list_for_each_entry(group, &ctx->old_groups[prio], run_node) { in tick_ctx_apply()
2229 /* This group is gone. Process interrupts to clear in tick_ctx_apply()
2231 * group. in tick_ctx_apply()
2233 if (group->csg_id >= 0) in tick_ctx_apply()
2234 sched_process_csg_irq_locked(ptdev, group->csg_id); in tick_ctx_apply()
2236 group_unbind_locked(group); in tick_ctx_apply()
2241 if (!sched->csg_slots[i].group) in tick_ctx_apply()
2250 list_for_each_entry(group, &ctx->groups[prio], run_node) { in tick_ctx_apply()
2251 int csg_id = group->csg_id; in tick_ctx_apply()
2265 group_bind_locked(group, csg_id); in tick_ctx_apply()
2268 group->state == PANTHOR_CS_GROUP_SUSPENDED ? in tick_ctx_apply()
2286 list_for_each_entry_safe(group, tmp, &ctx->groups[prio], run_node) { in tick_ctx_apply()
2287 list_del_init(&group->run_node); in tick_ctx_apply()
2289 /* If the group has been destroyed while we were in tick_ctx_apply()
2292 * this dangling group. in tick_ctx_apply()
2294 if (group->destroyed) in tick_ctx_apply()
2296 group_put(group); in tick_ctx_apply()
2304 list_for_each_entry_safe(group, tmp, &ctx->old_groups[prio], run_node) { in tick_ctx_apply()
2305 if (!group_can_run(group)) in tick_ctx_apply()
2308 if (group_is_idle(group)) in tick_ctx_apply()
2309 list_move_tail(&group->run_node, &sched->groups.idle[prio]); in tick_ctx_apply()
2311 list_move_tail(&group->run_node, &sched->groups.runnable[prio]); in tick_ctx_apply()
2312 group_put(group); in tick_ctx_apply()
2329 * event happens (group unblocked, new job submitted, ...). in tick_ctx_update_resched_target()
2459 static int panthor_queue_eval_syncwait(struct panthor_group *group, u8 queue_idx) in panthor_queue_eval_syncwait() argument
2461 struct panthor_queue *queue = group->queues[queue_idx]; in panthor_queue_eval_syncwait()
2469 syncobj = panthor_queue_get_syncwait_obj(group, queue); in panthor_queue_eval_syncwait()
2493 struct panthor_group *group, *tmp; in sync_upd_work() local
2497 list_for_each_entry_safe(group, tmp, &sched->groups.waiting, wait_node) { in sync_upd_work()
2498 u32 tested_queues = group->blocked_queues; in sync_upd_work()
2505 ret = panthor_queue_eval_syncwait(group, cs_id); in sync_upd_work()
2506 drm_WARN_ON(&group->ptdev->base, ret < 0); in sync_upd_work()
2514 group->blocked_queues &= ~unblocked_queues; in sync_upd_work()
2516 if (group->csg_id < 0) { in sync_upd_work()
2517 list_move(&group->run_node, in sync_upd_work()
2518 &sched->groups.runnable[group->priority]); in sync_upd_work()
2519 if (group->priority == PANTHOR_CSG_PRIORITY_RT) in sync_upd_work()
2524 if (!group->blocked_queues) in sync_upd_work()
2525 list_del_init(&group->wait_node); in sync_upd_work()
2533 static void group_schedule_locked(struct panthor_group *group, u32 queue_mask) in group_schedule_locked() argument
2535 struct panthor_device *ptdev = group->ptdev; in group_schedule_locked()
2537 struct list_head *queue = &sched->groups.runnable[group->priority]; in group_schedule_locked()
2542 if (!group_can_run(group)) in group_schedule_locked()
2546 if ((queue_mask & group->blocked_queues) == queue_mask) in group_schedule_locked()
2549 was_idle = group_is_idle(group); in group_schedule_locked()
2550 group->idle_queues &= ~queue_mask; in group_schedule_locked()
2556 if (was_idle && !group_is_idle(group)) in group_schedule_locked()
2557 list_move_tail(&group->run_node, queue); in group_schedule_locked()
2560 if (group->priority == PANTHOR_CSG_PRIORITY_RT) { in group_schedule_locked()
2611 static void panthor_group_stop(struct panthor_group *group) in panthor_group_stop() argument
2613 struct panthor_scheduler *sched = group->ptdev->scheduler; in panthor_group_stop()
2617 for (u32 i = 0; i < group->queue_count; i++) in panthor_group_stop()
2618 queue_stop(group->queues[i], NULL); in panthor_group_stop()
2620 group_get(group); in panthor_group_stop()
2621 list_move_tail(&group->run_node, &sched->reset.stopped_groups); in panthor_group_stop()
2624 static void panthor_group_start(struct panthor_group *group) in panthor_group_start() argument
2626 struct panthor_scheduler *sched = group->ptdev->scheduler; in panthor_group_start()
2628 lockdep_assert_held(&group->ptdev->scheduler->reset.lock); in panthor_group_start()
2630 for (u32 i = 0; i < group->queue_count; i++) in panthor_group_start()
2631 queue_start(group->queues[i]); in panthor_group_start()
2633 if (group_can_run(group)) { in panthor_group_start()
2634 list_move_tail(&group->run_node, in panthor_group_start()
2635 group_is_idle(group) ? in panthor_group_start()
2636 &sched->groups.idle[group->priority] : in panthor_group_start()
2637 &sched->groups.runnable[group->priority]); in panthor_group_start()
2639 list_del_init(&group->run_node); in panthor_group_start()
2640 list_del_init(&group->wait_node); in panthor_group_start()
2641 group_queue_work(group, term); in panthor_group_start()
2644 group_put(group); in panthor_group_start()
2674 struct panthor_group *group; in panthor_sched_suspend() local
2683 if (csg_slot->group) { in panthor_sched_suspend()
2685 group_can_run(csg_slot->group) ? in panthor_sched_suspend()
2705 /* If the group was still usable before that point, we consider in panthor_sched_suspend()
2708 if (group_can_run(csg_slot->group)) in panthor_sched_suspend()
2709 csg_slot->group->innocent = true; in panthor_sched_suspend()
2711 /* We consider group suspension failures as fatal and flag the in panthor_sched_suspend()
2712 * group as unusable by setting timedout=true. in panthor_sched_suspend()
2714 csg_slot->group->timedout = true; in panthor_sched_suspend()
2733 if (csg_slot->group->state != PANTHOR_CS_GROUP_TERMINATED) in panthor_sched_suspend()
2734 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; in panthor_sched_suspend()
2754 csg_slot->group->state = PANTHOR_CS_GROUP_TERMINATED; in panthor_sched_suspend()
2765 group = csg_slot->group; in panthor_sched_suspend()
2766 if (!group) in panthor_sched_suspend()
2769 group_get(group); in panthor_sched_suspend()
2771 if (group->csg_id >= 0) in panthor_sched_suspend()
2772 sched_process_csg_irq_locked(ptdev, group->csg_id); in panthor_sched_suspend()
2774 group_unbind_locked(group); in panthor_sched_suspend()
2776 drm_WARN_ON(&group->ptdev->base, !list_empty(&group->run_node)); in panthor_sched_suspend()
2778 if (group_can_run(group)) { in panthor_sched_suspend()
2779 list_add(&group->run_node, in panthor_sched_suspend()
2780 &sched->groups.idle[group->priority]); in panthor_sched_suspend()
2782 /* We don't bother stopping the scheduler if the group is in panthor_sched_suspend()
2783 * faulty, the group termination work will finish the job. in panthor_sched_suspend()
2785 list_del_init(&group->wait_node); in panthor_sched_suspend()
2786 group_queue_work(group, term); in panthor_sched_suspend()
2788 group_put(group); in panthor_sched_suspend()
2796 struct panthor_group *group, *group_tmp; in panthor_sched_pre_reset() local
2816 list_for_each_entry_safe(group, group_tmp, &sched->groups.runnable[i], run_node) in panthor_sched_pre_reset()
2817 panthor_group_stop(group); in panthor_sched_pre_reset()
2821 list_for_each_entry_safe(group, group_tmp, &sched->groups.idle[i], run_node) in panthor_sched_pre_reset()
2822 panthor_group_stop(group); in panthor_sched_pre_reset()
2831 struct panthor_group *group, *group_tmp; in panthor_sched_post_reset() local
2835 list_for_each_entry_safe(group, group_tmp, &sched->reset.stopped_groups, run_node) { in panthor_sched_post_reset()
2836 /* Consider all previously running group as terminated if the in panthor_sched_post_reset()
2840 group->state = PANTHOR_CS_GROUP_TERMINATED; in panthor_sched_post_reset()
2842 panthor_group_start(group); in panthor_sched_post_reset()
2860 struct panthor_group *group = job->group; in update_fdinfo_stats() local
2861 struct panthor_queue *queue = group->queues[job->queue_idx]; in update_fdinfo_stats()
2862 struct panthor_gpu_usage *fdinfo = &group->fdinfo.data; in update_fdinfo_stats()
2866 scoped_guard(spinlock, &group->fdinfo.lock) { in update_fdinfo_stats()
2877 struct panthor_group *group; in panthor_fdinfo_gather_group_samples() local
2884 xa_for_each(&gpool->xa, i, group) { in panthor_fdinfo_gather_group_samples()
2885 guard(spinlock)(&group->fdinfo.lock); in panthor_fdinfo_gather_group_samples()
2886 pfile->stats.cycles += group->fdinfo.data.cycles; in panthor_fdinfo_gather_group_samples()
2887 pfile->stats.time += group->fdinfo.data.time; in panthor_fdinfo_gather_group_samples()
2888 group->fdinfo.data.cycles = 0; in panthor_fdinfo_gather_group_samples()
2889 group->fdinfo.data.time = 0; in panthor_fdinfo_gather_group_samples()
2896 struct panthor_group *group = in group_sync_upd_work() local
2904 for (queue_idx = 0; queue_idx < group->queue_count; queue_idx++) { in group_sync_upd_work()
2905 struct panthor_queue *queue = group->queues[queue_idx]; in group_sync_upd_work()
2911 syncobj = group->syncobjs->kmap + (queue_idx * sizeof(*syncobj)); in group_sync_upd_work()
2932 group_put(group); in group_sync_upd_work()
2990 struct panthor_group *group = job->group; in get_job_cs_params() local
2991 struct panthor_queue *queue = group->queues[job->queue_idx]; in get_job_cs_params()
2992 struct panthor_device *ptdev = group->ptdev; in get_job_cs_params()
3001 params->sync_addr = panthor_kernel_bo_gpuva(group->syncobjs) + in get_job_cs_params()
3119 struct panthor_group *group = job->group; in queue_run_job() local
3120 struct panthor_queue *queue = group->queues[job->queue_idx]; in queue_run_job()
3121 struct panthor_device *ptdev = group->ptdev; in queue_run_job()
3142 if (!group_can_run(group)) { in queue_run_job()
3178 if (group->csg_id < 0) { in queue_run_job()
3180 * can detect unbounded waits and kill the group when that happens. in queue_run_job()
3184 if (!(group->blocked_queues & BIT(job->queue_idx)) && in queue_run_job()
3190 group_schedule_locked(group, BIT(job->queue_idx)); in queue_run_job()
3194 !(group->blocked_queues & BIT(job->queue_idx))) { in queue_run_job()
3219 struct panthor_group *group = job->group; in queue_timedout_job() local
3220 struct panthor_device *ptdev = group->ptdev; in queue_timedout_job()
3222 struct panthor_queue *queue = group->queues[job->queue_idx]; in queue_timedout_job()
3231 group->timedout = true; in queue_timedout_job()
3232 if (group->csg_id >= 0) { in queue_timedout_job()
3236 * pick the group on the next tick. in queue_timedout_job()
3238 list_del_init(&group->run_node); in queue_timedout_job()
3239 list_del_init(&group->wait_node); in queue_timedout_job()
3241 group_queue_work(group, term); in queue_timedout_job()
3290 group_create_queue(struct panthor_group *group, in group_create_queue() argument
3317 queue->ringbuf = panthor_kernel_bo_create(group->ptdev, group->vm, in group_create_queue()
3332 queue->iface.mem = panthor_fw_alloc_queue_iface_mem(group->ptdev, in group_create_queue()
3343 calc_profiling_ringbuf_num_slots(group->ptdev, args->ringbuf_size); in group_create_queue()
3346 panthor_kernel_bo_create(group->ptdev, group->vm, in group_create_queue()
3369 group->ptdev->scheduler->wq, 1, in group_create_queue()
3372 group->ptdev->reset.wq, in group_create_queue()
3373 NULL, "panthor-queue", group->ptdev->base.dev); in group_create_queue()
3383 group_free_queue(group, queue); in group_create_queue()
3388 struct panthor_group *group) in add_group_kbo_sizes() argument
3393 if (drm_WARN_ON(&ptdev->base, IS_ERR_OR_NULL(group))) in add_group_kbo_sizes()
3395 if (drm_WARN_ON(&ptdev->base, ptdev != group->ptdev)) in add_group_kbo_sizes()
3398 group->fdinfo.kbo_sizes += group->suspend_buf->obj->size; in add_group_kbo_sizes()
3399 group->fdinfo.kbo_sizes += group->protm_suspend_buf->obj->size; in add_group_kbo_sizes()
3400 group->fdinfo.kbo_sizes += group->syncobjs->obj->size; in add_group_kbo_sizes()
3402 for (i = 0; i < group->queue_count; i++) { in add_group_kbo_sizes()
3403 queue = group->queues[i]; in add_group_kbo_sizes()
3404 group->fdinfo.kbo_sizes += queue->ringbuf->obj->size; in add_group_kbo_sizes()
3405 group->fdinfo.kbo_sizes += queue->iface.mem->obj->size; in add_group_kbo_sizes()
3406 group->fdinfo.kbo_sizes += queue->profiling.slots->obj->size; in add_group_kbo_sizes()
3420 struct panthor_group *group = NULL; in panthor_group_create() local
3440 group = kzalloc(sizeof(*group), GFP_KERNEL); in panthor_group_create()
3441 if (!group) in panthor_group_create()
3444 spin_lock_init(&group->fatal_lock); in panthor_group_create()
3445 kref_init(&group->refcount); in panthor_group_create()
3446 group->state = PANTHOR_CS_GROUP_CREATED; in panthor_group_create()
3447 group->csg_id = -1; in panthor_group_create()
3449 group->ptdev = ptdev; in panthor_group_create()
3450 group->max_compute_cores = group_args->max_compute_cores; in panthor_group_create()
3451 group->compute_core_mask = group_args->compute_core_mask; in panthor_group_create()
3452 group->max_fragment_cores = group_args->max_fragment_cores; in panthor_group_create()
3453 group->fragment_core_mask = group_args->fragment_core_mask; in panthor_group_create()
3454 group->max_tiler_cores = group_args->max_tiler_cores; in panthor_group_create()
3455 group->tiler_core_mask = group_args->tiler_core_mask; in panthor_group_create()
3456 group->priority = group_args->priority; in panthor_group_create()
3458 INIT_LIST_HEAD(&group->wait_node); in panthor_group_create()
3459 INIT_LIST_HEAD(&group->run_node); in panthor_group_create()
3460 INIT_WORK(&group->term_work, group_term_work); in panthor_group_create()
3461 INIT_WORK(&group->sync_upd_work, group_sync_upd_work); in panthor_group_create()
3462 INIT_WORK(&group->tiler_oom_work, group_tiler_oom_work); in panthor_group_create()
3463 INIT_WORK(&group->release_work, group_release_work); in panthor_group_create()
3465 group->vm = panthor_vm_pool_get_vm(pfile->vms, group_args->vm_id); in panthor_group_create()
3466 if (!group->vm) { in panthor_group_create()
3472 group->suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); in panthor_group_create()
3473 if (IS_ERR(group->suspend_buf)) { in panthor_group_create()
3474 ret = PTR_ERR(group->suspend_buf); in panthor_group_create()
3475 group->suspend_buf = NULL; in panthor_group_create()
3480 group->protm_suspend_buf = panthor_fw_alloc_suspend_buf_mem(ptdev, suspend_size); in panthor_group_create()
3481 if (IS_ERR(group->protm_suspend_buf)) { in panthor_group_create()
3482 ret = PTR_ERR(group->protm_suspend_buf); in panthor_group_create()
3483 group->protm_suspend_buf = NULL; in panthor_group_create()
3487 group->syncobjs = panthor_kernel_bo_create(ptdev, group->vm, in panthor_group_create()
3494 if (IS_ERR(group->syncobjs)) { in panthor_group_create()
3495 ret = PTR_ERR(group->syncobjs); in panthor_group_create()
3499 ret = panthor_kernel_bo_vmap(group->syncobjs); in panthor_group_create()
3503 memset(group->syncobjs->kmap, 0, in panthor_group_create()
3507 group->queues[i] = group_create_queue(group, &queue_args[i]); in panthor_group_create()
3508 if (IS_ERR(group->queues[i])) { in panthor_group_create()
3509 ret = PTR_ERR(group->queues[i]); in panthor_group_create()
3510 group->queues[i] = NULL; in panthor_group_create()
3514 group->queue_count++; in panthor_group_create()
3517 group->idle_queues = GENMASK(group->queue_count - 1, 0); in panthor_group_create()
3519 ret = xa_alloc(&gpool->xa, &gid, group, XA_LIMIT(1, MAX_GROUPS_PER_POOL), GFP_KERNEL); in panthor_group_create()
3525 panthor_group_stop(group); in panthor_group_create()
3528 list_add_tail(&group->run_node, in panthor_group_create()
3529 &sched->groups.idle[group->priority]); in panthor_group_create()
3534 add_group_kbo_sizes(group->ptdev, group); in panthor_group_create()
3535 spin_lock_init(&group->fdinfo.lock); in panthor_group_create()
3540 group_put(group); in panthor_group_create()
3549 struct panthor_group *group; in panthor_group_destroy() local
3551 group = xa_erase(&gpool->xa, group_handle); in panthor_group_destroy()
3552 if (!group) in panthor_group_destroy()
3555 for (u32 i = 0; i < group->queue_count; i++) { in panthor_group_destroy()
3556 if (group->queues[i]) in panthor_group_destroy()
3557 drm_sched_entity_destroy(&group->queues[i]->entity); in panthor_group_destroy()
3562 group->destroyed = true; in panthor_group_destroy()
3563 if (group->csg_id >= 0) { in panthor_group_destroy()
3567 * pick the group on the next tick. in panthor_group_destroy()
3569 list_del_init(&group->run_node); in panthor_group_destroy()
3570 list_del_init(&group->wait_node); in panthor_group_destroy()
3571 group_queue_work(group, term); in panthor_group_destroy()
3576 group_put(group); in panthor_group_destroy()
3583 struct panthor_group *group; in group_from_handle() local
3586 group = group_get(xa_load(&pool->xa, group_handle)); in group_from_handle()
3589 return group; in group_from_handle()
3598 struct panthor_group *group; in panthor_group_get_state() local
3603 group = group_from_handle(gpool, get_state->group_handle); in panthor_group_get_state()
3604 if (!group) in panthor_group_get_state()
3610 if (group->timedout) in panthor_group_get_state()
3612 if (group->fatal_queues) { in panthor_group_get_state()
3614 get_state->fatal_queues = group->fatal_queues; in panthor_group_get_state()
3616 if (group->innocent) in panthor_group_get_state()
3620 group_put(group); in panthor_group_get_state()
3640 struct panthor_group *group; in panthor_group_pool_destroy() local
3646 xa_for_each(&gpool->xa, i, group) in panthor_group_pool_destroy()
3666 struct panthor_group *group; in panthor_fdinfo_gather_group_mem_info() local
3673 xa_for_each(&gpool->xa, i, group) { in panthor_fdinfo_gather_group_mem_info()
3674 stats->resident += group->fdinfo.kbo_sizes; in panthor_fdinfo_gather_group_mem_info()
3675 if (group->csg_id >= 0) in panthor_fdinfo_gather_group_mem_info()
3676 stats->active += group->fdinfo.kbo_sizes; in panthor_fdinfo_gather_group_mem_info()
3685 drm_WARN_ON(&job->group->ptdev->base, !list_empty(&job->node)); in job_release()
3695 group_put(job->group); in job_release()
3723 return job->group->vm; in panthor_job_vm()
3764 job->group = group_from_handle(gpool, group_handle); in panthor_job_create()
3765 if (!job->group) { in panthor_job_create()
3770 if (!group_can_run(job->group)) { in panthor_job_create()
3775 if (job->queue_idx >= job->group->queue_count || in panthor_job_create()
3776 !job->group->queues[job->queue_idx]) { in panthor_job_create()
3800 &job->group->queues[job->queue_idx]->entity, in panthor_job_create()
3801 credits, job->group); in panthor_job_create()
3816 panthor_vm_update_resvs(job->group->vm, exec, &sched_job->s_fence->finished, in panthor_job_update_resvs()
3878 * of the resources being allocated to one group and the other part to in panthor_sched_init()
3879 * the other group, both groups waiting for the remaining resources to in panthor_sched_init()