Lines Matching +full:current +full:- +full:mode
1 // SPDX-License-Identifier: GPL-2.0-only
64 * been used to perform kernel mode NEON in the meantime.
67 * the id of the current CPU every time the state is loaded onto a CPU. For (b),
68 * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the
70 * the most recently, or NULL if kernel mode NEON has been performed after that.
79 * indicate whether or not the userland FPSIMD state of the current task is
81 * CPU currently contain the most recent userland FPSIMD state of the current
84 * loaded on the CPU, allowing the state to be saved if a FPSIMD-aware
97 * - the task gets scheduled in; if both the task's fpsimd_cpu field
98 * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu
102 * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's
104 * fpsimd_cpu field is set to the id of the current CPU, the current
108 * - the task executes an ordinary syscall; upon return to userland, the
112 * - the task executes a syscall which executes some NEON instructions; this is
114 * register contents to memory, clears the fpsimd_last_state per-cpu variable
117 * - the task gets preempted after kernel_neon_end() is called; as we have not
251 return task->thread.vl[type]; in task_get_vl()
257 task->thread.vl[type] = vl; in task_set_vl()
263 return task->thread.vl_onexec[type]; in task_get_vl_onexec()
269 task->thread.vl_onexec[type] = vl; in task_set_vl_onexec()
276 * and the SVE registers for both streaming and non-streaming modes.
312 * * FPSIMD only - FP_STATE_FPSIMD:
314 * When the FPSIMD only state stored task->thread.fp_type is set to
315 * FP_STATE_FPSIMD, the FPSIMD registers V0-V31 are encoded in
316 * task->thread.uw.fpsimd_state; bits [max : 128] for each of Z0-Z31 are
317 * logically zero but not stored anywhere; P0-P15 and FFR are not
322 * task->thread.sve_state does not need to be non-NULL, valid or any
326 * * SVE state - FP_STATE_SVE:
328 * When the full SVE state is stored task->thread.fp_type is set to
329 * FP_STATE_SVE and Z0-Z31 (incorporating Vn in bits[127:0] or the
330 * corresponding Zn), P0-P15 and FFR are encoded in in
331 * task->thread.sve_state, formatted appropriately for vector
332 * length task->thread.sve_vl or, if SVCR.SM is set,
333 * task->thread.sme_vl. The storage for the vector registers in
334 * task->thread.uw.fpsimd_state should be ignored.
336 * task->thread.sve_state must point to a valid buffer at least
338 * task->thread.uw.fpsimd_state.vregs should be considered stale
341 * * FPSR and FPCR are always stored in task->thread.uw.fpsimd_state
347 * Update current's FPSIMD/SVE registers from thread_struct.
363 write_sysreg_s(current->thread.uw.fpmr, SYS_FPMR); in task_fpsimd_load()
366 switch (current->thread.fp_type) { in task_fpsimd_load()
373 if (!thread_sm_enabled(¤t->thread) && in task_fpsimd_load()
378 sve_set_vq(sve_vq_from_vl(task_get_sve_vl(current)) - 1); in task_fpsimd_load()
400 unsigned long sme_vl = task_get_sme_vl(current); in task_fpsimd_load()
404 sme_set_vq(sve_vq_from_vl(sme_vl) - 1); in task_fpsimd_load()
406 write_sysreg_s(current->thread.svcr, SYS_SVCR); in task_fpsimd_load()
408 if (thread_za_enabled(¤t->thread)) in task_fpsimd_load()
409 sme_load_state(current->thread.sme_state, in task_fpsimd_load()
412 if (thread_sm_enabled(¤t->thread)) in task_fpsimd_load()
417 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_SVE); in task_fpsimd_load()
418 sve_load_state(sve_pffr(¤t->thread), in task_fpsimd_load()
419 ¤t->thread.uw.fpsimd_state.fpsr, in task_fpsimd_load()
422 WARN_ON_ONCE(current->thread.fp_type != FP_STATE_FPSIMD); in task_fpsimd_load()
423 fpsimd_load_state(¤t->thread.uw.fpsimd_state); in task_fpsimd_load()
430 * current context is the context last bound to the CPU stored in
432 * than the host thread for the VM pointed to by current. This means
434 * than via current, if we are saving KVM state then it will have
435 * ensured that the type of registers to save is set in last->to_save.
453 *(last->fpmr) = read_sysreg_s(SYS_FPMR); in fpsimd_save_user_state()
460 if ((last->to_save == FP_STATE_CURRENT && test_thread_flag(TIF_SVE) && in fpsimd_save_user_state()
462 last->to_save == FP_STATE_SVE) { in fpsimd_save_user_state()
465 vl = last->sve_vl; in fpsimd_save_user_state()
469 u64 *svcr = last->svcr; in fpsimd_save_user_state()
474 sme_save_state(last->sme_state, in fpsimd_save_user_state()
477 /* If we are in streaming mode override regular SVE. */ in fpsimd_save_user_state()
481 vl = last->sme_vl; in fpsimd_save_user_state()
489 * Can't save the user regs, so current would in fpsimd_save_user_state()
490 * re-enter user with corrupt state. in fpsimd_save_user_state()
497 sve_save_state((char *)last->sve_state + in fpsimd_save_user_state()
499 &last->st->fpsr, save_ffr); in fpsimd_save_user_state()
500 *last->fp_type = FP_STATE_SVE; in fpsimd_save_user_state()
502 fpsimd_save_state(last->st); in fpsimd_save_user_state()
503 *last->fp_type = FP_STATE_FPSIMD; in fpsimd_save_user_state()
509 * We're on a slow path, so some sanity-checks are included.
518 int max_vl = info->max_vl; in find_supported_vector_length()
521 vl = info->min_vl; in find_supported_vector_length()
524 max_vl = info->min_vl; in find_supported_vector_length()
528 if (vl < info->min_vl) in find_supported_vector_length()
529 vl = info->min_vl; in find_supported_vector_length()
531 bit = find_next_bit(info->vq_map, SVE_VQ_MAX, in find_supported_vector_length()
541 struct vl_info *info = table->extra1; in vec_proc_do_default_vl()
542 enum vec_type type = info->type; in vec_proc_do_default_vl()
554 /* Writing -1 has the special meaning "set to max": */ in vec_proc_do_default_vl()
555 if (vl == -1) in vec_proc_do_default_vl()
556 vl = info->max_vl; in vec_proc_do_default_vl()
559 return -EINVAL; in vec_proc_do_default_vl()
568 .mode = 0644,
578 return -EINVAL; in sve_sysctl_init()
591 .mode = 0644,
601 return -EINVAL; in sme_sysctl_init()
611 (SVE_SIG_ZREG_OFFSET(vq, n) - SVE_SIG_REGS_OFFSET))
638 *p = arm64_cpu_to_le128(fst->vregs[i]); in __fpsimd_to_sve()
643 * Transfer the FPSIMD state in task->thread.uw.fpsimd_state to
644 * task->thread.sve_state.
646 * Task can be a non-runnable task, or current. In the latter case,
649 * task->thread.sve_state must point to at least sve_state_size(task)
651 * task->thread.uw.fpsimd_state must be up to date before calling this
657 void *sst = task->thread.sve_state; in fpsimd_to_sve()
658 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; in fpsimd_to_sve()
663 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); in fpsimd_to_sve()
668 * Transfer the SVE state in task->thread.sve_state to
669 * task->thread.uw.fpsimd_state.
671 * Task can be a non-runnable task, or current. In the latter case,
674 * task->thread.sve_state must point to at least sve_state_size(task)
676 * task->thread.sve_state must be up to date before calling this function.
681 void const *sst = task->thread.sve_state; in sve_to_fpsimd()
682 struct user_fpsimd_state *fst = &task->thread.uw.fpsimd_state; in sve_to_fpsimd()
689 vl = thread_get_cur_vl(&task->thread); in sve_to_fpsimd()
693 fst->vregs[i] = arm64_le128_to_cpu(*p); in sve_to_fpsimd()
710 kfree(task->thread.sve_state); in __sve_free()
711 task->thread.sve_state = NULL; in __sve_free()
738 * Ensure that task->thread.sve_state is allocated and sufficiently large.
741 * task->thread.sve_state with new data. The memory is always zeroed
749 if (task->thread.sve_state) { in sve_alloc()
751 memset(task->thread.sve_state, 0, in sve_alloc()
757 task->thread.sve_state = in sve_alloc()
764 * even if the SVE state is the current active state.
766 * This should only be called by ptrace. task must be non-runnable.
767 * task->thread.sve_state must point to at least sve_state_size(task)
776 * Ensure that task->thread.sve_state is up to date with respect to
779 * This should only be called by ptrace. task must be non-runnable.
780 * task->thread.sve_state must point to at least sve_state_size(task)
786 !thread_sm_enabled(&task->thread)) in fpsimd_sync_to_sve()
791 * Ensure that task->thread.uw.fpsimd_state is up to date with respect to
794 * This should only be called by ptrace. task must be non-runnable.
795 * task->thread.sve_state must point to at least sve_state_size(task)
800 if (task->thread.fp_type == FP_STATE_SVE) in sve_sync_to_fpsimd()
805 * Ensure that task->thread.sve_state is up to date with respect to
806 * the task->thread.uw.fpsimd_state.
810 * task must be non-runnable.
811 * task->thread.sve_state must point to at least sve_state_size(task)
813 * task->thread.uw.fpsimd_state must already have been initialised with
819 void *sst = task->thread.sve_state; in sve_sync_from_fpsimd_zeropad()
820 struct user_fpsimd_state const *fst = &task->thread.uw.fpsimd_state; in sve_sync_from_fpsimd_zeropad()
823 !thread_sm_enabled(&task->thread)) in sve_sync_from_fpsimd_zeropad()
826 vq = sve_vq_from_vl(thread_get_cur_vl(&task->thread)); in sve_sync_from_fpsimd_zeropad()
839 return -EINVAL; in vec_set_vector_length()
842 return -EINVAL; in vec_set_vector_length()
845 * Clamp to the maximum vector length that VL-agnostic code in vec_set_vector_length()
874 if (task == current) { in vec_set_vector_length()
882 thread_sm_enabled(&task->thread)) { in vec_set_vector_length()
884 task->thread.fp_type = FP_STATE_FPSIMD; in vec_set_vector_length()
889 !(task->thread.svcr & (SVCR_SM_MASK | SVCR_ZA_MASK))) { in vec_set_vector_length()
895 task->thread.svcr &= ~(SVCR_SM_MASK | in vec_set_vector_length()
902 if (task == current) in vec_set_vector_length()
911 * mode. in vec_set_vector_length()
927 * Encode the current vector length and flags for return.
938 ret = task_get_vl_onexec(current, type); in vec_prctl_status()
940 ret = task_get_vl(current, type); in vec_prctl_status()
958 return -EINVAL; in sve_set_current_vl()
960 ret = vec_set_vector_length(current, ARM64_VEC_SVE, vl, flags); in sve_set_current_vl()
971 return -EINVAL; in sve_get_current_vl()
987 return -EINVAL; in sme_set_current_vl()
989 ret = vec_set_vector_length(current, ARM64_VEC_SME, vl, flags); in sme_set_current_vl()
1000 return -EINVAL; in sme_get_current_vl()
1013 for (vq = SVE_VQ_MAX; vq >= SVE_VQ_MIN; --vq) { in vec_probe_vqs()
1014 write_vl(info->type, vq - 1); /* self-syncing */ in vec_probe_vqs()
1016 switch (info->type) { in vec_probe_vqs()
1044 vec_probe_vqs(info, info->vq_map); in vec_init_vq_map()
1045 bitmap_copy(info->vq_partial_map, info->vq_map, SVE_VQ_MAX); in vec_init_vq_map()
1050 * those not supported by the current CPU.
1051 * This function is called during the bring-up of early secondary CPUs only.
1059 bitmap_and(info->vq_map, info->vq_map, tmp_map, SVE_VQ_MAX); in vec_update_vq_map()
1060 bitmap_or(info->vq_partial_map, info->vq_partial_map, tmp_map, in vec_update_vq_map()
1065 * Check whether the current CPU supports all VQs in the committed set.
1066 * This function is called during the bring-up of late secondary CPUs only.
1077 if (bitmap_intersects(tmp_map, info->vq_map, SVE_VQ_MAX)) { in vec_verify_vq_map()
1079 info->name, smp_processor_id()); in vec_verify_vq_map()
1080 return -EINVAL; in vec_verify_vq_map()
1095 bitmap_andnot(tmp_map, tmp_map, info->vq_map, SVE_VQ_MAX); in vec_verify_vq_map()
1106 if (sve_vl_from_vq(__bit_to_vq(b)) <= info->max_virtualisable_vl) { in vec_verify_vq_map()
1108 info->name, smp_processor_id()); in vec_verify_vq_map()
1109 return -EINVAL; in vec_verify_vq_map()
1164 * The SVE architecture mandates support for 128-bit vectors, in sve_setup()
1168 if (WARN_ON(!test_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map))) in sve_setup()
1169 set_bit(__vq_to_bit(SVE_VQ_MIN), info->vq_map); in sve_setup()
1171 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX); in sve_setup()
1172 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit)); in sve_setup()
1180 bitmap_andnot(tmp_map, info->vq_partial_map, info->vq_map, in sve_setup()
1185 /* No non-virtualisable VLs found */ in sve_setup()
1186 info->max_virtualisable_vl = SVE_VQ_MAX; in sve_setup()
1187 else if (WARN_ON(b == SVE_VQ_MAX - 1)) in sve_setup()
1189 info->max_virtualisable_vl = SVE_VQ_MIN; in sve_setup()
1191 info->max_virtualisable_vl = sve_vl_from_vq(__bit_to_vq(b + 1)); in sve_setup()
1193 if (info->max_virtualisable_vl > info->max_vl) in sve_setup()
1194 info->max_virtualisable_vl = info->max_vl; in sve_setup()
1197 info->name, info->max_vl); in sve_setup()
1199 info->name, get_sve_default_vl()); in sve_setup()
1204 info->name); in sve_setup()
1224 * Ensure that task->thread.sme_state is allocated and sufficiently large.
1227 * task->thread.sme_state with new data. The memory is always zeroed
1234 if (task->thread.sme_state) { in sme_alloc()
1236 memset(task->thread.sme_state, 0, in sme_alloc()
1242 task->thread.sme_state = in sme_alloc()
1248 kfree(task->thread.sme_state); in sme_free()
1249 task->thread.sme_state = NULL; in sme_free()
1305 WARN_ON(bitmap_empty(info->vq_map, SVE_VQ_MAX)); in sme_setup()
1307 min_bit = find_last_bit(info->vq_map, SVE_VQ_MAX); in sme_setup()
1308 info->min_vl = sve_vl_from_vq(__bit_to_vq(min_bit)); in sme_setup()
1310 max_bit = find_first_bit(info->vq_map, SVE_VQ_MAX); in sme_setup()
1311 info->max_vl = sve_vl_from_vq(__bit_to_vq(max_bit)); in sme_setup()
1313 WARN_ON(info->min_vl > info->max_vl); in sme_setup()
1318 * grow the signal frame when in streaming mode, otherwise the in sme_setup()
1324 info->min_vl); in sme_setup()
1326 info->max_vl); in sme_setup()
1353 * is not shared with FPSIMD. If (as is likely) the current in sve_init_regs()
1355 * update our metadata for the current task including in sve_init_regs()
1356 * disabling the trap, otherwise update our in-memory copy. in sve_init_regs()
1357 * We are guaranteed to not be in streaming mode, we can only in sve_init_regs()
1358 * take a SVE trap when not in streaming mode and we can't be in sve_init_regs()
1359 * in streaming mode when taking a SME trap. in sve_init_regs()
1363 sve_vq_from_vl(task_get_sve_vl(current)) - 1; in sve_init_regs()
1368 fpsimd_to_sve(current); in sve_init_regs()
1369 current->thread.fp_type = FP_STATE_SVE; in sve_init_regs()
1370 fpsimd_flush_task_state(current); in sve_init_regs()
1377 * Storage is allocated for the full SVE state, the current FPSIMD
1389 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_sve_acc()
1393 sve_alloc(current, true); in do_sve_acc()
1394 if (!current->thread.sve_state) { in do_sve_acc()
1405 * Even if the task can have used streaming mode we can only in do_sve_acc()
1406 * generate SVE access traps in normal SVE mode and in do_sve_acc()
1407 * transitioning out of streaming mode may discard any in do_sve_acc()
1408 * streaming mode state. Always clear the high bits to avoid in do_sve_acc()
1419 * Storage is allocated for the full SVE and SME state, the current
1431 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_sme_acc()
1437 * is being used in the wrong mode, report as SIGILL. in do_sme_acc()
1440 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_sme_acc()
1444 sve_alloc(current, false); in do_sme_acc()
1445 sme_alloc(current, true); in do_sme_acc()
1446 if (!current->thread.sve_state || !current->thread.sme_state) { in do_sme_acc()
1459 sve_vq_from_vl(task_get_sme_vl(current)) - 1; in do_sme_acc()
1475 force_signal_inject(SIGILL, ILL_ILLOPC, regs->pc, 0); in do_fpsimd_acc()
1487 * Raise a SIGFPE for the current process.
1508 current); in do_fpsimd_exc()
1516 * Elide the load if this CPU holds the most recent kernel mode in fpsimd_load_kernel_state()
1517 * FPSIMD context of the current task. in fpsimd_load_kernel_state()
1519 if (last->st == &task->thread.kernel_fpsimd_state && in fpsimd_load_kernel_state()
1520 task->thread.kernel_fpsimd_cpu == smp_processor_id()) in fpsimd_load_kernel_state()
1523 fpsimd_load_state(&task->thread.kernel_fpsimd_state); in fpsimd_load_kernel_state()
1529 .st = &task->thread.kernel_fpsimd_state, in fpsimd_save_kernel_state()
1533 fpsimd_save_state(&task->thread.kernel_fpsimd_state); in fpsimd_save_kernel_state()
1536 task->thread.kernel_fpsimd_cpu = smp_processor_id(); in fpsimd_save_kernel_state()
1550 * Leaving streaming mode enabled will cause issues for any kernel in fpsimd_flush_cpu_state()
1551 * NEON and leaving streaming mode or ZA enabled may increase power in fpsimd_flush_cpu_state()
1571 fpsimd_save_kernel_state(current); in fpsimd_thread_switch()
1582 * loaded with user mode FPSIMD state and so wrong_task and in fpsimd_thread_switch()
1586 &next->thread.uw.fpsimd_state; in fpsimd_thread_switch()
1587 wrong_cpu = next->thread.fpsimd_cpu != smp_processor_id(); in fpsimd_thread_switch()
1608 vl = task_get_vl_onexec(current, type); in fpsimd_flush_thread_vl()
1619 task_set_vl(current, type, vl); in fpsimd_flush_thread_vl()
1626 task_set_vl_onexec(current, type, 0); in fpsimd_flush_thread_vl()
1639 fpsimd_flush_task_state(current); in fpsimd_flush_thread()
1640 memset(¤t->thread.uw.fpsimd_state, 0, in fpsimd_flush_thread()
1641 sizeof(current->thread.uw.fpsimd_state)); in fpsimd_flush_thread()
1647 sve_state = current->thread.sve_state; in fpsimd_flush_thread()
1648 current->thread.sve_state = NULL; in fpsimd_flush_thread()
1657 sme_state = current->thread.sme_state; in fpsimd_flush_thread()
1658 current->thread.sme_state = NULL; in fpsimd_flush_thread()
1661 current->thread.svcr = 0; in fpsimd_flush_thread()
1664 current->thread.fp_type = FP_STATE_FPSIMD; in fpsimd_flush_thread()
1672 * Save the userland FPSIMD state of 'current' to memory, but only if the state
1673 * currently held in the registers does in fact belong to 'current'
1687 * current->thread.uw.fpsimd_state is updated so that it can be copied to
1693 if (current->thread.fp_type == FP_STATE_SVE) in fpsimd_signal_preserve_current_state()
1694 sve_to_fpsimd(current); in fpsimd_signal_preserve_current_state()
1698 * Associate current's FPSIMD context with this cpu
1707 last->st = ¤t->thread.uw.fpsimd_state; in fpsimd_bind_task_to_cpu()
1708 last->sve_state = current->thread.sve_state; in fpsimd_bind_task_to_cpu()
1709 last->sme_state = current->thread.sme_state; in fpsimd_bind_task_to_cpu()
1710 last->sve_vl = task_get_sve_vl(current); in fpsimd_bind_task_to_cpu()
1711 last->sme_vl = task_get_sme_vl(current); in fpsimd_bind_task_to_cpu()
1712 last->svcr = ¤t->thread.svcr; in fpsimd_bind_task_to_cpu()
1713 last->fpmr = ¤t->thread.uw.fpmr; in fpsimd_bind_task_to_cpu()
1714 last->fp_type = ¤t->thread.fp_type; in fpsimd_bind_task_to_cpu()
1715 last->to_save = FP_STATE_CURRENT; in fpsimd_bind_task_to_cpu()
1716 current->thread.fpsimd_cpu = smp_processor_id(); in fpsimd_bind_task_to_cpu()
1748 * Load the userland FPSIMD state of 'current' from memory, but only if the
1750 * state of 'current'. This is called when we are preparing to return to
1762 * TIF_FOREIGN_FPSTATE via flush_thread() -> fpsimd_flush_thread(), in fpsimd_restore_current_state()
1790 * Load an updated userland FPSIMD state for 'current' from memory and set the
1792 * FPSIMD state of 'current'. This is used by the signal code to restore the
1803 current->thread.uw.fpsimd_state = *state; in fpsimd_update_current_state()
1805 fpsimd_to_sve(current); in fpsimd_update_current_state()
1821 * that TIF_FOREIGN_FPSTATE cannot be spuriously re-cleared.
1828 t->thread.fpsimd_cpu = NR_CPUS; in fpsimd_flush_task_state()
1862 * Kernel-side NEON support functions
1890 fpsimd_save_kernel_state(current); in kernel_neon_begin()
1895 * Set the thread flag so that the kernel mode FPSIMD state in kernel_neon_begin()
1899 * On non-PREEMPT_RT, softirqs may interrupt task level kernel in kernel_neon_begin()
1900 * mode FPSIMD, but the task will not be preemptible so setting in kernel_neon_begin()
1908 * mode in task context. So in this case, setting the flag here in kernel_neon_begin()
1923 * kernel_neon_end(): give the CPU FPSIMD registers back to the current task
1937 * If we are returning from a nested use of kernel mode FPSIMD, restore in kernel_neon_end()
1938 * the task context kernel mode FPSIMD state. This can only happen when in kernel_neon_end()
1939 * running in softirq context on non-PREEMPT_RT. in kernel_neon_end()
1943 fpsimd_load_kernel_state(current); in kernel_neon_end()
1966 * Do not use them for any other purpose -- if tempted to do so, you are
2002 * exist in streaming mode. in __efi_fpsimd_begin()
2009 &this_cpu_ptr(&efi_fpsimd_state)->fpsr, in __efi_fpsimd_begin()
2041 * Restore streaming mode; EFI calls are in __efi_fpsimd_end()
2043 * streaming mode. in __efi_fpsimd_end()
2053 * exist in streaming mode. in __efi_fpsimd_end()
2061 &this_cpu_ptr(&efi_fpsimd_state)->fpsr, in __efi_fpsimd_end()
2138 pr_notice("Floating-point is not implemented\n"); in fpsimd_init()