Lines Matching +full:synquacer +full:- +full:pre +full:- +full:its
1 // SPDX-License-Identifier: GPL-2.0-only
3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
37 #include <linux/irqchip/arm-gic-v3.h>
38 #include <linux/irqchip/arm-gic-v4.h>
43 #include "irq-gic-common.h"
44 #include "irq-msi-lib.h"
71 * Collection structure - just an ID, and a redistributor address to
81 * The ITS_BASER structure - contains memory information, cached
82 * value of BASER register configuration and ITS page size.
94 * The ITS structure - contains most of the infrastructure, with the
95 * top-level MSI domain, the command queue, the collections, and the
124 u32 pre_its_base; /* for Socionext Synquacer */
128 #define is_v4(its) (!!((its)->typer & GITS_TYPER_VLPIS)) argument
129 #define is_v4_1(its) (!!((its)->typer & GITS_TYPER_VMAPP)) argument
130 #define device_ids(its) (FIELD_GET(GITS_TYPER_DEVBITS, (its)->typer) + 1) argument
138 if (gic_rdists->has_rvpeid && \
139 gic_rdists->gicd_typer2 & GICD_TYPER2_VIL) \
140 nvpeid = 1 + (gic_rdists->gicd_typer2 & \
162 * The ITS view of a device - belongs to an ITS, owns an interrupt
163 * translation table, and a list of interrupts. If it some of its
169 struct its_node *its; member
203 #define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
204 #define gic_data_rdist_cpu(cpu) (per_cpu_ptr(gic_rdists->rdist, cpu))
205 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
295 static bool require_its_list_vmovp(struct its_vm *vm, struct its_node *its) in require_its_list_vmovp() argument
297 return (gic_rdists->has_rvpeid || vm->vlpi_count[its->list_nr]); in require_its_list_vmovp()
302 return !(gic_rdists->flags & RDIST_FLAGS_FORCE_NON_SHAREABLE); in rdists_support_shareable()
307 struct its_node *its; in get_its_list() local
310 list_for_each_entry(its, &its_nodes, entry) { in get_its_list()
311 if (!is_v4(its)) in get_its_list()
314 if (require_its_list_vmovp(vm, its)) in get_its_list()
315 __set_bit(its->list_nr, &its_list); in get_its_list()
324 return d->hwirq - its_dev->event_map.lpi_base; in its_get_event_id()
330 struct its_node *its = its_dev->its; in dev_event_to_col() local
332 return its->collections + its_dev->event_map.col_map[event]; in dev_event_to_col()
338 if (WARN_ON_ONCE(event >= its_dev->event_map.nr_lpis)) in dev_event_to_vlpi_map()
341 return &its_dev->event_map.vlpi_maps[event]; in dev_event_to_vlpi_map()
358 raw_spin_lock_irqsave(&vpe->vpe_lock, *flags); in vpe_to_cpuid_lock()
359 return vpe->col_idx; in vpe_to_cpuid_lock()
364 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); in vpe_to_cpuid_unlock()
374 if (d->chip == &its_vpe_irq_chip) { in irq_to_cpuid_lock()
379 vpe = map->vpe; in irq_to_cpuid_lock()
387 cpu = its_dev->event_map.col_map[its_get_event_id(d)]; in irq_to_cpuid_lock()
399 if (d->chip == &its_vpe_irq_chip) { in irq_to_cpuid_unlock()
404 vpe = map->vpe; in irq_to_cpuid_unlock()
413 if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(15, 0))) in valid_col()
419 static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe) in valid_vpe() argument
421 if (valid_col(its->collections + vpe->col_idx)) in valid_vpe()
428 * ITS command descriptors - parameters to be encoded in a command
527 * The ITS command block, which is what the ITS actually parses.
556 its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0); in its_encode_cmd()
561 its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32); in its_encode_devid()
566 its_mask_encode(&cmd->raw_cmd[1], id, 31, 0); in its_encode_event_id()
571 its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32); in its_encode_phys_id()
576 its_mask_encode(&cmd->raw_cmd[1], size, 4, 0); in its_encode_size()
581 its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8); in its_encode_itt()
586 its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63); in its_encode_valid()
591 its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16); in its_encode_target()
596 its_mask_encode(&cmd->raw_cmd[2], col, 15, 0); in its_encode_collection()
601 its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32); in its_encode_vpeid()
606 its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0); in its_encode_virt_id()
611 its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32); in its_encode_db_phys_id()
616 its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0); in its_encode_db_valid()
621 its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32); in its_encode_seq_num()
626 its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0); in its_encode_its_list()
631 its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16); in its_encode_vpt_addr()
636 its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0); in its_encode_vpt_size()
641 its_mask_encode(&cmd->raw_cmd[0], vconf_pa >> 16, 51, 16); in its_encode_vconf_addr()
646 its_mask_encode(&cmd->raw_cmd[0], alloc, 8, 8); in its_encode_alloc()
651 its_mask_encode(&cmd->raw_cmd[0], ptz, 9, 9); in its_encode_ptz()
657 its_mask_encode(&cmd->raw_cmd[1], vpe_db_lpi, 31, 0); in its_encode_vmapp_default_db()
663 its_mask_encode(&cmd->raw_cmd[3], vpe_db_lpi, 31, 0); in its_encode_vmovp_default_db()
668 its_mask_encode(&cmd->raw_cmd[2], db, 63, 63); in its_encode_db()
673 its_mask_encode(&cmd->raw_cmd[0], sgi, 35, 32); in its_encode_sgi_intid()
678 its_mask_encode(&cmd->raw_cmd[0], prio >> 4, 23, 20); in its_encode_sgi_priority()
683 its_mask_encode(&cmd->raw_cmd[0], grp, 10, 10); in its_encode_sgi_group()
688 its_mask_encode(&cmd->raw_cmd[0], clr, 9, 9); in its_encode_sgi_clear()
693 its_mask_encode(&cmd->raw_cmd[0], en, 8, 8); in its_encode_sgi_enable()
699 cmd->raw_cmd_le[0] = cpu_to_le64(cmd->raw_cmd[0]); in its_fixup_cmd()
700 cmd->raw_cmd_le[1] = cpu_to_le64(cmd->raw_cmd[1]); in its_fixup_cmd()
701 cmd->raw_cmd_le[2] = cpu_to_le64(cmd->raw_cmd[2]); in its_fixup_cmd()
702 cmd->raw_cmd_le[3] = cpu_to_le64(cmd->raw_cmd[3]); in its_fixup_cmd()
705 static struct its_collection *its_build_mapd_cmd(struct its_node *its, in its_build_mapd_cmd() argument
710 u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites); in its_build_mapd_cmd()
712 itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt); in its_build_mapd_cmd()
715 its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id); in its_build_mapd_cmd()
716 its_encode_size(cmd, size - 1); in its_build_mapd_cmd()
718 its_encode_valid(cmd, desc->its_mapd_cmd.valid); in its_build_mapd_cmd()
725 static struct its_collection *its_build_mapc_cmd(struct its_node *its, in its_build_mapc_cmd() argument
730 its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id); in its_build_mapc_cmd()
731 its_encode_target(cmd, desc->its_mapc_cmd.col->target_address); in its_build_mapc_cmd()
732 its_encode_valid(cmd, desc->its_mapc_cmd.valid); in its_build_mapc_cmd()
736 return desc->its_mapc_cmd.col; in its_build_mapc_cmd()
739 static struct its_collection *its_build_mapti_cmd(struct its_node *its, in its_build_mapti_cmd() argument
745 col = dev_event_to_col(desc->its_mapti_cmd.dev, in its_build_mapti_cmd()
746 desc->its_mapti_cmd.event_id); in its_build_mapti_cmd()
749 its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id); in its_build_mapti_cmd()
750 its_encode_event_id(cmd, desc->its_mapti_cmd.event_id); in its_build_mapti_cmd()
751 its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id); in its_build_mapti_cmd()
752 its_encode_collection(cmd, col->col_id); in its_build_mapti_cmd()
759 static struct its_collection *its_build_movi_cmd(struct its_node *its, in its_build_movi_cmd() argument
765 col = dev_event_to_col(desc->its_movi_cmd.dev, in its_build_movi_cmd()
766 desc->its_movi_cmd.event_id); in its_build_movi_cmd()
769 its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id); in its_build_movi_cmd()
770 its_encode_event_id(cmd, desc->its_movi_cmd.event_id); in its_build_movi_cmd()
771 its_encode_collection(cmd, desc->its_movi_cmd.col->col_id); in its_build_movi_cmd()
778 static struct its_collection *its_build_discard_cmd(struct its_node *its, in its_build_discard_cmd() argument
784 col = dev_event_to_col(desc->its_discard_cmd.dev, in its_build_discard_cmd()
785 desc->its_discard_cmd.event_id); in its_build_discard_cmd()
788 its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id); in its_build_discard_cmd()
789 its_encode_event_id(cmd, desc->its_discard_cmd.event_id); in its_build_discard_cmd()
796 static struct its_collection *its_build_inv_cmd(struct its_node *its, in its_build_inv_cmd() argument
802 col = dev_event_to_col(desc->its_inv_cmd.dev, in its_build_inv_cmd()
803 desc->its_inv_cmd.event_id); in its_build_inv_cmd()
806 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); in its_build_inv_cmd()
807 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); in its_build_inv_cmd()
814 static struct its_collection *its_build_int_cmd(struct its_node *its, in its_build_int_cmd() argument
820 col = dev_event_to_col(desc->its_int_cmd.dev, in its_build_int_cmd()
821 desc->its_int_cmd.event_id); in its_build_int_cmd()
824 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); in its_build_int_cmd()
825 its_encode_event_id(cmd, desc->its_int_cmd.event_id); in its_build_int_cmd()
832 static struct its_collection *its_build_clear_cmd(struct its_node *its, in its_build_clear_cmd() argument
838 col = dev_event_to_col(desc->its_clear_cmd.dev, in its_build_clear_cmd()
839 desc->its_clear_cmd.event_id); in its_build_clear_cmd()
842 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); in its_build_clear_cmd()
843 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); in its_build_clear_cmd()
850 static struct its_collection *its_build_invall_cmd(struct its_node *its, in its_build_invall_cmd() argument
855 its_encode_collection(cmd, desc->its_invall_cmd.col->col_id); in its_build_invall_cmd()
859 return desc->its_invall_cmd.col; in its_build_invall_cmd()
862 static struct its_vpe *its_build_vinvall_cmd(struct its_node *its, in its_build_vinvall_cmd() argument
867 its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id); in its_build_vinvall_cmd()
871 return valid_vpe(its, desc->its_vinvall_cmd.vpe); in its_build_vinvall_cmd()
874 static struct its_vpe *its_build_vmapp_cmd(struct its_node *its, in its_build_vmapp_cmd() argument
878 struct its_vpe *vpe = valid_vpe(its, desc->its_vmapp_cmd.vpe); in its_build_vmapp_cmd()
884 its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id); in its_build_vmapp_cmd()
885 its_encode_valid(cmd, desc->its_vmapp_cmd.valid); in its_build_vmapp_cmd()
887 if (!desc->its_vmapp_cmd.valid) { in its_build_vmapp_cmd()
888 alloc = !atomic_dec_return(&desc->its_vmapp_cmd.vpe->vmapp_count); in its_build_vmapp_cmd()
889 if (is_v4_1(its)) { in its_build_vmapp_cmd()
892 * Unmapping a VPE is self-synchronizing on GICv4.1, in its_build_vmapp_cmd()
901 vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page)); in its_build_vmapp_cmd()
902 target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset; in its_build_vmapp_cmd()
906 its_encode_vpt_size(cmd, LPI_NRBITS - 1); in its_build_vmapp_cmd()
908 alloc = !atomic_fetch_inc(&desc->its_vmapp_cmd.vpe->vmapp_count); in its_build_vmapp_cmd()
910 if (!is_v4_1(its)) in its_build_vmapp_cmd()
913 vconf_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->its_vm->vprop_page)); in its_build_vmapp_cmd()
925 its_encode_vmapp_default_db(cmd, desc->its_vmapp_cmd.vpe->vpe_db_lpi); in its_build_vmapp_cmd()
933 static struct its_vpe *its_build_vmapti_cmd(struct its_node *its, in its_build_vmapti_cmd() argument
939 if (!is_v4_1(its) && desc->its_vmapti_cmd.db_enabled) in its_build_vmapti_cmd()
940 db = desc->its_vmapti_cmd.vpe->vpe_db_lpi; in its_build_vmapti_cmd()
945 its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id); in its_build_vmapti_cmd()
946 its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id); in its_build_vmapti_cmd()
947 its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id); in its_build_vmapti_cmd()
949 its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id); in its_build_vmapti_cmd()
953 return valid_vpe(its, desc->its_vmapti_cmd.vpe); in its_build_vmapti_cmd()
956 static struct its_vpe *its_build_vmovi_cmd(struct its_node *its, in its_build_vmovi_cmd() argument
962 if (!is_v4_1(its) && desc->its_vmovi_cmd.db_enabled) in its_build_vmovi_cmd()
963 db = desc->its_vmovi_cmd.vpe->vpe_db_lpi; in its_build_vmovi_cmd()
968 its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id); in its_build_vmovi_cmd()
969 its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id); in its_build_vmovi_cmd()
970 its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id); in its_build_vmovi_cmd()
976 return valid_vpe(its, desc->its_vmovi_cmd.vpe); in its_build_vmovi_cmd()
979 static struct its_vpe *its_build_vmovp_cmd(struct its_node *its, in its_build_vmovp_cmd() argument
985 target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset; in its_build_vmovp_cmd()
987 its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num); in its_build_vmovp_cmd()
988 its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list); in its_build_vmovp_cmd()
989 its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id); in its_build_vmovp_cmd()
992 if (is_v4_1(its)) { in its_build_vmovp_cmd()
994 its_encode_vmovp_default_db(cmd, desc->its_vmovp_cmd.vpe->vpe_db_lpi); in its_build_vmovp_cmd()
999 return valid_vpe(its, desc->its_vmovp_cmd.vpe); in its_build_vmovp_cmd()
1002 static struct its_vpe *its_build_vinv_cmd(struct its_node *its, in its_build_vinv_cmd() argument
1008 map = dev_event_to_vlpi_map(desc->its_inv_cmd.dev, in its_build_vinv_cmd()
1009 desc->its_inv_cmd.event_id); in its_build_vinv_cmd()
1012 its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id); in its_build_vinv_cmd()
1013 its_encode_event_id(cmd, desc->its_inv_cmd.event_id); in its_build_vinv_cmd()
1017 return valid_vpe(its, map->vpe); in its_build_vinv_cmd()
1020 static struct its_vpe *its_build_vint_cmd(struct its_node *its, in its_build_vint_cmd() argument
1026 map = dev_event_to_vlpi_map(desc->its_int_cmd.dev, in its_build_vint_cmd()
1027 desc->its_int_cmd.event_id); in its_build_vint_cmd()
1030 its_encode_devid(cmd, desc->its_int_cmd.dev->device_id); in its_build_vint_cmd()
1031 its_encode_event_id(cmd, desc->its_int_cmd.event_id); in its_build_vint_cmd()
1035 return valid_vpe(its, map->vpe); in its_build_vint_cmd()
1038 static struct its_vpe *its_build_vclear_cmd(struct its_node *its, in its_build_vclear_cmd() argument
1044 map = dev_event_to_vlpi_map(desc->its_clear_cmd.dev, in its_build_vclear_cmd()
1045 desc->its_clear_cmd.event_id); in its_build_vclear_cmd()
1048 its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id); in its_build_vclear_cmd()
1049 its_encode_event_id(cmd, desc->its_clear_cmd.event_id); in its_build_vclear_cmd()
1053 return valid_vpe(its, map->vpe); in its_build_vclear_cmd()
1056 static struct its_vpe *its_build_invdb_cmd(struct its_node *its, in its_build_invdb_cmd() argument
1060 if (WARN_ON(!is_v4_1(its))) in its_build_invdb_cmd()
1064 its_encode_vpeid(cmd, desc->its_invdb_cmd.vpe->vpe_id); in its_build_invdb_cmd()
1068 return valid_vpe(its, desc->its_invdb_cmd.vpe); in its_build_invdb_cmd()
1071 static struct its_vpe *its_build_vsgi_cmd(struct its_node *its, in its_build_vsgi_cmd() argument
1075 if (WARN_ON(!is_v4_1(its))) in its_build_vsgi_cmd()
1079 its_encode_vpeid(cmd, desc->its_vsgi_cmd.vpe->vpe_id); in its_build_vsgi_cmd()
1080 its_encode_sgi_intid(cmd, desc->its_vsgi_cmd.sgi); in its_build_vsgi_cmd()
1081 its_encode_sgi_priority(cmd, desc->its_vsgi_cmd.priority); in its_build_vsgi_cmd()
1082 its_encode_sgi_group(cmd, desc->its_vsgi_cmd.group); in its_build_vsgi_cmd()
1083 its_encode_sgi_clear(cmd, desc->its_vsgi_cmd.clear); in its_build_vsgi_cmd()
1084 its_encode_sgi_enable(cmd, desc->its_vsgi_cmd.enable); in its_build_vsgi_cmd()
1088 return valid_vpe(its, desc->its_vsgi_cmd.vpe); in its_build_vsgi_cmd()
1091 static u64 its_cmd_ptr_to_offset(struct its_node *its, in its_cmd_ptr_to_offset() argument
1094 return (ptr - its->cmd_base) * sizeof(*ptr); in its_cmd_ptr_to_offset()
1097 static int its_queue_full(struct its_node *its) in its_queue_full() argument
1102 widx = its->cmd_write - its->cmd_base; in its_queue_full()
1103 ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block); in its_queue_full()
1105 /* This is incredibly unlikely to happen, unless the ITS locks up. */ in its_queue_full()
1112 static struct its_cmd_block *its_allocate_entry(struct its_node *its) in its_allocate_entry() argument
1117 while (its_queue_full(its)) { in its_allocate_entry()
1118 count--; in its_allocate_entry()
1120 pr_err_ratelimited("ITS queue not draining\n"); in its_allocate_entry()
1127 cmd = its->cmd_write++; in its_allocate_entry()
1130 if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES)) in its_allocate_entry()
1131 its->cmd_write = its->cmd_base; in its_allocate_entry()
1134 cmd->raw_cmd[0] = 0; in its_allocate_entry()
1135 cmd->raw_cmd[1] = 0; in its_allocate_entry()
1136 cmd->raw_cmd[2] = 0; in its_allocate_entry()
1137 cmd->raw_cmd[3] = 0; in its_allocate_entry()
1142 static struct its_cmd_block *its_post_commands(struct its_node *its) in its_post_commands() argument
1144 u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write); in its_post_commands()
1146 writel_relaxed(wr, its->base + GITS_CWRITER); in its_post_commands()
1148 return its->cmd_write; in its_post_commands()
1151 static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd) in its_flush_cmd() argument
1155 * the ITS. in its_flush_cmd()
1157 if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING) in its_flush_cmd()
1163 static int its_wait_for_range_completion(struct its_node *its, in its_wait_for_range_completion() argument
1171 to_idx = its_cmd_ptr_to_offset(its, to); in its_wait_for_range_completion()
1180 rd_idx = readl_relaxed(its->base + GITS_CREADR); in its_wait_for_range_completion()
1184 * potential wrap-around into account. in its_wait_for_range_completion()
1186 delta = rd_idx - prev_idx; in its_wait_for_range_completion()
1194 count--; in its_wait_for_range_completion()
1196 pr_err_ratelimited("ITS queue timeout (%llu %llu)\n", in its_wait_for_range_completion()
1198 return -1; in its_wait_for_range_completion()
1210 void name(struct its_node *its, \
1219 raw_spin_lock_irqsave(&its->lock, flags); \
1221 cmd = its_allocate_entry(its); \
1223 raw_spin_unlock_irqrestore(&its->lock, flags); \
1226 sync_obj = builder(its, cmd, desc); \
1227 its_flush_cmd(its, cmd); \
1230 sync_cmd = its_allocate_entry(its); \
1234 buildfn(its, sync_cmd, sync_obj); \
1235 its_flush_cmd(its, sync_cmd); \
1239 rd_idx = readl_relaxed(its->base + GITS_CREADR); \
1240 next_cmd = its_post_commands(its); \
1241 raw_spin_unlock_irqrestore(&its->lock, flags); \
1243 if (its_wait_for_range_completion(its, rd_idx, next_cmd)) \
1244 pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
1247 static void its_build_sync_cmd(struct its_node *its, in its_build_sync_cmd() argument
1252 its_encode_target(sync_cmd, sync_col->target_address); in its_build_sync_cmd()
1260 static void its_build_vsync_cmd(struct its_node *its, in BUILD_SINGLE_CMD_FUNC()
1265 its_encode_vpeid(sync_cmd, sync_vpe->vpe_id); in BUILD_SINGLE_CMD_FUNC()
1280 its_send_single_command(dev->its, its_build_int_cmd, &desc); in BUILD_SINGLE_CMD_FUNC()
1290 its_send_single_command(dev->its, its_build_clear_cmd, &desc); in its_send_clear()
1300 its_send_single_command(dev->its, its_build_inv_cmd, &desc); in its_send_inv()
1310 its_send_single_command(dev->its, its_build_mapd_cmd, &desc); in its_send_mapd()
1313 static void its_send_mapc(struct its_node *its, struct its_collection *col, in its_send_mapc() argument
1321 its_send_single_command(its, its_build_mapc_cmd, &desc); in its_send_mapc()
1332 its_send_single_command(dev->its, its_build_mapti_cmd, &desc); in its_send_mapti()
1344 its_send_single_command(dev->its, its_build_movi_cmd, &desc); in its_send_movi()
1354 its_send_single_command(dev->its, its_build_discard_cmd, &desc); in its_send_discard()
1357 static void its_send_invall(struct its_node *its, struct its_collection *col) in its_send_invall() argument
1363 its_send_single_command(its, its_build_invall_cmd, &desc); in its_send_invall()
1371 desc.its_vmapti_cmd.vpe = map->vpe; in its_send_vmapti()
1373 desc.its_vmapti_cmd.virt_id = map->vintid; in its_send_vmapti()
1375 desc.its_vmapti_cmd.db_enabled = map->db_enabled; in its_send_vmapti()
1377 its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc); in its_send_vmapti()
1385 desc.its_vmovi_cmd.vpe = map->vpe; in its_send_vmovi()
1388 desc.its_vmovi_cmd.db_enabled = map->db_enabled; in its_send_vmovi()
1390 its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc); in its_send_vmovi()
1393 static void its_send_vmapp(struct its_node *its, in its_send_vmapp() argument
1400 desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx]; in its_send_vmapp()
1402 its_send_single_vcommand(its, its_build_vmapp_cmd, &desc); in its_send_vmapp()
1408 struct its_node *its; in its_send_vmovp() local
1409 int col_id = vpe->col_idx; in its_send_vmovp()
1414 its = list_first_entry(&its_nodes, struct its_node, entry); in its_send_vmovp()
1415 desc.its_vmovp_cmd.col = &its->collections[col_id]; in its_send_vmovp()
1416 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); in its_send_vmovp()
1426 * Wall <-- Head. in its_send_vmovp()
1430 desc.its_vmovp_cmd.its_list = get_its_list(vpe->its_vm); in its_send_vmovp()
1433 list_for_each_entry(its, &its_nodes, entry) { in its_send_vmovp()
1434 if (!is_v4(its)) in its_send_vmovp()
1437 if (!require_its_list_vmovp(vpe->its_vm, its)) in its_send_vmovp()
1440 desc.its_vmovp_cmd.col = &its->collections[col_id]; in its_send_vmovp()
1441 its_send_single_vcommand(its, its_build_vmovp_cmd, &desc); in its_send_vmovp()
1445 static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe) in its_send_vinvall() argument
1450 its_send_single_vcommand(its, its_build_vinvall_cmd, &desc); in its_send_vinvall()
1464 its_send_single_vcommand(dev->its, its_build_vinv_cmd, &desc); in its_send_vinv()
1478 its_send_single_vcommand(dev->its, its_build_vint_cmd, &desc); in its_send_vint()
1492 its_send_single_vcommand(dev->its, its_build_vclear_cmd, &desc); in its_send_vclear()
1495 static void its_send_invdb(struct its_node *its, struct its_vpe *vpe) in its_send_invdb() argument
1500 its_send_single_vcommand(its, its_build_invdb_cmd, &desc); in its_send_invdb()
1504 * irqchip functions - assumes MSI, mostly.
1514 va = page_address(map->vm->vprop_page); in lpi_write_config()
1515 hwirq = map->vintid; in lpi_write_config()
1518 map->properties &= ~clr; in lpi_write_config()
1519 map->properties |= set | LPI_PROP_GROUP1; in lpi_write_config()
1521 va = gic_rdists->prop_table_va; in lpi_write_config()
1522 hwirq = d->hwirq; in lpi_write_config()
1525 cfg = va + hwirq - 8192; in lpi_write_config()
1534 if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING) in lpi_write_config()
1554 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); in __direct_lpi_inv()
1556 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; in __direct_lpi_inv()
1560 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); in __direct_lpi_inv()
1572 WARN_ON(!is_v4_1(its_dev->its)); in direct_lpi_inv()
1575 val |= FIELD_PREP(GICR_INVLPIR_VPEID, map->vpe->vpe_id); in direct_lpi_inv()
1576 val |= FIELD_PREP(GICR_INVLPIR_INTID, map->vintid); in direct_lpi_inv()
1578 val = d->hwirq; in direct_lpi_inv()
1589 if (gic_rdists->has_direct_lpi && in lpi_update_config()
1590 (is_v4_1(its_dev->its) || !irqd_is_forwarded_to_vcpu(d))) in lpi_update_config()
1605 * GICv4.1 does away with the per-LPI nonsense, nothing to do in its_vlpi_set_doorbell()
1608 if (is_v4_1(its_dev->its)) in its_vlpi_set_doorbell()
1613 if (map->db_enabled == enable) in its_vlpi_set_doorbell()
1616 map->db_enabled = enable; in its_vlpi_set_doorbell()
1621 * Ideally, we'd issue a VMAPTI to set the doorbell to its LPI in its_vlpi_set_doorbell()
1650 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); in its_read_lpi_count()
1652 return atomic_read(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); in its_read_lpi_count()
1658 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); in its_inc_lpi_count()
1660 atomic_inc(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); in its_inc_lpi_count()
1666 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->managed); in its_dec_lpi_count()
1668 atomic_dec(&per_cpu_ptr(&cpu_lpi_count, cpu)->unmanaged); in its_dec_lpi_count()
1701 node = its_dev->its->numa_node; in its_select_cpu()
1723 * ITS placed next to two NUMA nodes. in its_select_cpu()
1733 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)) in its_select_cpu()
1751 if ((its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) && in its_select_cpu()
1760 pr_debug("IRQ%d -> %*pbl CPU%d\n", d->irq, cpumask_pr_args(aff_mask), cpu); in its_select_cpu()
1774 return -EINVAL; in its_set_affinity()
1776 prev_cpu = its_dev->event_map.col_map[id]; in its_set_affinity()
1789 target_col = &its_dev->its->collections[cpu]; in its_set_affinity()
1791 its_dev->event_map.col_map[id] = cpu; in its_set_affinity()
1801 return -EINVAL; in its_set_affinity()
1806 struct its_node *its = its_dev->its; in its_irq_get_msi_base() local
1808 return its->phys_base + GITS_TRANSLATER; in its_irq_get_msi_base()
1814 struct its_node *its; in its_irq_compose_msi_msg() local
1817 its = its_dev->its; in its_irq_compose_msi_msg()
1818 addr = its->get_msi_base(its_dev); in its_irq_compose_msi_msg()
1820 msg->address_lo = lower_32_bits(addr); in its_irq_compose_msi_msg()
1821 msg->address_hi = upper_32_bits(addr); in its_irq_compose_msi_msg()
1822 msg->data = its_get_event_id(d); in its_irq_compose_msi_msg()
1835 return -EINVAL; in its_irq_set_irqchip_state()
1871 if (!its_list_map || gic_rdists->has_rvpeid) in gic_requires_eager_mapping()
1877 static void its_map_vm(struct its_node *its, struct its_vm *vm) in its_map_vm() argument
1882 guard(raw_spinlock_irqsave)(&vm->vmapp_lock); in its_map_vm()
1888 vm->vlpi_count[its->list_nr]++; in its_map_vm()
1890 if (vm->vlpi_count[its->list_nr] == 1) { in its_map_vm()
1893 for (i = 0; i < vm->nr_vpes; i++) { in its_map_vm()
1894 struct its_vpe *vpe = vm->vpes[i]; in its_map_vm()
1896 scoped_guard(raw_spinlock, &vpe->vpe_lock) in its_map_vm()
1897 its_send_vmapp(its, vpe, true); in its_map_vm()
1899 its_send_vinvall(its, vpe); in its_map_vm()
1904 static void its_unmap_vm(struct its_node *its, struct its_vm *vm) in its_unmap_vm() argument
1906 /* Not using the ITS list? Everything is always mapped. */ in its_unmap_vm()
1910 guard(raw_spinlock_irqsave)(&vm->vmapp_lock); in its_unmap_vm()
1912 if (!--vm->vlpi_count[its->list_nr]) { in its_unmap_vm()
1915 for (i = 0; i < vm->nr_vpes; i++) { in its_unmap_vm()
1916 guard(raw_spinlock)(&vm->vpes[i]->vpe_lock); in its_unmap_vm()
1917 its_send_vmapp(its, vm->vpes[i], false); in its_unmap_vm()
1927 if (!info->map) in its_vlpi_map()
1928 return -EINVAL; in its_vlpi_map()
1930 if (!its_dev->event_map.vm) { in its_vlpi_map()
1933 maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps), in its_vlpi_map()
1936 return -ENOMEM; in its_vlpi_map()
1938 its_dev->event_map.vm = info->map->vm; in its_vlpi_map()
1939 its_dev->event_map.vlpi_maps = maps; in its_vlpi_map()
1940 } else if (its_dev->event_map.vm != info->map->vm) { in its_vlpi_map()
1941 return -EINVAL; in its_vlpi_map()
1945 its_dev->event_map.vlpi_maps[event] = *info->map; in its_vlpi_map()
1951 /* Ensure all the VPEs are mapped on this ITS */ in its_vlpi_map()
1952 its_map_vm(its_dev->its, info->map->vm); in its_vlpi_map()
1961 lpi_write_config(d, 0xff, info->map->properties); in its_vlpi_map()
1970 its_dev->event_map.nr_vlpis++; in its_vlpi_map()
1983 if (!its_dev->event_map.vm || !map) in its_vlpi_get()
1984 return -EINVAL; in its_vlpi_get()
1987 *info->map = *map; in its_vlpi_get()
1997 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) in its_vlpi_unmap()
1998 return -EINVAL; in its_vlpi_unmap()
2005 its_send_mapti(its_dev, d->hwirq, event); in its_vlpi_unmap()
2010 /* Potentially unmap the VM from this ITS */ in its_vlpi_unmap()
2011 its_unmap_vm(its_dev->its, its_dev->event_map.vm); in its_vlpi_unmap()
2017 if (!--its_dev->event_map.nr_vlpis) { in its_vlpi_unmap()
2018 its_dev->event_map.vm = NULL; in its_vlpi_unmap()
2019 kfree(its_dev->event_map.vlpi_maps); in its_vlpi_unmap()
2029 if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) in its_vlpi_prop_update()
2030 return -EINVAL; in its_vlpi_prop_update()
2032 if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI) in its_vlpi_prop_update()
2033 lpi_update_config(d, 0xff, info->config); in its_vlpi_prop_update()
2035 lpi_write_config(d, 0xff, info->config); in its_vlpi_prop_update()
2036 its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED)); in its_vlpi_prop_update()
2046 /* Need a v4 ITS */ in its_irq_set_vcpu_affinity()
2047 if (!is_v4(its_dev->its)) in its_irq_set_vcpu_affinity()
2048 return -EINVAL; in its_irq_set_vcpu_affinity()
2050 guard(raw_spinlock)(&its_dev->event_map.vlpi_lock); in its_irq_set_vcpu_affinity()
2056 switch (info->cmd_type) { in its_irq_set_vcpu_affinity()
2068 return -EINVAL; in its_irq_set_vcpu_affinity()
2073 .name = "ITS",
2117 range->base_id = base; in mk_lpi_range()
2118 range->span = span; in mk_lpi_range()
2127 int err = -ENOSPC; in alloc_lpi_range()
2132 if (range->span >= nr_lpis) { in alloc_lpi_range()
2133 *base = range->base_id; in alloc_lpi_range()
2134 range->base_id += nr_lpis; in alloc_lpi_range()
2135 range->span -= nr_lpis; in alloc_lpi_range()
2137 if (range->span == 0) { in alloc_lpi_range()
2138 list_del(&range->entry); in alloc_lpi_range()
2149 pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis); in alloc_lpi_range()
2155 if (&a->entry == &lpi_range_list || &b->entry == &lpi_range_list) in merge_lpi_ranges()
2157 if (a->base_id + a->span != b->base_id) in merge_lpi_ranges()
2159 b->base_id = a->base_id; in merge_lpi_ranges()
2160 b->span += a->span; in merge_lpi_ranges()
2161 list_del(&a->entry); in merge_lpi_ranges()
2171 return -ENOMEM; in free_lpi_range()
2176 if (old->base_id < base) in free_lpi_range()
2180 * old is the last element with ->base_id smaller than base, in free_lpi_range()
2182 * ->base_id smaller than base, &old->entry ends up pointing in free_lpi_range()
2186 list_add(&new->entry, &old->entry); in free_lpi_range()
2200 u32 lpis = (1UL << id_bits) - 8192; in its_lpi_init()
2204 numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer); in its_lpi_init()
2208 pr_info("ITS: Using hypervisor restricted LPI range [%u]\n", in its_lpi_init()
2217 pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis); in its_lpi_init()
2235 err = -ENOSPC; in its_lpi_alloc()
2261 /* Regular IRQ priority, Group-1, disabled */ in gic_reset_prop_table()
2300 addr_end = addr + size - 1; in gic_check_reserved_range()
2324 if (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) { in its_setup_lpi_prop_table()
2330 gic_rdists->prop_table_pa = val & GENMASK_ULL(51, 12); in its_setup_lpi_prop_table()
2331 gic_rdists->prop_table_va = memremap(gic_rdists->prop_table_pa, in its_setup_lpi_prop_table()
2334 gic_reset_prop_table(gic_rdists->prop_table_va); in its_setup_lpi_prop_table()
2339 GICD_TYPER_ID_BITS(gic_rdists->gicd_typer), in its_setup_lpi_prop_table()
2344 return -ENOMEM; in its_setup_lpi_prop_table()
2347 gic_rdists->prop_table_pa = page_to_phys(page); in its_setup_lpi_prop_table()
2348 gic_rdists->prop_table_va = page_address(page); in its_setup_lpi_prop_table()
2349 WARN_ON(gic_reserve_range(gic_rdists->prop_table_pa, in its_setup_lpi_prop_table()
2354 &gic_rdists->prop_table_pa); in its_setup_lpi_prop_table()
2369 static u64 its_read_baser(struct its_node *its, struct its_baser *baser) in its_read_baser() argument
2371 u32 idx = baser - its->tables; in its_read_baser()
2373 return gits_read_baser(its->base + GITS_BASER + (idx << 3)); in its_read_baser()
2376 static void its_write_baser(struct its_node *its, struct its_baser *baser, in its_write_baser() argument
2379 u32 idx = baser - its->tables; in its_write_baser()
2381 gits_write_baser(val, its->base + GITS_BASER + (idx << 3)); in its_write_baser()
2382 baser->val = its_read_baser(its, baser); in its_write_baser()
2385 static int its_setup_baser(struct its_node *its, struct its_baser *baser, in its_setup_baser() argument
2388 u64 val = its_read_baser(its, baser); in its_setup_baser()
2396 psz = baser->psz; in its_setup_baser()
2399 pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n", in its_setup_baser()
2400 &its->phys_base, its_base_type_string[type], in its_setup_baser()
2406 page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, order); in its_setup_baser()
2408 return -ENOMEM; in its_setup_baser()
2418 pr_err("ITS: no 52bit PA support when psz=%d\n", psz); in its_setup_baser()
2420 return -ENXIO; in its_setup_baser()
2430 ((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) | in its_setup_baser()
2431 ((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) | in its_setup_baser()
2453 its_write_baser(its, baser, val); in its_setup_baser()
2454 tmp = baser->val; in its_setup_baser()
2462 * non-cacheable as well. in its_setup_baser()
2472 pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n", in its_setup_baser()
2473 &its->phys_base, its_base_type_string[type], in its_setup_baser()
2476 return -ENXIO; in its_setup_baser()
2479 baser->order = order; in its_setup_baser()
2480 baser->base = base; in its_setup_baser()
2481 baser->psz = psz; in its_setup_baser()
2484 pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n", in its_setup_baser()
2485 &its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp), in its_setup_baser()
2494 static bool its_parse_indirect_baser(struct its_node *its, in its_parse_indirect_baser() argument
2498 u64 tmp = its_read_baser(its, baser); in its_parse_indirect_baser()
2503 u32 psz = baser->psz; in its_parse_indirect_baser()
2509 * Find out whether hw supports a single or two-level table by in its_parse_indirect_baser()
2512 its_write_baser(its, baser, val | GITS_BASER_INDIRECT); in its_parse_indirect_baser()
2513 indirect = !!(baser->val & GITS_BASER_INDIRECT); in its_parse_indirect_baser()
2517 * The size of the lvl2 table is equal to ITS page size in its_parse_indirect_baser()
2520 * which is reported by ITS hardware times lvl1 table in its_parse_indirect_baser()
2523 ids -= ilog2(psz / (int)esz); in its_parse_indirect_baser()
2530 * range of device IDs that the ITS can grok... The ID in its_parse_indirect_baser()
2532 * massive waste of memory if two-level device table in its_parse_indirect_baser()
2539 pr_warn("ITS@%pa: %s Table too large, reduce ids %llu->%u\n", in its_parse_indirect_baser()
2540 &its->phys_base, its_base_type_string[type], in its_parse_indirect_baser()
2541 device_ids(its), ids); in its_parse_indirect_baser()
2559 static u32 compute_its_aff(struct its_node *its) in compute_its_aff() argument
2565 * Reencode the ITS SVPET and MPIDR as a GICR_TYPER, and compute in compute_its_aff()
2569 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); in compute_its_aff()
2571 val |= FIELD_PREP(GICR_TYPER_AFFINITY, its->mpidr); in compute_its_aff()
2577 struct its_node *its; in find_sibling_its() local
2580 if (!FIELD_GET(GITS_TYPER_SVPET, cur_its->typer)) in find_sibling_its()
2585 list_for_each_entry(its, &its_nodes, entry) { in find_sibling_its()
2588 if (!is_v4_1(its) || its == cur_its) in find_sibling_its()
2591 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) in find_sibling_its()
2594 if (aff != compute_its_aff(its)) in find_sibling_its()
2598 baser = its->tables[2].val; in find_sibling_its()
2602 return its; in find_sibling_its()
2608 static void its_free_tables(struct its_node *its) in its_free_tables() argument
2613 if (its->tables[i].base) { in its_free_tables()
2614 its_free_pages(its->tables[i].base, its->tables[i].order); in its_free_tables()
2615 its->tables[i].base = NULL; in its_free_tables()
2620 static int its_probe_baser_psz(struct its_node *its, struct its_baser *baser) in its_probe_baser_psz() argument
2627 val = its_read_baser(its, baser); in its_probe_baser_psz()
2646 its_write_baser(its, baser, val); in its_probe_baser_psz()
2648 if (FIELD_GET(GITS_BASER_PAGE_SIZE_MASK, baser->val) == gpsz) in its_probe_baser_psz()
2660 return -1; in its_probe_baser_psz()
2664 baser->psz = psz; in its_probe_baser_psz()
2668 static int its_alloc_tables(struct its_node *its) in its_alloc_tables() argument
2674 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375) in its_alloc_tables()
2678 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) { in its_alloc_tables()
2684 struct its_baser *baser = its->tables + i; in its_alloc_tables()
2685 u64 val = its_read_baser(its, baser); in its_alloc_tables()
2693 if (its_probe_baser_psz(its, baser)) { in its_alloc_tables()
2694 its_free_tables(its); in its_alloc_tables()
2695 return -ENXIO; in its_alloc_tables()
2698 order = get_order(baser->psz); in its_alloc_tables()
2702 indirect = its_parse_indirect_baser(its, baser, &order, in its_alloc_tables()
2703 device_ids(its)); in its_alloc_tables()
2707 if (is_v4_1(its)) { in its_alloc_tables()
2711 if ((sibling = find_sibling_its(its))) { in its_alloc_tables()
2712 *baser = sibling->tables[2]; in its_alloc_tables()
2713 its_write_baser(its, baser, baser->val); in its_alloc_tables()
2718 indirect = its_parse_indirect_baser(its, baser, &order, in its_alloc_tables()
2723 err = its_setup_baser(its, baser, cache, shr, order, indirect); in its_alloc_tables()
2725 its_free_tables(its); in its_alloc_tables()
2730 cache = baser->val & GITS_BASER_CACHEABILITY_MASK; in its_alloc_tables()
2731 shr = baser->val & GITS_BASER_SHAREABILITY_MASK; in its_alloc_tables()
2739 struct its_node *its; in inherit_vpe_l1_table_from_its() local
2746 list_for_each_entry(its, &its_nodes, entry) { in inherit_vpe_l1_table_from_its()
2749 if (!is_v4_1(its)) in inherit_vpe_l1_table_from_its()
2752 if (!FIELD_GET(GITS_TYPER_SVPET, its->typer)) in inherit_vpe_l1_table_from_its()
2755 if (aff != compute_its_aff(its)) in inherit_vpe_l1_table_from_its()
2759 baser = its->tables[2].val; in inherit_vpe_l1_table_from_its()
2764 gic_data_rdist()->vpe_l1_base = its->tables[2].base; in inherit_vpe_l1_table_from_its()
2786 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, GITS_BASER_NR_PAGES(baser) - 1); in inherit_vpe_l1_table_from_its()
2804 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; in inherit_vpe_l1_table_from_rd()
2816 * ours wrt CommonLPIAff. Let's use its own VPROPBASER. in inherit_vpe_l1_table_from_rd()
2822 gic_data_rdist()->vpe_l1_base = gic_data_rdist_cpu(cpu)->vpe_l1_base; in inherit_vpe_l1_table_from_rd()
2823 *mask = gic_data_rdist_cpu(cpu)->vpe_table_mask; in inherit_vpe_l1_table_from_rd()
2833 void __iomem *base = gic_data_rdist_cpu(cpu)->rd_base; in allocate_vpe_l2_table()
2839 if (!gic_rdists->has_rvpeid) in allocate_vpe_l2_table()
2842 /* Skip non-present CPUs */ in allocate_vpe_l2_table()
2876 table = gic_data_rdist_cpu(cpu)->vpe_l1_base; in allocate_vpe_l2_table()
2909 if (!gic_rdists->has_rvpeid) in allocate_vpe_l1_table()
2928 val = inherit_vpe_l1_table_from_rd(&gic_data_rdist()->vpe_table_mask); in allocate_vpe_l1_table()
2932 gic_data_rdist()->vpe_table_mask = kzalloc(sizeof(cpumask_t), GFP_ATOMIC); in allocate_vpe_l1_table()
2933 if (!gic_data_rdist()->vpe_table_mask) in allocate_vpe_l1_table()
2934 return -ENOMEM; in allocate_vpe_l1_table()
2992 val |= FIELD_PREP(GICR_VPROPBASER_4_1_SIZE, npg - 1); in allocate_vpe_l1_table()
3001 return -ENOMEM; in allocate_vpe_l1_table()
3003 gic_data_rdist()->vpe_l1_base = page_address(page); in allocate_vpe_l1_table()
3017 cpumask_set_cpu(smp_processor_id(), gic_data_rdist()->vpe_table_mask); in allocate_vpe_l1_table()
3021 cpumask_pr_args(gic_data_rdist()->vpe_table_mask)); in allocate_vpe_l1_table()
3026 static int its_alloc_collections(struct its_node *its) in its_alloc_collections() argument
3030 its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections), in its_alloc_collections()
3032 if (!its->collections) in its_alloc_collections()
3033 return -ENOMEM; in its_alloc_collections()
3036 its->collections[i].target_address = ~0ULL; in its_alloc_collections()
3049 /* Make sure the GIC will observe the zero-ed page */ in its_allocate_pending_table()
3083 * flag the RD tables as pre-allocated if the stars do align. in allocate_lpi_tables()
3087 gic_rdists->flags |= (RDIST_FLAGS_RD_TABLES_PREALLOCATED | in allocate_lpi_tables()
3107 return -ENOMEM; in allocate_lpi_tables()
3110 gic_data_rdist_cpu(cpu)->pend_page = pend_page; in allocate_lpi_tables()
3126 count--; in read_vpend_dirty_clear()
3133 pr_err_ratelimited("ITS virtual pending table not cleaning\n"); in read_vpend_dirty_clear()
3163 if (gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) in its_cpu_init_lpis()
3167 if ((gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED) && in its_cpu_init_lpis()
3175 if (WARN_ON(gic_rdists->prop_table_pa != paddr)) in its_cpu_init_lpis()
3182 gic_data_rdist()->flags |= RD_LOCAL_PENDTABLE_PREALLOCATED; in its_cpu_init_lpis()
3187 pend_page = gic_data_rdist()->pend_page; in its_cpu_init_lpis()
3191 val = (gic_rdists->prop_table_pa | in its_cpu_init_lpis()
3194 ((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK)); in its_cpu_init_lpis()
3205 * The HW reports non-shareable, we must in its_cpu_init_lpis()
3215 gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING; in its_cpu_init_lpis()
3231 * The HW reports non-shareable, we must remove the in its_cpu_init_lpis()
3246 if (gic_rdists->has_vlpis && !gic_rdists->has_rvpeid) { in its_cpu_init_lpis()
3256 val = (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; in its_cpu_init_lpis()
3275 gic_rdists->has_rvpeid = false; in its_cpu_init_lpis()
3276 gic_rdists->has_vlpis = false; in its_cpu_init_lpis()
3281 gic_data_rdist()->flags |= RD_LOCAL_LPI_ENABLED; in its_cpu_init_lpis()
3284 gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED ? in its_cpu_init_lpis()
3289 static void its_cpu_init_collection(struct its_node *its) in its_cpu_init_collection() argument
3294 /* avoid cross node collections and its mapping */ in its_cpu_init_collection()
3295 if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) { in its_cpu_init_collection()
3299 if (its->numa_node != NUMA_NO_NODE && in its_cpu_init_collection()
3300 its->numa_node != of_node_to_nid(cpu_node)) in its_cpu_init_collection()
3305 * We now have to bind each collection to its target in its_cpu_init_collection()
3308 if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) { in its_cpu_init_collection()
3310 * This ITS wants the physical address of the in its_cpu_init_collection()
3313 target = gic_data_rdist()->phys_base; in its_cpu_init_collection()
3315 /* This ITS wants a linear CPU number. */ in its_cpu_init_collection()
3321 its->collections[cpu].target_address = target; in its_cpu_init_collection()
3322 its->collections[cpu].col_id = cpu; in its_cpu_init_collection()
3324 its_send_mapc(its, &its->collections[cpu], 1); in its_cpu_init_collection()
3325 its_send_invall(its, &its->collections[cpu]); in its_cpu_init_collection()
3330 struct its_node *its; in its_cpu_init_collections() local
3334 list_for_each_entry(its, &its_nodes, entry) in its_cpu_init_collections()
3335 its_cpu_init_collection(its); in its_cpu_init_collections()
3340 static struct its_device *its_find_device(struct its_node *its, u32 dev_id) in its_find_device() argument
3345 raw_spin_lock_irqsave(&its->lock, flags); in its_find_device()
3347 list_for_each_entry(tmp, &its->its_device_list, entry) { in its_find_device()
3348 if (tmp->device_id == dev_id) { in its_find_device()
3354 raw_spin_unlock_irqrestore(&its->lock, flags); in its_find_device()
3359 static struct its_baser *its_get_baser(struct its_node *its, u32 type) in its_get_baser() argument
3364 if (GITS_BASER_TYPE(its->tables[i].val) == type) in its_get_baser()
3365 return &its->tables[i]; in its_get_baser()
3371 static bool its_alloc_table_entry(struct its_node *its, in its_alloc_table_entry() argument
3379 esz = GITS_BASER_ENTRY_SIZE(baser->val); in its_alloc_table_entry()
3380 if (!(baser->val & GITS_BASER_INDIRECT)) in its_alloc_table_entry()
3381 return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz)); in its_alloc_table_entry()
3384 idx = id >> ilog2(baser->psz / esz); in its_alloc_table_entry()
3385 if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE)) in its_alloc_table_entry()
3388 table = baser->base; in its_alloc_table_entry()
3392 page = its_alloc_pages_node(its->numa_node, GFP_KERNEL | __GFP_ZERO, in its_alloc_table_entry()
3393 get_order(baser->psz)); in its_alloc_table_entry()
3398 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) in its_alloc_table_entry()
3399 gic_flush_dcache_to_poc(page_address(page), baser->psz); in its_alloc_table_entry()
3404 if (!(baser->val & GITS_BASER_SHAREABILITY_MASK)) in its_alloc_table_entry()
3407 /* Ensure updated table contents are visible to ITS hardware */ in its_alloc_table_entry()
3414 static bool its_alloc_device_table(struct its_node *its, u32 dev_id) in its_alloc_device_table() argument
3418 baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE); in its_alloc_device_table()
3420 /* Don't allow device id that exceeds ITS hardware limit */ in its_alloc_device_table()
3422 return (ilog2(dev_id) < device_ids(its)); in its_alloc_device_table()
3424 return its_alloc_table_entry(its, baser, dev_id); in its_alloc_device_table()
3429 struct its_node *its; in its_alloc_vpe_table() local
3439 list_for_each_entry(its, &its_nodes, entry) { in its_alloc_vpe_table()
3442 if (!is_v4(its)) in its_alloc_vpe_table()
3445 baser = its_get_baser(its, GITS_BASER_TYPE_VCPU); in its_alloc_vpe_table()
3449 if (!its_alloc_table_entry(its, baser, vpe_id)) in its_alloc_vpe_table()
3454 if (!gic_rdists->has_rvpeid) in its_alloc_vpe_table()
3469 static struct its_device *its_create_device(struct its_node *its, u32 dev_id, in its_create_device() argument
3482 if (!its_alloc_device_table(its, dev_id)) in its_create_device()
3493 sz = nr_ites * (FIELD_GET(GITS_TYPER_ITT_ENTRY_SIZE, its->typer) + 1); in its_create_device()
3496 itt = itt_alloc_pool(its->numa_node, sz); in its_create_device()
3521 dev->its = its; in its_create_device()
3522 dev->itt = itt; in its_create_device()
3523 dev->itt_sz = sz; in its_create_device()
3524 dev->nr_ites = nr_ites; in its_create_device()
3525 dev->event_map.lpi_map = lpi_map; in its_create_device()
3526 dev->event_map.col_map = col_map; in its_create_device()
3527 dev->event_map.lpi_base = lpi_base; in its_create_device()
3528 dev->event_map.nr_lpis = nr_lpis; in its_create_device()
3529 raw_spin_lock_init(&dev->event_map.vlpi_lock); in its_create_device()
3530 dev->device_id = dev_id; in its_create_device()
3531 INIT_LIST_HEAD(&dev->entry); in its_create_device()
3533 raw_spin_lock_irqsave(&its->lock, flags); in its_create_device()
3534 list_add(&dev->entry, &its->its_device_list); in its_create_device()
3535 raw_spin_unlock_irqrestore(&its->lock, flags); in its_create_device()
3537 /* Map device to its ITT */ in its_create_device()
3547 raw_spin_lock_irqsave(&its_dev->its->lock, flags); in its_free_device()
3548 list_del(&its_dev->entry); in its_free_device()
3549 raw_spin_unlock_irqrestore(&its_dev->its->lock, flags); in its_free_device()
3550 kfree(its_dev->event_map.col_map); in its_free_device()
3551 itt_free_pool(its_dev->itt, its_dev->itt_sz); in its_free_device()
3560 idx = bitmap_find_free_region(dev->event_map.lpi_map, in its_alloc_device_irq()
3561 dev->event_map.nr_lpis, in its_alloc_device_irq()
3564 return -ENOSPC; in its_alloc_device_irq()
3566 *hwirq = dev->event_map.lpi_base + idx; in its_alloc_device_irq()
3574 struct its_node *its; in its_msi_prepare() local
3584 * are built on top of the ITS. in its_msi_prepare()
3586 dev_id = info->scratchpad[0].ul; in its_msi_prepare()
3589 its = msi_info->data; in its_msi_prepare()
3591 if (!gic_rdists->has_direct_lpi && in its_msi_prepare()
3593 vpe_proxy.dev->its == its && in its_msi_prepare()
3594 dev_id == vpe_proxy.dev->device_id) { in its_msi_prepare()
3598 return -EINVAL; in its_msi_prepare()
3601 mutex_lock(&its->dev_alloc_lock); in its_msi_prepare()
3602 its_dev = its_find_device(its, dev_id); in its_msi_prepare()
3609 its_dev->shared = true; in its_msi_prepare()
3614 its_dev = its_create_device(its, dev_id, nvec, true); in its_msi_prepare()
3616 err = -ENOMEM; in its_msi_prepare()
3620 if (info->flags & MSI_ALLOC_FLAGS_PROXY_DEVICE) in its_msi_prepare()
3621 its_dev->shared = true; in its_msi_prepare()
3625 mutex_unlock(&its->dev_alloc_lock); in its_msi_prepare()
3626 info->scratchpad[0].ptr = its_dev; in its_msi_prepare()
3640 if (irq_domain_get_of_node(domain->parent)) { in its_irq_gic_domain_alloc()
3641 fwspec.fwnode = domain->parent->fwnode; in its_irq_gic_domain_alloc()
3646 } else if (is_fwnode_irqchip(domain->parent->fwnode)) { in its_irq_gic_domain_alloc()
3647 fwspec.fwnode = domain->parent->fwnode; in its_irq_gic_domain_alloc()
3652 return -EINVAL; in its_irq_gic_domain_alloc()
3662 struct its_device *its_dev = info->scratchpad[0].ptr; in its_irq_domain_alloc()
3663 struct its_node *its = its_dev->its; in its_irq_domain_alloc() local
3673 err = iommu_dma_prepare_msi(info->desc, its->get_msi_base(its_dev)); in its_irq_domain_alloc()
3689 (int)(hwirq + i - its_dev->event_map.lpi_base), in its_irq_domain_alloc()
3705 return -EINVAL; in its_irq_domain_activate()
3708 its_dev->event_map.col_map[event] = cpu; in its_irq_domain_activate()
3712 its_send_mapti(its_dev, d->hwirq, event); in its_irq_domain_activate()
3722 its_dec_lpi_count(d, its_dev->event_map.col_map[event]); in its_irq_domain_deactivate()
3732 struct its_node *its = its_dev->its; in its_irq_domain_free() local
3735 bitmap_release_region(its_dev->event_map.lpi_map, in its_irq_domain_free()
3746 mutex_lock(&its->dev_alloc_lock); in its_irq_domain_free()
3752 if (!its_dev->shared && in its_irq_domain_free()
3753 bitmap_empty(its_dev->event_map.lpi_map, in its_irq_domain_free()
3754 its_dev->event_map.nr_lpis)) { in its_irq_domain_free()
3755 its_lpi_free(its_dev->event_map.lpi_map, in its_irq_domain_free()
3756 its_dev->event_map.lpi_base, in its_irq_domain_free()
3757 its_dev->event_map.nr_lpis); in its_irq_domain_free()
3764 mutex_unlock(&its->dev_alloc_lock); in its_irq_domain_free()
3799 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_unmap_locked()
3803 if (vpe->vpe_proxy_event == -1) in its_vpe_db_proxy_unmap_locked()
3806 its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event); in its_vpe_db_proxy_unmap_locked()
3807 vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL; in its_vpe_db_proxy_unmap_locked()
3817 vpe_proxy.next_victim = vpe->vpe_proxy_event; in its_vpe_db_proxy_unmap_locked()
3819 vpe->vpe_proxy_event = -1; in its_vpe_db_proxy_unmap_locked()
3825 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_unmap()
3828 if (!gic_rdists->has_direct_lpi) { in its_vpe_db_proxy_unmap()
3840 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_map_locked()
3844 if (vpe->vpe_proxy_event != -1) in its_vpe_db_proxy_map_locked()
3853 vpe->vpe_proxy_event = vpe_proxy.next_victim; in its_vpe_db_proxy_map_locked()
3854 vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites; in its_vpe_db_proxy_map_locked()
3856 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx; in its_vpe_db_proxy_map_locked()
3857 its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event); in its_vpe_db_proxy_map_locked()
3866 if (gic_rdists->has_rvpeid) in its_vpe_db_proxy_move()
3869 if (gic_rdists->has_direct_lpi) { in its_vpe_db_proxy_move()
3872 rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base; in its_vpe_db_proxy_move()
3873 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); in its_vpe_db_proxy_move()
3883 target_col = &vpe_proxy.dev->its->collections[to]; in its_vpe_db_proxy_move()
3884 its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event); in its_vpe_db_proxy_move()
3885 vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to; in its_vpe_db_proxy_move()
3896 val |= FIELD_PREP(GICR_INVALLR_VPEID, vpe->vpe_id); in its_vpe_4_1_invall_locked()
3898 guard(raw_spinlock)(&gic_data_rdist_cpu(cpu)->rd_lock); in its_vpe_4_1_invall_locked()
3899 rdbase = per_cpu_ptr(gic_rdists->rdist, cpu)->rd_base; in its_vpe_4_1_invall_locked()
3911 struct its_node *its; in its_vpe_set_affinity() local
3918 if (!atomic_read(&vpe->vmapp_count)) { in its_vpe_set_affinity()
3920 return -EINVAL; in its_vpe_set_affinity()
3935 * interrupt to its new location. in its_vpe_set_affinity()
3940 * protect us, and that we must ensure nobody samples vpe->col_idx in its_vpe_set_affinity()
3942 * taken on any vLPI handling path that evaluates vpe->col_idx. in its_vpe_set_affinity()
3945 * the mapping state on this VM should the ITS list be in use (see in its_vpe_set_affinity()
3949 raw_spin_lock(&vpe->its_vm->vmapp_lock); in its_vpe_set_affinity()
3952 table_mask = gic_data_rdist_cpu(from)->vpe_table_mask; in its_vpe_set_affinity()
3955 * If we are offered another CPU in the same GICv4.1 ITS in its_vpe_set_affinity()
3971 vpe->col_idx = cpu; in its_vpe_set_affinity()
3975 its = find_4_1_its(); in its_vpe_set_affinity()
3976 if (its && its->flags & ITS_FLAGS_WORKAROUND_HISILICON_162100801) in its_vpe_set_affinity()
3986 raw_spin_unlock(&vpe->its_vm->vmapp_lock); in its_vpe_set_affinity()
3996 if (!gic_rdists->has_vpend_valid_dirty) in its_wait_vpt_parse_complete()
4011 val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) & in its_vpe_schedule()
4013 val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK; in its_vpe_schedule()
4020 val = virt_to_phys(page_address(vpe->vpt_page)) & in its_vpe_schedule()
4029 * easily. So in the end, vpe->pending_last is only an in its_vpe_schedule()
4032 * would be able to read its coarse map pretty quickly anyway, in its_vpe_schedule()
4036 val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0; in its_vpe_schedule()
4048 vpe->idai = !!(val & GICR_VPENDBASER_IDAI); in its_vpe_deschedule()
4049 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); in its_vpe_deschedule()
4054 struct its_node *its; in its_vpe_invall() local
4056 guard(raw_spinlock_irqsave)(&vpe->its_vm->vmapp_lock); in its_vpe_invall()
4058 list_for_each_entry(its, &its_nodes, entry) { in its_vpe_invall()
4059 if (!is_v4(its)) in its_vpe_invall()
4062 if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr]) in its_vpe_invall()
4066 * Sending a VINVALL to a single ITS is enough, as all in its_vpe_invall()
4069 its_send_vinvall(its, vpe); in its_vpe_invall()
4079 switch (info->cmd_type) { in its_vpe_set_vcpu_affinity()
4097 return -EINVAL; in its_vpe_set_vcpu_affinity()
4109 cmd(vpe_proxy.dev, vpe->vpe_proxy_event); in its_vpe_send_cmd()
4118 if (gic_rdists->has_direct_lpi) in its_vpe_send_inv()
4119 __direct_lpi_inv(d, d->parent_data->hwirq); in its_vpe_send_inv()
4132 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); in its_vpe_mask_irq()
4139 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); in its_vpe_unmask_irq()
4150 return -EINVAL; in its_vpe_set_irqchip_state()
4152 if (gic_rdists->has_direct_lpi) { in its_vpe_set_irqchip_state()
4155 rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base; in its_vpe_set_irqchip_state()
4157 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR); in its_vpe_set_irqchip_state()
4159 gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR); in its_vpe_set_irqchip_state()
4178 .name = "GICv4-vpe",
4190 static struct its_node *its = NULL; in find_4_1_its() local
4192 if (!its) { in find_4_1_its()
4193 list_for_each_entry(its, &its_nodes, entry) { in find_4_1_its()
4194 if (is_v4_1(its)) in find_4_1_its()
4195 return its; in find_4_1_its()
4199 its = NULL; in find_4_1_its()
4202 return its; in find_4_1_its()
4208 struct its_node *its; in its_vpe_4_1_send_inv() local
4213 * it to the first valid ITS, and let the HW do its magic. in its_vpe_4_1_send_inv()
4215 its = find_4_1_its(); in its_vpe_4_1_send_inv()
4216 if (its) in its_vpe_4_1_send_inv()
4217 its_send_invdb(its, vpe); in its_vpe_4_1_send_inv()
4222 lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0); in its_vpe_4_1_mask_irq()
4228 lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED); in its_vpe_4_1_unmask_irq()
4240 val |= info->g0en ? GICR_VPENDBASER_4_1_VGRP0EN : 0; in its_vpe_4_1_schedule()
4241 val |= info->g1en ? GICR_VPENDBASER_4_1_VGRP1EN : 0; in its_vpe_4_1_schedule()
4242 val |= FIELD_PREP(GICR_VPENDBASER_4_1_VPEID, vpe->vpe_id); in its_vpe_4_1_schedule()
4253 if (info->req_db) { in its_vpe_4_1_deschedule()
4257 * vPE is going to block: make the vPE non-resident with in its_vpe_4_1_deschedule()
4259 * we read-back PendingLast clear, then a doorbell will be in its_vpe_4_1_deschedule()
4266 raw_spin_lock_irqsave(&vpe->vpe_lock, flags); in its_vpe_4_1_deschedule()
4270 vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast); in its_vpe_4_1_deschedule()
4271 raw_spin_unlock_irqrestore(&vpe->vpe_lock, flags); in its_vpe_4_1_deschedule()
4274 * We're not blocking, so just make the vPE non-resident in its_vpe_4_1_deschedule()
4280 vpe->pending_last = true; in its_vpe_4_1_deschedule()
4300 switch (info->cmd_type) { in its_vpe_4_1_set_vcpu_affinity()
4318 return -EINVAL; in its_vpe_4_1_set_vcpu_affinity()
4323 .name = "GICv4.1-vpe",
4337 desc.its_vsgi_cmd.sgi = d->hwirq; in its_configure_sgi()
4338 desc.its_vsgi_cmd.priority = vpe->sgi_config[d->hwirq].priority; in its_configure_sgi()
4339 desc.its_vsgi_cmd.enable = vpe->sgi_config[d->hwirq].enabled; in its_configure_sgi()
4340 desc.its_vsgi_cmd.group = vpe->sgi_config[d->hwirq].group; in its_configure_sgi()
4344 * GICv4.1 allows us to send VSGI commands to any ITS as long as the in its_configure_sgi()
4346 * activation time, we're pretty sure the first GICv4.1 ITS will do. in its_configure_sgi()
4355 vpe->sgi_config[d->hwirq].enabled = false; in its_sgi_mask_irq()
4363 vpe->sgi_config[d->hwirq].enabled = true; in its_sgi_unmask_irq()
4385 return -EINVAL; in its_sgi_set_irqchip_state()
4389 struct its_node *its = find_4_1_its(); in its_sgi_set_irqchip_state() local
4392 val = FIELD_PREP(GITS_SGIR_VPEID, vpe->vpe_id); in its_sgi_set_irqchip_state()
4393 val |= FIELD_PREP(GITS_SGIR_VINTID, d->hwirq); in its_sgi_set_irqchip_state()
4394 writeq_relaxed(val, its->sgir_base + GITS_SGIR - SZ_128K); in its_sgi_set_irqchip_state()
4413 return -EINVAL; in its_sgi_get_irqchip_state()
4418 * - Concurrent vPE affinity change: we must make sure it cannot in its_sgi_get_irqchip_state()
4422 * - Concurrent VSGIPENDR access: As it involves accessing two in its_sgi_get_irqchip_state()
4426 raw_spin_lock(&gic_data_rdist_cpu(cpu)->rd_lock); in its_sgi_get_irqchip_state()
4427 base = gic_data_rdist_cpu(cpu)->rd_base + SZ_128K; in its_sgi_get_irqchip_state()
4428 writel_relaxed(vpe->vpe_id, base + GICR_VSGIR); in its_sgi_get_irqchip_state()
4434 count--; in its_sgi_get_irqchip_state()
4444 raw_spin_unlock(&gic_data_rdist_cpu(cpu)->rd_lock); in its_sgi_get_irqchip_state()
4448 return -ENXIO; in its_sgi_get_irqchip_state()
4450 *val = !!(status & (1 << d->hwirq)); in its_sgi_get_irqchip_state()
4460 switch (info->cmd_type) { in its_sgi_set_vcpu_affinity()
4462 vpe->sgi_config[d->hwirq].priority = info->priority; in its_sgi_set_vcpu_affinity()
4463 vpe->sgi_config[d->hwirq].group = info->group; in its_sgi_set_vcpu_affinity()
4468 return -EINVAL; in its_sgi_set_vcpu_affinity()
4473 .name = "GICv4.1-sgi",
4493 vpe->sgi_config[i].priority = 0; in its_sgi_irq_domain_alloc()
4494 vpe->sgi_config[i].enabled = false; in its_sgi_irq_domain_alloc()
4495 vpe->sgi_config[i].group = false; in its_sgi_irq_domain_alloc()
4528 * - To change the configuration, CLEAR must be set to false, in its_sgi_irq_domain_deactivate()
4530 * - To clear the pending bit, CLEAR must be set to true, leaving in its_sgi_irq_domain_deactivate()
4535 vpe->sgi_config[d->hwirq].enabled = false; in its_sgi_irq_domain_deactivate()
4549 return ida_alloc_max(&its_vpeid_ida, ITS_MAX_VPEID - 1, GFP_KERNEL); in its_vpe_id_alloc()
4571 return -ENOMEM; in its_vpe_init()
4577 return -ENOMEM; in its_vpe_init()
4580 raw_spin_lock_init(&vpe->vpe_lock); in its_vpe_init()
4581 vpe->vpe_id = vpe_id; in its_vpe_init()
4582 vpe->vpt_page = vpt_page; in its_vpe_init()
4583 atomic_set(&vpe->vmapp_count, 0); in its_vpe_init()
4584 if (!gic_rdists->has_rvpeid) in its_vpe_init()
4585 vpe->vpe_proxy_event = -1; in its_vpe_init()
4593 its_vpe_id_free(vpe->vpe_id); in its_vpe_teardown()
4594 its_free_pending_table(vpe->vpt_page); in its_vpe_teardown()
4601 struct its_vm *vm = domain->host_data; in its_vpe_irq_domain_free()
4611 BUG_ON(vm != vpe->its_vm); in its_vpe_irq_domain_free()
4613 clear_bit(data->hwirq, vm->db_bitmap); in its_vpe_irq_domain_free()
4618 if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) { in its_vpe_irq_domain_free()
4619 its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis); in its_vpe_irq_domain_free()
4620 its_free_prop_table(vm->vprop_page); in its_vpe_irq_domain_free()
4635 return -ENOMEM; in its_vpe_irq_domain_alloc()
4639 return -ENOMEM; in its_vpe_irq_domain_alloc()
4645 return -ENOMEM; in its_vpe_irq_domain_alloc()
4648 vm->db_bitmap = bitmap; in its_vpe_irq_domain_alloc()
4649 vm->db_lpi_base = base; in its_vpe_irq_domain_alloc()
4650 vm->nr_db_lpis = nr_ids; in its_vpe_irq_domain_alloc()
4651 vm->vprop_page = vprop_page; in its_vpe_irq_domain_alloc()
4652 raw_spin_lock_init(&vm->vmapp_lock); in its_vpe_irq_domain_alloc()
4654 if (gic_rdists->has_rvpeid) in its_vpe_irq_domain_alloc()
4658 vm->vpes[i]->vpe_db_lpi = base + i; in its_vpe_irq_domain_alloc()
4659 err = its_vpe_init(vm->vpes[i]); in its_vpe_irq_domain_alloc()
4663 vm->vpes[i]->vpe_db_lpi); in its_vpe_irq_domain_alloc()
4667 irqchip, vm->vpes[i]); in its_vpe_irq_domain_alloc()
4682 struct its_node *its; in its_vpe_irq_domain_activate() local
4685 vpe->col_idx = cpumask_first(cpu_online_mask); in its_vpe_irq_domain_activate()
4686 irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx)); in its_vpe_irq_domain_activate()
4696 list_for_each_entry(its, &its_nodes, entry) { in its_vpe_irq_domain_activate()
4697 if (!is_v4(its)) in its_vpe_irq_domain_activate()
4700 its_send_vmapp(its, vpe, true); in its_vpe_irq_domain_activate()
4701 its_send_vinvall(its, vpe); in its_vpe_irq_domain_activate()
4711 struct its_node *its; in its_vpe_irq_domain_deactivate() local
4720 list_for_each_entry(its, &its_nodes, entry) { in its_vpe_irq_domain_deactivate()
4721 if (!is_v4(its)) in its_vpe_irq_domain_deactivate()
4724 its_send_vmapp(its, vpe, false); in its_vpe_irq_domain_deactivate()
4732 if (find_4_1_its() && !atomic_read(&vpe->vmapp_count)) in its_vpe_irq_domain_deactivate()
4733 gic_flush_dcache_to_poc(page_address(vpe->vpt_page), in its_vpe_irq_domain_deactivate()
4751 * GIC architecture specification requires the ITS to be both in its_force_quiescent()
4758 /* Disable the generation of all interrupts to this ITS */ in its_force_quiescent()
4762 /* Poll GITS_CTLR and wait until ITS becomes quiescent */ in its_force_quiescent()
4768 count--; in its_force_quiescent()
4770 return -EBUSY; in its_force_quiescent()
4779 struct its_node *its = data; in its_enable_quirk_cavium_22375() local
4782 its->typer &= ~GITS_TYPER_DEVBITS; in its_enable_quirk_cavium_22375()
4783 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, 20 - 1); in its_enable_quirk_cavium_22375()
4784 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375; in its_enable_quirk_cavium_22375()
4791 struct its_node *its = data; in its_enable_quirk_cavium_23144() local
4793 its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144; in its_enable_quirk_cavium_23144()
4800 struct its_node *its = data; in its_enable_quirk_qdf2400_e0065() local
4803 its->typer &= ~GITS_TYPER_ITT_ENTRY_SIZE; in its_enable_quirk_qdf2400_e0065()
4804 its->typer |= FIELD_PREP(GITS_TYPER_ITT_ENTRY_SIZE, 16 - 1); in its_enable_quirk_qdf2400_e0065()
4811 struct its_node *its = its_dev->its; in its_irq_get_msi_base_pre_its() local
4814 * The Socionext Synquacer SoC has a so-called 'pre-ITS', in its_irq_get_msi_base_pre_its()
4815 * which maps 32-bit writes targeted at a separate window of in its_irq_get_msi_base_pre_its()
4820 return its->pre_its_base + (its_dev->device_id << 2); in its_irq_get_msi_base_pre_its()
4825 struct its_node *its = data; in its_enable_quirk_socionext_synquacer() local
4829 if (!fwnode_property_read_u32_array(its->fwnode_handle, in its_enable_quirk_socionext_synquacer()
4830 "socionext,synquacer-pre-its", in its_enable_quirk_socionext_synquacer()
4834 its->pre_its_base = pre_its_window[0]; in its_enable_quirk_socionext_synquacer()
4835 its->get_msi_base = its_irq_get_msi_base_pre_its; in its_enable_quirk_socionext_synquacer()
4837 ids = ilog2(pre_its_window[1]) - 2; in its_enable_quirk_socionext_synquacer()
4838 if (device_ids(its) > ids) { in its_enable_quirk_socionext_synquacer()
4839 its->typer &= ~GITS_TYPER_DEVBITS; in its_enable_quirk_socionext_synquacer()
4840 its->typer |= FIELD_PREP(GITS_TYPER_DEVBITS, ids - 1); in its_enable_quirk_socionext_synquacer()
4843 /* the pre-ITS breaks isolation, so disable MSI remapping */ in its_enable_quirk_socionext_synquacer()
4844 its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_ISOLATED_MSI; in its_enable_quirk_socionext_synquacer()
4852 struct its_node *its = data; in its_enable_quirk_hip07_161600802() local
4858 its->vlpi_redist_offset = SZ_128K; in its_enable_quirk_hip07_161600802()
4864 struct its_node *its = data; in its_enable_rk3588001() local
4870 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; in its_enable_rk3588001()
4871 gic_rdists->flags |= RDIST_FLAGS_FORCE_NON_SHAREABLE; in its_enable_rk3588001()
4878 struct its_node *its = data; in its_set_non_coherent() local
4880 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; in its_set_non_coherent()
4886 struct its_node *its = data; in its_enable_quirk_hip09_162100801() local
4888 its->flags |= ITS_FLAGS_WORKAROUND_HISILICON_162100801; in its_enable_quirk_hip09_162100801()
4906 .desc = "ITS: Cavium errata 22375, 24313",
4914 .desc = "ITS: Cavium erratum 23144",
4922 .desc = "ITS: QDF2400 erratum 0065",
4923 .iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
4931 * The Socionext Synquacer SoC incorporates ARM's own GIC-500
4932 * implementation, but with a 'pre-ITS' added that requires
4935 .desc = "ITS: Socionext Synquacer pre-ITS",
4943 .desc = "ITS: Hip07 erratum 161600802",
4951 .desc = "ITS: Hip09 erratum 162100801",
4959 .desc = "ITS: Rockchip erratum RK3588001",
4966 .desc = "ITS: non-coherent attribute",
4967 .property = "dma-noncoherent",
4972 .desc = "ITS: Rockchip erratum RK3568002",
4982 static void its_enable_quirks(struct its_node *its) in its_enable_quirks() argument
4984 u32 iidr = readl_relaxed(its->base + GITS_IIDR); in its_enable_quirks()
4986 gic_enable_quirks(iidr, its_quirks, its); in its_enable_quirks()
4988 if (is_of_node(its->fwnode_handle)) in its_enable_quirks()
4989 gic_enable_of_quirks(to_of_node(its->fwnode_handle), in its_enable_quirks()
4990 its_quirks, its); in its_enable_quirks()
4995 struct its_node *its; in its_save_disable() local
4999 list_for_each_entry(its, &its_nodes, entry) { in its_save_disable()
5002 base = its->base; in its_save_disable()
5003 its->ctlr_save = readl_relaxed(base + GITS_CTLR); in its_save_disable()
5006 pr_err("ITS@%pa: failed to quiesce: %d\n", in its_save_disable()
5007 &its->phys_base, err); in its_save_disable()
5008 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_save_disable()
5012 its->cbaser_save = gits_read_cbaser(base + GITS_CBASER); in its_save_disable()
5017 list_for_each_entry_continue_reverse(its, &its_nodes, entry) { in its_save_disable()
5020 base = its->base; in its_save_disable()
5021 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_save_disable()
5031 struct its_node *its; in its_restore_enable() local
5035 list_for_each_entry(its, &its_nodes, entry) { in its_restore_enable()
5039 base = its->base; in its_restore_enable()
5042 * Make sure that the ITS is disabled. If it fails to quiesce, in its_restore_enable()
5044 * registers is undefined according to the GIC v3 ITS in its_restore_enable()
5047 * Firmware resuming with the ITS enabled is terminally broken. in its_restore_enable()
5052 pr_err("ITS@%pa: failed to quiesce on resume: %d\n", in its_restore_enable()
5053 &its->phys_base, ret); in its_restore_enable()
5057 gits_write_cbaser(its->cbaser_save, base + GITS_CBASER); in its_restore_enable()
5063 its->cmd_write = its->cmd_base; in its_restore_enable()
5068 struct its_baser *baser = &its->tables[i]; in its_restore_enable()
5070 if (!(baser->val & GITS_BASER_VALID)) in its_restore_enable()
5073 its_write_baser(its, baser, baser->val); in its_restore_enable()
5075 writel_relaxed(its->ctlr_save, base + GITS_CTLR); in its_restore_enable()
5078 * Reinit the collection if it's stored in the ITS. This is in its_restore_enable()
5082 if (its->collections[smp_processor_id()].col_id < in its_restore_enable()
5084 its_cpu_init_collection(its); in its_restore_enable()
5099 its_base = ioremap(res->start, SZ_64K); in its_map_one()
5101 pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start); in its_map_one()
5102 *err = -ENOMEM; in its_map_one()
5108 pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start); in its_map_one()
5109 *err = -ENODEV; in its_map_one()
5115 pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start); in its_map_one()
5126 static int its_init_domain(struct its_node *its) in its_init_domain() argument
5133 return -ENOMEM; in its_init_domain()
5135 info->ops = &its_msi_domain_ops; in its_init_domain()
5136 info->data = its; in its_init_domain()
5139 its->msi_domain_flags, 0, in its_init_domain()
5140 its->fwnode_handle, &its_domain_ops, in its_init_domain()
5144 return -ENOMEM; in its_init_domain()
5149 inner_domain->msi_parent_ops = &gic_v3_its_msi_parent_ops; in its_init_domain()
5150 inner_domain->flags |= IRQ_DOMAIN_FLAG_MSI_PARENT; in its_init_domain()
5157 struct its_node *its; in its_init_vpe_domain() local
5161 if (gic_rdists->has_direct_lpi) { in its_init_vpe_domain()
5162 pr_info("ITS: Using DirectLPI for VPE invalidation\n"); in its_init_vpe_domain()
5166 /* Any ITS will do, even if not v4 */ in its_init_vpe_domain()
5167 its = list_first_entry(&its_nodes, struct its_node, entry); in its_init_vpe_domain()
5173 return -ENOMEM; in its_init_vpe_domain()
5176 devid = GENMASK(device_ids(its) - 1, 0); in its_init_vpe_domain()
5177 vpe_proxy.dev = its_create_device(its, devid, entries, false); in its_init_vpe_domain()
5180 pr_err("ITS: Can't allocate GICv4 proxy device\n"); in its_init_vpe_domain()
5181 return -ENOMEM; in its_init_vpe_domain()
5184 BUG_ON(entries > vpe_proxy.dev->nr_ites); in its_init_vpe_domain()
5188 pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n", in its_init_vpe_domain()
5189 devid, vpe_proxy.dev->nr_ites); in its_init_vpe_domain()
5194 static int __init its_compute_its_list_map(struct its_node *its) in its_compute_its_list_map() argument
5201 * guaranteed to be single-threaded, hence no in its_compute_its_list_map()
5207 pr_err("ITS@%pa: No ITSList entry available!\n", in its_compute_its_list_map()
5208 &its->phys_base); in its_compute_its_list_map()
5209 return -EINVAL; in its_compute_its_list_map()
5212 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_compute_its_list_map()
5215 writel_relaxed(ctlr, its->base + GITS_CTLR); in its_compute_its_list_map()
5216 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_compute_its_list_map()
5223 pr_err("ITS@%pa: Duplicate ITSList entry %d\n", in its_compute_its_list_map()
5224 &its->phys_base, its_number); in its_compute_its_list_map()
5225 return -EINVAL; in its_compute_its_list_map()
5231 static int __init its_probe_one(struct its_node *its) in its_probe_one() argument
5238 its_enable_quirks(its); in its_probe_one()
5240 if (is_v4(its)) { in its_probe_one()
5241 if (!(its->typer & GITS_TYPER_VMOVP)) { in its_probe_one()
5242 err = its_compute_its_list_map(its); in its_probe_one()
5246 its->list_nr = err; in its_probe_one()
5248 pr_info("ITS@%pa: Using ITS number %d\n", in its_probe_one()
5249 &its->phys_base, err); in its_probe_one()
5251 pr_info("ITS@%pa: Single VMOVP capable\n", &its->phys_base); in its_probe_one()
5254 if (is_v4_1(its)) { in its_probe_one()
5255 u32 svpet = FIELD_GET(GITS_TYPER_SVPET, its->typer); in its_probe_one()
5257 its->sgir_base = ioremap(its->phys_base + SZ_128K, SZ_64K); in its_probe_one()
5258 if (!its->sgir_base) { in its_probe_one()
5259 err = -ENOMEM; in its_probe_one()
5263 its->mpidr = readl_relaxed(its->base + GITS_MPIDR); in its_probe_one()
5265 pr_info("ITS@%pa: Using GICv4.1 mode %08x %08x\n", in its_probe_one()
5266 &its->phys_base, its->mpidr, svpet); in its_probe_one()
5270 page = its_alloc_pages_node(its->numa_node, in its_probe_one()
5274 err = -ENOMEM; in its_probe_one()
5277 its->cmd_base = (void *)page_address(page); in its_probe_one()
5278 its->cmd_write = its->cmd_base; in its_probe_one()
5280 err = its_alloc_tables(its); in its_probe_one()
5284 err = its_alloc_collections(its); in its_probe_one()
5288 baser = (virt_to_phys(its->cmd_base) | in its_probe_one()
5291 (ITS_CMD_QUEUE_SZ / SZ_4K - 1) | in its_probe_one()
5294 gits_write_cbaser(baser, its->base + GITS_CBASER); in its_probe_one()
5295 tmp = gits_read_cbaser(its->base + GITS_CBASER); in its_probe_one()
5297 if (its->flags & ITS_FLAGS_FORCE_NON_SHAREABLE) in its_probe_one()
5303 * The HW reports non-shareable, we must in its_probe_one()
5310 gits_write_cbaser(baser, its->base + GITS_CBASER); in its_probe_one()
5312 pr_info("ITS: using cache flushing for cmd queue\n"); in its_probe_one()
5313 its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING; in its_probe_one()
5316 gits_write_cwriter(0, its->base + GITS_CWRITER); in its_probe_one()
5317 ctlr = readl_relaxed(its->base + GITS_CTLR); in its_probe_one()
5319 if (is_v4(its)) in its_probe_one()
5321 writel_relaxed(ctlr, its->base + GITS_CTLR); in its_probe_one()
5323 err = its_init_domain(its); in its_probe_one()
5328 list_add(&its->entry, &its_nodes); in its_probe_one()
5334 its_free_tables(its); in its_probe_one()
5336 its_free_pages(its->cmd_base, get_order(ITS_CMD_QUEUE_SZ)); in its_probe_one()
5338 if (its->sgir_base) in its_probe_one()
5339 iounmap(its->sgir_base); in its_probe_one()
5341 pr_err("ITS@%pa: failed probing (%d)\n", &its->phys_base, err); in its_probe_one()
5358 return -ENXIO; in redist_disable_lpis()
5367 * LPIs before trying to re-enable them. They are already in redist_disable_lpis()
5372 if ((gic_data_rdist()->flags & RD_LOCAL_LPI_ENABLED) || in redist_disable_lpis()
5373 (gic_rdists->flags & RDIST_FLAGS_RD_TABLES_PREALLOCATED)) in redist_disable_lpis()
5399 return -ETIMEDOUT; in redist_disable_lpis()
5402 timeout--; in redist_disable_lpis()
5412 return -EBUSY; in redist_disable_lpis()
5436 cpuhp_remove_state_nocalls(gic_rdists->cpuhp_memreserve_state); in rdist_memreserve_cpuhp_cleanup_workfn()
5437 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; in rdist_memreserve_cpuhp_cleanup_workfn()
5449 if (gic_data_rdist()->flags & RD_LOCAL_MEMRESERVE_DONE) in its_cpu_memreserve_lpi()
5452 pend_page = gic_data_rdist()->pend_page; in its_cpu_memreserve_lpi()
5454 ret = -ENOMEM; in its_cpu_memreserve_lpi()
5458 * If the pending table was pre-programmed, free the memory we in its_cpu_memreserve_lpi()
5462 if (gic_data_rdist()->flags & RD_LOCAL_PENDTABLE_PREALLOCATED) { in its_cpu_memreserve_lpi()
5464 gic_data_rdist()->pend_page = NULL; in its_cpu_memreserve_lpi()
5476 gic_data_rdist()->flags |= RD_LOCAL_MEMRESERVE_DONE; in its_cpu_memreserve_lpi()
5498 { .compatible = "arm,gic-v3-its", },
5506 struct its_node *its; in its_node_init() local
5513 pr_info("ITS %pR\n", res); in its_node_init()
5515 its = kzalloc(sizeof(*its), GFP_KERNEL); in its_node_init()
5516 if (!its) in its_node_init()
5519 raw_spin_lock_init(&its->lock); in its_node_init()
5520 mutex_init(&its->dev_alloc_lock); in its_node_init()
5521 INIT_LIST_HEAD(&its->entry); in its_node_init()
5522 INIT_LIST_HEAD(&its->its_device_list); in its_node_init()
5524 its->typer = gic_read_typer(its_base + GITS_TYPER); in its_node_init()
5525 its->base = its_base; in its_node_init()
5526 its->phys_base = res->start; in its_node_init()
5527 its->get_msi_base = its_irq_get_msi_base; in its_node_init()
5528 its->msi_domain_flags = IRQ_DOMAIN_FLAG_ISOLATED_MSI; in its_node_init()
5530 its->numa_node = numa_node; in its_node_init()
5531 its->fwnode_handle = handle; in its_node_init()
5533 return its; in its_node_init()
5540 static void its_node_destroy(struct its_node *its) in its_node_destroy() argument
5542 iounmap(its->base); in its_node_destroy()
5543 kfree(its); in its_node_destroy()
5553 * Make sure *all* the ITS are reset before we probe any, as in its_of_probe()
5554 * they may be sharing memory. If any of the ITS fails to in its_of_probe()
5561 !of_property_read_bool(np, "msi-controller") || in its_of_probe()
5572 struct its_node *its; in its_of_probe() local
5576 if (!of_property_read_bool(np, "msi-controller")) { in its_of_probe()
5577 pr_warn("%pOF: no msi-controller property, ITS ignored\n", in its_of_probe()
5588 its = its_node_init(&res, &np->fwnode, of_node_to_nid(np)); in its_of_probe()
5589 if (!its) in its_of_probe()
5590 return -ENOMEM; in its_of_probe()
5592 err = its_probe_one(its); in its_of_probe()
5594 its_node_destroy(its); in its_of_probe()
5609 /* GIC ITS ID */
5641 return -EINVAL; in gic_acpi_parse_srat_its()
5643 if (its_affinity->header.length < sizeof(*its_affinity)) { in gic_acpi_parse_srat_its()
5644 pr_err("SRAT: Invalid header length %d in ITS affinity\n", in gic_acpi_parse_srat_its()
5645 its_affinity->header.length); in gic_acpi_parse_srat_its()
5646 return -EINVAL; in gic_acpi_parse_srat_its()
5654 node = pxm_to_node(its_affinity->proximity_domain); in gic_acpi_parse_srat_its()
5657 pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node); in gic_acpi_parse_srat_its()
5662 its_srat_maps[its_in_srat].its_id = its_affinity->its_id; in gic_acpi_parse_srat_its()
5664 pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n", in gic_acpi_parse_srat_its()
5665 its_affinity->proximity_domain, its_affinity->its_id, node); in gic_acpi_parse_srat_its()
5692 /* free the its_srat_maps after ITS probing */
5708 struct its_node *its; in gic_acpi_parse_madt_its() local
5714 res.start = its_entry->base_address; in gic_acpi_parse_madt_its()
5715 res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1; in gic_acpi_parse_madt_its()
5720 pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n", in gic_acpi_parse_madt_its()
5722 return -ENOMEM; in gic_acpi_parse_madt_its()
5725 err = iort_register_domain_token(its_entry->translation_id, res.start, in gic_acpi_parse_madt_its()
5728 pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n", in gic_acpi_parse_madt_its()
5729 &res.start, its_entry->translation_id); in gic_acpi_parse_madt_its()
5733 its = its_node_init(&res, dom_handle, in gic_acpi_parse_madt_its()
5734 acpi_get_its_numa_node(its_entry->translation_id)); in gic_acpi_parse_madt_its()
5735 if (!its) { in gic_acpi_parse_madt_its()
5736 err = -ENOMEM; in gic_acpi_parse_madt_its()
5741 (its_entry->flags & ACPI_MADT_ITS_NON_COHERENT)) in gic_acpi_parse_madt_its()
5742 its->flags |= ITS_FLAGS_FORCE_NON_SHAREABLE; in gic_acpi_parse_madt_its()
5744 err = its_probe_one(its); in gic_acpi_parse_madt_its()
5749 iort_deregister_domain_token(its_entry->translation_id); in gic_acpi_parse_madt_its()
5763 .start = its_entry->base_address, in its_acpi_reset()
5764 .end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1, in its_acpi_reset()
5775 * Make sure *all* the ITS are reset before we probe any, as in its_acpi_probe()
5776 * they may be sharing memory. If any of the ITS fails to in its_acpi_probe()
5800 gic_rdists->cpuhp_memreserve_state = CPUHP_INVALID; in its_lpi_memreserve_init()
5808 gic_rdists->cpuhp_memreserve_state = state; in its_lpi_memreserve_init()
5817 struct its_node *its; in its_init() local
5822 itt_pool = gen_pool_create(get_order(ITS_ITT_ALIGN), -1); in its_init()
5824 return -ENOMEM; in its_init()
5837 pr_warn("ITS: No ITS available, not enabling LPIs\n"); in its_init()
5838 return -ENXIO; in its_init()
5845 list_for_each_entry(its, &its_nodes, entry) { in its_init()
5846 has_v4 |= is_v4(its); in its_init()
5847 has_v4_1 |= is_v4_1(its); in its_init()
5851 if (WARN_ON(!has_v4_1 && rdists->has_rvpeid)) in its_init()
5852 rdists->has_rvpeid = false; in its_init()
5854 if (has_v4 & rdists->has_vlpis) { in its_init()
5864 rdists->has_vlpis = false; in its_init()
5865 pr_err("ITS: Disabling GICv4 support\n"); in its_init()