Lines Matching full:smmu
33 #include "arm-smmu-v3.h"
111 static void parse_driver_options(struct arm_smmu_device *smmu) in parse_driver_options() argument
116 if (of_property_read_bool(smmu->dev->of_node, in parse_driver_options()
118 smmu->options |= arm_smmu_options[i].opt; in parse_driver_options()
119 dev_notice(smmu->dev, "option %s\n", in parse_driver_options()
214 static void queue_poll_init(struct arm_smmu_device *smmu, in queue_poll_init() argument
219 qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV); in queue_poll_init()
371 static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu, in arm_smmu_get_cmdq() argument
376 if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq) in arm_smmu_get_cmdq()
377 cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent); in arm_smmu_get_cmdq()
379 return cmdq ?: &smmu->cmdq; in arm_smmu_get_cmdq()
382 static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu, in arm_smmu_cmdq_needs_busy_polling() argument
385 if (cmdq == &smmu->cmdq) in arm_smmu_cmdq_needs_busy_polling()
388 return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV; in arm_smmu_cmdq_needs_busy_polling()
391 static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu, in arm_smmu_cmdq_build_sync_cmd() argument
403 if (smmu->options & ARM_SMMU_OPT_MSIPOLL) { in arm_smmu_cmdq_build_sync_cmd()
409 if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) in arm_smmu_cmdq_build_sync_cmd()
413 void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu, in __arm_smmu_cmdq_skip_err() argument
432 dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons, in __arm_smmu_cmdq_skip_err()
437 dev_err(smmu->dev, "retrying command fetch\n"); in __arm_smmu_cmdq_skip_err()
459 dev_err(smmu->dev, "skipping command in error state:\n"); in __arm_smmu_cmdq_skip_err()
461 dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]); in __arm_smmu_cmdq_skip_err()
465 if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) in __arm_smmu_cmdq_skip_err()
471 static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu) in arm_smmu_cmdq_skip_err() argument
473 __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq); in arm_smmu_cmdq_skip_err()
567 * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
637 static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu, in arm_smmu_cmdq_poll_until_not_full() argument
656 queue_poll_init(smmu, &qp); in arm_smmu_cmdq_poll_until_not_full()
669 * Wait until the SMMU signals a CMD_SYNC completion MSI.
672 static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu, in __arm_smmu_cmdq_poll_until_msi() argument
680 queue_poll_init(smmu, &qp); in __arm_smmu_cmdq_poll_until_msi()
693 * Wait until the SMMU cons index passes llq->prod.
696 static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu, in __arm_smmu_cmdq_poll_until_consumed() argument
704 queue_poll_init(smmu, &qp); in __arm_smmu_cmdq_poll_until_consumed()
746 static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu, in arm_smmu_cmdq_poll_until_sync() argument
750 if (smmu->options & ARM_SMMU_OPT_MSIPOLL && in arm_smmu_cmdq_poll_until_sync()
751 !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq)) in arm_smmu_cmdq_poll_until_sync()
752 return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq); in arm_smmu_cmdq_poll_until_sync()
754 return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq); in arm_smmu_cmdq_poll_until_sync()
790 int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, in arm_smmu_cmdq_issue_cmdlist() argument
811 if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq)) in arm_smmu_cmdq_issue_cmdlist()
812 dev_err_ratelimited(smmu->dev, "CMDQ timeout\n"); in arm_smmu_cmdq_issue_cmdlist()
837 arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod); in arm_smmu_cmdq_issue_cmdlist()
853 /* 4. If we are the owner, take control of the SMMU hardware */ in arm_smmu_cmdq_issue_cmdlist()
887 ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq); in arm_smmu_cmdq_issue_cmdlist()
889 dev_err_ratelimited(smmu->dev, in arm_smmu_cmdq_issue_cmdlist()
910 static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, in __arm_smmu_cmdq_issue_cmd() argument
917 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", in __arm_smmu_cmdq_issue_cmd()
923 smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync); in __arm_smmu_cmdq_issue_cmd()
926 static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu, in arm_smmu_cmdq_issue_cmd() argument
929 return __arm_smmu_cmdq_issue_cmd(smmu, ent, false); in arm_smmu_cmdq_issue_cmd()
932 static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu, in arm_smmu_cmdq_issue_cmd_with_sync() argument
935 return __arm_smmu_cmdq_issue_cmd(smmu, ent, true); in arm_smmu_cmdq_issue_cmd_with_sync()
938 static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu, in arm_smmu_cmdq_batch_init() argument
943 cmds->cmdq = arm_smmu_get_cmdq(smmu, ent); in arm_smmu_cmdq_batch_init()
946 static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu, in arm_smmu_cmdq_batch_add() argument
952 (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC); in arm_smmu_cmdq_batch_add()
956 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, in arm_smmu_cmdq_batch_add()
958 arm_smmu_cmdq_batch_init(smmu, cmds, cmd); in arm_smmu_cmdq_batch_add()
962 arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, in arm_smmu_cmdq_batch_add()
964 arm_smmu_cmdq_batch_init(smmu, cmds, cmd); in arm_smmu_cmdq_batch_add()
969 dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n", in arm_smmu_cmdq_batch_add()
977 static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu, in arm_smmu_cmdq_batch_submit() argument
980 return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds, in arm_smmu_cmdq_batch_submit()
1009 arm_smmu_cmdq_issue_cmd(master->smmu, &cmd); in arm_smmu_page_response()
1019 void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid) in arm_smmu_tlb_inv_asid() argument
1022 .opcode = smmu->features & ARM_SMMU_FEAT_E2H ? in arm_smmu_tlb_inv_asid()
1027 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); in arm_smmu_tlb_inv_asid()
1225 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_sync_cd() local
1234 arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd); in arm_smmu_sync_cd()
1237 arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd); in arm_smmu_sync_cd()
1240 arm_smmu_cmdq_batch_submit(smmu, &cmds); in arm_smmu_sync_cd()
1279 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_alloc_cd_ptr() local
1296 *l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr), in arm_smmu_alloc_cd_ptr()
1435 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_alloc_cd_tables() local
1441 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) || in arm_smmu_alloc_cd_tables()
1447 cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size, in arm_smmu_alloc_cd_tables()
1464 cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size, in arm_smmu_alloc_cd_tables()
1483 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_free_cd_tables() local
1491 dma_free_coherent(smmu->dev, in arm_smmu_free_cd_tables()
1498 dma_free_coherent(smmu->dev, in arm_smmu_free_cd_tables()
1503 dma_free_coherent(smmu->dev, in arm_smmu_free_cd_tables()
1540 arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd); in arm_smmu_ste_writer_sync_entry()
1552 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_write_ste() local
1564 if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) { in arm_smmu_write_ste()
1571 arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd); in arm_smmu_write_ste()
1585 void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu, in arm_smmu_make_bypass_ste() argument
1593 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) in arm_smmu_make_bypass_ste()
1605 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_make_cdtable_ste() local
1620 ((smmu->features & ARM_SMMU_FEAT_STALLS && in arm_smmu_make_cdtable_ste()
1627 if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) && in arm_smmu_make_cdtable_ste()
1632 if (smmu->features & ARM_SMMU_FEAT_E2H) { in arm_smmu_make_cdtable_ste()
1668 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_make_s2_domain_ste() local
1681 if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) in arm_smmu_make_s2_domain_ste()
1710 * because the STE table has not been installed in the SMMU yet.
1723 static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_init_l2_strtab() argument
1726 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_l2_strtab()
1733 *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table), in arm_smmu_init_l2_strtab()
1736 dev_err(smmu->dev, in arm_smmu_init_l2_strtab()
1770 arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_find_master() argument
1774 lockdep_assert_held(&smmu->streams_mutex); in arm_smmu_find_master()
1776 node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key); in arm_smmu_find_master()
1783 static void arm_smmu_decode_event(struct arm_smmu_device *smmu, u64 *raw, in arm_smmu_decode_event() argument
1809 mutex_lock(&smmu->streams_mutex); in arm_smmu_decode_event()
1810 master = arm_smmu_find_master(smmu, event->sid); in arm_smmu_decode_event()
1813 mutex_unlock(&smmu->streams_mutex); in arm_smmu_decode_event()
1816 static int arm_smmu_handle_event(struct arm_smmu_device *smmu, in arm_smmu_handle_event() argument
1862 mutex_lock(&smmu->streams_mutex); in arm_smmu_handle_event()
1863 master = arm_smmu_find_master(smmu, event->sid); in arm_smmu_handle_event()
1871 mutex_unlock(&smmu->streams_mutex); in arm_smmu_handle_event()
1875 static void arm_smmu_dump_raw_event(struct arm_smmu_device *smmu, u64 *raw, in arm_smmu_dump_raw_event() argument
1880 dev_err(smmu->dev, "event 0x%02x received:\n", event->id); in arm_smmu_dump_raw_event()
1883 dev_err(smmu->dev, "\t0x%016llx\n", raw[i]); in arm_smmu_dump_raw_event()
1890 static void arm_smmu_dump_event(struct arm_smmu_device *smmu, u64 *raw, in arm_smmu_dump_event() argument
1897 arm_smmu_dump_raw_event(smmu, raw, evt); in arm_smmu_dump_event()
1904 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x iova: %#llx ipa: %#llx", in arm_smmu_dump_event()
1908 dev_err(smmu->dev, "%s %s %s %s \"%s\"%s%s stag: %#x", in arm_smmu_dump_event()
1921 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x fetch_addr: %#llx", in arm_smmu_dump_event()
1928 dev_err(smmu->dev, "event: %s client: %s sid: %#x ssid: %#x", in arm_smmu_dump_event()
1938 struct arm_smmu_device *smmu = dev; in arm_smmu_evtq_thread() local
1939 struct arm_smmu_queue *q = &smmu->evtq.q; in arm_smmu_evtq_thread()
1946 arm_smmu_decode_event(smmu, evt, &event); in arm_smmu_evtq_thread()
1947 if (arm_smmu_handle_event(smmu, &event)) in arm_smmu_evtq_thread()
1948 arm_smmu_dump_event(smmu, evt, &event, &rs); in arm_smmu_evtq_thread()
1959 dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n"); in arm_smmu_evtq_thread()
1967 static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt) in arm_smmu_handle_ppr() argument
1979 dev_info(smmu->dev, "unexpected PRI request received:\n"); in arm_smmu_handle_ppr()
1980 dev_info(smmu->dev, in arm_smmu_handle_ppr()
2001 arm_smmu_cmdq_issue_cmd(smmu, &cmd); in arm_smmu_handle_ppr()
2007 struct arm_smmu_device *smmu = dev; in arm_smmu_priq_thread() local
2008 struct arm_smmu_queue *q = &smmu->priq.q; in arm_smmu_priq_thread()
2014 arm_smmu_handle_ppr(smmu, evt); in arm_smmu_priq_thread()
2017 dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n"); in arm_smmu_priq_thread()
2025 static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
2030 struct arm_smmu_device *smmu = dev; in arm_smmu_gerror_handler() local
2032 gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR); in arm_smmu_gerror_handler()
2033 gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN); in arm_smmu_gerror_handler()
2039 dev_warn(smmu->dev, in arm_smmu_gerror_handler()
2044 dev_err(smmu->dev, "device has entered Service Failure Mode!\n"); in arm_smmu_gerror_handler()
2045 arm_smmu_device_disable(smmu); in arm_smmu_gerror_handler()
2049 dev_warn(smmu->dev, "GERROR MSI write aborted\n"); in arm_smmu_gerror_handler()
2052 dev_warn(smmu->dev, "PRIQ MSI write aborted\n"); in arm_smmu_gerror_handler()
2055 dev_warn(smmu->dev, "EVTQ MSI write aborted\n"); in arm_smmu_gerror_handler()
2058 dev_warn(smmu->dev, "CMDQ MSI write aborted\n"); in arm_smmu_gerror_handler()
2061 dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n"); in arm_smmu_gerror_handler()
2064 dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n"); in arm_smmu_gerror_handler()
2067 arm_smmu_cmdq_skip_err(smmu); in arm_smmu_gerror_handler()
2069 writel(gerror, smmu->base + ARM_SMMU_GERRORN); in arm_smmu_gerror_handler()
2075 struct arm_smmu_device *smmu = dev; in arm_smmu_combined_irq_thread() local
2078 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_combined_irq_thread()
2166 arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd); in arm_smmu_atc_inv_master()
2169 arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd); in arm_smmu_atc_inv_master()
2172 return arm_smmu_cmdq_batch_submit(master->smmu, &cmds); in arm_smmu_atc_inv_master()
2186 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS)) in arm_smmu_atc_inv_domain()
2206 arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd); in arm_smmu_atc_inv_domain()
2229 arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd); in arm_smmu_atc_inv_domain()
2234 return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds); in arm_smmu_atc_inv_domain()
2241 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_tlb_inv_context() local
2247 * to the SMMU. We are relying on the dma_wmb() implicit during cmd in arm_smmu_tlb_inv_context()
2252 arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid); in arm_smmu_tlb_inv_context()
2256 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); in arm_smmu_tlb_inv_context()
2266 struct arm_smmu_device *smmu = smmu_domain->smmu; in __arm_smmu_tlb_inv_range() local
2274 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { in __arm_smmu_tlb_inv_range()
2297 arm_smmu_cmdq_batch_init(smmu, &cmds, cmd); in __arm_smmu_tlb_inv_range()
2300 if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) { in __arm_smmu_tlb_inv_range()
2326 arm_smmu_cmdq_batch_add(smmu, &cmds, cmd); in __arm_smmu_tlb_inv_range()
2329 arm_smmu_cmdq_batch_submit(smmu, &cmds); in __arm_smmu_tlb_inv_range()
2343 cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? in arm_smmu_tlb_inv_range_domain()
2358 arm_smmu_cmdq_issue_cmd_with_sync(smmu_domain->smmu, &cmd); in arm_smmu_tlb_inv_range_domain()
2373 .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ? in arm_smmu_tlb_inv_range_asid()
2406 static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu) in arm_smmu_dbm_capable() argument
2410 return (smmu->features & features) == features; in arm_smmu_dbm_capable()
2421 return master->smmu->features & ARM_SMMU_FEAT_COHERENCY; in arm_smmu_capable()
2428 return arm_smmu_dbm_capable(master->smmu); in arm_smmu_capable()
2471 struct arm_smmu_device *smmu = smmu_domain->smmu; in arm_smmu_domain_free_paging() local
2484 ida_free(&smmu->vmid_map, cfg->vmid); in arm_smmu_domain_free_paging()
2490 static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu, in arm_smmu_domain_finalise_s1() argument
2500 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); in arm_smmu_domain_finalise_s1()
2506 static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu, in arm_smmu_domain_finalise_s2() argument
2513 vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1, in arm_smmu_domain_finalise_s2()
2523 struct arm_smmu_device *smmu, u32 flags) in arm_smmu_domain_finalise() argument
2529 int (*finalise_stage_fn)(struct arm_smmu_device *smmu, in arm_smmu_domain_finalise()
2534 .pgsize_bitmap = smmu->pgsize_bitmap, in arm_smmu_domain_finalise()
2535 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY, in arm_smmu_domain_finalise()
2537 .iommu_dev = smmu->dev, in arm_smmu_domain_finalise()
2542 unsigned long ias = (smmu->features & in arm_smmu_domain_finalise()
2546 pgtbl_cfg.oas = smmu->ias; in arm_smmu_domain_finalise()
2556 pgtbl_cfg.ias = smmu->ias; in arm_smmu_domain_finalise()
2557 pgtbl_cfg.oas = smmu->oas; in arm_smmu_domain_finalise()
2560 if ((smmu->features & ARM_SMMU_FEAT_S2FWB) && in arm_smmu_domain_finalise()
2578 ret = finalise_stage_fn(smmu, smmu_domain); in arm_smmu_domain_finalise()
2585 smmu_domain->smmu = smmu; in arm_smmu_domain_finalise()
2590 arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_get_step_for_sid() argument
2592 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_get_step_for_sid()
2594 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_get_step_for_sid()
2608 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_install_ste_for_dev() local
2620 arm_smmu_get_step_for_sid(smmu, sid); in arm_smmu_install_ste_for_dev()
2636 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_ats_supported() local
2639 if (!(smmu->features & ARM_SMMU_FEAT_ATS)) in arm_smmu_ats_supported()
2652 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_enable_ats() local
2655 stu = __ffs(smmu->pgsize_bitmap); in arm_smmu_enable_ats()
2693 master->smmu->ssid_bits); in arm_smmu_enable_pasid()
2816 * The SMMU does not support enabling ATS with bypass/abort. in arm_smmu_attach_prepare()
2901 * SMMU is translating for the new domain and both the old&new in arm_smmu_attach_commit()
2919 struct arm_smmu_device *smmu; in arm_smmu_attach_dev() local
2932 smmu = master->smmu; in arm_smmu_attach_dev()
2934 if (smmu_domain->smmu != smmu) in arm_smmu_attach_dev()
2989 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_s1_set_dev_pasid() local
2992 if (smmu_domain->smmu != smmu) in arm_smmu_s1_set_dev_pasid()
3047 if (smmu_domain->smmu != master->smmu) in arm_smmu_set_pasid()
3165 arm_smmu_make_bypass_ste(master->smmu, &ste); in arm_smmu_attach_dev_identity()
3205 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_domain_alloc_paging_flags() local
3224 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1) in arm_smmu_domain_alloc_paging_flags()
3230 if (!(smmu->features & ARM_SMMU_FEAT_NESTING)) { in arm_smmu_domain_alloc_paging_flags()
3240 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1)) { in arm_smmu_domain_alloc_paging_flags()
3253 ret = arm_smmu_domain_finalise(smmu_domain, smmu, flags); in arm_smmu_domain_alloc_paging_flags()
3292 if (smmu_domain->smmu) in arm_smmu_flush_iotlb_all()
3331 static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_sid_in_range() argument
3333 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_sid_in_range()
3334 return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents; in arm_smmu_sid_in_range()
3335 return sid < smmu->strtab_cfg.linear.num_ents; in arm_smmu_sid_in_range()
3338 static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid) in arm_smmu_init_sid_strtab() argument
3340 /* Check the SIDs are in range of the SMMU and our stream table */ in arm_smmu_init_sid_strtab()
3341 if (!arm_smmu_sid_in_range(smmu, sid)) in arm_smmu_init_sid_strtab()
3345 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_init_sid_strtab()
3346 return arm_smmu_init_l2_strtab(smmu, sid); in arm_smmu_init_sid_strtab()
3351 static int arm_smmu_insert_master(struct arm_smmu_device *smmu, in arm_smmu_insert_master() argument
3364 mutex_lock(&smmu->streams_mutex); in arm_smmu_insert_master()
3372 ret = arm_smmu_init_sid_strtab(smmu, sid); in arm_smmu_insert_master()
3377 if (rb_find_add(&new_stream->node, &smmu->streams, in arm_smmu_insert_master()
3388 rb_erase(&master->streams[i].node, &smmu->streams); in arm_smmu_insert_master()
3391 mutex_unlock(&smmu->streams_mutex); in arm_smmu_insert_master()
3399 struct arm_smmu_device *smmu = master->smmu; in arm_smmu_remove_master() local
3402 if (!smmu || !master->streams) in arm_smmu_remove_master()
3405 mutex_lock(&smmu->streams_mutex); in arm_smmu_remove_master()
3407 rb_erase(&master->streams[i].node, &smmu->streams); in arm_smmu_remove_master()
3408 mutex_unlock(&smmu->streams_mutex); in arm_smmu_remove_master()
3416 struct arm_smmu_device *smmu; in arm_smmu_probe_device() local
3423 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode); in arm_smmu_probe_device()
3424 if (!smmu) in arm_smmu_probe_device()
3432 master->smmu = smmu; in arm_smmu_probe_device()
3435 ret = arm_smmu_insert_master(smmu, master); in arm_smmu_probe_device()
3440 master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits); in arm_smmu_probe_device()
3452 if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB)) in arm_smmu_probe_device()
3456 if ((smmu->features & ARM_SMMU_FEAT_STALLS && in arm_smmu_probe_device()
3458 smmu->features & ARM_SMMU_FEAT_STALL_FORCE) in arm_smmu_probe_device()
3462 unsigned int stu = __ffs(smmu->pgsize_bitmap); in arm_smmu_probe_device()
3467 return &smmu->iommu; in arm_smmu_probe_device()
3479 iopf_queue_remove_device(master->smmu->evtq.iopf, dev); in arm_smmu_release_device()
3665 int arm_smmu_init_one_queue(struct arm_smmu_device *smmu, in arm_smmu_init_one_queue() argument
3674 q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma, in arm_smmu_init_one_queue()
3683 dev_err(smmu->dev, in arm_smmu_init_one_queue()
3690 dev_info(smmu->dev, "allocated %u entries for %s\n", in arm_smmu_init_one_queue()
3706 int arm_smmu_cmdq_init(struct arm_smmu_device *smmu, in arm_smmu_cmdq_init() argument
3714 cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents, in arm_smmu_cmdq_init()
3722 static int arm_smmu_init_queues(struct arm_smmu_device *smmu) in arm_smmu_init_queues() argument
3727 ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base, in arm_smmu_init_queues()
3733 ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq); in arm_smmu_init_queues()
3738 ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1, in arm_smmu_init_queues()
3744 if ((smmu->features & ARM_SMMU_FEAT_SVA) && in arm_smmu_init_queues()
3745 (smmu->features & ARM_SMMU_FEAT_STALLS)) { in arm_smmu_init_queues()
3746 smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev)); in arm_smmu_init_queues()
3747 if (!smmu->evtq.iopf) in arm_smmu_init_queues()
3752 if (!(smmu->features & ARM_SMMU_FEAT_PRI)) in arm_smmu_init_queues()
3755 return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1, in arm_smmu_init_queues()
3760 static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) in arm_smmu_init_strtab_2lvl() argument
3763 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_strtab_2lvl()
3765 arm_smmu_strtab_l1_idx((1ULL << smmu->sid_bits) - 1); in arm_smmu_init_strtab_2lvl()
3770 dev_warn(smmu->dev, in arm_smmu_init_strtab_2lvl()
3773 smmu->sid_bits); in arm_smmu_init_strtab_2lvl()
3776 cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma, in arm_smmu_init_strtab_2lvl()
3779 dev_err(smmu->dev, in arm_smmu_init_strtab_2lvl()
3785 cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents, in arm_smmu_init_strtab_2lvl()
3793 static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) in arm_smmu_init_strtab_linear() argument
3796 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_init_strtab_linear()
3798 size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste); in arm_smmu_init_strtab_linear()
3799 cfg->linear.table = dmam_alloc_coherent(smmu->dev, size, in arm_smmu_init_strtab_linear()
3803 dev_err(smmu->dev, in arm_smmu_init_strtab_linear()
3808 cfg->linear.num_ents = 1 << smmu->sid_bits; in arm_smmu_init_strtab_linear()
3814 static int arm_smmu_init_strtab(struct arm_smmu_device *smmu) in arm_smmu_init_strtab() argument
3818 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) in arm_smmu_init_strtab()
3819 ret = arm_smmu_init_strtab_2lvl(smmu); in arm_smmu_init_strtab()
3821 ret = arm_smmu_init_strtab_linear(smmu); in arm_smmu_init_strtab()
3825 ida_init(&smmu->vmid_map); in arm_smmu_init_strtab()
3830 static int arm_smmu_init_structures(struct arm_smmu_device *smmu) in arm_smmu_init_structures() argument
3834 mutex_init(&smmu->streams_mutex); in arm_smmu_init_structures()
3835 smmu->streams = RB_ROOT; in arm_smmu_init_structures()
3837 ret = arm_smmu_init_queues(smmu); in arm_smmu_init_structures()
3841 ret = arm_smmu_init_strtab(smmu); in arm_smmu_init_structures()
3845 if (smmu->impl_ops && smmu->impl_ops->init_structures) in arm_smmu_init_structures()
3846 return smmu->impl_ops->init_structures(smmu); in arm_smmu_init_structures()
3851 static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, in arm_smmu_write_reg_sync() argument
3856 writel_relaxed(val, smmu->base + reg_off); in arm_smmu_write_reg_sync()
3857 return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val, in arm_smmu_write_reg_sync()
3862 static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr) in arm_smmu_update_gbpa() argument
3865 u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA; in arm_smmu_update_gbpa()
3879 dev_err(smmu->dev, "GBPA not responding to update\n"); in arm_smmu_update_gbpa()
3894 struct arm_smmu_device *smmu = dev_get_drvdata(dev); in arm_smmu_write_msi_msg() local
3900 writeq_relaxed(doorbell, smmu->base + cfg[0]); in arm_smmu_write_msi_msg()
3901 writel_relaxed(msg->data, smmu->base + cfg[1]); in arm_smmu_write_msi_msg()
3902 writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]); in arm_smmu_write_msi_msg()
3905 static void arm_smmu_setup_msis(struct arm_smmu_device *smmu) in arm_smmu_setup_msis() argument
3908 struct device *dev = smmu->dev; in arm_smmu_setup_msis()
3911 writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0); in arm_smmu_setup_msis()
3912 writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0); in arm_smmu_setup_msis()
3914 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_setup_msis()
3915 writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0); in arm_smmu_setup_msis()
3919 if (!(smmu->features & ARM_SMMU_FEAT_MSI)) in arm_smmu_setup_msis()
3923 dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n"); in arm_smmu_setup_msis()
3934 smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX); in arm_smmu_setup_msis()
3935 smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX); in arm_smmu_setup_msis()
3936 smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX); in arm_smmu_setup_msis()
3942 static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) in arm_smmu_setup_unique_irqs() argument
3946 arm_smmu_setup_msis(smmu); in arm_smmu_setup_unique_irqs()
3949 irq = smmu->evtq.q.irq; in arm_smmu_setup_unique_irqs()
3951 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, in arm_smmu_setup_unique_irqs()
3954 "arm-smmu-v3-evtq", smmu); in arm_smmu_setup_unique_irqs()
3956 dev_warn(smmu->dev, "failed to enable evtq irq\n"); in arm_smmu_setup_unique_irqs()
3958 dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n"); in arm_smmu_setup_unique_irqs()
3961 irq = smmu->gerr_irq; in arm_smmu_setup_unique_irqs()
3963 ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler, in arm_smmu_setup_unique_irqs()
3964 0, "arm-smmu-v3-gerror", smmu); in arm_smmu_setup_unique_irqs()
3966 dev_warn(smmu->dev, "failed to enable gerror irq\n"); in arm_smmu_setup_unique_irqs()
3968 dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n"); in arm_smmu_setup_unique_irqs()
3971 if (smmu->features & ARM_SMMU_FEAT_PRI) { in arm_smmu_setup_unique_irqs()
3972 irq = smmu->priq.q.irq; in arm_smmu_setup_unique_irqs()
3974 ret = devm_request_threaded_irq(smmu->dev, irq, NULL, in arm_smmu_setup_unique_irqs()
3977 "arm-smmu-v3-priq", in arm_smmu_setup_unique_irqs()
3978 smmu); in arm_smmu_setup_unique_irqs()
3980 dev_warn(smmu->dev, in arm_smmu_setup_unique_irqs()
3983 dev_warn(smmu->dev, "no priq irq - PRI will be broken\n"); in arm_smmu_setup_unique_irqs()
3988 static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu) in arm_smmu_setup_irqs() argument
3994 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL, in arm_smmu_setup_irqs()
3997 dev_err(smmu->dev, "failed to disable irqs\n"); in arm_smmu_setup_irqs()
4001 irq = smmu->combined_irq; in arm_smmu_setup_irqs()
4007 ret = devm_request_threaded_irq(smmu->dev, irq, in arm_smmu_setup_irqs()
4011 "arm-smmu-v3-combined-irq", smmu); in arm_smmu_setup_irqs()
4013 dev_warn(smmu->dev, "failed to enable combined irq\n"); in arm_smmu_setup_irqs()
4015 arm_smmu_setup_unique_irqs(smmu); in arm_smmu_setup_irqs()
4017 if (smmu->features & ARM_SMMU_FEAT_PRI) in arm_smmu_setup_irqs()
4020 /* Enable interrupt generation on the SMMU */ in arm_smmu_setup_irqs()
4021 ret = arm_smmu_write_reg_sync(smmu, irqen_flags, in arm_smmu_setup_irqs()
4024 dev_warn(smmu->dev, "failed to enable irqs\n"); in arm_smmu_setup_irqs()
4029 static int arm_smmu_device_disable(struct arm_smmu_device *smmu) in arm_smmu_device_disable() argument
4033 ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK); in arm_smmu_device_disable()
4035 dev_err(smmu->dev, "failed to clear cr0\n"); in arm_smmu_device_disable()
4040 static void arm_smmu_write_strtab(struct arm_smmu_device *smmu) in arm_smmu_write_strtab() argument
4042 struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg; in arm_smmu_write_strtab()
4046 if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { in arm_smmu_write_strtab()
4056 FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); in arm_smmu_write_strtab()
4060 smmu->base + ARM_SMMU_STRTAB_BASE); in arm_smmu_write_strtab()
4061 writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG); in arm_smmu_write_strtab()
4064 static int arm_smmu_device_reset(struct arm_smmu_device *smmu) in arm_smmu_device_reset() argument
4070 /* Clear CR0 and sync (disables SMMU and queue processing) */ in arm_smmu_device_reset()
4071 reg = readl_relaxed(smmu->base + ARM_SMMU_CR0); in arm_smmu_device_reset()
4073 dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n"); in arm_smmu_device_reset()
4074 arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0); in arm_smmu_device_reset()
4077 ret = arm_smmu_device_disable(smmu); in arm_smmu_device_reset()
4088 writel_relaxed(reg, smmu->base + ARM_SMMU_CR1); in arm_smmu_device_reset()
4093 if (smmu->features & ARM_SMMU_FEAT_E2H) in arm_smmu_device_reset()
4096 writel_relaxed(reg, smmu->base + ARM_SMMU_CR2); in arm_smmu_device_reset()
4099 arm_smmu_write_strtab(smmu); in arm_smmu_device_reset()
4102 writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE); in arm_smmu_device_reset()
4103 writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD); in arm_smmu_device_reset()
4104 writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS); in arm_smmu_device_reset()
4107 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
4110 dev_err(smmu->dev, "failed to enable command queue\n"); in arm_smmu_device_reset()
4116 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); in arm_smmu_device_reset()
4119 if (smmu->features & ARM_SMMU_FEAT_HYP) { in arm_smmu_device_reset()
4121 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); in arm_smmu_device_reset()
4125 arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd); in arm_smmu_device_reset()
4128 writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE); in arm_smmu_device_reset()
4129 writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD); in arm_smmu_device_reset()
4130 writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS); in arm_smmu_device_reset()
4133 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
4136 dev_err(smmu->dev, "failed to enable event queue\n"); in arm_smmu_device_reset()
4141 if (smmu->features & ARM_SMMU_FEAT_PRI) { in arm_smmu_device_reset()
4142 writeq_relaxed(smmu->priq.q.q_base, in arm_smmu_device_reset()
4143 smmu->base + ARM_SMMU_PRIQ_BASE); in arm_smmu_device_reset()
4144 writel_relaxed(smmu->priq.q.llq.prod, in arm_smmu_device_reset()
4145 smmu->page1 + ARM_SMMU_PRIQ_PROD); in arm_smmu_device_reset()
4146 writel_relaxed(smmu->priq.q.llq.cons, in arm_smmu_device_reset()
4147 smmu->page1 + ARM_SMMU_PRIQ_CONS); in arm_smmu_device_reset()
4150 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
4153 dev_err(smmu->dev, "failed to enable PRI queue\n"); in arm_smmu_device_reset()
4158 if (smmu->features & ARM_SMMU_FEAT_ATS) { in arm_smmu_device_reset()
4160 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
4163 dev_err(smmu->dev, "failed to enable ATS check\n"); in arm_smmu_device_reset()
4168 ret = arm_smmu_setup_irqs(smmu); in arm_smmu_device_reset()
4170 dev_err(smmu->dev, "failed to setup irqs\n"); in arm_smmu_device_reset()
4177 /* Enable the SMMU interface */ in arm_smmu_device_reset()
4179 ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0, in arm_smmu_device_reset()
4182 dev_err(smmu->dev, "failed to enable SMMU interface\n"); in arm_smmu_device_reset()
4186 if (smmu->impl_ops && smmu->impl_ops->device_reset) { in arm_smmu_device_reset()
4187 ret = smmu->impl_ops->device_reset(smmu); in arm_smmu_device_reset()
4189 dev_err(smmu->dev, "failed to reset impl\n"); in arm_smmu_device_reset()
4201 static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu) in arm_smmu_device_iidr_probe() argument
4206 reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR); in arm_smmu_device_iidr_probe()
4218 smmu->features &= ~ARM_SMMU_FEAT_SEV; in arm_smmu_device_iidr_probe()
4221 smmu->features &= ~ARM_SMMU_FEAT_NESTING; in arm_smmu_device_iidr_probe()
4225 smmu->features &= ~ARM_SMMU_FEAT_BTM; in arm_smmu_device_iidr_probe()
4226 smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC; in arm_smmu_device_iidr_probe()
4228 smmu->features &= ~ARM_SMMU_FEAT_NESTING; in arm_smmu_device_iidr_probe()
4235 static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg) in arm_smmu_get_httu() argument
4237 u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD); in arm_smmu_get_httu()
4248 if (smmu->dev->of_node) in arm_smmu_get_httu()
4249 smmu->features |= hw_features; in arm_smmu_get_httu()
4252 dev_warn(smmu->dev, in arm_smmu_get_httu()
4257 static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu) in arm_smmu_device_hw_probe() argument
4260 bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_hw_probe()
4263 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0); in arm_smmu_device_hw_probe()
4267 smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB; in arm_smmu_device_hw_probe()
4270 smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB; in arm_smmu_device_hw_probe()
4279 smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE; in arm_smmu_device_hw_probe()
4283 smmu->features |= ARM_SMMU_FEAT_TT_BE; in arm_smmu_device_hw_probe()
4287 smmu->features |= ARM_SMMU_FEAT_TT_LE; in arm_smmu_device_hw_probe()
4291 dev_err(smmu->dev, "unknown/unsupported TT endianness!\n"); in arm_smmu_device_hw_probe()
4297 smmu->features |= ARM_SMMU_FEAT_PRI; in arm_smmu_device_hw_probe()
4300 smmu->features |= ARM_SMMU_FEAT_ATS; in arm_smmu_device_hw_probe()
4303 smmu->features |= ARM_SMMU_FEAT_SEV; in arm_smmu_device_hw_probe()
4306 smmu->features |= ARM_SMMU_FEAT_MSI; in arm_smmu_device_hw_probe()
4308 smmu->options |= ARM_SMMU_OPT_MSIPOLL; in arm_smmu_device_hw_probe()
4312 smmu->features |= ARM_SMMU_FEAT_HYP; in arm_smmu_device_hw_probe()
4314 smmu->features |= ARM_SMMU_FEAT_E2H; in arm_smmu_device_hw_probe()
4317 arm_smmu_get_httu(smmu, reg); in arm_smmu_device_hw_probe()
4324 dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n", in arm_smmu_device_hw_probe()
4329 smmu->features |= ARM_SMMU_FEAT_STALL_FORCE; in arm_smmu_device_hw_probe()
4332 smmu->features |= ARM_SMMU_FEAT_STALLS; in arm_smmu_device_hw_probe()
4336 smmu->features |= ARM_SMMU_FEAT_TRANS_S1; in arm_smmu_device_hw_probe()
4339 smmu->features |= ARM_SMMU_FEAT_TRANS_S2; in arm_smmu_device_hw_probe()
4342 dev_err(smmu->dev, "no translation support!\n"); in arm_smmu_device_hw_probe()
4349 smmu->ias = 40; in arm_smmu_device_hw_probe()
4354 dev_err(smmu->dev, "AArch64 table format not supported!\n"); in arm_smmu_device_hw_probe()
4359 smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8; in arm_smmu_device_hw_probe()
4360 smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8; in arm_smmu_device_hw_probe()
4363 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1); in arm_smmu_device_hw_probe()
4365 dev_err(smmu->dev, "embedded implementation not supported\n"); in arm_smmu_device_hw_probe()
4370 smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR; in arm_smmu_device_hw_probe()
4373 smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
4375 if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) { in arm_smmu_device_hw_probe()
4382 dev_err(smmu->dev, "command queue size <= %d entries not supported\n", in arm_smmu_device_hw_probe()
4387 smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
4389 smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT, in arm_smmu_device_hw_probe()
4393 smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg); in arm_smmu_device_hw_probe()
4394 smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg); in arm_smmu_device_hw_probe()
4395 smmu->iommu.max_pasids = 1UL << smmu->ssid_bits; in arm_smmu_device_hw_probe()
4398 * If the SMMU supports fewer bits than would fill a single L2 stream in arm_smmu_device_hw_probe()
4401 if (smmu->sid_bits <= STRTAB_SPLIT) in arm_smmu_device_hw_probe()
4402 smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB; in arm_smmu_device_hw_probe()
4405 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3); in arm_smmu_device_hw_probe()
4407 smmu->features |= ARM_SMMU_FEAT_RANGE_INV; in arm_smmu_device_hw_probe()
4410 reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5); in arm_smmu_device_hw_probe()
4413 smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg); in arm_smmu_device_hw_probe()
4417 smmu->pgsize_bitmap |= SZ_64K | SZ_512M; in arm_smmu_device_hw_probe()
4419 smmu->pgsize_bitmap |= SZ_16K | SZ_32M; in arm_smmu_device_hw_probe()
4421 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G; in arm_smmu_device_hw_probe()
4425 smmu->features |= ARM_SMMU_FEAT_VAX; in arm_smmu_device_hw_probe()
4430 smmu->oas = 32; in arm_smmu_device_hw_probe()
4433 smmu->oas = 36; in arm_smmu_device_hw_probe()
4436 smmu->oas = 40; in arm_smmu_device_hw_probe()
4439 smmu->oas = 42; in arm_smmu_device_hw_probe()
4442 smmu->oas = 44; in arm_smmu_device_hw_probe()
4445 smmu->oas = 52; in arm_smmu_device_hw_probe()
4446 smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */ in arm_smmu_device_hw_probe()
4449 dev_info(smmu->dev, in arm_smmu_device_hw_probe()
4453 smmu->oas = 48; in arm_smmu_device_hw_probe()
4457 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap; in arm_smmu_device_hw_probe()
4459 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap; in arm_smmu_device_hw_probe()
4462 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas))) in arm_smmu_device_hw_probe()
4463 dev_warn(smmu->dev, in arm_smmu_device_hw_probe()
4466 smmu->ias = max(smmu->ias, smmu->oas); in arm_smmu_device_hw_probe()
4468 if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) && in arm_smmu_device_hw_probe()
4469 (smmu->features & ARM_SMMU_FEAT_TRANS_S2)) in arm_smmu_device_hw_probe()
4470 smmu->features |= ARM_SMMU_FEAT_NESTING; in arm_smmu_device_hw_probe()
4472 arm_smmu_device_iidr_probe(smmu); in arm_smmu_device_hw_probe()
4474 if (arm_smmu_sva_supported(smmu)) in arm_smmu_device_hw_probe()
4475 smmu->features |= ARM_SMMU_FEAT_SVA; in arm_smmu_device_hw_probe()
4477 dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n", in arm_smmu_device_hw_probe()
4478 smmu->ias, smmu->oas, smmu->features); in arm_smmu_device_hw_probe()
4485 struct arm_smmu_device *smmu) in acpi_smmu_dsdt_probe_tegra241_cmdqv() argument
4490 /* Look for an NVDA200C node whose _UID matches the SMMU node ID */ in acpi_smmu_dsdt_probe_tegra241_cmdqv()
4494 smmu->impl_dev = &adev->dev; in acpi_smmu_dsdt_probe_tegra241_cmdqv()
4495 smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV; in acpi_smmu_dsdt_probe_tegra241_cmdqv()
4496 dev_info(smmu->dev, "found companion CMDQV device: %s\n", in acpi_smmu_dsdt_probe_tegra241_cmdqv()
4497 dev_name(smmu->impl_dev)); in acpi_smmu_dsdt_probe_tegra241_cmdqv()
4503 struct arm_smmu_device *smmu) in acpi_smmu_dsdt_probe_tegra241_cmdqv() argument
4509 struct arm_smmu_device *smmu) in acpi_smmu_iort_probe_model() argument
4516 smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY; in acpi_smmu_iort_probe_model()
4519 smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH; in acpi_smmu_iort_probe_model()
4523 * Tegra241 implementation stores its SMMU options and impl_dev in acpi_smmu_iort_probe_model()
4526 acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu); in acpi_smmu_iort_probe_model()
4530 dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options); in acpi_smmu_iort_probe_model()
4535 struct arm_smmu_device *smmu) in arm_smmu_device_acpi_probe() argument
4538 struct device *dev = smmu->dev; in arm_smmu_device_acpi_probe()
4547 smmu->features |= ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_acpi_probe()
4551 smmu->features |= ARM_SMMU_FEAT_HD; in arm_smmu_device_acpi_probe()
4554 smmu->features |= ARM_SMMU_FEAT_HA; in arm_smmu_device_acpi_probe()
4557 return acpi_smmu_iort_probe_model(node, smmu); in arm_smmu_device_acpi_probe()
4561 struct arm_smmu_device *smmu) in arm_smmu_device_acpi_probe() argument
4568 struct arm_smmu_device *smmu) in arm_smmu_device_dt_probe() argument
4581 parse_driver_options(smmu); in arm_smmu_device_dt_probe()
4584 smmu->features |= ARM_SMMU_FEAT_COHERENCY; in arm_smmu_device_dt_probe()
4589 static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu) in arm_smmu_resource_size() argument
4591 if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY) in arm_smmu_resource_size()
4605 static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu) in arm_smmu_rmr_install_bypass_ste() argument
4611 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); in arm_smmu_rmr_install_bypass_ste()
4619 ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]); in arm_smmu_rmr_install_bypass_ste()
4621 dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n", in arm_smmu_rmr_install_bypass_ste()
4630 arm_smmu_make_bypass_ste(smmu, in arm_smmu_rmr_install_bypass_ste()
4631 arm_smmu_get_step_for_sid(smmu, rmr->sids[i])); in arm_smmu_rmr_install_bypass_ste()
4635 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list); in arm_smmu_rmr_install_bypass_ste()
4640 struct arm_smmu_device *smmu = data; in arm_smmu_impl_remove() local
4642 if (smmu->impl_ops && smmu->impl_ops->device_remove) in arm_smmu_impl_remove()
4643 smmu->impl_ops->device_remove(smmu); in arm_smmu_impl_remove()
4651 static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu) in arm_smmu_impl_probe() argument
4656 if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV)) in arm_smmu_impl_probe()
4657 new_smmu = tegra241_cmdqv_probe(smmu); in arm_smmu_impl_probe()
4660 return smmu; in arm_smmu_impl_probe()
4676 struct arm_smmu_device *smmu; in arm_smmu_device_probe() local
4679 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL); in arm_smmu_device_probe()
4680 if (!smmu) in arm_smmu_device_probe()
4682 smmu->dev = dev; in arm_smmu_device_probe()
4685 ret = arm_smmu_device_dt_probe(pdev, smmu); in arm_smmu_device_probe()
4687 ret = arm_smmu_device_acpi_probe(pdev, smmu); in arm_smmu_device_probe()
4692 smmu = arm_smmu_impl_probe(smmu); in arm_smmu_device_probe()
4693 if (IS_ERR(smmu)) in arm_smmu_device_probe()
4694 return PTR_ERR(smmu); in arm_smmu_device_probe()
4700 if (resource_size(res) < arm_smmu_resource_size(smmu)) { in arm_smmu_device_probe()
4710 smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ); in arm_smmu_device_probe()
4711 if (IS_ERR(smmu->base)) in arm_smmu_device_probe()
4712 return PTR_ERR(smmu->base); in arm_smmu_device_probe()
4714 if (arm_smmu_resource_size(smmu) > SZ_64K) { in arm_smmu_device_probe()
4715 smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K, in arm_smmu_device_probe()
4717 if (IS_ERR(smmu->page1)) in arm_smmu_device_probe()
4718 return PTR_ERR(smmu->page1); in arm_smmu_device_probe()
4720 smmu->page1 = smmu->base; in arm_smmu_device_probe()
4727 smmu->combined_irq = irq; in arm_smmu_device_probe()
4731 smmu->evtq.q.irq = irq; in arm_smmu_device_probe()
4735 smmu->priq.q.irq = irq; in arm_smmu_device_probe()
4739 smmu->gerr_irq = irq; in arm_smmu_device_probe()
4742 ret = arm_smmu_device_hw_probe(smmu); in arm_smmu_device_probe()
4747 ret = arm_smmu_init_structures(smmu); in arm_smmu_device_probe()
4752 platform_set_drvdata(pdev, smmu); in arm_smmu_device_probe()
4755 arm_smmu_rmr_install_bypass_ste(smmu); in arm_smmu_device_probe()
4758 ret = arm_smmu_device_reset(smmu); in arm_smmu_device_probe()
4763 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, in arm_smmu_device_probe()
4768 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev); in arm_smmu_device_probe()
4777 iommu_device_sysfs_remove(&smmu->iommu); in arm_smmu_device_probe()
4779 arm_smmu_device_disable(smmu); in arm_smmu_device_probe()
4781 iopf_queue_free(smmu->evtq.iopf); in arm_smmu_device_probe()
4787 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); in arm_smmu_device_remove() local
4789 iommu_device_unregister(&smmu->iommu); in arm_smmu_device_remove()
4790 iommu_device_sysfs_remove(&smmu->iommu); in arm_smmu_device_remove()
4791 arm_smmu_device_disable(smmu); in arm_smmu_device_remove()
4792 iopf_queue_free(smmu->evtq.iopf); in arm_smmu_device_remove()
4793 ida_destroy(&smmu->vmid_map); in arm_smmu_device_remove()
4798 struct arm_smmu_device *smmu = platform_get_drvdata(pdev); in arm_smmu_device_shutdown() local
4800 arm_smmu_device_disable(smmu); in arm_smmu_device_shutdown()
4804 { .compatible = "arm,smmu-v3", },
4817 .name = "arm-smmu-v3",
4830 MODULE_ALIAS("platform:arm-smmu-v3");