diff options
Diffstat (limited to 'drivers/iommu')
| -rw-r--r-- | drivers/iommu/Kconfig | 42 | ||||
| -rw-r--r-- | drivers/iommu/Makefile | 2 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu.c | 6 | ||||
| -rw-r--r-- | drivers/iommu/amd_iommu_init.c | 63 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu-v3.c | 50 | ||||
| -rw-r--r-- | drivers/iommu/arm-smmu.c | 79 | ||||
| -rw-r--r-- | drivers/iommu/dmar.c | 7 | ||||
| -rw-r--r-- | drivers/iommu/exynos-iommu.c | 608 | ||||
| -rw-r--r-- | drivers/iommu/intel-iommu.c | 6 | ||||
| -rw-r--r-- | drivers/iommu/intel-svm.c | 37 | ||||
| -rw-r--r-- | drivers/iommu/intel_irq_remapping.c | 2 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable-arm-v7s.c | 846 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 35 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable.c | 5 | ||||
| -rw-r--r-- | drivers/iommu/io-pgtable.h | 53 | ||||
| -rw-r--r-- | drivers/iommu/iommu.c | 3 | ||||
| -rw-r--r-- | drivers/iommu/mtk_iommu.c | 736 | ||||
| -rw-r--r-- | drivers/iommu/of_iommu.c | 1 |
18 files changed, 2234 insertions, 347 deletions
diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index a1e75cba18e0..dd1dc39f84ff 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -39,6 +39,25 @@ config IOMMU_IO_PGTABLE_LPAE_SELFTEST If unsure, say N here. +config IOMMU_IO_PGTABLE_ARMV7S + bool "ARMv7/v8 Short Descriptor Format" + select IOMMU_IO_PGTABLE + depends on HAS_DMA && (ARM || ARM64 || COMPILE_TEST) + help + Enable support for the ARM Short-descriptor pagetable format. + This supports 32-bit virtual and physical addresses mapped using + 2-level tables with 4KB pages/1MB sections, and contiguous entries + for 64KB pages/16MB supersections if indicated by the IOMMU driver. + +config IOMMU_IO_PGTABLE_ARMV7S_SELFTEST + bool "ARMv7s selftests" + depends on IOMMU_IO_PGTABLE_ARMV7S + help + Enable self-tests for ARMv7s page table allocator. This performs + a series of page-table consistency checks during boot. + + If unsure, say N here. + endmenu config IOMMU_IOVA @@ -51,9 +70,9 @@ config OF_IOMMU # IOMMU-agnostic DMA-mapping layer config IOMMU_DMA bool - depends on NEED_SG_DMA_LENGTH select IOMMU_API select IOMMU_IOVA + select NEED_SG_DMA_LENGTH config FSL_PAMU bool "Freescale IOMMU support" @@ -243,7 +262,7 @@ config TEGRA_IOMMU_SMMU config EXYNOS_IOMMU bool "Exynos IOMMU Support" - depends on ARCH_EXYNOS && ARM && MMU + depends on ARCH_EXYNOS && MMU select IOMMU_API select ARM_DMA_USE_IOMMU help @@ -266,7 +285,7 @@ config EXYNOS_IOMMU_DEBUG config IPMMU_VMSA bool "Renesas VMSA-compatible IPMMU" depends on ARM_LPAE - depends on ARCH_SHMOBILE || COMPILE_TEST + depends on ARCH_RENESAS || COMPILE_TEST select IOMMU_API select IOMMU_IO_PGTABLE_LPAE select ARM_DMA_USE_IOMMU @@ -318,4 +337,21 @@ config S390_IOMMU help Support for the IOMMU API for s390 PCI devices. +config MTK_IOMMU + bool "MTK IOMMU Support" + depends on ARM || ARM64 + depends on ARCH_MEDIATEK || COMPILE_TEST + select ARM_DMA_USE_IOMMU + select IOMMU_API + select IOMMU_DMA + select IOMMU_IO_PGTABLE_ARMV7S + select MEMORY + select MTK_SMI + help + Support for the M4U on certain Mediatek SOCs. M4U is MultiMedia + Memory Management Unit. This option enables remapping of DMA memory + accesses for the multimedia subsystem. + + If unsure, say N here. + endif # IOMMU_SUPPORT diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index 42fc0c25cf1a..c6edb31bf8c6 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_IOMMU_API) += iommu-traces.o obj-$(CONFIG_IOMMU_API) += iommu-sysfs.o obj-$(CONFIG_IOMMU_DMA) += dma-iommu.o obj-$(CONFIG_IOMMU_IO_PGTABLE) += io-pgtable.o +obj-$(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) += io-pgtable-arm-v7s.o obj-$(CONFIG_IOMMU_IO_PGTABLE_LPAE) += io-pgtable-arm.o obj-$(CONFIG_IOMMU_IOVA) += iova.o obj-$(CONFIG_OF_IOMMU) += of_iommu.o @@ -16,6 +17,7 @@ obj-$(CONFIG_INTEL_IOMMU) += intel-iommu.o obj-$(CONFIG_INTEL_IOMMU_SVM) += intel-svm.o obj-$(CONFIG_IPMMU_VMSA) += ipmmu-vmsa.o obj-$(CONFIG_IRQ_REMAP) += intel_irq_remapping.o irq_remapping.o +obj-$(CONFIG_MTK_IOMMU) += mtk_iommu.o obj-$(CONFIG_OMAP_IOMMU) += omap-iommu.o obj-$(CONFIG_OMAP_IOMMU_DEBUG) += omap-iommu-debug.o obj-$(CONFIG_ROCKCHIP_IOMMU) += rockchip-iommu.o diff --git a/drivers/iommu/amd_iommu.c b/drivers/iommu/amd_iommu.c index 539b0dea8034..374c129219ef 100644 --- a/drivers/iommu/amd_iommu.c +++ b/drivers/iommu/amd_iommu.c @@ -114,6 +114,7 @@ struct kmem_cache *amd_iommu_irq_cache; static void update_domain(struct protection_domain *domain); static int protection_domain_init(struct protection_domain *domain); +static void detach_device(struct device *dev); /* * For dynamic growth the aperture size is split into ranges of 128MB of @@ -384,6 +385,9 @@ static void iommu_uninit_device(struct device *dev) if (!dev_data) return; + if (dev_data->domain) + detach_device(dev); + iommu_device_unlink(amd_iommu_rlookup_table[dev_data->devid]->iommu_dev, dev); @@ -2049,7 +2053,7 @@ static void do_attach(struct iommu_dev_data *dev_data, /* Update device table */ set_dte_entry(dev_data->devid, domain, ats); if (alias != dev_data->devid) - set_dte_entry(dev_data->devid, domain, ats); + set_dte_entry(alias, domain, ats); device_flush_dte(dev_data); } diff --git a/drivers/iommu/amd_iommu_init.c b/drivers/iommu/amd_iommu_init.c index 013bdfff2d4d..bf4959f4225b 100644 --- a/drivers/iommu/amd_iommu_init.c +++ b/drivers/iommu/amd_iommu_init.c @@ -228,6 +228,10 @@ static int amd_iommu_enable_interrupts(void); static int __init iommu_go_to_state(enum iommu_init_state state); static void init_device_table_dma(void); +static int iommu_pc_get_set_reg_val(struct amd_iommu *iommu, + u8 bank, u8 cntr, u8 fxn, + u64 *value, bool is_write); + static inline void update_last_devid(u16 devid) { if (devid > amd_iommu_last_bdf) @@ -1016,6 +1020,34 @@ static void amd_iommu_erratum_746_workaround(struct amd_iommu *iommu) } /* + * Family15h Model 30h-3fh (IOMMU Mishandles ATS Write Permission) + * Workaround: + * BIOS should enable ATS write permission check by setting + * L2_DEBUG_3[AtsIgnoreIWDis](D0F2xF4_x47[0]) = 1b + */ +static void amd_iommu_ats_write_check_workaround(struct amd_iommu *iommu) +{ + u32 value; + + if ((boot_cpu_data.x86 != 0x15) || + (boot_cpu_data.x86_model < 0x30) || + (boot_cpu_data.x86_model > 0x3f)) + return; + + /* Test L2_DEBUG_3[AtsIgnoreIWDis] == 1 */ + value = iommu_read_l2(iommu, 0x47); + + if (value & BIT(0)) + return; + + /* Set L2_DEBUG_3[AtsIgnoreIWDis] = 1 */ + iommu_write_l2(iommu, 0x47, value | BIT(0)); + + pr_info("AMD-Vi: Applying ATS write check workaround for IOMMU at %s\n", + dev_name(&iommu->dev->dev)); +} + +/* * This function clues the initialization function for one IOMMU * together and also allocates the command buffer and programs the * hardware. It does NOT enable the IOMMU. This is done afterwards. @@ -1142,8 +1174,8 @@ static void init_iommu_perf_ctr(struct amd_iommu *iommu) amd_iommu_pc_present = true; /* Check if the performance counters can be written to */ - if ((0 != amd_iommu_pc_get_set_reg_val(0, 0, 0, 0, &val, true)) || - (0 != amd_iommu_pc_get_set_reg_val(0, 0, 0, 0, &val2, false)) || + if ((0 != iommu_pc_get_set_reg_val(iommu, 0, 0, 0, &val, true)) || + (0 != iommu_pc_get_set_reg_val(iommu, 0, 0, 0, &val2, false)) || (val != val2)) { pr_err("AMD-Vi: Unable to write to IOMMU perf counter.\n"); amd_iommu_pc_present = false; @@ -1284,6 +1316,7 @@ static int iommu_init_pci(struct amd_iommu *iommu) } amd_iommu_erratum_746_workaround(iommu); + amd_iommu_ats_write_check_workaround(iommu); iommu->iommu_dev = iommu_device_create(&iommu->dev->dev, iommu, amd_iommu_groups, "ivhd%d", @@ -2283,22 +2316,15 @@ u8 amd_iommu_pc_get_max_counters(u16 devid) } EXPORT_SYMBOL(amd_iommu_pc_get_max_counters); -int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn, +static int iommu_pc_get_set_reg_val(struct amd_iommu *iommu, + u8 bank, u8 cntr, u8 fxn, u64 *value, bool is_write) { - struct amd_iommu *iommu; u32 offset; u32 max_offset_lim; - /* Make sure the IOMMU PC resource is available */ - if (!amd_iommu_pc_present) - return -ENODEV; - - /* Locate the iommu associated with the device ID */ - iommu = amd_iommu_rlookup_table[devid]; - /* Check for valid iommu and pc register indexing */ - if (WARN_ON((iommu == NULL) || (fxn > 0x28) || (fxn & 7))) + if (WARN_ON((fxn > 0x28) || (fxn & 7))) return -ENODEV; offset = (u32)(((0x40|bank) << 12) | (cntr << 8) | fxn); @@ -2322,3 +2348,16 @@ int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn, return 0; } EXPORT_SYMBOL(amd_iommu_pc_get_set_reg_val); + +int amd_iommu_pc_get_set_reg_val(u16 devid, u8 bank, u8 cntr, u8 fxn, + u64 *value, bool is_write) +{ + struct amd_iommu *iommu = amd_iommu_rlookup_table[devid]; + + /* Make sure the IOMMU PC resource is available */ + if (!amd_iommu_pc_present || iommu == NULL) + return -ENODEV; + + return iommu_pc_get_set_reg_val(iommu, bank, cntr, fxn, + value, is_write); +} diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c index 20875341c865..4ff73ff64e49 100644 --- a/drivers/iommu/arm-smmu-v3.c +++ b/drivers/iommu/arm-smmu-v3.c @@ -21,6 +21,7 @@ */ #include <linux/delay.h> +#include <linux/dma-iommu.h> #include <linux/err.h> #include <linux/interrupt.h> #include <linux/iommu.h> @@ -1396,7 +1397,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { struct arm_smmu_domain *smmu_domain; - if (type != IOMMU_DOMAIN_UNMANAGED) + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; /* @@ -1408,6 +1409,12 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) if (!smmu_domain) return NULL; + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&smmu_domain->domain)) { + kfree(smmu_domain); + return NULL; + } + mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); return &smmu_domain->domain; @@ -1436,6 +1443,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); struct arm_smmu_device *smmu = smmu_domain->smmu; + iommu_put_dma_cookie(domain); free_io_pgtable_ops(smmu_domain->pgtbl_ops); /* Free the CD and ASID, if we allocated them */ @@ -1630,6 +1638,17 @@ static int arm_smmu_install_ste_for_group(struct arm_smmu_group *smmu_group) return 0; } +static void arm_smmu_detach_dev(struct device *dev) +{ + struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); + + smmu_group->ste.bypass = true; + if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group))) + dev_warn(dev, "failed to install bypass STE\n"); + + smmu_group->domain = NULL; +} + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret = 0; @@ -1642,7 +1661,7 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) /* Already attached to a different domain? */ if (smmu_group->domain && smmu_group->domain != smmu_domain) - return -EEXIST; + arm_smmu_detach_dev(dev); smmu = smmu_group->smmu; mutex_lock(&smmu_domain->init_mutex); @@ -1668,7 +1687,12 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) goto out_unlock; smmu_group->domain = smmu_domain; - smmu_group->ste.bypass = false; + + /* + * FIXME: This should always be "false" once we have IOMMU-backed + * DMA ops for all devices behind the SMMU. + */ + smmu_group->ste.bypass = domain->type == IOMMU_DOMAIN_DMA; ret = arm_smmu_install_ste_for_group(smmu_group); if (IS_ERR_VALUE(ret)) @@ -1679,25 +1703,6 @@ out_unlock: return ret; } -static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_group *smmu_group = arm_smmu_group_get(dev); - - BUG_ON(!smmu_domain); - BUG_ON(!smmu_group); - - mutex_lock(&smmu_domain->init_mutex); - BUG_ON(smmu_group->domain != smmu_domain); - - smmu_group->ste.bypass = true; - if (IS_ERR_VALUE(arm_smmu_install_ste_for_group(smmu_group))) - dev_warn(dev, "failed to install bypass STE\n"); - - smmu_group->domain = NULL; - mutex_unlock(&smmu_domain->init_mutex); -} - static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -1935,7 +1940,6 @@ static struct iommu_ops arm_smmu_ops = { .domain_alloc = arm_smmu_domain_alloc, .domain_free = arm_smmu_domain_free, .attach_dev = arm_smmu_attach_dev, - .detach_dev = arm_smmu_detach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, .iova_to_phys = arm_smmu_iova_to_phys, diff --git a/drivers/iommu/arm-smmu.c b/drivers/iommu/arm-smmu.c index 59ee4b8a3236..2409e3bd3df2 100644 --- a/drivers/iommu/arm-smmu.c +++ b/drivers/iommu/arm-smmu.c @@ -29,6 +29,7 @@ #define pr_fmt(fmt) "arm-smmu: " fmt #include <linux/delay.h> +#include <linux/dma-iommu.h> #include <linux/dma-mapping.h> #include <linux/err.h> #include <linux/interrupt.h> @@ -167,6 +168,9 @@ #define S2CR_TYPE_BYPASS (1 << S2CR_TYPE_SHIFT) #define S2CR_TYPE_FAULT (2 << S2CR_TYPE_SHIFT) +#define S2CR_PRIVCFG_SHIFT 24 +#define S2CR_PRIVCFG_UNPRIV (2 << S2CR_PRIVCFG_SHIFT) + /* Context bank attribute registers */ #define ARM_SMMU_GR1_CBAR(n) (0x0 + ((n) << 2)) #define CBAR_VMID_SHIFT 0 @@ -257,9 +261,13 @@ #define FSYNR0_WNR (1 << 4) static int force_stage; -module_param_named(force_stage, force_stage, int, S_IRUGO); +module_param(force_stage, int, S_IRUGO); MODULE_PARM_DESC(force_stage, "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation."); +static bool disable_bypass; +module_param(disable_bypass, bool, S_IRUGO); +MODULE_PARM_DESC(disable_bypass, + "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU."); enum arm_smmu_arch_version { ARM_SMMU_V1 = 1, @@ -963,7 +971,7 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) { struct arm_smmu_domain *smmu_domain; - if (type != IOMMU_DOMAIN_UNMANAGED) + if (type != IOMMU_DOMAIN_UNMANAGED && type != IOMMU_DOMAIN_DMA) return NULL; /* * Allocate the domain and initialise some of its data structures. @@ -974,6 +982,12 @@ static struct iommu_domain *arm_smmu_domain_alloc(unsigned type) if (!smmu_domain) return NULL; + if (type == IOMMU_DOMAIN_DMA && + iommu_get_dma_cookie(&smmu_domain->domain)) { + kfree(smmu_domain); + return NULL; + } + mutex_init(&smmu_domain->init_mutex); spin_lock_init(&smmu_domain->pgtbl_lock); @@ -988,6 +1002,7 @@ static void arm_smmu_domain_free(struct iommu_domain *domain) * Free the domain resources. We assume that all devices have * already been detached. */ + iommu_put_dma_cookie(domain); arm_smmu_destroy_domain_context(domain); kfree(smmu_domain); } @@ -1079,11 +1094,18 @@ static int arm_smmu_domain_add_master(struct arm_smmu_domain *smmu_domain, if (ret) return ret == -EEXIST ? 0 : ret; + /* + * FIXME: This won't be needed once we have IOMMU-backed DMA ops + * for all devices behind the SMMU. + */ + if (smmu_domain->domain.type == IOMMU_DOMAIN_DMA) + return 0; + for (i = 0; i < cfg->num_streamids; ++i) { u32 idx, s2cr; idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; - s2cr = S2CR_TYPE_TRANS | + s2cr = S2CR_TYPE_TRANS | S2CR_PRIVCFG_UNPRIV | (smmu_domain->cfg.cbndx << S2CR_CBNDX_SHIFT); writel_relaxed(s2cr, gr0_base + ARM_SMMU_GR0_S2CR(idx)); } @@ -1108,14 +1130,24 @@ static void arm_smmu_domain_remove_master(struct arm_smmu_domain *smmu_domain, */ for (i = 0; i < cfg->num_streamids; ++i) { u32 idx = cfg->smrs ? cfg->smrs[i].idx : cfg->streamids[i]; + u32 reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS; - writel_relaxed(S2CR_TYPE_BYPASS, - gr0_base + ARM_SMMU_GR0_S2CR(idx)); + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(idx)); } arm_smmu_master_free_smrs(smmu, cfg); } +static void arm_smmu_detach_dev(struct device *dev, + struct arm_smmu_master_cfg *cfg) +{ + struct iommu_domain *domain = dev->archdata.iommu; + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + + dev->archdata.iommu = NULL; + arm_smmu_domain_remove_master(smmu_domain, cfg); +} + static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) { int ret; @@ -1129,11 +1161,6 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return -ENXIO; } - if (dev->archdata.iommu) { - dev_err(dev, "already attached to IOMMU domain\n"); - return -EEXIST; - } - /* Ensure that the domain is finalised */ ret = arm_smmu_init_domain_context(domain, smmu); if (IS_ERR_VALUE(ret)) @@ -1155,25 +1182,16 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) if (!cfg) return -ENODEV; + /* Detach the dev from its current domain */ + if (dev->archdata.iommu) + arm_smmu_detach_dev(dev, cfg); + ret = arm_smmu_domain_add_master(smmu_domain, cfg); if (!ret) dev->archdata.iommu = domain; return ret; } -static void arm_smmu_detach_dev(struct iommu_domain *domain, struct device *dev) -{ - struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); - struct arm_smmu_master_cfg *cfg; - - cfg = find_smmu_master_cfg(dev); - if (!cfg) - return; - - dev->archdata.iommu = NULL; - arm_smmu_domain_remove_master(smmu_domain, cfg); -} - static int arm_smmu_map(struct iommu_domain *domain, unsigned long iova, phys_addr_t paddr, size_t size, int prot) { @@ -1449,7 +1467,6 @@ static struct iommu_ops arm_smmu_ops = { .domain_alloc = arm_smmu_domain_alloc, .domain_free = arm_smmu_domain_free, .attach_dev = arm_smmu_attach_dev, - .detach_dev = arm_smmu_detach_dev, .map = arm_smmu_map, .unmap = arm_smmu_unmap, .map_sg = default_iommu_map_sg, @@ -1473,11 +1490,11 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR); - /* Mark all SMRn as invalid and all S2CRn as bypass */ + /* Mark all SMRn as invalid and all S2CRn as bypass unless overridden */ + reg = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS; for (i = 0; i < smmu->num_mapping_groups; ++i) { writel_relaxed(0, gr0_base + ARM_SMMU_GR0_SMR(i)); - writel_relaxed(S2CR_TYPE_BYPASS, - gr0_base + ARM_SMMU_GR0_S2CR(i)); + writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_S2CR(i)); } /* Make sure all context banks are disabled and clear CB_FSR */ @@ -1499,8 +1516,12 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu) /* Disable TLB broadcasting. */ reg |= (sCR0_VMIDPNE | sCR0_PTM); - /* Enable client access, but bypass when no mapping is found */ - reg &= ~(sCR0_CLIENTPD | sCR0_USFCFG); + /* Enable client access, handling unmatched streams as appropriate */ + reg &= ~sCR0_CLIENTPD; + if (disable_bypass) + reg |= sCR0_USFCFG; + else + reg &= ~sCR0_USFCFG; /* Disable forced broadcasting */ reg &= ~sCR0_FB; diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 62a400c5ba06..8ffd7568fc91 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -329,7 +329,8 @@ static int dmar_pci_bus_notifier(struct notifier_block *nb, /* Only care about add/remove events for physical functions */ if (pdev->is_virtfn) return NOTIFY_DONE; - if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE) + if (action != BUS_NOTIFY_ADD_DEVICE && + action != BUS_NOTIFY_REMOVED_DEVICE) return NOTIFY_DONE; info = dmar_alloc_pci_notify_info(pdev, action); @@ -339,7 +340,7 @@ static int dmar_pci_bus_notifier(struct notifier_block *nb, down_write(&dmar_global_lock); if (action == BUS_NOTIFY_ADD_DEVICE) dmar_pci_bus_add_dev(info); - else if (action == BUS_NOTIFY_DEL_DEVICE) + else if (action == BUS_NOTIFY_REMOVED_DEVICE) dmar_pci_bus_del_dev(info); up_write(&dmar_global_lock); @@ -1353,7 +1354,7 @@ void dmar_disable_qi(struct intel_iommu *iommu) raw_spin_lock_irqsave(&iommu->register_lock, flags); - sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); + sts = readl(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_QIES)) goto end; diff --git a/drivers/iommu/exynos-iommu.c b/drivers/iommu/exynos-iommu.c index 97c41b8ab5d9..5ecc86cb74c8 100644 --- a/drivers/iommu/exynos-iommu.c +++ b/drivers/iommu/exynos-iommu.c @@ -1,6 +1,5 @@ -/* linux/drivers/iommu/exynos_iommu.c - * - * Copyright (c) 2011 Samsung Electronics Co., Ltd. +/* + * Copyright (c) 2011,2016 Samsung Electronics Co., Ltd. * http://www.samsung.com * * This program is free software; you can redistribute it and/or modify @@ -25,10 +24,7 @@ #include <linux/platform_device.h> #include <linux/pm_runtime.h> #include <linux/slab.h> - -#include <asm/cacheflush.h> -#include <asm/dma-iommu.h> -#include <asm/pgtable.h> +#include <linux/dma-iommu.h> typedef u32 sysmmu_iova_t; typedef u32 sysmmu_pte_t; @@ -58,17 +54,25 @@ typedef u32 sysmmu_pte_t; #define lv2ent_small(pent) ((*(pent) & 2) == 2) #define lv2ent_large(pent) ((*(pent) & 3) == 1) -static u32 sysmmu_page_offset(sysmmu_iova_t iova, u32 size) -{ - return iova & (size - 1); -} - -#define section_phys(sent) (*(sent) & SECT_MASK) -#define section_offs(iova) sysmmu_page_offset((iova), SECT_SIZE) -#define lpage_phys(pent) (*(pent) & LPAGE_MASK) -#define lpage_offs(iova) sysmmu_page_offset((iova), LPAGE_SIZE) -#define spage_phys(pent) (*(pent) & SPAGE_MASK) -#define spage_offs(iova) sysmmu_page_offset((iova), SPAGE_SIZE) +/* + * v1.x - v3.x SYSMMU supports 32bit physical and 32bit virtual address spaces + * v5.0 introduced support for 36bit physical address space by shifting + * all page entry values by 4 bits. + * All SYSMMU controllers in the system support the address spaces of the same + * size, so PG_ENT_SHIFT can be initialized on first SYSMMU probe to proper + * value (0 or 4). + */ +static short PG_ENT_SHIFT = -1; +#define SYSMMU_PG_ENT_SHIFT 0 +#define SYSMMU_V5_PG_ENT_SHIFT 4 + +#define sect_to_phys(ent) (((phys_addr_t) ent) << PG_ENT_SHIFT) +#define section_phys(sent) (sect_to_phys(*(sent)) & SECT_MASK) +#define section_offs(iova) (iova & (SECT_SIZE - 1)) +#define lpage_phys(pent) (sect_to_phys(*(pent)) & LPAGE_MASK) +#define lpage_offs(iova) (iova & (LPAGE_SIZE - 1)) +#define spage_phys(pent) (sect_to_phys(*(pent)) & SPAGE_MASK) +#define spage_offs(iova) (iova & (SPAGE_SIZE - 1)) #define NUM_LV1ENTRIES 4096 #define NUM_LV2ENTRIES (SECT_SIZE / SPAGE_SIZE) @@ -83,16 +87,16 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) return (iova >> SPAGE_ORDER) & (NUM_LV2ENTRIES - 1); } +#define LV1TABLE_SIZE (NUM_LV1ENTRIES * sizeof(sysmmu_pte_t)) #define LV2TABLE_SIZE (NUM_LV2ENTRIES * sizeof(sysmmu_pte_t)) #define SPAGES_PER_LPAGE (LPAGE_SIZE / SPAGE_SIZE) +#define lv2table_base(sent) (sect_to_phys(*(sent) & 0xFFFFFFC0)) -#define lv2table_base(sent) (*(sent) & 0xFFFFFC00) - -#define mk_lv1ent_sect(pa) ((pa) | 2) -#define mk_lv1ent_page(pa) ((pa) | 1) -#define mk_lv2ent_lpage(pa) ((pa) | 1) -#define mk_lv2ent_spage(pa) ((pa) | 2) +#define mk_lv1ent_sect(pa) ((pa >> PG_ENT_SHIFT) | 2) +#define mk_lv1ent_page(pa) ((pa >> PG_ENT_SHIFT) | 1) +#define mk_lv2ent_lpage(pa) ((pa >> PG_ENT_SHIFT) | 1) +#define mk_lv2ent_spage(pa) ((pa >> PG_ENT_SHIFT) | 2) #define CTRL_ENABLE 0x5 #define CTRL_BLOCK 0x7 @@ -100,14 +104,23 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define CFG_LRU 0x1 #define CFG_QOS(n) ((n & 0xF) << 7) -#define CFG_MASK 0x0150FFFF /* Selecting bit 0-15, 20, 22 and 24 */ #define CFG_ACGEN (1 << 24) /* System MMU 3.3 only */ #define CFG_SYSSEL (1 << 22) /* System MMU 3.2 only */ #define CFG_FLPDCACHE (1 << 20) /* System MMU 3.2+ only */ +/* common registers */ #define REG_MMU_CTRL 0x000 #define REG_MMU_CFG 0x004 #define REG_MMU_STATUS 0x008 +#define REG_MMU_VERSION 0x034 + +#define MMU_MAJ_VER(val) ((val) >> 7) +#define MMU_MIN_VER(val) ((val) & 0x7F) +#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ + +#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) + +/* v1.x - v3.x registers */ #define REG_MMU_FLUSH 0x00C #define REG_MMU_FLUSH_ENTRY 0x010 #define REG_PT_BASE_ADDR 0x014 @@ -119,21 +132,18 @@ static u32 lv2ent_offset(sysmmu_iova_t iova) #define REG_AR_FAULT_ADDR 0x02C #define REG_DEFAULT_SLAVE_ADDR 0x030 -#define REG_MMU_VERSION 0x034 - -#define MMU_MAJ_VER(val) ((val) >> 7) -#define MMU_MIN_VER(val) ((val) & 0x7F) -#define MMU_RAW_VER(reg) (((reg) >> 21) & ((1 << 11) - 1)) /* 11 bits */ - -#define MAKE_MMU_VER(maj, min) ((((maj) & 0xF) << 7) | ((min) & 0x7F)) - -#define REG_PB0_SADDR 0x04C -#define REG_PB0_EADDR 0x050 -#define REG_PB1_SADDR 0x054 -#define REG_PB1_EADDR 0x058 +/* v5.x registers */ +#define REG_V5_PT_BASE_PFN 0x00C +#define REG_V5_MMU_FLUSH_ALL 0x010 +#define REG_V5_MMU_FLUSH_ENTRY 0x014 +#define REG_V5_INT_STATUS 0x060 +#define REG_V5_INT_CLEAR 0x064 +#define REG_V5_FAULT_AR_VA 0x070 +#define REG_V5_FAULT_AW_VA 0x080 #define has_sysmmu(dev) (dev->archdata.iommu != NULL) +static struct device *dma_dev; static struct kmem_cache *lv2table_kmem_cache; static sysmmu_pte_t *zero_lv2_table; #define ZERO_LV2LINK mk_lv1ent_page(virt_to_phys(zero_lv2_table)) @@ -149,40 +159,38 @@ static sysmmu_pte_t *page_entry(sysmmu_pte_t *sent, sysmmu_iova_t iova) lv2table_base(sent)) + lv2ent_offset(iova); } -enum exynos_sysmmu_inttype { - SYSMMU_PAGEFAULT, - SYSMMU_AR_MULTIHIT, - SYSMMU_AW_MULTIHIT, - SYSMMU_BUSERROR, - SYSMMU_AR_SECURITY, - SYSMMU_AR_ACCESS, - SYSMMU_AW_SECURITY, - SYSMMU_AW_PROTECTION, /* 7 */ - SYSMMU_FAULT_UNKNOWN, - SYSMMU_FAULTS_NUM +/* + * IOMMU fault information register + */ +struct sysmmu_fault_info { + unsigned int bit; /* bit number in STATUS register */ + unsigned short addr_reg; /* register to read VA fault address */ + const char *name; /* human readable fault name */ + unsigned int type; /* fault type for report_iommu_fault */ }; -static unsigned short fault_reg_offset[SYSMMU_FAULTS_NUM] = { - REG_PAGE_FAULT_ADDR, - REG_AR_FAULT_ADDR, - REG_AW_FAULT_ADDR, - REG_DEFAULT_SLAVE_ADDR, - REG_AR_FAULT_ADDR, - REG_AR_FAULT_ADDR, - REG_AW_FAULT_ADDR, - REG_AW_FAULT_ADDR +static const struct sysmmu_fault_info sysmmu_faults[] = { + { 0, REG_PAGE_FAULT_ADDR, "PAGE", IOMMU_FAULT_READ }, + { 1, REG_AR_FAULT_ADDR, "AR MULTI-HIT", IOMMU_FAULT_READ }, + { 2, REG_AW_FAULT_ADDR, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, + { 3, REG_DEFAULT_SLAVE_ADDR, "BUS ERROR", IOMMU_FAULT_READ }, + { 4, REG_AR_FAULT_ADDR, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, + { 5, REG_AR_FAULT_ADDR, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, + { 6, REG_AW_FAULT_ADDR, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, + { 7, REG_AW_FAULT_ADDR, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, }; -static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { - "PAGE FAULT", - "AR MULTI-HIT FAULT", - "AW MULTI-HIT FAULT", - "BUS ERROR", - "AR SECURITY PROTECTION FAULT", - "AR ACCESS PROTECTION FAULT", - "AW SECURITY PROTECTION FAULT", - "AW ACCESS PROTECTION FAULT", - "UNKNOWN FAULT" +static const struct sysmmu_fault_info sysmmu_v5_faults[] = { + { 0, REG_V5_FAULT_AR_VA, "AR PTW", IOMMU_FAULT_READ }, + { 1, REG_V5_FAULT_AR_VA, "AR PAGE", IOMMU_FAULT_READ }, + { 2, REG_V5_FAULT_AR_VA, "AR MULTI-HIT", IOMMU_FAULT_READ }, + { 3, REG_V5_FAULT_AR_VA, "AR ACCESS PROTECTION", IOMMU_FAULT_READ }, + { 4, REG_V5_FAULT_AR_VA, "AR SECURITY PROTECTION", IOMMU_FAULT_READ }, + { 16, REG_V5_FAULT_AW_VA, "AW PTW", IOMMU_FAULT_WRITE }, + { 17, REG_V5_FAULT_AW_VA, "AW PAGE", IOMMU_FAULT_WRITE }, + { 18, REG_V5_FAULT_AW_VA, "AW MULTI-HIT", IOMMU_FAULT_WRITE }, + { 19, REG_V5_FAULT_AW_VA, "AW ACCESS PROTECTION", IOMMU_FAULT_WRITE }, + { 20, REG_V5_FAULT_AW_VA, "AW SECURITY PROTECTION", IOMMU_FAULT_WRITE }, }; /* @@ -193,6 +201,7 @@ static char *sysmmu_fault_name[SYSMMU_FAULTS_NUM] = { */ struct exynos_iommu_owner { struct list_head controllers; /* list of sysmmu_drvdata.owner_node */ + struct iommu_domain *domain; /* domain this device is attached */ }; /* @@ -221,6 +230,8 @@ struct sysmmu_drvdata { struct device *master; /* master device (owner) */ void __iomem *sfrbase; /* our registers */ struct clk *clk; /* SYSMMU's clock */ + struct clk *aclk; /* SYSMMU's aclk clock */ + struct clk *pclk; /* SYSMMU's pclk clock */ struct clk *clk_master; /* master's device clock */ int activations; /* number of calls to sysmmu_enable */ spinlock_t lock; /* lock for modyfying state */ @@ -255,70 +266,101 @@ static bool is_sysmmu_active(struct sysmmu_drvdata *data) return data->activations > 0; } -static void sysmmu_unblock(void __iomem *sfrbase) +static void sysmmu_unblock(struct sysmmu_drvdata *data) { - __raw_writel(CTRL_ENABLE, sfrbase + REG_MMU_CTRL); + writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); } -static bool sysmmu_block(void __iomem *sfrbase) +static bool sysmmu_block(struct sysmmu_drvdata *data) { int i = 120; - __raw_writel(CTRL_BLOCK, sfrbase + REG_MMU_CTRL); - while ((i > 0) && !(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) + writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + while ((i > 0) && !(readl(data->sfrbase + REG_MMU_STATUS) & 1)) --i; - if (!(__raw_readl(sfrbase + REG_MMU_STATUS) & 1)) { - sysmmu_unblock(sfrbase); + if (!(readl(data->sfrbase + REG_MMU_STATUS) & 1)) { + sysmmu_unblock(data); return false; } return true; } -static void __sysmmu_tlb_invalidate(void __iomem *sfrbase) +static void __sysmmu_tlb_invalidate(struct sysmmu_drvdata *data) { - __raw_writel(0x1, sfrbase + REG_MMU_FLUSH); + if (MMU_MAJ_VER(data->version) < 5) + writel(0x1, data->sfrbase + REG_MMU_FLUSH); + else + writel(0x1, data->sfrbase + REG_V5_MMU_FLUSH_ALL); } -static void __sysmmu_tlb_invalidate_entry(void __iomem *sfrbase, +static void __sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, sysmmu_iova_t iova, unsigned int num_inv) { unsigned int i; for (i = 0; i < num_inv; i++) { - __raw_writel((iova & SPAGE_MASK) | 1, - sfrbase + REG_MMU_FLUSH_ENTRY); + if (MMU_MAJ_VER(data->version) < 5) + writel((iova & SPAGE_MASK) | 1, + data->sfrbase + REG_MMU_FLUSH_ENTRY); + else + writel((iova & SPAGE_MASK) | 1, + data->sfrbase + REG_V5_MMU_FLUSH_ENTRY); iova += SPAGE_SIZE; } } -static void __sysmmu_set_ptbase(void __iomem *sfrbase, - phys_addr_t pgd) +static void __sysmmu_set_ptbase(struct sysmmu_drvdata *data, phys_addr_t pgd) { - __raw_writel(pgd, sfrbase + REG_PT_BASE_ADDR); + if (MMU_MAJ_VER(data->version) < 5) + writel(pgd, data->sfrbase + REG_PT_BASE_ADDR); + else + writel(pgd >> PAGE_SHIFT, + data->sfrbase + REG_V5_PT_BASE_PFN); - __sysmmu_tlb_invalidate(sfrbase); + __sysmmu_tlb_invalidate(data); } -static void show_fault_information(const char *name, - enum exynos_sysmmu_inttype itype, - phys_addr_t pgtable_base, sysmmu_iova_t fault_addr) +static void __sysmmu_get_version(struct sysmmu_drvdata *data) { - sysmmu_pte_t *ent; + u32 ver; + + clk_enable(data->clk_master); + clk_enable(data->clk); + clk_enable(data->pclk); + clk_enable(data->aclk); + + ver = readl(data->sfrbase + REG_MMU_VERSION); - if ((itype >= SYSMMU_FAULTS_NUM) || (itype < SYSMMU_PAGEFAULT)) - itype = SYSMMU_FAULT_UNKNOWN; + /* controllers on some SoCs don't report proper version */ + if (ver == 0x80000001u) + data->version = MAKE_MMU_VER(1, 0); + else + data->version = MMU_RAW_VER(ver); + + dev_dbg(data->sysmmu, "hardware version: %d.%d\n", + MMU_MAJ_VER(data->version), MMU_MIN_VER(data->version)); - pr_err("%s occurred at %#x by %s(Page table base: %pa)\n", - sysmmu_fault_name[itype], fault_addr, name, &pgtable_base); + clk_disable(data->aclk); + clk_disable(data->pclk); + clk_disable(data->clk); + clk_disable(data->clk_master); +} - ent = section_entry(phys_to_virt(pgtable_base), fault_addr); - pr_err("\tLv1 entry: %#x\n", *ent); +static void show_fault_information(struct sysmmu_drvdata *data, + const struct sysmmu_fault_info *finfo, + sysmmu_iova_t fault_addr) +{ + sysmmu_pte_t *ent; + dev_err(data->sysmmu, "%s FAULT occurred at %#x (page table base: %pa)\n", + finfo->name, fault_addr, &data->pgtable); + ent = section_entry(phys_to_virt(data->pgtable), fault_addr); + dev_err(data->sysmmu, "\tLv1 entry: %#x\n", *ent); if (lv1ent_page(ent)) { ent = page_entry(ent, fault_addr); - pr_err("\t Lv2 entry: %#x\n", *ent); + dev_err(data->sysmmu, "\t Lv2 entry: %#x\n", *ent); } } @@ -326,49 +368,52 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) { /* SYSMMU is in blocked state when interrupt occurred. */ struct sysmmu_drvdata *data = dev_id; - enum exynos_sysmmu_inttype itype; - sysmmu_iova_t addr = -1; + const struct sysmmu_fault_info *finfo; + unsigned int i, n, itype; + sysmmu_iova_t fault_addr = -1; + unsigned short reg_status, reg_clear; int ret = -ENOSYS; WARN_ON(!is_sysmmu_active(data)); + if (MMU_MAJ_VER(data->version) < 5) { + reg_status = REG_INT_STATUS; + reg_clear = REG_INT_CLEAR; + finfo = sysmmu_faults; + n = ARRAY_SIZE(sysmmu_faults); + } else { + reg_status = REG_V5_INT_STATUS; + reg_clear = REG_V5_INT_CLEAR; + finfo = sysmmu_v5_faults; + n = ARRAY_SIZE(sysmmu_v5_faults); + } + spin_lock(&data->lock); - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); - itype = (enum exynos_sysmmu_inttype) - __ffs(__raw_readl(data->sfrbase + REG_INT_STATUS)); - if (WARN_ON(!((itype >= 0) && (itype < SYSMMU_FAULT_UNKNOWN)))) - itype = SYSMMU_FAULT_UNKNOWN; - else - addr = __raw_readl(data->sfrbase + fault_reg_offset[itype]); + itype = __ffs(readl(data->sfrbase + reg_status)); + for (i = 0; i < n; i++, finfo++) + if (finfo->bit == itype) + break; + /* unknown/unsupported fault */ + BUG_ON(i == n); - if (itype == SYSMMU_FAULT_UNKNOWN) { - pr_err("%s: Fault is not occurred by System MMU '%s'!\n", - __func__, dev_name(data->sysmmu)); - pr_err("%s: Please check if IRQ is correctly configured.\n", - __func__); - BUG(); - } else { - unsigned int base = - __raw_readl(data->sfrbase + REG_PT_BASE_ADDR); - show_fault_information(dev_name(data->sysmmu), - itype, base, addr); - if (data->domain) - ret = report_iommu_fault(&data->domain->domain, - data->master, addr, itype); - } + /* print debug message */ + fault_addr = readl(data->sfrbase + finfo->addr_reg); + show_fault_information(data, finfo, fault_addr); + if (data->domain) + ret = report_iommu_fault(&data->domain->domain, + data->master, fault_addr, finfo->type); /* fault is not recovered by fault handler */ BUG_ON(ret != 0); - __raw_writel(1 << itype, data->sfrbase + REG_INT_CLEAR); + writel(1 << itype, data->sfrbase + reg_clear); - sysmmu_unblock(data->sfrbase); + sysmmu_unblock(data); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); spin_unlock(&data->lock); @@ -377,15 +422,15 @@ static irqreturn_t exynos_sysmmu_irq(int irq, void *dev_id) static void __sysmmu_disable_nocount(struct sysmmu_drvdata *data) { - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); - __raw_writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); - __raw_writel(0, data->sfrbase + REG_MMU_CFG); + writel(CTRL_DISABLE, data->sfrbase + REG_MMU_CTRL); + writel(0, data->sfrbase + REG_MMU_CFG); + clk_disable(data->aclk); + clk_disable(data->pclk); clk_disable(data->clk); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } static bool __sysmmu_disable(struct sysmmu_drvdata *data) @@ -416,42 +461,34 @@ static bool __sysmmu_disable(struct sysmmu_drvdata *data) static void __sysmmu_init_config(struct sysmmu_drvdata *data) { - unsigned int cfg = CFG_LRU | CFG_QOS(15); - unsigned int ver; - - ver = MMU_RAW_VER(__raw_readl(data->sfrbase + REG_MMU_VERSION)); - if (MMU_MAJ_VER(ver) == 3) { - if (MMU_MIN_VER(ver) >= 2) { - cfg |= CFG_FLPDCACHE; - if (MMU_MIN_VER(ver) == 3) { - cfg |= CFG_ACGEN; - cfg &= ~CFG_LRU; - } else { - cfg |= CFG_SYSSEL; - } - } - } + unsigned int cfg; - __raw_writel(cfg, data->sfrbase + REG_MMU_CFG); - data->version = ver; + if (data->version <= MAKE_MMU_VER(3, 1)) + cfg = CFG_LRU | CFG_QOS(15); + else if (data->version <= MAKE_MMU_VER(3, 2)) + cfg = CFG_LRU | CFG_QOS(15) | CFG_FLPDCACHE | CFG_SYSSEL; + else + cfg = CFG_QOS(15) | CFG_FLPDCACHE | CFG_ACGEN; + + writel(cfg, data->sfrbase + REG_MMU_CFG); } static void __sysmmu_enable_nocount(struct sysmmu_drvdata *data) { - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); clk_enable(data->clk); + clk_enable(data->pclk); + clk_enable(data->aclk); - __raw_writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); + writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL); __sysmmu_init_config(data); - __sysmmu_set_ptbase(data->sfrbase, data->pgtable); + __sysmmu_set_ptbase(data, data->pgtable); - __raw_writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); + writel(CTRL_ENABLE, data->sfrbase + REG_MMU_CTRL); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, @@ -482,28 +519,21 @@ static int __sysmmu_enable(struct sysmmu_drvdata *data, phys_addr_t pgtable, return ret; } -static void __sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, - sysmmu_iova_t iova) -{ - if (data->version == MAKE_MMU_VER(3, 3)) - __raw_writel(iova | 0x1, data->sfrbase + REG_MMU_FLUSH_ENTRY); -} - static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data, sysmmu_iova_t iova) { unsigned long flags; - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); spin_lock_irqsave(&data->lock, flags); - if (is_sysmmu_active(data)) - __sysmmu_tlb_invalidate_flpdcache(data, iova); + if (is_sysmmu_active(data)) { + if (data->version >= MAKE_MMU_VER(3, 3)) + __sysmmu_tlb_invalidate_entry(data, iova, 1); + } spin_unlock_irqrestore(&data->lock, flags); - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, @@ -515,8 +545,7 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, if (is_sysmmu_active(data)) { unsigned int num_inv = 1; - if (!IS_ERR(data->clk_master)) - clk_enable(data->clk_master); + clk_enable(data->clk_master); /* * L2TLB invalidation required @@ -531,13 +560,11 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data, if (MMU_MAJ_VER(data->version) == 2) num_inv = min_t(unsigned int, size / PAGE_SIZE, 64); - if (sysmmu_block(data->sfrbase)) { - __sysmmu_tlb_invalidate_entry( - data->sfrbase, iova, num_inv); - sysmmu_unblock(data->sfrbase); + if (sysmmu_block(data)) { + __sysmmu_tlb_invalidate_entry(data, iova, num_inv); + sysmmu_unblock(data); } - if (!IS_ERR(data->clk_master)) - clk_disable(data->clk_master); + clk_disable(data->clk_master); } else { dev_dbg(data->master, "disabled. Skipping TLB invalidation @ %#x\n", iova); @@ -575,25 +602,52 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) } data->clk = devm_clk_get(dev, "sysmmu"); - if (IS_ERR(data->clk)) { - dev_err(dev, "Failed to get clock!\n"); - return PTR_ERR(data->clk); - } else { + if (!IS_ERR(data->clk)) { ret = clk_prepare(data->clk); if (ret) { dev_err(dev, "Failed to prepare clk\n"); return ret; } + } else { + data->clk = NULL; + } + + data->aclk = devm_clk_get(dev, "aclk"); + if (!IS_ERR(data->aclk)) { + ret = clk_prepare(data->aclk); + if (ret) { + dev_err(dev, "Failed to prepare aclk\n"); + return ret; + } + } else { + data->aclk = NULL; + } + + data->pclk = devm_clk_get(dev, "pclk"); + if (!IS_ERR(data->pclk)) { + ret = clk_prepare(data->pclk); + if (ret) { + dev_err(dev, "Failed to prepare pclk\n"); + return ret; + } + } else { + data->pclk = NULL; + } + + if (!data->clk && (!data->aclk || !data->pclk)) { + dev_err(dev, "Failed to get device clock(s)!\n"); + return -ENOSYS; } data->clk_master = devm_clk_get(dev, "master"); if (!IS_ERR(data->clk_master)) { ret = clk_prepare(data->clk_master); if (ret) { - clk_unprepare(data->clk); dev_err(dev, "Failed to prepare master's clk\n"); return ret; } + } else { + data->clk_master = NULL; } data->sysmmu = dev; @@ -601,6 +655,14 @@ static int __init exynos_sysmmu_probe(struct platform_device *pdev) platform_set_drvdata(pdev, data); + __sysmmu_get_version(data); + if (PG_ENT_SHIFT < 0) { + if (MMU_MAJ_VER(data->version) < 5) + PG_ENT_SHIFT = SYSMMU_PG_ENT_SHIFT; + else + PG_ENT_SHIFT = SYSMMU_V5_PG_ENT_SHIFT; + } + pm_runtime_enable(dev); return 0; @@ -650,28 +712,38 @@ static struct platform_driver exynos_sysmmu_driver __refdata = { } }; -static inline void pgtable_flush(void *vastart, void *vaend) +static inline void update_pte(sysmmu_pte_t *ent, sysmmu_pte_t val) { - dmac_flush_range(vastart, vaend); - outer_flush_range(virt_to_phys(vastart), - virt_to_phys(vaend)); + dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), sizeof(*ent), + DMA_TO_DEVICE); + *ent = val; + dma_sync_single_for_device(dma_dev, virt_to_phys(ent), sizeof(*ent), + DMA_TO_DEVICE); } static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) { struct exynos_iommu_domain *domain; + dma_addr_t handle; int i; - if (type != IOMMU_DOMAIN_UNMANAGED) - return NULL; + /* Check if correct PTE offsets are initialized */ + BUG_ON(PG_ENT_SHIFT < 0 || !dma_dev); domain = kzalloc(sizeof(*domain), GFP_KERNEL); if (!domain) return NULL; + if (type == IOMMU_DOMAIN_DMA) { + if (iommu_get_dma_cookie(&domain->domain) != 0) + goto err_pgtable; + } else if (type != IOMMU_DOMAIN_UNMANAGED) { + goto err_pgtable; + } + domain->pgtable = (sysmmu_pte_t *)__get_free_pages(GFP_KERNEL, 2); if (!domain->pgtable) - goto err_pgtable; + goto err_dma_cookie; domain->lv2entcnt = (short *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); if (!domain->lv2entcnt) @@ -689,7 +761,10 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) domain->pgtable[i + 7] = ZERO_LV2LINK; } - pgtable_flush(domain->pgtable, domain->pgtable + NUM_LV1ENTRIES); + handle = dma_map_single(dma_dev, domain->pgtable, LV1TABLE_SIZE, + DMA_TO_DEVICE); + /* For mapping page table entries we rely on dma == phys */ + BUG_ON(handle != virt_to_phys(domain->pgtable)); spin_lock_init(&domain->lock); spin_lock_init(&domain->pgtablelock); @@ -703,6 +778,9 @@ static struct iommu_domain *exynos_iommu_domain_alloc(unsigned type) err_counter: free_pages((unsigned long)domain->pgtable, 2); +err_dma_cookie: + if (type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(&domain->domain); err_pgtable: kfree(domain); return NULL; @@ -727,16 +805,62 @@ static void exynos_iommu_domain_free(struct iommu_domain *iommu_domain) spin_unlock_irqrestore(&domain->lock, flags); + if (iommu_domain->type == IOMMU_DOMAIN_DMA) + iommu_put_dma_cookie(iommu_domain); + + dma_unmap_single(dma_dev, virt_to_phys(domain->pgtable), LV1TABLE_SIZE, + DMA_TO_DEVICE); + for (i = 0; i < NUM_LV1ENTRIES; i++) - if (lv1ent_page(domain->pgtable + i)) + if (lv1ent_page(domain->pgtable + i)) { + phys_addr_t base = lv2table_base(domain->pgtable + i); + + dma_unmap_single(dma_dev, base, LV2TABLE_SIZE, + DMA_TO_DEVICE); kmem_cache_free(lv2table_kmem_cache, - phys_to_virt(lv2table_base(domain->pgtable + i))); + phys_to_virt(base)); + } free_pages((unsigned long)domain->pgtable, 2); free_pages((unsigned long)domain->lv2entcnt, 1); kfree(domain); } +static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, + struct device *dev) +{ + struct exynos_iommu_owner *owner = dev->archdata.iommu; + struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); + phys_addr_t pagetable = virt_to_phys(domain->pgtable); + struct sysmmu_drvdata *data, *next; + unsigned long flags; + bool found = false; + + if (!has_sysmmu(dev) || owner->domain != iommu_domain) + return; + + spin_lock_irqsave(&domain->lock, flags); + list_for_each_entry_safe(data, next, &domain->clients, domain_node) { + if (data->master == dev) { + if (__sysmmu_disable(data)) { + data->master = NULL; + list_del_init(&data->domain_node); + } + pm_runtime_put(data->sysmmu); + found = true; + } + } + spin_unlock_irqrestore(&domain->lock, flags); + + owner->domain = NULL; + + if (found) + dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", + __func__, &pagetable); + else + dev_err(dev, "%s: No IOMMU is attached\n", __func__); +} + static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, struct device *dev) { @@ -750,6 +874,9 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, if (!has_sysmmu(dev)) return -ENODEV; + if (owner->domain) + exynos_iommu_detach_device(owner->domain, dev); + list_for_each_entry(data, &owner->controllers, owner_node) { pm_runtime_get_sync(data->sysmmu); ret = __sysmmu_enable(data, pagetable, domain); @@ -768,44 +895,13 @@ static int exynos_iommu_attach_device(struct iommu_domain *iommu_domain, return ret; } + owner->domain = iommu_domain; dev_dbg(dev, "%s: Attached IOMMU with pgtable %pa %s\n", __func__, &pagetable, (ret == 0) ? "" : ", again"); return ret; } -static void exynos_iommu_detach_device(struct iommu_domain *iommu_domain, - struct device *dev) -{ - struct exynos_iommu_domain *domain = to_exynos_domain(iommu_domain); - phys_addr_t pagetable = virt_to_phys(domain->pgtable); - struct sysmmu_drvdata *data, *next; - unsigned long flags; - bool found = false; - - if (!has_sysmmu(dev)) - return; - - spin_lock_irqsave(&domain->lock, flags); - list_for_each_entry_safe(data, next, &domain->clients, domain_node) { - if (data->master == dev) { - if (__sysmmu_disable(data)) { - data->master = NULL; - list_del_init(&data->domain_node); - } - pm_runtime_put(data->sysmmu); - found = true; - } - } - spin_unlock_irqrestore(&domain->lock, flags); - - if (found) - dev_dbg(dev, "%s: Detached IOMMU with pgtable %pa\n", - __func__, &pagetable); - else - dev_err(dev, "%s: No IOMMU is attached\n", __func__); -} - static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, sysmmu_pte_t *sent, sysmmu_iova_t iova, short *pgcounter) { @@ -819,15 +915,14 @@ static sysmmu_pte_t *alloc_lv2entry(struct exynos_iommu_domain *domain, bool need_flush_flpd_cache = lv1ent_zero(sent); pent = kmem_cache_zalloc(lv2table_kmem_cache, GFP_ATOMIC); - BUG_ON((unsigned int)pent & (LV2TABLE_SIZE - 1)); + BUG_ON((uintptr_t)pent & (LV2TABLE_SIZE - 1)); if (!pent) return ERR_PTR(-ENOMEM); - *sent = mk_lv1ent_page(virt_to_phys(pent)); + update_pte(sent, mk_lv1ent_page(virt_to_phys(pent))); kmemleak_ignore(pent); *pgcounter = NUM_LV2ENTRIES; - pgtable_flush(pent, pent + NUM_LV2ENTRIES); - pgtable_flush(sent, sent + 1); + dma_map_single(dma_dev, pent, LV2TABLE_SIZE, DMA_TO_DEVICE); /* * If pre-fetched SLPD is a faulty SLPD in zero_l2_table, @@ -880,9 +975,7 @@ static int lv1set_section(struct exynos_iommu_domain *domain, *pgcnt = 0; } - *sent = mk_lv1ent_sect(paddr); - - pgtable_flush(sent, sent + 1); + update_pte(sent, mk_lv1ent_sect(paddr)); spin_lock(&domain->lock); if (lv1ent_page_zero(sent)) { @@ -906,12 +999,15 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, if (WARN_ON(!lv2ent_fault(pent))) return -EADDRINUSE; - *pent = mk_lv2ent_spage(paddr); - pgtable_flush(pent, pent + 1); + update_pte(pent, mk_lv2ent_spage(paddr)); *pgcnt -= 1; } else { /* size == LPAGE_SIZE */ int i; + dma_addr_t pent_base = virt_to_phys(pent); + dma_sync_single_for_cpu(dma_dev, pent_base, + sizeof(*pent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); for (i = 0; i < SPAGES_PER_LPAGE; i++, pent++) { if (WARN_ON(!lv2ent_fault(pent))) { if (i > 0) @@ -921,7 +1017,9 @@ static int lv2set_page(sysmmu_pte_t *pent, phys_addr_t paddr, size_t size, *pent = mk_lv2ent_lpage(paddr); } - pgtable_flush(pent - SPAGES_PER_LPAGE, pent); + dma_sync_single_for_device(dma_dev, pent_base, + sizeof(*pent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); *pgcnt -= SPAGES_PER_LPAGE; } @@ -1031,8 +1129,7 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, } /* workaround for h/w bug in System MMU v3.3 */ - *ent = ZERO_LV2LINK; - pgtable_flush(ent, ent + 1); + update_pte(ent, ZERO_LV2LINK); size = SECT_SIZE; goto done; } @@ -1053,9 +1150,8 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, } if (lv2ent_small(ent)) { - *ent = 0; + update_pte(ent, 0); size = SPAGE_SIZE; - pgtable_flush(ent, ent + 1); domain->lv2entcnt[lv1ent_offset(iova)] += 1; goto done; } @@ -1066,9 +1162,13 @@ static size_t exynos_iommu_unmap(struct iommu_domain *iommu_domain, goto err; } + dma_sync_single_for_cpu(dma_dev, virt_to_phys(ent), + sizeof(*ent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); memset(ent, 0, sizeof(*ent) * SPAGES_PER_LPAGE); - pgtable_flush(ent, ent + SPAGES_PER_LPAGE); - + dma_sync_single_for_device(dma_dev, virt_to_phys(ent), + sizeof(*ent) * SPAGES_PER_LPAGE, + DMA_TO_DEVICE); size = LPAGE_SIZE; domain->lv2entcnt[lv1ent_offset(iova)] += SPAGES_PER_LPAGE; done: @@ -1114,28 +1214,32 @@ static phys_addr_t exynos_iommu_iova_to_phys(struct iommu_domain *iommu_domain, return phys; } +static struct iommu_group *get_device_iommu_group(struct device *dev) +{ + struct iommu_group *group; + + group = iommu_group_get(dev); + if (!group) + group = iommu_group_alloc(); + + return group; +} + static int exynos_iommu_add_device(struct device *dev) { struct iommu_group *group; - int ret; if (!has_sysmmu(dev)) return -ENODEV; - group = iommu_group_get(dev); + group = iommu_group_get_for_dev(dev); - if (!group) { - group = iommu_group_alloc(); - if (IS_ERR(group)) { - dev_err(dev, "Failed to allocate IOMMU group\n"); - return PTR_ERR(group); - } - } + if (IS_ERR(group)) + return PTR_ERR(group); - ret = iommu_group_add_device(group, dev); iommu_group_put(group); - return ret; + return 0; } static void exynos_iommu_remove_device(struct device *dev) @@ -1182,6 +1286,7 @@ static struct iommu_ops exynos_iommu_ops = { .unmap = exynos_iommu_unmap, .map_sg = default_iommu_map_sg, .iova_to_phys = exynos_iommu_iova_to_phys, + .device_group = get_device_iommu_group, .add_device = exynos_iommu_add_device, .remove_device = exynos_iommu_remove_device, .pgsize_bitmap = SECT_SIZE | LPAGE_SIZE | SPAGE_SIZE, @@ -1245,6 +1350,13 @@ static int __init exynos_iommu_of_setup(struct device_node *np) if (IS_ERR(pdev)) return PTR_ERR(pdev); + /* + * use the first registered sysmmu device for performing + * dma mapping operations on iommu page tables (cpu cache flush) + */ + if (!dma_dev) + dma_dev = &pdev->dev; + of_iommu_set_ops(np, &exynos_iommu_ops); return 0; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index ac7387686ddc..a2e1b7f14df2 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -1489,7 +1489,7 @@ static void iommu_disable_dev_iotlb(struct device_domain_info *info) { struct pci_dev *pdev; - if (dev_is_pci(info->dev)) + if (!dev_is_pci(info->dev)) return; pdev = to_pci_dev(info->dev); @@ -4367,7 +4367,7 @@ int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) rmrru->devices_cnt); if(ret < 0) return ret; - } else if (info->event == BUS_NOTIFY_DEL_DEVICE) { + } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) { dmar_remove_dev_scope(info, rmrr->segment, rmrru->devices, rmrru->devices_cnt); } @@ -4387,7 +4387,7 @@ int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) break; else if(ret < 0) return ret; - } else if (info->event == BUS_NOTIFY_DEL_DEVICE) { + } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) { if (dmar_remove_dev_scope(info, atsr->segment, atsru->devices, atsru->devices_cnt)) break; diff --git a/drivers/iommu/intel-svm.c b/drivers/iommu/intel-svm.c index 50464833d0b8..d9939fa9b588 100644 --- a/drivers/iommu/intel-svm.c +++ b/drivers/iommu/intel-svm.c @@ -249,12 +249,30 @@ static void intel_flush_pasid_dev(struct intel_svm *svm, struct intel_svm_dev *s static void intel_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) { struct intel_svm *svm = container_of(mn, struct intel_svm, notifier); + struct intel_svm_dev *sdev; + /* This might end up being called from exit_mmap(), *before* the page + * tables are cleared. And __mmu_notifier_release() will delete us from + * the list of notifiers so that our invalidate_range() callback doesn't + * get called when the page tables are cleared. So we need to protect + * against hardware accessing those page tables. + * + * We do it by clearing the entry in the PASID table and then flushing + * the IOTLB and the PASID table caches. This might upset hardware; + * perhaps we'll want to point the PASID to a dummy PGD (like the zero + * page) so that we end up taking a fault that the hardware really + * *has* to handle gracefully without affecting other processes. + */ svm->iommu->pasid_table[svm->pasid].val = 0; + wmb(); + + rcu_read_lock(); + list_for_each_entry_rcu(sdev, &svm->devs, list) { + intel_flush_pasid_dev(svm, sdev, svm->pasid); + intel_flush_svm_range_dev(svm, sdev, 0, -1, 0, !svm->mm); + } + rcu_read_unlock(); - /* There's no need to do any flush because we can't get here if there - * are any devices left anyway. */ - WARN_ON(!list_empty(&svm->devs)); } static const struct mmu_notifier_ops intel_mmuops = { @@ -379,7 +397,6 @@ int intel_svm_bind_mm(struct device *dev, int *pasid, int flags, struct svm_dev_ goto out; } iommu->pasid_table[svm->pasid].val = (u64)__pa(mm->pgd) | 1; - mm = NULL; } else iommu->pasid_table[svm->pasid].val = (u64)__pa(init_mm.pgd) | 1 | (1ULL << 11); wmb(); @@ -442,11 +459,11 @@ int intel_svm_unbind_mm(struct device *dev, int pasid) kfree_rcu(sdev, rcu); if (list_empty(&svm->devs)) { - mmu_notifier_unregister(&svm->notifier, svm->mm); idr_remove(&svm->iommu->pasid_idr, svm->pasid); if (svm->mm) - mmput(svm->mm); + mmu_notifier_unregister(&svm->notifier, svm->mm); + /* We mandate that no page faults may be outstanding * for the PASID when intel_svm_unbind_mm() is called. * If that is not obeyed, subtle errors will happen. @@ -507,6 +524,10 @@ static irqreturn_t prq_event_thread(int irq, void *d) struct intel_svm *svm = NULL; int head, tail, handled = 0; + /* Clear PPR bit before reading head/tail registers, to + * ensure that we get a new interrupt if needed. */ + writel(DMA_PRS_PPR, iommu->reg + DMAR_PRS_REG); + tail = dmar_readq(iommu->reg + DMAR_PQT_REG) & PRQ_RING_MASK; head = dmar_readq(iommu->reg + DMAR_PQH_REG) & PRQ_RING_MASK; while (head != tail) { @@ -551,6 +572,9 @@ static irqreturn_t prq_event_thread(int irq, void *d) * any faults on kernel addresses. */ if (!svm->mm) goto bad_req; + /* If the mm is already defunct, don't handle faults. */ + if (!atomic_inc_not_zero(&svm->mm->mm_users)) + goto bad_req; down_read(&svm->mm->mmap_sem); vma = find_extend_vma(svm->mm, address); if (!vma || address < vma->vm_start) @@ -567,6 +591,7 @@ static irqreturn_t prq_event_thread(int irq, void *d) result = QI_RESP_SUCCESS; invalid: up_read(&svm->mm->mmap_sem); + mmput(svm->mm); bad_req: /* Accounting for major/minor faults? */ rcu_read_lock(); diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index c12ba4516df2..ac596928f6b4 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -629,7 +629,7 @@ static void iommu_disable_irq_remapping(struct intel_iommu *iommu) raw_spin_lock_irqsave(&iommu->register_lock, flags); - sts = dmar_readq(iommu->reg + DMAR_GSTS_REG); + sts = readl(iommu->reg + DMAR_GSTS_REG); if (!(sts & DMA_GSTS_IRES)) goto end; diff --git a/drivers/iommu/io-pgtable-arm-v7s.c b/drivers/iommu/io-pgtable-arm-v7s.c new file mode 100644 index 000000000000..9488e3c97bcb --- /dev/null +++ b/drivers/iommu/io-pgtable-arm-v7s.c @@ -0,0 +1,846 @@ +/* + * CPU-agnostic ARM page table allocator. + * + * ARMv7 Short-descriptor format, supporting + * - Basic memory attributes + * - Simplified access permissions (AP[2:1] model) + * - Backwards-compatible TEX remap + * - Large pages/supersections (if indicated by the caller) + * + * Not supporting: + * - Legacy access permissions (AP[2:0] model) + * + * Almost certainly never supporting: + * - PXN + * - Domains + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see <http://www.gnu.org/licenses/>. + * + * Copyright (C) 2014-2015 ARM Limited + * Copyright (c) 2014-2015 MediaTek Inc. + */ + +#define pr_fmt(fmt) "arm-v7s io-pgtable: " fmt + +#include <linux/dma-mapping.h> +#include <linux/gfp.h> +#include <linux/iommu.h> +#include <linux/kernel.h> +#include <linux/kmemleak.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/types.h> + +#include <asm/barrier.h> + +#include "io-pgtable.h" + +/* Struct accessors */ +#define io_pgtable_to_data(x) \ + container_of((x), struct arm_v7s_io_pgtable, iop) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +/* + * We have 32 bits total; 12 bits resolved at level 1, 8 bits at level 2, + * and 12 bits in a page. With some carefully-chosen coefficients we can + * hide the ugly inconsistencies behind these macros and at least let the + * rest of the code pretend to be somewhat sane. + */ +#define ARM_V7S_ADDR_BITS 32 +#define _ARM_V7S_LVL_BITS(lvl) (16 - (lvl) * 4) +#define ARM_V7S_LVL_SHIFT(lvl) (ARM_V7S_ADDR_BITS - (4 + 8 * (lvl))) +#define ARM_V7S_TABLE_SHIFT 10 + +#define ARM_V7S_PTES_PER_LVL(lvl) (1 << _ARM_V7S_LVL_BITS(lvl)) +#define ARM_V7S_TABLE_SIZE(lvl) \ + (ARM_V7S_PTES_PER_LVL(lvl) * sizeof(arm_v7s_iopte)) + +#define ARM_V7S_BLOCK_SIZE(lvl) (1UL << ARM_V7S_LVL_SHIFT(lvl)) +#define ARM_V7S_LVL_MASK(lvl) ((u32)(~0U << ARM_V7S_LVL_SHIFT(lvl))) +#define ARM_V7S_TABLE_MASK ((u32)(~0U << ARM_V7S_TABLE_SHIFT)) +#define _ARM_V7S_IDX_MASK(lvl) (ARM_V7S_PTES_PER_LVL(lvl) - 1) +#define ARM_V7S_LVL_IDX(addr, lvl) ({ \ + int _l = lvl; \ + ((u32)(addr) >> ARM_V7S_LVL_SHIFT(_l)) & _ARM_V7S_IDX_MASK(_l); \ +}) + +/* + * Large page/supersection entries are effectively a block of 16 page/section + * entries, along the lines of the LPAE contiguous hint, but all with the + * same output address. For want of a better common name we'll call them + * "contiguous" versions of their respective page/section entries here, but + * noting the distinction (WRT to TLB maintenance) that they represent *one* + * entry repeated 16 times, not 16 separate entries (as in the LPAE case). + */ +#define ARM_V7S_CONT_PAGES 16 + +/* PTE type bits: these are all mixed up with XN/PXN bits in most cases */ +#define ARM_V7S_PTE_TYPE_TABLE 0x1 +#define ARM_V7S_PTE_TYPE_PAGE 0x2 +#define ARM_V7S_PTE_TYPE_CONT_PAGE 0x1 + +#define ARM_V7S_PTE_IS_VALID(pte) (((pte) & 0x3) != 0) +#define ARM_V7S_PTE_IS_TABLE(pte, lvl) (lvl == 1 && ((pte) & ARM_V7S_PTE_TYPE_TABLE)) + +/* Page table bits */ +#define ARM_V7S_ATTR_XN(lvl) BIT(4 * (2 - (lvl))) +#define ARM_V7S_ATTR_B BIT(2) +#define ARM_V7S_ATTR_C BIT(3) +#define ARM_V7S_ATTR_NS_TABLE BIT(3) +#define ARM_V7S_ATTR_NS_SECTION BIT(19) + +#define ARM_V7S_CONT_SECTION BIT(18) +#define ARM_V7S_CONT_PAGE_XN_SHIFT 15 + +/* + * The attribute bits are consistently ordered*, but occupy bits [17:10] of + * a level 1 PTE vs. bits [11:4] at level 2. Thus we define the individual + * fields relative to that 8-bit block, plus a total shift relative to the PTE. + */ +#define ARM_V7S_ATTR_SHIFT(lvl) (16 - (lvl) * 6) + +#define ARM_V7S_ATTR_MASK 0xff +#define ARM_V7S_ATTR_AP0 BIT(0) +#define ARM_V7S_ATTR_AP1 BIT(1) +#define ARM_V7S_ATTR_AP2 BIT(5) +#define ARM_V7S_ATTR_S BIT(6) +#define ARM_V7S_ATTR_NG BIT(7) +#define ARM_V7S_TEX_SHIFT 2 +#define ARM_V7S_TEX_MASK 0x7 +#define ARM_V7S_ATTR_TEX(val) (((val) & ARM_V7S_TEX_MASK) << ARM_V7S_TEX_SHIFT) + +/* *well, except for TEX on level 2 large pages, of course :( */ +#define ARM_V7S_CONT_PAGE_TEX_SHIFT 6 +#define ARM_V7S_CONT_PAGE_TEX_MASK (ARM_V7S_TEX_MASK << ARM_V7S_CONT_PAGE_TEX_SHIFT) + +/* Simplified access permissions */ +#define ARM_V7S_PTE_AF ARM_V7S_ATTR_AP0 +#define ARM_V7S_PTE_AP_UNPRIV ARM_V7S_ATTR_AP1 +#define ARM_V7S_PTE_AP_RDONLY ARM_V7S_ATTR_AP2 + +/* Register bits */ +#define ARM_V7S_RGN_NC 0 +#define ARM_V7S_RGN_WBWA 1 +#define ARM_V7S_RGN_WT 2 +#define ARM_V7S_RGN_WB 3 + +#define ARM_V7S_PRRR_TYPE_DEVICE 1 +#define ARM_V7S_PRRR_TYPE_NORMAL 2 +#define ARM_V7S_PRRR_TR(n, type) (((type) & 0x3) << ((n) * 2)) +#define ARM_V7S_PRRR_DS0 BIT(16) +#define ARM_V7S_PRRR_DS1 BIT(17) +#define ARM_V7S_PRRR_NS0 BIT(18) +#define ARM_V7S_PRRR_NS1 BIT(19) +#define ARM_V7S_PRRR_NOS(n) BIT((n) + 24) + +#define ARM_V7S_NMRR_IR(n, attr) (((attr) & 0x3) << ((n) * 2)) +#define ARM_V7S_NMRR_OR(n, attr) (((attr) & 0x3) << ((n) * 2 + 16)) + +#define ARM_V7S_TTBR_S BIT(1) +#define ARM_V7S_TTBR_NOS BIT(5) +#define ARM_V7S_TTBR_ORGN_ATTR(attr) (((attr) & 0x3) << 3) +#define ARM_V7S_TTBR_IRGN_ATTR(attr) \ + ((((attr) & 0x1) << 6) | (((attr) & 0x2) >> 1)) + +#define ARM_V7S_TCR_PD1 BIT(5) + +typedef u32 arm_v7s_iopte; + +static bool selftest_running; + +struct arm_v7s_io_pgtable { + struct io_pgtable iop; + + arm_v7s_iopte *pgd; + struct kmem_cache *l2_tables; +}; + +static dma_addr_t __arm_v7s_dma_addr(void *pages) +{ + return (dma_addr_t)virt_to_phys(pages); +} + +static arm_v7s_iopte *iopte_deref(arm_v7s_iopte pte, int lvl) +{ + if (ARM_V7S_PTE_IS_TABLE(pte, lvl)) + pte &= ARM_V7S_TABLE_MASK; + else + pte &= ARM_V7S_LVL_MASK(lvl); + return phys_to_virt(pte); +} + +static void *__arm_v7s_alloc_table(int lvl, gfp_t gfp, + struct arm_v7s_io_pgtable *data) +{ + struct device *dev = data->iop.cfg.iommu_dev; + dma_addr_t dma; + size_t size = ARM_V7S_TABLE_SIZE(lvl); + void *table = NULL; + + if (lvl == 1) + table = (void *)__get_dma_pages(__GFP_ZERO, get_order(size)); + else if (lvl == 2) + table = kmem_cache_zalloc(data->l2_tables, gfp | GFP_DMA); + if (table && !selftest_running) { + dma = dma_map_single(dev, table, size, DMA_TO_DEVICE); + if (dma_mapping_error(dev, dma)) + goto out_free; + /* + * We depend on the IOMMU being able to work with any physical + * address directly, so if the DMA layer suggests otherwise by + * translating or truncating them, that bodes very badly... + */ + if (dma != virt_to_phys(table)) + goto out_unmap; + } + kmemleak_ignore(table); + return table; + +out_unmap: + dev_err(dev, "Cannot accommodate DMA translation for IOMMU page tables\n"); + dma_unmap_single(dev, dma, size, DMA_TO_DEVICE); +out_free: + if (lvl == 1) + free_pages((unsigned long)table, get_order(size)); + else + kmem_cache_free(data->l2_tables, table); + return NULL; +} + +static void __arm_v7s_free_table(void *table, int lvl, + struct arm_v7s_io_pgtable *data) +{ + struct device *dev = data->iop.cfg.iommu_dev; + size_t size = ARM_V7S_TABLE_SIZE(lvl); + + if (!selftest_running) + dma_unmap_single(dev, __arm_v7s_dma_addr(table), size, + DMA_TO_DEVICE); + if (lvl == 1) + free_pages((unsigned long)table, get_order(size)); + else + kmem_cache_free(data->l2_tables, table); +} + +static void __arm_v7s_pte_sync(arm_v7s_iopte *ptep, int num_entries, + struct io_pgtable_cfg *cfg) +{ + if (selftest_running) + return; + + dma_sync_single_for_device(cfg->iommu_dev, __arm_v7s_dma_addr(ptep), + num_entries * sizeof(*ptep), DMA_TO_DEVICE); +} +static void __arm_v7s_set_pte(arm_v7s_iopte *ptep, arm_v7s_iopte pte, + int num_entries, struct io_pgtable_cfg *cfg) +{ + int i; + + for (i = 0; i < num_entries; i++) + ptep[i] = pte; + + __arm_v7s_pte_sync(ptep, num_entries, cfg); +} + +static arm_v7s_iopte arm_v7s_prot_to_pte(int prot, int lvl, + struct io_pgtable_cfg *cfg) +{ + bool ap = !(cfg->quirks & IO_PGTABLE_QUIRK_NO_PERMS); + arm_v7s_iopte pte = ARM_V7S_ATTR_NG | ARM_V7S_ATTR_S | + ARM_V7S_ATTR_TEX(1); + + if (ap) { + pte |= ARM_V7S_PTE_AF | ARM_V7S_PTE_AP_UNPRIV; + if (!(prot & IOMMU_WRITE)) + pte |= ARM_V7S_PTE_AP_RDONLY; + } + pte <<= ARM_V7S_ATTR_SHIFT(lvl); + + if ((prot & IOMMU_NOEXEC) && ap) + pte |= ARM_V7S_ATTR_XN(lvl); + if (prot & IOMMU_CACHE) + pte |= ARM_V7S_ATTR_B | ARM_V7S_ATTR_C; + + return pte; +} + +static int arm_v7s_pte_to_prot(arm_v7s_iopte pte, int lvl) +{ + int prot = IOMMU_READ; + + if (pte & (ARM_V7S_PTE_AP_RDONLY << ARM_V7S_ATTR_SHIFT(lvl))) + prot |= IOMMU_WRITE; + if (pte & ARM_V7S_ATTR_C) + prot |= IOMMU_CACHE; + + return prot; +} + +static arm_v7s_iopte arm_v7s_pte_to_cont(arm_v7s_iopte pte, int lvl) +{ + if (lvl == 1) { + pte |= ARM_V7S_CONT_SECTION; + } else if (lvl == 2) { + arm_v7s_iopte xn = pte & ARM_V7S_ATTR_XN(lvl); + arm_v7s_iopte tex = pte & ARM_V7S_CONT_PAGE_TEX_MASK; + + pte ^= xn | tex | ARM_V7S_PTE_TYPE_PAGE; + pte |= (xn << ARM_V7S_CONT_PAGE_XN_SHIFT) | + (tex << ARM_V7S_CONT_PAGE_TEX_SHIFT) | + ARM_V7S_PTE_TYPE_CONT_PAGE; + } + return pte; +} + +static arm_v7s_iopte arm_v7s_cont_to_pte(arm_v7s_iopte pte, int lvl) +{ + if (lvl == 1) { + pte &= ~ARM_V7S_CONT_SECTION; + } else if (lvl == 2) { + arm_v7s_iopte xn = pte & BIT(ARM_V7S_CONT_PAGE_XN_SHIFT); + arm_v7s_iopte tex = pte & (ARM_V7S_CONT_PAGE_TEX_MASK << + ARM_V7S_CONT_PAGE_TEX_SHIFT); + + pte ^= xn | tex | ARM_V7S_PTE_TYPE_CONT_PAGE; + pte |= (xn >> ARM_V7S_CONT_PAGE_XN_SHIFT) | + (tex >> ARM_V7S_CONT_PAGE_TEX_SHIFT) | + ARM_V7S_PTE_TYPE_PAGE; + } + return pte; +} + +static bool arm_v7s_pte_is_cont(arm_v7s_iopte pte, int lvl) +{ + if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte, lvl)) + return pte & ARM_V7S_CONT_SECTION; + else if (lvl == 2) + return !(pte & ARM_V7S_PTE_TYPE_PAGE); + return false; +} + +static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *, unsigned long, + size_t, int, arm_v7s_iopte *); + +static int arm_v7s_init_pte(struct arm_v7s_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, int prot, + int lvl, int num_entries, arm_v7s_iopte *ptep) +{ + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_v7s_iopte pte = arm_v7s_prot_to_pte(prot, lvl, cfg); + int i; + + for (i = 0; i < num_entries; i++) + if (ARM_V7S_PTE_IS_TABLE(ptep[i], lvl)) { + /* + * We need to unmap and free the old table before + * overwriting it with a block entry. + */ + arm_v7s_iopte *tblp; + size_t sz = ARM_V7S_BLOCK_SIZE(lvl); + + tblp = ptep - ARM_V7S_LVL_IDX(iova, lvl); + if (WARN_ON(__arm_v7s_unmap(data, iova + i * sz, + sz, lvl, tblp) != sz)) + return -EINVAL; + } else if (ptep[i]) { + /* We require an unmap first */ + WARN_ON(!selftest_running); + return -EEXIST; + } + + pte |= ARM_V7S_PTE_TYPE_PAGE; + if (lvl == 1 && (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS)) + pte |= ARM_V7S_ATTR_NS_SECTION; + + if (num_entries > 1) + pte = arm_v7s_pte_to_cont(pte, lvl); + + pte |= paddr & ARM_V7S_LVL_MASK(lvl); + + __arm_v7s_set_pte(ptep, pte, num_entries, cfg); + return 0; +} + +static int __arm_v7s_map(struct arm_v7s_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, int prot, + int lvl, arm_v7s_iopte *ptep) +{ + struct io_pgtable_cfg *cfg = &data->iop.cfg; + arm_v7s_iopte pte, *cptep; + int num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); + + /* Find our entry at the current level */ + ptep += ARM_V7S_LVL_IDX(iova, lvl); + + /* If we can install a leaf entry at this level, then do so */ + if (num_entries) + return arm_v7s_init_pte(data, iova, paddr, prot, + lvl, num_entries, ptep); + + /* We can't allocate tables at the final level */ + if (WARN_ON(lvl == 2)) + return -EINVAL; + + /* Grab a pointer to the next level */ + pte = *ptep; + if (!pte) { + cptep = __arm_v7s_alloc_table(lvl + 1, GFP_ATOMIC, data); + if (!cptep) + return -ENOMEM; + + pte = virt_to_phys(cptep) | ARM_V7S_PTE_TYPE_TABLE; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_NS) + pte |= ARM_V7S_ATTR_NS_TABLE; + + __arm_v7s_set_pte(ptep, pte, 1, cfg); + } else { + cptep = iopte_deref(pte, lvl); + } + + /* Rinse, repeat */ + return __arm_v7s_map(data, iova, paddr, size, prot, lvl + 1, cptep); +} + +static int arm_v7s_map(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable *iop = &data->iop; + int ret; + + /* If no access, then nothing to do */ + if (!(prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + ret = __arm_v7s_map(data, iova, paddr, size, prot, 1, data->pgd); + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + if (iop->cfg.quirks & IO_PGTABLE_QUIRK_TLBI_ON_MAP) { + io_pgtable_tlb_add_flush(iop, iova, size, + ARM_V7S_BLOCK_SIZE(2), false); + io_pgtable_tlb_sync(iop); + } else { + wmb(); + } + + return ret; +} + +static void arm_v7s_free_pgtable(struct io_pgtable *iop) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_to_data(iop); + int i; + + for (i = 0; i < ARM_V7S_PTES_PER_LVL(1); i++) { + arm_v7s_iopte pte = data->pgd[i]; + + if (ARM_V7S_PTE_IS_TABLE(pte, 1)) + __arm_v7s_free_table(iopte_deref(pte, 1), 2, data); + } + __arm_v7s_free_table(data->pgd, 1, data); + kmem_cache_destroy(data->l2_tables); + kfree(data); +} + +static void arm_v7s_split_cont(struct arm_v7s_io_pgtable *data, + unsigned long iova, int idx, int lvl, + arm_v7s_iopte *ptep) +{ + struct io_pgtable *iop = &data->iop; + arm_v7s_iopte pte; + size_t size = ARM_V7S_BLOCK_SIZE(lvl); + int i; + + ptep -= idx & (ARM_V7S_CONT_PAGES - 1); + pte = arm_v7s_cont_to_pte(*ptep, lvl); + for (i = 0; i < ARM_V7S_CONT_PAGES; i++) { + ptep[i] = pte; + pte += size; + } + + __arm_v7s_pte_sync(ptep, ARM_V7S_CONT_PAGES, &iop->cfg); + + size *= ARM_V7S_CONT_PAGES; + io_pgtable_tlb_add_flush(iop, iova, size, size, true); + io_pgtable_tlb_sync(iop); +} + +static int arm_v7s_split_blk_unmap(struct arm_v7s_io_pgtable *data, + unsigned long iova, size_t size, + arm_v7s_iopte *ptep) +{ + unsigned long blk_start, blk_end, blk_size; + phys_addr_t blk_paddr; + arm_v7s_iopte table = 0; + int prot = arm_v7s_pte_to_prot(*ptep, 1); + + blk_size = ARM_V7S_BLOCK_SIZE(1); + blk_start = iova & ARM_V7S_LVL_MASK(1); + blk_end = blk_start + ARM_V7S_BLOCK_SIZE(1); + blk_paddr = *ptep & ARM_V7S_LVL_MASK(1); + + for (; blk_start < blk_end; blk_start += size, blk_paddr += size) { + arm_v7s_iopte *tablep; + + /* Unmap! */ + if (blk_start == iova) + continue; + + /* __arm_v7s_map expects a pointer to the start of the table */ + tablep = &table - ARM_V7S_LVL_IDX(blk_start, 1); + if (__arm_v7s_map(data, blk_start, blk_paddr, size, prot, 1, + tablep) < 0) { + if (table) { + /* Free the table we allocated */ + tablep = iopte_deref(table, 1); + __arm_v7s_free_table(tablep, 2, data); + } + return 0; /* Bytes unmapped */ + } + } + + __arm_v7s_set_pte(ptep, table, 1, &data->iop.cfg); + iova &= ~(blk_size - 1); + io_pgtable_tlb_add_flush(&data->iop, iova, blk_size, blk_size, true); + return size; +} + +static int __arm_v7s_unmap(struct arm_v7s_io_pgtable *data, + unsigned long iova, size_t size, int lvl, + arm_v7s_iopte *ptep) +{ + arm_v7s_iopte pte[ARM_V7S_CONT_PAGES]; + struct io_pgtable *iop = &data->iop; + int idx, i = 0, num_entries = size >> ARM_V7S_LVL_SHIFT(lvl); + + /* Something went horribly wrong and we ran out of page table */ + if (WARN_ON(lvl > 2)) + return 0; + + idx = ARM_V7S_LVL_IDX(iova, lvl); + ptep += idx; + do { + if (WARN_ON(!ARM_V7S_PTE_IS_VALID(ptep[i]))) + return 0; + pte[i] = ptep[i]; + } while (++i < num_entries); + + /* + * If we've hit a contiguous 'large page' entry at this level, it + * needs splitting first, unless we're unmapping the whole lot. + */ + if (num_entries <= 1 && arm_v7s_pte_is_cont(pte[0], lvl)) + arm_v7s_split_cont(data, iova, idx, lvl, ptep); + + /* If the size matches this level, we're in the right place */ + if (num_entries) { + size_t blk_size = ARM_V7S_BLOCK_SIZE(lvl); + + __arm_v7s_set_pte(ptep, 0, num_entries, &iop->cfg); + + for (i = 0; i < num_entries; i++) { + if (ARM_V7S_PTE_IS_TABLE(pte[i], lvl)) { + /* Also flush any partial walks */ + io_pgtable_tlb_add_flush(iop, iova, blk_size, + ARM_V7S_BLOCK_SIZE(lvl + 1), false); + io_pgtable_tlb_sync(iop); + ptep = iopte_deref(pte[i], lvl); + __arm_v7s_free_table(ptep, lvl + 1, data); + } else { + io_pgtable_tlb_add_flush(iop, iova, blk_size, + blk_size, true); + } + iova += blk_size; + } + return size; + } else if (lvl == 1 && !ARM_V7S_PTE_IS_TABLE(pte[0], lvl)) { + /* + * Insert a table at the next level to map the old region, + * minus the part we want to unmap + */ + return arm_v7s_split_blk_unmap(data, iova, size, ptep); + } + + /* Keep on walkin' */ + ptep = iopte_deref(pte[0], lvl); + return __arm_v7s_unmap(data, iova, size, lvl + 1, ptep); +} + +static int arm_v7s_unmap(struct io_pgtable_ops *ops, unsigned long iova, + size_t size) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); + size_t unmapped; + + unmapped = __arm_v7s_unmap(data, iova, size, 1, data->pgd); + if (unmapped) + io_pgtable_tlb_sync(&data->iop); + + return unmapped; +} + +static phys_addr_t arm_v7s_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct arm_v7s_io_pgtable *data = io_pgtable_ops_to_data(ops); + arm_v7s_iopte *ptep = data->pgd, pte; + int lvl = 0; + u32 mask; + + do { + pte = ptep[ARM_V7S_LVL_IDX(iova, ++lvl)]; + ptep = iopte_deref(pte, lvl); + } while (ARM_V7S_PTE_IS_TABLE(pte, lvl)); + + if (!ARM_V7S_PTE_IS_VALID(pte)) + return 0; + + mask = ARM_V7S_LVL_MASK(lvl); + if (arm_v7s_pte_is_cont(pte, lvl)) + mask *= ARM_V7S_CONT_PAGES; + return (pte & mask) | (iova & ~mask); +} + +static struct io_pgtable *arm_v7s_alloc_pgtable(struct io_pgtable_cfg *cfg, + void *cookie) +{ + struct arm_v7s_io_pgtable *data; + + if (cfg->ias > ARM_V7S_ADDR_BITS || cfg->oas > ARM_V7S_ADDR_BITS) + return NULL; + + if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | + IO_PGTABLE_QUIRK_NO_PERMS | + IO_PGTABLE_QUIRK_TLBI_ON_MAP)) + return NULL; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + + data->l2_tables = kmem_cache_create("io-pgtable_armv7s_l2", + ARM_V7S_TABLE_SIZE(2), + ARM_V7S_TABLE_SIZE(2), + SLAB_CACHE_DMA, NULL); + if (!data->l2_tables) + goto out_free_data; + + data->iop.ops = (struct io_pgtable_ops) { + .map = arm_v7s_map, + .unmap = arm_v7s_unmap, + .iova_to_phys = arm_v7s_iova_to_phys, + }; + + /* We have to do this early for __arm_v7s_alloc_table to work... */ + data->iop.cfg = *cfg; + + /* + * Unless the IOMMU driver indicates supersection support by + * having SZ_16M set in the initial bitmap, they won't be used. + */ + cfg->pgsize_bitmap &= SZ_4K | SZ_64K | SZ_1M | SZ_16M; + + /* TCR: T0SZ=0, disable TTBR1 */ + cfg->arm_v7s_cfg.tcr = ARM_V7S_TCR_PD1; + + /* + * TEX remap: the indices used map to the closest equivalent types + * under the non-TEX-remap interpretation of those attribute bits, + * excepting various implementation-defined aspects of shareability. + */ + cfg->arm_v7s_cfg.prrr = ARM_V7S_PRRR_TR(1, ARM_V7S_PRRR_TYPE_DEVICE) | + ARM_V7S_PRRR_TR(4, ARM_V7S_PRRR_TYPE_NORMAL) | + ARM_V7S_PRRR_TR(7, ARM_V7S_PRRR_TYPE_NORMAL) | + ARM_V7S_PRRR_DS0 | ARM_V7S_PRRR_DS1 | + ARM_V7S_PRRR_NS1 | ARM_V7S_PRRR_NOS(7); + cfg->arm_v7s_cfg.nmrr = ARM_V7S_NMRR_IR(7, ARM_V7S_RGN_WBWA) | + ARM_V7S_NMRR_OR(7, ARM_V7S_RGN_WBWA); + + /* Looking good; allocate a pgd */ + data->pgd = __arm_v7s_alloc_table(1, GFP_KERNEL, data); + if (!data->pgd) + goto out_free_data; + + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); + + /* TTBRs */ + cfg->arm_v7s_cfg.ttbr[0] = virt_to_phys(data->pgd) | + ARM_V7S_TTBR_S | ARM_V7S_TTBR_NOS | + ARM_V7S_TTBR_IRGN_ATTR(ARM_V7S_RGN_WBWA) | + ARM_V7S_TTBR_ORGN_ATTR(ARM_V7S_RGN_WBWA); + cfg->arm_v7s_cfg.ttbr[1] = 0; + return &data->iop; + +out_free_data: + kmem_cache_destroy(data->l2_tables); + kfree(data); + return NULL; +} + +struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns = { + .alloc = arm_v7s_alloc_pgtable, + .free = arm_v7s_free_pgtable, +}; + +#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S_SELFTEST + +static struct io_pgtable_cfg *cfg_cookie; + +static void dummy_tlb_flush_all(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static void dummy_tlb_add_flush(unsigned long iova, size_t size, + size_t granule, bool leaf, void *cookie) +{ + WARN_ON(cookie != cfg_cookie); + WARN_ON(!(size & cfg_cookie->pgsize_bitmap)); +} + +static void dummy_tlb_sync(void *cookie) +{ + WARN_ON(cookie != cfg_cookie); +} + +static struct iommu_gather_ops dummy_tlb_ops = { + .tlb_flush_all = dummy_tlb_flush_all, + .tlb_add_flush = dummy_tlb_add_flush, + .tlb_sync = dummy_tlb_sync, +}; + +#define __FAIL(ops) ({ \ + WARN(1, "selftest: test failed\n"); \ + selftest_running = false; \ + -EFAULT; \ +}) + +static int __init arm_v7s_do_selftests(void) +{ + struct io_pgtable_ops *ops; + struct io_pgtable_cfg cfg = { + .tlb = &dummy_tlb_ops, + .oas = 32, + .ias = 32, + .quirks = IO_PGTABLE_QUIRK_ARM_NS, + .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, + }; + unsigned int iova, size, iova_start; + unsigned int i, loopnr = 0; + + selftest_running = true; + + cfg_cookie = &cfg; + + ops = alloc_io_pgtable_ops(ARM_V7S, &cfg, &cfg); + if (!ops) { + pr_err("selftest: failed to allocate io pgtable ops\n"); + return -EINVAL; + } + + /* + * Initial sanity checks. + * Empty page tables shouldn't provide any translations. + */ + if (ops->iova_to_phys(ops, 42)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, SZ_1G + 42)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, SZ_2G + 42)) + return __FAIL(ops); + + /* + * Distinct mappings of different granule sizes. + */ + iova = 0; + i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG); + while (i != BITS_PER_LONG) { + size = 1UL << i; + if (ops->map(ops, iova, iova, size, IOMMU_READ | + IOMMU_WRITE | + IOMMU_NOEXEC | + IOMMU_CACHE)) + return __FAIL(ops); + + /* Overlapping mappings */ + if (!ops->map(ops, iova, iova + size, size, + IOMMU_READ | IOMMU_NOEXEC)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops); + + iova += SZ_16M; + i++; + i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i); + loopnr++; + } + + /* Partial unmap */ + i = 1; + size = 1UL << __ffs(cfg.pgsize_bitmap); + while (i < loopnr) { + iova_start = i * SZ_16M; + if (ops->unmap(ops, iova_start + size, size) != size) + return __FAIL(ops); + + /* Remap of partial unmap */ + if (ops->map(ops, iova_start + size, size, size, IOMMU_READ)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova_start + size + 42) + != (size + 42)) + return __FAIL(ops); + i++; + } + + /* Full unmap */ + iova = 0; + i = find_first_bit(&cfg.pgsize_bitmap, BITS_PER_LONG); + while (i != BITS_PER_LONG) { + size = 1UL << i; + + if (ops->unmap(ops, iova, size) != size) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova + 42)) + return __FAIL(ops); + + /* Remap full block */ + if (ops->map(ops, iova, iova, size, IOMMU_WRITE)) + return __FAIL(ops); + + if (ops->iova_to_phys(ops, iova + 42) != (iova + 42)) + return __FAIL(ops); + + iova += SZ_16M; + i++; + i = find_next_bit(&cfg.pgsize_bitmap, BITS_PER_LONG, i); + } + + free_io_pgtable_ops(ops); + + selftest_running = false; + + pr_info("self test ok\n"); + return 0; +} +subsys_initcall(arm_v7s_do_selftests); +#endif diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index 8bbcbfe7695c..f433b516098a 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -25,6 +25,7 @@ #include <linux/sizes.h> #include <linux/slab.h> #include <linux/types.h> +#include <linux/dma-mapping.h> #include <asm/barrier.h> @@ -445,7 +446,6 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, unsigned long blk_start, blk_end; phys_addr_t blk_paddr; arm_lpae_iopte table = 0; - struct io_pgtable_cfg *cfg = &data->iop.cfg; blk_start = iova & ~(blk_size - 1); blk_end = blk_start + blk_size; @@ -471,9 +471,9 @@ static int arm_lpae_split_blk_unmap(struct arm_lpae_io_pgtable *data, } } - __arm_lpae_set_pte(ptep, table, cfg); + __arm_lpae_set_pte(ptep, table, &data->iop.cfg); iova &= ~(blk_size - 1); - cfg->tlb->tlb_add_flush(iova, blk_size, blk_size, true, data->iop.cookie); + io_pgtable_tlb_add_flush(&data->iop, iova, blk_size, blk_size, true); return size; } @@ -482,8 +482,7 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, arm_lpae_iopte *ptep) { arm_lpae_iopte pte; - const struct iommu_gather_ops *tlb = data->iop.cfg.tlb; - void *cookie = data->iop.cookie; + struct io_pgtable *iop = &data->iop; size_t blk_size = ARM_LPAE_BLOCK_SIZE(lvl, data); /* Something went horribly wrong and we ran out of page table */ @@ -497,17 +496,17 @@ static int __arm_lpae_unmap(struct arm_lpae_io_pgtable *data, /* If the size matches this level, we're in the right place */ if (size == blk_size) { - __arm_lpae_set_pte(ptep, 0, &data->iop.cfg); + __arm_lpae_set_pte(ptep, 0, &iop->cfg); if (!iopte_leaf(pte, lvl)) { /* Also flush any partial walks */ - tlb->tlb_add_flush(iova, size, ARM_LPAE_GRANULE(data), - false, cookie); - tlb->tlb_sync(cookie); + io_pgtable_tlb_add_flush(iop, iova, size, + ARM_LPAE_GRANULE(data), false); + io_pgtable_tlb_sync(iop); ptep = iopte_deref(pte, data); __arm_lpae_free_pgtable(data, lvl + 1, ptep); } else { - tlb->tlb_add_flush(iova, size, size, true, cookie); + io_pgtable_tlb_add_flush(iop, iova, size, size, true); } return size; @@ -531,13 +530,12 @@ static int arm_lpae_unmap(struct io_pgtable_ops *ops, unsigned long iova, { size_t unmapped; struct arm_lpae_io_pgtable *data = io_pgtable_ops_to_data(ops); - struct io_pgtable *iop = &data->iop; arm_lpae_iopte *ptep = data->pgd; int lvl = ARM_LPAE_START_LVL(data); unmapped = __arm_lpae_unmap(data, iova, size, lvl, ptep); if (unmapped) - iop->cfg.tlb->tlb_sync(iop->cookie); + io_pgtable_tlb_sync(&data->iop); return unmapped; } @@ -661,8 +659,12 @@ static struct io_pgtable * arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg; - struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); + struct arm_lpae_io_pgtable *data; + if (cfg->quirks & ~IO_PGTABLE_QUIRK_ARM_NS) + return NULL; + + data = arm_lpae_alloc_pgtable(cfg); if (!data) return NULL; @@ -745,8 +747,13 @@ static struct io_pgtable * arm_64_lpae_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) { u64 reg, sl; - struct arm_lpae_io_pgtable *data = arm_lpae_alloc_pgtable(cfg); + struct arm_lpae_io_pgtable *data; + + /* The NS quirk doesn't apply at stage 2 */ + if (cfg->quirks) + return NULL; + data = arm_lpae_alloc_pgtable(cfg); if (!data) return NULL; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 6f2e319d4f04..876f6a76d288 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -33,6 +33,9 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = [ARM_64_LPAE_S1] = &io_pgtable_arm_64_lpae_s1_init_fns, [ARM_64_LPAE_S2] = &io_pgtable_arm_64_lpae_s2_init_fns, #endif +#ifdef CONFIG_IOMMU_IO_PGTABLE_ARMV7S + [ARM_V7S] = &io_pgtable_arm_v7s_init_fns, +#endif }; struct io_pgtable_ops *alloc_io_pgtable_ops(enum io_pgtable_fmt fmt, @@ -72,6 +75,6 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops) return; iop = container_of(ops, struct io_pgtable, ops); - iop->cfg.tlb->tlb_flush_all(iop->cookie); + io_pgtable_tlb_flush_all(iop); io_pgtable_init_table[iop->fmt]->free(iop); } diff --git a/drivers/iommu/io-pgtable.h b/drivers/iommu/io-pgtable.h index 36673c83de58..d4f502742e3b 100644 --- a/drivers/iommu/io-pgtable.h +++ b/drivers/iommu/io-pgtable.h @@ -1,5 +1,6 @@ #ifndef __IO_PGTABLE_H #define __IO_PGTABLE_H +#include <linux/bitops.h> /* * Public API for use by IOMMU drivers @@ -9,6 +10,7 @@ enum io_pgtable_fmt { ARM_32_LPAE_S2, ARM_64_LPAE_S1, ARM_64_LPAE_S2, + ARM_V7S, IO_PGTABLE_NUM_FMTS, }; @@ -45,8 +47,24 @@ struct iommu_gather_ops { * page table walker. */ struct io_pgtable_cfg { - #define IO_PGTABLE_QUIRK_ARM_NS (1 << 0) /* Set NS bit in PTEs */ - int quirks; + /* + * IO_PGTABLE_QUIRK_ARM_NS: (ARM formats) Set NS and NSTABLE bits in + * stage 1 PTEs, for hardware which insists on validating them + * even in non-secure state where they should normally be ignored. + * + * IO_PGTABLE_QUIRK_NO_PERMS: Ignore the IOMMU_READ, IOMMU_WRITE and + * IOMMU_NOEXEC flags and map everything with full access, for + * hardware which does not implement the permissions of a given + * format, and/or requires some format-specific default value. + * + * IO_PGTABLE_QUIRK_TLBI_ON_MAP: If the format forbids caching invalid + * (unmapped) entries but the hardware might do so anyway, perform + * TLB maintenance when mapping as well as when unmapping. + */ + #define IO_PGTABLE_QUIRK_ARM_NS BIT(0) + #define IO_PGTABLE_QUIRK_NO_PERMS BIT(1) + #define IO_PGTABLE_QUIRK_TLBI_ON_MAP BIT(2) + unsigned long quirks; unsigned long pgsize_bitmap; unsigned int ias; unsigned int oas; @@ -65,6 +83,13 @@ struct io_pgtable_cfg { u64 vttbr; u64 vtcr; } arm_lpae_s2_cfg; + + struct { + u32 ttbr[2]; + u32 tcr; + u32 nmrr; + u32 prrr; + } arm_v7s_cfg; }; }; @@ -121,18 +146,41 @@ void free_io_pgtable_ops(struct io_pgtable_ops *ops); * @fmt: The page table format. * @cookie: An opaque token provided by the IOMMU driver and passed back to * any callback routines. + * @tlb_sync_pending: Private flag for optimising out redundant syncs. * @cfg: A copy of the page table configuration. * @ops: The page table operations in use for this set of page tables. */ struct io_pgtable { enum io_pgtable_fmt fmt; void *cookie; + bool tlb_sync_pending; struct io_pgtable_cfg cfg; struct io_pgtable_ops ops; }; #define io_pgtable_ops_to_pgtable(x) container_of((x), struct io_pgtable, ops) +static inline void io_pgtable_tlb_flush_all(struct io_pgtable *iop) +{ + iop->cfg.tlb->tlb_flush_all(iop->cookie); + iop->tlb_sync_pending = true; +} + +static inline void io_pgtable_tlb_add_flush(struct io_pgtable *iop, + unsigned long iova, size_t size, size_t granule, bool leaf) +{ + iop->cfg.tlb->tlb_add_flush(iova, size, granule, leaf, iop->cookie); + iop->tlb_sync_pending = true; +} + +static inline void io_pgtable_tlb_sync(struct io_pgtable *iop) +{ + if (iop->tlb_sync_pending) { + iop->cfg.tlb->tlb_sync(iop->cookie); + iop->tlb_sync_pending = false; + } +} + /** * struct io_pgtable_init_fns - Alloc/free a set of page tables for a * particular format. @@ -149,5 +197,6 @@ extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_32_lpae_s2_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s1_init_fns; extern struct io_pgtable_init_fns io_pgtable_arm_64_lpae_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_arm_v7s_init_fns; #endif /* __IO_PGTABLE_H */ diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 0e3b0092ec92..bfd4f7c3b1d8 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -1314,6 +1314,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, unsigned long orig_iova = iova; unsigned int min_pagesz; size_t orig_size = size; + phys_addr_t orig_paddr = paddr; int ret = 0; if (unlikely(domain->ops->map == NULL || @@ -1358,7 +1359,7 @@ int iommu_map(struct iommu_domain *domain, unsigned long iova, if (ret) iommu_unmap(domain, orig_iova, orig_size - size); else - trace_map(orig_iova, paddr, orig_size); + trace_map(orig_iova, orig_paddr, orig_size); return ret; } diff --git a/drivers/iommu/mtk_iommu.c b/drivers/iommu/mtk_iommu.c new file mode 100644 index 000000000000..929a66a81b2b --- /dev/null +++ b/drivers/iommu/mtk_iommu.c @@ -0,0 +1,736 @@ +/* + * Copyright (c) 2015-2016 MediaTek Inc. + * Author: Yong Wu <yong.wu@mediatek.com> + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ +#include <linux/bug.h> +#include <linux/clk.h> +#include <linux/component.h> +#include <linux/device.h> +#include <linux/dma-iommu.h> +#include <linux/err.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iommu.h> +#include <linux/iopoll.h> +#include <linux/list.h> +#include <linux/of_address.h> +#include <linux/of_iommu.h> +#include <linux/of_irq.h> +#include <linux/of_platform.h> +#include <linux/platform_device.h> +#include <linux/slab.h> +#include <linux/spinlock.h> +#include <asm/barrier.h> +#include <dt-bindings/memory/mt8173-larb-port.h> +#include <soc/mediatek/smi.h> + +#include "io-pgtable.h" + +#define REG_MMU_PT_BASE_ADDR 0x000 + +#define REG_MMU_INVALIDATE 0x020 +#define F_ALL_INVLD 0x2 +#define F_MMU_INV_RANGE 0x1 + +#define REG_MMU_INVLD_START_A 0x024 +#define REG_MMU_INVLD_END_A 0x028 + +#define REG_MMU_INV_SEL 0x038 +#define F_INVLD_EN0 BIT(0) +#define F_INVLD_EN1 BIT(1) + +#define REG_MMU_STANDARD_AXI_MODE 0x048 +#define REG_MMU_DCM_DIS 0x050 + +#define REG_MMU_CTRL_REG 0x110 +#define F_MMU_PREFETCH_RT_REPLACE_MOD BIT(4) +#define F_MMU_TF_PROTECT_SEL(prot) (((prot) & 0x3) << 5) + +#define REG_MMU_IVRP_PADDR 0x114 +#define F_MMU_IVRP_PA_SET(pa) ((pa) >> 1) + +#define REG_MMU_INT_CONTROL0 0x120 +#define F_L2_MULIT_HIT_EN BIT(0) +#define F_TABLE_WALK_FAULT_INT_EN BIT(1) +#define F_PREETCH_FIFO_OVERFLOW_INT_EN BIT(2) +#define F_MISS_FIFO_OVERFLOW_INT_EN BIT(3) +#define F_PREFETCH_FIFO_ERR_INT_EN BIT(5) +#define F_MISS_FIFO_ERR_INT_EN BIT(6) +#define F_INT_CLR_BIT BIT(12) + +#define REG_MMU_INT_MAIN_CONTROL 0x124 +#define F_INT_TRANSLATION_FAULT BIT(0) +#define F_INT_MAIN_MULTI_HIT_FAULT BIT(1) +#define F_INT_INVALID_PA_FAULT BIT(2) +#define F_INT_ENTRY_REPLACEMENT_FAULT BIT(3) +#define F_INT_TLB_MISS_FAULT BIT(4) +#define F_INT_MISS_TRANSACTION_FIFO_FAULT BIT(5) +#define F_INT_PRETETCH_TRANSATION_FIFO_FAULT BIT(6) + +#define REG_MMU_CPE_DONE 0x12C + +#define REG_MMU_FAULT_ST1 0x134 + +#define REG_MMU_FAULT_VA 0x13c +#define F_MMU_FAULT_VA_MSK 0xfffff000 +#define F_MMU_FAULT_VA_WRITE_BIT BIT(1) +#define F_MMU_FAULT_VA_LAYER_BIT BIT(0) + +#define REG_MMU_INVLD_PA 0x140 +#define REG_MMU_INT_ID 0x150 +#define F_MMU0_INT_ID_LARB_ID(a) (((a) >> 7) & 0x7) +#define F_MMU0_INT_ID_PORT_ID(a) (((a) >> 2) & 0x1f) + +#define MTK_PROTECT_PA_ALIGN 128 + +struct mtk_iommu_suspend_reg { + u32 standard_axi_mode; + u32 dcm_dis; + u32 ctrl_reg; + u32 int_control0; + u32 int_main_control; +}; + +struct mtk_iommu_client_priv { + struct list_head client; + unsigned int mtk_m4u_id; + struct device *m4udev; +}; + +struct mtk_iommu_domain { + spinlock_t pgtlock; /* lock for page table */ + + struct io_pgtable_cfg cfg; + struct io_pgtable_ops *iop; + + struct iommu_domain domain; +}; + +struct mtk_iommu_data { + void __iomem *base; + int irq; + struct device *dev; + struct clk *bclk; + phys_addr_t protect_base; /* protect memory base */ + struct mtk_iommu_suspend_reg reg; + struct mtk_iommu_domain *m4u_dom; + struct iommu_group *m4u_group; + struct mtk_smi_iommu smi_imu; /* SMI larb iommu info */ +}; + +static struct iommu_ops mtk_iommu_ops; + +static struct mtk_iommu_domain *to_mtk_domain(struct iommu_domain *dom) +{ + return container_of(dom, struct mtk_iommu_domain, domain); +} + +static void mtk_iommu_tlb_flush_all(void *cookie) +{ + struct mtk_iommu_data *data = cookie; + + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); + writel_relaxed(F_ALL_INVLD, data->base + REG_MMU_INVALIDATE); + wmb(); /* Make sure the tlb flush all done */ +} + +static void mtk_iommu_tlb_add_flush_nosync(unsigned long iova, size_t size, + size_t granule, bool leaf, + void *cookie) +{ + struct mtk_iommu_data *data = cookie; + + writel_relaxed(F_INVLD_EN1 | F_INVLD_EN0, data->base + REG_MMU_INV_SEL); + + writel_relaxed(iova, data->base + REG_MMU_INVLD_START_A); + writel_relaxed(iova + size - 1, data->base + REG_MMU_INVLD_END_A); + writel_relaxed(F_MMU_INV_RANGE, data->base + REG_MMU_INVALIDATE); +} + +static void mtk_iommu_tlb_sync(void *cookie) +{ + struct mtk_iommu_data *data = cookie; + int ret; + u32 tmp; + + ret = readl_poll_timeout_atomic(data->base + REG_MMU_CPE_DONE, tmp, + tmp != 0, 10, 100000); + if (ret) { + dev_warn(data->dev, + "Partial TLB flush timed out, falling back to full flush\n"); + mtk_iommu_tlb_flush_all(cookie); + } + /* Clear the CPE status */ + writel_relaxed(0, data->base + REG_MMU_CPE_DONE); +} + +static const struct iommu_gather_ops mtk_iommu_gather_ops = { + .tlb_flush_all = mtk_iommu_tlb_flush_all, + .tlb_add_flush = mtk_iommu_tlb_add_flush_nosync, + .tlb_sync = mtk_iommu_tlb_sync, +}; + +static irqreturn_t mtk_iommu_isr(int irq, void *dev_id) +{ + struct mtk_iommu_data *data = dev_id; + struct mtk_iommu_domain *dom = data->m4u_dom; + u32 int_state, regval, fault_iova, fault_pa; + unsigned int fault_larb, fault_port; + bool layer, write; + + /* Read error info from registers */ + int_state = readl_relaxed(data->base + REG_MMU_FAULT_ST1); + fault_iova = readl_relaxed(data->base + REG_MMU_FAULT_VA); + layer = fault_iova & F_MMU_FAULT_VA_LAYER_BIT; + write = fault_iova & F_MMU_FAULT_VA_WRITE_BIT; + fault_iova &= F_MMU_FAULT_VA_MSK; + fault_pa = readl_relaxed(data->base + REG_MMU_INVLD_PA); + regval = readl_relaxed(data->base + REG_MMU_INT_ID); + fault_larb = F_MMU0_INT_ID_LARB_ID(regval); + fault_port = F_MMU0_INT_ID_PORT_ID(regval); + + if (report_iommu_fault(&dom->domain, data->dev, fault_iova, + write ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ)) { + dev_err_ratelimited( + data->dev, + "fault type=0x%x iova=0x%x pa=0x%x larb=%d port=%d layer=%d %s\n", + int_state, fault_iova, fault_pa, fault_larb, fault_port, + layer, write ? "write" : "read"); + } + + /* Interrupt clear */ + regval = readl_relaxed(data->base + REG_MMU_INT_CONTROL0); + regval |= F_INT_CLR_BIT; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + + mtk_iommu_tlb_flush_all(data); + + return IRQ_HANDLED; +} + +static void mtk_iommu_config(struct mtk_iommu_data *data, + struct device *dev, bool enable) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + struct mtk_smi_larb_iommu *larb_mmu; + unsigned int larbid, portid; + + head = dev->archdata.iommu; + list_for_each_entry_safe(cur, next, &head->client, client) { + larbid = MTK_M4U_TO_LARB(cur->mtk_m4u_id); + portid = MTK_M4U_TO_PORT(cur->mtk_m4u_id); + larb_mmu = &data->smi_imu.larb_imu[larbid]; + + dev_dbg(dev, "%s iommu port: %d\n", + enable ? "enable" : "disable", portid); + + if (enable) + larb_mmu->mmu |= MTK_SMI_MMU_EN(portid); + else + larb_mmu->mmu &= ~MTK_SMI_MMU_EN(portid); + } +} + +static int mtk_iommu_domain_finalise(struct mtk_iommu_data *data) +{ + struct mtk_iommu_domain *dom = data->m4u_dom; + + spin_lock_init(&dom->pgtlock); + + dom->cfg = (struct io_pgtable_cfg) { + .quirks = IO_PGTABLE_QUIRK_ARM_NS | + IO_PGTABLE_QUIRK_NO_PERMS | + IO_PGTABLE_QUIRK_TLBI_ON_MAP, + .pgsize_bitmap = mtk_iommu_ops.pgsize_bitmap, + .ias = 32, + .oas = 32, + .tlb = &mtk_iommu_gather_ops, + .iommu_dev = data->dev, + }; + + dom->iop = alloc_io_pgtable_ops(ARM_V7S, &dom->cfg, data); + if (!dom->iop) { + dev_err(data->dev, "Failed to alloc io pgtable\n"); + return -EINVAL; + } + + /* Update our support page sizes bitmap */ + mtk_iommu_ops.pgsize_bitmap = dom->cfg.pgsize_bitmap; + + writel(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], + data->base + REG_MMU_PT_BASE_ADDR); + return 0; +} + +static struct iommu_domain *mtk_iommu_domain_alloc(unsigned type) +{ + struct mtk_iommu_domain *dom; + + if (type != IOMMU_DOMAIN_DMA) + return NULL; + + dom = kzalloc(sizeof(*dom), GFP_KERNEL); + if (!dom) + return NULL; + + if (iommu_get_dma_cookie(&dom->domain)) { + kfree(dom); + return NULL; + } + + dom->domain.geometry.aperture_start = 0; + dom->domain.geometry.aperture_end = DMA_BIT_MASK(32); + dom->domain.geometry.force_aperture = true; + + return &dom->domain; +} + +static void mtk_iommu_domain_free(struct iommu_domain *domain) +{ + iommu_put_dma_cookie(domain); + kfree(to_mtk_domain(domain)); +} + +static int mtk_iommu_attach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + int ret; + + if (!priv) + return -ENODEV; + + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_dom) { + data->m4u_dom = dom; + ret = mtk_iommu_domain_finalise(data); + if (ret) { + data->m4u_dom = NULL; + return ret; + } + } else if (data->m4u_dom != dom) { + /* All the client devices should be in the same m4u domain */ + dev_err(dev, "try to attach into the error iommu domain\n"); + return -EPERM; + } + + mtk_iommu_config(data, dev, true); + return 0; +} + +static void mtk_iommu_detach_device(struct iommu_domain *domain, + struct device *dev) +{ + struct mtk_iommu_client_priv *priv = dev->archdata.iommu; + struct mtk_iommu_data *data; + + if (!priv) + return; + + data = dev_get_drvdata(priv->m4udev); + mtk_iommu_config(data, dev, false); +} + +static int mtk_iommu_map(struct iommu_domain *domain, unsigned long iova, + phys_addr_t paddr, size_t size, int prot) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + int ret; + + spin_lock_irqsave(&dom->pgtlock, flags); + ret = dom->iop->map(dom->iop, iova, paddr, size, prot); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return ret; +} + +static size_t mtk_iommu_unmap(struct iommu_domain *domain, + unsigned long iova, size_t size) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + size_t unmapsz; + + spin_lock_irqsave(&dom->pgtlock, flags); + unmapsz = dom->iop->unmap(dom->iop, iova, size); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return unmapsz; +} + +static phys_addr_t mtk_iommu_iova_to_phys(struct iommu_domain *domain, + dma_addr_t iova) +{ + struct mtk_iommu_domain *dom = to_mtk_domain(domain); + unsigned long flags; + phys_addr_t pa; + + spin_lock_irqsave(&dom->pgtlock, flags); + pa = dom->iop->iova_to_phys(dom->iop, iova); + spin_unlock_irqrestore(&dom->pgtlock, flags); + + return pa; +} + +static int mtk_iommu_add_device(struct device *dev) +{ + struct iommu_group *group; + + if (!dev->archdata.iommu) /* Not a iommu client device */ + return -ENODEV; + + group = iommu_group_get_for_dev(dev); + if (IS_ERR(group)) + return PTR_ERR(group); + + iommu_group_put(group); + return 0; +} + +static void mtk_iommu_remove_device(struct device *dev) +{ + struct mtk_iommu_client_priv *head, *cur, *next; + + head = dev->archdata.iommu; + if (!head) + return; + + list_for_each_entry_safe(cur, next, &head->client, client) { + list_del(&cur->client); + kfree(cur); + } + kfree(head); + dev->archdata.iommu = NULL; + + iommu_group_remove_device(dev); +} + +static struct iommu_group *mtk_iommu_device_group(struct device *dev) +{ + struct mtk_iommu_data *data; + struct mtk_iommu_client_priv *priv; + + priv = dev->archdata.iommu; + if (!priv) + return ERR_PTR(-ENODEV); + + /* All the client devices are in the same m4u iommu-group */ + data = dev_get_drvdata(priv->m4udev); + if (!data->m4u_group) { + data->m4u_group = iommu_group_alloc(); + if (IS_ERR(data->m4u_group)) + dev_err(dev, "Failed to allocate M4U IOMMU group\n"); + } + return data->m4u_group; +} + +static int mtk_iommu_of_xlate(struct device *dev, struct of_phandle_args *args) +{ + struct mtk_iommu_client_priv *head, *priv, *next; + struct platform_device *m4updev; + + if (args->args_count != 1) { + dev_err(dev, "invalid #iommu-cells(%d) property for IOMMU\n", + args->args_count); + return -EINVAL; + } + + if (!dev->archdata.iommu) { + /* Get the m4u device */ + m4updev = of_find_device_by_node(args->np); + of_node_put(args->np); + if (WARN_ON(!m4updev)) + return -EINVAL; + + head = kzalloc(sizeof(*head), GFP_KERNEL); + if (!head) + return -ENOMEM; + + dev->archdata.iommu = head; + INIT_LIST_HEAD(&head->client); + head->m4udev = &m4updev->dev; + } else { + head = dev->archdata.iommu; + } + + priv = kzalloc(sizeof(*priv), GFP_KERNEL); + if (!priv) + goto err_free_mem; + + priv->mtk_m4u_id = args->args[0]; + list_add_tail(&priv->client, &head->client); + + return 0; + +err_free_mem: + list_for_each_entry_safe(priv, next, &head->client, client) + kfree(priv); + kfree(head); + dev->archdata.iommu = NULL; + return -ENOMEM; +} + +static struct iommu_ops mtk_iommu_ops = { + .domain_alloc = mtk_iommu_domain_alloc, + .domain_free = mtk_iommu_domain_free, + .attach_dev = mtk_iommu_attach_device, + .detach_dev = mtk_iommu_detach_device, + .map = mtk_iommu_map, + .unmap = mtk_iommu_unmap, + .map_sg = default_iommu_map_sg, + .iova_to_phys = mtk_iommu_iova_to_phys, + .add_device = mtk_iommu_add_device, + .remove_device = mtk_iommu_remove_device, + .device_group = mtk_iommu_device_group, + .of_xlate = mtk_iommu_of_xlate, + .pgsize_bitmap = SZ_4K | SZ_64K | SZ_1M | SZ_16M, +}; + +static int mtk_iommu_hw_init(const struct mtk_iommu_data *data) +{ + u32 regval; + int ret; + + ret = clk_prepare_enable(data->bclk); + if (ret) { + dev_err(data->dev, "Failed to enable iommu bclk(%d)\n", ret); + return ret; + } + + regval = F_MMU_PREFETCH_RT_REPLACE_MOD | + F_MMU_TF_PROTECT_SEL(2); + writel_relaxed(regval, data->base + REG_MMU_CTRL_REG); + + regval = F_L2_MULIT_HIT_EN | + F_TABLE_WALK_FAULT_INT_EN | + F_PREETCH_FIFO_OVERFLOW_INT_EN | + F_MISS_FIFO_OVERFLOW_INT_EN | + F_PREFETCH_FIFO_ERR_INT_EN | + F_MISS_FIFO_ERR_INT_EN; + writel_relaxed(regval, data->base + REG_MMU_INT_CONTROL0); + + regval = F_INT_TRANSLATION_FAULT | + F_INT_MAIN_MULTI_HIT_FAULT | + F_INT_INVALID_PA_FAULT | + F_INT_ENTRY_REPLACEMENT_FAULT | + F_INT_TLB_MISS_FAULT | + F_INT_MISS_TRANSACTION_FIFO_FAULT | + F_INT_PRETETCH_TRANSATION_FIFO_FAULT; + writel_relaxed(regval, data->base + REG_MMU_INT_MAIN_CONTROL); + + writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base), + data->base + REG_MMU_IVRP_PADDR); + + writel_relaxed(0, data->base + REG_MMU_DCM_DIS); + writel_relaxed(0, data->base + REG_MMU_STANDARD_AXI_MODE); + + if (devm_request_irq(data->dev, data->irq, mtk_iommu_isr, 0, + dev_name(data->dev), (void *)data)) { + writel_relaxed(0, data->base + REG_MMU_PT_BASE_ADDR); + clk_disable_unprepare(data->bclk); + dev_err(data->dev, "Failed @ IRQ-%d Request\n", data->irq); + return -ENODEV; + } + + return 0; +} + +static int compare_of(struct device *dev, void *data) +{ + return dev->of_node == data; +} + +static int mtk_iommu_bind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + return component_bind_all(dev, &data->smi_imu); +} + +static void mtk_iommu_unbind(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + + component_unbind_all(dev, &data->smi_imu); +} + +static const struct component_master_ops mtk_iommu_com_ops = { + .bind = mtk_iommu_bind, + .unbind = mtk_iommu_unbind, +}; + +static int mtk_iommu_probe(struct platform_device *pdev) +{ + struct mtk_iommu_data *data; + struct device *dev = &pdev->dev; + struct resource *res; + struct component_match *match = NULL; + void *protect; + int i, larb_nr, ret; + + data = devm_kzalloc(dev, sizeof(*data), GFP_KERNEL); + if (!data) + return -ENOMEM; + data->dev = dev; + + /* Protect memory. HW will access here while translation fault.*/ + protect = devm_kzalloc(dev, MTK_PROTECT_PA_ALIGN * 2, GFP_KERNEL); + if (!protect) + return -ENOMEM; + data->protect_base = ALIGN(virt_to_phys(protect), MTK_PROTECT_PA_ALIGN); + + res = platform_get_resource(pdev, IORESOURCE_MEM, 0); + data->base = devm_ioremap_resource(dev, res); + if (IS_ERR(data->base)) + return PTR_ERR(data->base); + + data->irq = platform_get_irq(pdev, 0); + if (data->irq < 0) + return data->irq; + + data->bclk = devm_clk_get(dev, "bclk"); + if (IS_ERR(data->bclk)) + return PTR_ERR(data->bclk); + + larb_nr = of_count_phandle_with_args(dev->of_node, + "mediatek,larbs", NULL); + if (larb_nr < 0) + return larb_nr; + data->smi_imu.larb_nr = larb_nr; + + for (i = 0; i < larb_nr; i++) { + struct device_node *larbnode; + struct platform_device *plarbdev; + + larbnode = of_parse_phandle(dev->of_node, "mediatek,larbs", i); + if (!larbnode) + return -EINVAL; + + if (!of_device_is_available(larbnode)) + continue; + + plarbdev = of_find_device_by_node(larbnode); + of_node_put(larbnode); + if (!plarbdev) { + plarbdev = of_platform_device_create( + larbnode, NULL, + platform_bus_type.dev_root); + if (!plarbdev) + return -EPROBE_DEFER; + } + data->smi_imu.larb_imu[i].dev = &plarbdev->dev; + + component_match_add(dev, &match, compare_of, larbnode); + } + + platform_set_drvdata(pdev, data); + + ret = mtk_iommu_hw_init(data); + if (ret) + return ret; + + if (!iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, &mtk_iommu_ops); + + return component_master_add_with_match(dev, &mtk_iommu_com_ops, match); +} + +static int mtk_iommu_remove(struct platform_device *pdev) +{ + struct mtk_iommu_data *data = platform_get_drvdata(pdev); + + if (iommu_present(&platform_bus_type)) + bus_set_iommu(&platform_bus_type, NULL); + + free_io_pgtable_ops(data->m4u_dom->iop); + clk_disable_unprepare(data->bclk); + devm_free_irq(&pdev->dev, data->irq, data); + component_master_del(&pdev->dev, &mtk_iommu_com_ops); + return 0; +} + +static int __maybe_unused mtk_iommu_suspend(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + reg->standard_axi_mode = readl_relaxed(base + + REG_MMU_STANDARD_AXI_MODE); + reg->dcm_dis = readl_relaxed(base + REG_MMU_DCM_DIS); + reg->ctrl_reg = readl_relaxed(base + REG_MMU_CTRL_REG); + reg->int_control0 = readl_relaxed(base + REG_MMU_INT_CONTROL0); + reg->int_main_control = readl_relaxed(base + REG_MMU_INT_MAIN_CONTROL); + return 0; +} + +static int __maybe_unused mtk_iommu_resume(struct device *dev) +{ + struct mtk_iommu_data *data = dev_get_drvdata(dev); + struct mtk_iommu_suspend_reg *reg = &data->reg; + void __iomem *base = data->base; + + writel_relaxed(data->m4u_dom->cfg.arm_v7s_cfg.ttbr[0], + base + REG_MMU_PT_BASE_ADDR); + writel_relaxed(reg->standard_axi_mode, + base + REG_MMU_STANDARD_AXI_MODE); + writel_relaxed(reg->dcm_dis, base + REG_MMU_DCM_DIS); + writel_relaxed(reg->ctrl_reg, base + REG_MMU_CTRL_REG); + writel_relaxed(reg->int_control0, base + REG_MMU_INT_CONTROL0); + writel_relaxed(reg->int_main_control, base + REG_MMU_INT_MAIN_CONTROL); + writel_relaxed(F_MMU_IVRP_PA_SET(data->protect_base), + base + REG_MMU_IVRP_PADDR); + return 0; +} + +const struct dev_pm_ops mtk_iommu_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(mtk_iommu_suspend, mtk_iommu_resume) +}; + +static const struct of_device_id mtk_iommu_of_ids[] = { + { .compatible = "mediatek,mt8173-m4u", }, + {} +}; + +static struct platform_driver mtk_iommu_driver = { + .probe = mtk_iommu_probe, + .remove = mtk_iommu_remove, + .driver = { + .name = "mtk-iommu", + .of_match_table = mtk_iommu_of_ids, + .pm = &mtk_iommu_pm_ops, + } +}; + +static int mtk_iommu_init_fn(struct device_node *np) +{ + int ret; + struct platform_device *pdev; + + pdev = of_platform_device_create(np, NULL, platform_bus_type.dev_root); + if (!pdev) + return -ENOMEM; + + ret = platform_driver_register(&mtk_iommu_driver); + if (ret) { + pr_err("%s: Failed to register driver\n", __func__); + return ret; + } + + of_iommu_set_ops(np, &mtk_iommu_ops); + return 0; +} + +IOMMU_OF_DECLARE(mtkm4u, "mediatek,mt8173-m4u", mtk_iommu_init_fn); diff --git a/drivers/iommu/of_iommu.c b/drivers/iommu/of_iommu.c index 60ba238090d9..5fea665af99d 100644 --- a/drivers/iommu/of_iommu.c +++ b/drivers/iommu/of_iommu.c @@ -110,6 +110,7 @@ void of_iommu_set_ops(struct device_node *np, struct iommu_ops *ops) if (WARN_ON(!iommu)) return; + of_node_get(np); INIT_LIST_HEAD(&iommu->list); iommu->np = np; iommu->ops = ops; |