diff options
Diffstat (limited to 'drivers/iommu')
-rw-r--r-- | drivers/iommu/amd/amd_iommu_types.h | 8 | ||||
-rw-r--r-- | drivers/iommu/amd/init.c | 27 | ||||
-rw-r--r-- | drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c | 4 | ||||
-rw-r--r-- | drivers/iommu/intel/dmar.c | 24 | ||||
-rw-r--r-- | drivers/iommu/intel/iommu.c | 17 | ||||
-rw-r--r-- | drivers/iommu/intel/svm.c | 8 | ||||
-rw-r--r-- | drivers/iommu/io-pgtable-arm.c | 21 | ||||
-rw-r--r-- | drivers/iommu/iommu.c | 12 |
8 files changed, 96 insertions, 25 deletions
diff --git a/drivers/iommu/amd/amd_iommu_types.h b/drivers/iommu/amd/amd_iommu_types.h index ba74a722a400..52d70ca36169 100644 --- a/drivers/iommu/amd/amd_iommu_types.h +++ b/drivers/iommu/amd/amd_iommu_types.h @@ -257,7 +257,7 @@ #define DTE_IRQ_REMAP_INTCTL_MASK (0x3ULL << 60) #define DTE_IRQ_TABLE_LEN_MASK (0xfULL << 1) #define DTE_IRQ_REMAP_INTCTL (2ULL << 60) -#define DTE_IRQ_TABLE_LEN (8ULL << 1) +#define DTE_IRQ_TABLE_LEN (9ULL << 1) #define DTE_IRQ_REMAP_ENABLE 1ULL #define PAGE_MODE_NONE 0x00 @@ -409,7 +409,11 @@ extern bool amd_iommu_np_cache; /* Only true if all IOMMUs support device IOTLBs */ extern bool amd_iommu_iotlb_sup; -#define MAX_IRQS_PER_TABLE 256 +/* + * AMD IOMMU hardware only support 512 IRTEs despite + * the architectural limitation of 2048 entries. + */ +#define MAX_IRQS_PER_TABLE 512 #define IRQ_TABLE_ALIGNMENT 128 struct irq_remap_table { diff --git a/drivers/iommu/amd/init.c b/drivers/iommu/amd/init.c index 07d1f9913ce4..5ff5687a87c7 100644 --- a/drivers/iommu/amd/init.c +++ b/drivers/iommu/amd/init.c @@ -29,6 +29,7 @@ #include <asm/iommu_table.h> #include <asm/io_apic.h> #include <asm/irq_remapping.h> +#include <asm/set_memory.h> #include <linux/crash_dump.h> @@ -672,11 +673,27 @@ static void __init free_command_buffer(struct amd_iommu *iommu) free_pages((unsigned long)iommu->cmd_buf, get_order(CMD_BUFFER_SIZE)); } +static void *__init iommu_alloc_4k_pages(struct amd_iommu *iommu, + gfp_t gfp, size_t size) +{ + int order = get_order(size); + void *buf = (void *)__get_free_pages(gfp, order); + + if (buf && + iommu_feature(iommu, FEATURE_SNP) && + set_memory_4k((unsigned long)buf, (1 << order))) { + free_pages((unsigned long)buf, order); + buf = NULL; + } + + return buf; +} + /* allocates the memory where the IOMMU will log its events to */ static int __init alloc_event_buffer(struct amd_iommu *iommu) { - iommu->evt_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(EVT_BUFFER_SIZE)); + iommu->evt_buf = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, + EVT_BUFFER_SIZE); return iommu->evt_buf ? 0 : -ENOMEM; } @@ -715,8 +732,8 @@ static void __init free_event_buffer(struct amd_iommu *iommu) /* allocates the memory where the IOMMU will log its events to */ static int __init alloc_ppr_log(struct amd_iommu *iommu) { - iommu->ppr_log = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, - get_order(PPR_LOG_SIZE)); + iommu->ppr_log = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, + PPR_LOG_SIZE); return iommu->ppr_log ? 0 : -ENOMEM; } @@ -838,7 +855,7 @@ static int iommu_init_ga(struct amd_iommu *iommu) static int __init alloc_cwwb_sem(struct amd_iommu *iommu) { - iommu->cmd_sem = (void *)get_zeroed_page(GFP_KERNEL); + iommu->cmd_sem = iommu_alloc_4k_pages(iommu, GFP_KERNEL | __GFP_ZERO, 1); return iommu->cmd_sem ? 0 : -ENOMEM; } diff --git a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c index be4318044f96..702fbaa6c9ad 100644 --- a/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c +++ b/drivers/iommu/arm/arm-smmu/arm-smmu-qcom.c @@ -69,6 +69,10 @@ struct arm_smmu_device *qcom_smmu_impl_init(struct arm_smmu_device *smmu) { struct qcom_smmu *qsmmu; + /* Check to make sure qcom_scm has finished probing */ + if (!qcom_scm_is_available()) + return ERR_PTR(-EPROBE_DEFER); + qsmmu = devm_kzalloc(smmu->dev, sizeof(*qsmmu), GFP_KERNEL); if (!qsmmu) return ERR_PTR(-ENOMEM); diff --git a/drivers/iommu/intel/dmar.c b/drivers/iommu/intel/dmar.c index 404b40af31cb..b46dbfa6d0ed 100644 --- a/drivers/iommu/intel/dmar.c +++ b/drivers/iommu/intel/dmar.c @@ -333,6 +333,13 @@ static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info) dmar_iommu_notify_scope_dev(info); } +static inline void vf_inherit_msi_domain(struct pci_dev *pdev) +{ + struct pci_dev *physfn = pci_physfn(pdev); + + dev_set_msi_domain(&pdev->dev, dev_get_msi_domain(&physfn->dev)); +} + static int dmar_pci_bus_notifier(struct notifier_block *nb, unsigned long action, void *data) { @@ -342,8 +349,20 @@ static int dmar_pci_bus_notifier(struct notifier_block *nb, /* Only care about add/remove events for physical functions. * For VFs we actually do the lookup based on the corresponding * PF in device_to_iommu() anyway. */ - if (pdev->is_virtfn) + if (pdev->is_virtfn) { + /* + * Ensure that the VF device inherits the irq domain of the + * PF device. Ideally the device would inherit the domain + * from the bus, but DMAR can have multiple units per bus + * which makes this impossible. The VF 'bus' could inherit + * from the PF device, but that's yet another x86'sism to + * inflict on everybody else. + */ + if (action == BUS_NOTIFY_ADD_DEVICE) + vf_inherit_msi_domain(pdev); return NOTIFY_DONE; + } + if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_REMOVED_DEVICE) return NOTIFY_DONE; @@ -967,7 +986,8 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr) warn_invalid_dmar(phys_addr, " returns all ones"); goto unmap; } - iommu->vccap = dmar_readq(iommu->reg + DMAR_VCCAP_REG); + if (ecap_vcs(iommu->ecap)) + iommu->vccap = dmar_readq(iommu->reg + DMAR_VCCAP_REG); /* the registers might be more than one page */ map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap), diff --git a/drivers/iommu/intel/iommu.c b/drivers/iommu/intel/iommu.c index 8651f6d4dfa0..a49afa11673c 100644 --- a/drivers/iommu/intel/iommu.c +++ b/drivers/iommu/intel/iommu.c @@ -179,7 +179,7 @@ static int rwbf_quirk; * (used when kernel is launched w/ TXT) */ static int force_on = 0; -int intel_iommu_tboot_noforce; +static int intel_iommu_tboot_noforce; static int no_platform_optin; #define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry)) @@ -1833,7 +1833,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu) if (ecap_prs(iommu->ecap)) intel_svm_finish_prq(iommu); } - if (ecap_vcs(iommu->ecap) && vccap_pasid(iommu->vccap)) + if (vccap_pasid(iommu->vccap)) ioasid_unregister_allocator(&iommu->pasid_allocator); #endif @@ -2525,6 +2525,9 @@ struct dmar_domain *find_domain(struct device *dev) { struct device_domain_info *info; + if (unlikely(!dev || !dev->iommu)) + return NULL; + if (unlikely(attach_deferred(dev))) return NULL; @@ -3209,7 +3212,7 @@ static void register_pasid_allocator(struct intel_iommu *iommu) * is active. All vIOMMU allocators will eventually be calling the same * host allocator. */ - if (!ecap_vcs(iommu->ecap) || !vccap_pasid(iommu->vccap)) + if (!vccap_pasid(iommu->vccap)) return; pr_info("Register custom PASID allocator\n"); @@ -3815,9 +3818,8 @@ bounce_map_single(struct device *dev, phys_addr_t paddr, size_t size, * page aligned, we don't need to use a bounce page. */ if (!IS_ALIGNED(paddr | size, VTD_PAGE_SIZE)) { - tlb_addr = swiotlb_tbl_map_single(dev, - phys_to_dma_unencrypted(dev, io_tlb_start), - paddr, size, aligned_size, dir, attrs); + tlb_addr = swiotlb_tbl_map_single(dev, paddr, size, + aligned_size, dir, attrs); if (tlb_addr == DMA_MAPPING_ERROR) { goto swiotlb_error; } else { @@ -4882,7 +4884,8 @@ int __init intel_iommu_init(void) * Intel IOMMU is required for a TXT/tboot launch or platform * opt in, so enforce that. */ - force_on = tboot_force_iommu() || platform_optin_force_iommu(); + force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) || + platform_optin_force_iommu(); if (iommu_init_mempool()) { if (force_on) diff --git a/drivers/iommu/intel/svm.c b/drivers/iommu/intel/svm.c index f1861fa3d0e4..3242ebd0bca3 100644 --- a/drivers/iommu/intel/svm.c +++ b/drivers/iommu/intel/svm.c @@ -279,6 +279,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL); struct intel_svm_dev *sdev = NULL; struct dmar_domain *dmar_domain; + struct device_domain_info *info; struct intel_svm *svm = NULL; int ret = 0; @@ -310,6 +311,10 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, if (data->hpasid <= 0 || data->hpasid >= PASID_MAX) return -EINVAL; + info = get_domain_info(dev); + if (!info) + return -EINVAL; + dmar_domain = to_dmar_domain(domain); mutex_lock(&pasid_mutex); @@ -357,6 +362,7 @@ int intel_svm_bind_gpasid(struct iommu_domain *domain, struct device *dev, goto out; } sdev->dev = dev; + sdev->sid = PCI_DEVID(info->bus, info->devfn); /* Only count users if device has aux domains */ if (iommu_dev_feature_enabled(dev, IOMMU_DEV_FEAT_AUX)) @@ -1029,7 +1035,7 @@ no_pasid: resp.qw0 = QI_PGRP_PASID(req->pasid) | QI_PGRP_DID(req->rid) | QI_PGRP_PASID_P(req->pasid_present) | - QI_PGRP_PDP(req->pasid_present) | + QI_PGRP_PDP(req->priv_data_present) | QI_PGRP_RESP_CODE(result) | QI_PGRP_RESP_TYPE; resp.qw1 = QI_PGRP_IDX(req->prg_index) | diff --git a/drivers/iommu/io-pgtable-arm.c b/drivers/iommu/io-pgtable-arm.c index a7a9bc08dcd1..b34b00fadc45 100644 --- a/drivers/iommu/io-pgtable-arm.c +++ b/drivers/iommu/io-pgtable-arm.c @@ -417,7 +417,13 @@ static arm_lpae_iopte arm_lpae_prot_to_pte(struct arm_lpae_io_pgtable *data, << ARM_LPAE_PTE_ATTRINDX_SHIFT); } - if (prot & IOMMU_CACHE) + /* + * Also Mali has its own notions of shareability wherein its Inner + * domain covers the cores within the GPU, and its Outer domain is + * "outside the GPU" (i.e. either the Inner or System domain in CPU + * terms, depending on coherency). + */ + if (prot & IOMMU_CACHE && data->iop.fmt != ARM_MALI_LPAE) pte |= ARM_LPAE_PTE_SH_IS; else pte |= ARM_LPAE_PTE_SH_OS; @@ -761,7 +767,8 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) if (cfg->quirks & ~(IO_PGTABLE_QUIRK_ARM_NS | IO_PGTABLE_QUIRK_NON_STRICT | - IO_PGTABLE_QUIRK_ARM_TTBR1)) + IO_PGTABLE_QUIRK_ARM_TTBR1 | + IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) return NULL; data = arm_lpae_alloc_pgtable(cfg); @@ -773,10 +780,15 @@ arm_64_lpae_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) tcr->sh = ARM_LPAE_TCR_SH_IS; tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + if (cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA) + goto out_free_data; } else { tcr->sh = ARM_LPAE_TCR_SH_OS; tcr->irgn = ARM_LPAE_TCR_RGN_NC; - tcr->orgn = ARM_LPAE_TCR_RGN_NC; + if (!(cfg->quirks & IO_PGTABLE_QUIRK_ARM_OUTER_WBWA)) + tcr->orgn = ARM_LPAE_TCR_RGN_NC; + else + tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; } tg1 = cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1; @@ -1021,6 +1033,9 @@ arm_mali_lpae_alloc_pgtable(struct io_pgtable_cfg *cfg, void *cookie) cfg->arm_mali_lpae_cfg.transtab = virt_to_phys(data->pgd) | ARM_MALI_LPAE_TTBR_READ_INNER | ARM_MALI_LPAE_TTBR_ADRMODE_TABLE; + if (cfg->coherent_walk) + cfg->arm_mali_lpae_cfg.transtab |= ARM_MALI_LPAE_TTBR_SHARE_OUTER; + return &data->iop; out_free_data: diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index 8c470f451a32..0f4dc25d46c9 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -264,16 +264,18 @@ int iommu_probe_device(struct device *dev) */ iommu_alloc_default_domain(group, dev); - if (group->default_domain) + if (group->default_domain) { ret = __iommu_attach_device(group->default_domain, dev); + if (ret) { + iommu_group_put(group); + goto err_release; + } + } iommu_create_device_direct_mappings(group, dev); iommu_group_put(group); - if (ret) - goto err_release; - if (ops->probe_finalize) ops->probe_finalize(dev); @@ -2071,7 +2073,7 @@ EXPORT_SYMBOL_GPL(iommu_uapi_cache_invalidate); static int iommu_check_bind_data(struct iommu_gpasid_bind_data *data) { - u32 mask; + u64 mask; int i; if (data->version != IOMMU_GPASID_BIND_VERSION_1) |