From bb3a6b7845851d23cb826040b4c3c9c294e7cfb4 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:24 +0800 Subject: iommu/vt-d: Factor out dmar_alloc_dev_scope() for later reuse Factor out function dmar_alloc_dev_scope() from dmar_parse_dev_scope() for later reuse. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- include/linux/dmar.h | 1 + 1 file changed, 1 insertion(+) (limited to 'include') diff --git a/include/linux/dmar.h b/include/linux/dmar.h index eccb0c0c6cf6..1b08ce80bfbf 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -69,6 +69,7 @@ extern int dmar_table_init(void); extern int dmar_dev_scope_init(void); extern int dmar_parse_dev_scope(void *start, void *end, int *cnt, struct pci_dev ***devices, u16 segment); +extern void *dmar_alloc_dev_scope(void *start, void *end, int *cnt); extern void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt); /* Intel IOMMU detection */ -- cgit v1.2.3 From b94e4117f8c4ffb591b1e462364d725e3a1c63c4 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:25 +0800 Subject: iommu/vt-d: Move private structures and variables into intel-iommu.c Move private structures and variables into intel-iommu.c, which will help to simplify locking policy for hotplug. Also delete redundant declarations. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 31 +++++++++++++++++++++++++------ include/linux/dmar.h | 23 +---------------------- 2 files changed, 26 insertions(+), 28 deletions(-) (limited to 'include') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 5a411e83433c..fffe3d166662 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -380,6 +380,29 @@ struct device_domain_info { struct dmar_domain *domain; /* pointer to domain */ }; +struct dmar_rmrr_unit { + struct list_head list; /* list of rmrr units */ + struct acpi_dmar_header *hdr; /* ACPI header */ + u64 base_address; /* reserved base address*/ + u64 end_address; /* reserved end address */ + struct pci_dev **devices; /* target devices */ + int devices_cnt; /* target device count */ +}; + +struct dmar_atsr_unit { + struct list_head list; /* list of ATSR units */ + struct acpi_dmar_header *hdr; /* ACPI header */ + struct pci_dev **devices; /* target devices */ + int devices_cnt; /* target device count */ + u8 include_all:1; /* include all ports */ +}; + +static LIST_HEAD(dmar_atsr_units); +static LIST_HEAD(dmar_rmrr_units); + +#define for_each_rmrr_units(rmrr) \ + list_for_each_entry(rmrr, &dmar_rmrr_units, list) + static void flush_unmaps_timeout(unsigned long data); static DEFINE_TIMER(unmap_timer, flush_unmaps_timeout, 0, 0); @@ -403,6 +426,8 @@ static int timer_on; static long list_size; static void domain_remove_dev_info(struct dmar_domain *domain); +static void domain_remove_one_dev_info(struct dmar_domain *domain, + struct pci_dev *pdev); #ifdef CONFIG_INTEL_IOMMU_DEFAULT_ON int dmar_disabled = 0; @@ -2243,8 +2268,6 @@ static int __init si_domain_init(int hw) return 0; } -static void domain_remove_one_dev_info(struct dmar_domain *domain, - struct pci_dev *pdev); static int identity_mapping(struct pci_dev *pdev) { struct device_domain_info *info; @@ -3432,8 +3455,6 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ -LIST_HEAD(dmar_rmrr_units); - static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) { list_add(&rmrr->list, &dmar_rmrr_units); @@ -3470,8 +3491,6 @@ rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) rmrr->segment); } -static LIST_HEAD(dmar_atsr_units); - int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) { struct acpi_dmar_atsr *atsr; diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 1b08ce80bfbf..ea599d4ca9e0 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -139,28 +139,7 @@ extern int arch_setup_dmar_msi(unsigned int irq); #ifdef CONFIG_INTEL_IOMMU extern int iommu_detected, no_iommu; -extern struct list_head dmar_rmrr_units; -struct dmar_rmrr_unit { - struct list_head list; /* list of rmrr units */ - struct acpi_dmar_header *hdr; /* ACPI header */ - u64 base_address; /* reserved base address*/ - u64 end_address; /* reserved end address */ - struct pci_dev **devices; /* target devices */ - int devices_cnt; /* target device count */ -}; - -#define for_each_rmrr_units(rmrr) \ - list_for_each_entry(rmrr, &dmar_rmrr_units, list) - -struct dmar_atsr_unit { - struct list_head list; /* list of ATSR units */ - struct acpi_dmar_header *hdr; /* ACPI header */ - struct pci_dev **devices; /* target devices */ - int devices_cnt; /* target device count */ - u8 include_all:1; /* include all ports */ -}; - -int dmar_parse_rmrr_atsr_dev(void); +extern int dmar_parse_rmrr_atsr_dev(void); extern int dmar_parse_one_rmrr(struct acpi_dmar_header *header); extern int dmar_parse_one_atsr(struct acpi_dmar_header *header); extern int intel_iommu_init(void); -- cgit v1.2.3 From b683b230a244c3b2b3f6f3292e59d4a63298528b Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:32 +0800 Subject: iommu/vt-d: Introduce macro for_each_dev_scope() to walk device scope entries Introduce for_each_dev_scope()/for_each_active_dev_scope() to walk {active} device scope entries. This will help following RCU lock related patches. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 14 ++++--- drivers/iommu/intel-iommu.c | 100 ++++++++++++++++++++++---------------------- include/linux/dmar.h | 6 +++ 3 files changed, 64 insertions(+), 56 deletions(-) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 58dde75c5229..4ae6df27ad5d 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -170,9 +170,12 @@ int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt) { + int i; + struct pci_dev *tmp_dev; + if (*devices && *cnt) { - while (--*cnt >= 0) - pci_dev_put((*devices)[*cnt]); + for_each_active_dev_scope(*devices, *cnt, i, tmp_dev) + pci_dev_put(tmp_dev); kfree(*devices); *devices = NULL; *cnt = 0; @@ -402,10 +405,11 @@ static int dmar_pci_device_match(struct pci_dev *devices[], int cnt, struct pci_dev *dev) { int index; + struct pci_dev *tmp; while (dev) { - for (index = 0; index < cnt; index++) - if (dev == devices[index]) + for_each_active_dev_scope(devices, cnt, index, tmp) + if (dev == tmp) return 1; /* Check our parent */ @@ -452,7 +456,7 @@ int __init dmar_dev_scope_init(void) if (list_empty(&dmar_drhd_units)) goto fail; - list_for_each_entry(drhd, &dmar_drhd_units, list) { + for_each_drhd_unit(drhd) { ret = dmar_parse_dev(drhd); if (ret) goto fail; diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 7732c43871fa..bb98e37f2cf7 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -654,29 +654,31 @@ static void domain_update_iommu_cap(struct dmar_domain *domain) static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn) { struct dmar_drhd_unit *drhd = NULL; + struct intel_iommu *iommu; + struct pci_dev *dev; int i; - for_each_active_drhd_unit(drhd) { + for_each_active_iommu(iommu, drhd) { if (segment != drhd->segment) continue; - for (i = 0; i < drhd->devices_cnt; i++) { - if (drhd->devices[i] && - drhd->devices[i]->bus->number == bus && - drhd->devices[i]->devfn == devfn) - return drhd->iommu; - if (drhd->devices[i] && - drhd->devices[i]->subordinate && - drhd->devices[i]->subordinate->number <= bus && - drhd->devices[i]->subordinate->busn_res.end >= bus) - return drhd->iommu; + for_each_active_dev_scope(drhd->devices, + drhd->devices_cnt, i, dev) { + if (dev->bus->number == bus && dev->devfn == devfn) + goto out; + if (dev->subordinate && + dev->subordinate->number <= bus && + dev->subordinate->busn_res.end >= bus) + goto out; } if (drhd->include_all) - return drhd->iommu; + goto out; } + iommu = NULL; +out: - return NULL; + return iommu; } static void domain_flush_cache(struct dmar_domain *domain, @@ -2333,17 +2335,19 @@ static int domain_add_dev_info(struct dmar_domain *domain, static bool device_has_rmrr(struct pci_dev *dev) { struct dmar_rmrr_unit *rmrr; + struct pci_dev *tmp; int i; for_each_rmrr_units(rmrr) { - for (i = 0; i < rmrr->devices_cnt; i++) { - /* - * Return TRUE if this RMRR contains the device that - * is passed in. - */ - if (rmrr->devices[i] == dev) + /* + * Return TRUE if this RMRR contains the device that + * is passed in. + */ + for_each_active_dev_scope(rmrr->devices, + rmrr->devices_cnt, i, tmp) + if (tmp == dev) { return true; - } + } } return false; } @@ -2593,14 +2597,9 @@ static int __init init_dmars(void) */ printk(KERN_INFO "IOMMU: Setting RMRR:\n"); for_each_rmrr_units(rmrr) { - for (i = 0; i < rmrr->devices_cnt; i++) { - pdev = rmrr->devices[i]; - /* - * some BIOS lists non-exist devices in DMAR - * table. - */ - if (!pdev) - continue; + /* some BIOS lists non-exist devices in DMAR table. */ + for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, + i, pdev) { ret = iommu_prepare_rmrr_dev(rmrr, pdev); if (ret) printk(KERN_ERR @@ -3288,13 +3287,14 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quir static void __init init_no_remapping_devices(void) { struct dmar_drhd_unit *drhd; + struct pci_dev *dev; + int i; for_each_drhd_unit(drhd) { if (!drhd->include_all) { - int i; - for (i = 0; i < drhd->devices_cnt; i++) - if (drhd->devices[i] != NULL) - break; + for_each_active_dev_scope(drhd->devices, + drhd->devices_cnt, i, dev) + break; /* ignore DMAR unit if no pci devices exist */ if (i == drhd->devices_cnt) drhd->ignored = 1; @@ -3302,15 +3302,13 @@ static void __init init_no_remapping_devices(void) } for_each_active_drhd_unit(drhd) { - int i; if (drhd->include_all) continue; - for (i = 0; i < drhd->devices_cnt; i++) - if (drhd->devices[i] && - !IS_GFX_DEVICE(drhd->devices[i])) + for_each_active_dev_scope(drhd->devices, + drhd->devices_cnt, i, dev) + if (!IS_GFX_DEVICE(dev)) break; - if (i < drhd->devices_cnt) continue; @@ -3320,11 +3318,9 @@ static void __init init_no_remapping_devices(void) intel_iommu_gfx_mapped = 1; } else { drhd->ignored = 1; - for (i = 0; i < drhd->devices_cnt; i++) { - if (!drhd->devices[i]) - continue; - drhd->devices[i]->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; - } + for_each_active_dev_scope(drhd->devices, + drhd->devices_cnt, i, dev) + dev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; } } } @@ -3560,9 +3556,9 @@ static void intel_iommu_free_dmars(void) int dmar_find_matched_atsr_unit(struct pci_dev *dev) { - int i; + int i, ret = 1; struct pci_bus *bus; - struct pci_dev *bridge = NULL; + struct pci_dev *bridge = NULL, *tmp; struct acpi_dmar_atsr *atsr; struct dmar_atsr_unit *atsru; @@ -3583,22 +3579,24 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) if (atsr->segment != pci_domain_nr(dev->bus)) continue; - for (i = 0; i < atsru->devices_cnt; i++) - if (atsru->devices[i] == bridge) - return 1; + for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp) + if (tmp == bridge) + goto out; if (atsru->include_all) - return 1; + goto out; } + ret = 0; +out: - return 0; + return ret; } int __init dmar_parse_rmrr_atsr_dev(void) { struct dmar_rmrr_unit *rmrr; struct dmar_atsr_unit *atsr; - int ret = 0; + int ret; list_for_each_entry(rmrr, &dmar_rmrr_units, list) { ret = rmrr_parse_dev(rmrr); @@ -3612,7 +3610,7 @@ int __init dmar_parse_rmrr_atsr_dev(void) return ret; } - return ret; + return 0; } /* diff --git a/include/linux/dmar.h b/include/linux/dmar.h index ea599d4ca9e0..4b77fd8fde76 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -65,6 +65,12 @@ extern struct list_head dmar_drhd_units; list_for_each_entry(drhd, &dmar_drhd_units, list) \ if (i=drhd->iommu, 0) {} else +#define for_each_dev_scope(a, c, p, d) \ + for ((p) = 0; ((d) = (p) < (c) ? (a)[(p)] : NULL, (p) < (c)); (p)++) + +#define for_each_active_dev_scope(a, c, p, d) \ + for_each_dev_scope((a), (c), (p), (d)) if (!(d)) { continue; } else + extern int dmar_table_init(void); extern int dmar_dev_scope_init(void); extern int dmar_parse_dev_scope(void *start, void *end, int *cnt, -- cgit v1.2.3 From 3a5670e8ac932c10a3e50d9dc0ab1da4cc3041d7 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:33 +0800 Subject: iommu/vt-d: Introduce a rwsem to protect global data structures Introduce a global rwsem dmar_global_lock, which will be used to protect DMAR related global data structures from DMAR/PCI/memory device hotplug operations in process context. DMA and interrupt remapping related data structures are read most, and only change when memory/PCI/DMAR hotplug event happens. So a global rwsem solution is adopted for balance between simplicity and performance. For interrupt remapping driver, function intel_irq_remapping_supported(), dmar_table_init(), intel_enable_irq_remapping(), disable_irq_remapping(), reenable_irq_remapping() and enable_drhd_fault_handling() etc are called during booting, suspending and resuming with interrupt disabled, so no need to take the global lock. For interrupt remapping entry allocation, the locking model is: down_read(&dmar_global_lock); /* Find corresponding iommu */ iommu = map_hpet_to_ir(id); if (iommu) /* * Allocate remapping entry and mark entry busy, * the IOMMU won't be hot-removed until the * allocated entry has been released. */ index = alloc_irte(iommu, irq, 1); up_read(&dmar_global_lock); For DMA remmaping driver, we only uses the dmar_global_lock rwsem to protect functions which are only called in process context. For any function which may be called in interrupt context, we will use RCU to protect them in following patches. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 19 ++++++- drivers/iommu/intel-iommu.c | 22 +++++--- drivers/iommu/intel_irq_remapping.c | 108 ++++++++++++++++++++++++------------ include/linux/dmar.h | 2 + 4 files changed, 103 insertions(+), 48 deletions(-) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 4ae6df27ad5d..c9aca8841fa0 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -43,10 +43,19 @@ #include "irq_remapping.h" -/* No locks are needed as DMA remapping hardware unit - * list is constructed at boot time and hotplug of - * these units are not supported by the architecture. +/* + * Assumptions: + * 1) The hotplug framework guarentees that DMAR unit will be hot-added + * before IO devices managed by that unit. + * 2) The hotplug framework guarantees that DMAR unit will be hot-removed + * after IO devices managed by that unit. + * 3) Hotplug events are rare. + * + * Locking rules for DMA and interrupt remapping related global data structures: + * 1) Use dmar_global_lock in process context + * 2) Use RCU in interrupt context */ +DECLARE_RWSEM(dmar_global_lock); LIST_HEAD(dmar_drhd_units); struct acpi_table_header * __initdata dmar_tbl; @@ -565,6 +574,7 @@ int __init detect_intel_iommu(void) { int ret; + down_write(&dmar_global_lock); ret = dmar_table_detect(); if (ret) ret = check_zero_address(); @@ -582,6 +592,7 @@ int __init detect_intel_iommu(void) } early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size); dmar_tbl = NULL; + up_write(&dmar_global_lock); return ret ? 1 : -ENODEV; } @@ -1394,10 +1405,12 @@ static int __init dmar_free_unused_resources(void) if (irq_remapping_enabled || intel_iommu_enabled) return 0; + down_write(&dmar_global_lock); list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) { list_del(&dmaru->list); dmar_free_drhd(dmaru); } + up_write(&dmar_global_lock); return 0; } diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index bb98e37f2cf7..50d639a2df88 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3637,11 +3637,13 @@ static int device_notifier(struct notifier_block *nb, if (!domain) return 0; + down_read(&dmar_global_lock); domain_remove_one_dev_info(domain, pdev); if (!(domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE) && !(domain->flags & DOMAIN_FLAG_STATIC_IDENTITY) && list_empty(&domain->devices)) domain_exit(domain); + up_read(&dmar_global_lock); return 0; } @@ -3659,6 +3661,13 @@ int __init intel_iommu_init(void) /* VT-d is required for a TXT/tboot launch, so enforce that */ force_on = tboot_force_iommu(); + if (iommu_init_mempool()) { + if (force_on) + panic("tboot: Failed to initialize iommu memory\n"); + return -ENOMEM; + } + + down_write(&dmar_global_lock); if (dmar_table_init()) { if (force_on) panic("tboot: Failed to initialize DMAR table\n"); @@ -3681,12 +3690,6 @@ int __init intel_iommu_init(void) if (no_iommu || dmar_disabled) goto out_free_dmar; - if (iommu_init_mempool()) { - if (force_on) - panic("tboot: Failed to initialize iommu memory\n"); - goto out_free_dmar; - } - if (list_empty(&dmar_rmrr_units)) printk(KERN_INFO "DMAR: No RMRR found\n"); @@ -3696,7 +3699,7 @@ int __init intel_iommu_init(void) if (dmar_init_reserved_ranges()) { if (force_on) panic("tboot: Failed to reserve iommu ranges\n"); - goto out_free_mempool; + goto out_free_reserved_range; } init_no_remapping_devices(); @@ -3708,6 +3711,7 @@ int __init intel_iommu_init(void) printk(KERN_ERR "IOMMU: dmar init failed\n"); goto out_free_reserved_range; } + up_write(&dmar_global_lock); printk(KERN_INFO "PCI-DMA: Intel(R) Virtualization Technology for Directed I/O\n"); @@ -3729,10 +3733,10 @@ int __init intel_iommu_init(void) out_free_reserved_range: put_iova_domain(&reserved_iova_list); -out_free_mempool: - iommu_exit_mempool(); out_free_dmar: intel_iommu_free_dmars(); + up_write(&dmar_global_lock); + iommu_exit_mempool(); return ret; } diff --git a/drivers/iommu/intel_irq_remapping.c b/drivers/iommu/intel_irq_remapping.c index ef5f65dbafe9..9b174893f0f5 100644 --- a/drivers/iommu/intel_irq_remapping.c +++ b/drivers/iommu/intel_irq_remapping.c @@ -38,6 +38,17 @@ static struct ioapic_scope ir_ioapic[MAX_IO_APICS]; static struct hpet_scope ir_hpet[MAX_HPET_TBS]; static int ir_ioapic_num, ir_hpet_num; +/* + * Lock ordering: + * ->dmar_global_lock + * ->irq_2_ir_lock + * ->qi->q_lock + * ->iommu->register_lock + * Note: + * intel_irq_remap_ops.{supported,prepare,enable,disable,reenable} are called + * in single-threaded environment with interrupt disabled, so no need to tabke + * the dmar_global_lock. + */ static DEFINE_RAW_SPINLOCK(irq_2_ir_lock); static int __init parse_ioapics_under_ir(void); @@ -307,12 +318,14 @@ static int set_ioapic_sid(struct irte *irte, int apic) if (!irte) return -1; + down_read(&dmar_global_lock); for (i = 0; i < MAX_IO_APICS; i++) { if (ir_ioapic[i].id == apic) { sid = (ir_ioapic[i].bus << 8) | ir_ioapic[i].devfn; break; } } + up_read(&dmar_global_lock); if (sid == 0) { pr_warning("Failed to set source-id of IOAPIC (%d)\n", apic); @@ -332,12 +345,14 @@ static int set_hpet_sid(struct irte *irte, u8 id) if (!irte) return -1; + down_read(&dmar_global_lock); for (i = 0; i < MAX_HPET_TBS; i++) { if (ir_hpet[i].id == id) { sid = (ir_hpet[i].bus << 8) | ir_hpet[i].devfn; break; } } + up_read(&dmar_global_lock); if (sid == 0) { pr_warning("Failed to set source-id of HPET block (%d)\n", id); @@ -794,10 +809,16 @@ static int __init parse_ioapics_under_ir(void) static int __init ir_dev_scope_init(void) { + int ret; + if (!irq_remapping_enabled) return 0; - return dmar_dev_scope_init(); + down_write(&dmar_global_lock); + ret = dmar_dev_scope_init(); + up_write(&dmar_global_lock); + + return ret; } rootfs_initcall(ir_dev_scope_init); @@ -878,23 +899,27 @@ static int intel_setup_ioapic_entry(int irq, struct io_apic_irq_attr *attr) { int ioapic_id = mpc_ioapic_id(attr->ioapic); - struct intel_iommu *iommu = map_ioapic_to_ir(ioapic_id); + struct intel_iommu *iommu; struct IR_IO_APIC_route_entry *entry; struct irte irte; int index; + down_read(&dmar_global_lock); + iommu = map_ioapic_to_ir(ioapic_id); if (!iommu) { pr_warn("No mapping iommu for ioapic %d\n", ioapic_id); - return -ENODEV; - } - - entry = (struct IR_IO_APIC_route_entry *)route_entry; - - index = alloc_irte(iommu, irq, 1); - if (index < 0) { - pr_warn("Failed to allocate IRTE for ioapic %d\n", ioapic_id); - return -ENOMEM; + index = -ENODEV; + } else { + index = alloc_irte(iommu, irq, 1); + if (index < 0) { + pr_warn("Failed to allocate IRTE for ioapic %d\n", + ioapic_id); + index = -ENOMEM; + } } + up_read(&dmar_global_lock); + if (index < 0) + return index; prepare_irte(&irte, vector, destination); @@ -913,6 +938,7 @@ static int intel_setup_ioapic_entry(int irq, irte.avail, irte.vector, irte.dest_id, irte.sid, irte.sq, irte.svt); + entry = (struct IR_IO_APIC_route_entry *)route_entry; memset(entry, 0, sizeof(*entry)); entry->index2 = (index >> 15) & 0x1; @@ -1043,20 +1069,23 @@ static int intel_msi_alloc_irq(struct pci_dev *dev, int irq, int nvec) struct intel_iommu *iommu; int index; + down_read(&dmar_global_lock); iommu = map_dev_to_ir(dev); if (!iommu) { printk(KERN_ERR "Unable to map PCI %s to iommu\n", pci_name(dev)); - return -ENOENT; + index = -ENOENT; + } else { + index = alloc_irte(iommu, irq, nvec); + if (index < 0) { + printk(KERN_ERR + "Unable to allocate %d IRTE for PCI %s\n", + nvec, pci_name(dev)); + index = -ENOSPC; + } } + up_read(&dmar_global_lock); - index = alloc_irte(iommu, irq, nvec); - if (index < 0) { - printk(KERN_ERR - "Unable to allocate %d IRTE for PCI %s\n", nvec, - pci_name(dev)); - return -ENOSPC; - } return index; } @@ -1064,33 +1093,40 @@ static int intel_msi_setup_irq(struct pci_dev *pdev, unsigned int irq, int index, int sub_handle) { struct intel_iommu *iommu; + int ret = -ENOENT; + down_read(&dmar_global_lock); iommu = map_dev_to_ir(pdev); - if (!iommu) - return -ENOENT; - /* - * setup the mapping between the irq and the IRTE - * base index, the sub_handle pointing to the - * appropriate interrupt remap table entry. - */ - set_irte_irq(irq, iommu, index, sub_handle); + if (iommu) { + /* + * setup the mapping between the irq and the IRTE + * base index, the sub_handle pointing to the + * appropriate interrupt remap table entry. + */ + set_irte_irq(irq, iommu, index, sub_handle); + ret = 0; + } + up_read(&dmar_global_lock); - return 0; + return ret; } static int intel_setup_hpet_msi(unsigned int irq, unsigned int id) { - struct intel_iommu *iommu = map_hpet_to_ir(id); + int ret = -1; + struct intel_iommu *iommu; int index; - if (!iommu) - return -1; - - index = alloc_irte(iommu, irq, 1); - if (index < 0) - return -1; + down_read(&dmar_global_lock); + iommu = map_hpet_to_ir(id); + if (iommu) { + index = alloc_irte(iommu, irq, 1); + if (index >= 0) + ret = 0; + } + up_read(&dmar_global_lock); - return 0; + return ret; } struct irq_remap_ops intel_irq_remap_ops = { diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 4b77fd8fde76..8f06a0135a84 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -25,6 +25,7 @@ #include #include #include +#include struct acpi_dmar_header; @@ -48,6 +49,7 @@ struct dmar_drhd_unit { struct intel_iommu *iommu; }; +extern struct rw_semaphore dmar_global_lock; extern struct list_head dmar_drhd_units; #define for_each_drhd_unit(drhd) \ -- cgit v1.2.3 From 0e242612d9cdb46e878ed1f126c78fe68492af00 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:34 +0800 Subject: iommu/vt-d: Use RCU to protect global resources in interrupt context Global DMA and interrupt remapping resources may be accessed in interrupt context, so use RCU instead of rwsem to protect them in such cases. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 33 ++++++++++++++++++++------------- drivers/iommu/intel-iommu.c | 20 ++++++++++++++++---- include/linux/dmar.h | 23 +++++++++++++++++------ 3 files changed, 53 insertions(+), 23 deletions(-) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index c9aca8841fa0..6e4d851991f1 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -71,13 +71,13 @@ static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd) * the very end. */ if (drhd->include_all) - list_add_tail(&drhd->list, &dmar_drhd_units); + list_add_tail_rcu(&drhd->list, &dmar_drhd_units); else - list_add(&drhd->list, &dmar_drhd_units); + list_add_rcu(&drhd->list, &dmar_drhd_units); } static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope, - struct pci_dev **dev, u16 segment) + struct pci_dev __rcu **dev, u16 segment) { struct pci_bus *bus; struct pci_dev *pdev = NULL; @@ -122,7 +122,9 @@ static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope, pci_name(pdev)); return -EINVAL; } - *dev = pdev; + + rcu_assign_pointer(*dev, pdev); + return 0; } @@ -149,7 +151,7 @@ void *dmar_alloc_dev_scope(void *start, void *end, int *cnt) } int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, - struct pci_dev ***devices, u16 segment) + struct pci_dev __rcu ***devices, u16 segment) { struct acpi_dmar_device_scope *scope; int index, ret; @@ -177,7 +179,7 @@ int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, return 0; } -void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt) +void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt) { int i; struct pci_dev *tmp_dev; @@ -186,9 +188,10 @@ void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt) for_each_active_dev_scope(*devices, *cnt, i, tmp_dev) pci_dev_put(tmp_dev); kfree(*devices); - *devices = NULL; - *cnt = 0; } + + *devices = NULL; + *cnt = 0; } /** @@ -410,7 +413,7 @@ parse_dmar_table(void) return ret; } -static int dmar_pci_device_match(struct pci_dev *devices[], int cnt, +static int dmar_pci_device_match(struct pci_dev __rcu *devices[], int cnt, struct pci_dev *dev) { int index; @@ -431,11 +434,12 @@ static int dmar_pci_device_match(struct pci_dev *devices[], int cnt, struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct pci_dev *dev) { - struct dmar_drhd_unit *dmaru = NULL; + struct dmar_drhd_unit *dmaru; struct acpi_dmar_hardware_unit *drhd; dev = pci_physfn(dev); + rcu_read_lock(); for_each_drhd_unit(dmaru) { drhd = container_of(dmaru->hdr, struct acpi_dmar_hardware_unit, @@ -443,14 +447,17 @@ dmar_find_matched_drhd_unit(struct pci_dev *dev) if (dmaru->include_all && drhd->segment == pci_domain_nr(dev->bus)) - return dmaru; + goto out; if (dmar_pci_device_match(dmaru->devices, dmaru->devices_cnt, dev)) - return dmaru; + goto out; } + dmaru = NULL; +out: + rcu_read_unlock(); - return NULL; + return dmaru; } int __init dmar_dev_scope_init(void) diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 50d639a2df88..e1679a6fe468 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -385,14 +385,14 @@ struct dmar_rmrr_unit { struct acpi_dmar_header *hdr; /* ACPI header */ u64 base_address; /* reserved base address*/ u64 end_address; /* reserved end address */ - struct pci_dev **devices; /* target devices */ + struct pci_dev __rcu **devices; /* target devices */ int devices_cnt; /* target device count */ }; struct dmar_atsr_unit { struct list_head list; /* list of ATSR units */ struct acpi_dmar_header *hdr; /* ACPI header */ - struct pci_dev **devices; /* target devices */ + struct pci_dev __rcu **devices; /* target devices */ int devices_cnt; /* target device count */ u8 include_all:1; /* include all ports */ }; @@ -634,12 +634,15 @@ static void domain_update_iommu_superpage(struct dmar_domain *domain) } /* set iommu_superpage to the smallest common denominator */ + rcu_read_lock(); for_each_active_iommu(iommu, drhd) { mask &= cap_super_page_val(iommu->cap); if (!mask) { break; } } + rcu_read_unlock(); + domain->iommu_superpage = fls(mask); } @@ -658,6 +661,7 @@ static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn) struct pci_dev *dev; int i; + rcu_read_lock(); for_each_active_iommu(iommu, drhd) { if (segment != drhd->segment) continue; @@ -677,6 +681,7 @@ static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn) } iommu = NULL; out: + rcu_read_unlock(); return iommu; } @@ -1535,10 +1540,12 @@ static void domain_exit(struct dmar_domain *domain) dma_pte_free_pagetable(domain, 0, DOMAIN_MAX_PFN(domain->gaw)); /* clear attached or cached domains */ + rcu_read_lock(); for_each_active_iommu(iommu, drhd) if (domain->flags & DOMAIN_FLAG_VIRTUAL_MACHINE || test_bit(iommu->seq_id, domain->iommu_bmp)) iommu_detach_domain(domain, iommu); + rcu_read_unlock(); free_domain_mem(domain); } @@ -2338,6 +2345,7 @@ static bool device_has_rmrr(struct pci_dev *dev) struct pci_dev *tmp; int i; + rcu_read_lock(); for_each_rmrr_units(rmrr) { /* * Return TRUE if this RMRR contains the device that @@ -2346,9 +2354,11 @@ static bool device_has_rmrr(struct pci_dev *dev) for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, tmp) if (tmp == dev) { + rcu_read_unlock(); return true; } } + rcu_read_unlock(); return false; } @@ -3512,7 +3522,7 @@ int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) atsru->hdr = hdr; atsru->include_all = atsr->flags & 0x1; - list_add(&atsru->list, &dmar_atsr_units); + list_add_rcu(&atsru->list, &dmar_atsr_units); return 0; } @@ -3574,6 +3584,7 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) if (!bridge) return 0; + rcu_read_lock(); list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) { atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); if (atsr->segment != pci_domain_nr(dev->bus)) @@ -3588,6 +3599,7 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) } ret = 0; out: + rcu_read_unlock(); return ret; } @@ -3604,7 +3616,7 @@ int __init dmar_parse_rmrr_atsr_dev(void) return ret; } - list_for_each_entry(atsr, &dmar_atsr_units, list) { + list_for_each_entry_rcu(atsr, &dmar_atsr_units, list) { ret = atsr_parse_dev(atsr); if (ret) return ret; diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 8f06a0135a84..bedebab934b4 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -26,6 +26,7 @@ #include #include #include +#include struct acpi_dmar_header; @@ -41,7 +42,7 @@ struct dmar_drhd_unit { struct list_head list; /* list of drhd units */ struct acpi_dmar_header *hdr; /* ACPI header */ u64 reg_base_addr; /* register base address*/ - struct pci_dev **devices; /* target device array */ + struct pci_dev __rcu **devices;/* target device array */ int devices_cnt; /* target device count */ u16 segment; /* PCI domain */ u8 ignored:1; /* ignore drhd */ @@ -53,22 +54,31 @@ extern struct rw_semaphore dmar_global_lock; extern struct list_head dmar_drhd_units; #define for_each_drhd_unit(drhd) \ - list_for_each_entry(drhd, &dmar_drhd_units, list) + list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) #define for_each_active_drhd_unit(drhd) \ - list_for_each_entry(drhd, &dmar_drhd_units, list) \ + list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) \ if (drhd->ignored) {} else #define for_each_active_iommu(i, drhd) \ - list_for_each_entry(drhd, &dmar_drhd_units, list) \ + list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) \ if (i=drhd->iommu, drhd->ignored) {} else #define for_each_iommu(i, drhd) \ - list_for_each_entry(drhd, &dmar_drhd_units, list) \ + list_for_each_entry_rcu(drhd, &dmar_drhd_units, list) \ if (i=drhd->iommu, 0) {} else +static inline bool dmar_rcu_check(void) +{ + return rwsem_is_locked(&dmar_global_lock) || + system_state == SYSTEM_BOOTING; +} + +#define dmar_rcu_dereference(p) rcu_dereference_check((p), dmar_rcu_check()) + #define for_each_dev_scope(a, c, p, d) \ - for ((p) = 0; ((d) = (p) < (c) ? (a)[(p)] : NULL, (p) < (c)); (p)++) + for ((p) = 0; ((d) = (p) < (c) ? dmar_rcu_dereference((a)[(p)]) : \ + NULL, (p) < (c)); (p)++) #define for_each_active_dev_scope(a, c, p, d) \ for_each_dev_scope((a), (c), (p), (d)) if (!(d)) { continue; } else @@ -78,6 +88,7 @@ extern int dmar_dev_scope_init(void); extern int dmar_parse_dev_scope(void *start, void *end, int *cnt, struct pci_dev ***devices, u16 segment); extern void *dmar_alloc_dev_scope(void *start, void *end, int *cnt); +extern void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt); extern void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt); /* Intel IOMMU detection */ -- cgit v1.2.3 From 59ce0515cdaf3b7d47893d12f61e51d691863788 Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:35 +0800 Subject: iommu/vt-d: Update DRHD/RMRR/ATSR device scope caches when PCI hotplug happens Current Intel DMAR/IOMMU driver assumes that all PCI devices associated with DMAR/RMRR/ATSR device scope arrays are created at boot time and won't change at runtime, so it caches pointers of associated PCI device object. That assumption may be wrong now due to: 1) introduction of PCI host bridge hotplug 2) PCI device hotplug through sysfs interfaces. Wang Yijing has tried to solve this issue by caching tupple instead of the PCI device object pointer, but that's still unreliable because PCI bus number may change in case of hotplug. Please refer to http://lkml.org/lkml/2013/11/5/64 Message from Yingjing's mail: after remove and rescan a pci device [ 611.857095] dmar: DRHD: handling fault status reg 2 [ 611.857109] dmar: DMAR:[DMA Read] Request device [86:00.3] fault addr ffff7000 [ 611.857109] DMAR:[fault reason 02] Present bit in context entry is clear [ 611.857524] dmar: DRHD: handling fault status reg 102 [ 611.857534] dmar: DMAR:[DMA Read] Request device [86:00.3] fault addr ffff6000 [ 611.857534] DMAR:[fault reason 02] Present bit in context entry is clear [ 611.857936] dmar: DRHD: handling fault status reg 202 [ 611.857947] dmar: DMAR:[DMA Read] Request device [86:00.3] fault addr ffff5000 [ 611.857947] DMAR:[fault reason 02] Present bit in context entry is clear [ 611.858351] dmar: DRHD: handling fault status reg 302 [ 611.858362] dmar: DMAR:[DMA Read] Request device [86:00.3] fault addr ffff4000 [ 611.858362] DMAR:[fault reason 02] Present bit in context entry is clear [ 611.860819] IPv6: ADDRCONF(NETDEV_UP): eth3: link is not ready [ 611.860983] dmar: DRHD: handling fault status reg 402 [ 611.860995] dmar: INTR-REMAP: Request device [[86:00.3] fault index a4 [ 611.860995] INTR-REMAP:[fault reason 34] Present field in the IRTE entry is clear This patch introduces a new mechanism to update the DRHD/RMRR/ATSR device scope caches by hooking PCI bus notification. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 207 ++++++++++++++++++++++++++++++++++++++++++++ drivers/iommu/intel-iommu.c | 54 ++++++++++++ include/linux/dmar.h | 24 ++++- 3 files changed, 283 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 6e4d851991f1..bf6bfd1f69aa 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -194,6 +194,209 @@ void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt) *cnt = 0; } +/* Optimize out kzalloc()/kfree() for normal cases */ +static char dmar_pci_notify_info_buf[64]; + +static struct dmar_pci_notify_info * +dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event) +{ + int level = 0; + size_t size; + struct pci_dev *tmp; + struct dmar_pci_notify_info *info; + + BUG_ON(dev->is_virtfn); + + /* Only generate path[] for device addition event */ + if (event == BUS_NOTIFY_ADD_DEVICE) + for (tmp = dev; tmp; tmp = tmp->bus->self) + level++; + + size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path); + if (size <= sizeof(dmar_pci_notify_info_buf)) { + info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf; + } else { + info = kzalloc(size, GFP_KERNEL); + if (!info) { + pr_warn("Out of memory when allocating notify_info " + "for %s.\n", pci_name(dev)); + return NULL; + } + } + + info->event = event; + info->dev = dev; + info->seg = pci_domain_nr(dev->bus); + info->level = level; + if (event == BUS_NOTIFY_ADD_DEVICE) { + for (tmp = dev, level--; tmp; tmp = tmp->bus->self) { + info->path[level].device = PCI_SLOT(tmp->devfn); + info->path[level].function = PCI_FUNC(tmp->devfn); + if (pci_is_root_bus(tmp->bus)) + info->bus = tmp->bus->number; + } + } + + return info; +} + +static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info) +{ + if ((void *)info != dmar_pci_notify_info_buf) + kfree(info); +} + +static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus, + struct acpi_dmar_pci_path *path, int count) +{ + int i; + + if (info->bus != bus) + return false; + if (info->level != count) + return false; + + for (i = 0; i < count; i++) { + if (path[i].device != info->path[i].device || + path[i].function != info->path[i].function) + return false; + } + + return true; +} + +/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */ +int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, + void *start, void*end, u16 segment, + struct pci_dev __rcu **devices, int devices_cnt) +{ + int i, level; + struct pci_dev *tmp, *dev = info->dev; + struct acpi_dmar_device_scope *scope; + struct acpi_dmar_pci_path *path; + + if (segment != info->seg) + return 0; + + for (; start < end; start += scope->length) { + scope = start; + if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT && + scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE) + continue; + + path = (struct acpi_dmar_pci_path *)(scope + 1); + level = (scope->length - sizeof(*scope)) / sizeof(*path); + if (!dmar_match_pci_path(info, scope->bus, path, level)) + continue; + + if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^ + (dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) { + pr_warn("Device scope type does not match for %s\n", + pci_name(dev)); + return -EINVAL; + } + + for_each_dev_scope(devices, devices_cnt, i, tmp) + if (tmp == NULL) { + rcu_assign_pointer(devices[i], + pci_dev_get(dev)); + return 1; + } + BUG_ON(i >= devices_cnt); + } + + return 0; +} + +int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment, + struct pci_dev __rcu **devices, int count) +{ + int index; + struct pci_dev *tmp; + + if (info->seg != segment) + return 0; + + for_each_active_dev_scope(devices, count, index, tmp) + if (tmp == info->dev) { + rcu_assign_pointer(devices[index], NULL); + synchronize_rcu(); + pci_dev_put(tmp); + return 1; + } + + return 0; +} + +static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info) +{ + int ret = 0; + struct dmar_drhd_unit *dmaru; + struct acpi_dmar_hardware_unit *drhd; + + for_each_drhd_unit(dmaru) { + if (dmaru->include_all) + continue; + + drhd = container_of(dmaru->hdr, + struct acpi_dmar_hardware_unit, header); + ret = dmar_insert_dev_scope(info, (void *)(drhd + 1), + ((void *)drhd) + drhd->header.length, + dmaru->segment, + dmaru->devices, dmaru->devices_cnt); + if (ret != 0) + break; + } + if (ret >= 0) + ret = dmar_iommu_notify_scope_dev(info); + + return ret; +} + +static void dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info) +{ + struct dmar_drhd_unit *dmaru; + + for_each_drhd_unit(dmaru) + if (dmar_remove_dev_scope(info, dmaru->segment, + dmaru->devices, dmaru->devices_cnt)) + break; + dmar_iommu_notify_scope_dev(info); +} + +static int dmar_pci_bus_notifier(struct notifier_block *nb, + unsigned long action, void *data) +{ + struct pci_dev *pdev = to_pci_dev(data); + struct dmar_pci_notify_info *info; + + /* Only care about add/remove events for physical functions */ + if (pdev->is_virtfn) + return NOTIFY_DONE; + if (action != BUS_NOTIFY_ADD_DEVICE && action != BUS_NOTIFY_DEL_DEVICE) + return NOTIFY_DONE; + + info = dmar_alloc_pci_notify_info(pdev, action); + if (!info) + return NOTIFY_DONE; + + down_write(&dmar_global_lock); + if (action == BUS_NOTIFY_ADD_DEVICE) + dmar_pci_bus_add_dev(info); + else if (action == BUS_NOTIFY_DEL_DEVICE) + dmar_pci_bus_del_dev(info); + up_write(&dmar_global_lock); + + dmar_free_pci_notify_info(info); + + return NOTIFY_OK; +} + +static struct notifier_block dmar_pci_bus_nb = { + .notifier_call = dmar_pci_bus_notifier, + .priority = INT_MIN, +}; + /** * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition * structure which uniquely represent one DMA remapping hardware unit @@ -482,6 +685,8 @@ int __init dmar_dev_scope_init(void) if (ret) goto fail; + bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb); + dmar_dev_scope_initialized = 1; return 0; @@ -1412,6 +1617,8 @@ static int __init dmar_free_unused_resources(void) if (irq_remapping_enabled || intel_iommu_enabled) return 0; + bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb); + down_write(&dmar_global_lock); list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) { list_del(&dmaru->list); diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index e1679a6fe468..d9c0dc5a5d35 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3625,6 +3625,60 @@ int __init dmar_parse_rmrr_atsr_dev(void) return 0; } +int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) +{ + int ret = 0; + struct dmar_rmrr_unit *rmrru; + struct dmar_atsr_unit *atsru; + struct acpi_dmar_atsr *atsr; + struct acpi_dmar_reserved_memory *rmrr; + + if (!intel_iommu_enabled && system_state != SYSTEM_BOOTING) + return 0; + + list_for_each_entry(rmrru, &dmar_rmrr_units, list) { + rmrr = container_of(rmrru->hdr, + struct acpi_dmar_reserved_memory, header); + if (info->event == BUS_NOTIFY_ADD_DEVICE) { + ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1), + ((void *)rmrr) + rmrr->header.length, + rmrr->segment, rmrru->devices, + rmrru->devices_cnt); + if (ret > 0) + break; + else if(ret < 0) + return ret; + } else if (info->event == BUS_NOTIFY_DEL_DEVICE) { + if (dmar_remove_dev_scope(info, rmrr->segment, + rmrru->devices, rmrru->devices_cnt)) + break; + } + } + + list_for_each_entry(atsru, &dmar_atsr_units, list) { + if (atsru->include_all) + continue; + + atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); + if (info->event == BUS_NOTIFY_ADD_DEVICE) { + ret = dmar_insert_dev_scope(info, (void *)(atsr + 1), + (void *)atsr + atsr->header.length, + atsr->segment, atsru->devices, + atsru->devices_cnt); + if (ret > 0) + break; + else if(ret < 0) + return ret; + } else if (info->event == BUS_NOTIFY_DEL_DEVICE) { + if (dmar_remove_dev_scope(info, atsr->segment, + atsru->devices, atsru->devices_cnt)) + break; + } + } + + return 0; +} + /* * Here we only respond to action of unbound device from driver. * diff --git a/include/linux/dmar.h b/include/linux/dmar.h index bedebab934b4..4e196430f1b2 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -50,6 +50,15 @@ struct dmar_drhd_unit { struct intel_iommu *iommu; }; +struct dmar_pci_notify_info { + struct pci_dev *dev; + unsigned long event; + int bus; + u16 seg; + u16 level; + struct acpi_dmar_pci_path path[]; +} __attribute__((packed)); + extern struct rw_semaphore dmar_global_lock; extern struct list_head dmar_drhd_units; @@ -89,12 +98,18 @@ extern int dmar_parse_dev_scope(void *start, void *end, int *cnt, struct pci_dev ***devices, u16 segment); extern void *dmar_alloc_dev_scope(void *start, void *end, int *cnt); extern void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt); -extern void dmar_free_dev_scope(struct pci_dev ***devices, int *cnt); - +extern int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, + void *start, void*end, u16 segment, + struct pci_dev __rcu **devices, + int devices_cnt); +extern int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, + u16 segment, struct pci_dev __rcu **devices, + int count); /* Intel IOMMU detection */ extern int detect_intel_iommu(void); extern int enable_drhd_fault_handling(void); #else +struct dmar_pci_notify_info; static inline int detect_intel_iommu(void) { return -ENODEV; @@ -161,6 +176,7 @@ extern int iommu_detected, no_iommu; extern int dmar_parse_rmrr_atsr_dev(void); extern int dmar_parse_one_rmrr(struct acpi_dmar_header *header); extern int dmar_parse_one_atsr(struct acpi_dmar_header *header); +extern int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info); extern int intel_iommu_init(void); #else /* !CONFIG_INTEL_IOMMU: */ static inline int intel_iommu_init(void) { return -ENODEV; } @@ -176,6 +192,10 @@ static inline int dmar_parse_rmrr_atsr_dev(void) { return 0; } +static inline int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) +{ + return 0; +} #endif /* CONFIG_INTEL_IOMMU */ #endif /* __DMAR_H__ */ -- cgit v1.2.3 From 2e45528930388658603ea24d49cf52867b928d3e Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:36 +0800 Subject: iommu/vt-d: Unify the way to process DMAR device scope array Now we have a PCI bus notification based mechanism to update DMAR device scope array, we could extend the mechanism to support boot time initialization too, which will help to unify and simplify the implementation. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/dmar.c | 163 ++++++++++++-------------------------------- drivers/iommu/intel-iommu.c | 71 +++++-------------- include/linux/dmar.h | 5 -- 3 files changed, 61 insertions(+), 178 deletions(-) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index bf6bfd1f69aa..b19f9f4c3584 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -60,6 +60,7 @@ LIST_HEAD(dmar_drhd_units); struct acpi_table_header * __initdata dmar_tbl; static acpi_size dmar_tbl_size; +static int dmar_dev_scope_status = 1; static int alloc_iommu(struct dmar_drhd_unit *drhd); static void free_iommu(struct intel_iommu *iommu); @@ -76,58 +77,6 @@ static void __init dmar_register_drhd_unit(struct dmar_drhd_unit *drhd) list_add_rcu(&drhd->list, &dmar_drhd_units); } -static int __init dmar_parse_one_dev_scope(struct acpi_dmar_device_scope *scope, - struct pci_dev __rcu **dev, u16 segment) -{ - struct pci_bus *bus; - struct pci_dev *pdev = NULL; - struct acpi_dmar_pci_path *path; - int count; - - bus = pci_find_bus(segment, scope->bus); - path = (struct acpi_dmar_pci_path *)(scope + 1); - count = (scope->length - sizeof(struct acpi_dmar_device_scope)) - / sizeof(struct acpi_dmar_pci_path); - - while (count) { - if (pdev) - pci_dev_put(pdev); - /* - * Some BIOSes list non-exist devices in DMAR table, just - * ignore it - */ - if (!bus) { - pr_warn("Device scope bus [%d] not found\n", scope->bus); - break; - } - pdev = pci_get_slot(bus, PCI_DEVFN(path->device, path->function)); - if (!pdev) { - /* warning will be printed below */ - break; - } - path ++; - count --; - bus = pdev->subordinate; - } - if (!pdev) { - pr_warn("Device scope device [%04x:%02x:%02x.%02x] not found\n", - segment, scope->bus, path->device, path->function); - return 0; - } - if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT && \ - pdev->subordinate) || (scope->entry_type == \ - ACPI_DMAR_SCOPE_TYPE_BRIDGE && !pdev->subordinate)) { - pci_dev_put(pdev); - pr_warn("Device scope type does not match for %s\n", - pci_name(pdev)); - return -EINVAL; - } - - rcu_assign_pointer(*dev, pdev); - - return 0; -} - void *dmar_alloc_dev_scope(void *start, void *end, int *cnt) { struct acpi_dmar_device_scope *scope; @@ -150,35 +99,6 @@ void *dmar_alloc_dev_scope(void *start, void *end, int *cnt) return kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL); } -int __init dmar_parse_dev_scope(void *start, void *end, int *cnt, - struct pci_dev __rcu ***devices, u16 segment) -{ - struct acpi_dmar_device_scope *scope; - int index, ret; - - *devices = dmar_alloc_dev_scope(start, end, cnt); - if (*cnt == 0) - return 0; - else if (!*devices) - return -ENOMEM; - - for (index = 0; start < end; start += scope->length) { - scope = start; - if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT || - scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE) { - ret = dmar_parse_one_dev_scope(scope, - &(*devices)[index], segment); - if (ret) { - dmar_free_dev_scope(devices, cnt); - return ret; - } - index ++; - } - } - - return 0; -} - void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt) { int i; @@ -220,6 +140,8 @@ dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event) if (!info) { pr_warn("Out of memory when allocating notify_info " "for %s.\n", pci_name(dev)); + if (dmar_dev_scope_status == 0) + dmar_dev_scope_status = -ENOMEM; return NULL; } } @@ -349,6 +271,8 @@ static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info) } if (ret >= 0) ret = dmar_iommu_notify_scope_dev(info); + if (ret < 0 && dmar_dev_scope_status == 0) + dmar_dev_scope_status = ret; return ret; } @@ -418,9 +342,21 @@ dmar_parse_one_drhd(struct acpi_dmar_header *header) dmaru->reg_base_addr = drhd->address; dmaru->segment = drhd->segment; dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */ + if (!dmaru->include_all) { + dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1), + ((void *)drhd) + drhd->header.length, + &dmaru->devices_cnt); + if (dmaru->devices_cnt && dmaru->devices == NULL) { + kfree(dmaru); + return -ENOMEM; + } + } ret = alloc_iommu(dmaru); if (ret) { + if (!dmaru->include_all) + dmar_free_dev_scope(&dmaru->devices, + &dmaru->devices_cnt); kfree(dmaru); return ret; } @@ -437,21 +373,6 @@ static void dmar_free_drhd(struct dmar_drhd_unit *dmaru) kfree(dmaru); } -static int __init dmar_parse_dev(struct dmar_drhd_unit *dmaru) -{ - struct acpi_dmar_hardware_unit *drhd; - - drhd = (struct acpi_dmar_hardware_unit *) dmaru->hdr; - - if (dmaru->include_all) - return 0; - - return dmar_parse_dev_scope((void *)(drhd + 1), - ((void *)drhd) + drhd->header.length, - &dmaru->devices_cnt, &dmaru->devices, - drhd->segment); -} - #ifdef CONFIG_ACPI_NUMA static int __init dmar_parse_one_rhsa(struct acpi_dmar_header *header) @@ -665,34 +586,35 @@ out: int __init dmar_dev_scope_init(void) { - static int dmar_dev_scope_initialized; - struct dmar_drhd_unit *drhd; - int ret = -ENODEV; + struct pci_dev *dev = NULL; + struct dmar_pci_notify_info *info; - if (dmar_dev_scope_initialized) - return dmar_dev_scope_initialized; + if (dmar_dev_scope_status != 1) + return dmar_dev_scope_status; - if (list_empty(&dmar_drhd_units)) - goto fail; + if (list_empty(&dmar_drhd_units)) { + dmar_dev_scope_status = -ENODEV; + } else { + dmar_dev_scope_status = 0; + + for_each_pci_dev(dev) { + if (dev->is_virtfn) + continue; + + info = dmar_alloc_pci_notify_info(dev, + BUS_NOTIFY_ADD_DEVICE); + if (!info) { + return dmar_dev_scope_status; + } else { + dmar_pci_bus_add_dev(info); + dmar_free_pci_notify_info(info); + } + } - for_each_drhd_unit(drhd) { - ret = dmar_parse_dev(drhd); - if (ret) - goto fail; + bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb); } - ret = dmar_parse_rmrr_atsr_dev(); - if (ret) - goto fail; - - bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb); - - dmar_dev_scope_initialized = 1; - return 0; - -fail: - dmar_dev_scope_initialized = ret; - return ret; + return dmar_dev_scope_status; } @@ -1617,7 +1539,8 @@ static int __init dmar_free_unused_resources(void) if (irq_remapping_enabled || intel_iommu_enabled) return 0; - bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb); + if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units)) + bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb); down_write(&dmar_global_lock); list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) { diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index d9c0dc5a5d35..dd576c067d0d 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -3473,11 +3473,6 @@ static void __init init_iommu_pm_ops(void) static inline void init_iommu_pm_ops(void) {} #endif /* CONFIG_PM */ -static void __init dmar_register_rmrr_unit(struct dmar_rmrr_unit *rmrr) -{ - list_add(&rmrr->list, &dmar_rmrr_units); -} - int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header) { @@ -3492,21 +3487,17 @@ int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header) rmrr = (struct acpi_dmar_reserved_memory *)header; rmrru->base_address = rmrr->base_address; rmrru->end_address = rmrr->end_address; + rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1), + ((void *)rmrr) + rmrr->header.length, + &rmrru->devices_cnt); + if (rmrru->devices_cnt && rmrru->devices == NULL) { + kfree(rmrru); + return -ENOMEM; + } - dmar_register_rmrr_unit(rmrru); - return 0; -} + list_add(&rmrru->list, &dmar_rmrr_units); -static int __init -rmrr_parse_dev(struct dmar_rmrr_unit *rmrru) -{ - struct acpi_dmar_reserved_memory *rmrr; - - rmrr = (struct acpi_dmar_reserved_memory *) rmrru->hdr; - return dmar_parse_dev_scope((void *)(rmrr + 1), - ((void *)rmrr) + rmrr->header.length, - &rmrru->devices_cnt, &rmrru->devices, - rmrr->segment); + return 0; } int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) @@ -3521,26 +3512,21 @@ int __init dmar_parse_one_atsr(struct acpi_dmar_header *hdr) atsru->hdr = hdr; atsru->include_all = atsr->flags & 0x1; + if (!atsru->include_all) { + atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1), + (void *)atsr + atsr->header.length, + &atsru->devices_cnt); + if (atsru->devices_cnt && atsru->devices == NULL) { + kfree(atsru); + return -ENOMEM; + } + } list_add_rcu(&atsru->list, &dmar_atsr_units); return 0; } -static int __init atsr_parse_dev(struct dmar_atsr_unit *atsru) -{ - struct acpi_dmar_atsr *atsr; - - if (atsru->include_all) - return 0; - - atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header); - return dmar_parse_dev_scope((void *)(atsr + 1), - (void *)atsr + atsr->header.length, - &atsru->devices_cnt, &atsru->devices, - atsr->segment); -} - static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru) { dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt); @@ -3604,27 +3590,6 @@ out: return ret; } -int __init dmar_parse_rmrr_atsr_dev(void) -{ - struct dmar_rmrr_unit *rmrr; - struct dmar_atsr_unit *atsr; - int ret; - - list_for_each_entry(rmrr, &dmar_rmrr_units, list) { - ret = rmrr_parse_dev(rmrr); - if (ret) - return ret; - } - - list_for_each_entry_rcu(atsr, &dmar_atsr_units, list) { - ret = atsr_parse_dev(atsr); - if (ret) - return ret; - } - - return 0; -} - int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) { int ret = 0; diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 4e196430f1b2..0a92e4d978bc 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -173,7 +173,6 @@ extern int arch_setup_dmar_msi(unsigned int irq); #ifdef CONFIG_INTEL_IOMMU extern int iommu_detected, no_iommu; -extern int dmar_parse_rmrr_atsr_dev(void); extern int dmar_parse_one_rmrr(struct acpi_dmar_header *header); extern int dmar_parse_one_atsr(struct acpi_dmar_header *header); extern int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info); @@ -188,10 +187,6 @@ static inline int dmar_parse_one_atsr(struct acpi_dmar_header *header) { return 0; } -static inline int dmar_parse_rmrr_atsr_dev(void) -{ - return 0; -} static inline int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info) { return 0; -- cgit v1.2.3 From 75f05569d0e51f6332a291c82abbeb7c8262e32d Mon Sep 17 00:00:00 2001 From: Jiang Liu Date: Wed, 19 Feb 2014 14:07:37 +0800 Subject: iommu/vt-d: Update IOMMU state when memory hotplug happens If static identity domain is created, IOMMU driver needs to update si_domain page table when memory hotplug event happens. Otherwise PCI device DMA operations can't access the hot-added memory regions. Signed-off-by: Jiang Liu Signed-off-by: Joerg Roedel --- drivers/iommu/intel-iommu.c | 71 ++++++++++++++++++++++++++++++++++++++++++++- drivers/iommu/iova.c | 64 ++++++++++++++++++++++++++++++++++++---- include/linux/iova.h | 2 ++ 3 files changed, 130 insertions(+), 7 deletions(-) (limited to 'include') diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index dd576c067d0d..484d669d2720 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include #include @@ -3683,6 +3684,73 @@ static struct notifier_block device_nb = { .notifier_call = device_notifier, }; +static int intel_iommu_memory_notifier(struct notifier_block *nb, + unsigned long val, void *v) +{ + struct memory_notify *mhp = v; + unsigned long long start, end; + unsigned long start_vpfn, last_vpfn; + + switch (val) { + case MEM_GOING_ONLINE: + start = mhp->start_pfn << PAGE_SHIFT; + end = ((mhp->start_pfn + mhp->nr_pages) << PAGE_SHIFT) - 1; + if (iommu_domain_identity_map(si_domain, start, end)) { + pr_warn("dmar: failed to build identity map for [%llx-%llx]\n", + start, end); + return NOTIFY_BAD; + } + break; + + case MEM_OFFLINE: + case MEM_CANCEL_ONLINE: + start_vpfn = mm_to_dma_pfn(mhp->start_pfn); + last_vpfn = mm_to_dma_pfn(mhp->start_pfn + mhp->nr_pages - 1); + while (start_vpfn <= last_vpfn) { + struct iova *iova; + struct dmar_drhd_unit *drhd; + struct intel_iommu *iommu; + + iova = find_iova(&si_domain->iovad, start_vpfn); + if (iova == NULL) { + pr_debug("dmar: failed get IOVA for PFN %lx\n", + start_vpfn); + break; + } + + iova = split_and_remove_iova(&si_domain->iovad, iova, + start_vpfn, last_vpfn); + if (iova == NULL) { + pr_warn("dmar: failed to split IOVA PFN [%lx-%lx]\n", + start_vpfn, last_vpfn); + return NOTIFY_BAD; + } + + rcu_read_lock(); + for_each_active_iommu(iommu, drhd) + iommu_flush_iotlb_psi(iommu, si_domain->id, + iova->pfn_lo, + iova->pfn_hi - iova->pfn_lo + 1, 0); + rcu_read_unlock(); + dma_pte_clear_range(si_domain, iova->pfn_lo, + iova->pfn_hi); + dma_pte_free_pagetable(si_domain, iova->pfn_lo, + iova->pfn_hi); + + start_vpfn = iova->pfn_hi + 1; + free_iova_mem(iova); + } + break; + } + + return NOTIFY_OK; +} + +static struct notifier_block intel_iommu_memory_nb = { + .notifier_call = intel_iommu_memory_notifier, + .priority = 0 +}; + int __init intel_iommu_init(void) { int ret = -ENODEV; @@ -3755,8 +3823,9 @@ int __init intel_iommu_init(void) init_iommu_pm_ops(); bus_set_iommu(&pci_bus_type, &intel_iommu_ops); - bus_register_notifier(&pci_bus_type, &device_nb); + if (si_domain && !hw_pass_through) + register_memory_notifier(&intel_iommu_memory_nb); intel_iommu_enabled = 1; diff --git a/drivers/iommu/iova.c b/drivers/iommu/iova.c index 67da6cff74e8..f6b17e6af2fb 100644 --- a/drivers/iommu/iova.c +++ b/drivers/iommu/iova.c @@ -342,19 +342,30 @@ __is_range_overlap(struct rb_node *node, return 0; } +static inline struct iova * +alloc_and_init_iova(unsigned long pfn_lo, unsigned long pfn_hi) +{ + struct iova *iova; + + iova = alloc_iova_mem(); + if (iova) { + iova->pfn_lo = pfn_lo; + iova->pfn_hi = pfn_hi; + } + + return iova; +} + static struct iova * __insert_new_range(struct iova_domain *iovad, unsigned long pfn_lo, unsigned long pfn_hi) { struct iova *iova; - iova = alloc_iova_mem(); - if (!iova) - return iova; + iova = alloc_and_init_iova(pfn_lo, pfn_hi); + if (iova) + iova_insert_rbtree(&iovad->rbroot, iova); - iova->pfn_hi = pfn_hi; - iova->pfn_lo = pfn_lo; - iova_insert_rbtree(&iovad->rbroot, iova); return iova; } @@ -433,3 +444,44 @@ copy_reserved_iova(struct iova_domain *from, struct iova_domain *to) } spin_unlock_irqrestore(&from->iova_rbtree_lock, flags); } + +struct iova * +split_and_remove_iova(struct iova_domain *iovad, struct iova *iova, + unsigned long pfn_lo, unsigned long pfn_hi) +{ + unsigned long flags; + struct iova *prev = NULL, *next = NULL; + + spin_lock_irqsave(&iovad->iova_rbtree_lock, flags); + if (iova->pfn_lo < pfn_lo) { + prev = alloc_and_init_iova(iova->pfn_lo, pfn_lo - 1); + if (prev == NULL) + goto error; + } + if (iova->pfn_hi > pfn_hi) { + next = alloc_and_init_iova(pfn_hi + 1, iova->pfn_hi); + if (next == NULL) + goto error; + } + + __cached_rbnode_delete_update(iovad, iova); + rb_erase(&iova->node, &iovad->rbroot); + + if (prev) { + iova_insert_rbtree(&iovad->rbroot, prev); + iova->pfn_lo = pfn_lo; + } + if (next) { + iova_insert_rbtree(&iovad->rbroot, next); + iova->pfn_hi = pfn_hi; + } + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + + return iova; + +error: + spin_unlock_irqrestore(&iovad->iova_rbtree_lock, flags); + if (prev) + free_iova_mem(prev); + return NULL; +} diff --git a/include/linux/iova.h b/include/linux/iova.h index 76a0759e88ec..3277f4711349 100644 --- a/include/linux/iova.h +++ b/include/linux/iova.h @@ -47,5 +47,7 @@ void copy_reserved_iova(struct iova_domain *from, struct iova_domain *to); void init_iova_domain(struct iova_domain *iovad, unsigned long pfn_32bit); struct iova *find_iova(struct iova_domain *iovad, unsigned long pfn); void put_iova_domain(struct iova_domain *iovad); +struct iova *split_and_remove_iova(struct iova_domain *iovad, + struct iova *iova, unsigned long pfn_lo, unsigned long pfn_hi); #endif -- cgit v1.2.3 From 86a54dcce6ccb0d4a5810ed8112011039d855bee Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 7 Mar 2014 12:43:40 +0000 Subject: iommu/vt-d: Add ACPI namespace device reporting structures Signed-off-by: David Woodhouse --- include/acpi/actbl2.h | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/include/acpi/actbl2.h b/include/acpi/actbl2.h index 094a906a0e98..da5b057d775d 100644 --- a/include/acpi/actbl2.h +++ b/include/acpi/actbl2.h @@ -424,7 +424,8 @@ enum acpi_dmar_type { ACPI_DMAR_TYPE_RESERVED_MEMORY = 1, ACPI_DMAR_TYPE_ATSR = 2, ACPI_DMAR_HARDWARE_AFFINITY = 3, - ACPI_DMAR_TYPE_RESERVED = 4 /* 4 and greater are reserved */ + ACPI_DMAR_TYPE_ANDD = 4, + ACPI_DMAR_TYPE_RESERVED = 5 /* 5 and greater are reserved */ }; /* DMAR Device Scope structure */ @@ -445,7 +446,8 @@ enum acpi_dmar_scope_type { ACPI_DMAR_SCOPE_TYPE_BRIDGE = 2, ACPI_DMAR_SCOPE_TYPE_IOAPIC = 3, ACPI_DMAR_SCOPE_TYPE_HPET = 4, - ACPI_DMAR_SCOPE_TYPE_RESERVED = 5 /* 5 and greater are reserved */ + ACPI_DMAR_SCOPE_TYPE_ACPI = 5, + ACPI_DMAR_SCOPE_TYPE_RESERVED = 6 /* 6 and greater are reserved */ }; struct acpi_dmar_pci_path { @@ -507,6 +509,15 @@ struct acpi_dmar_rhsa { u32 proximity_domain; }; +/* 4: ACPI Namespace Device Declaration Structure */ + +struct acpi_dmar_andd { + struct acpi_dmar_header header; + u8 reserved[3]; + u8 device_number; + u8 object_name[]; +}; + /******************************************************************************* * * HPET - High Precision Event Timer table -- cgit v1.2.3 From 832bd858674023b2415c7585db3beba345ef807f Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Fri, 7 Mar 2014 15:08:36 +0000 Subject: iommu/vt-d: Change scope lists to struct device, bus, devfn It's not only for PCI devices any more, and the scope information for an ACPI device provides the bus and devfn so that has to be stored here too. It is the device pointer itself which needs to be protected with RCU, so the __rcu annotation follows it into the definition of struct dmar_dev_scope, since we're no longer just passing arrays of device pointers around. Signed-off-by: David Woodhouse --- drivers/iommu/dmar.c | 41 ++++++++++++++++++++++------------------- drivers/iommu/intel-iommu.c | 43 +++++++++++++++++++++++++------------------ include/linux/dmar.h | 18 ++++++++++++------ 3 files changed, 59 insertions(+), 43 deletions(-) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 4c6297d1b421..c1e2e0c82e69 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -97,17 +97,17 @@ void *dmar_alloc_dev_scope(void *start, void *end, int *cnt) if (*cnt == 0) return NULL; - return kcalloc(*cnt, sizeof(struct pci_dev *), GFP_KERNEL); + return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL); } -void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt) +void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt) { int i; - struct pci_dev *tmp_dev; + struct device *tmp_dev; if (*devices && *cnt) { for_each_active_dev_scope(*devices, *cnt, i, tmp_dev) - pci_dev_put(tmp_dev); + put_device(tmp_dev); kfree(*devices); } @@ -191,10 +191,11 @@ static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus, /* Return: > 0 if match found, 0 if no match found, < 0 if error happens */ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, void *start, void*end, u16 segment, - struct pci_dev __rcu **devices, int devices_cnt) + struct dmar_dev_scope *devices, + int devices_cnt) { int i, level; - struct pci_dev *tmp, *dev = info->dev; + struct device *tmp, *dev = &info->dev->dev; struct acpi_dmar_device_scope *scope; struct acpi_dmar_pci_path *path; @@ -213,16 +214,18 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, continue; if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^ - (dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) { + (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) { pr_warn("Device scope type does not match for %s\n", - pci_name(dev)); + pci_name(info->dev)); return -EINVAL; } for_each_dev_scope(devices, devices_cnt, i, tmp) if (tmp == NULL) { - rcu_assign_pointer(devices[i], - pci_dev_get(dev)); + devices[i].bus = info->dev->bus->number; + devices[i].devfn = info->dev->devfn; + rcu_assign_pointer(devices[i].dev, + get_device(dev)); return 1; } BUG_ON(i >= devices_cnt); @@ -232,19 +235,19 @@ int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, } int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment, - struct pci_dev __rcu **devices, int count) + struct dmar_dev_scope *devices, int count) { int index; - struct pci_dev *tmp; + struct device *tmp; if (info->seg != segment) return 0; for_each_active_dev_scope(devices, count, index, tmp) - if (tmp == info->dev) { - rcu_assign_pointer(devices[index], NULL); + if (tmp == &info->dev->dev) { + rcu_assign_pointer(devices[index].dev, NULL); synchronize_rcu(); - pci_dev_put(tmp); + put_device(tmp); return 1; } @@ -562,15 +565,15 @@ parse_dmar_table(void) return ret; } -static int dmar_pci_device_match(struct pci_dev __rcu *devices[], int cnt, - struct pci_dev *dev) +static int dmar_pci_device_match(struct dmar_dev_scope devices[], + int cnt, struct pci_dev *dev) { int index; - struct pci_dev *tmp; + struct device *tmp; while (dev) { for_each_active_dev_scope(devices, cnt, index, tmp) - if (dev == tmp) + if (dev_is_pci(tmp) && dev == to_pci_dev(tmp)) return 1; /* Check our parent */ diff --git a/drivers/iommu/intel-iommu.c b/drivers/iommu/intel-iommu.c index 1599cb1e041c..ace088eebb45 100644 --- a/drivers/iommu/intel-iommu.c +++ b/drivers/iommu/intel-iommu.c @@ -382,14 +382,14 @@ struct dmar_rmrr_unit { struct acpi_dmar_header *hdr; /* ACPI header */ u64 base_address; /* reserved base address*/ u64 end_address; /* reserved end address */ - struct pci_dev __rcu **devices; /* target devices */ + struct dmar_dev_scope *devices; /* target devices */ int devices_cnt; /* target device count */ }; struct dmar_atsr_unit { struct list_head list; /* list of ATSR units */ struct acpi_dmar_header *hdr; /* ACPI header */ - struct pci_dev __rcu **devices; /* target devices */ + struct dmar_dev_scope *devices; /* target devices */ int devices_cnt; /* target device count */ u8 include_all:1; /* include all ports */ }; @@ -669,7 +669,8 @@ static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn) { struct dmar_drhd_unit *drhd = NULL; struct intel_iommu *iommu; - struct pci_dev *dev; + struct device *dev; + struct pci_dev *pdev; int i; rcu_read_lock(); @@ -679,11 +680,14 @@ static struct intel_iommu *device_to_iommu(int segment, u8 bus, u8 devfn) for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) { - if (dev->bus->number == bus && dev->devfn == devfn) + if (!dev_is_pci(dev)) + continue; + pdev = to_pci_dev(dev); + if (pdev->bus->number == bus && pdev->devfn == devfn) goto out; - if (dev->subordinate && - dev->subordinate->number <= bus && - dev->subordinate->busn_res.end >= bus) + if (pdev->subordinate && + pdev->subordinate->number <= bus && + pdev->subordinate->busn_res.end >= bus) goto out; } @@ -2479,7 +2483,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, static bool device_has_rmrr(struct pci_dev *dev) { struct dmar_rmrr_unit *rmrr; - struct pci_dev *tmp; + struct device *tmp; int i; rcu_read_lock(); @@ -2490,7 +2494,7 @@ static bool device_has_rmrr(struct pci_dev *dev) */ for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, i, tmp) - if (tmp == dev) { + if (tmp == &dev->dev) { rcu_read_unlock(); return true; } @@ -2602,7 +2606,7 @@ static int __init init_dmars(void) { struct dmar_drhd_unit *drhd; struct dmar_rmrr_unit *rmrr; - struct pci_dev *pdev; + struct device *dev; struct intel_iommu *iommu; int i, ret; @@ -2746,8 +2750,10 @@ static int __init init_dmars(void) for_each_rmrr_units(rmrr) { /* some BIOS lists non-exist devices in DMAR table. */ for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt, - i, pdev) { - ret = iommu_prepare_rmrr_dev(rmrr, pdev); + i, dev) { + if (!dev_is_pci(dev)) + continue; + ret = iommu_prepare_rmrr_dev(rmrr, to_pci_dev(dev)); if (ret) printk(KERN_ERR "IOMMU: mapping reserved region failed\n"); @@ -3434,7 +3440,7 @@ DECLARE_PCI_FIXUP_ENABLE(PCI_VENDOR_ID_INTEL, PCI_DEVICE_ID_INTEL_IOAT_SNB, quir static void __init init_no_remapping_devices(void) { struct dmar_drhd_unit *drhd; - struct pci_dev *dev; + struct device *dev; int i; for_each_drhd_unit(drhd) { @@ -3442,7 +3448,7 @@ static void __init init_no_remapping_devices(void) for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) break; - /* ignore DMAR unit if no pci devices exist */ + /* ignore DMAR unit if no devices exist */ if (i == drhd->devices_cnt) drhd->ignored = 1; } @@ -3454,7 +3460,7 @@ static void __init init_no_remapping_devices(void) for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) - if (!IS_GFX_DEVICE(dev)) + if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev))) break; if (i < drhd->devices_cnt) continue; @@ -3467,7 +3473,7 @@ static void __init init_no_remapping_devices(void) drhd->ignored = 1; for_each_active_dev_scope(drhd->devices, drhd->devices_cnt, i, dev) - dev->dev.archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; + dev->archdata.iommu = DUMMY_DEVICE_DOMAIN_INFO; } } } @@ -3691,7 +3697,8 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) { int i, ret = 1; struct pci_bus *bus; - struct pci_dev *bridge = NULL, *tmp; + struct pci_dev *bridge = NULL; + struct device *tmp; struct acpi_dmar_atsr *atsr; struct dmar_atsr_unit *atsru; @@ -3714,7 +3721,7 @@ int dmar_find_matched_atsr_unit(struct pci_dev *dev) continue; for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp) - if (tmp == bridge) + if (tmp == &bridge->dev) goto out; if (atsru->include_all) diff --git a/include/linux/dmar.h b/include/linux/dmar.h index 0a92e4d978bc..23c8db129560 100644 --- a/include/linux/dmar.h +++ b/include/linux/dmar.h @@ -36,13 +36,19 @@ struct acpi_dmar_header; struct intel_iommu; +struct dmar_dev_scope { + struct device __rcu *dev; + u8 bus; + u8 devfn; +}; + #ifdef CONFIG_DMAR_TABLE extern struct acpi_table_header *dmar_tbl; struct dmar_drhd_unit { struct list_head list; /* list of drhd units */ struct acpi_dmar_header *hdr; /* ACPI header */ u64 reg_base_addr; /* register base address*/ - struct pci_dev __rcu **devices;/* target device array */ + struct dmar_dev_scope *devices;/* target device array */ int devices_cnt; /* target device count */ u16 segment; /* PCI domain */ u8 ignored:1; /* ignore drhd */ @@ -86,7 +92,7 @@ static inline bool dmar_rcu_check(void) #define dmar_rcu_dereference(p) rcu_dereference_check((p), dmar_rcu_check()) #define for_each_dev_scope(a, c, p, d) \ - for ((p) = 0; ((d) = (p) < (c) ? dmar_rcu_dereference((a)[(p)]) : \ + for ((p) = 0; ((d) = (p) < (c) ? dmar_rcu_dereference((a)[(p)].dev) : \ NULL, (p) < (c)); (p)++) #define for_each_active_dev_scope(a, c, p, d) \ @@ -95,15 +101,15 @@ static inline bool dmar_rcu_check(void) extern int dmar_table_init(void); extern int dmar_dev_scope_init(void); extern int dmar_parse_dev_scope(void *start, void *end, int *cnt, - struct pci_dev ***devices, u16 segment); + struct dmar_dev_scope **devices, u16 segment); extern void *dmar_alloc_dev_scope(void *start, void *end, int *cnt); -extern void dmar_free_dev_scope(struct pci_dev __rcu ***devices, int *cnt); +extern void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt); extern int dmar_insert_dev_scope(struct dmar_pci_notify_info *info, void *start, void*end, u16 segment, - struct pci_dev __rcu **devices, + struct dmar_dev_scope *devices, int devices_cnt); extern int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, - u16 segment, struct pci_dev __rcu **devices, + u16 segment, struct dmar_dev_scope *devices, int count); /* Intel IOMMU detection */ extern int detect_intel_iommu(void); -- cgit v1.2.3 From 67ccac41fafda88492620f4c0a30d4ccb2eb7767 Mon Sep 17 00:00:00 2001 From: David Woodhouse Date: Sun, 9 Mar 2014 13:49:45 -0700 Subject: iommu/vt-d: Store PCI segment number in struct intel_iommu Signed-off-by: David Woodhouse --- drivers/iommu/dmar.c | 1 + include/linux/intel-iommu.h | 1 + 2 files changed, 2 insertions(+) (limited to 'include') diff --git a/drivers/iommu/dmar.c b/drivers/iommu/dmar.c index 7ea086f61073..56e1c79dc77f 100644 --- a/drivers/iommu/dmar.c +++ b/drivers/iommu/dmar.c @@ -952,6 +952,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd) } iommu->agaw = agaw; iommu->msagaw = msagaw; + iommu->segment = drhd->segment; iommu->node = -1; diff --git a/include/linux/intel-iommu.h b/include/linux/intel-iommu.h index 2c4bed593b32..0a2da5188217 100644 --- a/include/linux/intel-iommu.h +++ b/include/linux/intel-iommu.h @@ -319,6 +319,7 @@ struct intel_iommu { int agaw; /* agaw of this iommu */ int msagaw; /* max sagaw of this iommu */ unsigned int irq; + u16 segment; /* PCI segment# */ unsigned char name[13]; /* Device Name */ #ifdef CONFIG_INTEL_IOMMU -- cgit v1.2.3