diff options
Diffstat (limited to 'drivers/pci/msi/msi.c')
| -rw-r--r-- | drivers/pci/msi/msi.c | 1078 |
1 files changed, 425 insertions, 653 deletions
diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index fdd2ec09651e..1f716624ca56 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -13,82 +13,114 @@ #include "../pci.h" #include "msi.h" -static int pci_msi_enable = 1; +int pci_msi_enable = 1; int pci_msi_ignore_mask; -static noinline void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) +/** + * pci_msi_supported - check whether MSI may be enabled on a device + * @dev: pointer to the pci_dev data structure of MSI device function + * @nvec: how many MSIs have been requested? + * + * Look at global flags, the device itself, and its parent buses + * to determine if MSI/-X are supported for the device. If MSI/-X is + * supported return 1, else return 0. + **/ +static int pci_msi_supported(struct pci_dev *dev, int nvec) { - raw_spinlock_t *lock = &to_pci_dev(desc->dev)->msi_lock; - unsigned long flags; + struct pci_bus *bus; - if (!desc->pci.msi_attrib.can_mask) - return; + /* MSI must be globally enabled and supported by the device */ + if (!pci_msi_enable) + return 0; - raw_spin_lock_irqsave(lock, flags); - desc->pci.msi_mask &= ~clear; - desc->pci.msi_mask |= set; - pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->pci.mask_pos, - desc->pci.msi_mask); - raw_spin_unlock_irqrestore(lock, flags); -} + if (!dev || dev->no_msi) + return 0; -static inline void pci_msi_mask(struct msi_desc *desc, u32 mask) -{ - pci_msi_update_mask(desc, 0, mask); -} + /* + * You can't ask to have 0 or less MSIs configured. + * a) it's stupid .. + * b) the list manipulation code assumes nvec >= 1. + */ + if (nvec < 1) + return 0; -static inline void pci_msi_unmask(struct msi_desc *desc, u32 mask) -{ - pci_msi_update_mask(desc, mask, 0); + /* + * Any bridge which does NOT route MSI transactions from its + * secondary bus to its primary bus must set NO_MSI flag on + * the secondary pci_bus. + * + * The NO_MSI flag can either be set directly by: + * - arch-specific PCI host bus controller drivers (deprecated) + * - quirks for specific PCI bridges + * + * or indirectly by platform-specific PCI host bridge drivers by + * advertising the 'msi_domain' property, which results in + * the NO_MSI flag when no MSI domain is found for this bridge + * at probe time. + */ + for (bus = dev->bus; bus; bus = bus->parent) + if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) + return 0; + + return 1; } -static inline void __iomem *pci_msix_desc_addr(struct msi_desc *desc) +static void pcim_msi_release(void *pcidev) { - return desc->pci.mask_base + desc->msi_index * PCI_MSIX_ENTRY_SIZE; + struct pci_dev *dev = pcidev; + + dev->is_msi_managed = false; + pci_free_irq_vectors(dev); } /* - * This internal function does not flush PCI writes to the device. All - * users must ensure that they read from the device before either assuming - * that the device state is up to date, or returning out of this file. - * It does not affect the msi_desc::msix_ctrl cache either. Use with care! + * Needs to be separate from pcim_release to prevent an ordering problem + * vs. msi_device_data_release() in the MSI core code. */ -static void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) +static int pcim_setup_msi_release(struct pci_dev *dev) { - void __iomem *desc_addr = pci_msix_desc_addr(desc); + int ret; - if (desc->pci.msi_attrib.can_mask) - writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); -} + if (!pci_is_managed(dev) || dev->is_msi_managed) + return 0; -static inline void pci_msix_mask(struct msi_desc *desc) -{ - desc->pci.msix_ctrl |= PCI_MSIX_ENTRY_CTRL_MASKBIT; - pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); - /* Flush write to device */ - readl(desc->pci.mask_base); + ret = devm_add_action(&dev->dev, pcim_msi_release, dev); + if (!ret) + dev->is_msi_managed = true; + return ret; } -static inline void pci_msix_unmask(struct msi_desc *desc) +/* + * Ordering vs. devres: msi device data has to be installed first so that + * pcim_msi_release() is invoked before it on device release. + */ +static int pci_setup_msi_context(struct pci_dev *dev) { - desc->pci.msix_ctrl &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT; - pci_msix_write_vector_ctrl(desc, desc->pci.msix_ctrl); -} + int ret = msi_setup_device_data(&dev->dev); -static void __pci_msi_mask_desc(struct msi_desc *desc, u32 mask) -{ - if (desc->pci.msi_attrib.is_msix) - pci_msix_mask(desc); - else - pci_msi_mask(desc, mask); + if (!ret) + ret = pcim_setup_msi_release(dev); + return ret; } -static void __pci_msi_unmask_desc(struct msi_desc *desc, u32 mask) +/* + * Helper functions for mask/unmask and MSI message handling + */ + +void pci_msi_update_mask(struct msi_desc *desc, u32 clear, u32 set) { - if (desc->pci.msi_attrib.is_msix) - pci_msix_unmask(desc); - else - pci_msi_unmask(desc, mask); + raw_spinlock_t *lock = &to_pci_dev(desc->dev)->msi_lock; + unsigned long flags; + + if (!desc->pci.msi_attrib.can_mask) + return; + + raw_spin_lock_irqsave(lock, flags); + desc->pci.msi_mask &= ~clear; + desc->pci.msi_mask |= set; + pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->pci.mask_pos, + desc->pci.msi_mask); + raw_spin_unlock_irqrestore(lock, flags); } /** @@ -148,6 +180,58 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) } } +static inline void pci_write_msg_msi(struct pci_dev *dev, struct msi_desc *desc, + struct msi_msg *msg) +{ + int pos = dev->msi_cap; + u16 msgctl; + + pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); + msgctl &= ~PCI_MSI_FLAGS_QSIZE; + msgctl |= desc->pci.msi_attrib.multiple << 4; + pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); + + pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, msg->address_lo); + if (desc->pci.msi_attrib.is_64) { + pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, msg->address_hi); + pci_write_config_word(dev, pos + PCI_MSI_DATA_64, msg->data); + } else { + pci_write_config_word(dev, pos + PCI_MSI_DATA_32, msg->data); + } + /* Ensure that the writes are visible in the device */ + pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); +} + +static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg) +{ + void __iomem *base = pci_msix_desc_addr(desc); + u32 ctrl = desc->pci.msix_ctrl; + bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); + + if (desc->pci.msi_attrib.is_virtual) + return; + /* + * The specification mandates that the entry is masked + * when the message is modified: + * + * "If software changes the Address or Data value of an + * entry while the entry is unmasked, the result is + * undefined." + */ + if (unmasked) + pci_msix_write_vector_ctrl(desc, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); + + writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); + writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); + writel(msg->data, base + PCI_MSIX_ENTRY_DATA); + + if (unmasked) + pci_msix_write_vector_ctrl(desc, ctrl); + + /* Ensure that the writes are visible in the device */ + readl(base + PCI_MSIX_ENTRY_DATA); +} + void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) { struct pci_dev *dev = msi_desc_to_pci_dev(entry); @@ -155,63 +239,15 @@ void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg) if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) { /* Don't touch the hardware now */ } else if (entry->pci.msi_attrib.is_msix) { - void __iomem *base = pci_msix_desc_addr(entry); - u32 ctrl = entry->pci.msix_ctrl; - bool unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); - - if (entry->pci.msi_attrib.is_virtual) - goto skip; - - /* - * The specification mandates that the entry is masked - * when the message is modified: - * - * "If software changes the Address or Data value of an - * entry while the entry is unmasked, the result is - * undefined." - */ - if (unmasked) - pci_msix_write_vector_ctrl(entry, ctrl | PCI_MSIX_ENTRY_CTRL_MASKBIT); - - writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); - writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); - writel(msg->data, base + PCI_MSIX_ENTRY_DATA); - - if (unmasked) - pci_msix_write_vector_ctrl(entry, ctrl); - - /* Ensure that the writes are visible in the device */ - readl(base + PCI_MSIX_ENTRY_DATA); + pci_write_msg_msix(entry, msg); } else { - int pos = dev->msi_cap; - u16 msgctl; - - pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); - msgctl &= ~PCI_MSI_FLAGS_QSIZE; - msgctl |= entry->pci.msi_attrib.multiple << 4; - pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl); - - pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO, - msg->address_lo); - if (entry->pci.msi_attrib.is_64) { - pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI, - msg->address_hi); - pci_write_config_word(dev, pos + PCI_MSI_DATA_64, - msg->data); - } else { - pci_write_config_word(dev, pos + PCI_MSI_DATA_32, - msg->data); - } - /* Ensure that the writes are visible in the device */ - pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl); + pci_write_msg_msi(dev, entry, msg); } -skip: entry->msg = *msg; if (entry->write_msi_msg) entry->write_msi_msg(entry, entry->write_msi_msg_data); - } void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) @@ -222,15 +258,8 @@ void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg) } EXPORT_SYMBOL_GPL(pci_write_msi_msg); -static void free_msi_irqs(struct pci_dev *dev) -{ - pci_msi_teardown_msi_irqs(dev); - if (dev->msix_base) { - iounmap(dev->msix_base); - dev->msix_base = NULL; - } -} +/* PCI/MSI specific functionality */ static void pci_intx_for_msi(struct pci_dev *dev, int enable) { @@ -249,118 +278,6 @@ static void pci_msi_set_enable(struct pci_dev *dev, int enable) pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); } -/* - * Architecture override returns true when the PCI MSI message should be - * written by the generic restore function. - */ -bool __weak arch_restore_msi_irqs(struct pci_dev *dev) -{ - return true; -} - -static void __pci_restore_msi_state(struct pci_dev *dev) -{ - struct msi_desc *entry; - u16 control; - - if (!dev->msi_enabled) - return; - - entry = irq_get_msi_desc(dev->irq); - - pci_intx_for_msi(dev, 0); - pci_msi_set_enable(dev, 0); - if (arch_restore_msi_irqs(dev)) - __pci_write_msi_msg(entry, &entry->msg); - - pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); - pci_msi_update_mask(entry, 0, 0); - control &= ~PCI_MSI_FLAGS_QSIZE; - control |= (entry->pci.msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; - pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); -} - -static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) -{ - u16 ctrl; - - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); - ctrl &= ~clear; - ctrl |= set; - pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl); -} - -static void __pci_restore_msix_state(struct pci_dev *dev) -{ - struct msi_desc *entry; - bool write_msg; - - if (!dev->msix_enabled) - return; - - /* route the table */ - pci_intx_for_msi(dev, 0); - pci_msix_clear_and_set_ctrl(dev, 0, - PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); - - write_msg = arch_restore_msi_irqs(dev); - - msi_lock_descs(&dev->dev); - msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { - if (write_msg) - __pci_write_msi_msg(entry, &entry->msg); - pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl); - } - msi_unlock_descs(&dev->dev); - - pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); -} - -void pci_restore_msi_state(struct pci_dev *dev) -{ - __pci_restore_msi_state(dev); - __pci_restore_msix_state(dev); -} -EXPORT_SYMBOL_GPL(pci_restore_msi_state); - -static void pcim_msi_release(void *pcidev) -{ - struct pci_dev *dev = pcidev; - - dev->is_msi_managed = false; - pci_free_irq_vectors(dev); -} - -/* - * Needs to be separate from pcim_release to prevent an ordering problem - * vs. msi_device_data_release() in the MSI core code. - */ -static int pcim_setup_msi_release(struct pci_dev *dev) -{ - int ret; - - if (!pci_is_managed(dev) || dev->is_msi_managed) - return 0; - - ret = devm_add_action(&dev->dev, pcim_msi_release, dev); - if (!ret) - dev->is_msi_managed = true; - return ret; -} - -/* - * Ordering vs. devres: msi device data has to be installed first so that - * pcim_msi_release() is invoked before it on device release. - */ -static int pci_setup_msi_context(struct pci_dev *dev) -{ - int ret = msi_setup_device_data(&dev->dev); - - if (!ret) - ret = pcim_setup_msi_release(dev); - return ret; -} - static int msi_setup_msi_desc(struct pci_dev *dev, int nvec, struct irq_affinity_desc *masks) { @@ -395,7 +312,7 @@ static int msi_setup_msi_desc(struct pci_dev *dev, int nvec, if (desc.pci.msi_attrib.can_mask) pci_read_config_dword(dev, desc.pci.mask_pos, &desc.pci.msi_mask); - return msi_add_msi_desc(&dev->dev, &desc); + return msi_insert_msi_desc(&dev->dev, &desc); } static int msi_verify_entries(struct pci_dev *dev) @@ -434,6 +351,10 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, struct msi_desc *entry; int ret; + /* Reject multi-MSI early on irq domain enabled architectures */ + if (nvec > 1 && !pci_msi_domain_supports(dev, MSI_FLAG_MULTI_PCI_MSI, ALLOW_LEGACY)) + return 1; + /* * Disable MSI during setup in the hardware, but mark it enabled * so that setup code can evaluate it. @@ -472,7 +393,7 @@ static int msi_capability_init(struct pci_dev *dev, int nvec, err: pci_msi_unmask(entry, msi_multi_mask(entry)); - free_msi_irqs(dev); + pci_free_msi_irqs(dev); fail: dev->msi_enabled = 0; unlock: @@ -481,6 +402,152 @@ unlock: return ret; } +int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, + struct irq_affinity *affd) +{ + int nvec; + int rc; + + if (!pci_msi_supported(dev, minvec) || dev->current_state != PCI_D0) + return -EINVAL; + + /* Check whether driver already requested MSI-X IRQs */ + if (dev->msix_enabled) { + pci_info(dev, "can't enable MSI (MSI-X already enabled)\n"); + return -EINVAL; + } + + if (maxvec < minvec) + return -ERANGE; + + if (WARN_ON_ONCE(dev->msi_enabled)) + return -EINVAL; + + nvec = pci_msi_vec_count(dev); + if (nvec < 0) + return nvec; + if (nvec < minvec) + return -ENOSPC; + + if (nvec > maxvec) + nvec = maxvec; + + rc = pci_setup_msi_context(dev); + if (rc) + return rc; + + if (!pci_setup_msi_device_domain(dev)) + return -ENODEV; + + for (;;) { + if (affd) { + nvec = irq_calc_affinity_vectors(minvec, nvec, affd); + if (nvec < minvec) + return -ENOSPC; + } + + rc = msi_capability_init(dev, nvec, affd); + if (rc == 0) + return nvec; + + if (rc < 0) + return rc; + if (rc < minvec) + return -ENOSPC; + + nvec = rc; + } +} + +/** + * pci_msi_vec_count - Return the number of MSI vectors a device can send + * @dev: device to report about + * + * This function returns the number of MSI vectors a device requested via + * Multiple Message Capable register. It returns a negative errno if the + * device is not capable sending MSI interrupts. Otherwise, the call succeeds + * and returns a power of two, up to a maximum of 2^5 (32), according to the + * MSI specification. + **/ +int pci_msi_vec_count(struct pci_dev *dev) +{ + int ret; + u16 msgctl; + + if (!dev->msi_cap) + return -EINVAL; + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl); + ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); + + return ret; +} +EXPORT_SYMBOL(pci_msi_vec_count); + +/* + * Architecture override returns true when the PCI MSI message should be + * written by the generic restore function. + */ +bool __weak arch_restore_msi_irqs(struct pci_dev *dev) +{ + return true; +} + +void __pci_restore_msi_state(struct pci_dev *dev) +{ + struct msi_desc *entry; + u16 control; + + if (!dev->msi_enabled) + return; + + entry = irq_get_msi_desc(dev->irq); + + pci_intx_for_msi(dev, 0); + pci_msi_set_enable(dev, 0); + if (arch_restore_msi_irqs(dev)) + __pci_write_msi_msg(entry, &entry->msg); + + pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control); + pci_msi_update_mask(entry, 0, 0); + control &= ~PCI_MSI_FLAGS_QSIZE; + control |= (entry->pci.msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE; + pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control); +} + +void pci_msi_shutdown(struct pci_dev *dev) +{ + struct msi_desc *desc; + + if (!pci_msi_enable || !dev || !dev->msi_enabled) + return; + + pci_msi_set_enable(dev, 0); + pci_intx_for_msi(dev, 1); + dev->msi_enabled = 0; + + /* Return the device with MSI unmasked as initial states */ + desc = msi_first_desc(&dev->dev, MSI_DESC_ALL); + if (!WARN_ON_ONCE(!desc)) + pci_msi_unmask(desc, msi_multi_mask(desc)); + + /* Restore dev->irq to its default pin-assertion IRQ */ + dev->irq = desc->pci.msi_attrib.default_irq; + pcibios_alloc_irq(dev); +} + +/* PCI/MSI-X specific functionality */ + +static void pci_msix_clear_and_set_ctrl(struct pci_dev *dev, u16 clear, u16 set) +{ + u16 ctrl; + + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &ctrl); + ctrl &= ~clear; + ctrl |= set; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, ctrl); +} + static void __iomem *msix_map_region(struct pci_dev *dev, unsigned int nr_entries) { @@ -502,36 +569,58 @@ static void __iomem *msix_map_region(struct pci_dev *dev, return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); } -static int msix_setup_msi_descs(struct pci_dev *dev, void __iomem *base, - struct msix_entry *entries, int nvec, - struct irq_affinity_desc *masks) +/** + * msix_prepare_msi_desc - Prepare a half initialized MSI descriptor for operation + * @dev: The PCI device for which the descriptor is prepared + * @desc: The MSI descriptor for preparation + * + * This is separate from msix_setup_msi_descs() below to handle dynamic + * allocations for MSI-X after initial enablement. + * + * Ideally the whole MSI-X setup would work that way, but there is no way to + * support this for the legacy arch_setup_msi_irqs() mechanism and for the + * fake irq domains like the x86 XEN one. Sigh... + * + * The descriptor is zeroed and only @desc::msi_index and @desc::affinity + * are set. When called from msix_setup_msi_descs() then the is_virtual + * attribute is initialized as well. + * + * Fill in the rest. + */ +void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) +{ + desc->nvec_used = 1; + desc->pci.msi_attrib.is_msix = 1; + desc->pci.msi_attrib.is_64 = 1; + desc->pci.msi_attrib.default_irq = dev->irq; + desc->pci.mask_base = dev->msix_base; + desc->pci.msi_attrib.can_mask = !pci_msi_ignore_mask && + !desc->pci.msi_attrib.is_virtual; + + if (desc->pci.msi_attrib.can_mask) { + void __iomem *addr = pci_msix_desc_addr(desc); + + desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); + } +} + +static int msix_setup_msi_descs(struct pci_dev *dev, struct msix_entry *entries, + int nvec, struct irq_affinity_desc *masks) { int ret = 0, i, vec_count = pci_msix_vec_count(dev); struct irq_affinity_desc *curmsk; struct msi_desc desc; - void __iomem *addr; memset(&desc, 0, sizeof(desc)); - desc.nvec_used = 1; - desc.pci.msi_attrib.is_msix = 1; - desc.pci.msi_attrib.is_64 = 1; - desc.pci.msi_attrib.default_irq = dev->irq; - desc.pci.mask_base = base; - for (i = 0, curmsk = masks; i < nvec; i++, curmsk++) { desc.msi_index = entries ? entries[i].entry : i; desc.affinity = masks ? curmsk : NULL; desc.pci.msi_attrib.is_virtual = desc.msi_index >= vec_count; - desc.pci.msi_attrib.can_mask = !pci_msi_ignore_mask && - !desc.pci.msi_attrib.is_virtual; - if (desc.pci.msi_attrib.can_mask) { - addr = pci_msix_desc_addr(&desc); - desc.pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); - } + msix_prepare_msi_desc(dev, &desc); - ret = msi_add_msi_desc(&dev->dev, &desc); + ret = msi_insert_msi_desc(&dev->dev, &desc); if (ret) break; } @@ -562,9 +651,8 @@ static void msix_mask_all(void __iomem *base, int tsize) writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL); } -static int msix_setup_interrupts(struct pci_dev *dev, void __iomem *base, - struct msix_entry *entries, int nvec, - struct irq_affinity *affd) +static int msix_setup_interrupts(struct pci_dev *dev, struct msix_entry *entries, + int nvec, struct irq_affinity *affd) { struct irq_affinity_desc *masks = NULL; int ret; @@ -573,7 +661,7 @@ static int msix_setup_interrupts(struct pci_dev *dev, void __iomem *base, masks = irq_create_affinity_masks(nvec, affd); msi_lock_descs(&dev->dev); - ret = msix_setup_msi_descs(dev, base, entries, nvec, masks); + ret = msix_setup_msi_descs(dev, entries, nvec, masks); if (ret) goto out_free; @@ -590,7 +678,7 @@ static int msix_setup_interrupts(struct pci_dev *dev, void __iomem *base, goto out_unlock; out_free: - free_msi_irqs(dev); + pci_free_msi_irqs(dev); out_unlock: msi_unlock_descs(&dev->dev); kfree(masks); @@ -611,7 +699,6 @@ out_unlock: static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, int nvec, struct irq_affinity *affd) { - void __iomem *base; int ret, tsize; u16 control; @@ -629,15 +716,13 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); /* Request & Map MSI-X table region */ tsize = msix_table_size(control); - base = msix_map_region(dev, tsize); - if (!base) { + dev->msix_base = msix_map_region(dev, tsize); + if (!dev->msix_base) { ret = -ENOMEM; goto out_disable; } - dev->msix_base = base; - - ret = msix_setup_interrupts(dev, base, entries, nvec, affd); + ret = msix_setup_interrupts(dev, entries, nvec, affd); if (ret) goto out_disable; @@ -652,7 +737,7 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, * which takes the MSI-X mask bits into account even * when MSI-X is disabled, which prevents MSI delivery. */ - msix_mask_all(base, tsize); + msix_mask_all(dev->msix_base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); pcibios_free_irq(dev); @@ -665,236 +750,82 @@ out_disable: return ret; } -/** - * pci_msi_supported - check whether MSI may be enabled on a device - * @dev: pointer to the pci_dev data structure of MSI device function - * @nvec: how many MSIs have been requested? - * - * Look at global flags, the device itself, and its parent buses - * to determine if MSI/-X are supported for the device. If MSI/-X is - * supported return 1, else return 0. - **/ -static int pci_msi_supported(struct pci_dev *dev, int nvec) -{ - struct pci_bus *bus; - - /* MSI must be globally enabled and supported by the device */ - if (!pci_msi_enable) - return 0; - - if (!dev || dev->no_msi) - return 0; - - /* - * You can't ask to have 0 or less MSIs configured. - * a) it's stupid .. - * b) the list manipulation code assumes nvec >= 1. - */ - if (nvec < 1) - return 0; - - /* - * Any bridge which does NOT route MSI transactions from its - * secondary bus to its primary bus must set NO_MSI flag on - * the secondary pci_bus. - * - * The NO_MSI flag can either be set directly by: - * - arch-specific PCI host bus controller drivers (deprecated) - * - quirks for specific PCI bridges - * - * or indirectly by platform-specific PCI host bridge drivers by - * advertising the 'msi_domain' property, which results in - * the NO_MSI flag when no MSI domain is found for this bridge - * at probe time. - */ - for (bus = dev->bus; bus; bus = bus->parent) - if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI) - return 0; - - return 1; -} - -/** - * pci_msi_vec_count - Return the number of MSI vectors a device can send - * @dev: device to report about - * - * This function returns the number of MSI vectors a device requested via - * Multiple Message Capable register. It returns a negative errno if the - * device is not capable sending MSI interrupts. Otherwise, the call succeeds - * and returns a power of two, up to a maximum of 2^5 (32), according to the - * MSI specification. - **/ -int pci_msi_vec_count(struct pci_dev *dev) -{ - int ret; - u16 msgctl; - - if (!dev->msi_cap) - return -EINVAL; - - pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl); - ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1); - - return ret; -} -EXPORT_SYMBOL(pci_msi_vec_count); - -static void pci_msi_shutdown(struct pci_dev *dev) -{ - struct msi_desc *desc; - - if (!pci_msi_enable || !dev || !dev->msi_enabled) - return; - - pci_msi_set_enable(dev, 0); - pci_intx_for_msi(dev, 1); - dev->msi_enabled = 0; - - /* Return the device with MSI unmasked as initial states */ - desc = msi_first_desc(&dev->dev, MSI_DESC_ALL); - if (!WARN_ON_ONCE(!desc)) - pci_msi_unmask(desc, msi_multi_mask(desc)); - - /* Restore dev->irq to its default pin-assertion IRQ */ - dev->irq = desc->pci.msi_attrib.default_irq; - pcibios_alloc_irq(dev); -} - -void pci_disable_msi(struct pci_dev *dev) -{ - if (!pci_msi_enable || !dev || !dev->msi_enabled) - return; - - msi_lock_descs(&dev->dev); - pci_msi_shutdown(dev); - free_msi_irqs(dev); - msi_unlock_descs(&dev->dev); -} -EXPORT_SYMBOL(pci_disable_msi); - -/** - * pci_msix_vec_count - return the number of device's MSI-X table entries - * @dev: pointer to the pci_dev data structure of MSI-X device function - * This function returns the number of device's MSI-X table entries and - * therefore the number of MSI-X vectors device is capable of sending. - * It returns a negative errno if the device is not capable of sending MSI-X - * interrupts. - **/ -int pci_msix_vec_count(struct pci_dev *dev) -{ - u16 control; - - if (!dev->msix_cap) - return -EINVAL; - - pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control); - return msix_table_size(control); -} -EXPORT_SYMBOL(pci_msix_vec_count); - -static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries, - int nvec, struct irq_affinity *affd, int flags) +static bool pci_msix_validate_entries(struct pci_dev *dev, struct msix_entry *entries, + int nvec, int hwsize) { - int nr_entries; + bool nogap; int i, j; - if (!pci_msi_supported(dev, nvec) || dev->current_state != PCI_D0) - return -EINVAL; + if (!entries) + return true; - nr_entries = pci_msix_vec_count(dev); - if (nr_entries < 0) - return nr_entries; - if (nvec > nr_entries && !(flags & PCI_IRQ_VIRTUAL)) - return nr_entries; + nogap = pci_msi_domain_supports(dev, MSI_FLAG_MSIX_CONTIGUOUS, DENY_LEGACY); - if (entries) { - /* Check for any invalid entries */ - for (i = 0; i < nvec; i++) { - if (entries[i].entry >= nr_entries) - return -EINVAL; /* invalid entry */ - for (j = i + 1; j < nvec; j++) { - if (entries[i].entry == entries[j].entry) - return -EINVAL; /* duplicate entry */ - } - } - } + for (i = 0; i < nvec; i++) { + /* Entry within hardware limit? */ + if (entries[i].entry >= hwsize) + return false; - /* Check whether driver already requested for MSI IRQ */ - if (dev->msi_enabled) { - pci_info(dev, "can't enable MSI-X (MSI IRQ already assigned)\n"); - return -EINVAL; + /* Check for duplicate entries */ + for (j = i + 1; j < nvec; j++) { + if (entries[i].entry == entries[j].entry) + return false; + } + /* Check for unsupported gaps */ + if (nogap && entries[i].entry != i) + return false; } - return msix_capability_init(dev, entries, nvec, affd); + return true; } -static void pci_msix_shutdown(struct pci_dev *dev) +int __pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, int minvec, + int maxvec, struct irq_affinity *affd, int flags) { - struct msi_desc *desc; + int hwsize, rc, nvec = maxvec; - if (!pci_msi_enable || !dev || !dev->msix_enabled) - return; + if (maxvec < minvec) + return -ERANGE; - if (pci_dev_is_disconnected(dev)) { - dev->msix_enabled = 0; - return; + if (dev->msi_enabled) { + pci_info(dev, "can't enable MSI-X (MSI already enabled)\n"); + return -EINVAL; } - /* Return the device with MSI-X masked as initial states */ - msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) - pci_msix_mask(desc); + if (WARN_ON_ONCE(dev->msix_enabled)) + return -EINVAL; - pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); - pci_intx_for_msi(dev, 1); - dev->msix_enabled = 0; - pcibios_alloc_irq(dev); -} + /* Check MSI-X early on irq domain enabled architectures */ + if (!pci_msi_domain_supports(dev, MSI_FLAG_PCI_MSIX, ALLOW_LEGACY)) + return -ENOTSUPP; -void pci_disable_msix(struct pci_dev *dev) -{ - if (!pci_msi_enable || !dev || !dev->msix_enabled) - return; - - msi_lock_descs(&dev->dev); - pci_msix_shutdown(dev); - free_msi_irqs(dev); - msi_unlock_descs(&dev->dev); -} -EXPORT_SYMBOL(pci_disable_msix); + if (!pci_msi_supported(dev, nvec) || dev->current_state != PCI_D0) + return -EINVAL; -static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, - struct irq_affinity *affd) -{ - int nvec; - int rc; + hwsize = pci_msix_vec_count(dev); + if (hwsize < 0) + return hwsize; - if (!pci_msi_supported(dev, minvec) || dev->current_state != PCI_D0) + if (!pci_msix_validate_entries(dev, entries, nvec, hwsize)) return -EINVAL; - /* Check whether driver already requested MSI-X IRQs */ - if (dev->msix_enabled) { - pci_info(dev, "can't enable MSI (MSI-X already enabled)\n"); - return -EINVAL; + if (hwsize < nvec) { + /* Keep the IRQ virtual hackery working */ + if (flags & PCI_IRQ_VIRTUAL) + hwsize = nvec; + else + nvec = hwsize; } - if (maxvec < minvec) - return -ERANGE; - - if (WARN_ON_ONCE(dev->msi_enabled)) - return -EINVAL; - - nvec = pci_msi_vec_count(dev); - if (nvec < 0) - return nvec; if (nvec < minvec) return -ENOSPC; - if (nvec > maxvec) - nvec = maxvec; - rc = pci_setup_msi_context(dev); if (rc) return rc; + if (!pci_setup_msix_device_domain(dev, hwsize)) + return -ENODEV; + for (;;) { if (affd) { nvec = irq_calc_affinity_vectors(minvec, nvec, affd); @@ -902,7 +833,7 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, return -ENOSPC; } - rc = msi_capability_init(dev, nvec, affd); + rc = msix_capability_init(dev, entries, nvec, affd); if (rc == 0) return nvec; @@ -915,214 +846,67 @@ static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec, } } -/* deprecated, don't use */ -int pci_enable_msi(struct pci_dev *dev) -{ - int rc = __pci_enable_msi_range(dev, 1, 1, NULL); - if (rc < 0) - return rc; - return 0; -} -EXPORT_SYMBOL(pci_enable_msi); - -static int __pci_enable_msix_range(struct pci_dev *dev, - struct msix_entry *entries, int minvec, - int maxvec, struct irq_affinity *affd, - int flags) +void __pci_restore_msix_state(struct pci_dev *dev) { - int rc, nvec = maxvec; - - if (maxvec < minvec) - return -ERANGE; - - if (WARN_ON_ONCE(dev->msix_enabled)) - return -EINVAL; - - rc = pci_setup_msi_context(dev); - if (rc) - return rc; + struct msi_desc *entry; + bool write_msg; - for (;;) { - if (affd) { - nvec = irq_calc_affinity_vectors(minvec, nvec, affd); - if (nvec < minvec) - return -ENOSPC; - } + if (!dev->msix_enabled) + return; - rc = __pci_enable_msix(dev, entries, nvec, affd, flags); - if (rc == 0) - return nvec; + /* route the table */ + pci_intx_for_msi(dev, 0); + pci_msix_clear_and_set_ctrl(dev, 0, + PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL); - if (rc < 0) - return rc; - if (rc < minvec) - return -ENOSPC; + write_msg = arch_restore_msi_irqs(dev); - nvec = rc; + msi_lock_descs(&dev->dev); + msi_for_each_desc(entry, &dev->dev, MSI_DESC_ALL) { + if (write_msg) + __pci_write_msi_msg(entry, &entry->msg); + pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl); } -} + msi_unlock_descs(&dev->dev); -/** - * pci_enable_msix_range - configure device's MSI-X capability structure - * @dev: pointer to the pci_dev data structure of MSI-X device function - * @entries: pointer to an array of MSI-X entries - * @minvec: minimum number of MSI-X IRQs requested - * @maxvec: maximum number of MSI-X IRQs requested - * - * Setup the MSI-X capability structure of device function with a maximum - * possible number of interrupts in the range between @minvec and @maxvec - * upon its software driver call to request for MSI-X mode enabled on its - * hardware device function. It returns a negative errno if an error occurs. - * If it succeeds, it returns the actual number of interrupts allocated and - * indicates the successful configuration of MSI-X capability structure - * with new allocated MSI-X interrupts. - **/ -int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries, - int minvec, int maxvec) -{ - return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL, 0); + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); } -EXPORT_SYMBOL(pci_enable_msix_range); -/** - * pci_alloc_irq_vectors_affinity - allocate multiple IRQs for a device - * @dev: PCI device to operate on - * @min_vecs: minimum number of vectors required (must be >= 1) - * @max_vecs: maximum (desired) number of vectors - * @flags: flags or quirks for the allocation - * @affd: optional description of the affinity requirements - * - * Allocate up to @max_vecs interrupt vectors for @dev, using MSI-X or MSI - * vectors if available, and fall back to a single legacy vector - * if neither is available. Return the number of vectors allocated, - * (which might be smaller than @max_vecs) if successful, or a negative - * error code on error. If less than @min_vecs interrupt vectors are - * available for @dev the function will fail with -ENOSPC. - * - * To get the Linux IRQ number used for a vector that can be passed to - * request_irq() use the pci_irq_vector() helper. - */ -int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs, - unsigned int max_vecs, unsigned int flags, - struct irq_affinity *affd) +void pci_msix_shutdown(struct pci_dev *dev) { - struct irq_affinity msi_default_affd = {0}; - int nvecs = -ENOSPC; - - if (flags & PCI_IRQ_AFFINITY) { - if (!affd) - affd = &msi_default_affd; - } else { - if (WARN_ON(affd)) - affd = NULL; - } + struct msi_desc *desc; - if (flags & PCI_IRQ_MSIX) { - nvecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs, - affd, flags); - if (nvecs > 0) - return nvecs; - } + if (!pci_msi_enable || !dev || !dev->msix_enabled) + return; - if (flags & PCI_IRQ_MSI) { - nvecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd); - if (nvecs > 0) - return nvecs; + if (pci_dev_is_disconnected(dev)) { + dev->msix_enabled = 0; + return; } - /* use legacy IRQ if allowed */ - if (flags & PCI_IRQ_LEGACY) { - if (min_vecs == 1 && dev->irq) { - /* - * Invoke the affinity spreading logic to ensure that - * the device driver can adjust queue configuration - * for the single interrupt case. - */ - if (affd) - irq_create_affinity_masks(1, affd); - pci_intx(dev, 1); - return 1; - } - } + /* Return the device with MSI-X masked as initial states */ + msi_for_each_desc(desc, &dev->dev, MSI_DESC_ALL) + pci_msix_mask(desc); - return nvecs; + pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0); + pci_intx_for_msi(dev, 1); + dev->msix_enabled = 0; + pcibios_alloc_irq(dev); } -EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity); -/** - * pci_free_irq_vectors - free previously allocated IRQs for a device - * @dev: PCI device to operate on - * - * Undoes the allocations and enabling in pci_alloc_irq_vectors(). - */ -void pci_free_irq_vectors(struct pci_dev *dev) -{ - pci_disable_msix(dev); - pci_disable_msi(dev); -} -EXPORT_SYMBOL(pci_free_irq_vectors); +/* Common interfaces */ -/** - * pci_irq_vector - return Linux IRQ number of a device vector - * @dev: PCI device to operate on - * @nr: Interrupt vector index (0-based) - * - * @nr has the following meanings depending on the interrupt mode: - * MSI-X: The index in the MSI-X vector table - * MSI: The index of the enabled MSI vectors - * INTx: Must be 0 - * - * Return: The Linux interrupt number or -EINVAl if @nr is out of range. - */ -int pci_irq_vector(struct pci_dev *dev, unsigned int nr) +void pci_free_msi_irqs(struct pci_dev *dev) { - unsigned int irq; - - if (!dev->msi_enabled && !dev->msix_enabled) - return !nr ? dev->irq : -EINVAL; + pci_msi_teardown_msi_irqs(dev); - irq = msi_get_virq(&dev->dev, nr); - return irq ? irq : -EINVAL; + if (dev->msix_base) { + iounmap(dev->msix_base); + dev->msix_base = NULL; + } } -EXPORT_SYMBOL(pci_irq_vector); -/** - * pci_irq_get_affinity - return the affinity of a particular MSI vector - * @dev: PCI device to operate on - * @nr: device-relative interrupt vector index (0-based). - * - * @nr has the following meanings depending on the interrupt mode: - * MSI-X: The index in the MSI-X vector table - * MSI: The index of the enabled MSI vectors - * INTx: Must be 0 - * - * Return: A cpumask pointer or NULL if @nr is out of range - */ -const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr) -{ - int idx, irq = pci_irq_vector(dev, nr); - struct msi_desc *desc; - - if (WARN_ON_ONCE(irq <= 0)) - return NULL; - - desc = irq_get_msi_desc(irq); - /* Non-MSI does not have the information handy */ - if (!desc) - return cpu_possible_mask; - - /* MSI[X] interrupts can be allocated without affinity descriptor */ - if (!desc->affinity) - return NULL; - - /* - * MSI has a mask array in the descriptor. - * MSI-X has a single mask. - */ - idx = dev->msi_enabled ? nr : 0; - return &desc->affinity[idx].mask; -} -EXPORT_SYMBOL(pci_irq_get_affinity); +/* Misc. infrastructure */ struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc) { @@ -1134,15 +918,3 @@ void pci_no_msi(void) { pci_msi_enable = 0; } - -/** - * pci_msi_enabled - is MSI enabled? - * - * Returns true if MSI has not been disabled by the command-line option - * pci=nomsi. - **/ -int pci_msi_enabled(void) -{ - return pci_msi_enable; -} -EXPORT_SYMBOL(pci_msi_enabled); |