From a430739009384ba2c4804f3a427334ff395433cd Mon Sep 17 00:00:00 2001 From: Frederic Barrat Date: Wed, 15 Jun 2016 16:42:16 +0200 Subject: cxl: Make vPHB device node match adapter's On bare-metal, when a device is attached to the cxl card, lsvpd shows a location code such as (with cxlflash): # lsvpd -l sg22 ... *YL U78CB.001.WZS0073-P1-C33-B0-T0-L0 which makes it hard to easily identify the cxl adapter owning the flash device, since in this example C33 refers to a P8 processor. lsvpd looks in the parent devices until it finds a location code, so the device node for the vPHB ends up being used. By reusing the device node of the adapter for the vPHB, lsvpd shows: # lsvpd -l sg16 ... *YL U78C9.001.WZS09XA-P1-C7-B1-T0-L3 where C7 is the PCI slot of the cxl adapter. On powerVM, the vPHB was already using the adapter device node, so there's no change there. Tested by cxlflash on bare-metal and powerVM. Signed-off-by: Frederic Barrat Reviewed-by: Matthew R. Ochs Acked-by: Ian Munsie Signed-off-by: Michael Ellerman --- drivers/misc/cxl/vphb.c | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) (limited to 'drivers/misc/cxl/vphb.c') diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c index cdc7723b845d..012b6aa9fb3e 100644 --- a/drivers/misc/cxl/vphb.c +++ b/drivers/misc/cxl/vphb.c @@ -208,20 +208,19 @@ static struct pci_controller_ops cxl_pci_controller_ops = int cxl_pci_vphb_add(struct cxl_afu *afu) { - struct pci_dev *phys_dev; - struct pci_controller *phb, *phys_phb; + struct pci_controller *phb; struct device_node *vphb_dn; struct device *parent; - if (cpu_has_feature(CPU_FTR_HVMODE)) { - phys_dev = to_pci_dev(afu->adapter->dev.parent); - phys_phb = pci_bus_to_host(phys_dev->bus); - vphb_dn = phys_phb->dn; - parent = &phys_dev->dev; - } else { - vphb_dn = afu->adapter->dev.parent->of_node; - parent = afu->adapter->dev.parent; - } + /* The parent device is the adapter. Reuse the device node of + * the adapter. + * We don't seem to care what device node is used for the vPHB, + * but tools such as lsvpd walk up the device parents looking + * for a valid location code, so we might as well show devices + * attached to the adapter as being located on that adapter. + */ + parent = afu->adapter->dev.parent; + vphb_dn = parent->of_node; /* Alloc and setup PHB data structure */ phb = pcibios_alloc_controller(vphb_dn); -- cgit v1.2.3 From a19bd79e31769626d288cc016e21a31b6f47bf6f Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:04 +1000 Subject: cxl: Allow a default context to be associated with an external pci_dev The cxl kernel API has a concept of a default context associated with each PCI device under the virtual PHB. The Mellanox CX4 will also use the cxl kernel API, but it does not use a virtual PHB - rather, the AFU appears as a physical function as a peer to the networking functions. In order to allow the kernel API to work with those networking functions, we will need to associate a default context with them as well. To this end, refactor the corresponding code to do this in vphb.c and export it so that it can be called from the PHB code. Signed-off-by: Ian Munsie Reviewed-by: Frederic Barrat Reviewed-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- drivers/misc/cxl/Makefile | 2 +- drivers/misc/cxl/base.c | 35 +++++++++++++++++++++++++++++++++++ drivers/misc/cxl/cxl.h | 6 ++++++ drivers/misc/cxl/main.c | 2 ++ drivers/misc/cxl/phb.c | 44 ++++++++++++++++++++++++++++++++++++++++++++ drivers/misc/cxl/vphb.c | 30 +++--------------------------- include/misc/cxl-base.h | 6 ++++++ 7 files changed, 97 insertions(+), 28 deletions(-) create mode 100644 drivers/misc/cxl/phb.c (limited to 'drivers/misc/cxl/vphb.c') diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile index 8a55c1aa11aa..56e9a4732ef0 100644 --- a/drivers/misc/cxl/Makefile +++ b/drivers/misc/cxl/Makefile @@ -3,7 +3,7 @@ ccflags-$(CONFIG_PPC_WERROR) += -Werror cxl-y += main.o file.o irq.o fault.o native.o cxl-y += context.o sysfs.o debugfs.o pci.o trace.o -cxl-y += vphb.o api.o +cxl-y += vphb.o phb.o api.o cxl-$(CONFIG_PPC_PSERIES) += flash.o guest.o of.o hcalls.o obj-$(CONFIG_CXL) += cxl.o obj-$(CONFIG_CXL_BASE) += base.o diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index 7557835cdfcd..e1e80cb99ad9 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -106,6 +106,41 @@ int cxl_update_properties(struct device_node *dn, } EXPORT_SYMBOL_GPL(cxl_update_properties); +/* + * API calls into the driver that may be called from the PHB code and must be + * built in. + */ +bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) +{ + bool ret; + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return false; + + ret = calls->cxl_pci_associate_default_context(dev, afu); + + cxl_calls_put(calls); + + return ret; +} +EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context); + +void cxl_pci_disable_device(struct pci_dev *dev) +{ + struct cxl_calls *calls; + + calls = cxl_calls_get(); + if (!calls) + return; + + calls->cxl_pci_disable_device(dev); + + cxl_calls_put(calls); +} +EXPORT_SYMBOL_GPL(cxl_pci_disable_device); + static int __init cxl_base_init(void) { struct device_node *np; diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h index d4aae6f855a8..b81f476a2b9f 100644 --- a/drivers/misc/cxl/cxl.h +++ b/drivers/misc/cxl/cxl.h @@ -719,9 +719,15 @@ static inline u64 cxl_p2n_read(struct cxl_afu *afu, cxl_p2n_reg_t reg) ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf, loff_t off, size_t count); +/* Internal functions wrapped in cxl_base to allow PHB to call them */ +bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu); +void _cxl_pci_disable_device(struct pci_dev *dev); struct cxl_calls { void (*cxl_slbia)(struct mm_struct *mm); + bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu); + void (*cxl_pci_disable_device)(struct pci_dev *dev); + struct module *owner; }; int register_cxl_calls(struct cxl_calls *calls); diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c index ae68c3201156..4e5474b00b28 100644 --- a/drivers/misc/cxl/main.c +++ b/drivers/misc/cxl/main.c @@ -110,6 +110,8 @@ static inline void cxl_slbia_core(struct mm_struct *mm) static struct cxl_calls cxl_calls = { .cxl_slbia = cxl_slbia_core, + .cxl_pci_associate_default_context = _cxl_pci_associate_default_context, + .cxl_pci_disable_device = _cxl_pci_disable_device, .owner = THIS_MODULE, }; diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c new file mode 100644 index 000000000000..0935d44c1770 --- /dev/null +++ b/drivers/misc/cxl/phb.c @@ -0,0 +1,44 @@ +/* + * Copyright 2014-2016 IBM Corp. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include +#include "cxl.h" + +bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) +{ + struct cxl_context *ctx; + + /* + * Allocate a context to do cxl things to. This is used for interrupts + * in the peer model using a real phb, and if we eventually do DMA ops + * in the virtual phb, we'll need a default context to attach them to. + */ + ctx = cxl_dev_context_init(dev); + if (!ctx) + return false; + dev->dev.archdata.cxl_ctx = ctx; + + return (cxl_ops->afu_check_and_enable(afu) == 0); +} +/* exported via cxl_base */ + +void _cxl_pci_disable_device(struct pci_dev *dev) +{ + struct cxl_context *ctx = cxl_get_context(dev); + + if (ctx) { + if (ctx->status == STARTED) { + dev_err(&dev->dev, "Default context started\n"); + return; + } + dev->dev.archdata.cxl_ctx = NULL; + cxl_release_context(ctx); + } +} +/* exported via cxl_base */ diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c index 012b6aa9fb3e..c8a759f4ccfa 100644 --- a/drivers/misc/cxl/vphb.c +++ b/drivers/misc/cxl/vphb.c @@ -44,7 +44,6 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev) { struct pci_controller *phb; struct cxl_afu *afu; - struct cxl_context *ctx; phb = pci_bus_to_host(dev->bus); afu = (struct cxl_afu *)phb->private_data; @@ -57,30 +56,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev) set_dma_ops(&dev->dev, &dma_direct_ops); set_dma_offset(&dev->dev, PAGE_OFFSET); - /* - * Allocate a context to do cxl things too. If we eventually do real - * DMA ops, we'll need a default context to attach them to - */ - ctx = cxl_dev_context_init(dev); - if (!ctx) - return false; - dev->dev.archdata.cxl_ctx = ctx; - - return (cxl_ops->afu_check_and_enable(afu) == 0); -} - -static void cxl_pci_disable_device(struct pci_dev *dev) -{ - struct cxl_context *ctx = cxl_get_context(dev); - - if (ctx) { - if (ctx->status == STARTED) { - dev_err(&dev->dev, "Default context started\n"); - return; - } - dev->dev.archdata.cxl_ctx = NULL; - cxl_release_context(ctx); - } + return _cxl_pci_associate_default_context(dev, afu); } static resource_size_t cxl_pci_window_alignment(struct pci_bus *bus, @@ -197,8 +173,8 @@ static struct pci_controller_ops cxl_pci_controller_ops = { .probe_mode = cxl_pci_probe_mode, .enable_device_hook = cxl_pci_enable_device_hook, - .disable_device = cxl_pci_disable_device, - .release_device = cxl_pci_disable_device, + .disable_device = _cxl_pci_disable_device, + .release_device = _cxl_pci_disable_device, .window_alignment = cxl_pci_window_alignment, .reset_secondary_bus = cxl_pci_reset_secondary_bus, .setup_msi_irqs = cxl_setup_msi_irqs, diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h index f53808fa638a..bb7e629ae492 100644 --- a/include/misc/cxl-base.h +++ b/include/misc/cxl-base.h @@ -10,6 +10,8 @@ #ifndef _MISC_CXL_BASE_H #define _MISC_CXL_BASE_H +#include + #ifdef CONFIG_CXL_BASE #define CXL_IRQ_RANGES 4 @@ -39,6 +41,8 @@ static inline void cxl_ctx_put(void) struct cxl_afu *cxl_afu_get(struct cxl_afu *afu); void cxl_afu_put(struct cxl_afu *afu); void cxl_slbia(struct mm_struct *mm); +bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu); +void cxl_pci_disable_device(struct pci_dev *dev); #else /* CONFIG_CXL_BASE */ @@ -46,6 +50,8 @@ static inline bool cxl_ctx_in_use(void) { return false; } static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) { return NULL; } static inline void cxl_afu_put(struct cxl_afu *afu) {} static inline void cxl_slbia(struct mm_struct *mm) {} +static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; } +static inline void cxl_pci_disable_device(struct pci_dev *dev) {} #endif /* CONFIG_CXL_BASE */ -- cgit v1.2.3 From e4f5fc001a6cb82bef910372457ca7754defa84d Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:05 +1000 Subject: cxl: Do not create vPHB if there are no AFU configuration records The vPHB model of the cxl kernel API is a hierarchy where the AFU is represented by the vPHB, and it's AFU configuration records are exposed as functions under that vPHB. If there are no AFU configuration records we will create a vPHB with nothing under it, which is a waste of resources and will opt us into EEH handling despite not having anything special to handle. This also does not make sense for cards using the peer model of the cxl kernel API, where the other functions of the device are exposed via additional peer physical functions rather than AFU configuration records. This model will also not work with the existing EEH handling in the cxl driver, as that is designed around the vPHB model. Skip creating the vPHB for AFUs without any AFU configuration records, and opt out of EEH handling for them. Signed-off-by: Ian Munsie Reviewed-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- drivers/misc/cxl/pci.c | 3 +++ drivers/misc/cxl/vphb.c | 11 +++++++++++ 2 files changed, 14 insertions(+) (limited to 'drivers/misc/cxl/vphb.c') diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index deef9c75e73a..dd7ff221288b 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -1572,6 +1572,9 @@ static pci_ers_result_t cxl_pci_error_detected(struct pci_dev *pdev, */ for (i = 0; i < adapter->slices; i++) { afu = adapter->afu[i]; + /* Only participate in EEH if we are on a virtual PHB */ + if (afu->phb == NULL) + return PCI_ERS_RESULT_NONE; cxl_vphb_error_detected(afu, state); } return PCI_ERS_RESULT_DISCONNECT; diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c index c8a759f4ccfa..8865e8d9b3c5 100644 --- a/drivers/misc/cxl/vphb.c +++ b/drivers/misc/cxl/vphb.c @@ -188,6 +188,17 @@ int cxl_pci_vphb_add(struct cxl_afu *afu) struct device_node *vphb_dn; struct device *parent; + /* + * If there are no AFU configuration records we won't have anything to + * expose under the vPHB, so skip creating one, returning success since + * this is still a valid case. This will also opt us out of EEH + * handling since we won't have anything special to do if there are no + * kernel drivers attached to the vPHB, and EEH handling is not yet + * supported in the peer model. + */ + if (!afu->crs_num) + return 0; + /* The parent device is the adapter. Reuse the device node of * the adapter. * We don't seem to care what device node is used for the vPHB, -- cgit v1.2.3 From 317f5ef1b363417b6f1e93b90dfd2ffd6be6e867 Mon Sep 17 00:00:00 2001 From: Ian Munsie Date: Thu, 14 Jul 2016 07:17:07 +1000 Subject: cxl: Add support for using the kernel API with a real PHB This hooks up support for using the kernel API with a real PHB. After the AFU initialisation has completed it calls into the PHB code to pass it the AFU that will be used by other peer physical functions on the adapter. The cxl_pci_to_afu API is extended to work with peer PCI devices, retrieving the peer AFU from the PHB. This API may also now return an error if it is called on a PCI device that is not associated with either a cxl vPHB or a peer PCI device to an AFU, and this error is propagated down. Signed-off-by: Ian Munsie Reviewed-by: Andrew Donnellan Signed-off-by: Michael Ellerman --- drivers/misc/cxl/api.c | 5 +++++ drivers/misc/cxl/pci.c | 3 +++ drivers/misc/cxl/vphb.c | 16 ++++++++++++++-- 3 files changed, 22 insertions(+), 2 deletions(-) (limited to 'drivers/misc/cxl/vphb.c') diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c index 7707055d33ab..6a030bf71655 100644 --- a/drivers/misc/cxl/api.c +++ b/drivers/misc/cxl/api.c @@ -13,6 +13,7 @@ #include #include #include +#include #include "cxl.h" @@ -24,6 +25,8 @@ struct cxl_context *cxl_dev_context_init(struct pci_dev *dev) int rc; afu = cxl_pci_to_afu(dev); + if (IS_ERR(afu)) + return ERR_CAST(afu); ctx = cxl_context_alloc(); if (IS_ERR(ctx)) { @@ -438,6 +441,8 @@ EXPORT_SYMBOL_GPL(cxl_perst_reloads_same_image); ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count) { struct cxl_afu *afu = cxl_pci_to_afu(dev); + if (IS_ERR(afu)) + return -ENODEV; return cxl_ops->read_adapter_vpd(afu->adapter, buf, count); } diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index dd7ff221288b..cb5d172fec1b 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -1502,6 +1502,9 @@ static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id) dev_err(&dev->dev, "AFU %i failed to start: %i\n", slice, rc); } + if (pnv_pci_on_cxl_phb(dev) && adapter->slices >= 1) + pnv_cxl_phb_set_peer_afu(dev, adapter->afu[0]); + return 0; } diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c index 8865e8d9b3c5..dee8def1c193 100644 --- a/drivers/misc/cxl/vphb.c +++ b/drivers/misc/cxl/vphb.c @@ -9,6 +9,7 @@ #include #include +#include #include "cxl.h" static int cxl_dma_set_mask(struct pci_dev *pdev, u64 dma_mask) @@ -258,13 +259,18 @@ void cxl_pci_vphb_remove(struct cxl_afu *afu) pcibios_free_controller(phb); } +static bool _cxl_pci_is_vphb_device(struct pci_controller *phb) +{ + return (phb->ops == &cxl_pcie_pci_ops); +} + bool cxl_pci_is_vphb_device(struct pci_dev *dev) { struct pci_controller *phb; phb = pci_bus_to_host(dev->bus); - return (phb->ops == &cxl_pcie_pci_ops); + return _cxl_pci_is_vphb_device(phb); } struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev) @@ -273,7 +279,13 @@ struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev) phb = pci_bus_to_host(dev->bus); - return (struct cxl_afu *)phb->private_data; + if (_cxl_pci_is_vphb_device(phb)) + return (struct cxl_afu *)phb->private_data; + + if (pnv_pci_on_cxl_phb(dev)) + return pnv_cxl_phb_to_afu(phb); + + return ERR_PTR(-ENODEV); } EXPORT_SYMBOL_GPL(cxl_pci_to_afu); -- cgit v1.2.3