From 9f38abefd37af8726d59706b9b84530630b6b620 Mon Sep 17 00:00:00 2001 From: Mauro Carvalho Chehab Date: Fri, 23 Oct 2020 18:33:17 +0200 Subject: uio: fix some kernel-doc markups The definitions for (devm_)uio_register_device should be at the header file, as the macros are there. The ones inside uio.c refer, instead, to __(devm_)uio_register_device. Update them and add new kernel-doc markups for the macros. Signed-off-by: Mauro Carvalho Chehab Link: https://lore.kernel.org/r/82ab7b68d271aeda7396e369ff8a629491b9d628.1603469755.git.mchehab+huawei@kernel.org Signed-off-by: Greg Kroah-Hartman --- include/linux/uio_driver.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'include') diff --git a/include/linux/uio_driver.h b/include/linux/uio_driver.h index 54bf6b118401..47c5962b876b 100644 --- a/include/linux/uio_driver.h +++ b/include/linux/uio_driver.h @@ -117,6 +117,14 @@ extern int __must_check struct uio_info *info); /* use a define to avoid include chaining to get THIS_MODULE */ + +/** + * uio_register_device - register a new userspace IO device + * @parent: parent device + * @info: UIO device capabilities + * + * returns zero on success or a negative error code. + */ #define uio_register_device(parent, info) \ __uio_register_device(THIS_MODULE, parent, info) @@ -129,6 +137,14 @@ extern int __must_check struct uio_info *info); /* use a define to avoid include chaining to get THIS_MODULE */ + +/** + * devm_uio_register_device - Resource managed uio_register_device() + * @parent: parent device + * @info: UIO device capabilities + * + * returns zero on success or a negative error code. + */ #define devm_uio_register_device(parent, info) \ __devm_uio_register_device(THIS_MODULE, parent, info) -- cgit v1.2.3 From 78e1d22687ff1fd320abac12e8a607ea79782c48 Mon Sep 17 00:00:00 2001 From: Bhaumik Bhatt Date: Fri, 6 Nov 2020 09:44:47 -0800 Subject: bus: mhi: core: Expose mhi_get_exec_env() API for controllers The mhi_get_exec_env() APIs can be used by the controller drivers to query the execution environment of the MHI device. Expose it so it can be used in some scenarios to determine behavior of controllers. Signed-off-by: Bhaumik Bhatt Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/internal.h | 1 - drivers/bus/mhi/core/main.c | 1 + include/linux/mhi.h | 6 ++++++ 3 files changed, 7 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/bus/mhi/core/internal.h b/drivers/bus/mhi/core/internal.h index 78e4e84d6743..d8af8a702493 100644 --- a/drivers/bus/mhi/core/internal.h +++ b/drivers/bus/mhi/core/internal.h @@ -609,7 +609,6 @@ enum mhi_pm_state __must_check mhi_tryset_pm_state( struct mhi_controller *mhi_cntrl, enum mhi_pm_state state); const char *to_mhi_pm_state_str(enum mhi_pm_state state); -enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl); int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl, enum dev_st_transition state); void mhi_pm_st_worker(struct work_struct *work); diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 6ecaacaa8b54..f953e2a6d58a 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -123,6 +123,7 @@ enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl) return (ret) ? MHI_EE_MAX : exec; } +EXPORT_SYMBOL_GPL(mhi_get_exec_env); enum mhi_state mhi_get_mhi_state(struct mhi_controller *mhi_cntrl) { diff --git a/include/linux/mhi.h b/include/linux/mhi.h index d4841e5a5f45..9225d5551d69 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -658,6 +658,12 @@ int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic); */ int mhi_force_rddm_mode(struct mhi_controller *mhi_cntrl); +/** + * mhi_get_exec_env - Get BHI execution environment of the device + * @mhi_cntrl: MHI controller + */ +enum mhi_ee_type mhi_get_exec_env(struct mhi_controller *mhi_cntrl); + /** * mhi_get_mhi_state - Get MHI state of the device * @mhi_cntrl: MHI controller -- cgit v1.2.3 From 9e1660e5c396ee081907386d0b95b8e0804a6c86 Mon Sep 17 00:00:00 2001 From: Bhaumik Bhatt Date: Fri, 6 Nov 2020 09:44:49 -0800 Subject: bus: mhi: core: Rename RDDM download function to use proper words mhi_download_rddm_img() uses a shorter version of the word image. Expand it and rename the function to mhi_download_rddm_image(). Signed-off-by: Bhaumik Bhatt Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/boot.c | 4 ++-- include/linux/mhi.h | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'include') diff --git a/drivers/bus/mhi/core/boot.c b/drivers/bus/mhi/core/boot.c index 6b6fd9668c3b..16244cc8fbe7 100644 --- a/drivers/bus/mhi/core/boot.c +++ b/drivers/bus/mhi/core/boot.c @@ -147,7 +147,7 @@ static int __mhi_download_rddm_in_panic(struct mhi_controller *mhi_cntrl) } /* Download RDDM image from device */ -int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic) +int mhi_download_rddm_image(struct mhi_controller *mhi_cntrl, bool in_panic) { void __iomem *base = mhi_cntrl->bhie; struct device *dev = &mhi_cntrl->mhi_dev->dev; @@ -169,7 +169,7 @@ int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic) return (rx_status == BHIE_RXVECSTATUS_STATUS_XFER_COMPL) ? 0 : -EIO; } -EXPORT_SYMBOL_GPL(mhi_download_rddm_img); +EXPORT_SYMBOL_GPL(mhi_download_rddm_image); static int mhi_fw_load_amss(struct mhi_controller *mhi_cntrl, const struct mhi_buf *mhi_buf) diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 9225d5551d69..52b3c60bf9bb 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -645,12 +645,12 @@ int mhi_pm_suspend(struct mhi_controller *mhi_cntrl); int mhi_pm_resume(struct mhi_controller *mhi_cntrl); /** - * mhi_download_rddm_img - Download ramdump image from device for - * debugging purpose. + * mhi_download_rddm_image - Download ramdump image from device for + * debugging purpose. * @mhi_cntrl: MHI controller * @in_panic: Download rddm image during kernel panic */ -int mhi_download_rddm_img(struct mhi_controller *mhi_cntrl, bool in_panic); +int mhi_download_rddm_image(struct mhi_controller *mhi_cntrl, bool in_panic); /** * mhi_force_rddm_mode - Force device into rddm mode -- cgit v1.2.3 From 8f70397876872789b2a5deba804eb6216fb5deb7 Mon Sep 17 00:00:00 2001 From: Bhaumik Bhatt Date: Mon, 9 Nov 2020 12:47:21 -0800 Subject: bus: mhi: core: Move to using high priority workqueue MHI work is currently scheduled on the global/system workqueue and can encounter delays on a stressed system. To avoid those unforeseen delays which can hamper bootup or shutdown times, use a dedicated high priority workqueue instead of the global/system workqueue. Signed-off-by: Bhaumik Bhatt Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/init.c | 9 +++++++++ drivers/bus/mhi/core/pm.c | 2 +- include/linux/mhi.h | 2 ++ 3 files changed, 12 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index 8cefa359fccd..877e40c86801 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -880,6 +880,13 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, INIT_WORK(&mhi_cntrl->st_worker, mhi_pm_st_worker); init_waitqueue_head(&mhi_cntrl->state_event); + mhi_cntrl->hiprio_wq = alloc_ordered_workqueue + ("mhi_hiprio_wq", WQ_MEM_RECLAIM | WQ_HIGHPRI); + if (!mhi_cntrl->hiprio_wq) { + dev_err(mhi_cntrl->cntrl_dev, "Failed to allocate workqueue\n"); + goto error_alloc_cmd; + } + mhi_cmd = mhi_cntrl->mhi_cmd; for (i = 0; i < NR_OF_CMD_RINGS; i++, mhi_cmd++) spin_lock_init(&mhi_cmd->lock); @@ -969,6 +976,7 @@ error_alloc_dev: error_alloc_cmd: vfree(mhi_cntrl->mhi_chan); kfree(mhi_cntrl->mhi_event); + destroy_workqueue(mhi_cntrl->hiprio_wq); return ret; } @@ -982,6 +990,7 @@ void mhi_unregister_controller(struct mhi_controller *mhi_cntrl) mhi_destroy_debugfs(mhi_cntrl); + destroy_workqueue(mhi_cntrl->hiprio_wq); kfree(mhi_cntrl->mhi_cmd); kfree(mhi_cntrl->mhi_event); diff --git a/drivers/bus/mhi/core/pm.c b/drivers/bus/mhi/core/pm.c index 3de7b1639ec6..805b6fa748f0 100644 --- a/drivers/bus/mhi/core/pm.c +++ b/drivers/bus/mhi/core/pm.c @@ -597,7 +597,7 @@ int mhi_queue_state_transition(struct mhi_controller *mhi_cntrl, list_add_tail(&item->node, &mhi_cntrl->transition_list); spin_unlock_irqrestore(&mhi_cntrl->transition_lock, flags); - schedule_work(&mhi_cntrl->st_worker); + queue_work(mhi_cntrl->hiprio_wq, &mhi_cntrl->st_worker); return 0; } diff --git a/include/linux/mhi.h b/include/linux/mhi.h index 52b3c60bf9bb..1ed5f2aa224b 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -337,6 +337,7 @@ struct mhi_controller_config { * @wlock: Lock for protecting device wakeup * @mhi_link_info: Device bandwidth info * @st_worker: State transition worker + * @hiprio_wq: High priority workqueue for MHI work such as state transitions * @state_event: State change event * @status_cb: CB function to notify power states of the device (required) * @wake_get: CB function to assert device wake (optional) @@ -419,6 +420,7 @@ struct mhi_controller { spinlock_t wlock; struct mhi_link_info mhi_link_info; struct work_struct st_worker; + struct workqueue_struct *hiprio_wq; wait_queue_head_t state_event; void (*status_cb)(struct mhi_controller *mhi_cntrl, -- cgit v1.2.3 From 2fb94784952e4b290c392b74c2c67b4afa672523 Mon Sep 17 00:00:00 2001 From: Pierre-Louis Bossart Date: Tue, 24 Nov 2020 09:33:16 +0800 Subject: soundwire: registers: add definitions for clearable interrupt fields DP0 has reserved fields and the read-only SDCA_CASCADE bit. We should not try to write values in these fields, so add a formal definition for clearable interrupts to be used in DP0 interrupt handling. DPN also has reserved fields so add definitions for clearable interrupts as well. Signed-off-by: Pierre-Louis Bossart Reviewed-by: Guennadi Liakhovetski Signed-off-by: Bard Liao Link: https://lore.kernel.org/r/20201124013318.8963-4-yung-chuan.liao@linux.intel.com Signed-off-by: Vinod Koul --- include/linux/soundwire/sdw_registers.h | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'include') diff --git a/include/linux/soundwire/sdw_registers.h b/include/linux/soundwire/sdw_registers.h index f420e8059779..0cb1a22685b8 100644 --- a/include/linux/soundwire/sdw_registers.h +++ b/include/linux/soundwire/sdw_registers.h @@ -41,6 +41,12 @@ #define SDW_DP0_INT_IMPDEF1 BIT(5) #define SDW_DP0_INT_IMPDEF2 BIT(6) #define SDW_DP0_INT_IMPDEF3 BIT(7) +#define SDW_DP0_INTERRUPTS (SDW_DP0_INT_TEST_FAIL | \ + SDW_DP0_INT_PORT_READY | \ + SDW_DP0_INT_BRA_FAILURE | \ + SDW_DP0_INT_IMPDEF1 | \ + SDW_DP0_INT_IMPDEF2 | \ + SDW_DP0_INT_IMPDEF3) #define SDW_DP0_PORTCTRL_DATAMODE GENMASK(3, 2) #define SDW_DP0_PORTCTRL_NXTINVBANK BIT(4) @@ -241,6 +247,11 @@ #define SDW_DPN_INT_IMPDEF1 BIT(5) #define SDW_DPN_INT_IMPDEF2 BIT(6) #define SDW_DPN_INT_IMPDEF3 BIT(7) +#define SDW_DPN_INTERRUPTS (SDW_DPN_INT_TEST_FAIL | \ + SDW_DPN_INT_PORT_READY | \ + SDW_DPN_INT_IMPDEF1 | \ + SDW_DPN_INT_IMPDEF2 | \ + SDW_DPN_INT_IMPDEF3) #define SDW_DPN_PORTCTRL_FLOWMODE GENMASK(1, 0) #define SDW_DPN_PORTCTRL_DATAMODE GENMASK(3, 2) -- cgit v1.2.3 From fd3bb8f54a88107570334c156efb0c724a261003 Mon Sep 17 00:00:00 2001 From: Evan Green Date: Fri, 27 Nov 2020 10:28:34 +0000 Subject: nvmem: core: Add support for keepout regions Introduce support into the nvmem core for arrays of register ranges that should not result in actual device access. For these regions a constant byte (repeated) is returned instead on read, and writes are quietly ignored and returned as successful. This is useful for instance if certain efuse regions are protected from access by Linux because they contain secret info to another part of the system (like an integrated modem). Signed-off-by: Evan Green Signed-off-by: Srinivas Kandagatla Link: https://lore.kernel.org/r/20201127102837.19366-3-srinivas.kandagatla@linaro.org Signed-off-by: Greg Kroah-Hartman --- drivers/nvmem/core.c | 153 +++++++++++++++++++++++++++++++++++++++-- include/linux/nvmem-provider.h | 17 +++++ 2 files changed, 166 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/drivers/nvmem/core.c b/drivers/nvmem/core.c index a09ff8409f60..177f5bf27c6d 100644 --- a/drivers/nvmem/core.c +++ b/drivers/nvmem/core.c @@ -34,6 +34,8 @@ struct nvmem_device { struct bin_attribute eeprom; struct device *base_dev; struct list_head cells; + const struct nvmem_keepout *keepout; + unsigned int nkeepout; nvmem_reg_read_t reg_read; nvmem_reg_write_t reg_write; struct gpio_desc *wp_gpio; @@ -66,8 +68,8 @@ static LIST_HEAD(nvmem_lookup_list); static BLOCKING_NOTIFIER_HEAD(nvmem_notifier); -static int nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset, - void *val, size_t bytes) +static int __nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset, + void *val, size_t bytes) { if (nvmem->reg_read) return nvmem->reg_read(nvmem->priv, offset, val, bytes); @@ -75,8 +77,8 @@ static int nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset, return -EINVAL; } -static int nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset, - void *val, size_t bytes) +static int __nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset, + void *val, size_t bytes) { int ret; @@ -90,6 +92,88 @@ static int nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset, return -EINVAL; } +static int nvmem_access_with_keepouts(struct nvmem_device *nvmem, + unsigned int offset, void *val, + size_t bytes, int write) +{ + + unsigned int end = offset + bytes; + unsigned int kend, ksize; + const struct nvmem_keepout *keepout = nvmem->keepout; + const struct nvmem_keepout *keepoutend = keepout + nvmem->nkeepout; + int rc; + + /* + * Skip all keepouts before the range being accessed. + * Keepouts are sorted. + */ + while ((keepout < keepoutend) && (keepout->end <= offset)) + keepout++; + + while ((offset < end) && (keepout < keepoutend)) { + /* Access the valid portion before the keepout. */ + if (offset < keepout->start) { + kend = min(end, keepout->start); + ksize = kend - offset; + if (write) + rc = __nvmem_reg_write(nvmem, offset, val, ksize); + else + rc = __nvmem_reg_read(nvmem, offset, val, ksize); + + if (rc) + return rc; + + offset += ksize; + val += ksize; + } + + /* + * Now we're aligned to the start of this keepout zone. Go + * through it. + */ + kend = min(end, keepout->end); + ksize = kend - offset; + if (!write) + memset(val, keepout->value, ksize); + + val += ksize; + offset += ksize; + keepout++; + } + + /* + * If we ran out of keepouts but there's still stuff to do, send it + * down directly + */ + if (offset < end) { + ksize = end - offset; + if (write) + return __nvmem_reg_write(nvmem, offset, val, ksize); + else + return __nvmem_reg_read(nvmem, offset, val, ksize); + } + + return 0; +} + +static int nvmem_reg_read(struct nvmem_device *nvmem, unsigned int offset, + void *val, size_t bytes) +{ + if (!nvmem->nkeepout) + return __nvmem_reg_read(nvmem, offset, val, bytes); + + return nvmem_access_with_keepouts(nvmem, offset, val, bytes, false); +} + +static int nvmem_reg_write(struct nvmem_device *nvmem, unsigned int offset, + void *val, size_t bytes) +{ + if (!nvmem->nkeepout) + return __nvmem_reg_write(nvmem, offset, val, bytes); + + return nvmem_access_with_keepouts(nvmem, offset, val, bytes, true); +} + #ifdef CONFIG_NVMEM_SYSFS static const char * const nvmem_type_str[] = { [NVMEM_TYPE_UNKNOWN] = "Unknown", @@ -533,6 +617,59 @@ nvmem_find_cell_by_name(struct nvmem_device *nvmem, const char *cell_id) return cell; } +static int nvmem_validate_keepouts(struct nvmem_device *nvmem) +{ + unsigned int cur = 0; + const struct nvmem_keepout *keepout = nvmem->keepout; + const struct nvmem_keepout *keepoutend = keepout + nvmem->nkeepout; + + while (keepout < keepoutend) { + /* Ensure keepouts are sorted and don't overlap. */ + if (keepout->start < cur) { + dev_err(&nvmem->dev, + "Keepout regions aren't sorted or overlap.\n"); + + return -ERANGE; + } + + if (keepout->end < keepout->start) { + dev_err(&nvmem->dev, + "Invalid keepout region.\n"); + + return -EINVAL; + } + + /* + * Validate keepouts (and holes between) don't violate + * word_size constraints. + */ + if ((keepout->end - keepout->start < nvmem->word_size) || + ((keepout->start != cur) && + (keepout->start - cur < nvmem->word_size))) { + + dev_err(&nvmem->dev, + "Keepout regions violate word_size constraints.\n"); + + return -ERANGE; + } + + /* Validate keepouts don't violate stride (alignment). */ + if (!IS_ALIGNED(keepout->start, nvmem->stride) || + !IS_ALIGNED(keepout->end, nvmem->stride)) { + + dev_err(&nvmem->dev, + "Keepout regions violate stride.\n"); + + return -EINVAL; + } + + cur = keepout->end; + keepout++; + } + + return 0; +} + static int nvmem_add_cells_from_of(struct nvmem_device *nvmem) { struct device_node *parent, *child; @@ -647,6 +784,8 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) nvmem->type = config->type; nvmem->reg_read = config->reg_read; nvmem->reg_write = config->reg_write; + nvmem->keepout = config->keepout; + nvmem->nkeepout = config->nkeepout; if (!config->no_of_node) nvmem->dev.of_node = config->dev->of_node; @@ -671,6 +810,12 @@ struct nvmem_device *nvmem_register(const struct nvmem_config *config) nvmem->dev.groups = nvmem_dev_groups; #endif + if (nvmem->nkeepout) { + rval = nvmem_validate_keepouts(nvmem); + if (rval) + goto err_put_device; + } + dev_dbg(&nvmem->dev, "Registering nvmem device %s\n", config->name); rval = device_register(&nvmem->dev); diff --git a/include/linux/nvmem-provider.h b/include/linux/nvmem-provider.h index 06409a6c40bc..e162b757b6d5 100644 --- a/include/linux/nvmem-provider.h +++ b/include/linux/nvmem-provider.h @@ -30,6 +30,19 @@ enum nvmem_type { #define NVMEM_DEVID_NONE (-1) #define NVMEM_DEVID_AUTO (-2) +/** + * struct nvmem_keepout - NVMEM register keepout range. + * + * @start: The first byte offset to avoid. + * @end: One beyond the last byte offset to avoid. + * @value: The byte to fill reads with for this region. + */ +struct nvmem_keepout { + unsigned int start; + unsigned int end; + unsigned char value; +}; + /** * struct nvmem_config - NVMEM device configuration * @@ -39,6 +52,8 @@ enum nvmem_type { * @owner: Pointer to exporter module. Used for refcounting. * @cells: Optional array of pre-defined NVMEM cells. * @ncells: Number of elements in cells. + * @keepout: Optional array of keepout ranges (sorted ascending by start). + * @nkeepout: Number of elements in the keepout array. * @type: Type of the nvmem storage * @read_only: Device is read-only. * @root_only: Device is accessibly to root only. @@ -66,6 +81,8 @@ struct nvmem_config { struct gpio_desc *wp_gpio; const struct nvmem_cell_info *cells; int ncells; + const struct nvmem_keepout *keepout; + unsigned int nkeepout; enum nvmem_type type; bool read_only; bool root_only; -- cgit v1.2.3 From 206e7383b34316cf56cdde96eab9d97e9a1dbd70 Mon Sep 17 00:00:00 2001 From: Loic Poulain Date: Thu, 26 Nov 2020 11:20:35 +0100 Subject: bus: mhi: core: Indexed MHI controller name Today the MHI controller name is simply cloned from the underlying bus device (its parent), that gives the following device structure for e.g. a MHI/PCI controller: devices/pci0000:00/0000:00:01.2/0000:02:00.0/0000:02:00.0 devices/pci0000:00/0000:00:01.2/0000:02:00.0/0000:02:00.0/0000:02:00.0_IPCR ... That's quite misleading/confusing and can cause device registering issues because of duplicate dev name (e.g. if a PCI device register two different MHI instances). This patch changes MHI core to create indexed mhi controller names (mhi0, mhi1...) in the same way as other busses (i2c0, usb0...). The previous example becomes: devices/pci0000:00/0000:00:01.2/0000:02:00.0/mhi0 devices/pci0000:00/0000:00:01.2/0000:02:00.0/mhi0/mhi0_IPCR ... v2: move index field at the end of mhi_controller struct (before bool) to avoid breaking well packed alignment. Signed-off-by: Loic Poulain Reviewed-by: Jeffrey Hugo Reviewed-by: Manivannan Sadhasivam Signed-off-by: Manivannan Sadhasivam --- drivers/bus/mhi/core/init.c | 18 ++++++++++++++++-- drivers/bus/mhi/core/main.c | 2 +- include/linux/mhi.h | 2 ++ 3 files changed, 19 insertions(+), 3 deletions(-) (limited to 'include') diff --git a/drivers/bus/mhi/core/init.c b/drivers/bus/mhi/core/init.c index 655d539c6808..1d6f7b6c1fcd 100644 --- a/drivers/bus/mhi/core/init.c +++ b/drivers/bus/mhi/core/init.c @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,8 @@ #include #include "internal.h" +static DEFINE_IDA(mhi_controller_ida); + const char * const mhi_ee_str[MHI_EE_MAX] = { [MHI_EE_PBL] = "PBL", [MHI_EE_SBL] = "SBL", @@ -940,6 +943,12 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, mhi_cntrl->minor_version = (soc_info & SOC_HW_VERSION_MINOR_VER_BMSK) >> SOC_HW_VERSION_MINOR_VER_SHFT; + mhi_cntrl->index = ida_alloc(&mhi_controller_ida, GFP_KERNEL); + if (mhi_cntrl->index < 0) { + ret = mhi_cntrl->index; + goto error_ida_alloc; + } + /* Register controller with MHI bus */ mhi_dev = mhi_alloc_device(mhi_cntrl); if (IS_ERR(mhi_dev)) { @@ -950,8 +959,8 @@ int mhi_register_controller(struct mhi_controller *mhi_cntrl, mhi_dev->dev_type = MHI_DEVICE_CONTROLLER; mhi_dev->mhi_cntrl = mhi_cntrl; - dev_set_name(&mhi_dev->dev, "%s", dev_name(mhi_cntrl->cntrl_dev)); - mhi_dev->name = dev_name(mhi_cntrl->cntrl_dev); + dev_set_name(&mhi_dev->dev, "mhi%d", mhi_cntrl->index); + mhi_dev->name = dev_name(&mhi_dev->dev); /* Init wakeup source */ device_init_wakeup(&mhi_dev->dev, true); @@ -970,6 +979,9 @@ error_add_dev: put_device(&mhi_dev->dev); error_alloc_dev: + ida_free(&mhi_controller_ida, mhi_cntrl->index); + +error_ida_alloc: kfree(mhi_cntrl->mhi_cmd); error_alloc_cmd: @@ -1004,6 +1016,8 @@ void mhi_unregister_controller(struct mhi_controller *mhi_cntrl) device_del(&mhi_dev->dev); put_device(&mhi_dev->dev); + + ida_free(&mhi_controller_ida, mhi_cntrl->index); } EXPORT_SYMBOL_GPL(mhi_unregister_controller); diff --git a/drivers/bus/mhi/core/main.c b/drivers/bus/mhi/core/main.c index 4eb93d8bea1d..702c31b6aefa 100644 --- a/drivers/bus/mhi/core/main.c +++ b/drivers/bus/mhi/core/main.c @@ -331,7 +331,7 @@ void mhi_create_devices(struct mhi_controller *mhi_cntrl) /* Channel name is same for both UL and DL */ mhi_dev->name = mhi_chan->name; dev_set_name(&mhi_dev->dev, "%s_%s", - dev_name(mhi_cntrl->cntrl_dev), + dev_name(&mhi_cntrl->mhi_dev->dev), mhi_dev->name); /* Init wakeup source if available */ diff --git a/include/linux/mhi.h b/include/linux/mhi.h index d31efcf02ae7..aa9757e71f1f 100644 --- a/include/linux/mhi.h +++ b/include/linux/mhi.h @@ -348,6 +348,7 @@ struct mhi_controller_config { * @read_reg: Read a MHI register via the physical link (required) * @write_reg: Write a MHI register via the physical link (required) * @buffer_len: Bounce buffer length + * @index: Index of the MHI controller instance * @bounce_buf: Use of bounce buffer * @fbc_download: MHI host needs to do complete image transfer (optional) * @pre_init: MHI host needs to do pre-initialization before power up @@ -438,6 +439,7 @@ struct mhi_controller { u32 val); size_t buffer_len; + int index; bool bounce_buf; bool fbc_download; bool pre_init; -- cgit v1.2.3 From 4bb1f2f3fb31ed60a23064a8fc4d5ecde5d1002d Mon Sep 17 00:00:00 2001 From: Tal Cohen Date: Wed, 3 Jun 2020 09:25:27 +0300 Subject: habanalabs: use enum for CB allocation options In the future there will be situations where queues can accept either kernel allocated CBs or user allocated CBs, depending on different states. Therefore, instead of using a boolean variable of kernel/user allocated CB, we need to use a bitmask to indicate that, which will allow to combine the two options. Add a flag to the uapi so the user will be able to indicate whether the CB was allocated by kernel or by user. Of course the driver validates that. Signed-off-by: Tal Cohen Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 31 ++++++++++++++++++++-- drivers/misc/habanalabs/common/habanalabs.h | 19 ++++++++++--- drivers/misc/habanalabs/gaudi/gaudi.c | 13 ++++++--- drivers/misc/habanalabs/goya/goya.c | 6 ++--- include/uapi/misc/habanalabs.h | 16 +++++++++++ 5 files changed, 73 insertions(+), 12 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 0e37aad85930..cd3422bfe6f8 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -568,9 +568,36 @@ static int validate_queue_index(struct hl_device *hdev, return -EINVAL; } - *queue_type = hw_queue_prop->type; - *is_kernel_allocated_cb = !!hw_queue_prop->requires_kernel_cb; + /* When hw queue type isn't QUEUE_TYPE_HW, + * USER_ALLOC_CB flag shall be referred as "don't care". + */ + if (hw_queue_prop->type == QUEUE_TYPE_HW) { + if (chunk->cs_chunk_flags & HL_CS_CHUNK_FLAGS_USER_ALLOC_CB) { + if (!(hw_queue_prop->cb_alloc_flags & CB_ALLOC_USER)) { + dev_err(hdev->dev, + "Queue index %d doesn't support user CB\n", + chunk->queue_index); + return -EINVAL; + } + *is_kernel_allocated_cb = false; + } else { + if (!(hw_queue_prop->cb_alloc_flags & + CB_ALLOC_KERNEL)) { + dev_err(hdev->dev, + "Queue index %d doesn't support kernel CB\n", + chunk->queue_index); + return -EINVAL; + } + + *is_kernel_allocated_cb = true; + } + } else { + *is_kernel_allocated_cb = !!(hw_queue_prop->cb_alloc_flags + & CB_ALLOC_KERNEL); + } + + *queue_type = hw_queue_prop->type; return 0; } diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index b5a34936e22d..0823798f292e 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -206,6 +206,17 @@ struct hl_outbound_pci_region { u64 size; }; +/* + * enum queue_cb_alloc_flags - Indicates queue support for CBs that + * allocated by Kernel or by User + * @CB_ALLOC_KERNEL: support only CBs that allocated by Kernel + * @CB_ALLOC_USER: support only CBs that allocated by User + */ +enum queue_cb_alloc_flags { + CB_ALLOC_KERNEL = 0x1, + CB_ALLOC_USER = 0x2 +}; + /* * struct hl_hw_sob - H/W SOB info. * @hdev: habanalabs device structure. @@ -223,16 +234,18 @@ struct hl_hw_sob { /** * struct hw_queue_properties - queue information. * @type: queue type. + * @queue_cb_alloc_flags: bitmap which indicates if the hw queue supports CB + * that allocated by the Kernel driver and therefore, + * a CB handle can be provided for jobs on this queue. + * Otherwise, a CB address must be provided. * @driver_only: true if only the driver is allowed to send a job to this queue, * false otherwise. - * @requires_kernel_cb: true if a CB handle must be provided for jobs on this - * queue, false otherwise (a CB address must be provided). * @supports_sync_stream: True if queue supports sync stream */ struct hw_queue_properties { enum hl_queue_type type; + enum queue_cb_alloc_flags cb_alloc_flags; u8 driver_only; - u8 requires_kernel_cb; u8 supports_sync_stream; }; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 2dd9b732299a..9393e34b9719 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -381,23 +381,28 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) if (gaudi_queue_type[i] == QUEUE_TYPE_EXT) { prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; prop->hw_queues_props[i].driver_only = 0; - prop->hw_queues_props[i].requires_kernel_cb = 1; prop->hw_queues_props[i].supports_sync_stream = 1; + prop->hw_queues_props[i].cb_alloc_flags = + CB_ALLOC_KERNEL; num_sync_stream_queues++; } else if (gaudi_queue_type[i] == QUEUE_TYPE_CPU) { prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; prop->hw_queues_props[i].driver_only = 1; - prop->hw_queues_props[i].requires_kernel_cb = 0; prop->hw_queues_props[i].supports_sync_stream = 0; + prop->hw_queues_props[i].cb_alloc_flags = + CB_ALLOC_KERNEL; } else if (gaudi_queue_type[i] == QUEUE_TYPE_INT) { prop->hw_queues_props[i].type = QUEUE_TYPE_INT; prop->hw_queues_props[i].driver_only = 0; - prop->hw_queues_props[i].requires_kernel_cb = 0; + prop->hw_queues_props[i].supports_sync_stream = 0; + prop->hw_queues_props[i].cb_alloc_flags = + CB_ALLOC_USER; } else if (gaudi_queue_type[i] == QUEUE_TYPE_NA) { prop->hw_queues_props[i].type = QUEUE_TYPE_NA; prop->hw_queues_props[i].driver_only = 0; - prop->hw_queues_props[i].requires_kernel_cb = 0; prop->hw_queues_props[i].supports_sync_stream = 0; + prop->hw_queues_props[i].cb_alloc_flags = + CB_ALLOC_USER; } } diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index d873f613acb0..74c44278166b 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -373,20 +373,20 @@ int goya_get_fixed_properties(struct hl_device *hdev) for (i = 0 ; i < NUMBER_OF_EXT_HW_QUEUES ; i++) { prop->hw_queues_props[i].type = QUEUE_TYPE_EXT; prop->hw_queues_props[i].driver_only = 0; - prop->hw_queues_props[i].requires_kernel_cb = 1; + prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; } for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES ; i++) { prop->hw_queues_props[i].type = QUEUE_TYPE_CPU; prop->hw_queues_props[i].driver_only = 1; - prop->hw_queues_props[i].requires_kernel_cb = 0; + prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_KERNEL; } for (; i < NUMBER_OF_EXT_HW_QUEUES + NUMBER_OF_CPU_HW_QUEUES + NUMBER_OF_INT_HW_QUEUES; i++) { prop->hw_queues_props[i].type = QUEUE_TYPE_INT; prop->hw_queues_props[i].driver_only = 0; - prop->hw_queues_props[i].requires_kernel_cb = 0; + prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER; } prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 9705b8adb60c..5753157e71b3 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -490,6 +490,22 @@ union hl_cb_args { struct hl_cb_out out; }; +/* HL_CS_CHUNK_FLAGS_ values + * + * HL_CS_CHUNK_FLAGS_USER_ALLOC_CB: + * Indicates if the CB was allocated and mapped by userspace. + * User allocated CB is a command buffer allocated by the user, via malloc + * (or similar). After allocating the CB, the user invokes “memory ioctl” + * to map the user memory into a device virtual address. The user provides + * this address via the cb_handle field. The interface provides the + * ability to create a large CBs, Which aren’t limited to + * “HL_MAX_CB_SIZE”. Therefore, it increases the PCI-DMA queues + * throughput. This CB allocation method also reduces the use of Linux + * DMA-able memory pool. Which are limited and used by other Linux + * sub-systems. + */ +#define HL_CS_CHUNK_FLAGS_USER_ALLOC_CB 0x1 + /* * This structure size must always be fixed to 64-bytes for backward * compatibility -- cgit v1.2.3 From 5fe1c17ddf2e5e5d05e983b56ebbc0d1c702b16a Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 10 Sep 2020 10:10:55 +0300 Subject: habanalabs: sync stream collective infrastructure Define new API for collective wait support and modify sync stream common flow. In addition add kernel CB allocation support for internal queues. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 38 ++++++++++++--- drivers/misc/habanalabs/common/habanalabs.h | 55 +++++++++++++++++++--- drivers/misc/habanalabs/common/hw_queue.c | 44 +++++++++++++++-- drivers/misc/habanalabs/gaudi/gaudi.c | 22 ++++++++- drivers/misc/habanalabs/goya/goya.c | 22 ++++++++- include/uapi/misc/habanalabs.h | 17 +++++-- 6 files changed, 176 insertions(+), 22 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index cd3422bfe6f8..2dbd42b6ad0c 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -85,7 +85,8 @@ static void hl_fence_release(struct kref *kref) goto free; if ((hl_cs_cmpl->type == CS_TYPE_SIGNAL) || - (hl_cs_cmpl->type == CS_TYPE_WAIT)) { + (hl_cs_cmpl->type == CS_TYPE_WAIT) || + (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT)) { dev_dbg(hdev->dev, "CS 0x%llx type %d finished, sob_id: %d, sob_val: 0x%x\n", @@ -112,6 +113,10 @@ static void hl_fence_release(struct kref *kref) * hence the above scenario is avoided. */ kref_put(&hl_cs_cmpl->hw_sob->kref, hl_sob_reset); + + if (hl_cs_cmpl->type == CS_TYPE_COLLECTIVE_WAIT) + hdev->asic_funcs->reset_sob_group(hdev, + hl_cs_cmpl->sob_group); } free: @@ -247,9 +252,11 @@ static void free_job(struct hl_device *hdev, struct hl_cs_job *job) /* For H/W queue jobs, if a user CB was allocated by driver and MMU is * enabled, the user CB isn't released in cs_parser() and thus should be * released here. + * This is also true for INT queues jobs which were allocated by driver */ - if (job->queue_type == QUEUE_TYPE_HW && - job->is_kernel_allocated_cb && hdev->mmu_enable) { + if (job->is_kernel_allocated_cb && + ((job->queue_type == QUEUE_TYPE_HW && hdev->mmu_enable) || + job->queue_type == QUEUE_TYPE_INT)) { spin_lock(&job->user_cb->lock); job->user_cb->cs_cnt--; spin_unlock(&job->user_cb->lock); @@ -931,7 +938,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, struct hl_cs_compl *sig_waitcs_cmpl; struct hl_cs *cs; enum hl_queue_type q_type; - u32 size_to_copy, q_idx; + u32 size_to_copy, q_idx, collective_engine_id; u64 signal_seq; int rc; @@ -981,7 +988,18 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, goto free_cs_chunk_array; } - if (cs_type == CS_TYPE_WAIT) { + if (cs_type == CS_TYPE_COLLECTIVE_WAIT) { + if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) { + dev_err(hdev->dev, + "Queue index %d is invalid\n", q_idx); + rc = -EINVAL; + goto free_cs_chunk_array; + } + + collective_engine_id = chunk->collective_engine_id; + } + + if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) { rc = cs_ioctl_extract_signal_seq(hdev, chunk, &signal_seq); if (rc) goto free_cs_chunk_array; @@ -1026,7 +1044,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, rc = allocate_cs(hdev, ctx, cs_type, &cs); if (rc) { - if (cs_type == CS_TYPE_WAIT) + if (cs_type == CS_TYPE_WAIT || + cs_type == CS_TYPE_COLLECTIVE_WAIT) hl_fence_put(sig_fence); hl_ctx_put(ctx); goto free_cs_chunk_array; @@ -1036,7 +1055,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, * Save the signal CS fence for later initialization right before * hanging the wait CS on the queue. */ - if (cs_type == CS_TYPE_WAIT) + if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_COLLECTIVE_WAIT) cs->signal_fence = sig_fence; hl_debugfs_add_cs(cs); @@ -1046,6 +1065,9 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, if (cs_type == CS_TYPE_WAIT || cs_type == CS_TYPE_SIGNAL) rc = cs_ioctl_signal_wait_create_jobs(hdev, ctx, cs, q_type, q_idx); + else + rc = hdev->asic_funcs->collective_wait_create_jobs(hdev, ctx, + cs, q_idx, collective_engine_id); if (rc) goto put_cs; @@ -1120,6 +1142,8 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) cs_type = CS_TYPE_SIGNAL; else if (args->in.cs_flags & HL_CS_FLAGS_WAIT) cs_type = CS_TYPE_WAIT; + else if (args->in.cs_flags & HL_CS_FLAGS_COLLECTIVE_WAIT) + cs_type = CS_TYPE_COLLECTIVE_WAIT; else cs_type = CS_TYPE_DEFAULT; diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 0823798f292e..98249a2c97e7 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -68,6 +68,11 @@ #define HL_RSVD_SOBS 4 #define HL_RSVD_MONS 2 +/* + * HL_COLLECTIVE_RSVD_MSTR_MONS 'collective' reserved monitors per QMAN stream + */ +#define HL_COLLECTIVE_RSVD_MSTR_MONS 2 + #define HL_MAX_SOB_VAL (1 << 15) #define IS_POWER_OF_2(n) (n != 0 && ((n & (n - 1)) == 0)) @@ -177,7 +182,8 @@ enum hl_queue_type { enum hl_cs_type { CS_TYPE_DEFAULT, CS_TYPE_SIGNAL, - CS_TYPE_WAIT + CS_TYPE_WAIT, + CS_TYPE_COLLECTIVE_WAIT }; /* @@ -231,6 +237,12 @@ struct hl_hw_sob { u32 q_idx; }; +enum hl_collective_mode { + HL_COLLECTIVE_NOT_SUPPORTED = 0x0, + HL_COLLECTIVE_MASTER = 0x1, + HL_COLLECTIVE_SLAVE = 0x2 +}; + /** * struct hw_queue_properties - queue information. * @type: queue type. @@ -238,6 +250,7 @@ struct hl_hw_sob { * that allocated by the Kernel driver and therefore, * a CB handle can be provided for jobs on this queue. * Otherwise, a CB address must be provided. + * @collective_mode: collective mode of current queue * @driver_only: true if only the driver is allowed to send a job to this queue, * false otherwise. * @supports_sync_stream: True if queue supports sync stream @@ -245,6 +258,7 @@ struct hl_hw_sob { struct hw_queue_properties { enum hl_queue_type type; enum queue_cb_alloc_flags cb_alloc_flags; + enum hl_collective_mode collective_mode; u8 driver_only; u8 supports_sync_stream; }; @@ -358,6 +372,8 @@ struct hl_mmu_properties { * @cb_pool_cb_size: size of each CB in the CB pool. * @max_pending_cs: maximum of concurrent pending command submissions * @max_queues: maximum amount of queues in the system + * @collective_first_sob: first sync object available for collective use + * @collective_first_mon: first monitor available for collective use * @sync_stream_first_sob: first sync object available for sync stream use * @sync_stream_first_mon: first monitor available for sync stream use * @first_available_user_sob: first sob available for the user @@ -410,6 +426,8 @@ struct asic_fixed_properties { u32 cb_pool_cb_size; u32 max_pending_cs; u32 max_queues; + u16 collective_first_sob; + u16 collective_first_mon; u16 sync_stream_first_sob; u16 sync_stream_first_mon; u16 first_available_user_sob[HL_MAX_DCORES]; @@ -441,6 +459,7 @@ struct hl_fence { * @cs_seq: command submission sequence number. * @type: type of the CS - signal/wait. * @sob_val: the SOB value that is used in this signal/wait CS. + * @sob_group: the SOB group that is used in this collective wait CS. */ struct hl_cs_compl { struct hl_fence base_fence; @@ -450,6 +469,7 @@ struct hl_cs_compl { u64 cs_seq; enum hl_cs_type type; u16 sob_val; + u16 sob_group; }; /* @@ -512,6 +532,7 @@ struct hl_cb { * QUEUES */ +struct hl_cs; struct hl_cs_job; /* Queue length of external and HW queues */ @@ -540,15 +561,24 @@ struct hl_cs_job; * @next_sob_val: the next value to use for the currently used SOB. * @base_sob_id: the base SOB id of the SOBs used by this queue. * @base_mon_id: the base MON id of the MONs used by this queue. + * @collective_mstr_mon_id: the MON ids of the MONs used by this master queue + * in order to sync with all slave queues. + * @collective_slave_mon_id: the MON id used by this slave queue in order to + * sync with its master queue. + * @collective_sob_id: current SOB id used by this collective slave queue + * to signal its collective master queue upon completion. * @curr_sob_offset: the id offset to the currently used SOB from the * HL_RSVD_SOBS that are being used by this queue. */ struct hl_sync_stream_properties { - struct hl_hw_sob hw_sob[HL_RSVD_SOBS]; - u16 next_sob_val; - u16 base_sob_id; - u16 base_mon_id; - u8 curr_sob_offset; + struct hl_hw_sob hw_sob[HL_RSVD_SOBS]; + u16 next_sob_val; + u16 base_sob_id; + u16 base_mon_id; + u16 collective_mstr_mon_id[HL_COLLECTIVE_RSVD_MSTR_MONS]; + u16 collective_slave_mon_id; + u16 collective_sob_id; + u8 curr_sob_offset; }; /** @@ -556,6 +586,7 @@ struct hl_sync_stream_properties { * @shadow_queue: pointer to a shadow queue that holds pointers to jobs. * @sync_stream_prop: sync stream queue properties * @queue_type: type of queue. + * @collective_mode: collective mode of current queue * @kernel_address: holds the queue's kernel virtual address. * @bus_address: holds the queue's DMA address. * @pi: holds the queue's pi value. @@ -572,6 +603,7 @@ struct hl_hw_queue { struct hl_cs_job **shadow_queue; struct hl_sync_stream_properties sync_stream_prop; enum hl_queue_type queue_type; + enum hl_collective_mode collective_mode; void *kernel_address; dma_addr_t bus_address; u32 pi; @@ -764,9 +796,13 @@ enum div_select_defs { * @gen_signal_cb: Generate a signal CB. * @gen_wait_cb: Generate a wait CB. * @reset_sob: Reset a SOB. + * @reset_sob_group: Reset SOB group * @set_dma_mask_from_fw: set the DMA mask in the driver according to the * firmware configuration * @get_device_time: Get the device time. + * @collective_wait_init_cs: Generate collective master/slave packets + * and place them in the relevant cs jobs + * @collective_wait_create_jobs: allocate collective wait cs jobs */ struct hl_asic_funcs { int (*early_init)(struct hl_device *hdev); @@ -868,8 +904,13 @@ struct hl_asic_funcs { u32 (*gen_wait_cb)(struct hl_device *hdev, struct hl_gen_wait_properties *prop); void (*reset_sob)(struct hl_device *hdev, void *data); + void (*reset_sob_group)(struct hl_device *hdev, u16 sob_group); void (*set_dma_mask_from_fw)(struct hl_device *hdev); u64 (*get_device_time)(struct hl_device *hdev); + void (*collective_wait_init_cs)(struct hl_cs *cs); + int (*collective_wait_create_jobs)(struct hl_device *hdev, + struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, + u32 collective_engine_id); }; @@ -1656,6 +1697,7 @@ struct hl_mmu_funcs { * @stop_on_err: true if engines should stop on error. * @supports_sync_stream: is sync stream supported. * @sync_stream_queue_idx: helper index for sync stream queues initialization. + * @collective_mon_idx: helper index for collective initialization * @supports_coresight: is CoreSight supported. * @supports_soft_reset: is soft reset supported. * @supports_cb_mapping: is mapping a CB to the device's MMU supported. @@ -1756,6 +1798,7 @@ struct hl_device { u8 stop_on_err; u8 supports_sync_stream; u8 sync_stream_queue_idx; + u8 collective_mon_idx; u8 supports_coresight; u8 supports_soft_reset; u8 supports_cb_mapping; diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index be1d0e2c99d8..d9448375beac 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -333,7 +333,14 @@ static void int_queue_schedule_job(struct hl_cs_job *job) bd.ctl = 0; bd.len = cpu_to_le32(job->job_cb_size); - bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb); + + if (job->is_kernel_allocated_cb) + /* bus_address is actually a mmu mapped address + * allocated from an internal pool + */ + bd.ptr = cpu_to_le64(job->user_cb->bus_address); + else + bd.ptr = cpu_to_le64((u64) (uintptr_t) job->user_cb); pi = q->kernel_address + (q->pi & (q->int_queue_len - 1)) * sizeof(bd); @@ -562,6 +569,8 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) if ((cs->type == CS_TYPE_SIGNAL) || (cs->type == CS_TYPE_WAIT)) init_signal_wait_cs(cs); + else if (cs->type == CS_TYPE_COLLECTIVE_WAIT) + hdev->asic_funcs->collective_wait_init_cs(cs); spin_lock(&hdev->hw_queues_mirror_lock); list_add_tail(&cs->mirror_node, &hdev->hw_queues_mirror_list); @@ -741,12 +750,40 @@ static void sync_stream_queue_init(struct hl_device *hdev, u32 q_idx) struct hl_sync_stream_properties *sync_stream_prop; struct asic_fixed_properties *prop = &hdev->asic_prop; struct hl_hw_sob *hw_sob; - int sob, queue_idx; + int sob, reserved_mon_idx, queue_idx; + + sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop; + + /* We use 'collective_mon_idx' as a running index in order to reserve + * monitors for collective master/slave queues. + * collective master queue gets 2 reserved monitors + * collective slave queue gets 1 reserved monitor + */ + if (hdev->kernel_queues[q_idx].collective_mode == + HL_COLLECTIVE_MASTER) { + reserved_mon_idx = hdev->collective_mon_idx; + + /* reserve the first monitor for collective master queue */ + sync_stream_prop->collective_mstr_mon_id[0] = + prop->collective_first_mon + reserved_mon_idx; + + /* reserve the second monitor for collective master queue */ + sync_stream_prop->collective_mstr_mon_id[1] = + prop->collective_first_mon + reserved_mon_idx + 1; + + hdev->collective_mon_idx += HL_COLLECTIVE_RSVD_MSTR_MONS; + } else if (hdev->kernel_queues[q_idx].collective_mode == + HL_COLLECTIVE_SLAVE) { + reserved_mon_idx = hdev->collective_mon_idx++; + + /* reserve a monitor for collective slave queue */ + sync_stream_prop->collective_slave_mon_id = + prop->collective_first_mon + reserved_mon_idx; + } if (!hdev->kernel_queues[q_idx].supports_sync_stream) return; - sync_stream_prop = &hdev->kernel_queues[q_idx].sync_stream_prop; queue_idx = hdev->sync_stream_queue_idx++; sync_stream_prop->base_sob_id = prop->sync_stream_first_sob + @@ -897,6 +934,7 @@ int hl_hw_queues_create(struct hl_device *hdev) q->queue_type = asic->hw_queues_props[i].type; q->supports_sync_stream = asic->hw_queues_props[i].supports_sync_stream; + q->collective_mode = asic->hw_queues_props[i].collective_mode; rc = queue_init(hdev, q, i); if (rc) { dev_err(hdev->dev, diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 9393e34b9719..d4218c4b4887 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -793,6 +793,23 @@ out: return rc; } +static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_groupt) +{ + +} + +static void gaudi_collective_wait_init_cs(struct hl_cs *cs) +{ + +} + +static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, + struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, + u32 collective_engine_id) +{ + return -EINVAL; +} + static int gaudi_late_init(struct hl_device *hdev) { struct gaudi_device *gaudi = hdev->asic_specific; @@ -7358,8 +7375,11 @@ static const struct hl_asic_funcs gaudi_funcs = { .gen_signal_cb = gaudi_gen_signal_cb, .gen_wait_cb = gaudi_gen_wait_cb, .reset_sob = gaudi_reset_sob, + .reset_sob_group = gaudi_reset_sob_group, .set_dma_mask_from_fw = gaudi_set_dma_mask_from_fw, - .get_device_time = gaudi_get_device_time + .get_device_time = gaudi_get_device_time, + .collective_wait_init_cs = gaudi_collective_wait_init_cs, + .collective_wait_create_jobs = gaudi_collective_wait_create_jobs }; /** diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 74c44278166b..9332580b038d 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5305,6 +5305,11 @@ static void goya_reset_sob(struct hl_device *hdev, void *data) } +void goya_reset_sob_group(struct hl_device *hdev, u16 sob_group) +{ + +} + static void goya_set_dma_mask_from_fw(struct hl_device *hdev) { if (RREG32(mmPSOC_GLOBAL_CONF_NON_RST_FLOPS_0) == @@ -5326,6 +5331,18 @@ u64 goya_get_device_time(struct hl_device *hdev) return device_time | RREG32(mmPSOC_TIMESTAMP_CNTCVL); } +void goya_collective_wait_init_cs(struct hl_cs *cs) +{ + +} + +int goya_collective_wait_create_jobs(struct hl_device *hdev, + struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, + u32 collective_engine_id) +{ + return -EINVAL; +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5397,8 +5414,11 @@ static const struct hl_asic_funcs goya_funcs = { .gen_signal_cb = goya_gen_signal_cb, .gen_wait_cb = goya_gen_wait_cb, .reset_sob = goya_reset_sob, + .reset_sob_group = goya_reset_sob_group, .set_dma_mask_from_fw = goya_set_dma_mask_from_fw, - .get_device_time = goya_get_device_time + .get_device_time = goya_get_device_time, + .collective_wait_init_cs = goya_collective_wait_init_cs, + .collective_wait_create_jobs = goya_collective_wait_create_jobs }; /* diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 5753157e71b3..2b244d0bdc26 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -523,7 +523,8 @@ struct hl_cs_chunk { */ __u64 cb_handle; - /* Relevant only when HL_CS_FLAGS_WAIT is set. + /* Relevant only when HL_CS_FLAGS_WAIT or + * HL_CS_FLAGS_COLLECTIVE_WAIT is set. * This holds address of array of u64 values that contain * signal CS sequence numbers. The wait described by this job * will listen on all those signals (wait event per signal) @@ -541,7 +542,8 @@ struct hl_cs_chunk { */ __u32 cb_size; - /* Relevant only when HL_CS_FLAGS_WAIT is set. + /* Relevant only when HL_CS_FLAGS_WAIT or + * HL_CS_FLAGS_COLLECTIVE_WAIT is set. * Number of entries in signal_seq_arr */ __u32 num_signal_seq_arr; @@ -550,14 +552,21 @@ struct hl_cs_chunk { /* HL_CS_CHUNK_FLAGS_* */ __u32 cs_chunk_flags; + /* Relevant only when HL_CS_FLAGS_COLLECTIVE_WAIT is set. + * This holds the collective engine ID. The wait described by this job + * will sync with this engine and with all NICs before completion. + */ + __u32 collective_engine_id; + /* Align structure to 64 bytes */ - __u32 pad[11]; + __u32 pad[10]; }; -/* SIGNAL and WAIT flags are mutually exclusive */ +/* SIGNAL and WAIT/COLLECTIVE_WAIT flags are mutually exclusive */ #define HL_CS_FLAGS_FORCE_RESTORE 0x1 #define HL_CS_FLAGS_SIGNAL 0x2 #define HL_CS_FLAGS_WAIT 0x4 +#define HL_CS_FLAGS_COLLECTIVE_WAIT 0x8 #define HL_CS_STATUS_SUCCESS 0 -- cgit v1.2.3 From 0940cabafde98466bec8ba32dd567f0ec060478d Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Mon, 31 Aug 2020 08:52:56 +0300 Subject: habanalabs/gaudi: Set DMA5 QMAN internal DMA5 QMAN is designated to be used for reduction process, hence it will be no longer configured as external queue. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/gaudi/gaudi.c | 31 +++++++++++++------------------ drivers/misc/habanalabs/gaudi/gaudiP.h | 8 ++++---- include/uapi/misc/habanalabs.h | 12 ++++++------ 3 files changed, 23 insertions(+), 28 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index d4218c4b4887..7e1557f8a73f 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -38,7 +38,7 @@ * * MMU is always enabled. * - * QMAN DMA channels 0,1,5 (PCI DMAN): + * QMAN DMA channels 0,1 (PCI DMAN): * - DMA is not secured. * - PQ and CQ are secured. * - CP is secured: The driver needs to parse CB but WREG should be allowed @@ -55,7 +55,7 @@ * idle) * - MMU page tables area clear (happens on init) * - * QMAN DMA 2-4,6,7, TPC, MME, NIC: + * QMAN DMA 2-7, TPC, MME, NIC: * PQ is secured and is located on the Host (HBM CON TPC3 bug) * CQ, CP and the engine are not secured * @@ -113,12 +113,12 @@ static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = { static const u8 gaudi_dma_assignment[GAUDI_DMA_MAX] = { [GAUDI_PCI_DMA_1] = GAUDI_ENGINE_ID_DMA_0, [GAUDI_PCI_DMA_2] = GAUDI_ENGINE_ID_DMA_1, - [GAUDI_PCI_DMA_3] = GAUDI_ENGINE_ID_DMA_5, [GAUDI_HBM_DMA_1] = GAUDI_ENGINE_ID_DMA_2, [GAUDI_HBM_DMA_2] = GAUDI_ENGINE_ID_DMA_3, [GAUDI_HBM_DMA_3] = GAUDI_ENGINE_ID_DMA_4, - [GAUDI_HBM_DMA_4] = GAUDI_ENGINE_ID_DMA_6, - [GAUDI_HBM_DMA_5] = GAUDI_ENGINE_ID_DMA_7 + [GAUDI_HBM_DMA_4] = GAUDI_ENGINE_ID_DMA_5, + [GAUDI_HBM_DMA_5] = GAUDI_ENGINE_ID_DMA_6, + [GAUDI_HBM_DMA_6] = GAUDI_ENGINE_ID_DMA_7 }; static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = { @@ -130,10 +130,6 @@ static const u8 gaudi_cq_assignment[NUMBER_OF_CMPLT_QUEUES] = { [5] = GAUDI_QUEUE_ID_DMA_1_1, [6] = GAUDI_QUEUE_ID_DMA_1_2, [7] = GAUDI_QUEUE_ID_DMA_1_3, - [8] = GAUDI_QUEUE_ID_DMA_5_0, - [9] = GAUDI_QUEUE_ID_DMA_5_1, - [10] = GAUDI_QUEUE_ID_DMA_5_2, - [11] = GAUDI_QUEUE_ID_DMA_5_3 }; static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = { @@ -249,10 +245,10 @@ static enum hl_queue_type gaudi_queue_type[GAUDI_QUEUE_ID_SIZE] = { QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_1 */ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_2 */ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_4_3 */ - QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_0 */ - QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_1 */ - QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_2 */ - QUEUE_TYPE_EXT, /* GAUDI_QUEUE_ID_DMA_5_3 */ + QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_0 */ + QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_1 */ + QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_2 */ + QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_5_3 */ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_0 */ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_1 */ QUEUE_TYPE_INT, /* GAUDI_QUEUE_ID_DMA_6_2 */ @@ -978,8 +974,7 @@ static int gaudi_alloc_internal_qmans_pq_mem(struct hl_device *hdev) q = &gaudi->internal_qmans[i]; switch (i) { - case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_4_3: - case GAUDI_QUEUE_ID_DMA_6_0 ... GAUDI_QUEUE_ID_DMA_7_3: + case GAUDI_QUEUE_ID_DMA_2_0 ... GAUDI_QUEUE_ID_DMA_7_3: q->pq_size = HBM_DMA_QMAN_SIZE_IN_BYTES; break; case GAUDI_QUEUE_ID_MME_0_0 ... GAUDI_QUEUE_ID_MME_1_3: @@ -3424,21 +3419,21 @@ static void gaudi_ring_doorbell(struct hl_device *hdev, u32 hw_queue_id, u32 pi) break; case GAUDI_QUEUE_ID_DMA_5_0...GAUDI_QUEUE_ID_DMA_5_3: - dma_id = gaudi_dma_assignment[GAUDI_PCI_DMA_3]; + dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4]; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; break; case GAUDI_QUEUE_ID_DMA_6_0...GAUDI_QUEUE_ID_DMA_6_3: - dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_4]; + dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5]; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; break; case GAUDI_QUEUE_ID_DMA_7_0...GAUDI_QUEUE_ID_DMA_7_3: - dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_5]; + dma_id = gaudi_dma_assignment[GAUDI_HBM_DMA_6]; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; q_off = dma_qm_offset + ((hw_queue_id - 1) & 0x3) * 4; db_reg_offset = mmDMA0_QM_PQ_PI_0 + q_off; diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index 277c391272ac..16871d9ff88b 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -15,7 +15,7 @@ #include "../include/gaudi/gaudi.h" #include "../include/gaudi/gaudi_async_events.h" -#define NUMBER_OF_EXT_HW_QUEUES 12 +#define NUMBER_OF_EXT_HW_QUEUES 8 #define NUMBER_OF_CMPLT_QUEUES NUMBER_OF_EXT_HW_QUEUES #define NUMBER_OF_CPU_HW_QUEUES 1 #define NUMBER_OF_INT_HW_QUEUES 100 @@ -62,8 +62,8 @@ #error "GAUDI_MAX_PENDING_CS must be power of 2 and greater than 1" #endif -#define PCI_DMA_NUMBER_OF_CHNLS 3 -#define HBM_DMA_NUMBER_OF_CHNLS 5 +#define PCI_DMA_NUMBER_OF_CHNLS 2 +#define HBM_DMA_NUMBER_OF_CHNLS 6 #define DMA_NUMBER_OF_CHNLS (PCI_DMA_NUMBER_OF_CHNLS + \ HBM_DMA_NUMBER_OF_CHNLS) @@ -205,12 +205,12 @@ enum gaudi_dma_channels { GAUDI_PCI_DMA_1, GAUDI_PCI_DMA_2, - GAUDI_PCI_DMA_3, GAUDI_HBM_DMA_1, GAUDI_HBM_DMA_2, GAUDI_HBM_DMA_3, GAUDI_HBM_DMA_4, GAUDI_HBM_DMA_5, + GAUDI_HBM_DMA_6, GAUDI_DMA_MAX }; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 2b244d0bdc26..4661a74f0425 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -18,8 +18,8 @@ #define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000 /* 32KB */ #define GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START 0x80 /* 128 bytes */ -#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 48 -#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 24 +#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 32 +#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 16 /* * Goya queue Numbering * @@ -76,10 +76,10 @@ enum gaudi_queue_id { GAUDI_QUEUE_ID_DMA_4_1 = 18, /* internal */ GAUDI_QUEUE_ID_DMA_4_2 = 19, /* internal */ GAUDI_QUEUE_ID_DMA_4_3 = 20, /* internal */ - GAUDI_QUEUE_ID_DMA_5_0 = 21, /* external */ - GAUDI_QUEUE_ID_DMA_5_1 = 22, /* external */ - GAUDI_QUEUE_ID_DMA_5_2 = 23, /* external */ - GAUDI_QUEUE_ID_DMA_5_3 = 24, /* external */ + GAUDI_QUEUE_ID_DMA_5_0 = 21, /* internal */ + GAUDI_QUEUE_ID_DMA_5_1 = 22, /* internal */ + GAUDI_QUEUE_ID_DMA_5_2 = 23, /* internal */ + GAUDI_QUEUE_ID_DMA_5_3 = 24, /* internal */ GAUDI_QUEUE_ID_DMA_6_0 = 25, /* internal */ GAUDI_QUEUE_ID_DMA_6_1 = 26, /* internal */ GAUDI_QUEUE_ID_DMA_6_2 = 27, /* internal */ -- cgit v1.2.3 From 5de406c0b5c747c40277861ecf204ebfa095caa5 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Thu, 10 Sep 2020 10:56:26 +0300 Subject: habanalabs: sync stream collective support Implement sync stream collective for GAUDI. Need to allocate additional resources for that and add ctx_fini() to clean up those resources. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 11 +- drivers/misc/habanalabs/common/context.c | 1 + drivers/misc/habanalabs/common/habanalabs.h | 7 +- drivers/misc/habanalabs/gaudi/gaudi.c | 846 +++++++++++++++++++-- drivers/misc/habanalabs/gaudi/gaudiP.h | 40 +- drivers/misc/habanalabs/goya/goya.c | 6 + include/uapi/misc/habanalabs.h | 14 +- 7 files changed, 862 insertions(+), 63 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 2dbd42b6ad0c..5ece52588ec6 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -142,7 +142,7 @@ static void hl_fence_init(struct hl_fence *fence) init_completion(&fence->completion); } -static void cs_get(struct hl_cs *cs) +void cs_get(struct hl_cs *cs) { kref_get(&cs->refcount); } @@ -917,6 +917,9 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, job->job_cb_size = job->user_cb_size; hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); + /* increment refcount as for external queues we get completion */ + cs_get(cs); + cs->jobs_in_queue_cnt[job->hw_queue_id]++; list_add_tail(&job->cs_node, &cs->job_list); @@ -1070,11 +1073,7 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, cs, q_idx, collective_engine_id); if (rc) - goto put_cs; - - - /* increment refcount as for external queues we get completion */ - cs_get(cs); + goto free_cs_object; rc = hl_hw_queue_schedule_cs(cs); if (rc) { diff --git a/drivers/misc/habanalabs/common/context.c b/drivers/misc/habanalabs/common/context.c index 7a59dd7c6450..2077bbe3606a 100644 --- a/drivers/misc/habanalabs/common/context.c +++ b/drivers/misc/habanalabs/common/context.c @@ -40,6 +40,7 @@ static void hl_ctx_fini(struct hl_ctx *ctx) if ((hdev->in_debug) && (hdev->compute_ctx == ctx)) hl_device_set_debug_mode(hdev, false); + hdev->asic_funcs->ctx_fini(ctx); hl_cb_va_pool_fini(ctx); hl_vm_ctx_fini(ctx); hl_asid_free(hdev, ctx->asid); diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 98249a2c97e7..d6eb5c6a2873 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -65,8 +65,8 @@ * HL_RSVD_SOBS 'sync stream' reserved sync objects per QMAN stream * HL_RSVD_MONS 'sync stream' reserved monitors per QMAN stream */ -#define HL_RSVD_SOBS 4 -#define HL_RSVD_MONS 2 +#define HL_RSVD_SOBS 2 +#define HL_RSVD_MONS 1 /* * HL_COLLECTIVE_RSVD_MSTR_MONS 'collective' reserved monitors per QMAN stream @@ -785,6 +785,7 @@ enum div_select_defs { * @wreg: Write a register. Needed for simulator support. * @halt_coresight: stop the ETF and ETR traces. * @ctx_init: context dependent initialization. + * @ctx_fini: context dependent cleanup. * @get_clk_rate: Retrieve the ASIC current and maximum clock rate in MHz * @get_queue_id_for_cq: Get the H/W queue id related to the given CQ index. * @read_device_fw_version: read the device's firmware versions that are @@ -891,6 +892,7 @@ struct hl_asic_funcs { void (*wreg)(struct hl_device *hdev, u32 reg, u32 val); void (*halt_coresight)(struct hl_device *hdev); int (*ctx_init)(struct hl_ctx *ctx); + void (*ctx_fini)(struct hl_ctx *ctx); int (*get_clk_rate)(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk); u32 (*get_queue_id_for_cq)(struct hl_device *hdev, u32 cq_idx); void (*read_device_fw_version)(struct hl_device *hdev, @@ -1992,6 +1994,7 @@ void hl_sob_reset_error(struct kref *ref); int hl_gen_sob_mask(u16 sob_base, u8 sob_mask, u8 *mask); void hl_fence_put(struct hl_fence *fence); void hl_fence_get(struct hl_fence *fence); +void cs_get(struct hl_cs *cs); void goya_set_asic_funcs(struct hl_device *hdev); void gaudi_set_asic_funcs(struct hl_device *hdev); diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 7e1557f8a73f..c8e59a8338fb 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -358,6 +358,31 @@ static int gaudi_mmu_clear_pgt_range(struct hl_device *hdev); static int gaudi_cpucp_info_get(struct hl_device *hdev); static void gaudi_disable_clock_gating(struct hl_device *hdev); static void gaudi_mmu_prepare(struct hl_device *hdev, u32 asid); +static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, + u32 size); +static u32 gaudi_gen_wait_cb(struct hl_device *hdev, + struct hl_gen_wait_properties *prop); + +static inline enum hl_collective_mode +get_collective_mode(struct hl_device *hdev, u32 queue_id) +{ + if (gaudi_queue_type[queue_id] == QUEUE_TYPE_EXT) + return HL_COLLECTIVE_MASTER; + + if (queue_id >= GAUDI_QUEUE_ID_DMA_5_0 && + queue_id <= GAUDI_QUEUE_ID_DMA_5_3) + return HL_COLLECTIVE_SLAVE; + + if (queue_id >= GAUDI_QUEUE_ID_TPC_7_0 && + queue_id <= GAUDI_QUEUE_ID_TPC_7_3) + return HL_COLLECTIVE_SLAVE; + + if (queue_id >= GAUDI_QUEUE_ID_NIC_0_0 && + queue_id <= GAUDI_QUEUE_ID_NIC_9_3) + return HL_COLLECTIVE_SLAVE; + + return HL_COLLECTIVE_NOT_SUPPORTED; +} static int gaudi_get_fixed_properties(struct hl_device *hdev) { @@ -393,18 +418,28 @@ static int gaudi_get_fixed_properties(struct hl_device *hdev) prop->hw_queues_props[i].supports_sync_stream = 0; prop->hw_queues_props[i].cb_alloc_flags = CB_ALLOC_USER; - } else if (gaudi_queue_type[i] == QUEUE_TYPE_NA) { - prop->hw_queues_props[i].type = QUEUE_TYPE_NA; - prop->hw_queues_props[i].driver_only = 0; - prop->hw_queues_props[i].supports_sync_stream = 0; - prop->hw_queues_props[i].cb_alloc_flags = - CB_ALLOC_USER; + } + prop->hw_queues_props[i].collective_mode = + get_collective_mode(hdev, i); } prop->completion_queues_count = NUMBER_OF_CMPLT_QUEUES; - prop->sync_stream_first_sob = 0; - prop->sync_stream_first_mon = 0; + prop->collective_first_sob = 0; + prop->collective_first_mon = 0; + + /* 2 SOBs per internal queue stream are reserved for collective */ + prop->sync_stream_first_sob = + ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR) + * QMAN_STREAMS * HL_RSVD_SOBS; + + /* 1 monitor per internal queue stream are reserved for collective + * 2 monitors per external queue stream are reserved for collective + */ + prop->sync_stream_first_mon = + (NUMBER_OF_COLLECTIVE_QUEUES * QMAN_STREAMS) + + (NUMBER_OF_EXT_HW_QUEUES * 2); + prop->dram_base_address = DRAM_PHYS_BASE; prop->dram_size = GAUDI_HBM_SIZE_32GB; prop->dram_end_address = prop->dram_base_address + @@ -789,21 +824,451 @@ out: return rc; } -static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_groupt) +static void gaudi_collective_map_sobs(struct hl_device *hdev, u32 stream) +{ + struct gaudi_device *gaudi = hdev->asic_specific; + struct gaudi_collective_properties *prop = &gaudi->collective_props; + struct hl_hw_queue *q; + u32 i, sob_id, sob_group_id, queue_id; + + /* Iterate through SOB groups and assign a SOB for each slave queue */ + sob_group_id = + stream * HL_RSVD_SOBS + prop->curr_sob_group_idx[stream]; + sob_id = prop->hw_sob_group[sob_group_id].base_sob_id; + + queue_id = GAUDI_QUEUE_ID_NIC_0_0 + stream; + for (i = 0 ; i < NIC_NUMBER_OF_ENGINES ; i++) { + q = &hdev->kernel_queues[queue_id + (4 * i)]; + q->sync_stream_prop.collective_sob_id = sob_id + i; + } + + /* Both DMA5 and TPC7 use the same resources since only a single + * engine need to participate in the reduction process + */ + queue_id = GAUDI_QUEUE_ID_DMA_5_0 + stream; + q = &hdev->kernel_queues[queue_id]; + q->sync_stream_prop.collective_sob_id = + sob_id + NIC_NUMBER_OF_ENGINES; + + queue_id = GAUDI_QUEUE_ID_TPC_7_0 + stream; + q = &hdev->kernel_queues[queue_id]; + q->sync_stream_prop.collective_sob_id = + sob_id + NIC_NUMBER_OF_ENGINES; +} + +static void gaudi_sob_group_hw_reset(struct kref *ref) +{ + struct gaudi_hw_sob_group *hw_sob_group = + container_of(ref, struct gaudi_hw_sob_group, kref); + struct hl_device *hdev = hw_sob_group->hdev; + int i; + + for (i = 0 ; i < NUMBER_OF_SOBS_IN_GRP ; i++) + WREG32(mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0 + + (hw_sob_group->base_sob_id + i) * 4, 0); + + kref_init(&hw_sob_group->kref); +} + +static void gaudi_sob_group_reset_error(struct kref *ref) +{ + struct gaudi_hw_sob_group *hw_sob_group = + container_of(ref, struct gaudi_hw_sob_group, kref); + struct hl_device *hdev = hw_sob_group->hdev; + + dev_crit(hdev->dev, + "SOB release shouldn't be called here, base_sob_id: %d\n", + hw_sob_group->base_sob_id); +} + +static int gaudi_collective_init(struct hl_device *hdev) +{ + u32 i, master_monitor_sobs, sob_id, reserved_sobs_per_group; + struct gaudi_collective_properties *prop; + struct gaudi_device *gaudi; + + gaudi = hdev->asic_specific; + prop = &gaudi->collective_props; + sob_id = hdev->asic_prop.collective_first_sob; + + /* First sob in group must be aligned to HL_MAX_SOBS_PER_MONITOR */ + reserved_sobs_per_group = + ALIGN(NUMBER_OF_SOBS_IN_GRP, HL_MAX_SOBS_PER_MONITOR); + + /* Init SOB groups */ + for (i = 0 ; i < NUM_SOB_GROUPS; i++) { + prop->hw_sob_group[i].hdev = hdev; + prop->hw_sob_group[i].base_sob_id = sob_id; + sob_id += reserved_sobs_per_group; + gaudi_sob_group_hw_reset(&prop->hw_sob_group[i].kref); + } + + for (i = 0 ; i < QMAN_STREAMS; i++) { + prop->next_sob_group_val[i] = 1; + prop->curr_sob_group_idx[i] = 0; + gaudi_collective_map_sobs(hdev, i); + } + + prop->mstr_sob_mask[0] = 0; + master_monitor_sobs = HL_MAX_SOBS_PER_MONITOR; + for (i = 0 ; i < master_monitor_sobs ; i++) + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) + prop->mstr_sob_mask[0] |= BIT(i); + + prop->mstr_sob_mask[1] = 0; + master_monitor_sobs = + NIC_NUMBER_OF_ENGINES - HL_MAX_SOBS_PER_MONITOR; + for (i = 0 ; i < master_monitor_sobs; i++) { + if (gaudi->hw_cap_initialized & BIT(HW_CAP_NIC_SHIFT + i)) + prop->mstr_sob_mask[1] |= BIT(i); + } + + /* Set collective engine bit */ + prop->mstr_sob_mask[1] |= BIT(i); + + return 0; +} + +static void gaudi_reset_sob_group(struct hl_device *hdev, u16 sob_group) +{ + struct gaudi_device *gaudi = hdev->asic_specific; + struct gaudi_collective_properties *cprop = &gaudi->collective_props; + + kref_put(&cprop->hw_sob_group[sob_group].kref, + gaudi_sob_group_hw_reset); +} + +static void gaudi_collective_master_init_job(struct hl_device *hdev, + struct hl_cs_job *job, u32 stream, u32 sob_group_offset) +{ + u32 master_sob_base, master_monitor, queue_id, cb_size = 0; + struct gaudi_collective_properties *cprop; + struct hl_gen_wait_properties wait_prop; + struct hl_sync_stream_properties *prop; + struct gaudi_device *gaudi; + + gaudi = hdev->asic_specific; + cprop = &gaudi->collective_props; + queue_id = job->hw_queue_id; + prop = &hdev->kernel_queues[queue_id].sync_stream_prop; + + master_sob_base = + cprop->hw_sob_group[sob_group_offset].base_sob_id; + master_monitor = prop->collective_mstr_mon_id[0]; + + dev_dbg(hdev->dev, + "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n", + master_sob_base, cprop->mstr_sob_mask[0], + cprop->next_sob_group_val[stream], + master_monitor, queue_id); + + wait_prop.data = (void *) job->patched_cb; + wait_prop.sob_base = master_sob_base; + wait_prop.sob_mask = cprop->mstr_sob_mask[0]; + wait_prop.sob_val = cprop->next_sob_group_val[stream]; + wait_prop.mon_id = master_monitor; + wait_prop.q_idx = queue_id; + wait_prop.size = cb_size; + cb_size += gaudi_gen_wait_cb(hdev, &wait_prop); + + master_sob_base += HL_MAX_SOBS_PER_MONITOR; + master_monitor = prop->collective_mstr_mon_id[1]; + + dev_dbg(hdev->dev, + "Generate master wait CBs, sob %d (mask %#x), val:0x%x, mon %u, q %d\n", + master_sob_base, cprop->mstr_sob_mask[1], + cprop->next_sob_group_val[stream], + master_monitor, queue_id); + + wait_prop.sob_base = master_sob_base; + wait_prop.sob_mask = cprop->mstr_sob_mask[1]; + wait_prop.mon_id = master_monitor; + wait_prop.size = cb_size; + cb_size += gaudi_gen_wait_cb(hdev, &wait_prop); +} + +static void gaudi_collective_slave_init_job(struct hl_device *hdev, + struct hl_cs_job *job, struct hl_cs_compl *cs_cmpl) { + struct hl_gen_wait_properties wait_prop; + struct hl_sync_stream_properties *prop; + u32 queue_id, cb_size = 0; + + queue_id = job->hw_queue_id; + prop = &hdev->kernel_queues[queue_id].sync_stream_prop; + + /* Add to wait CBs using slave monitor */ + wait_prop.data = (void *) job->user_cb; + wait_prop.sob_base = cs_cmpl->hw_sob->sob_id; + wait_prop.sob_mask = 0x1; + wait_prop.sob_val = cs_cmpl->sob_val; + wait_prop.mon_id = prop->collective_slave_mon_id; + wait_prop.q_idx = queue_id; + wait_prop.size = cb_size; + + dev_dbg(hdev->dev, + "Generate slave wait CB, sob %d, val:0x%x, mon %d, q %d\n", + cs_cmpl->hw_sob->sob_id, cs_cmpl->sob_val, + prop->collective_slave_mon_id, queue_id); + + cb_size += gaudi_gen_wait_cb(hdev, &wait_prop); + + dev_dbg(hdev->dev, + "generate signal CB, sob_id: %d, sob val: 1, q_idx: %d\n", + prop->collective_sob_id, queue_id); + cb_size += gaudi_gen_signal_cb(hdev, job->user_cb, + prop->collective_sob_id, cb_size); } static void gaudi_collective_wait_init_cs(struct hl_cs *cs) { + struct hl_cs_compl *signal_cs_cmpl = + container_of(cs->signal_fence, struct hl_cs_compl, base_fence); + struct hl_cs_compl *cs_cmpl = + container_of(cs->fence, struct hl_cs_compl, base_fence); + struct gaudi_collective_properties *cprop; + u32 stream, queue_id, sob_group_offset; + struct gaudi_device *gaudi; + struct hl_device *hdev; + struct hl_cs_job *job; + struct hl_ctx *ctx; + + ctx = cs->ctx; + hdev = ctx->hdev; + gaudi = hdev->asic_specific; + cprop = &gaudi->collective_props; + + /* copy the SOB id and value of the signal CS */ + cs_cmpl->hw_sob = signal_cs_cmpl->hw_sob; + cs_cmpl->sob_val = signal_cs_cmpl->sob_val; + + /* Calculate the stream from collective master queue (1st job) */ + job = list_first_entry(&cs->job_list, struct hl_cs_job, cs_node); + stream = job->hw_queue_id % 4; + sob_group_offset = + stream * HL_RSVD_SOBS + cprop->curr_sob_group_idx[stream]; + + list_for_each_entry(job, &cs->job_list, cs_node) { + queue_id = job->hw_queue_id; + + if (hdev->kernel_queues[queue_id].collective_mode == + HL_COLLECTIVE_MASTER) + gaudi_collective_master_init_job(hdev, job, stream, + sob_group_offset); + else + gaudi_collective_slave_init_job(hdev, job, cs_cmpl); + } + + cs_cmpl->sob_group = sob_group_offset; + + /* Handle sob group kref and wraparound */ + kref_get(&cprop->hw_sob_group[sob_group_offset].kref); + cprop->next_sob_group_val[stream]++; + if (cprop->next_sob_group_val[stream] == HL_MAX_SOB_VAL) { + /* + * Decrement as we reached the max value. + * The release function won't be called here as we've + * just incremented the refcount. + */ + kref_put(&cprop->hw_sob_group[sob_group_offset].kref, + gaudi_sob_group_reset_error); + cprop->next_sob_group_val[stream] = 1; + /* only two SOBs are currently in use */ + cprop->curr_sob_group_idx[stream] = + (cprop->curr_sob_group_idx[stream] + 1) & + (HL_RSVD_SOBS - 1); + + gaudi_collective_map_sobs(hdev, stream); + + dev_dbg(hdev->dev, "switched to SOB group %d, stream: %d\n", + cprop->curr_sob_group_idx[stream], stream); + } + + /* Increment kref since all slave queues are now waiting on it */ + kref_get(&cs_cmpl->hw_sob->kref); + /* + * Must put the signal fence after the SOB refcnt increment so + * the SOB refcnt won't turn 0 and reset the SOB before the + * wait CS was submitted. + */ + mb(); + hl_fence_put(cs->signal_fence); + cs->signal_fence = NULL; +} + +static int gaudi_collective_wait_create_job(struct hl_device *hdev, + struct hl_ctx *ctx, struct hl_cs *cs, + enum hl_collective_mode mode, u32 queue_id, u32 wait_queue_id) +{ + struct hw_queue_properties *hw_queue_prop; + struct hl_cs_counters_atomic *cntr; + struct hl_cs_job *job; + struct hl_cb *cb; + u32 cb_size; + bool patched_cb; + + cntr = &hdev->aggregated_cs_counters; + + if (mode == HL_COLLECTIVE_MASTER) { + /* CB size of collective master queue contains + * 4 msg short packets for monitor 1 configuration + * 1 fence packet + * 4 msg short packets for monitor 2 configuration + * 1 fence packet + * 2 msg prot packets for completion and MSI-X + */ + cb_size = sizeof(struct packet_msg_short) * 8 + + sizeof(struct packet_fence) * 2 + + sizeof(struct packet_msg_prot) * 2; + patched_cb = true; + } else { + /* CB size of collective slave queues contains + * 4 msg short packets for monitor configuration + * 1 fence packet + * 1 additional msg short packet for sob signal + */ + cb_size = sizeof(struct packet_msg_short) * 5 + + sizeof(struct packet_fence); + patched_cb = false; + } + + hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id]; + job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); + if (!job) { + ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc(&cntr->out_of_mem_drop_cnt); + dev_err(hdev->dev, "Failed to allocate a new job\n"); + return -ENOMEM; + } + + /* Allocate internal mapped CB for non patched CBs */ + cb = hl_cb_kernel_create(hdev, cb_size, + hdev->mmu_enable && !patched_cb); + if (!cb) { + ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc(&cntr->out_of_mem_drop_cnt); + kfree(job); + return -EFAULT; + } + + job->id = 0; + job->cs = cs; + job->user_cb = cb; + job->user_cb->cs_cnt++; + job->user_cb_size = cb_size; + job->hw_queue_id = queue_id; + + /* + * No need in parsing, user CB is the patched CB. + * We call hl_cb_destroy() out of two reasons - we don't need + * the CB in the CB idr anymore and to decrement its refcount as + * it was incremented inside hl_cb_kernel_create(). + */ + if (patched_cb) + job->patched_cb = job->user_cb; + else + job->patched_cb = NULL; + + job->job_cb_size = job->user_cb_size; + hl_cb_destroy(hdev, &hdev->kernel_cb_mgr, cb->id << PAGE_SHIFT); + + /* increment refcount as for external queues we get completion */ + if (hw_queue_prop->type == QUEUE_TYPE_EXT) + cs_get(cs); + + cs->jobs_in_queue_cnt[job->hw_queue_id]++; + + list_add_tail(&job->cs_node, &cs->job_list); + + hl_debugfs_add_job(hdev, job); + + return 0; } static int gaudi_collective_wait_create_jobs(struct hl_device *hdev, struct hl_ctx *ctx, struct hl_cs *cs, u32 wait_queue_id, u32 collective_engine_id) { - return -EINVAL; + struct gaudi_device *gaudi = hdev->asic_specific; + struct hw_queue_properties *hw_queue_prop; + u32 queue_id, collective_queue, num_jobs; + u32 stream, nic_queue, nic_idx = 0; + bool skip; + int i, rc; + + /* Verify wait queue id is configured as master */ + hw_queue_prop = &hdev->asic_prop.hw_queues_props[wait_queue_id]; + if (!(hw_queue_prop->collective_mode == HL_COLLECTIVE_MASTER)) { + dev_err(hdev->dev, + "Queue %d is not configured as collective master\n", + wait_queue_id); + return -EINVAL; + } + + /* Verify engine id is supported */ + if (collective_engine_id != GAUDI_ENGINE_ID_DMA_5 && + collective_engine_id != GAUDI_ENGINE_ID_TPC_7) { + dev_err(hdev->dev, + "Collective wait does not support engine %u\n", + collective_engine_id); + return -EINVAL; + } + + stream = wait_queue_id % 4; + + if (collective_engine_id == GAUDI_ENGINE_ID_DMA_5) + collective_queue = GAUDI_QUEUE_ID_DMA_5_0 + stream; + else if (collective_engine_id == GAUDI_ENGINE_ID_TPC_7) + collective_queue = GAUDI_QUEUE_ID_TPC_7_0 + stream; + else + return -EINVAL; + + num_jobs = NUMBER_OF_SOBS_IN_GRP + 1; + nic_queue = GAUDI_QUEUE_ID_NIC_0_0 + stream; + + /* First job goes to the collective master queue, it will wait for + * the collective slave queues to finish execution. + * The synchronization is done using two monitors: + * First monitor for NICs 0-7, second monitor for NICs 8-9 and the + * reduction engine (DMA5/TPC7). + * + * Rest of the jobs goes to the collective slave queues which will + * all wait for the user to signal sob 'cs_cmpl->sob_val'. + */ + for (i = 0 ; i < num_jobs ; i++) { + if (i == 0) { + queue_id = wait_queue_id; + rc = gaudi_collective_wait_create_job(hdev, ctx, cs, + HL_COLLECTIVE_MASTER, queue_id, wait_queue_id); + } else { + if (nic_idx < NIC_NUMBER_OF_ENGINES) { + if (gaudi->hw_cap_initialized & + BIT(HW_CAP_NIC_SHIFT + nic_idx)) + skip = false; + else + skip = true; + + queue_id = nic_queue; + nic_queue += 4; + nic_idx++; + + if (skip) + continue; + } else { + queue_id = collective_queue; + } + + rc = gaudi_collective_wait_create_job(hdev, ctx, cs, + HL_COLLECTIVE_SLAVE, queue_id, wait_queue_id); + } + + if (rc) + return rc; + } + + return rc; } static int gaudi_late_init(struct hl_device *hdev) @@ -860,6 +1325,12 @@ static int gaudi_late_init(struct hl_device *hdev) goto disable_pci_access; } + rc = gaudi_collective_init(hdev); + if (rc) { + dev_err(hdev->dev, "Failed to init collective\n"); + goto disable_pci_access; + } + return 0; disable_pci_access: @@ -2041,21 +2512,29 @@ static void gaudi_init_pci_dma_qmans(struct hl_device *hdev) static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, int qman_id, u64 qman_base_addr) { - u32 mtr_base_lo, mtr_base_hi; - u32 so_base_lo, so_base_hi; + u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; + u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; u32 q_off, dma_qm_offset; u32 dma_qm_err_cfg; dma_qm_offset = dma_id * DMA_QMAN_OFFSET; - mtr_base_lo = lower_32_bits(CFG_BASE + - mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); - mtr_base_hi = upper_32_bits(CFG_BASE + + mtr_base_en_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + mtr_base_en_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); - so_base_lo = lower_32_bits(CFG_BASE + + so_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); - so_base_hi = upper_32_bits(CFG_BASE + + so_base_en_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); + mtr_base_ws_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + mtr_base_ws_hi = upper_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + so_base_ws_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); + so_base_ws_hi = upper_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); q_off = dma_qm_offset + qman_id * 4; @@ -2113,10 +2592,22 @@ static void gaudi_init_hbm_dma_qman(struct hl_device *hdev, int dma_id, QMAN_INTERNAL_MAKE_TRUSTED); } - WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo); - WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi); - WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo); - WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi); + WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); + WREG32(mmDMA0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); + WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); + WREG32(mmDMA0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); + + /* Configure DMA5 CP_MSG_BASE 2/3 for sync stream collective */ + if (gaudi_dma_assignment[dma_id] == GAUDI_ENGINE_ID_DMA_5) { + WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, + mtr_base_ws_lo); + WREG32(mmDMA0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, + mtr_base_ws_hi); + WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, + so_base_ws_lo); + WREG32(mmDMA0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, + so_base_ws_hi); + } } static void gaudi_init_hbm_dma_qmans(struct hl_device *hdev) @@ -2279,22 +2770,33 @@ static void gaudi_init_mme_qmans(struct hl_device *hdev) static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, int qman_id, u64 qman_base_addr) { - u32 mtr_base_lo, mtr_base_hi; - u32 so_base_lo, so_base_hi; + u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; + u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; u32 q_off, tpc_id; u32 tpc_qm_err_cfg; - mtr_base_lo = lower_32_bits(CFG_BASE + - mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); - mtr_base_hi = upper_32_bits(CFG_BASE + + mtr_base_en_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + mtr_base_en_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); - so_base_lo = lower_32_bits(CFG_BASE + + so_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); - so_base_hi = upper_32_bits(CFG_BASE + + so_base_en_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); + mtr_base_ws_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + mtr_base_ws_hi = upper_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + so_base_ws_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); + so_base_ws_hi = upper_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); q_off = tpc_offset + qman_id * 4; + tpc_id = tpc_offset / + (mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0); + if (qman_id < 4) { WREG32(mmTPC0_QM_PQ_BASE_LO_0 + q_off, lower_32_bits(qman_base_addr)); @@ -2320,9 +2822,6 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, QMAN_LDMA_DST_OFFSET); /* Configure RAZWI IRQ */ - tpc_id = tpc_offset / - (mmTPC1_QM_GLBL_CFG0 - mmTPC0_QM_GLBL_CFG0); - tpc_qm_err_cfg = TPC_QMAN_GLBL_ERR_CFG_MSG_EN_MASK; if (hdev->stop_on_err) { tpc_qm_err_cfg |= @@ -2352,10 +2851,22 @@ static void gaudi_init_tpc_qman(struct hl_device *hdev, u32 tpc_offset, QMAN_INTERNAL_MAKE_TRUSTED); } - WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo); - WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi); - WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo); - WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi); + WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); + WREG32(mmTPC0_QM_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); + WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); + WREG32(mmTPC0_QM_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); + + /* Configure TPC7 CP_MSG_BASE 2/3 for sync stream collective */ + if (tpc_id == 6) { + WREG32(mmTPC0_QM_CP_MSG_BASE2_ADDR_LO_0 + q_off, + mtr_base_ws_lo); + WREG32(mmTPC0_QM_CP_MSG_BASE2_ADDR_HI_0 + q_off, + mtr_base_ws_hi); + WREG32(mmTPC0_QM_CP_MSG_BASE3_ADDR_LO_0 + q_off, + so_base_ws_lo); + WREG32(mmTPC0_QM_CP_MSG_BASE3_ADDR_HI_0 + q_off, + so_base_ws_hi); + } } static void gaudi_init_tpc_qmans(struct hl_device *hdev) @@ -2406,19 +2917,27 @@ static void gaudi_init_tpc_qmans(struct hl_device *hdev) static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, int qman_id, u64 qman_base_addr, int nic_id) { - u32 mtr_base_lo, mtr_base_hi; - u32 so_base_lo, so_base_hi; + u32 mtr_base_en_lo, mtr_base_en_hi, mtr_base_ws_lo, mtr_base_ws_hi; + u32 so_base_en_lo, so_base_en_hi, so_base_ws_lo, so_base_ws_hi; u32 q_off; u32 nic_qm_err_cfg; - mtr_base_lo = lower_32_bits(CFG_BASE + - mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); - mtr_base_hi = upper_32_bits(CFG_BASE + + mtr_base_en_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + mtr_base_en_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); - so_base_lo = lower_32_bits(CFG_BASE + + so_base_en_lo = lower_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); - so_base_hi = upper_32_bits(CFG_BASE + + so_base_en_hi = upper_32_bits(CFG_BASE + mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0); + mtr_base_ws_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + mtr_base_ws_hi = upper_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_MON_PAY_ADDRL_0); + so_base_ws_lo = lower_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); + so_base_ws_hi = upper_32_bits(CFG_BASE + + mmSYNC_MNGR_W_S_SYNC_MNGR_OBJS_SOB_OBJ_0); q_off = nic_offset + qman_id * 4; @@ -2429,14 +2948,23 @@ static void gaudi_init_nic_qman(struct hl_device *hdev, u32 nic_offset, WREG32(mmNIC0_QM0_PQ_PI_0 + q_off, 0); WREG32(mmNIC0_QM0_PQ_CI_0 + q_off, 0); - WREG32(mmNIC0_QM0_CP_LDMA_TSIZE_OFFSET_0 + q_off, 0x74); - WREG32(mmNIC0_QM0_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, 0x14); - WREG32(mmNIC0_QM0_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, 0x1C); + WREG32(mmNIC0_QM0_CP_LDMA_TSIZE_OFFSET_0 + q_off, + QMAN_LDMA_SIZE_OFFSET); + WREG32(mmNIC0_QM0_CP_LDMA_SRC_BASE_LO_OFFSET_0 + q_off, + QMAN_LDMA_SRC_OFFSET); + WREG32(mmNIC0_QM0_CP_LDMA_DST_BASE_LO_OFFSET_0 + q_off, + QMAN_LDMA_DST_OFFSET); + + WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_en_lo); + WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_en_hi); + WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_en_lo); + WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_en_hi); - WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_LO_0 + q_off, mtr_base_lo); - WREG32(mmNIC0_QM0_CP_MSG_BASE0_ADDR_HI_0 + q_off, mtr_base_hi); - WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_LO_0 + q_off, so_base_lo); - WREG32(mmNIC0_QM0_CP_MSG_BASE1_ADDR_HI_0 + q_off, so_base_hi); + /* Configure NIC CP_MSG_BASE 2/3 for sync stream collective */ + WREG32(mmNIC0_QM0_CP_MSG_BASE2_ADDR_LO_0 + q_off, mtr_base_ws_lo); + WREG32(mmNIC0_QM0_CP_MSG_BASE2_ADDR_HI_0 + q_off, mtr_base_ws_hi); + WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_LO_0 + q_off, so_base_ws_lo); + WREG32(mmNIC0_QM0_CP_MSG_BASE3_ADDR_HI_0 + q_off, so_base_ws_hi); if (qman_id == 0) { /* Configure RAZWI IRQ */ @@ -7011,11 +7539,152 @@ static enum hl_device_hw_state gaudi_get_hw_state(struct hl_device *hdev) return RREG32(mmHW_STATE); } +static int gaudi_internal_cb_pool_init(struct hl_device *hdev, + struct hl_ctx *ctx) +{ + struct gaudi_device *gaudi = hdev->asic_specific; + bool flush_pte; + u64 va, pa; + s64 off; + int min_alloc_order, rc, collective_cb_size; + + if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) + return 0; + + hdev->internal_cb_pool_virt_addr = + hdev->asic_funcs->asic_dma_alloc_coherent(hdev, + HOST_SPACE_INTERNAL_CB_SZ, + &hdev->internal_cb_pool_dma_addr, + GFP_KERNEL | __GFP_ZERO); + + if (!hdev->internal_cb_pool_virt_addr) + return -ENOMEM; + + collective_cb_size = sizeof(struct packet_msg_short) * 5 + + sizeof(struct packet_fence); + min_alloc_order = ilog2(collective_cb_size); + + hdev->internal_cb_pool = gen_pool_create(min_alloc_order, -1); + if (!hdev->internal_cb_pool) { + dev_err(hdev->dev, + "Failed to create internal CB pool\n"); + rc = -ENOMEM; + goto free_internal_cb_pool; + } + + rc = gen_pool_add(hdev->internal_cb_pool, + (uintptr_t) hdev->internal_cb_pool_virt_addr, + HOST_SPACE_INTERNAL_CB_SZ, -1); + if (rc) { + dev_err(hdev->dev, + "Failed to add memory to internal CB pool\n"); + rc = -EFAULT; + goto destroy_internal_cb_pool; + } + + hdev->internal_cb_va_base = VA_HOST_SPACE_INTERNAL_CB_START; + + mutex_lock(&ctx->mmu_lock); + + /* The mapping is done page by page since we can't assure allocated ptr + * is aligned to HOST_SPACE_INTERNAL_CB_SZ + */ + for (off = 0 ; off < HOST_SPACE_INTERNAL_CB_SZ ; off += PAGE_SIZE_4KB) { + va = VA_HOST_SPACE_INTERNAL_CB_START + off; + pa = hdev->internal_cb_pool_dma_addr + off; + flush_pte = (off + PAGE_SIZE_4KB) >= HOST_SPACE_INTERNAL_CB_SZ; + rc = hl_mmu_map(ctx, va, pa, PAGE_SIZE_4KB, flush_pte); + if (rc) { + dev_err(hdev->dev, + "Map failed for va 0x%llx to pa 0x%llx\n", + va, pa); + goto unmap; + } + } + + hdev->asic_funcs->mmu_invalidate_cache(hdev, false, VM_TYPE_USERPTR); + + mutex_unlock(&ctx->mmu_lock); + + return 0; + +unmap: + for (; off >= 0 ; off -= PAGE_SIZE_4KB) { + va = VA_HOST_SPACE_INTERNAL_CB_START + off; + flush_pte = (off - (s32) PAGE_SIZE_4KB) < 0; + if (hl_mmu_unmap(ctx, va, PAGE_SIZE_4KB, flush_pte)) + dev_warn_ratelimited(hdev->dev, + "failed to unmap va 0x%llx\n", va); + } + + hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR); + + mutex_unlock(&ctx->mmu_lock); + +destroy_internal_cb_pool: + gen_pool_destroy(hdev->internal_cb_pool); + +free_internal_cb_pool: + hdev->asic_funcs->asic_dma_free_coherent(hdev, + HOST_SPACE_INTERNAL_CB_SZ, + hdev->internal_cb_pool_virt_addr, + hdev->internal_cb_pool_dma_addr); + + return rc; +} + +static void gaudi_internal_cb_pool_fini(struct hl_device *hdev, + struct hl_ctx *ctx) +{ + struct gaudi_device *gaudi = hdev->asic_specific; + bool flush_pte = false; + u64 va, off; + + if (!(gaudi->hw_cap_initialized & HW_CAP_MMU)) + return; + + mutex_lock(&ctx->mmu_lock); + + for (off = 0 ; off < HOST_SPACE_INTERNAL_CB_SZ ; off += PAGE_SIZE_4KB) { + va = VA_HOST_SPACE_INTERNAL_CB_START + off; + + if (off + PAGE_SIZE_4KB >= HOST_SPACE_INTERNAL_CB_SZ) + flush_pte = true; + + if (hl_mmu_unmap(ctx, va, PAGE_SIZE_4KB, flush_pte)) + dev_warn_ratelimited(hdev->dev, + "failed to unmap va 0x%llx\n", va); + } + + hdev->asic_funcs->mmu_invalidate_cache(hdev, true, VM_TYPE_USERPTR); + + mutex_unlock(&ctx->mmu_lock); + + gen_pool_destroy(hdev->internal_cb_pool); + + hdev->asic_funcs->asic_dma_free_coherent(hdev, + HOST_SPACE_INTERNAL_CB_SZ, + hdev->internal_cb_pool_virt_addr, + hdev->internal_cb_pool_dma_addr); +} + static int gaudi_ctx_init(struct hl_ctx *ctx) { gaudi_mmu_prepare(ctx->hdev, ctx->asid); + return gaudi_internal_cb_pool_init(ctx->hdev, ctx); +} - return 0; +void gaudi_ctx_fini(struct hl_ctx *ctx) +{ + struct hl_device *hdev = ctx->hdev; + + /* Gaudi will NEVER support more then a single compute context. + * Therefore, don't clear anything unless it is the compute context + */ + if (hdev->compute_ctx != ctx) + return; + + gaudi_internal_cb_pool_fini(ctx->hdev, ctx); } static u32 gaudi_get_queue_id_for_cq(struct hl_device *hdev, u32 cq_idx) @@ -7037,7 +7706,7 @@ static u32 gaudi_get_wait_cb_size(struct hl_device *hdev) } static u32 gaudi_gen_signal_cb(struct hl_device *hdev, void *data, u16 sob_id, - u32 size) + u32 size) { struct hl_cb *cb = (struct hl_cb *) data; struct packet_msg_short *pkt; @@ -7156,7 +7825,7 @@ static u32 gaudi_add_fence_pkt(struct packet_fence *pkt) static int gaudi_get_fence_addr(struct hl_device *hdev, u32 queue_id, u64 *addr) { - u32 offset; + u32 offset, nic_index; switch (queue_id) { case GAUDI_QUEUE_ID_DMA_0_0: @@ -7195,6 +7864,78 @@ static int gaudi_get_fence_addr(struct hl_device *hdev, u32 queue_id, u64 *addr) case GAUDI_QUEUE_ID_DMA_5_3: offset = mmDMA5_QM_CP_FENCE2_RDATA_3; break; + case GAUDI_QUEUE_ID_TPC_7_0: + offset = mmTPC7_QM_CP_FENCE2_RDATA_0; + break; + case GAUDI_QUEUE_ID_TPC_7_1: + offset = mmTPC7_QM_CP_FENCE2_RDATA_1; + break; + case GAUDI_QUEUE_ID_TPC_7_2: + offset = mmTPC7_QM_CP_FENCE2_RDATA_2; + break; + case GAUDI_QUEUE_ID_TPC_7_3: + offset = mmTPC7_QM_CP_FENCE2_RDATA_3; + break; + case GAUDI_QUEUE_ID_NIC_0_0: + case GAUDI_QUEUE_ID_NIC_1_0: + case GAUDI_QUEUE_ID_NIC_2_0: + case GAUDI_QUEUE_ID_NIC_3_0: + case GAUDI_QUEUE_ID_NIC_4_0: + case GAUDI_QUEUE_ID_NIC_5_0: + case GAUDI_QUEUE_ID_NIC_6_0: + case GAUDI_QUEUE_ID_NIC_7_0: + case GAUDI_QUEUE_ID_NIC_8_0: + case GAUDI_QUEUE_ID_NIC_9_0: + nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_0) >> 2; + offset = mmNIC0_QM0_CP_FENCE2_RDATA_0 + + (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + + (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; + break; + case GAUDI_QUEUE_ID_NIC_0_1: + case GAUDI_QUEUE_ID_NIC_1_1: + case GAUDI_QUEUE_ID_NIC_2_1: + case GAUDI_QUEUE_ID_NIC_3_1: + case GAUDI_QUEUE_ID_NIC_4_1: + case GAUDI_QUEUE_ID_NIC_5_1: + case GAUDI_QUEUE_ID_NIC_6_1: + case GAUDI_QUEUE_ID_NIC_7_1: + case GAUDI_QUEUE_ID_NIC_8_1: + case GAUDI_QUEUE_ID_NIC_9_1: + nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_1) >> 2; + offset = mmNIC0_QM0_CP_FENCE2_RDATA_1 + + (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + + (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; + break; + case GAUDI_QUEUE_ID_NIC_0_2: + case GAUDI_QUEUE_ID_NIC_1_2: + case GAUDI_QUEUE_ID_NIC_2_2: + case GAUDI_QUEUE_ID_NIC_3_2: + case GAUDI_QUEUE_ID_NIC_4_2: + case GAUDI_QUEUE_ID_NIC_5_2: + case GAUDI_QUEUE_ID_NIC_6_2: + case GAUDI_QUEUE_ID_NIC_7_2: + case GAUDI_QUEUE_ID_NIC_8_2: + case GAUDI_QUEUE_ID_NIC_9_2: + nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_2) >> 2; + offset = mmNIC0_QM0_CP_FENCE2_RDATA_2 + + (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + + (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; + break; + case GAUDI_QUEUE_ID_NIC_0_3: + case GAUDI_QUEUE_ID_NIC_1_3: + case GAUDI_QUEUE_ID_NIC_2_3: + case GAUDI_QUEUE_ID_NIC_3_3: + case GAUDI_QUEUE_ID_NIC_4_3: + case GAUDI_QUEUE_ID_NIC_5_3: + case GAUDI_QUEUE_ID_NIC_6_3: + case GAUDI_QUEUE_ID_NIC_7_3: + case GAUDI_QUEUE_ID_NIC_8_3: + case GAUDI_QUEUE_ID_NIC_9_3: + nic_index = (queue_id - GAUDI_QUEUE_ID_NIC_0_3) >> 2; + offset = mmNIC0_QM0_CP_FENCE2_RDATA_3 + + (nic_index >> 1) * NIC_MACRO_QMAN_OFFSET + + (nic_index & 0x1) * NIC_ENGINE_QMAN_OFFSET; + break; default: return -EINVAL; } @@ -7360,6 +8101,7 @@ static const struct hl_asic_funcs gaudi_funcs = { .wreg = hl_wreg, .halt_coresight = gaudi_halt_coresight, .ctx_init = gaudi_ctx_init, + .ctx_fini = gaudi_ctx_fini, .get_clk_rate = gaudi_get_clk_rate, .get_queue_id_for_cq = gaudi_get_queue_id_for_cq, .read_device_fw_version = gaudi_read_device_fw_version, diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index 16871d9ff88b..297a96dbf4e5 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -23,6 +23,10 @@ NUMBER_OF_CPU_HW_QUEUES + \ NUMBER_OF_INT_HW_QUEUES) +/* 10 NIC QMANs, DMA5 QMAN, TPC7 QMAN */ +#define NUMBER_OF_COLLECTIVE_QUEUES 12 +#define NUMBER_OF_SOBS_IN_GRP 11 + /* * Number of MSI interrupts IDS: * Each completion queue has 1 ID @@ -149,10 +153,14 @@ /* Virtual address space */ #define VA_HOST_SPACE_START 0x1000000000000ull /* 256TB */ -#define VA_HOST_SPACE_END 0x3FF8000000000ull /* 1PB - 1TB */ +#define VA_HOST_SPACE_END 0x3FF7FFFE00000ull /* 1PB - 1TB */ #define VA_HOST_SPACE_SIZE (VA_HOST_SPACE_END - \ VA_HOST_SPACE_START) /* 767TB */ +#define VA_HOST_SPACE_INTERNAL_CB_START 0x3FF7FFFE00000ull /* 1PB - 1TB - 2MB */ +#define VA_HOST_SPACE_INTERNAL_CB_END 0x3FF8000000000ull /* 1PB - 1TB */ +#define HOST_SPACE_INTERNAL_CB_SZ SZ_2M + #define HW_CAP_PLL BIT(0) #define HW_CAP_HBM BIT(1) #define HW_CAP_MMU BIT(2) @@ -240,6 +248,34 @@ enum gaudi_nic_mask { GAUDI_NIC_MASK_ALL = 0x3FF }; +/* + * struct gaudi_hw_sob_group - H/W SOB group info. + * @hdev: habanalabs device structure. + * @kref: refcount of this SOB group. group will reset once refcount is zero. + * @base_sob_id: base sob id of this SOB group. + */ +struct gaudi_hw_sob_group { + struct hl_device *hdev; + struct kref kref; + u32 base_sob_id; +}; + +#define NUM_SOB_GROUPS (HL_RSVD_SOBS * QMAN_STREAMS) +/** + * struct gaudi_collective_properties - + * holds all SOB groups and queues info reserved for the collective + * @hw_sob_group: H/W SOB groups. + * @next_sob_group_val: the next value to use for the currently used SOB group. + * @curr_sob_group_idx: the index of the currently used SOB group. + * @mstr_sob_mask: pre-defined masks for collective master monitors + */ +struct gaudi_collective_properties { + struct gaudi_hw_sob_group hw_sob_group[NUM_SOB_GROUPS]; + u16 next_sob_group_val[QMAN_STREAMS]; + u8 curr_sob_group_idx[QMAN_STREAMS]; + u8 mstr_sob_mask[HL_COLLECTIVE_RSVD_MSTR_MONS]; +}; + /** * struct gaudi_internal_qman_info - Internal QMAN information. * @pq_kernel_addr: Kernel address of the PQ memory area in the host. @@ -285,6 +321,8 @@ struct gaudi_device { struct gaudi_internal_qman_info internal_qmans[GAUDI_QUEUE_ID_SIZE]; + struct gaudi_collective_properties collective_props; + u64 hbm_bar_cur_addr; u64 max_freq_value; diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 9332580b038d..374881ff551d 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5343,6 +5343,11 @@ int goya_collective_wait_create_jobs(struct hl_device *hdev, return -EINVAL; } +static void goya_ctx_fini(struct hl_ctx *ctx) +{ + +} + static const struct hl_asic_funcs goya_funcs = { .early_init = goya_early_init, .early_fini = goya_early_fini, @@ -5404,6 +5409,7 @@ static const struct hl_asic_funcs goya_funcs = { .wreg = hl_wreg, .halt_coresight = goya_halt_coresight, .ctx_init = goya_ctx_init, + .ctx_fini = goya_ctx_fini, .get_clk_rate = goya_get_clk_rate, .get_queue_id_for_cq = goya_get_queue_id_for_cq, .read_device_fw_version = goya_read_device_fw_version, diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 4661a74f0425..0185311b679b 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -18,8 +18,18 @@ #define GOYA_KMD_SRAM_RESERVED_SIZE_FROM_START 0x8000 /* 32KB */ #define GAUDI_DRIVER_SRAM_RESERVED_SIZE_FROM_START 0x80 /* 128 bytes */ -#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 32 -#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 16 +/* + * 128 SOBs reserved for collective wait + * 16 SOBs reserved for sync stream + */ +#define GAUDI_FIRST_AVAILABLE_W_S_SYNC_OBJECT 144 + +/* + * 64 monitors reserved for collective wait + * 8 monitors reserved for sync stream + */ +#define GAUDI_FIRST_AVAILABLE_W_S_MONITOR 72 + /* * Goya queue Numbering * -- cgit v1.2.3 From e753643d516c7c38f69f3d73169bb00cd70a60b9 Mon Sep 17 00:00:00 2001 From: farah kassabri Date: Mon, 12 Oct 2020 14:30:26 +0300 Subject: habanalabs: fix cs counters structure Fix cs counters structure in uapi to be one flat structure instead of two instances of the same other structure. use atomic read/increment for context counters so we could use one structure for both aggregated and context counters. Signed-off-by: farah kassabri Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 18 +++--- drivers/misc/habanalabs/common/habanalabs.h | 73 +++++++++++----------- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 35 +++++++---- drivers/misc/habanalabs/common/hw_queue.c | 5 +- drivers/misc/habanalabs/gaudi/gaudi.c | 4 +- include/uapi/misc/habanalabs.h | 35 ++++++----- 6 files changed, 95 insertions(+), 75 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 26822cfd1491..e123101b74d6 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -462,7 +462,7 @@ static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, if (other && !completion_done(&other->completion)) { dev_dbg_ratelimited(hdev->dev, "Rejecting CS because of too many in-flights CS\n"); - ctx->cs_counters.max_cs_in_flight_drop_cnt++; + atomic64_inc(&ctx->cs_counters.max_cs_in_flight_drop_cnt); atomic64_inc(&cntr->max_cs_in_flight_drop_cnt); rc = -EAGAIN; goto free_fence; @@ -720,7 +720,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, rc = validate_queue_index(hdev, chunk, &queue_type, &is_kernel_allocated_cb); if (rc) { - hpriv->ctx->cs_counters.parsing_drop_cnt++; + atomic64_inc(&hpriv->ctx->cs_counters.parsing_drop_cnt); atomic64_inc(&cntr->parsing_drop_cnt); goto free_cs_object; } @@ -728,7 +728,8 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, if (is_kernel_allocated_cb) { cb = get_cb_from_cs_chunk(hdev, &hpriv->cb_mgr, chunk); if (!cb) { - hpriv->ctx->cs_counters.parsing_drop_cnt++; + atomic64_inc( + &hpriv->ctx->cs_counters.parsing_drop_cnt); atomic64_inc(&cntr->parsing_drop_cnt); rc = -EINVAL; goto free_cs_object; @@ -743,7 +744,8 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, job = hl_cs_allocate_job(hdev, queue_type, is_kernel_allocated_cb); if (!job) { - hpriv->ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc( + &hpriv->ctx->cs_counters.out_of_mem_drop_cnt); atomic64_inc(&cntr->out_of_mem_drop_cnt); dev_err(hdev->dev, "Failed to allocate a new job\n"); rc = -ENOMEM; @@ -777,7 +779,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, rc = cs_parser(hpriv, job); if (rc) { - hpriv->ctx->cs_counters.parsing_drop_cnt++; + atomic64_inc(&hpriv->ctx->cs_counters.parsing_drop_cnt); atomic64_inc(&cntr->parsing_drop_cnt); dev_err(hdev->dev, "Failed to parse JOB %d.%llu.%d, err %d, rejecting the CS\n", @@ -787,7 +789,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, } if (int_queues_only) { - hpriv->ctx->cs_counters.parsing_drop_cnt++; + atomic64_inc(&hpriv->ctx->cs_counters.parsing_drop_cnt); atomic64_inc(&cntr->parsing_drop_cnt); dev_err(hdev->dev, "Reject CS %d.%llu because only internal queues jobs are present\n", @@ -880,7 +882,7 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, job = hl_cs_allocate_job(hdev, q_type, true); if (!job) { - ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); atomic64_inc(&cntr->out_of_mem_drop_cnt); dev_err(hdev->dev, "Failed to allocate a new job\n"); return -ENOMEM; @@ -894,7 +896,7 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, cb = hl_cb_kernel_create(hdev, cb_size, q_type == QUEUE_TYPE_HW && hdev->mmu_enable); if (!cb) { - ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); atomic64_inc(&cntr->out_of_mem_drop_cnt); kfree(job); return -EFAULT; diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 7f1522b101b4..b1f20f225ff9 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -936,6 +936,22 @@ struct hl_va_range { u64 end_addr; }; +/** + * struct hl_cs_counters_atomic - command submission counters + * @out_of_mem_drop_cnt: dropped due to memory allocation issue + * @parsing_drop_cnt: dropped due to error in packet parsing + * @queue_full_drop_cnt: dropped due to queue full + * @device_in_reset_drop_cnt: dropped due to device in reset + * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight + */ +struct hl_cs_counters_atomic { + atomic64_t out_of_mem_drop_cnt; + atomic64_t parsing_drop_cnt; + atomic64_t queue_full_drop_cnt; + atomic64_t device_in_reset_drop_cnt; + atomic64_t max_cs_in_flight_drop_cnt; +}; + /** * struct hl_ctx - user/kernel context. * @mem_hash: holds mapping from virtual address to virtual memory area @@ -954,6 +970,7 @@ struct hl_va_range { * @mmu_lock: protects the MMU page tables. Any change to the PGT, modifying the * MMU hash or walking the PGT requires talking this lock. * @debugfs_list: node in debugfs list of contexts. + * @cs_counters: context command submission counters. * @cb_va_pool: device VA pool for command buffers which are mapped to the * device's MMU. * @cs_sequence: sequence number for CS. Value is assigned to a CS and passed @@ -976,26 +993,26 @@ struct hl_va_range { struct hl_ctx { DECLARE_HASHTABLE(mem_hash, MEM_HASH_TABLE_BITS); DECLARE_HASHTABLE(mmu_shadow_hash, MMU_HASH_TABLE_BITS); - struct hl_fpriv *hpriv; - struct hl_device *hdev; - struct kref refcount; - struct hl_fence **cs_pending; - struct hl_va_range *host_va_range; - struct hl_va_range *host_huge_va_range; - struct hl_va_range *dram_va_range; - struct mutex mem_hash_lock; - struct mutex mmu_lock; - struct list_head debugfs_list; - struct hl_cs_counters cs_counters; - struct gen_pool *cb_va_pool; - u64 cs_sequence; - u64 *dram_default_hops; - spinlock_t cs_lock; - atomic64_t dram_phys_mem; - atomic_t thread_ctx_switch_token; - u32 thread_ctx_switch_wait_token; - u32 asid; - u32 handle; + struct hl_fpriv *hpriv; + struct hl_device *hdev; + struct kref refcount; + struct hl_fence **cs_pending; + struct hl_va_range *host_va_range; + struct hl_va_range *host_huge_va_range; + struct hl_va_range *dram_va_range; + struct mutex mem_hash_lock; + struct mutex mmu_lock; + struct list_head debugfs_list; + struct hl_cs_counters_atomic cs_counters; + struct gen_pool *cb_va_pool; + u64 cs_sequence; + u64 *dram_default_hops; + spinlock_t cs_lock; + atomic64_t dram_phys_mem; + atomic_t thread_ctx_switch_token; + u32 thread_ctx_switch_wait_token; + u32 asid; + u32 handle; }; /** @@ -1164,22 +1181,6 @@ struct hl_cs_parser { u8 contains_dma_pkt; }; -/** - * struct hl_info_cs_counters - command submission counters - * @out_of_mem_drop_cnt: dropped due to memory allocation issue - * @parsing_drop_cnt: dropped due to error in packet parsing - * @queue_full_drop_cnt: dropped due to queue full - * @device_in_reset_drop_cnt: dropped due to device in reset - * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight - */ -struct hl_cs_counters_atomic { - atomic64_t out_of_mem_drop_cnt; - atomic64_t parsing_drop_cnt; - atomic64_t queue_full_drop_cnt; - atomic64_t device_in_reset_drop_cnt; - atomic64_t max_cs_in_flight_drop_cnt; -}; - /* * MEMORY STRUCTURE */ diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 350a768309bd..1d8bea626e78 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -315,7 +315,7 @@ static int clk_throttle_info(struct hl_fpriv *hpriv, struct hl_info_args *args) static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args) { void __user *out = (void __user *) (uintptr_t) args->return_pointer; - struct hl_info_cs_counters cs_counters = { {0} }; + struct hl_info_cs_counters cs_counters = {0}; struct hl_device *hdev = hpriv->hdev; struct hl_cs_counters_atomic *cntr; u32 max_size = args->return_size; @@ -325,23 +325,34 @@ static int cs_counters_info(struct hl_fpriv *hpriv, struct hl_info_args *args) if ((!max_size) || (!out)) return -EINVAL; - memcpy(&cs_counters.cs_counters, &hdev->aggregated_cs_counters, - sizeof(struct hl_cs_counters)); - - cs_counters.cs_counters.out_of_mem_drop_cnt = + cs_counters.total_out_of_mem_drop_cnt = atomic64_read(&cntr->out_of_mem_drop_cnt); - cs_counters.cs_counters.parsing_drop_cnt = + cs_counters.total_parsing_drop_cnt = atomic64_read(&cntr->parsing_drop_cnt); - cs_counters.cs_counters.queue_full_drop_cnt = + cs_counters.total_queue_full_drop_cnt = atomic64_read(&cntr->queue_full_drop_cnt); - cs_counters.cs_counters.device_in_reset_drop_cnt = + cs_counters.total_device_in_reset_drop_cnt = atomic64_read(&cntr->device_in_reset_drop_cnt); - cs_counters.cs_counters.max_cs_in_flight_drop_cnt = + cs_counters.total_max_cs_in_flight_drop_cnt = atomic64_read(&cntr->max_cs_in_flight_drop_cnt); - if (hpriv->ctx) - memcpy(&cs_counters.ctx_cs_counters, &hpriv->ctx->cs_counters, - sizeof(struct hl_cs_counters)); + if (hpriv->ctx) { + cs_counters.ctx_out_of_mem_drop_cnt = + atomic64_read( + &hpriv->ctx->cs_counters.out_of_mem_drop_cnt); + cs_counters.ctx_parsing_drop_cnt = + atomic64_read( + &hpriv->ctx->cs_counters.parsing_drop_cnt); + cs_counters.ctx_queue_full_drop_cnt = + atomic64_read( + &hpriv->ctx->cs_counters.queue_full_drop_cnt); + cs_counters.ctx_device_in_reset_drop_cnt = + atomic64_read( + &hpriv->ctx->cs_counters.device_in_reset_drop_cnt); + cs_counters.ctx_max_cs_in_flight_drop_cnt = + atomic64_read( + &hpriv->ctx->cs_counters.max_cs_in_flight_drop_cnt); + } return copy_to_user(out, &cs_counters, min((size_t) max_size, sizeof(cs_counters))) ? -EFAULT : 0; diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index d9448375beac..44155a6e557f 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -523,7 +523,7 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) hdev->asic_funcs->hw_queues_lock(hdev); if (hl_device_disabled_or_in_reset(hdev)) { - ctx->cs_counters.device_in_reset_drop_cnt++; + atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt); atomic64_inc(&cntr->device_in_reset_drop_cnt); dev_err(hdev->dev, "device is disabled or in reset, CS rejected!\n"); @@ -557,7 +557,8 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) } if (rc) { - ctx->cs_counters.queue_full_drop_cnt++; + atomic64_inc( + &ctx->cs_counters.queue_full_drop_cnt); atomic64_inc(&cntr->queue_full_drop_cnt); goto unroll_cq_resv; } diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index ab8c9463932f..e640c9fcc932 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -1137,7 +1137,7 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, hw_queue_prop = &hdev->asic_prop.hw_queues_props[queue_id]; job = hl_cs_allocate_job(hdev, hw_queue_prop->type, true); if (!job) { - ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); atomic64_inc(&cntr->out_of_mem_drop_cnt); dev_err(hdev->dev, "Failed to allocate a new job\n"); return -ENOMEM; @@ -1147,7 +1147,7 @@ static int gaudi_collective_wait_create_job(struct hl_device *hdev, cb = hl_cb_kernel_create(hdev, cb_size, hdev->mmu_enable && !patched_cb); if (!cb) { - ctx->cs_counters.out_of_mem_drop_cnt++; + atomic64_inc(&ctx->cs_counters.out_of_mem_drop_cnt); atomic64_inc(&cntr->out_of_mem_drop_cnt); kfree(job); return -EFAULT; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 0185311b679b..61f8f9144b54 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -407,23 +407,28 @@ struct hl_info_sync_manager { /** * struct hl_info_cs_counters - command submission counters - * @out_of_mem_drop_cnt: dropped due to memory allocation issue - * @parsing_drop_cnt: dropped due to error in packet parsing - * @queue_full_drop_cnt: dropped due to queue full - * @device_in_reset_drop_cnt: dropped due to device in reset - * @max_cs_in_flight_drop_cnt: dropped due to maximum CS in-flight + * @total_out_of_mem_drop_cnt: total dropped due to memory allocation issue + * @ctx_out_of_mem_drop_cnt: context dropped due to memory allocation issue + * @total_parsing_drop_cnt: total dropped due to error in packet parsing + * @ctx_parsing_drop_cnt: context dropped due to error in packet parsing + * @total_queue_full_drop_cnt: total dropped due to queue full + * @ctx_queue_full_drop_cnt: context dropped due to queue full + * @total_device_in_reset_drop_cnt: total dropped due to device in reset + * @ctx_device_in_reset_drop_cnt: context dropped due to device in reset + * @total_max_cs_in_flight_drop_cnt: total dropped due to maximum CS in-flight + * @ctx_max_cs_in_flight_drop_cnt: context dropped due to maximum CS in-flight */ -struct hl_cs_counters { - __u64 out_of_mem_drop_cnt; - __u64 parsing_drop_cnt; - __u64 queue_full_drop_cnt; - __u64 device_in_reset_drop_cnt; - __u64 max_cs_in_flight_drop_cnt; -}; - struct hl_info_cs_counters { - struct hl_cs_counters cs_counters; - struct hl_cs_counters ctx_cs_counters; + __u64 total_out_of_mem_drop_cnt; + __u64 ctx_out_of_mem_drop_cnt; + __u64 total_parsing_drop_cnt; + __u64 ctx_parsing_drop_cnt; + __u64 total_queue_full_drop_cnt; + __u64 ctx_queue_full_drop_cnt; + __u64 total_device_in_reset_drop_cnt; + __u64 ctx_device_in_reset_drop_cnt; + __u64 total_max_cs_in_flight_drop_cnt; + __u64 ctx_max_cs_in_flight_drop_cnt; }; enum gaudi_dcores { -- cgit v1.2.3 From 66a76401c50b2638fd95dd31f365fd64be307d6a Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Mon, 5 Oct 2020 14:40:10 +0300 Subject: habanalabs: add 'needs reset' state in driver The new state indicates that device should be reset in order to re-gain funcionality. This unique state can occur if reset_on_lockup is disabled and an actual lockup has occurred. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/command_buffer.c | 5 ++-- .../misc/habanalabs/common/command_submission.c | 7 +++-- drivers/misc/habanalabs/common/debugfs.c | 6 ++-- drivers/misc/habanalabs/common/device.c | 33 ++++++++++++++++------ drivers/misc/habanalabs/common/habanalabs.h | 14 +++++++-- drivers/misc/habanalabs/common/habanalabs_drv.c | 17 ++++++++--- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 12 +++++--- drivers/misc/habanalabs/common/hw_queue.c | 6 ++-- drivers/misc/habanalabs/common/hwmon.c | 4 +-- drivers/misc/habanalabs/common/memory.c | 5 ++-- drivers/misc/habanalabs/common/sysfs.c | 8 ++++-- drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c | 8 +++--- drivers/misc/habanalabs/goya/goya_hwmgr.c | 28 +++++++++--------- include/uapi/misc/habanalabs.h | 3 +- 14 files changed, 101 insertions(+), 55 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c index 075679626c7b..03ffcead1855 100644 --- a/drivers/misc/habanalabs/common/command_buffer.c +++ b/drivers/misc/habanalabs/common/command_buffer.c @@ -379,13 +379,14 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) { union hl_cb_args *args = data; struct hl_device *hdev = hpriv->hdev; + enum hl_device_status status; u64 handle = 0; int rc; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { dev_warn_ratelimited(hdev->dev, "Device is %s. Can't execute CB IOCTL\n", - atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); + hdev->status[status]); return -EBUSY; } diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 91bdd6d8b020..20b34fb054ee 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -427,6 +427,8 @@ static void cs_timedout(struct work_struct *work) if (hdev->reset_on_lockup) hl_device_reset(hdev, false, false); + else + hdev->needs_reset = true; } static int allocate_cs(struct hl_device *hdev, struct hl_ctx *ctx, @@ -689,12 +691,13 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; u32 cs_type_flags, num_chunks; + enum hl_device_status status; enum hl_cs_type cs_type; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { dev_warn_ratelimited(hdev->dev, "Device is %s. Can't submit new CS\n", - atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); + hdev->status[status]); return -EBUSY; } diff --git a/drivers/misc/habanalabs/common/debugfs.c b/drivers/misc/habanalabs/common/debugfs.c index b44193ec3d12..104b9686e57b 100644 --- a/drivers/misc/habanalabs/common/debugfs.c +++ b/drivers/misc/habanalabs/common/debugfs.c @@ -24,7 +24,7 @@ static int hl_debugfs_i2c_read(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr, struct cpucp_packet pkt; int rc; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -EBUSY; memset(&pkt, 0, sizeof(pkt)); @@ -50,7 +50,7 @@ static int hl_debugfs_i2c_write(struct hl_device *hdev, u8 i2c_bus, u8 i2c_addr, struct cpucp_packet pkt; int rc; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -EBUSY; memset(&pkt, 0, sizeof(pkt)); @@ -76,7 +76,7 @@ static void hl_debugfs_led_set(struct hl_device *hdev, u8 led, u8 state) struct cpucp_packet pkt; int rc; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return; memset(&pkt, 0, sizeof(pkt)); diff --git a/drivers/misc/habanalabs/common/device.c b/drivers/misc/habanalabs/common/device.c index 3b82020648c7..59308a612b36 100644 --- a/drivers/misc/habanalabs/common/device.c +++ b/drivers/misc/habanalabs/common/device.c @@ -15,14 +15,6 @@ #define HL_PLDM_PENDING_RESET_PER_SEC (HL_PENDING_RESET_PER_SEC * 10) -bool hl_device_disabled_or_in_reset(struct hl_device *hdev) -{ - if ((hdev->disabled) || (atomic_read(&hdev->in_reset))) - return true; - else - return false; -} - enum hl_device_status hl_device_status(struct hl_device *hdev) { enum hl_device_status status; @@ -31,12 +23,34 @@ enum hl_device_status hl_device_status(struct hl_device *hdev) status = HL_DEVICE_STATUS_MALFUNCTION; else if (atomic_read(&hdev->in_reset)) status = HL_DEVICE_STATUS_IN_RESET; + else if (hdev->needs_reset) + status = HL_DEVICE_STATUS_NEEDS_RESET; else status = HL_DEVICE_STATUS_OPERATIONAL; return status; } +bool hl_device_operational(struct hl_device *hdev, + enum hl_device_status *status) +{ + enum hl_device_status current_status; + + current_status = hl_device_status(hdev); + if (status) + *status = current_status; + + switch (current_status) { + case HL_DEVICE_STATUS_IN_RESET: + case HL_DEVICE_STATUS_MALFUNCTION: + case HL_DEVICE_STATUS_NEEDS_RESET: + return false; + case HL_DEVICE_STATUS_OPERATIONAL: + default: + return true; + } +} + static void hpriv_release(struct kref *ref) { struct hl_fpriv *hpriv; @@ -411,7 +425,7 @@ static void hl_device_heartbeat(struct work_struct *work) struct hl_device *hdev = container_of(work, struct hl_device, work_heartbeat.work); - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) goto reschedule; if (!hdev->asic_funcs->send_heartbeat(hdev)) @@ -1091,6 +1105,7 @@ again: } atomic_set(&hdev->in_reset, 0); + hdev->needs_reset = false; if (hard_reset) hdev->hard_reset_cnt++; diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 81ff340b6a5b..9c7594d0ca07 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -1432,6 +1432,10 @@ struct hl_dbg_device_entry { * DEVICES */ +#define HL_STR_MAX 32 + +#define HL_DEV_STS_MAX (HL_DEVICE_STATUS_NEEDS_RESET + 1) + /* Theoretical limit only. A single host can only contain up to 4 or 8 PCIe * x16 cards. In extreme cases, there are hosts that can accommodate 16 cards. */ @@ -1706,6 +1710,7 @@ struct hl_mmu_funcs { * @hwmon_dev: H/W monitor device. * @pm_mng_profile: current power management profile. * @hl_chip_info: ASIC's sensors information. + * @device_status_description: device status description. * @hl_debugfs: device's debugfs manager. * @cb_pool: list of preallocated CBs. * @cb_pool_lock: protects the CB pool. @@ -1774,6 +1779,8 @@ struct hl_mmu_funcs { * @supports_coresight: is CoreSight supported. * @supports_soft_reset: is soft reset supported. * @supports_cb_mapping: is mapping a CB to the device's MMU supported. + * @needs_reset: true if reset_on_lockup is false and device should be reset + * due to lockup. */ struct hl_device { struct pci_dev *pdev; @@ -1786,7 +1793,8 @@ struct hl_device { struct device *dev_ctrl; struct delayed_work work_freq; struct delayed_work work_heartbeat; - char asic_name[32]; + char asic_name[HL_STR_MAX]; + char status[HL_DEV_STS_MAX][HL_STR_MAX]; enum hl_asic_type asic_type; struct hl_cq *completion_queue; struct workqueue_struct **cq_wq; @@ -1876,6 +1884,7 @@ struct hl_device { u8 supports_coresight; u8 supports_soft_reset; u8 supports_cb_mapping; + u8 needs_reset; /* Parameters for bring-up */ u64 nic_ports_mask; @@ -1978,7 +1987,8 @@ static inline bool hl_mem_area_crosses_range(u64 address, u32 size, int hl_device_open(struct inode *inode, struct file *filp); int hl_device_open_ctrl(struct inode *inode, struct file *filp); -bool hl_device_disabled_or_in_reset(struct hl_device *hdev); +bool hl_device_operational(struct hl_device *hdev, + enum hl_device_status *status); enum hl_device_status hl_device_status(struct hl_device *hdev); int hl_device_set_debug_mode(struct hl_device *hdev, bool enable); int create_hdev(struct hl_device **dev, struct pci_dev *pdev, diff --git a/drivers/misc/habanalabs/common/habanalabs_drv.c b/drivers/misc/habanalabs/common/habanalabs_drv.c index 20458bd82c5a..aac798f3296e 100644 --- a/drivers/misc/habanalabs/common/habanalabs_drv.c +++ b/drivers/misc/habanalabs/common/habanalabs_drv.c @@ -92,6 +92,7 @@ static enum hl_asic_type get_asic_type(u16 device) */ int hl_device_open(struct inode *inode, struct file *filp) { + enum hl_device_status status; struct hl_device *hdev; struct hl_fpriv *hpriv; int rc; @@ -124,10 +125,10 @@ int hl_device_open(struct inode *inode, struct file *filp) mutex_lock(&hdev->fpriv_list_lock); - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { dev_err_ratelimited(hdev->dev, - "Can't open %s because it is disabled or in reset\n", - dev_name(hdev->dev)); + "Can't open %s because it is %s\n", + dev_name(hdev->dev), hdev->status[status]); rc = -EPERM; goto out_err; } @@ -204,7 +205,7 @@ int hl_device_open_ctrl(struct inode *inode, struct file *filp) mutex_lock(&hdev->fpriv_list_lock); - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { dev_err_ratelimited(hdev->dev_ctrl, "Can't open %s because it is disabled or in reset\n", dev_name(hdev->dev_ctrl)); @@ -287,6 +288,14 @@ int create_hdev(struct hl_device **dev, struct pci_dev *pdev, hdev->asic_type = asic_type; } + /* Assign status description string */ + strncpy(hdev->status[HL_DEVICE_STATUS_MALFUNCTION], + "disabled", HL_STR_MAX); + strncpy(hdev->status[HL_DEVICE_STATUS_IN_RESET], + "in reset", HL_STR_MAX); + strncpy(hdev->status[HL_DEVICE_STATUS_NEEDS_RESET], + "needs reset", HL_STR_MAX); + hdev->major = hl_major; hdev->reset_on_lockup = reset_on_lockup; hdev->memory_scrub = memory_scrub; diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index 1d8bea626e78..0729cd43f297 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -406,8 +406,10 @@ static int total_energy_consumption_info(struct hl_fpriv *hpriv, static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { + enum hl_device_status status; struct hl_info_args *args = data; struct hl_device *hdev = hpriv->hdev; + int rc; /* @@ -428,10 +430,10 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, break; } - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { dev_warn_ratelimited(dev, "Device is %s. Can't execute INFO IOCTL\n", - atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); + hdev->status[status]); return -EBUSY; } @@ -501,12 +503,14 @@ static int hl_debug_ioctl(struct hl_fpriv *hpriv, void *data) { struct hl_debug_args *args = data; struct hl_device *hdev = hpriv->hdev; + enum hl_device_status status; + int rc = 0; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { dev_warn_ratelimited(hdev->dev, "Device is %s. Can't execute DEBUG IOCTL\n", - atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); + hdev->status[status]); return -EBUSY; } diff --git a/drivers/misc/habanalabs/common/hw_queue.c b/drivers/misc/habanalabs/common/hw_queue.c index e808e668a007..f9550fcf5500 100644 --- a/drivers/misc/habanalabs/common/hw_queue.c +++ b/drivers/misc/habanalabs/common/hw_queue.c @@ -515,6 +515,7 @@ static void init_signal_wait_cs(struct hl_cs *cs) */ int hl_hw_queue_schedule_cs(struct hl_cs *cs) { + enum hl_device_status status; struct hl_cs_counters_atomic *cntr; struct hl_ctx *ctx = cs->ctx; struct hl_device *hdev = ctx->hdev; @@ -527,11 +528,10 @@ int hl_hw_queue_schedule_cs(struct hl_cs *cs) hdev->asic_funcs->hw_queues_lock(hdev); - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { atomic64_inc(&ctx->cs_counters.device_in_reset_drop_cnt); - atomic64_inc(&cntr->device_in_reset_drop_cnt); dev_err(hdev->dev, - "device is disabled or in reset, CS rejected!\n"); + "device is %s, CS rejected!\n", hdev->status[status]); rc = -EPERM; goto out; } diff --git a/drivers/misc/habanalabs/common/hwmon.c b/drivers/misc/habanalabs/common/hwmon.c index 892a5e2b0b9d..ab96401c3752 100644 --- a/drivers/misc/habanalabs/common/hwmon.c +++ b/drivers/misc/habanalabs/common/hwmon.c @@ -114,7 +114,7 @@ static int hl_read(struct device *dev, enum hwmon_sensor_types type, struct hl_device *hdev = dev_get_drvdata(dev); int rc; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; switch (type) { @@ -192,7 +192,7 @@ static int hl_write(struct device *dev, enum hwmon_sensor_types type, { struct hl_device *hdev = dev_get_drvdata(dev); - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; switch (type) { diff --git a/drivers/misc/habanalabs/common/memory.c b/drivers/misc/habanalabs/common/memory.c index 5c1dae6aaf4d..e00ad11dc5f7 100644 --- a/drivers/misc/habanalabs/common/memory.c +++ b/drivers/misc/habanalabs/common/memory.c @@ -1237,6 +1237,7 @@ out: int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) { + enum hl_device_status status; union hl_mem_args *args = data; struct hl_device *hdev = hpriv->hdev; struct hl_ctx *ctx = hpriv->ctx; @@ -1244,10 +1245,10 @@ int hl_mem_ioctl(struct hl_fpriv *hpriv, void *data) u32 handle = 0; int rc; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, &status)) { dev_warn_ratelimited(hdev->dev, "Device is %s. Can't execute MEMORY IOCTL\n", - atomic_read(&hdev->in_reset) ? "in_reset" : "disabled"); + hdev->status[status]); return -EBUSY; } diff --git a/drivers/misc/habanalabs/common/sysfs.c b/drivers/misc/habanalabs/common/sysfs.c index 3ceae87016b1..94ca68e62000 100644 --- a/drivers/misc/habanalabs/common/sysfs.c +++ b/drivers/misc/habanalabs/common/sysfs.c @@ -276,6 +276,8 @@ static ssize_t status_show(struct device *dev, struct device_attribute *attr, str = "In reset"; else if (hdev->disabled) str = "Malfunction"; + else if (hdev->needs_reset) + str = "Needs Reset"; else str = "Operational"; @@ -304,7 +306,7 @@ static ssize_t max_power_show(struct device *dev, struct device_attribute *attr, struct hl_device *hdev = dev_get_drvdata(dev); long val; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; val = hl_get_max_power(hdev); @@ -319,7 +321,7 @@ static ssize_t max_power_store(struct device *dev, unsigned long value; int rc; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto out; } @@ -347,7 +349,7 @@ static ssize_t eeprom_read_handler(struct file *filp, struct kobject *kobj, char *data; int rc; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; if (!max_size) diff --git a/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c b/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c index 1076b4932ce2..8c49da4bcbd5 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c +++ b/drivers/misc/habanalabs/gaudi/gaudi_hwmgr.c @@ -20,7 +20,7 @@ int gaudi_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk) { long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, MME_PLL, false); @@ -54,7 +54,7 @@ static ssize_t clk_max_freq_mhz_show(struct device *dev, struct gaudi_device *gaudi = hdev->asic_specific; long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, MME_PLL, false); @@ -72,7 +72,7 @@ static ssize_t clk_max_freq_mhz_store(struct device *dev, int rc; u64 value; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto fail; } @@ -97,7 +97,7 @@ static ssize_t clk_cur_freq_mhz_show(struct device *dev, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, MME_PLL, true); diff --git a/drivers/misc/habanalabs/goya/goya_hwmgr.c b/drivers/misc/habanalabs/goya/goya_hwmgr.c index cdd4903e48fa..3acb36a1a902 100644 --- a/drivers/misc/habanalabs/goya/goya_hwmgr.c +++ b/drivers/misc/habanalabs/goya/goya_hwmgr.c @@ -36,7 +36,7 @@ int goya_get_clk_rate(struct hl_device *hdev, u32 *cur_clk, u32 *max_clk) { long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, MME_PLL, false); @@ -69,7 +69,7 @@ static ssize_t mme_clk_show(struct device *dev, struct device_attribute *attr, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, MME_PLL, false); @@ -88,7 +88,7 @@ static ssize_t mme_clk_store(struct device *dev, struct device_attribute *attr, int rc; long value; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto fail; } @@ -118,7 +118,7 @@ static ssize_t tpc_clk_show(struct device *dev, struct device_attribute *attr, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, TPC_PLL, false); @@ -137,7 +137,7 @@ static ssize_t tpc_clk_store(struct device *dev, struct device_attribute *attr, int rc; long value; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto fail; } @@ -167,7 +167,7 @@ static ssize_t ic_clk_show(struct device *dev, struct device_attribute *attr, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, IC_PLL, false); @@ -186,7 +186,7 @@ static ssize_t ic_clk_store(struct device *dev, struct device_attribute *attr, int rc; long value; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto fail; } @@ -216,7 +216,7 @@ static ssize_t mme_clk_curr_show(struct device *dev, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, MME_PLL, true); @@ -233,7 +233,7 @@ static ssize_t tpc_clk_curr_show(struct device *dev, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, TPC_PLL, true); @@ -250,7 +250,7 @@ static ssize_t ic_clk_curr_show(struct device *dev, struct hl_device *hdev = dev_get_drvdata(dev); long value; - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; value = hl_get_frequency(hdev, IC_PLL, true); @@ -266,7 +266,7 @@ static ssize_t pm_mng_profile_show(struct device *dev, { struct hl_device *hdev = dev_get_drvdata(dev); - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; return sprintf(buf, "%s\n", @@ -280,7 +280,7 @@ static ssize_t pm_mng_profile_store(struct device *dev, { struct hl_device *hdev = dev_get_drvdata(dev); - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto out; } @@ -335,7 +335,7 @@ static ssize_t high_pll_show(struct device *dev, struct device_attribute *attr, { struct hl_device *hdev = dev_get_drvdata(dev); - if (hl_device_disabled_or_in_reset(hdev)) + if (!hl_device_operational(hdev, NULL)) return -ENODEV; return sprintf(buf, "%u\n", hdev->high_pll); @@ -348,7 +348,7 @@ static ssize_t high_pll_store(struct device *dev, struct device_attribute *attr, long value; int rc; - if (hl_device_disabled_or_in_reset(hdev)) { + if (!hl_device_operational(hdev, NULL)) { count = -ENODEV; goto out; } diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 61f8f9144b54..d9cc782aba21 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -242,7 +242,8 @@ enum gaudi_engine_id { enum hl_device_status { HL_DEVICE_STATUS_OPERATIONAL, HL_DEVICE_STATUS_IN_RESET, - HL_DEVICE_STATUS_MALFUNCTION + HL_DEVICE_STATUS_MALFUNCTION, + HL_DEVICE_STATUS_NEEDS_RESET }; /* Opcode for management ioctl -- cgit v1.2.3 From 4147864e8d65a0d57dd8573cf306382653616ac2 Mon Sep 17 00:00:00 2001 From: Alon Mizrahi Date: Tue, 17 Nov 2020 14:25:14 +0200 Subject: habanalabs: fetch pll frequency from firmware Once firmware security is enabled, driver must fetch pll frequencies through the firmware message interface instead of reading the registers directly. Signed-off-by: Alon Mizrahi Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/firmware_if.c | 24 ++-- drivers/misc/habanalabs/common/habanalabs.h | 9 +- drivers/misc/habanalabs/common/habanalabs_ioctl.c | 22 ++++ drivers/misc/habanalabs/gaudi/gaudi.c | 135 ++++++++++++++------- drivers/misc/habanalabs/gaudi/gaudiP.h | 8 ++ drivers/misc/habanalabs/goya/goya.c | 2 +- drivers/misc/habanalabs/include/common/cpucp_if.h | 40 +++++- .../misc/habanalabs/include/common/hl_boot_if.h | 4 + .../habanalabs/include/gaudi/asic_reg/gaudi_regs.h | 14 ++- .../include/gaudi/asic_reg/psoc_hbm_pll_regs.h | 114 ----------------- .../include/gaudi/asic_reg/psoc_pci_pll_regs.h | 114 ----------------- include/uapi/misc/habanalabs.h | 9 ++ 12 files changed, 199 insertions(+), 296 deletions(-) delete mode 100644 drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h delete mode 100644 drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h (limited to 'include') diff --git a/drivers/misc/habanalabs/common/firmware_if.c b/drivers/misc/habanalabs/common/firmware_if.c index 8f70d0bbe5e1..c4a8d6ca34bb 100644 --- a/drivers/misc/habanalabs/common/firmware_if.c +++ b/drivers/misc/habanalabs/common/firmware_if.c @@ -279,7 +279,8 @@ int hl_fw_send_heartbeat(struct hl_device *hdev) return rc; } -int hl_fw_cpucp_info_get(struct hl_device *hdev) +int hl_fw_cpucp_info_get(struct hl_device *hdev, + u32 cpu_security_boot_status_reg) { struct asic_fixed_properties *prop = &hdev->asic_prop; struct cpucp_packet pkt = {}; @@ -324,6 +325,11 @@ int hl_fw_cpucp_info_get(struct hl_device *hdev) goto out; } + /* Read FW application security bits again */ + if (hdev->asic_prop.fw_security_status_valid) + hdev->asic_prop.fw_app_security_map = + RREG32(cpu_security_boot_status_reg); + out: hdev->asic_funcs->cpu_accessible_dma_pool_free(hdev, sizeof(struct cpucp_info), cpucp_info_cpu_addr); @@ -446,10 +452,8 @@ int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy) return rc; } -int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, - enum cpucp_pll_type_attributes pll_type, - enum cpucp_pll_reg_attributes pll_reg, - u32 *pll_info) +int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index, + u16 *pll_freq_arr) { struct cpucp_packet pkt; u64 result; @@ -457,17 +461,19 @@ int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, memset(&pkt, 0, sizeof(pkt)); - pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_REG_GET << + pkt.ctl = cpu_to_le32(CPUCP_PACKET_PLL_INFO_GET << CPUCP_PKT_CTL_OPCODE_SHIFT); - pkt.pll_type = __cpu_to_le16(pll_type); - pkt.pll_reg = __cpu_to_le16(pll_reg); + pkt.pll_type = __cpu_to_le16(pll_index); rc = hdev->asic_funcs->send_cpu_message(hdev, (u32 *) &pkt, sizeof(pkt), HL_CPUCP_INFO_TIMEOUT_USEC, &result); if (rc) dev_err(hdev->dev, "Failed to read PLL info, error %d\n", rc); - *pll_info = (u32) result; + pll_freq_arr[0] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT0_MASK, result); + pll_freq_arr[1] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT1_MASK, result); + pll_freq_arr[2] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT2_MASK, result); + pll_freq_arr[3] = FIELD_GET(CPUCP_PKT_RES_PLL_OUT3_MASK, result); return rc; } diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index e1db8301ecbd..9c9c8b24c47a 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -2191,16 +2191,15 @@ void *hl_fw_cpu_accessible_dma_pool_alloc(struct hl_device *hdev, size_t size, void hl_fw_cpu_accessible_dma_pool_free(struct hl_device *hdev, size_t size, void *vaddr); int hl_fw_send_heartbeat(struct hl_device *hdev); -int hl_fw_cpucp_info_get(struct hl_device *hdev); +int hl_fw_cpucp_info_get(struct hl_device *hdev, + u32 cpu_security_boot_status_reg); int hl_fw_get_eeprom_data(struct hl_device *hdev, void *data, size_t max_size); int hl_fw_cpucp_pci_counters_get(struct hl_device *hdev, struct hl_info_pci_counters *counters); int hl_fw_cpucp_total_energy_get(struct hl_device *hdev, u64 *total_energy); -int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, - enum cpucp_pll_type_attributes pll_type, - enum cpucp_pll_reg_attributes pll_reg, - u32 *pll_info); +int hl_fw_cpucp_pll_info_get(struct hl_device *hdev, u16 pll_index, + u16 *pll_freq_arr); int hl_fw_init_cpu(struct hl_device *hdev, u32 cpu_boot_status_reg, u32 msg_to_cpu_reg, u32 cpu_msg_status_reg, u32 cpu_security_boot_status_reg, u32 boot_err0_reg, diff --git a/drivers/misc/habanalabs/common/habanalabs_ioctl.c b/drivers/misc/habanalabs/common/habanalabs_ioctl.c index ba8217fc9425..32e6af1db4e3 100644 --- a/drivers/misc/habanalabs/common/habanalabs_ioctl.c +++ b/drivers/misc/habanalabs/common/habanalabs_ioctl.c @@ -403,6 +403,25 @@ static int total_energy_consumption_info(struct hl_fpriv *hpriv, min((size_t) max_size, sizeof(total_energy))) ? -EFAULT : 0; } +static int pll_frequency_info(struct hl_fpriv *hpriv, struct hl_info_args *args) +{ + struct hl_device *hdev = hpriv->hdev; + struct hl_pll_frequency_info freq_info = {0}; + u32 max_size = args->return_size; + void __user *out = (void __user *) (uintptr_t) args->return_pointer; + int rc; + + if ((!max_size) || (!out)) + return -EINVAL; + + rc = hl_fw_cpucp_pll_info_get(hdev, args->pll_index, freq_info.output); + if (rc) + return rc; + + return copy_to_user(out, &freq_info, + min((size_t) max_size, sizeof(freq_info))) ? -EFAULT : 0; +} + static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, struct device *dev) { @@ -480,6 +499,9 @@ static int _hl_info_ioctl(struct hl_fpriv *hpriv, void *data, case HL_INFO_TOTAL_ENERGY: return total_energy_consumption_info(hpriv, args); + case HL_INFO_PLL_FREQUENCY: + return pll_frequency_info(hpriv, args); + default: dev_err(dev, "Invalid request %d\n", args->op); rc = -ENOTTY; diff --git a/drivers/misc/habanalabs/gaudi/gaudi.c b/drivers/misc/habanalabs/gaudi/gaudi.c index 49d4b5dda115..732559053133 100644 --- a/drivers/misc/habanalabs/gaudi/gaudi.c +++ b/drivers/misc/habanalabs/gaudi/gaudi.c @@ -103,6 +103,8 @@ #define HBM_SCRUBBING_TIMEOUT_US 1000000 /* 1s */ +#define GAUDI_PLL_MAX 10 + static const char gaudi_irq_name[GAUDI_MSI_ENTRIES][GAUDI_MAX_STRING_LEN] = { "gaudi cq 0_0", "gaudi cq 0_1", "gaudi cq 0_2", "gaudi cq 0_3", "gaudi cq 1_0", "gaudi cq 1_1", "gaudi cq 1_2", "gaudi cq 1_3", @@ -149,6 +151,19 @@ static const u16 gaudi_packet_sizes[MAX_PACKET_ID] = { [PACKET_LOAD_AND_EXE] = sizeof(struct packet_load_and_exe) }; +static const u32 gaudi_pll_base_addresses[GAUDI_PLL_MAX] = { + [CPU_PLL] = mmPSOC_CPU_PLL_NR, + [PCI_PLL] = mmPSOC_PCI_PLL_NR, + [SRAM_PLL] = mmSRAM_W_PLL_NR, + [HBM_PLL] = mmPSOC_HBM_PLL_NR, + [NIC_PLL] = mmNIC0_PLL_NR, + [DMA_PLL] = mmDMA_W_PLL_NR, + [MESH_PLL] = mmMESH_W_PLL_NR, + [MME_PLL] = mmPSOC_MME_PLL_NR, + [TPC_PLL] = mmPSOC_TPC_PLL_NR, + [IF_PLL] = mmIF_W_PLL_NR +}; + static inline bool validate_packet_id(enum packet_id id) { switch (id) { @@ -688,61 +703,93 @@ static int gaudi_early_fini(struct hl_device *hdev) } /** - * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values + * gaudi_fetch_pll_frequency - Fetch PLL frequency values * * @hdev: pointer to hl_device structure + * @pll_index: index of the pll to fetch frequency from + * @pll_freq: pointer to store the pll frequency in MHz in each of the available + * outputs. if a certain output is not available a 0 will be set * */ -static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) +static int gaudi_fetch_pll_frequency(struct hl_device *hdev, + enum gaudi_pll_index pll_index, + u16 *pll_freq_arr) { - struct asic_fixed_properties *prop = &hdev->asic_prop; - u32 trace_freq = 0, pll_clk = 0; - u32 div_fctr, div_sel, nr, nf, od; - int rc; + u32 nr = 0, nf = 0, od = 0, pll_clk = 0, div_fctr, div_sel, + pll_base_addr = gaudi_pll_base_addresses[pll_index]; + u16 freq = 0; + int i, rc; + + if (hdev->asic_prop.fw_security_status_valid && + (hdev->asic_prop.fw_app_security_map & + CPU_BOOT_DEV_STS0_PLL_INFO_EN)) { + rc = hl_fw_cpucp_pll_info_get(hdev, pll_index, pll_freq_arr); - if (hdev->asic_prop.fw_security_disabled) { - div_fctr = RREG32(mmPSOC_CPU_PLL_DIV_FACTOR_2); - div_sel = RREG32(mmPSOC_CPU_PLL_DIV_SEL_2); - nr = RREG32(mmPSOC_CPU_PLL_NR); - nf = RREG32(mmPSOC_CPU_PLL_NF); - od = RREG32(mmPSOC_CPU_PLL_OD); - } else { - rc = hl_fw_cpucp_pll_info_get(hdev, cpucp_pll_cpu, - cpucp_pll_div_factor_reg, &div_fctr); - rc |= hl_fw_cpucp_pll_info_get(hdev, cpucp_pll_cpu, - cpucp_pll_div_sel_reg, &div_sel); - rc |= hl_fw_cpucp_pll_info_get(hdev, cpucp_pll_cpu, - cpucp_pll_nr_reg, &nr); - rc |= hl_fw_cpucp_pll_info_get(hdev, cpucp_pll_cpu, - cpucp_pll_nf_reg, &nf); - rc |= hl_fw_cpucp_pll_info_get(hdev, cpucp_pll_cpu, - cpucp_pll_od_reg, &od); if (rc) return rc; - } - - if (div_sel == DIV_SEL_REF_CLK || div_sel == DIV_SEL_DIVIDED_REF) { - if (div_sel == DIV_SEL_REF_CLK) - trace_freq = PLL_REF_CLK; - else - trace_freq = PLL_REF_CLK / (div_fctr + 1); - } else if (div_sel == DIV_SEL_PLL_CLK || + } else if (hdev->asic_prop.fw_security_disabled) { + /* Backward compatibility */ + nr = RREG32(pll_base_addr + PLL_NR_OFFSET); + nf = RREG32(pll_base_addr + PLL_NF_OFFSET); + od = RREG32(pll_base_addr + PLL_OD_OFFSET); + + for (i = 0; i < HL_PLL_NUM_OUTPUTS; i++) { + div_fctr = RREG32(pll_base_addr + + PLL_DIV_FACTOR_0_OFFSET + i * 4); + div_sel = RREG32(pll_base_addr + + PLL_DIV_SEL_0_OFFSET + i * 4); + + if (div_sel == DIV_SEL_REF_CLK || + div_sel == DIV_SEL_DIVIDED_REF) { + if (div_sel == DIV_SEL_REF_CLK) + freq = PLL_REF_CLK; + else + freq = PLL_REF_CLK / (div_fctr + 1); + } else if (div_sel == DIV_SEL_PLL_CLK || div_sel == DIV_SEL_DIVIDED_PLL) { - pll_clk = PLL_REF_CLK * (nf + 1) / ((nr + 1) * (od + 1)); - if (div_sel == DIV_SEL_PLL_CLK) - trace_freq = pll_clk; - else - trace_freq = pll_clk / (div_fctr + 1); + pll_clk = PLL_REF_CLK * (nf + 1) / + ((nr + 1) * (od + 1)); + if (div_sel == DIV_SEL_PLL_CLK) + freq = pll_clk; + else + freq = pll_clk / (div_fctr + 1); + } else { + dev_warn(hdev->dev, + "Received invalid div select value: %d", + div_sel); + } + + pll_freq_arr[i] = freq; + } } else { - dev_warn(hdev->dev, - "Received invalid div select value: %d", div_sel); + dev_err(hdev->dev, "Failed to fetch PLL frequency values\n"); + return -EIO; } - prop->psoc_timestamp_frequency = trace_freq; - prop->psoc_pci_pll_nr = nr; - prop->psoc_pci_pll_nf = nf; - prop->psoc_pci_pll_od = od; - prop->psoc_pci_pll_div_factor = div_fctr; + return 0; +} + +/** + * gaudi_fetch_psoc_frequency - Fetch PSOC frequency values + * + * @hdev: pointer to hl_device structure + * + */ +static int gaudi_fetch_psoc_frequency(struct hl_device *hdev) +{ + struct asic_fixed_properties *prop = &hdev->asic_prop; + u16 pll_freq[HL_PLL_NUM_OUTPUTS]; + int rc; + + rc = gaudi_fetch_pll_frequency(hdev, CPU_PLL, pll_freq); + if (rc) + return rc; + + prop->psoc_timestamp_frequency = pll_freq[2]; + prop->psoc_pci_pll_nr = 0; + prop->psoc_pci_pll_nf = 0; + prop->psoc_pci_pll_od = 0; + prop->psoc_pci_pll_div_factor = 0; return 0; } @@ -7438,7 +7485,7 @@ static int gaudi_cpucp_info_get(struct hl_device *hdev) if (!(gaudi->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_info_get(hdev); + rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0); if (rc) return rc; diff --git a/drivers/misc/habanalabs/gaudi/gaudiP.h b/drivers/misc/habanalabs/gaudi/gaudiP.h index 1a5e681c720d..f2d91f4fcffe 100644 --- a/drivers/misc/habanalabs/gaudi/gaudiP.h +++ b/drivers/misc/habanalabs/gaudi/gaudiP.h @@ -14,6 +14,7 @@ #include "../include/gaudi/gaudi_packets.h" #include "../include/gaudi/gaudi.h" #include "../include/gaudi/gaudi_async_events.h" +#include "../include/gaudi/gaudi_fw_if.h" #define NUMBER_OF_EXT_HW_QUEUES 8 #define NUMBER_OF_CMPLT_QUEUES NUMBER_OF_EXT_HW_QUEUES @@ -104,6 +105,13 @@ #define MME_ACC_OFFSET (mmMME1_ACC_BASE - mmMME0_ACC_BASE) #define SRAM_BANK_OFFSET (mmSRAM_Y0_X1_RTR_BASE - mmSRAM_Y0_X0_RTR_BASE) +#define PLL_NR_OFFSET 0 +#define PLL_NF_OFFSET (mmPSOC_CPU_PLL_NF - mmPSOC_CPU_PLL_NR) +#define PLL_OD_OFFSET (mmPSOC_CPU_PLL_OD - mmPSOC_CPU_PLL_NR) +#define PLL_DIV_FACTOR_0_OFFSET (mmPSOC_CPU_PLL_DIV_FACTOR_0 - \ + mmPSOC_CPU_PLL_NR) +#define PLL_DIV_SEL_0_OFFSET (mmPSOC_CPU_PLL_DIV_SEL_0 - mmPSOC_CPU_PLL_NR) + #define NUM_OF_SOB_IN_BLOCK \ (((mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_2047 - \ mmSYNC_MNGR_E_N_SYNC_MNGR_OBJS_SOB_OBJ_0) + 4) >> 2) diff --git a/drivers/misc/habanalabs/goya/goya.c b/drivers/misc/habanalabs/goya/goya.c index 342227b93778..d91f553b8595 100644 --- a/drivers/misc/habanalabs/goya/goya.c +++ b/drivers/misc/habanalabs/goya/goya.c @@ -5156,7 +5156,7 @@ int goya_cpucp_info_get(struct hl_device *hdev) if (!(goya->hw_cap_initialized & HW_CAP_CPU_Q)) return 0; - rc = hl_fw_cpucp_info_get(hdev); + rc = hl_fw_cpucp_info_get(hdev, mmCPU_BOOT_DEV_STS0); if (rc) return rc; diff --git a/drivers/misc/habanalabs/include/common/cpucp_if.h b/drivers/misc/habanalabs/include/common/cpucp_if.h index 759c068b2b7a..554f82271d5f 100644 --- a/drivers/misc/habanalabs/include/common/cpucp_if.h +++ b/drivers/misc/habanalabs/include/common/cpucp_if.h @@ -252,10 +252,26 @@ enum pq_init_status { * The packet's arguments specify the desired sensor and the field to * set. * - * CPUCP_PACKET_PLL_REG_GET - * Fetch register of PLL from the required PLL IP. - * The packet's arguments specify the PLL IP and the register to get. - * Each register is 32-bit value which is returned in result field. + * CPUCP_PACKET_PCIE_THROUGHPUT_GET + * Get throughput of PCIe. + * The packet's arguments specify the transaction direction (TX/RX). + * The window measurement is 10[msec], and the return value is in KB/sec. + * + * CPUCP_PACKET_PCIE_REPLAY_CNT_GET + * Replay count measures number of "replay" events, which is basicly + * number of retries done by PCIe. + * + * CPUCP_PACKET_TOTAL_ENERGY_GET + * Total Energy is measurement of energy from the time FW Linux + * is loaded. It is calculated by multiplying the average power + * by time (passed from armcp start). The units are in MilliJouls. + * + * CPUCP_PACKET_PLL_INFO_GET + * Fetch frequencies of PLL from the required PLL IP. + * The packet's arguments specify the device PLL type + * Pll type is the PLL from device pll_index enum. + * The result is composed of 4 outputs, each is 16-bit + * frequency in MHz. * */ @@ -289,7 +305,7 @@ enum cpucp_packet_id { CPUCP_PACKET_PCIE_THROUGHPUT_GET, /* internal */ CPUCP_PACKET_PCIE_REPLAY_CNT_GET, /* internal */ CPUCP_PACKET_TOTAL_ENERGY_GET, /* internal */ - CPUCP_PACKET_PLL_REG_GET, /* internal */ + CPUCP_PACKET_PLL_INFO_GET, /* internal */ }; #define CPUCP_PACKET_FENCE_VAL 0xFE8CE7A5 @@ -300,6 +316,15 @@ enum cpucp_packet_id { #define CPUCP_PKT_CTL_OPCODE_SHIFT 16 #define CPUCP_PKT_CTL_OPCODE_MASK 0x1FFF0000 +#define CPUCP_PKT_RES_PLL_OUT0_SHIFT 0 +#define CPUCP_PKT_RES_PLL_OUT0_MASK 0x000000000000FFFF +#define CPUCP_PKT_RES_PLL_OUT1_SHIFT 16 +#define CPUCP_PKT_RES_PLL_OUT1_MASK 0x00000000FFFF0000 +#define CPUCP_PKT_RES_PLL_OUT2_SHIFT 32 +#define CPUCP_PKT_RES_PLL_OUT2_MASK 0x0000FFFF00000000 +#define CPUCP_PKT_RES_PLL_OUT3_SHIFT 48 +#define CPUCP_PKT_RES_PLL_OUT3_MASK 0xFFFF000000000000 + struct cpucp_packet { union { __le64 value; /* For SET packets */ @@ -324,8 +349,9 @@ struct cpucp_packet { __u8 pad; /* unused */ }; - struct {/* For PLL register fetch */ + struct {/* For PLL info fetch */ __le16 pll_type; + /* TODO pll_reg is kept temporary before removal */ __le16 pll_reg; }; @@ -404,6 +430,7 @@ enum cpucp_pcie_throughput_attributes { cpucp_pcie_throughput_rx }; +/* TODO temporary kept before removal */ enum cpucp_pll_reg_attributes { cpucp_pll_nr_reg, cpucp_pll_nf_reg, @@ -412,6 +439,7 @@ enum cpucp_pll_reg_attributes { cpucp_pll_div_sel_reg }; +/* TODO temporary kept before removal */ enum cpucp_pll_type_attributes { cpucp_pll_cpu, cpucp_pll_pci, diff --git a/drivers/misc/habanalabs/include/common/hl_boot_if.h b/drivers/misc/habanalabs/include/common/hl_boot_if.h index 60916780df35..68ac15c53f37 100644 --- a/drivers/misc/habanalabs/include/common/hl_boot_if.h +++ b/drivers/misc/habanalabs/include/common/hl_boot_if.h @@ -131,6 +131,9 @@ * receiving the halt-machine event. * Initialized in: linux * + * CPU_BOOT_DEV_STS0_PLL_INFO_EN FW retrieval of PLL info is enabled. + * Initialized in: linux + * * CPU_BOOT_DEV_STS0_ENABLED Device status register enabled. * This is a main indication that the * running FW populates the device status @@ -150,6 +153,7 @@ #define CPU_BOOT_DEV_STS0_SRAM_SCR_EN (1 << 8) #define CPU_BOOT_DEV_STS0_DRAM_SCR_EN (1 << 9) #define CPU_BOOT_DEV_STS0_FW_HARD_RST_EN (1 << 10) +#define CPU_BOOT_DEV_STS0_PLL_INFO_EN (1 << 11) #define CPU_BOOT_DEV_STS0_ENABLED (1 << 31) enum cpu_boot_status { diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h index df21a40691e5..5bb54b34a8ae 100644 --- a/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h +++ b/drivers/misc/habanalabs/include/gaudi/asic_reg/gaudi_regs.h @@ -81,6 +81,7 @@ #include "sif_rtr_ctrl_6_regs.h" #include "sif_rtr_ctrl_7_regs.h" #include "psoc_etr_regs.h" +#include "psoc_cpu_pll_regs.h" #include "dma0_qm_masks.h" #include "mme0_qm_masks.h" @@ -102,9 +103,6 @@ #include "nic0_qm0_masks.h" -#include "psoc_hbm_pll_regs.h" -#include "psoc_cpu_pll_regs.h" - #define GAUDI_ECC_MEM_SEL_OFFSET 0xF18 #define GAUDI_ECC_ADDRESS_OFFSET 0xF1C #define GAUDI_ECC_SYNDROME_OFFSET 0xF20 @@ -307,4 +305,14 @@ #define mmPCIE_AUX_FLR_CTRL 0xC07394 #define mmPCIE_AUX_DBI 0xC07490 +#define mmPSOC_PCI_PLL_NR 0xC72100 +#define mmSRAM_W_PLL_NR 0x4C8100 +#define mmPSOC_HBM_PLL_NR 0xC74100 +#define mmNIC0_PLL_NR 0xCF9100 +#define mmDMA_W_PLL_NR 0x487100 +#define mmMESH_W_PLL_NR 0x4C7100 +#define mmPSOC_MME_PLL_NR 0xC71100 +#define mmPSOC_TPC_PLL_NR 0xC73100 +#define mmIF_W_PLL_NR 0x488100 + #endif /* ASIC_REG_GAUDI_REGS_H_ */ diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h deleted file mode 100644 index 687e2255cb19..000000000000 --- a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_hbm_pll_regs.h +++ /dev/null @@ -1,114 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * Copyright 2016-2018 HabanaLabs, Ltd. - * All Rights Reserved. - * - */ - -/************************************ - ** This is an auto-generated file ** - ** DO NOT EDIT BELOW ** - ************************************/ - -#ifndef ASIC_REG_PSOC_HBM_PLL_REGS_H_ -#define ASIC_REG_PSOC_HBM_PLL_REGS_H_ - -/* - ***************************************** - * PSOC_HBM_PLL (Prototype: PLL) - ***************************************** - */ - -#define mmPSOC_HBM_PLL_NR 0xC74100 - -#define mmPSOC_HBM_PLL_NF 0xC74104 - -#define mmPSOC_HBM_PLL_OD 0xC74108 - -#define mmPSOC_HBM_PLL_NB 0xC7410C - -#define mmPSOC_HBM_PLL_CFG 0xC74110 - -#define mmPSOC_HBM_PLL_LOSE_MASK 0xC74120 - -#define mmPSOC_HBM_PLL_LOCK_INTR 0xC74128 - -#define mmPSOC_HBM_PLL_LOCK_BYPASS 0xC7412C - -#define mmPSOC_HBM_PLL_DATA_CHNG 0xC74130 - -#define mmPSOC_HBM_PLL_RST 0xC74134 - -#define mmPSOC_HBM_PLL_SLIP_WD_CNTR 0xC74150 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_0 0xC74200 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_1 0xC74204 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_2 0xC74208 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_3 0xC7420C - -#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_0 0xC74220 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_1 0xC74224 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_2 0xC74228 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_CMD_3 0xC7422C - -#define mmPSOC_HBM_PLL_DIV_SEL_0 0xC74280 - -#define mmPSOC_HBM_PLL_DIV_SEL_1 0xC74284 - -#define mmPSOC_HBM_PLL_DIV_SEL_2 0xC74288 - -#define mmPSOC_HBM_PLL_DIV_SEL_3 0xC7428C - -#define mmPSOC_HBM_PLL_DIV_EN_0 0xC742A0 - -#define mmPSOC_HBM_PLL_DIV_EN_1 0xC742A4 - -#define mmPSOC_HBM_PLL_DIV_EN_2 0xC742A8 - -#define mmPSOC_HBM_PLL_DIV_EN_3 0xC742AC - -#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_0 0xC742C0 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_1 0xC742C4 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_2 0xC742C8 - -#define mmPSOC_HBM_PLL_DIV_FACTOR_BUSY_3 0xC742CC - -#define mmPSOC_HBM_PLL_CLK_GATER 0xC74300 - -#define mmPSOC_HBM_PLL_CLK_RLX_0 0xC74310 - -#define mmPSOC_HBM_PLL_CLK_RLX_1 0xC74314 - -#define mmPSOC_HBM_PLL_CLK_RLX_2 0xC74318 - -#define mmPSOC_HBM_PLL_CLK_RLX_3 0xC7431C - -#define mmPSOC_HBM_PLL_REF_CNTR_PERIOD 0xC74400 - -#define mmPSOC_HBM_PLL_REF_LOW_THRESHOLD 0xC74410 - -#define mmPSOC_HBM_PLL_REF_HIGH_THRESHOLD 0xC74420 - -#define mmPSOC_HBM_PLL_PLL_NOT_STABLE 0xC74430 - -#define mmPSOC_HBM_PLL_FREQ_CALC_EN 0xC74440 - -#define mmPSOC_HBM_PLL_RLX_BITMAP_CFG 0xC74500 - -#define mmPSOC_HBM_PLL_RLX_BITMAP_0 0xC74510 - -#define mmPSOC_HBM_PLL_RLX_BITMAP_1 0xC74514 - -#define mmPSOC_HBM_PLL_RLX_BITMAP_2 0xC74518 - -#define mmPSOC_HBM_PLL_RLX_BITMAP_3 0xC7451C - -#endif /* ASIC_REG_PSOC_HBM_PLL_REGS_H_ */ diff --git a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h b/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h deleted file mode 100644 index 3dc9bb4542dd..000000000000 --- a/drivers/misc/habanalabs/include/gaudi/asic_reg/psoc_pci_pll_regs.h +++ /dev/null @@ -1,114 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 - * - * Copyright 2016-2018 HabanaLabs, Ltd. - * All Rights Reserved. - * - */ - -/************************************ - ** This is an auto-generated file ** - ** DO NOT EDIT BELOW ** - ************************************/ - -#ifndef ASIC_REG_PSOC_PCI_PLL_REGS_H_ -#define ASIC_REG_PSOC_PCI_PLL_REGS_H_ - -/* - ***************************************** - * PSOC_PCI_PLL (Prototype: PLL) - ***************************************** - */ - -#define mmPSOC_PCI_PLL_NR 0xC72100 - -#define mmPSOC_PCI_PLL_NF 0xC72104 - -#define mmPSOC_PCI_PLL_OD 0xC72108 - -#define mmPSOC_PCI_PLL_NB 0xC7210C - -#define mmPSOC_PCI_PLL_CFG 0xC72110 - -#define mmPSOC_PCI_PLL_LOSE_MASK 0xC72120 - -#define mmPSOC_PCI_PLL_LOCK_INTR 0xC72128 - -#define mmPSOC_PCI_PLL_LOCK_BYPASS 0xC7212C - -#define mmPSOC_PCI_PLL_DATA_CHNG 0xC72130 - -#define mmPSOC_PCI_PLL_RST 0xC72134 - -#define mmPSOC_PCI_PLL_SLIP_WD_CNTR 0xC72150 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_0 0xC72200 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_1 0xC72204 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_2 0xC72208 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_3 0xC7220C - -#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_0 0xC72220 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_1 0xC72224 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_2 0xC72228 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_CMD_3 0xC7222C - -#define mmPSOC_PCI_PLL_DIV_SEL_0 0xC72280 - -#define mmPSOC_PCI_PLL_DIV_SEL_1 0xC72284 - -#define mmPSOC_PCI_PLL_DIV_SEL_2 0xC72288 - -#define mmPSOC_PCI_PLL_DIV_SEL_3 0xC7228C - -#define mmPSOC_PCI_PLL_DIV_EN_0 0xC722A0 - -#define mmPSOC_PCI_PLL_DIV_EN_1 0xC722A4 - -#define mmPSOC_PCI_PLL_DIV_EN_2 0xC722A8 - -#define mmPSOC_PCI_PLL_DIV_EN_3 0xC722AC - -#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_0 0xC722C0 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_1 0xC722C4 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_2 0xC722C8 - -#define mmPSOC_PCI_PLL_DIV_FACTOR_BUSY_3 0xC722CC - -#define mmPSOC_PCI_PLL_CLK_GATER 0xC72300 - -#define mmPSOC_PCI_PLL_CLK_RLX_0 0xC72310 - -#define mmPSOC_PCI_PLL_CLK_RLX_1 0xC72314 - -#define mmPSOC_PCI_PLL_CLK_RLX_2 0xC72318 - -#define mmPSOC_PCI_PLL_CLK_RLX_3 0xC7231C - -#define mmPSOC_PCI_PLL_REF_CNTR_PERIOD 0xC72400 - -#define mmPSOC_PCI_PLL_REF_LOW_THRESHOLD 0xC72410 - -#define mmPSOC_PCI_PLL_REF_HIGH_THRESHOLD 0xC72420 - -#define mmPSOC_PCI_PLL_PLL_NOT_STABLE 0xC72430 - -#define mmPSOC_PCI_PLL_FREQ_CALC_EN 0xC72440 - -#define mmPSOC_PCI_PLL_RLX_BITMAP_CFG 0xC72500 - -#define mmPSOC_PCI_PLL_RLX_BITMAP_0 0xC72510 - -#define mmPSOC_PCI_PLL_RLX_BITMAP_1 0xC72514 - -#define mmPSOC_PCI_PLL_RLX_BITMAP_2 0xC72518 - -#define mmPSOC_PCI_PLL_RLX_BITMAP_3 0xC7251C - -#endif /* ASIC_REG_PSOC_PCI_PLL_REGS_H_ */ diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index d9cc782aba21..96eea49f48bc 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -295,6 +295,7 @@ enum hl_device_status { #define HL_INFO_CLK_THROTTLE_REASON 13 #define HL_INFO_SYNC_MANAGER 14 #define HL_INFO_TOTAL_ENERGY 15 +#define HL_INFO_PLL_FREQUENCY 16 #define HL_INFO_VERSION_MAX_LEN 128 #define HL_INFO_CARD_NAME_MAX_LEN 16 @@ -396,6 +397,12 @@ struct hl_info_energy { __u64 total_energy_consumption; }; +#define HL_PLL_NUM_OUTPUTS 4 + +struct hl_pll_frequency_info { + __u16 output[HL_PLL_NUM_OUTPUTS]; +}; + /** * struct hl_info_sync_manager - sync manager information * @first_available_sync_object: first available sob @@ -465,6 +472,8 @@ struct hl_info_args { * resolution. */ __u32 period_ms; + /* PLL frequency retrieval */ + __u32 pll_index; }; __u32 pad; -- cgit v1.2.3 From 9d127ad5719a865bac668a506dfe924ac11cd9bb Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Tue, 10 Nov 2020 16:30:53 +0200 Subject: habanalabs: indicate to user that a cs is gone We want to indicate to the user that a certain command submission is finished long time ago and it is no longer in database. This means no further information regarding this cs can be obtained. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 68 ++++++++++++++++------ include/uapi/misc/habanalabs.h | 5 +- 2 files changed, 54 insertions(+), 19 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index 7309dd2b88a9..f91b17480588 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -11,9 +11,22 @@ #include #include +/** + * enum hl_cs_wait_status - cs wait status + * @CS_WAIT_STATUS_BUSY: cs was not completed yet + * @CS_WAIT_STATUS_COMPLETED: cs completed + * @CS_WAIT_STATUS_GONE: cs completed but fence is already gone + */ +enum hl_cs_wait_status { + CS_WAIT_STATUS_BUSY, + CS_WAIT_STATUS_COMPLETED, + CS_WAIT_STATUS_GONE +}; + static void job_wq_completion(struct work_struct *work); -static long _hl_cs_wait_ioctl(struct hl_device *hdev, - struct hl_ctx *ctx, u64 timeout_us, u64 seq); +static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, + u64 timeout_us, u64 seq, + enum hl_cs_wait_status *status); static void cs_do_release(struct kref *ref); static void hl_sob_reset(struct kref *ref) @@ -942,7 +955,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, int rc = 0, do_ctx_switch; void __user *chunks; u32 num_chunks, tmp; - long ret; + int ret; do_ctx_switch = atomic_cmpxchg(&ctx->thread_ctx_switch_token, 1, 0); @@ -996,18 +1009,19 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, /* Need to wait for restore completion before execution phase */ if (num_chunks) { + enum hl_cs_wait_status status; wait_again: ret = _hl_cs_wait_ioctl(hdev, ctx, jiffies_to_usecs(hdev->timeout_jiffies), - *cs_seq); - if (ret <= 0) { + *cs_seq, &status); + if (ret) { if (ret == -ERESTARTSYS) { usleep_range(100, 200); goto wait_again; } dev_err(hdev->dev, - "Restore CS for context %d failed to complete %ld\n", + "Restore CS for context %d failed to complete %d\n", ctx->asid, ret); rc = -ENOEXEC; goto out; @@ -1337,12 +1351,14 @@ out: return rc; } -static long _hl_cs_wait_ioctl(struct hl_device *hdev, - struct hl_ctx *ctx, u64 timeout_us, u64 seq) +static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, + u64 timeout_us, u64 seq, + enum hl_cs_wait_status *status) { struct hl_fence *fence; unsigned long timeout; - long rc; + int rc = 0; + long completion_rc; if (timeout_us == MAX_SCHEDULE_TIMEOUT) timeout = timeout_us; @@ -1360,11 +1376,17 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev, seq, ctx->cs_sequence); } else if (fence) { if (!timeout_us) - rc = completion_done(&fence->completion); + completion_rc = completion_done(&fence->completion); else - rc = wait_for_completion_interruptible_timeout( + completion_rc = + wait_for_completion_interruptible_timeout( &fence->completion, timeout); + if (completion_rc > 0) + *status = CS_WAIT_STATUS_COMPLETED; + else + *status = CS_WAIT_STATUS_BUSY; + if (fence->error == -ETIMEDOUT) rc = -ETIMEDOUT; else if (fence->error == -EIO) @@ -1375,7 +1397,7 @@ static long _hl_cs_wait_ioctl(struct hl_device *hdev, dev_dbg(hdev->dev, "Can't wait on seq %llu because current CS is at seq %llu (Fence is gone)\n", seq, ctx->cs_sequence); - rc = 1; + *status = CS_WAIT_STATUS_GONE; } hl_ctx_put(ctx); @@ -1387,14 +1409,16 @@ int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data) { struct hl_device *hdev = hpriv->hdev; union hl_wait_cs_args *args = data; + enum hl_cs_wait_status status; u64 seq = args->in.seq; - long rc; + int rc; - rc = _hl_cs_wait_ioctl(hdev, hpriv->ctx, args->in.timeout_us, seq); + rc = _hl_cs_wait_ioctl(hdev, hpriv->ctx, args->in.timeout_us, seq, + &status); memset(args, 0, sizeof(*args)); - if (rc < 0) { + if (rc) { if (rc == -ERESTARTSYS) { dev_err_ratelimited(hdev->dev, "user process got signal while waiting for CS handle %llu\n", @@ -1415,10 +1439,18 @@ int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data) return rc; } - if (rc == 0) - args->out.status = HL_WAIT_CS_STATUS_BUSY; - else + switch (status) { + case CS_WAIT_STATUS_GONE: + args->out.flags |= HL_WAIT_CS_STATUS_FLAG_GONE; + fallthrough; + case CS_WAIT_STATUS_COMPLETED: args->out.status = HL_WAIT_CS_STATUS_COMPLETED; + break; + case CS_WAIT_STATUS_BUSY: + default: + args->out.status = HL_WAIT_CS_STATUS_BUSY; + break; + } return 0; } diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 96eea49f48bc..808d20da024a 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -662,10 +662,13 @@ struct hl_wait_cs_in { #define HL_WAIT_CS_STATUS_ABORTED 3 #define HL_WAIT_CS_STATUS_INTERRUPTED 4 +#define HL_WAIT_CS_STATUS_FLAG_GONE 0x1 + struct hl_wait_cs_out { /* HL_WAIT_CS_STATUS_* */ __u32 status; - __u32 pad; + /* HL_WAIT_CS_STATUS_FLAG* */ + __u32 flags; }; union hl_wait_cs_args { -- cgit v1.2.3 From bd2f477f2037d2638464dc105f16994994308c20 Mon Sep 17 00:00:00 2001 From: Ofir Bitton Date: Tue, 10 Nov 2020 17:26:22 +0200 Subject: habanalabs: add support for cs with timestamp add support for user to request a timestamp upon cs completion. Signed-off-by: Ofir Bitton Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- .../misc/habanalabs/common/command_submission.c | 44 ++++++++++++++++------ drivers/misc/habanalabs/common/habanalabs.h | 4 ++ include/uapi/misc/habanalabs.h | 6 ++- 3 files changed, 41 insertions(+), 13 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_submission.c b/drivers/misc/habanalabs/common/command_submission.c index f91b17480588..bd2f54399020 100644 --- a/drivers/misc/habanalabs/common/command_submission.c +++ b/drivers/misc/habanalabs/common/command_submission.c @@ -26,7 +26,7 @@ enum hl_cs_wait_status { static void job_wq_completion(struct work_struct *work); static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, u64 timeout_us, u64 seq, - enum hl_cs_wait_status *status); + enum hl_cs_wait_status *status, s64 *timestamp); static void cs_do_release(struct kref *ref); static void hl_sob_reset(struct kref *ref) @@ -150,6 +150,7 @@ static void hl_fence_init(struct hl_fence *fence) { kref_init(&fence->refcount); fence->error = 0; + fence->timestamp = ktime_set(0, 0); init_completion(&fence->completion); } @@ -404,6 +405,8 @@ out: else if (!cs->submitted) cs->fence->error = -EBUSY; + if (cs->timestamp) + cs->fence->timestamp = ktime_get(); complete_all(&cs->fence->completion); hl_fence_put(cs->fence); @@ -734,7 +737,8 @@ static int hl_cs_sanity_checks(struct hl_fpriv *hpriv, union hl_cs_args *args) return -EBUSY; } - cs_type_flags = args->in.cs_flags & ~HL_CS_FLAGS_FORCE_RESTORE; + cs_type_flags = args->in.cs_flags & + ~(HL_CS_FLAGS_FORCE_RESTORE | HL_CS_FLAGS_TIMESTAMP); if (unlikely(cs_type_flags && !is_power_of_2(cs_type_flags))) { dev_err(hdev->dev, @@ -798,7 +802,7 @@ static int hl_cs_copy_chunk_array(struct hl_device *hdev, } static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, - u32 num_chunks, u64 *cs_seq) + u32 num_chunks, u64 *cs_seq, bool timestamp) { bool int_queues_only = true; struct hl_device *hdev = hpriv->hdev; @@ -825,6 +829,7 @@ static int cs_ioctl_default(struct hl_fpriv *hpriv, void __user *chunks, goto free_cs_chunk_array; } + cs->timestamp = !!timestamp; *cs_seq = cs->sequence; hl_debugfs_add_cs(cs); @@ -995,7 +1000,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, rc = 0; } else { rc = cs_ioctl_default(hpriv, chunks, num_chunks, - cs_seq); + cs_seq, false); } mutex_unlock(&hpriv->restore_phase_mutex); @@ -1013,7 +1018,7 @@ static int hl_cs_ctx_switch(struct hl_fpriv *hpriv, union hl_cs_args *args, wait_again: ret = _hl_cs_wait_ioctl(hdev, ctx, jiffies_to_usecs(hdev->timeout_jiffies), - *cs_seq, &status); + *cs_seq, &status, NULL); if (ret) { if (ret == -ERESTARTSYS) { usleep_range(100, 200); @@ -1154,7 +1159,7 @@ static int cs_ioctl_signal_wait_create_jobs(struct hl_device *hdev, static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, void __user *chunks, u32 num_chunks, - u64 *cs_seq) + u64 *cs_seq, bool timestamp) { struct hl_cs_chunk *cs_chunk_array, *chunk; struct hw_queue_properties *hw_queue_prop; @@ -1259,6 +1264,8 @@ static int cs_ioctl_signal_wait(struct hl_fpriv *hpriv, enum hl_cs_type cs_type, goto free_cs_chunk_array; } + cs->timestamp = !!timestamp; + /* * Save the signal CS fence for later initialization right before * hanging the wait CS on the queue. @@ -1334,10 +1341,11 @@ int hl_cs_ioctl(struct hl_fpriv *hpriv, void *data) case CS_TYPE_WAIT: case CS_TYPE_COLLECTIVE_WAIT: rc = cs_ioctl_signal_wait(hpriv, cs_type, chunks, num_chunks, - &cs_seq); + &cs_seq, args->in.cs_flags & HL_CS_FLAGS_TIMESTAMP); break; default: - rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq); + rc = cs_ioctl_default(hpriv, chunks, num_chunks, &cs_seq, + args->in.cs_flags & HL_CS_FLAGS_TIMESTAMP); break; } @@ -1353,13 +1361,16 @@ out: static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, u64 timeout_us, u64 seq, - enum hl_cs_wait_status *status) + enum hl_cs_wait_status *status, s64 *timestamp) { struct hl_fence *fence; unsigned long timeout; int rc = 0; long completion_rc; + if (timestamp) + *timestamp = 0; + if (timeout_us == MAX_SCHEDULE_TIMEOUT) timeout = timeout_us; else @@ -1382,10 +1393,13 @@ static int _hl_cs_wait_ioctl(struct hl_device *hdev, struct hl_ctx *ctx, wait_for_completion_interruptible_timeout( &fence->completion, timeout); - if (completion_rc > 0) + if (completion_rc > 0) { *status = CS_WAIT_STATUS_COMPLETED; - else + if (timestamp) + *timestamp = ktime_to_ns(fence->timestamp); + } else { *status = CS_WAIT_STATUS_BUSY; + } if (fence->error == -ETIMEDOUT) rc = -ETIMEDOUT; @@ -1411,10 +1425,11 @@ int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data) union hl_wait_cs_args *args = data; enum hl_cs_wait_status status; u64 seq = args->in.seq; + s64 timestamp; int rc; rc = _hl_cs_wait_ioctl(hdev, hpriv->ctx, args->in.timeout_us, seq, - &status); + &status, ×tamp); memset(args, 0, sizeof(*args)); @@ -1439,6 +1454,11 @@ int hl_cs_wait_ioctl(struct hl_fpriv *hpriv, void *data) return rc; } + if (timestamp) { + args->out.flags |= HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD; + args->out.timestamp_nsec = timestamp; + } + switch (status) { case CS_WAIT_STATUS_GONE: args->out.flags |= HL_WAIT_CS_STATUS_FLAG_GONE; diff --git a/drivers/misc/habanalabs/common/habanalabs.h b/drivers/misc/habanalabs/common/habanalabs.h index 9c9c8b24c47a..8e2d164d97e8 100644 --- a/drivers/misc/habanalabs/common/habanalabs.h +++ b/drivers/misc/habanalabs/common/habanalabs.h @@ -481,12 +481,14 @@ struct asic_fixed_properties { * @completion: fence is implemented using completion * @refcount: refcount for this fence * @error: mark this fence with error + * @timestamp: timestamp upon completion * */ struct hl_fence { struct completion completion; struct kref refcount; int error; + ktime_t timestamp; }; /** @@ -1127,6 +1129,7 @@ struct hl_userptr { * @tdr_active: true if TDR was activated for this CS (to prevent * double TDR activation). * @aborted: true if CS was aborted due to some device error. + * @timestamp: true if a timestmap must be captured upon completion */ struct hl_cs { u16 *jobs_in_queue_cnt; @@ -1147,6 +1150,7 @@ struct hl_cs { u8 timedout; u8 tdr_active; u8 aborted; + u8 timestamp; }; /** diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 808d20da024a..6eff4e05eccb 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -592,6 +592,7 @@ struct hl_cs_chunk { #define HL_CS_FLAGS_SIGNAL 0x2 #define HL_CS_FLAGS_WAIT 0x4 #define HL_CS_FLAGS_COLLECTIVE_WAIT 0x8 +#define HL_CS_FLAGS_TIMESTAMP 0x20 #define HL_CS_STATUS_SUCCESS 0 @@ -662,13 +663,16 @@ struct hl_wait_cs_in { #define HL_WAIT_CS_STATUS_ABORTED 3 #define HL_WAIT_CS_STATUS_INTERRUPTED 4 -#define HL_WAIT_CS_STATUS_FLAG_GONE 0x1 +#define HL_WAIT_CS_STATUS_FLAG_GONE 0x1 +#define HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD 0x2 struct hl_wait_cs_out { /* HL_WAIT_CS_STATUS_* */ __u32 status; /* HL_WAIT_CS_STATUS_FLAG* */ __u32 flags; + /* valid only if HL_WAIT_CS_STATUS_FLAG_TIMESTAMP_VLD is set */ + __s64 timestamp_nsec; }; union hl_wait_cs_args { -- cgit v1.2.3 From f44afb5b5a5d04448da843b2fe872e01669bc317 Mon Sep 17 00:00:00 2001 From: Tomer Tayar Date: Wed, 2 Sep 2020 13:43:32 +0300 Subject: habanalabs: Add CB IOCTL opcode to retrieve CB information Add a new CB IOCTL opcode that enables a user to query about a CB and get its usage count. Signed-off-by: Tomer Tayar Reviewed-by: Oded Gabbay Signed-off-by: Oded Gabbay --- drivers/misc/habanalabs/common/command_buffer.c | 38 +++++++++++++++++++++++++ include/uapi/misc/habanalabs.h | 15 ++++++++-- 2 files changed, 51 insertions(+), 2 deletions(-) (limited to 'include') diff --git a/drivers/misc/habanalabs/common/command_buffer.c b/drivers/misc/habanalabs/common/command_buffer.c index 2856bb3423ee..6f6a904ab6ca 100644 --- a/drivers/misc/habanalabs/common/command_buffer.c +++ b/drivers/misc/habanalabs/common/command_buffer.c @@ -375,12 +375,43 @@ int hl_cb_destroy(struct hl_device *hdev, struct hl_cb_mgr *mgr, u64 cb_handle) return rc; } +static int hl_cb_info(struct hl_device *hdev, struct hl_cb_mgr *mgr, + u64 cb_handle, u32 *usage_cnt) +{ + struct hl_cb *cb; + u32 handle; + int rc = 0; + + /* The CB handle was given to user to do mmap, so need to shift it back + * to the value which was allocated by the IDR module. + */ + cb_handle >>= PAGE_SHIFT; + handle = (u32) cb_handle; + + spin_lock(&mgr->cb_lock); + + cb = idr_find(&mgr->cb_handles, handle); + if (!cb) { + dev_err(hdev->dev, + "CB info failed, no match to handle 0x%x\n", handle); + rc = -EINVAL; + goto out; + } + + *usage_cnt = atomic_read(&cb->cs_cnt); + +out: + spin_unlock(&mgr->cb_lock); + return rc; +} + int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) { union hl_cb_args *args = data; struct hl_device *hdev = hpriv->hdev; enum hl_device_status status; u64 handle = 0; + u32 usage_cnt = 0; int rc; if (!hl_device_operational(hdev, &status)) { @@ -413,6 +444,13 @@ int hl_cb_ioctl(struct hl_fpriv *hpriv, void *data) args->in.cb_handle); break; + case HL_CB_OP_INFO: + rc = hl_cb_info(hdev, &hpriv->cb_mgr, args->in.cb_handle, + &usage_cnt); + memset(args, 0, sizeof(*args)); + args->out.usage_cnt = usage_cnt; + break; + default: rc = -ENOTTY; break; diff --git a/include/uapi/misc/habanalabs.h b/include/uapi/misc/habanalabs.h index 6eff4e05eccb..8c15a7d336a0 100644 --- a/include/uapi/misc/habanalabs.h +++ b/include/uapi/misc/habanalabs.h @@ -483,6 +483,8 @@ struct hl_info_args { #define HL_CB_OP_CREATE 0 /* Opcode to destroy previously created command buffer */ #define HL_CB_OP_DESTROY 1 +/* Opcode to retrieve information about a command buffer */ +#define HL_CB_OP_INFO 2 /* 2MB minus 32 bytes for 2xMSG_PROT */ #define HL_MAX_CB_SIZE (0x200000 - 32) @@ -506,8 +508,17 @@ struct hl_cb_in { }; struct hl_cb_out { - /* Handle of CB */ - __u64 cb_handle; + union { + /* Handle of CB */ + __u64 cb_handle; + + /* Information about CB */ + struct { + /* Usage count of CB */ + __u32 usage_cnt; + __u32 pad; + }; + }; }; union hl_cb_args { -- cgit v1.2.3 From 57d9352b6c651b090179f8b223b6681275c64a4f Mon Sep 17 00:00:00 2001 From: Moritz Fischer Date: Sun, 15 Nov 2020 11:51:18 -0800 Subject: fpga: fpga-mgr: Add devm_fpga_mgr_register() API Add a devm_fpga_mgr_register() API that can be used to register a FPGA Manager that was created using devm_fpga_mgr_create(). Introduce a struct fpga_mgr_devres that makes the devres allocation a little bit more readable and gets reused for devm_fpga_mgr_create() devm_fpga_mgr_register(). Reviewed-by: Tom Rix Signed-off-by: Moritz Fischer Link: https://lore.kernel.org/r/20201115195127.284487-2-mdf@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/fpga/fpga-mgr.c | 81 ++++++++++++++++++++++++++++++++++++------- include/linux/fpga/fpga-mgr.h | 2 ++ 2 files changed, 71 insertions(+), 12 deletions(-) (limited to 'include') diff --git a/drivers/fpga/fpga-mgr.c b/drivers/fpga/fpga-mgr.c index f38bab01432e..b85bc47c91a9 100644 --- a/drivers/fpga/fpga-mgr.c +++ b/drivers/fpga/fpga-mgr.c @@ -21,6 +21,10 @@ static DEFINE_IDA(fpga_mgr_ida); static struct class *fpga_mgr_class; +struct fpga_mgr_devres { + struct fpga_manager *mgr; +}; + /** * fpga_image_info_alloc - Allocate a FPGA image info struct * @dev: owning device @@ -625,9 +629,9 @@ EXPORT_SYMBOL_GPL(fpga_mgr_free); static void devm_fpga_mgr_release(struct device *dev, void *res) { - struct fpga_manager *mgr = *(struct fpga_manager **)res; + struct fpga_mgr_devres *dr = res; - fpga_mgr_free(mgr); + fpga_mgr_free(dr->mgr); } /** @@ -651,21 +655,21 @@ struct fpga_manager *devm_fpga_mgr_create(struct device *dev, const char *name, const struct fpga_manager_ops *mops, void *priv) { - struct fpga_manager **ptr, *mgr; + struct fpga_mgr_devres *dr; - ptr = devres_alloc(devm_fpga_mgr_release, sizeof(*ptr), GFP_KERNEL); - if (!ptr) + dr = devres_alloc(devm_fpga_mgr_release, sizeof(*dr), GFP_KERNEL); + if (!dr) return NULL; - mgr = fpga_mgr_create(dev, name, mops, priv); - if (!mgr) { - devres_free(ptr); - } else { - *ptr = mgr; - devres_add(dev, ptr); + dr->mgr = fpga_mgr_create(dev, name, mops, priv); + if (!dr->mgr) { + devres_free(dr); + return NULL; } - return mgr; + devres_add(dev, dr); + + return dr->mgr; } EXPORT_SYMBOL_GPL(devm_fpga_mgr_create); @@ -722,6 +726,59 @@ void fpga_mgr_unregister(struct fpga_manager *mgr) } EXPORT_SYMBOL_GPL(fpga_mgr_unregister); +static int fpga_mgr_devres_match(struct device *dev, void *res, + void *match_data) +{ + struct fpga_mgr_devres *dr = res; + + return match_data == dr->mgr; +} + +static void devm_fpga_mgr_unregister(struct device *dev, void *res) +{ + struct fpga_mgr_devres *dr = res; + + fpga_mgr_unregister(dr->mgr); +} + +/** + * devm_fpga_mgr_register - resource managed variant of fpga_mgr_register() + * @dev: managing device for this FPGA manager + * @mgr: fpga manager struct + * + * This is the devres variant of fpga_mgr_register() for which the unregister + * function will be called automatically when the managing device is detached. + */ +int devm_fpga_mgr_register(struct device *dev, struct fpga_manager *mgr) +{ + struct fpga_mgr_devres *dr; + int ret; + + /* + * Make sure that the struct fpga_manager * that is passed in is + * managed itself. + */ + if (WARN_ON(!devres_find(dev, devm_fpga_mgr_release, + fpga_mgr_devres_match, mgr))) + return -EINVAL; + + dr = devres_alloc(devm_fpga_mgr_unregister, sizeof(*dr), GFP_KERNEL); + if (!dr) + return -ENOMEM; + + ret = fpga_mgr_register(mgr); + if (ret) { + devres_free(dr); + return ret; + } + + dr->mgr = mgr; + devres_add(dev, dr); + + return 0; +} +EXPORT_SYMBOL_GPL(devm_fpga_mgr_register); + static void fpga_mgr_dev_release(struct device *dev) { } diff --git a/include/linux/fpga/fpga-mgr.h b/include/linux/fpga/fpga-mgr.h index e8ca62b2cb5b..2bc3030a69e5 100644 --- a/include/linux/fpga/fpga-mgr.h +++ b/include/linux/fpga/fpga-mgr.h @@ -198,6 +198,8 @@ void fpga_mgr_free(struct fpga_manager *mgr); int fpga_mgr_register(struct fpga_manager *mgr); void fpga_mgr_unregister(struct fpga_manager *mgr); +int devm_fpga_mgr_register(struct device *dev, struct fpga_manager *mgr); + struct fpga_manager *devm_fpga_mgr_create(struct device *dev, const char *name, const struct fpga_manager_ops *mops, void *priv); -- cgit v1.2.3 From 85261c1ff156eb60fc26c378748387f2e85c6878 Mon Sep 17 00:00:00 2001 From: Alexander Usyskin Date: Mon, 16 Nov 2020 14:56:11 +0200 Subject: mei: bus: add vtag support Add API to support vtag in communication on mei bus. Add mei_cldev_send_vtag, mei_cldev_recv_vtag and mei_cldev_recv_nonblock_vtag functions to allow sending a message with vtag set and to receive vtag of an incoming message. Cc: Sean Z Huang Signed-off-by: Alexander Usyskin Signed-off-by: Tomas Winkler Link: https://lore.kernel.org/r/20201116125612.1660971-1-tomas.winkler@intel.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/mei/bus-fixup.c | 13 +++--- drivers/misc/mei/bus.c | 101 +++++++++++++++++++++++++++++++++++-------- drivers/misc/mei/client.c | 6 ++- drivers/misc/mei/mei_dev.h | 4 +- include/linux/mei_cl_bus.h | 6 +++ 5 files changed, 104 insertions(+), 26 deletions(-) (limited to 'include') diff --git a/drivers/misc/mei/bus-fixup.c b/drivers/misc/mei/bus-fixup.c index 4e30fa98fe7d..6cc3145bb716 100644 --- a/drivers/misc/mei/bus-fixup.c +++ b/drivers/misc/mei/bus-fixup.c @@ -148,7 +148,7 @@ static int mei_osver(struct mei_cl_device *cldev) os_ver = (struct mei_os_ver *)fwcaps->data; os_ver->os_type = OSTYPE_LINUX; - return __mei_cl_send(cldev->cl, buf, size, mode); + return __mei_cl_send(cldev->cl, buf, size, 0, mode); } #define MKHI_FWVER_BUF_LEN (sizeof(struct mkhi_msg_hdr) + \ @@ -169,7 +169,7 @@ static int mei_fwver(struct mei_cl_device *cldev) req.hdr.group_id = MKHI_GEN_GROUP_ID; req.hdr.command = MKHI_GEN_GET_FW_VERSION_CMD; - ret = __mei_cl_send(cldev->cl, (u8 *)&req, sizeof(req), + ret = __mei_cl_send(cldev->cl, (u8 *)&req, sizeof(req), 0, MEI_CL_IO_TX_BLOCKING); if (ret < 0) { dev_err(&cldev->dev, "Could not send ReqFWVersion cmd\n"); @@ -177,7 +177,7 @@ static int mei_fwver(struct mei_cl_device *cldev) } ret = 0; - bytes_recv = __mei_cl_recv(cldev->cl, buf, sizeof(buf), 0, + bytes_recv = __mei_cl_recv(cldev->cl, buf, sizeof(buf), NULL, 0, MKHI_RCV_TIMEOUT); if (bytes_recv < 0 || (size_t)bytes_recv < MKHI_FWVER_LEN(1)) { /* @@ -324,13 +324,15 @@ static int mei_nfc_if_version(struct mei_cl *cl, }; struct mei_nfc_reply *reply = NULL; size_t if_version_length; + u8 vtag; int bytes_recv, ret; bus = cl->dev; WARN_ON(mutex_is_locked(&bus->device_lock)); - ret = __mei_cl_send(cl, (u8 *)&cmd, sizeof(cmd), MEI_CL_IO_TX_BLOCKING); + ret = __mei_cl_send(cl, (u8 *)&cmd, sizeof(cmd), 0, + MEI_CL_IO_TX_BLOCKING); if (ret < 0) { dev_err(bus->dev, "Could not send IF version cmd\n"); return ret; @@ -344,7 +346,8 @@ static int mei_nfc_if_version(struct mei_cl *cl, return -ENOMEM; ret = 0; - bytes_recv = __mei_cl_recv(cl, (u8 *)reply, if_version_length, 0, 0); + bytes_recv = __mei_cl_recv(cl, (u8 *)reply, if_version_length, &vtag, + 0, 0); if (bytes_recv < 0 || (size_t)bytes_recv < if_version_length) { dev_err(bus->dev, "Could not read IF version\n"); ret = -EIO; diff --git a/drivers/misc/mei/bus.c b/drivers/misc/mei/bus.c index 7fe48baa103a..2907db260fba 100644 --- a/drivers/misc/mei/bus.c +++ b/drivers/misc/mei/bus.c @@ -26,11 +26,12 @@ * @cl: host client * @buf: buffer to send * @length: buffer length + * @vtag: virtual tag * @mode: sending mode * * Return: written size bytes or < 0 on error */ -ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, +ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, u8 vtag, unsigned int mode) { struct mei_device *bus; @@ -86,6 +87,7 @@ ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, rets = -ENOMEM; goto out; } + cb->vtag = vtag; cb->internal = !!(mode & MEI_CL_IO_TX_INTERNAL); cb->blocking = !!(mode & MEI_CL_IO_TX_BLOCKING); @@ -106,11 +108,12 @@ out: * @buf: buffer to receive * @length: buffer length * @mode: io mode + * @vtag: virtual tag * @timeout: recv timeout, 0 for infinite timeout * * Return: read size in bytes of < 0 on error */ -ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length, +ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length, u8 *vtag, unsigned int mode, unsigned long timeout) { struct mei_device *bus; @@ -196,6 +199,8 @@ copy: r_length = min_t(size_t, length, cb->buf_idx); memcpy(buf, cb->buf.data, r_length); rets = r_length; + if (vtag) + *vtag = cb->vtag; free: mei_cl_del_rd_completed(cl, cb); @@ -206,40 +211,87 @@ out: } /** - * mei_cldev_send - me device send (write) + * mei_cldev_send_vtag - me device send with vtag (write) * * @cldev: me client device * @buf: buffer to send * @length: buffer length + * @vtag: virtual tag * - * Return: written size in bytes or < 0 on error + * Return: + * * written size in bytes + * * < 0 on error */ -ssize_t mei_cldev_send(struct mei_cl_device *cldev, u8 *buf, size_t length) + +ssize_t mei_cldev_send_vtag(struct mei_cl_device *cldev, u8 *buf, size_t length, + u8 vtag) { struct mei_cl *cl = cldev->cl; - return __mei_cl_send(cl, buf, length, MEI_CL_IO_TX_BLOCKING); + return __mei_cl_send(cl, buf, length, vtag, MEI_CL_IO_TX_BLOCKING); } -EXPORT_SYMBOL_GPL(mei_cldev_send); +EXPORT_SYMBOL_GPL(mei_cldev_send_vtag); /** - * mei_cldev_recv_nonblock - non block client receive (read) + * mei_cldev_recv_vtag - client receive with vtag (read) * * @cldev: me client device * @buf: buffer to receive * @length: buffer length + * @vtag: virtual tag * - * Return: read size in bytes of < 0 on error - * -EAGAIN if function will block. + * Return: + * * read size in bytes + * * < 0 on error */ -ssize_t mei_cldev_recv_nonblock(struct mei_cl_device *cldev, u8 *buf, - size_t length) + +ssize_t mei_cldev_recv_vtag(struct mei_cl_device *cldev, u8 *buf, size_t length, + u8 *vtag) { struct mei_cl *cl = cldev->cl; - return __mei_cl_recv(cl, buf, length, MEI_CL_IO_RX_NONBLOCK, 0); + return __mei_cl_recv(cl, buf, length, vtag, 0, 0); } -EXPORT_SYMBOL_GPL(mei_cldev_recv_nonblock); +EXPORT_SYMBOL_GPL(mei_cldev_recv_vtag); + +/** + * mei_cldev_recv_nonblock_vtag - non block client receive with vtag (read) + * + * @cldev: me client device + * @buf: buffer to receive + * @length: buffer length + * @vtag: virtual tag + * + * Return: + * * read size in bytes + * * -EAGAIN if function will block. + * * < 0 on other error + */ +ssize_t mei_cldev_recv_nonblock_vtag(struct mei_cl_device *cldev, u8 *buf, + size_t length, u8 *vtag) +{ + struct mei_cl *cl = cldev->cl; + + return __mei_cl_recv(cl, buf, length, vtag, MEI_CL_IO_RX_NONBLOCK, 0); +} +EXPORT_SYMBOL_GPL(mei_cldev_recv_nonblock_vtag); + +/** + * mei_cldev_send - me device send (write) + * + * @cldev: me client device + * @buf: buffer to send + * @length: buffer length + * + * Return: + * * written size in bytes + * * < 0 on error + */ +ssize_t mei_cldev_send(struct mei_cl_device *cldev, u8 *buf, size_t length) +{ + return mei_cldev_send_vtag(cldev, buf, length, 0); +} +EXPORT_SYMBOL_GPL(mei_cldev_send); /** * mei_cldev_recv - client receive (read) @@ -252,12 +304,27 @@ EXPORT_SYMBOL_GPL(mei_cldev_recv_nonblock); */ ssize_t mei_cldev_recv(struct mei_cl_device *cldev, u8 *buf, size_t length) { - struct mei_cl *cl = cldev->cl; - - return __mei_cl_recv(cl, buf, length, 0, 0); + return mei_cldev_recv_vtag(cldev, buf, length, NULL); } EXPORT_SYMBOL_GPL(mei_cldev_recv); +/** + * mei_cldev_recv_nonblock - non block client receive (read) + * + * @cldev: me client device + * @buf: buffer to receive + * @length: buffer length + * + * Return: read size in bytes of < 0 on error + * -EAGAIN if function will block. + */ +ssize_t mei_cldev_recv_nonblock(struct mei_cl_device *cldev, u8 *buf, + size_t length) +{ + return mei_cldev_recv_nonblock_vtag(cldev, buf, length, NULL); +} +EXPORT_SYMBOL_GPL(mei_cldev_recv_nonblock); + /** * mei_cl_bus_rx_work - dispatch rx event for a bus device * diff --git a/drivers/misc/mei/client.c b/drivers/misc/mei/client.c index d5c3f7d54634..a56d41321f32 100644 --- a/drivers/misc/mei/client.c +++ b/drivers/misc/mei/client.c @@ -1306,7 +1306,7 @@ struct mei_cl_vtag *mei_cl_vtag_alloc(struct file *fp, u8 vtag) * mei_cl_fp_by_vtag - obtain the file pointer by vtag * * @cl: host client - * @vtag: vm tag + * @vtag: virtual tag * * Return: * * A file pointer - on success @@ -1317,7 +1317,9 @@ const struct file *mei_cl_fp_by_vtag(const struct mei_cl *cl, u8 vtag) struct mei_cl_vtag *vtag_l; list_for_each_entry(vtag_l, &cl->vtag_map, list) - if (vtag_l->vtag == vtag) + /* The client on bus has one fixed fp */ + if ((cl->cldev && mei_cldev_enabled(cl->cldev)) || + vtag_l->vtag == vtag) return vtag_l->fp; return ERR_PTR(-ENOENT); diff --git a/drivers/misc/mei/mei_dev.h b/drivers/misc/mei/mei_dev.h index 2f4cc1a8aae8..8c395bfdf6f3 100644 --- a/drivers/misc/mei/mei_dev.h +++ b/drivers/misc/mei/mei_dev.h @@ -340,9 +340,9 @@ struct mei_hw_ops { /* MEI bus API*/ void mei_cl_bus_rescan_work(struct work_struct *work); void mei_cl_bus_dev_fixup(struct mei_cl_device *dev); -ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, +ssize_t __mei_cl_send(struct mei_cl *cl, u8 *buf, size_t length, u8 vtag, unsigned int mode); -ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length, +ssize_t __mei_cl_recv(struct mei_cl *cl, u8 *buf, size_t length, u8 *vtag, unsigned int mode, unsigned long timeout); bool mei_cl_bus_rx_event(struct mei_cl *cl); bool mei_cl_bus_notify_event(struct mei_cl *cl); diff --git a/include/linux/mei_cl_bus.h b/include/linux/mei_cl_bus.h index 52aa4821093a..959ad7d850b4 100644 --- a/include/linux/mei_cl_bus.h +++ b/include/linux/mei_cl_bus.h @@ -95,6 +95,12 @@ ssize_t mei_cldev_send(struct mei_cl_device *cldev, u8 *buf, size_t length); ssize_t mei_cldev_recv(struct mei_cl_device *cldev, u8 *buf, size_t length); ssize_t mei_cldev_recv_nonblock(struct mei_cl_device *cldev, u8 *buf, size_t length); +ssize_t mei_cldev_send_vtag(struct mei_cl_device *cldev, u8 *buf, size_t length, + u8 vtag); +ssize_t mei_cldev_recv_vtag(struct mei_cl_device *cldev, u8 *buf, size_t length, + u8 *vtag); +ssize_t mei_cldev_recv_nonblock_vtag(struct mei_cl_device *cldev, u8 *buf, + size_t length, u8 *vtag); int mei_cldev_register_rx_cb(struct mei_cl_device *cldev, mei_cldev_cb_t rx_cb); int mei_cldev_register_notif_cb(struct mei_cl_device *cldev, -- cgit v1.2.3 From 0f966cba95c78029f491b433ea95ff38f414a761 Mon Sep 17 00:00:00 2001 From: Todd Kjos Date: Fri, 20 Nov 2020 15:37:43 -0800 Subject: binder: add flag to clear buffer on txn complete Add a per-transaction flag to indicate that the buffer must be cleared when the transaction is complete to prevent copies of sensitive data from being preserved in memory. Signed-off-by: Todd Kjos Link: https://lore.kernel.org/r/20201120233743.3617529-1-tkjos@google.com Cc: stable Signed-off-by: Greg Kroah-Hartman --- drivers/android/binder.c | 1 + drivers/android/binder_alloc.c | 48 +++++++++++++++++++++++++++++++++++++ drivers/android/binder_alloc.h | 4 +++- include/uapi/linux/android/binder.h | 1 + 4 files changed, 53 insertions(+), 1 deletion(-) (limited to 'include') diff --git a/drivers/android/binder.c b/drivers/android/binder.c index 20b08f52e788..1338209f9f86 100644 --- a/drivers/android/binder.c +++ b/drivers/android/binder.c @@ -2756,6 +2756,7 @@ static void binder_transaction(struct binder_proc *proc, t->buffer->debug_id = t->debug_id; t->buffer->transaction = t; t->buffer->target_node = target_node; + t->buffer->clear_on_free = !!(t->flags & TF_CLEAR_BUF); trace_binder_transaction_alloc_buf(t->buffer); if (binder_alloc_copy_user_to_buffer( diff --git a/drivers/android/binder_alloc.c b/drivers/android/binder_alloc.c index 2f846b7ae8b8..7caf74ad2405 100644 --- a/drivers/android/binder_alloc.c +++ b/drivers/android/binder_alloc.c @@ -696,6 +696,8 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, binder_insert_free_buffer(alloc, buffer); } +static void binder_alloc_clear_buf(struct binder_alloc *alloc, + struct binder_buffer *buffer); /** * binder_alloc_free_buf() - free a binder buffer * @alloc: binder_alloc for this proc @@ -706,6 +708,18 @@ static void binder_free_buf_locked(struct binder_alloc *alloc, void binder_alloc_free_buf(struct binder_alloc *alloc, struct binder_buffer *buffer) { + /* + * We could eliminate the call to binder_alloc_clear_buf() + * from binder_alloc_deferred_release() by moving this to + * binder_alloc_free_buf_locked(). However, that could + * increase contention for the alloc mutex if clear_on_free + * is used frequently for large buffers. The mutex is not + * needed for correctness here. + */ + if (buffer->clear_on_free) { + binder_alloc_clear_buf(alloc, buffer); + buffer->clear_on_free = false; + } mutex_lock(&alloc->mutex); binder_free_buf_locked(alloc, buffer); mutex_unlock(&alloc->mutex); @@ -802,6 +816,10 @@ void binder_alloc_deferred_release(struct binder_alloc *alloc) /* Transaction should already have been freed */ BUG_ON(buffer->transaction); + if (buffer->clear_on_free) { + binder_alloc_clear_buf(alloc, buffer); + buffer->clear_on_free = false; + } binder_free_buf_locked(alloc, buffer); buffers++; } @@ -1135,6 +1153,36 @@ static struct page *binder_alloc_get_page(struct binder_alloc *alloc, return lru_page->page_ptr; } +/** + * binder_alloc_clear_buf() - zero out buffer + * @alloc: binder_alloc for this proc + * @buffer: binder buffer to be cleared + * + * memset the given buffer to 0 + */ +static void binder_alloc_clear_buf(struct binder_alloc *alloc, + struct binder_buffer *buffer) +{ + size_t bytes = binder_alloc_buffer_size(alloc, buffer); + binder_size_t buffer_offset = 0; + + while (bytes) { + unsigned long size; + struct page *page; + pgoff_t pgoff; + void *kptr; + + page = binder_alloc_get_page(alloc, buffer, + buffer_offset, &pgoff); + size = min_t(size_t, bytes, PAGE_SIZE - pgoff); + kptr = kmap(page) + pgoff; + memset(kptr, 0, size); + kunmap(page); + bytes -= size; + buffer_offset += size; + } +} + /** * binder_alloc_copy_user_to_buffer() - copy src user to tgt user * @alloc: binder_alloc for this proc diff --git a/drivers/android/binder_alloc.h b/drivers/android/binder_alloc.h index 55d8b4106766..6e8e001381af 100644 --- a/drivers/android/binder_alloc.h +++ b/drivers/android/binder_alloc.h @@ -23,6 +23,7 @@ struct binder_transaction; * @entry: entry alloc->buffers * @rb_node: node for allocated_buffers/free_buffers rb trees * @free: %true if buffer is free + * @clear_on_free: %true if buffer must be zeroed after use * @allow_user_free: %true if user is allowed to free buffer * @async_transaction: %true if buffer is in use for an async txn * @debug_id: unique ID for debugging @@ -41,9 +42,10 @@ struct binder_buffer { struct rb_node rb_node; /* free entry by size or allocated entry */ /* by address */ unsigned free:1; + unsigned clear_on_free:1; unsigned allow_user_free:1; unsigned async_transaction:1; - unsigned debug_id:29; + unsigned debug_id:28; struct binder_transaction *transaction; diff --git a/include/uapi/linux/android/binder.h b/include/uapi/linux/android/binder.h index f1ce2c4c077e..ec84ad106568 100644 --- a/include/uapi/linux/android/binder.h +++ b/include/uapi/linux/android/binder.h @@ -248,6 +248,7 @@ enum transaction_flags { TF_ROOT_OBJECT = 0x04, /* contents are the component's root object */ TF_STATUS_CODE = 0x08, /* contents are a 32-bit status code */ TF_ACCEPT_FDS = 0x10, /* allow replies with file descriptors */ + TF_CLEAR_BUF = 0x20, /* clear buffer on txn complete */ }; struct binder_transaction_data { -- cgit v1.2.3 From 5b4258f6721f41b092c63f6ee71be76e9616718b Mon Sep 17 00:00:00 2001 From: Ricky Wu Date: Wed, 2 Dec 2020 14:58:57 +0800 Subject: misc: rtsx: rts5249 support runtime PM rtsx_pcr: add callback functions to support runtime PM add delay_work to put device to D3 after idle over 10 sec rts5249: add extra init flow for rtd3 and set rtd3_en from config setting rtsx_pci_sdmmc: child device support autosuspend Signed-off-by: Ricky Wu Link: https://lore.kernel.org/r/20201202065857.19412-1-ricky_wu@realtek.com Signed-off-by: Greg Kroah-Hartman --- drivers/misc/cardreader/rts5249.c | 26 +++++++-- drivers/misc/cardreader/rtsx_pcr.c | 106 ++++++++++++++++++++++++++++++++++++- drivers/misc/cardreader/rtsx_pcr.h | 1 + drivers/mmc/host/rtsx_pci_sdmmc.c | 16 ++++++ include/linux/rtsx_pci.h | 2 + 5 files changed, 145 insertions(+), 6 deletions(-) (limited to 'include') diff --git a/drivers/misc/cardreader/rts5249.c b/drivers/misc/cardreader/rts5249.c index b85279f1fc5e..b2676e7f5027 100644 --- a/drivers/misc/cardreader/rts5249.c +++ b/drivers/misc/cardreader/rts5249.c @@ -73,6 +73,9 @@ static void rtsx_base_fetch_vendor_settings(struct rtsx_pcr *pcr) pci_read_config_dword(pdev, PCR_SETTING_REG2, ®); pcr_dbg(pcr, "Cfg 0x%x: 0x%x\n", PCR_SETTING_REG2, reg); + + pcr->rtd3_en = rtsx_reg_to_rtd3_uhsii(reg); + if (rtsx_check_mmc_support(reg)) pcr->extra_caps |= EXTRA_CAPS_NO_MMC; pcr->sd30_drive_sel_3v3 = rtsx_reg_to_sd30_drive_sel_3v3(reg); @@ -278,15 +281,28 @@ static int rts5249_extra_init_hw(struct rtsx_pcr *pcr) rtsx_pci_send_cmd(pcr, CMD_TIMEOUT_DEF); - if (CHK_PCI_PID(pcr, PID_524A) || CHK_PCI_PID(pcr, PID_525A)) { + if (CHK_PCI_PID(pcr, PID_524A) || CHK_PCI_PID(pcr, PID_525A)) rtsx_pci_write_register(pcr, REG_VREF, PWD_SUSPND_EN, PWD_SUSPND_EN); - rtsx_pci_write_register(pcr, RTS524A_PM_CTRL3, 0x01, 0x00); - rtsx_pci_write_register(pcr, RTS524A_PME_FORCE_CTL, 0x30, 0x20); + + if (pcr->rtd3_en) { + if (CHK_PCI_PID(pcr, PID_524A) || CHK_PCI_PID(pcr, PID_525A)) { + rtsx_pci_write_register(pcr, RTS524A_PM_CTRL3, 0x01, 0x01); + rtsx_pci_write_register(pcr, RTS524A_PME_FORCE_CTL, 0x30, 0x30); + } else { + rtsx_pci_write_register(pcr, PM_CTRL3, 0x01, 0x01); + rtsx_pci_write_register(pcr, PME_FORCE_CTL, 0xFF, 0x33); + } } else { - rtsx_pci_write_register(pcr, PME_FORCE_CTL, 0xFF, 0x30); - rtsx_pci_write_register(pcr, PM_CTRL3, 0x01, 0x00); + if (CHK_PCI_PID(pcr, PID_524A) || CHK_PCI_PID(pcr, PID_525A)) { + rtsx_pci_write_register(pcr, RTS524A_PM_CTRL3, 0x01, 0x00); + rtsx_pci_write_register(pcr, RTS524A_PME_FORCE_CTL, 0x30, 0x20); + } else { + rtsx_pci_write_register(pcr, PME_FORCE_CTL, 0xFF, 0x30); + rtsx_pci_write_register(pcr, PM_CTRL3, 0x01, 0x00); + } } + /* * If u_force_clkreq_0 is enabled, CLKREQ# PIN will be forced * to drive low, and we forcibly request clock. diff --git a/drivers/misc/cardreader/rtsx_pcr.c b/drivers/misc/cardreader/rtsx_pcr.c index 3612063cab09..2700d1997750 100644 --- a/drivers/misc/cardreader/rtsx_pcr.c +++ b/drivers/misc/cardreader/rtsx_pcr.c @@ -20,6 +20,8 @@ #include #include #include +#include +#include #include "rtsx_pcr.h" #include "rts5261.h" @@ -150,6 +152,12 @@ void rtsx_pci_start_run(struct rtsx_pcr *pcr) if (pcr->remove_pci) return; + if (pcr->rtd3_en) + if (pcr->is_runtime_suspended) { + pm_runtime_get(&(pcr->pci->dev)); + pcr->is_runtime_suspended = false; + } + if (pcr->state != PDEV_STAT_RUN) { pcr->state = PDEV_STAT_RUN; if (pcr->ops->enable_auto_blink) @@ -1081,6 +1089,16 @@ static void rtsx_pm_power_saving(struct rtsx_pcr *pcr) rtsx_comm_pm_power_saving(pcr); } +static void rtsx_pci_rtd3_work(struct work_struct *work) +{ + struct delayed_work *dwork = to_delayed_work(work); + struct rtsx_pcr *pcr = container_of(dwork, struct rtsx_pcr, rtd3_work); + + pcr_dbg(pcr, "--> %s\n", __func__); + if (!pcr->is_runtime_suspended) + pm_runtime_put(&(pcr->pci->dev)); +} + static void rtsx_pci_idle_work(struct work_struct *work) { struct delayed_work *dwork = to_delayed_work(work); @@ -1100,6 +1118,9 @@ static void rtsx_pci_idle_work(struct work_struct *work) rtsx_pm_power_saving(pcr); mutex_unlock(&pcr->pcr_mutex); + + if (pcr->rtd3_en) + mod_delayed_work(system_wq, &pcr->rtd3_work, msecs_to_jiffies(10000)); } static void rtsx_base_force_power_down(struct rtsx_pcr *pcr, u8 pm_state) @@ -1579,6 +1600,15 @@ static int rtsx_pci_probe(struct pci_dev *pcidev, rtsx_pcr_cells[i].platform_data = handle; rtsx_pcr_cells[i].pdata_size = sizeof(*handle); } + + if (pcr->rtd3_en) { + INIT_DELAYED_WORK(&pcr->rtd3_work, rtsx_pci_rtd3_work); + pm_runtime_allow(&pcidev->dev); + pm_runtime_enable(&pcidev->dev); + pcr->is_runtime_suspended = false; + } + + ret = mfd_add_devices(&pcidev->dev, pcr->id, rtsx_pcr_cells, ARRAY_SIZE(rtsx_pcr_cells), NULL, 0, NULL); if (ret < 0) @@ -1616,6 +1646,9 @@ static void rtsx_pci_remove(struct pci_dev *pcidev) struct pcr_handle *handle = pci_get_drvdata(pcidev); struct rtsx_pcr *pcr = handle->pcr; + if (pcr->rtd3_en) + pm_runtime_get_noresume(&pcr->pci->dev); + pcr->remove_pci = true; /* Disable interrupts at the pcr level */ @@ -1626,6 +1659,8 @@ static void rtsx_pci_remove(struct pci_dev *pcidev) cancel_delayed_work_sync(&pcr->carddet_work); cancel_delayed_work_sync(&pcr->idle_work); + if (pcr->rtd3_en) + cancel_delayed_work_sync(&pcr->rtd3_work); mfd_remove_devices(&pcidev->dev); @@ -1643,6 +1678,11 @@ static void rtsx_pci_remove(struct pci_dev *pcidev) idr_remove(&rtsx_pci_idr, pcr->id); spin_unlock(&rtsx_pci_lock); + if (pcr->rtd3_en) { + pm_runtime_disable(&pcr->pci->dev); + pm_runtime_put_noidle(&pcr->pci->dev); + } + kfree(pcr->slots); kfree(pcr); kfree(handle); @@ -1724,13 +1764,77 @@ static void rtsx_pci_shutdown(struct pci_dev *pcidev) pci_disable_msi(pcr->pci); } +static int rtsx_pci_runtime_suspend(struct device *device) +{ + struct pci_dev *pcidev = to_pci_dev(device); + struct pcr_handle *handle; + struct rtsx_pcr *pcr; + + handle = pci_get_drvdata(pcidev); + pcr = handle->pcr; + dev_dbg(&(pcidev->dev), "--> %s\n", __func__); + + cancel_delayed_work(&pcr->carddet_work); + cancel_delayed_work(&pcr->rtd3_work); + cancel_delayed_work(&pcr->idle_work); + + mutex_lock(&pcr->pcr_mutex); + rtsx_pci_power_off(pcr, HOST_ENTER_S3); + + free_irq(pcr->irq, (void *)pcr); + + mutex_unlock(&pcr->pcr_mutex); + + pcr->is_runtime_suspended = true; + + return 0; +} + +static int rtsx_pci_runtime_resume(struct device *device) +{ + struct pci_dev *pcidev = to_pci_dev(device); + struct pcr_handle *handle; + struct rtsx_pcr *pcr; + int ret = 0; + + handle = pci_get_drvdata(pcidev); + pcr = handle->pcr; + dev_dbg(&(pcidev->dev), "--> %s\n", __func__); + + mutex_lock(&pcr->pcr_mutex); + + rtsx_pci_write_register(pcr, HOST_SLEEP_STATE, 0x03, 0x00); + rtsx_pci_acquire_irq(pcr); + synchronize_irq(pcr->irq); + + if (pcr->ops->fetch_vendor_settings) + pcr->ops->fetch_vendor_settings(pcr); + + rtsx_pci_init_hw(pcr); + + if (pcr->slots[RTSX_SD_CARD].p_dev != NULL) { + pcr->slots[RTSX_SD_CARD].card_event( + pcr->slots[RTSX_SD_CARD].p_dev); + } + + schedule_delayed_work(&pcr->idle_work, msecs_to_jiffies(200)); + + mutex_unlock(&pcr->pcr_mutex); + return ret; +} + #else /* CONFIG_PM */ #define rtsx_pci_shutdown NULL +#define rtsx_pci_runtime_suspend NULL +#define rtsx_pic_runtime_resume NULL #endif /* CONFIG_PM */ -static SIMPLE_DEV_PM_OPS(rtsx_pci_pm_ops, rtsx_pci_suspend, rtsx_pci_resume); +static const struct dev_pm_ops rtsx_pci_pm_ops = { + SET_SYSTEM_SLEEP_PM_OPS(rtsx_pci_suspend, rtsx_pci_resume) + SET_RUNTIME_PM_OPS(rtsx_pci_runtime_suspend, rtsx_pci_runtime_resume, NULL) +}; static struct pci_driver rtsx_pci_driver = { .name = DRV_NAME_RTSX_PCI, diff --git a/drivers/misc/cardreader/rtsx_pcr.h b/drivers/misc/cardreader/rtsx_pcr.h index fe5f4ca0f937..daf057c4eea6 100644 --- a/drivers/misc/cardreader/rtsx_pcr.h +++ b/drivers/misc/cardreader/rtsx_pcr.h @@ -90,6 +90,7 @@ static inline u8 map_sd_drive(int idx) #define rtsx_check_mmc_support(reg) ((reg) & 0x10) #define rtsx_reg_to_rtd3(reg) ((reg) & 0x02) +#define rtsx_reg_to_rtd3_uhsii(reg) ((reg) & 0x04) #define rtsx_reg_to_aspm(reg) (((reg) >> 28) & 0x03) #define rtsx_reg_to_sd30_drive_sel_1v8(reg) (((reg) >> 26) & 0x03) #define rtsx_reg_to_sd30_drive_sel_3v3(reg) (((reg) >> 5) & 0x03) diff --git a/drivers/mmc/host/rtsx_pci_sdmmc.c b/drivers/mmc/host/rtsx_pci_sdmmc.c index eb395e144207..a7b5ad17bcf5 100644 --- a/drivers/mmc/host/rtsx_pci_sdmmc.c +++ b/drivers/mmc/host/rtsx_pci_sdmmc.c @@ -20,6 +20,7 @@ #include #include #include +#include struct realtek_pci_sdmmc { struct platform_device *pdev; @@ -1343,6 +1344,7 @@ static void init_extra_caps(struct realtek_pci_sdmmc *host) static void realtek_init_host(struct realtek_pci_sdmmc *host) { struct mmc_host *mmc = host->mmc; + struct rtsx_pcr *pcr = host->pcr; mmc->f_min = 250000; mmc->f_max = 208000000; @@ -1350,6 +1352,8 @@ static void realtek_init_host(struct realtek_pci_sdmmc *host) mmc->caps = MMC_CAP_4_BIT_DATA | MMC_CAP_SD_HIGHSPEED | MMC_CAP_MMC_HIGHSPEED | MMC_CAP_BUS_WIDTH_TEST | MMC_CAP_UHS_SDR12 | MMC_CAP_UHS_SDR25; + if (pcr->rtd3_en) + mmc->caps = mmc->caps | MMC_CAP_AGGRESSIVE_PM; mmc->caps2 = MMC_CAP2_NO_PRESCAN_POWERUP | MMC_CAP2_FULL_PWR_CYCLE; mmc->max_current_330 = 400; mmc->max_current_180 = 800; @@ -1407,6 +1411,13 @@ static int rtsx_pci_sdmmc_drv_probe(struct platform_device *pdev) realtek_init_host(host); + if (pcr->rtd3_en) { + pm_runtime_set_autosuspend_delay(&pdev->dev, 5000); + pm_runtime_use_autosuspend(&pdev->dev); + pm_runtime_enable(&pdev->dev); + } + + mmc_add_host(mmc); return 0; @@ -1426,6 +1437,11 @@ static int rtsx_pci_sdmmc_drv_remove(struct platform_device *pdev) pcr->slots[RTSX_SD_CARD].card_event = NULL; mmc = host->mmc; + if (pcr->rtd3_en) { + pm_runtime_dont_use_autosuspend(&pdev->dev); + pm_runtime_disable(&pdev->dev); + } + cancel_work_sync(&host->work); mutex_lock(&host->host_mutex); diff --git a/include/linux/rtsx_pci.h b/include/linux/rtsx_pci.h index 745f5e73f99a..f895ccabbe29 100644 --- a/include/linux/rtsx_pci.h +++ b/include/linux/rtsx_pci.h @@ -1174,6 +1174,7 @@ struct rtsx_pcr { struct delayed_work carddet_work; struct delayed_work idle_work; + struct delayed_work rtd3_work; spinlock_t lock; struct mutex pcr_mutex; @@ -1183,6 +1184,7 @@ struct rtsx_pcr { unsigned int cur_clock; bool remove_pci; bool msi_en; + bool is_runtime_suspended; #define EXTRA_CAPS_SD_SDR50 (1 << 0) #define EXTRA_CAPS_SD_SDR104 (1 << 1) -- cgit v1.2.3 From d40c2d4ed62df64ce603c208bceff25245380157 Mon Sep 17 00:00:00 2001 From: Hsin-Hsiung Wang Date: Wed, 9 Dec 2020 18:33:43 -0800 Subject: spmi: Add driver shutdown support Add new shutdown() method. Use it in the standard driver model style. Link: https://lore.kernel.org/r/1603187810-30481-2-git-send-email-hsin-hsiung.wang@mediatek.com Signed-off-by: Hsin-Hsiung Wang Signed-off-by: Stephen Boyd Link: https://lore.kernel.org/r/20201210023344.2838141-4-sboyd@kernel.org Signed-off-by: Greg Kroah-Hartman --- drivers/spmi/spmi.c | 9 +++++++++ include/linux/spmi.h | 1 + 2 files changed, 10 insertions(+) (limited to 'include') diff --git a/drivers/spmi/spmi.c b/drivers/spmi/spmi.c index 253340e10dab..51f5aeb65b3b 100644 --- a/drivers/spmi/spmi.c +++ b/drivers/spmi/spmi.c @@ -359,6 +359,14 @@ static int spmi_drv_remove(struct device *dev) return 0; } +static void spmi_drv_shutdown(struct device *dev) +{ + const struct spmi_driver *sdrv = to_spmi_driver(dev->driver); + + if (sdrv && sdrv->shutdown) + sdrv->shutdown(to_spmi_device(dev)); +} + static int spmi_drv_uevent(struct device *dev, struct kobj_uevent_env *env) { int ret; @@ -375,6 +383,7 @@ static struct bus_type spmi_bus_type = { .match = spmi_device_match, .probe = spmi_drv_probe, .remove = spmi_drv_remove, + .shutdown = spmi_drv_shutdown, .uevent = spmi_drv_uevent, }; diff --git a/include/linux/spmi.h b/include/linux/spmi.h index 394a3f68bad5..729bcbf9f5ad 100644 --- a/include/linux/spmi.h +++ b/include/linux/spmi.h @@ -138,6 +138,7 @@ struct spmi_driver { struct device_driver driver; int (*probe)(struct spmi_device *sdev); void (*remove)(struct spmi_device *sdev); + void (*shutdown)(struct spmi_device *sdev); }; static inline struct spmi_driver *to_spmi_driver(struct device_driver *d) -- cgit v1.2.3 From 1c12c27086dcef853832a7cbebcb48bdac8104b6 Mon Sep 17 00:00:00 2001 From: Uwe Kleine-König Date: Wed, 25 Nov 2020 10:31:06 +0100 Subject: siox: Make remove callback return void MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The driver core ignores the return value of the remove callback, so don't give siox drivers the chance to provide a value. All siox drivers only allocate devm-managed resources in .probe, so there is no .remove callback to fix. Tested-by: Thorsten Scherer Acked-by: Thorsten Scherer Signed-off-by: Uwe Kleine-König Link: https://lore.kernel.org/r/20201125093106.240643-3-u.kleine-koenig@pengutronix.de Signed-off-by: Greg Kroah-Hartman --- drivers/siox/siox-core.c | 5 ++--- include/linux/siox.h | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) (limited to 'include') diff --git a/drivers/siox/siox-core.c b/drivers/siox/siox-core.c index b56cdcb52967..1794ff0106bc 100644 --- a/drivers/siox/siox-core.c +++ b/drivers/siox/siox-core.c @@ -525,12 +525,11 @@ static int siox_remove(struct device *dev) struct siox_driver *sdriver = container_of(dev->driver, struct siox_driver, driver); struct siox_device *sdevice = to_siox_device(dev); - int ret = 0; if (sdriver->remove) - ret = sdriver->remove(sdevice); + sdriver->remove(sdevice); - return ret; + return 0; } static void siox_shutdown(struct device *dev) diff --git a/include/linux/siox.h b/include/linux/siox.h index da7225bf1877..6bfbda3f634c 100644 --- a/include/linux/siox.h +++ b/include/linux/siox.h @@ -36,7 +36,7 @@ bool siox_device_connected(struct siox_device *sdevice); struct siox_driver { int (*probe)(struct siox_device *sdevice); - int (*remove)(struct siox_device *sdevice); + void (*remove)(struct siox_device *sdevice); void (*shutdown)(struct siox_device *sdevice); /* -- cgit v1.2.3