diff options
| -rw-r--r-- | Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt | 7 | ||||
| -rw-r--r-- | Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt | 5 | ||||
| -rw-r--r-- | Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt | 13 | ||||
| -rw-r--r-- | drivers/remoteproc/Kconfig | 1 | ||||
| -rw-r--r-- | drivers/remoteproc/qcom_adsp_pil.c | 3 | ||||
| -rw-r--r-- | drivers/remoteproc/qcom_common.c | 49 | ||||
| -rw-r--r-- | drivers/remoteproc/qcom_common.h | 11 | ||||
| -rw-r--r-- | drivers/rpmsg/Kconfig | 16 | ||||
| -rw-r--r-- | drivers/rpmsg/Makefile | 2 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_native.c | 1612 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_native.h | 45 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_rpm.c | 1026 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_glink_smem.c | 316 | ||||
| -rw-r--r-- | drivers/rpmsg/qcom_smd.c | 1 | ||||
| -rw-r--r-- | drivers/rpmsg/virtio_rpmsg_bus.c | 47 | ||||
| -rw-r--r-- | include/linux/rpmsg/qcom_glink.h | 27 |
16 files changed, 2210 insertions, 971 deletions
diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt index 75ad7b8df0b1..728e4193f7a6 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,adsp.txt @@ -63,9 +63,10 @@ on the Qualcomm ADSP Hexagon core. = SUBNODES -The adsp node may have an subnode named "smd-edge" that describes the SMD edge, -channels and devices related to the ADSP. See ../soc/qcom/qcom,smd.txt for -details on how to describe the SMD edge. +The adsp node may have an subnode named either "smd-edge" or "glink-edge" that +describes the communication edge, channels and devices related to the ADSP. +See ../soc/qcom/qcom,smd.txt and ../soc/qcom/qcom,glink.txt for details on how +to describe these. = EXAMPLE diff --git a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt index 92347fe6890e..7ff3f7903f26 100644 --- a/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt +++ b/Documentation/devicetree/bindings/remoteproc/qcom,q6v5.txt @@ -90,6 +90,11 @@ the memory regions used by the Hexagon firmware. Each sub-node must contain: Value type: <phandle> Definition: reference to the reserved-memory for the region +The Hexagon node may also have an subnode named either "smd-edge" or +"glink-edge" that describes the communication edge, channels and devices +related to the Hexagon. See ../soc/qcom/qcom,smd.txt and +../soc/qcom/qcom,glink.txt for details on how to describe these. + = EXAMPLE The following example describes the resources needed to boot control the Hexagon, as it is found on MSM8974 boards. diff --git a/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt b/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt index 50fc20c6ce91..b277eca861f7 100644 --- a/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt +++ b/Documentation/devicetree/bindings/soc/qcom/qcom,glink.txt @@ -1,11 +1,12 @@ -Qualcomm RPM GLINK binding +Qualcomm GLINK edge binding -This binding describes the Qualcomm RPM GLINK, a fifo based mechanism for -communication with the Resource Power Management system on various Qualcomm -platforms. +This binding describes a Qualcomm GLINK edge, a fifo based mechanism for +communication between subsystem-pairs on various Qualcomm platforms. Two types +of edges can be described by the binding; the GLINK RPM edge and a SMEM based +edge. - compatible: - Usage: required + Usage: required for glink-rpm Value type: <stringlist> Definition: must be "qcom,glink-rpm" @@ -16,7 +17,7 @@ platforms. signal this processor about communication related events - qcom,rpm-msg-ram: - Usage: required + Usage: required for glink-rpm Value type: <prop-encoded-array> Definition: handle to RPM message memory resource diff --git a/drivers/remoteproc/Kconfig b/drivers/remoteproc/Kconfig index 79f903af6504..df63e44526ac 100644 --- a/drivers/remoteproc/Kconfig +++ b/drivers/remoteproc/Kconfig @@ -92,6 +92,7 @@ config QCOM_ADSP_PIL depends on OF && ARCH_QCOM depends on QCOM_SMEM depends on RPMSG_QCOM_SMD || (COMPILE_TEST && RPMSG_QCOM_SMD=n) + depends on RPMSG_QCOM_GLINK_SMEM || RPMSG_QCOM_GLINK_SMEM=n select MFD_SYSCON select QCOM_MDT_LOADER select QCOM_RPROC_COMMON diff --git a/drivers/remoteproc/qcom_adsp_pil.c b/drivers/remoteproc/qcom_adsp_pil.c index d01a8daf948a..3f6af54dbc96 100644 --- a/drivers/remoteproc/qcom_adsp_pil.c +++ b/drivers/remoteproc/qcom_adsp_pil.c @@ -72,6 +72,7 @@ struct qcom_adsp { void *mem_region; size_t mem_size; + struct qcom_rproc_glink glink_subdev; struct qcom_rproc_subdev smd_subdev; struct qcom_rproc_ssr ssr_subdev; }; @@ -400,6 +401,7 @@ static int adsp_probe(struct platform_device *pdev) goto free_rproc; } + qcom_add_glink_subdev(rproc, &adsp->glink_subdev); qcom_add_smd_subdev(rproc, &adsp->smd_subdev); qcom_add_ssr_subdev(rproc, &adsp->ssr_subdev, desc->ssr_name); @@ -422,6 +424,7 @@ static int adsp_remove(struct platform_device *pdev) qcom_smem_state_put(adsp->state); rproc_del(adsp->rproc); + qcom_remove_glink_subdev(adsp->rproc, &adsp->glink_subdev); qcom_remove_smd_subdev(adsp->rproc, &adsp->smd_subdev); qcom_remove_ssr_subdev(adsp->rproc, &adsp->ssr_subdev); rproc_free(adsp->rproc); diff --git a/drivers/remoteproc/qcom_common.c b/drivers/remoteproc/qcom_common.c index 257680f82b27..d487040b528b 100644 --- a/drivers/remoteproc/qcom_common.c +++ b/drivers/remoteproc/qcom_common.c @@ -20,11 +20,13 @@ #include <linux/module.h> #include <linux/notifier.h> #include <linux/remoteproc.h> +#include <linux/rpmsg/qcom_glink.h> #include <linux/rpmsg/qcom_smd.h> #include "remoteproc_internal.h" #include "qcom_common.h" +#define to_glink_subdev(d) container_of(d, struct qcom_rproc_glink, subdev) #define to_smd_subdev(d) container_of(d, struct qcom_rproc_subdev, subdev) #define to_ssr_subdev(d) container_of(d, struct qcom_rproc_ssr, subdev) @@ -49,6 +51,53 @@ struct resource_table *qcom_mdt_find_rsc_table(struct rproc *rproc, } EXPORT_SYMBOL_GPL(qcom_mdt_find_rsc_table); +static int glink_subdev_probe(struct rproc_subdev *subdev) +{ + struct qcom_rproc_glink *glink = to_glink_subdev(subdev); + + glink->edge = qcom_glink_smem_register(glink->dev, glink->node); + + return IS_ERR(glink->edge) ? PTR_ERR(glink->edge) : 0; +} + +static void glink_subdev_remove(struct rproc_subdev *subdev) +{ + struct qcom_rproc_glink *glink = to_glink_subdev(subdev); + + qcom_glink_smem_unregister(glink->edge); + glink->edge = NULL; +} + +/** + * qcom_add_glink_subdev() - try to add a GLINK subdevice to rproc + * @rproc: rproc handle to parent the subdevice + * @glink: reference to a GLINK subdev context + */ +void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink) +{ + struct device *dev = &rproc->dev; + + glink->node = of_get_child_by_name(dev->parent->of_node, "glink-edge"); + if (!glink->node) + return; + + glink->dev = dev; + rproc_add_subdev(rproc, &glink->subdev, glink_subdev_probe, glink_subdev_remove); +} +EXPORT_SYMBOL_GPL(qcom_add_glink_subdev); + +/** + * qcom_remove_glink_subdev() - remove a GLINK subdevice from rproc + * @rproc: rproc handle + * @glink: reference to a GLINK subdev context + */ +void qcom_remove_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink) +{ + rproc_remove_subdev(rproc, &glink->subdev); + of_node_put(glink->node); +} +EXPORT_SYMBOL_GPL(qcom_remove_glink_subdev); + static int smd_subdev_probe(struct rproc_subdev *subdev) { struct qcom_rproc_subdev *smd = to_smd_subdev(subdev); diff --git a/drivers/remoteproc/qcom_common.h b/drivers/remoteproc/qcom_common.h index fab28b64b8ea..4f8bc168473c 100644 --- a/drivers/remoteproc/qcom_common.h +++ b/drivers/remoteproc/qcom_common.h @@ -4,6 +4,14 @@ #include <linux/remoteproc.h> #include "remoteproc_internal.h" +struct qcom_rproc_glink { + struct rproc_subdev subdev; + + struct device *dev; + struct device_node *node; + struct qcom_glink *edge; +}; + struct qcom_rproc_subdev { struct rproc_subdev subdev; @@ -22,6 +30,9 @@ struct resource_table *qcom_mdt_find_rsc_table(struct rproc *rproc, const struct firmware *fw, int *tablesz); +void qcom_add_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink); +void qcom_remove_glink_subdev(struct rproc *rproc, struct qcom_rproc_glink *glink); + void qcom_add_smd_subdev(struct rproc *rproc, struct qcom_rproc_subdev *smd); void qcom_remove_smd_subdev(struct rproc *rproc, struct qcom_rproc_subdev *smd); diff --git a/drivers/rpmsg/Kconfig b/drivers/rpmsg/Kconfig index 1323a245763b..0fe6eac46512 100644 --- a/drivers/rpmsg/Kconfig +++ b/drivers/rpmsg/Kconfig @@ -13,9 +13,13 @@ config RPMSG_CHAR in /dev. They make it possible for user-space programs to send and receive rpmsg packets. +config RPMSG_QCOM_GLINK_NATIVE + tristate + select RPMSG + config RPMSG_QCOM_GLINK_RPM tristate "Qualcomm RPM Glink driver" - select RPMSG + select RPMSG_QCOM_GLINK_NATIVE depends on HAS_IOMEM depends on MAILBOX help @@ -23,6 +27,16 @@ config RPMSG_QCOM_GLINK_RPM which serves as a channel for communication with the RPM in GLINK enabled systems. +config RPMSG_QCOM_GLINK_SMEM + tristate "Qualcomm SMEM Glink driver" + select RPMSG_QCOM_GLINK_NATIVE + depends on MAILBOX + depends on QCOM_SMEM + help + Say y here to enable support for the GLINK SMEM communication driver, + which provides support for using the GLINK communication protocol + over SMEM. + config RPMSG_QCOM_SMD tristate "Qualcomm Shared Memory Driver (SMD)" depends on QCOM_SMEM diff --git a/drivers/rpmsg/Makefile b/drivers/rpmsg/Makefile index 28cc19088cc0..c71f4ab1ae17 100644 --- a/drivers/rpmsg/Makefile +++ b/drivers/rpmsg/Makefile @@ -1,5 +1,7 @@ obj-$(CONFIG_RPMSG) += rpmsg_core.o obj-$(CONFIG_RPMSG_CHAR) += rpmsg_char.o obj-$(CONFIG_RPMSG_QCOM_GLINK_RPM) += qcom_glink_rpm.o +obj-$(CONFIG_RPMSG_QCOM_GLINK_NATIVE) += qcom_glink_native.o +obj-$(CONFIG_RPMSG_QCOM_GLINK_SMEM) += qcom_glink_smem.o obj-$(CONFIG_RPMSG_QCOM_SMD) += qcom_smd.o obj-$(CONFIG_RPMSG_VIRTIO) += virtio_rpmsg_bus.o diff --git a/drivers/rpmsg/qcom_glink_native.c b/drivers/rpmsg/qcom_glink_native.c new file mode 100644 index 000000000000..5a5e927ea50f --- /dev/null +++ b/drivers/rpmsg/qcom_glink_native.c @@ -0,0 +1,1612 @@ +/* + * Copyright (c) 2016-2017, Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/idr.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/list.h> +#include <linux/mfd/syscon.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/of_irq.h> +#include <linux/platform_device.h> +#include <linux/regmap.h> +#include <linux/rpmsg.h> +#include <linux/sizes.h> +#include <linux/slab.h> +#include <linux/workqueue.h> +#include <linux/mailbox_client.h> + +#include "rpmsg_internal.h" +#include "qcom_glink_native.h" + +#define GLINK_NAME_SIZE 32 +#define GLINK_VERSION_1 1 + +#define RPM_GLINK_CID_MIN 1 +#define RPM_GLINK_CID_MAX 65536 + +struct glink_msg { + __le16 cmd; + __le16 param1; + __le32 param2; + u8 data[]; +} __packed; + +/** + * struct glink_defer_cmd - deferred incoming control message + * @node: list node + * @msg: message header + * data: payload of the message + * + * Copy of a received control message, to be added to @rx_queue and processed + * by @rx_work of @qcom_glink. + */ +struct glink_defer_cmd { + struct list_head node; + + struct glink_msg msg; + u8 data[]; +}; + +/** + * struct glink_core_rx_intent - RX intent + * RX intent + * + * data: pointer to the data (may be NULL for zero-copy) + * id: remote or local intent ID + * size: size of the original intent (do not modify) + * reuse: To mark if the intent can be reused after first use + * in_use: To mark if intent is already in use for the channel + * offset: next write offset (initially 0) + */ +struct glink_core_rx_intent { + void *data; + u32 id; + size_t size; + bool reuse; + bool in_use; + u32 offset; + + struct list_head node; +}; + +/** + * struct qcom_glink - driver context, relates to one remote subsystem + * @dev: reference to the associated struct device + * @mbox_client: mailbox client + * @mbox_chan: mailbox channel + * @rx_pipe: pipe object for receive FIFO + * @tx_pipe: pipe object for transmit FIFO + * @irq: IRQ for signaling incoming events + * @rx_work: worker for handling received control messages + * @rx_lock: protects the @rx_queue + * @rx_queue: queue of received control messages to be processed in @rx_work + * @tx_lock: synchronizes operations on the tx fifo + * @idr_lock: synchronizes @lcids and @rcids modifications + * @lcids: idr of all channels with a known local channel id + * @rcids: idr of all channels with a known remote channel id + */ +struct qcom_glink { + struct device *dev; + + struct mbox_client mbox_client; + struct mbox_chan *mbox_chan; + + struct qcom_glink_pipe *rx_pipe; + struct qcom_glink_pipe *tx_pipe; + + int irq; + + struct work_struct rx_work; + spinlock_t rx_lock; + struct list_head rx_queue; + + struct mutex tx_lock; + + spinlock_t idr_lock; + struct idr lcids; + struct idr rcids; + unsigned long features; + + bool intentless; +}; + +enum { + GLINK_STATE_CLOSED, + GLINK_STATE_OPENING, + GLINK_STATE_OPEN, + GLINK_STATE_CLOSING, +}; + +/** + * struct glink_channel - internal representation of a channel + * @rpdev: rpdev reference, only used for primary endpoints + * @ept: rpmsg endpoint this channel is associated with + * @glink: qcom_glink context handle + * @refcount: refcount for the channel object + * @recv_lock: guard for @ept.cb + * @name: unique channel name/identifier + * @lcid: channel id, in local space + * @rcid: channel id, in remote space + * @intent_lock: lock for protection of @liids, @riids + * @liids: idr of all local intents + * @riids: idr of all remote intents + * @intent_work: worker responsible for transmitting rx_done packets + * @done_intents: list of intents that needs to be announced rx_done + * @buf: receive buffer, for gathering fragments + * @buf_offset: write offset in @buf + * @buf_size: size of current @buf + * @open_ack: completed once remote has acked the open-request + * @open_req: completed once open-request has been received + * @intent_req_lock: Synchronises multiple intent requests + * @intent_req_result: Result of intent request + * @intent_req_comp: Completion for intent_req signalling + */ +struct glink_channel { + struct rpmsg_endpoint ept; + + struct rpmsg_device *rpdev; + struct qcom_glink *glink; + + struct kref refcount; + + spinlock_t recv_lock; + + char *name; + unsigned int lcid; + unsigned int rcid; + + spinlock_t intent_lock; + struct idr liids; + struct idr riids; + struct work_struct intent_work; + struct list_head done_intents; + + struct glink_core_rx_intent *buf; + int buf_offset; + int buf_size; + + struct completion open_ack; + struct completion open_req; + + struct mutex intent_req_lock; + bool intent_req_result; + struct completion intent_req_comp; +}; + +#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) + +static const struct rpmsg_endpoint_ops glink_endpoint_ops; + +#define RPM_CMD_VERSION 0 +#define RPM_CMD_VERSION_ACK 1 +#define RPM_CMD_OPEN 2 +#define RPM_CMD_CLOSE 3 +#define RPM_CMD_OPEN_ACK 4 +#define RPM_CMD_INTENT 5 +#define RPM_CMD_RX_DONE 6 +#define RPM_CMD_RX_INTENT_REQ 7 +#define RPM_CMD_RX_INTENT_REQ_ACK 8 +#define RPM_CMD_TX_DATA 9 +#define RPM_CMD_CLOSE_ACK 11 +#define RPM_CMD_TX_DATA_CONT 12 +#define RPM_CMD_READ_NOTIF 13 +#define RPM_CMD_RX_DONE_W_REUSE 14 + +#define GLINK_FEATURE_INTENTLESS BIT(1) + +static void qcom_glink_rx_done_work(struct work_struct *work); + +static struct glink_channel *qcom_glink_alloc_channel(struct qcom_glink *glink, + const char *name) +{ + struct glink_channel *channel; + + channel = kzalloc(sizeof(*channel), GFP_KERNEL); + if (!channel) + return ERR_PTR(-ENOMEM); + + /* Setup glink internal glink_channel data */ + spin_lock_init(&channel->recv_lock); + spin_lock_init(&channel->intent_lock); + + channel->glink = glink; + channel->name = kstrdup(name, GFP_KERNEL); + + init_completion(&channel->open_req); + init_completion(&channel->open_ack); + + INIT_LIST_HEAD(&channel->done_intents); + INIT_WORK(&channel->intent_work, qcom_glink_rx_done_work); + + idr_init(&channel->liids); + idr_init(&channel->riids); + kref_init(&channel->refcount); + + return channel; +} + +static void qcom_glink_channel_release(struct kref *ref) +{ + struct glink_channel *channel = container_of(ref, struct glink_channel, + refcount); + unsigned long flags; + + spin_lock_irqsave(&channel->intent_lock, flags); + idr_destroy(&channel->liids); + idr_destroy(&channel->riids); + spin_unlock_irqrestore(&channel->intent_lock, flags); + + kfree(channel->name); + kfree(channel); +} + +static size_t qcom_glink_rx_avail(struct qcom_glink *glink) +{ + return glink->rx_pipe->avail(glink->rx_pipe); +} + +static void qcom_glink_rx_peak(struct qcom_glink *glink, + void *data, unsigned int offset, size_t count) +{ + glink->rx_pipe->peak(glink->rx_pipe, data, offset, count); +} + +static void qcom_glink_rx_advance(struct qcom_glink *glink, size_t count) +{ + glink->rx_pipe->advance(glink->rx_pipe, count); +} + +static size_t qcom_glink_tx_avail(struct qcom_glink *glink) +{ + return glink->tx_pipe->avail(glink->tx_pipe); +} + +static void qcom_glink_tx_write(struct qcom_glink *glink, + const void *hdr, size_t hlen, + const void *data, size_t dlen) +{ + glink->tx_pipe->write(glink->tx_pipe, hdr, hlen, data, dlen); +} + +static int qcom_glink_tx(struct qcom_glink *glink, + const void *hdr, size_t hlen, + const void *data, size_t dlen, bool wait) +{ + unsigned int tlen = hlen + dlen; + int ret; + + /* Reject packets that are too big */ + if (tlen >= glink->tx_pipe->length) + return -EINVAL; + + ret = mutex_lock_interruptible(&glink->tx_lock); + if (ret) + return ret; + + while (qcom_glink_tx_avail(glink) < tlen) { + if (!wait) { + ret = -EAGAIN; + goto out; + } + + usleep_range(10000, 15000); + } + + qcom_glink_tx_write(glink, hdr, hlen, data, dlen); + + mbox_send_message(glink->mbox_chan, NULL); + mbox_client_txdone(glink->mbox_chan, 0); + +out: + mutex_unlock(&glink->tx_lock); + + return ret; +} + +static int qcom_glink_send_version(struct qcom_glink *glink) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(RPM_CMD_VERSION); + msg.param1 = cpu_to_le16(GLINK_VERSION_1); + msg.param2 = cpu_to_le32(glink->features); + + return qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_send_version_ack(struct qcom_glink *glink) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK); + msg.param1 = cpu_to_le16(GLINK_VERSION_1); + msg.param2 = cpu_to_le32(glink->features); + + qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_send_open_ack(struct qcom_glink *glink, + struct glink_channel *channel) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK); + msg.param1 = cpu_to_le16(channel->rcid); + msg.param2 = cpu_to_le32(0); + + qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); +} + +static void qcom_glink_handle_intent_req_ack(struct qcom_glink *glink, + unsigned int cid, bool granted) +{ + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "unable to find channel\n"); + return; + } + + channel->intent_req_result = granted; + complete(&channel->intent_req_comp); +} + +/** + * qcom_glink_send_open_req() - send a RPM_CMD_OPEN request to the remote + * @glink: Ptr to the glink edge + * @channel: Ptr to the channel that the open req is sent + * + * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote. + * Will return with refcount held, regardless of outcome. + * + * Returns 0 on success, negative errno otherwise. + */ +static int qcom_glink_send_open_req(struct qcom_glink *glink, + struct glink_channel *channel) +{ + struct { + struct glink_msg msg; + u8 name[GLINK_NAME_SIZE]; + } __packed req; + int name_len = strlen(channel->name) + 1; + int req_len = ALIGN(sizeof(req.msg) + name_len, 8); + int ret; + unsigned long flags; + + kref_get(&channel->refcount); + + spin_lock_irqsave(&glink->idr_lock, flags); + ret = idr_alloc_cyclic(&glink->lcids, channel, + RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, + GFP_ATOMIC); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (ret < 0) + return ret; + + channel->lcid = ret; + + req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN); + req.msg.param1 = cpu_to_le16(channel->lcid); + req.msg.param2 = cpu_to_le32(name_len); + strcpy(req.name, channel->name); + + ret = qcom_glink_tx(glink, &req, req_len, NULL, 0, true); + if (ret) + goto remove_idr; + + return 0; + +remove_idr: + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->lcids, channel->lcid); + channel->lcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + return ret; +} + +static void qcom_glink_send_close_req(struct qcom_glink *glink, + struct glink_channel *channel) +{ + struct glink_msg req; + + req.cmd = cpu_to_le16(RPM_CMD_CLOSE); + req.param1 = cpu_to_le16(channel->lcid); + req.param2 = 0; + + qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); +} + +static void qcom_glink_send_close_ack(struct qcom_glink *glink, + unsigned int rcid) +{ + struct glink_msg req; + + req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK); + req.param1 = cpu_to_le16(rcid); + req.param2 = 0; + + qcom_glink_tx(glink, &req, sizeof(req), NULL, 0, true); +} + +static void qcom_glink_rx_done_work(struct work_struct *work) +{ + struct glink_channel *channel = container_of(work, struct glink_channel, + intent_work); + struct qcom_glink *glink = channel->glink; + struct glink_core_rx_intent *intent, *tmp; + struct { + u16 id; + u16 lcid; + u32 liid; + } __packed cmd; + + unsigned int cid = channel->lcid; + unsigned int iid; + bool reuse; + unsigned long flags; + + spin_lock_irqsave(&channel->intent_lock, flags); + list_for_each_entry_safe(intent, tmp, &channel->done_intents, node) { + list_del(&intent->node); + spin_unlock_irqrestore(&channel->intent_lock, flags); + iid = intent->id; + reuse = intent->reuse; + + cmd.id = reuse ? RPM_CMD_RX_DONE_W_REUSE : RPM_CMD_RX_DONE; + cmd.lcid = cid; + cmd.liid = iid; + + qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); + if (!reuse) { + kfree(intent->data); + kfree(intent); + } + spin_lock_irqsave(&channel->intent_lock, flags); + } + spin_unlock_irqrestore(&channel->intent_lock, flags); +} + +static void qcom_glink_rx_done(struct qcom_glink *glink, + struct glink_channel *channel, + struct glink_core_rx_intent *intent) +{ + /* We don't send RX_DONE to intentless systems */ + if (glink->intentless) { + kfree(intent->data); + kfree(intent); + return; + } + + /* Take it off the tree of receive intents */ + if (!intent->reuse) { + spin_lock(&channel->intent_lock); + idr_remove(&channel->liids, intent->id); + spin_unlock(&channel->intent_lock); + } + + /* Schedule the sending of a rx_done indication */ + spin_lock(&channel->intent_lock); + list_add_tail(&intent->node, &channel->done_intents); + spin_unlock(&channel->intent_lock); + + schedule_work(&channel->intent_work); +} + +/** + * qcom_glink_receive_version() - receive version/features from remote system + * + * @glink: pointer to transport interface + * @r_version: remote version + * @r_features: remote features + * + * This function is called in response to a remote-initiated version/feature + * negotiation sequence. + */ +static void qcom_glink_receive_version(struct qcom_glink *glink, + u32 version, + u32 features) +{ + switch (version) { + case 0: + break; + case GLINK_VERSION_1: + glink->features &= features; + /* FALLTHROUGH */ + default: + qcom_glink_send_version_ack(glink); + break; + } +} + +/** + * qcom_glink_receive_version_ack() - receive negotiation ack from remote system + * + * @glink: pointer to transport interface + * @r_version: remote version response + * @r_features: remote features response + * + * This function is called in response to a local-initiated version/feature + * negotiation sequence and is the counter-offer from the remote side based + * upon the initial version and feature set requested. + */ +static void qcom_glink_receive_version_ack(struct qcom_glink *glink, + u32 version, + u32 features) +{ + switch (version) { + case 0: + /* Version negotiation failed */ + break; + case GLINK_VERSION_1: + if (features == glink->features) + break; + + glink->features &= features; + /* FALLTHROUGH */ + default: + qcom_glink_send_version(glink); + break; + } +} + +/** + * qcom_glink_send_intent_req_ack() - convert an rx intent request ack cmd to + wire format and transmit + * @glink: The transport to transmit on. + * @channel: The glink channel + * @granted: The request response to encode. + * + * Return: 0 on success or standard Linux error code. + */ +static int qcom_glink_send_intent_req_ack(struct qcom_glink *glink, + struct glink_channel *channel, + bool granted) +{ + struct glink_msg msg; + + msg.cmd = cpu_to_le16(RPM_CMD_RX_INTENT_REQ_ACK); + msg.param1 = cpu_to_le16(channel->lcid); + msg.param2 = cpu_to_le32(granted); + + qcom_glink_tx(glink, &msg, sizeof(msg), NULL, 0, true); + + return 0; +} + +/** + * qcom_glink_advertise_intent - convert an rx intent cmd to wire format and + * transmit + * @glink: The transport to transmit on. + * @channel: The local channel + * @size: The intent to pass on to remote. + * + * Return: 0 on success or standard Linux error code. + */ +static int qcom_glink_advertise_intent(struct qcom_glink *glink, + struct glink_channel *channel, + struct glink_core_rx_intent *intent) +{ + struct command { + u16 id; + u16 lcid; + u32 count; + u32 size; + u32 liid; + } __packed; + struct command cmd; + + cmd.id = cpu_to_le16(RPM_CMD_INTENT); + cmd.lcid = cpu_to_le16(channel->lcid); + cmd.count = cpu_to_le32(1); + cmd.size = cpu_to_le32(intent->size); + cmd.liid = cpu_to_le32(intent->id); + + qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); + + return 0; +} + +static struct glink_core_rx_intent * +qcom_glink_alloc_intent(struct qcom_glink *glink, + struct glink_channel *channel, + size_t size, + bool reuseable) +{ + struct glink_core_rx_intent *intent; + int ret; + unsigned long flags; + + intent = kzalloc(sizeof(*intent), GFP_KERNEL); + + if (!intent) + return NULL; + + intent->data = kzalloc(size, GFP_KERNEL); + if (!intent->data) + return NULL; + + spin_lock_irqsave(&channel->intent_lock, flags); + ret = idr_alloc_cyclic(&channel->liids, intent, 1, -1, GFP_ATOMIC); + if (ret < 0) { + spin_unlock_irqrestore(&channel->intent_lock, flags); + return NULL; + } + spin_unlock_irqrestore(&channel->intent_lock, flags); + + intent->id = ret; + intent->size = size; + intent->reuse = reuseable; + + return intent; +} + +static void qcom_glink_handle_rx_done(struct qcom_glink *glink, + u32 cid, uint32_t iid, + bool reuse) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "invalid channel id received\n"); + return; + } + + spin_lock_irqsave(&channel->intent_lock, flags); + intent = idr_find(&channel->riids, iid); + + if (!intent) { + spin_unlock_irqrestore(&channel->intent_lock, flags); + dev_err(glink->dev, "invalid intent id received\n"); + return; + } + + intent->in_use = false; + + if (!reuse) { + idr_remove(&channel->riids, intent->id); + kfree(intent); + } + spin_unlock_irqrestore(&channel->intent_lock, flags); +} + +/** + * qcom_glink_handle_intent_req() - Receive a request for rx_intent + * from remote side + * if_ptr: Pointer to the transport interface + * rcid: Remote channel ID + * size: size of the intent + * + * The function searches for the local channel to which the request for + * rx_intent has arrived and allocates and notifies the remote back + */ +static void qcom_glink_handle_intent_req(struct qcom_glink *glink, + u32 cid, size_t size) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + + if (!channel) { + pr_err("%s channel not found for cid %d\n", __func__, cid); + return; + } + + intent = qcom_glink_alloc_intent(glink, channel, size, false); + if (intent) + qcom_glink_advertise_intent(glink, channel, intent); + + qcom_glink_send_intent_req_ack(glink, channel, !!intent); +} + +static int qcom_glink_rx_defer(struct qcom_glink *glink, size_t extra) +{ + struct glink_defer_cmd *dcmd; + + extra = ALIGN(extra, 8); + + if (qcom_glink_rx_avail(glink) < sizeof(struct glink_msg) + extra) { + dev_dbg(glink->dev, "Insufficient data in rx fifo"); + return -ENXIO; + } + + dcmd = kzalloc(sizeof(*dcmd) + extra, GFP_ATOMIC); + if (!dcmd) + return -ENOMEM; + + INIT_LIST_HEAD(&dcmd->node); + + qcom_glink_rx_peak(glink, &dcmd->msg, 0, sizeof(dcmd->msg) + extra); + + spin_lock(&glink->rx_lock); + list_add_tail(&dcmd->node, &glink->rx_queue); + spin_unlock(&glink->rx_lock); + + schedule_work(&glink->rx_work); + qcom_glink_rx_advance(glink, sizeof(dcmd->msg) + extra); + + return 0; +} + +static int qcom_glink_rx_data(struct qcom_glink *glink, size_t avail) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + struct { + struct glink_msg msg; + __le32 chunk_size; + __le32 left_size; + } __packed hdr; + unsigned int chunk_size; + unsigned int left_size; + unsigned int rcid; + unsigned int liid; + int ret = 0; + unsigned long flags; + + if (avail < sizeof(hdr)) { + dev_dbg(glink->dev, "Not enough data in fifo\n"); + return -EAGAIN; + } + + qcom_glink_rx_peak(glink, &hdr, 0, sizeof(hdr)); + chunk_size = le32_to_cpu(hdr.chunk_size); + left_size = le32_to_cpu(hdr.left_size); + + if (avail < sizeof(hdr) + chunk_size) { + dev_dbg(glink->dev, "Payload not yet in fifo\n"); + return -EAGAIN; + } + + if (WARN(chunk_size % 4, "Incoming data must be word aligned\n")) + return -EINVAL; + + rcid = le16_to_cpu(hdr.msg.param1); + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, rcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_dbg(glink->dev, "Data on non-existing channel\n"); + + /* Drop the message */ + goto advance_rx; + } + + if (glink->intentless) { + /* Might have an ongoing, fragmented, message to append */ + if (!channel->buf) { + intent = kzalloc(sizeof(*intent), GFP_ATOMIC); + if (!intent) + return -ENOMEM; + + intent->data = kmalloc(chunk_size + left_size, + GFP_ATOMIC); + if (!intent->data) { + kfree(intent); + return -ENOMEM; + } + + intent->id = 0xbabababa; + intent->size = chunk_size + left_size; + intent->offset = 0; + + channel->buf = intent; + } else { + intent = channel->buf; + } + } else { + liid = le32_to_cpu(hdr.msg.param2); + + spin_lock_irqsave(&channel->intent_lock, flags); + intent = idr_find(&channel->liids, liid); + spin_unlock_irqrestore(&channel->intent_lock, flags); + + if (!intent) { + dev_err(glink->dev, + "no intent found for channel %s intent %d", + channel->name, liid); + goto advance_rx; + } + } + + if (intent->size - intent->offset < chunk_size) { + dev_err(glink->dev, "Insufficient space in intent\n"); + + /* The packet header lied, drop payload */ + goto advance_rx; + } + + qcom_glink_rx_peak(glink, intent->data + intent->offset, + sizeof(hdr), chunk_size); + intent->offset += chunk_size; + + /* Handle message when no fragments remain to be received */ + if (!left_size) { + spin_lock(&channel->recv_lock); + if (channel->ept.cb) { + channel->ept.cb(channel->ept.rpdev, + intent->data, + intent->offset, + channel->ept.priv, + RPMSG_ADDR_ANY); + } + spin_unlock(&channel->recv_lock); + + intent->offset = 0; + channel->buf = NULL; + + qcom_glink_rx_done(glink, channel, intent); + } + +advance_rx: + qcom_glink_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); + + return ret; +} + +static void qcom_glink_handle_intent(struct qcom_glink *glink, + unsigned int cid, + unsigned int count, + size_t avail) +{ + struct glink_core_rx_intent *intent; + struct glink_channel *channel; + struct intent_pair { + __le32 size; + __le32 iid; + }; + + struct { + struct glink_msg msg; + struct intent_pair intents[]; + } __packed * msg; + + const size_t msglen = sizeof(*msg) + sizeof(struct intent_pair) * count; + int ret; + int i; + unsigned long flags; + + if (avail < msglen) { + dev_dbg(glink->dev, "Not enough data in fifo\n"); + return; + } + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, cid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (!channel) { + dev_err(glink->dev, "intents for non-existing channel\n"); + return; + } + + msg = kmalloc(msglen, GFP_ATOMIC); + if (!msg) + return; + + qcom_glink_rx_peak(glink, msg, 0, msglen); + + for (i = 0; i < count; ++i) { + intent = kzalloc(sizeof(*intent), GFP_ATOMIC); + if (!intent) + break; + + intent->id = le32_to_cpu(msg->intents[i].iid); + intent->size = le32_to_cpu(msg->intents[i].size); + + spin_lock_irqsave(&channel->intent_lock, flags); + ret = idr_alloc(&channel->riids, intent, + intent->id, intent->id + 1, GFP_ATOMIC); + spin_unlock_irqrestore(&channel->intent_lock, flags); + + if (ret < 0) + dev_err(glink->dev, "failed to store remote intent\n"); + } + + kfree(msg); + qcom_glink_rx_advance(glink, ALIGN(msglen, 8)); +} + +static int qcom_glink_rx_open_ack(struct qcom_glink *glink, unsigned int lcid) +{ + struct glink_channel *channel; + + spin_lock(&glink->idr_lock); + channel = idr_find(&glink->lcids, lcid); + spin_unlock(&glink->idr_lock); + if (!channel) { + dev_err(glink->dev, "Invalid open ack packet\n"); + return -EINVAL; + } + + complete(&channel->open_ack); + + return 0; +} + +static irqreturn_t qcom_glink_native_intr(int irq, void *data) +{ + struct qcom_glink *glink = data; + struct glink_msg msg; + unsigned int param1; + unsigned int param2; + unsigned int avail; + unsigned int cmd; + int ret = 0; + + for (;;) { + avail = qcom_glink_rx_avail(glink); + if (avail < sizeof(msg)) + break; + + qcom_glink_rx_peak(glink, &msg, 0, sizeof(msg)); + + cmd = le16_to_cpu(msg.cmd); + param1 = le16_to_cpu(msg.param1); + param2 = le32_to_cpu(msg.param2); + + switch (cmd) { + case RPM_CMD_VERSION: + case RPM_CMD_VERSION_ACK: + case RPM_CMD_CLOSE: + case RPM_CMD_CLOSE_ACK: + case RPM_CMD_RX_INTENT_REQ: + ret = qcom_glink_rx_defer(glink, 0); + break; + case RPM_CMD_OPEN_ACK: + ret = qcom_glink_rx_open_ack(glink, param1); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case RPM_CMD_OPEN: + ret = qcom_glink_rx_defer(glink, param2); + break; + case RPM_CMD_TX_DATA: + case RPM_CMD_TX_DATA_CONT: + ret = qcom_glink_rx_data(glink, avail); + break; + case RPM_CMD_READ_NOTIF: + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + + mbox_send_message(glink->mbox_chan, NULL); + mbox_client_txdone(glink->mbox_chan, 0); + break; + case RPM_CMD_INTENT: + qcom_glink_handle_intent(glink, param1, param2, avail); + break; + case RPM_CMD_RX_DONE: + qcom_glink_handle_rx_done(glink, param1, param2, false); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case RPM_CMD_RX_DONE_W_REUSE: + qcom_glink_handle_rx_done(glink, param1, param2, true); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + case RPM_CMD_RX_INTENT_REQ_ACK: + qcom_glink_handle_intent_req_ack(glink, param1, param2); + qcom_glink_rx_advance(glink, ALIGN(sizeof(msg), 8)); + break; + default: + dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); + ret = -EINVAL; + break; + } + + if (ret) + break; + } + + return IRQ_HANDLED; +} + +/* Locally initiated rpmsg_create_ept */ +static struct glink_channel *qcom_glink_create_local(struct qcom_glink *glink, + const char *name) +{ + struct glink_channel *channel; + int ret; + unsigned long flags; + + channel = qcom_glink_alloc_channel(glink, name); + if (IS_ERR(channel)) + return ERR_CAST(channel); + + ret = qcom_glink_send_open_req(glink, channel); + if (ret) + goto release_channel; + + ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); + if (!ret) + goto err_timeout; + + ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); + if (!ret) + goto err_timeout; + + qcom_glink_send_open_ack(glink, channel); + + return channel; + +err_timeout: + /* qcom_glink_send_open_req() did register the channel in lcids*/ + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->lcids, channel->lcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + +release_channel: + /* Release qcom_glink_send_open_req() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + /* Release qcom_glink_alloc_channel() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + + return ERR_PTR(-ETIMEDOUT); +} + +/* Remote initiated rpmsg_create_ept */ +static int qcom_glink_create_remote(struct qcom_glink *glink, + struct glink_channel *channel) +{ + int ret; + + qcom_glink_send_open_ack(glink, channel); + + ret = qcom_glink_send_open_req(glink, channel); + if (ret) + goto close_link; + + ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); + if (!ret) { + ret = -ETIMEDOUT; + goto close_link; + } + + return 0; + +close_link: + /* + * Send a close request to "undo" our open-ack. The close-ack will + * release the last reference. + */ + qcom_glink_send_close_req(glink, channel); + + /* Release qcom_glink_send_open_req() reference */ + kref_put(&channel->refcount, qcom_glink_channel_release); + + return ret; +} + +static struct rpmsg_endpoint *qcom_glink_create_ept(struct rpmsg_device *rpdev, + rpmsg_rx_cb_t cb, + void *priv, + struct rpmsg_channel_info + chinfo) +{ + struct glink_channel *parent = to_glink_channel(rpdev->ept); + struct glink_channel *channel; + struct qcom_glink *glink = parent->glink; + struct rpmsg_endpoint *ept; + const char *name = chinfo.name; + int cid; + int ret; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + idr_for_each_entry(&glink->rcids, channel, cid) { + if (!strcmp(channel->name, name)) + break; + } + spin_unlock_irqrestore(&glink->idr_lock, flags); + + if (!channel) { + channel = qcom_glink_create_local(glink, name); + if (IS_ERR(channel)) + return NULL; + } else { + ret = qcom_glink_create_remote(glink, channel); + if (ret) + return NULL; + } + + ept = &channel->ept; + ept->rpdev = rpdev; + ept->cb = cb; + ept->priv = priv; + ept->ops = &glink_endpoint_ops; + + return ept; +} + +static int qcom_glink_announce_create(struct rpmsg_device *rpdev) +{ + struct glink_channel *channel = to_glink_channel(rpdev->ept); + struct glink_core_rx_intent *intent; + struct qcom_glink *glink = channel->glink; + int num_intents = glink->intentless ? 0 : 5; + + /* Channel is now open, advertise base set of intents */ + while (num_intents--) { + intent = qcom_glink_alloc_intent(glink, channel, SZ_1K, true); + if (!intent) + break; + + qcom_glink_advertise_intent(glink, channel, intent); + } + + return 0; +} + +static void qcom_glink_destroy_ept(struct rpmsg_endpoint *ept) +{ + struct glink_channel *channel = to_glink_channel(ept); + struct qcom_glink *glink = channel->glink; + unsigned long flags; + + spin_lock_irqsave(&channel->recv_lock, flags); + channel->ept.cb = NULL; + spin_unlock_irqrestore(&channel->recv_lock, flags); + + /* Decouple the potential rpdev from the channel */ + channel->rpdev = NULL; + + qcom_glink_send_close_req(glink, channel); +} + +static int qcom_glink_request_intent(struct qcom_glink *glink, + struct glink_channel *channel, + size_t size) +{ + struct { + u16 id; + u16 cid; + u32 size; + } __packed cmd; + + int ret; + + mutex_lock(&channel->intent_req_lock); + + reinit_completion(&channel->intent_req_comp); + + cmd.id = RPM_CMD_RX_INTENT_REQ; + cmd.cid = channel->lcid; + cmd.size = size; + + ret = qcom_glink_tx(glink, &cmd, sizeof(cmd), NULL, 0, true); + if (ret) + return ret; + + ret = wait_for_completion_timeout(&channel->intent_req_comp, 10 * HZ); + if (!ret) { + dev_err(glink->dev, "intent request timed out\n"); + ret = -ETIMEDOUT; + } else { + ret = channel->intent_req_result ? 0 : -ECANCELED; + } + + mutex_unlock(&channel->intent_req_lock); + return ret; +} + +static int __qcom_glink_send(struct glink_channel *channel, + void *data, int len, bool wait) +{ + struct qcom_glink *glink = channel->glink; + struct glink_core_rx_intent *intent = NULL; + struct glink_core_rx_intent *tmp; + int iid = 0; + struct { + struct glink_msg msg; + __le32 chunk_size; + __le32 left_size; + } __packed req; + int ret; + unsigned long flags; + + if (!glink->intentless) { + while (!intent) { + spin_lock_irqsave(&channel->intent_lock, flags); + idr_for_each_entry(&channel->riids, tmp, iid) { + if (tmp->size >= len && !tmp->in_use) { + tmp->in_use = true; + intent = tmp; + break; + } + } + spin_unlock_irqrestore(&channel->intent_lock, flags); + + /* We found an available intent */ + if (intent) + break; + + if (!wait) + return -EBUSY; + + ret = qcom_glink_request_intent(glink, channel, len); + if (ret < 0) + return ret; + } + + iid = intent->id; + } + + req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA); + req.msg.param1 = cpu_to_le16(channel->lcid); + req.msg.param2 = cpu_to_le32(iid); + req.chunk_size = cpu_to_le32(len); + req.left_size = cpu_to_le32(0); + + ret = qcom_glink_tx(glink, &req, sizeof(req), data, len, wait); + + /* Mark intent available if we failed */ + if (ret && intent) + intent->in_use = false; + + return ret; +} + +static int qcom_glink_send(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct glink_channel *channel = to_glink_channel(ept); + + return __qcom_glink_send(channel, data, len, true); +} + +static int qcom_glink_trysend(struct rpmsg_endpoint *ept, void *data, int len) +{ + struct glink_channel *channel = to_glink_channel(ept); + + return __qcom_glink_send(channel, data, len, false); +} + +/* + * Finds the device_node for the glink child interested in this channel. + */ +static struct device_node *qcom_glink_match_channel(struct device_node *node, + const char *channel) +{ + struct device_node *child; + const char *name; + const char *key; + int ret; + + for_each_available_child_of_node(node, child) { + key = "qcom,glink-channels"; + ret = of_property_read_string(child, key, &name); + if (ret) + continue; + + if (strcmp(name, channel) == 0) + return child; + } + + return NULL; +} + +static const struct rpmsg_device_ops glink_device_ops = { + .create_ept = qcom_glink_create_ept, + .announce_create = qcom_glink_announce_create, +}; + +static const struct rpmsg_endpoint_ops glink_endpoint_ops = { + .destroy_ept = qcom_glink_destroy_ept, + .send = qcom_glink_send, + .trysend = qcom_glink_trysend, +}; + +static void qcom_glink_rpdev_release(struct device *dev) +{ + struct rpmsg_device *rpdev = to_rpmsg_device(dev); + struct glink_channel *channel = to_glink_channel(rpdev->ept); + + channel->rpdev = NULL; + kfree(rpdev); +} + +static int qcom_glink_rx_open(struct qcom_glink *glink, unsigned int rcid, + char *name) +{ + struct glink_channel *channel; + struct rpmsg_device *rpdev; + bool create_device = false; + struct device_node *node; + int lcid; + int ret; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + idr_for_each_entry(&glink->lcids, channel, lcid) { + if (!strcmp(channel->name, name)) + break; + } + spin_unlock_irqrestore(&glink->idr_lock, flags); + + if (!channel) { + channel = qcom_glink_alloc_channel(glink, name); + if (IS_ERR(channel)) + return PTR_ERR(channel); + + /* The opening dance was initiated by the remote */ + create_device = true; + } + + spin_lock_irqsave(&glink->idr_lock, flags); + ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_ATOMIC); + if (ret < 0) { + dev_err(glink->dev, "Unable to insert channel into rcid list\n"); + spin_unlock_irqrestore(&glink->idr_lock, flags); + goto free_channel; + } + channel->rcid = ret; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + complete(&channel->open_req); + + if (create_device) { + rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); + if (!rpdev) { + ret = -ENOMEM; + goto rcid_remove; + } + + rpdev->ept = &channel->ept; + strncpy(rpdev->id.name, name, RPMSG_NAME_SIZE); + rpdev->src = RPMSG_ADDR_ANY; + rpdev->dst = RPMSG_ADDR_ANY; + rpdev->ops = &glink_device_ops; + + node = qcom_glink_match_channel(glink->dev->of_node, name); + rpdev->dev.of_node = node; + rpdev->dev.parent = glink->dev; + rpdev->dev.release = qcom_glink_rpdev_release; + + ret = rpmsg_register_device(rpdev); + if (ret) + goto free_rpdev; + + channel->rpdev = rpdev; + } + + return 0; + +free_rpdev: + kfree(rpdev); +rcid_remove: + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->rcids, channel->rcid); + channel->rcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); +free_channel: + /* Release the reference, iff we took it */ + if (create_device) + kref_put(&channel->refcount, qcom_glink_channel_release); + + return ret; +} + +static void qcom_glink_rx_close(struct qcom_glink *glink, unsigned int rcid) +{ + struct rpmsg_channel_info chinfo; + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->rcids, rcid); + spin_unlock_irqrestore(&glink->idr_lock, flags); + if (WARN(!channel, "close request on unknown channel\n")) + return; + + /* cancel pending rx_done work */ + cancel_work_sync(&channel->intent_work); + + if (channel->rpdev) { + strncpy(chinfo.name, channel->name, sizeof(chinfo.name)); + chinfo.src = RPMSG_ADDR_ANY; + chinfo.dst = RPMSG_ADDR_ANY; + + rpmsg_unregister_device(glink->dev, &chinfo); + } + + qcom_glink_send_close_ack(glink, channel->rcid); + + spin_lock_irqsave(&glink->idr_lock, flags); + idr_remove(&glink->rcids, channel->rcid); + channel->rcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + kref_put(&channel->refcount, qcom_glink_channel_release); +} + +static void qcom_glink_rx_close_ack(struct qcom_glink *glink, unsigned int lcid) +{ + struct glink_channel *channel; + unsigned long flags; + + spin_lock_irqsave(&glink->idr_lock, flags); + channel = idr_find(&glink->lcids, lcid); + if (WARN(!channel, "close ack on unknown channel\n")) { + spin_unlock_irqrestore(&glink->idr_lock, flags); + return; + } + + idr_remove(&glink->lcids, channel->lcid); + channel->lcid = 0; + spin_unlock_irqrestore(&glink->idr_lock, flags); + + kref_put(&channel->refcount, qcom_glink_channel_release); +} + +static void qcom_glink_work(struct work_struct *work) +{ + struct qcom_glink *glink = container_of(work, struct qcom_glink, + rx_work); + struct glink_defer_cmd *dcmd; + struct glink_msg *msg; + unsigned long flags; + unsigned int param1; + unsigned int param2; + unsigned int cmd; + + for (;;) { + spin_lock_irqsave(&glink->rx_lock, flags); + if (list_empty(&glink->rx_queue)) { + spin_unlock_irqrestore(&glink->rx_lock, flags); + break; + } + dcmd = list_first_entry(&glink->rx_queue, + struct glink_defer_cmd, node); + list_del(&dcmd->node); + spin_unlock_irqrestore(&glink->rx_lock, flags); + + msg = &dcmd->msg; + cmd = le16_to_cpu(msg->cmd); + param1 = le16_to_cpu(msg->param1); + param2 = le32_to_cpu(msg->param2); + + switch (cmd) { + case RPM_CMD_VERSION: + qcom_glink_receive_version(glink, param1, param2); + break; + case RPM_CMD_VERSION_ACK: + qcom_glink_receive_version_ack(glink, param1, param2); + break; + case RPM_CMD_OPEN: + qcom_glink_rx_open(glink, param1, msg->data); + break; + case RPM_CMD_CLOSE: + qcom_glink_rx_close(glink, param1); + break; + case RPM_CMD_CLOSE_ACK: + qcom_glink_rx_close_ack(glink, param1); + break; + case RPM_CMD_RX_INTENT_REQ: + qcom_glink_handle_intent_req(glink, param1, param2); + break; + default: + WARN(1, "Unknown defer object %d\n", cmd); + break; + } + + kfree(dcmd); + } +} + +struct qcom_glink *qcom_glink_native_probe(struct device *dev, + unsigned long features, + struct qcom_glink_pipe *rx, + struct qcom_glink_pipe *tx, + bool intentless) +{ + int irq; + int ret; + struct qcom_glink *glink; + + glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); + if (!glink) + return ERR_PTR(-ENOMEM); + + glink->dev = dev; + glink->tx_pipe = tx; + glink->rx_pipe = rx; + + glink->features = features; + glink->intentless = intentless; + + mutex_init(&glink->tx_lock); + spin_lock_init(&glink->rx_lock); + INIT_LIST_HEAD(&glink->rx_queue); + INIT_WORK(&glink->rx_work, qcom_glink_work); + + spin_lock_init(&glink->idr_lock); + idr_init(&glink->lcids); + idr_init(&glink->rcids); + + glink->mbox_client.dev = dev; + glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0); + if (IS_ERR(glink->mbox_chan)) { + if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER) + dev_err(dev, "failed to acquire IPC channel\n"); + return ERR_CAST(glink->mbox_chan); + } + + irq = of_irq_get(dev->of_node, 0); + ret = devm_request_irq(dev, irq, + qcom_glink_native_intr, + IRQF_NO_SUSPEND | IRQF_SHARED, + "glink-native", glink); + if (ret) { + dev_err(dev, "failed to request IRQ\n"); + return ERR_PTR(ret); + } + + glink->irq = irq; + + ret = qcom_glink_send_version(glink); + if (ret) + return ERR_PTR(ret); + + return glink; +} +EXPORT_SYMBOL_GPL(qcom_glink_native_probe); + +static int qcom_glink_remove_device(struct device *dev, void *data) +{ + device_unregister(dev); + + return 0; +} + +void qcom_glink_native_remove(struct qcom_glink *glink) +{ + struct glink_channel *channel; + int cid; + int ret; + unsigned long flags; + + disable_irq(glink->irq); + cancel_work_sync(&glink->rx_work); + + ret = device_for_each_child(glink->dev, NULL, qcom_glink_remove_device); + if (ret) + dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); + + spin_lock_irqsave(&glink->idr_lock, flags); + /* Release any defunct local channels, waiting for close-ack */ + idr_for_each_entry(&glink->lcids, channel, cid) + kref_put(&channel->refcount, qcom_glink_channel_release); + + idr_destroy(&glink->lcids); + idr_destroy(&glink->rcids); + spin_unlock_irqrestore(&glink->idr_lock, flags); + mbox_free_channel(glink->mbox_chan); +} +EXPORT_SYMBOL_GPL(qcom_glink_native_remove); + +void qcom_glink_native_unregister(struct qcom_glink *glink) +{ + device_unregister(glink->dev); +} +EXPORT_SYMBOL_GPL(qcom_glink_native_unregister); diff --git a/drivers/rpmsg/qcom_glink_native.h b/drivers/rpmsg/qcom_glink_native.h new file mode 100644 index 000000000000..0cae8a8199f8 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_native.h @@ -0,0 +1,45 @@ +/* + * Copyright (c) 2016-2017, Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#ifndef __QCOM_GLINK_NATIVE_H__ +#define __QCOM_GLINK_NATIVE_H__ + +#define GLINK_FEATURE_INTENT_REUSE BIT(0) +#define GLINK_FEATURE_MIGRATION BIT(1) +#define GLINK_FEATURE_TRACER_PKT BIT(2) + +struct qcom_glink_pipe { + size_t length; + + size_t (*avail)(struct qcom_glink_pipe *glink_pipe); + + void (*peak)(struct qcom_glink_pipe *glink_pipe, void *data, + unsigned int offset, size_t count); + void (*advance)(struct qcom_glink_pipe *glink_pipe, size_t count); + + void (*write)(struct qcom_glink_pipe *glink_pipe, + const void *hdr, size_t hlen, + const void *data, size_t dlen); +}; + +struct qcom_glink; + +struct qcom_glink *qcom_glink_native_probe(struct device *dev, + unsigned long features, + struct qcom_glink_pipe *rx, + struct qcom_glink_pipe *tx, + bool intentless); +void qcom_glink_native_remove(struct qcom_glink *glink); + +void qcom_glink_native_unregister(struct qcom_glink *glink); +#endif diff --git a/drivers/rpmsg/qcom_glink_rpm.c b/drivers/rpmsg/qcom_glink_rpm.c index 3559a3e84c1e..69b25d157d0f 100644 --- a/drivers/rpmsg/qcom_glink_rpm.c +++ b/drivers/rpmsg/qcom_glink_rpm.c @@ -27,6 +27,7 @@ #include <linux/mailbox_client.h> #include "rpmsg_internal.h" +#include "qcom_glink_native.h" #define RPM_TOC_SIZE 256 #define RPM_TOC_MAGIC 0x67727430 /* grt0 */ @@ -36,10 +37,7 @@ #define RPM_TX_FIFO_ID 0x61703272 /* ap2r */ #define RPM_RX_FIFO_ID 0x72326170 /* r2ap */ -#define GLINK_NAME_SIZE 32 - -#define RPM_GLINK_CID_MIN 1 -#define RPM_GLINK_CID_MAX 65536 +#define to_rpm_pipe(p) container_of(p, struct glink_rpm_pipe, native) struct rpm_toc_entry { __le32 id; @@ -54,170 +52,18 @@ struct rpm_toc { struct rpm_toc_entry entries[]; } __packed; -struct glink_msg { - __le16 cmd; - __le16 param1; - __le32 param2; - u8 data[]; -} __packed; - struct glink_rpm_pipe { + struct qcom_glink_pipe native; + void __iomem *tail; void __iomem *head; void __iomem *fifo; - - size_t length; -}; - -/** - * struct glink_defer_cmd - deferred incoming control message - * @node: list node - * @msg: message header - * data: payload of the message - * - * Copy of a received control message, to be added to @rx_queue and processed - * by @rx_work of @glink_rpm. - */ -struct glink_defer_cmd { - struct list_head node; - - struct glink_msg msg; - u8 data[]; -}; - -/** - * struct glink_rpm - driver context, relates to one remote subsystem - * @dev: reference to the associated struct device - * @doorbell: "rpm_hlos" ipc doorbell - * @rx_pipe: pipe object for receive FIFO - * @tx_pipe: pipe object for transmit FIFO - * @irq: IRQ for signaling incoming events - * @rx_work: worker for handling received control messages - * @rx_lock: protects the @rx_queue - * @rx_queue: queue of received control messages to be processed in @rx_work - * @tx_lock: synchronizes operations on the tx fifo - * @idr_lock: synchronizes @lcids and @rcids modifications - * @lcids: idr of all channels with a known local channel id - * @rcids: idr of all channels with a known remote channel id - */ -struct glink_rpm { - struct device *dev; - - struct mbox_client mbox_client; - struct mbox_chan *mbox_chan; - - struct glink_rpm_pipe rx_pipe; - struct glink_rpm_pipe tx_pipe; - - int irq; - - struct work_struct rx_work; - spinlock_t rx_lock; - struct list_head rx_queue; - - struct mutex tx_lock; - - struct mutex idr_lock; - struct idr lcids; - struct idr rcids; -}; - -enum { - GLINK_STATE_CLOSED, - GLINK_STATE_OPENING, - GLINK_STATE_OPEN, - GLINK_STATE_CLOSING, }; -/** - * struct glink_channel - internal representation of a channel - * @rpdev: rpdev reference, only used for primary endpoints - * @ept: rpmsg endpoint this channel is associated with - * @glink: glink_rpm context handle - * @refcount: refcount for the channel object - * @recv_lock: guard for @ept.cb - * @name: unique channel name/identifier - * @lcid: channel id, in local space - * @rcid: channel id, in remote space - * @buf: receive buffer, for gathering fragments - * @buf_offset: write offset in @buf - * @buf_size: size of current @buf - * @open_ack: completed once remote has acked the open-request - * @open_req: completed once open-request has been received - */ -struct glink_channel { - struct rpmsg_endpoint ept; - - struct rpmsg_device *rpdev; - struct glink_rpm *glink; - - struct kref refcount; - - spinlock_t recv_lock; - - char *name; - unsigned int lcid; - unsigned int rcid; - - void *buf; - int buf_offset; - int buf_size; - - struct completion open_ack; - struct completion open_req; -}; - -#define to_glink_channel(_ept) container_of(_ept, struct glink_channel, ept) - -static const struct rpmsg_endpoint_ops glink_endpoint_ops; - -#define RPM_CMD_VERSION 0 -#define RPM_CMD_VERSION_ACK 1 -#define RPM_CMD_OPEN 2 -#define RPM_CMD_CLOSE 3 -#define RPM_CMD_OPEN_ACK 4 -#define RPM_CMD_TX_DATA 9 -#define RPM_CMD_CLOSE_ACK 11 -#define RPM_CMD_TX_DATA_CONT 12 -#define RPM_CMD_READ_NOTIF 13 - -#define GLINK_FEATURE_INTENTLESS BIT(1) - -static struct glink_channel *glink_rpm_alloc_channel(struct glink_rpm *glink, - const char *name) -{ - struct glink_channel *channel; - - channel = kzalloc(sizeof(*channel), GFP_KERNEL); - if (!channel) - return ERR_PTR(-ENOMEM); - - /* Setup glink internal glink_channel data */ - spin_lock_init(&channel->recv_lock); - channel->glink = glink; - channel->name = kstrdup(name, GFP_KERNEL); - - init_completion(&channel->open_req); - init_completion(&channel->open_ack); - - kref_init(&channel->refcount); - - return channel; -} - -static void glink_rpm_channel_release(struct kref *ref) -{ - struct glink_channel *channel = container_of(ref, struct glink_channel, - refcount); - - kfree(channel->name); - kfree(channel); -} - -static size_t glink_rpm_rx_avail(struct glink_rpm *glink) +static size_t glink_rpm_rx_avail(struct qcom_glink_pipe *glink_pipe) { - struct glink_rpm_pipe *pipe = &glink->rx_pipe; + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); unsigned int head; unsigned int tail; @@ -225,21 +71,24 @@ static size_t glink_rpm_rx_avail(struct glink_rpm *glink) tail = readl(pipe->tail); if (head < tail) - return pipe->length - tail + head; + return pipe->native.length - tail + head; else return head - tail; } -static void glink_rpm_rx_peak(struct glink_rpm *glink, - void *data, size_t count) +static void glink_rpm_rx_peak(struct qcom_glink_pipe *glink_pipe, + void *data, unsigned int offset, size_t count) { - struct glink_rpm_pipe *pipe = &glink->rx_pipe; + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); unsigned int tail; size_t len; tail = readl(pipe->tail); + tail += offset; + if (tail >= pipe->native.length) + tail -= pipe->native.length; - len = min_t(size_t, count, pipe->length - tail); + len = min_t(size_t, count, pipe->native.length - tail); if (len) { __ioread32_copy(data, pipe->fifo + tail, len / sizeof(u32)); @@ -251,24 +100,24 @@ static void glink_rpm_rx_peak(struct glink_rpm *glink, } } -static void glink_rpm_rx_advance(struct glink_rpm *glink, +static void glink_rpm_rx_advance(struct qcom_glink_pipe *glink_pipe, size_t count) { - struct glink_rpm_pipe *pipe = &glink->rx_pipe; + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); unsigned int tail; tail = readl(pipe->tail); tail += count; - if (tail >= pipe->length) - tail -= pipe->length; + if (tail >= pipe->native.length) + tail -= pipe->native.length; writel(tail, pipe->tail); } -static size_t glink_rpm_tx_avail(struct glink_rpm *glink) +static size_t glink_rpm_tx_avail(struct qcom_glink_pipe *glink_pipe) { - struct glink_rpm_pipe *pipe = &glink->tx_pipe; + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); unsigned int head; unsigned int tail; @@ -276,19 +125,18 @@ static size_t glink_rpm_tx_avail(struct glink_rpm *glink) tail = readl(pipe->tail); if (tail <= head) - return pipe->length - head + tail; + return pipe->native.length - head + tail; else return tail - head; } -static unsigned int glink_rpm_tx_write(struct glink_rpm *glink, - unsigned int head, - const void *data, size_t count) +static unsigned int glink_rpm_tx_write_one(struct glink_rpm_pipe *pipe, + unsigned int head, + const void *data, size_t count) { - struct glink_rpm_pipe *pipe = &glink->tx_pipe; size_t len; - len = min_t(size_t, count, pipe->length - head); + len = min_t(size_t, count, pipe->native.length - head); if (len) { __iowrite32_copy(pipe->fifo + head, data, len / sizeof(u32)); @@ -300,725 +148,43 @@ static unsigned int glink_rpm_tx_write(struct glink_rpm *glink, } head += count; - if (head >= pipe->length) - head -= pipe->length; + if (head >= pipe->native.length) + head -= pipe->native.length; return head; } -static int glink_rpm_tx(struct glink_rpm *glink, - const void *hdr, size_t hlen, - const void *data, size_t dlen, bool wait) +static void glink_rpm_tx_write(struct qcom_glink_pipe *glink_pipe, + const void *hdr, size_t hlen, + const void *data, size_t dlen) { - struct glink_rpm_pipe *pipe = &glink->tx_pipe; + struct glink_rpm_pipe *pipe = to_rpm_pipe(glink_pipe); + size_t tlen = hlen + dlen; + size_t aligned_dlen; unsigned int head; - unsigned int tlen = hlen + dlen; - int ret; - - /* Reject packets that are too big */ - if (tlen >= glink->tx_pipe.length) - return -EINVAL; - - if (WARN(tlen % 8, "Unaligned TX request")) - return -EINVAL; - - ret = mutex_lock_interruptible(&glink->tx_lock); - if (ret) - return ret; - - while (glink_rpm_tx_avail(glink) < tlen) { - if (!wait) { - ret = -ENOMEM; - goto out; - } - - msleep(10); - } - - head = readl(pipe->head); - head = glink_rpm_tx_write(glink, head, hdr, hlen); - head = glink_rpm_tx_write(glink, head, data, dlen); - writel(head, pipe->head); - - mbox_send_message(glink->mbox_chan, NULL); - mbox_client_txdone(glink->mbox_chan, 0); - -out: - mutex_unlock(&glink->tx_lock); + char padding[8] = {0}; + size_t pad; - return ret; -} - -static int glink_rpm_send_version(struct glink_rpm *glink) -{ - struct glink_msg msg; - - msg.cmd = cpu_to_le16(RPM_CMD_VERSION); - msg.param1 = cpu_to_le16(1); - msg.param2 = cpu_to_le32(GLINK_FEATURE_INTENTLESS); - - return glink_rpm_tx(glink, &msg, sizeof(msg), NULL, 0, true); -} - -static void glink_rpm_send_version_ack(struct glink_rpm *glink) -{ - struct glink_msg msg; - - msg.cmd = cpu_to_le16(RPM_CMD_VERSION_ACK); - msg.param1 = cpu_to_le16(1); - msg.param2 = cpu_to_le32(0); - - glink_rpm_tx(glink, &msg, sizeof(msg), NULL, 0, true); -} - -static void glink_rpm_send_open_ack(struct glink_rpm *glink, - struct glink_channel *channel) -{ - struct glink_msg msg; - - msg.cmd = cpu_to_le16(RPM_CMD_OPEN_ACK); - msg.param1 = cpu_to_le16(channel->rcid); - msg.param2 = cpu_to_le32(0); - - glink_rpm_tx(glink, &msg, sizeof(msg), NULL, 0, true); -} - -/** - * glink_rpm_send_open_req() - send a RPM_CMD_OPEN request to the remote - * @glink: - * @channel: - * - * Allocates a local channel id and sends a RPM_CMD_OPEN message to the remote. - * Will return with refcount held, regardless of outcome. - * - * Returns 0 on success, negative errno otherwise. - */ -static int glink_rpm_send_open_req(struct glink_rpm *glink, - struct glink_channel *channel) -{ - struct { - struct glink_msg msg; - u8 name[GLINK_NAME_SIZE]; - } __packed req; - int name_len = strlen(channel->name) + 1; - int req_len = ALIGN(sizeof(req.msg) + name_len, 8); - int ret; - - kref_get(&channel->refcount); - - mutex_lock(&glink->idr_lock); - ret = idr_alloc_cyclic(&glink->lcids, channel, - RPM_GLINK_CID_MIN, RPM_GLINK_CID_MAX, GFP_KERNEL); - mutex_unlock(&glink->idr_lock); - if (ret < 0) - return ret; - - channel->lcid = ret; - - req.msg.cmd = cpu_to_le16(RPM_CMD_OPEN); - req.msg.param1 = cpu_to_le16(channel->lcid); - req.msg.param2 = cpu_to_le32(name_len); - strcpy(req.name, channel->name); - - ret = glink_rpm_tx(glink, &req, req_len, NULL, 0, true); - if (ret) - goto remove_idr; - - return 0; - -remove_idr: - mutex_lock(&glink->idr_lock); - idr_remove(&glink->lcids, channel->lcid); - channel->lcid = 0; - mutex_unlock(&glink->idr_lock); - - return ret; -} - -static void glink_rpm_send_close_req(struct glink_rpm *glink, - struct glink_channel *channel) -{ - struct glink_msg req; - - req.cmd = cpu_to_le16(RPM_CMD_CLOSE); - req.param1 = cpu_to_le16(channel->lcid); - req.param2 = 0; - - glink_rpm_tx(glink, &req, sizeof(req), NULL, 0, true); -} - -static void glink_rpm_send_close_ack(struct glink_rpm *glink, unsigned int rcid) -{ - struct glink_msg req; - - req.cmd = cpu_to_le16(RPM_CMD_CLOSE_ACK); - req.param1 = cpu_to_le16(rcid); - req.param2 = 0; - - glink_rpm_tx(glink, &req, sizeof(req), NULL, 0, true); -} - -static int glink_rpm_rx_defer(struct glink_rpm *glink, size_t extra) -{ - struct glink_defer_cmd *dcmd; - - extra = ALIGN(extra, 8); - - if (glink_rpm_rx_avail(glink) < sizeof(struct glink_msg) + extra) { - dev_dbg(glink->dev, "Insufficient data in rx fifo"); - return -ENXIO; - } - - dcmd = kzalloc(sizeof(*dcmd) + extra, GFP_ATOMIC); - if (!dcmd) - return -ENOMEM; - - INIT_LIST_HEAD(&dcmd->node); - - glink_rpm_rx_peak(glink, &dcmd->msg, sizeof(dcmd->msg) + extra); - - spin_lock(&glink->rx_lock); - list_add_tail(&dcmd->node, &glink->rx_queue); - spin_unlock(&glink->rx_lock); - - schedule_work(&glink->rx_work); - glink_rpm_rx_advance(glink, sizeof(dcmd->msg) + extra); - - return 0; -} - -static int glink_rpm_rx_data(struct glink_rpm *glink, size_t avail) -{ - struct glink_channel *channel; - struct { - struct glink_msg msg; - __le32 chunk_size; - __le32 left_size; - } __packed hdr; - unsigned int chunk_size; - unsigned int left_size; - unsigned int rcid; - - if (avail < sizeof(hdr)) { - dev_dbg(glink->dev, "Not enough data in fifo\n"); - return -EAGAIN; - } - - glink_rpm_rx_peak(glink, &hdr, sizeof(hdr)); - chunk_size = le32_to_cpu(hdr.chunk_size); - left_size = le32_to_cpu(hdr.left_size); - - if (avail < sizeof(hdr) + chunk_size) { - dev_dbg(glink->dev, "Payload not yet in fifo\n"); - return -EAGAIN; - } - - if (WARN(chunk_size % 4, "Incoming data must be word aligned\n")) - return -EINVAL; - - rcid = le16_to_cpu(hdr.msg.param1); - channel = idr_find(&glink->rcids, rcid); - if (!channel) { - dev_dbg(glink->dev, "Data on non-existing channel\n"); - - /* Drop the message */ - glink_rpm_rx_advance(glink, ALIGN(sizeof(hdr) + chunk_size, 8)); - return 0; - } - - /* Might have an ongoing, fragmented, message to append */ - if (!channel->buf) { - channel->buf = kmalloc(chunk_size + left_size, GFP_ATOMIC); - if (!channel->buf) - return -ENOMEM; - - channel->buf_size = chunk_size + left_size; - channel->buf_offset = 0; - } - - glink_rpm_rx_advance(glink, sizeof(hdr)); - - if (channel->buf_size - channel->buf_offset < chunk_size) { - dev_err(glink->dev, "Insufficient space in input buffer\n"); - - /* The packet header lied, drop payload */ - glink_rpm_rx_advance(glink, chunk_size); - return -ENOMEM; - } - - glink_rpm_rx_peak(glink, channel->buf + channel->buf_offset, chunk_size); - channel->buf_offset += chunk_size; - - /* Handle message when no fragments remain to be received */ - if (!left_size) { - spin_lock(&channel->recv_lock); - if (channel->ept.cb) { - channel->ept.cb(channel->ept.rpdev, - channel->buf, - channel->buf_offset, - channel->ept.priv, - RPMSG_ADDR_ANY); - } - spin_unlock(&channel->recv_lock); - - kfree(channel->buf); - channel->buf = NULL; - channel->buf_size = 0; - } - - /* Each message starts at 8 byte aligned address */ - glink_rpm_rx_advance(glink, ALIGN(chunk_size, 8)); - - return 0; -} - -static int glink_rpm_rx_open_ack(struct glink_rpm *glink, unsigned int lcid) -{ - struct glink_channel *channel; - - channel = idr_find(&glink->lcids, lcid); - if (!channel) { - dev_err(glink->dev, "Invalid open ack packet\n"); - return -EINVAL; - } - - complete(&channel->open_ack); - - return 0; -} - -static irqreturn_t glink_rpm_intr(int irq, void *data) -{ - struct glink_rpm *glink = data; - struct glink_msg msg; - unsigned int param1; - unsigned int param2; - unsigned int avail; - unsigned int cmd; - int ret; - - for (;;) { - avail = glink_rpm_rx_avail(glink); - if (avail < sizeof(msg)) - break; - - glink_rpm_rx_peak(glink, &msg, sizeof(msg)); - - cmd = le16_to_cpu(msg.cmd); - param1 = le16_to_cpu(msg.param1); - param2 = le32_to_cpu(msg.param2); - - switch (cmd) { - case RPM_CMD_VERSION: - case RPM_CMD_VERSION_ACK: - case RPM_CMD_CLOSE: - case RPM_CMD_CLOSE_ACK: - ret = glink_rpm_rx_defer(glink, 0); - break; - case RPM_CMD_OPEN_ACK: - ret = glink_rpm_rx_open_ack(glink, param1); - glink_rpm_rx_advance(glink, ALIGN(sizeof(msg), 8)); - break; - case RPM_CMD_OPEN: - ret = glink_rpm_rx_defer(glink, param2); - break; - case RPM_CMD_TX_DATA: - case RPM_CMD_TX_DATA_CONT: - ret = glink_rpm_rx_data(glink, avail); - break; - case RPM_CMD_READ_NOTIF: - glink_rpm_rx_advance(glink, ALIGN(sizeof(msg), 8)); - - mbox_send_message(glink->mbox_chan, NULL); - mbox_client_txdone(glink->mbox_chan, 0); - - ret = 0; - break; - default: - dev_err(glink->dev, "unhandled rx cmd: %d\n", cmd); - ret = -EINVAL; - break; - } - - if (ret) - break; - } - - return IRQ_HANDLED; -} - -/* Locally initiated rpmsg_create_ept */ -static struct glink_channel *glink_rpm_create_local(struct glink_rpm *glink, - const char *name) -{ - struct glink_channel *channel; - int ret; - - channel = glink_rpm_alloc_channel(glink, name); - if (IS_ERR(channel)) - return ERR_CAST(channel); - - ret = glink_rpm_send_open_req(glink, channel); - if (ret) - goto release_channel; - - ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); - if (!ret) - goto err_timeout; - - ret = wait_for_completion_timeout(&channel->open_req, 5 * HZ); - if (!ret) - goto err_timeout; - - glink_rpm_send_open_ack(glink, channel); - - return channel; - -err_timeout: - /* glink_rpm_send_open_req() did register the channel in lcids*/ - mutex_lock(&glink->idr_lock); - idr_remove(&glink->lcids, channel->lcid); - mutex_unlock(&glink->idr_lock); - -release_channel: - /* Release glink_rpm_send_open_req() reference */ - kref_put(&channel->refcount, glink_rpm_channel_release); - /* Release glink_rpm_alloc_channel() reference */ - kref_put(&channel->refcount, glink_rpm_channel_release); - - return ERR_PTR(-ETIMEDOUT); -} - -/* Remote initiated rpmsg_create_ept */ -static int glink_rpm_create_remote(struct glink_rpm *glink, - struct glink_channel *channel) -{ - int ret; - - glink_rpm_send_open_ack(glink, channel); - - ret = glink_rpm_send_open_req(glink, channel); - if (ret) - goto close_link; - - ret = wait_for_completion_timeout(&channel->open_ack, 5 * HZ); - if (!ret) { - ret = -ETIMEDOUT; - goto close_link; - } - - return 0; + /* Header length comes from glink native and is always 4 byte aligned */ + if (WARN(hlen % 4, "Glink Header length must be 4 bytes aligned\n")) + return; -close_link: /* - * Send a close request to "undo" our open-ack. The close-ack will - * release the last reference. + * Move the unaligned tail of the message to the padding chunk, to + * ensure word aligned accesses */ - glink_rpm_send_close_req(glink, channel); - - /* Release glink_rpm_send_open_req() reference */ - kref_put(&channel->refcount, glink_rpm_channel_release); - - return ret; -} - -static struct rpmsg_endpoint *glink_rpm_create_ept(struct rpmsg_device *rpdev, - rpmsg_rx_cb_t cb, void *priv, - struct rpmsg_channel_info chinfo) -{ - struct glink_channel *parent = to_glink_channel(rpdev->ept); - struct glink_channel *channel; - struct glink_rpm *glink = parent->glink; - struct rpmsg_endpoint *ept; - const char *name = chinfo.name; - int cid; - int ret; - - idr_for_each_entry(&glink->rcids, channel, cid) { - if (!strcmp(channel->name, name)) - break; - } - - if (!channel) { - channel = glink_rpm_create_local(glink, name); - if (IS_ERR(channel)) - return NULL; - } else { - ret = glink_rpm_create_remote(glink, channel); - if (ret) - return NULL; - } - - ept = &channel->ept; - ept->rpdev = rpdev; - ept->cb = cb; - ept->priv = priv; - ept->ops = &glink_endpoint_ops; - - return ept; -} - -static void glink_rpm_destroy_ept(struct rpmsg_endpoint *ept) -{ - struct glink_channel *channel = to_glink_channel(ept); - struct glink_rpm *glink = channel->glink; - unsigned long flags; - - spin_lock_irqsave(&channel->recv_lock, flags); - channel->ept.cb = NULL; - spin_unlock_irqrestore(&channel->recv_lock, flags); - - /* Decouple the potential rpdev from the channel */ - channel->rpdev = NULL; - - glink_rpm_send_close_req(glink, channel); -} - -static int __glink_rpm_send(struct glink_channel *channel, - void *data, int len, bool wait) -{ - struct glink_rpm *glink = channel->glink; - struct { - struct glink_msg msg; - __le32 chunk_size; - __le32 left_size; - } __packed req; - - if (WARN(len % 8, "RPM GLINK expects 8 byte aligned messages\n")) - return -EINVAL; - - req.msg.cmd = cpu_to_le16(RPM_CMD_TX_DATA); - req.msg.param1 = cpu_to_le16(channel->lcid); - req.msg.param2 = cpu_to_le32(channel->rcid); - req.chunk_size = cpu_to_le32(len); - req.left_size = cpu_to_le32(0); - - return glink_rpm_tx(glink, &req, sizeof(req), data, len, wait); -} - -static int glink_rpm_send(struct rpmsg_endpoint *ept, void *data, int len) -{ - struct glink_channel *channel = to_glink_channel(ept); - - return __glink_rpm_send(channel, data, len, true); -} - -static int glink_rpm_trysend(struct rpmsg_endpoint *ept, void *data, int len) -{ - struct glink_channel *channel = to_glink_channel(ept); - - return __glink_rpm_send(channel, data, len, false); -} - -/* - * Finds the device_node for the glink child interested in this channel. - */ -static struct device_node *glink_rpm_match_channel(struct device_node *node, - const char *channel) -{ - struct device_node *child; - const char *name; - const char *key; - int ret; - - for_each_available_child_of_node(node, child) { - key = "qcom,glink-channels"; - ret = of_property_read_string(child, key, &name); - if (ret) - continue; - - if (strcmp(name, channel) == 0) - return child; - } - - return NULL; -} - -static const struct rpmsg_device_ops glink_device_ops = { - .create_ept = glink_rpm_create_ept, -}; - -static const struct rpmsg_endpoint_ops glink_endpoint_ops = { - .destroy_ept = glink_rpm_destroy_ept, - .send = glink_rpm_send, - .trysend = glink_rpm_trysend, -}; - -static void glink_rpm_rpdev_release(struct device *dev) -{ - struct rpmsg_device *rpdev = to_rpmsg_device(dev); - struct glink_channel *channel = to_glink_channel(rpdev->ept); - - channel->rpdev = NULL; - kfree(rpdev); -} - -static int glink_rpm_rx_open(struct glink_rpm *glink, unsigned int rcid, - char *name) -{ - struct glink_channel *channel; - struct rpmsg_device *rpdev; - bool create_device = false; - int lcid; - int ret; - - idr_for_each_entry(&glink->lcids, channel, lcid) { - if (!strcmp(channel->name, name)) - break; - } - - if (!channel) { - channel = glink_rpm_alloc_channel(glink, name); - if (IS_ERR(channel)) - return PTR_ERR(channel); - - /* The opening dance was initiated by the remote */ - create_device = true; - } - - mutex_lock(&glink->idr_lock); - ret = idr_alloc(&glink->rcids, channel, rcid, rcid + 1, GFP_KERNEL); - if (ret < 0) { - dev_err(glink->dev, "Unable to insert channel into rcid list\n"); - mutex_unlock(&glink->idr_lock); - goto free_channel; - } - channel->rcid = ret; - mutex_unlock(&glink->idr_lock); - - complete(&channel->open_req); - - if (create_device) { - rpdev = kzalloc(sizeof(*rpdev), GFP_KERNEL); - if (!rpdev) { - ret = -ENOMEM; - goto rcid_remove; - } - - rpdev->ept = &channel->ept; - strncpy(rpdev->id.name, name, RPMSG_NAME_SIZE); - rpdev->src = RPMSG_ADDR_ANY; - rpdev->dst = RPMSG_ADDR_ANY; - rpdev->ops = &glink_device_ops; - - rpdev->dev.of_node = glink_rpm_match_channel(glink->dev->of_node, name); - rpdev->dev.parent = glink->dev; - rpdev->dev.release = glink_rpm_rpdev_release; - - ret = rpmsg_register_device(rpdev); - if (ret) - goto free_rpdev; - - channel->rpdev = rpdev; - } - - return 0; - -free_rpdev: - kfree(rpdev); -rcid_remove: - mutex_lock(&glink->idr_lock); - idr_remove(&glink->rcids, channel->rcid); - channel->rcid = 0; - mutex_unlock(&glink->idr_lock); -free_channel: - /* Release the reference, iff we took it */ - if (create_device) - kref_put(&channel->refcount, glink_rpm_channel_release); - - return ret; -} - -static void glink_rpm_rx_close(struct glink_rpm *glink, unsigned int rcid) -{ - struct rpmsg_channel_info chinfo; - struct glink_channel *channel; - - channel = idr_find(&glink->rcids, rcid); - if (WARN(!channel, "close request on unknown channel\n")) - return; - - if (channel->rpdev) { - strncpy(chinfo.name, channel->name, sizeof(chinfo.name)); - chinfo.src = RPMSG_ADDR_ANY; - chinfo.dst = RPMSG_ADDR_ANY; - - rpmsg_unregister_device(glink->dev, &chinfo); - } - - glink_rpm_send_close_ack(glink, channel->rcid); - - mutex_lock(&glink->idr_lock); - idr_remove(&glink->rcids, channel->rcid); - channel->rcid = 0; - mutex_unlock(&glink->idr_lock); - - kref_put(&channel->refcount, glink_rpm_channel_release); -} + aligned_dlen = ALIGN_DOWN(dlen, 4); + if (aligned_dlen != dlen) + memcpy(padding, data + aligned_dlen, dlen - aligned_dlen); -static void glink_rpm_rx_close_ack(struct glink_rpm *glink, unsigned int lcid) -{ - struct glink_channel *channel; - - channel = idr_find(&glink->lcids, lcid); - if (WARN(!channel, "close ack on unknown channel\n")) - return; - - mutex_lock(&glink->idr_lock); - idr_remove(&glink->lcids, channel->lcid); - channel->lcid = 0; - mutex_unlock(&glink->idr_lock); - - kref_put(&channel->refcount, glink_rpm_channel_release); -} - -static void glink_rpm_work(struct work_struct *work) -{ - struct glink_rpm *glink = container_of(work, struct glink_rpm, rx_work); - struct glink_defer_cmd *dcmd; - struct glink_msg *msg; - unsigned long flags; - unsigned int param1; - unsigned int param2; - unsigned int cmd; - - for (;;) { - spin_lock_irqsave(&glink->rx_lock, flags); - if (list_empty(&glink->rx_queue)) { - spin_unlock_irqrestore(&glink->rx_lock, flags); - break; - } - dcmd = list_first_entry(&glink->rx_queue, struct glink_defer_cmd, node); - list_del(&dcmd->node); - spin_unlock_irqrestore(&glink->rx_lock, flags); - - msg = &dcmd->msg; - cmd = le16_to_cpu(msg->cmd); - param1 = le16_to_cpu(msg->param1); - param2 = le32_to_cpu(msg->param2); - - switch (cmd) { - case RPM_CMD_VERSION: - glink_rpm_send_version_ack(glink); - break; - case RPM_CMD_VERSION_ACK: - break; - case RPM_CMD_OPEN: - glink_rpm_rx_open(glink, param1, msg->data); - break; - case RPM_CMD_CLOSE: - glink_rpm_rx_close(glink, param1); - break; - case RPM_CMD_CLOSE_ACK: - glink_rpm_rx_close_ack(glink, param1); - break; - default: - WARN(1, "Unknown defer object %d\n", cmd); - break; - } + head = readl(pipe->head); + head = glink_rpm_tx_write_one(pipe, head, hdr, hlen); + head = glink_rpm_tx_write_one(pipe, head, data, aligned_dlen); - kfree(dcmd); - } + pad = ALIGN(tlen, 8) - ALIGN_DOWN(tlen, 4); + if (pad) + head = glink_rpm_tx_write_one(pipe, head, padding, pad); + writel(head, pipe->head); } static int glink_rpm_parse_toc(struct device *dev, @@ -1067,14 +233,14 @@ static int glink_rpm_parse_toc(struct device *dev, switch (id) { case RPM_RX_FIFO_ID: - rx->length = size; + rx->native.length = size; rx->tail = msg_ram + offset; rx->head = msg_ram + offset + sizeof(u32); rx->fifo = msg_ram + offset + 2 * sizeof(u32); break; case RPM_TX_FIFO_ID: - tx->length = size; + tx->native.length = size; tx->tail = msg_ram + offset; tx->head = msg_ram + offset + sizeof(u32); @@ -1098,38 +264,21 @@ err_inval: static int glink_rpm_probe(struct platform_device *pdev) { - struct glink_rpm *glink; + struct qcom_glink *glink; + struct glink_rpm_pipe *rx_pipe; + struct glink_rpm_pipe *tx_pipe; struct device_node *np; void __iomem *msg_ram; size_t msg_ram_size; struct device *dev = &pdev->dev; struct resource r; - int irq; int ret; - glink = devm_kzalloc(dev, sizeof(*glink), GFP_KERNEL); - if (!glink) + rx_pipe = devm_kzalloc(&pdev->dev, sizeof(*rx_pipe), GFP_KERNEL); + tx_pipe = devm_kzalloc(&pdev->dev, sizeof(*tx_pipe), GFP_KERNEL); + if (!rx_pipe || !tx_pipe) return -ENOMEM; - glink->dev = dev; - - mutex_init(&glink->tx_lock); - spin_lock_init(&glink->rx_lock); - INIT_LIST_HEAD(&glink->rx_queue); - INIT_WORK(&glink->rx_work, glink_rpm_work); - - mutex_init(&glink->idr_lock); - idr_init(&glink->lcids); - idr_init(&glink->rcids); - - glink->mbox_client.dev = &pdev->dev; - glink->mbox_chan = mbox_request_channel(&glink->mbox_client, 0); - if (IS_ERR(glink->mbox_chan)) { - if (PTR_ERR(glink->mbox_chan) != -EPROBE_DEFER) - dev_err(&pdev->dev, "failed to acquire IPC channel\n"); - return PTR_ERR(glink->mbox_chan); - } - np = of_parse_phandle(dev->of_node, "qcom,rpm-msg-ram", 0); ret = of_address_to_resource(np, 0, &r); of_node_put(np); @@ -1142,61 +291,38 @@ static int glink_rpm_probe(struct platform_device *pdev) return -ENOMEM; ret = glink_rpm_parse_toc(dev, msg_ram, msg_ram_size, - &glink->rx_pipe, &glink->tx_pipe); + rx_pipe, tx_pipe); if (ret) return ret; - writel(0, glink->tx_pipe.head); - writel(0, glink->rx_pipe.tail); + /* Pipe specific accessors */ + rx_pipe->native.avail = glink_rpm_rx_avail; + rx_pipe->native.peak = glink_rpm_rx_peak; + rx_pipe->native.advance = glink_rpm_rx_advance; + tx_pipe->native.avail = glink_rpm_tx_avail; + tx_pipe->native.write = glink_rpm_tx_write; - irq = platform_get_irq(pdev, 0); - ret = devm_request_irq(dev, irq, - glink_rpm_intr, - IRQF_NO_SUSPEND | IRQF_SHARED, - "glink-rpm", glink); - if (ret) { - dev_err(dev, "Failed to request IRQ\n"); - return ret; - } - - glink->irq = irq; + writel(0, tx_pipe->head); + writel(0, rx_pipe->tail); - ret = glink_rpm_send_version(glink); - if (ret) - return ret; + glink = qcom_glink_native_probe(&pdev->dev, + 0, + &rx_pipe->native, + &tx_pipe->native, + true); + if (IS_ERR(glink)) + return PTR_ERR(glink); platform_set_drvdata(pdev, glink); return 0; } -static int glink_rpm_remove_device(struct device *dev, void *data) -{ - device_unregister(dev); - - return 0; -} - static int glink_rpm_remove(struct platform_device *pdev) { - struct glink_rpm *glink = platform_get_drvdata(pdev); - struct glink_channel *channel; - int cid; - int ret; - - disable_irq(glink->irq); - cancel_work_sync(&glink->rx_work); - - ret = device_for_each_child(glink->dev, NULL, glink_rpm_remove_device); - if (ret) - dev_warn(glink->dev, "Can't remove GLINK devices: %d\n", ret); - - /* Release any defunct local channels, waiting for close-ack */ - idr_for_each_entry(&glink->lcids, channel, cid) - kref_put(&channel->refcount, glink_rpm_channel_release); + struct qcom_glink *glink = platform_get_drvdata(pdev); - idr_destroy(&glink->lcids); - idr_destroy(&glink->rcids); + qcom_glink_native_remove(glink); return 0; } diff --git a/drivers/rpmsg/qcom_glink_smem.c b/drivers/rpmsg/qcom_glink_smem.c new file mode 100644 index 000000000000..5cdaa5f8fb61 --- /dev/null +++ b/drivers/rpmsg/qcom_glink_smem.c @@ -0,0 +1,316 @@ +/* + * Copyright (c) 2016, Linaro Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 and + * only version 2 as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + */ + +#include <linux/io.h> +#include <linux/module.h> +#include <linux/of.h> +#include <linux/of_address.h> +#include <linux/interrupt.h> +#include <linux/platform_device.h> +#include <linux/mfd/syscon.h> +#include <linux/slab.h> +#include <linux/rpmsg.h> +#include <linux/idr.h> +#include <linux/circ_buf.h> +#include <linux/soc/qcom/smem.h> +#include <linux/sizes.h> +#include <linux/delay.h> +#include <linux/regmap.h> +#include <linux/workqueue.h> +#include <linux/list.h> + +#include <linux/delay.h> +#include <linux/rpmsg.h> +#include <linux/rpmsg/qcom_glink.h> + +#include "qcom_glink_native.h" + +#define FIFO_FULL_RESERVE 8 +#define FIFO_ALIGNMENT 8 +#define TX_BLOCKED_CMD_RESERVE 8 /* size of struct read_notif_request */ + +#define SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR 478 +#define SMEM_GLINK_NATIVE_XPRT_FIFO_0 479 +#define SMEM_GLINK_NATIVE_XPRT_FIFO_1 480 + +struct glink_smem_pipe { + struct qcom_glink_pipe native; + + __le32 *tail; + __le32 *head; + + void *fifo; + + int remote_pid; +}; + +#define to_smem_pipe(p) container_of(p, struct glink_smem_pipe, native) + +static size_t glink_smem_rx_avail(struct qcom_glink_pipe *np) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + size_t len; + void *fifo; + u32 head; + u32 tail; + + if (!pipe->fifo) { + fifo = qcom_smem_get(pipe->remote_pid, + SMEM_GLINK_NATIVE_XPRT_FIFO_1, &len); + if (IS_ERR(fifo)) { + pr_err("failed to acquire RX fifo handle: %ld\n", + PTR_ERR(fifo)); + return 0; + } + + pipe->fifo = fifo; + pipe->native.length = len; + } + + head = le32_to_cpu(*pipe->head); + tail = le32_to_cpu(*pipe->tail); + + if (head < tail) + return pipe->native.length - tail + head; + else + return head - tail; +} + +static void glink_smem_rx_peak(struct qcom_glink_pipe *np, + void *data, unsigned int offset, size_t count) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + size_t len; + u32 tail; + + tail = le32_to_cpu(*pipe->tail); + tail += offset; + if (tail >= pipe->native.length) + tail -= pipe->native.length; + + len = min_t(size_t, count, pipe->native.length - tail); + if (len) { + __ioread32_copy(data, pipe->fifo + tail, + len / sizeof(u32)); + } + + if (len != count) { + __ioread32_copy(data + len, pipe->fifo, + (count - len) / sizeof(u32)); + } +} + +static void glink_smem_rx_advance(struct qcom_glink_pipe *np, + size_t count) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + u32 tail; + + tail = le32_to_cpu(*pipe->tail); + + tail += count; + if (tail > pipe->native.length) + tail -= pipe->native.length; + + *pipe->tail = cpu_to_le32(tail); +} + +static size_t glink_smem_tx_avail(struct qcom_glink_pipe *np) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(np); + u32 head; + u32 tail; + u32 avail; + + head = le32_to_cpu(*pipe->head); + tail = le32_to_cpu(*pipe->tail); + + if (tail <= head) + avail = pipe->native.length - head + tail; + else + avail = tail - head; + + if (avail < (FIFO_FULL_RESERVE + TX_BLOCKED_CMD_RESERVE)) + avail = 0; + else + avail -= FIFO_FULL_RESERVE + TX_BLOCKED_CMD_RESERVE; + + return avail; +} + +static unsigned int glink_smem_tx_write_one(struct glink_smem_pipe *pipe, + unsigned int head, + const void *data, size_t count) +{ + size_t len; + + len = min_t(size_t, count, pipe->native.length - head); + if (len) + memcpy(pipe->fifo + head, data, len); + + if (len != count) + memcpy(pipe->fifo, data + len, count - len); + + head += count; + if (head >= pipe->native.length) + head -= pipe->native.length; + + return head; +} + +static void glink_smem_tx_write(struct qcom_glink_pipe *glink_pipe, + const void *hdr, size_t hlen, + const void *data, size_t dlen) +{ + struct glink_smem_pipe *pipe = to_smem_pipe(glink_pipe); + unsigned int head; + + head = le32_to_cpu(*pipe->head); + + head = glink_smem_tx_write_one(pipe, head, hdr, hlen); + head = glink_smem_tx_write_one(pipe, head, data, dlen); + + /* Ensure head is always aligned to 8 bytes */ + head = ALIGN(head, 8); + if (head >= pipe->native.length) + head -= pipe->native.length; + + *pipe->head = cpu_to_le32(head); +} + +static void qcom_glink_smem_release(struct device *dev) +{ + kfree(dev); +} + +struct qcom_glink *qcom_glink_smem_register(struct device *parent, + struct device_node *node) +{ + struct glink_smem_pipe *rx_pipe; + struct glink_smem_pipe *tx_pipe; + struct qcom_glink *glink; + struct device *dev; + u32 remote_pid; + __le32 *descs; + size_t size; + int ret; + + dev = kzalloc(sizeof(*dev), GFP_KERNEL); + if (!dev) + return ERR_PTR(-ENOMEM); + + dev->parent = parent; + dev->of_node = node; + dev->release = qcom_glink_smem_release; + dev_set_name(dev, "%s:%s", node->parent->name, node->name); + ret = device_register(dev); + if (ret) { + pr_err("failed to register glink edge\n"); + return ERR_PTR(ret); + } + + ret = of_property_read_u32(dev->of_node, "qcom,remote-pid", + &remote_pid); + if (ret) { + dev_err(dev, "failed to parse qcom,remote-pid\n"); + goto err_put_dev; + } + + rx_pipe = devm_kzalloc(dev, sizeof(*rx_pipe), GFP_KERNEL); + tx_pipe = devm_kzalloc(dev, sizeof(*tx_pipe), GFP_KERNEL); + if (!rx_pipe || !tx_pipe) { + ret = -ENOMEM; + goto err_put_dev; + } + + ret = qcom_smem_alloc(remote_pid, + SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR, 32); + if (ret && ret != -EEXIST) { + dev_err(dev, "failed to allocate glink descriptors\n"); + goto err_put_dev; + } + + descs = qcom_smem_get(remote_pid, + SMEM_GLINK_NATIVE_XPRT_DESCRIPTOR, &size); + if (IS_ERR(descs)) { + dev_err(dev, "failed to acquire xprt descriptor\n"); + ret = PTR_ERR(descs); + goto err_put_dev; + } + + if (size != 32) { + dev_err(dev, "glink descriptor of invalid size\n"); + ret = -EINVAL; + goto err_put_dev; + } + + tx_pipe->tail = &descs[0]; + tx_pipe->head = &descs[1]; + rx_pipe->tail = &descs[2]; + rx_pipe->head = &descs[3]; + + ret = qcom_smem_alloc(remote_pid, SMEM_GLINK_NATIVE_XPRT_FIFO_0, + SZ_16K); + if (ret && ret != -EEXIST) { + dev_err(dev, "failed to allocate TX fifo\n"); + goto err_put_dev; + } + + tx_pipe->fifo = qcom_smem_get(remote_pid, SMEM_GLINK_NATIVE_XPRT_FIFO_0, + &tx_pipe->native.length); + if (IS_ERR(tx_pipe->fifo)) { + dev_err(dev, "failed to acquire TX fifo\n"); + ret = PTR_ERR(tx_pipe->fifo); + goto err_put_dev; + } + + rx_pipe->native.avail = glink_smem_rx_avail; + rx_pipe->native.peak = glink_smem_rx_peak; + rx_pipe->native.advance = glink_smem_rx_advance; + rx_pipe->remote_pid = remote_pid; + + tx_pipe->native.avail = glink_smem_tx_avail; + tx_pipe->native.write = glink_smem_tx_write; + tx_pipe->remote_pid = remote_pid; + + *rx_pipe->tail = 0; + *tx_pipe->head = 0; + + glink = qcom_glink_native_probe(dev, + GLINK_FEATURE_INTENT_REUSE, + &rx_pipe->native, &tx_pipe->native, + false); + if (IS_ERR(glink)) { + ret = PTR_ERR(glink); + goto err_put_dev; + } + + return glink; + +err_put_dev: + put_device(dev); + + return ERR_PTR(ret); +} +EXPORT_SYMBOL_GPL(qcom_glink_smem_register); + +void qcom_glink_smem_unregister(struct qcom_glink *glink) +{ + qcom_glink_native_remove(glink); + qcom_glink_native_unregister(glink); +} +EXPORT_SYMBOL_GPL(qcom_glink_smem_unregister); + +MODULE_AUTHOR("Bjorn Andersson <bjorn.andersson@linaro.org>"); +MODULE_DESCRIPTION("Qualcomm GLINK SMEM driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/rpmsg/qcom_smd.c b/drivers/rpmsg/qcom_smd.c index a0a39a8821a3..b01774e9fac0 100644 --- a/drivers/rpmsg/qcom_smd.c +++ b/drivers/rpmsg/qcom_smd.c @@ -1368,6 +1368,7 @@ struct qcom_smd_edge *qcom_smd_register_edge(struct device *parent, edge->dev.parent = parent; edge->dev.release = qcom_smd_edge_release; + edge->dev.of_node = node; edge->dev.groups = qcom_smd_edge_groups; dev_set_name(&edge->dev, "%s:%s", dev_name(parent), node->name); ret = device_register(&edge->dev); diff --git a/drivers/rpmsg/virtio_rpmsg_bus.c b/drivers/rpmsg/virtio_rpmsg_bus.c index eee2a9f77d37..82b83002fcba 100644 --- a/drivers/rpmsg/virtio_rpmsg_bus.c +++ b/drivers/rpmsg/virtio_rpmsg_bus.c @@ -45,6 +45,7 @@ * @rbufs: kernel address of rx buffers * @sbufs: kernel address of tx buffers * @num_bufs: total number of buffers for rx and tx + * @buf_size: size of one rx or tx buffer * @last_sbuf: index of last tx buffer used * @bufs_dma: dma base addr of the buffers * @tx_lock: protects svq, sbufs and sleepers, to allow concurrent senders. @@ -65,6 +66,7 @@ struct virtproc_info { struct virtqueue *rvq, *svq; void *rbufs, *sbufs; unsigned int num_bufs; + unsigned int buf_size; int last_sbuf; dma_addr_t bufs_dma; struct mutex tx_lock; @@ -158,7 +160,7 @@ struct virtio_rpmsg_channel { * processor. */ #define MAX_RPMSG_NUM_BUFS (512) -#define RPMSG_BUF_SIZE (512) +#define MAX_RPMSG_BUF_SIZE (512) /* * Local addresses are dynamically allocated on-demand. @@ -193,6 +195,28 @@ static const struct rpmsg_endpoint_ops virtio_endpoint_ops = { }; /** + * rpmsg_sg_init - initialize scatterlist according to cpu address location + * @sg: scatterlist to fill + * @cpu_addr: virtual address of the buffer + * @len: buffer length + * + * An internal function filling scatterlist according to virtual address + * location (in vmalloc or in kernel). + */ +static void +rpmsg_sg_init(struct scatterlist *sg, void *cpu_addr, unsigned int len) +{ + if (is_vmalloc_addr(cpu_addr)) { + sg_init_table(sg, 1); + sg_set_page(sg, vmalloc_to_page(cpu_addr), len, + offset_in_page(cpu_addr)); + } else { + WARN_ON(!virt_addr_valid(cpu_addr)); + sg_init_one(sg, cpu_addr, len); + } +} + +/** * __ept_release() - deallocate an rpmsg endpoint * @kref: the ept's reference count * @@ -435,7 +459,7 @@ static void *get_a_tx_buf(struct virtproc_info *vrp) * (half of our buffers are used for sending messages) */ if (vrp->last_sbuf < vrp->num_bufs / 2) - ret = vrp->sbufs + RPMSG_BUF_SIZE * vrp->last_sbuf++; + ret = vrp->sbufs + vrp->buf_size * vrp->last_sbuf++; /* or recycle a used one */ else ret = virtqueue_get_buf(vrp->svq, &len); @@ -561,7 +585,7 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev, * messaging), or to improve the buffer allocator, to support * variable-length buffer sizes. */ - if (len > RPMSG_BUF_SIZE - sizeof(struct rpmsg_hdr)) { + if (len > vrp->buf_size - sizeof(struct rpmsg_hdr)) { dev_err(dev, "message is too big (%d)\n", len); return -EMSGSIZE; } @@ -610,7 +634,7 @@ static int rpmsg_send_offchannel_raw(struct rpmsg_device *rpdev, msg, sizeof(*msg) + msg->len, true); #endif - sg_init_one(&sg, msg, sizeof(*msg) + len); + rpmsg_sg_init(&sg, msg, sizeof(*msg) + len); mutex_lock(&vrp->tx_lock); @@ -632,7 +656,6 @@ out: mutex_unlock(&vrp->tx_lock); return err; } -EXPORT_SYMBOL(rpmsg_send_offchannel_raw); static int virtio_rpmsg_send(struct rpmsg_endpoint *ept, void *data, int len) { @@ -702,7 +725,7 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev, * We currently use fixed-sized buffers, so trivially sanitize * the reported payload length. */ - if (len > RPMSG_BUF_SIZE || + if (len > vrp->buf_size || msg->len > (len - sizeof(struct rpmsg_hdr))) { dev_warn(dev, "inbound msg too big: (%d, %d)\n", len, msg->len); return -EINVAL; @@ -735,7 +758,7 @@ static int rpmsg_recv_single(struct virtproc_info *vrp, struct device *dev, dev_warn(dev, "msg received with no recipient\n"); /* publish the real size of the buffer */ - sg_init_one(&sg, msg, RPMSG_BUF_SIZE); + rpmsg_sg_init(&sg, msg, vrp->buf_size); /* add the buffer back to the remote processor's virtqueue */ err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, msg, GFP_KERNEL); @@ -892,7 +915,9 @@ static int rpmsg_probe(struct virtio_device *vdev) else vrp->num_bufs = MAX_RPMSG_NUM_BUFS; - total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE; + vrp->buf_size = MAX_RPMSG_BUF_SIZE; + + total_buf_space = vrp->num_bufs * vrp->buf_size; /* allocate coherent memory for the buffers */ bufs_va = dma_alloc_coherent(vdev->dev.parent->parent, @@ -915,9 +940,9 @@ static int rpmsg_probe(struct virtio_device *vdev) /* set up the receive buffers */ for (i = 0; i < vrp->num_bufs / 2; i++) { struct scatterlist sg; - void *cpu_addr = vrp->rbufs + i * RPMSG_BUF_SIZE; + void *cpu_addr = vrp->rbufs + i * vrp->buf_size; - sg_init_one(&sg, cpu_addr, RPMSG_BUF_SIZE); + rpmsg_sg_init(&sg, cpu_addr, vrp->buf_size); err = virtqueue_add_inbuf(vrp->rvq, &sg, 1, cpu_addr, GFP_KERNEL); @@ -982,7 +1007,7 @@ static int rpmsg_remove_device(struct device *dev, void *data) static void rpmsg_remove(struct virtio_device *vdev) { struct virtproc_info *vrp = vdev->priv; - size_t total_buf_space = vrp->num_bufs * RPMSG_BUF_SIZE; + size_t total_buf_space = vrp->num_bufs * vrp->buf_size; int ret; vdev->config->reset(vdev); diff --git a/include/linux/rpmsg/qcom_glink.h b/include/linux/rpmsg/qcom_glink.h new file mode 100644 index 000000000000..a622f029836e --- /dev/null +++ b/include/linux/rpmsg/qcom_glink.h @@ -0,0 +1,27 @@ +#ifndef _LINUX_RPMSG_QCOM_GLINK_H +#define _LINUX_RPMSG_QCOM_GLINK_H + +#include <linux/device.h> + +struct qcom_glink; + +#if IS_ENABLED(CONFIG_RPMSG_QCOM_GLINK_SMEM) + +struct qcom_glink *qcom_glink_smem_register(struct device *parent, + struct device_node *node); +void qcom_glink_smem_unregister(struct qcom_glink *glink); + +#else + +static inline struct qcom_glink * +qcom_glink_smem_register(struct device *parent, + struct device_node *node) +{ + return NULL; +} + +static inline void qcom_glink_smem_unregister(struct qcom_glink *glink) {} + +#endif + +#endif |