From 90c311b0eeead647b708a723dbdde1eda3dcad05 Mon Sep 17 00:00:00 2001 From: Vineeth Remanan Pillai Date: Thu, 19 Jan 2017 08:35:39 -0800 Subject: xen-netfront: Fix Rx stall during network stress and OOM During an OOM scenario, request slots could not be created as skb allocation fails. So the netback cannot pass in packets and netfront wrongly assumes that there is no more work to be done and it disables polling. This causes Rx to stall. The issue is with the retry logic which schedules the timer if the created slots are less than NET_RX_SLOTS_MIN. The count of new request slots to be pushed are calculated as a difference between new req_prod and rsp_cons which could be more than the actual slots, if there are unconsumed responses. The fix is to calculate the count of newly created slots as the difference between new req_prod and old req_prod. Signed-off-by: Vineeth Remanan Pillai Reviewed-by: Juergen Gross Signed-off-by: David S. Miller --- drivers/net/xen-netfront.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'drivers/net/xen-netfront.c') diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index a479cd99911d..8315fe73ecd0 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -321,7 +321,7 @@ static void xennet_alloc_rx_buffers(struct netfront_queue *queue) queue->rx.req_prod_pvt = req_prod; /* Not enough requests? Try again later. */ - if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN) { + if (req_prod - queue->rx.sring->req_prod < NET_RX_SLOTS_MIN) { mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); return; } -- cgit v1.2.3 From 538d92912d3190a1dd809233a0d57277459f37b2 Mon Sep 17 00:00:00 2001 From: Vineeth Remanan Pillai Date: Tue, 7 Feb 2017 18:59:01 +0000 Subject: xen-netfront: Rework the fix for Rx stall during OOM and network stress The commit 90c311b0eeea ("xen-netfront: Fix Rx stall during network stress and OOM") caused the refill timer to be triggerred almost on all invocations of xennet_alloc_rx_buffers for certain workloads. This reworks the fix by reverting to the old behaviour and taking into consideration the skb allocation failure. Refill timer is now triggered on insufficient requests or skb allocation failure. Signed-off-by: Vineeth Remanan Pillai Fixes: 90c311b0eeea (xen-netfront: Fix Rx stall during network stress and OOM) Reported-by: Boris Ostrovsky Reviewed-by: Boris Ostrovsky Signed-off-by: David S. Miller --- drivers/net/xen-netfront.c | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'drivers/net/xen-netfront.c') diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 8315fe73ecd0..9dba69731f30 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -281,6 +281,7 @@ static void xennet_alloc_rx_buffers(struct netfront_queue *queue) { RING_IDX req_prod = queue->rx.req_prod_pvt; int notify; + int err = 0; if (unlikely(!netif_carrier_ok(queue->info->netdev))) return; @@ -295,8 +296,10 @@ static void xennet_alloc_rx_buffers(struct netfront_queue *queue) struct xen_netif_rx_request *req; skb = xennet_alloc_one_rx_buffer(queue); - if (!skb) + if (!skb) { + err = -ENOMEM; break; + } id = xennet_rxidx(req_prod); @@ -320,8 +323,13 @@ static void xennet_alloc_rx_buffers(struct netfront_queue *queue) queue->rx.req_prod_pvt = req_prod; - /* Not enough requests? Try again later. */ - if (req_prod - queue->rx.sring->req_prod < NET_RX_SLOTS_MIN) { + /* Try again later if there are not enough requests or skb allocation + * failed. + * Enough requests is quantified as the sum of newly created slots and + * the unconsumed slots at the backend. + */ + if (req_prod - queue->rx.rsp_cons < NET_RX_SLOTS_MIN || + unlikely(err)) { mod_timer(&queue->rx_refill_timer, jiffies + (HZ/10)); return; } -- cgit v1.2.3 From e2e004acc7cbe3c531e752a270a74e95cde3ea48 Mon Sep 17 00:00:00 2001 From: Ross Lagerwall Date: Wed, 8 Feb 2017 10:57:37 +0000 Subject: xen-netfront: Improve error handling during initialization This fixes a crash when running out of grant refs when creating many queues across many netdevs. * If creating queues fails (i.e. there are no grant refs available), call xenbus_dev_fatal() to ensure that the xenbus device is set to the closed state. * If no queues are created, don't call xennet_disconnect_backend as netdev->real_num_tx_queues will not have been set correctly. * If setup_netfront() fails, ensure that all the queues created are cleaned up, not just those that have been set up. * If any queues were set up and an error occurs, call xennet_destroy_queues() to clean up the napi context. * If any fatal error occurs, unregister and destroy the netdev to avoid leaving around a half setup network device. Signed-off-by: Ross Lagerwall Reviewed-by: Boris Ostrovsky Signed-off-by: David S. Miller --- drivers/net/xen-netfront.c | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) (limited to 'drivers/net/xen-netfront.c') diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index 9dba69731f30..d3812581c6c0 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1830,27 +1830,19 @@ static int talk_to_netback(struct xenbus_device *dev, xennet_destroy_queues(info); err = xennet_create_queues(info, &num_queues); - if (err < 0) - goto destroy_ring; + if (err < 0) { + xenbus_dev_fatal(dev, err, "creating queues"); + kfree(info->queues); + info->queues = NULL; + goto out; + } /* Create shared ring, alloc event channel -- for each queue */ for (i = 0; i < num_queues; ++i) { queue = &info->queues[i]; err = setup_netfront(dev, queue, feature_split_evtchn); - if (err) { - /* setup_netfront() will tidy up the current - * queue on error, but we need to clean up - * those already allocated. - */ - if (i > 0) { - rtnl_lock(); - netif_set_real_num_tx_queues(info->netdev, i); - rtnl_unlock(); - goto destroy_ring; - } else { - goto out; - } - } + if (err) + goto destroy_ring; } again: @@ -1940,9 +1932,10 @@ abort_transaction_no_dev_fatal: xenbus_transaction_end(xbt, 1); destroy_ring: xennet_disconnect_backend(info); - kfree(info->queues); - info->queues = NULL; + xennet_destroy_queues(info); out: + unregister_netdev(info->netdev); + xennet_free_netdev(info->netdev); return err; } -- cgit v1.2.3 From 74470954857c264168d2b5a113904cf0cfd27d18 Mon Sep 17 00:00:00 2001 From: Boris Ostrovsky Date: Mon, 30 Jan 2017 12:45:46 -0500 Subject: xen-netfront: Delete rx_refill_timer in xennet_disconnect_backend() rx_refill_timer should be deleted as soon as we disconnect from the backend since otherwise it is possible for the timer to go off before we get to xennet_destroy_queues(). If this happens we may dereference queue->rx.sring which is set to NULL in xennet_disconnect_backend(). Signed-off-by: Boris Ostrovsky CC: stable@vger.kernel.org Reviewed-by: Juergen Gross Signed-off-by: David S. Miller --- drivers/net/xen-netfront.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'drivers/net/xen-netfront.c') diff --git a/drivers/net/xen-netfront.c b/drivers/net/xen-netfront.c index d3812581c6c0..1e4125a98291 100644 --- a/drivers/net/xen-netfront.c +++ b/drivers/net/xen-netfront.c @@ -1387,6 +1387,8 @@ static void xennet_disconnect_backend(struct netfront_info *info) for (i = 0; i < num_queues && info->queues; ++i) { struct netfront_queue *queue = &info->queues[i]; + del_timer_sync(&queue->rx_refill_timer); + if (queue->tx_irq && (queue->tx_irq == queue->rx_irq)) unbind_from_irqhandler(queue->tx_irq, queue); if (queue->tx_irq && (queue->tx_irq != queue->rx_irq)) { @@ -1741,7 +1743,6 @@ static void xennet_destroy_queues(struct netfront_info *info) if (netif_running(info->netdev)) napi_disable(&queue->napi); - del_timer_sync(&queue->rx_refill_timer); netif_napi_del(&queue->napi); } -- cgit v1.2.3