diff options
Diffstat (limited to 'drivers/net/ethernet/amd/xgbe')
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe-common.h | 63 | ||||
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe-desc.c | 208 | ||||
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe-dev.c | 446 | ||||
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe-drv.c | 535 | ||||
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c | 89 | ||||
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe-main.c | 86 | ||||
| -rw-r--r-- | drivers/net/ethernet/amd/xgbe/xgbe.h | 112 |
7 files changed, 1206 insertions, 333 deletions
diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-common.h b/drivers/net/ethernet/amd/xgbe/xgbe-common.h index caade30820d5..75b08c63d39f 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-common.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe-common.h @@ -125,9 +125,6 @@ #define DMA_AXIAWCR 0x3018 #define DMA_DSR0 0x3020 #define DMA_DSR1 0x3024 -#define DMA_DSR2 0x3028 -#define DMA_DSR3 0x302c -#define DMA_DSR4 0x3030 /* DMA register entry bit positions and sizes */ #define DMA_AXIARCR_DRC_INDEX 0 @@ -158,10 +155,6 @@ #define DMA_AXIAWCR_TDC_WIDTH 4 #define DMA_AXIAWCR_TDD_INDEX 28 #define DMA_AXIAWCR_TDD_WIDTH 2 -#define DMA_DSR0_RPS_INDEX 8 -#define DMA_DSR0_RPS_WIDTH 4 -#define DMA_DSR0_TPS_INDEX 12 -#define DMA_DSR0_TPS_WIDTH 4 #define DMA_ISR_MACIS_INDEX 17 #define DMA_ISR_MACIS_WIDTH 1 #define DMA_ISR_MTLIS_INDEX 16 @@ -175,6 +168,20 @@ #define DMA_SBMR_UNDEF_INDEX 0 #define DMA_SBMR_UNDEF_WIDTH 1 +/* DMA register values */ +#define DMA_DSR_RPS_WIDTH 4 +#define DMA_DSR_TPS_WIDTH 4 +#define DMA_DSR_Q_WIDTH (DMA_DSR_RPS_WIDTH + DMA_DSR_TPS_WIDTH) +#define DMA_DSR0_RPS_START 8 +#define DMA_DSR0_TPS_START 12 +#define DMA_DSRX_FIRST_QUEUE 3 +#define DMA_DSRX_INC 4 +#define DMA_DSRX_QPR 4 +#define DMA_DSRX_RPS_START 0 +#define DMA_DSRX_TPS_START 4 +#define DMA_TPS_STOPPED 0x00 +#define DMA_TPS_SUSPENDED 0x06 + /* DMA channel register offsets * Multiple channels can be active. The first channel has registers * that begin at 0x3100. Each subsequent channel has registers that @@ -207,6 +214,8 @@ /* DMA channel register entry bit positions and sizes */ #define DMA_CH_CR_PBLX8_INDEX 16 #define DMA_CH_CR_PBLX8_WIDTH 1 +#define DMA_CH_CR_SPH_INDEX 24 +#define DMA_CH_CR_SPH_WIDTH 1 #define DMA_CH_IER_AIE_INDEX 15 #define DMA_CH_IER_AIE_WIDTH 1 #define DMA_CH_IER_FBEE_INDEX 12 @@ -306,6 +315,9 @@ #define MAC_MACA0LR 0x0304 #define MAC_MACA1HR 0x0308 #define MAC_MACA1LR 0x030c +#define MAC_RSSCR 0x0c80 +#define MAC_RSSAR 0x0c88 +#define MAC_RSSDR 0x0c8c #define MAC_TSCR 0x0d00 #define MAC_SSIR 0x0d04 #define MAC_STSR 0x0d08 @@ -429,6 +441,8 @@ #define MAC_RCR_CST_WIDTH 1 #define MAC_RCR_DCRCC_INDEX 3 #define MAC_RCR_DCRCC_WIDTH 1 +#define MAC_RCR_HDSMS_INDEX 12 +#define MAC_RCR_HDSMS_WIDTH 3 #define MAC_RCR_IPC_INDEX 9 #define MAC_RCR_IPC_WIDTH 1 #define MAC_RCR_JE_INDEX 8 @@ -445,6 +459,24 @@ #define MAC_RFCR_UP_WIDTH 1 #define MAC_RQC0R_RXQ0EN_INDEX 0 #define MAC_RQC0R_RXQ0EN_WIDTH 2 +#define MAC_RSSAR_ADDRT_INDEX 2 +#define MAC_RSSAR_ADDRT_WIDTH 1 +#define MAC_RSSAR_CT_INDEX 1 +#define MAC_RSSAR_CT_WIDTH 1 +#define MAC_RSSAR_OB_INDEX 0 +#define MAC_RSSAR_OB_WIDTH 1 +#define MAC_RSSAR_RSSIA_INDEX 8 +#define MAC_RSSAR_RSSIA_WIDTH 8 +#define MAC_RSSCR_IP2TE_INDEX 1 +#define MAC_RSSCR_IP2TE_WIDTH 1 +#define MAC_RSSCR_RSSE_INDEX 0 +#define MAC_RSSCR_RSSE_WIDTH 1 +#define MAC_RSSCR_TCP4TE_INDEX 2 +#define MAC_RSSCR_TCP4TE_WIDTH 1 +#define MAC_RSSCR_UDP4TE_INDEX 3 +#define MAC_RSSCR_UDP4TE_WIDTH 1 +#define MAC_RSSDR_DMCH_INDEX 0 +#define MAC_RSSDR_DMCH_WIDTH 4 #define MAC_SSIR_SNSINC_INDEX 8 #define MAC_SSIR_SNSINC_WIDTH 8 #define MAC_SSIR_SSINC_INDEX 16 @@ -844,9 +876,13 @@ #define RX_PACKET_ATTRIBUTES_CONTEXT_WIDTH 1 #define RX_PACKET_ATTRIBUTES_RX_TSTAMP_INDEX 5 #define RX_PACKET_ATTRIBUTES_RX_TSTAMP_WIDTH 1 +#define RX_PACKET_ATTRIBUTES_RSS_HASH_INDEX 6 +#define RX_PACKET_ATTRIBUTES_RSS_HASH_WIDTH 1 #define RX_NORMAL_DESC0_OVT_INDEX 0 #define RX_NORMAL_DESC0_OVT_WIDTH 16 +#define RX_NORMAL_DESC2_HL_INDEX 0 +#define RX_NORMAL_DESC2_HL_WIDTH 10 #define RX_NORMAL_DESC3_CDA_INDEX 27 #define RX_NORMAL_DESC3_CDA_WIDTH 1 #define RX_NORMAL_DESC3_CTXT_INDEX 30 @@ -855,14 +891,27 @@ #define RX_NORMAL_DESC3_ES_WIDTH 1 #define RX_NORMAL_DESC3_ETLT_INDEX 16 #define RX_NORMAL_DESC3_ETLT_WIDTH 4 +#define RX_NORMAL_DESC3_FD_INDEX 29 +#define RX_NORMAL_DESC3_FD_WIDTH 1 #define RX_NORMAL_DESC3_INTE_INDEX 30 #define RX_NORMAL_DESC3_INTE_WIDTH 1 +#define RX_NORMAL_DESC3_L34T_INDEX 20 +#define RX_NORMAL_DESC3_L34T_WIDTH 4 #define RX_NORMAL_DESC3_LD_INDEX 28 #define RX_NORMAL_DESC3_LD_WIDTH 1 #define RX_NORMAL_DESC3_OWN_INDEX 31 #define RX_NORMAL_DESC3_OWN_WIDTH 1 #define RX_NORMAL_DESC3_PL_INDEX 0 #define RX_NORMAL_DESC3_PL_WIDTH 14 +#define RX_NORMAL_DESC3_RSV_INDEX 26 +#define RX_NORMAL_DESC3_RSV_WIDTH 1 + +#define RX_DESC3_L34T_IPV4_TCP 1 +#define RX_DESC3_L34T_IPV4_UDP 2 +#define RX_DESC3_L34T_IPV4_ICMP 3 +#define RX_DESC3_L34T_IPV6_TCP 9 +#define RX_DESC3_L34T_IPV6_UDP 10 +#define RX_DESC3_L34T_IPV6_ICMP 11 #define RX_CONTEXT_DESC3_TSA_INDEX 4 #define RX_CONTEXT_DESC3_TSA_WIDTH 1 diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c index 6fc5da01437d..a50891f52197 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-desc.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-desc.c @@ -117,7 +117,7 @@ #include "xgbe.h" #include "xgbe-common.h" -static void xgbe_unmap_skb(struct xgbe_prv_data *, struct xgbe_ring_data *); +static void xgbe_unmap_rdata(struct xgbe_prv_data *, struct xgbe_ring_data *); static void xgbe_free_ring(struct xgbe_prv_data *pdata, struct xgbe_ring *ring) @@ -131,13 +131,35 @@ static void xgbe_free_ring(struct xgbe_prv_data *pdata, if (ring->rdata) { for (i = 0; i < ring->rdesc_count; i++) { rdata = XGBE_GET_DESC_DATA(ring, i); - xgbe_unmap_skb(pdata, rdata); + xgbe_unmap_rdata(pdata, rdata); } kfree(ring->rdata); ring->rdata = NULL; } + if (ring->rx_hdr_pa.pages) { + dma_unmap_page(pdata->dev, ring->rx_hdr_pa.pages_dma, + ring->rx_hdr_pa.pages_len, DMA_FROM_DEVICE); + put_page(ring->rx_hdr_pa.pages); + + ring->rx_hdr_pa.pages = NULL; + ring->rx_hdr_pa.pages_len = 0; + ring->rx_hdr_pa.pages_offset = 0; + ring->rx_hdr_pa.pages_dma = 0; + } + + if (ring->rx_buf_pa.pages) { + dma_unmap_page(pdata->dev, ring->rx_buf_pa.pages_dma, + ring->rx_buf_pa.pages_len, DMA_FROM_DEVICE); + put_page(ring->rx_buf_pa.pages); + + ring->rx_buf_pa.pages = NULL; + ring->rx_buf_pa.pages_len = 0; + ring->rx_buf_pa.pages_offset = 0; + ring->rx_buf_pa.pages_dma = 0; + } + if (ring->rdesc) { dma_free_coherent(pdata->dev, (sizeof(struct xgbe_ring_desc) * @@ -233,6 +255,96 @@ err_ring: return ret; } +static int xgbe_alloc_pages(struct xgbe_prv_data *pdata, + struct xgbe_page_alloc *pa, gfp_t gfp, int order) +{ + struct page *pages = NULL; + dma_addr_t pages_dma; + int ret; + + /* Try to obtain pages, decreasing order if necessary */ + gfp |= __GFP_COLD | __GFP_COMP; + while (order >= 0) { + pages = alloc_pages(gfp, order); + if (pages) + break; + + order--; + } + if (!pages) + return -ENOMEM; + + /* Map the pages */ + pages_dma = dma_map_page(pdata->dev, pages, 0, + PAGE_SIZE << order, DMA_FROM_DEVICE); + ret = dma_mapping_error(pdata->dev, pages_dma); + if (ret) { + put_page(pages); + return ret; + } + + pa->pages = pages; + pa->pages_len = PAGE_SIZE << order; + pa->pages_offset = 0; + pa->pages_dma = pages_dma; + + return 0; +} + +static void xgbe_set_buffer_data(struct xgbe_buffer_data *bd, + struct xgbe_page_alloc *pa, + unsigned int len) +{ + get_page(pa->pages); + bd->pa = *pa; + + bd->dma = pa->pages_dma + pa->pages_offset; + bd->dma_len = len; + + pa->pages_offset += len; + if ((pa->pages_offset + len) > pa->pages_len) { + /* This data descriptor is responsible for unmapping page(s) */ + bd->pa_unmap = *pa; + + /* Get a new allocation next time */ + pa->pages = NULL; + pa->pages_len = 0; + pa->pages_offset = 0; + pa->pages_dma = 0; + } +} + +static int xgbe_map_rx_buffer(struct xgbe_prv_data *pdata, + struct xgbe_ring *ring, + struct xgbe_ring_data *rdata) +{ + int order, ret; + + if (!ring->rx_hdr_pa.pages) { + ret = xgbe_alloc_pages(pdata, &ring->rx_hdr_pa, GFP_ATOMIC, 0); + if (ret) + return ret; + } + + if (!ring->rx_buf_pa.pages) { + order = max_t(int, PAGE_ALLOC_COSTLY_ORDER - 1, 0); + ret = xgbe_alloc_pages(pdata, &ring->rx_buf_pa, GFP_ATOMIC, + order); + if (ret) + return ret; + } + + /* Set up the header page info */ + xgbe_set_buffer_data(&rdata->rx.hdr, &ring->rx_hdr_pa, + XGBE_SKB_ALLOC_SIZE); + + /* Set up the buffer page info */ + xgbe_set_buffer_data(&rdata->rx.buf, &ring->rx_buf_pa, + pdata->rx_buf_size); + + return 0; +} + static void xgbe_wrapper_tx_descriptor_init(struct xgbe_prv_data *pdata) { struct xgbe_hw_if *hw_if = &pdata->hw_if; @@ -266,7 +378,7 @@ static void xgbe_wrapper_tx_descriptor_init(struct xgbe_prv_data *pdata) ring->cur = 0; ring->dirty = 0; - ring->tx.queue_stopped = 0; + memset(&ring->tx, 0, sizeof(ring->tx)); hw_if->tx_desc_init(channel); } @@ -281,8 +393,7 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) struct xgbe_ring *ring; struct xgbe_ring_desc *rdesc; struct xgbe_ring_data *rdata; - dma_addr_t rdesc_dma, skb_dma; - struct sk_buff *skb = NULL; + dma_addr_t rdesc_dma; unsigned int i, j; DBGPR("-->xgbe_wrapper_rx_descriptor_init\n"); @@ -302,22 +413,8 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) rdata->rdesc = rdesc; rdata->rdesc_dma = rdesc_dma; - /* Allocate skb & assign to each rdesc */ - skb = dev_alloc_skb(pdata->rx_buf_size); - if (skb == NULL) - break; - skb_dma = dma_map_single(pdata->dev, skb->data, - pdata->rx_buf_size, - DMA_FROM_DEVICE); - if (dma_mapping_error(pdata->dev, skb_dma)) { - netdev_alert(pdata->netdev, - "failed to do the dma map\n"); - dev_kfree_skb_any(skb); + if (xgbe_map_rx_buffer(pdata, ring, rdata)) break; - } - rdata->skb = skb; - rdata->skb_dma = skb_dma; - rdata->skb_dma_len = pdata->rx_buf_size; rdesc++; rdesc_dma += sizeof(struct xgbe_ring_desc); @@ -325,8 +422,7 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) ring->cur = 0; ring->dirty = 0; - ring->rx.realloc_index = 0; - ring->rx.realloc_threshold = 0; + memset(&ring->rx, 0, sizeof(ring->rx)); hw_if->rx_desc_init(channel); } @@ -334,8 +430,8 @@ static void xgbe_wrapper_rx_descriptor_init(struct xgbe_prv_data *pdata) DBGPR("<--xgbe_wrapper_rx_descriptor_init\n"); } -static void xgbe_unmap_skb(struct xgbe_prv_data *pdata, - struct xgbe_ring_data *rdata) +static void xgbe_unmap_rdata(struct xgbe_prv_data *pdata, + struct xgbe_ring_data *rdata) { if (rdata->skb_dma) { if (rdata->mapped_as_page) { @@ -354,9 +450,29 @@ static void xgbe_unmap_skb(struct xgbe_prv_data *pdata, rdata->skb = NULL; } - rdata->tso_header = 0; - rdata->len = 0; - rdata->interrupt = 0; + if (rdata->rx.hdr.pa.pages) + put_page(rdata->rx.hdr.pa.pages); + + if (rdata->rx.hdr.pa_unmap.pages) { + dma_unmap_page(pdata->dev, rdata->rx.hdr.pa_unmap.pages_dma, + rdata->rx.hdr.pa_unmap.pages_len, + DMA_FROM_DEVICE); + put_page(rdata->rx.hdr.pa_unmap.pages); + } + + if (rdata->rx.buf.pa.pages) + put_page(rdata->rx.buf.pa.pages); + + if (rdata->rx.buf.pa_unmap.pages) { + dma_unmap_page(pdata->dev, rdata->rx.buf.pa_unmap.pages_dma, + rdata->rx.buf.pa_unmap.pages_len, + DMA_FROM_DEVICE); + put_page(rdata->rx.buf.pa_unmap.pages); + } + + memset(&rdata->tx, 0, sizeof(rdata->tx)); + memset(&rdata->rx, 0, sizeof(rdata->rx)); + rdata->mapped_as_page = 0; if (rdata->state_saved) { @@ -414,7 +530,6 @@ static int xgbe_map_tx_skb(struct xgbe_channel *channel, struct sk_buff *skb) } rdata->skb_dma = skb_dma; rdata->skb_dma_len = packet->header_len; - rdata->tso_header = 1; offset = packet->header_len; @@ -481,7 +596,11 @@ static int xgbe_map_tx_skb(struct xgbe_channel *channel, struct sk_buff *skb) } } - /* Save the skb address in the last entry */ + /* Save the skb address in the last entry. We always have some data + * that has been mapped so rdata is always advanced past the last + * piece of mapped data - use the entry pointed to by cur_index - 1. + */ + rdata = XGBE_GET_DESC_DATA(ring, cur_index - 1); rdata->skb = skb; /* Save the number of descriptor entries used */ @@ -494,7 +613,7 @@ static int xgbe_map_tx_skb(struct xgbe_channel *channel, struct sk_buff *skb) err_out: while (start_index < cur_index) { rdata = XGBE_GET_DESC_DATA(ring, start_index++); - xgbe_unmap_skb(pdata, rdata); + xgbe_unmap_rdata(pdata, rdata); } DBGPR("<--xgbe_map_tx_skb: count=0\n"); @@ -502,40 +621,25 @@ err_out: return 0; } -static void xgbe_realloc_skb(struct xgbe_channel *channel) +static void xgbe_realloc_rx_buffer(struct xgbe_channel *channel) { struct xgbe_prv_data *pdata = channel->pdata; struct xgbe_hw_if *hw_if = &pdata->hw_if; struct xgbe_ring *ring = channel->rx_ring; struct xgbe_ring_data *rdata; - struct sk_buff *skb = NULL; - dma_addr_t skb_dma; int i; - DBGPR("-->xgbe_realloc_skb: rx_ring->rx.realloc_index = %u\n", + DBGPR("-->xgbe_realloc_rx_buffer: rx_ring->rx.realloc_index = %u\n", ring->rx.realloc_index); for (i = 0; i < ring->dirty; i++) { rdata = XGBE_GET_DESC_DATA(ring, ring->rx.realloc_index); /* Reset rdata values */ - xgbe_unmap_skb(pdata, rdata); + xgbe_unmap_rdata(pdata, rdata); - /* Allocate skb & assign to each rdesc */ - skb = dev_alloc_skb(pdata->rx_buf_size); - if (skb == NULL) + if (xgbe_map_rx_buffer(pdata, ring, rdata)) break; - skb_dma = dma_map_single(pdata->dev, skb->data, - pdata->rx_buf_size, DMA_FROM_DEVICE); - if (dma_mapping_error(pdata->dev, skb_dma)) { - netdev_alert(pdata->netdev, - "failed to do the dma map\n"); - dev_kfree_skb_any(skb); - break; - } - rdata->skb = skb; - rdata->skb_dma = skb_dma; - rdata->skb_dma_len = pdata->rx_buf_size; hw_if->rx_desc_reset(rdata); @@ -543,7 +647,7 @@ static void xgbe_realloc_skb(struct xgbe_channel *channel) } ring->dirty = 0; - DBGPR("<--xgbe_realloc_skb\n"); + DBGPR("<--xgbe_realloc_rx_buffer\n"); } void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *desc_if) @@ -553,8 +657,8 @@ void xgbe_init_function_ptrs_desc(struct xgbe_desc_if *desc_if) desc_if->alloc_ring_resources = xgbe_alloc_ring_resources; desc_if->free_ring_resources = xgbe_free_ring_resources; desc_if->map_tx_skb = xgbe_map_tx_skb; - desc_if->realloc_skb = xgbe_realloc_skb; - desc_if->unmap_skb = xgbe_unmap_skb; + desc_if->realloc_rx_buffer = xgbe_realloc_rx_buffer; + desc_if->unmap_rdata = xgbe_unmap_rdata; desc_if->wrapper_tx_desc_init = xgbe_wrapper_tx_descriptor_init; desc_if->wrapper_rx_desc_init = xgbe_wrapper_rx_descriptor_init; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c index 9da3a03e8c07..53f5f66ec2ee 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-dev.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-dev.c @@ -335,6 +335,161 @@ static void xgbe_config_tso_mode(struct xgbe_prv_data *pdata) } } +static void xgbe_config_sph_mode(struct xgbe_prv_data *pdata) +{ + struct xgbe_channel *channel; + unsigned int i; + + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (!channel->rx_ring) + break; + + XGMAC_DMA_IOWRITE_BITS(channel, DMA_CH_CR, SPH, 1); + } + + XGMAC_IOWRITE_BITS(pdata, MAC_RCR, HDSMS, XGBE_SPH_HDSMS_SIZE); +} + +static int xgbe_write_rss_reg(struct xgbe_prv_data *pdata, unsigned int type, + unsigned int index, unsigned int val) +{ + unsigned int wait; + int ret = 0; + + mutex_lock(&pdata->rss_mutex); + + if (XGMAC_IOREAD_BITS(pdata, MAC_RSSAR, OB)) { + ret = -EBUSY; + goto unlock; + } + + XGMAC_IOWRITE(pdata, MAC_RSSDR, val); + + XGMAC_IOWRITE_BITS(pdata, MAC_RSSAR, RSSIA, index); + XGMAC_IOWRITE_BITS(pdata, MAC_RSSAR, ADDRT, type); + XGMAC_IOWRITE_BITS(pdata, MAC_RSSAR, CT, 0); + XGMAC_IOWRITE_BITS(pdata, MAC_RSSAR, OB, 1); + + wait = 1000; + while (wait--) { + if (!XGMAC_IOREAD_BITS(pdata, MAC_RSSAR, OB)) + goto unlock; + + usleep_range(1000, 1500); + } + + ret = -EBUSY; + +unlock: + mutex_unlock(&pdata->rss_mutex); + + return ret; +} + +static int xgbe_write_rss_hash_key(struct xgbe_prv_data *pdata) +{ + unsigned int key_regs = sizeof(pdata->rss_key) / sizeof(u32); + unsigned int *key = (unsigned int *)&pdata->rss_key; + int ret; + + while (key_regs--) { + ret = xgbe_write_rss_reg(pdata, XGBE_RSS_HASH_KEY_TYPE, + key_regs, *key++); + if (ret) + return ret; + } + + return 0; +} + +static int xgbe_write_rss_lookup_table(struct xgbe_prv_data *pdata) +{ + unsigned int i; + int ret; + + for (i = 0; i < ARRAY_SIZE(pdata->rss_table); i++) { + ret = xgbe_write_rss_reg(pdata, + XGBE_RSS_LOOKUP_TABLE_TYPE, i, + pdata->rss_table[i]); + if (ret) + return ret; + } + + return 0; +} + +static int xgbe_set_rss_hash_key(struct xgbe_prv_data *pdata, const u8 *key) +{ + memcpy(pdata->rss_key, key, sizeof(pdata->rss_key)); + + return xgbe_write_rss_hash_key(pdata); +} + +static int xgbe_set_rss_lookup_table(struct xgbe_prv_data *pdata, + const u32 *table) +{ + unsigned int i; + + for (i = 0; i < ARRAY_SIZE(pdata->rss_table); i++) + XGMAC_SET_BITS(pdata->rss_table[i], MAC_RSSDR, DMCH, table[i]); + + return xgbe_write_rss_lookup_table(pdata); +} + +static int xgbe_enable_rss(struct xgbe_prv_data *pdata) +{ + int ret; + + if (!pdata->hw_feat.rss) + return -EOPNOTSUPP; + + /* Program the hash key */ + ret = xgbe_write_rss_hash_key(pdata); + if (ret) + return ret; + + /* Program the lookup table */ + ret = xgbe_write_rss_lookup_table(pdata); + if (ret) + return ret; + + /* Set the RSS options */ + XGMAC_IOWRITE(pdata, MAC_RSSCR, pdata->rss_options); + + /* Enable RSS */ + XGMAC_IOWRITE_BITS(pdata, MAC_RSSCR, RSSE, 1); + + return 0; +} + +static int xgbe_disable_rss(struct xgbe_prv_data *pdata) +{ + if (!pdata->hw_feat.rss) + return -EOPNOTSUPP; + + XGMAC_IOWRITE_BITS(pdata, MAC_RSSCR, RSSE, 0); + + return 0; +} + +static void xgbe_config_rss(struct xgbe_prv_data *pdata) +{ + int ret; + + if (!pdata->hw_feat.rss) + return; + + if (pdata->netdev->features & NETIF_F_RXHASH) + ret = xgbe_enable_rss(pdata); + else + ret = xgbe_disable_rss(pdata); + + if (ret) + netdev_err(pdata->netdev, + "error configuring RSS, RSS disabled\n"); +} + static int xgbe_disable_tx_flow_control(struct xgbe_prv_data *pdata) { unsigned int max_q_count, q_count; @@ -465,17 +620,21 @@ static void xgbe_enable_dma_interrupts(struct xgbe_prv_data *pdata) if (channel->tx_ring) { /* Enable the following Tx interrupts - * TIE - Transmit Interrupt Enable (unless polling) + * TIE - Transmit Interrupt Enable (unless using + * per channel interrupts) */ - XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1); + if (!pdata->per_channel_irq) + XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, TIE, 1); } if (channel->rx_ring) { /* Enable following Rx interrupts * RBUE - Receive Buffer Unavailable Enable - * RIE - Receive Interrupt Enable + * RIE - Receive Interrupt Enable (unless using + * per channel interrupts) */ XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RBUE, 1); - XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1); + if (!pdata->per_channel_irq) + XGMAC_SET_BITS(dma_ch_ier, DMA_CH_IER, RIE, 1); } XGMAC_DMA_IOWRITE(channel, DMA_CH_IER, dma_ch_ier); @@ -880,13 +1039,15 @@ static void xgbe_tx_desc_reset(struct xgbe_ring_data *rdata) rdesc->desc1 = 0; rdesc->desc2 = 0; rdesc->desc3 = 0; + + /* Make sure ownership is written to the descriptor */ + wmb(); } static void xgbe_tx_desc_init(struct xgbe_channel *channel) { struct xgbe_ring *ring = channel->tx_ring; struct xgbe_ring_data *rdata; - struct xgbe_ring_desc *rdesc; int i; int start_index = ring->cur; @@ -895,26 +1056,11 @@ static void xgbe_tx_desc_init(struct xgbe_channel *channel) /* Initialze all descriptors */ for (i = 0; i < ring->rdesc_count; i++) { rdata = XGBE_GET_DESC_DATA(ring, i); - rdesc = rdata->rdesc; - /* Initialize Tx descriptor - * Set buffer 1 (lo) address to zero - * Set buffer 1 (hi) address to zero - * Reset all other control bits (IC, TTSE, B2L & B1L) - * Reset all other control bits (OWN, CTXT, FD, LD, CPC, CIC, - * etc) - */ - rdesc->desc0 = 0; - rdesc->desc1 = 0; - rdesc->desc2 = 0; - rdesc->desc3 = 0; + /* Initialize Tx descriptor */ + xgbe_tx_desc_reset(rdata); } - /* Make sure everything is written to the descriptor(s) before - * telling the device about them - */ - wmb(); - /* Update the total number of Tx descriptors */ XGMAC_DMA_IOWRITE(channel, DMA_CH_TDRLR, ring->rdesc_count - 1); @@ -933,19 +1079,19 @@ static void xgbe_rx_desc_reset(struct xgbe_ring_data *rdata) struct xgbe_ring_desc *rdesc = rdata->rdesc; /* Reset the Rx descriptor - * Set buffer 1 (lo) address to dma address (lo) - * Set buffer 1 (hi) address to dma address (hi) - * Set buffer 2 (lo) address to zero - * Set buffer 2 (hi) address to zero and set control bits - * OWN and INTE + * Set buffer 1 (lo) address to header dma address (lo) + * Set buffer 1 (hi) address to header dma address (hi) + * Set buffer 2 (lo) address to buffer dma address (lo) + * Set buffer 2 (hi) address to buffer dma address (hi) and + * set control bits OWN and INTE */ - rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->skb_dma)); - rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->skb_dma)); - rdesc->desc2 = 0; + rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->rx.hdr.dma)); + rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->rx.hdr.dma)); + rdesc->desc2 = cpu_to_le32(lower_32_bits(rdata->rx.buf.dma)); + rdesc->desc3 = cpu_to_le32(upper_32_bits(rdata->rx.buf.dma)); - rdesc->desc3 = 0; - if (rdata->interrupt) - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, 1); + XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, + rdata->interrupt ? 1 : 0); /* Since the Rx DMA engine is likely running, make sure everything * is written to the descriptor(s) before setting the OWN bit @@ -964,7 +1110,6 @@ static void xgbe_rx_desc_init(struct xgbe_channel *channel) struct xgbe_prv_data *pdata = channel->pdata; struct xgbe_ring *ring = channel->rx_ring; struct xgbe_ring_data *rdata; - struct xgbe_ring_desc *rdesc; unsigned int start_index = ring->cur; unsigned int rx_coalesce, rx_frames; unsigned int i; @@ -977,34 +1122,16 @@ static void xgbe_rx_desc_init(struct xgbe_channel *channel) /* Initialize all descriptors */ for (i = 0; i < ring->rdesc_count; i++) { rdata = XGBE_GET_DESC_DATA(ring, i); - rdesc = rdata->rdesc; - /* Initialize Rx descriptor - * Set buffer 1 (lo) address to dma address (lo) - * Set buffer 1 (hi) address to dma address (hi) - * Set buffer 2 (lo) address to zero - * Set buffer 2 (hi) address to zero and set control - * bits OWN and INTE appropriateley - */ - rdesc->desc0 = cpu_to_le32(lower_32_bits(rdata->skb_dma)); - rdesc->desc1 = cpu_to_le32(upper_32_bits(rdata->skb_dma)); - rdesc->desc2 = 0; - rdesc->desc3 = 0; - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, OWN, 1); - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, 1); - rdata->interrupt = 1; - if (rx_coalesce && (!rx_frames || ((i + 1) % rx_frames))) { - /* Clear interrupt on completion bit */ - XGMAC_SET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, INTE, - 0); + /* Set interrupt on completion bit as appropriate */ + if (rx_coalesce && (!rx_frames || ((i + 1) % rx_frames))) rdata->interrupt = 0; - } - } + else + rdata->interrupt = 1; - /* Make sure everything is written to the descriptors before - * telling the device about them - */ - wmb(); + /* Initialize Rx descriptor */ + xgbe_rx_desc_reset(rdata); + } /* Update the total number of Rx descriptors */ XGMAC_DMA_IOWRITE(channel, DMA_CH_RDRLR, ring->rdesc_count - 1); @@ -1198,7 +1325,30 @@ static void xgbe_config_dcb_pfc(struct xgbe_prv_data *pdata) xgbe_config_flow_control(pdata); } -static void xgbe_pre_xmit(struct xgbe_channel *channel) +static void xgbe_tx_start_xmit(struct xgbe_channel *channel, + struct xgbe_ring *ring) +{ + struct xgbe_prv_data *pdata = channel->pdata; + struct xgbe_ring_data *rdata; + + /* Issue a poll command to Tx DMA by writing address + * of next immediate free descriptor */ + rdata = XGBE_GET_DESC_DATA(ring, ring->cur); + XGMAC_DMA_IOWRITE(channel, DMA_CH_TDTR_LO, + lower_32_bits(rdata->rdesc_dma)); + + /* Start the Tx coalescing timer */ + if (pdata->tx_usecs && !channel->tx_timer_active) { + channel->tx_timer_active = 1; + hrtimer_start(&channel->tx_timer, + ktime_set(0, pdata->tx_usecs * NSEC_PER_USEC), + HRTIMER_MODE_REL); + } + + ring->tx.xmit_more = 0; +} + +static void xgbe_dev_xmit(struct xgbe_channel *channel) { struct xgbe_prv_data *pdata = channel->pdata; struct xgbe_ring *ring = channel->tx_ring; @@ -1207,11 +1357,11 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) struct xgbe_packet_data *packet = &ring->packet_data; unsigned int csum, tso, vlan; unsigned int tso_context, vlan_context; - unsigned int tx_coalesce, tx_frames; + unsigned int tx_set_ic; int start_index = ring->cur; int i; - DBGPR("-->xgbe_pre_xmit\n"); + DBGPR("-->xgbe_dev_xmit\n"); csum = XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, CSUM_ENABLE); @@ -1230,10 +1380,26 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) else vlan_context = 0; - tx_coalesce = (pdata->tx_usecs || pdata->tx_frames) ? 1 : 0; - tx_frames = pdata->tx_frames; - if (tx_coalesce && !channel->tx_timer_active) - ring->coalesce_count = 0; + /* Determine if an interrupt should be generated for this Tx: + * Interrupt: + * - Tx frame count exceeds the frame count setting + * - Addition of Tx frame count to the frame count since the + * last interrupt was set exceeds the frame count setting + * No interrupt: + * - No frame count setting specified (ethtool -C ethX tx-frames 0) + * - Addition of Tx frame count to the frame count since the + * last interrupt was set does not exceed the frame count setting + */ + ring->coalesce_count += packet->tx_packets; + if (!pdata->tx_frames) + tx_set_ic = 0; + else if (packet->tx_packets > pdata->tx_frames) + tx_set_ic = 1; + else if ((ring->coalesce_count % pdata->tx_frames) < + packet->tx_packets) + tx_set_ic = 1; + else + tx_set_ic = 0; rdata = XGBE_GET_DESC_DATA(ring, ring->cur); rdesc = rdata->rdesc; @@ -1300,13 +1466,6 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) if (XGMAC_GET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, PTP)) XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, TTSE, 1); - /* Set IC bit based on Tx coalescing settings */ - XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, IC, 1); - if (tx_coalesce && (!tx_frames || - (++ring->coalesce_count % tx_frames))) - /* Clear IC bit */ - XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, IC, 0); - /* Mark it as First Descriptor */ XGMAC_SET_BITS_LE(rdesc->desc3, TX_NORMAL_DESC3, FD, 1); @@ -1351,13 +1510,6 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, HL_B1L, rdata->skb_dma_len); - /* Set IC bit based on Tx coalescing settings */ - XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, IC, 1); - if (tx_coalesce && (!tx_frames || - (++ring->coalesce_count % tx_frames))) - /* Clear IC bit */ - XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, IC, 0); - /* Set OWN bit */ XGMAC_SET_BITS_LE(rdesc->desc3, TX_NORMAL_DESC3, OWN, 1); @@ -1373,6 +1525,14 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) /* Set LAST bit for the last descriptor */ XGMAC_SET_BITS_LE(rdesc->desc3, TX_NORMAL_DESC3, LD, 1); + /* Set IC bit based on Tx coalescing settings */ + if (tx_set_ic) + XGMAC_SET_BITS_LE(rdesc->desc2, TX_NORMAL_DESC2, IC, 1); + + /* Save the Tx info to report back during cleanup */ + rdata->tx.packets = packet->tx_packets; + rdata->tx.bytes = packet->tx_bytes; + /* In case the Tx DMA engine is running, make sure everything * is written to the descriptor(s) before setting the OWN bit * for the first descriptor @@ -1391,26 +1551,19 @@ static void xgbe_pre_xmit(struct xgbe_channel *channel) /* Make sure ownership is written to the descriptor */ wmb(); - /* Issue a poll command to Tx DMA by writing address - * of next immediate free descriptor */ ring->cur++; - rdata = XGBE_GET_DESC_DATA(ring, ring->cur); - XGMAC_DMA_IOWRITE(channel, DMA_CH_TDTR_LO, - lower_32_bits(rdata->rdesc_dma)); - - /* Start the Tx coalescing timer */ - if (tx_coalesce && !channel->tx_timer_active) { - channel->tx_timer_active = 1; - hrtimer_start(&channel->tx_timer, - ktime_set(0, pdata->tx_usecs * NSEC_PER_USEC), - HRTIMER_MODE_REL); - } + if (!packet->skb->xmit_more || + netif_xmit_stopped(netdev_get_tx_queue(pdata->netdev, + channel->queue_index))) + xgbe_tx_start_xmit(channel, ring); + else + ring->tx.xmit_more = 1; DBGPR(" %s: descriptors %u to %u written\n", channel->name, start_index & (ring->rdesc_count - 1), (ring->cur - 1) & (ring->rdesc_count - 1)); - DBGPR("<--xgbe_pre_xmit\n"); + DBGPR("<--xgbe_dev_xmit\n"); } static int xgbe_dev_read(struct xgbe_channel *channel) @@ -1420,7 +1573,7 @@ static int xgbe_dev_read(struct xgbe_channel *channel) struct xgbe_ring_desc *rdesc; struct xgbe_packet_data *packet = &ring->packet_data; struct net_device *netdev = channel->pdata->netdev; - unsigned int err, etlt; + unsigned int err, etlt, l34t; DBGPR("-->xgbe_dev_read: cur = %d\n", ring->cur); @@ -1431,6 +1584,9 @@ static int xgbe_dev_read(struct xgbe_channel *channel) if (XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, OWN)) return 1; + /* Make sure descriptor fields are read after reading the OWN bit */ + rmb(); + #ifdef XGMAC_ENABLE_RX_DESC_DUMP xgbe_dump_rx_desc(ring, rdesc, ring->cur); #endif @@ -1454,8 +1610,33 @@ static int xgbe_dev_read(struct xgbe_channel *channel) XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, CONTEXT_NEXT, 1); + /* Get the header length */ + if (XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, FD)) + rdata->rx.hdr_len = XGMAC_GET_BITS_LE(rdesc->desc2, + RX_NORMAL_DESC2, HL); + + /* Get the RSS hash */ + if (XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, RSV)) { + XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, + RSS_HASH, 1); + + packet->rss_hash = le32_to_cpu(rdesc->desc1); + + l34t = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, L34T); + switch (l34t) { + case RX_DESC3_L34T_IPV4_TCP: + case RX_DESC3_L34T_IPV4_UDP: + case RX_DESC3_L34T_IPV6_TCP: + case RX_DESC3_L34T_IPV6_UDP: + packet->rss_hash_type = PKT_HASH_TYPE_L4; + break; + default: + packet->rss_hash_type = PKT_HASH_TYPE_L3; + } + } + /* Get the packet length */ - rdata->len = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, PL); + rdata->rx.len = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, PL); if (!XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, LD)) { /* Not all the data has been transferred for this packet */ @@ -1478,7 +1659,8 @@ static int xgbe_dev_read(struct xgbe_channel *channel) etlt = XGMAC_GET_BITS_LE(rdesc->desc3, RX_NORMAL_DESC3, ETLT); DBGPR(" err=%u, etlt=%#x\n", err, etlt); - if (!err || (err && !etlt)) { + if (!err || !etlt) { + /* No error if err is 0 or etlt is 0 */ if ((etlt == 0x09) && (netdev->features & NETIF_F_HW_VLAN_CTAG_RX)) { XGMAC_SET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, @@ -2298,6 +2480,47 @@ static void xgbe_config_mmc(struct xgbe_prv_data *pdata) XGMAC_IOWRITE_BITS(pdata, MMC_CR, CR, 1); } +static void xgbe_prepare_tx_stop(struct xgbe_prv_data *pdata, + struct xgbe_channel *channel) +{ + unsigned int tx_dsr, tx_pos, tx_qidx; + unsigned int tx_status; + unsigned long tx_timeout; + + /* Calculate the status register to read and the position within */ + if (channel->queue_index < DMA_DSRX_FIRST_QUEUE) { + tx_dsr = DMA_DSR0; + tx_pos = (channel->queue_index * DMA_DSR_Q_WIDTH) + + DMA_DSR0_TPS_START; + } else { + tx_qidx = channel->queue_index - DMA_DSRX_FIRST_QUEUE; + + tx_dsr = DMA_DSR1 + ((tx_qidx / DMA_DSRX_QPR) * DMA_DSRX_INC); + tx_pos = ((tx_qidx % DMA_DSRX_QPR) * DMA_DSR_Q_WIDTH) + + DMA_DSRX_TPS_START; + } + + /* The Tx engine cannot be stopped if it is actively processing + * descriptors. Wait for the Tx engine to enter the stopped or + * suspended state. Don't wait forever though... + */ + tx_timeout = jiffies + (XGBE_DMA_STOP_TIMEOUT * HZ); + while (time_before(jiffies, tx_timeout)) { + tx_status = XGMAC_IOREAD(pdata, tx_dsr); + tx_status = GET_BITS(tx_status, tx_pos, DMA_DSR_TPS_WIDTH); + if ((tx_status == DMA_TPS_STOPPED) || + (tx_status == DMA_TPS_SUSPENDED)) + break; + + usleep_range(500, 1000); + } + + if (!time_before(jiffies, tx_timeout)) + netdev_info(pdata->netdev, + "timed out waiting for Tx DMA channel %u to stop\n", + channel->queue_index); +} + static void xgbe_enable_tx(struct xgbe_prv_data *pdata) { struct xgbe_channel *channel; @@ -2326,6 +2549,15 @@ static void xgbe_disable_tx(struct xgbe_prv_data *pdata) struct xgbe_channel *channel; unsigned int i; + /* Prepare for Tx DMA channel stop */ + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (!channel->tx_ring) + break; + + xgbe_prepare_tx_stop(pdata, channel); + } + /* Disable MAC Tx */ XGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 0); @@ -2417,6 +2649,15 @@ static void xgbe_powerdown_tx(struct xgbe_prv_data *pdata) struct xgbe_channel *channel; unsigned int i; + /* Prepare for Tx DMA channel stop */ + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (!channel->tx_ring) + break; + + xgbe_prepare_tx_stop(pdata, channel); + } + /* Disable MAC Tx */ XGMAC_IOWRITE_BITS(pdata, MAC_TCR, TE, 0); @@ -2485,6 +2726,8 @@ static int xgbe_init(struct xgbe_prv_data *pdata) xgbe_config_tx_coalesce(pdata); xgbe_config_rx_buffer_size(pdata); xgbe_config_tso_mode(pdata); + xgbe_config_sph_mode(pdata); + xgbe_config_rss(pdata); desc_if->wrapper_tx_desc_init(pdata); desc_if->wrapper_rx_desc_init(pdata); xgbe_enable_dma_interrupts(pdata); @@ -2561,7 +2804,7 @@ void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if) hw_if->powerup_rx = xgbe_powerup_rx; hw_if->powerdown_rx = xgbe_powerdown_rx; - hw_if->pre_xmit = xgbe_pre_xmit; + hw_if->dev_xmit = xgbe_dev_xmit; hw_if->dev_read = xgbe_dev_read; hw_if->enable_int = xgbe_enable_int; hw_if->disable_int = xgbe_disable_int; @@ -2575,6 +2818,7 @@ void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if) hw_if->rx_desc_reset = xgbe_rx_desc_reset; hw_if->is_last_desc = xgbe_is_last_desc; hw_if->is_context_desc = xgbe_is_context_desc; + hw_if->tx_start_xmit = xgbe_tx_start_xmit; /* For FLOW ctrl */ hw_if->config_tx_flow_control = xgbe_config_tx_flow_control; @@ -2620,5 +2864,11 @@ void xgbe_init_function_ptrs_dev(struct xgbe_hw_if *hw_if) hw_if->config_dcb_tc = xgbe_config_dcb_tc; hw_if->config_dcb_pfc = xgbe_config_dcb_pfc; + /* For Receive Side Scaling */ + hw_if->enable_rss = xgbe_enable_rss; + hw_if->disable_rss = xgbe_disable_rss; + hw_if->set_rss_hash_key = xgbe_set_rss_hash_key; + hw_if->set_rss_lookup_table = xgbe_set_rss_lookup_table; + DBGPR("<--xgbe_init_function_ptrs\n"); } diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c index 2349ea970255..7bb5f07dbeef 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-drv.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-drv.c @@ -114,6 +114,7 @@ * THE POSSIBILITY OF SUCH DAMAGE. */ +#include <linux/platform_device.h> #include <linux/spinlock.h> #include <linux/tcp.h> #include <linux/if_vlan.h> @@ -126,14 +127,126 @@ #include "xgbe.h" #include "xgbe-common.h" -static int xgbe_poll(struct napi_struct *, int); +static int xgbe_one_poll(struct napi_struct *, int); +static int xgbe_all_poll(struct napi_struct *, int); static void xgbe_set_rx_mode(struct net_device *); +static int xgbe_alloc_channels(struct xgbe_prv_data *pdata) +{ + struct xgbe_channel *channel_mem, *channel; + struct xgbe_ring *tx_ring, *rx_ring; + unsigned int count, i; + int ret = -ENOMEM; + + count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count); + + channel_mem = kcalloc(count, sizeof(struct xgbe_channel), GFP_KERNEL); + if (!channel_mem) + goto err_channel; + + tx_ring = kcalloc(pdata->tx_ring_count, sizeof(struct xgbe_ring), + GFP_KERNEL); + if (!tx_ring) + goto err_tx_ring; + + rx_ring = kcalloc(pdata->rx_ring_count, sizeof(struct xgbe_ring), + GFP_KERNEL); + if (!rx_ring) + goto err_rx_ring; + + for (i = 0, channel = channel_mem; i < count; i++, channel++) { + snprintf(channel->name, sizeof(channel->name), "channel-%d", i); + channel->pdata = pdata; + channel->queue_index = i; + channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE + + (DMA_CH_INC * i); + + if (pdata->per_channel_irq) { + /* Get the DMA interrupt (offset 1) */ + ret = platform_get_irq(pdata->pdev, i + 1); + if (ret < 0) { + netdev_err(pdata->netdev, + "platform_get_irq %u failed\n", + i + 1); + goto err_irq; + } + + channel->dma_irq = ret; + } + + if (i < pdata->tx_ring_count) { + spin_lock_init(&tx_ring->lock); + channel->tx_ring = tx_ring++; + } + + if (i < pdata->rx_ring_count) { + spin_lock_init(&rx_ring->lock); + channel->rx_ring = rx_ring++; + } + + DBGPR(" %s: queue=%u, dma_regs=%p, dma_irq=%d, tx=%p, rx=%p\n", + channel->name, channel->queue_index, channel->dma_regs, + channel->dma_irq, channel->tx_ring, channel->rx_ring); + } + + pdata->channel = channel_mem; + pdata->channel_count = count; + + return 0; + +err_irq: + kfree(rx_ring); + +err_rx_ring: + kfree(tx_ring); + +err_tx_ring: + kfree(channel_mem); + +err_channel: + return ret; +} + +static void xgbe_free_channels(struct xgbe_prv_data *pdata) +{ + if (!pdata->channel) + return; + + kfree(pdata->channel->rx_ring); + kfree(pdata->channel->tx_ring); + kfree(pdata->channel); + + pdata->channel = NULL; + pdata->channel_count = 0; +} + static inline unsigned int xgbe_tx_avail_desc(struct xgbe_ring *ring) { return (ring->rdesc_count - (ring->cur - ring->dirty)); } +static int xgbe_maybe_stop_tx_queue(struct xgbe_channel *channel, + struct xgbe_ring *ring, unsigned int count) +{ + struct xgbe_prv_data *pdata = channel->pdata; + + if (count > xgbe_tx_avail_desc(ring)) { + DBGPR(" Tx queue stopped, not enough descriptors available\n"); + netif_stop_subqueue(pdata->netdev, channel->queue_index); + ring->tx.queue_stopped = 1; + + /* If we haven't notified the hardware because of xmit_more + * support, tell it now + */ + if (ring->tx.xmit_more) + pdata->hw_if.tx_start_xmit(channel, ring); + + return NETDEV_TX_BUSY; + } + + return 0; +} + static int xgbe_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu) { unsigned int rx_buf_size; @@ -144,8 +257,8 @@ static int xgbe_calc_rx_buf_size(struct net_device *netdev, unsigned int mtu) } rx_buf_size = mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN; - if (rx_buf_size < XGBE_RX_MIN_BUF_SIZE) - rx_buf_size = XGBE_RX_MIN_BUF_SIZE; + rx_buf_size = clamp_val(rx_buf_size, XGBE_RX_MIN_BUF_SIZE, PAGE_SIZE); + rx_buf_size = (rx_buf_size + XGBE_RX_BUF_ALIGN - 1) & ~(XGBE_RX_BUF_ALIGN - 1); @@ -213,11 +326,7 @@ static irqreturn_t xgbe_isr(int irq, void *data) if (!dma_isr) goto isr_done; - DBGPR("-->xgbe_isr\n"); - DBGPR(" DMA_ISR = %08x\n", dma_isr); - DBGPR(" DMA_DS0 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR0)); - DBGPR(" DMA_DS1 = %08x\n", XGMAC_IOREAD(pdata, DMA_DSR1)); for (i = 0; i < pdata->channel_count; i++) { if (!(dma_isr & (1 << i))) @@ -228,6 +337,10 @@ static irqreturn_t xgbe_isr(int irq, void *data) dma_ch_isr = XGMAC_DMA_IOREAD(channel, DMA_CH_SR); DBGPR(" DMA_CH%u_ISR = %08x\n", i, dma_ch_isr); + /* If we get a TI or RI interrupt that means per channel DMA + * interrupts are not enabled, so we use the private data napi + * structure, not the per channel napi structure + */ if (XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, TI) || XGMAC_GET_BITS(dma_ch_isr, DMA_CH_SR, RI)) { if (napi_schedule_prep(&pdata->napi)) { @@ -270,12 +383,28 @@ static irqreturn_t xgbe_isr(int irq, void *data) DBGPR(" DMA_ISR = %08x\n", XGMAC_IOREAD(pdata, DMA_ISR)); - DBGPR("<--xgbe_isr\n"); - isr_done: return IRQ_HANDLED; } +static irqreturn_t xgbe_dma_isr(int irq, void *data) +{ + struct xgbe_channel *channel = data; + + /* Per channel DMA interrupts are enabled, so we use the per + * channel napi structure and not the private data napi structure + */ + if (napi_schedule_prep(&channel->napi)) { + /* Disable Tx and Rx interrupts */ + disable_irq_nosync(channel->dma_irq); + + /* Turn on polling */ + __napi_schedule(&channel->napi); + } + + return IRQ_HANDLED; +} + static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer) { struct xgbe_channel *channel = container_of(timer, @@ -283,18 +412,24 @@ static enum hrtimer_restart xgbe_tx_timer(struct hrtimer *timer) tx_timer); struct xgbe_ring *ring = channel->tx_ring; struct xgbe_prv_data *pdata = channel->pdata; + struct napi_struct *napi; unsigned long flags; DBGPR("-->xgbe_tx_timer\n"); + napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi; + spin_lock_irqsave(&ring->lock, flags); - if (napi_schedule_prep(&pdata->napi)) { + if (napi_schedule_prep(napi)) { /* Disable Tx and Rx interrupts */ - xgbe_disable_rx_tx_ints(pdata); + if (pdata->per_channel_irq) + disable_irq(channel->dma_irq); + else + xgbe_disable_rx_tx_ints(pdata); /* Turn on polling */ - __napi_schedule(&pdata->napi); + __napi_schedule(napi); } channel->tx_timer_active = 0; @@ -430,18 +565,46 @@ void xgbe_get_all_hw_features(struct xgbe_prv_data *pdata) static void xgbe_napi_enable(struct xgbe_prv_data *pdata, unsigned int add) { - if (add) - netif_napi_add(pdata->netdev, &pdata->napi, xgbe_poll, - NAPI_POLL_WEIGHT); - napi_enable(&pdata->napi); + struct xgbe_channel *channel; + unsigned int i; + + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (add) + netif_napi_add(pdata->netdev, &channel->napi, + xgbe_one_poll, NAPI_POLL_WEIGHT); + + napi_enable(&channel->napi); + } + } else { + if (add) + netif_napi_add(pdata->netdev, &pdata->napi, + xgbe_all_poll, NAPI_POLL_WEIGHT); + + napi_enable(&pdata->napi); + } } static void xgbe_napi_disable(struct xgbe_prv_data *pdata, unsigned int del) { - napi_disable(&pdata->napi); + struct xgbe_channel *channel; + unsigned int i; + + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + napi_disable(&channel->napi); - if (del) - netif_napi_del(&pdata->napi); + if (del) + netif_napi_del(&channel->napi); + } + } else { + napi_disable(&pdata->napi); + + if (del) + netif_napi_del(&pdata->napi); + } } void xgbe_init_tx_coalesce(struct xgbe_prv_data *pdata) @@ -472,7 +635,7 @@ void xgbe_init_rx_coalesce(struct xgbe_prv_data *pdata) DBGPR("<--xgbe_init_rx_coalesce\n"); } -static void xgbe_free_tx_skbuff(struct xgbe_prv_data *pdata) +static void xgbe_free_tx_data(struct xgbe_prv_data *pdata) { struct xgbe_desc_if *desc_if = &pdata->desc_if; struct xgbe_channel *channel; @@ -480,7 +643,7 @@ static void xgbe_free_tx_skbuff(struct xgbe_prv_data *pdata) struct xgbe_ring_data *rdata; unsigned int i, j; - DBGPR("-->xgbe_free_tx_skbuff\n"); + DBGPR("-->xgbe_free_tx_data\n"); channel = pdata->channel; for (i = 0; i < pdata->channel_count; i++, channel++) { @@ -490,14 +653,14 @@ static void xgbe_free_tx_skbuff(struct xgbe_prv_data *pdata) for (j = 0; j < ring->rdesc_count; j++) { rdata = XGBE_GET_DESC_DATA(ring, j); - desc_if->unmap_skb(pdata, rdata); + desc_if->unmap_rdata(pdata, rdata); } } - DBGPR("<--xgbe_free_tx_skbuff\n"); + DBGPR("<--xgbe_free_tx_data\n"); } -static void xgbe_free_rx_skbuff(struct xgbe_prv_data *pdata) +static void xgbe_free_rx_data(struct xgbe_prv_data *pdata) { struct xgbe_desc_if *desc_if = &pdata->desc_if; struct xgbe_channel *channel; @@ -505,7 +668,7 @@ static void xgbe_free_rx_skbuff(struct xgbe_prv_data *pdata) struct xgbe_ring_data *rdata; unsigned int i, j; - DBGPR("-->xgbe_free_rx_skbuff\n"); + DBGPR("-->xgbe_free_rx_data\n"); channel = pdata->channel; for (i = 0; i < pdata->channel_count; i++, channel++) { @@ -515,11 +678,11 @@ static void xgbe_free_rx_skbuff(struct xgbe_prv_data *pdata) for (j = 0; j < ring->rdesc_count; j++) { rdata = XGBE_GET_DESC_DATA(ring, j); - desc_if->unmap_skb(pdata, rdata); + desc_if->unmap_rdata(pdata, rdata); } } - DBGPR("<--xgbe_free_rx_skbuff\n"); + DBGPR("<--xgbe_free_rx_data\n"); } static void xgbe_adjust_link(struct net_device *netdev) @@ -735,7 +898,10 @@ static int xgbe_start(struct xgbe_prv_data *pdata) static void xgbe_stop(struct xgbe_prv_data *pdata) { struct xgbe_hw_if *hw_if = &pdata->hw_if; + struct xgbe_channel *channel; struct net_device *netdev = pdata->netdev; + struct netdev_queue *txq; + unsigned int i; DBGPR("-->xgbe_stop\n"); @@ -749,12 +915,23 @@ static void xgbe_stop(struct xgbe_prv_data *pdata) hw_if->disable_tx(pdata); hw_if->disable_rx(pdata); + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + if (!channel->tx_ring) + continue; + + txq = netdev_get_tx_queue(netdev, channel->queue_index); + netdev_tx_reset_queue(txq); + } + DBGPR("<--xgbe_stop\n"); } static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset) { + struct xgbe_channel *channel; struct xgbe_hw_if *hw_if = &pdata->hw_if; + unsigned int i; DBGPR("-->xgbe_restart_dev\n"); @@ -763,10 +940,15 @@ static void xgbe_restart_dev(struct xgbe_prv_data *pdata, unsigned int reset) return; xgbe_stop(pdata); - synchronize_irq(pdata->irq_number); + synchronize_irq(pdata->dev_irq); + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) + synchronize_irq(channel->dma_irq); + } - xgbe_free_tx_skbuff(pdata); - xgbe_free_rx_skbuff(pdata); + xgbe_free_tx_data(pdata); + xgbe_free_rx_data(pdata); /* Issue software reset to device if requested */ if (reset) @@ -1008,6 +1190,12 @@ static int xgbe_prep_tso(struct sk_buff *skb, struct xgbe_packet_data *packet) packet->tcp_header_len, packet->tcp_payload_len); DBGPR(" packet->mss=%u\n", packet->mss); + /* Update the number of packets that will ultimately be transmitted + * along with the extra bytes for each extra packet + */ + packet->tx_packets = skb_shinfo(skb)->gso_segs; + packet->tx_bytes += (packet->tx_packets - 1) * packet->header_len; + return 0; } @@ -1033,17 +1221,22 @@ static void xgbe_packet_info(struct xgbe_prv_data *pdata, unsigned int len; unsigned int i; + packet->skb = skb; + context_desc = 0; packet->rdesc_count = 0; + packet->tx_packets = 1; + packet->tx_bytes = skb->len; + if (xgbe_is_tso(skb)) { - /* TSO requires an extra desriptor if mss is different */ + /* TSO requires an extra descriptor if mss is different */ if (skb_shinfo(skb)->gso_size != ring->tx.cur_mss) { context_desc = 1; packet->rdesc_count++; } - /* TSO requires an extra desriptor for TSO header */ + /* TSO requires an extra descriptor for TSO header */ packet->rdesc_count++; XGMAC_SET_BITS(packet->attributes, TX_PACKET_ATTRIBUTES, @@ -1091,6 +1284,8 @@ static int xgbe_open(struct net_device *netdev) struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; struct xgbe_desc_if *desc_if = &pdata->desc_if; + struct xgbe_channel *channel = NULL; + unsigned int i = 0; int ret; DBGPR("-->xgbe_open\n"); @@ -1119,24 +1314,48 @@ static int xgbe_open(struct net_device *netdev) goto err_ptpclk; pdata->rx_buf_size = ret; + /* Allocate the channel and ring structures */ + ret = xgbe_alloc_channels(pdata); + if (ret) + goto err_ptpclk; + /* Allocate the ring descriptors and buffers */ ret = desc_if->alloc_ring_resources(pdata); if (ret) - goto err_ptpclk; + goto err_channels; /* Initialize the device restart and Tx timestamp work struct */ INIT_WORK(&pdata->restart_work, xgbe_restart); INIT_WORK(&pdata->tx_tstamp_work, xgbe_tx_tstamp); /* Request interrupts */ - ret = devm_request_irq(pdata->dev, netdev->irq, xgbe_isr, 0, + ret = devm_request_irq(pdata->dev, pdata->dev_irq, xgbe_isr, 0, netdev->name, pdata); if (ret) { netdev_alert(netdev, "error requesting irq %d\n", - pdata->irq_number); - goto err_irq; + pdata->dev_irq); + goto err_rings; + } + + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) { + snprintf(channel->dma_irq_name, + sizeof(channel->dma_irq_name) - 1, + "%s-TxRx-%u", netdev_name(netdev), + channel->queue_index); + + ret = devm_request_irq(pdata->dev, channel->dma_irq, + xgbe_dma_isr, 0, + channel->dma_irq_name, channel); + if (ret) { + netdev_alert(netdev, + "error requesting irq %d\n", + channel->dma_irq); + goto err_irq; + } + } } - pdata->irq_number = netdev->irq; ret = xgbe_start(pdata); if (ret) @@ -1149,12 +1368,21 @@ static int xgbe_open(struct net_device *netdev) err_start: hw_if->exit(pdata); - devm_free_irq(pdata->dev, pdata->irq_number, pdata); - pdata->irq_number = 0; - err_irq: + if (pdata->per_channel_irq) { + /* Using an unsigned int, 'i' will go to UINT_MAX and exit */ + for (i--, channel--; i < pdata->channel_count; i--, channel--) + devm_free_irq(pdata->dev, channel->dma_irq, channel); + } + + devm_free_irq(pdata->dev, pdata->dev_irq, pdata); + +err_rings: desc_if->free_ring_resources(pdata); +err_channels: + xgbe_free_channels(pdata); + err_ptpclk: clk_disable_unprepare(pdata->ptpclk); @@ -1172,6 +1400,8 @@ static int xgbe_close(struct net_device *netdev) struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; struct xgbe_desc_if *desc_if = &pdata->desc_if; + struct xgbe_channel *channel; + unsigned int i; DBGPR("-->xgbe_close\n"); @@ -1181,15 +1411,20 @@ static int xgbe_close(struct net_device *netdev) /* Issue software reset to device */ hw_if->exit(pdata); - /* Free all the ring data */ + /* Free the ring descriptors and buffers */ desc_if->free_ring_resources(pdata); - /* Release the interrupt */ - if (pdata->irq_number != 0) { - devm_free_irq(pdata->dev, pdata->irq_number, pdata); - pdata->irq_number = 0; + /* Release the interrupts */ + devm_free_irq(pdata->dev, pdata->dev_irq, pdata); + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) + devm_free_irq(pdata->dev, channel->dma_irq, channel); } + /* Free the channel and ring structures */ + xgbe_free_channels(pdata); + /* Disable the clocks */ clk_disable_unprepare(pdata->ptpclk); clk_disable_unprepare(pdata->sysclk); @@ -1210,12 +1445,14 @@ static int xgbe_xmit(struct sk_buff *skb, struct net_device *netdev) struct xgbe_channel *channel; struct xgbe_ring *ring; struct xgbe_packet_data *packet; + struct netdev_queue *txq; unsigned long flags; int ret; DBGPR("-->xgbe_xmit: skb->len = %d\n", skb->len); channel = pdata->channel + skb->queue_mapping; + txq = netdev_get_tx_queue(netdev, channel->queue_index); ring = channel->tx_ring; packet = &ring->packet_data; @@ -1234,13 +1471,9 @@ static int xgbe_xmit(struct sk_buff *skb, struct net_device *netdev) xgbe_packet_info(pdata, ring, skb, packet); /* Check that there are enough descriptors available */ - if (packet->rdesc_count > xgbe_tx_avail_desc(ring)) { - DBGPR(" Tx queue stopped, not enough descriptors available\n"); - netif_stop_subqueue(netdev, channel->queue_index); - ring->tx.queue_stopped = 1; - ret = NETDEV_TX_BUSY; + ret = xgbe_maybe_stop_tx_queue(channel, ring, packet->rdesc_count); + if (ret) goto tx_netdev_return; - } ret = xgbe_prep_tso(skb, packet); if (ret) { @@ -1257,13 +1490,21 @@ static int xgbe_xmit(struct sk_buff *skb, struct net_device *netdev) xgbe_prep_tx_tstamp(pdata, skb, packet); + /* Report on the actual number of bytes (to be) sent */ + netdev_tx_sent_queue(txq, packet->tx_bytes); + /* Configure required descriptor fields for transmission */ - hw_if->pre_xmit(channel); + hw_if->dev_xmit(channel); #ifdef XGMAC_ENABLE_TX_PKT_DUMP xgbe_print_pkt(netdev, skb, true); #endif + /* Stop the queue in advance if there may not be enough descriptors */ + xgbe_maybe_stop_tx_queue(channel, ring, XGBE_TX_MAX_DESCS); + + ret = NETDEV_TX_OK; + tx_netdev_return: spin_unlock_irqrestore(&ring->lock, flags); @@ -1420,14 +1661,20 @@ static int xgbe_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto, static void xgbe_poll_controller(struct net_device *netdev) { struct xgbe_prv_data *pdata = netdev_priv(netdev); + struct xgbe_channel *channel; + unsigned int i; DBGPR("-->xgbe_poll_controller\n"); - disable_irq(pdata->irq_number); - - xgbe_isr(pdata->irq_number, pdata); - - enable_irq(pdata->irq_number); + if (pdata->per_channel_irq) { + channel = pdata->channel; + for (i = 0; i < pdata->channel_count; i++, channel++) + xgbe_dma_isr(channel->dma_irq, channel); + } else { + disable_irq(pdata->dev_irq); + xgbe_isr(pdata->dev_irq, pdata); + enable_irq(pdata->dev_irq); + } DBGPR("<--xgbe_poll_controller\n"); } @@ -1465,12 +1712,21 @@ static int xgbe_set_features(struct net_device *netdev, { struct xgbe_prv_data *pdata = netdev_priv(netdev); struct xgbe_hw_if *hw_if = &pdata->hw_if; - netdev_features_t rxcsum, rxvlan, rxvlan_filter; + netdev_features_t rxhash, rxcsum, rxvlan, rxvlan_filter; + int ret = 0; + rxhash = pdata->netdev_features & NETIF_F_RXHASH; rxcsum = pdata->netdev_features & NETIF_F_RXCSUM; rxvlan = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_RX; rxvlan_filter = pdata->netdev_features & NETIF_F_HW_VLAN_CTAG_FILTER; + if ((features & NETIF_F_RXHASH) && !rxhash) + ret = hw_if->enable_rss(pdata); + else if (!(features & NETIF_F_RXHASH) && rxhash) + ret = hw_if->disable_rss(pdata); + if (ret) + return ret; + if ((features & NETIF_F_RXCSUM) && !rxcsum) hw_if->enable_rx_csum(pdata); else if (!(features & NETIF_F_RXCSUM) && rxcsum) @@ -1524,7 +1780,7 @@ static void xgbe_rx_refresh(struct xgbe_channel *channel) struct xgbe_ring *ring = channel->rx_ring; struct xgbe_ring_data *rdata; - desc_if->realloc_skb(channel); + desc_if->realloc_rx_buffer(channel); /* Update the Rx Tail Pointer Register with address of * the last cleaned entry */ @@ -1533,6 +1789,31 @@ static void xgbe_rx_refresh(struct xgbe_channel *channel) lower_32_bits(rdata->rdesc_dma)); } +static struct sk_buff *xgbe_create_skb(struct xgbe_prv_data *pdata, + struct xgbe_ring_data *rdata, + unsigned int *len) +{ + struct net_device *netdev = pdata->netdev; + struct sk_buff *skb; + u8 *packet; + unsigned int copy_len; + + skb = netdev_alloc_skb_ip_align(netdev, rdata->rx.hdr.dma_len); + if (!skb) + return NULL; + + packet = page_address(rdata->rx.hdr.pa.pages) + + rdata->rx.hdr.pa.pages_offset; + copy_len = (rdata->rx.hdr_len) ? rdata->rx.hdr_len : *len; + copy_len = min(rdata->rx.hdr.dma_len, copy_len); + skb_copy_to_linear_data(skb, packet, copy_len); + skb_put(skb, copy_len); + + *len -= copy_len; + + return skb; +} + static int xgbe_tx_poll(struct xgbe_channel *channel) { struct xgbe_prv_data *pdata = channel->pdata; @@ -1542,8 +1823,10 @@ static int xgbe_tx_poll(struct xgbe_channel *channel) struct xgbe_ring_data *rdata; struct xgbe_ring_desc *rdesc; struct net_device *netdev = pdata->netdev; + struct netdev_queue *txq; unsigned long flags; int processed = 0; + unsigned int tx_packets = 0, tx_bytes = 0; DBGPR("-->xgbe_tx_poll\n"); @@ -1551,36 +1834,53 @@ static int xgbe_tx_poll(struct xgbe_channel *channel) if (!ring) return 0; + txq = netdev_get_tx_queue(netdev, channel->queue_index); + spin_lock_irqsave(&ring->lock, flags); while ((processed < XGBE_TX_DESC_MAX_PROC) && - (ring->dirty < ring->cur)) { + (ring->dirty != ring->cur)) { rdata = XGBE_GET_DESC_DATA(ring, ring->dirty); rdesc = rdata->rdesc; if (!hw_if->tx_complete(rdesc)) break; + /* Make sure descriptor fields are read after reading the OWN + * bit */ + rmb(); + #ifdef XGMAC_ENABLE_TX_DESC_DUMP xgbe_dump_tx_desc(ring, ring->dirty, 1, 0); #endif + if (hw_if->is_last_desc(rdesc)) { + tx_packets += rdata->tx.packets; + tx_bytes += rdata->tx.bytes; + } + /* Free the SKB and reset the descriptor for re-use */ - desc_if->unmap_skb(pdata, rdata); + desc_if->unmap_rdata(pdata, rdata); hw_if->tx_desc_reset(rdata); processed++; ring->dirty++; } + if (!processed) + goto unlock; + + netdev_tx_completed_queue(txq, tx_packets, tx_bytes); + if ((ring->tx.queue_stopped == 1) && (xgbe_tx_avail_desc(ring) > XGBE_TX_DESC_MIN_FREE)) { ring->tx.queue_stopped = 0; - netif_wake_subqueue(netdev, channel->queue_index); + netif_tx_wake_queue(txq); } DBGPR("<--xgbe_tx_poll: processed=%d\n", processed); +unlock: spin_unlock_irqrestore(&ring->lock, flags); return processed; @@ -1594,6 +1894,7 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) struct xgbe_ring_data *rdata; struct xgbe_packet_data *packet; struct net_device *netdev = pdata->netdev; + struct napi_struct *napi; struct sk_buff *skb; struct skb_shared_hwtstamps *hwtstamps; unsigned int incomplete, error, context_next, context; @@ -1607,6 +1908,8 @@ static int xgbe_rx_poll(struct xgbe_channel *channel, int budget) if (!ring) return 0; + napi = (pdata->per_channel_irq) ? &channel->napi : &pdata->napi; + rdata = XGBE_GET_DESC_DATA(ring, ring->cur); packet = &ring->packet_data; while (packet_count < budget) { @@ -1641,10 +1944,6 @@ read_again: ring->cur++; ring->dirty++; - dma_unmap_single(pdata->dev, rdata->skb_dma, - rdata->skb_dma_len, DMA_FROM_DEVICE); - rdata->skb_dma = 0; - incomplete = XGMAC_GET_BITS(packet->attributes, RX_PACKET_ATTRIBUTES, INCOMPLETE); @@ -1667,33 +1966,40 @@ read_again: } if (!context) { - put_len = rdata->len - len; - if (skb) { - if (pskb_expand_head(skb, 0, put_len, - GFP_ATOMIC)) { - DBGPR("pskb_expand_head error\n"); - if (incomplete) { - error = 1; - goto read_again; - } - - dev_kfree_skb(skb); - goto next_packet; + put_len = rdata->rx.len - len; + len += put_len; + + if (!skb) { + dma_sync_single_for_cpu(pdata->dev, + rdata->rx.hdr.dma, + rdata->rx.hdr.dma_len, + DMA_FROM_DEVICE); + + skb = xgbe_create_skb(pdata, rdata, &put_len); + if (!skb) { + error = 1; + goto skip_data; } - memcpy(skb_tail_pointer(skb), rdata->skb->data, - put_len); - } else { - skb = rdata->skb; - rdata->skb = NULL; } - skb_put(skb, put_len); - len += put_len; + + if (put_len) { + dma_sync_single_for_cpu(pdata->dev, + rdata->rx.buf.dma, + rdata->rx.buf.dma_len, + DMA_FROM_DEVICE); + + skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, + rdata->rx.buf.pa.pages, + rdata->rx.buf.pa.pages_offset, + put_len, rdata->rx.buf.dma_len); + rdata->rx.buf.pa.pages = NULL; + } } +skip_data: if (incomplete || context_next) goto read_again; - /* Stray Context Descriptor? */ if (!skb) goto next_packet; @@ -1733,13 +2039,18 @@ read_again: hwtstamps->hwtstamp = ns_to_ktime(nsec); } + if (XGMAC_GET_BITS(packet->attributes, + RX_PACKET_ATTRIBUTES, RSS_HASH)) + skb_set_hash(skb, packet->rss_hash, + packet->rss_hash_type); + skb->dev = netdev; skb->protocol = eth_type_trans(skb, netdev); skb_record_rx_queue(skb, channel->queue_index); - skb_mark_napi_id(skb, &pdata->napi); + skb_mark_napi_id(skb, napi); netdev->last_rx = jiffies; - napi_gro_receive(&pdata->napi, skb); + napi_gro_receive(napi, skb); next_packet: packet_count++; @@ -1761,7 +2072,35 @@ next_packet: return packet_count; } -static int xgbe_poll(struct napi_struct *napi, int budget) +static int xgbe_one_poll(struct napi_struct *napi, int budget) +{ + struct xgbe_channel *channel = container_of(napi, struct xgbe_channel, + napi); + int processed = 0; + + DBGPR("-->xgbe_one_poll: budget=%d\n", budget); + + /* Cleanup Tx ring first */ + xgbe_tx_poll(channel); + + /* Process Rx ring next */ + processed = xgbe_rx_poll(channel, budget); + + /* If we processed everything, we are done */ + if (processed < budget) { + /* Turn off polling */ + napi_complete(napi); + + /* Enable Tx and Rx interrupts */ + enable_irq(channel->dma_irq); + } + + DBGPR("<--xgbe_one_poll: received = %d\n", processed); + + return processed; +} + +static int xgbe_all_poll(struct napi_struct *napi, int budget) { struct xgbe_prv_data *pdata = container_of(napi, struct xgbe_prv_data, napi); @@ -1770,7 +2109,7 @@ static int xgbe_poll(struct napi_struct *napi, int budget) int processed, last_processed; unsigned int i; - DBGPR("-->xgbe_poll: budget=%d\n", budget); + DBGPR("-->xgbe_all_poll: budget=%d\n", budget); processed = 0; ring_budget = budget / pdata->rx_ring_count; @@ -1798,7 +2137,7 @@ static int xgbe_poll(struct napi_struct *napi, int budget) xgbe_enable_rx_tx_ints(pdata); } - DBGPR("<--xgbe_poll: received = %d\n", processed); + DBGPR("<--xgbe_all_poll: received = %d\n", processed); return processed; } @@ -1812,10 +2151,10 @@ void xgbe_dump_tx_desc(struct xgbe_ring *ring, unsigned int idx, while (count--) { rdata = XGBE_GET_DESC_DATA(ring, idx); rdesc = rdata->rdesc; - DBGPR("TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx, - (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE", - le32_to_cpu(rdesc->desc0), le32_to_cpu(rdesc->desc1), - le32_to_cpu(rdesc->desc2), le32_to_cpu(rdesc->desc3)); + pr_alert("TX_NORMAL_DESC[%d %s] = %08x:%08x:%08x:%08x\n", idx, + (flag == 1) ? "QUEUED FOR TX" : "TX BY DEVICE", + le32_to_cpu(rdesc->desc0), le32_to_cpu(rdesc->desc1), + le32_to_cpu(rdesc->desc2), le32_to_cpu(rdesc->desc3)); idx++; } } @@ -1823,9 +2162,9 @@ void xgbe_dump_tx_desc(struct xgbe_ring *ring, unsigned int idx, void xgbe_dump_rx_desc(struct xgbe_ring *ring, struct xgbe_ring_desc *desc, unsigned int idx) { - DBGPR("RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n", idx, - le32_to_cpu(desc->desc0), le32_to_cpu(desc->desc1), - le32_to_cpu(desc->desc2), le32_to_cpu(desc->desc3)); + pr_alert("RX_NORMAL_DESC[%d RX BY DEVICE] = %08x:%08x:%08x:%08x\n", idx, + le32_to_cpu(desc->desc0), le32_to_cpu(desc->desc1), + le32_to_cpu(desc->desc2), le32_to_cpu(desc->desc3)); } void xgbe_print_pkt(struct net_device *netdev, struct sk_buff *skb, bool tx_rx) diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c index 49508ec98b72..ebf489351555 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-ethtool.c @@ -452,9 +452,9 @@ static int xgbe_set_coalesce(struct net_device *netdev, rx_usecs); return -EINVAL; } - if (rx_frames > pdata->channel->rx_ring->rdesc_count) { + if (rx_frames > pdata->rx_desc_count) { netdev_alert(netdev, "rx-frames is limited to %d frames\n", - pdata->channel->rx_ring->rdesc_count); + pdata->rx_desc_count); return -EINVAL; } @@ -462,9 +462,9 @@ static int xgbe_set_coalesce(struct net_device *netdev, tx_frames = ec->tx_max_coalesced_frames; /* Check the bounds of values for Tx */ - if (tx_frames > pdata->channel->tx_ring->rdesc_count) { + if (tx_frames > pdata->tx_desc_count) { netdev_alert(netdev, "tx-frames is limited to %d frames\n", - pdata->channel->tx_ring->rdesc_count); + pdata->tx_desc_count); return -EINVAL; } @@ -481,6 +481,82 @@ static int xgbe_set_coalesce(struct net_device *netdev, return 0; } +static int xgbe_get_rxnfc(struct net_device *netdev, + struct ethtool_rxnfc *rxnfc, u32 *rule_locs) +{ + struct xgbe_prv_data *pdata = netdev_priv(netdev); + + switch (rxnfc->cmd) { + case ETHTOOL_GRXRINGS: + rxnfc->data = pdata->rx_ring_count; + break; + default: + return -EOPNOTSUPP; + } + + return 0; +} + +static u32 xgbe_get_rxfh_key_size(struct net_device *netdev) +{ + struct xgbe_prv_data *pdata = netdev_priv(netdev); + + return sizeof(pdata->rss_key); +} + +static u32 xgbe_get_rxfh_indir_size(struct net_device *netdev) +{ + struct xgbe_prv_data *pdata = netdev_priv(netdev); + + return ARRAY_SIZE(pdata->rss_table); +} + +static int xgbe_get_rxfh(struct net_device *netdev, u32 *indir, u8 *key, + u8 *hfunc) +{ + struct xgbe_prv_data *pdata = netdev_priv(netdev); + unsigned int i; + + if (indir) { + for (i = 0; i < ARRAY_SIZE(pdata->rss_table); i++) + indir[i] = XGMAC_GET_BITS(pdata->rss_table[i], + MAC_RSSDR, DMCH); + } + + if (key) + memcpy(key, pdata->rss_key, sizeof(pdata->rss_key)); + + if (hfunc) + *hfunc = ETH_RSS_HASH_TOP; + + return 0; +} + +static int xgbe_set_rxfh(struct net_device *netdev, const u32 *indir, + const u8 *key, const u8 hfunc) +{ + struct xgbe_prv_data *pdata = netdev_priv(netdev); + struct xgbe_hw_if *hw_if = &pdata->hw_if; + unsigned int ret; + + if (hfunc != ETH_RSS_HASH_NO_CHANGE && hfunc != ETH_RSS_HASH_TOP) + return -EOPNOTSUPP; + + if (indir) { + ret = hw_if->set_rss_lookup_table(pdata, indir); + if (ret) + return ret; + } + + if (key) { + ret = hw_if->set_rss_hash_key(pdata, key); + if (ret) + return ret; + } + + return 0; +} + static int xgbe_get_ts_info(struct net_device *netdev, struct ethtool_ts_info *ts_info) { @@ -526,6 +602,11 @@ static const struct ethtool_ops xgbe_ethtool_ops = { .get_strings = xgbe_get_strings, .get_ethtool_stats = xgbe_get_ethtool_stats, .get_sset_count = xgbe_get_sset_count, + .get_rxnfc = xgbe_get_rxnfc, + .get_rxfh_key_size = xgbe_get_rxfh_key_size, + .get_rxfh_indir_size = xgbe_get_rxfh_indir_size, + .get_rxfh = xgbe_get_rxfh, + .set_rxfh = xgbe_set_rxfh, .get_ts_info = xgbe_get_ts_info, }; diff --git a/drivers/net/ethernet/amd/xgbe/xgbe-main.c b/drivers/net/ethernet/amd/xgbe/xgbe-main.c index f5a8fa03921a..dbd3850b8b0a 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe-main.c +++ b/drivers/net/ethernet/amd/xgbe/xgbe-main.c @@ -133,60 +133,6 @@ MODULE_LICENSE("Dual BSD/GPL"); MODULE_VERSION(XGBE_DRV_VERSION); MODULE_DESCRIPTION(XGBE_DRV_DESC); -static struct xgbe_channel *xgbe_alloc_rings(struct xgbe_prv_data *pdata) -{ - struct xgbe_channel *channel_mem, *channel; - struct xgbe_ring *tx_ring, *rx_ring; - unsigned int count, i; - - DBGPR("-->xgbe_alloc_rings\n"); - - count = max_t(unsigned int, pdata->tx_ring_count, pdata->rx_ring_count); - - channel_mem = devm_kcalloc(pdata->dev, count, - sizeof(struct xgbe_channel), GFP_KERNEL); - if (!channel_mem) - return NULL; - - tx_ring = devm_kcalloc(pdata->dev, pdata->tx_ring_count, - sizeof(struct xgbe_ring), GFP_KERNEL); - if (!tx_ring) - return NULL; - - rx_ring = devm_kcalloc(pdata->dev, pdata->rx_ring_count, - sizeof(struct xgbe_ring), GFP_KERNEL); - if (!rx_ring) - return NULL; - - for (i = 0, channel = channel_mem; i < count; i++, channel++) { - snprintf(channel->name, sizeof(channel->name), "channel-%d", i); - channel->pdata = pdata; - channel->queue_index = i; - channel->dma_regs = pdata->xgmac_regs + DMA_CH_BASE + - (DMA_CH_INC * i); - - if (i < pdata->tx_ring_count) { - spin_lock_init(&tx_ring->lock); - channel->tx_ring = tx_ring++; - } - - if (i < pdata->rx_ring_count) { - spin_lock_init(&rx_ring->lock); - channel->rx_ring = rx_ring++; - } - - DBGPR(" %s - queue_index=%u, dma_regs=%p, tx=%p, rx=%p\n", - channel->name, channel->queue_index, channel->dma_regs, - channel->tx_ring, channel->rx_ring); - } - - pdata->channel_count = count; - - DBGPR("<--xgbe_alloc_rings\n"); - - return channel_mem; -} - static void xgbe_default_config(struct xgbe_prv_data *pdata) { DBGPR("-->xgbe_default_config\n"); @@ -224,6 +170,7 @@ static int xgbe_probe(struct platform_device *pdev) struct device *dev = &pdev->dev; struct resource *res; const u8 *mac_addr; + unsigned int i; int ret; DBGPR("--> xgbe_probe\n"); @@ -244,6 +191,7 @@ static int xgbe_probe(struct platform_device *pdev) spin_lock_init(&pdata->lock); mutex_init(&pdata->xpcs_mutex); + mutex_init(&pdata->rss_mutex); spin_lock_init(&pdata->tstamp_lock); /* Set and validate the number of descriptors for a ring */ @@ -318,12 +266,18 @@ static int xgbe_probe(struct platform_device *pdev) pdata->awcache = XGBE_DMA_SYS_AWCACHE; } + /* Check for per channel interrupt support */ + if (of_property_read_bool(dev->of_node, XGBE_DMA_IRQS)) + pdata->per_channel_irq = 1; + ret = platform_get_irq(pdev, 0); if (ret < 0) { - dev_err(dev, "platform_get_irq failed\n"); + dev_err(dev, "platform_get_irq 0 failed\n"); goto err_io; } - netdev->irq = ret; + pdata->dev_irq = ret; + + netdev->irq = pdata->dev_irq; netdev->base_addr = (unsigned long)pdata->xgmac_regs; /* Set all the function pointers */ @@ -383,13 +337,16 @@ static int xgbe_probe(struct platform_device *pdev) goto err_io; } - /* Allocate the rings for the DMA channels */ - pdata->channel = xgbe_alloc_rings(pdata); - if (!pdata->channel) { - dev_err(dev, "ring allocation failed\n"); - ret = -ENOMEM; - goto err_io; - } + /* Initialize RSS hash key and lookup table */ + netdev_rss_key_fill(pdata->rss_key, sizeof(pdata->rss_key)); + + for (i = 0; i < XGBE_RSS_MAX_TABLE_SIZE; i++) + XGMAC_SET_BITS(pdata->rss_table[i], MAC_RSSDR, DMCH, + i % pdata->rx_ring_count); + + XGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, IP2TE, 1); + XGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, TCP4TE, 1); + XGMAC_SET_BITS(pdata->rss_options, MAC_RSSCR, UDP4TE, 1); /* Prepare to regsiter with MDIO */ pdata->mii_bus_id = kasprintf(GFP_KERNEL, "%s", pdev->name); @@ -421,6 +378,9 @@ static int xgbe_probe(struct platform_device *pdev) NETIF_F_HW_VLAN_CTAG_TX | NETIF_F_HW_VLAN_CTAG_FILTER; + if (pdata->hw_feat.rss) + netdev->hw_features |= NETIF_F_RXHASH; + netdev->vlan_features |= NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM | diff --git a/drivers/net/ethernet/amd/xgbe/xgbe.h b/drivers/net/ethernet/amd/xgbe/xgbe.h index 789957d43a13..f9ec762ac3f0 100644 --- a/drivers/net/ethernet/amd/xgbe/xgbe.h +++ b/drivers/net/ethernet/amd/xgbe/xgbe.h @@ -140,11 +140,25 @@ #define XGBE_TX_MAX_BUF_SIZE (0x3fff & ~(64 - 1)) +/* Descriptors required for maximum contigous TSO/GSO packet */ +#define XGBE_TX_MAX_SPLIT ((GSO_MAX_SIZE / XGBE_TX_MAX_BUF_SIZE) + 1) + +/* Maximum possible descriptors needed for an SKB: + * - Maximum number of SKB frags + * - Maximum descriptors for contiguous TSO/GSO packet + * - Possible context descriptor + * - Possible TSO header descriptor + */ +#define XGBE_TX_MAX_DESCS (MAX_SKB_FRAGS + XGBE_TX_MAX_SPLIT + 2) + #define XGBE_RX_MIN_BUF_SIZE (ETH_FRAME_LEN + ETH_FCS_LEN + VLAN_HLEN) #define XGBE_RX_BUF_ALIGN 64 +#define XGBE_SKB_ALLOC_SIZE 256 +#define XGBE_SPH_HDSMS_SIZE 2 /* Keep in sync with SKB_ALLOC_SIZE */ #define XGBE_MAX_DMA_CHANNELS 16 #define XGBE_MAX_QUEUES 16 +#define XGBE_DMA_STOP_TIMEOUT 5 /* DMA cache settings - Outer sharable, write-back, write-allocate */ #define XGBE_DMA_OS_AXDOMAIN 0x2 @@ -171,6 +185,7 @@ /* Device-tree clock names */ #define XGBE_DMA_CLOCK "dma_clk" #define XGBE_PTP_CLOCK "ptp_clk" +#define XGBE_DMA_IRQS "amd,per-channel-interrupt" /* Timestamp support - values based on 50MHz PTP clock * 50MHz => 20 nsec @@ -212,9 +227,17 @@ /* Maximum MAC address hash table size (256 bits = 8 bytes) */ #define XGBE_MAC_HASH_TABLE_SIZE 8 +/* Receive Side Scaling */ +#define XGBE_RSS_HASH_KEY_SIZE 40 +#define XGBE_RSS_MAX_TABLE_SIZE 256 +#define XGBE_RSS_LOOKUP_TABLE_TYPE 0 +#define XGBE_RSS_HASH_KEY_TYPE 1 + struct xgbe_prv_data; struct xgbe_packet_data { + struct sk_buff *skb; + unsigned int attributes; unsigned int errors; @@ -230,14 +253,53 @@ struct xgbe_packet_data { unsigned short vlan_ctag; u64 rx_tstamp; + + u32 rss_hash; + enum pkt_hash_types rss_hash_type; + + unsigned int tx_packets; + unsigned int tx_bytes; }; /* Common Rx and Tx descriptor mapping */ struct xgbe_ring_desc { - unsigned int desc0; - unsigned int desc1; - unsigned int desc2; - unsigned int desc3; + __le32 desc0; + __le32 desc1; + __le32 desc2; + __le32 desc3; +}; + +/* Page allocation related values */ +struct xgbe_page_alloc { + struct page *pages; + unsigned int pages_len; + unsigned int pages_offset; + + dma_addr_t pages_dma; +}; + +/* Ring entry buffer data */ +struct xgbe_buffer_data { + struct xgbe_page_alloc pa; + struct xgbe_page_alloc pa_unmap; + + dma_addr_t dma; + unsigned int dma_len; +}; + +/* Tx-related ring data */ +struct xgbe_tx_ring_data { + unsigned int packets; /* BQL packet count */ + unsigned int bytes; /* BQL byte count */ +}; + +/* Rx-related ring data */ +struct xgbe_rx_ring_data { + struct xgbe_buffer_data hdr; /* Header locations */ + struct xgbe_buffer_data buf; /* Payload locations */ + + unsigned short hdr_len; /* Length of received header */ + unsigned short len; /* Length of received packet */ }; /* Structure used to hold information related to the descriptor @@ -251,9 +313,9 @@ struct xgbe_ring_data { struct sk_buff *skb; /* Virtual address of SKB */ dma_addr_t skb_dma; /* DMA address of SKB data */ unsigned int skb_dma_len; /* Length of SKB DMA area */ - unsigned int tso_header; /* TSO header indicator */ - unsigned short len; /* Length of received Rx packet */ + struct xgbe_tx_ring_data tx; /* Tx-related data */ + struct xgbe_rx_ring_data rx; /* Rx-related data */ unsigned int interrupt; /* Interrupt indicator */ @@ -291,6 +353,10 @@ struct xgbe_ring { */ struct xgbe_ring_data *rdata; + /* Page allocation for RX buffers */ + struct xgbe_page_alloc rx_hdr_pa; + struct xgbe_page_alloc rx_buf_pa; + /* Ring index values * cur - Tx: index of descriptor to be used for current transfer * Rx: index of descriptor to check for packet availability @@ -307,6 +373,7 @@ struct xgbe_ring { union { struct { unsigned int queue_stopped; + unsigned int xmit_more; unsigned short cur_mss; unsigned short cur_vlan_ctag; } tx; @@ -331,6 +398,13 @@ struct xgbe_channel { unsigned int queue_index; void __iomem *dma_regs; + /* Per channel interrupt irq number */ + int dma_irq; + char dma_irq_name[IFNAMSIZ + 32]; + + /* Netdev related settings */ + struct napi_struct napi; + unsigned int saved_ier; unsigned int tx_timer_active; @@ -456,7 +530,7 @@ struct xgbe_hw_if { int (*enable_int)(struct xgbe_channel *, enum xgbe_int); int (*disable_int)(struct xgbe_channel *, enum xgbe_int); - void (*pre_xmit)(struct xgbe_channel *); + void (*dev_xmit)(struct xgbe_channel *); int (*dev_read)(struct xgbe_channel *); void (*tx_desc_init)(struct xgbe_channel *); void (*rx_desc_init)(struct xgbe_channel *); @@ -464,6 +538,7 @@ struct xgbe_hw_if { void (*tx_desc_reset)(struct xgbe_ring_data *); int (*is_last_desc)(struct xgbe_ring_desc *); int (*is_context_desc)(struct xgbe_ring_desc *); + void (*tx_start_xmit)(struct xgbe_channel *, struct xgbe_ring *); /* For FLOW ctrl */ int (*config_tx_flow_control)(struct xgbe_prv_data *); @@ -509,14 +584,20 @@ struct xgbe_hw_if { /* For Data Center Bridging config */ void (*config_dcb_tc)(struct xgbe_prv_data *); void (*config_dcb_pfc)(struct xgbe_prv_data *); + + /* For Receive Side Scaling */ + int (*enable_rss)(struct xgbe_prv_data *); + int (*disable_rss)(struct xgbe_prv_data *); + int (*set_rss_hash_key)(struct xgbe_prv_data *, const u8 *); + int (*set_rss_lookup_table)(struct xgbe_prv_data *, const u32 *); }; struct xgbe_desc_if { int (*alloc_ring_resources)(struct xgbe_prv_data *); void (*free_ring_resources)(struct xgbe_prv_data *); int (*map_tx_skb)(struct xgbe_channel *, struct sk_buff *); - void (*realloc_skb)(struct xgbe_channel *); - void (*unmap_skb)(struct xgbe_prv_data *, struct xgbe_ring_data *); + void (*realloc_rx_buffer)(struct xgbe_channel *); + void (*unmap_rdata)(struct xgbe_prv_data *, struct xgbe_ring_data *); void (*wrapper_tx_desc_init)(struct xgbe_prv_data *); void (*wrapper_rx_desc_init)(struct xgbe_prv_data *); }; @@ -581,7 +662,11 @@ struct xgbe_prv_data { /* XPCS indirect addressing mutex */ struct mutex xpcs_mutex; - int irq_number; + /* RSS addressing mutex */ + struct mutex rss_mutex; + + int dev_irq; + unsigned int per_channel_irq; struct xgbe_hw_if hw_if; struct xgbe_desc_if desc_if; @@ -624,7 +709,7 @@ struct xgbe_prv_data { unsigned int rx_riwt; unsigned int rx_frames; - /* Current MTU */ + /* Current Rx buffer size */ unsigned int rx_buf_size; /* Flow control settings */ @@ -632,6 +717,11 @@ struct xgbe_prv_data { unsigned int tx_pause; unsigned int rx_pause; + /* Receive Side Scaling settings */ + u8 rss_key[XGBE_RSS_HASH_KEY_SIZE]; + u32 rss_table[XGBE_RSS_MAX_TABLE_SIZE]; + u32 rss_options; + /* MDIO settings */ struct module *phy_module; char *mii_bus_id; |