summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c2
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h35
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en.h28
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_main.c67
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rep.c39
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_rx.c154
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_stats.c6
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_stats.h1
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/en_tc.c29
-rw-r--r--drivers/net/ethernet/mellanox/mlx5/core/fs_core.c2
10 files changed, 211 insertions, 152 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
index 424457ff9759..8ecac81a385d 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.c
@@ -258,6 +258,8 @@ const char *parse_fs_dst(struct trace_seq *p,
return ret;
}
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_add_ft);
+EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_del_ft);
EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_add_fg);
EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_del_fg);
EXPORT_TRACEPOINT_SYMBOL(mlx5_fs_set_fte);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h
index d027ce00c8ce..a4cf123e3f17 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/diag/fs_tracepoint.h
@@ -61,6 +61,41 @@ const char *parse_fs_dst(struct trace_seq *p,
const struct mlx5_flow_destination *dst,
u32 counter_id);
+TRACE_EVENT(mlx5_fs_add_ft,
+ TP_PROTO(const struct mlx5_flow_table *ft),
+ TP_ARGS(ft),
+ TP_STRUCT__entry(
+ __field(const struct mlx5_flow_table *, ft)
+ __field(u32, id)
+ __field(u32, level)
+ __field(u32, type)
+ ),
+ TP_fast_assign(
+ __entry->ft = ft;
+ __entry->id = ft->id;
+ __entry->level = ft->level;
+ __entry->type = ft->type;
+ ),
+ TP_printk("ft=%p id=%u level=%u type=%u \n",
+ __entry->ft, __entry->id, __entry->level, __entry->type)
+ );
+
+TRACE_EVENT(mlx5_fs_del_ft,
+ TP_PROTO(const struct mlx5_flow_table *ft),
+ TP_ARGS(ft),
+ TP_STRUCT__entry(
+ __field(const struct mlx5_flow_table *, ft)
+ __field(u32, id)
+ ),
+ TP_fast_assign(
+ __entry->ft = ft;
+ __entry->id = ft->id;
+
+ ),
+ TP_printk("ft=%p id=%u\n",
+ __entry->ft, __entry->id)
+ );
+
TRACE_EVENT(mlx5_fs_add_fg,
TP_PROTO(const struct mlx5_flow_group *fg),
TP_ARGS(fg),
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h
index 8fa8fdd30b85..6dd74ef69389 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h
@@ -76,15 +76,14 @@ struct page_pool;
#define MLX5_SKB_FRAG_SZ(len) (SKB_DATA_ALIGN(len) + \
SKB_DATA_ALIGN(sizeof(struct skb_shared_info)))
+#define MLX5E_RX_MAX_HEAD (256)
+
#define MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev) \
(6 + MLX5_CAP_GEN(mdev, cache_line_128byte)) /* HW restriction */
#define MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, req) \
max_t(u32, MLX5_MPWRQ_MIN_LOG_STRIDE_SZ(mdev), req)
-#define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, 6)
-#define MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(mdev) MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, 8)
-#define MLX5E_MPWQE_STRIDE_SZ(mdev, cqe_cmprs) \
- (cqe_cmprs ? MLX5_MPWRQ_CQE_CMPRS_LOG_STRIDE_SZ(mdev) : \
- MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev))
+#define MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev) \
+ MLX5_MPWRQ_LOG_STRIDE_SZ(mdev, order_base_2(MLX5E_RX_MAX_HEAD))
#define MLX5_MPWRQ_LOG_WQE_SZ 18
#define MLX5_MPWRQ_WQE_PAGE_ORDER (MLX5_MPWRQ_LOG_WQE_SZ - PAGE_SHIFT > 0 ? \
@@ -119,8 +118,6 @@ struct page_pool;
#define MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE_MPW 0x2
-#define MLX5E_RX_MAX_HEAD (256)
-
#define MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ (64 * 1024)
#define MLX5E_DEFAULT_LRO_TIMEOUT 32
#define MLX5E_LRO_TIMEOUT_ARR_SIZE 4
@@ -309,16 +306,18 @@ struct mlx5e_cq {
struct mlx5_core_cq mcq;
struct mlx5e_channel *channel;
+ /* control */
+ struct mlx5_core_dev *mdev;
+ struct mlx5_wq_ctrl wq_ctrl;
+} ____cacheline_aligned_in_smp;
+
+struct mlx5e_cq_decomp {
/* cqe decompression */
struct mlx5_cqe64 title;
struct mlx5_mini_cqe8 mini_arr[MLX5_MINI_CQE_ARRAY_SIZE];
u8 mini_arr_idx;
- u16 decmprs_left;
- u16 decmprs_wqe_counter;
-
- /* control */
- struct mlx5_core_dev *mdev;
- struct mlx5_wq_ctrl wq_ctrl;
+ u16 left;
+ u16 wqe_counter;
} ____cacheline_aligned_in_smp;
struct mlx5e_tx_wqe_info {
@@ -581,6 +580,7 @@ struct mlx5e_rq {
struct net_device *netdev;
struct mlx5e_rq_stats *stats;
struct mlx5e_cq cq;
+ struct mlx5e_cq_decomp cqd;
struct mlx5e_page_cache page_cache;
struct hwtstamp_config *tstamp;
struct mlx5_clock *clock;
@@ -638,6 +638,7 @@ struct mlx5e_channel {
struct hwtstamp_config *tstamp;
int ix;
int cpu;
+ cpumask_var_t xps_cpumask;
};
struct mlx5e_channels {
@@ -803,6 +804,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
void mlx5e_update_stats(struct mlx5e_priv *priv);
void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats);
+void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s);
void mlx5e_init_l2_addr(struct mlx5e_priv *priv);
int mlx5e_self_test_num(struct mlx5e_priv *priv);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
index 8cfd2ec7c0a2..17b6babbed8a 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c
@@ -171,8 +171,7 @@ static u8 mlx5e_mpwqe_get_log_stride_size(struct mlx5_core_dev *mdev,
if (mlx5e_rx_mpwqe_is_linear_skb(mdev, params))
return order_base_2(mlx5e_rx_get_linear_frag_sz(params));
- return MLX5E_MPWQE_STRIDE_SZ(mdev,
- MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS));
+ return MLX5_MPWRQ_DEF_LOG_STRIDE_SZ(mdev);
}
static u8 mlx5e_mpwqe_get_log_num_strides(struct mlx5_core_dev *mdev,
@@ -1950,6 +1949,29 @@ static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate)
return err;
}
+static int mlx5e_alloc_xps_cpumask(struct mlx5e_channel *c,
+ struct mlx5e_params *params)
+{
+ int num_comp_vectors = mlx5_comp_vectors_count(c->mdev);
+ int irq;
+
+ if (!zalloc_cpumask_var(&c->xps_cpumask, GFP_KERNEL))
+ return -ENOMEM;
+
+ for (irq = c->ix; irq < num_comp_vectors; irq += params->num_channels) {
+ int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(c->mdev, irq));
+
+ cpumask_set_cpu(cpu, c->xps_cpumask);
+ }
+
+ return 0;
+}
+
+static void mlx5e_free_xps_cpumask(struct mlx5e_channel *c)
+{
+ free_cpumask_var(c->xps_cpumask);
+}
+
static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
struct mlx5e_params *params,
struct mlx5e_channel_param *cparam,
@@ -1982,9 +2004,12 @@ static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix,
c->num_tc = params->num_tc;
c->xdp = !!params->xdp_prog;
c->stats = &priv->channel_stats[ix].ch;
-
c->irq_desc = irq_to_desc(irq);
+ err = mlx5e_alloc_xps_cpumask(c, params);
+ if (err)
+ goto err_free_channel;
+
netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64);
err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq);
@@ -2067,6 +2092,9 @@ err_close_icosq_cq:
err_napi_del:
netif_napi_del(&c->napi);
+ mlx5e_free_xps_cpumask(c);
+
+err_free_channel:
kvfree(c);
return err;
@@ -2079,7 +2107,7 @@ static void mlx5e_activate_channel(struct mlx5e_channel *c)
for (tc = 0; tc < c->num_tc; tc++)
mlx5e_activate_txqsq(&c->sq[tc]);
mlx5e_activate_rq(&c->rq);
- netif_set_xps_queue(c->netdev, get_cpu_mask(c->cpu), c->ix);
+ netif_set_xps_queue(c->netdev, c->xps_cpumask, c->ix);
}
static void mlx5e_deactivate_channel(struct mlx5e_channel *c)
@@ -2107,6 +2135,7 @@ static void mlx5e_close_channel(struct mlx5e_channel *c)
mlx5e_close_tx_cqs(c);
mlx5e_close_cq(&c->icosq.cq);
netif_napi_del(&c->napi);
+ mlx5e_free_xps_cpumask(c);
kvfree(c);
}
@@ -3492,11 +3521,32 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type,
}
}
+void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s)
+{
+ int i;
+
+ for (i = 0; i < mlx5e_get_netdev_max_channels(priv->netdev); i++) {
+ struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i];
+ struct mlx5e_rq_stats *rq_stats = &channel_stats->rq;
+ int j;
+
+ s->rx_packets += rq_stats->packets;
+ s->rx_bytes += rq_stats->bytes;
+
+ for (j = 0; j < priv->max_opened_tc; j++) {
+ struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j];
+
+ s->tx_packets += sq_stats->packets;
+ s->tx_bytes += sq_stats->bytes;
+ s->tx_dropped += sq_stats->dropped;
+ }
+ }
+}
+
void
mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
{
struct mlx5e_priv *priv = netdev_priv(dev);
- struct mlx5e_sw_stats *sstats = &priv->stats.sw;
struct mlx5e_vport_stats *vstats = &priv->stats.vport;
struct mlx5e_pport_stats *pstats = &priv->stats.pport;
@@ -3511,12 +3561,7 @@ mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats)
stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok);
stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok);
} else {
- mlx5e_grp_sw_update_stats(priv);
- stats->rx_packets = sstats->rx_packets;
- stats->rx_bytes = sstats->rx_bytes;
- stats->tx_packets = sstats->tx_packets;
- stats->tx_bytes = sstats->tx_bytes;
- stats->tx_dropped = sstats->tx_queue_dropped;
+ mlx5e_fold_sw_stats64(priv, stats);
}
stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer;
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
index 04736212a21c..edb34b397c53 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c
@@ -162,27 +162,16 @@ static void mlx5e_rep_update_hw_counters(struct mlx5e_priv *priv)
static void mlx5e_rep_update_sw_counters(struct mlx5e_priv *priv)
{
struct mlx5e_sw_stats *s = &priv->stats.sw;
- struct mlx5e_rq_stats *rq_stats;
- struct mlx5e_sq_stats *sq_stats;
- int i, j;
+ struct rtnl_link_stats64 stats64 = {};
memset(s, 0, sizeof(*s));
- for (i = 0; i < priv->channels.num; i++) {
- struct mlx5e_channel *c = priv->channels.c[i];
-
- rq_stats = c->rq.stats;
-
- s->rx_packets += rq_stats->packets;
- s->rx_bytes += rq_stats->bytes;
+ mlx5e_fold_sw_stats64(priv, &stats64);
- for (j = 0; j < priv->channels.params.num_tc; j++) {
- sq_stats = c->sq[j].stats;
-
- s->tx_packets += sq_stats->packets;
- s->tx_bytes += sq_stats->bytes;
- s->tx_queue_dropped += sq_stats->dropped;
- }
- }
+ s->rx_packets = stats64.rx_packets;
+ s->rx_bytes = stats64.rx_bytes;
+ s->tx_packets = stats64.tx_packets;
+ s->tx_bytes = stats64.tx_bytes;
+ s->tx_queue_dropped = stats64.tx_dropped;
}
static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
@@ -195,8 +184,7 @@ static void mlx5e_rep_get_ethtool_stats(struct net_device *dev,
return;
mutex_lock(&priv->state_lock);
- if (test_bit(MLX5E_STATE_OPENED, &priv->state))
- mlx5e_rep_update_sw_counters(priv);
+ mlx5e_rep_update_sw_counters(priv);
mlx5e_rep_update_hw_counters(priv);
mutex_unlock(&priv->state_lock);
@@ -1229,17 +1217,8 @@ mlx5e_get_sw_stats64(const struct net_device *dev,
struct rtnl_link_stats64 *stats)
{
struct mlx5e_priv *priv = netdev_priv(dev);
- struct mlx5e_sw_stats *sstats = &priv->stats.sw;
-
- mlx5e_rep_update_sw_counters(priv);
-
- stats->rx_packets = sstats->rx_packets;
- stats->rx_bytes = sstats->rx_bytes;
- stats->tx_packets = sstats->tx_packets;
- stats->tx_bytes = sstats->tx_bytes;
-
- stats->tx_dropped = sstats->tx_queue_dropped;
+ mlx5e_fold_sw_stats64(priv, stats);
return 0;
}
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
index f86e4804e83e..be396e5e4e39 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rx.c
@@ -52,40 +52,45 @@ static inline bool mlx5e_rx_hw_stamp(struct hwtstamp_config *config)
return config->rx_filter == HWTSTAMP_FILTER_ALL;
}
-static inline void mlx5e_read_cqe_slot(struct mlx5e_cq *cq, u32 cqcc,
- void *data)
+static inline void mlx5e_read_cqe_slot(struct mlx5_cqwq *wq,
+ u32 cqcc, void *data)
{
- u32 ci = mlx5_cqwq_ctr2ix(&cq->wq, cqcc);
+ u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
- memcpy(data, mlx5_cqwq_get_wqe(&cq->wq, ci), sizeof(struct mlx5_cqe64));
+ memcpy(data, mlx5_cqwq_get_wqe(wq, ci), sizeof(struct mlx5_cqe64));
}
static inline void mlx5e_read_title_slot(struct mlx5e_rq *rq,
- struct mlx5e_cq *cq, u32 cqcc)
+ struct mlx5_cqwq *wq,
+ u32 cqcc)
{
- mlx5e_read_cqe_slot(cq, cqcc, &cq->title);
- cq->decmprs_left = be32_to_cpu(cq->title.byte_cnt);
- cq->decmprs_wqe_counter = be16_to_cpu(cq->title.wqe_counter);
+ struct mlx5e_cq_decomp *cqd = &rq->cqd;
+ struct mlx5_cqe64 *title = &cqd->title;
+
+ mlx5e_read_cqe_slot(wq, cqcc, title);
+ cqd->left = be32_to_cpu(title->byte_cnt);
+ cqd->wqe_counter = be16_to_cpu(title->wqe_counter);
rq->stats->cqe_compress_blks++;
}
-static inline void mlx5e_read_mini_arr_slot(struct mlx5e_cq *cq, u32 cqcc)
+static inline void mlx5e_read_mini_arr_slot(struct mlx5_cqwq *wq,
+ struct mlx5e_cq_decomp *cqd,
+ u32 cqcc)
{
- mlx5e_read_cqe_slot(cq, cqcc, cq->mini_arr);
- cq->mini_arr_idx = 0;
+ mlx5e_read_cqe_slot(wq, cqcc, cqd->mini_arr);
+ cqd->mini_arr_idx = 0;
}
-static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)
+static inline void mlx5e_cqes_update_owner(struct mlx5_cqwq *wq, int n)
{
- struct mlx5_cqwq *wq = &cq->wq;
-
+ u32 cqcc = wq->cc;
u8 op_own = mlx5_cqwq_get_ctr_wrap_cnt(wq, cqcc) & 1;
u32 ci = mlx5_cqwq_ctr2ix(wq, cqcc);
u32 wq_sz = mlx5_cqwq_get_size(wq);
u32 ci_top = min_t(u32, wq_sz, ci + n);
for (; ci < ci_top; ci++, n--) {
- struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, ci);
+ struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
cqe->op_own = op_own;
}
@@ -93,7 +98,7 @@ static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)
if (unlikely(ci == wq_sz)) {
op_own = !op_own;
for (ci = 0; ci < n; ci++) {
- struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, ci);
+ struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(wq, ci);
cqe->op_own = op_own;
}
@@ -101,68 +106,79 @@ static inline void mlx5e_cqes_update_owner(struct mlx5e_cq *cq, u32 cqcc, int n)
}
static inline void mlx5e_decompress_cqe(struct mlx5e_rq *rq,
- struct mlx5e_cq *cq, u32 cqcc)
+ struct mlx5_cqwq *wq,
+ u32 cqcc)
{
- cq->title.byte_cnt = cq->mini_arr[cq->mini_arr_idx].byte_cnt;
- cq->title.check_sum = cq->mini_arr[cq->mini_arr_idx].checksum;
- cq->title.op_own &= 0xf0;
- cq->title.op_own |= 0x01 & (cqcc >> cq->wq.fbc.log_sz);
- cq->title.wqe_counter = cpu_to_be16(cq->decmprs_wqe_counter);
+ struct mlx5e_cq_decomp *cqd = &rq->cqd;
+ struct mlx5_mini_cqe8 *mini_cqe = &cqd->mini_arr[cqd->mini_arr_idx];
+ struct mlx5_cqe64 *title = &cqd->title;
+
+ title->byte_cnt = mini_cqe->byte_cnt;
+ title->check_sum = mini_cqe->checksum;
+ title->op_own &= 0xf0;
+ title->op_own |= 0x01 & (cqcc >> wq->fbc.log_sz);
+ title->wqe_counter = cpu_to_be16(cqd->wqe_counter);
if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ)
- cq->decmprs_wqe_counter +=
- mpwrq_get_cqe_consumed_strides(&cq->title);
+ cqd->wqe_counter += mpwrq_get_cqe_consumed_strides(title);
else
- cq->decmprs_wqe_counter =
- mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, cq->decmprs_wqe_counter + 1);
+ cqd->wqe_counter =
+ mlx5_wq_cyc_ctr2ix(&rq->wqe.wq, cqd->wqe_counter + 1);
}
static inline void mlx5e_decompress_cqe_no_hash(struct mlx5e_rq *rq,
- struct mlx5e_cq *cq, u32 cqcc)
+ struct mlx5_cqwq *wq,
+ u32 cqcc)
{
- mlx5e_decompress_cqe(rq, cq, cqcc);
- cq->title.rss_hash_type = 0;
- cq->title.rss_hash_result = 0;
+ struct mlx5e_cq_decomp *cqd = &rq->cqd;
+
+ mlx5e_decompress_cqe(rq, wq, cqcc);
+ cqd->title.rss_hash_type = 0;
+ cqd->title.rss_hash_result = 0;
}
static inline u32 mlx5e_decompress_cqes_cont(struct mlx5e_rq *rq,
- struct mlx5e_cq *cq,
+ struct mlx5_cqwq *wq,
int update_owner_only,
int budget_rem)
{
- u32 cqcc = cq->wq.cc + update_owner_only;
+ struct mlx5e_cq_decomp *cqd = &rq->cqd;
+ u32 cqcc = wq->cc + update_owner_only;
u32 cqe_count;
u32 i;
- cqe_count = min_t(u32, cq->decmprs_left, budget_rem);
+ cqe_count = min_t(u32, cqd->left, budget_rem);
for (i = update_owner_only; i < cqe_count;
- i++, cq->mini_arr_idx++, cqcc++) {
- if (cq->mini_arr_idx == MLX5_MINI_CQE_ARRAY_SIZE)
- mlx5e_read_mini_arr_slot(cq, cqcc);
+ i++, cqd->mini_arr_idx++, cqcc++) {
+ if (cqd->mini_arr_idx == MLX5_MINI_CQE_ARRAY_SIZE)
+ mlx5e_read_mini_arr_slot(wq, cqd, cqcc);
- mlx5e_decompress_cqe_no_hash(rq, cq, cqcc);
- rq->handle_rx_cqe(rq, &cq->title);
+ mlx5e_decompress_cqe_no_hash(rq, wq, cqcc);
+ rq->handle_rx_cqe(rq, &cqd->title);
}
- mlx5e_cqes_update_owner(cq, cq->wq.cc, cqcc - cq->wq.cc);
- cq->wq.cc = cqcc;
- cq->decmprs_left -= cqe_count;
+ mlx5e_cqes_update_owner(wq, cqcc - wq->cc);
+ wq->cc = cqcc;
+ cqd->left -= cqe_count;
rq->stats->cqe_compress_pkts += cqe_count;
return cqe_count;
}
static inline u32 mlx5e_decompress_cqes_start(struct mlx5e_rq *rq,
- struct mlx5e_cq *cq,
+ struct mlx5_cqwq *wq,
int budget_rem)
{
- mlx5e_read_title_slot(rq, cq, cq->wq.cc);
- mlx5e_read_mini_arr_slot(cq, cq->wq.cc + 1);
- mlx5e_decompress_cqe(rq, cq, cq->wq.cc);
- rq->handle_rx_cqe(rq, &cq->title);
- cq->mini_arr_idx++;
+ struct mlx5e_cq_decomp *cqd = &rq->cqd;
+ u32 cc = wq->cc;
- return mlx5e_decompress_cqes_cont(rq, cq, 1, budget_rem) - 1;
+ mlx5e_read_title_slot(rq, wq, cc);
+ mlx5e_read_mini_arr_slot(wq, cqd, cc + 1);
+ mlx5e_decompress_cqe(rq, wq, cc);
+ rq->handle_rx_cqe(rq, &cqd->title);
+ cqd->mini_arr_idx++;
+
+ return mlx5e_decompress_cqes_cont(rq, wq, 1, budget_rem) - 1;
}
static inline bool mlx5e_page_is_reserved(struct page *page)
@@ -369,7 +385,7 @@ mlx5e_add_skb_frag(struct mlx5e_rq *rq, struct sk_buff *skb,
static inline void
mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
struct mlx5e_dma_info *dma_info,
- int offset_from, int offset_to, u32 headlen)
+ int offset_from, u32 headlen)
{
const void *from = page_address(dma_info->page) + offset_from;
/* Aligning len to sizeof(long) optimizes memcpy performance */
@@ -377,24 +393,7 @@ mlx5e_copy_skb_header(struct device *pdev, struct sk_buff *skb,
dma_sync_single_for_cpu(pdev, dma_info->addr + offset_from, len,
DMA_FROM_DEVICE);
- skb_copy_to_linear_data_offset(skb, offset_to, from, len);
-}
-
-static inline void
-mlx5e_copy_skb_header_mpwqe(struct device *pdev,
- struct sk_buff *skb,
- struct mlx5e_dma_info *dma_info,
- u32 offset, u32 headlen)
-{
- u16 headlen_pg = min_t(u32, headlen, PAGE_SIZE - offset);
-
- mlx5e_copy_skb_header(pdev, skb, dma_info, offset, 0, headlen_pg);
-
- if (unlikely(offset + headlen > PAGE_SIZE)) {
- dma_info++;
- mlx5e_copy_skb_header(pdev, skb, dma_info, 0, headlen_pg,
- headlen - headlen_pg);
- }
+ skb_copy_to_linear_data(skb, from, len);
}
static void
@@ -973,8 +972,7 @@ mlx5e_skb_from_cqe_nonlinear(struct mlx5e_rq *rq, struct mlx5_cqe64 *cqe,
}
/* copy header */
- mlx5e_copy_skb_header(rq->pdev, skb, head_wi->di, head_wi->offset,
- 0, headlen);
+ mlx5e_copy_skb_header(rq->pdev, skb, head_wi->di, head_wi->offset, headlen);
/* skb linear part was allocated with headlen and aligned to long */
skb->tail += headlen;
skb->len += headlen;
@@ -1096,8 +1094,7 @@ mlx5e_skb_from_cqe_mpwrq_nonlinear(struct mlx5e_rq *rq, struct mlx5e_mpw_info *w
di++;
}
/* copy header */
- mlx5e_copy_skb_header_mpwqe(rq->pdev, skb, head_di,
- head_offset, headlen);
+ mlx5e_copy_skb_header(rq->pdev, skb, head_di, head_offset, headlen);
/* skb linear part was allocated with headlen and aligned to long */
skb->tail += headlen;
skb->len += headlen;
@@ -1203,16 +1200,17 @@ mpwrq_cqe_out:
int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
{
struct mlx5e_rq *rq = container_of(cq, struct mlx5e_rq, cq);
+ struct mlx5_cqwq *cqwq = &cq->wq;
struct mlx5_cqe64 *cqe;
int work_done = 0;
if (unlikely(!test_bit(MLX5E_RQ_STATE_ENABLED, &rq->state)))
return 0;
- if (cq->decmprs_left)
- work_done += mlx5e_decompress_cqes_cont(rq, cq, 0, budget);
+ if (rq->cqd.left)
+ work_done += mlx5e_decompress_cqes_cont(rq, cqwq, 0, budget);
- cqe = mlx5_cqwq_get_cqe(&cq->wq);
+ cqe = mlx5_cqwq_get_cqe(cqwq);
if (!cqe) {
if (unlikely(work_done))
goto out;
@@ -1222,21 +1220,21 @@ int mlx5e_poll_rx_cq(struct mlx5e_cq *cq, int budget)
do {
if (mlx5_get_cqe_format(cqe) == MLX5_COMPRESSED) {
work_done +=
- mlx5e_decompress_cqes_start(rq, cq,
+ mlx5e_decompress_cqes_start(rq, cqwq,
budget - work_done);
continue;
}
- mlx5_cqwq_pop(&cq->wq);
+ mlx5_cqwq_pop(cqwq);
rq->handle_rx_cqe(rq, cqe);
- } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(&cq->wq)));
+ } while ((++work_done < budget) && (cqe = mlx5_cqwq_get_cqe(cqwq)));
out:
if (rq->xdp_prog)
mlx5e_xdp_rx_poll_complete(rq);
- mlx5_cqwq_update_db_record(&cq->wq);
+ mlx5_cqwq_update_db_record(cqwq);
/* ensure cq space is freed before enabling more cqes */
wmb();
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
index d3fe48ff9da9..1a78e05cbba8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c
@@ -127,9 +127,9 @@ static int mlx5e_grp_sw_fill_stats(struct mlx5e_priv *priv, u64 *data, int idx)
return idx;
}
-void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
+static void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
{
- struct mlx5e_sw_stats temp, *s = &temp;
+ struct mlx5e_sw_stats *s = &priv->stats.sw;
int i;
memset(s, 0, sizeof(*s));
@@ -212,8 +212,6 @@ void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv)
s->tx_cqes += sq_stats->cqes;
}
}
-
- memcpy(&priv->stats.sw, s, sizeof(*s));
}
static const struct counter_desc q_stats_desc[] = {
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
index fe91ec06e3c7..4640d4f986f8 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h
@@ -277,7 +277,6 @@ struct mlx5e_stats_grp {
extern const struct mlx5e_stats_grp mlx5e_stats_grps[];
extern const int mlx5e_num_stats_grps;
-void mlx5e_grp_sw_update_stats(struct mlx5e_priv *priv);
void mlx5e_grp_802_3_update_stats(struct mlx5e_priv *priv);
#endif /* __MLX5_EN_STATS_H__ */
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
index cae6c6d48984..74159d39dd66 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c
@@ -2767,14 +2767,13 @@ err_free:
return err;
}
-static int
+static struct mlx5e_tc_flow *
__mlx5e_add_fdb_flow(struct mlx5e_priv *priv,
struct tc_cls_flower_offload *f,
u16 flow_flags,
struct net_device *filter_dev,
struct mlx5_eswitch_rep *in_rep,
- struct mlx5_core_dev *in_mdev,
- struct mlx5e_tc_flow **__flow)
+ struct mlx5_core_dev *in_mdev)
{
struct netlink_ext_ack *extack = f->common.extack;
struct mlx5_eswitch *esw = priv->mdev->priv.eswitch;
@@ -2814,15 +2813,13 @@ __mlx5e_add_fdb_flow(struct mlx5e_priv *priv,
if (err)
goto err_free;
- *__flow = flow;
-
- return 0;
+ return flow;
err_free:
kfree(flow);
kvfree(parse_attr);
out:
- return err;
+ return ERR_PTR(err);
}
static int mlx5e_tc_add_fdb_peer_flow(struct tc_cls_flower_offload *f,
@@ -2855,11 +2852,13 @@ static int mlx5e_tc_add_fdb_peer_flow(struct tc_cls_flower_offload *f,
in_mdev = priv->mdev;
parse_attr = flow->esw_attr->parse_attr;
- err = __mlx5e_add_fdb_flow(peer_priv, f, flow->flags,
- parse_attr->filter_dev,
- flow->esw_attr->in_rep, in_mdev, &peer_flow);
- if (err)
+ peer_flow = __mlx5e_add_fdb_flow(peer_priv, f, flow->flags,
+ parse_attr->filter_dev,
+ flow->esw_attr->in_rep, in_mdev);
+ if (IS_ERR(peer_flow)) {
+ err = PTR_ERR(peer_flow);
goto out;
+ }
flow->peer_flow = peer_flow;
flow->flags |= MLX5E_TC_FLOW_DUP;
@@ -2885,10 +2884,10 @@ mlx5e_add_fdb_flow(struct mlx5e_priv *priv,
struct mlx5e_tc_flow *flow;
int err;
- err = __mlx5e_add_fdb_flow(priv, f, flow_flags, filter_dev, in_rep,
- in_mdev, &flow);
- if (err)
- goto out;
+ flow = __mlx5e_add_fdb_flow(priv, f, flow_flags, filter_dev, in_rep,
+ in_mdev);
+ if (IS_ERR(flow))
+ return PTR_ERR(flow);
if (is_peer_flow_needed(flow)) {
err = mlx5e_tc_add_fdb_peer_flow(f, flow);
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
index 79f122b45def..b2df7c3749fb 100644
--- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
+++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c
@@ -397,6 +397,7 @@ static void del_hw_flow_table(struct fs_node *node)
fs_get_obj(ft, node);
dev = get_dev(&ft->node);
root = find_root(&ft->node);
+ trace_mlx5_fs_del_ft(ft);
if (node->active) {
err = root->cmds->destroy_flow_table(dev, ft);
@@ -1019,6 +1020,7 @@ static struct mlx5_flow_table *__mlx5_create_flow_table(struct mlx5_flow_namespa
fs_prio->num_ft++;
up_write_ref_node(&fs_prio->node);
mutex_unlock(&root->chain_lock);
+ trace_mlx5_fs_add_ft(ft);
return ft;
destroy_ft:
root->cmds->destroy_flow_table(root->dev, ft);
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-13 12:15:13 +0000