diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5')
79 files changed, 5279 insertions, 1731 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/Makefile b/drivers/net/ethernet/mellanox/mlx5/core/Makefile index 9ea867a45764..a3773a8177ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/Makefile +++ b/drivers/net/ethernet/mellanox/mlx5/core/Makefile @@ -17,7 +17,7 @@ mlx5_core-y := main.o cmd.o debugfs.o fw.o eq.o uar.o pagealloc.o \ fs_counters.o fs_ft_pool.o rl.o lag/debugfs.o lag/lag.o dev.o events.o wq.o lib/gid.o \ lib/devcom.o lib/pci_vsc.o lib/dm.o lib/fs_ttc.o diag/fs_tracepoint.o \ diag/fw_tracer.o diag/crdump.o devlink.o diag/rsc_dump.o \ - fw_reset.o qos.o lib/tout.o + fw_reset.o qos.o lib/tout.o lib/aso.o # # Netdev basic @@ -28,7 +28,8 @@ mlx5_core-$(CONFIG_MLX5_CORE_EN) += en/rqt.o en/tir.o en/rss.o en/rx_res.o \ en_selftest.o en/port.o en/monitor_stats.o en/health.o \ en/reporter_tx.o en/reporter_rx.o en/params.o en/xsk/pool.o \ en/xsk/setup.o en/xsk/rx.o en/xsk/tx.o en/devlink.o en/ptp.o \ - en/qos.o en/trap.o en/fs_tt_redirect.o en/selq.o lib/crypto.o + en/qos.o en/htb.o en/trap.o en/fs_tt_redirect.o en/selq.o \ + lib/crypto.o # # Netdev extra @@ -45,7 +46,8 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en_tc.o en/rep/tc.o en/rep/neigh.o \ esw/indir_table.o en/tc_tun_encap.o \ en/tc_tun_vxlan.o en/tc_tun_gre.o en/tc_tun_geneve.o \ en/tc_tun_mplsoudp.o diag/en_tc_tracepoint.o \ - en/tc/post_act.o en/tc/int_port.o + en/tc/post_act.o en/tc/int_port.o en/tc/meter.o \ + en/tc/post_meter.o mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/act/trap.o \ en/tc/act/accept.o en/tc/act/mark.o en/tc/act/goto.o \ @@ -53,7 +55,7 @@ mlx5_core-$(CONFIG_MLX5_CLS_ACT) += en/tc/act/act.o en/tc/act/drop.o en/tc/a en/tc/act/vlan.o en/tc/act/vlan_mangle.o en/tc/act/mpls.o \ en/tc/act/mirred.o en/tc/act/mirred_nic.o \ en/tc/act/ct.o en/tc/act/sample.o en/tc/act/ptype.o \ - en/tc/act/redirect_ingress.o + en/tc/act/redirect_ingress.o en/tc/act/police.o ifneq ($(CONFIG_MLX5_TC_CT),) mlx5_core-y += en/tc_ct.o en/tc/ct_fs_dmfs.o @@ -67,7 +69,7 @@ mlx5_core-$(CONFIG_MLX5_TC_SAMPLE) += en/tc/sample.o # mlx5_core-$(CONFIG_MLX5_ESWITCH) += eswitch.o eswitch_offloads.o eswitch_offloads_termtbl.o \ ecpf.o rdma.o esw/legacy.o \ - esw/devlink_port.o esw/vporttbl.o esw/qos.o + esw/debugfs.o esw/devlink_port.o esw/vporttbl.o esw/qos.o mlx5_core-$(CONFIG_MLX5_ESWITCH) += esw/acl/helper.o \ esw/acl/egress_lgcy.o esw/acl/egress_ofld.o \ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c index 9caa1b52321b..3e232a65a0c3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/debugfs.c @@ -166,6 +166,28 @@ static const struct file_operations stats_fops = { .write = average_write, }; +static ssize_t slots_read(struct file *filp, char __user *buf, size_t count, + loff_t *pos) +{ + struct mlx5_cmd *cmd; + char tbuf[6]; + int weight; + int field; + int ret; + + cmd = filp->private_data; + weight = bitmap_weight(&cmd->bitmask, cmd->max_reg_cmds); + field = cmd->max_reg_cmds - weight; + ret = snprintf(tbuf, sizeof(tbuf), "%d\n", field); + return simple_read_from_buffer(buf, count, pos, tbuf, ret); +} + +static const struct file_operations slots_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = slots_read, +}; + void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) { struct mlx5_cmd_stats *stats; @@ -176,6 +198,8 @@ void mlx5_cmdif_debugfs_init(struct mlx5_core_dev *dev) cmd = &dev->priv.dbg.cmdif_debugfs; *cmd = debugfs_create_dir("commands", dev->priv.dbg.dbg_root); + debugfs_create_file("slots_inuse", 0400, *cmd, &dev->cmd, &slots_fops); + for (i = 0; i < MLX5_CMD_OP_MAX; i++) { stats = &dev->cmd.stats[i]; namep = mlx5_command_str(i); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/dev.c b/drivers/net/ethernet/mellanox/mlx5/core/dev.c index 50422b56a64d..0571e40c6ee5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/dev.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/dev.c @@ -340,8 +340,10 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) struct auxiliary_driver *adrv; int ret = 0, i; + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); priv->flags &= ~MLX5_PRIV_FLAGS_DETACH; + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = 0; i < ARRAY_SIZE(mlx5_adev_devices); i++) { if (!priv->adev[i]) { bool is_supported = false; @@ -389,6 +391,7 @@ int mlx5_attach_device(struct mlx5_core_dev *dev) break; } } + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; mutex_unlock(&mlx5_intf_mutex); return ret; } @@ -401,7 +404,9 @@ void mlx5_detach_device(struct mlx5_core_dev *dev) pm_message_t pm = {}; int i; + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; for (i = ARRAY_SIZE(mlx5_adev_devices) - 1; i >= 0; i--) { if (!priv->adev[i]) continue; @@ -430,6 +435,7 @@ skip_suspend: del_adev(&priv->adev[i]->adev); priv->adev[i] = NULL; } + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; priv->flags |= MLX5_PRIV_FLAGS_DETACH; mutex_unlock(&mlx5_intf_mutex); } @@ -438,6 +444,7 @@ int mlx5_register_device(struct mlx5_core_dev *dev) { int ret; + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); dev->priv.flags &= ~MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; ret = mlx5_rescan_drivers_locked(dev); @@ -450,6 +457,7 @@ int mlx5_register_device(struct mlx5_core_dev *dev) void mlx5_unregister_device(struct mlx5_core_dev *dev) { + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&mlx5_intf_mutex); dev->priv.flags = MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV; mlx5_rescan_drivers_locked(dev); @@ -526,16 +534,22 @@ del_adev: int mlx5_rescan_drivers_locked(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; + int err = 0; lockdep_assert_held(&mlx5_intf_mutex); if (priv->flags & MLX5_PRIV_FLAGS_DETACH) return 0; + priv->flags |= MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; delete_drivers(dev); if (priv->flags & MLX5_PRIV_FLAGS_DISABLE_ALL_ADEV) - return 0; + goto out; + + err = add_drivers(dev); - return add_drivers(dev); +out: + priv->flags &= ~MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW; + return err; } bool mlx5_same_hw_devs(struct mlx5_core_dev *dev, struct mlx5_core_dev *peer_dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c index f85166e587f2..66c6a7017695 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/devlink.c @@ -104,7 +104,16 @@ static int mlx5_devlink_reload_fw_activate(struct devlink *devlink, struct netli if (err) return err; - return mlx5_fw_reset_wait_reset_done(dev); + err = mlx5_fw_reset_wait_reset_done(dev); + if (err) + return err; + + mlx5_unload_one_devl_locked(dev); + err = mlx5_health_wait_pci_up(dev); + if (err) + NL_SET_ERR_MSG_MOD(extack, "FW activate aborted, PCI reads fail after reset"); + + return err; } static int mlx5_devlink_trigger_fw_live_patch(struct devlink *devlink, @@ -134,6 +143,7 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, struct mlx5_core_dev *dev = devlink_priv(devlink); struct pci_dev *pdev = dev->pdev; bool sf_dev_allocated; + int ret = 0; sf_dev_allocated = mlx5_sf_dev_allocated(dev); if (sf_dev_allocated) { @@ -156,17 +166,21 @@ static int mlx5_devlink_reload_down(struct devlink *devlink, bool netns_change, switch (action) { case DEVLINK_RELOAD_ACTION_DRIVER_REINIT: - mlx5_unload_one(dev); - return 0; + mlx5_unload_one_devl_locked(dev); + break; case DEVLINK_RELOAD_ACTION_FW_ACTIVATE: if (limit == DEVLINK_RELOAD_LIMIT_NO_RESET) - return mlx5_devlink_trigger_fw_live_patch(devlink, extack); - return mlx5_devlink_reload_fw_activate(devlink, extack); + ret = mlx5_devlink_trigger_fw_live_patch(devlink, extack); + else + ret = mlx5_devlink_reload_fw_activate(devlink, extack); + break; default: /* Unsupported action should not get to this function */ WARN_ON(1); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; } + + return ret; } static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_action action, @@ -174,24 +188,27 @@ static int mlx5_devlink_reload_up(struct devlink *devlink, enum devlink_reload_a struct netlink_ext_ack *extack) { struct mlx5_core_dev *dev = devlink_priv(devlink); + int ret = 0; *actions_performed = BIT(action); switch (action) { case DEVLINK_RELOAD_ACTION_DRIVER_REINIT: - return mlx5_load_one(dev, false); + ret = mlx5_load_one_devl_locked(dev, false); + break; case DEVLINK_RELOAD_ACTION_FW_ACTIVATE: if (limit == DEVLINK_RELOAD_LIMIT_NO_RESET) break; /* On fw_activate action, also driver is reloaded and reinit performed */ *actions_performed |= BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT); - return mlx5_load_one(dev, false); + ret = mlx5_load_one_devl_locked(dev, false); + break; default: /* Unsupported action should not get to this function */ WARN_ON(1); - return -EOPNOTSUPP; + ret = -EOPNOTSUPP; } - return 0; + return ret; } static struct mlx5_devlink_trap *mlx5_find_trap_by_id(struct mlx5_core_dev *dev, int trap_id) @@ -828,28 +845,28 @@ static int mlx5_devlink_traps_register(struct devlink *devlink) struct mlx5_core_dev *core_dev = devlink_priv(devlink); int err; - err = devlink_trap_groups_register(devlink, mlx5_trap_groups_arr, - ARRAY_SIZE(mlx5_trap_groups_arr)); + err = devl_trap_groups_register(devlink, mlx5_trap_groups_arr, + ARRAY_SIZE(mlx5_trap_groups_arr)); if (err) return err; - err = devlink_traps_register(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr), - &core_dev->priv); + err = devl_traps_register(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr), + &core_dev->priv); if (err) goto err_trap_group; return 0; err_trap_group: - devlink_trap_groups_unregister(devlink, mlx5_trap_groups_arr, - ARRAY_SIZE(mlx5_trap_groups_arr)); + devl_trap_groups_unregister(devlink, mlx5_trap_groups_arr, + ARRAY_SIZE(mlx5_trap_groups_arr)); return err; } static void mlx5_devlink_traps_unregister(struct devlink *devlink) { - devlink_traps_unregister(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr)); - devlink_trap_groups_unregister(devlink, mlx5_trap_groups_arr, - ARRAY_SIZE(mlx5_trap_groups_arr)); + devl_traps_unregister(devlink, mlx5_traps_arr, ARRAY_SIZE(mlx5_traps_arr)); + devl_trap_groups_unregister(devlink, mlx5_trap_groups_arr, + ARRAY_SIZE(mlx5_trap_groups_arr)); } int mlx5_devlink_register(struct devlink *devlink) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en.h b/drivers/net/ethernet/mellanox/mlx5/core/en.h index 29b10ef787b9..a560df446bac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en.h @@ -322,7 +322,8 @@ struct mlx5e_params { u8 num_tc; struct netdev_tc_txq tc_to_txq[TC_MAX_QUEUE]; struct { - struct mlx5e_mqprio_rl *rl; + u64 max_rate[TC_MAX_QUEUE]; + u32 hw_id[TC_MAX_QUEUE]; } channel; } mqprio; bool rx_cqe_compress_def; @@ -899,16 +900,8 @@ struct mlx5e_scratchpad { cpumask_var_t cpumask; }; -struct mlx5e_htb { - DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); - DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); - struct mlx5e_sq_stats **qos_sq_stats; - u16 max_qos_sqs; - u16 maj_id; - u16 defcls; -}; - struct mlx5e_trap; +struct mlx5e_htb; struct mlx5e_priv { /* priv data path fields - start */ @@ -929,7 +922,7 @@ struct mlx5e_priv { struct mlx5e_rx_res *rx_res; u32 *tx_rates; - struct mlx5e_flow_steering fs; + struct mlx5e_flow_steering *fs; struct workqueue_struct *wq; struct work_struct update_carrier_work; @@ -946,6 +939,8 @@ struct mlx5e_priv { struct mlx5e_channel_stats **channel_stats; struct mlx5e_channel_stats trap_stats; struct mlx5e_ptp_stats ptp_stats; + struct mlx5e_sq_stats **htb_qos_sq_stats; + u16 htb_max_qos_sqs; u16 stats_nch; u16 max_nch; u8 max_opened_tc; @@ -977,7 +972,7 @@ struct mlx5e_priv { struct mlx5e_hv_vhca_stats_agent stats_agent; #endif struct mlx5e_scratchpad scratchpad; - struct mlx5e_htb htb; + struct mlx5e_htb *htb; struct mlx5e_mqprio_rl *mqprio_rl; }; @@ -993,6 +988,8 @@ enum mlx5e_profile_feature { MLX5E_PROFILE_FEATURE_PTP_RX, MLX5E_PROFILE_FEATURE_PTP_TX, MLX5E_PROFILE_FEATURE_QOS_HTB, + MLX5E_PROFILE_FEATURE_FS_VLAN, + MLX5E_PROFILE_FEATURE_FS_TC, }; struct mlx5e_profile { @@ -1028,7 +1025,6 @@ void mlx5e_shampo_dealloc_hd(struct mlx5e_rq *rq, u16 len, u16 start, bool close void mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats); void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s); -void mlx5e_init_l2_addr(struct mlx5e_priv *priv); int mlx5e_self_test_num(struct mlx5e_priv *priv); int mlx5e_self_test_fill_strings(struct mlx5e_priv *priv, u8 *data); void mlx5e_self_test(struct net_device *ndev, struct ethtool_test *etest, @@ -1182,7 +1178,8 @@ int mlx5e_ethtool_get_sset_count(struct mlx5e_priv *priv, int sset); void mlx5e_ethtool_get_ethtool_stats(struct mlx5e_priv *priv, struct ethtool_stats *stats, u64 *data); void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, - struct ethtool_ringparam *param); + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param); int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, struct ethtool_ringparam *param); void mlx5e_ethtool_get_channels(struct mlx5e_priv *priv, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c index ae52e7f38306..b69f9d10ccbd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/devlink.c @@ -21,6 +21,7 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) struct netdev_phys_item_id ppid = {}; struct devlink_port *dl_port; unsigned int dl_port_index; + int ret; if (mlx5_core_is_pf(priv->mdev)) { attrs.flavour = DEVLINK_PORT_FLAVOUR_PHYSICAL; @@ -41,7 +42,13 @@ int mlx5e_devlink_port_register(struct mlx5e_priv *priv) memset(dl_port, 0, sizeof(*dl_port)); devlink_port_attrs_set(dl_port, &attrs); - return devlink_port_register(devlink, dl_port, dl_port_index); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_lock(devlink); + ret = devl_port_register(devlink, dl_port, dl_port_index); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_unlock(devlink); + + return ret; } void mlx5e_devlink_port_type_eth_set(struct mlx5e_priv *priv) @@ -54,8 +61,13 @@ void mlx5e_devlink_port_type_eth_set(struct mlx5e_priv *priv) void mlx5e_devlink_port_unregister(struct mlx5e_priv *priv) { struct devlink_port *dl_port = mlx5e_devlink_get_dl_port(priv); + struct devlink *devlink = priv_to_devlink(priv->mdev); - devlink_port_unregister(dl_port); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_lock(devlink); + devl_port_unregister(dl_port); + if (!(priv->mdev->priv.flags & MLX5_PRIV_FLAGS_MLX5E_LOCKED_FLOW)) + devl_unlock(devlink); } struct devlink_port *mlx5e_get_devlink_port(struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h index 6e3a90a959e9..9b8cdf2e68ad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs.h @@ -15,29 +15,6 @@ enum { MLX5E_TC_MISS_LEVEL, }; -struct mlx5e_tc_table { - /* Protects the dynamic assignment of the t parameter - * which is the nic tc root table. - */ - struct mutex t_lock; - struct mlx5_flow_table *t; - struct mlx5_flow_table *miss_t; - struct mlx5_fs_chains *chains; - struct mlx5e_post_act *post_act; - - struct rhashtable ht; - - struct mod_hdr_tbl mod_hdr; - struct mutex hairpin_tbl_lock; /* protects hairpin_tbl */ - DECLARE_HASHTABLE(hairpin_tbl, 8); - - struct notifier_block netdevice_nb; - struct netdev_net_notifier netdevice_nn; - - struct mlx5_tc_ct_priv *ct; - struct mapping_ctx *mapping; -}; - struct mlx5e_flow_table { int num_groups; struct mlx5_flow_table *t; @@ -160,16 +137,20 @@ static inline int mlx5e_arfs_disable(struct mlx5e_priv *priv) { return -EOPNOTSU struct mlx5e_accel_fs_tcp; #endif +struct mlx5e_profile; struct mlx5e_fs_udp; struct mlx5e_fs_any; struct mlx5e_ptp_fs; struct mlx5e_flow_steering { + bool state_destroy; + bool vlan_strip_disable; + struct mlx5_core_dev *mdev; struct mlx5_flow_namespace *ns; #ifdef CONFIG_MLX5_EN_RXNFC struct mlx5e_ethtool_steering ethtool; #endif - struct mlx5e_tc_table tc; + struct mlx5e_tc_table *tc; struct mlx5e_promisc_table promisc; struct mlx5e_vlan_table *vlan; struct mlx5e_l2_table l2; @@ -200,13 +181,22 @@ void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv); int mlx5e_create_flow_steering(struct mlx5e_priv *priv); void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv); -int mlx5e_fs_init(struct mlx5e_priv *priv); -void mlx5e_fs_cleanup(struct mlx5e_priv *priv); +struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile, + struct mlx5_core_dev *mdev, + bool state_destroy); +void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs); int mlx5e_add_vlan_trap(struct mlx5e_priv *priv, int trap_id, int tir_num); void mlx5e_remove_vlan_trap(struct mlx5e_priv *priv); int mlx5e_add_mac_trap(struct mlx5e_priv *priv, int trap_id, int tir_num); void mlx5e_remove_mac_trap(struct mlx5e_priv *priv); - +void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, struct net_device *netdev); +int mlx5e_fs_vlan_rx_add_vid(struct mlx5e_flow_steering *fs, + struct net_device *netdev, + __be16 proto, u16 vid); +int mlx5e_fs_vlan_rx_kill_vid(struct mlx5e_flow_steering *fs, + struct net_device *netdev, + __be16 proto, u16 vid); +void mlx5e_fs_init_l2_addr(struct mlx5e_flow_steering *fs, struct net_device *netdev); #endif /* __MLX5E_FLOW_STEER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c index 7aa25a5e29d7..e153d6119e02 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/fs_tt_redirect.c @@ -94,7 +94,7 @@ mlx5e_fs_tt_redirect_udp_add_rule(struct mlx5e_priv *priv, if (!spec) return ERR_PTR(-ENOMEM); - fs_udp = priv->fs.udp; + fs_udp = priv->fs->udp; ft = fs_udp->tables[type].t; fs_udp_set_dport_flow(spec, type, d_port); @@ -121,10 +121,10 @@ static int fs_udp_add_default_rule(struct mlx5e_priv *priv, enum fs_udp_type typ struct mlx5e_fs_udp *fs_udp; int err; - fs_udp = priv->fs.udp; + fs_udp = priv->fs->udp; fs_udp_t = &fs_udp->tables[type]; - dest = mlx5_ttc_get_default_dest(priv->fs.ttc, fs_udp2tt(type)); + dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_udp2tt(type)); rule = mlx5_add_flow_rules(fs_udp_t->t, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -208,7 +208,7 @@ out: static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type) { - struct mlx5e_flow_table *ft = &priv->fs.udp->tables[type]; + struct mlx5e_flow_table *ft = &priv->fs->udp->tables[type]; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -218,7 +218,7 @@ static int fs_udp_create_table(struct mlx5e_priv *priv, enum fs_udp_type type) ft_attr.level = MLX5E_FS_TT_UDP_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -259,7 +259,7 @@ static int fs_udp_disable(struct mlx5e_priv *priv) for (i = 0; i < FS_UDP_NUM_TYPES; i++) { /* Modify ttc rules destination to point back to the indir TIRs */ - err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_udp2tt(i)); + err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_udp2tt(i)); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] default destination failed, err(%d)\n", @@ -278,10 +278,10 @@ static int fs_udp_enable(struct mlx5e_priv *priv) dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < FS_UDP_NUM_TYPES; i++) { - dest.ft = priv->fs.udp->tables[i].t; + dest.ft = priv->fs->udp->tables[i].t; /* Modify ttc rules destination to point on the accel_fs FTs */ - err = mlx5_ttc_fwd_dest(priv->fs.ttc, fs_udp2tt(i), &dest); + err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_udp2tt(i), &dest); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] destination to accel failed, err(%d)\n", @@ -294,7 +294,7 @@ static int fs_udp_enable(struct mlx5e_priv *priv) void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv) { - struct mlx5e_fs_udp *fs_udp = priv->fs.udp; + struct mlx5e_fs_udp *fs_udp = priv->fs->udp; int i; if (!fs_udp) @@ -309,20 +309,20 @@ void mlx5e_fs_tt_redirect_udp_destroy(struct mlx5e_priv *priv) fs_udp_destroy_table(fs_udp, i); kfree(fs_udp); - priv->fs.udp = NULL; + priv->fs->udp = NULL; } int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv) { int i, err; - if (priv->fs.udp) { - priv->fs.udp->ref_cnt++; + if (priv->fs->udp) { + priv->fs->udp->ref_cnt++; return 0; } - priv->fs.udp = kzalloc(sizeof(*priv->fs.udp), GFP_KERNEL); - if (!priv->fs.udp) + priv->fs->udp = kzalloc(sizeof(*priv->fs->udp), GFP_KERNEL); + if (!priv->fs->udp) return -ENOMEM; for (i = 0; i < FS_UDP_NUM_TYPES; i++) { @@ -335,16 +335,16 @@ int mlx5e_fs_tt_redirect_udp_create(struct mlx5e_priv *priv) if (err) goto err_destroy_tables; - priv->fs.udp->ref_cnt = 1; + priv->fs->udp->ref_cnt = 1; return 0; err_destroy_tables: while (--i >= 0) - fs_udp_destroy_table(priv->fs.udp, i); + fs_udp_destroy_table(priv->fs->udp, i); - kfree(priv->fs.udp); - priv->fs.udp = NULL; + kfree(priv->fs->udp); + priv->fs->udp = NULL; return err; } @@ -371,7 +371,7 @@ mlx5e_fs_tt_redirect_any_add_rule(struct mlx5e_priv *priv, if (!spec) return ERR_PTR(-ENOMEM); - fs_any = priv->fs.any; + fs_any = priv->fs->any; ft = fs_any->table.t; fs_any_set_ethertype_flow(spec, ether_type); @@ -398,10 +398,10 @@ static int fs_any_add_default_rule(struct mlx5e_priv *priv) struct mlx5e_fs_any *fs_any; int err; - fs_any = priv->fs.any; + fs_any = priv->fs->any; fs_any_t = &fs_any->table; - dest = mlx5_ttc_get_default_dest(priv->fs.ttc, MLX5_TT_ANY); + dest = mlx5_ttc_get_default_dest(priv->fs->ttc, MLX5_TT_ANY); rule = mlx5_add_flow_rules(fs_any_t->t, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -474,7 +474,7 @@ err: static int fs_any_create_table(struct mlx5e_priv *priv) { - struct mlx5e_flow_table *ft = &priv->fs.any->table; + struct mlx5e_flow_table *ft = &priv->fs->any->table; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -484,7 +484,7 @@ static int fs_any_create_table(struct mlx5e_priv *priv) ft_attr.level = MLX5E_FS_TT_ANY_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -514,7 +514,7 @@ static int fs_any_disable(struct mlx5e_priv *priv) int err; /* Modify ttc rules destination to point back to the indir TIRs */ - err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, MLX5_TT_ANY); + err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, MLX5_TT_ANY); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] default destination failed, err(%d)\n", @@ -530,10 +530,10 @@ static int fs_any_enable(struct mlx5e_priv *priv) int err; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = priv->fs.any->table.t; + dest.ft = priv->fs->any->table.t; /* Modify ttc rules destination to point on the accel_fs FTs */ - err = mlx5_ttc_fwd_dest(priv->fs.ttc, MLX5_TT_ANY, &dest); + err = mlx5_ttc_fwd_dest(priv->fs->ttc, MLX5_TT_ANY, &dest); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] destination to accel failed, err(%d)\n", @@ -555,7 +555,7 @@ static void fs_any_destroy_table(struct mlx5e_fs_any *fs_any) void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv) { - struct mlx5e_fs_any *fs_any = priv->fs.any; + struct mlx5e_fs_any *fs_any = priv->fs->any; if (!fs_any) return; @@ -568,20 +568,20 @@ void mlx5e_fs_tt_redirect_any_destroy(struct mlx5e_priv *priv) fs_any_destroy_table(fs_any); kfree(fs_any); - priv->fs.any = NULL; + priv->fs->any = NULL; } int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv) { int err; - if (priv->fs.any) { - priv->fs.any->ref_cnt++; + if (priv->fs->any) { + priv->fs->any->ref_cnt++; return 0; } - priv->fs.any = kzalloc(sizeof(*priv->fs.any), GFP_KERNEL); - if (!priv->fs.any) + priv->fs->any = kzalloc(sizeof(*priv->fs->any), GFP_KERNEL); + if (!priv->fs->any) return -ENOMEM; err = fs_any_create_table(priv); @@ -592,14 +592,14 @@ int mlx5e_fs_tt_redirect_any_create(struct mlx5e_priv *priv) if (err) goto err_destroy_table; - priv->fs.any->ref_cnt = 1; + priv->fs->any->ref_cnt = 1; return 0; err_destroy_table: - fs_any_destroy_table(priv->fs.any); + fs_any_destroy_table(priv->fs->any); - kfree(priv->fs.any); - priv->fs.any = NULL; + kfree(priv->fs->any); + priv->fs->any = NULL; return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c new file mode 100644 index 000000000000..6dac76fa58a3 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.c @@ -0,0 +1,722 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <net/pkt_cls.h> +#include "htb.h" +#include "en.h" +#include "../qos.h" + +struct mlx5e_qos_node { + struct hlist_node hnode; + struct mlx5e_qos_node *parent; + u64 rate; + u32 bw_share; + u32 max_average_bw; + u32 hw_id; + u32 classid; /* 16-bit, except root. */ + u16 qid; +}; + +struct mlx5e_htb { + DECLARE_HASHTABLE(qos_tc2node, order_base_2(MLX5E_QOS_MAX_LEAF_NODES)); + DECLARE_BITMAP(qos_used_qids, MLX5E_QOS_MAX_LEAF_NODES); + struct mlx5_core_dev *mdev; + struct net_device *netdev; + struct mlx5e_priv *priv; + struct mlx5e_selq *selq; +}; + +#define MLX5E_QOS_QID_INNER 0xffff +#define MLX5E_HTB_CLASSID_ROOT 0xffffffff + +/* Software representation of the QoS tree */ + +int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data) +{ + struct mlx5e_qos_node *node = NULL; + int bkt, err; + + hash_for_each(htb->qos_tc2node, bkt, node, hnode) { + if (node->qid == MLX5E_QOS_QID_INNER) + continue; + err = callback(data, node->qid, node->hw_id); + if (err) + return err; + } + return 0; +} + +int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb) +{ + int last; + + last = find_last_bit(htb->qos_used_qids, mlx5e_qos_max_leaf_nodes(htb->mdev)); + return last == mlx5e_qos_max_leaf_nodes(htb->mdev) ? 0 : last + 1; +} + +static int mlx5e_htb_find_unused_qos_qid(struct mlx5e_htb *htb) +{ + int size = mlx5e_qos_max_leaf_nodes(htb->mdev); + struct mlx5e_priv *priv = htb->priv; + int res; + + WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); + res = find_first_zero_bit(htb->qos_used_qids, size); + + return res == size ? -ENOSPC : res; +} + +static struct mlx5e_qos_node * +mlx5e_htb_node_create_leaf(struct mlx5e_htb *htb, u16 classid, u16 qid, + struct mlx5e_qos_node *parent) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->parent = parent; + + node->qid = qid; + __set_bit(qid, htb->qos_used_qids); + + node->classid = classid; + hash_add_rcu(htb->qos_tc2node, &node->hnode, classid); + + mlx5e_update_tx_netdev_queues(htb->priv); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_create_root(struct mlx5e_htb *htb) +{ + struct mlx5e_qos_node *node; + + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return ERR_PTR(-ENOMEM); + + node->qid = MLX5E_QOS_QID_INNER; + node->classid = MLX5E_HTB_CLASSID_ROOT; + hash_add_rcu(htb->qos_tc2node, &node->hnode, node->classid); + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find(struct mlx5e_htb *htb, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible(htb->qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find_rcu(struct mlx5e_htb *htb, u32 classid) +{ + struct mlx5e_qos_node *node = NULL; + + hash_for_each_possible_rcu(htb->qos_tc2node, node, hnode, classid) { + if (node->classid == classid) + break; + } + + return node; +} + +static void mlx5e_htb_node_delete(struct mlx5e_htb *htb, struct mlx5e_qos_node *node) +{ + hash_del_rcu(&node->hnode); + if (node->qid != MLX5E_QOS_QID_INNER) { + __clear_bit(node->qid, htb->qos_used_qids); + mlx5e_update_tx_netdev_queues(htb->priv); + } + /* Make sure this qid is no longer selected by mlx5e_select_queue, so + * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. + */ + synchronize_net(); + kfree(node); +} + +/* TX datapath API */ + +int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid) +{ + struct mlx5e_qos_node *node; + u16 qid; + int res; + + rcu_read_lock(); + + node = mlx5e_htb_node_find_rcu(htb, classid); + if (!node) { + res = -ENOENT; + goto out; + } + qid = READ_ONCE(node->qid); + if (qid == MLX5E_QOS_QID_INNER) { + res = -EINVAL; + goto out; + } + res = mlx5e_qid_from_qos(&htb->priv->channels, qid); + +out: + rcu_read_unlock(); + return res; +} + +/* HTB TC handlers */ + +static int +mlx5e_htb_root_add(struct mlx5e_htb *htb, u16 htb_maj_id, u16 htb_defcls, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *root; + bool opened; + int err; + + qos_dbg(htb->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); + + mlx5e_selq_prepare_htb(htb->selq, htb_maj_id, htb_defcls); + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + err = mlx5e_qos_alloc_queues(priv, &priv->channels); + if (err) + goto err_cancel_selq; + } + + root = mlx5e_htb_node_create_root(htb); + if (IS_ERR(root)) { + err = PTR_ERR(root); + goto err_free_queues; + } + + err = mlx5_qos_create_root_node(htb->mdev, &root->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); + goto err_sw_node_delete; + } + + mlx5e_selq_apply(htb->selq); + + return 0; + +err_sw_node_delete: + mlx5e_htb_node_delete(htb, root); + +err_free_queues: + if (opened) + mlx5e_qos_close_all_queues(&priv->channels); +err_cancel_selq: + mlx5e_selq_cancel(htb->selq); + return err; +} + +static int mlx5e_htb_root_del(struct mlx5e_htb *htb) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *root; + int err; + + qos_dbg(htb->mdev, "TC_HTB_DESTROY\n"); + + /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, + * so that we can safely switch to its non-HTB non-PTP fastpath. + */ + synchronize_net(); + + mlx5e_selq_prepare_htb(htb->selq, 0, 0); + mlx5e_selq_apply(htb->selq); + + root = mlx5e_htb_node_find(htb, MLX5E_HTB_CLASSID_ROOT); + if (!root) { + qos_err(htb->mdev, "Failed to find the root node in the QoS tree\n"); + return -ENOENT; + } + err = mlx5_qos_destroy_node(htb->mdev, root->hw_id); + if (err) + qos_err(htb->mdev, "Failed to destroy root node %u, err = %d\n", + root->hw_id, err); + mlx5e_htb_node_delete(htb, root); + + mlx5e_qos_deactivate_all_queues(&priv->channels); + mlx5e_qos_close_all_queues(&priv->channels); + + return err; +} + +static int mlx5e_htb_convert_rate(struct mlx5e_htb *htb, u64 rate, + struct mlx5e_qos_node *parent, u32 *bw_share) +{ + u64 share = 0; + + while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) + parent = parent->parent; + + if (parent->max_average_bw) + share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), + parent->max_average_bw); + else + share = 101; + + *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; + + qos_dbg(htb->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", + rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); + + return 0; +} + +static void mlx5e_htb_convert_ceil(struct mlx5e_htb *htb, u64 ceil, u32 *max_average_bw) +{ + /* Hardware treats 0 as "unlimited", set at least 1. */ + *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); + + qos_dbg(htb->mdev, "Convert: ceil %llu -> max_average_bw %u\n", + ceil, *max_average_bw); +} + +int +mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + struct mlx5e_priv *priv = htb->priv; + int qid; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", + classid, parent_classid, rate, ceil); + + qid = mlx5e_htb_find_unused_qos_qid(htb); + if (qid < 0) { + NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); + return qid; + } + + parent = mlx5e_htb_node_find(htb, parent_classid); + if (!parent) + return -EINVAL; + + node = mlx5e_htb_node_create_leaf(htb, classid, qid, parent); + if (IS_ERR(node)) + return PTR_ERR(node); + + node->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node->parent, &node->bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &node->max_average_bw); + + err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + mlx5e_htb_node_delete(htb, node); + return err; + } + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + return mlx5e_qid_from_qos(&priv->channels, node->qid); +} + +int +mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *child; + struct mlx5e_priv *priv = htb->priv; + int err, tmp_err; + u32 new_hw_id; + u16 qid; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", + classid, child_classid, rate, ceil); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_inner_node(htb->mdev, node->parent->hw_id, + node->bw_share, node->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); + qos_err(htb->mdev, "Failed to create an inner node (class %04x), err = %d\n", + classid, err); + return err; + } + + /* Intentionally reuse the qid for the upcoming first child. */ + child = mlx5e_htb_node_create_leaf(htb, child_classid, node->qid, node); + if (IS_ERR(child)) { + err = PTR_ERR(child); + goto err_destroy_hw_node; + } + + child->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node, &child->bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &child->max_average_bw); + + err = mlx5_qos_create_leaf_node(htb->mdev, new_hw_id, child->bw_share, + child->max_average_bw, &child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + goto err_delete_sw_node; + } + + /* No fail point. */ + + qid = node->qid; + /* Pairs with mlx5e_htb_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, child->qid, child->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, child->qid, child->hw_id); + } + } + + return 0; + +err_delete_sw_node: + child->qid = MLX5E_QOS_QID_INNER; + mlx5e_htb_node_delete(htb, child); + +err_destroy_hw_node: + tmp_err = mlx5_qos_destroy_node(htb->mdev, new_hw_id); + if (tmp_err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", + new_hw_id, classid, tmp_err); + return err; +} + +static struct mlx5e_qos_node *mlx5e_htb_node_find_by_qid(struct mlx5e_htb *htb, u16 qid) +{ + struct mlx5e_qos_node *node = NULL; + int bkt; + + hash_for_each(htb->qos_tc2node, bkt, node, hnode) + if (node->qid == qid) + break; + + return node; +} + +int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, + struct netlink_ext_ack *extack) +{ + struct mlx5e_priv *priv = htb->priv; + struct mlx5e_qos_node *node; + struct netdev_queue *txq; + u16 qid, moved_qid; + bool opened; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); + + node = mlx5e_htb_node_find(htb, *classid); + if (!node) + return -ENOENT; + + /* Store qid for reuse. */ + qid = node->qid; + + opened = test_bit(MLX5E_STATE_OPENED, &priv->state); + if (opened) { + txq = netdev_get_tx_queue(htb->netdev, + mlx5e_qid_from_qos(&priv->channels, qid)); + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, *classid, err); + + mlx5e_htb_node_delete(htb, node); + + moved_qid = mlx5e_htb_cur_leaf_nodes(htb); + + if (moved_qid == 0) { + /* The last QoS SQ was just destroyed. */ + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + moved_qid--; + + if (moved_qid < qid) { + /* The highest QoS SQ was just destroyed. */ + WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", + qid, moved_qid); + if (opened) + mlx5e_reactivate_qos_sq(priv, qid, txq); + return 0; + } + + WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); + qos_dbg(htb->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); + + node = mlx5e_htb_node_find_by_qid(htb, moved_qid); + WARN(!node, "Could not find a node with qid %u to move to queue %u", + moved_qid, qid); + + /* Stop traffic to the old queue. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + __clear_bit(moved_qid, priv->htb->qos_used_qids); + + if (opened) { + txq = netdev_get_tx_queue(htb->netdev, + mlx5e_qid_from_qos(&priv->channels, moved_qid)); + mlx5e_deactivate_qos_sq(priv, moved_qid); + mlx5e_close_qos_sq(priv, moved_qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(htb->netdev, moved_qid); + + __set_bit(qid, htb->qos_used_qids); + WRITE_ONCE(node->qid, qid); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", + node->classid, moved_qid, qid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + mlx5e_update_tx_netdev_queues(priv); + if (opened) + mlx5e_reactivate_qos_sq(priv, moved_qid, txq); + + *classid = node->classid; + return 0; +} + +int +mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *node, *parent; + struct mlx5e_priv *priv = htb->priv; + u32 old_hw_id, new_hw_id; + int err, saved_err = 0; + u16 qid; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", + force ? "_FORCE" : "", classid); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + err = mlx5_qos_create_leaf_node(htb->mdev, node->parent->parent->hw_id, + node->parent->bw_share, + node->parent->max_average_bw, + &new_hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); + qos_err(htb->mdev, "Failed to create a leaf node (class %04x), err = %d\n", + classid, err); + if (!force) + return err; + saved_err = err; + } + + /* Store qid for reuse and prevent clearing the bit. */ + qid = node->qid; + /* Pairs with mlx5e_htb_get_txq_by_classid. */ + WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + mlx5e_deactivate_qos_sq(priv, qid); + mlx5e_close_qos_sq(priv, qid); + } + + /* Prevent packets from the old class from getting into the new one. */ + mlx5e_reset_qdisc(htb->netdev, qid); + + err = mlx5_qos_destroy_node(htb->mdev, node->hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + parent = node->parent; + mlx5e_htb_node_delete(htb, node); + + node = parent; + WRITE_ONCE(node->qid, qid); + + /* Early return on error in force mode. Parent will still be an inner + * node to be deleted by a following delete operation. + */ + if (saved_err) + return saved_err; + + old_hw_id = node->hw_id; + node->hw_id = new_hw_id; + + if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { + err = mlx5e_open_qos_sq(priv, &priv->channels, node->qid, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); + qos_warn(htb->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", + classid, err); + } else { + mlx5e_activate_qos_sq(priv, node->qid, node->hw_id); + } + } + + err = mlx5_qos_destroy_node(htb->mdev, old_hw_id); + if (err) /* Not fatal. */ + qos_warn(htb->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", + node->hw_id, classid, err); + + return 0; +} + +static int +mlx5e_htb_update_children(struct mlx5e_htb *htb, struct mlx5e_qos_node *node, + struct netlink_ext_ack *extack) +{ + struct mlx5e_qos_node *child; + int err = 0; + int bkt; + + hash_for_each(htb->qos_tc2node, bkt, child, hnode) { + u32 old_bw_share = child->bw_share; + int err_one; + + if (child->parent != node) + continue; + + mlx5e_htb_convert_rate(htb, child->rate, node, &child->bw_share); + if (child->bw_share == old_bw_share) + continue; + + err_one = mlx5_qos_update_node(htb->mdev, child->hw_id, child->bw_share, + child->max_average_bw, child->hw_id); + if (!err && err_one) { + err = err_one; + + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); + qos_err(htb->mdev, "Failed to modify a child node (class %04x), err = %d\n", + node->classid, err); + } + } + + return err; +} + +int +mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack) +{ + u32 bw_share, max_average_bw; + struct mlx5e_qos_node *node; + bool ceil_changed = false; + int err; + + qos_dbg(htb->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", + classid, rate, ceil); + + node = mlx5e_htb_node_find(htb, classid); + if (!node) + return -ENOENT; + + node->rate = rate; + mlx5e_htb_convert_rate(htb, rate, node->parent, &bw_share); + mlx5e_htb_convert_ceil(htb, ceil, &max_average_bw); + + err = mlx5_qos_update_node(htb->mdev, node->parent->hw_id, bw_share, + max_average_bw, node->hw_id); + if (err) { + NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); + qos_err(htb->mdev, "Failed to modify a node (class %04x), err = %d\n", + classid, err); + return err; + } + + if (max_average_bw != node->max_average_bw) + ceil_changed = true; + + node->bw_share = bw_share; + node->max_average_bw = max_average_bw; + + if (ceil_changed) + err = mlx5e_htb_update_children(htb, node, extack); + + return err; +} + +struct mlx5e_htb *mlx5e_htb_alloc(void) +{ + return kvzalloc(sizeof(struct mlx5e_htb), GFP_KERNEL); +} + +void mlx5e_htb_free(struct mlx5e_htb *htb) +{ + kvfree(htb); +} + +int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, + struct net_device *netdev, struct mlx5_core_dev *mdev, + struct mlx5e_selq *selq, struct mlx5e_priv *priv) +{ + htb->mdev = mdev; + htb->netdev = netdev; + htb->selq = selq; + htb->priv = priv; + hash_init(htb->qos_tc2node); + return mlx5e_htb_root_add(htb, htb_qopt->parent_classid, htb_qopt->classid, + htb_qopt->extack); +} + +void mlx5e_htb_cleanup(struct mlx5e_htb *htb) +{ + mlx5e_htb_root_del(htb); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h new file mode 100644 index 000000000000..8386f1ea4559 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/htb.h @@ -0,0 +1,46 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5E_EN_HTB_H_ +#define __MLX5E_EN_HTB_H_ + +#include "qos.h" + +#define MLX5E_QOS_MAX_LEAF_NODES 256 + +struct mlx5e_selq; +struct mlx5e_htb; + +typedef int (*mlx5e_fp_htb_enumerate)(void *data, u16 qid, u32 hw_id); +int mlx5e_htb_enumerate_leaves(struct mlx5e_htb *htb, mlx5e_fp_htb_enumerate callback, void *data); + +int mlx5e_htb_cur_leaf_nodes(struct mlx5e_htb *htb); + +/* TX datapath API */ +int mlx5e_htb_get_txq_by_classid(struct mlx5e_htb *htb, u16 classid); + +/* HTB TC handlers */ + +int +mlx5e_htb_leaf_alloc_queue(struct mlx5e_htb *htb, u16 classid, + u32 parent_classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +int +mlx5e_htb_leaf_to_inner(struct mlx5e_htb *htb, u16 classid, u16 child_classid, + u64 rate, u64 ceil, struct netlink_ext_ack *extack); +int mlx5e_htb_leaf_del(struct mlx5e_htb *htb, u16 *classid, + struct netlink_ext_ack *extack); +int +mlx5e_htb_leaf_del_last(struct mlx5e_htb *htb, u16 classid, bool force, + struct netlink_ext_ack *extack); +int +mlx5e_htb_node_modify(struct mlx5e_htb *htb, u16 classid, u64 rate, u64 ceil, + struct netlink_ext_ack *extack); +struct mlx5e_htb *mlx5e_htb_alloc(void); +void mlx5e_htb_free(struct mlx5e_htb *htb); +int mlx5e_htb_init(struct mlx5e_htb *htb, struct tc_htb_qopt_offload *htb_qopt, + struct net_device *netdev, struct mlx5_core_dev *mdev, + struct mlx5e_selq *selq, struct mlx5e_priv *priv); +void mlx5e_htb_cleanup(struct mlx5e_htb *htb); +#endif + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c index 047f88f09203..903de88bab53 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.c @@ -79,19 +79,49 @@ void mlx5e_skb_cb_hwtstamp_handler(struct sk_buff *skb, int hwtstamp_type, memset(skb->cb, 0, sizeof(struct mlx5e_skb_cb_hwtstamp)); } +#define PTP_WQE_CTR2IDX(val) ((val) & ptpsq->ts_cqe_ctr_mask) + +static bool mlx5e_ptp_ts_cqe_drop(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id) +{ + return (ptpsq->ts_cqe_ctr_mask && (skb_cc != skb_id)); +} + +static void mlx5e_ptp_skb_fifo_ts_cqe_resync(struct mlx5e_ptpsq *ptpsq, u16 skb_cc, u16 skb_id) +{ + struct skb_shared_hwtstamps hwts = {}; + struct sk_buff *skb; + + ptpsq->cq_stats->resync_event++; + + while (skb_cc != skb_id) { + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); + hwts.hwtstamp = mlx5e_skb_cb_get_hwts(skb)->cqe_hwtstamp; + skb_tstamp_tx(skb, &hwts); + ptpsq->cq_stats->resync_cqe++; + skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc); + } +} + static void mlx5e_ptp_handle_ts_cqe(struct mlx5e_ptpsq *ptpsq, struct mlx5_cqe64 *cqe, int budget) { - struct sk_buff *skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); + u16 skb_id = PTP_WQE_CTR2IDX(be16_to_cpu(cqe->wqe_counter)); + u16 skb_cc = PTP_WQE_CTR2IDX(ptpsq->skb_fifo_cc); struct mlx5e_txqsq *sq = &ptpsq->txqsq; + struct sk_buff *skb; ktime_t hwtstamp; if (unlikely(MLX5E_RX_ERR_CQE(cqe))) { + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); ptpsq->cq_stats->err_cqe++; goto out; } + if (mlx5e_ptp_ts_cqe_drop(ptpsq, skb_cc, skb_id)) + mlx5e_ptp_skb_fifo_ts_cqe_resync(ptpsq, skb_cc, skb_id); + + skb = mlx5e_skb_fifo_pop(&ptpsq->skb_fifo); hwtstamp = mlx5e_cqe_ts_to_ns(sq->ptp_cyc2time, sq->clock, get_cqe_ts(cqe)); mlx5e_skb_cb_hwtstamp_handler(skb, MLX5E_SKB_CB_PORT_HWTSTAMP, hwtstamp, ptpsq->cq_stats); @@ -241,6 +271,7 @@ static void mlx5e_ptp_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa) { int wq_sz = mlx5_wq_cyc_get_size(&ptpsq->txqsq.wq); + struct mlx5_core_dev *mdev = ptpsq->txqsq.mdev; ptpsq->skb_fifo.fifo = kvzalloc_node(array_size(wq_sz, sizeof(*ptpsq->skb_fifo.fifo)), GFP_KERNEL, numa); @@ -250,7 +281,9 @@ static int mlx5e_ptp_alloc_traffic_db(struct mlx5e_ptpsq *ptpsq, int numa) ptpsq->skb_fifo.pc = &ptpsq->skb_fifo_pc; ptpsq->skb_fifo.cc = &ptpsq->skb_fifo_cc; ptpsq->skb_fifo.mask = wq_sz - 1; - + if (MLX5_CAP_GEN_2(mdev, ts_cqe_metadata_size2wqe_counter)) + ptpsq->ts_cqe_ctr_mask = + (1 << MLX5_CAP_GEN_2(mdev, ts_cqe_metadata_size2wqe_counter)) - 1; return 0; } @@ -591,7 +624,7 @@ static int mlx5e_ptp_set_state(struct mlx5e_ptp *c, struct mlx5e_params *params) static void mlx5e_ptp_rx_unset_fs(struct mlx5e_priv *priv) { - struct mlx5e_ptp_fs *ptp_fs = priv->fs.ptp_fs; + struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs; if (!ptp_fs->valid) return; @@ -608,7 +641,7 @@ static void mlx5e_ptp_rx_unset_fs(struct mlx5e_priv *priv) static int mlx5e_ptp_rx_set_fs(struct mlx5e_priv *priv) { u32 tirn = mlx5e_rx_res_get_tirn_ptp(priv->rx_res); - struct mlx5e_ptp_fs *ptp_fs = priv->fs.ptp_fs; + struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs; struct mlx5_flow_handle *rule; int err; @@ -775,13 +808,13 @@ int mlx5e_ptp_alloc_rx_fs(struct mlx5e_priv *priv) if (!ptp_fs) return -ENOMEM; - priv->fs.ptp_fs = ptp_fs; + priv->fs->ptp_fs = ptp_fs; return 0; } void mlx5e_ptp_free_rx_fs(struct mlx5e_priv *priv) { - struct mlx5e_ptp_fs *ptp_fs = priv->fs.ptp_fs; + struct mlx5e_ptp_fs *ptp_fs = priv->fs->ptp_fs; if (!mlx5e_profile_feature_cap(priv->profile, PTP_RX)) return; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h index a71a32e00ebb..92dbbec472ec 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/ptp.h @@ -17,6 +17,7 @@ struct mlx5e_ptpsq { u16 skb_fifo_pc; struct mlx5e_skb_fifo skb_fifo; struct mlx5e_ptp_cq_stats *cq_stats; + u16 ts_cqe_ctr_mask; }; enum { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c index 9db677e9ca9c..2842195ee548 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.c @@ -2,11 +2,16 @@ /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ #include <net/sch_generic.h> +#include <net/pkt_cls.h> #include "en.h" #include "params.h" #include "../qos.h" +#include "en/htb.h" -#define BYTES_IN_MBIT 125000 +struct qos_sq_callback_params { + struct mlx5e_priv *priv; + struct mlx5e_channels *chs; +}; int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes) { @@ -28,124 +33,14 @@ int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev) return min(MLX5E_QOS_MAX_LEAF_NODES, mlx5_qos_max_leaf_nodes(mdev)); } -int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv) -{ - int last = find_last_bit(priv->htb.qos_used_qids, mlx5e_qos_max_leaf_nodes(priv->mdev)); - - return last == mlx5e_qos_max_leaf_nodes(priv->mdev) ? 0 : last + 1; -} - -/* Software representation of the QoS tree (internal to this file) */ - -static int mlx5e_find_unused_qos_qid(struct mlx5e_priv *priv) -{ - int size = mlx5e_qos_max_leaf_nodes(priv->mdev); - int res; - - WARN_ONCE(!mutex_is_locked(&priv->state_lock), "%s: state_lock is not held\n", __func__); - res = find_first_zero_bit(priv->htb.qos_used_qids, size); - - return res == size ? -ENOSPC : res; -} - -struct mlx5e_qos_node { - struct hlist_node hnode; - struct mlx5e_qos_node *parent; - u64 rate; - u32 bw_share; - u32 max_average_bw; - u32 hw_id; - u32 classid; /* 16-bit, except root. */ - u16 qid; -}; - -#define MLX5E_QOS_QID_INNER 0xffff -#define MLX5E_HTB_CLASSID_ROOT 0xffffffff - -static struct mlx5e_qos_node * -mlx5e_sw_node_create_leaf(struct mlx5e_priv *priv, u16 classid, u16 qid, - struct mlx5e_qos_node *parent) -{ - struct mlx5e_qos_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->parent = parent; - - node->qid = qid; - __set_bit(qid, priv->htb.qos_used_qids); - - node->classid = classid; - hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, classid); - - mlx5e_update_tx_netdev_queues(priv); - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_create_root(struct mlx5e_priv *priv) -{ - struct mlx5e_qos_node *node; - - node = kzalloc(sizeof(*node), GFP_KERNEL); - if (!node) - return ERR_PTR(-ENOMEM); - - node->qid = MLX5E_QOS_QID_INNER; - node->classid = MLX5E_HTB_CLASSID_ROOT; - hash_add_rcu(priv->htb.qos_tc2node, &node->hnode, node->classid); - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find(struct mlx5e_priv *priv, u32 classid) -{ - struct mlx5e_qos_node *node = NULL; - - hash_for_each_possible(priv->htb.qos_tc2node, node, hnode, classid) { - if (node->classid == classid) - break; - } - - return node; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find_rcu(struct mlx5e_priv *priv, u32 classid) -{ - struct mlx5e_qos_node *node = NULL; - - hash_for_each_possible_rcu(priv->htb.qos_tc2node, node, hnode, classid) { - if (node->classid == classid) - break; - } - - return node; -} - -static void mlx5e_sw_node_delete(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) -{ - hash_del_rcu(&node->hnode); - if (node->qid != MLX5E_QOS_QID_INNER) { - __clear_bit(node->qid, priv->htb.qos_used_qids); - mlx5e_update_tx_netdev_queues(priv); - } - /* Make sure this qid is no longer selected by mlx5e_select_queue, so - * that mlx5e_reactivate_qos_sq can safely restart the netdev TX queue. - */ - synchronize_net(); - kfree(node); -} - /* TX datapath API */ -static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) +u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) { /* These channel params are safe to access from the datapath, because: - * 1. This function is called only after checking priv->htb.maj_id != 0, + * 1. This function is called only after checking selq->htb_maj_id != 0, * and the number of queues can't change while HTB offload is active. - * 2. When priv->htb.maj_id becomes 0, synchronize_rcu waits for + * 2. When selq->htb_maj_id becomes 0, synchronize_rcu waits for * mlx5e_select_queue to finish while holding priv->state_lock, * preventing other code from changing the number of queues. */ @@ -154,30 +49,7 @@ static u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid) return (chs->params.num_channels + is_ptp) * mlx5e_get_dcb_num_tc(&chs->params) + qid; } -int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid) -{ - struct mlx5e_qos_node *node; - u16 qid; - int res; - - rcu_read_lock(); - - node = mlx5e_sw_node_find_rcu(priv, classid); - if (!node) { - res = -ENOENT; - goto out; - } - qid = READ_ONCE(node->qid); - if (qid == MLX5E_QOS_QID_INNER) { - res = -EINVAL; - goto out; - } - res = mlx5e_qid_from_qos(&priv->channels, qid); - -out: - rcu_read_unlock(); - return res; -} +/* SQ lifecycle */ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) { @@ -194,10 +66,8 @@ static struct mlx5e_txqsq *mlx5e_get_qos_sq(struct mlx5e_priv *priv, int qid) return mlx5e_state_dereference(priv, qos_sqs[qid]); } -/* SQ lifecycle */ - -static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, - struct mlx5e_qos_node *node) +int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + u16 node_qid, u32 hw_id) { struct mlx5e_create_cq_param ccp = {}; struct mlx5e_txqsq __rcu **qos_sqs; @@ -210,13 +80,13 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs params = &chs->params; - txq_ix = mlx5e_qid_from_qos(chs, node->qid); + txq_ix = mlx5e_qid_from_qos(chs, node_qid); - WARN_ON(node->qid > priv->htb.max_qos_sqs); - if (node->qid == priv->htb.max_qos_sqs) { + WARN_ON(node_qid > priv->htb_max_qos_sqs); + if (node_qid == priv->htb_max_qos_sqs) { struct mlx5e_sq_stats *stats, **stats_list = NULL; - if (priv->htb.max_qos_sqs == 0) { + if (priv->htb_max_qos_sqs == 0) { stats_list = kvcalloc(mlx5e_qos_max_leaf_nodes(priv->mdev), sizeof(*stats_list), GFP_KERNEL); @@ -229,16 +99,16 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs return -ENOMEM; } if (stats_list) - WRITE_ONCE(priv->htb.qos_sq_stats, stats_list); - WRITE_ONCE(priv->htb.qos_sq_stats[node->qid], stats); - /* Order max_qos_sqs increment after writing the array pointer. + WRITE_ONCE(priv->htb_qos_sq_stats, stats_list); + WRITE_ONCE(priv->htb_qos_sq_stats[node_qid], stats); + /* Order htb_max_qos_sqs increment after writing the array pointer. * Pairs with smp_load_acquire in en_stats.c. */ - smp_store_release(&priv->htb.max_qos_sqs, priv->htb.max_qos_sqs + 1); + smp_store_release(&priv->htb_max_qos_sqs, priv->htb_max_qos_sqs + 1); } - ix = node->qid % params->num_channels; - qid = node->qid / params->num_channels; + ix = node_qid % params->num_channels; + qid = node_qid / params->num_channels; c = chs->c[ix]; qos_sqs = mlx5e_state_dereference(priv, c->qos_sqs); @@ -257,8 +127,8 @@ static int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs if (err) goto err_free_sq; err = mlx5e_open_txqsq(c, priv->tisn[c->lag_port][0], txq_ix, params, - ¶m_sq, sq, 0, node->hw_id, - priv->htb.qos_sq_stats[node->qid]); + ¶m_sq, sq, 0, hw_id, + priv->htb_qos_sq_stats[node_qid]); if (err) goto err_close_cq; @@ -273,14 +143,22 @@ err_free_sq: return err; } -static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node *node) +static int mlx5e_open_qos_sq_cb_wrapper(void *data, u16 node_qid, u32 hw_id) +{ + struct qos_sq_callback_params *cb_params = data; + + return mlx5e_open_qos_sq(cb_params->priv, cb_params->chs, node_qid, hw_id); +} + +int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id) { + struct mlx5e_priv *priv = data; struct mlx5e_txqsq *sq; u16 qid; - sq = mlx5e_get_qos_sq(priv, node->qid); + sq = mlx5e_get_qos_sq(priv, node_qid); - qid = mlx5e_qid_from_qos(&priv->channels, node->qid); + qid = mlx5e_qid_from_qos(&priv->channels, node_qid); /* If it's a new queue, it will be marked as started at this point. * Stop it before updating txq2sq. @@ -295,11 +173,13 @@ static void mlx5e_activate_qos_sq(struct mlx5e_priv *priv, struct mlx5e_qos_node */ smp_wmb(); - qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node->qid); + qos_dbg(priv->mdev, "Activate QoS SQ qid %u\n", node_qid); mlx5e_activate_txqsq(sq); + + return 0; } -static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) +void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) { struct mlx5e_txqsq *sq; @@ -319,7 +199,7 @@ static void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid) smp_wmb(); } -static void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) +void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid) { struct mlx5e_txqsq __rcu **qos_sqs; struct mlx5e_params *params; @@ -369,7 +249,7 @@ void mlx5e_qos_close_queues(struct mlx5e_channel *c) kvfree(qos_sqs); } -static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) +void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) { int i; @@ -377,7 +257,7 @@ static void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs) mlx5e_qos_close_queues(chs->c[i]); } -static int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) +int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { u16 qos_sqs_size; int i; @@ -413,24 +293,20 @@ err_free: int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) { - struct mlx5e_qos_node *node = NULL; - int bkt, err; - - if (!priv->htb.maj_id) - return 0; + struct qos_sq_callback_params callback_params; + int err; err = mlx5e_qos_alloc_queues(priv, chs); if (err) return err; - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { - if (node->qid == MLX5E_QOS_QID_INNER) - continue; - err = mlx5e_open_qos_sq(priv, chs, node); - if (err) { - mlx5e_qos_close_all_queues(chs); - return err; - } + callback_params.priv = priv; + callback_params.chs = chs; + + err = mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_open_qos_sq_cb_wrapper, &callback_params); + if (err) { + mlx5e_qos_close_all_queues(chs); + return err; } return 0; @@ -438,14 +314,7 @@ int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs) void mlx5e_qos_activate_queues(struct mlx5e_priv *priv) { - struct mlx5e_qos_node *node = NULL; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) { - if (node->qid == MLX5E_QOS_QID_INNER) - continue; - mlx5e_activate_qos_sq(priv, node); - } + mlx5e_htb_enumerate_leaves(priv->htb, mlx5e_activate_qos_sq, priv); } void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) @@ -474,7 +343,7 @@ void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c) } } -static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) +void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) { int i; @@ -482,293 +351,14 @@ static void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs) mlx5e_qos_deactivate_queues(chs->c[i]); } -/* HTB API */ - -int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *root; - bool opened; - int err; - - qos_dbg(priv->mdev, "TC_HTB_CREATE handle %04x:, default :%04x\n", htb_maj_id, htb_defcls); - - if (!mlx5_qos_is_supported(priv->mdev)) { - NL_SET_ERR_MSG_MOD(extack, - "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); - return -EOPNOTSUPP; - } - - opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (opened) { - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, true); - - err = mlx5e_qos_alloc_queues(priv, &priv->channels); - if (err) - goto err_cancel_selq; - } - - root = mlx5e_sw_node_create_root(priv); - if (IS_ERR(root)) { - err = PTR_ERR(root); - goto err_free_queues; - } - - err = mlx5_qos_create_root_node(priv->mdev, &root->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error. Try upgrading firmware."); - goto err_sw_node_delete; - } - - WRITE_ONCE(priv->htb.defcls, htb_defcls); - /* Order maj_id after defcls - pairs with - * mlx5e_select_queue/mlx5e_select_htb_queues. - */ - smp_store_release(&priv->htb.maj_id, htb_maj_id); - - if (opened) - mlx5e_selq_apply(&priv->selq); - - return 0; - -err_sw_node_delete: - mlx5e_sw_node_delete(priv, root); - -err_free_queues: - if (opened) - mlx5e_qos_close_all_queues(&priv->channels); -err_cancel_selq: - mlx5e_selq_cancel(&priv->selq); - return err; -} - -int mlx5e_htb_root_del(struct mlx5e_priv *priv) -{ - struct mlx5e_qos_node *root; - int err; - - qos_dbg(priv->mdev, "TC_HTB_DESTROY\n"); - - /* Wait until real_num_tx_queues is updated for mlx5e_select_queue, - * so that we can safely switch to its non-HTB non-PTP fastpath. - */ - synchronize_net(); - - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, false); - mlx5e_selq_apply(&priv->selq); - - WRITE_ONCE(priv->htb.maj_id, 0); - - root = mlx5e_sw_node_find(priv, MLX5E_HTB_CLASSID_ROOT); - if (!root) { - qos_err(priv->mdev, "Failed to find the root node in the QoS tree\n"); - return -ENOENT; - } - err = mlx5_qos_destroy_node(priv->mdev, root->hw_id); - if (err) - qos_err(priv->mdev, "Failed to destroy root node %u, err = %d\n", - root->hw_id, err); - mlx5e_sw_node_delete(priv, root); - - mlx5e_qos_deactivate_all_queues(&priv->channels); - mlx5e_qos_close_all_queues(&priv->channels); - - return err; -} - -static int mlx5e_htb_convert_rate(struct mlx5e_priv *priv, u64 rate, - struct mlx5e_qos_node *parent, u32 *bw_share) -{ - u64 share = 0; - - while (parent->classid != MLX5E_HTB_CLASSID_ROOT && !parent->max_average_bw) - parent = parent->parent; - - if (parent->max_average_bw) - share = div64_u64(div_u64(rate * 100, BYTES_IN_MBIT), - parent->max_average_bw); - else - share = 101; - - *bw_share = share == 0 ? 1 : share > 100 ? 0 : share; - - qos_dbg(priv->mdev, "Convert: rate %llu, parent ceil %llu -> bw_share %u\n", - rate, (u64)parent->max_average_bw * BYTES_IN_MBIT, *bw_share); - - return 0; -} - -static void mlx5e_htb_convert_ceil(struct mlx5e_priv *priv, u64 ceil, u32 *max_average_bw) -{ - /* Hardware treats 0 as "unlimited", set at least 1. */ - *max_average_bw = max_t(u32, div_u64(ceil, BYTES_IN_MBIT), 1); - - qos_dbg(priv->mdev, "Convert: ceil %llu -> max_average_bw %u\n", - ceil, *max_average_bw); -} - -int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, - u32 parent_classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *parent; - int qid; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_ALLOC_QUEUE classid %04x, parent %04x, rate %llu, ceil %llu\n", - classid, parent_classid, rate, ceil); - - qid = mlx5e_find_unused_qos_qid(priv); - if (qid < 0) { - NL_SET_ERR_MSG_MOD(extack, "Maximum amount of leaf classes is reached."); - return qid; - } - - parent = mlx5e_sw_node_find(priv, parent_classid); - if (!parent) - return -EINVAL; - - node = mlx5e_sw_node_create_leaf(priv, classid, qid, parent); - if (IS_ERR(node)) - return PTR_ERR(node); - - node->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node->parent, &node->bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &node->max_average_bw); - - err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->hw_id, - node->bw_share, node->max_average_bw, - &node->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - mlx5e_sw_node_delete(priv, node); - return err; - } - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, node); - } - } - - return mlx5e_qid_from_qos(&priv->channels, node->qid); -} - -int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, - u64 rate, u64 ceil, struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *child; - int err, tmp_err; - u32 new_hw_id; - u16 qid; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_TO_INNER classid %04x, upcoming child %04x, rate %llu, ceil %llu\n", - classid, child_classid, rate, ceil); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - err = mlx5_qos_create_inner_node(priv->mdev, node->parent->hw_id, - node->bw_share, node->max_average_bw, - &new_hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating an inner node."); - qos_err(priv->mdev, "Failed to create an inner node (class %04x), err = %d\n", - classid, err); - return err; - } - - /* Intentionally reuse the qid for the upcoming first child. */ - child = mlx5e_sw_node_create_leaf(priv, child_classid, node->qid, node); - if (IS_ERR(child)) { - err = PTR_ERR(child); - goto err_destroy_hw_node; - } - - child->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node, &child->bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &child->max_average_bw); - - err = mlx5_qos_create_leaf_node(priv->mdev, new_hw_id, child->bw_share, - child->max_average_bw, &child->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - goto err_delete_sw_node; - } - - /* No fail point. */ - - qid = node->qid; - /* Pairs with mlx5e_get_txq_by_classid. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - node->hw_id = new_hw_id; - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, child); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, child); - } - } - - return 0; - -err_delete_sw_node: - child->qid = MLX5E_QOS_QID_INNER; - mlx5e_sw_node_delete(priv, child); - -err_destroy_hw_node: - tmp_err = mlx5_qos_destroy_node(priv->mdev, new_hw_id); - if (tmp_err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to roll back creation of an inner node %u (class %04x), err = %d\n", - new_hw_id, classid, tmp_err); - return err; -} - -static struct mlx5e_qos_node *mlx5e_sw_node_find_by_qid(struct mlx5e_priv *priv, u16 qid) -{ - struct mlx5e_qos_node *node = NULL; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, node, hnode) - if (node->qid == qid) - break; - - return node; -} - -static void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) +void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq) { qos_dbg(priv->mdev, "Reactivate QoS SQ qid %u\n", qid); netdev_tx_reset_queue(txq); netif_tx_start_queue(txq); } -static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) +void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) { struct netdev_queue *dev_queue = netdev_get_tx_queue(dev, qid); struct Qdisc *qdisc = dev_queue->qdisc_sleeping; @@ -781,251 +371,65 @@ static void mlx5e_reset_qdisc(struct net_device *dev, u16 qid) spin_unlock_bh(qdisc_lock(qdisc)); } -int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 *classid, - struct netlink_ext_ack *extack) +int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb_qopt) { - struct mlx5e_qos_node *node; - struct netdev_queue *txq; - u16 qid, moved_qid; - bool opened; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL classid %04x\n", *classid); - - node = mlx5e_sw_node_find(priv, *classid); - if (!node) - return -ENOENT; - - /* Store qid for reuse. */ - qid = node->qid; - - opened = test_bit(MLX5E_STATE_OPENED, &priv->state); - if (opened) { - txq = netdev_get_tx_queue(priv->netdev, - mlx5e_qid_from_qos(&priv->channels, qid)); - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, *classid, err); - - mlx5e_sw_node_delete(priv, node); - - moved_qid = mlx5e_qos_cur_leaf_nodes(priv); - - if (moved_qid == 0) { - /* The last QoS SQ was just destroyed. */ - if (opened) - mlx5e_reactivate_qos_sq(priv, qid, txq); - return 0; - } - moved_qid--; - - if (moved_qid < qid) { - /* The highest QoS SQ was just destroyed. */ - WARN(moved_qid != qid - 1, "Gaps in queue numeration: destroyed queue %u, the highest queue is %u", - qid, moved_qid); - if (opened) - mlx5e_reactivate_qos_sq(priv, qid, txq); - return 0; - } - - WARN(moved_qid == qid, "Can't move node with qid %u to itself", qid); - qos_dbg(priv->mdev, "Moving QoS SQ %u to %u\n", moved_qid, qid); - - node = mlx5e_sw_node_find_by_qid(priv, moved_qid); - WARN(!node, "Could not find a node with qid %u to move to queue %u", - moved_qid, qid); - - /* Stop traffic to the old queue. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - __clear_bit(moved_qid, priv->htb.qos_used_qids); - - if (opened) { - txq = netdev_get_tx_queue(priv->netdev, - mlx5e_qid_from_qos(&priv->channels, moved_qid)); - mlx5e_deactivate_qos_sq(priv, moved_qid); - mlx5e_close_qos_sq(priv, moved_qid); - } - - /* Prevent packets from the old class from getting into the new one. */ - mlx5e_reset_qdisc(priv->netdev, moved_qid); - - __set_bit(qid, priv->htb.qos_used_qids); - WRITE_ONCE(node->qid, qid); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x) while moving qid %u to %u, err = %d\n", - node->classid, moved_qid, qid, err); - } else { - mlx5e_activate_qos_sq(priv, node); - } - } - - mlx5e_update_tx_netdev_queues(priv); - if (opened) - mlx5e_reactivate_qos_sq(priv, moved_qid, txq); - - *classid = node->classid; - return 0; -} - -int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *node, *parent; - u32 old_hw_id, new_hw_id; - int err, saved_err = 0; - u16 qid; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_DEL_LAST%s classid %04x\n", - force ? "_FORCE" : "", classid); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - err = mlx5_qos_create_leaf_node(priv->mdev, node->parent->parent->hw_id, - node->parent->bw_share, - node->parent->max_average_bw, - &new_hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when creating a leaf node."); - qos_err(priv->mdev, "Failed to create a leaf node (class %04x), err = %d\n", - classid, err); - if (!force) - return err; - saved_err = err; - } - - /* Store qid for reuse and prevent clearing the bit. */ - qid = node->qid; - /* Pairs with mlx5e_get_txq_by_classid. */ - WRITE_ONCE(node->qid, MLX5E_QOS_QID_INNER); - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - mlx5e_deactivate_qos_sq(priv, qid); - mlx5e_close_qos_sq(priv, qid); - } - - /* Prevent packets from the old class from getting into the new one. */ - mlx5e_reset_qdisc(priv->netdev, qid); - - err = mlx5_qos_destroy_node(priv->mdev, node->hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - parent = node->parent; - mlx5e_sw_node_delete(priv, node); + struct mlx5e_htb *htb = priv->htb; + int res; - node = parent; - WRITE_ONCE(node->qid, qid); + if (!htb && htb_qopt->command != TC_HTB_CREATE) + return -EINVAL; - /* Early return on error in force mode. Parent will still be an inner - * node to be deleted by a following delete operation. - */ - if (saved_err) - return saved_err; - - old_hw_id = node->hw_id; - node->hw_id = new_hw_id; - - if (test_bit(MLX5E_STATE_OPENED, &priv->state)) { - err = mlx5e_open_qos_sq(priv, &priv->channels, node); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Error creating an SQ."); - qos_warn(priv->mdev, "Failed to create a QoS SQ (class %04x), err = %d\n", - classid, err); - } else { - mlx5e_activate_qos_sq(priv, node); + switch (htb_qopt->command) { + case TC_HTB_CREATE: + if (!mlx5_qos_is_supported(priv->mdev)) { + NL_SET_ERR_MSG_MOD(htb_qopt->extack, + "Missing QoS capabilities. Try disabling SRIOV or use a supported device."); + return -EOPNOTSUPP; } - } - - err = mlx5_qos_destroy_node(priv->mdev, old_hw_id); - if (err) /* Not fatal. */ - qos_warn(priv->mdev, "Failed to destroy leaf node %u (class %04x), err = %d\n", - node->hw_id, classid, err); - - return 0; -} - -static int mlx5e_qos_update_children(struct mlx5e_priv *priv, struct mlx5e_qos_node *node, - struct netlink_ext_ack *extack) -{ - struct mlx5e_qos_node *child; - int err = 0; - int bkt; - - hash_for_each(priv->htb.qos_tc2node, bkt, child, hnode) { - u32 old_bw_share = child->bw_share; - int err_one; - - if (child->parent != node) - continue; - - mlx5e_htb_convert_rate(priv, child->rate, node, &child->bw_share); - if (child->bw_share == old_bw_share) - continue; - - err_one = mlx5_qos_update_node(priv->mdev, child->hw_id, child->bw_share, - child->max_average_bw, child->hw_id); - if (!err && err_one) { - err = err_one; - - NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a child node."); - qos_err(priv->mdev, "Failed to modify a child node (class %04x), err = %d\n", - node->classid, err); + priv->htb = mlx5e_htb_alloc(); + htb = priv->htb; + if (!htb) + return -ENOMEM; + res = mlx5e_htb_init(htb, htb_qopt, priv->netdev, priv->mdev, &priv->selq, priv); + if (res) { + mlx5e_htb_free(htb); + priv->htb = NULL; } + return res; + case TC_HTB_DESTROY: + mlx5e_htb_cleanup(htb); + mlx5e_htb_free(htb); + priv->htb = NULL; + return 0; + case TC_HTB_LEAF_ALLOC_QUEUE: + res = mlx5e_htb_leaf_alloc_queue(htb, htb_qopt->classid, htb_qopt->parent_classid, + htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); + if (res < 0) + return res; + htb_qopt->qid = res; + return 0; + case TC_HTB_LEAF_TO_INNER: + return mlx5e_htb_leaf_to_inner(htb, htb_qopt->parent_classid, htb_qopt->classid, + htb_qopt->rate, htb_qopt->ceil, htb_qopt->extack); + case TC_HTB_LEAF_DEL: + return mlx5e_htb_leaf_del(htb, &htb_qopt->classid, htb_qopt->extack); + case TC_HTB_LEAF_DEL_LAST: + case TC_HTB_LEAF_DEL_LAST_FORCE: + return mlx5e_htb_leaf_del_last(htb, htb_qopt->classid, + htb_qopt->command == TC_HTB_LEAF_DEL_LAST_FORCE, + htb_qopt->extack); + case TC_HTB_NODE_MODIFY: + return mlx5e_htb_node_modify(htb, htb_qopt->classid, htb_qopt->rate, htb_qopt->ceil, + htb_qopt->extack); + case TC_HTB_LEAF_QUERY_QUEUE: + res = mlx5e_htb_get_txq_by_classid(htb, htb_qopt->classid); + if (res < 0) + return res; + htb_qopt->qid = res; + return 0; + default: + return -EOPNOTSUPP; } - - return err; -} - -int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack) -{ - u32 bw_share, max_average_bw; - struct mlx5e_qos_node *node; - bool ceil_changed = false; - int err; - - qos_dbg(priv->mdev, "TC_HTB_LEAF_MODIFY classid %04x, rate %llu, ceil %llu\n", - classid, rate, ceil); - - node = mlx5e_sw_node_find(priv, classid); - if (!node) - return -ENOENT; - - node->rate = rate; - mlx5e_htb_convert_rate(priv, rate, node->parent, &bw_share); - mlx5e_htb_convert_ceil(priv, ceil, &max_average_bw); - - err = mlx5_qos_update_node(priv->mdev, node->parent->hw_id, bw_share, - max_average_bw, node->hw_id); - if (err) { - NL_SET_ERR_MSG_MOD(extack, "Firmware error when modifying a node."); - qos_err(priv->mdev, "Failed to modify a node (class %04x), err = %d\n", - classid, err); - return err; - } - - if (max_average_bw != node->max_average_bw) - ceil_changed = true; - - node->bw_share = bw_share; - node->max_average_bw = max_average_bw; - - if (ceil_changed) - err = mlx5e_qos_update_children(priv, node, extack); - - return err; } struct mlx5e_mqprio_rl { @@ -1111,3 +515,4 @@ int mlx5e_mqprio_rl_get_node_hw_id(struct mlx5e_mqprio_rl *rl, int tc, u32 *hw_i *hw_id = rl->leaves_id[tc]; return 0; } + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h index 5d9bd91d86c2..4947afa23b73 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/qos.h @@ -6,40 +6,39 @@ #include <linux/mlx5/driver.h> -#define MLX5E_QOS_MAX_LEAF_NODES 256 +#define BYTES_IN_MBIT 125000 struct mlx5e_priv; +struct mlx5e_htb; struct mlx5e_channels; struct mlx5e_channel; +struct tc_htb_qopt_offload; int mlx5e_qos_bytes_rate_check(struct mlx5_core_dev *mdev, u64 nbytes); int mlx5e_qos_max_leaf_nodes(struct mlx5_core_dev *mdev); -int mlx5e_qos_cur_leaf_nodes(struct mlx5e_priv *priv); - -/* TX datapath API */ -int mlx5e_get_txq_by_classid(struct mlx5e_priv *priv, u16 classid); /* SQ lifecycle */ +int mlx5e_open_qos_sq(struct mlx5e_priv *priv, struct mlx5e_channels *chs, + u16 node_qid, u32 hw_id); +int mlx5e_activate_qos_sq(void *data, u16 node_qid, u32 hw_id); +void mlx5e_deactivate_qos_sq(struct mlx5e_priv *priv, u16 qid); +void mlx5e_close_qos_sq(struct mlx5e_priv *priv, u16 qid); +void mlx5e_reactivate_qos_sq(struct mlx5e_priv *priv, u16 qid, struct netdev_queue *txq); +void mlx5e_reset_qdisc(struct net_device *dev, u16 qid); + int mlx5e_qos_open_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); void mlx5e_qos_activate_queues(struct mlx5e_priv *priv); void mlx5e_qos_deactivate_queues(struct mlx5e_channel *c); +void mlx5e_qos_deactivate_all_queues(struct mlx5e_channels *chs); void mlx5e_qos_close_queues(struct mlx5e_channel *c); +void mlx5e_qos_close_all_queues(struct mlx5e_channels *chs); +int mlx5e_qos_alloc_queues(struct mlx5e_priv *priv, struct mlx5e_channels *chs); + +/* TX datapath API */ +u16 mlx5e_qid_from_qos(struct mlx5e_channels *chs, u16 qid); /* HTB API */ -int mlx5e_htb_root_add(struct mlx5e_priv *priv, u16 htb_maj_id, u16 htb_defcls, - struct netlink_ext_ack *extack); -int mlx5e_htb_root_del(struct mlx5e_priv *priv); -int mlx5e_htb_leaf_alloc_queue(struct mlx5e_priv *priv, u16 classid, - u32 parent_classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_to_inner(struct mlx5e_priv *priv, u16 classid, u16 child_classid, - u64 rate, u64 ceil, struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_del(struct mlx5e_priv *priv, u16 *classid, - struct netlink_ext_ack *extack); -int mlx5e_htb_leaf_del_last(struct mlx5e_priv *priv, u16 classid, bool force, - struct netlink_ext_ack *extack); -int mlx5e_htb_node_modify(struct mlx5e_priv *priv, u16 classid, u64 rate, u64 ceil, - struct netlink_ext_ack *extack); +int mlx5e_htb_setup_tc(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb); /* MQPRIO TX rate limit */ struct mlx5e_mqprio_rl; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c index 48dc121b2cb4..39ef2a2561a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/bridge.c @@ -269,6 +269,12 @@ mlx5_esw_bridge_port_obj_attr_set(struct net_device *dev, err = mlx5_esw_bridge_vlan_filtering_set(vport_num, esw_owner_vhca_id, attr->u.vlan_filtering, br_offloads); break; + case SWITCHDEV_ATTR_ID_BRIDGE_VLAN_PROTOCOL: + err = mlx5_esw_bridge_vlan_proto_set(vport_num, + esw_owner_vhca_id, + attr->u.vlan_protocol, + br_offloads); + break; default: err = -EOPNOTSUPP; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c index 86fa0bdbee36..fac7e3ff2674 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/rep/tc.c @@ -21,6 +21,7 @@ #include "en/tc/sample.h" #include "en_accel/ipsec_rxtx.h" #include "en/tc/int_port.h" +#include "en/tc/act/act.h" struct mlx5e_rep_indr_block_priv { struct net_device *netdev; @@ -511,6 +512,120 @@ mlx5e_rep_indr_setup_block(struct net_device *netdev, struct Qdisc *sch, return 0; } +static int +mlx5e_rep_indr_replace_act(struct mlx5e_rep_priv *rpriv, + struct flow_offload_action *fl_act) + +{ + struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + enum mlx5_flow_namespace_type ns_type; + struct flow_action_entry *action; + struct mlx5e_tc_act *act; + bool add = false; + int i; + + /* There is no use case currently for more than one action (e.g. pedit). + * when there will be, need to handle cleaning multiple actions on err. + */ + if (!flow_offload_has_one_action(&fl_act->action)) + return -EOPNOTSUPP; + + if (esw && esw->mode == MLX5_ESWITCH_OFFLOADS) + ns_type = MLX5_FLOW_NAMESPACE_FDB; + else + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + + flow_action_for_each(i, action, &fl_act->action) { + act = mlx5e_tc_act_get(action->id, ns_type); + if (!act) + continue; + + if (!act->offload_action) + continue; + + if (!act->offload_action(priv, fl_act, action)) + add = true; + } + + return add ? 0 : -EOPNOTSUPP; +} + +static int +mlx5e_rep_indr_destroy_act(struct mlx5e_rep_priv *rpriv, + struct flow_offload_action *fl_act) +{ + struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + enum mlx5_flow_namespace_type ns_type; + struct mlx5e_tc_act *act; + + if (esw && esw->mode == MLX5_ESWITCH_OFFLOADS) + ns_type = MLX5_FLOW_NAMESPACE_FDB; + else + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + + act = mlx5e_tc_act_get(fl_act->id, ns_type); + if (!act || !act->destroy_action) + return -EOPNOTSUPP; + + return act->destroy_action(priv, fl_act); +} + +static int +mlx5e_rep_indr_stats_act(struct mlx5e_rep_priv *rpriv, + struct flow_offload_action *fl_act) + +{ + struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); + struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + enum mlx5_flow_namespace_type ns_type; + struct mlx5e_tc_act *act; + + if (esw && esw->mode == MLX5_ESWITCH_OFFLOADS) + ns_type = MLX5_FLOW_NAMESPACE_FDB; + else + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + + act = mlx5e_tc_act_get(fl_act->id, ns_type); + if (!act || !act->stats_action) + return -EOPNOTSUPP; + + return act->stats_action(priv, fl_act); +} + +static int +mlx5e_rep_indr_setup_act(struct mlx5e_rep_priv *rpriv, + struct flow_offload_action *fl_act) +{ + switch (fl_act->command) { + case FLOW_ACT_REPLACE: + return mlx5e_rep_indr_replace_act(rpriv, fl_act); + case FLOW_ACT_DESTROY: + return mlx5e_rep_indr_destroy_act(rpriv, fl_act); + case FLOW_ACT_STATS: + return mlx5e_rep_indr_stats_act(rpriv, fl_act); + default: + return -EOPNOTSUPP; + } +} + +static int +mlx5e_rep_indr_no_dev_setup(struct mlx5e_rep_priv *rpriv, + enum tc_setup_type type, + void *data) +{ + if (!data) + return -EOPNOTSUPP; + + switch (type) { + case TC_SETUP_ACT: + return mlx5e_rep_indr_setup_act(rpriv, data); + default: + return -EOPNOTSUPP; + } +} + static int mlx5e_rep_indr_setup_cb(struct net_device *netdev, struct Qdisc *sch, void *cb_priv, enum tc_setup_type type, void *type_data, @@ -518,7 +633,7 @@ int mlx5e_rep_indr_setup_cb(struct net_device *netdev, struct Qdisc *sch, void * void (*cleanup)(struct flow_block_cb *block_cb)) { if (!netdev) - return -EOPNOTSUPP; + return mlx5e_rep_indr_no_dev_setup(cb_priv, type, data); switch (type) { case TC_SETUP_BLOCK: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c index d98a277eb7f8..f675b1926340 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.c @@ -7,6 +7,7 @@ #include <linux/rcupdate.h> #include "en.h" #include "en/ptp.h" +#include "en/htb.h" struct mlx5e_selq_params { unsigned int num_regular_queues; @@ -19,6 +20,8 @@ struct mlx5e_selq_params { bool is_ptp : 1; }; }; + u16 htb_maj_id; + u16 htb_defcls; }; int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock) @@ -44,6 +47,8 @@ int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock) .num_tcs = 1, .is_htb = false, .is_ptp = false, + .htb_maj_id = 0, + .htb_defcls = 0, }; rcu_assign_pointer(selq->active, init_params); @@ -64,21 +69,50 @@ void mlx5e_selq_cleanup(struct mlx5e_selq *selq) selq->standby = NULL; } -void mlx5e_selq_prepare(struct mlx5e_selq *selq, struct mlx5e_params *params, bool htb) +void mlx5e_selq_prepare_params(struct mlx5e_selq *selq, struct mlx5e_params *params) { + struct mlx5e_selq_params *selq_active; + lockdep_assert_held(selq->state_lock); WARN_ON_ONCE(selq->is_prepared); selq->is_prepared = true; + selq_active = rcu_dereference_protected(selq->active, + lockdep_is_held(selq->state_lock)); + *selq->standby = *selq_active; selq->standby->num_channels = params->num_channels; selq->standby->num_tcs = mlx5e_get_dcb_num_tc(params); selq->standby->num_regular_queues = selq->standby->num_channels * selq->standby->num_tcs; - selq->standby->is_htb = htb; selq->standby->is_ptp = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_TX_PORT_TS); } +bool mlx5e_selq_is_htb_enabled(struct mlx5e_selq *selq) +{ + struct mlx5e_selq_params *selq_active = + rcu_dereference_protected(selq->active, lockdep_is_held(selq->state_lock)); + + return selq_active->htb_maj_id; +} + +void mlx5e_selq_prepare_htb(struct mlx5e_selq *selq, u16 htb_maj_id, u16 htb_defcls) +{ + struct mlx5e_selq_params *selq_active; + + lockdep_assert_held(selq->state_lock); + WARN_ON_ONCE(selq->is_prepared); + + selq->is_prepared = true; + + selq_active = rcu_dereference_protected(selq->active, + lockdep_is_held(selq->state_lock)); + *selq->standby = *selq_active; + selq->standby->is_htb = htb_maj_id; + selq->standby->htb_maj_id = htb_maj_id; + selq->standby->htb_defcls = htb_defcls; +} + void mlx5e_selq_apply(struct mlx5e_selq *selq) { struct mlx5e_selq_params *old_params; @@ -137,20 +171,21 @@ static u16 mlx5e_select_ptpsq(struct net_device *dev, struct sk_buff *skb, return selq->num_regular_queues + up; } -static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb) +static int mlx5e_select_htb_queue(struct mlx5e_priv *priv, struct sk_buff *skb, + struct mlx5e_selq_params *selq) { u16 classid; /* Order maj_id before defcls - pairs with mlx5e_htb_root_add. */ - if ((TC_H_MAJ(skb->priority) >> 16) == smp_load_acquire(&priv->htb.maj_id)) + if ((TC_H_MAJ(skb->priority) >> 16) == selq->htb_maj_id) classid = TC_H_MIN(skb->priority); else - classid = READ_ONCE(priv->htb.defcls); + classid = selq->htb_defcls; if (!classid) return 0; - return mlx5e_get_txq_by_classid(priv, classid); + return mlx5e_htb_get_txq_by_classid(priv->htb, classid); } u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, @@ -187,10 +222,10 @@ u16 mlx5e_select_queue(struct net_device *dev, struct sk_buff *skb, up * selq->num_channels; } - if (unlikely(selq->is_htb)) { + if (unlikely(selq->htb_maj_id)) { /* num_tcs == 1, shortcut for PTP */ - txq_ix = mlx5e_select_htb_queue(priv, skb); + txq_ix = mlx5e_select_htb_queue(priv, skb, selq); if (txq_ix > 0) return txq_ix; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h index 6c070141d8f1..fd590f80e4d1 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/selq.h @@ -21,7 +21,9 @@ struct sk_buff; int mlx5e_selq_init(struct mlx5e_selq *selq, struct mutex *state_lock); void mlx5e_selq_cleanup(struct mlx5e_selq *selq); -void mlx5e_selq_prepare(struct mlx5e_selq *selq, struct mlx5e_params *params, bool htb); +void mlx5e_selq_prepare_params(struct mlx5e_selq *selq, struct mlx5e_params *params); +void mlx5e_selq_prepare_htb(struct mlx5e_selq *selq, u16 htb_maj_id, u16 htb_defcls); +bool mlx5e_selq_is_htb_enabled(struct mlx5e_selq *selq); void mlx5e_selq_apply(struct mlx5e_selq *selq); void mlx5e_selq_cancel(struct mlx5e_selq *selq); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c index 2755c25ba324..305fde62a78d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.c @@ -30,7 +30,7 @@ static struct mlx5e_tc_act *tc_acts_fdb[NUM_FLOW_ACTIONS] = { NULL, /* FLOW_ACTION_WAKE, */ NULL, /* FLOW_ACTION_QUEUE, */ &mlx5e_tc_act_sample, - NULL, /* FLOW_ACTION_POLICE, */ + &mlx5e_tc_act_police, &mlx5e_tc_act_ct, NULL, /* FLOW_ACTION_CT_METADATA, */ &mlx5e_tc_act_mpls_push, @@ -106,8 +106,8 @@ mlx5e_tc_act_init_parse_state(struct mlx5e_tc_act_parse_state *parse_state, { memset(parse_state, 0, sizeof(*parse_state)); parse_state->flow = flow; - parse_state->num_actions = flow_action->num_entries; parse_state->extack = extack; + parse_state->flow_action = flow_action; } void diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h index f34714c5ddd4..e1570ff056ae 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/act.h @@ -13,7 +13,7 @@ struct mlx5_flow_attr; struct mlx5e_tc_act_parse_state { - unsigned int num_actions; + struct flow_action *flow_action; struct mlx5e_tc_flow *flow; struct netlink_ext_ack *extack; u32 actions; @@ -50,6 +50,16 @@ struct mlx5e_tc_act { bool (*is_multi_table_act)(struct mlx5e_priv *priv, const struct flow_action_entry *act, struct mlx5_flow_attr *attr); + + int (*offload_action)(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act, + struct flow_action_entry *act); + + int (*destroy_action)(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act); + + int (*stats_action)(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act); }; struct mlx5e_tc_flow_action { @@ -76,6 +86,7 @@ extern struct mlx5e_tc_act mlx5e_tc_act_ct; extern struct mlx5e_tc_act mlx5e_tc_act_sample; extern struct mlx5e_tc_act mlx5e_tc_act_ptype; extern struct mlx5e_tc_act mlx5e_tc_act_redirect_ingress; +extern struct mlx5e_tc_act mlx5e_tc_act_police; struct mlx5e_tc_act * mlx5e_tc_act_get(enum flow_action_id act_id, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c index 4726bcb46eec..69949ab830b6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/goto.c @@ -21,7 +21,7 @@ validate_goto_chain(struct mlx5e_priv *priv, u32 max_chain; esw = priv->mdev->priv.eswitch; - chains = is_esw ? esw_chains(esw) : mlx5e_nic_chains(priv); + chains = is_esw ? esw_chains(esw) : mlx5e_nic_chains(priv->fs->tc); max_chain = mlx5_chains_get_chain_range(chains); reformat_and_fwd = is_esw ? MLX5_CAP_ESW_FLOWTABLE_FDB(priv->mdev, reformat_and_fwd_to_table) : diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c new file mode 100644 index 000000000000..37522352e4b2 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/police.c @@ -0,0 +1,153 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "act.h" +#include "en/tc_priv.h" + +static bool +tc_act_can_offload_police(struct mlx5e_tc_act_parse_state *parse_state, + const struct flow_action_entry *act, + int act_index, + struct mlx5_flow_attr *attr) +{ + if (act->police.notexceed.act_id != FLOW_ACTION_PIPE && + act->police.notexceed.act_id != FLOW_ACTION_ACCEPT) { + NL_SET_ERR_MSG_MOD(parse_state->extack, + "Offload not supported when conform action is not pipe or ok"); + return false; + } + if (mlx5e_policer_validate(parse_state->flow_action, act, + parse_state->extack)) + return false; + + return !!mlx5e_get_flow_meters(parse_state->flow->priv->mdev); +} + +static int +fill_meter_params_from_act(const struct flow_action_entry *act, + struct mlx5e_flow_meter_params *params) +{ + params->index = act->hw_index; + if (act->police.rate_bytes_ps) { + params->mode = MLX5_RATE_LIMIT_BPS; + /* change rate to bits per second */ + params->rate = act->police.rate_bytes_ps << 3; + params->burst = act->police.burst; + } else if (act->police.rate_pkt_ps) { + params->mode = MLX5_RATE_LIMIT_PPS; + params->rate = act->police.rate_pkt_ps; + params->burst = act->police.burst_pkt; + } else { + return -EOPNOTSUPP; + } + + return 0; +} + +static int +tc_act_parse_police(struct mlx5e_tc_act_parse_state *parse_state, + const struct flow_action_entry *act, + struct mlx5e_priv *priv, + struct mlx5_flow_attr *attr) +{ + int err; + + err = fill_meter_params_from_act(act, &attr->meter_attr.params); + if (err) + return err; + + attr->action |= MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO; + attr->exe_aso_type = MLX5_EXE_ASO_FLOW_METER; + + return 0; +} + +static bool +tc_act_is_multi_table_act_police(struct mlx5e_priv *priv, + const struct flow_action_entry *act, + struct mlx5_flow_attr *attr) +{ + return true; +} + +static int +tc_act_police_offload(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act, + struct flow_action_entry *act) +{ + struct mlx5e_flow_meter_params params = {}; + struct mlx5e_flow_meter_handle *meter; + int err = 0; + + err = fill_meter_params_from_act(act, ¶ms); + if (err) + return err; + + meter = mlx5e_tc_meter_get(priv->mdev, ¶ms); + if (IS_ERR(meter) && PTR_ERR(meter) == -ENOENT) { + meter = mlx5e_tc_meter_replace(priv->mdev, ¶ms); + } else if (!IS_ERR(meter)) { + err = mlx5e_tc_meter_update(meter, ¶ms); + mlx5e_tc_meter_put(meter); + } + + if (IS_ERR(meter)) { + NL_SET_ERR_MSG_MOD(fl_act->extack, "Failed to get flow meter"); + mlx5_core_err(priv->mdev, "Failed to get flow meter %d\n", params.index); + err = PTR_ERR(meter); + } + + return err; +} + +static int +tc_act_police_destroy(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act) +{ + struct mlx5e_flow_meter_params params = {}; + struct mlx5e_flow_meter_handle *meter; + + params.index = fl_act->index; + meter = mlx5e_tc_meter_get(priv->mdev, ¶ms); + if (IS_ERR(meter)) { + NL_SET_ERR_MSG_MOD(fl_act->extack, "Failed to get flow meter"); + mlx5_core_err(priv->mdev, "Failed to get flow meter %d\n", params.index); + return PTR_ERR(meter); + } + /* first put for the get and second for cleanup */ + mlx5e_tc_meter_put(meter); + mlx5e_tc_meter_put(meter); + return 0; +} + +static int +tc_act_police_stats(struct mlx5e_priv *priv, + struct flow_offload_action *fl_act) +{ + struct mlx5e_flow_meter_params params = {}; + struct mlx5e_flow_meter_handle *meter; + u64 bytes, packets, drops, lastuse; + + params.index = fl_act->index; + meter = mlx5e_tc_meter_get(priv->mdev, ¶ms); + if (IS_ERR(meter)) { + NL_SET_ERR_MSG_MOD(fl_act->extack, "Failed to get flow meter"); + mlx5_core_err(priv->mdev, "Failed to get flow meter %d\n", params.index); + return PTR_ERR(meter); + } + + mlx5e_tc_meter_get_stats(meter, &bytes, &packets, &drops, &lastuse); + flow_stats_update(&fl_act->stats, bytes, packets, drops, lastuse, + FLOW_ACTION_HW_STATS_DELAYED); + mlx5e_tc_meter_put(meter); + return 0; +} + +struct mlx5e_tc_act mlx5e_tc_act_police = { + .can_offload = tc_act_can_offload_police, + .parse_action = tc_act_parse_police, + .is_multi_table_act = tc_act_is_multi_table_act_police, + .offload_action = tc_act_police_offload, + .destroy_action = tc_act_police_destroy, + .stats_action = tc_act_police_stats, +}; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c index a7d9eab19e4a..53b270f652b9 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/act/trap.c @@ -12,7 +12,7 @@ tc_act_can_offload_trap(struct mlx5e_tc_act_parse_state *parse_state, { struct netlink_ext_ack *extack = parse_state->extack; - if (parse_state->num_actions != 1) { + if (parse_state->flow_action->num_entries != 1) { NL_SET_ERR_MSG_MOD(extack, "action trap is supported as a sole action only"); return false; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c new file mode 100644 index 000000000000..a53e205f4a89 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.c @@ -0,0 +1,579 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include <linux/math64.h> +#include "lib/aso.h" +#include "en/tc/post_act.h" +#include "meter.h" +#include "en/tc_priv.h" + +#define MLX5_START_COLOR_SHIFT 28 +#define MLX5_METER_MODE_SHIFT 24 +#define MLX5_CBS_EXP_SHIFT 24 +#define MLX5_CBS_MAN_SHIFT 16 +#define MLX5_CIR_EXP_SHIFT 8 + +/* cir = 8*(10^9)*cir_mantissa/(2^cir_exponent)) bits/s */ +#define MLX5_CONST_CIR 8000000000ULL +#define MLX5_CALC_CIR(m, e) ((MLX5_CONST_CIR * (m)) >> (e)) +#define MLX5_MAX_CIR ((MLX5_CONST_CIR * 0x100) - 1) + +/* cbs = cbs_mantissa*2^cbs_exponent */ +#define MLX5_CALC_CBS(m, e) ((m) << (e)) +#define MLX5_MAX_CBS ((0x100ULL << 0x1F) - 1) +#define MLX5_MAX_HW_CBS 0x7FFFFFFF + +struct mlx5e_flow_meter_aso_obj { + struct list_head entry; + int base_id; + int total_meters; + + unsigned long meters_map[0]; /* must be at the end of this struct */ +}; + +struct mlx5e_flow_meters { + enum mlx5_flow_namespace_type ns_type; + struct mlx5_aso *aso; + struct mutex aso_lock; /* Protects aso operations */ + int log_granularity; + u32 pdn; + + DECLARE_HASHTABLE(hashtbl, 8); + + struct mutex sync_lock; /* protect flow meter operations */ + struct list_head partial_list; + struct list_head full_list; + + struct mlx5_core_dev *mdev; + struct mlx5e_post_act *post_act; +}; + +static void +mlx5e_flow_meter_cir_calc(u64 cir, u8 *man, u8 *exp) +{ + s64 _cir, _delta, delta = S64_MAX; + u8 e, _man = 0, _exp = 0; + u64 m; + + for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ + m = cir << e; + if ((s64)m < 0) /* overflow */ + break; + m = div64_u64(m, MLX5_CONST_CIR); + if (m > 0xFF) /* man width 8 bit */ + continue; + _cir = MLX5_CALC_CIR(m, e); + _delta = cir - _cir; + if (_delta < delta) { + _man = m; + _exp = e; + if (!_delta) + goto found; + delta = _delta; + } + } + +found: + *man = _man; + *exp = _exp; +} + +static void +mlx5e_flow_meter_cbs_calc(u64 cbs, u8 *man, u8 *exp) +{ + s64 _cbs, _delta, delta = S64_MAX; + u8 e, _man = 0, _exp = 0; + u64 m; + + for (e = 0; e <= 0x1F; e++) { /* exp width 5bit */ + m = cbs >> e; + if (m > 0xFF) /* man width 8 bit */ + continue; + _cbs = MLX5_CALC_CBS(m, e); + _delta = cbs - _cbs; + if (_delta < delta) { + _man = m; + _exp = e; + if (!_delta) + goto found; + delta = _delta; + } + } + +found: + *man = _man; + *exp = _exp; +} + +int +mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, + struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *meter_params) +{ + struct mlx5_wqe_aso_ctrl_seg *aso_ctrl; + struct mlx5_wqe_aso_data_seg *aso_data; + struct mlx5e_flow_meters *flow_meters; + u8 cir_man, cir_exp, cbs_man, cbs_exp; + struct mlx5_aso_wqe *aso_wqe; + struct mlx5_aso *aso; + u64 rate, burst; + u8 ds_cnt; + int err; + + rate = meter_params->rate; + burst = meter_params->burst; + + /* HW treats each packet as 128 bytes in PPS mode */ + if (meter_params->mode == MLX5_RATE_LIMIT_PPS) { + rate <<= 10; + burst <<= 7; + } + + if (!rate || rate > MLX5_MAX_CIR || !burst || burst > MLX5_MAX_CBS) + return -EINVAL; + + /* HW has limitation of total 31 bits for cbs */ + if (burst > MLX5_MAX_HW_CBS) { + mlx5_core_warn(mdev, + "burst(%lld) is too large, use HW allowed value(%d)\n", + burst, MLX5_MAX_HW_CBS); + burst = MLX5_MAX_HW_CBS; + } + + mlx5_core_dbg(mdev, "meter mode=%d\n", meter_params->mode); + mlx5e_flow_meter_cir_calc(rate, &cir_man, &cir_exp); + mlx5_core_dbg(mdev, "rate=%lld, cir=%lld, exp=%d, man=%d\n", + rate, MLX5_CALC_CIR(cir_man, cir_exp), cir_exp, cir_man); + mlx5e_flow_meter_cbs_calc(burst, &cbs_man, &cbs_exp); + mlx5_core_dbg(mdev, "burst=%lld, cbs=%lld, exp=%d, man=%d\n", + burst, MLX5_CALC_CBS((u64)cbs_man, cbs_exp), cbs_exp, cbs_man); + + if (!cir_man || !cbs_man) + return -EINVAL; + + flow_meters = meter->flow_meters; + aso = flow_meters->aso; + + mutex_lock(&flow_meters->aso_lock); + aso_wqe = mlx5_aso_get_wqe(aso); + ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_DS); + mlx5_aso_build_wqe(aso, ds_cnt, aso_wqe, meter->obj_id, + MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER); + + aso_ctrl = &aso_wqe->aso_ctrl; + memset(aso_ctrl, 0, sizeof(*aso_ctrl)); + aso_ctrl->data_mask_mode = MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE << 6; + aso_ctrl->condition_1_0_operand = MLX5_ASO_ALWAYS_TRUE | + MLX5_ASO_ALWAYS_TRUE << 4; + aso_ctrl->data_offset_condition_operand = MLX5_ASO_LOGICAL_OR << 6; + aso_ctrl->data_mask = cpu_to_be64(0x80FFFFFFULL << (meter->idx ? 0 : 32)); + + aso_data = (struct mlx5_wqe_aso_data_seg *)(aso_wqe + 1); + memset(aso_data, 0, sizeof(*aso_data)); + aso_data->bytewise_data[meter->idx * 8] = cpu_to_be32((0x1 << 31) | /* valid */ + (MLX5_FLOW_METER_COLOR_GREEN << MLX5_START_COLOR_SHIFT)); + if (meter_params->mode == MLX5_RATE_LIMIT_PPS) + aso_data->bytewise_data[meter->idx * 8] |= + cpu_to_be32(MLX5_FLOW_METER_MODE_NUM_PACKETS << MLX5_METER_MODE_SHIFT); + else + aso_data->bytewise_data[meter->idx * 8] |= + cpu_to_be32(MLX5_FLOW_METER_MODE_BYTES_IP_LENGTH << MLX5_METER_MODE_SHIFT); + + aso_data->bytewise_data[meter->idx * 8 + 2] = cpu_to_be32((cbs_exp << MLX5_CBS_EXP_SHIFT) | + (cbs_man << MLX5_CBS_MAN_SHIFT) | + (cir_exp << MLX5_CIR_EXP_SHIFT) | + cir_man); + + mlx5_aso_post_wqe(aso, true, &aso_wqe->ctrl); + + /* With newer FW, the wait for the first ASO WQE is more than 2us, put the wait 10ms. */ + err = mlx5_aso_poll_cq(aso, true, 10); + mutex_unlock(&flow_meters->aso_lock); + + return err; +} + +static int +mlx5e_flow_meter_create_aso_obj(struct mlx5e_flow_meters *flow_meters, int *obj_id) +{ + u32 in[MLX5_ST_SZ_DW(create_flow_meter_aso_obj_in)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + struct mlx5_core_dev *mdev = flow_meters->mdev; + void *obj; + int err; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); + MLX5_SET(general_obj_in_cmd_hdr, in, log_obj_range, flow_meters->log_granularity); + + obj = MLX5_ADDR_OF(create_flow_meter_aso_obj_in, in, flow_meter_aso_obj); + MLX5_SET(flow_meter_aso_obj, obj, meter_aso_access_pd, flow_meters->pdn); + + err = mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + if (!err) { + *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id); + mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) created\n", *obj_id); + } + + return err; +} + +static void +mlx5e_flow_meter_destroy_aso_obj(struct mlx5_core_dev *mdev, u32 obj_id) +{ + u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {}; + u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)]; + + MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, + MLX5_GENERAL_OBJECT_TYPES_FLOW_METER_ASO); + MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, obj_id); + + mlx5_cmd_exec(mdev, in, sizeof(in), out, sizeof(out)); + mlx5_core_dbg(mdev, "flow meter aso obj(0x%x) destroyed\n", obj_id); +} + +static struct mlx5e_flow_meter_handle * +__mlx5e_flow_meter_alloc(struct mlx5e_flow_meters *flow_meters) +{ + struct mlx5_core_dev *mdev = flow_meters->mdev; + struct mlx5e_flow_meter_aso_obj *meters_obj; + struct mlx5e_flow_meter_handle *meter; + struct mlx5_fc *counter; + int err, pos, total; + u32 id; + + meter = kzalloc(sizeof(*meter), GFP_KERNEL); + if (!meter) + return ERR_PTR(-ENOMEM); + + counter = mlx5_fc_create(mdev, true); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_red_counter; + } + meter->red_counter = counter; + + counter = mlx5_fc_create(mdev, true); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_green_counter; + } + meter->green_counter = counter; + + meters_obj = list_first_entry_or_null(&flow_meters->partial_list, + struct mlx5e_flow_meter_aso_obj, + entry); + /* 2 meters in one object */ + total = 1 << (flow_meters->log_granularity + 1); + if (!meters_obj) { + err = mlx5e_flow_meter_create_aso_obj(flow_meters, &id); + if (err) { + mlx5_core_err(mdev, "Failed to create flow meter ASO object\n"); + goto err_create; + } + + meters_obj = kzalloc(sizeof(*meters_obj) + BITS_TO_BYTES(total), + GFP_KERNEL); + if (!meters_obj) { + err = -ENOMEM; + goto err_mem; + } + + meters_obj->base_id = id; + meters_obj->total_meters = total; + list_add(&meters_obj->entry, &flow_meters->partial_list); + pos = 0; + } else { + pos = find_first_zero_bit(meters_obj->meters_map, total); + if (bitmap_weight(meters_obj->meters_map, total) == total - 1) { + list_del(&meters_obj->entry); + list_add(&meters_obj->entry, &flow_meters->full_list); + } + } + + bitmap_set(meters_obj->meters_map, pos, 1); + meter->flow_meters = flow_meters; + meter->meters_obj = meters_obj; + meter->obj_id = meters_obj->base_id + pos / 2; + meter->idx = pos % 2; + + mlx5_core_dbg(mdev, "flow meter allocated, obj_id=0x%x, index=%d\n", + meter->obj_id, meter->idx); + + return meter; + +err_mem: + mlx5e_flow_meter_destroy_aso_obj(mdev, id); +err_create: + mlx5_fc_destroy(mdev, meter->green_counter); +err_green_counter: + mlx5_fc_destroy(mdev, meter->red_counter); +err_red_counter: + kfree(meter); + return ERR_PTR(err); +} + +static void +__mlx5e_flow_meter_free(struct mlx5e_flow_meter_handle *meter) +{ + struct mlx5e_flow_meters *flow_meters = meter->flow_meters; + struct mlx5_core_dev *mdev = flow_meters->mdev; + struct mlx5e_flow_meter_aso_obj *meters_obj; + int n, pos; + + mlx5_fc_destroy(mdev, meter->green_counter); + mlx5_fc_destroy(mdev, meter->red_counter); + + meters_obj = meter->meters_obj; + pos = (meter->obj_id - meters_obj->base_id) * 2 + meter->idx; + bitmap_clear(meters_obj->meters_map, pos, 1); + n = bitmap_weight(meters_obj->meters_map, meters_obj->total_meters); + if (n == 0) { + list_del(&meters_obj->entry); + mlx5e_flow_meter_destroy_aso_obj(mdev, meters_obj->base_id); + kfree(meters_obj); + } else if (n == meters_obj->total_meters - 1) { + list_del(&meters_obj->entry); + list_add(&meters_obj->entry, &flow_meters->partial_list); + } + + mlx5_core_dbg(mdev, "flow meter freed, obj_id=0x%x, index=%d\n", + meter->obj_id, meter->idx); + kfree(meter); +} + +static struct mlx5e_flow_meter_handle * +__mlx5e_tc_meter_get(struct mlx5e_flow_meters *flow_meters, u32 index) +{ + struct mlx5e_flow_meter_handle *meter; + + hash_for_each_possible(flow_meters->hashtbl, meter, hlist, index) + if (meter->params.index == index) + goto add_ref; + + return ERR_PTR(-ENOENT); + +add_ref: + meter->refcnt++; + + return meter; +} + +struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) +{ + struct mlx5e_flow_meters *flow_meters; + struct mlx5e_flow_meter_handle *meter; + + flow_meters = mlx5e_get_flow_meters(mdev); + if (!flow_meters) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&flow_meters->sync_lock); + meter = __mlx5e_tc_meter_get(flow_meters, params->index); + mutex_unlock(&flow_meters->sync_lock); + + return meter; +} + +static void +__mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) +{ + if (--meter->refcnt == 0) { + hash_del(&meter->hlist); + __mlx5e_flow_meter_free(meter); + } +} + +void +mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter) +{ + struct mlx5e_flow_meters *flow_meters = meter->flow_meters; + + mutex_lock(&flow_meters->sync_lock); + __mlx5e_tc_meter_put(meter); + mutex_unlock(&flow_meters->sync_lock); +} + +static struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_alloc(struct mlx5e_flow_meters *flow_meters, + struct mlx5e_flow_meter_params *params) +{ + struct mlx5e_flow_meter_handle *meter; + + meter = __mlx5e_flow_meter_alloc(flow_meters); + if (IS_ERR(meter)) + return meter; + + hash_add(flow_meters->hashtbl, &meter->hlist, params->index); + meter->params.index = params->index; + meter->refcnt++; + + return meter; +} + +static int +__mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *params) +{ + struct mlx5_core_dev *mdev = meter->flow_meters->mdev; + int err = 0; + + if (meter->params.mode != params->mode || meter->params.rate != params->rate || + meter->params.burst != params->burst) { + err = mlx5e_tc_meter_modify(mdev, meter, params); + if (err) + goto out; + + meter->params.mode = params->mode; + meter->params.rate = params->rate; + meter->params.burst = params->burst; + } + +out: + return err; +} + +int +mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *params) +{ + struct mlx5_core_dev *mdev = meter->flow_meters->mdev; + struct mlx5e_flow_meters *flow_meters; + int err; + + flow_meters = mlx5e_get_flow_meters(mdev); + if (!flow_meters) + return -EOPNOTSUPP; + + mutex_lock(&flow_meters->sync_lock); + err = __mlx5e_tc_meter_update(meter, params); + mutex_unlock(&flow_meters->sync_lock); + return err; +} + +struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_replace(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params) +{ + struct mlx5e_flow_meters *flow_meters; + struct mlx5e_flow_meter_handle *meter; + int err; + + flow_meters = mlx5e_get_flow_meters(mdev); + if (!flow_meters) + return ERR_PTR(-EOPNOTSUPP); + + mutex_lock(&flow_meters->sync_lock); + meter = __mlx5e_tc_meter_get(flow_meters, params->index); + if (IS_ERR(meter)) { + meter = mlx5e_tc_meter_alloc(flow_meters, params); + if (IS_ERR(meter)) { + err = PTR_ERR(meter); + goto err_get; + } + } + + err = __mlx5e_tc_meter_update(meter, params); + if (err) + goto err_update; + + mutex_unlock(&flow_meters->sync_lock); + return meter; + +err_update: + __mlx5e_tc_meter_put(meter); +err_get: + mutex_unlock(&flow_meters->sync_lock); + return ERR_PTR(err); +} + +enum mlx5_flow_namespace_type +mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters *flow_meters) +{ + return flow_meters->ns_type; +} + +struct mlx5e_flow_meters * +mlx5e_flow_meters_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_flow_meters *flow_meters; + int err; + + if (!(MLX5_CAP_GEN_64(mdev, general_obj_types) & + MLX5_HCA_CAP_GENERAL_OBJECT_TYPES_FLOW_METER_ASO)) + return ERR_PTR(-EOPNOTSUPP); + + if (IS_ERR_OR_NULL(post_act)) { + netdev_dbg(priv->netdev, + "flow meter offload is not supported, post action is missing\n"); + return ERR_PTR(-EOPNOTSUPP); + } + + flow_meters = kzalloc(sizeof(*flow_meters), GFP_KERNEL); + if (!flow_meters) + return ERR_PTR(-ENOMEM); + + err = mlx5_core_alloc_pd(mdev, &flow_meters->pdn); + if (err) { + mlx5_core_err(mdev, "Failed to alloc pd for flow meter aso, err=%d\n", err); + goto err_out; + } + + flow_meters->aso = mlx5_aso_create(mdev, flow_meters->pdn); + if (IS_ERR(flow_meters->aso)) { + mlx5_core_warn(mdev, "Failed to create aso wqe for flow meter\n"); + err = PTR_ERR(flow_meters->aso); + goto err_sq; + } + + mutex_init(&flow_meters->sync_lock); + INIT_LIST_HEAD(&flow_meters->partial_list); + INIT_LIST_HEAD(&flow_meters->full_list); + + flow_meters->ns_type = ns_type; + flow_meters->mdev = mdev; + flow_meters->post_act = post_act; + mutex_init(&flow_meters->aso_lock); + flow_meters->log_granularity = min_t(int, 6, + MLX5_CAP_QOS(mdev, log_meter_aso_max_alloc)); + + return flow_meters; + +err_sq: + mlx5_core_dealloc_pd(mdev, flow_meters->pdn); +err_out: + kfree(flow_meters); + return ERR_PTR(err); +} + +void +mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters) +{ + if (IS_ERR_OR_NULL(flow_meters)) + return; + + mlx5_aso_destroy(flow_meters->aso); + mlx5_core_dealloc_pd(flow_meters->mdev, flow_meters->pdn); + kfree(flow_meters); +} + +void +mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter, + u64 *bytes, u64 *packets, u64 *drops, u64 *lastuse) +{ + u64 bytes1, packets1, lastuse1; + u64 bytes2, packets2, lastuse2; + + mlx5_fc_query_cached(meter->green_counter, &bytes1, &packets1, &lastuse1); + mlx5_fc_query_cached(meter->red_counter, &bytes2, &packets2, &lastuse2); + + *bytes = bytes1 + bytes2; + *packets = packets1 + packets2; + *drops = packets2; + *lastuse = max_t(u64, lastuse1, lastuse2); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h new file mode 100644 index 000000000000..6de6e8a16327 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/meter.h @@ -0,0 +1,74 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_FLOW_METER_H__ +#define __MLX5_EN_FLOW_METER_H__ + +struct mlx5e_post_meter_priv; +struct mlx5e_flow_meter_aso_obj; +struct mlx5e_flow_meters; +struct mlx5_flow_attr; + +enum mlx5e_flow_meter_mode { + MLX5_RATE_LIMIT_BPS, + MLX5_RATE_LIMIT_PPS, +}; + +struct mlx5e_flow_meter_params { + enum mlx5e_flow_meter_mode mode; + /* police action index */ + u32 index; + u64 rate; + u64 burst; +}; + +struct mlx5e_flow_meter_handle { + struct mlx5e_flow_meters *flow_meters; + struct mlx5e_flow_meter_aso_obj *meters_obj; + u32 obj_id; + u8 idx; + + int refcnt; + struct hlist_node hlist; + struct mlx5e_flow_meter_params params; + + struct mlx5_fc *green_counter; + struct mlx5_fc *red_counter; +}; + +struct mlx5e_meter_attr { + struct mlx5e_flow_meter_params params; + struct mlx5e_flow_meter_handle *meter; + struct mlx5e_post_meter_priv *post_meter; +}; + +int +mlx5e_tc_meter_modify(struct mlx5_core_dev *mdev, + struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *meter_params); + +struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_get(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params); +void +mlx5e_tc_meter_put(struct mlx5e_flow_meter_handle *meter); +int +mlx5e_tc_meter_update(struct mlx5e_flow_meter_handle *meter, + struct mlx5e_flow_meter_params *params); +struct mlx5e_flow_meter_handle * +mlx5e_tc_meter_replace(struct mlx5_core_dev *mdev, struct mlx5e_flow_meter_params *params); + +enum mlx5_flow_namespace_type +mlx5e_tc_meter_get_namespace(struct mlx5e_flow_meters *flow_meters); + +struct mlx5e_flow_meters * +mlx5e_flow_meters_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_action); +void +mlx5e_flow_meters_cleanup(struct mlx5e_flow_meters *flow_meters); + +void +mlx5e_tc_meter_get_stats(struct mlx5e_flow_meter_handle *meter, + u64 *bytes, u64 *packets, u64 *drops, u64 *lastuse); + +#endif /* __MLX5_EN_FLOW_METER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c index 2b64dd557b5d..4e48946c4c2a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_act.c @@ -22,9 +22,9 @@ struct mlx5e_post_act_handle { u32 id; }; -#define MLX5_POST_ACTION_BITS (mlx5e_tc_attr_to_reg_mappings[FTEID_TO_REG].mlen) -#define MLX5_POST_ACTION_MAX GENMASK(MLX5_POST_ACTION_BITS - 1, 0) -#define MLX5_POST_ACTION_MASK MLX5_POST_ACTION_MAX +#define MLX5_POST_ACTION_BITS MLX5_REG_MAPPING_MBITS(FTEID_TO_REG) +#define MLX5_POST_ACTION_MASK MLX5_REG_MAPPING_MASK(FTEID_TO_REG) +#define MLX5_POST_ACTION_MAX MLX5_POST_ACTION_MASK struct mlx5e_post_act * mlx5e_tc_post_act_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, @@ -36,7 +36,7 @@ mlx5e_tc_post_act_init(struct mlx5e_priv *priv, struct mlx5_fs_chains *chains, int err; if (!MLX5_CAP_FLOWTABLE_TYPE(priv->mdev, ignore_flow_level, table_type)) { - if (priv->mdev->coredev_type != MLX5_COREDEV_VF) + if (priv->mdev->coredev_type == MLX5_COREDEV_PF) mlx5_core_warn(priv->mdev, "firmware level support is missing\n"); err = -EOPNOTSUPP; goto err_check; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c new file mode 100644 index 000000000000..8b77e822810e --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.c @@ -0,0 +1,209 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include "en/tc_priv.h" +#include "post_meter.h" +#include "en/tc/post_act.h" + +#define MLX5_PACKET_COLOR_BITS MLX5_REG_MAPPING_MBITS(PACKET_COLOR_TO_REG) +#define MLX5_PACKET_COLOR_MASK MLX5_REG_MAPPING_MASK(PACKET_COLOR_TO_REG) + +struct mlx5e_post_meter_priv { + struct mlx5_flow_table *ft; + struct mlx5_flow_group *fg; + struct mlx5_flow_handle *fwd_green_rule; + struct mlx5_flow_handle *drop_red_rule; +}; + +struct mlx5_flow_table * +mlx5e_post_meter_get_ft(struct mlx5e_post_meter_priv *post_meter) +{ + return post_meter->ft; +} + +static int +mlx5e_post_meter_table_create(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_meter_priv *post_meter) +{ + struct mlx5_flow_table_attr ft_attr = {}; + struct mlx5_flow_namespace *root_ns; + + root_ns = mlx5_get_flow_namespace(priv->mdev, ns_type); + if (!root_ns) { + mlx5_core_warn(priv->mdev, "Failed to get namespace for flow meter\n"); + return -EOPNOTSUPP; + } + + ft_attr.flags = MLX5_FLOW_TABLE_UNMANAGED; + ft_attr.prio = FDB_SLOW_PATH; + ft_attr.max_fte = 2; + ft_attr.level = 1; + + post_meter->ft = mlx5_create_flow_table(root_ns, &ft_attr); + if (IS_ERR(post_meter->ft)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter table\n"); + return PTR_ERR(post_meter->ft); + } + + return 0; +} + +static int +mlx5e_post_meter_fg_create(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter) +{ + int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); + void *misc2, *match_criteria; + u32 *flow_group_in; + int err = 0; + + flow_group_in = kvzalloc(inlen, GFP_KERNEL); + if (!flow_group_in) + return -ENOMEM; + + MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, + MLX5_MATCH_MISC_PARAMETERS_2); + match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, + match_criteria); + misc2 = MLX5_ADDR_OF(fte_match_param, match_criteria, misc_parameters_2); + MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_5, MLX5_PACKET_COLOR_MASK); + MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); + MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1); + + post_meter->fg = mlx5_create_flow_group(post_meter->ft, flow_group_in); + if (IS_ERR(post_meter->fg)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter flow group\n"); + err = PTR_ERR(post_meter->fg); + } + + kvfree(flow_group_in); + return err; +} + +static int +mlx5e_post_meter_rules_create(struct mlx5e_priv *priv, + struct mlx5e_post_meter_priv *post_meter, + struct mlx5e_post_act *post_act, + struct mlx5_fc *green_counter, + struct mlx5_fc *red_counter) +{ + struct mlx5_flow_destination dest[2] = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + int err; + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) + return -ENOMEM; + + mlx5e_tc_match_to_reg_match(spec, PACKET_COLOR_TO_REG, + MLX5_FLOW_METER_COLOR_RED, MLX5_PACKET_COLOR_MASK); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; + dest[0].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[0].counter_id = mlx5_fc_id(red_counter); + + rule = mlx5_add_flow_rules(post_meter->ft, spec, &flow_act, dest, 1); + if (IS_ERR(rule)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter flow drop rule\n"); + err = PTR_ERR(rule); + goto err_red; + } + post_meter->drop_red_rule = rule; + + mlx5e_tc_match_to_reg_match(spec, PACKET_COLOR_TO_REG, + MLX5_FLOW_METER_COLOR_GREEN, MLX5_PACKET_COLOR_MASK); + flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[0].ft = mlx5e_tc_post_act_get_ft(post_act); + dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[1].counter_id = mlx5_fc_id(green_counter); + + rule = mlx5_add_flow_rules(post_meter->ft, spec, &flow_act, dest, 2); + if (IS_ERR(rule)) { + mlx5_core_warn(priv->mdev, "Failed to create post_meter flow fwd rule\n"); + err = PTR_ERR(rule); + goto err_green; + } + post_meter->fwd_green_rule = rule; + + kvfree(spec); + return 0; + +err_green: + mlx5_del_flow_rules(post_meter->drop_red_rule); +err_red: + kvfree(spec); + return err; +} + +static void +mlx5e_post_meter_rules_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_del_flow_rules(post_meter->drop_red_rule); + mlx5_del_flow_rules(post_meter->fwd_green_rule); +} + +static void +mlx5e_post_meter_fg_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_destroy_flow_group(post_meter->fg); +} + +static void +mlx5e_post_meter_table_destroy(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5_destroy_flow_table(post_meter->ft); +} + +struct mlx5e_post_meter_priv * +mlx5e_post_meter_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act, + struct mlx5_fc *green_counter, + struct mlx5_fc *red_counter) +{ + struct mlx5e_post_meter_priv *post_meter; + int err; + + post_meter = kzalloc(sizeof(*post_meter), GFP_KERNEL); + if (!post_meter) + return ERR_PTR(-ENOMEM); + + err = mlx5e_post_meter_table_create(priv, ns_type, post_meter); + if (err) + goto err_ft; + + err = mlx5e_post_meter_fg_create(priv, post_meter); + if (err) + goto err_fg; + + err = mlx5e_post_meter_rules_create(priv, post_meter, post_act, green_counter, + red_counter); + if (err) + goto err_rules; + + return post_meter; + +err_rules: + mlx5e_post_meter_fg_destroy(post_meter); +err_fg: + mlx5e_post_meter_table_destroy(post_meter); +err_ft: + kfree(post_meter); + return ERR_PTR(err); +} + +void +mlx5e_post_meter_cleanup(struct mlx5e_post_meter_priv *post_meter) +{ + mlx5e_post_meter_rules_destroy(post_meter); + mlx5e_post_meter_fg_destroy(post_meter); + mlx5e_post_meter_table_destroy(post_meter); + kfree(post_meter); +} + diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h new file mode 100644 index 000000000000..34d0e4b9fc7a --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc/post_meter.h @@ -0,0 +1,29 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2021, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_EN_POST_METER_H__ +#define __MLX5_EN_POST_METER_H__ + +#define packet_color_to_reg { \ + .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_5, \ + .moffset = 0, \ + .mlen = 8, \ + .soffset = MLX5_BYTE_OFF(fte_match_param, \ + misc_parameters_2.metadata_reg_c_5), \ +} + +struct mlx5e_post_meter_priv; + +struct mlx5_flow_table * +mlx5e_post_meter_get_ft(struct mlx5e_post_meter_priv *post_meter); + +struct mlx5e_post_meter_priv * +mlx5e_post_meter_init(struct mlx5e_priv *priv, + enum mlx5_flow_namespace_type ns_type, + struct mlx5e_post_act *post_act, + struct mlx5_fc *green_counter, + struct mlx5_fc *red_counter); +void +mlx5e_post_meter_cleanup(struct mlx5e_post_meter_priv *post_meter); + +#endif /* __MLX5_EN_POST_METER_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c index ba171c7f0a67..864ce0c393e6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.c @@ -36,8 +36,8 @@ #define MLX5_CT_STATE_RELATED_BIT BIT(5) #define MLX5_CT_STATE_INVALID_BIT BIT(6) -#define MLX5_CT_LABELS_BITS (mlx5e_tc_attr_to_reg_mappings[LABELS_TO_REG].mlen) -#define MLX5_CT_LABELS_MASK GENMASK(MLX5_CT_LABELS_BITS - 1, 0) +#define MLX5_CT_LABELS_BITS MLX5_REG_MAPPING_MBITS(LABELS_TO_REG) +#define MLX5_CT_LABELS_MASK MLX5_REG_MAPPING_MASK(LABELS_TO_REG) /* Statically allocate modify actions for * ipv6 and port nat (5) + tuple fields (4) + nic mode zone restore (1) = 10. @@ -2062,7 +2062,7 @@ mlx5_tc_ct_init_check_support(struct mlx5e_priv *priv, /* Ignore_flow_level support isn't supported by default for VFs and so post_act * won't be supported. Skip showing error msg. */ - if (priv->mdev->coredev_type != MLX5_COREDEV_VF) + if (priv->mdev->coredev_type == MLX5_COREDEV_PF) err_msg = "post action is missing"; err = -EOPNOTSUPP; goto out_err; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h index 00a3ba862afb..5bbd6b92840f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_ct.h @@ -62,10 +62,11 @@ struct mlx5_ct_attr { misc_parameters_2.metadata_reg_c_4),\ } +/* 8 LSB of metadata C5 are reserved for packet color */ #define fteid_to_reg_ct {\ .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_5,\ - .moffset = 0,\ - .mlen = 32,\ + .moffset = 8,\ + .mlen = 24,\ .soffset = MLX5_BYTE_OFF(fte_match_param,\ misc_parameters_2.metadata_reg_c_5),\ } @@ -84,10 +85,8 @@ struct mlx5_ct_attr { .mlen = ESW_ZONE_ID_BITS,\ } -#define REG_MAPPING_MLEN(reg) (mlx5e_tc_attr_to_reg_mappings[reg].mlen) -#define REG_MAPPING_MOFFSET(reg) (mlx5e_tc_attr_to_reg_mappings[reg].moffset) -#define MLX5_CT_ZONE_BITS (mlx5e_tc_attr_to_reg_mappings[ZONE_TO_REG].mlen) -#define MLX5_CT_ZONE_MASK GENMASK(MLX5_CT_ZONE_BITS - 1, 0) +#define MLX5_CT_ZONE_BITS MLX5_REG_MAPPING_MBITS(ZONE_TO_REG) +#define MLX5_CT_ZONE_MASK MLX5_REG_MAPPING_MASK(ZONE_TO_REG) #if IS_ENABLED(CONFIG_MLX5_TC_CT) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h index 3b74a6fd5c43..10c9a8a79d00 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en/tc_priv.h @@ -11,7 +11,6 @@ #define MLX5E_TC_MAX_SPLITS 1 -#define mlx5e_nic_chains(priv) ((priv)->fs.tc.chains) enum { MLX5E_TC_FLOW_FLAG_INGRESS = MLX5E_TC_FLAG_INGRESS_BIT, @@ -44,6 +43,8 @@ struct mlx5e_tc_flow_parse_attr { struct mlx5e_tc_act_parse_state parse_state; }; +struct mlx5_fs_chains *mlx5e_nic_chains(struct mlx5e_tc_table *tc); + /* Helper struct for accessing a struct containing list_head array. * Containing struct * |- Helper array @@ -203,7 +204,13 @@ struct mlx5_fc *mlx5e_tc_get_counter(struct mlx5e_tc_flow *flow); struct mlx5e_tc_int_port_priv * mlx5e_get_int_port_priv(struct mlx5e_priv *priv); +struct mlx5e_flow_meters *mlx5e_get_flow_meters(struct mlx5_core_dev *dev); + void *mlx5e_get_match_headers_value(u32 flags, struct mlx5_flow_spec *spec); void *mlx5e_get_match_headers_criteria(u32 flags, struct mlx5_flow_spec *spec); +int mlx5e_policer_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack); + #endif /* __MLX5_EN_TC_PRIV_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h index 04c0a5e1c89a..1839f1ab1ddd 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/en_accel.h @@ -194,4 +194,14 @@ static inline void mlx5e_accel_cleanup_rx(struct mlx5e_priv *priv) { mlx5e_ktls_cleanup_rx(priv); } + +static inline int mlx5e_accel_init_tx(struct mlx5e_priv *priv) +{ + return mlx5e_ktls_init_tx(priv); +} + +static inline void mlx5e_accel_cleanup_tx(struct mlx5e_priv *priv) +{ + mlx5e_ktls_cleanup_tx(priv); +} #endif /* __MLX5E_EN_ACCEL_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c index 3ae6067c7e6b..20a4f1e585af 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/fs_tcp.c @@ -86,7 +86,7 @@ struct mlx5_flow_handle *mlx5e_accel_fs_add_sk(struct mlx5e_priv *priv, if (!spec) return ERR_PTR(-ENOMEM); - fs_tcp = priv->fs.accel_tcp; + fs_tcp = priv->fs->accel_tcp; spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; @@ -158,10 +158,10 @@ static int accel_fs_tcp_add_default_rule(struct mlx5e_priv *priv, struct mlx5_flow_handle *rule; int err = 0; - fs_tcp = priv->fs.accel_tcp; + fs_tcp = priv->fs->accel_tcp; accel_fs_t = &fs_tcp->tables[type]; - dest = mlx5_ttc_get_default_dest(priv->fs.ttc, fs_accel2tt(type)); + dest = mlx5_ttc_get_default_dest(priv->fs->ttc, fs_accel2tt(type)); rule = mlx5_add_flow_rules(accel_fs_t->t, NULL, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -267,7 +267,7 @@ out: static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_type type) { - struct mlx5e_flow_table *ft = &priv->fs.accel_tcp->tables[type]; + struct mlx5e_flow_table *ft = &priv->fs->accel_tcp->tables[type]; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -277,7 +277,7 @@ static int accel_fs_tcp_create_table(struct mlx5e_priv *priv, enum accel_fs_tcp_ ft_attr.level = MLX5E_ACCEL_FS_TCP_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -307,7 +307,7 @@ static int accel_fs_tcp_disable(struct mlx5e_priv *priv) for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { /* Modify ttc rules destination to point back to the indir TIRs */ - err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_accel2tt(i)); + err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_accel2tt(i)); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] default destination failed, err(%d)\n", @@ -326,10 +326,10 @@ static int accel_fs_tcp_enable(struct mlx5e_priv *priv) dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { - dest.ft = priv->fs.accel_tcp->tables[i].t; + dest.ft = priv->fs->accel_tcp->tables[i].t; /* Modify ttc rules destination to point on the accel_fs FTs */ - err = mlx5_ttc_fwd_dest(priv->fs.ttc, fs_accel2tt(i), &dest); + err = mlx5_ttc_fwd_dest(priv->fs->ttc, fs_accel2tt(i), &dest); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] destination to accel failed, err(%d)\n", @@ -344,7 +344,7 @@ static void accel_fs_tcp_destroy_table(struct mlx5e_priv *priv, int i) { struct mlx5e_accel_fs_tcp *fs_tcp; - fs_tcp = priv->fs.accel_tcp; + fs_tcp = priv->fs->accel_tcp; if (IS_ERR_OR_NULL(fs_tcp->tables[i].t)) return; @@ -357,7 +357,7 @@ void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) { int i; - if (!priv->fs.accel_tcp) + if (!priv->fs->accel_tcp) return; accel_fs_tcp_disable(priv); @@ -365,8 +365,8 @@ void mlx5e_accel_fs_tcp_destroy(struct mlx5e_priv *priv) for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) accel_fs_tcp_destroy_table(priv, i); - kfree(priv->fs.accel_tcp); - priv->fs.accel_tcp = NULL; + kfree(priv->fs->accel_tcp); + priv->fs->accel_tcp = NULL; } int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) @@ -376,8 +376,8 @@ int mlx5e_accel_fs_tcp_create(struct mlx5e_priv *priv) if (!MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version)) return -EOPNOTSUPP; - priv->fs.accel_tcp = kzalloc(sizeof(*priv->fs.accel_tcp), GFP_KERNEL); - if (!priv->fs.accel_tcp) + priv->fs->accel_tcp = kzalloc(sizeof(*priv->fs->accel_tcp), GFP_KERNEL); + if (!priv->fs->accel_tcp) return -ENOMEM; for (i = 0; i < ACCEL_FS_TCP_NUM_TYPES; i++) { @@ -396,7 +396,7 @@ err_destroy_tables: while (--i >= 0) accel_fs_tcp_destroy_table(priv, i); - kfree(priv->fs.accel_tcp); - priv->fs.accel_tcp = NULL; + kfree(priv->fs->accel_tcp); + priv->fs->accel_tcp = NULL; return err; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index 8315e8f603d7..f8113fd23265 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -184,13 +184,13 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) fs_prot = &accel_esp->fs_prot[type]; fs_prot->default_dest = - mlx5_ttc_get_default_dest(priv->fs.ttc, fs_esp2tt(type)); + mlx5_ttc_get_default_dest(priv->fs->ttc, fs_esp2tt(type)); ft_attr.max_fte = 1; ft_attr.autogroup.max_num_groups = 1; ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft = mlx5_create_auto_grouped_flow_table(priv->fs.ns, &ft_attr); + ft = mlx5_create_auto_grouped_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft)) return PTR_ERR(ft); @@ -205,7 +205,7 @@ static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) ft_attr.prio = MLX5E_NIC_PRIO; ft_attr.autogroup.num_reserved_entries = 1; ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(priv->fs.ns, &ft_attr); + ft = mlx5_create_auto_grouped_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; @@ -249,7 +249,7 @@ static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) /* connect */ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; dest.ft = fs_prot->ft; - mlx5_ttc_fwd_dest(priv->fs.ttc, fs_esp2tt(type), &dest); + mlx5_ttc_fwd_dest(priv->fs->ttc, fs_esp2tt(type), &dest); skip: fs_prot->refcnt++; @@ -271,7 +271,7 @@ static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) goto out; /* disconnect */ - mlx5_ttc_fwd_default_dest(priv->fs.ttc, fs_esp2tt(type)); + mlx5_ttc_fwd_default_dest(priv->fs->ttc, fs_esp2tt(type)); /* remove FT */ rx_destroy(priv, type); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h deleted file mode 100644 index e4eeb2ba21c7..000000000000 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.h +++ /dev/null @@ -1,21 +0,0 @@ -/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ -/* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ - -#ifndef __MLX5_IPSEC_STEERING_H__ -#define __MLX5_IPSEC_STEERING_H__ - -#include "en.h" -#include "ipsec.h" -#include "ipsec_offload.h" -#include "en/fs.h" - -void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec); -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5_accel_esp_xfrm_attrs *attrs, - u32 ipsec_obj_id, - struct mlx5e_ipsec_rule *ipsec_rule); -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5_accel_esp_xfrm_attrs *attrs, - struct mlx5e_ipsec_rule *ipsec_rule); -#endif /* __MLX5_IPSEC_STEERING_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h index d016624fbc9d..948400dee525 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls.h @@ -42,6 +42,8 @@ static inline bool mlx5e_ktls_type_check(struct mlx5_core_dev *mdev, } void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv); +int mlx5e_ktls_init_tx(struct mlx5e_priv *priv); +void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv); int mlx5e_ktls_init_rx(struct mlx5e_priv *priv); void mlx5e_ktls_cleanup_rx(struct mlx5e_priv *priv); int mlx5e_ktls_set_feature_rx(struct net_device *netdev, bool enable); @@ -62,6 +64,8 @@ static inline bool mlx5e_is_ktls_rx(struct mlx5_core_dev *mdev) struct mlx5e_tls_sw_stats { atomic64_t tx_tls_ctx; atomic64_t tx_tls_del; + atomic64_t tx_tls_pool_alloc; + atomic64_t tx_tls_pool_free; atomic64_t rx_tls_ctx; atomic64_t rx_tls_del; }; @@ -69,6 +73,7 @@ struct mlx5e_tls_sw_stats { struct mlx5e_tls { struct mlx5e_tls_sw_stats sw_stats; struct workqueue_struct *rx_wq; + struct mlx5e_tls_tx_pool *tx_pool; }; int mlx5e_ktls_init(struct mlx5e_priv *priv); @@ -83,6 +88,15 @@ static inline void mlx5e_ktls_build_netdev(struct mlx5e_priv *priv) { } +static inline int mlx5e_ktls_init_tx(struct mlx5e_priv *priv) +{ + return 0; +} + +static inline void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv) +{ +} + static inline int mlx5e_ktls_init_rx(struct mlx5e_priv *priv) { return 0; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c index 2ab46c4247ff..7c1c0eb16787 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_stats.c @@ -41,6 +41,8 @@ static const struct counter_desc mlx5e_ktls_sw_stats_desc[] = { { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_ctx) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_del) }, + { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_pool_alloc) }, + { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, tx_tls_pool_free) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, rx_tls_ctx) }, { MLX5E_DECLARE_STAT(struct mlx5e_tls_sw_stats, rx_tls_del) }, }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c index f239fb2e832f..6b6c7044b64a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ktls_tx.c @@ -35,30 +35,70 @@ u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *pa stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS); stop_room += num_dumps * mlx5e_stop_room_for_wqe(mdev, MLX5E_KTLS_DUMP_WQEBBS); + stop_room += 1; /* fence nop */ return stop_room; } +static void mlx5e_ktls_set_tisc(struct mlx5_core_dev *mdev, void *tisc) +{ + MLX5_SET(tisc, tisc, tls_en, 1); + MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn); + MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); +} + static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) { u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; - void *tisc; - tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); + mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx)); - MLX5_SET(tisc, tisc, tls_en, 1); + return mlx5_core_create_tis(mdev, in, tisn); +} + +static int mlx5e_ktls_create_tis_cb(struct mlx5_core_dev *mdev, + struct mlx5_async_ctx *async_ctx, + u32 *out, int outlen, + mlx5_async_cbk_t callback, + struct mlx5_async_work *context) +{ + u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; + + mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx)); + MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); + + return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in), + out, outlen, callback, context); +} + +static int mlx5e_ktls_destroy_tis_cb(struct mlx5_core_dev *mdev, u32 tisn, + struct mlx5_async_ctx *async_ctx, + u32 *out, int outlen, + mlx5_async_cbk_t callback, + struct mlx5_async_work *context) +{ + u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; + + MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); + MLX5_SET(destroy_tis_in, in, tisn, tisn); - return mlx5e_create_tis(mdev, in, tisn); + return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in), + out, outlen, callback, context); } struct mlx5e_ktls_offload_context_tx { - struct tls_offload_context_tx *tx_ctx; - struct tls12_crypto_info_aes_gcm_128 crypto_info; - struct mlx5e_tls_sw_stats *sw_stats; + /* fast path */ u32 expected_seq; u32 tisn; - u32 key_id; bool ctx_post_pending; + /* control / resync */ + struct list_head list_node; /* member of the pool */ + struct tls12_crypto_info_aes_gcm_128 crypto_info; + struct tls_offload_context_tx *tx_ctx; + struct mlx5_core_dev *mdev; + struct mlx5e_tls_sw_stats *sw_stats; + u32 key_id; + u8 create_err : 1; }; static void @@ -82,28 +122,368 @@ mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) return *ctx; } +/* struct for callback API management */ +struct mlx5e_async_ctx { + struct mlx5_async_work context; + struct mlx5_async_ctx async_ctx; + struct work_struct work; + struct mlx5e_ktls_offload_context_tx *priv_tx; + struct completion complete; + int err; + union { + u32 out_create[MLX5_ST_SZ_DW(create_tis_out)]; + u32 out_destroy[MLX5_ST_SZ_DW(destroy_tis_out)]; + }; +}; + +static struct mlx5e_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, int n) +{ + struct mlx5e_async_ctx *bulk_async; + int i; + + bulk_async = kvcalloc(n, sizeof(struct mlx5e_async_ctx), GFP_KERNEL); + if (!bulk_async) + return NULL; + + for (i = 0; i < n; i++) { + struct mlx5e_async_ctx *async = &bulk_async[i]; + + mlx5_cmd_init_async_ctx(mdev, &async->async_ctx); + init_completion(&async->complete); + } + + return bulk_async; +} + +static void mlx5e_bulk_async_cleanup(struct mlx5e_async_ctx *bulk_async, int n) +{ + int i; + + for (i = 0; i < n; i++) { + struct mlx5e_async_ctx *async = &bulk_async[i]; + + mlx5_cmd_cleanup_async_ctx(&async->async_ctx); + } + kvfree(bulk_async); +} + +static void create_tis_callback(int status, struct mlx5_async_work *context) +{ + struct mlx5e_async_ctx *async = + container_of(context, struct mlx5e_async_ctx, context); + struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; + + if (status) { + async->err = status; + priv_tx->create_err = 1; + goto out; + } + + priv_tx->tisn = MLX5_GET(create_tis_out, async->out_create, tisn); +out: + complete(&async->complete); +} + +static void destroy_tis_callback(int status, struct mlx5_async_work *context) +{ + struct mlx5e_async_ctx *async = + container_of(context, struct mlx5e_async_ctx, context); + struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; + + complete(&async->complete); + kfree(priv_tx); +} + +static struct mlx5e_ktls_offload_context_tx * +mlx5e_tls_priv_tx_init(struct mlx5_core_dev *mdev, struct mlx5e_tls_sw_stats *sw_stats, + struct mlx5e_async_ctx *async) +{ + struct mlx5e_ktls_offload_context_tx *priv_tx; + int err; + + priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); + if (!priv_tx) + return ERR_PTR(-ENOMEM); + + priv_tx->mdev = mdev; + priv_tx->sw_stats = sw_stats; + + if (!async) { + err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); + if (err) + goto err_out; + } else { + async->priv_tx = priv_tx; + err = mlx5e_ktls_create_tis_cb(mdev, &async->async_ctx, + async->out_create, sizeof(async->out_create), + create_tis_callback, &async->context); + if (err) + goto err_out; + } + + return priv_tx; + +err_out: + kfree(priv_tx); + return ERR_PTR(err); +} + +static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv_tx, + struct mlx5e_async_ctx *async) +{ + if (priv_tx->create_err) { + complete(&async->complete); + kfree(priv_tx); + return; + } + async->priv_tx = priv_tx; + mlx5e_ktls_destroy_tis_cb(priv_tx->mdev, priv_tx->tisn, + &async->async_ctx, + async->out_destroy, sizeof(async->out_destroy), + destroy_tis_callback, &async->context); +} + +static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, + struct list_head *list, int size) +{ + struct mlx5e_ktls_offload_context_tx *obj; + struct mlx5e_async_ctx *bulk_async; + int i; + + bulk_async = mlx5e_bulk_async_init(mdev, size); + if (!bulk_async) + return; + + i = 0; + list_for_each_entry(obj, list, list_node) { + mlx5e_tls_priv_tx_cleanup(obj, &bulk_async[i]); + i++; + } + + for (i = 0; i < size; i++) { + struct mlx5e_async_ctx *async = &bulk_async[i]; + + wait_for_completion(&async->complete); + } + mlx5e_bulk_async_cleanup(bulk_async, size); +} + +/* Recycling pool API */ + +#define MLX5E_TLS_TX_POOL_BULK (16) +#define MLX5E_TLS_TX_POOL_HIGH (4 * 1024) +#define MLX5E_TLS_TX_POOL_LOW (MLX5E_TLS_TX_POOL_HIGH / 4) + +struct mlx5e_tls_tx_pool { + struct mlx5_core_dev *mdev; + struct mlx5e_tls_sw_stats *sw_stats; + struct mutex lock; /* Protects access to the pool */ + struct list_head list; + size_t size; + + struct workqueue_struct *wq; + struct work_struct create_work; + struct work_struct destroy_work; +}; + +static void create_work(struct work_struct *work) +{ + struct mlx5e_tls_tx_pool *pool = + container_of(work, struct mlx5e_tls_tx_pool, create_work); + struct mlx5e_ktls_offload_context_tx *obj; + struct mlx5e_async_ctx *bulk_async; + LIST_HEAD(local_list); + int i, j, err = 0; + + bulk_async = mlx5e_bulk_async_init(pool->mdev, MLX5E_TLS_TX_POOL_BULK); + if (!bulk_async) + return; + + for (i = 0; i < MLX5E_TLS_TX_POOL_BULK; i++) { + obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, &bulk_async[i]); + if (IS_ERR(obj)) { + err = PTR_ERR(obj); + break; + } + list_add(&obj->list_node, &local_list); + } + + for (j = 0; j < i; j++) { + struct mlx5e_async_ctx *async = &bulk_async[j]; + + wait_for_completion(&async->complete); + if (!err && async->err) + err = async->err; + } + atomic64_add(i, &pool->sw_stats->tx_tls_pool_alloc); + mlx5e_bulk_async_cleanup(bulk_async, MLX5E_TLS_TX_POOL_BULK); + if (err) + goto err_out; + + mutex_lock(&pool->lock); + if (pool->size + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH) { + mutex_unlock(&pool->lock); + goto err_out; + } + list_splice(&local_list, &pool->list); + pool->size += MLX5E_TLS_TX_POOL_BULK; + if (pool->size <= MLX5E_TLS_TX_POOL_LOW) + queue_work(pool->wq, work); + mutex_unlock(&pool->lock); + return; + +err_out: + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, i); + atomic64_add(i, &pool->sw_stats->tx_tls_pool_free); +} + +static void destroy_work(struct work_struct *work) +{ + struct mlx5e_tls_tx_pool *pool = + container_of(work, struct mlx5e_tls_tx_pool, destroy_work); + struct mlx5e_ktls_offload_context_tx *obj; + LIST_HEAD(local_list); + int i = 0; + + mutex_lock(&pool->lock); + if (pool->size < MLX5E_TLS_TX_POOL_HIGH) { + mutex_unlock(&pool->lock); + return; + } + + list_for_each_entry(obj, &pool->list, list_node) + if (++i == MLX5E_TLS_TX_POOL_BULK) + break; + + list_cut_position(&local_list, &pool->list, &obj->list_node); + pool->size -= MLX5E_TLS_TX_POOL_BULK; + if (pool->size >= MLX5E_TLS_TX_POOL_HIGH) + queue_work(pool->wq, work); + mutex_unlock(&pool->lock); + + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK); + atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free); +} + +static struct mlx5e_tls_tx_pool *mlx5e_tls_tx_pool_init(struct mlx5_core_dev *mdev, + struct mlx5e_tls_sw_stats *sw_stats) +{ + struct mlx5e_tls_tx_pool *pool; + + BUILD_BUG_ON(MLX5E_TLS_TX_POOL_LOW + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH); + + pool = kvzalloc(sizeof(*pool), GFP_KERNEL); + if (!pool) + return NULL; + + pool->wq = create_singlethread_workqueue("mlx5e_tls_tx_pool"); + if (!pool->wq) + goto err_free; + + INIT_LIST_HEAD(&pool->list); + mutex_init(&pool->lock); + + INIT_WORK(&pool->create_work, create_work); + INIT_WORK(&pool->destroy_work, destroy_work); + + pool->mdev = mdev; + pool->sw_stats = sw_stats; + + return pool; + +err_free: + kvfree(pool); + return NULL; +} + +static void mlx5e_tls_tx_pool_list_cleanup(struct mlx5e_tls_tx_pool *pool) +{ + while (pool->size > MLX5E_TLS_TX_POOL_BULK) { + struct mlx5e_ktls_offload_context_tx *obj; + LIST_HEAD(local_list); + int i = 0; + + list_for_each_entry(obj, &pool->list, list_node) + if (++i == MLX5E_TLS_TX_POOL_BULK) + break; + + list_cut_position(&local_list, &pool->list, &obj->list_node); + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK); + atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free); + pool->size -= MLX5E_TLS_TX_POOL_BULK; + } + if (pool->size) { + mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &pool->list, pool->size); + atomic64_add(pool->size, &pool->sw_stats->tx_tls_pool_free); + } +} + +static void mlx5e_tls_tx_pool_cleanup(struct mlx5e_tls_tx_pool *pool) +{ + mlx5e_tls_tx_pool_list_cleanup(pool); + destroy_workqueue(pool->wq); + kvfree(pool); +} + +static void pool_push(struct mlx5e_tls_tx_pool *pool, struct mlx5e_ktls_offload_context_tx *obj) +{ + mutex_lock(&pool->lock); + list_add(&obj->list_node, &pool->list); + if (++pool->size == MLX5E_TLS_TX_POOL_HIGH) + queue_work(pool->wq, &pool->destroy_work); + mutex_unlock(&pool->lock); +} + +static struct mlx5e_ktls_offload_context_tx *pool_pop(struct mlx5e_tls_tx_pool *pool) +{ + struct mlx5e_ktls_offload_context_tx *obj; + + mutex_lock(&pool->lock); + if (unlikely(pool->size == 0)) { + /* pool is empty: + * - trigger the populating work, and + * - serve the current context via the regular blocking api. + */ + queue_work(pool->wq, &pool->create_work); + mutex_unlock(&pool->lock); + obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, NULL); + if (!IS_ERR(obj)) + atomic64_inc(&pool->sw_stats->tx_tls_pool_alloc); + return obj; + } + + obj = list_first_entry(&pool->list, struct mlx5e_ktls_offload_context_tx, + list_node); + list_del(&obj->list_node); + if (--pool->size == MLX5E_TLS_TX_POOL_LOW) + queue_work(pool->wq, &pool->create_work); + mutex_unlock(&pool->lock); + return obj; +} + +/* End of pool API */ + int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn) { struct mlx5e_ktls_offload_context_tx *priv_tx; + struct mlx5e_tls_tx_pool *pool; struct tls_context *tls_ctx; - struct mlx5_core_dev *mdev; struct mlx5e_priv *priv; int err; tls_ctx = tls_get_ctx(sk); priv = netdev_priv(netdev); - mdev = priv->mdev; + pool = priv->tls->tx_pool; - priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); - if (!priv_tx) - return -ENOMEM; + priv_tx = pool_pop(pool); + if (IS_ERR(priv_tx)) + return PTR_ERR(priv_tx); - err = mlx5_ktls_create_key(mdev, crypto_info, &priv_tx->key_id); + err = mlx5_ktls_create_key(pool->mdev, crypto_info, &priv_tx->key_id); if (err) goto err_create_key; - priv_tx->sw_stats = &priv->tls->sw_stats; priv_tx->expected_seq = start_offload_tcp_sn; priv_tx->crypto_info = *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; @@ -111,36 +491,29 @@ int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); - err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); - if (err) - goto err_create_tis; - priv_tx->ctx_post_pending = true; atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx); return 0; -err_create_tis: - mlx5_ktls_destroy_key(mdev, priv_tx->key_id); err_create_key: - kfree(priv_tx); + pool_push(pool, priv_tx); return err; } void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) { struct mlx5e_ktls_offload_context_tx *priv_tx; - struct mlx5_core_dev *mdev; + struct mlx5e_tls_tx_pool *pool; struct mlx5e_priv *priv; priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); priv = netdev_priv(netdev); - mdev = priv->mdev; + pool = priv->tls->tx_pool; atomic64_inc(&priv_tx->sw_stats->tx_tls_del); - mlx5e_destroy_tis(mdev, priv_tx->tisn); - mlx5_ktls_destroy_key(mdev, priv_tx->key_id); - kfree(priv_tx); + mlx5_ktls_destroy_key(priv_tx->mdev, priv_tx->key_id); + pool_push(pool, priv_tx); } static void tx_fill_wi(struct mlx5e_txqsq *sq, @@ -201,6 +574,16 @@ post_progress_params(struct mlx5e_txqsq *sq, sq->pc += num_wqebbs; } +static void tx_post_fence_nop(struct mlx5e_txqsq *sq) +{ + struct mlx5_wq_cyc *wq = &sq->wq; + u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); + + tx_fill_wi(sq, pi, 1, 0, NULL); + + mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc); +} + static void mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, struct mlx5e_ktls_offload_context_tx *priv_tx, @@ -212,6 +595,7 @@ mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, post_static_params(sq, priv_tx, fence_first_post); post_progress_params(sq, priv_tx, progress_fence); + tx_post_fence_nop(sq); } struct tx_sync_info { @@ -304,7 +688,7 @@ tx_post_resync_params(struct mlx5e_txqsq *sq, } static int -tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool first) +tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn) { struct mlx5_wqe_ctrl_seg *cseg; struct mlx5_wqe_data_seg *dseg; @@ -326,7 +710,6 @@ tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn, bool fir cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP); cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); cseg->tis_tir_num = cpu_to_be32(tisn << 8); - cseg->fm_ce_se = first ? MLX5_FENCE_MODE_INITIATOR_SMALL : 0; fsz = skb_frag_size(frag); dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, @@ -361,67 +744,39 @@ void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, stats->tls_dump_bytes += wi->num_bytes; } -static void tx_post_fence_nop(struct mlx5e_txqsq *sq) -{ - struct mlx5_wq_cyc *wq = &sq->wq; - u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); - - tx_fill_wi(sq, pi, 1, 0, NULL); - - mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc); -} - static enum mlx5e_ktls_sync_retval mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx, struct mlx5e_txqsq *sq, int datalen, u32 seq) { - struct mlx5e_sq_stats *stats = sq->stats; enum mlx5e_ktls_sync_retval ret; struct tx_sync_info info = {}; - int i = 0; + int i; ret = tx_sync_info_get(priv_tx, seq, datalen, &info); - if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) { - if (ret == MLX5E_KTLS_SYNC_SKIP_NO_DATA) { - stats->tls_skip_no_sync_data++; - return MLX5E_KTLS_SYNC_SKIP_NO_DATA; - } - /* We might get here if a retransmission reaches the driver - * after the relevant record is acked. + if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) + /* We might get here with ret == FAIL if a retransmission + * reaches the driver after the relevant record is acked. * It should be safe to drop the packet in this case */ - stats->tls_drop_no_sync_data++; - goto err_out; - } - - stats->tls_ooo++; + return ret; tx_post_resync_params(sq, priv_tx, info.rcd_sn); - /* If no dump WQE was sent, we need to have a fence NOP WQE before the - * actual data xmit. - */ - if (!info.nr_frags) { - tx_post_fence_nop(sq); - return MLX5E_KTLS_SYNC_DONE; - } - - for (; i < info.nr_frags; i++) { + for (i = 0; i < info.nr_frags; i++) { unsigned int orig_fsz, frag_offset = 0, n = 0; skb_frag_t *f = &info.frags[i]; orig_fsz = skb_frag_size(f); do { - bool fence = !(i || frag_offset); unsigned int fsz; n++; fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset); skb_frag_size_set(f, fsz); - if (tx_post_resync_dump(sq, f, priv_tx->tisn, fence)) { + if (tx_post_resync_dump(sq, f, priv_tx->tisn)) { page_ref_add(skb_frag_page(f), n - 1); goto err_out; } @@ -457,7 +812,7 @@ bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, int datalen; u32 seq; - datalen = skb->len - (skb_transport_offset(skb) + tcp_hdrlen(skb)); + datalen = skb->len - skb_tcp_all_headers(skb); if (!datalen) return true; @@ -469,24 +824,27 @@ bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); - if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) { + if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false); - } seq = ntohl(tcp_hdr(skb)->seq); if (unlikely(priv_tx->expected_seq != seq)) { enum mlx5e_ktls_sync_retval ret = mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq); + stats->tls_ooo++; + switch (ret) { case MLX5E_KTLS_SYNC_DONE: break; case MLX5E_KTLS_SYNC_SKIP_NO_DATA: + stats->tls_skip_no_sync_data++; if (likely(!skb->decrypted)) goto out; WARN_ON_ONCE(1); - fallthrough; + goto err_out; case MLX5E_KTLS_SYNC_FAIL: + stats->tls_drop_no_sync_data++; goto err_out; } } @@ -505,3 +863,24 @@ err_out: dev_kfree_skb_any(skb); return false; } + +int mlx5e_ktls_init_tx(struct mlx5e_priv *priv) +{ + if (!mlx5e_is_ktls_tx(priv->mdev)) + return 0; + + priv->tls->tx_pool = mlx5e_tls_tx_pool_init(priv->mdev, &priv->tls->sw_stats); + if (!priv->tls->tx_pool) + return -ENOMEM; + + return 0; +} + +void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv) +{ + if (!mlx5e_is_ktls_tx(priv->mdev)) + return; + + mlx5e_tls_tx_pool_cleanup(priv->tls->tx_pool); + priv->tls->tx_pool = NULL; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c index 49cca6bd49a1..cd7f245dcf14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_arfs.c @@ -120,7 +120,7 @@ static int arfs_disable(struct mlx5e_priv *priv) for (i = 0; i < ARFS_NUM_TYPES; i++) { /* Modify ttc rules destination back to their default */ - err = mlx5_ttc_fwd_default_dest(priv->fs.ttc, arfs_get_tt(i)); + err = mlx5_ttc_fwd_default_dest(priv->fs->ttc, arfs_get_tt(i)); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] default destination failed, err(%d)\n", @@ -147,9 +147,9 @@ int mlx5e_arfs_enable(struct mlx5e_priv *priv) dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; for (i = 0; i < ARFS_NUM_TYPES; i++) { - dest.ft = priv->fs.arfs->arfs_tables[i].ft.t; + dest.ft = priv->fs->arfs->arfs_tables[i].ft.t; /* Modify ttc rules destination to point on the aRFS FTs */ - err = mlx5_ttc_fwd_dest(priv->fs.ttc, arfs_get_tt(i), &dest); + err = mlx5_ttc_fwd_dest(priv->fs->ttc, arfs_get_tt(i), &dest); if (err) { netdev_err(priv->netdev, "%s: modify ttc[%d] dest to arfs, failed err(%d)\n", @@ -172,10 +172,10 @@ static void _mlx5e_cleanup_tables(struct mlx5e_priv *priv) int i; arfs_del_rules(priv); - destroy_workqueue(priv->fs.arfs->wq); + destroy_workqueue(priv->fs->arfs->wq); for (i = 0; i < ARFS_NUM_TYPES; i++) { - if (!IS_ERR_OR_NULL(priv->fs.arfs->arfs_tables[i].ft.t)) - arfs_destroy_table(&priv->fs.arfs->arfs_tables[i]); + if (!IS_ERR_OR_NULL(priv->fs->arfs->arfs_tables[i].ft.t)) + arfs_destroy_table(&priv->fs->arfs->arfs_tables[i]); } } @@ -185,13 +185,13 @@ void mlx5e_arfs_destroy_tables(struct mlx5e_priv *priv) return; _mlx5e_cleanup_tables(priv); - kvfree(priv->fs.arfs); + kvfree(priv->fs->arfs); } static int arfs_add_default_rule(struct mlx5e_priv *priv, enum arfs_type type) { - struct arfs_table *arfs_t = &priv->fs.arfs->arfs_tables[type]; + struct arfs_table *arfs_t = &priv->fs->arfs->arfs_tables[type]; struct mlx5_flow_destination dest = {}; MLX5_DECLARE_FLOW_ACT(flow_act); enum mlx5_traffic_types tt; @@ -321,7 +321,7 @@ out: static int arfs_create_table(struct mlx5e_priv *priv, enum arfs_type type) { - struct mlx5e_arfs_tables *arfs = priv->fs.arfs; + struct mlx5e_arfs_tables *arfs = priv->fs->arfs; struct mlx5e_flow_table *ft = &arfs->arfs_tables[type].ft; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -332,7 +332,7 @@ static int arfs_create_table(struct mlx5e_priv *priv, ft_attr.level = MLX5E_ARFS_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -361,14 +361,14 @@ int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) if (!(priv->netdev->hw_features & NETIF_F_NTUPLE)) return 0; - priv->fs.arfs = kvzalloc(sizeof(*priv->fs.arfs), GFP_KERNEL); - if (!priv->fs.arfs) + priv->fs->arfs = kvzalloc(sizeof(*priv->fs->arfs), GFP_KERNEL); + if (!priv->fs->arfs) return -ENOMEM; - spin_lock_init(&priv->fs.arfs->arfs_lock); - INIT_LIST_HEAD(&priv->fs.arfs->rules); - priv->fs.arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); - if (!priv->fs.arfs->wq) + spin_lock_init(&priv->fs->arfs->arfs_lock); + INIT_LIST_HEAD(&priv->fs->arfs->rules); + priv->fs->arfs->wq = create_singlethread_workqueue("mlx5e_arfs"); + if (!priv->fs->arfs->wq) goto err; for (i = 0; i < ARFS_NUM_TYPES; i++) { @@ -381,7 +381,7 @@ int mlx5e_arfs_create_tables(struct mlx5e_priv *priv) err_des: _mlx5e_cleanup_tables(priv); err: - kvfree(priv->fs.arfs); + kvfree(priv->fs->arfs); return err; } @@ -396,8 +396,8 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv) int i; int j; - spin_lock_bh(&priv->fs.arfs->arfs_lock); - mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs.arfs->arfs_tables, i, j) { + spin_lock_bh(&priv->fs->arfs->arfs_lock); + mlx5e_for_each_arfs_rule(arfs_rule, htmp, priv->fs->arfs->arfs_tables, i, j) { if (!work_pending(&arfs_rule->arfs_work) && rps_may_expire_flow(priv->netdev, arfs_rule->rxq, arfs_rule->flow_id, @@ -408,7 +408,7 @@ static void arfs_may_expire_flow(struct mlx5e_priv *priv) break; } } - spin_unlock_bh(&priv->fs.arfs->arfs_lock); + spin_unlock_bh(&priv->fs->arfs->arfs_lock); hlist_for_each_entry_safe(arfs_rule, htmp, &del_list, hlist) { if (arfs_rule->rule) mlx5_del_flow_rules(arfs_rule->rule); @@ -425,12 +425,12 @@ static void arfs_del_rules(struct mlx5e_priv *priv) int i; int j; - spin_lock_bh(&priv->fs.arfs->arfs_lock); - mlx5e_for_each_arfs_rule(rule, htmp, priv->fs.arfs->arfs_tables, i, j) { + spin_lock_bh(&priv->fs->arfs->arfs_lock); + mlx5e_for_each_arfs_rule(rule, htmp, priv->fs->arfs->arfs_tables, i, j) { hlist_del_init(&rule->hlist); hlist_add_head(&rule->hlist, &del_list); } - spin_unlock_bh(&priv->fs.arfs->arfs_lock); + spin_unlock_bh(&priv->fs->arfs->arfs_lock); hlist_for_each_entry_safe(rule, htmp, &del_list, hlist) { cancel_work_sync(&rule->arfs_work); @@ -474,7 +474,7 @@ static struct arfs_table *arfs_get_table(struct mlx5e_arfs_tables *arfs, static struct mlx5_flow_handle *arfs_add_rule(struct mlx5e_priv *priv, struct arfs_rule *arfs_rule) { - struct mlx5e_arfs_tables *arfs = priv->fs.arfs; + struct mlx5e_arfs_tables *arfs = priv->fs->arfs; struct arfs_tuple *tuple = &arfs_rule->tuple; struct mlx5_flow_handle *rule = NULL; struct mlx5_flow_destination dest = {}; @@ -592,9 +592,9 @@ static void arfs_handle_work(struct work_struct *work) mutex_lock(&priv->state_lock); if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { - spin_lock_bh(&priv->fs.arfs->arfs_lock); + spin_lock_bh(&priv->fs->arfs->arfs_lock); hlist_del(&arfs_rule->hlist); - spin_unlock_bh(&priv->fs.arfs->arfs_lock); + spin_unlock_bh(&priv->fs->arfs->arfs_lock); mutex_unlock(&priv->state_lock); kfree(arfs_rule); @@ -647,7 +647,7 @@ static struct arfs_rule *arfs_alloc_rule(struct mlx5e_priv *priv, tuple->dst_port = fk->ports.dst; rule->flow_id = flow_id; - rule->filter_id = priv->fs.arfs->last_filter_id++ % RPS_NO_FILTER; + rule->filter_id = priv->fs->arfs->last_filter_id++ % RPS_NO_FILTER; hlist_add_head(&rule->hlist, arfs_hash_bucket(arfs_t, tuple->src_port, @@ -691,7 +691,7 @@ int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, u16 rxq_index, u32 flow_id) { struct mlx5e_priv *priv = netdev_priv(dev); - struct mlx5e_arfs_tables *arfs = priv->fs.arfs; + struct mlx5e_arfs_tables *arfs = priv->fs->arfs; struct arfs_table *arfs_t; struct arfs_rule *arfs_rule; struct flow_keys fk; @@ -725,7 +725,7 @@ int mlx5e_rx_flow_steer(struct net_device *dev, const struct sk_buff *skb, return -ENOMEM; } } - queue_work(priv->fs.arfs->wq, &arfs_rule->arfs_work); + queue_work(priv->fs->arfs->wq, &arfs_rule->arfs_work); spin_unlock_bh(&arfs->arfs_lock); return arfs_rule->filter_id; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c index 6e80585d731f..b811207fe5ed 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_ethtool.c @@ -30,6 +30,8 @@ * SOFTWARE. */ +#include <linux/ethtool_netlink.h> + #include "en.h" #include "en/port.h" #include "en/params.h" @@ -305,12 +307,18 @@ static void mlx5e_get_ethtool_stats(struct net_device *dev, } void mlx5e_ethtool_get_ringparam(struct mlx5e_priv *priv, - struct ethtool_ringparam *param) + struct ethtool_ringparam *param, + struct kernel_ethtool_ringparam *kernel_param) { param->rx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_RQ_SIZE; param->tx_max_pending = 1 << MLX5E_PARAMS_MAXIMUM_LOG_SQ_SIZE; param->rx_pending = 1 << priv->channels.params.log_rq_mtu_frames; param->tx_pending = 1 << priv->channels.params.log_sq_size; + + kernel_param->tcp_data_split = + (priv->channels.params.packet_merge.type == MLX5E_PACKET_MERGE_SHAMPO) ? + ETHTOOL_TCP_DATA_SPLIT_ENABLED : + ETHTOOL_TCP_DATA_SPLIT_DISABLED; } static void mlx5e_get_ringparam(struct net_device *dev, @@ -320,7 +328,7 @@ static void mlx5e_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = netdev_priv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } int mlx5e_ethtool_set_ringparam(struct mlx5e_priv *priv, @@ -451,7 +459,7 @@ int mlx5e_ethtool_set_channels(struct mlx5e_priv *priv, * because the numeration of the QoS SQs will change, while per-queue * qdiscs are attached. */ - if (priv->htb.maj_id) { + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { err = -EINVAL; netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the number of channels\n", __func__); @@ -2067,7 +2075,7 @@ static int set_pflag_tx_port_ts(struct net_device *netdev, bool enable) * the numeration of the QoS SQs will change, while per-queue qdiscs are * attached. */ - if (priv->htb.maj_id) { + if (mlx5e_selq_is_htb_enabled(&priv->selq)) { netdev_err(priv->netdev, "%s: HTB offload is active, cannot change the PTP state\n", __func__); return -EINVAL; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c index d2f0773f95c6..e2a9b9be5c1f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs.c @@ -37,13 +37,13 @@ #include <linux/mlx5/fs.h> #include <linux/mlx5/mpfs.h> #include "en.h" -#include "en_rep.h" +#include "en_tc.h" #include "lib/mpfs.h" #include "en/ptp.h" -static int mlx5e_add_l2_flow_rule(struct mlx5e_priv *priv, +static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs, struct mlx5e_l2_rule *ai, int type); -static void mlx5e_del_l2_flow_rule(struct mlx5e_priv *priv, +static void mlx5e_del_l2_flow_rule(struct mlx5e_flow_steering *fs, struct mlx5e_l2_rule *ai); enum { @@ -132,9 +132,8 @@ struct mlx5_flow_table *mlx5e_vlan_get_flowtable(struct mlx5e_vlan_table *vlan) return vlan->ft.t; } -static int mlx5e_vport_context_update_vlans(struct mlx5e_priv *priv) +static int mlx5e_vport_context_update_vlans(struct mlx5e_flow_steering *fs) { - struct net_device *ndev = priv->netdev; int max_list_size; int list_size; u16 *vlans; @@ -143,15 +142,15 @@ static int mlx5e_vport_context_update_vlans(struct mlx5e_priv *priv) int i; list_size = 0; - for_each_set_bit(vlan, priv->fs.vlan->active_cvlans, VLAN_N_VID) + for_each_set_bit(vlan, fs->vlan->active_cvlans, VLAN_N_VID) list_size++; - max_list_size = 1 << MLX5_CAP_GEN(priv->mdev, log_max_vlan_list); + max_list_size = 1 << MLX5_CAP_GEN(fs->mdev, log_max_vlan_list); if (list_size > max_list_size) { - netdev_warn(ndev, - "netdev vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n", - list_size, max_list_size); + mlx5_core_warn(fs->mdev, + "netdev vlans list size (%d) > (%d) max vport list size, some vlans will be dropped\n", + list_size, max_list_size); list_size = max_list_size; } @@ -160,16 +159,16 @@ static int mlx5e_vport_context_update_vlans(struct mlx5e_priv *priv) return -ENOMEM; i = 0; - for_each_set_bit(vlan, priv->fs.vlan->active_cvlans, VLAN_N_VID) { + for_each_set_bit(vlan, fs->vlan->active_cvlans, VLAN_N_VID) { if (i >= list_size) break; vlans[i++] = vlan; } - err = mlx5_modify_nic_vport_vlans(priv->mdev, vlans, list_size); + err = mlx5_modify_nic_vport_vlans(fs->mdev, vlans, list_size); if (err) - netdev_err(ndev, "Failed to modify vport vlans list err(%d)\n", - err); + mlx5_core_err(fs->mdev, "Failed to modify vport vlans list err(%d)\n", + err); kvfree(vlans); return err; @@ -183,18 +182,18 @@ enum mlx5e_vlan_rule_type { MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, }; -static int __mlx5e_add_vlan_rule(struct mlx5e_priv *priv, +static int __mlx5e_add_vlan_rule(struct mlx5e_flow_steering *fs, enum mlx5e_vlan_rule_type rule_type, u16 vid, struct mlx5_flow_spec *spec) { - struct mlx5_flow_table *ft = priv->fs.vlan->ft.t; + struct mlx5_flow_table *ft = fs->vlan->ft.t; struct mlx5_flow_destination dest = {}; struct mlx5_flow_handle **rule_p; MLX5_DECLARE_FLOW_ACT(flow_act); int err = 0; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = priv->fs.l2.ft.t; + dest.ft = fs->l2.ft.t; spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; @@ -204,24 +203,24 @@ static int __mlx5e_add_vlan_rule(struct mlx5e_priv *priv, * disabled in match value means both S & C tags * don't exist (untagged of both) */ - rule_p = &priv->fs.vlan->untagged_rule; + rule_p = &fs->vlan->untagged_rule; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); break; case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID: - rule_p = &priv->fs.vlan->any_cvlan_rule; + rule_p = &fs->vlan->any_cvlan_rule; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); MLX5_SET(fte_match_param, spec->match_value, outer_headers.cvlan_tag, 1); break; case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID: - rule_p = &priv->fs.vlan->any_svlan_rule; + rule_p = &fs->vlan->any_svlan_rule; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.svlan_tag); MLX5_SET(fte_match_param, spec->match_value, outer_headers.svlan_tag, 1); break; case MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID: - rule_p = &priv->fs.vlan->active_svlans_rule[vid]; + rule_p = &fs->vlan->active_svlans_rule[vid]; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.svlan_tag); MLX5_SET(fte_match_param, spec->match_value, outer_headers.svlan_tag, 1); @@ -231,7 +230,7 @@ static int __mlx5e_add_vlan_rule(struct mlx5e_priv *priv, vid); break; default: /* MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID */ - rule_p = &priv->fs.vlan->active_cvlans_rule[vid]; + rule_p = &fs->vlan->active_cvlans_rule[vid]; MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.cvlan_tag); MLX5_SET(fte_match_param, spec->match_value, outer_headers.cvlan_tag, 1); @@ -250,13 +249,13 @@ static int __mlx5e_add_vlan_rule(struct mlx5e_priv *priv, if (IS_ERR(*rule_p)) { err = PTR_ERR(*rule_p); *rule_p = NULL; - netdev_err(priv->netdev, "%s: add rule failed\n", __func__); + mlx5_core_err(fs->mdev, "%s: add rule failed\n", __func__); } return err; } -static int mlx5e_add_vlan_rule(struct mlx5e_priv *priv, +static int mlx5e_add_vlan_rule(struct mlx5e_flow_steering *fs, enum mlx5e_vlan_rule_type rule_type, u16 vid) { struct mlx5_flow_spec *spec; @@ -267,68 +266,68 @@ static int mlx5e_add_vlan_rule(struct mlx5e_priv *priv, return -ENOMEM; if (rule_type == MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID) - mlx5e_vport_context_update_vlans(priv); + mlx5e_vport_context_update_vlans(fs); - err = __mlx5e_add_vlan_rule(priv, rule_type, vid, spec); + err = __mlx5e_add_vlan_rule(fs, rule_type, vid, spec); kvfree(spec); return err; } -static void mlx5e_del_vlan_rule(struct mlx5e_priv *priv, - enum mlx5e_vlan_rule_type rule_type, u16 vid) +static void mlx5e_fs_del_vlan_rule(struct mlx5e_flow_steering *fs, + enum mlx5e_vlan_rule_type rule_type, u16 vid) { switch (rule_type) { case MLX5E_VLAN_RULE_TYPE_UNTAGGED: - if (priv->fs.vlan->untagged_rule) { - mlx5_del_flow_rules(priv->fs.vlan->untagged_rule); - priv->fs.vlan->untagged_rule = NULL; + if (fs->vlan->untagged_rule) { + mlx5_del_flow_rules(fs->vlan->untagged_rule); + fs->vlan->untagged_rule = NULL; } break; case MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID: - if (priv->fs.vlan->any_cvlan_rule) { - mlx5_del_flow_rules(priv->fs.vlan->any_cvlan_rule); - priv->fs.vlan->any_cvlan_rule = NULL; + if (fs->vlan->any_cvlan_rule) { + mlx5_del_flow_rules(fs->vlan->any_cvlan_rule); + fs->vlan->any_cvlan_rule = NULL; } break; case MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID: - if (priv->fs.vlan->any_svlan_rule) { - mlx5_del_flow_rules(priv->fs.vlan->any_svlan_rule); - priv->fs.vlan->any_svlan_rule = NULL; + if (fs->vlan->any_svlan_rule) { + mlx5_del_flow_rules(fs->vlan->any_svlan_rule); + fs->vlan->any_svlan_rule = NULL; } break; case MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID: - if (priv->fs.vlan->active_svlans_rule[vid]) { - mlx5_del_flow_rules(priv->fs.vlan->active_svlans_rule[vid]); - priv->fs.vlan->active_svlans_rule[vid] = NULL; + if (fs->vlan->active_svlans_rule[vid]) { + mlx5_del_flow_rules(fs->vlan->active_svlans_rule[vid]); + fs->vlan->active_svlans_rule[vid] = NULL; } break; case MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID: - if (priv->fs.vlan->active_cvlans_rule[vid]) { - mlx5_del_flow_rules(priv->fs.vlan->active_cvlans_rule[vid]); - priv->fs.vlan->active_cvlans_rule[vid] = NULL; + if (fs->vlan->active_cvlans_rule[vid]) { + mlx5_del_flow_rules(fs->vlan->active_cvlans_rule[vid]); + fs->vlan->active_cvlans_rule[vid] = NULL; } - mlx5e_vport_context_update_vlans(priv); + mlx5e_vport_context_update_vlans(fs); break; } } -static void mlx5e_del_any_vid_rules(struct mlx5e_priv *priv) +static void mlx5e_fs_del_any_vid_rules(struct mlx5e_flow_steering *fs) { - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0); + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0); } -static int mlx5e_add_any_vid_rules(struct mlx5e_priv *priv) +static int mlx5e_fs_add_any_vid_rules(struct mlx5e_flow_steering *fs) { int err; - err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); + err = mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); if (err) return err; - return mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0); + return mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_ANY_STAG_VID, 0); } static struct mlx5_flow_handle * @@ -354,101 +353,101 @@ mlx5e_add_trap_rule(struct mlx5_flow_table *ft, int trap_id, int tir_num) int mlx5e_add_vlan_trap(struct mlx5e_priv *priv, int trap_id, int tir_num) { - struct mlx5_flow_table *ft = priv->fs.vlan->ft.t; + struct mlx5_flow_table *ft = priv->fs->vlan->ft.t; struct mlx5_flow_handle *rule; int err; rule = mlx5e_add_trap_rule(ft, trap_id, tir_num); if (IS_ERR(rule)) { err = PTR_ERR(rule); - priv->fs.vlan->trap_rule = NULL; - netdev_err(priv->netdev, "%s: add VLAN trap rule failed, err %d\n", - __func__, err); + priv->fs->vlan->trap_rule = NULL; + mlx5_core_err(priv->fs->mdev, "%s: add VLAN trap rule failed, err %d\n", + __func__, err); return err; } - priv->fs.vlan->trap_rule = rule; + priv->fs->vlan->trap_rule = rule; return 0; } void mlx5e_remove_vlan_trap(struct mlx5e_priv *priv) { - if (priv->fs.vlan->trap_rule) { - mlx5_del_flow_rules(priv->fs.vlan->trap_rule); - priv->fs.vlan->trap_rule = NULL; + if (priv->fs->vlan->trap_rule) { + mlx5_del_flow_rules(priv->fs->vlan->trap_rule); + priv->fs->vlan->trap_rule = NULL; } } int mlx5e_add_mac_trap(struct mlx5e_priv *priv, int trap_id, int tir_num) { - struct mlx5_flow_table *ft = priv->fs.l2.ft.t; + struct mlx5_flow_table *ft = priv->fs->l2.ft.t; struct mlx5_flow_handle *rule; int err; rule = mlx5e_add_trap_rule(ft, trap_id, tir_num); if (IS_ERR(rule)) { err = PTR_ERR(rule); - priv->fs.l2.trap_rule = NULL; - netdev_err(priv->netdev, "%s: add MAC trap rule failed, err %d\n", - __func__, err); + priv->fs->l2.trap_rule = NULL; + mlx5_core_err(priv->fs->mdev, "%s: add MAC trap rule failed, err %d\n", + __func__, err); return err; } - priv->fs.l2.trap_rule = rule; + priv->fs->l2.trap_rule = rule; return 0; } void mlx5e_remove_mac_trap(struct mlx5e_priv *priv) { - if (priv->fs.l2.trap_rule) { - mlx5_del_flow_rules(priv->fs.l2.trap_rule); - priv->fs.l2.trap_rule = NULL; + if (priv->fs->l2.trap_rule) { + mlx5_del_flow_rules(priv->fs->l2.trap_rule); + priv->fs->l2.trap_rule = NULL; } } void mlx5e_enable_cvlan_filter(struct mlx5e_priv *priv) { - if (!priv->fs.vlan->cvlan_filter_disabled) + if (!priv->fs->vlan->cvlan_filter_disabled) return; - priv->fs.vlan->cvlan_filter_disabled = false; + priv->fs->vlan->cvlan_filter_disabled = false; if (priv->netdev->flags & IFF_PROMISC) return; - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); + mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); } void mlx5e_disable_cvlan_filter(struct mlx5e_priv *priv) { - if (priv->fs.vlan->cvlan_filter_disabled) + if (priv->fs->vlan->cvlan_filter_disabled) return; - priv->fs.vlan->cvlan_filter_disabled = true; + priv->fs->vlan->cvlan_filter_disabled = true; if (priv->netdev->flags & IFF_PROMISC) return; - mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); + mlx5e_add_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_ANY_CTAG_VID, 0); } -static int mlx5e_vlan_rx_add_cvid(struct mlx5e_priv *priv, u16 vid) +static int mlx5e_vlan_rx_add_cvid(struct mlx5e_flow_steering *fs, u16 vid) { int err; - set_bit(vid, priv->fs.vlan->active_cvlans); + set_bit(vid, fs->vlan->active_cvlans); - err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, vid); + err = mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, vid); if (err) - clear_bit(vid, priv->fs.vlan->active_cvlans); + clear_bit(vid, fs->vlan->active_cvlans); return err; } -static int mlx5e_vlan_rx_add_svid(struct mlx5e_priv *priv, u16 vid) +static int mlx5e_vlan_rx_add_svid(struct mlx5e_flow_steering *fs, + struct net_device *netdev, u16 vid) { - struct net_device *netdev = priv->netdev; int err; - set_bit(vid, priv->fs.vlan->active_svlans); + set_bit(vid, fs->vlan->active_svlans); - err = mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, vid); + err = mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, vid); if (err) { - clear_bit(vid, priv->fs.vlan->active_svlans); + clear_bit(vid, fs->vlan->active_svlans); return err; } @@ -457,86 +456,91 @@ static int mlx5e_vlan_rx_add_svid(struct mlx5e_priv *priv, u16 vid) return err; } -int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) +int mlx5e_fs_vlan_rx_add_vid(struct mlx5e_flow_steering *fs, + struct net_device *netdev, + __be16 proto, u16 vid) { - struct mlx5e_priv *priv = netdev_priv(dev); - if (mlx5e_is_uplink_rep(priv)) - return 0; /* no vlan table for uplink rep */ + if (!fs->vlan) { + mlx5_core_err(fs->mdev, "Vlan doesn't exist\n"); + return -EINVAL; + } if (be16_to_cpu(proto) == ETH_P_8021Q) - return mlx5e_vlan_rx_add_cvid(priv, vid); + return mlx5e_vlan_rx_add_cvid(fs, vid); else if (be16_to_cpu(proto) == ETH_P_8021AD) - return mlx5e_vlan_rx_add_svid(priv, vid); + return mlx5e_vlan_rx_add_svid(fs, netdev, vid); return -EOPNOTSUPP; } -int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) +int mlx5e_fs_vlan_rx_kill_vid(struct mlx5e_flow_steering *fs, + struct net_device *netdev, + __be16 proto, u16 vid) { - struct mlx5e_priv *priv = netdev_priv(dev); - - if (mlx5e_is_uplink_rep(priv)) - return 0; /* no vlan table for uplink rep */ + if (!fs->vlan) { + mlx5_core_err(fs->mdev, "Vlan doesn't exist\n"); + return -EINVAL; + } if (be16_to_cpu(proto) == ETH_P_8021Q) { - clear_bit(vid, priv->fs.vlan->active_cvlans); - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, vid); + clear_bit(vid, fs->vlan->active_cvlans); + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, vid); } else if (be16_to_cpu(proto) == ETH_P_8021AD) { - clear_bit(vid, priv->fs.vlan->active_svlans); - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, vid); - netdev_update_features(dev); + clear_bit(vid, fs->vlan->active_svlans); + mlx5e_fs_del_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, vid); + netdev_update_features(netdev); } return 0; } -static void mlx5e_add_vlan_rules(struct mlx5e_priv *priv) +static void mlx5e_fs_add_vlan_rules(struct mlx5e_flow_steering *fs) { int i; - mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); + mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); - for_each_set_bit(i, priv->fs.vlan->active_cvlans, VLAN_N_VID) { - mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i); + for_each_set_bit(i, fs->vlan->active_cvlans, VLAN_N_VID) { + mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i); } - for_each_set_bit(i, priv->fs.vlan->active_svlans, VLAN_N_VID) - mlx5e_add_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i); + for_each_set_bit(i, fs->vlan->active_svlans, VLAN_N_VID) + mlx5e_add_vlan_rule(fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i); - if (priv->fs.vlan->cvlan_filter_disabled) - mlx5e_add_any_vid_rules(priv); + if (fs->vlan->cvlan_filter_disabled) + mlx5e_fs_add_any_vid_rules(fs); } static void mlx5e_del_vlan_rules(struct mlx5e_priv *priv) { int i; - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); + mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_UNTAGGED, 0); - for_each_set_bit(i, priv->fs.vlan->active_cvlans, VLAN_N_VID) { - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i); + for_each_set_bit(i, priv->fs->vlan->active_cvlans, VLAN_N_VID) { + mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_MATCH_CTAG_VID, i); } - for_each_set_bit(i, priv->fs.vlan->active_svlans, VLAN_N_VID) - mlx5e_del_vlan_rule(priv, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i); + for_each_set_bit(i, priv->fs->vlan->active_svlans, VLAN_N_VID) + mlx5e_fs_del_vlan_rule(priv->fs, MLX5E_VLAN_RULE_TYPE_MATCH_STAG_VID, i); - WARN_ON_ONCE(!(test_bit(MLX5E_STATE_DESTROYING, &priv->state))); + WARN_ON_ONCE(priv->fs->state_destroy); mlx5e_remove_vlan_trap(priv); /* must be called after DESTROY bit is set and * set_rx_mode is called and flushed */ - if (priv->fs.vlan->cvlan_filter_disabled) - mlx5e_del_any_vid_rules(priv); + if (priv->fs->vlan->cvlan_filter_disabled) + mlx5e_fs_del_any_vid_rules(priv->fs); } #define mlx5e_for_each_hash_node(hn, tmp, hash, i) \ for (i = 0; i < MLX5E_L2_ADDR_HASH_SIZE; i++) \ hlist_for_each_entry_safe(hn, tmp, &hash[i], hlist) -static void mlx5e_execute_l2_action(struct mlx5e_priv *priv, +static void mlx5e_execute_l2_action(struct mlx5e_flow_steering *fs, struct mlx5e_l2_hash_node *hn) { u8 action = hn->action; @@ -547,9 +551,9 @@ static void mlx5e_execute_l2_action(struct mlx5e_priv *priv, switch (action) { case MLX5E_ACTION_ADD: - mlx5e_add_l2_flow_rule(priv, &hn->ai, MLX5E_FULLMATCH); + mlx5e_add_l2_flow_rule(fs, &hn->ai, MLX5E_FULLMATCH); if (!is_multicast_ether_addr(mac_addr)) { - l2_err = mlx5_mpfs_add_mac(priv->mdev, mac_addr); + l2_err = mlx5_mpfs_add_mac(fs->mdev, mac_addr); hn->mpfs = !l2_err; } hn->action = MLX5E_ACTION_NONE; @@ -557,52 +561,50 @@ static void mlx5e_execute_l2_action(struct mlx5e_priv *priv, case MLX5E_ACTION_DEL: if (!is_multicast_ether_addr(mac_addr) && hn->mpfs) - l2_err = mlx5_mpfs_del_mac(priv->mdev, mac_addr); - mlx5e_del_l2_flow_rule(priv, &hn->ai); + l2_err = mlx5_mpfs_del_mac(fs->mdev, mac_addr); + mlx5e_del_l2_flow_rule(fs, &hn->ai); mlx5e_del_l2_from_hash(hn); break; } if (l2_err) - netdev_warn(priv->netdev, "MPFS, failed to %s mac %pM, err(%d)\n", - action == MLX5E_ACTION_ADD ? "add" : "del", mac_addr, l2_err); + mlx5_core_warn(fs->mdev, "MPFS, failed to %s mac %pM, err(%d)\n", + action == MLX5E_ACTION_ADD ? "add" : "del", mac_addr, l2_err); } -static void mlx5e_sync_netdev_addr(struct mlx5e_priv *priv) +static void mlx5e_sync_netdev_addr(struct mlx5e_flow_steering *fs, + struct net_device *netdev) { - struct net_device *netdev = priv->netdev; struct netdev_hw_addr *ha; netif_addr_lock_bh(netdev); - mlx5e_add_l2_to_hash(priv->fs.l2.netdev_uc, - priv->netdev->dev_addr); - + mlx5e_add_l2_to_hash(fs->l2.netdev_uc, netdev->dev_addr); netdev_for_each_uc_addr(ha, netdev) - mlx5e_add_l2_to_hash(priv->fs.l2.netdev_uc, ha->addr); + mlx5e_add_l2_to_hash(fs->l2.netdev_uc, ha->addr); netdev_for_each_mc_addr(ha, netdev) - mlx5e_add_l2_to_hash(priv->fs.l2.netdev_mc, ha->addr); + mlx5e_add_l2_to_hash(fs->l2.netdev_mc, ha->addr); netif_addr_unlock_bh(netdev); } -static void mlx5e_fill_addr_array(struct mlx5e_priv *priv, int list_type, +static void mlx5e_fill_addr_array(struct mlx5e_flow_steering *fs, int list_type, + struct net_device *ndev, u8 addr_array[][ETH_ALEN], int size) { bool is_uc = (list_type == MLX5_NVPRT_LIST_TYPE_UC); - struct net_device *ndev = priv->netdev; struct mlx5e_l2_hash_node *hn; struct hlist_head *addr_list; struct hlist_node *tmp; int i = 0; int hi; - addr_list = is_uc ? priv->fs.l2.netdev_uc : priv->fs.l2.netdev_mc; + addr_list = is_uc ? fs->l2.netdev_uc : fs->l2.netdev_mc; if (is_uc) /* Make sure our own address is pushed first */ ether_addr_copy(addr_array[i++], ndev->dev_addr); - else if (priv->fs.l2.broadcast_enabled) + else if (fs->l2.broadcast_enabled) ether_addr_copy(addr_array[i++], ndev->broadcast); mlx5e_for_each_hash_node(hn, tmp, addr_list, hi) { @@ -614,7 +616,8 @@ static void mlx5e_fill_addr_array(struct mlx5e_priv *priv, int list_type, } } -static void mlx5e_vport_context_update_addr_list(struct mlx5e_priv *priv, +static void mlx5e_vport_context_update_addr_list(struct mlx5e_flow_steering *fs, + struct net_device *netdev, int list_type) { bool is_uc = (list_type == MLX5_NVPRT_LIST_TYPE_UC); @@ -627,19 +630,19 @@ static void mlx5e_vport_context_update_addr_list(struct mlx5e_priv *priv, int err; int hi; - size = is_uc ? 0 : (priv->fs.l2.broadcast_enabled ? 1 : 0); + size = is_uc ? 0 : (fs->l2.broadcast_enabled ? 1 : 0); max_size = is_uc ? - 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_uc_list) : - 1 << MLX5_CAP_GEN(priv->mdev, log_max_current_mc_list); + 1 << MLX5_CAP_GEN(fs->mdev, log_max_current_uc_list) : + 1 << MLX5_CAP_GEN(fs->mdev, log_max_current_mc_list); - addr_list = is_uc ? priv->fs.l2.netdev_uc : priv->fs.l2.netdev_mc; + addr_list = is_uc ? fs->l2.netdev_uc : fs->l2.netdev_mc; mlx5e_for_each_hash_node(hn, tmp, addr_list, hi) size++; if (size > max_size) { - netdev_warn(priv->netdev, - "netdev %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n", - is_uc ? "UC" : "MC", size, max_size); + mlx5_core_warn(fs->mdev, + "mdev %s list size (%d) > (%d) max vport list size, some addresses will be dropped\n", + is_uc ? "UC" : "MC", size, max_size); size = max_size; } @@ -649,65 +652,67 @@ static void mlx5e_vport_context_update_addr_list(struct mlx5e_priv *priv, err = -ENOMEM; goto out; } - mlx5e_fill_addr_array(priv, list_type, addr_array, size); + mlx5e_fill_addr_array(fs, list_type, netdev, addr_array, size); } - err = mlx5_modify_nic_vport_mac_list(priv->mdev, list_type, addr_array, size); + err = mlx5_modify_nic_vport_mac_list(fs->mdev, list_type, addr_array, size); out: if (err) - netdev_err(priv->netdev, - "Failed to modify vport %s list err(%d)\n", - is_uc ? "UC" : "MC", err); + mlx5_core_err(fs->mdev, + "Failed to modify vport %s list err(%d)\n", + is_uc ? "UC" : "MC", err); kfree(addr_array); } -static void mlx5e_vport_context_update(struct mlx5e_priv *priv) +static void mlx5e_vport_context_update(struct mlx5e_flow_steering *fs, + struct net_device *netdev) { - struct mlx5e_l2_table *ea = &priv->fs.l2; + struct mlx5e_l2_table *ea = &fs->l2; - mlx5e_vport_context_update_addr_list(priv, MLX5_NVPRT_LIST_TYPE_UC); - mlx5e_vport_context_update_addr_list(priv, MLX5_NVPRT_LIST_TYPE_MC); - mlx5_modify_nic_vport_promisc(priv->mdev, 0, + mlx5e_vport_context_update_addr_list(fs, netdev, MLX5_NVPRT_LIST_TYPE_UC); + mlx5e_vport_context_update_addr_list(fs, netdev, MLX5_NVPRT_LIST_TYPE_MC); + mlx5_modify_nic_vport_promisc(fs->mdev, 0, ea->allmulti_enabled, ea->promisc_enabled); } -static void mlx5e_apply_netdev_addr(struct mlx5e_priv *priv) +static void mlx5e_apply_netdev_addr(struct mlx5e_flow_steering *fs) { struct mlx5e_l2_hash_node *hn; struct hlist_node *tmp; int i; - mlx5e_for_each_hash_node(hn, tmp, priv->fs.l2.netdev_uc, i) - mlx5e_execute_l2_action(priv, hn); + mlx5e_for_each_hash_node(hn, tmp, fs->l2.netdev_uc, i) + mlx5e_execute_l2_action(fs, hn); - mlx5e_for_each_hash_node(hn, tmp, priv->fs.l2.netdev_mc, i) - mlx5e_execute_l2_action(priv, hn); + mlx5e_for_each_hash_node(hn, tmp, fs->l2.netdev_mc, i) + mlx5e_execute_l2_action(fs, hn); } -static void mlx5e_handle_netdev_addr(struct mlx5e_priv *priv) +static void mlx5e_handle_netdev_addr(struct mlx5e_flow_steering *fs, + struct net_device *netdev) { struct mlx5e_l2_hash_node *hn; struct hlist_node *tmp; int i; - mlx5e_for_each_hash_node(hn, tmp, priv->fs.l2.netdev_uc, i) + mlx5e_for_each_hash_node(hn, tmp, fs->l2.netdev_uc, i) hn->action = MLX5E_ACTION_DEL; - mlx5e_for_each_hash_node(hn, tmp, priv->fs.l2.netdev_mc, i) + mlx5e_for_each_hash_node(hn, tmp, fs->l2.netdev_mc, i) hn->action = MLX5E_ACTION_DEL; - if (!test_bit(MLX5E_STATE_DESTROYING, &priv->state)) - mlx5e_sync_netdev_addr(priv); + if (fs->state_destroy) + mlx5e_sync_netdev_addr(fs, netdev); - mlx5e_apply_netdev_addr(priv); + mlx5e_apply_netdev_addr(fs); } #define MLX5E_PROMISC_GROUP0_SIZE BIT(0) #define MLX5E_PROMISC_TABLE_SIZE MLX5E_PROMISC_GROUP0_SIZE -static int mlx5e_add_promisc_rule(struct mlx5e_priv *priv) +static int mlx5e_add_promisc_rule(struct mlx5e_flow_steering *fs) { - struct mlx5_flow_table *ft = priv->fs.promisc.ft.t; + struct mlx5_flow_table *ft = fs->promisc.ft.t; struct mlx5_flow_destination dest = {}; struct mlx5_flow_handle **rule_p; MLX5_DECLARE_FLOW_ACT(flow_act); @@ -718,22 +723,22 @@ static int mlx5e_add_promisc_rule(struct mlx5e_priv *priv) if (!spec) return -ENOMEM; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = mlx5_get_ttc_flow_table(priv->fs.ttc); + dest.ft = mlx5_get_ttc_flow_table(fs->ttc); - rule_p = &priv->fs.promisc.rule; + rule_p = &fs->promisc.rule; *rule_p = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); if (IS_ERR(*rule_p)) { err = PTR_ERR(*rule_p); *rule_p = NULL; - netdev_err(priv->netdev, "%s: add promiscuous rule failed\n", __func__); + mlx5_core_err(fs->mdev, "%s: add promiscuous rule failed\n", __func__); } kvfree(spec); return err; } -static int mlx5e_create_promisc_table(struct mlx5e_priv *priv) +static int mlx5e_create_promisc_table(struct mlx5e_flow_steering *fs) { - struct mlx5e_flow_table *ft = &priv->fs.promisc.ft; + struct mlx5e_flow_table *ft = &fs->promisc.ft; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -742,14 +747,14 @@ static int mlx5e_create_promisc_table(struct mlx5e_priv *priv) ft_attr.level = MLX5E_PROMISC_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_auto_grouped_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_auto_grouped_flow_table(fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); - netdev_err(priv->netdev, "fail to create promisc table err=%d\n", err); + mlx5_core_err(fs->mdev, "fail to create promisc table err=%d\n", err); return err; } - err = mlx5e_add_promisc_rule(priv); + err = mlx5e_add_promisc_rule(fs); if (err) goto err_destroy_promisc_table; @@ -762,34 +767,31 @@ err_destroy_promisc_table: return err; } -static void mlx5e_del_promisc_rule(struct mlx5e_priv *priv) +static void mlx5e_del_promisc_rule(struct mlx5e_flow_steering *fs) { - if (WARN(!priv->fs.promisc.rule, "Trying to remove non-existing promiscuous rule")) + if (WARN(!fs->promisc.rule, "Trying to remove non-existing promiscuous rule")) return; - mlx5_del_flow_rules(priv->fs.promisc.rule); - priv->fs.promisc.rule = NULL; + mlx5_del_flow_rules(fs->promisc.rule); + fs->promisc.rule = NULL; } -static void mlx5e_destroy_promisc_table(struct mlx5e_priv *priv) +static void mlx5e_destroy_promisc_table(struct mlx5e_flow_steering *fs) { - if (WARN(!priv->fs.promisc.ft.t, "Trying to remove non-existing promiscuous table")) + if (WARN(!fs->promisc.ft.t, "Trying to remove non-existing promiscuous table")) return; - mlx5e_del_promisc_rule(priv); - mlx5_destroy_flow_table(priv->fs.promisc.ft.t); - priv->fs.promisc.ft.t = NULL; + mlx5e_del_promisc_rule(fs); + mlx5_destroy_flow_table(fs->promisc.ft.t); + fs->promisc.ft.t = NULL; } -void mlx5e_set_rx_mode_work(struct work_struct *work) +void mlx5e_fs_set_rx_mode_work(struct mlx5e_flow_steering *fs, + struct net_device *netdev) { - struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, - set_rx_mode_work); + struct mlx5e_l2_table *ea = &fs->l2; - struct mlx5e_l2_table *ea = &priv->fs.l2; - struct net_device *ndev = priv->netdev; - - bool rx_mode_enable = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); - bool promisc_enabled = rx_mode_enable && (ndev->flags & IFF_PROMISC); - bool allmulti_enabled = rx_mode_enable && (ndev->flags & IFF_ALLMULTI); + bool rx_mode_enable = fs->state_destroy; + bool promisc_enabled = rx_mode_enable && (netdev->flags & IFF_PROMISC); + bool allmulti_enabled = rx_mode_enable && (netdev->flags & IFF_ALLMULTI); bool broadcast_enabled = rx_mode_enable; bool enable_promisc = !ea->promisc_enabled && promisc_enabled; @@ -801,32 +803,32 @@ void mlx5e_set_rx_mode_work(struct work_struct *work) int err; if (enable_promisc) { - err = mlx5e_create_promisc_table(priv); + err = mlx5e_create_promisc_table(fs); if (err) enable_promisc = false; - if (!priv->channels.params.vlan_strip_disable && !err) - netdev_warn_once(ndev, - "S-tagged traffic will be dropped while C-tag vlan stripping is enabled\n"); + if (!fs->vlan_strip_disable && !err) + mlx5_core_warn_once(fs->mdev, + "S-tagged traffic will be dropped while C-tag vlan stripping is enabled\n"); } if (enable_allmulti) - mlx5e_add_l2_flow_rule(priv, &ea->allmulti, MLX5E_ALLMULTI); + mlx5e_add_l2_flow_rule(fs, &ea->allmulti, MLX5E_ALLMULTI); if (enable_broadcast) - mlx5e_add_l2_flow_rule(priv, &ea->broadcast, MLX5E_FULLMATCH); + mlx5e_add_l2_flow_rule(fs, &ea->broadcast, MLX5E_FULLMATCH); - mlx5e_handle_netdev_addr(priv); + mlx5e_handle_netdev_addr(fs, netdev); if (disable_broadcast) - mlx5e_del_l2_flow_rule(priv, &ea->broadcast); + mlx5e_del_l2_flow_rule(fs, &ea->broadcast); if (disable_allmulti) - mlx5e_del_l2_flow_rule(priv, &ea->allmulti); + mlx5e_del_l2_flow_rule(fs, &ea->allmulti); if (disable_promisc) - mlx5e_destroy_promisc_table(priv); + mlx5e_destroy_promisc_table(fs); ea->promisc_enabled = promisc_enabled; ea->allmulti_enabled = allmulti_enabled; ea->broadcast_enabled = broadcast_enabled; - mlx5e_vport_context_update(priv); + mlx5e_vport_context_update(fs, netdev); } static void mlx5e_destroy_groups(struct mlx5e_flow_table *ft) @@ -841,9 +843,9 @@ static void mlx5e_destroy_groups(struct mlx5e_flow_table *ft) ft->num_groups = 0; } -void mlx5e_init_l2_addr(struct mlx5e_priv *priv) +void mlx5e_fs_init_l2_addr(struct mlx5e_flow_steering *fs, struct net_device *netdev) { - ether_addr_copy(priv->fs.l2.broadcast.addr, priv->netdev->broadcast); + ether_addr_copy(fs->l2.broadcast.addr, netdev->broadcast); } void mlx5e_destroy_flow_table(struct mlx5e_flow_table *ft) @@ -861,7 +863,7 @@ static void mlx5e_set_inner_ttc_params(struct mlx5e_priv *priv, int tt; memset(ttc_params, 0, sizeof(*ttc_params)); - ttc_params->ns = mlx5_get_flow_namespace(priv->mdev, + ttc_params->ns = mlx5_get_flow_namespace(priv->fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL); ft_attr->level = MLX5E_INNER_TTC_FT_LEVEL; ft_attr->prio = MLX5E_NIC_PRIO; @@ -884,7 +886,7 @@ void mlx5e_set_ttc_params(struct mlx5e_priv *priv, int tt; memset(ttc_params, 0, sizeof(*ttc_params)); - ttc_params->ns = mlx5_get_flow_namespace(priv->mdev, + ttc_params->ns = mlx5_get_flow_namespace(priv->fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL); ft_attr->level = MLX5E_TTC_FT_LEVEL; ft_attr->prio = MLX5E_NIC_PRIO; @@ -898,18 +900,18 @@ void mlx5e_set_ttc_params(struct mlx5e_priv *priv, } ttc_params->inner_ttc = tunnel; - if (!tunnel || !mlx5_tunnel_inner_ft_supported(priv->mdev)) + if (!tunnel || !mlx5_tunnel_inner_ft_supported(priv->fs->mdev)) return; for (tt = 0; tt < MLX5_NUM_TUNNEL_TT; tt++) { ttc_params->tunnel_dests[tt].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; ttc_params->tunnel_dests[tt].ft = - mlx5_get_ttc_flow_table(priv->fs.inner_ttc); + mlx5_get_ttc_flow_table(priv->fs->inner_ttc); } } -static void mlx5e_del_l2_flow_rule(struct mlx5e_priv *priv, +static void mlx5e_del_l2_flow_rule(struct mlx5e_flow_steering *fs, struct mlx5e_l2_rule *ai) { if (!IS_ERR_OR_NULL(ai->rule)) { @@ -918,10 +920,10 @@ static void mlx5e_del_l2_flow_rule(struct mlx5e_priv *priv, } } -static int mlx5e_add_l2_flow_rule(struct mlx5e_priv *priv, +static int mlx5e_add_l2_flow_rule(struct mlx5e_flow_steering *fs, struct mlx5e_l2_rule *ai, int type) { - struct mlx5_flow_table *ft = priv->fs.l2.ft.t; + struct mlx5_flow_table *ft = fs->l2.ft.t; struct mlx5_flow_destination dest = {}; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_spec *spec; @@ -939,7 +941,7 @@ static int mlx5e_add_l2_flow_rule(struct mlx5e_priv *priv, outer_headers.dmac_47_16); dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = mlx5_get_ttc_flow_table(priv->fs.ttc); + dest.ft = mlx5_get_ttc_flow_table(fs->ttc); switch (type) { case MLX5E_FULLMATCH: @@ -957,8 +959,8 @@ static int mlx5e_add_l2_flow_rule(struct mlx5e_priv *priv, ai->rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); if (IS_ERR(ai->rule)) { - netdev_err(priv->netdev, "%s: add l2 rule(mac:%pM) failed\n", - __func__, mv_dmac); + mlx5_core_err(fs->mdev, "%s: add l2 rule(mac:%pM) failed\n", + __func__, mv_dmac); err = PTR_ERR(ai->rule); ai->rule = NULL; } @@ -1044,12 +1046,12 @@ err_destroy_groups: static void mlx5e_destroy_l2_table(struct mlx5e_priv *priv) { - mlx5e_destroy_flow_table(&priv->fs.l2.ft); + mlx5e_destroy_flow_table(&priv->fs->l2.ft); } static int mlx5e_create_l2_table(struct mlx5e_priv *priv) { - struct mlx5e_l2_table *l2_table = &priv->fs.l2; + struct mlx5e_l2_table *l2_table = &priv->fs->l2; struct mlx5e_flow_table *ft = &l2_table->ft; struct mlx5_flow_table_attr ft_attr = {}; int err; @@ -1060,7 +1062,7 @@ static int mlx5e_create_l2_table(struct mlx5e_priv *priv) ft_attr.level = MLX5E_L2_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_flow_table(priv->fs->ns, &ft_attr); if (IS_ERR(ft->t)) { err = PTR_ERR(ft->t); ft->t = NULL; @@ -1180,20 +1182,20 @@ static int mlx5e_create_vlan_table_groups(struct mlx5e_flow_table *ft) return err; } -static int mlx5e_create_vlan_table(struct mlx5e_priv *priv) +static int mlx5e_fs_create_vlan_table(struct mlx5e_flow_steering *fs) { struct mlx5_flow_table_attr ft_attr = {}; struct mlx5e_flow_table *ft; int err; - ft = &priv->fs.vlan->ft; + ft = &fs->vlan->ft; ft->num_groups = 0; ft_attr.max_fte = MLX5E_VLAN_TABLE_SIZE; ft_attr.level = MLX5E_VLAN_FT_LEVEL; ft_attr.prio = MLX5E_NIC_PRIO; - ft->t = mlx5_create_flow_table(priv->fs.ns, &ft_attr); + ft->t = mlx5_create_flow_table(fs->ns, &ft_attr); if (IS_ERR(ft->t)) return PTR_ERR(ft->t); @@ -1207,7 +1209,7 @@ static int mlx5e_create_vlan_table(struct mlx5e_priv *priv) if (err) goto err_free_g; - mlx5e_add_vlan_rules(priv); + mlx5e_fs_add_vlan_rules(fs); return 0; @@ -1222,33 +1224,33 @@ err_destroy_vlan_table: static void mlx5e_destroy_vlan_table(struct mlx5e_priv *priv) { mlx5e_del_vlan_rules(priv); - mlx5e_destroy_flow_table(&priv->fs.vlan->ft); + mlx5e_destroy_flow_table(&priv->fs->vlan->ft); } static void mlx5e_destroy_inner_ttc_table(struct mlx5e_priv *priv) { - if (!mlx5_tunnel_inner_ft_supported(priv->mdev)) + if (!mlx5_tunnel_inner_ft_supported(priv->fs->mdev)) return; - mlx5_destroy_ttc_table(priv->fs.inner_ttc); + mlx5_destroy_ttc_table(priv->fs->inner_ttc); } void mlx5e_destroy_ttc_table(struct mlx5e_priv *priv) { - mlx5_destroy_ttc_table(priv->fs.ttc); + mlx5_destroy_ttc_table(priv->fs->ttc); } static int mlx5e_create_inner_ttc_table(struct mlx5e_priv *priv) { struct ttc_params ttc_params = {}; - if (!mlx5_tunnel_inner_ft_supported(priv->mdev)) + if (!mlx5_tunnel_inner_ft_supported(priv->fs->mdev)) return 0; mlx5e_set_inner_ttc_params(priv, &ttc_params); - priv->fs.inner_ttc = mlx5_create_inner_ttc_table(priv->mdev, - &ttc_params); - if (IS_ERR(priv->fs.inner_ttc)) - return PTR_ERR(priv->fs.inner_ttc); + priv->fs->inner_ttc = mlx5_create_inner_ttc_table(priv->fs->mdev, + &ttc_params); + if (IS_ERR(priv->fs->inner_ttc)) + return PTR_ERR(priv->fs->inner_ttc); return 0; } @@ -1257,9 +1259,9 @@ int mlx5e_create_ttc_table(struct mlx5e_priv *priv) struct ttc_params ttc_params = {}; mlx5e_set_ttc_params(priv, &ttc_params, true); - priv->fs.ttc = mlx5_create_ttc_table(priv->mdev, &ttc_params); - if (IS_ERR(priv->fs.ttc)) - return PTR_ERR(priv->fs.ttc); + priv->fs->ttc = mlx5_create_ttc_table(priv->fs->mdev, &ttc_params); + if (IS_ERR(priv->fs->ttc)) + return PTR_ERR(priv->fs->ttc); return 0; } @@ -1267,45 +1269,44 @@ int mlx5e_create_flow_steering(struct mlx5e_priv *priv) { int err; - priv->fs.ns = mlx5_get_flow_namespace(priv->mdev, + priv->fs->ns = mlx5_get_flow_namespace(priv->fs->mdev, MLX5_FLOW_NAMESPACE_KERNEL); - if (!priv->fs.ns) + if (!priv->fs->ns) return -EOPNOTSUPP; err = mlx5e_arfs_create_tables(priv); if (err) { - netdev_err(priv->netdev, "Failed to create arfs tables, err=%d\n", - err); + mlx5_core_err(priv->fs->mdev, "Failed to create arfs tables, err=%d\n", + err); priv->netdev->hw_features &= ~NETIF_F_NTUPLE; } err = mlx5e_create_inner_ttc_table(priv); if (err) { - netdev_err(priv->netdev, - "Failed to create inner ttc table, err=%d\n", - err); + mlx5_core_err(priv->fs->mdev, + "Failed to create inner ttc table, err=%d\n", err); goto err_destroy_arfs_tables; } err = mlx5e_create_ttc_table(priv); if (err) { - netdev_err(priv->netdev, "Failed to create ttc table, err=%d\n", - err); + mlx5_core_err(priv->fs->mdev, "Failed to create ttc table, err=%d\n", + err); goto err_destroy_inner_ttc_table; } err = mlx5e_create_l2_table(priv); if (err) { - netdev_err(priv->netdev, "Failed to create l2 table, err=%d\n", - err); + mlx5_core_err(priv->fs->mdev, "Failed to create l2 table, err=%d\n", + err); goto err_destroy_ttc_table; } - err = mlx5e_create_vlan_table(priv); + err = mlx5e_fs_create_vlan_table(priv->fs); if (err) { - netdev_err(priv->netdev, "Failed to create vlan table, err=%d\n", - err); + mlx5_core_err(priv->fs->mdev, "Failed to create vlan table, err=%d\n", + err); goto err_destroy_l2_table; } @@ -1342,16 +1343,69 @@ void mlx5e_destroy_flow_steering(struct mlx5e_priv *priv) mlx5e_ethtool_cleanup_steering(priv); } -int mlx5e_fs_init(struct mlx5e_priv *priv) +static int mlx5e_fs_vlan_alloc(struct mlx5e_flow_steering *fs) +{ + fs->vlan = kvzalloc(sizeof(*fs->vlan), GFP_KERNEL); + if (!fs->vlan) + return -ENOMEM; + return 0; +} + +static void mlx5e_fs_vlan_free(struct mlx5e_flow_steering *fs) { - priv->fs.vlan = kvzalloc(sizeof(*priv->fs.vlan), GFP_KERNEL); - if (!priv->fs.vlan) + kvfree(fs->vlan); +} + +static int mlx5e_fs_tc_alloc(struct mlx5e_flow_steering *fs) +{ + fs->tc = mlx5e_tc_table_alloc(); + if (IS_ERR(fs->tc)) return -ENOMEM; return 0; } -void mlx5e_fs_cleanup(struct mlx5e_priv *priv) +static void mlx5e_fs_tc_free(struct mlx5e_flow_steering *fs) +{ + mlx5e_tc_table_free(fs->tc); +} + +struct mlx5e_flow_steering *mlx5e_fs_init(const struct mlx5e_profile *profile, + struct mlx5_core_dev *mdev, + bool state_destroy) +{ + struct mlx5e_flow_steering *fs; + int err; + + fs = kvzalloc(sizeof(*fs), GFP_KERNEL); + if (!fs) + goto err; + + fs->mdev = mdev; + fs->state_destroy = state_destroy; + if (mlx5e_profile_feature_cap(profile, FS_VLAN)) { + err = mlx5e_fs_vlan_alloc(fs); + if (err) + goto err_free_fs; + } + + if (mlx5e_profile_feature_cap(profile, FS_TC)) { + err = mlx5e_fs_tc_alloc(fs); + if (err) + goto err_free_vlan; + } + + return fs; +err_free_fs: + kvfree(fs); +err_free_vlan: + mlx5e_fs_vlan_free(fs); +err: + return NULL; +} + +void mlx5e_fs_cleanup(struct mlx5e_flow_steering *fs) { - kvfree(priv->fs.vlan); - priv->fs.vlan = NULL; + mlx5e_fs_tc_free(fs); + mlx5e_fs_vlan_free(fs); + kvfree(fs); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c index ad0d234632a3..3e4bc7836ef4 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_fs_ethtool.c @@ -81,18 +81,18 @@ static struct mlx5e_ethtool_table *get_flow_table(struct mlx5e_priv *priv, case UDP_V6_FLOW: max_tuples = ETHTOOL_NUM_L3_L4_FTS; prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples); - eth_ft = &priv->fs.ethtool.l3_l4_ft[prio]; + eth_ft = &priv->fs->ethtool.l3_l4_ft[prio]; break; case IP_USER_FLOW: case IPV6_USER_FLOW: max_tuples = ETHTOOL_NUM_L3_L4_FTS; prio = MLX5E_ETHTOOL_L3_L4_PRIO + (max_tuples - num_tuples); - eth_ft = &priv->fs.ethtool.l3_l4_ft[prio]; + eth_ft = &priv->fs->ethtool.l3_l4_ft[prio]; break; case ETHER_FLOW: max_tuples = ETHTOOL_NUM_L2_FTS; prio = max_tuples - num_tuples; - eth_ft = &priv->fs.ethtool.l2_ft[prio]; + eth_ft = &priv->fs->ethtool.l2_ft[prio]; prio += MLX5E_ETHTOOL_L2_PRIO; break; default: @@ -383,14 +383,14 @@ static void add_rule_to_list(struct mlx5e_priv *priv, struct mlx5e_ethtool_rule *rule) { struct mlx5e_ethtool_rule *iter; - struct list_head *head = &priv->fs.ethtool.rules; + struct list_head *head = &priv->fs->ethtool.rules; - list_for_each_entry(iter, &priv->fs.ethtool.rules, list) { + list_for_each_entry(iter, &priv->fs->ethtool.rules, list) { if (iter->flow_spec.location > rule->flow_spec.location) break; head = &iter->list; } - priv->fs.ethtool.tot_num_rules++; + priv->fs->ethtool.tot_num_rules++; list_add(&rule->list, head); } @@ -507,7 +507,7 @@ static void del_ethtool_rule(struct mlx5e_priv *priv, if (eth_rule->rss) mlx5e_rss_refcnt_dec(eth_rule->rss); list_del(ð_rule->list); - priv->fs.ethtool.tot_num_rules--; + priv->fs->ethtool.tot_num_rules--; put_flow_table(eth_rule->eth_ft); kfree(eth_rule); } @@ -517,7 +517,7 @@ static struct mlx5e_ethtool_rule *find_ethtool_rule(struct mlx5e_priv *priv, { struct mlx5e_ethtool_rule *iter; - list_for_each_entry(iter, &priv->fs.ethtool.rules, list) { + list_for_each_entry(iter, &priv->fs->ethtool.rules, list) { if (iter->flow_spec.location == location) return iter; } @@ -742,10 +742,7 @@ mlx5e_ethtool_flow_replace(struct mlx5e_priv *priv, eth_rule->flow_spec = *fs; eth_rule->eth_ft = eth_ft; - if (!eth_ft->ft) { - err = -EINVAL; - goto del_ethtool_rule; - } + rule = add_ethtool_flow_rule(priv, eth_rule, eth_ft->ft, fs, rss_context); if (IS_ERR(rule)) { err = PTR_ERR(rule); @@ -791,7 +788,7 @@ mlx5e_ethtool_get_flow(struct mlx5e_priv *priv, if (location < 0 || location >= MAX_NUM_OF_ETHTOOL_RULES) return -EINVAL; - list_for_each_entry(eth_rule, &priv->fs.ethtool.rules, list) { + list_for_each_entry(eth_rule, &priv->fs->ethtool.rules, list) { int index; if (eth_rule->flow_spec.location != location) @@ -834,13 +831,13 @@ void mlx5e_ethtool_cleanup_steering(struct mlx5e_priv *priv) struct mlx5e_ethtool_rule *iter; struct mlx5e_ethtool_rule *temp; - list_for_each_entry_safe(iter, temp, &priv->fs.ethtool.rules, list) + list_for_each_entry_safe(iter, temp, &priv->fs->ethtool.rules, list) del_ethtool_rule(priv, iter); } void mlx5e_ethtool_init_steering(struct mlx5e_priv *priv) { - INIT_LIST_HEAD(&priv->fs.ethtool.rules); + INIT_LIST_HEAD(&priv->fs->ethtool.rules); } static int flow_type_to_traffic_type(u32 flow_type) @@ -966,7 +963,7 @@ int mlx5e_ethtool_get_rxnfc(struct mlx5e_priv *priv, switch (info->cmd) { case ETHTOOL_GRXCLSRLCNT: - info->rule_cnt = priv->fs.ethtool.tot_num_rules; + info->rule_cnt = priv->fs->ethtool.tot_num_rules; break; case ETHTOOL_GRXCLSRULE: err = mlx5e_ethtool_get_flow(priv, info, info->fs.location); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c index 087952b84ccb..d858667736a3 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_main.c @@ -31,7 +31,6 @@ */ #include <net/tc_act/tc_gact.h> -#include <net/pkt_cls.h> #include <linux/mlx5/fs.h> #include <net/vxlan.h> #include <net/geneve.h> @@ -64,6 +63,7 @@ #include "en/devlink.h" #include "lib/mlx5.h" #include "en/ptp.h" +#include "en/htb.h" #include "qos.h" #include "en/trap.h" @@ -1912,8 +1912,7 @@ static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, { int tc; - if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL || - !params->mqprio.channel.rl) { + if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) { *hw_id = 0; return 0; } @@ -1922,7 +1921,14 @@ static int mlx5e_txq_get_qos_node_hw_id(struct mlx5e_params *params, int txq_ix, if (tc < 0) return tc; - return mlx5e_mqprio_rl_get_node_hw_id(params->mqprio.channel.rl, tc, hw_id); + if (tc >= params->mqprio.num_tc) { + WARN(1, "Unexpected TCs configuration. tc %d is out of range of %u", + tc, params->mqprio.num_tc); + return -EINVAL; + } + + *hw_id = params->mqprio.channel.hw_id[tc]; + return 0; } static int mlx5e_open_sqs(struct mlx5e_channel *c, @@ -2383,9 +2389,11 @@ int mlx5e_open_channels(struct mlx5e_priv *priv, goto err_close_channels; } - err = mlx5e_qos_open_queues(priv, chs); - if (err) - goto err_close_ptp; + if (priv->htb) { + err = mlx5e_qos_open_queues(priv, chs); + if (err) + goto err_close_ptp; + } mlx5e_health_channels_update(priv); kvfree(cparam); @@ -2567,9 +2575,11 @@ static int mlx5e_netdev_set_tcs(struct net_device *netdev, u16 nch, u8 ntc, int mlx5e_update_tx_netdev_queues(struct mlx5e_priv *priv) { - int qos_queues, nch, ntc, num_txqs, err; + int nch, ntc, num_txqs, err; + int qos_queues = 0; - qos_queues = mlx5e_qos_cur_leaf_nodes(priv); + if (priv->htb) + qos_queues = mlx5e_htb_cur_leaf_nodes(priv->htb); nch = priv->channels.params.num_channels; ntc = mlx5e_get_dcb_num_tc(&priv->channels.params); @@ -2615,13 +2625,6 @@ static int mlx5e_update_netdev_queues(struct mlx5e_priv *priv) netdev_warn(netdev, "netif_set_real_num_rx_queues failed, %d\n", err); goto err_txqs; } - if (priv->mqprio_rl != priv->channels.params.mqprio.channel.rl) { - if (priv->mqprio_rl) { - mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); - mlx5e_mqprio_rl_free(priv->mqprio_rl); - } - priv->mqprio_rl = priv->channels.params.mqprio.channel.rl; - } return 0; @@ -2724,7 +2727,8 @@ void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) { mlx5e_build_txq_maps(priv); mlx5e_activate_channels(&priv->channels); - mlx5e_qos_activate_queues(priv); + if (priv->htb) + mlx5e_qos_activate_queues(priv); mlx5e_xdp_tx_enable(priv); /* dev_watchdog() wants all TX queues to be started when the carrier is @@ -2841,7 +2845,7 @@ int mlx5e_safe_switch_params(struct mlx5e_priv *priv, new_chs.params = *params; - mlx5e_selq_prepare(&priv->selq, &new_chs.params, !!priv->htb.maj_id); + mlx5e_selq_prepare_params(&priv->selq, &new_chs.params); err = mlx5e_open_channels(priv, &new_chs); if (err) @@ -2897,7 +2901,7 @@ int mlx5e_open_locked(struct net_device *netdev) struct mlx5e_priv *priv = netdev_priv(netdev); int err; - mlx5e_selq_prepare(&priv->selq, &priv->channels.params, !!priv->htb.maj_id); + mlx5e_selq_prepare_params(&priv->selq, &priv->channels.params); set_bit(MLX5E_STATE_OPENED, &priv->state); @@ -3135,6 +3139,12 @@ err_close_tises: static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) { + if (priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + priv->mqprio_rl = NULL; + } + mlx5e_accel_cleanup_tx(priv); mlx5e_destroy_tises(priv); } @@ -3203,19 +3213,38 @@ static void mlx5e_params_mqprio_dcb_set(struct mlx5e_params *params, u8 num_tc) { params->mqprio.mode = TC_MQPRIO_MODE_DCB; params->mqprio.num_tc = num_tc; - params->mqprio.channel.rl = NULL; mlx5e_mqprio_build_default_tc_to_txq(params->mqprio.tc_to_txq, num_tc, params->num_channels); } +static void mlx5e_mqprio_rl_update_params(struct mlx5e_params *params, + struct mlx5e_mqprio_rl *rl) +{ + int tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) { + u32 hw_id = 0; + + if (rl) + mlx5e_mqprio_rl_get_node_hw_id(rl, tc, &hw_id); + params->mqprio.channel.hw_id[tc] = hw_id; + } +} + static void mlx5e_params_mqprio_channel_set(struct mlx5e_params *params, - struct tc_mqprio_qopt *qopt, + struct tc_mqprio_qopt_offload *mqprio, struct mlx5e_mqprio_rl *rl) { + int tc; + params->mqprio.mode = TC_MQPRIO_MODE_CHANNEL; - params->mqprio.num_tc = qopt->num_tc; - params->mqprio.channel.rl = rl; - mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, qopt); + params->mqprio.num_tc = mqprio->qopt.num_tc; + + for (tc = 0; tc < TC_MAX_QUEUE; tc++) + params->mqprio.channel.max_rate[tc] = mqprio->max_rate[tc]; + + mlx5e_mqprio_rl_update_params(params, rl); + mlx5e_mqprio_build_tc_to_txq(params->mqprio.tc_to_txq, &mqprio->qopt); } static void mlx5e_params_mqprio_reset(struct mlx5e_params *params) @@ -3241,6 +3270,12 @@ static int mlx5e_setup_tc_mqprio_dcb(struct mlx5e_priv *priv, err = mlx5e_safe_switch_params(priv, &new_params, mlx5e_num_channels_changed_ctx, NULL, true); + if (!err && priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + priv->mqprio_rl = NULL; + } + priv->max_opened_tc = max_t(u8, priv->max_opened_tc, mlx5e_get_dcb_num_tc(&priv->channels.params)); return err; @@ -3299,16 +3334,38 @@ static int mlx5e_mqprio_channel_validate(struct mlx5e_priv *priv, return 0; } -static bool mlx5e_mqprio_rate_limit(struct tc_mqprio_qopt_offload *mqprio) +static bool mlx5e_mqprio_rate_limit(u8 num_tc, u64 max_rate[]) { int tc; - for (tc = 0; tc < mqprio->qopt.num_tc; tc++) - if (mqprio->max_rate[tc]) + for (tc = 0; tc < num_tc; tc++) + if (max_rate[tc]) return true; return false; } +static struct mlx5e_mqprio_rl *mlx5e_mqprio_rl_create(struct mlx5_core_dev *mdev, + u8 num_tc, u64 max_rate[]) +{ + struct mlx5e_mqprio_rl *rl; + int err; + + if (!mlx5e_mqprio_rate_limit(num_tc, max_rate)) + return NULL; + + rl = mlx5e_mqprio_rl_alloc(); + if (!rl) + return ERR_PTR(-ENOMEM); + + err = mlx5e_mqprio_rl_init(rl, mdev, num_tc, max_rate); + if (err) { + mlx5e_mqprio_rl_free(rl); + return ERR_PTR(err); + } + + return rl; +} + static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, struct tc_mqprio_qopt_offload *mqprio) { @@ -3322,32 +3379,32 @@ static int mlx5e_setup_tc_mqprio_channel(struct mlx5e_priv *priv, if (err) return err; - rl = NULL; - if (mlx5e_mqprio_rate_limit(mqprio)) { - rl = mlx5e_mqprio_rl_alloc(); - if (!rl) - return -ENOMEM; - err = mlx5e_mqprio_rl_init(rl, priv->mdev, mqprio->qopt.num_tc, - mqprio->max_rate); - if (err) { - mlx5e_mqprio_rl_free(rl); - return err; - } - } + rl = mlx5e_mqprio_rl_create(priv->mdev, mqprio->qopt.num_tc, mqprio->max_rate); + if (IS_ERR(rl)) + return PTR_ERR(rl); new_params = priv->channels.params; - mlx5e_params_mqprio_channel_set(&new_params, &mqprio->qopt, rl); + mlx5e_params_mqprio_channel_set(&new_params, mqprio, rl); nch_changed = mlx5e_get_dcb_num_tc(&priv->channels.params) > 1; preactivate = nch_changed ? mlx5e_num_channels_changed_ctx : mlx5e_update_netdev_queues_ctx; err = mlx5e_safe_switch_params(priv, &new_params, preactivate, NULL, true); - if (err && rl) { - mlx5e_mqprio_rl_cleanup(rl); - mlx5e_mqprio_rl_free(rl); + if (err) { + if (rl) { + mlx5e_mqprio_rl_cleanup(rl); + mlx5e_mqprio_rl_free(rl); + } + return err; } - return err; + if (priv->mqprio_rl) { + mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); + mlx5e_mqprio_rl_free(priv->mqprio_rl); + } + priv->mqprio_rl = rl; + + return 0; } static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, @@ -3356,7 +3413,7 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, /* MQPRIO is another toplevel qdisc that can't be attached * simultaneously with the offloaded HTB. */ - if (WARN_ON(priv->htb.maj_id)) + if (WARN_ON(mlx5e_selq_is_htb_enabled(&priv->selq))) return -EINVAL; switch (mqprio->mode) { @@ -3369,47 +3426,6 @@ static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, } } -static int mlx5e_setup_tc_htb(struct mlx5e_priv *priv, struct tc_htb_qopt_offload *htb) -{ - int res; - - switch (htb->command) { - case TC_HTB_CREATE: - return mlx5e_htb_root_add(priv, htb->parent_classid, htb->classid, - htb->extack); - case TC_HTB_DESTROY: - return mlx5e_htb_root_del(priv); - case TC_HTB_LEAF_ALLOC_QUEUE: - res = mlx5e_htb_leaf_alloc_queue(priv, htb->classid, htb->parent_classid, - htb->rate, htb->ceil, htb->extack); - if (res < 0) - return res; - htb->qid = res; - return 0; - case TC_HTB_LEAF_TO_INNER: - return mlx5e_htb_leaf_to_inner(priv, htb->parent_classid, htb->classid, - htb->rate, htb->ceil, htb->extack); - case TC_HTB_LEAF_DEL: - return mlx5e_htb_leaf_del(priv, &htb->classid, htb->extack); - case TC_HTB_LEAF_DEL_LAST: - case TC_HTB_LEAF_DEL_LAST_FORCE: - return mlx5e_htb_leaf_del_last(priv, htb->classid, - htb->command == TC_HTB_LEAF_DEL_LAST_FORCE, - htb->extack); - case TC_HTB_NODE_MODIFY: - return mlx5e_htb_node_modify(priv, htb->classid, htb->rate, htb->ceil, - htb->extack); - case TC_HTB_LEAF_QUERY_QUEUE: - res = mlx5e_get_txq_by_classid(priv, htb->classid); - if (res < 0) - return res; - htb->qid = res; - return 0; - default: - return -EOPNOTSUPP; - } -} - static LIST_HEAD(mlx5e_block_cb_list); static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, @@ -3443,7 +3459,7 @@ static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, return err; case TC_SETUP_QDISC_HTB: mutex_lock(&priv->state_lock); - err = mlx5e_setup_tc_htb(priv, type_data); + err = mlx5e_htb_setup_tc(priv, type_data); mutex_unlock(&priv->state_lock); return err; default: @@ -3594,20 +3610,7 @@ static int set_feature_lro(struct net_device *netdev, bool enable) mutex_lock(&priv->state_lock); - if (enable && priv->xsk.refcnt) { - netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", - priv->xsk.refcnt); - err = -EINVAL; - goto out; - } - cur_params = &priv->channels.params; - if (enable && !MLX5E_GET_PFLAG(cur_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { - netdev_warn(netdev, "can't set LRO with legacy RQ\n"); - err = -EINVAL; - goto out; - } - new_params = *cur_params; if (enable) @@ -3676,6 +3679,7 @@ static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) static int set_feature_hw_tc(struct net_device *netdev, bool enable) { struct mlx5e_priv *priv = netdev_priv(netdev); + int err = 0; #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { @@ -3685,12 +3689,14 @@ static int set_feature_hw_tc(struct net_device *netdev, bool enable) } #endif - if (!enable && priv->htb.maj_id) { + mutex_lock(&priv->state_lock); + if (!enable && mlx5e_selq_is_htb_enabled(&priv->selq)) { netdev_err(netdev, "Active HTB offload, can't turn hw_tc_offload off\n"); - return -EINVAL; + err = -EINVAL; } + mutex_unlock(&priv->state_lock); - return 0; + return err; } static int set_feature_rx_all(struct net_device *netdev, bool enable) @@ -3772,20 +3778,45 @@ static int set_feature_rx_vlan(struct net_device *netdev, bool enable) mutex_lock(&priv->state_lock); + priv->fs->vlan_strip_disable = !enable; priv->channels.params.vlan_strip_disable = !enable; + if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) goto unlock; err = mlx5e_modify_channels_vsd(&priv->channels, !enable); - if (err) + if (err) { + priv->fs->vlan_strip_disable = enable; priv->channels.params.vlan_strip_disable = enable; - + } unlock: mutex_unlock(&priv->state_lock); return err; } +int mlx5e_vlan_rx_add_vid(struct net_device *dev, __be16 proto, u16 vid) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5e_flow_steering *fs = priv->fs; + + if (mlx5e_is_uplink_rep(priv)) + return 0; /* no vlan table for uplink rep */ + + return mlx5e_fs_vlan_rx_add_vid(fs, dev, proto, vid); +} + +int mlx5e_vlan_rx_kill_vid(struct net_device *dev, __be16 proto, u16 vid) +{ + struct mlx5e_priv *priv = netdev_priv(dev); + struct mlx5e_flow_steering *fs = priv->fs; + + if (mlx5e_is_uplink_rep(priv)) + return 0; /* no vlan table for uplink rep */ + + return mlx5e_fs_vlan_rx_kill_vid(fs, dev, proto, vid); +} + #ifdef CONFIG_MLX5_EN_ARFS static int set_feature_arfs(struct net_device *netdev, bool enable) { @@ -3883,8 +3914,8 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, mutex_lock(&priv->state_lock); params = &priv->channels.params; - if (!priv->fs.vlan || - !bitmap_empty(mlx5e_vlan_get_active_svlans(priv->fs.vlan), VLAN_N_VID)) { + if (!priv->fs->vlan || + !bitmap_empty(mlx5e_vlan_get_active_svlans(priv->fs->vlan), VLAN_N_VID)) { /* HW strips the outer C-tag header, this is a problem * for S-tag traffic. */ @@ -3916,6 +3947,11 @@ static netdev_features_t mlx5e_fix_features(struct net_device *netdev, } if (priv->xsk.refcnt) { + if (features & NETIF_F_LRO) { + netdev_warn(netdev, "LRO is incompatible with AF_XDP (%u XSKs are active)\n", + priv->xsk.refcnt); + features &= ~NETIF_F_LRO; + } if (features & NETIF_F_GRO_HW) { netdev_warn(netdev, "HW GRO is incompatible with AF_XDP (%u XSKs are active)\n", priv->xsk.refcnt); @@ -5002,6 +5038,7 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, struct net_device *netdev) { struct mlx5e_priv *priv = netdev_priv(netdev); + struct mlx5e_flow_steering *fs; int err; mlx5e_build_nic_params(priv, &priv->xsk, netdev->mtu); @@ -5009,11 +5046,14 @@ static int mlx5e_nic_init(struct mlx5_core_dev *mdev, mlx5e_timestamp_init(priv); - err = mlx5e_fs_init(priv); - if (err) { + fs = mlx5e_fs_init(priv->profile, mdev, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); + if (!fs) { + err = -ENOMEM; mlx5_core_err(mdev, "FS initialization failed, %d\n", err); return err; } + priv->fs = fs; err = mlx5e_ipsec_init(priv); if (err) @@ -5032,7 +5072,7 @@ static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) mlx5e_health_destroy_reporters(priv); mlx5e_ktls_cleanup(priv); mlx5e_ipsec_cleanup(priv); - mlx5e_fs_cleanup(priv); + mlx5e_fs_cleanup(priv->fs); } static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) @@ -5110,6 +5150,23 @@ static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) priv->rx_res = NULL; } +static void mlx5e_set_mqprio_rl(struct mlx5e_priv *priv) +{ + struct mlx5e_params *params; + struct mlx5e_mqprio_rl *rl; + + params = &priv->channels.params; + if (params->mqprio.mode != TC_MQPRIO_MODE_CHANNEL) + return; + + rl = mlx5e_mqprio_rl_create(priv->mdev, params->mqprio.num_tc, + params->mqprio.channel.max_rate); + if (IS_ERR(rl)) + rl = NULL; + priv->mqprio_rl = rl; + mlx5e_mqprio_rl_update_params(params, rl); +} + static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) { int err; @@ -5120,8 +5177,17 @@ static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) return err; } + err = mlx5e_accel_init_tx(priv); + if (err) + goto err_destroy_tises; + + mlx5e_set_mqprio_rl(priv); mlx5e_dcbnl_initialize(priv); return 0; + +err_destroy_tises: + mlx5e_destroy_tises(priv); + return err; } static void mlx5e_nic_enable(struct mlx5e_priv *priv) @@ -5129,7 +5195,7 @@ static void mlx5e_nic_enable(struct mlx5e_priv *priv) struct net_device *netdev = priv->netdev; struct mlx5_core_dev *mdev = priv->mdev; - mlx5e_init_l2_addr(priv); + mlx5e_fs_init_l2_addr(priv->fs, netdev); /* Marking the link as currently not needed by the Driver */ if (!netif_running(netdev)) @@ -5214,7 +5280,9 @@ static const struct mlx5e_profile mlx5e_nic_profile = { .stats_grps_num = mlx5e_nic_stats_grps_num, .features = BIT(MLX5E_PROFILE_FEATURE_PTP_RX) | BIT(MLX5E_PROFILE_FEATURE_PTP_TX) | - BIT(MLX5E_PROFILE_FEATURE_QOS_HTB), + BIT(MLX5E_PROFILE_FEATURE_QOS_HTB) | + BIT(MLX5E_PROFILE_FEATURE_FS_VLAN) | + BIT(MLX5E_PROFILE_FEATURE_FS_TC), }; static int mlx5e_profile_max_num_channels(struct mlx5_core_dev *mdev, @@ -5264,6 +5332,14 @@ int mlx5e_get_pf_num_tirs(struct mlx5_core_dev *mdev) + mlx5e_profile_max_num_channels(mdev, &mlx5e_nic_profile); } +void mlx5e_set_rx_mode_work(struct work_struct *work) +{ + struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, + set_rx_mode_work); + + return mlx5e_fs_set_rx_mode_work(priv->fs, priv->netdev); +} + /* mlx5e generic netdev management API (move to en_common.c) */ int mlx5e_priv_init(struct mlx5e_priv *priv, const struct mlx5e_profile *profile, @@ -5293,7 +5369,6 @@ int mlx5e_priv_init(struct mlx5e_priv *priv, if (err) goto err_free_cpumask; - hash_init(priv->htb.qos_tc2node); INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); @@ -5350,14 +5425,9 @@ void mlx5e_priv_cleanup(struct mlx5e_priv *priv) mutex_unlock(&priv->state_lock); free_cpumask_var(priv->scratchpad.cpumask); - for (i = 0; i < priv->htb.max_qos_sqs; i++) - kfree(priv->htb.qos_sq_stats[i]); - kvfree(priv->htb.qos_sq_stats); - - if (priv->mqprio_rl) { - mlx5e_mqprio_rl_cleanup(priv->mqprio_rl); - mlx5e_mqprio_rl_free(priv->mqprio_rl); - } + for (i = 0; i < priv->htb_max_qos_sqs; i++) + kfree(priv->htb_qos_sq_stats[i]); + kvfree(priv->htb_qos_sq_stats); memset(priv, 0, sizeof(*priv)); } @@ -5447,6 +5517,8 @@ int mlx5e_attach_netdev(struct mlx5e_priv *priv) int err; clear_bit(MLX5E_STATE_DESTROYING, &priv->state); + if (priv->fs) + priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); /* max number of channels may have changed */ max_nch = mlx5e_calc_max_nch(priv->mdev, priv->netdev, profile); @@ -5506,6 +5578,8 @@ err_cleanup_tx: out: mlx5e_reset_channels(priv->netdev); set_bit(MLX5E_STATE_DESTROYING, &priv->state); + if (priv->fs) + priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); cancel_work_sync(&priv->update_stats_work); return err; } @@ -5515,6 +5589,8 @@ void mlx5e_detach_netdev(struct mlx5e_priv *priv) const struct mlx5e_profile *profile = priv->profile; set_bit(MLX5E_STATE_DESTROYING, &priv->state); + if (priv->fs) + priv->fs->state_destroy = !test_bit(MLX5E_STATE_DESTROYING, &priv->state); if (profile->disable) profile->disable(priv); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c index f797fd97d305..4c1599de652c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.c @@ -229,7 +229,7 @@ mlx5e_rep_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = netdev_priv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } static int @@ -718,6 +718,7 @@ static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, static void mlx5e_cleanup_rep(struct mlx5e_priv *priv) { + mlx5e_fs_cleanup(priv->fs); mlx5e_ipsec_cleanup(priv); } @@ -728,8 +729,8 @@ static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) struct ttc_params ttc_params = {}; int err; - priv->fs.ns = mlx5_get_flow_namespace(priv->mdev, - MLX5_FLOW_NAMESPACE_KERNEL); + priv->fs->ns = mlx5_get_flow_namespace(priv->mdev, + MLX5_FLOW_NAMESPACE_KERNEL); /* The inner_ttc in the ttc params is intentionally not set */ mlx5e_set_ttc_params(priv, &ttc_params, false); @@ -738,9 +739,9 @@ static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) /* To give uplik rep TTC a lower level for chaining from root ft */ ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1; - priv->fs.ttc = mlx5_create_ttc_table(priv->mdev, &ttc_params); - if (IS_ERR(priv->fs.ttc)) { - err = PTR_ERR(priv->fs.ttc); + priv->fs->ttc = mlx5_create_ttc_table(priv->mdev, &ttc_params); + if (IS_ERR(priv->fs->ttc)) { + err = PTR_ERR(priv->fs->ttc); netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n", err); return err; @@ -760,7 +761,7 @@ static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv) /* non uplik reps will skip any bypass tables and go directly to * their own ttc */ - rpriv->root_ft = mlx5_get_ttc_flow_table(priv->fs.ttc); + rpriv->root_ft = mlx5_get_ttc_flow_table(priv->fs->ttc); return 0; } @@ -835,11 +836,20 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) struct mlx5_core_dev *mdev = priv->mdev; int err; - priv->rx_res = mlx5e_rx_res_alloc(); - if (!priv->rx_res) + priv->fs = mlx5e_fs_init(priv->profile, mdev, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); + if (!priv->fs) { + netdev_err(priv->netdev, "FS allocation failed\n"); return -ENOMEM; + } + + priv->rx_res = mlx5e_rx_res_alloc(); + if (!priv->rx_res) { + err = -ENOMEM; + goto err_free_fs; + } - mlx5e_init_l2_addr(priv); + mlx5e_fs_init_l2_addr(priv->fs, priv->netdev); err = mlx5e_open_drop_rq(priv, &priv->drop_rq); if (err) { @@ -873,13 +883,15 @@ static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) err_destroy_root_ft: mlx5e_destroy_rep_root_ft(priv); err_destroy_ttc_table: - mlx5_destroy_ttc_table(priv->fs.ttc); + mlx5_destroy_ttc_table(priv->fs->ttc); err_destroy_rx_res: mlx5e_rx_res_destroy(priv->rx_res); err_close_drop_rq: mlx5e_close_drop_rq(&priv->drop_rq); mlx5e_rx_res_free(priv->rx_res); priv->rx_res = NULL; +err_free_fs: + mlx5e_fs_cleanup(priv->fs); return err; } @@ -888,7 +900,7 @@ static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) mlx5e_ethtool_cleanup_steering(priv); rep_vport_rx_rule_destroy(priv); mlx5e_destroy_rep_root_ft(priv); - mlx5_destroy_ttc_table(priv->fs.ttc); + mlx5_destroy_ttc_table(priv->fs->ttc); mlx5e_rx_res_destroy(priv->rx_res); mlx5e_close_drop_rq(&priv->drop_rq); mlx5e_rx_res_free(priv->rx_res); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h index adf5cc6a7b8c..dec183ccd4ac 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_rep.h @@ -62,6 +62,7 @@ struct mlx5_tc_int_port_priv; struct mlx5e_rep_bond; struct mlx5e_tc_tun_encap; struct mlx5e_post_act; +struct mlx5e_flow_meters; struct mlx5_rep_uplink_priv { /* indirect block callbacks are invoked on bind/unbind events @@ -97,6 +98,8 @@ struct mlx5_rep_uplink_priv { /* OVS internal port support */ struct mlx5e_tc_int_port_priv *int_port_priv; + + struct mlx5e_flow_meters *flow_meters; }; struct mlx5e_rep_priv { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c index 1e87bb2b7541..7409829d1201 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.c @@ -474,8 +474,8 @@ static void mlx5e_stats_grp_sw_update_stats_qos(struct mlx5e_priv *priv, int i; /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); - stats = READ_ONCE(priv->htb.qos_sq_stats); + max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); + stats = READ_ONCE(priv->htb_qos_sq_stats); for (i = 0; i < max_qos_sqs; i++) { mlx5e_stats_grp_sw_update_stats_sq(s, READ_ONCE(stats[i])); @@ -2100,6 +2100,8 @@ static const struct counter_desc ptp_cq_stats_desc[] = { { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, err_cqe) }, { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort) }, { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, abort_abs_diff_ns) }, + { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, resync_cqe) }, + { MLX5E_DECLARE_PTP_CQ_STAT(struct mlx5e_ptp_cq_stats, resync_event) }, }; static const struct counter_desc ptp_rq_stats_desc[] = { @@ -2184,13 +2186,13 @@ static const struct counter_desc qos_sq_stats_desc[] = { static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(qos) { /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb.max_qos_sqs); + return NUM_QOS_SQ_STATS * smp_load_acquire(&priv->htb_max_qos_sqs); } static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(qos) { /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - u16 max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); + u16 max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); int i, qid; for (qid = 0; qid < max_qos_sqs; qid++) @@ -2208,8 +2210,8 @@ static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(qos) int i, qid; /* Pairs with smp_store_release in mlx5e_open_qos_sq. */ - max_qos_sqs = smp_load_acquire(&priv->htb.max_qos_sqs); - stats = READ_ONCE(priv->htb.qos_sq_stats); + max_qos_sqs = smp_load_acquire(&priv->htb_max_qos_sqs); + stats = READ_ONCE(priv->htb_qos_sq_stats); for (qid = 0; qid < max_qos_sqs; qid++) { struct mlx5e_sq_stats *s = READ_ONCE(stats[qid]); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h index e48b15b55b6f..ed4fc940e4ef 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_stats.h @@ -453,6 +453,8 @@ struct mlx5e_ptp_cq_stats { u64 err_cqe; u64 abort; u64 abort_abs_diff_ns; + u64 resync_cqe; + u64 resync_event; }; struct mlx5e_stats { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c index 9ca2c8763237..f154bda668ad 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.c @@ -59,6 +59,7 @@ #include "en/tc_tun_encap.h" #include "en/tc/sample.h" #include "en/tc/act/act.h" +#include "en/tc/post_meter.h" #include "lib/devcom.h" #include "lib/geneve.h" #include "lib/fs_chains.h" @@ -70,6 +71,30 @@ #define MLX5E_TC_TABLE_NUM_GROUPS 4 #define MLX5E_TC_TABLE_MAX_GROUP_SIZE BIT(18) +struct mlx5e_tc_table { + /* Protects the dynamic assignment of the t parameter + * which is the nic tc root table. + */ + struct mutex t_lock; + struct mlx5e_priv *priv; + struct mlx5_flow_table *t; + struct mlx5_flow_table *miss_t; + struct mlx5_fs_chains *chains; + struct mlx5e_post_act *post_act; + + struct rhashtable ht; + + struct mod_hdr_tbl mod_hdr; + struct mutex hairpin_tbl_lock; /* protects hairpin_tbl */ + DECLARE_HASHTABLE(hairpin_tbl, 8); + + struct notifier_block netdevice_nb; + struct netdev_net_notifier netdevice_nn; + + struct mlx5_tc_ct_priv *ct; + struct mapping_ctx *mapping; +}; + struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { [CHAIN_TO_REG] = { .mfield = MLX5_ACTION_IN_FIELD_METADATA_REG_C_0, @@ -104,8 +129,27 @@ struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[] = { .mlen = 16, }, [NIC_ZONE_RESTORE_TO_REG] = nic_zone_restore_to_reg_ct, + [PACKET_COLOR_TO_REG] = packet_color_to_reg, }; +struct mlx5e_tc_table *mlx5e_tc_table_alloc(void) +{ + struct mlx5e_tc_table *tc; + + tc = kvzalloc(sizeof(*tc), GFP_KERNEL); + return tc ? tc : ERR_PTR(-ENOMEM); +} + +void mlx5e_tc_table_free(struct mlx5e_tc_table *tc) +{ + kvfree(tc); +} + +struct mlx5_fs_chains *mlx5e_nic_chains(struct mlx5e_tc_table *tc) +{ + return tc->chains; +} + /* To avoid false lock dependency warning set the tc_ht lock * class different than the lock class of the ht being used when deleting * last flow from a group and then deleting a group, we get into del_sw_flow_group() @@ -240,6 +284,30 @@ mlx5e_get_int_port_priv(struct mlx5e_priv *priv) return NULL; } +struct mlx5e_flow_meters * +mlx5e_get_flow_meters(struct mlx5_core_dev *dev) +{ + struct mlx5_eswitch *esw = dev->priv.eswitch; + struct mlx5_rep_uplink_priv *uplink_priv; + struct mlx5e_rep_priv *uplink_rpriv; + struct mlx5e_priv *priv; + + if (is_mdev_switchdev_mode(dev)) { + uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); + uplink_priv = &uplink_rpriv->uplink_priv; + priv = netdev_priv(uplink_rpriv->netdev); + if (!uplink_priv->flow_meters) + uplink_priv->flow_meters = + mlx5e_flow_meters_init(priv, + MLX5_FLOW_NAMESPACE_FDB, + uplink_priv->post_act); + if (!IS_ERR(uplink_priv->flow_meters)) + return uplink_priv->flow_meters; + } + + return NULL; +} + static struct mlx5_tc_ct_priv * get_ct_priv(struct mlx5e_priv *priv) { @@ -254,7 +322,7 @@ get_ct_priv(struct mlx5e_priv *priv) return uplink_priv->ct_priv; } - return priv->fs.tc.ct; + return priv->fs->tc->ct; } static struct mlx5e_tc_psample * @@ -288,7 +356,7 @@ get_post_action(struct mlx5e_priv *priv) return uplink_priv->post_act; } - return priv->fs.tc.post_act; + return priv->fs->tc->post_act; } struct mlx5_flow_handle * @@ -319,12 +387,62 @@ mlx5_tc_rule_delete(struct mlx5e_priv *priv, mlx5e_del_offloaded_nic_rule(priv, rule, attr); } +static bool +is_flow_meter_action(struct mlx5_flow_attr *attr) +{ + return ((attr->action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && + (attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER)); +} + +static int +mlx5e_tc_add_flow_meter(struct mlx5e_priv *priv, + struct mlx5_flow_attr *attr) +{ + struct mlx5e_post_act *post_act = get_post_action(priv); + struct mlx5e_post_meter_priv *post_meter; + enum mlx5_flow_namespace_type ns_type; + struct mlx5e_flow_meter_handle *meter; + + meter = mlx5e_tc_meter_replace(priv->mdev, &attr->meter_attr.params); + if (IS_ERR(meter)) { + mlx5_core_err(priv->mdev, "Failed to get flow meter\n"); + return PTR_ERR(meter); + } + + ns_type = mlx5e_tc_meter_get_namespace(meter->flow_meters); + post_meter = mlx5e_post_meter_init(priv, ns_type, post_act, meter->green_counter, + meter->red_counter); + if (IS_ERR(post_meter)) { + mlx5_core_err(priv->mdev, "Failed to init post meter\n"); + goto err_meter_init; + } + + attr->meter_attr.meter = meter; + attr->meter_attr.post_meter = post_meter; + attr->dest_ft = mlx5e_post_meter_get_ft(post_meter); + attr->action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + + return 0; + +err_meter_init: + mlx5e_tc_meter_put(meter); + return PTR_ERR(post_meter); +} + +static void +mlx5e_tc_del_flow_meter(struct mlx5_flow_attr *attr) +{ + mlx5e_post_meter_cleanup(attr->meter_attr.post_meter); + mlx5e_tc_meter_put(attr->meter_attr.meter); +} + struct mlx5_flow_handle * mlx5e_tc_rule_offload(struct mlx5e_priv *priv, struct mlx5_flow_spec *spec, struct mlx5_flow_attr *attr) { struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; + int err; if (attr->flags & MLX5_ATTR_FLAG_CT) { struct mlx5e_tc_mod_hdr_acts *mod_hdr_acts = @@ -341,6 +459,12 @@ mlx5e_tc_rule_offload(struct mlx5e_priv *priv, if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) return mlx5e_tc_sample_offload(get_sample_priv(priv), spec, attr); + if (is_flow_meter_action(attr)) { + err = mlx5e_tc_add_flow_meter(priv, attr); + if (err) + return ERR_PTR(err); + } + return mlx5_eswitch_add_offloaded_rule(esw, spec, attr); } @@ -367,6 +491,9 @@ mlx5e_tc_rule_unoffload(struct mlx5e_priv *priv, } mlx5_eswitch_del_offloaded_rule(esw, rule, attr); + + if (attr->meter_attr.meter) + mlx5e_tc_del_flow_meter(attr); } int @@ -484,7 +611,7 @@ get_mod_hdr_table(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) return mlx5e_get_flow_namespace(flow) == MLX5_FLOW_NAMESPACE_FDB ? &esw->offloads.mod_hdr : - &priv->fs.tc.mod_hdr; + &priv->fs->tc->mod_hdr; } static int mlx5e_attach_mod_hdr(struct mlx5e_priv *priv, @@ -702,7 +829,7 @@ static int mlx5e_hairpin_rss_init(struct mlx5e_hairpin *hp) netdev_dbg(priv->netdev, "add hairpin: using %d channels rss ttc table id %x\n", hp->num_channels, - mlx5_get_ttc_flow_table(priv->fs.ttc)->id); + mlx5_get_ttc_flow_table(priv->fs->ttc)->id); return 0; @@ -792,7 +919,7 @@ static struct mlx5e_hairpin_entry *mlx5e_hairpin_get(struct mlx5e_priv *priv, struct mlx5e_hairpin_entry *hpe; u32 hash_key = hash_hairpin_info(peer_vhca_id, prio); - hash_for_each_possible(priv->fs.tc.hairpin_tbl, hpe, + hash_for_each_possible(priv->fs->tc->hairpin_tbl, hpe, hairpin_hlist, hash_key) { if (hpe->peer_vhca_id == peer_vhca_id && hpe->prio == prio) { refcount_inc(&hpe->refcnt); @@ -807,10 +934,10 @@ static void mlx5e_hairpin_put(struct mlx5e_priv *priv, struct mlx5e_hairpin_entry *hpe) { /* no more hairpin flows for us, release the hairpin pair */ - if (!refcount_dec_and_mutex_lock(&hpe->refcnt, &priv->fs.tc.hairpin_tbl_lock)) + if (!refcount_dec_and_mutex_lock(&hpe->refcnt, &priv->fs->tc->hairpin_tbl_lock)) return; hash_del(&hpe->hairpin_hlist); - mutex_unlock(&priv->fs.tc.hairpin_tbl_lock); + mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); if (!IS_ERR_OR_NULL(hpe->hp)) { netdev_dbg(priv->netdev, "del hairpin: peer %s\n", @@ -894,10 +1021,10 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, if (err) return err; - mutex_lock(&priv->fs.tc.hairpin_tbl_lock); + mutex_lock(&priv->fs->tc->hairpin_tbl_lock); hpe = mlx5e_hairpin_get(priv, peer_id, match_prio); if (hpe) { - mutex_unlock(&priv->fs.tc.hairpin_tbl_lock); + mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); wait_for_completion(&hpe->res_ready); if (IS_ERR(hpe->hp)) { @@ -909,7 +1036,7 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, hpe = kzalloc(sizeof(*hpe), GFP_KERNEL); if (!hpe) { - mutex_unlock(&priv->fs.tc.hairpin_tbl_lock); + mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); return -ENOMEM; } @@ -921,9 +1048,9 @@ static int mlx5e_hairpin_flow_add(struct mlx5e_priv *priv, refcount_set(&hpe->refcnt, 1); init_completion(&hpe->res_ready); - hash_add(priv->fs.tc.hairpin_tbl, &hpe->hairpin_hlist, + hash_add(priv->fs->tc->hairpin_tbl, &hpe->hairpin_hlist, hash_hairpin_info(peer_id, match_prio)); - mutex_unlock(&priv->fs.tc.hairpin_tbl_lock); + mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); params.log_data_size = 16; params.log_data_size = min_t(u8, params.log_data_size, @@ -999,10 +1126,10 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, struct mlx5_flow_attr *attr) { struct mlx5_flow_context *flow_context = &spec->flow_context; - struct mlx5_fs_chains *nic_chains = mlx5e_nic_chains(priv); struct mlx5_nic_flow_attr *nic_attr = attr->nic_attr; - struct mlx5e_tc_table *tc = &priv->fs.tc; + struct mlx5e_tc_table *tc = priv->fs->tc; struct mlx5_flow_destination dest[2] = {}; + struct mlx5_fs_chains *nic_chains; struct mlx5_flow_act flow_act = { .action = attr->action, .flags = FLOW_ACT_NO_APPEND, @@ -1011,6 +1138,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, struct mlx5_flow_table *ft; int dest_ix = 0; + nic_chains = mlx5e_nic_chains(tc); flow_context->flags |= FLOW_CONTEXT_HAS_TAG; flow_context->flow_tag = nic_attr->flow_tag; @@ -1035,7 +1163,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, if (IS_ERR(dest[dest_ix].ft)) return ERR_CAST(dest[dest_ix].ft); } else { - dest[dest_ix].ft = mlx5e_vlan_get_flowtable(priv->fs.vlan); + dest[dest_ix].ft = mlx5e_vlan_get_flowtable(priv->fs->vlan); } dest_ix++; } @@ -1063,7 +1191,7 @@ mlx5e_add_offloaded_nic_rule(struct mlx5e_priv *priv, mutex_unlock(&tc->t_lock); netdev_err(priv->netdev, "Failed to create tc offload table\n"); - rule = ERR_CAST(priv->fs.tc.t); + rule = ERR_CAST(priv->fs->tc->t); goto err_ft_get; } } @@ -1165,7 +1293,7 @@ void mlx5e_del_offloaded_nic_rule(struct mlx5e_priv *priv, struct mlx5_flow_handle *rule, struct mlx5_flow_attr *attr) { - struct mlx5_fs_chains *nic_chains = mlx5e_nic_chains(priv); + struct mlx5_fs_chains *nic_chains = mlx5e_nic_chains(priv->fs->tc); mlx5_del_flow_rules(rule); @@ -1182,7 +1310,7 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv, struct mlx5e_tc_flow *flow) { struct mlx5_flow_attr *attr = flow->attr; - struct mlx5e_tc_table *tc = &priv->fs.tc; + struct mlx5e_tc_table *tc = priv->fs->tc; flow_flag_clear(flow, OFFLOADED); @@ -1194,13 +1322,13 @@ static void mlx5e_tc_del_nic_flow(struct mlx5e_priv *priv, /* Remove root table if no rules are left to avoid * extra steering hops. */ - mutex_lock(&priv->fs.tc.t_lock); + mutex_lock(&priv->fs->tc->t_lock); if (!mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD)) && !IS_ERR_OR_NULL(tc->t)) { - mlx5_chains_put_table(mlx5e_nic_chains(priv), 0, 1, MLX5E_TC_FT_LEVEL); - priv->fs.tc.t = NULL; + mlx5_chains_put_table(mlx5e_nic_chains(tc), 0, 1, MLX5E_TC_FT_LEVEL); + priv->fs->tc->t = NULL; } - mutex_unlock(&priv->fs.tc.t_lock); + mutex_unlock(&priv->fs->tc->t_lock); if (attr->action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) mlx5e_detach_mod_hdr(priv, flow); @@ -3936,7 +4064,7 @@ static struct rhashtable *get_tc_ht(struct mlx5e_priv *priv, rpriv = priv->ppriv; return &rpriv->tc_ht; } else /* NIC offload */ - return &priv->fs.tc.ht; + return &priv->fs->tc->ht; } static bool is_peer_flow_needed(struct mlx5e_tc_flow *flow) @@ -4519,9 +4647,9 @@ static int apply_police_params(struct mlx5e_priv *priv, u64 rate, return err; } -static int mlx5e_policer_validate(const struct flow_action *action, - const struct flow_action_entry *act, - struct netlink_ext_ack *extack) +int mlx5e_policer_validate(const struct flow_action *action, + const struct flow_action_entry *act, + struct netlink_ext_ack *extack) { if (act->police.exceed.act_id != FLOW_ACTION_DROP) { NL_SET_ERR_MSG_MOD(extack, @@ -4655,11 +4783,11 @@ static void mlx5e_tc_hairpin_update_dead_peer(struct mlx5e_priv *priv, peer_vhca_id = MLX5_CAP_GEN(peer_mdev, vhca_id); - mutex_lock(&priv->fs.tc.hairpin_tbl_lock); - hash_for_each(priv->fs.tc.hairpin_tbl, bkt, hpe, hairpin_hlist) + mutex_lock(&priv->fs->tc->hairpin_tbl_lock); + hash_for_each(priv->fs->tc->hairpin_tbl, bkt, hpe, hairpin_hlist) if (refcount_inc_not_zero(&hpe->refcnt)) list_add(&hpe->dead_peer_wait_list, &init_wait_list); - mutex_unlock(&priv->fs.tc.hairpin_tbl_lock); + mutex_unlock(&priv->fs->tc->hairpin_tbl_lock); list_for_each_entry_safe(hpe, tmp, &init_wait_list, dead_peer_wait_list) { wait_for_completion(&hpe->res_ready); @@ -4674,7 +4802,6 @@ static int mlx5e_tc_netdev_event(struct notifier_block *this, unsigned long event, void *ptr) { struct net_device *ndev = netdev_notifier_info_to_dev(ptr); - struct mlx5e_flow_steering *fs; struct mlx5e_priv *peer_priv; struct mlx5e_tc_table *tc; struct mlx5e_priv *priv; @@ -4685,8 +4812,7 @@ static int mlx5e_tc_netdev_event(struct notifier_block *this, return NOTIFY_DONE; tc = container_of(this, struct mlx5e_tc_table, netdevice_nb); - fs = container_of(tc, struct mlx5e_flow_steering, tc); - priv = container_of(fs, struct mlx5e_priv, fs); + priv = tc->priv; peer_priv = netdev_priv(ndev); if (priv == peer_priv || !(priv->netdev->features & NETIF_F_HW_TC)) @@ -4715,7 +4841,7 @@ static int mlx5e_tc_nic_get_ft_size(struct mlx5_core_dev *dev) static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv) { - struct mlx5_flow_table **ft = &priv->fs.tc.miss_t; + struct mlx5_flow_table **ft = &priv->fs->tc->miss_t; struct mlx5_flow_table_attr ft_attr = {}; struct mlx5_flow_namespace *ns; int err = 0; @@ -4737,12 +4863,12 @@ static int mlx5e_tc_nic_create_miss_table(struct mlx5e_priv *priv) static void mlx5e_tc_nic_destroy_miss_table(struct mlx5e_priv *priv) { - mlx5_destroy_flow_table(priv->fs.tc.miss_t); + mlx5_destroy_flow_table(priv->fs->tc->miss_t); } int mlx5e_tc_nic_init(struct mlx5e_priv *priv) { - struct mlx5e_tc_table *tc = &priv->fs.tc; + struct mlx5e_tc_table *tc = priv->fs->tc; struct mlx5_core_dev *dev = priv->mdev; struct mapping_ctx *chains_mapping; struct mlx5_chains_attr attr = {}; @@ -4753,6 +4879,7 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) mutex_init(&tc->t_lock); mutex_init(&tc->hairpin_tbl_lock); hash_init(tc->hairpin_tbl); + tc->priv = priv; err = rhashtable_init(&tc->ht, &tc_ht_params); if (err) @@ -4782,7 +4909,7 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) attr.ns = MLX5_FLOW_NAMESPACE_KERNEL; attr.max_ft_sz = mlx5e_tc_nic_get_ft_size(dev); attr.max_grp_num = MLX5E_TC_TABLE_NUM_GROUPS; - attr.default_ft = priv->fs.tc.miss_t; + attr.default_ft = priv->fs->tc->miss_t; attr.mapping = chains_mapping; tc->chains = mlx5_chains_create(dev, &attr); @@ -4792,7 +4919,7 @@ int mlx5e_tc_nic_init(struct mlx5e_priv *priv) } tc->post_act = mlx5e_tc_post_act_init(priv, tc->chains, MLX5_FLOW_NAMESPACE_KERNEL); - tc->ct = mlx5_tc_ct_init(priv, tc->chains, &priv->fs.tc.mod_hdr, + tc->ct = mlx5_tc_ct_init(priv, tc->chains, &tc->mod_hdr, MLX5_FLOW_NAMESPACE_KERNEL, tc->post_act); tc->netdevice_nb.notifier_call = mlx5e_tc_netdev_event; @@ -4831,7 +4958,7 @@ static void _mlx5e_tc_del_flow(void *ptr, void *arg) void mlx5e_tc_nic_cleanup(struct mlx5e_priv *priv) { - struct mlx5e_tc_table *tc = &priv->fs.tc; + struct mlx5e_tc_table *tc = priv->fs->tc; if (tc->netdevice_nb.notifier_call) unregister_netdevice_notifier_dev_net(priv->netdev, @@ -4955,6 +5082,7 @@ void mlx5e_tc_esw_cleanup(struct mlx5_rep_uplink_priv *uplink_priv) mlx5e_tc_sample_cleanup(uplink_priv->tc_psample); mlx5e_tc_int_port_cleanup(uplink_priv->int_port_priv); mlx5_tc_ct_clean(uplink_priv->ct_priv); + mlx5e_flow_meters_cleanup(uplink_priv->flow_meters); mlx5e_tc_post_act_destroy(uplink_priv->post_act); } @@ -5035,7 +5163,7 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) u32 chain = 0, chain_tag, reg_b, zone_restore_id; struct mlx5e_priv *priv = netdev_priv(skb->dev); - struct mlx5e_tc_table *tc = &priv->fs.tc; + struct mlx5e_tc_table *tc = priv->fs->tc; struct mlx5_mapped_obj mapped_obj; struct tc_skb_ext *tc_skb_ext; int err; @@ -5060,7 +5188,7 @@ bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, tc_skb_ext->chain = chain; - zone_restore_id = (reg_b >> REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) & + zone_restore_id = (reg_b >> MLX5_REG_MAPPING_MOFFSET(NIC_ZONE_RESTORE_TO_REG)) & ESW_ZONE_ID_MASK; if (!mlx5e_tc_ct_restore_flow(tc->ct, skb, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h index e2a1250aeca1..6ce1ab6b86b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tc.h @@ -39,6 +39,7 @@ #include "en/tc_ct.h" #include "en/tc_tun.h" #include "en/tc/int_port.h" +#include "en/tc/meter.h" #include "en_rep.h" #define MLX5E_TC_FLOW_ID_MASK 0x0000ffff @@ -71,6 +72,7 @@ struct mlx5_flow_attr { struct mlx5_modify_hdr *modify_hdr; struct mlx5_ct_attr ct_attr; struct mlx5e_sample_attr sample_attr; + struct mlx5e_meter_attr meter_attr; struct mlx5e_tc_flow_parse_attr *parse_attr; u32 chain; u16 prio; @@ -83,6 +85,7 @@ struct mlx5_flow_attr { u8 tun_ip_version; int tunnel_id; /* mapped tunnel id */ u32 flags; + u32 exe_aso_type; struct list_head list; struct mlx5e_post_act_handle *post_act_handle; struct { @@ -229,6 +232,7 @@ enum mlx5e_tc_attr_to_reg { FTEID_TO_REG, NIC_CHAIN_TO_REG, NIC_ZONE_RESTORE_TO_REG, + PACKET_COLOR_TO_REG, }; struct mlx5e_tc_attr_to_reg_mapping { @@ -241,6 +245,10 @@ struct mlx5e_tc_attr_to_reg_mapping { extern struct mlx5e_tc_attr_to_reg_mapping mlx5e_tc_attr_to_reg_mappings[]; +#define MLX5_REG_MAPPING_MOFFSET(reg_id) (mlx5e_tc_attr_to_reg_mappings[reg_id].moffset) +#define MLX5_REG_MAPPING_MBITS(reg_id) (mlx5e_tc_attr_to_reg_mappings[reg_id].mlen) +#define MLX5_REG_MAPPING_MASK(reg_id) (GENMASK(mlx5e_tc_attr_to_reg_mappings[reg_id].mlen - 1, 0)) + bool mlx5e_is_valid_eswitch_fwd_dev(struct mlx5e_priv *priv, struct net_device *out_dev); @@ -348,6 +356,8 @@ mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, void *cb_priv) #endif #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) +struct mlx5e_tc_table *mlx5e_tc_table_alloc(void); +void mlx5e_tc_table_free(struct mlx5e_tc_table *tc); static inline bool mlx5e_cqe_regb_chain(struct mlx5_cqe64 *cqe) { #if IS_ENABLED(CONFIG_NET_TC_SKB_EXT) @@ -368,6 +378,8 @@ static inline bool mlx5e_cqe_regb_chain(struct mlx5_cqe64 *cqe) bool mlx5e_tc_update_skb(struct mlx5_cqe64 *cqe, struct sk_buff *skb); #else /* CONFIG_MLX5_CLS_ACT */ +static inline struct mlx5e_tc_table *mlx5e_tc_table_alloc(void) { return NULL; } +static inline void mlx5e_tc_table_free(struct mlx5e_tc_table *tc) {} static inline bool mlx5e_cqe_regb_chain(struct mlx5_cqe64 *cqe) { return false; } static inline bool diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c index 9a7250be229f..27f791feb517 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_tx.c @@ -152,14 +152,14 @@ mlx5e_tx_get_gso_ihs(struct mlx5e_txqsq *sq, struct sk_buff *skb, int *hopbyhop) *hopbyhop = 0; if (skb->encapsulation) { - ihs = skb_inner_transport_offset(skb) + inner_tcp_hdrlen(skb); + ihs = skb_inner_tcp_all_headers(skb); stats->tso_inner_packets++; stats->tso_inner_bytes += skb->len - ihs; } else { if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) { ihs = skb_transport_offset(skb) + sizeof(struct udphdr); } else { - ihs = skb_transport_offset(skb) + tcp_hdrlen(skb); + ihs = skb_tcp_all_headers(skb); if (ipv6_has_hopopt_jumbo(skb)) { *hopbyhop = sizeof(struct hop_jumbo_hdr); ihs -= sizeof(struct hop_jumbo_hdr); @@ -631,12 +631,22 @@ void mlx5e_tx_mpwqe_ensure_complete(struct mlx5e_txqsq *sq) mlx5e_tx_mpwqe_session_complete(sq); } +static void mlx5e_cqe_ts_id_eseg(struct mlx5e_ptpsq *ptpsq, struct sk_buff *skb, + struct mlx5_wqe_eth_seg *eseg) +{ + if (ptpsq->ts_cqe_ctr_mask && unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) + eseg->flow_table_metadata = cpu_to_be32(ptpsq->skb_fifo_pc & + ptpsq->ts_cqe_ctr_mask); +} + static void mlx5e_txwqe_build_eseg(struct mlx5e_priv *priv, struct mlx5e_txqsq *sq, struct sk_buff *skb, struct mlx5e_accel_tx_state *accel, struct mlx5_wqe_eth_seg *eseg, u16 ihs) { mlx5e_accel_tx_eseg(priv, skb, eseg, ihs); mlx5e_txwqe_build_eseg_csum(sq, skb, accel, eseg); + if (unlikely(sq->ptpsq)) + mlx5e_cqe_ts_id_eseg(sq->ptpsq, skb, eseg); } netdev_tx_t mlx5e_xmit(struct sk_buff *skb, struct net_device *dev) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c index 05e08cec5a8c..4fbff7bcc155 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.c @@ -1,6 +1,7 @@ // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB /* Copyright (c) 2021 Mellanox Technologies. */ +#include <linux/build_bug.h> #include <linux/list.h> #include <linux/notifier.h> #include <net/netevent.h> @@ -12,26 +13,57 @@ #define CREATE_TRACE_POINTS #include "diag/bridge_tracepoint.h" -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE 64000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE 12000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE 16000 #define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM 0 -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE / 4 - 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_FROM \ +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM \ (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO \ - (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE / 2 - 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM \ - (MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO (MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE - 1) - -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE 64000 +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_INGRESS_TABLE_UNTAGGED_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE \ + (MLX5_ESW_BRIDGE_INGRESS_TABLE_MAC_GRP_IDX_TO + 1) +static_assert(MLX5_ESW_BRIDGE_INGRESS_TABLE_SIZE == 64000); + +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE 16000 +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE (32000 - 1) #define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM 0 -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE / 2 - 1) -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \ +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM \ (MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE - 2) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_SIZE - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO + 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_FROM + \ + MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_SIZE - 1) #define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM \ (MLX5_ESW_BRIDGE_EGRESS_TABLE_MAC_GRP_IDX_TO + 1) -#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO (MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE - 1) +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO \ + MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_FROM +#define MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE \ + (MLX5_ESW_BRIDGE_EGRESS_TABLE_MISS_GRP_IDX_TO + 1) +static_assert(MLX5_ESW_BRIDGE_EGRESS_TABLE_SIZE == 64000); #define MLX5_ESW_BRIDGE_SKIP_TABLE_SIZE 0 @@ -63,12 +95,14 @@ struct mlx5_esw_bridge { struct mlx5_flow_table *egress_ft; struct mlx5_flow_group *egress_vlan_fg; + struct mlx5_flow_group *egress_qinq_fg; struct mlx5_flow_group *egress_mac_fg; struct mlx5_flow_group *egress_miss_fg; struct mlx5_pkt_reformat *egress_miss_pkt_reformat; struct mlx5_flow_handle *egress_miss_handle; unsigned long ageing_time; u32 flags; + u16 vlan_proto; }; static void @@ -138,7 +172,9 @@ mlx5_esw_bridge_table_create(int max_fte, u32 level, struct mlx5_eswitch *esw) } static struct mlx5_flow_group * -mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft) +mlx5_esw_bridge_ingress_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto, + struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *fg; @@ -154,30 +190,53 @@ mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flo MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); - MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + if (vlan_proto == ETH_P_8021Q) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + else if (vlan_proto == ETH_P_8021AD) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid); MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); - MLX5_SET(create_flow_group_in, in, start_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM); - MLX5_SET(create_flow_group_in, in, end_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO); + MLX5_SET(create_flow_group_in, in, start_flow_index, from); + MLX5_SET(create_flow_group_in, in, end_flow_index, to); fg = mlx5_create_flow_group(ingress_ft, in); kvfree(in); if (IS_ERR(fg)) esw_warn(esw->dev, - "Failed to create VLAN flow group for bridge ingress table (err=%ld)\n", - PTR_ERR(fg)); + "Failed to create VLAN(proto=%x) flow group for bridge ingress table (err=%ld)\n", + vlan_proto, PTR_ERR(fg)); return fg; } static struct mlx5_flow_group * -mlx5_esw_bridge_ingress_filter_fg_create(struct mlx5_eswitch *esw, - struct mlx5_flow_table *ingress_ft) +mlx5_esw_bridge_ingress_vlan_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, ingress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_qinq_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw, + ingress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(unsigned int from, unsigned int to, + u16 vlan_proto, struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *fg; @@ -193,27 +252,48 @@ mlx5_esw_bridge_ingress_filter_fg_create(struct mlx5_eswitch *esw, MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_47_16); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.smac_15_0); - MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); - + if (vlan_proto == ETH_P_8021Q) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + else if (vlan_proto == ETH_P_8021AD) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag); MLX5_SET(fte_match_param, match, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); - MLX5_SET(create_flow_group_in, in, start_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_FROM); - MLX5_SET(create_flow_group_in, in, end_flow_index, - MLX5_ESW_BRIDGE_INGRESS_TABLE_FILTER_GRP_IDX_TO); + MLX5_SET(create_flow_group_in, in, start_flow_index, from); + MLX5_SET(create_flow_group_in, in, end_flow_index, to); fg = mlx5_create_flow_group(ingress_ft, in); if (IS_ERR(fg)) esw_warn(esw->dev, "Failed to create bridge ingress table VLAN filter flow group (err=%ld)\n", PTR_ERR(fg)); - kvfree(in); return fg; } static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_vlan_filter_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_VLAN_FILTER_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(from, to, ETH_P_8021Q, esw, + ingress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_ingress_qinq_filter_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *ingress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_INGRESS_TABLE_QINQ_FILTER_GRP_IDX_TO; + + return mlx5_esw_bridge_ingress_vlan_proto_filter_fg_create(from, to, ETH_P_8021AD, esw, + ingress_ft); +} + +static struct mlx5_flow_group * mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *ingress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); @@ -250,7 +330,9 @@ mlx5_esw_bridge_ingress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow } static struct mlx5_flow_group * -mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) +mlx5_esw_bridge_egress_vlan_proto_fg_create(unsigned int from, unsigned int to, u16 vlan_proto, + struct mlx5_eswitch *esw, + struct mlx5_flow_table *egress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); struct mlx5_flow_group *fg; @@ -265,13 +347,14 @@ mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_47_16); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.dmac_15_0); - MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + if (vlan_proto == ETH_P_8021Q) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.cvlan_tag); + else if (vlan_proto == ETH_P_8021AD) + MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.svlan_tag); MLX5_SET_TO_ONES(fte_match_param, match, outer_headers.first_vid); - MLX5_SET(create_flow_group_in, in, start_flow_index, - MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM); - MLX5_SET(create_flow_group_in, in, end_flow_index, - MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO); + MLX5_SET(create_flow_group_in, in, start_flow_index, from); + MLX5_SET(create_flow_group_in, in, end_flow_index, to); fg = mlx5_create_flow_group(egress_ft, in); if (IS_ERR(fg)) @@ -283,6 +366,25 @@ mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow } static struct mlx5_flow_group * +mlx5_esw_bridge_egress_vlan_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_EGRESS_TABLE_VLAN_GRP_IDX_TO; + + return mlx5_esw_bridge_egress_vlan_proto_fg_create(from, to, ETH_P_8021Q, esw, egress_ft); +} + +static struct mlx5_flow_group * +mlx5_esw_bridge_egress_qinq_fg_create(struct mlx5_eswitch *esw, + struct mlx5_flow_table *egress_ft) +{ + unsigned int from = MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_FROM; + unsigned int to = MLX5_ESW_BRIDGE_EGRESS_TABLE_QINQ_GRP_IDX_TO; + + return mlx5_esw_bridge_egress_vlan_proto_fg_create(from, to, ETH_P_8021AD, esw, egress_ft); +} + +static struct mlx5_flow_group * mlx5_esw_bridge_egress_mac_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *egress_ft) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); @@ -346,7 +448,7 @@ mlx5_esw_bridge_egress_miss_fg_create(struct mlx5_eswitch *esw, struct mlx5_flow static int mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) { - struct mlx5_flow_group *mac_fg, *filter_fg, *vlan_fg; + struct mlx5_flow_group *mac_fg, *qinq_filter_fg, *qinq_fg, *vlan_filter_fg, *vlan_fg; struct mlx5_flow_table *ingress_ft, *skip_ft; struct mlx5_eswitch *esw = br_offloads->esw; int err; @@ -374,10 +476,22 @@ mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) goto err_vlan_fg; } - filter_fg = mlx5_esw_bridge_ingress_filter_fg_create(esw, ingress_ft); - if (IS_ERR(filter_fg)) { - err = PTR_ERR(filter_fg); - goto err_filter_fg; + vlan_filter_fg = mlx5_esw_bridge_ingress_vlan_filter_fg_create(esw, ingress_ft); + if (IS_ERR(vlan_filter_fg)) { + err = PTR_ERR(vlan_filter_fg); + goto err_vlan_filter_fg; + } + + qinq_fg = mlx5_esw_bridge_ingress_qinq_fg_create(esw, ingress_ft); + if (IS_ERR(qinq_fg)) { + err = PTR_ERR(qinq_fg); + goto err_qinq_fg; + } + + qinq_filter_fg = mlx5_esw_bridge_ingress_qinq_filter_fg_create(esw, ingress_ft); + if (IS_ERR(qinq_filter_fg)) { + err = PTR_ERR(qinq_filter_fg); + goto err_qinq_filter_fg; } mac_fg = mlx5_esw_bridge_ingress_mac_fg_create(esw, ingress_ft); @@ -389,13 +503,19 @@ mlx5_esw_bridge_ingress_table_init(struct mlx5_esw_bridge_offloads *br_offloads) br_offloads->ingress_ft = ingress_ft; br_offloads->skip_ft = skip_ft; br_offloads->ingress_vlan_fg = vlan_fg; - br_offloads->ingress_filter_fg = filter_fg; + br_offloads->ingress_vlan_filter_fg = vlan_filter_fg; + br_offloads->ingress_qinq_fg = qinq_fg; + br_offloads->ingress_qinq_filter_fg = qinq_filter_fg; br_offloads->ingress_mac_fg = mac_fg; return 0; err_mac_fg: - mlx5_destroy_flow_group(filter_fg); -err_filter_fg: + mlx5_destroy_flow_group(qinq_filter_fg); +err_qinq_filter_fg: + mlx5_destroy_flow_group(qinq_fg); +err_qinq_fg: + mlx5_destroy_flow_group(vlan_filter_fg); +err_vlan_filter_fg: mlx5_destroy_flow_group(vlan_fg); err_vlan_fg: mlx5_destroy_flow_table(skip_ft); @@ -409,8 +529,12 @@ mlx5_esw_bridge_ingress_table_cleanup(struct mlx5_esw_bridge_offloads *br_offloa { mlx5_destroy_flow_group(br_offloads->ingress_mac_fg); br_offloads->ingress_mac_fg = NULL; - mlx5_destroy_flow_group(br_offloads->ingress_filter_fg); - br_offloads->ingress_filter_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_qinq_filter_fg); + br_offloads->ingress_qinq_filter_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_qinq_fg); + br_offloads->ingress_qinq_fg = NULL; + mlx5_destroy_flow_group(br_offloads->ingress_vlan_filter_fg); + br_offloads->ingress_vlan_filter_fg = NULL; mlx5_destroy_flow_group(br_offloads->ingress_vlan_fg); br_offloads->ingress_vlan_fg = NULL; mlx5_destroy_flow_table(br_offloads->skip_ft); @@ -428,7 +552,7 @@ static int mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, struct mlx5_esw_bridge *bridge) { - struct mlx5_flow_group *miss_fg = NULL, *mac_fg, *vlan_fg; + struct mlx5_flow_group *miss_fg = NULL, *mac_fg, *vlan_fg, *qinq_fg; struct mlx5_pkt_reformat *miss_pkt_reformat = NULL; struct mlx5_flow_handle *miss_handle = NULL; struct mlx5_eswitch *esw = br_offloads->esw; @@ -447,6 +571,12 @@ mlx5_esw_bridge_egress_table_init(struct mlx5_esw_bridge_offloads *br_offloads, goto err_vlan_fg; } + qinq_fg = mlx5_esw_bridge_egress_qinq_fg_create(esw, egress_ft); + if (IS_ERR(qinq_fg)) { + err = PTR_ERR(qinq_fg); + goto err_qinq_fg; + } + mac_fg = mlx5_esw_bridge_egress_mac_fg_create(esw, egress_ft); if (IS_ERR(mac_fg)) { err = PTR_ERR(mac_fg); @@ -491,6 +621,7 @@ skip_miss_flow: bridge->egress_ft = egress_ft; bridge->egress_vlan_fg = vlan_fg; + bridge->egress_qinq_fg = qinq_fg; bridge->egress_mac_fg = mac_fg; bridge->egress_miss_fg = miss_fg; bridge->egress_miss_pkt_reformat = miss_pkt_reformat; @@ -498,6 +629,8 @@ skip_miss_flow: return 0; err_mac_fg: + mlx5_destroy_flow_group(qinq_fg); +err_qinq_fg: mlx5_destroy_flow_group(vlan_fg); err_vlan_fg: mlx5_destroy_flow_table(egress_ft); @@ -515,6 +648,7 @@ mlx5_esw_bridge_egress_table_cleanup(struct mlx5_esw_bridge *bridge) if (bridge->egress_miss_fg) mlx5_destroy_flow_group(bridge->egress_miss_fg); mlx5_destroy_flow_group(bridge->egress_mac_fg); + mlx5_destroy_flow_group(bridge->egress_qinq_fg); mlx5_destroy_flow_group(bridge->egress_vlan_fg); mlx5_destroy_flow_table(bridge->egress_ft); } @@ -559,10 +693,17 @@ mlx5_esw_bridge_ingress_flow_with_esw_create(u16 vport_num, const unsigned char flow_act.pkt_reformat = vlan->pkt_reformat_push; flow_act.modify_hdr = vlan->pkt_mod_hdr_push_mark; } else if (vlan) { - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, - outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, - outer_headers.cvlan_tag); + if (bridge->vlan_proto == ETH_P_8021Q) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + } else if (bridge->vlan_proto == ETH_P_8021AD) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.svlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.svlan_tag); + } MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.first_vid); MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, @@ -645,10 +786,17 @@ mlx5_esw_bridge_ingress_filter_flow_create(u16 vport_num, const unsigned char *a MLX5_SET(fte_match_param, rule_spec->match_value, misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_for_match(br_offloads->esw, vport_num)); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, - outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, - outer_headers.cvlan_tag); + if (bridge->vlan_proto == ETH_P_8021Q) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + } else if (bridge->vlan_proto == ETH_P_8021AD) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.svlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.svlan_tag); + } handle = mlx5_add_flow_rules(br_offloads->ingress_ft, rule_spec, &flow_act, &dest, 1); @@ -696,10 +844,17 @@ mlx5_esw_bridge_egress_flow_create(u16 vport_num, u16 esw_owner_vhca_id, const u flow_act.pkt_reformat = vlan->pkt_reformat_pop; } - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, - outer_headers.cvlan_tag); - MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, - outer_headers.cvlan_tag); + if (bridge->vlan_proto == ETH_P_8021Q) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.cvlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.cvlan_tag); + } else if (bridge->vlan_proto == ETH_P_8021AD) { + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, + outer_headers.svlan_tag); + MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_value, + outer_headers.svlan_tag); + } MLX5_SET_TO_ONES(fte_match_param, rule_spec->match_criteria, outer_headers.first_vid); MLX5_SET(fte_match_param, rule_spec->match_value, outer_headers.first_vid, @@ -774,6 +929,7 @@ static struct mlx5_esw_bridge *mlx5_esw_bridge_create(int ifindex, bridge->ifindex = ifindex; bridge->refcnt = 1; bridge->ageing_time = clock_t_to_jiffies(BR_DEFAULT_AGEING_TIME); + bridge->vlan_proto = ETH_P_8021Q; list_add(&bridge->list, &br_offloads->bridges); return bridge; @@ -911,12 +1067,13 @@ mlx5_esw_bridge_vlan_lookup(u16 vid, struct mlx5_esw_bridge_port *port) } static int -mlx5_esw_bridge_vlan_push_create(struct mlx5_esw_bridge_vlan *vlan, struct mlx5_eswitch *esw) +mlx5_esw_bridge_vlan_push_create(u16 vlan_proto, struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_eswitch *esw) { struct { __be16 h_vlan_proto; __be16 h_vlan_TCI; - } vlan_hdr = { htons(ETH_P_8021Q), htons(vlan->vid) }; + } vlan_hdr = { htons(vlan_proto), htons(vlan->vid) }; struct mlx5_pkt_reformat_params reformat_params = {}; struct mlx5_pkt_reformat *pkt_reformat; @@ -1008,36 +1165,58 @@ mlx5_esw_bridge_vlan_push_mark_cleanup(struct mlx5_esw_bridge_vlan *vlan, struct vlan->pkt_mod_hdr_push_mark = NULL; } -static struct mlx5_esw_bridge_vlan * -mlx5_esw_bridge_vlan_create(u16 vid, u16 flags, struct mlx5_esw_bridge_port *port, - struct mlx5_eswitch *esw) +static int +mlx5_esw_bridge_vlan_push_pop_create(u16 vlan_proto, u16 flags, struct mlx5_esw_bridge_vlan *vlan, + struct mlx5_eswitch *esw) { - struct mlx5_esw_bridge_vlan *vlan; int err; - vlan = kvzalloc(sizeof(*vlan), GFP_KERNEL); - if (!vlan) - return ERR_PTR(-ENOMEM); - - vlan->vid = vid; - vlan->flags = flags; - INIT_LIST_HEAD(&vlan->fdb_list); - if (flags & BRIDGE_VLAN_INFO_PVID) { - err = mlx5_esw_bridge_vlan_push_create(vlan, esw); + err = mlx5_esw_bridge_vlan_push_create(vlan_proto, vlan, esw); if (err) - goto err_vlan_push; + return err; err = mlx5_esw_bridge_vlan_push_mark_create(vlan, esw); if (err) goto err_vlan_push_mark; } + if (flags & BRIDGE_VLAN_INFO_UNTAGGED) { err = mlx5_esw_bridge_vlan_pop_create(vlan, esw); if (err) goto err_vlan_pop; } + return 0; + +err_vlan_pop: + if (vlan->pkt_mod_hdr_push_mark) + mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw); +err_vlan_push_mark: + if (vlan->pkt_reformat_push) + mlx5_esw_bridge_vlan_push_cleanup(vlan, esw); + return err; +} + +static struct mlx5_esw_bridge_vlan * +mlx5_esw_bridge_vlan_create(u16 vlan_proto, u16 vid, u16 flags, struct mlx5_esw_bridge_port *port, + struct mlx5_eswitch *esw) +{ + struct mlx5_esw_bridge_vlan *vlan; + int err; + + vlan = kvzalloc(sizeof(*vlan), GFP_KERNEL); + if (!vlan) + return ERR_PTR(-ENOMEM); + + vlan->vid = vid; + vlan->flags = flags; + INIT_LIST_HEAD(&vlan->fdb_list); + + err = mlx5_esw_bridge_vlan_push_pop_create(vlan_proto, flags, vlan, esw); + if (err) + goto err_vlan_push_pop; + err = xa_insert(&port->vlans, vid, vlan, GFP_KERNEL); if (err) goto err_xa_insert; @@ -1048,13 +1227,11 @@ mlx5_esw_bridge_vlan_create(u16 vid, u16 flags, struct mlx5_esw_bridge_port *por err_xa_insert: if (vlan->pkt_reformat_pop) mlx5_esw_bridge_vlan_pop_cleanup(vlan, esw); -err_vlan_pop: if (vlan->pkt_mod_hdr_push_mark) mlx5_esw_bridge_vlan_push_mark_cleanup(vlan, esw); -err_vlan_push_mark: if (vlan->pkt_reformat_push) mlx5_esw_bridge_vlan_push_cleanup(vlan, esw); -err_vlan_push: +err_vlan_push_pop: kvfree(vlan); return ERR_PTR(err); } @@ -1102,6 +1279,50 @@ static void mlx5_esw_bridge_port_vlans_flush(struct mlx5_esw_bridge_port *port, mlx5_esw_bridge_vlan_cleanup(port, vlan, bridge); } +static int mlx5_esw_bridge_port_vlans_recreate(struct mlx5_esw_bridge_port *port, + struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads; + struct mlx5_esw_bridge_vlan *vlan; + unsigned long i; + int err; + + xa_for_each(&port->vlans, i, vlan) { + mlx5_esw_bridge_vlan_flush(vlan, bridge); + err = mlx5_esw_bridge_vlan_push_pop_create(bridge->vlan_proto, vlan->flags, vlan, + br_offloads->esw); + if (err) { + esw_warn(br_offloads->esw->dev, + "Failed to create VLAN=%u(proto=%x) push/pop actions (vport=%u,err=%d)\n", + vlan->vid, bridge->vlan_proto, port->vport_num, + err); + return err; + } + } + + return 0; +} + +static int +mlx5_esw_bridge_vlans_recreate(struct mlx5_esw_bridge *bridge) +{ + struct mlx5_esw_bridge_offloads *br_offloads = bridge->br_offloads; + struct mlx5_esw_bridge_port *port; + unsigned long i; + int err; + + xa_for_each(&br_offloads->ports, i, port) { + if (port->bridge != bridge) + continue; + + err = mlx5_esw_bridge_port_vlans_recreate(port, bridge); + if (err) + return err; + } + + return 0; +} + static struct mlx5_esw_bridge_vlan * mlx5_esw_bridge_port_vlan_lookup(u16 vid, u16 vport_num, u16 esw_owner_vhca_id, struct mlx5_esw_bridge *bridge, struct mlx5_eswitch *esw) @@ -1287,6 +1508,32 @@ int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, boo return 0; } +int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto, + struct mlx5_esw_bridge_offloads *br_offloads) +{ + struct mlx5_esw_bridge_port *port; + struct mlx5_esw_bridge *bridge; + + port = mlx5_esw_bridge_port_lookup(vport_num, esw_owner_vhca_id, + br_offloads); + if (!port) + return -EINVAL; + + bridge = port->bridge; + if (bridge->vlan_proto == proto) + return 0; + if (proto != ETH_P_8021Q && proto != ETH_P_8021AD) { + esw_warn(br_offloads->esw->dev, "Can't set unsupported VLAN protocol %x", proto); + return -EOPNOTSUPP; + } + + mlx5_esw_bridge_fdb_flush(bridge); + bridge->vlan_proto = proto; + mlx5_esw_bridge_vlans_recreate(bridge); + + return 0; +} + static int mlx5_esw_bridge_vport_init(u16 vport_num, u16 esw_owner_vhca_id, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct mlx5_esw_bridge *bridge) @@ -1434,7 +1681,8 @@ int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, mlx5_esw_bridge_vlan_cleanup(port, vlan, port->bridge); } - vlan = mlx5_esw_bridge_vlan_create(vid, flags, port, br_offloads->esw); + vlan = mlx5_esw_bridge_vlan_create(port->bridge->vlan_proto, vid, flags, port, + br_offloads->esw); if (IS_ERR(vlan)) { NL_SET_ERR_MSG_MOD(extack, "Failed to create VLAN entry"); return PTR_ERR(vlan); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h index efc39975226e..10851a515bca 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/bridge.h @@ -26,7 +26,9 @@ struct mlx5_esw_bridge_offloads { struct mlx5_flow_table *ingress_ft; struct mlx5_flow_group *ingress_vlan_fg; - struct mlx5_flow_group *ingress_filter_fg; + struct mlx5_flow_group *ingress_vlan_filter_fg; + struct mlx5_flow_group *ingress_qinq_fg; + struct mlx5_flow_group *ingress_qinq_filter_fg; struct mlx5_flow_group *ingress_mac_fg; struct mlx5_flow_table *skip_ft; @@ -60,6 +62,8 @@ int mlx5_esw_bridge_ageing_time_set(u16 vport_num, u16 esw_owner_vhca_id, unsign struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_vlan_filtering_set(u16 vport_num, u16 esw_owner_vhca_id, bool enable, struct mlx5_esw_bridge_offloads *br_offloads); +int mlx5_esw_bridge_vlan_proto_set(u16 vport_num, u16 esw_owner_vhca_id, u16 proto, + struct mlx5_esw_bridge_offloads *br_offloads); int mlx5_esw_bridge_port_vlan_add(u16 vport_num, u16 esw_owner_vhca_id, u16 vid, u16 flags, struct mlx5_esw_bridge_offloads *br_offloads, struct netlink_ext_ack *extack); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c new file mode 100644 index 000000000000..2db13c71e88c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/debugfs.c @@ -0,0 +1,182 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#include <linux/debugfs.h> +#include "eswitch.h" + +enum vnic_diag_counter { + MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE, + MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW, + MLX5_VNIC_DIAG_COMP_EQ_OVERRUN, + MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN, + MLX5_VNIC_DIAG_CQ_OVERRUN, + MLX5_VNIC_DIAG_INVALID_COMMAND, + MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND, +}; + +static int mlx5_esw_query_vnic_diag(struct mlx5_vport *vport, enum vnic_diag_counter counter, + u32 *val) +{ + u32 out[MLX5_ST_SZ_DW(query_vnic_env_out)] = {}; + u32 in[MLX5_ST_SZ_DW(query_vnic_env_in)] = {}; + struct mlx5_core_dev *dev = vport->dev; + u16 vport_num = vport->vport; + void *vnic_diag_out; + int err; + + MLX5_SET(query_vnic_env_in, in, opcode, MLX5_CMD_OP_QUERY_VNIC_ENV); + MLX5_SET(query_vnic_env_in, in, vport_number, vport_num); + if (!mlx5_esw_is_manager_vport(dev->priv.eswitch, vport_num)) + MLX5_SET(query_vnic_env_in, in, other_vport, 1); + + err = mlx5_cmd_exec(dev, in, sizeof(in), out, sizeof(out)); + if (err) + return err; + + vnic_diag_out = MLX5_ADDR_OF(query_vnic_env_out, out, vport_env); + switch (counter) { + case MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, total_error_queues); + break; + case MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, + send_queue_priority_update_flow); + break; + case MLX5_VNIC_DIAG_COMP_EQ_OVERRUN: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, comp_eq_overrun); + break; + case MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, async_eq_overrun); + break; + case MLX5_VNIC_DIAG_CQ_OVERRUN: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, cq_overrun); + break; + case MLX5_VNIC_DIAG_INVALID_COMMAND: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, invalid_command); + break; + case MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND: + *val = MLX5_GET(vnic_diagnostic_statistics, vnic_diag_out, quota_exceeded_command); + break; + } + + return 0; +} + +static int __show_vnic_diag(struct seq_file *file, struct mlx5_vport *vport, + enum vnic_diag_counter type) +{ + u32 val = 0; + int ret; + + ret = mlx5_esw_query_vnic_diag(vport, type, &val); + if (ret) + return ret; + + seq_printf(file, "%d\n", val); + return 0; +} + +static int total_q_under_processor_handle_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_TOTAL_Q_UNDER_PROCESSOR_HANDLE); +} + +static int send_queue_priority_update_flow_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, + MLX5_VNIC_DIAG_SEND_QUEUE_PRIORITY_UPDATE_FLOW); +} + +static int comp_eq_overrun_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_COMP_EQ_OVERRUN); +} + +static int async_eq_overrun_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_ASYNC_EQ_OVERRUN); +} + +static int cq_overrun_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_CQ_OVERRUN); +} + +static int invalid_command_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_INVALID_COMMAND); +} + +static int quota_exceeded_command_show(struct seq_file *file, void *priv) +{ + return __show_vnic_diag(file, file->private, MLX5_VNIC_DIAG_QOUTA_EXCEEDED_COMMAND); +} + +DEFINE_SHOW_ATTRIBUTE(total_q_under_processor_handle); +DEFINE_SHOW_ATTRIBUTE(send_queue_priority_update_flow); +DEFINE_SHOW_ATTRIBUTE(comp_eq_overrun); +DEFINE_SHOW_ATTRIBUTE(async_eq_overrun); +DEFINE_SHOW_ATTRIBUTE(cq_overrun); +DEFINE_SHOW_ATTRIBUTE(invalid_command); +DEFINE_SHOW_ATTRIBUTE(quota_exceeded_command); + +void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num) +{ + struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); + + debugfs_remove_recursive(vport->dbgfs); + vport->dbgfs = NULL; +} + +/* vnic diag dir name is "pf", "ecpf" or "{vf/sf}_xxxx" */ +#define VNIC_DIAG_DIR_NAME_MAX_LEN 8 + +void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool is_sf, u16 sf_num) +{ + struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); + struct dentry *vnic_diag; + char dir_name[VNIC_DIAG_DIR_NAME_MAX_LEN]; + int err; + + if (!MLX5_CAP_GEN(esw->dev, vport_group_manager)) + return; + + if (vport_num == MLX5_VPORT_PF) { + strcpy(dir_name, "pf"); + } else if (vport_num == MLX5_VPORT_ECPF) { + strcpy(dir_name, "ecpf"); + } else { + err = snprintf(dir_name, VNIC_DIAG_DIR_NAME_MAX_LEN, "%s_%d", is_sf ? "sf" : "vf", + is_sf ? sf_num : vport_num - MLX5_VPORT_FIRST_VF); + if (WARN_ON(err < 0)) + return; + } + + vport->dbgfs = debugfs_create_dir(dir_name, esw->dbgfs); + vnic_diag = debugfs_create_dir("vnic_diag", vport->dbgfs); + + if (MLX5_CAP_GEN(esw->dev, vnic_env_queue_counters)) { + debugfs_create_file("total_q_under_processor_handle", 0444, vnic_diag, vport, + &total_q_under_processor_handle_fops); + debugfs_create_file("send_queue_priority_update_flow", 0444, vnic_diag, vport, + &send_queue_priority_update_flow_fops); + } + + if (MLX5_CAP_GEN(esw->dev, eq_overrun_count)) { + debugfs_create_file("comp_eq_overrun", 0444, vnic_diag, vport, + &comp_eq_overrun_fops); + debugfs_create_file("async_eq_overrun", 0444, vnic_diag, vport, + &async_eq_overrun_fops); + } + + if (MLX5_CAP_GEN(esw->dev, vnic_env_cq_overrun)) + debugfs_create_file("cq_overrun", 0444, vnic_diag, vport, &cq_overrun_fops); + + if (MLX5_CAP_GEN(esw->dev, invalid_command_count)) + debugfs_create_file("invalid_command", 0444, vnic_diag, vport, + &invalid_command_fops); + + if (MLX5_CAP_GEN(esw->dev, quota_exceeded_count)) + debugfs_create_file("quota_exceeded_command", 0444, vnic_diag, vport, + "a_exceeded_command_fops); +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c index 7f9b96d9537e..9bc7be95db54 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/esw/devlink_port.c @@ -87,11 +87,11 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_ devlink = priv_to_devlink(dev); dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, vport_num); - err = devlink_port_register(devlink, dl_port, dl_port_index); + err = devl_port_register(devlink, dl_port, dl_port_index); if (err) goto reg_err; - err = devlink_rate_leaf_create(dl_port, vport); + err = devl_rate_leaf_create(dl_port, vport); if (err) goto rate_err; @@ -99,7 +99,7 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_ return 0; rate_err: - devlink_port_unregister(dl_port); + devl_port_unregister(dl_port); reg_err: mlx5_esw_dl_port_free(dl_port); return err; @@ -118,10 +118,10 @@ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vpo if (vport->dl_port->devlink_rate) { mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); - devlink_rate_leaf_destroy(vport->dl_port); + devl_rate_leaf_destroy(vport->dl_port); } - devlink_port_unregister(vport->dl_port); + devl_port_unregister(vport->dl_port); mlx5_esw_dl_port_free(vport->dl_port); vport->dl_port = NULL; } @@ -156,11 +156,11 @@ int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_p devlink_port_attrs_pci_sf_set(dl_port, controller, pfnum, sfnum, !!controller); devlink = priv_to_devlink(dev); dl_port_index = mlx5_esw_vport_to_devlink_port_index(dev, vport_num); - err = devlink_port_register(devlink, dl_port, dl_port_index); + err = devl_port_register(devlink, dl_port, dl_port_index); if (err) return err; - err = devlink_rate_leaf_create(dl_port, vport); + err = devl_rate_leaf_create(dl_port, vport); if (err) goto rate_err; @@ -168,7 +168,7 @@ int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_p return 0; rate_err: - devlink_port_unregister(dl_port); + devl_port_unregister(dl_port); return err; } @@ -182,9 +182,9 @@ void mlx5_esw_devlink_sf_port_unregister(struct mlx5_eswitch *esw, u16 vport_num if (vport->dl_port->devlink_rate) { mlx5_esw_qos_vport_update_group(esw, vport, NULL, NULL); - devlink_rate_leaf_destroy(vport->dl_port); + devl_rate_leaf_destroy(vport->dl_port); } - devlink_port_unregister(vport->dl_port); + devl_port_unregister(vport->dl_port); vport->dl_port = NULL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c index 719ef26d23c0..6aa58044b949 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.c @@ -36,6 +36,7 @@ #include <linux/mlx5/vport.h> #include <linux/mlx5/fs.h> #include <linux/mlx5/mpfs.h> +#include <linux/debugfs.h> #include "esw/acl/lgcy.h" #include "esw/legacy.h" #include "esw/qos.h" @@ -1002,6 +1003,7 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, if (err) return err; + mlx5_esw_vport_debugfs_create(esw, vport_num, false, 0); err = esw_offloads_load_rep(esw, vport_num); if (err) goto err_rep; @@ -1009,6 +1011,7 @@ int mlx5_eswitch_load_vport(struct mlx5_eswitch *esw, u16 vport_num, return err; err_rep: + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); return err; } @@ -1016,6 +1019,7 @@ err_rep: void mlx5_eswitch_unload_vport(struct mlx5_eswitch *esw, u16 vport_num) { esw_offloads_unload_rep(esw, vport_num); + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); } @@ -1152,8 +1156,6 @@ mlx5_eswitch_update_num_of_vfs(struct mlx5_eswitch *esw, int num_vfs) { const u32 *out; - WARN_ON_ONCE(esw->mode != MLX5_ESWITCH_NONE); - if (num_vfs < 0) return; @@ -1186,6 +1188,9 @@ static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) int total_vports; int err; + if (esw->flags & MLX5_ESWITCH_VPORT_ACL_NS_CREATED) + return 0; + total_vports = mlx5_eswitch_get_total_vports(dev); if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) { @@ -1203,6 +1208,7 @@ static int mlx5_esw_acls_ns_init(struct mlx5_eswitch *esw) } else { esw_warn(dev, "ingress ACL is not supported by FW\n"); } + esw->flags |= MLX5_ESWITCH_VPORT_ACL_NS_CREATED; return 0; err: @@ -1215,6 +1221,7 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) { struct mlx5_core_dev *dev = esw->dev; + esw->flags &= ~MLX5_ESWITCH_VPORT_ACL_NS_CREATED; if (MLX5_CAP_ESW_INGRESS_ACL(dev, ft_support)) mlx5_fs_ingress_acls_cleanup(dev); if (MLX5_CAP_ESW_EGRESS_ACL(dev, ft_support)) @@ -1224,7 +1231,6 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) /** * mlx5_eswitch_enable_locked - Enable eswitch * @esw: Pointer to eswitch - * @mode: Eswitch mode to enable * @num_vfs: Enable eswitch for given number of VFs. This is optional. * Valid value are 0, > 0 and MLX5_ESWITCH_IGNORE_NUM_VFS. * Caller should pass num_vfs > 0 when enabling eswitch for @@ -1238,7 +1244,7 @@ static void mlx5_esw_acls_ns_cleanup(struct mlx5_eswitch *esw) * mode. If num_vfs >=0 is provided, it setup VF related eswitch vports. * It returns 0 on success or error code on failure. */ -int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) +int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs) { int err; @@ -1257,9 +1263,7 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) mlx5_eswitch_update_num_of_vfs(esw, num_vfs); - esw->mode = mode; - - if (mode == MLX5_ESWITCH_LEGACY) { + if (esw->mode == MLX5_ESWITCH_LEGACY) { err = esw_legacy_enable(esw); } else { mlx5_rescan_drivers(esw->dev); @@ -1269,22 +1273,19 @@ int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs) if (err) goto abort; + esw->fdb_table.flags |= MLX5_ESW_FDB_CREATED; + mlx5_eswitch_event_handlers_register(esw); esw_info(esw->dev, "Enable: mode(%s), nvfs(%d), active vports(%d)\n", - mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", + esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); - mlx5_esw_mode_change_notify(esw, mode); + mlx5_esw_mode_change_notify(esw, esw->mode); return 0; abort: - esw->mode = MLX5_ESWITCH_NONE; - - if (mode == MLX5_ESWITCH_OFFLOADS) - mlx5_rescan_drivers(esw->dev); - mlx5_esw_acls_ns_cleanup(esw); return err; } @@ -1305,14 +1306,16 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) if (!mlx5_esw_allowed(esw)) return 0; - toggle_lag = esw->mode == MLX5_ESWITCH_NONE; + devl_assert_locked(priv_to_devlink(esw->dev)); + + toggle_lag = !mlx5_esw_is_fdb_created(esw); if (toggle_lag) mlx5_lag_disable_change(esw->dev); down_write(&esw->mode_lock); - if (esw->mode == MLX5_ESWITCH_NONE) { - ret = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, num_vfs); + if (!mlx5_esw_is_fdb_created(esw)) { + ret = mlx5_eswitch_enable_locked(esw, num_vfs); } else { enum mlx5_eswitch_vport_event vport_events; @@ -1330,55 +1333,81 @@ int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) return ret; } -void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf) +/* When disabling sriov, free driver level resources. */ +void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) { - struct devlink *devlink = priv_to_devlink(esw->dev); - int old_mode; - - lockdep_assert_held_write(&esw->mode_lock); - - if (esw->mode == MLX5_ESWITCH_NONE) + if (!mlx5_esw_allowed(esw)) return; - esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n", + devl_assert_locked(priv_to_devlink(esw->dev)); + down_write(&esw->mode_lock); + /* If driver is unloaded, this function is called twice by remove_one() + * and mlx5_unload(). Prevent the second call. + */ + if (!esw->esw_funcs.num_vfs && !clear_vf) + goto unlock; + + esw_info(esw->dev, "Unload vfs: mode(%s), nvfs(%d), active vports(%d)\n", esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", esw->esw_funcs.num_vfs, esw->enabled_vports); - /* Notify eswitch users that it is exiting from current mode. - * So that it can do necessary cleanup before the eswitch is disabled. + mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); + if (clear_vf) + mlx5_eswitch_clear_vf_vports_info(esw); + /* If disabling sriov in switchdev mode, free meta rules here + * because it depends on num_vfs. */ - mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_NONE); + if (esw->mode == MLX5_ESWITCH_OFFLOADS) { + struct devlink *devlink = priv_to_devlink(esw->dev); - mlx5_eswitch_event_handlers_unregister(esw); + esw_offloads_del_send_to_vport_meta_rules(esw); + devl_rate_nodes_destroy(devlink); + } - if (esw->mode == MLX5_ESWITCH_LEGACY) - esw_legacy_disable(esw); - else if (esw->mode == MLX5_ESWITCH_OFFLOADS) - esw_offloads_disable(esw); + esw->esw_funcs.num_vfs = 0; - old_mode = esw->mode; - esw->mode = MLX5_ESWITCH_NONE; +unlock: + up_write(&esw->mode_lock); +} - if (old_mode == MLX5_ESWITCH_OFFLOADS) - mlx5_rescan_drivers(esw->dev); +/* Free resources for corresponding eswitch mode. It is called by devlink + * when changing eswitch mode or modprobe when unloading driver. + */ +void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw) +{ + struct devlink *devlink = priv_to_devlink(esw->dev); + + /* Notify eswitch users that it is exiting from current mode. + * So that it can do necessary cleanup before the eswitch is disabled. + */ + mlx5_esw_mode_change_notify(esw, MLX5_ESWITCH_LEGACY); - devlink_rate_nodes_destroy(devlink); + mlx5_eswitch_event_handlers_unregister(esw); + esw_info(esw->dev, "Disable: mode(%s), nvfs(%d), active vports(%d)\n", + esw->mode == MLX5_ESWITCH_LEGACY ? "LEGACY" : "OFFLOADS", + esw->esw_funcs.num_vfs, esw->enabled_vports); + + esw->fdb_table.flags &= ~MLX5_ESW_FDB_CREATED; + if (esw->mode == MLX5_ESWITCH_OFFLOADS) + esw_offloads_disable(esw); + else if (esw->mode == MLX5_ESWITCH_LEGACY) + esw_legacy_disable(esw); mlx5_esw_acls_ns_cleanup(esw); - if (clear_vf) - mlx5_eswitch_clear_vf_vports_info(esw); + if (esw->mode == MLX5_ESWITCH_OFFLOADS) + devl_rate_nodes_destroy(devlink); } -void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) +void mlx5_eswitch_disable(struct mlx5_eswitch *esw) { if (!mlx5_esw_allowed(esw)) return; + devl_assert_locked(priv_to_devlink(esw->dev)); mlx5_lag_disable_change(esw->dev); down_write(&esw->mode_lock); - mlx5_eswitch_disable_locked(esw, clear_vf); - esw->esw_funcs.num_vfs = 0; + mlx5_eswitch_disable_locked(esw); up_write(&esw->mode_lock); mlx5_lag_enable_change(esw->dev); } @@ -1573,7 +1602,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) refcount_set(&esw->qos.refcnt, 0); esw->enabled_vports = 0; - esw->mode = MLX5_ESWITCH_NONE; + esw->mode = MLX5_ESWITCH_LEGACY; esw->offloads.inline_mode = MLX5_INLINE_MODE_NONE; if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) && MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap)) @@ -1587,6 +1616,7 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev) dev->priv.eswitch = esw; BLOCKING_INIT_NOTIFIER_HEAD(&esw->n_head); + esw->dbgfs = debugfs_create_dir("esw", mlx5_debugfs_get_dev_root(esw->dev)); esw_info(dev, "Total vports %d, per vport: max uc(%d) max mc(%d)\n", esw->total_vports, @@ -1610,6 +1640,7 @@ void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) esw_info(esw->dev, "cleanup\n"); + debugfs_remove_recursive(esw->dbgfs); esw->dev->priv.eswitch = NULL; destroy_workqueue(esw->work_queue); WARN_ON(refcount_read(&esw->qos.refcnt)); @@ -1875,7 +1906,7 @@ u8 mlx5_eswitch_mode(const struct mlx5_core_dev *dev) { struct mlx5_eswitch *esw = dev->priv.eswitch; - return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_NONE; + return mlx5_esw_allowed(esw) ? esw->mode : MLX5_ESWITCH_LEGACY; } EXPORT_SYMBOL_GPL(mlx5_eswitch_mode); @@ -1995,8 +2026,6 @@ int mlx5_esw_try_lock(struct mlx5_eswitch *esw) */ void mlx5_esw_unlock(struct mlx5_eswitch *esw) { - if (!mlx5_esw_allowed(esw)) - return; up_write(&esw->mode_lock); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h index 2754a732914d..87ce5a208cb5 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch.h @@ -191,6 +191,7 @@ struct mlx5_vport { enum mlx5_eswitch_vport_event enabled_events; int index; struct devlink_port *dl_port; + struct dentry *dbgfs; }; struct mlx5_esw_indir_table; @@ -282,10 +283,15 @@ struct mlx5_esw_functions { enum { MLX5_ESWITCH_VPORT_MATCH_METADATA = BIT(0), MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED = BIT(1), + MLX5_ESWITCH_VPORT_ACL_NS_CREATED = BIT(2), }; struct mlx5_esw_bridge_offloads; +enum { + MLX5_ESW_FDB_CREATED = BIT(0), +}; + struct mlx5_eswitch { struct mlx5_core_dev *dev; struct mlx5_nb nb; @@ -331,12 +337,14 @@ struct mlx5_eswitch { u32 large_group_num; } params; struct blocking_notifier_head n_head; + struct dentry *dbgfs; }; void esw_offloads_disable(struct mlx5_eswitch *esw); int esw_offloads_enable(struct mlx5_eswitch *esw); void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw); int esw_offloads_init_reps(struct mlx5_eswitch *esw); +void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw); bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw); int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable); @@ -350,10 +358,11 @@ int mlx5_eswitch_init(struct mlx5_core_dev *dev); void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw); #define MLX5_ESWITCH_IGNORE_NUM_VFS (-1) -int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int mode, int num_vfs); +int mlx5_eswitch_enable_locked(struct mlx5_eswitch *esw, int num_vfs); int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs); -void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw, bool clear_vf); -void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf); +void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf); +void mlx5_eswitch_disable_locked(struct mlx5_eswitch *esw); +void mlx5_eswitch_disable(struct mlx5_eswitch *esw); int mlx5_eswitch_set_vport_mac(struct mlx5_eswitch *esw, u16 vport, const u8 *mac); int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, @@ -575,6 +584,11 @@ mlx5_esw_devlink_port_index_to_vport_num(unsigned int dl_port_index) return dl_port_index & 0xffff; } +static inline bool mlx5_esw_is_fdb_created(struct mlx5_eswitch *esw) +{ + return esw->fdb_table.flags & MLX5_ESW_FDB_CREATED; +} + /* TODO: This mlx5e_tc function shouldn't be called by eswitch */ void mlx5e_tc_clean_fdb_peer_flows(struct mlx5_eswitch *esw); @@ -672,6 +686,9 @@ int mlx5_esw_offloads_devlink_port_register(struct mlx5_eswitch *esw, u16 vport_ void mlx5_esw_offloads_devlink_port_unregister(struct mlx5_eswitch *esw, u16 vport_num); struct devlink_port *mlx5_esw_offloads_devlink_port(struct mlx5_eswitch *esw, u16 vport_num); +void mlx5_esw_vport_debugfs_create(struct mlx5_eswitch *esw, u16 vport_num, bool is_sf, u16 sf_num); +void mlx5_esw_vport_debugfs_destroy(struct mlx5_eswitch *esw, u16 vport_num); + int mlx5_esw_devlink_sf_port_register(struct mlx5_eswitch *esw, struct devlink_port *dl_port, u16 vport_num, u32 controller, u32 sfnum); void mlx5_esw_devlink_sf_port_unregister(struct mlx5_eswitch *esw, u16 vport_num); @@ -719,7 +736,8 @@ int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw); static inline int mlx5_eswitch_init(struct mlx5_core_dev *dev) { return 0; } static inline void mlx5_eswitch_cleanup(struct mlx5_eswitch *esw) {} static inline int mlx5_eswitch_enable(struct mlx5_eswitch *esw, int num_vfs) { return 0; } -static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw, bool clear_vf) {} +static inline void mlx5_eswitch_disable_sriov(struct mlx5_eswitch *esw, bool clear_vf) {} +static inline void mlx5_eswitch_disable(struct mlx5_eswitch *esw) {} static inline bool mlx5_eswitch_is_funcs_handler(struct mlx5_core_dev *dev) { return false; } static inline int mlx5_eswitch_set_vport_state(struct mlx5_eswitch *esw, u16 vport, int link_state) { return 0; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c index eb79810199d3..ed73132129aa 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/eswitch_offloads.c @@ -523,6 +523,20 @@ esw_cleanup_dests(struct mlx5_eswitch *esw, } } +static void +esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) +{ + struct mlx5e_flow_meter_handle *meter; + + meter = attr->meter_attr.meter; + flow_act->exe_aso.type = attr->exe_aso_type; + flow_act->exe_aso.object_id = meter->obj_id; + flow_act->exe_aso.flow_meter.meter_idx = meter->idx; + flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; + /* use metadata reg 5 for packet color */ + flow_act->exe_aso.return_reg_id = 5; +} + struct mlx5_flow_handle * mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec, @@ -590,6 +604,10 @@ mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) flow_act.modify_hdr = attr->modify_hdr; + if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && + attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) + esw_setup_meter(attr, &flow_act); + if (split) { fwd_attr.chain = attr->chain; fwd_attr.prio = attr->prio; @@ -1051,6 +1069,15 @@ static void mlx5_eswitch_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) mlx5_del_flow_rules(flows[i]); kvfree(flows); + /* If changing eswitch mode from switchdev to legacy, but num_vfs is not 0, + * meta rules could be freed again. So set it to NULL. + */ + esw->fdb_table.offloads.send_to_vport_meta_rules = NULL; +} + +void esw_offloads_del_send_to_vport_meta_rules(struct mlx5_eswitch *esw) +{ + mlx5_eswitch_del_send_to_vport_meta_rules(esw); } static int @@ -2045,7 +2072,7 @@ static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) if (!MLX5_CAP_GEN(dev, vport_group_manager)) return -EOPNOTSUPP; - if (esw->mode == MLX5_ESWITCH_NONE) + if (!mlx5_esw_is_fdb_created(esw)) return -EOPNOTSUPP; switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { @@ -2181,18 +2208,18 @@ static int esw_offloads_start(struct mlx5_eswitch *esw, { int err, err1; - mlx5_eswitch_disable_locked(esw, false); - err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, - esw->dev->priv.sriov.num_vfs); + esw->mode = MLX5_ESWITCH_OFFLOADS; + err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); if (err) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to offloads"); - err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, - MLX5_ESWITCH_IGNORE_NUM_VFS); + esw->mode = MLX5_ESWITCH_LEGACY; + err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err1) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch back to legacy"); } + mlx5_rescan_drivers(esw->dev); } if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { if (mlx5_eswitch_inline_mode_get(esw, @@ -2905,7 +2932,7 @@ int mlx5_esw_offloads_vport_metadata_set(struct mlx5_eswitch *esw, bool enable) int err = 0; down_write(&esw->mode_lock); - if (esw->mode != MLX5_ESWITCH_NONE) { + if (mlx5_esw_is_fdb_created(esw)) { err = -EBUSY; goto done; } @@ -3066,6 +3093,7 @@ static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) static void esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) { + struct devlink *devlink; bool host_pf_disabled; u16 new_num_vfs; @@ -3077,6 +3105,8 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) return; + devlink = priv_to_devlink(esw->dev); + devl_lock(devlink); /* Number of VFs can only change from "0 to x" or "x to 0". */ if (esw->esw_funcs.num_vfs > 0) { mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); @@ -3089,6 +3119,7 @@ esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) return; } esw->esw_funcs.num_vfs = new_num_vfs; + devl_unlock(devlink); } static void esw_functions_changed_event_handler(struct work_struct *work) @@ -3240,13 +3271,12 @@ static int esw_offloads_stop(struct mlx5_eswitch *esw, { int err, err1; - mlx5_eswitch_disable_locked(esw, false); - err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_LEGACY, - MLX5_ESWITCH_IGNORE_NUM_VFS); + esw->mode = MLX5_ESWITCH_LEGACY; + err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); - err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_OFFLOADS, - MLX5_ESWITCH_IGNORE_NUM_VFS); + esw->mode = MLX5_ESWITCH_OFFLOADS; + err1 = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); if (err1) { NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch back to offloads"); @@ -3345,36 +3375,6 @@ static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) return 0; } -static int eswitch_devlink_esw_mode_check(const struct mlx5_eswitch *esw) -{ - /* devlink commands in NONE eswitch mode are currently supported only - * on ECPF. - */ - return (esw->mode == MLX5_ESWITCH_NONE && - !mlx5_core_is_ecpf_esw_manager(esw->dev)) ? -EOPNOTSUPP : 0; -} - -/* FIXME: devl_unlock() followed by devl_lock() inside driver callback - * is never correct and prone to races. It's a transitional workaround, - * never repeat this pattern. - * - * This code MUST be fixed before removing devlink_mutex as it is safe - * to do only because of that mutex. - */ -static void mlx5_eswtich_mode_callback_enter(struct devlink *devlink, - struct mlx5_eswitch *esw) -{ - devl_unlock(devlink); - down_write(&esw->mode_lock); -} - -static void mlx5_eswtich_mode_callback_exit(struct devlink *devlink, - struct mlx5_eswitch *esw) -{ - up_write(&esw->mode_lock); - devl_lock(devlink); -} - int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, struct netlink_ext_ack *extack) { @@ -3389,15 +3389,6 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (esw_mode_from_devlink(mode, &mlx5_mode)) return -EINVAL; - /* FIXME: devl_unlock() followed by devl_lock() inside driver callback - * is never correct and prone to races. It's a transitional workaround, - * never repeat this pattern. - * - * This code MUST be fixed before removing devlink_mutex as it is safe - * to do only because of that mutex. - */ - devl_unlock(devlink); - mlx5_lag_disable_change(esw->dev); err = mlx5_esw_try_lock(esw); if (err < 0) { @@ -3410,6 +3401,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, if (cur_mlx5_mode == mlx5_mode) goto unlock; + mlx5_eswitch_disable_locked(esw); if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { if (mlx5_devlink_trap_get_num_active(esw->dev)) { NL_SET_ERR_MSG_MOD(extack, @@ -3420,6 +3412,7 @@ int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, err = esw_offloads_start(esw, extack); } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { err = esw_offloads_stop(esw, extack); + mlx5_rescan_drivers(esw->dev); } else { err = -EINVAL; } @@ -3428,7 +3421,6 @@ unlock: mlx5_esw_unlock(esw); enable_lag: mlx5_lag_enable_change(esw->dev); - devl_lock(devlink); return err; } @@ -3441,14 +3433,9 @@ int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - + down_write(&esw->mode_lock); err = esw_mode_to_devlink(esw->mode, mode); -unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3495,10 +3482,7 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto out; + down_write(&esw->mode_lock); switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: @@ -3532,11 +3516,11 @@ int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, goto out; esw->offloads.inline_mode = mlx5_mode; - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return 0; out: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3549,14 +3533,9 @@ int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - + down_write(&esw->mode_lock); err = esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); -unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3566,16 +3545,13 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, { struct mlx5_core_dev *dev = devlink_priv(devlink); struct mlx5_eswitch *esw; - int err; + int err = 0; esw = mlx5_devlink_eswitch_get(devlink); if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; + down_write(&esw->mode_lock); if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || @@ -3618,7 +3594,7 @@ int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, } unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); + up_write(&esw->mode_lock); return err; } @@ -3626,21 +3602,15 @@ int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, enum devlink_eswitch_encap_mode *encap) { struct mlx5_eswitch *esw; - int err; esw = mlx5_devlink_eswitch_get(devlink); if (IS_ERR(esw)) return PTR_ERR(esw); - mlx5_eswtich_mode_callback_enter(devlink, esw); - err = eswitch_devlink_esw_mode_check(esw); - if (err) - goto unlock; - + down_write(&esw->mode_lock); *encap = esw->offloads.encap; -unlock: - mlx5_eswtich_mode_callback_exit(devlink, esw); - return err; + up_write(&esw->mode_lock); + return 0; } static bool @@ -3763,12 +3733,14 @@ int mlx5_esw_offloads_sf_vport_enable(struct mlx5_eswitch *esw, struct devlink_p if (err) goto devlink_err; + mlx5_esw_vport_debugfs_create(esw, vport_num, true, sfnum); err = mlx5_esw_offloads_rep_load(esw, vport_num); if (err) goto rep_err; return 0; rep_err: + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_devlink_sf_port_unregister(esw, vport_num); devlink_err: mlx5_esw_vport_disable(esw, vport_num); @@ -3778,6 +3750,7 @@ devlink_err: void mlx5_esw_offloads_sf_vport_disable(struct mlx5_eswitch *esw, u16 vport_num) { mlx5_esw_offloads_rep_unload(esw, vport_num); + mlx5_esw_vport_debugfs_destroy(esw, vport_num); mlx5_esw_devlink_sf_port_unregister(esw, vport_num); mlx5_esw_vport_disable(esw, vport_num); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c index 2ccf7bef9b05..735dc805dad7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_cmd.c @@ -479,6 +479,30 @@ static int mlx5_set_extended_dest(struct mlx5_core_dev *dev, return 0; } + +static void +mlx5_cmd_set_fte_flow_meter(struct fs_fte *fte, void *in_flow_context) +{ + void *exe_aso_ctrl; + void *execute_aso; + + execute_aso = MLX5_ADDR_OF(flow_context, in_flow_context, + execute_aso[0]); + MLX5_SET(execute_aso, execute_aso, valid, 1); + MLX5_SET(execute_aso, execute_aso, aso_object_id, + fte->action.exe_aso.object_id); + + exe_aso_ctrl = MLX5_ADDR_OF(execute_aso, execute_aso, exe_aso_ctrl); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, return_reg_id, + fte->action.exe_aso.return_reg_id); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, aso_type, + fte->action.exe_aso.type); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, init_color, + fte->action.exe_aso.flow_meter.init_color); + MLX5_SET(exe_aso_ctrl_flow_meter, exe_aso_ctrl, meter_id, + fte->action.exe_aso.flow_meter.meter_idx); +} + static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, int opmod, int modify_mask, struct mlx5_flow_table *ft, @@ -663,6 +687,15 @@ static int mlx5_cmd_set_fte(struct mlx5_core_dev *dev, list_size); } + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) { + if (fte->action.exe_aso.type == MLX5_EXE_ASO_FLOW_METER) { + mlx5_cmd_set_fte_flow_meter(fte, in_flow_context); + } else { + err = -EOPNOTSUPP; + goto err_out; + } + } + err = mlx5_cmd_exec(dev, in, inlen, out, sizeof(out)); err_out: kvfree(in); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c index 21e5c709b2d3..f1b908d40fa6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fs_core.c @@ -2895,6 +2895,14 @@ static int create_fdb_bypass(struct mlx5_flow_steering *steering) return 0; } +static void cleanup_fdb_root_ns(struct mlx5_flow_steering *steering) +{ + cleanup_root_ns(steering->fdb_root_ns); + steering->fdb_root_ns = NULL; + kfree(steering->fdb_sub_ns); + steering->fdb_sub_ns = NULL; +} + static int init_fdb_root_ns(struct mlx5_flow_steering *steering) { struct fs_prio *maj_prio; @@ -2945,10 +2953,7 @@ static int init_fdb_root_ns(struct mlx5_flow_steering *steering) return 0; out_err: - cleanup_root_ns(steering->fdb_root_ns); - kfree(steering->fdb_sub_ns); - steering->fdb_sub_ns = NULL; - steering->fdb_root_ns = NULL; + cleanup_fdb_root_ns(steering); return err; } @@ -3108,10 +3113,7 @@ void mlx5_fs_core_cleanup(struct mlx5_core_dev *dev) struct mlx5_flow_steering *steering = dev->priv.steering; cleanup_root_ns(steering->root_ns); - cleanup_root_ns(steering->fdb_root_ns); - steering->fdb_root_ns = NULL; - kfree(steering->fdb_sub_ns); - steering->fdb_sub_ns = NULL; + cleanup_fdb_root_ns(steering); cleanup_root_ns(steering->port_sel_root_ns); cleanup_root_ns(steering->sniffer_rx_root_ns); cleanup_root_ns(steering->sniffer_tx_root_ns); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw.c b/drivers/net/ethernet/mellanox/mlx5/core/fw.c index cfb8bedba512..079fa44ada71 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw.c @@ -289,6 +289,10 @@ int mlx5_cmd_init_hca(struct mlx5_core_dev *dev, uint32_t *sw_owner_id) sw_owner_id[i]); } + if (MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) && + dev->priv.sw_vhca_id > 0) + MLX5_SET(init_hca_in, in, sw_vhca_id, dev->priv.sw_vhca_id); + return mlx5_cmd_exec_in(dev, init_hca, in); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c index 052af4901c0b..e8896f368362 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/fw_reset.c @@ -149,6 +149,9 @@ static void mlx5_fw_reset_complete_reload(struct mlx5_core_dev *dev) if (test_bit(MLX5_FW_RESET_FLAGS_PENDING_COMP, &fw_reset->reset_flags)) { complete(&fw_reset->done); } else { + mlx5_unload_one(dev); + if (mlx5_health_wait_pci_up(dev)) + mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); mlx5_load_one(dev, false); devlink_remote_reload_actions_performed(priv_to_devlink(dev), 0, BIT(DEVLINK_RELOAD_ACTION_DRIVER_REINIT) | @@ -183,15 +186,9 @@ static void mlx5_sync_reset_reload_work(struct work_struct *work) struct mlx5_fw_reset *fw_reset = container_of(work, struct mlx5_fw_reset, reset_reload_work); struct mlx5_core_dev *dev = fw_reset->dev; - int err; mlx5_sync_reset_clear_reset_requested(dev, false); mlx5_enter_error_state(dev, true); - mlx5_unload_one(dev); - err = mlx5_health_wait_pci_up(dev); - if (err) - mlx5_core_err(dev, "reset reload flow aborted, PCI reads still not working\n"); - fw_reset->ret = err; mlx5_fw_reset_complete_reload(dev); } @@ -395,7 +392,6 @@ static void mlx5_sync_reset_now_event(struct work_struct *work) } mlx5_enter_error_state(dev, true); - mlx5_unload_one(dev); done: fw_reset->ret = err; mlx5_fw_reset_complete_reload(dev); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/health.c b/drivers/net/ethernet/mellanox/mlx5/core/health.c index 659021c31cbd..2cf2c9948446 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/health.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/health.c @@ -666,16 +666,20 @@ static void mlx5_fw_fatal_reporter_err_work(struct work_struct *work) struct mlx5_fw_reporter_ctx fw_reporter_ctx; struct mlx5_core_health *health; struct mlx5_core_dev *dev; + struct devlink *devlink; struct mlx5_priv *priv; health = container_of(work, struct mlx5_core_health, fatal_report_work); priv = container_of(health, struct mlx5_priv, health); dev = container_of(priv, struct mlx5_core_dev, priv); + devlink = priv_to_devlink(dev); enter_error_state(dev, false); if (IS_ERR_OR_NULL(health->fw_fatal_reporter)) { + devl_lock(devlink); if (mlx5_health_try_recover(dev)) mlx5_core_err(dev, "health recovery failed\n"); + devl_unlock(devlink); return; } fw_reporter_ctx.err_synd = health->synd; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c index 8da73ef5680f..ac3757beaea2 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ethtool.c @@ -83,7 +83,7 @@ static void mlx5i_get_ringparam(struct net_device *dev, { struct mlx5e_priv *priv = mlx5i_epriv(dev); - mlx5e_ethtool_get_ringparam(priv, param); + mlx5e_ethtool_get_ringparam(priv, param, kernel_param); } static int mlx5i_set_channels(struct net_device *dev, diff --git a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c index 0a99a020a3b2..c02b7b08fb4c 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/ipoib/ipoib.c @@ -322,10 +322,10 @@ static int mlx5i_create_flow_steering(struct mlx5e_priv *priv) { int err; - priv->fs.ns = mlx5_get_flow_namespace(priv->mdev, + priv->fs->ns = mlx5_get_flow_namespace(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL); - if (!priv->fs.ns) + if (!priv->fs->ns) return -EINVAL; err = mlx5e_arfs_create_tables(priv); @@ -364,9 +364,18 @@ static int mlx5i_init_rx(struct mlx5e_priv *priv) struct mlx5_core_dev *mdev = priv->mdev; int err; - priv->rx_res = mlx5e_rx_res_alloc(); - if (!priv->rx_res) + priv->fs = mlx5e_fs_init(priv->profile, mdev, + !test_bit(MLX5E_STATE_DESTROYING, &priv->state)); + if (!priv->fs) { + netdev_err(priv->netdev, "FS allocation failed\n"); return -ENOMEM; + } + + priv->rx_res = mlx5e_rx_res_alloc(); + if (!priv->rx_res) { + err = -ENOMEM; + goto err_free_fs; + } mlx5e_create_q_counters(priv); @@ -397,6 +406,8 @@ err_destroy_q_counters: mlx5e_destroy_q_counters(priv); mlx5e_rx_res_free(priv->rx_res); priv->rx_res = NULL; +err_free_fs: + mlx5e_fs_cleanup(priv->fs); return err; } @@ -408,6 +419,7 @@ static void mlx5i_cleanup_rx(struct mlx5e_priv *priv) mlx5e_destroy_q_counters(priv); mlx5e_rx_res_free(priv->rx_res); priv->rx_res = NULL; + mlx5e_fs_cleanup(priv->fs); } /* The stats groups order is opposite to the update_stats() order calls */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c index 5d41e19378e0..0f34e3c80d1f 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lag/lag.c @@ -638,6 +638,7 @@ static int mlx5_deactivate_lag(struct mlx5_lag *ldev) static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) { #ifdef CONFIG_MLX5_ESWITCH + struct mlx5_core_dev *dev; u8 mode; #endif int i; @@ -647,11 +648,11 @@ static bool mlx5_lag_check_prereq(struct mlx5_lag *ldev) return false; #ifdef CONFIG_MLX5_ESWITCH - mode = mlx5_eswitch_mode(ldev->pf[MLX5_LAG_P1].dev); - - if (mode != MLX5_ESWITCH_NONE && mode != MLX5_ESWITCH_OFFLOADS) + dev = ldev->pf[MLX5_LAG_P1].dev; + if ((mlx5_sriov_is_enabled(dev)) && !is_mdev_switchdev_mode(dev)) return false; + mode = mlx5_eswitch_mode(dev); for (i = 0; i < ldev->ports; i++) if (mlx5_eswitch_mode(ldev->pf[i].dev) != mode) return false; @@ -766,8 +767,7 @@ static bool mlx5_lag_is_roce_lag(struct mlx5_lag *ldev) #ifdef CONFIG_MLX5_ESWITCH for (i = 0; i < ldev->ports; i++) - roce_lag = roce_lag && - ldev->pf[i].dev->priv.eswitch->mode == MLX5_ESWITCH_NONE; + roce_lag = roce_lag && is_mdev_legacy_mode(ldev->pf[i].dev); #endif return roce_lag; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c new file mode 100644 index 000000000000..21e14507ff5c --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.c @@ -0,0 +1,433 @@ +// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB +// Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + +#include <linux/mlx5/device.h> +#include <linux/mlx5/transobj.h> +#include "aso.h" +#include "wq.h" + +struct mlx5_aso_cq { + /* data path - accessed per cqe */ + struct mlx5_cqwq wq; + + /* data path - accessed per napi poll */ + struct mlx5_core_cq mcq; + + /* control */ + struct mlx5_core_dev *mdev; + struct mlx5_wq_ctrl wq_ctrl; +} ____cacheline_aligned_in_smp; + +struct mlx5_aso { + /* data path */ + u16 cc; + u16 pc; + + struct mlx5_wqe_ctrl_seg *doorbell_cseg; + struct mlx5_aso_cq cq; + + /* read only */ + struct mlx5_wq_cyc wq; + void __iomem *uar_map; + u32 sqn; + + /* control path */ + struct mlx5_wq_ctrl wq_ctrl; + +} ____cacheline_aligned_in_smp; + +static void mlx5_aso_free_cq(struct mlx5_aso_cq *cq) +{ + mlx5_wq_destroy(&cq->wq_ctrl); +} + +static int mlx5_aso_alloc_cq(struct mlx5_core_dev *mdev, int numa_node, + void *cqc_data, struct mlx5_aso_cq *cq) +{ + struct mlx5_core_cq *mcq = &cq->mcq; + struct mlx5_wq_param param; + int err; + u32 i; + + param.buf_numa_node = numa_node; + param.db_numa_node = numa_node; + + err = mlx5_cqwq_create(mdev, ¶m, cqc_data, &cq->wq, &cq->wq_ctrl); + if (err) + return err; + + mcq->cqe_sz = 64; + mcq->set_ci_db = cq->wq_ctrl.db.db; + mcq->arm_db = cq->wq_ctrl.db.db + 1; + + for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { + struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); + + cqe->op_own = 0xf1; + } + + cq->mdev = mdev; + + return 0; +} + +static int create_aso_cq(struct mlx5_aso_cq *cq, void *cqc_data) +{ + u32 out[MLX5_ST_SZ_DW(create_cq_out)]; + struct mlx5_core_dev *mdev = cq->mdev; + struct mlx5_core_cq *mcq = &cq->mcq; + void *in, *cqc; + int inlen, eqn; + int err; + + err = mlx5_vector2eqn(mdev, 0, &eqn); + if (err) + return err; + + inlen = MLX5_ST_SZ_BYTES(create_cq_in) + + sizeof(u64) * cq->wq_ctrl.buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); + + memcpy(cqc, cqc_data, MLX5_ST_SZ_BYTES(cqc)); + + mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, + (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); + + MLX5_SET(cqc, cqc, cq_period_mode, DIM_CQ_PERIOD_MODE_START_FROM_EQE); + MLX5_SET(cqc, cqc, c_eqn_or_apu_element, eqn); + MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); + MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); + + err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); + + kvfree(in); + + return err; +} + +static void mlx5_aso_destroy_cq(struct mlx5_aso_cq *cq) +{ + mlx5_core_destroy_cq(cq->mdev, &cq->mcq); + mlx5_wq_destroy(&cq->wq_ctrl); +} + +static int mlx5_aso_create_cq(struct mlx5_core_dev *mdev, int numa_node, + struct mlx5_aso_cq *cq) +{ + void *cqc_data; + int err; + + cqc_data = kvzalloc(MLX5_ST_SZ_BYTES(cqc), GFP_KERNEL); + if (!cqc_data) + return -ENOMEM; + + MLX5_SET(cqc, cqc_data, log_cq_size, 1); + MLX5_SET(cqc, cqc_data, uar_page, mdev->priv.uar->index); + if (MLX5_CAP_GEN(mdev, cqe_128_always) && cache_line_size() >= 128) + MLX5_SET(cqc, cqc_data, cqe_sz, CQE_STRIDE_128_PAD); + + err = mlx5_aso_alloc_cq(mdev, numa_node, cqc_data, cq); + if (err) { + mlx5_core_err(mdev, "Failed to alloc aso wq cq, err=%d\n", err); + goto err_out; + } + + err = create_aso_cq(cq, cqc_data); + if (err) { + mlx5_core_err(mdev, "Failed to create aso wq cq, err=%d\n", err); + goto err_free_cq; + } + + kvfree(cqc_data); + return 0; + +err_free_cq: + mlx5_aso_free_cq(cq); +err_out: + kvfree(cqc_data); + return err; +} + +static int mlx5_aso_alloc_sq(struct mlx5_core_dev *mdev, int numa_node, + void *sqc_data, struct mlx5_aso *sq) +{ + void *sqc_wq = MLX5_ADDR_OF(sqc, sqc_data, wq); + struct mlx5_wq_cyc *wq = &sq->wq; + struct mlx5_wq_param param; + int err; + + sq->uar_map = mdev->mlx5e_res.hw_objs.bfreg.map; + + param.db_numa_node = numa_node; + param.buf_numa_node = numa_node; + err = mlx5_wq_cyc_create(mdev, ¶m, sqc_wq, wq, &sq->wq_ctrl); + if (err) + return err; + wq->db = &wq->db[MLX5_SND_DBR]; + + return 0; +} + +static int create_aso_sq(struct mlx5_core_dev *mdev, int pdn, + void *sqc_data, struct mlx5_aso *sq) +{ + void *in, *sqc, *wq; + int inlen, err; + + inlen = MLX5_ST_SZ_BYTES(create_sq_in) + + sizeof(u64) * sq->wq_ctrl.buf.npages; + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); + wq = MLX5_ADDR_OF(sqc, sqc, wq); + + memcpy(sqc, sqc_data, MLX5_ST_SZ_BYTES(sqc)); + MLX5_SET(sqc, sqc, cqn, sq->cq.mcq.cqn); + + MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); + MLX5_SET(sqc, sqc, flush_in_error_en, 1); + + MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); + MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.hw_objs.bfreg.index); + MLX5_SET(wq, wq, log_wq_pg_sz, sq->wq_ctrl.buf.page_shift - + MLX5_ADAPTER_PAGE_SHIFT); + MLX5_SET64(wq, wq, dbr_addr, sq->wq_ctrl.db.dma); + + mlx5_fill_page_frag_array(&sq->wq_ctrl.buf, + (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); + + err = mlx5_core_create_sq(mdev, in, inlen, &sq->sqn); + + kvfree(in); + + return err; +} + +static int mlx5_aso_set_sq_rdy(struct mlx5_core_dev *mdev, u32 sqn) +{ + void *in, *sqc; + int inlen, err; + + inlen = MLX5_ST_SZ_BYTES(modify_sq_in); + in = kvzalloc(inlen, GFP_KERNEL); + if (!in) + return -ENOMEM; + + MLX5_SET(modify_sq_in, in, sq_state, MLX5_SQC_STATE_RST); + sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); + MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RDY); + + err = mlx5_core_modify_sq(mdev, sqn, in); + + kvfree(in); + + return err; +} + +static int mlx5_aso_create_sq_rdy(struct mlx5_core_dev *mdev, u32 pdn, + void *sqc_data, struct mlx5_aso *sq) +{ + int err; + + err = create_aso_sq(mdev, pdn, sqc_data, sq); + if (err) + return err; + + err = mlx5_aso_set_sq_rdy(mdev, sq->sqn); + if (err) + mlx5_core_destroy_sq(mdev, sq->sqn); + + return err; +} + +static void mlx5_aso_free_sq(struct mlx5_aso *sq) +{ + mlx5_wq_destroy(&sq->wq_ctrl); +} + +static void mlx5_aso_destroy_sq(struct mlx5_aso *sq) +{ + mlx5_core_destroy_sq(sq->cq.mdev, sq->sqn); + mlx5_aso_free_sq(sq); +} + +static int mlx5_aso_create_sq(struct mlx5_core_dev *mdev, int numa_node, + u32 pdn, struct mlx5_aso *sq) +{ + void *sqc_data, *wq; + int err; + + sqc_data = kvzalloc(MLX5_ST_SZ_BYTES(sqc), GFP_KERNEL); + if (!sqc_data) + return -ENOMEM; + + wq = MLX5_ADDR_OF(sqc, sqc_data, wq); + MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); + MLX5_SET(wq, wq, pd, pdn); + MLX5_SET(wq, wq, log_wq_sz, 1); + + err = mlx5_aso_alloc_sq(mdev, numa_node, sqc_data, sq); + if (err) { + mlx5_core_err(mdev, "Failed to alloc aso wq sq, err=%d\n", err); + goto err_out; + } + + err = mlx5_aso_create_sq_rdy(mdev, pdn, sqc_data, sq); + if (err) { + mlx5_core_err(mdev, "Failed to open aso wq sq, err=%d\n", err); + goto err_free_asosq; + } + + mlx5_core_dbg(mdev, "aso sq->sqn = 0x%x\n", sq->sqn); + + kvfree(sqc_data); + return 0; + +err_free_asosq: + mlx5_aso_free_sq(sq); +err_out: + kvfree(sqc_data); + return err; +} + +struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn) +{ + int numa_node = dev_to_node(mlx5_core_dma_dev(mdev)); + struct mlx5_aso *aso; + int err; + + aso = kzalloc(sizeof(*aso), GFP_KERNEL); + if (!aso) + return ERR_PTR(-ENOMEM); + + err = mlx5_aso_create_cq(mdev, numa_node, &aso->cq); + if (err) + goto err_cq; + + err = mlx5_aso_create_sq(mdev, numa_node, pdn, aso); + if (err) + goto err_sq; + + return aso; + +err_sq: + mlx5_aso_destroy_cq(&aso->cq); +err_cq: + kfree(aso); + return ERR_PTR(err); +} + +void mlx5_aso_destroy(struct mlx5_aso *aso) +{ + if (IS_ERR_OR_NULL(aso)) + return; + + mlx5_aso_destroy_sq(aso); + mlx5_aso_destroy_cq(&aso->cq); + kfree(aso); +} + +void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, + struct mlx5_aso_wqe *aso_wqe, + u32 obj_id, u32 opc_mode) +{ + struct mlx5_wqe_ctrl_seg *cseg = &aso_wqe->ctrl; + + cseg->opmod_idx_opcode = cpu_to_be32((opc_mode << MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT) | + (aso->pc << MLX5_WQE_CTRL_WQE_INDEX_SHIFT) | + MLX5_OPCODE_ACCESS_ASO); + cseg->qpn_ds = cpu_to_be32((aso->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | ds_cnt); + cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; + cseg->general_id = cpu_to_be32(obj_id); +} + +void *mlx5_aso_get_wqe(struct mlx5_aso *aso) +{ + u16 pi; + + pi = mlx5_wq_cyc_ctr2ix(&aso->wq, aso->pc); + return mlx5_wq_cyc_get_wqe(&aso->wq, pi); +} + +void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, + struct mlx5_wqe_ctrl_seg *doorbell_cseg) +{ + doorbell_cseg->fm_ce_se |= MLX5_WQE_CTRL_CQ_UPDATE; + /* ensure wqe is visible to device before updating doorbell record */ + dma_wmb(); + + if (with_data) + aso->pc += MLX5_ASO_WQEBBS_DATA; + else + aso->pc += MLX5_ASO_WQEBBS; + *aso->wq.db = cpu_to_be32(aso->pc); + + /* ensure doorbell record is visible to device before ringing the + * doorbell + */ + wmb(); + + mlx5_write64((__be32 *)doorbell_cseg, aso->uar_map); + + /* Ensure doorbell is written on uar_page before poll_cq */ + WRITE_ONCE(doorbell_cseg, NULL); +} + +int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms) +{ + struct mlx5_aso_cq *cq = &aso->cq; + struct mlx5_cqe64 *cqe; + unsigned long expires; + + cqe = mlx5_cqwq_get_cqe(&cq->wq); + + expires = jiffies + msecs_to_jiffies(interval_ms); + while (!cqe && time_is_after_jiffies(expires)) { + usleep_range(2, 10); + cqe = mlx5_cqwq_get_cqe(&cq->wq); + } + + if (!cqe) + return -ETIMEDOUT; + + /* sq->cc must be updated only after mlx5_cqwq_update_db_record(), + * otherwise a cq overrun may occur + */ + mlx5_cqwq_pop(&cq->wq); + + if (unlikely(get_cqe_opcode(cqe) != MLX5_CQE_REQ)) { + struct mlx5_err_cqe *err_cqe; + + mlx5_core_err(cq->mdev, "Bad OP in ASOSQ CQE: 0x%x\n", + get_cqe_opcode(cqe)); + + err_cqe = (struct mlx5_err_cqe *)cqe; + mlx5_core_err(cq->mdev, "vendor_err_synd=%x\n", + err_cqe->vendor_err_synd); + mlx5_core_err(cq->mdev, "syndrome=%x\n", + err_cqe->syndrome); + print_hex_dump(KERN_WARNING, "", DUMP_PREFIX_OFFSET, + 16, 1, err_cqe, + sizeof(*err_cqe), false); + } + + mlx5_cqwq_update_db_record(&cq->wq); + + /* ensure cq space is freed before enabling more cqes */ + wmb(); + + if (with_data) + aso->cc += MLX5_ASO_WQEBBS_DATA; + else + aso->cc += MLX5_ASO_WQEBBS; + + return 0; +} diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h new file mode 100644 index 000000000000..b3bbf284fe71 --- /dev/null +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/aso.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB */ +/* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. */ + +#ifndef __MLX5_LIB_ASO_H__ +#define __MLX5_LIB_ASO_H__ + +#include <linux/mlx5/qp.h> +#include "mlx5_core.h" + +#define MLX5_ASO_WQEBBS \ + (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe), MLX5_SEND_WQE_BB)) +#define MLX5_ASO_WQEBBS_DATA \ + (DIV_ROUND_UP(sizeof(struct mlx5_aso_wqe_data), MLX5_SEND_WQE_BB)) +#define MLX5_WQE_CTRL_WQE_OPC_MOD_SHIFT 24 + +struct mlx5_wqe_aso_ctrl_seg { + __be32 va_h; + __be32 va_l; /* include read_enable */ + __be32 l_key; + u8 data_mask_mode; + u8 condition_1_0_operand; + u8 condition_1_0_offset; + u8 data_offset_condition_operand; + __be32 condition_0_data; + __be32 condition_0_mask; + __be32 condition_1_data; + __be32 condition_1_mask; + __be64 bitwise_data; + __be64 data_mask; +}; + +struct mlx5_wqe_aso_data_seg { + __be32 bytewise_data[16]; +}; + +struct mlx5_aso_wqe { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_aso_ctrl_seg aso_ctrl; +}; + +struct mlx5_aso_wqe_data { + struct mlx5_wqe_ctrl_seg ctrl; + struct mlx5_wqe_aso_ctrl_seg aso_ctrl; + struct mlx5_wqe_aso_data_seg aso_data; +}; + +enum { + MLX5_ASO_LOGICAL_AND, + MLX5_ASO_LOGICAL_OR, +}; + +enum { + MLX5_ASO_ALWAYS_FALSE, + MLX5_ASO_ALWAYS_TRUE, + MLX5_ASO_EQUAL, + MLX5_ASO_NOT_EQUAL, + MLX5_ASO_GREATER_OR_EQUAL, + MLX5_ASO_LESSER_OR_EQUAL, + MLX5_ASO_LESSER, + MLX5_ASO_GREATER, + MLX5_ASO_CYCLIC_GREATER, + MLX5_ASO_CYCLIC_LESSER, +}; + +enum { + MLX5_ASO_DATA_MASK_MODE_BITWISE_64BIT, + MLX5_ASO_DATA_MASK_MODE_BYTEWISE_64BYTE, + MLX5_ASO_DATA_MASK_MODE_CALCULATED_64BYTE, +}; + +enum { + MLX5_ACCESS_ASO_OPC_MOD_FLOW_METER = 0x2, +}; + +struct mlx5_aso; + +void *mlx5_aso_get_wqe(struct mlx5_aso *aso); +void mlx5_aso_build_wqe(struct mlx5_aso *aso, u8 ds_cnt, + struct mlx5_aso_wqe *aso_wqe, + u32 obj_id, u32 opc_mode); +void mlx5_aso_post_wqe(struct mlx5_aso *aso, bool with_data, + struct mlx5_wqe_ctrl_seg *doorbell_cseg); +int mlx5_aso_poll_cq(struct mlx5_aso *aso, bool with_data, u32 interval_ms); + +struct mlx5_aso *mlx5_aso_create(struct mlx5_core_dev *mdev, u32 pdn); +void mlx5_aso_destroy(struct mlx5_aso *aso); +#endif /* __MLX5_LIB_ASO_H__ */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c index 3d5e57ff558c..9482e51ac82a 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/lib/dm.c @@ -12,13 +12,16 @@ struct mlx5_dm { spinlock_t lock; unsigned long *steering_sw_icm_alloc_blocks; unsigned long *header_modify_sw_icm_alloc_blocks; + unsigned long *header_modify_pattern_sw_icm_alloc_blocks; }; struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) { + u64 header_modify_pattern_icm_blocks = 0; u64 header_modify_icm_blocks = 0; u64 steering_icm_blocks = 0; struct mlx5_dm *dm; + bool support_v2; if (!(MLX5_CAP_GEN_64(dev, general_obj_types) & MLX5_GENERAL_OBJ_TYPES_CAP_SW_ICM)) return NULL; @@ -35,8 +38,7 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); dm->steering_sw_icm_alloc_blocks = - kcalloc(BITS_TO_LONGS(steering_icm_blocks), - sizeof(unsigned long), GFP_KERNEL); + bitmap_zalloc(steering_icm_blocks, GFP_KERNEL); if (!dm->steering_sw_icm_alloc_blocks) goto err_steering; } @@ -47,16 +49,33 @@ struct mlx5_dm *mlx5_dm_create(struct mlx5_core_dev *dev) MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); dm->header_modify_sw_icm_alloc_blocks = - kcalloc(BITS_TO_LONGS(header_modify_icm_blocks), - sizeof(unsigned long), GFP_KERNEL); + bitmap_zalloc(header_modify_icm_blocks, GFP_KERNEL); if (!dm->header_modify_sw_icm_alloc_blocks) goto err_modify_hdr; } + support_v2 = MLX5_CAP_FLOWTABLE_NIC_RX(dev, sw_owner_v2) && + MLX5_CAP_FLOWTABLE_NIC_TX(dev, sw_owner_v2) && + MLX5_CAP64_DEV_MEM(dev, header_modify_pattern_sw_icm_start_address); + + if (support_v2) { + header_modify_pattern_icm_blocks = + BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_pattern_sw_icm_size) - + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)); + + dm->header_modify_pattern_sw_icm_alloc_blocks = + bitmap_zalloc(header_modify_pattern_icm_blocks, GFP_KERNEL); + if (!dm->header_modify_pattern_sw_icm_alloc_blocks) + goto err_pattern; + } + return dm; +err_pattern: + bitmap_free(dm->header_modify_sw_icm_alloc_blocks); + err_modify_hdr: - kfree(dm->steering_sw_icm_alloc_blocks); + bitmap_free(dm->steering_sw_icm_alloc_blocks); err_steering: kfree(dm); @@ -75,7 +94,7 @@ void mlx5_dm_cleanup(struct mlx5_core_dev *dev) WARN_ON(!bitmap_empty(dm->steering_sw_icm_alloc_blocks, BIT(MLX5_CAP_DEV_MEM(dev, log_steering_sw_icm_size) - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); - kfree(dm->steering_sw_icm_alloc_blocks); + bitmap_free(dm->steering_sw_icm_alloc_blocks); } if (dm->header_modify_sw_icm_alloc_blocks) { @@ -83,7 +102,15 @@ void mlx5_dm_cleanup(struct mlx5_core_dev *dev) BIT(MLX5_CAP_DEV_MEM(dev, log_header_modify_sw_icm_size) - MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); - kfree(dm->header_modify_sw_icm_alloc_blocks); + bitmap_free(dm->header_modify_sw_icm_alloc_blocks); + } + + if (dm->header_modify_pattern_sw_icm_alloc_blocks) { + WARN_ON(!bitmap_empty(dm->header_modify_pattern_sw_icm_alloc_blocks, + BIT(MLX5_CAP_DEV_MEM(dev, + log_header_modify_pattern_sw_icm_size) - + MLX5_LOG_SW_ICM_BLOCK_SIZE(dev)))); + bitmap_free(dm->header_modify_pattern_sw_icm_alloc_blocks); } kfree(dm); @@ -130,6 +157,13 @@ int mlx5_dm_sw_icm_alloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type, log_header_modify_sw_icm_size); block_map = dm->header_modify_sw_icm_alloc_blocks; break; + case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, + header_modify_pattern_sw_icm_start_address); + log_icm_size = MLX5_CAP_DEV_MEM(dev, + log_header_modify_pattern_sw_icm_size); + block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; + break; default: return -EINVAL; } @@ -203,6 +237,11 @@ int mlx5_dm_sw_icm_dealloc(struct mlx5_core_dev *dev, enum mlx5_sw_icm_type type icm_start_addr = MLX5_CAP64_DEV_MEM(dev, header_modify_sw_icm_start_address); block_map = dm->header_modify_sw_icm_alloc_blocks; break; + case MLX5_SW_ICM_TYPE_HEADER_MODIFY_PATTERN: + icm_start_addr = MLX5_CAP64_DEV_MEM(dev, + header_modify_pattern_sw_icm_start_address); + block_map = dm->header_modify_pattern_sw_icm_alloc_blocks; + break; default: return -EINVAL; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/main.c b/drivers/net/ethernet/mellanox/mlx5/core/main.c index ba2e5232b90b..bec8d6d0b5f6 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/main.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/main.c @@ -90,6 +90,8 @@ module_param_named(prof_sel, prof_sel, uint, 0444); MODULE_PARM_DESC(prof_sel, "profile selector. Valid range 0 - 2"); static u32 sw_owner_id[4]; +#define MAX_SW_VHCA_ID (BIT(__mlx5_bit_sz(cmd_hca_cap_2, sw_vhca_id)) - 1) +static DEFINE_IDA(sw_vhca_ida); enum { MLX5_ATOMIC_REQ_MODE_BE = 0x0, @@ -314,13 +316,6 @@ struct mlx5_reg_host_endianness { u8 rsvd[15]; }; -#define CAP_MASK(pos, size) ((u64)((1 << (size)) - 1) << (pos)) - -enum { - MLX5_CAP_BITS_RW_MASK = CAP_MASK(MLX5_CAP_OFF_CMDIF_CSUM, 2) | - MLX5_DEV_CAP_FLAG_DCT, -}; - static u16 to_fw_pkey_sz(struct mlx5_core_dev *dev, u32 size) { switch (size) { @@ -499,6 +494,31 @@ static int max_uc_list_get_devlink_param(struct mlx5_core_dev *dev) return err; } +static int handle_hca_cap_2(struct mlx5_core_dev *dev, void *set_ctx) +{ + void *set_hca_cap; + int err; + + if (!MLX5_CAP_GEN_MAX(dev, hca_cap_2)) + return 0; + + err = mlx5_core_get_caps(dev, MLX5_CAP_GENERAL_2); + if (err) + return err; + + if (!MLX5_CAP_GEN_2_MAX(dev, sw_vhca_id_valid) || + !(dev->priv.sw_vhca_id > 0)) + return 0; + + set_hca_cap = MLX5_ADDR_OF(set_hca_cap_in, set_ctx, + capability); + memcpy(set_hca_cap, dev->caps.hca[MLX5_CAP_GENERAL_2]->cur, + MLX5_ST_SZ_BYTES(cmd_hca_cap_2)); + MLX5_SET(cmd_hca_cap_2, set_hca_cap, sw_vhca_id_valid, 1); + + return set_caps(dev, set_ctx, MLX5_CAP_GENERAL_2); +} + static int handle_hca_cap(struct mlx5_core_dev *dev, void *set_ctx) { struct mlx5_profile *prof = &dev->profile; @@ -669,6 +689,13 @@ static int set_hca_cap(struct mlx5_core_dev *dev) goto out; } + memset(set_ctx, 0, set_sz); + err = handle_hca_cap_2(dev, set_ctx); + if (err) { + mlx5_core_err(dev, "handle_hca_cap_2 failed\n"); + goto out; + } + out: kfree(set_ctx); return err; @@ -1255,6 +1282,7 @@ static void mlx5_unload(struct mlx5_core_dev *dev) { mlx5_sf_dev_table_destroy(dev); mlx5_sriov_detach(dev); + mlx5_eswitch_disable(dev->priv.eswitch); mlx5_lag_remove_mdev(dev); mlx5_ec_cleanup(dev); mlx5_sf_hw_table_destroy(dev); @@ -1274,8 +1302,10 @@ static void mlx5_unload(struct mlx5_core_dev *dev) int mlx5_init_one(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); int err = 0; + devl_lock(devlink); mutex_lock(&dev->intf_state_mutex); dev->state = MLX5_DEVICE_STATE_UP; @@ -1304,6 +1334,7 @@ int mlx5_init_one(struct mlx5_core_dev *dev) goto err_register; mutex_unlock(&dev->intf_state_mutex); + devl_unlock(devlink); return 0; err_register: @@ -1318,11 +1349,15 @@ function_teardown: err_function: dev->state = MLX5_DEVICE_STATE_INTERNAL_ERROR; mutex_unlock(&dev->intf_state_mutex); + devl_unlock(devlink); return err; } void mlx5_uninit_one(struct mlx5_core_dev *dev) { + struct devlink *devlink = priv_to_devlink(dev); + + devl_lock(devlink); mutex_lock(&dev->intf_state_mutex); mlx5_unregister_device(dev); @@ -1341,13 +1376,15 @@ void mlx5_uninit_one(struct mlx5_core_dev *dev) mlx5_function_teardown(dev, true); out: mutex_unlock(&dev->intf_state_mutex); + devl_unlock(devlink); } -int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) +int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery) { int err = 0; u64 timeout; + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&dev->intf_state_mutex); if (test_bit(MLX5_INTERFACE_STATE_UP, &dev->intf_state)) { mlx5_core_warn(dev, "interface is up, NOP\n"); @@ -1389,8 +1426,20 @@ out: return err; } -void mlx5_unload_one(struct mlx5_core_dev *dev) +int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery) +{ + struct devlink *devlink = priv_to_devlink(dev); + int ret; + + devl_lock(devlink); + ret = mlx5_load_one_devl_locked(dev, recovery); + devl_unlock(devlink); + return ret; +} + +void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev) { + devl_assert_locked(priv_to_devlink(dev)); mutex_lock(&dev->intf_state_mutex); mlx5_detach_device(dev); @@ -1408,6 +1457,15 @@ out: mutex_unlock(&dev->intf_state_mutex); } +void mlx5_unload_one(struct mlx5_core_dev *dev) +{ + struct devlink *devlink = priv_to_devlink(dev); + + devl_lock(devlink); + mlx5_unload_one_devl_locked(dev); + devl_unlock(devlink); +} + static const int types[] = { MLX5_CAP_GENERAL, MLX5_CAP_GENERAL_2, @@ -1510,6 +1568,18 @@ int mlx5_mdev_init(struct mlx5_core_dev *dev, int profile_idx) if (err) goto err_hca_caps; + /* The conjunction of sw_vhca_id with sw_owner_id will be a global + * unique id per function which uses mlx5_core. + * Those values are supplied to FW as part of the init HCA command to + * be used by both driver and FW when it's applicable. + */ + dev->priv.sw_vhca_id = ida_alloc_range(&sw_vhca_ida, 1, + MAX_SW_VHCA_ID, + GFP_KERNEL); + if (dev->priv.sw_vhca_id < 0) + mlx5_core_err(dev, "failed to allocate sw_vhca_id, err=%d\n", + dev->priv.sw_vhca_id); + return 0; err_hca_caps: @@ -1534,6 +1604,9 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev) { struct mlx5_priv *priv = &dev->priv; + if (priv->sw_vhca_id > 0) + ida_free(&sw_vhca_ida, dev->priv.sw_vhca_id); + mlx5_hca_caps_free(dev); mlx5_adev_cleanup(dev); mlx5_pagealloc_cleanup(dev); @@ -1857,7 +1930,7 @@ MODULE_DEVICE_TABLE(pci, mlx5_core_pci_table); void mlx5_disable_device(struct mlx5_core_dev *dev) { mlx5_error_sw_reset(dev); - mlx5_unload_one(dev); + mlx5_unload_one_devl_locked(dev); } int mlx5_recover_device(struct mlx5_core_dev *dev) @@ -1868,7 +1941,7 @@ int mlx5_recover_device(struct mlx5_core_dev *dev) return -EIO; } - return mlx5_load_one(dev, true); + return mlx5_load_one_devl_locked(dev, true); } static struct pci_driver mlx5_core_driver = { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h index 9cc7afea2758..ad61b86d5769 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/mlx5_core.h @@ -290,7 +290,9 @@ void mlx5_mdev_uninit(struct mlx5_core_dev *dev); int mlx5_init_one(struct mlx5_core_dev *dev); void mlx5_uninit_one(struct mlx5_core_dev *dev); void mlx5_unload_one(struct mlx5_core_dev *dev); +void mlx5_unload_one_devl_locked(struct mlx5_core_dev *dev); int mlx5_load_one(struct mlx5_core_dev *dev, bool recovery); +int mlx5_load_one_devl_locked(struct mlx5_core_dev *dev, bool recovery); int mlx5_vport_get_other_func_cap(struct mlx5_core_dev *dev, u16 function_id, void *out); diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c index 3be659cd91f1..7d955a4d9f14 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sf/devlink.c @@ -501,7 +501,7 @@ static int mlx5_sf_esw_event(struct notifier_block *nb, unsigned long event, voi case MLX5_ESWITCH_OFFLOADS: mlx5_sf_table_enable(table); break; - case MLX5_ESWITCH_NONE: + case MLX5_ESWITCH_LEGACY: mlx5_sf_table_disable(table); break; default: diff --git a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c index 2935614f6fa9..ee2e1b7c1310 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/sriov.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/sriov.c @@ -145,8 +145,7 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf) sriov->vfs_ctx[vf].enabled = 0; } - if (MLX5_ESWITCH_MANAGER(dev)) - mlx5_eswitch_disable(dev->priv.eswitch, clear_vf); + mlx5_eswitch_disable_sriov(dev->priv.eswitch, clear_vf); if (mlx5_wait_for_pages(dev, &dev->priv.vfs_pages)) mlx5_core_warn(dev, "timeout reclaiming VFs pages\n"); @@ -155,13 +154,16 @@ mlx5_device_disable_sriov(struct mlx5_core_dev *dev, int num_vfs, bool clear_vf) static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); + struct devlink *devlink = priv_to_devlink(dev); int err; + devl_lock(devlink); err = mlx5_device_enable_sriov(dev, num_vfs); if (err) { mlx5_core_warn(dev, "mlx5_device_enable_sriov failed : %d\n", err); return err; } + devl_unlock(devlink); err = pci_enable_sriov(pdev, num_vfs); if (err) { @@ -174,10 +176,13 @@ static int mlx5_sriov_enable(struct pci_dev *pdev, int num_vfs) void mlx5_sriov_disable(struct pci_dev *pdev) { struct mlx5_core_dev *dev = pci_get_drvdata(pdev); + struct devlink *devlink = priv_to_devlink(dev); int num_vfs = pci_num_vf(dev->pdev); pci_disable_sriov(pdev); + devl_lock(devlink); mlx5_device_disable_sriov(dev, num_vfs, true); + devl_unlock(devlink); } int mlx5_core_sriov_configure(struct pci_dev *pdev, int num_vfs) diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c index 1383550f44c1..b1dfad274a39 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_action.c @@ -22,6 +22,7 @@ enum dr_action_valid_state { DR_ACTION_STATE_PUSH_VLAN, DR_ACTION_STATE_NON_TERM, DR_ACTION_STATE_TERM, + DR_ACTION_STATE_ASO, DR_ACTION_STATE_MAX, }; @@ -42,6 +43,7 @@ static const char * const action_type_to_str[] = { [DR_ACTION_TYP_SAMPLER] = "DR_ACTION_TYP_SAMPLER", [DR_ACTION_TYP_INSERT_HDR] = "DR_ACTION_TYP_INSERT_HDR", [DR_ACTION_TYP_REMOVE_HDR] = "DR_ACTION_TYP_REMOVE_HDR", + [DR_ACTION_TYP_ASO_FLOW_METER] = "DR_ACTION_TYP_ASO_FLOW_METER", [DR_ACTION_TYP_MAX] = "DR_ACTION_UNKNOWN", }; @@ -71,6 +73,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -85,6 +88,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -93,6 +97,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_TAG] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -105,6 +110,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -118,6 +124,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, @@ -128,6 +135,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -145,6 +153,13 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, + }, + [DR_ACTION_STATE_ASO] = { + [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_QP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_TERM] = { [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_TERM, @@ -163,18 +178,21 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -185,6 +203,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, @@ -196,6 +215,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -206,6 +226,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -219,6 +240,16 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, + }, + [DR_ACTION_STATE_ASO] = { + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, + [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, + [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, }, [DR_ACTION_STATE_TERM] = { [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_TERM, @@ -240,6 +271,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -253,6 +285,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -261,6 +294,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -272,6 +306,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -284,6 +319,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -296,6 +332,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -312,6 +349,13 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, + }, + [DR_ACTION_STATE_ASO] = { + [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_TERM] = { [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_TERM, @@ -331,6 +375,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_DECAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -338,6 +383,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_DECAP, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_ENCAP] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -345,6 +391,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_SAMPLER] = DR_ACTION_STATE_TERM, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_MODIFY_HDR] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -356,6 +403,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_POP_VLAN] = { [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, @@ -368,6 +416,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_PUSH_VLAN] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -379,6 +428,7 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_INSERT_HDR] = DR_ACTION_STATE_ENCAP, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_NON_TERM] = { [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, @@ -393,6 +443,17 @@ next_action_state[DR_ACTION_DOMAIN_MAX][DR_ACTION_STATE_MAX][DR_ACTION_TYP_MAX] [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, [DR_ACTION_TYP_POP_VLAN] = DR_ACTION_STATE_POP_VLAN, [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_ASO_FLOW_METER] = DR_ACTION_STATE_ASO, + }, + [DR_ACTION_STATE_ASO] = { + [DR_ACTION_TYP_L2_TO_TNL_L2] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_L2_TO_TNL_L3] = DR_ACTION_STATE_ENCAP, + [DR_ACTION_TYP_MODIFY_HDR] = DR_ACTION_STATE_MODIFY_HDR, + [DR_ACTION_TYP_PUSH_VLAN] = DR_ACTION_STATE_PUSH_VLAN, + [DR_ACTION_TYP_DROP] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_FT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_VPORT] = DR_ACTION_STATE_TERM, + [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_ASO, }, [DR_ACTION_STATE_TERM] = { [DR_ACTION_TYP_CTR] = DR_ACTION_STATE_TERM, @@ -738,6 +799,12 @@ int mlx5dr_actions_build_ste_arr(struct mlx5dr_matcher *matcher, attr.reformat.param_0 = action->reformat->param_0; attr.reformat.param_1 = action->reformat->param_1; break; + case DR_ACTION_TYP_ASO_FLOW_METER: + attr.aso_flow_meter.obj_id = action->aso->obj_id; + attr.aso_flow_meter.offset = action->aso->offset; + attr.aso_flow_meter.dest_reg_id = action->aso->dest_reg_id; + attr.aso_flow_meter.init_color = action->aso->init_color; + break; default: mlx5dr_err(dmn, "Unsupported action type %d\n", action_type); return -EINVAL; @@ -798,6 +865,7 @@ static unsigned int action_size[DR_ACTION_TYP_MAX] = { [DR_ACTION_TYP_INSERT_HDR] = sizeof(struct mlx5dr_action_reformat), [DR_ACTION_TYP_REMOVE_HDR] = sizeof(struct mlx5dr_action_reformat), [DR_ACTION_TYP_SAMPLER] = sizeof(struct mlx5dr_action_sampler), + [DR_ACTION_TYP_ASO_FLOW_METER] = sizeof(struct mlx5dr_action_aso_flow_meter), }; static struct mlx5dr_action * @@ -1830,6 +1898,34 @@ mlx5dr_action_create_dest_vport(struct mlx5dr_domain *dmn, return action; } +struct mlx5dr_action * +mlx5dr_action_create_aso(struct mlx5dr_domain *dmn, u32 obj_id, + u8 dest_reg_id, u8 aso_type, + u8 init_color, u8 meter_id) +{ + struct mlx5dr_action *action; + + if (aso_type != MLX5_EXE_ASO_FLOW_METER) + return NULL; + + if (init_color > MLX5_FLOW_METER_COLOR_UNDEFINED) + return NULL; + + action = dr_action_create_generic(DR_ACTION_TYP_ASO_FLOW_METER); + if (!action) + return NULL; + + action->aso->obj_id = obj_id; + action->aso->offset = meter_id; + action->aso->dest_reg_id = dest_reg_id; + action->aso->init_color = init_color; + action->aso->dmn = dmn; + + refcount_inc(&dmn->refcount); + + return action; +} + int mlx5dr_action_destroy(struct mlx5dr_action *action) { if (WARN_ON_ONCE(refcount_read(&action->refcount) > 1)) @@ -1881,6 +1977,9 @@ int mlx5dr_action_destroy(struct mlx5dr_action *action) case DR_ACTION_TYP_SAMPLER: refcount_dec(&action->sampler->dmn->refcount); break; + case DR_ACTION_TYP_ASO_FLOW_METER: + refcount_dec(&action->aso->dmn->refcount); + break; default: break; } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c index fcb962c6db2e..ee677a5c76be 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_ste_v1.c @@ -89,6 +89,7 @@ enum dr_ste_v1_action_id { DR_STE_V1_ACTION_ID_QUEUE_ID_SEL = 0x0d, DR_STE_V1_ACTION_ID_ACCELERATED_LIST = 0x0e, DR_STE_V1_ACTION_ID_MODIFY_LIST = 0x0f, + DR_STE_V1_ACTION_ID_ASO = 0x12, DR_STE_V1_ACTION_ID_TRAILER = 0x13, DR_STE_V1_ACTION_ID_COUNTER_ID = 0x14, DR_STE_V1_ACTION_ID_MAX = 0x21, @@ -129,6 +130,10 @@ enum { DR_STE_V1_ACTION_MDFY_FLD_REGISTER_0_1 = 0x91, }; +enum dr_ste_v1_aso_ctx_type { + DR_STE_V1_ASO_CTX_TYPE_POLICERS = 0x2, +}; + static const struct mlx5dr_ste_action_modify_field dr_ste_v1_action_modify_field_arr[] = { [MLX5_ACTION_IN_FIELD_OUT_SMAC_47_16] = { .hw_field = DR_STE_V1_ACTION_MDFY_FLD_SRC_L2_OUT_0, .start = 0, .end = 31, @@ -494,6 +499,27 @@ static void dr_ste_v1_set_rewrite_actions(u8 *hw_ste_p, dr_ste_v1_set_reparse(hw_ste_p); } +static void dr_ste_v1_set_aso_flow_meter(u8 *d_action, + u32 object_id, + u32 offset, + u8 dest_reg_id, + u8 init_color) +{ + MLX5_SET(ste_double_action_aso_v1, d_action, action_id, + DR_STE_V1_ACTION_ID_ASO); + MLX5_SET(ste_double_action_aso_v1, d_action, aso_context_number, + object_id + (offset / MLX5DR_ASO_FLOW_METER_NUM_PER_OBJ)); + /* Convert reg_c index to HW 64bit index */ + MLX5_SET(ste_double_action_aso_v1, d_action, dest_reg_id, + (dest_reg_id - 1) / 2); + MLX5_SET(ste_double_action_aso_v1, d_action, aso_context_type, + DR_STE_V1_ASO_CTX_TYPE_POLICERS); + MLX5_SET(ste_double_action_aso_v1, d_action, flow_meter.line_id, + offset % MLX5DR_ASO_FLOW_METER_NUM_PER_OBJ); + MLX5_SET(ste_double_action_aso_v1, d_action, flow_meter.initial_color, + init_color); +} + static void dr_ste_v1_arr_init_next_match(u8 **last_ste, u32 *added_stes, u16 gvmi) @@ -629,6 +655,21 @@ void dr_ste_v1_set_actions_tx(struct mlx5dr_domain *dmn, action += DR_STE_ACTION_SINGLE_SZ; } + if (action_type_set[DR_ACTION_TYP_ASO_FLOW_METER]) { + if (action_sz < DR_STE_ACTION_DOUBLE_SZ) { + dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); + action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); + action_sz = DR_STE_ACTION_TRIPLE_SZ; + } + dr_ste_v1_set_aso_flow_meter(action, + attr->aso_flow_meter.obj_id, + attr->aso_flow_meter.offset, + attr->aso_flow_meter.dest_reg_id, + attr->aso_flow_meter.init_color); + action_sz -= DR_STE_ACTION_DOUBLE_SZ; + action += DR_STE_ACTION_DOUBLE_SZ; + } + dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi); dr_ste_v1_set_hit_addr(last_ste, attr->final_icm_addr, 1); } @@ -802,6 +843,21 @@ void dr_ste_v1_set_actions_rx(struct mlx5dr_domain *dmn, action += DR_STE_ACTION_SINGLE_SZ; } + if (action_type_set[DR_ACTION_TYP_ASO_FLOW_METER]) { + if (action_sz < DR_STE_ACTION_DOUBLE_SZ) { + dr_ste_v1_arr_init_next_match(&last_ste, added_stes, attr->gvmi); + action = MLX5_ADDR_OF(ste_mask_and_match_v1, last_ste, action); + action_sz = DR_STE_ACTION_TRIPLE_SZ; + } + dr_ste_v1_set_aso_flow_meter(action, + attr->aso_flow_meter.obj_id, + attr->aso_flow_meter.offset, + attr->aso_flow_meter.dest_reg_id, + attr->aso_flow_meter.init_color); + action_sz -= DR_STE_ACTION_DOUBLE_SZ; + action += DR_STE_ACTION_DOUBLE_SZ; + } + dr_ste_v1_set_hit_gvmi(last_ste, attr->hit_gvmi); dr_ste_v1_set_hit_addr(last_ste, attr->final_icm_addr, 1); } diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h index 98320e3945ad..feed227944b7 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/dr_types.h @@ -127,6 +127,7 @@ enum mlx5dr_action_type { DR_ACTION_TYP_INSERT_HDR, DR_ACTION_TYP_REMOVE_HDR, DR_ACTION_TYP_SAMPLER, + DR_ACTION_TYP_ASO_FLOW_METER, DR_ACTION_TYP_MAX, }; @@ -271,6 +272,13 @@ struct mlx5dr_ste_actions_attr { int count; u32 headers[MLX5DR_MAX_VLANS]; } vlans; + + struct { + u32 obj_id; + u32 offset; + u8 dest_reg_id; + u8 init_color; + } aso_flow_meter; }; void mlx5dr_ste_set_actions_rx(struct mlx5dr_ste_ctx *ste_ctx, @@ -1035,6 +1043,14 @@ struct mlx5dr_rule_action_member { struct list_head list; }; +struct mlx5dr_action_aso_flow_meter { + struct mlx5dr_domain *dmn; + u32 obj_id; + u32 offset; + u8 dest_reg_id; + u8 init_color; +}; + struct mlx5dr_action { enum mlx5dr_action_type action_type; refcount_t refcount; @@ -1049,6 +1065,7 @@ struct mlx5dr_action { struct mlx5dr_action_vport *vport; struct mlx5dr_action_push_vlan *push_vlan; struct mlx5dr_action_flow_tag *flow_tag; + struct mlx5dr_action_aso_flow_meter *aso; }; }; diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c index 6a9abba92df6..75672663359d 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/fs_dr.c @@ -500,6 +500,27 @@ static int mlx5_cmd_dr_create_fte(struct mlx5_flow_root_namespace *ns, } } + if (fte->action.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) { + if (fte->action.exe_aso.type != MLX5_EXE_ASO_FLOW_METER) { + err = -EOPNOTSUPP; + goto free_actions; + } + + tmp_action = + mlx5dr_action_create_aso(domain, + fte->action.exe_aso.object_id, + fte->action.exe_aso.return_reg_id, + fte->action.exe_aso.type, + fte->action.exe_aso.flow_meter.init_color, + fte->action.exe_aso.flow_meter.meter_idx); + if (!tmp_action) { + err = -ENOMEM; + goto free_actions; + } + fs_dr_actions[fs_dr_num_actions++] = tmp_action; + actions[num_actions++] = tmp_action; + } + params.match_sz = match_sz; params.match_buf = (u64 *)fte->val; if (num_term_actions == 1) { diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h index 9604b2091358..fb078fa0f0cc 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5_ifc_dr.h @@ -574,4 +574,30 @@ struct mlx5_ifc_dr_action_hw_copy_bits { u8 reserved_at_38[0x8]; }; +enum { + MLX5DR_ASO_FLOW_METER_NUM_PER_OBJ = 2, +}; + +struct mlx5_ifc_ste_aso_flow_meter_action_bits { + u8 reserved_at_0[0xc]; + u8 action[0x1]; + u8 initial_color[0x2]; + u8 line_id[0x1]; +}; + +struct mlx5_ifc_ste_double_action_aso_v1_bits { + u8 action_id[0x8]; + u8 aso_context_number[0x18]; + + u8 dest_reg_id[0x2]; + u8 change_ordering_tag[0x1]; + u8 aso_check_ordering[0x1]; + u8 aso_context_type[0x4]; + u8 reserved_at_28[0x8]; + union { + u8 aso_fields[0x10]; + struct mlx5_ifc_ste_aso_flow_meter_action_bits flow_meter; + }; +}; + #endif /* MLX5_IFC_DR_H */ diff --git a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h index 7626c85643b1..ecffc2cbe6fe 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h +++ b/drivers/net/ethernet/mellanox/mlx5/core/steering/mlx5dr.h @@ -131,6 +131,14 @@ struct mlx5dr_action *mlx5dr_action_create_pop_vlan(void); struct mlx5dr_action * mlx5dr_action_create_push_vlan(struct mlx5dr_domain *domain, __be32 vlan_hdr); +struct mlx5dr_action * +mlx5dr_action_create_aso(struct mlx5dr_domain *dmn, + u32 obj_id, + u8 return_reg_id, + u8 aso_type, + u8 init_color, + u8 meter_id); + int mlx5dr_action_destroy(struct mlx5dr_action *action); static inline bool diff --git a/drivers/net/ethernet/mellanox/mlx5/core/vport.c b/drivers/net/ethernet/mellanox/mlx5/core/vport.c index ac020cb78072..d5c317325030 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/vport.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/vport.c @@ -1086,9 +1086,17 @@ int mlx5_nic_vport_affiliate_multiport(struct mlx5_core_dev *master_mdev, goto free; MLX5_SET(modify_nic_vport_context_in, in, field_select.affiliation, 1); - MLX5_SET(modify_nic_vport_context_in, in, - nic_vport_context.affiliated_vhca_id, - MLX5_CAP_GEN(master_mdev, vhca_id)); + if (MLX5_CAP_GEN_2(master_mdev, sw_vhca_id_valid)) { + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.vhca_id_type, VHCA_ID_TYPE_SW); + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.affiliated_vhca_id, + MLX5_CAP_GEN_2(master_mdev, sw_vhca_id)); + } else { + MLX5_SET(modify_nic_vport_context_in, in, + nic_vport_context.affiliated_vhca_id, + MLX5_CAP_GEN(master_mdev, vhca_id)); + } MLX5_SET(modify_nic_vport_context_in, in, nic_vport_context.affiliation_criteria, MLX5_CAP_GEN(port_mdev, affiliate_nic_vport_criteria)); |