diff options
Diffstat (limited to 'drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c')
| -rw-r--r-- | drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c | 1069 |
1 files changed, 764 insertions, 305 deletions
diff --git a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c index b859e4a4c744..9f19f4b59a70 100644 --- a/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c +++ b/drivers/net/ethernet/mellanox/mlx5/core/en_accel/ipsec_fs.c @@ -9,53 +9,67 @@ #define NUM_IPSEC_FTE BIT(15) -enum accel_fs_esp_type { - ACCEL_FS_ESP4, - ACCEL_FS_ESP6, - ACCEL_FS_ESP_NUM_TYPES, +struct mlx5e_ipsec_fc { + struct mlx5_fc *cnt; + struct mlx5_fc *drop; }; -struct mlx5e_ipsec_rx_err { - struct mlx5_flow_table *ft; - struct mlx5_flow_handle *rule; - struct mlx5_modify_hdr *copy_modify_hdr; +struct mlx5e_ipsec_ft { + struct mutex mutex; /* Protect changes to this struct */ + struct mlx5_flow_table *pol; + struct mlx5_flow_table *sa; + struct mlx5_flow_table *status; + u32 refcnt; }; -struct mlx5e_accel_fs_esp_prot { - struct mlx5_flow_table *ft; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; - struct mlx5_flow_destination default_dest; - struct mlx5e_ipsec_rx_err rx_err; - u32 refcnt; - struct mutex prot_mutex; /* protect ESP4/ESP6 protocol */ +struct mlx5e_ipsec_miss { + struct mlx5_flow_group *group; + struct mlx5_flow_handle *rule; }; -struct mlx5e_accel_fs_esp { - struct mlx5e_accel_fs_esp_prot fs_prot[ACCEL_FS_ESP_NUM_TYPES]; +struct mlx5e_ipsec_rx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; + struct mlx5e_ipsec_miss sa; + struct mlx5e_ipsec_rule status; + struct mlx5e_ipsec_fc *fc; }; struct mlx5e_ipsec_tx { + struct mlx5e_ipsec_ft ft; + struct mlx5e_ipsec_miss pol; struct mlx5_flow_namespace *ns; - struct mlx5_flow_table *ft; - struct mutex mutex; /* Protect IPsec TX steering */ - u32 refcnt; + struct mlx5e_ipsec_fc *fc; }; /* IPsec RX flow steering */ -static enum mlx5_traffic_types fs_esp2tt(enum accel_fs_esp_type i) +static enum mlx5_traffic_types family2tt(u32 family) { - if (i == ACCEL_FS_ESP4) + if (family == AF_INET) return MLX5_TT_IPV4_IPSEC_ESP; return MLX5_TT_IPV6_IPSEC_ESP; } -static int rx_err_add_rule(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot, - struct mlx5e_ipsec_rx_err *rx_err) +static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, + int level, int prio, + int max_num_groups) +{ + struct mlx5_flow_table_attr ft_attr = {}; + + ft_attr.autogroup.num_reserved_entries = 1; + ft_attr.autogroup.max_num_groups = max_num_groups; + ft_attr.max_fte = NUM_IPSEC_FTE; + ft_attr.level = level; + ft_attr.prio = prio; + + return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); +} + +static int ipsec_status_rule(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec_rx *rx, + struct mlx5_flow_destination *dest) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - struct mlx5_core_dev *mdev = priv->mdev; struct mlx5_flow_act flow_act = {}; struct mlx5_modify_hdr *modify_hdr; struct mlx5_flow_handle *fte; @@ -79,26 +93,26 @@ static int rx_err_add_rule(struct mlx5e_priv *priv, if (IS_ERR(modify_hdr)) { err = PTR_ERR(modify_hdr); - netdev_err(priv->netdev, - "fail to alloc ipsec copy modify_header_id err=%d\n", err); + mlx5_core_err(mdev, + "fail to alloc ipsec copy modify_header_id err=%d\n", err); goto out_spec; } /* create fte */ flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | - MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_COUNT; flow_act.modify_hdr = modify_hdr; - fte = mlx5_add_flow_rules(rx_err->ft, spec, &flow_act, - &fs_prot->default_dest, 1); + fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); if (IS_ERR(fte)) { err = PTR_ERR(fte); - netdev_err(priv->netdev, "fail to add ipsec rx err copy rule err=%d\n", err); + mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); goto out; } kvfree(spec); - rx_err->rule = fte; - rx_err->copy_modify_hdr = modify_hdr; + rx->status.rule = fte; + rx->status.modify_hdr = modify_hdr; return 0; out: @@ -108,13 +122,12 @@ out_spec: return err; } -static int rx_fs_create(struct mlx5e_priv *priv, - struct mlx5e_accel_fs_esp_prot *fs_prot) +static int ipsec_miss_create(struct mlx5_core_dev *mdev, + struct mlx5_flow_table *ft, + struct mlx5e_ipsec_miss *miss, + struct mlx5_flow_destination *dest) { int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); - struct mlx5_flow_table *ft = fs_prot->ft; - struct mlx5_flow_group *miss_group; - struct mlx5_flow_handle *miss_rule; MLX5_DECLARE_FLOW_ACT(flow_act); struct mlx5_flow_spec *spec; u32 *flow_group_in; @@ -130,450 +143,888 @@ static int rx_fs_create(struct mlx5e_priv *priv, /* Create miss_group */ MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); - miss_group = mlx5_create_flow_group(ft, flow_group_in); - if (IS_ERR(miss_group)) { - err = PTR_ERR(miss_group); - netdev_err(priv->netdev, "fail to create ipsec rx miss_group err=%d\n", err); + miss->group = mlx5_create_flow_group(ft, flow_group_in); + if (IS_ERR(miss->group)) { + err = PTR_ERR(miss->group); + mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", + err); goto out; } - fs_prot->miss_group = miss_group; /* Create miss rule */ - miss_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &fs_prot->default_dest, 1); - if (IS_ERR(miss_rule)) { - mlx5_destroy_flow_group(fs_prot->miss_group); - err = PTR_ERR(miss_rule); - netdev_err(priv->netdev, "fail to create ipsec rx miss_rule err=%d\n", err); + miss->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); + if (IS_ERR(miss->rule)) { + mlx5_destroy_flow_group(miss->group); + err = PTR_ERR(miss->rule); + mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", + err); goto out; } - fs_prot->miss_rule = miss_rule; out: kvfree(flow_group_in); kvfree(spec); return err; } -static void rx_destroy(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_rx *rx) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; - - accel_esp = priv->ipsec->rx_fs; - - /* The netdev unreg already happened, so all offloaded rule are already removed */ - fs_prot = &accel_esp->fs_prot[type]; + mlx5_del_flow_rules(rx->pol.rule); + mlx5_destroy_flow_group(rx->pol.group); + mlx5_destroy_flow_table(rx->ft.pol); - mlx5_del_flow_rules(fs_prot->miss_rule); - mlx5_destroy_flow_group(fs_prot->miss_group); - mlx5_destroy_flow_table(fs_prot->ft); + mlx5_del_flow_rules(rx->sa.rule); + mlx5_destroy_flow_group(rx->sa.group); + mlx5_destroy_flow_table(rx->ft.sa); - mlx5_del_flow_rules(fs_prot->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, fs_prot->rx_err.copy_modify_hdr); - mlx5_destroy_flow_table(fs_prot->rx_err.ft); + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); + mlx5_destroy_flow_table(rx->ft.status); } -static int rx_create(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + struct mlx5e_ipsec_rx *rx, u32 family) { - struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(priv->fs, false); - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + struct mlx5_flow_destination dest[2]; struct mlx5_flow_table *ft; int err; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - fs_prot->default_dest = - mlx5_ttc_get_default_dest(ttc, fs_esp2tt(type)); - - ft_attr.max_fte = 1; - ft_attr.autogroup.max_num_groups = 1; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL, + MLX5E_NIC_PRIO, 1); if (IS_ERR(ft)) return PTR_ERR(ft); - fs_prot->rx_err.ft = ft; - err = rx_err_add_rule(priv, fs_prot, &fs_prot->rx_err); + rx->ft.status = ft; + + dest[0] = mlx5_ttc_get_default_dest(ttc, family2tt(family)); + dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); + err = ipsec_status_rule(mdev, rx, dest); if (err) goto err_add; /* Create FT */ - ft_attr.max_fte = NUM_IPSEC_FTE; - ft_attr.level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; - ft_attr.prio = MLX5E_NIC_PRIO; - ft_attr.autogroup.num_reserved_entries = 1; - ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ns, &ft_attr); + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_ESP_FT_LEVEL, MLX5E_NIC_PRIO, + 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); goto err_fs_ft; } - fs_prot->ft = ft; + rx->ft.sa = ft; - err = rx_fs_create(priv, fs_prot); + err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, dest); if (err) goto err_fs; + ft = ipsec_ft_create(ns, MLX5E_ACCEL_FS_POL_FT_LEVEL, MLX5E_NIC_PRIO, + 2); + if (IS_ERR(ft)) { + err = PTR_ERR(ft); + goto err_pol_ft; + } + rx->ft.pol = ft; + memset(dest, 0x00, 2 * sizeof(*dest)); + dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[0].ft = rx->ft.sa; + err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, dest); + if (err) + goto err_pol_miss; + return 0; +err_pol_miss: + mlx5_destroy_flow_table(rx->ft.pol); +err_pol_ft: + mlx5_del_flow_rules(rx->sa.rule); + mlx5_destroy_flow_group(rx->sa.group); err_fs: - mlx5_destroy_flow_table(fs_prot->ft); + mlx5_destroy_flow_table(rx->ft.sa); err_fs_ft: - mlx5_del_flow_rules(fs_prot->rx_err.rule); - mlx5_modify_header_dealloc(priv->mdev, fs_prot->rx_err.copy_modify_hdr); + mlx5_del_flow_rules(rx->status.rule); + mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); err_add: - mlx5_destroy_flow_table(fs_prot->rx_err.ft); + mlx5_destroy_flow_table(rx->ft.status); return err; } -static int rx_ft_get(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec *ipsec, u32 family) { - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5e_ipsec_rx *rx; int err = 0; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - if (fs_prot->refcnt) + if (family == AF_INET) + rx = ipsec->rx_ipv4; + else + rx = ipsec->rx_ipv6; + + mutex_lock(&rx->ft.mutex); + if (rx->ft.refcnt) goto skip; /* create FT */ - err = rx_create(priv, type); + err = rx_create(mdev, ipsec, rx, family); if (err) goto out; /* connect */ dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - dest.ft = fs_prot->ft; - mlx5_ttc_fwd_dest(ttc, fs_esp2tt(type), &dest); + dest.ft = rx->ft.pol; + mlx5_ttc_fwd_dest(ttc, family2tt(family), &dest); skip: - fs_prot->refcnt++; + rx->ft.refcnt++; out: - mutex_unlock(&fs_prot->prot_mutex); - return err; + mutex_unlock(&rx->ft.mutex); + if (err) + return ERR_PTR(err); + return rx; } -static void rx_ft_put(struct mlx5e_priv *priv, enum accel_fs_esp_type type) +static void rx_ft_put(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, + u32 family) { - struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(priv->fs, false); - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; + struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); + struct mlx5e_ipsec_rx *rx; + + if (family == AF_INET) + rx = ipsec->rx_ipv4; + else + rx = ipsec->rx_ipv6; - accel_esp = priv->ipsec->rx_fs; - fs_prot = &accel_esp->fs_prot[type]; - mutex_lock(&fs_prot->prot_mutex); - fs_prot->refcnt--; - if (fs_prot->refcnt) + mutex_lock(&rx->ft.mutex); + rx->ft.refcnt--; + if (rx->ft.refcnt) goto out; /* disconnect */ - mlx5_ttc_fwd_default_dest(ttc, fs_esp2tt(type)); + mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); /* remove FT */ - rx_destroy(priv, type); + rx_destroy(mdev, rx); out: - mutex_unlock(&fs_prot->prot_mutex); + mutex_unlock(&rx->ft.mutex); } /* IPsec TX flow steering */ -static int tx_create(struct mlx5e_priv *priv) +static int tx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) { - struct mlx5_flow_table_attr ft_attr = {}; - struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5_flow_destination dest = {}; struct mlx5_flow_table *ft; int err; - ft_attr.max_fte = NUM_IPSEC_FTE; - ft_attr.autogroup.max_num_groups = 1; - ft = mlx5_create_auto_grouped_flow_table(ipsec->tx_fs->ns, &ft_attr); + ft = ipsec_ft_create(tx->ns, 1, 0, 4); + if (IS_ERR(ft)) + return PTR_ERR(ft); + + tx->ft.sa = ft; + + ft = ipsec_ft_create(tx->ns, 0, 0, 2); if (IS_ERR(ft)) { err = PTR_ERR(ft); - netdev_err(priv->netdev, "fail to create ipsec tx ft err=%d\n", err); - return err; + goto err_pol_ft; } - ipsec->tx_fs->ft = ft; + tx->ft.pol = ft; + dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest.ft = tx->ft.sa; + err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); + if (err) + goto err_pol_miss; return 0; + +err_pol_miss: + mlx5_destroy_flow_table(tx->ft.pol); +err_pol_ft: + mlx5_destroy_flow_table(tx->ft.sa); + return err; } -static int tx_ft_get(struct mlx5e_priv *priv) +static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, + struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_tx *tx_fs = priv->ipsec->tx_fs; + struct mlx5e_ipsec_tx *tx = ipsec->tx; int err = 0; - mutex_lock(&tx_fs->mutex); - if (tx_fs->refcnt) + mutex_lock(&tx->ft.mutex); + if (tx->ft.refcnt) goto skip; - err = tx_create(priv); + err = tx_create(mdev, tx); if (err) goto out; skip: - tx_fs->refcnt++; + tx->ft.refcnt++; out: - mutex_unlock(&tx_fs->mutex); - return err; + mutex_unlock(&tx->ft.mutex); + if (err) + return ERR_PTR(err); + return tx; } -static void tx_ft_put(struct mlx5e_priv *priv) +static void tx_ft_put(struct mlx5e_ipsec *ipsec) { - struct mlx5e_ipsec_tx *tx_fs = priv->ipsec->tx_fs; + struct mlx5e_ipsec_tx *tx = ipsec->tx; - mutex_lock(&tx_fs->mutex); - tx_fs->refcnt--; - if (tx_fs->refcnt) + mutex_lock(&tx->ft.mutex); + tx->ft.refcnt--; + if (tx->ft.refcnt) goto out; - mlx5_destroy_flow_table(tx_fs->ft); + mlx5_del_flow_rules(tx->pol.rule); + mlx5_destroy_flow_group(tx->pol.group); + mlx5_destroy_flow_table(tx->ft.pol); + mlx5_destroy_flow_table(tx->ft.sa); out: - mutex_unlock(&tx_fs->mutex); + mutex_unlock(&tx->ft.mutex); } -static void setup_fte_common(struct mlx5_accel_esp_xfrm_attrs *attrs, - u32 ipsec_obj_id, - struct mlx5_flow_spec *spec, - struct mlx5_flow_act *flow_act) +static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, + __be32 *daddr) { - u8 ip_version = attrs->is_ipv6 ? 6 : 4; - - spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS | MLX5_MATCH_MISC_PARAMETERS; + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; - /* ip_version */ MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); + + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); +} - /* Non fragmented */ - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); - MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); +static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, + __be32 *daddr) +{ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); + + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); + memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); + memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, + outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); +} + +static void setup_fte_esp(struct mlx5_flow_spec *spec) +{ /* ESP header */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); +} +static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi) +{ /* SPI number */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); + MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); +} + +static void setup_fte_no_frags(struct mlx5_flow_spec *spec) +{ + /* Non fragmented */ + spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; + + MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); + MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); +} + +static void setup_fte_reg_a(struct mlx5_flow_spec *spec) +{ + /* Add IPsec indicator in metadata_reg_a */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); MLX5_SET(fte_match_param, spec->match_value, - misc_parameters.outer_esp_spi, attrs->spi); - - if (ip_version == 4) { - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), - &attrs->saddr.a4, 4); - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), - &attrs->daddr.a4, 4); - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); - MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); - } else { - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), - &attrs->saddr.a6, 16); - memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), - &attrs->daddr.a6, 16); - memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), - 0xff, 16); - memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, - outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), - 0xff, 16); - } + misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); +} - flow_act->crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; - flow_act->crypto.obj_id = ipsec_obj_id; - flow_act->flags |= FLOW_ACT_NO_APPEND; +static void setup_fte_reg_c0(struct mlx5_flow_spec *spec, u32 reqid) +{ + /* Pass policy check before choosing this SA */ + spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; + + MLX5_SET(fte_match_param, spec->match_criteria, + misc_parameters_2.metadata_reg_c_0, reqid); + MLX5_SET(fte_match_param, spec->match_value, + misc_parameters_2.metadata_reg_c_0, reqid); } -static int rx_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int setup_modify_header(struct mlx5_core_dev *mdev, u32 val, u8 dir, + struct mlx5_flow_act *flow_act) { u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; - struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; + enum mlx5_flow_namespace_type ns_type; + struct mlx5_modify_hdr *modify_hdr; + + MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); + switch (dir) { + case XFRM_DEV_OFFLOAD_IN: + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_B); + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + break; + case XFRM_DEV_OFFLOAD_OUT: + MLX5_SET(set_action_in, action, field, + MLX5_ACTION_IN_FIELD_METADATA_REG_C_0); + ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + break; + default: + return -EINVAL; + } + + MLX5_SET(set_action_in, action, data, val); + MLX5_SET(set_action_in, action, offset, 0); + MLX5_SET(set_action_in, action, length, 32); + + modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); + if (IS_ERR(modify_hdr)) { + mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", + PTR_ERR(modify_hdr)); + return PTR_ERR(modify_hdr); + } + + flow_act->modify_hdr = modify_hdr; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + return 0; +} + +static int setup_pkt_reformat(struct mlx5_core_dev *mdev, + struct mlx5_accel_esp_xfrm_attrs *attrs, + struct mlx5_flow_act *flow_act) +{ + enum mlx5_flow_namespace_type ns_type = MLX5_FLOW_NAMESPACE_EGRESS; + struct mlx5_pkt_reformat_params reformat_params = {}; + struct mlx5_pkt_reformat *pkt_reformat; + u8 reformatbf[16] = {}; + __be32 spi; + + if (attrs->dir == XFRM_DEV_OFFLOAD_IN) { + reformat_params.type = MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; + ns_type = MLX5_FLOW_NAMESPACE_KERNEL; + goto cmd; + } + + if (attrs->family == AF_INET) + reformat_params.type = + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; + else + reformat_params.type = + MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; + + /* convert to network format */ + spi = htonl(attrs->spi); + memcpy(reformatbf, &spi, 4); + + reformat_params.param_0 = attrs->authsize; + reformat_params.size = sizeof(reformatbf); + reformat_params.data = &reformatbf; + +cmd: + pkt_reformat = + mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); + if (IS_ERR(pkt_reformat)) + return PTR_ERR(pkt_reformat); + + flow_act->pkt_reformat = pkt_reformat; + flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; + return 0; +} + +static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) +{ struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; - u32 ipsec_obj_id = sa_entry->ipsec_obj_id; - struct mlx5_modify_hdr *modify_hdr = NULL; - struct mlx5e_accel_fs_esp_prot *fs_prot; + struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; struct mlx5_flow_destination dest = {}; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; - enum accel_fs_esp_type type; struct mlx5_flow_spec *spec; - int err = 0; + struct mlx5e_ipsec_rx *rx; + int err; - accel_esp = priv->ipsec->rx_fs; - type = attrs->is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4; - fs_prot = &accel_esp->fs_prot[type]; - - err = rx_ft_get(priv, type); - if (err) - return err; + rx = rx_ft_get(mdev, ipsec, attrs->family); + if (IS_ERR(rx)) + return PTR_ERR(rx); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { err = -ENOMEM; - goto out_err; + goto err_alloc; } - setup_fte_common(attrs, ipsec_obj_id, spec, &flow_act); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); - /* Set bit[31] ipsec marker */ - /* Set bit[23-0] ipsec_obj_id */ - MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); - MLX5_SET(set_action_in, action, field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); - MLX5_SET(set_action_in, action, data, (ipsec_obj_id | BIT(31))); - MLX5_SET(set_action_in, action, offset, 0); - MLX5_SET(set_action_in, action, length, 32); + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_no_frags(spec); - modify_hdr = mlx5_modify_header_alloc(priv->mdev, MLX5_FLOW_NAMESPACE_KERNEL, - 1, action); - if (IS_ERR(modify_hdr)) { - err = PTR_ERR(modify_hdr); - netdev_err(priv->netdev, - "fail to alloc ipsec set modify_header_id err=%d\n", err); - modify_hdr = NULL; - goto out_err; + err = setup_modify_header(mdev, sa_entry->ipsec_obj_id | BIT(31), + XFRM_DEV_OFFLOAD_IN, &flow_act); + if (err) + goto err_mod_header; + + switch (attrs->type) { + case XFRM_DEV_OFFLOAD_PACKET: + err = setup_pkt_reformat(mdev, attrs, &flow_act); + if (err) + goto err_pkt_reformat; + break; + default: + break; } - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | - MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; + flow_act.flags |= FLOW_ACT_NO_APPEND; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT; dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; - flow_act.modify_hdr = modify_hdr; - dest.ft = fs_prot->rx_err.ft; - rule = mlx5_add_flow_rules(fs_prot->ft, spec, &flow_act, &dest, 1); + dest.ft = rx->ft.status; + rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", - attrs->action, err); - goto out_err; + mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); + goto err_add_flow; } + kvfree(spec); - ipsec_rule->rule = rule; - ipsec_rule->set_modify_hdr = modify_hdr; - goto out; - -out_err: - if (modify_hdr) - mlx5_modify_header_dealloc(priv->mdev, modify_hdr); - rx_ft_put(priv, type); + sa_entry->ipsec_rule.rule = rule; + sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; + sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; + return 0; -out: +err_add_flow: + if (flow_act.pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); +err_pkt_reformat: + mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); +err_mod_header: kvfree(spec); +err_alloc: + rx_ft_put(mdev, ipsec, attrs->family); return err; } -static int tx_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { + struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); + struct mlx5e_ipsec *ipsec = sa_entry->ipsec; + struct mlx5_flow_destination dest = {}; struct mlx5_flow_act flow_act = {}; struct mlx5_flow_handle *rule; struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_tx *tx; int err = 0; - err = tx_ft_get(priv); - if (err) - return err; + tx = tx_ft_get(mdev, ipsec); + if (IS_ERR(tx)) + return PTR_ERR(tx); spec = kvzalloc(sizeof(*spec), GFP_KERNEL); if (!spec) { err = -ENOMEM; - goto out; + goto err_alloc; } - setup_fte_common(&sa_entry->attrs, sa_entry->ipsec_obj_id, spec, - &flow_act); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + switch (attrs->type) { + case XFRM_DEV_OFFLOAD_CRYPTO: + setup_fte_spi(spec, attrs->spi); + setup_fte_esp(spec); + setup_fte_reg_a(spec); + break; + case XFRM_DEV_OFFLOAD_PACKET: + setup_fte_reg_c0(spec, attrs->reqid); + err = setup_pkt_reformat(mdev, attrs, &flow_act); + if (err) + goto err_pkt_reformat; + break; + default: + break; + } - /* Add IPsec indicator in metadata_reg_a */ - spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; - MLX5_SET(fte_match_param, spec->match_criteria, misc_parameters_2.metadata_reg_a, - MLX5_ETH_WQE_FT_META_IPSEC); - MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_a, - MLX5_ETH_WQE_FT_META_IPSEC); - - flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | - MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT; - rule = mlx5_add_flow_rules(priv->ipsec->tx_fs->ft, spec, &flow_act, NULL, 0); + flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; + flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; + flow_act.flags |= FLOW_ACT_NO_APPEND; + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_ALLOW | + MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest.counter_id = mlx5_fc_id(tx->fc->cnt); + rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, &dest, 1); if (IS_ERR(rule)) { err = PTR_ERR(rule); - netdev_err(priv->netdev, "fail to add ipsec rule attrs->action=0x%x, err=%d\n", - sa_entry->attrs.action, err); - goto out; + mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); + goto err_add_flow; } + kvfree(spec); sa_entry->ipsec_rule.rule = rule; + sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; + return 0; -out: +err_add_flow: + if (flow_act.pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); +err_pkt_reformat: kvfree(spec); +err_alloc: + tx_ft_put(ipsec); + return err; +} + +static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5_flow_destination dest[2] = {}; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_tx *tx; + int err, dstn = 0; + + tx = tx_ft_get(mdev, pol_entry->ipsec); + if (IS_ERR(tx)) + return PTR_ERR(tx); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto err_alloc; + } + + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + err = setup_modify_header(mdev, attrs->reqid, XFRM_DEV_OFFLOAD_OUT, + &flow_act); if (err) - tx_ft_put(priv); + goto err_mod_header; + + switch (attrs->action) { + case XFRM_POLICY_ALLOW: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + break; + case XFRM_POLICY_BLOCK: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | + MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); + dstn++; + break; + default: + WARN_ON(true); + err = -EINVAL; + goto err_action; + } + + flow_act.flags |= FLOW_ACT_NO_APPEND; + dest[dstn].ft = tx->ft.sa; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dstn++; + rule = mlx5_add_flow_rules(tx->ft.pol, spec, &flow_act, dest, dstn); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); + goto err_action; + } + + kvfree(spec); + pol_entry->ipsec_rule.rule = rule; + pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; + return 0; + +err_action: + mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); +err_mod_header: + kvfree(spec); +err_alloc: + tx_ft_put(pol_entry->ipsec); return err; } -int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) { - if (sa_entry->attrs.action == MLX5_ACCEL_ESP_ACTION_ENCRYPT) - return tx_add_rule(priv, sa_entry); + struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); + struct mlx5_flow_destination dest[2]; + struct mlx5_flow_act flow_act = {}; + struct mlx5_flow_handle *rule; + struct mlx5_flow_spec *spec; + struct mlx5e_ipsec_rx *rx; + int err, dstn = 0; + + rx = rx_ft_get(mdev, pol_entry->ipsec, attrs->family); + if (IS_ERR(rx)) + return PTR_ERR(rx); + + spec = kvzalloc(sizeof(*spec), GFP_KERNEL); + if (!spec) { + err = -ENOMEM; + goto err_alloc; + } - return rx_add_rule(priv, sa_entry); + if (attrs->family == AF_INET) + setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); + else + setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); + + setup_fte_no_frags(spec); + + switch (attrs->action) { + case XFRM_POLICY_ALLOW: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; + break; + case XFRM_POLICY_BLOCK: + flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; + dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); + dstn++; + break; + default: + WARN_ON(true); + err = -EINVAL; + goto err_action; + } + + flow_act.flags |= FLOW_ACT_NO_APPEND; + dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; + dest[dstn].ft = rx->ft.sa; + dstn++; + rule = mlx5_add_flow_rules(rx->ft.pol, spec, &flow_act, dest, dstn); + if (IS_ERR(rule)) { + err = PTR_ERR(rule); + mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); + goto err_action; + } + + kvfree(spec); + pol_entry->ipsec_rule.rule = rule; + return 0; + +err_action: + kvfree(spec); +err_alloc: + rx_ft_put(mdev, pol_entry->ipsec, attrs->family); + return err; } -void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_priv *priv, - struct mlx5e_ipsec_sa_entry *sa_entry) +static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx *tx = ipsec->tx; + + mlx5_fc_destroy(mdev, tx->fc->drop); + mlx5_fc_destroy(mdev, tx->fc->cnt); + kfree(tx->fc); + mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); + mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); + kfree(rx_ipv4->fc); +} + +static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) +{ + struct mlx5e_ipsec_rx *rx_ipv4 = ipsec->rx_ipv4; + struct mlx5e_ipsec_rx *rx_ipv6 = ipsec->rx_ipv6; + struct mlx5_core_dev *mdev = ipsec->mdev; + struct mlx5e_ipsec_tx *tx = ipsec->tx; + struct mlx5e_ipsec_fc *fc; + struct mlx5_fc *counter; + int err; + + fc = kzalloc(sizeof(*rx_ipv4->fc), GFP_KERNEL); + if (!fc) + return -ENOMEM; + + /* Both IPv4 and IPv6 point to same flow counters struct. */ + rx_ipv4->fc = fc; + rx_ipv6->fc = fc; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_rx_cnt; + } + + fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_rx_drop; + } + + fc->drop = counter; + fc = kzalloc(sizeof(*tx->fc), GFP_KERNEL); + if (!fc) { + err = -ENOMEM; + goto err_tx_fc; + } + + tx->fc = fc; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_tx_cnt; + } + + fc->cnt = counter; + counter = mlx5_fc_create(mdev, false); + if (IS_ERR(counter)) { + err = PTR_ERR(counter); + goto err_tx_drop; + } + + fc->drop = counter; + return 0; + +err_tx_drop: + mlx5_fc_destroy(mdev, tx->fc->cnt); +err_tx_cnt: + kfree(tx->fc); +err_tx_fc: + mlx5_fc_destroy(mdev, rx_ipv4->fc->drop); +err_rx_drop: + mlx5_fc_destroy(mdev, rx_ipv4->fc->cnt); +err_rx_cnt: + kfree(rx_ipv4->fc); + return err; +} + +void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) +{ + struct mlx5_core_dev *mdev = priv->mdev; + struct mlx5e_ipsec *ipsec = priv->ipsec; + struct mlx5e_ipsec_hw_stats *stats; + struct mlx5e_ipsec_fc *fc; + + stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; + + stats->ipsec_rx_pkts = 0; + stats->ipsec_rx_bytes = 0; + stats->ipsec_rx_drop_pkts = 0; + stats->ipsec_rx_drop_bytes = 0; + stats->ipsec_tx_pkts = 0; + stats->ipsec_tx_bytes = 0; + stats->ipsec_tx_drop_pkts = 0; + stats->ipsec_tx_drop_bytes = 0; + + fc = ipsec->rx_ipv4->fc; + mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_rx_pkts, &stats->ipsec_rx_bytes); + mlx5_fc_query(mdev, fc->drop, &stats->ipsec_rx_drop_pkts, + &stats->ipsec_rx_drop_bytes); + + fc = ipsec->tx->fc; + mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); + mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, + &stats->ipsec_tx_drop_bytes); +} + +int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) +{ + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) + return tx_add_rule(sa_entry); + + return rx_add_rule(sa_entry); +} + +void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) { struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); mlx5_del_flow_rules(ipsec_rule->rule); - if (sa_entry->attrs.action == MLX5_ACCEL_ESP_ACTION_ENCRYPT) { - tx_ft_put(priv); + if (ipsec_rule->pkt_reformat) + mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); + + if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { + tx_ft_put(sa_entry->ipsec); return; } - mlx5_modify_header_dealloc(mdev, ipsec_rule->set_modify_hdr); - rx_ft_put(priv, - sa_entry->attrs.is_ipv6 ? ACCEL_FS_ESP6 : ACCEL_FS_ESP4); + mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); + rx_ft_put(mdev, sa_entry->ipsec, sa_entry->attrs.family); } -void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) +int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; - enum accel_fs_esp_type i; + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) + return tx_add_policy(pol_entry); - if (!ipsec->rx_fs) - return; + return rx_add_policy(pol_entry); +} + +void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) +{ + struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; + struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); - mutex_destroy(&ipsec->tx_fs->mutex); - WARN_ON(ipsec->tx_fs->refcnt); - kfree(ipsec->tx_fs); + mlx5_del_flow_rules(ipsec_rule->rule); - accel_esp = ipsec->rx_fs; - for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { - fs_prot = &accel_esp->fs_prot[i]; - mutex_destroy(&fs_prot->prot_mutex); - WARN_ON(fs_prot->refcnt); + if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { + rx_ft_put(mdev, pol_entry->ipsec, pol_entry->attrs.family); + return; } - kfree(ipsec->rx_fs); + + mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); + tx_ft_put(pol_entry->ipsec); +} + +void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) +{ + if (!ipsec->tx) + return; + + ipsec_fs_destroy_counters(ipsec); + mutex_destroy(&ipsec->tx->ft.mutex); + WARN_ON(ipsec->tx->ft.refcnt); + kfree(ipsec->tx); + + mutex_destroy(&ipsec->rx_ipv4->ft.mutex); + WARN_ON(ipsec->rx_ipv4->ft.refcnt); + kfree(ipsec->rx_ipv4); + + mutex_destroy(&ipsec->rx_ipv6->ft.mutex); + WARN_ON(ipsec->rx_ipv6->ft.refcnt); + kfree(ipsec->rx_ipv6); } int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) { - struct mlx5e_accel_fs_esp_prot *fs_prot; - struct mlx5e_accel_fs_esp *accel_esp; struct mlx5_flow_namespace *ns; - enum accel_fs_esp_type i; int err = -ENOMEM; ns = mlx5_get_flow_namespace(ipsec->mdev, @@ -581,26 +1032,34 @@ int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec) if (!ns) return -EOPNOTSUPP; - ipsec->tx_fs = kzalloc(sizeof(*ipsec->tx_fs), GFP_KERNEL); - if (!ipsec->tx_fs) + ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); + if (!ipsec->tx) return -ENOMEM; - ipsec->rx_fs = kzalloc(sizeof(*ipsec->rx_fs), GFP_KERNEL); - if (!ipsec->rx_fs) - goto err_rx; + ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); + if (!ipsec->rx_ipv4) + goto err_rx_ipv4; - mutex_init(&ipsec->tx_fs->mutex); - ipsec->tx_fs->ns = ns; + ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); + if (!ipsec->rx_ipv6) + goto err_rx_ipv6; - accel_esp = ipsec->rx_fs; - for (i = 0; i < ACCEL_FS_ESP_NUM_TYPES; i++) { - fs_prot = &accel_esp->fs_prot[i]; - mutex_init(&fs_prot->prot_mutex); - } + err = ipsec_fs_init_counters(ipsec); + if (err) + goto err_counters; + + mutex_init(&ipsec->tx->ft.mutex); + mutex_init(&ipsec->rx_ipv4->ft.mutex); + mutex_init(&ipsec->rx_ipv6->ft.mutex); + ipsec->tx->ns = ns; return 0; -err_rx: - kfree(ipsec->tx_fs); +err_counters: + kfree(ipsec->rx_ipv6); +err_rx_ipv6: + kfree(ipsec->rx_ipv4); +err_rx_ipv4: + kfree(ipsec->tx); return err; } |