+int __skb_datagram_iter(const struct sk_buff *skb, int offset,

+ struct iov_iter *to, int len, bool fault_short,

+ size_t (*cb)(const void *, size_t, void *, struct iov_iter *),

+ void *data)

+ n = cb(skb->data + offset, copy, data, to);

+ struct page *page = skb_frag_page(frag);

+ u8 *vaddr = kmap(page);

+

+ n = cb(vaddr + frag->page_offset +

+ offset - start, copy, data, to);

+ kunmap(page);

+ if (__skb_datagram_iter(frag_iter, offset - start,

+ to, copy, fault_short, cb, data))

+ if (fault_short || iov_iter_count(to))

+

+/**

+ * skb_copy_and_hash_datagram_iter - Copy datagram to an iovec iterator

+ * and update a hash.

+ * @skb: buffer to copy

+ * @offset: offset in the buffer to start copying from

+ * @to: iovec iterator to copy to

+ * @len: amount of data to copy from buffer to iovec

+ * @hash: hash request to update

+ */

+int skb_copy_and_hash_datagram_iter(const struct sk_buff *skb, int offset,

+ struct iov_iter *to, int len,

+ struct ahash_request *hash)

+{

+ return __skb_datagram_iter(skb, offset, to, len, true,

+ hash_and_copy_to_iter, hash);

+}

+EXPORT_SYMBOL(skb_copy_and_hash_datagram_iter);

+

+static size_t simple_copy_to_iter(const void *addr, size_t bytes,

+ void *data __always_unused, struct iov_iter *i)

+{

+ return copy_to_iter(addr, bytes, i);

+}

+

+/**

+ * skb_copy_datagram_iter - Copy a datagram to an iovec iterator.

+ * @skb: buffer to copy

+ * @offset: offset in the buffer to start copying from

+ * @to: iovec iterator to copy to

+ * @len: amount of data to copy from buffer to iovec

+ */

+int skb_copy_datagram_iter(const struct sk_buff *skb, int offset,

+ struct iov_iter *to, int len)

+{

+ trace_skb_copy_datagram_iovec(skb, len);

+ return __skb_datagram_iter(skb, offset, to, len, false,

+ simple_copy_to_iter, NULL);

+}

+/**

+ * skb_copy_and_csum_datagram_iter - Copy datagram to an iovec iterator

+ * and update a checksum.

+ * @skb: buffer to copy

+ * @offset: offset in the buffer to start copying from

+ * @to: iovec iterator to copy to

+ * @len: amount of data to copy from buffer to iovec

+ * @csump: checksum pointer

+ */

+ return __skb_datagram_iter(skb, offset, to, len, true,

+ csum_and_copy_to_iter, csump);

+ netdev_rx_csum_fault(NULL, skb);

+#include <linux/indirect_call_wrapper.h>

+static int call_netdevice_notifiers_extack(unsigned long val,

+ struct net_device *dev,

+ struct netlink_ext_ack *extack);

+static int __dev_open(struct net_device *dev, struct netlink_ext_ack *extack)

+ ret = call_netdevice_notifiers_extack(NETDEV_PRE_UP, dev, extack);

+ * @dev: device to open

+ * @extack: netlink extended ack

+int dev_open(struct net_device *dev, struct netlink_ext_ack *extack)

+ ret = __dev_open(dev, extack);

+ N(PRE_CHANGEADDR)

+static int call_netdevice_notifiers_extack(unsigned long val,

+ struct net_device *dev,

+ struct netlink_ext_ack *extack)

+{

+ struct netdev_notifier_info info = {

+ .dev = dev,

+ .extack = extack,

+ };

+

+ return call_netdevice_notifiers_info(val, &info);

+}

+

+ return call_netdevice_notifiers_extack(val, dev, NULL);

+static void reset_xps_maps(struct net_device *dev,

+ struct xps_dev_maps *dev_maps,

+ bool is_rxqs_map)

+{

+ if (is_rxqs_map) {

+ static_key_slow_dec_cpuslocked(&xps_rxqs_needed);

+ RCU_INIT_POINTER(dev->xps_rxqs_map, NULL);

+ } else {

+ RCU_INIT_POINTER(dev->xps_cpus_map, NULL);

+ }

+ static_key_slow_dec_cpuslocked(&xps_needed);

+ kfree_rcu(dev_maps, rcu);

+}

+

+ if (!active)

+ reset_xps_maps(dev, dev_maps, is_rxqs_map);

+ if (!is_rxqs_map) {

+ for (i = offset + (count - 1); count--; i--) {

+ netdev_queue_numa_node_write(

+ netdev_get_tx_queue(dev, i),

+ NUMA_NO_NODE);

+ if (!dev_maps) {

+ /* Increment static keys at most once per type */

+ static_key_slow_inc_cpuslocked(&xps_needed);

+ if (is_rxqs_map)

+ static_key_slow_inc_cpuslocked(&xps_rxqs_needed);

+ }

+ if (!active)

+ reset_xps_maps(dev, dev_maps, is_rxqs_map);

+void netdev_rx_csum_fault(struct net_device *dev, struct sk_buff *skb)

+ if (dev)

+ pr_err("dev features: %pNF\n", &dev->features);

+ pr_err("skb len=%u data_len=%u pkt_type=%u gso_size=%u gso_type=%u nr_frags=%u ip_summed=%u csum=%x csum_complete_sw=%d csum_valid=%d csum_level=%u\n",

+ skb->len, skb->data_len, skb->pkt_type,

+ skb_shinfo(skb)->gso_size, skb_shinfo(skb)->gso_type,

+ skb_shinfo(skb)->nr_frags, skb->ip_summed, skb->csum,

+ skb->csum_complete_sw, skb->csum_valid, skb->csum_level);

+ if (netif_tx_queue_stopped(txq) && skb) {

+ int ret;

+

+ ret = netif_rx_internal(skb);

+ trace_netif_rx_exit(ret);

+

+ return ret;

+ trace_netif_rx_ni_exit(err);

+ __vlan_hwaccel_clear_tag(skb);

+ skb_list_del_init(skb);

+ skb_list_del_init(skb);

+ skb_list_del_init(skb);

+ skb_list_del_init(skb);

+ int ret;

+

+ ret = netif_receive_skb_internal(skb);

+ trace_netif_receive_skb_exit(ret);

+

+ return ret;

+ if (trace_netif_receive_skb_list_entry_enabled()) {

+ list_for_each_entry(skb, head, list)

+ trace_netif_receive_skb_list_entry(skb);

+ }

+ trace_netif_receive_skb_list_exit(0);

+INDIRECT_CALLABLE_DECLARE(int inet_gro_complete(struct sk_buff *, int));

+INDIRECT_CALLABLE_DECLARE(int ipv6_gro_complete(struct sk_buff *, int));

+ err = INDIRECT_CALL_INET(ptype->callbacks.gro_complete,

+ ipv6_gro_complete, inet_gro_complete,

+ skb, 0);

+ unsigned long bitmask = napi->gro_bitmask;

+ unsigned int i, base = ~0U;

+ while ((i = ffs(bitmask)) != 0) {

+ bitmask >>= i;

+ base += i;

+ __napi_gro_flush_chain(napi, base, flush_old);

+ diffs |= skb_vlan_tag_present(p) ^ skb_vlan_tag_present(skb);

+ if (skb_vlan_tag_present(p))

+ diffs |= p->vlan_tci ^ skb->vlan_tci;

+INDIRECT_CALLABLE_DECLARE(struct sk_buff *inet_gro_receive(struct list_head *,

+ struct sk_buff *));

+INDIRECT_CALLABLE_DECLARE(struct sk_buff *ipv6_gro_receive(struct list_head *,

+ struct sk_buff *));

+ pp = INDIRECT_CALL_INET(ptype->callbacks.gro_receive,

+ ipv6_gro_receive, inet_gro_receive,

+ gro_head, skb);

+ gro_result_t ret;

+

+ ret = napi_skb_finish(dev_gro_receive(napi, skb), skb);

+ trace_napi_gro_receive_exit(ret);

+

+ return ret;

+ __vlan_hwaccel_clear_tag(skb);

+

+ /* eth_type_trans() assumes pkt_type is PACKET_HOST */

+ skb->pkt_type = PACKET_HOST;

+

+ gro_result_t ret;

+ ret = napi_frags_finish(napi, skb, dev_gro_receive(napi, skb));

+ trace_napi_gro_frags_exit(ret);

+

+ return ret;

+ /* See comments in __skb_checksum_complete(). */

+ netdev_rx_csum_fault(skb->dev, skb);

+ /* When the NAPI instance uses a timeout and keeps postponing

+ * it, we need to bound somehow the time packets are kept in

+ * the GRO layer

+ */

+ napi_gro_flush(n, !!timeout);

+ netdev_err_once(dev, "%s() called with weight %d\n", __func__,

+ weight);

+int __dev_change_flags(struct net_device *dev, unsigned int flags,

+ struct netlink_ext_ack *extack)

+ ret = __dev_open(dev, extack);

+ * @extack: netlink extended ack

+int dev_change_flags(struct net_device *dev, unsigned int flags,

+ struct netlink_ext_ack *extack)

+ ret = __dev_change_flags(dev, flags, extack);

+ * dev_pre_changeaddr_notify - Call NETDEV_PRE_CHANGEADDR.

+ * @dev: device

+ * @addr: new address

+ * @extack: netlink extended ack

+ */

+int dev_pre_changeaddr_notify(struct net_device *dev, const char *addr,

+ struct netlink_ext_ack *extack)

+{

+ struct netdev_notifier_pre_changeaddr_info info = {

+ .info.dev = dev,

+ .info.extack = extack,

+ .dev_addr = addr,

+ };

+ int rc;

+

+ rc = call_netdevice_notifiers_info(NETDEV_PRE_CHANGEADDR, &info.info);

+ return notifier_to_errno(rc);

+}

+EXPORT_SYMBOL(dev_pre_changeaddr_notify);

+

+/**

+ * @extack: netlink extended ack

+int dev_set_mac_address(struct net_device *dev, struct sockaddr *sa,

+ struct netlink_ext_ack *extack)

+ err = dev_pre_changeaddr_notify(dev, sa->sa_data, extack);

+ if (err)

+ return err;

+ * __hw_addr_ref_sync_dev - Synchronize device's multicast address list taking

+ * into account references

+ * @list: address list to synchronize

+ * @dev: device to sync

+ * @sync: function to call if address or reference on it should be added

+ * @unsync: function to call if address or some reference on it should removed

+ *

+ * This function is intended to be called from the ndo_set_rx_mode

+ * function of devices that require explicit address or references on it

+ * add/remove notifications. The unsync function may be NULL in which case

+ * the addresses or references on it requiring removal will simply be

+ * removed without any notification to the device. That is responsibility of

+ * the driver to identify and distribute address or references on it between

+ * internal address tables.

+ **/

+int __hw_addr_ref_sync_dev(struct netdev_hw_addr_list *list,

+ struct net_device *dev,

+ int (*sync)(struct net_device *,

+ const unsigned char *, int),

+ int (*unsync)(struct net_device *,

+ const unsigned char *, int))

+{

+ struct netdev_hw_addr *ha, *tmp;

+ int err, ref_cnt;

+

+ /* first go through and flush out any unsynced/stale entries */

+ list_for_each_entry_safe(ha, tmp, &list->list, list) {

+ /* sync if address is not used */

+ if ((ha->sync_cnt << 1) <= ha->refcount)

+ continue;

+

+ /* if fails defer unsyncing address */

+ ref_cnt = ha->refcount - ha->sync_cnt;

+ if (unsync && unsync(dev, ha->addr, ref_cnt))

+ continue;

+

+ ha->refcount = (ref_cnt << 1) + 1;

+ ha->sync_cnt = ref_cnt;

+ __hw_addr_del_entry(list, ha, false, false);

+ }

+

+ /* go through and sync updated/new entries to the list */

+ list_for_each_entry_safe(ha, tmp, &list->list, list) {

+ /* sync if address added or reused */

+ if ((ha->sync_cnt << 1) >= ha->refcount)

+ continue;

+

+ ref_cnt = ha->refcount - ha->sync_cnt;

+ err = sync(dev, ha->addr, ref_cnt);

+ if (err)

+ return err;

+

+ ha->refcount = ref_cnt << 1;

+ ha->sync_cnt = ref_cnt;

+ }

+

+ return 0;

+}

+EXPORT_SYMBOL(__hw_addr_ref_sync_dev);

+

+/**

+ * __hw_addr_ref_unsync_dev - Remove synchronized addresses and references on

+ * it from device

+ * @list: address list to remove synchronized addresses (references on it) from

+ * @dev: device to sync

+ * @unsync: function to call if address and references on it should be removed

+ *

+ * Remove all addresses that were added to the device by

+ * __hw_addr_ref_sync_dev(). This function is intended to be called from the

+ * ndo_stop or ndo_open functions on devices that require explicit address (or

+ * references on it) add/remove notifications. If the unsync function pointer

+ * is NULL then this function can be used to just reset the sync_cnt for the

+ * addresses in the list.

+ **/

+void __hw_addr_ref_unsync_dev(struct netdev_hw_addr_list *list,

+ struct net_device *dev,

+ int (*unsync)(struct net_device *,

+ const unsigned char *, int))

+{

+ struct netdev_hw_addr *ha, *tmp;

+

+ list_for_each_entry_safe(ha, tmp, &list->list, list) {

+ if (!ha->sync_cnt)

+ continue;

+

+ /* if fails defer unsyncing address */

+ if (unsync && unsync(dev, ha->addr, ha->sync_cnt))

+ continue;

+

+ ha->refcount -= ha->sync_cnt - 1;

+ ha->sync_cnt = 0;

+ __hw_addr_del_entry(list, ha, false, false);

+ }

+}

+EXPORT_SYMBOL(__hw_addr_ref_unsync_dev);

+

+/**

+ err = dev_pre_changeaddr_notify(dev, addr, NULL);

+ if (err)

+ return err;

+ return dev_change_flags(dev, ifr->ifr_flags, NULL);

+ return dev_set_mac_address(dev, &ifr->ifr_hwaddr, NULL);

+ {

+ .id = DEVLINK_PARAM_GENERIC_ID_FW_LOAD_POLICY,

+ .name = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_NAME,

+ .type = DEVLINK_PARAM_GENERIC_FW_LOAD_POLICY_TYPE,

+ },

+ if (ops->get_regs_len) {

+ int ret = ops->get_regs_len(dev);

+

+ if (ret > 0)

+ info.regdump_len = ret;

+ }

+

+ if (reglen <= 0)

+ return reglen;

+

+ break;

+ case SKF_AD_VLAN_TAG_PRESENT:

+ *insn++ = BPF_LDX_MEM(BPF_B, dst_reg, src_reg, PKT_VLAN_PRESENT_OFFSET());

+ if (PKT_VLAN_PRESENT_BIT)

+ *insn++ = BPF_ALU32_IMM(BPF_RSH, dst_reg, PKT_VLAN_PRESENT_BIT);

+ if (PKT_VLAN_PRESENT_BIT < 7)

+ if (offset)

+ *insn++ = BPF_ALU64_IMM(BPF_SUB, BPF_REG_TMP, offset);

+static void sk_msg_shift_left(struct sk_msg *msg, int i)

+{

+ int prev;

+

+ do {

+ prev = i;

+ sk_msg_iter_var_next(i);

+ msg->sg.data[prev] = msg->sg.data[i];

+ } while (i != msg->sg.end);

+

+ sk_msg_iter_prev(msg, end);

+}

+

+static void sk_msg_shift_right(struct sk_msg *msg, int i)

+{

+ struct scatterlist tmp, sge;

+

+ sk_msg_iter_next(msg, end);

+ sge = sk_msg_elem_cpy(msg, i);

+ sk_msg_iter_var_next(i);

+ tmp = sk_msg_elem_cpy(msg, i);

+

+ while (i != msg->sg.end) {

+ msg->sg.data[i] = sge;

+ sk_msg_iter_var_next(i);

+ sge = tmp;

+ tmp = sk_msg_elem_cpy(msg, i);

+ }

+}

+

+BPF_CALL_4(bpf_msg_pop_data, struct sk_msg *, msg, u32, start,

+ u32, len, u64, flags)

+{

+ u32 i = 0, l, space, offset = 0;

+ u64 last = start + len;

+ int pop;

+

+ if (unlikely(flags))

+ return -EINVAL;

+

+ /* First find the starting scatterlist element */

+ i = msg->sg.start;

+ do {

+ l = sk_msg_elem(msg, i)->length;

+

+ if (start < offset + l)

+ break;

+ offset += l;

+ sk_msg_iter_var_next(i);

+ } while (i != msg->sg.end);

+

+ /* Bounds checks: start and pop must be inside message */

+ if (start >= offset + l || last >= msg->sg.size)

+ return -EINVAL;

+

+ space = MAX_MSG_FRAGS - sk_msg_elem_used(msg);

+

+ pop = len;

+ /* --------------| offset

+ * -| start |-------- len -------|

+ *

+ * |----- a ----|-------- pop -------|----- b ----|

+ * |______________________________________________| length

+ *

+ *

+ * a: region at front of scatter element to save

+ * b: region at back of scatter element to save when length > A + pop

+ * pop: region to pop from element, same as input 'pop' here will be

+ * decremented below per iteration.

+ *

+ * Two top-level cases to handle when start != offset, first B is non

+ * zero and second B is zero corresponding to when a pop includes more

+ * than one element.

+ *

+ * Then if B is non-zero AND there is no space allocate space and

+ * compact A, B regions into page. If there is space shift ring to

+ * the rigth free'ing the next element in ring to place B, leaving

+ * A untouched except to reduce length.

+ */

+ if (start != offset) {

+ struct scatterlist *nsge, *sge = sk_msg_elem(msg, i);

+ int a = start;

+ int b = sge->length - pop - a;

+

+ sk_msg_iter_var_next(i);

+

+ if (pop < sge->length - a) {

+ if (space) {

+ sge->length = a;

+ sk_msg_shift_right(msg, i);

+ nsge = sk_msg_elem(msg, i);

+ get_page(sg_page(sge));

+ sg_set_page(nsge,

+ sg_page(sge),

+ b, sge->offset + pop + a);

+ } else {

+ struct page *page, *orig;

+ u8 *to, *from;

+

+ page = alloc_pages(__GFP_NOWARN |

+ __GFP_COMP | GFP_ATOMIC,

+ get_order(a + b));

+ if (unlikely(!page))

+ return -ENOMEM;

+

+ sge->length = a;

+ orig = sg_page(sge);

+ from = sg_virt(sge);

+ to = page_address(page);

+ memcpy(to, from, a);

+ memcpy(to + a, from + a + pop, b);

+ sg_set_page(sge, page, a + b, 0);

+ put_page(orig);

+ }

+ pop = 0;

+ } else if (pop >= sge->length - a) {

+ sge->length = a;

+ pop -= (sge->length - a);

+ }

+ }

+

+ /* From above the current layout _must_ be as follows,

+ *

+ * -| offset

+ * -| start

+ *

+ * |---- pop ---|---------------- b ------------|

+ * |____________________________________________| length

+ *

+ * Offset and start of the current msg elem are equal because in the

+ * previous case we handled offset != start and either consumed the

+ * entire element and advanced to the next element OR pop == 0.

+ *

+ * Two cases to handle here are first pop is less than the length

+ * leaving some remainder b above. Simply adjust the element's layout

+ * in this case. Or pop >= length of the element so that b = 0. In this

+ * case advance to next element decrementing pop.

+ */

+ while (pop) {

+ struct scatterlist *sge = sk_msg_elem(msg, i);

+

+ if (pop < sge->length) {

+ sge->length -= pop;

+ sge->offset += pop;

+ pop = 0;

+ } else {

+ pop -= sge->length;

+ sk_msg_shift_left(msg, i);

+ }

+ sk_msg_iter_var_next(i);

+ }

+

+ sk_mem_uncharge(msg->sk, len - pop);

+ msg->sg.size -= (len - pop);

+ sk_msg_compute_data_pointers(msg);

+ return 0;

+}

+

+static const struct bpf_func_proto bpf_msg_pop_data_proto = {

+ .func = bpf_msg_pop_data,

+ .gpl_only = false,

+ .ret_type = RET_INTEGER,

+ .arg1_type = ARG_PTR_TO_CTX,

+ .arg2_type = ARG_ANYTHING,

+ .arg3_type = ARG_ANYTHING,

+ .arg4_type = ARG_ANYTHING,

+};

+

+BPF_CALL_5(bpf_sockopt_event_output, struct bpf_sock_ops_kern *, bpf_sock,

+ struct bpf_map *, map, u64, flags, void *, data, u64, size)

+{

+ if (unlikely(flags & ~(BPF_F_INDEX_MASK)))

+ return -EINVAL;

+

+ return bpf_event_output(map, flags, data, size, NULL, 0, NULL);

+}

+

+static const struct bpf_func_proto bpf_sockopt_event_output_proto = {

+ .func = bpf_sockopt_event_output,

+ .gpl_only = true,

+ .ret_type = RET_INTEGER,

+ .arg1_type = ARG_PTR_TO_CTX,

+ .arg2_type = ARG_CONST_MAP_PTR,

+ .arg3_type = ARG_ANYTHING,

+ .arg4_type = ARG_PTR_TO_MEM,

+ .arg5_type = ARG_CONST_SIZE_OR_ZERO,

+};

+

+ int dif, int sdif, u8 family, u8 proto)

+ sk = __inet_lookup(net, &tcp_hashinfo, NULL, 0,

+ dif, sdif, &udp_table, NULL);

+ sk = __inet6_lookup(net, &tcp_hashinfo, NULL, 0,

+ dst6, ntohs(tuple->ipv6.dport),

+ dst6, tuple->ipv6.dport,

+ &udp_table, NULL);

+__bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,

+ struct net *caller_net, u32 ifindex, u8 proto, u64 netns_id,

+ u64 flags)

+ int sdif;

+ if (unlikely(family == AF_UNSPEC || flags ||

+ !((s32)netns_id < 0 || netns_id <= S32_MAX)))

+ if (family == AF_INET)

+ sdif = inet_sdif(skb);

+ sdif = inet6_sdif(skb);

+

+ if ((s32)netns_id < 0) {

+ net = caller_net;

+ sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);

+ } else {

+ sk = sk_lookup(net, tuple, ifindex, sdif, family, proto);

+static unsigned long

+bpf_sk_lookup(struct sk_buff *skb, struct bpf_sock_tuple *tuple, u32 len,

+ u8 proto, u64 netns_id, u64 flags)

+{

+ struct net *caller_net;

+ int ifindex;

+

+ if (skb->dev) {

+ caller_net = dev_net(skb->dev);

+ ifindex = skb->dev->ifindex;

+ } else {

+ caller_net = sock_net(skb->sk);

+ ifindex = 0;

+ }

+

+ return __bpf_sk_lookup(skb, tuple, len, caller_net, ifindex,

+ proto, netns_id, flags);

+}

+

+

+BPF_CALL_5(bpf_xdp_sk_lookup_udp, struct xdp_buff *, ctx,

+ struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)

+{

+ struct net *caller_net = dev_net(ctx->rxq->dev);

+ int ifindex = ctx->rxq->dev->ifindex;

+

+ return __bpf_sk_lookup(NULL, tuple, len, caller_net, ifindex,

+ IPPROTO_UDP, netns_id, flags);

+}

+

+static const struct bpf_func_proto bpf_xdp_sk_lookup_udp_proto = {

+ .func = bpf_xdp_sk_lookup_udp,

+ .gpl_only = false,

+ .pkt_access = true,

+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,

+ .arg1_type = ARG_PTR_TO_CTX,

+ .arg2_type = ARG_PTR_TO_MEM,

+ .arg3_type = ARG_CONST_SIZE,

+ .arg4_type = ARG_ANYTHING,

+ .arg5_type = ARG_ANYTHING,

+};

+

+BPF_CALL_5(bpf_xdp_sk_lookup_tcp, struct xdp_buff *, ctx,

+ struct bpf_sock_tuple *, tuple, u32, len, u32, netns_id, u64, flags)

+{

+ struct net *caller_net = dev_net(ctx->rxq->dev);

+ int ifindex = ctx->rxq->dev->ifindex;

+

+ return __bpf_sk_lookup(NULL, tuple, len, caller_net, ifindex,

+ IPPROTO_TCP, netns_id, flags);

+}

+

+static const struct bpf_func_proto bpf_xdp_sk_lookup_tcp_proto = {

+ .func = bpf_xdp_sk_lookup_tcp,

+ .gpl_only = false,

+ .pkt_access = true,

+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,

+ .arg1_type = ARG_PTR_TO_CTX,

+ .arg2_type = ARG_PTR_TO_MEM,

+ .arg3_type = ARG_CONST_SIZE,

+ .arg4_type = ARG_ANYTHING,

+ .arg5_type = ARG_ANYTHING,

+};

+

+BPF_CALL_5(bpf_sock_addr_sk_lookup_tcp, struct bpf_sock_addr_kern *, ctx,

+ struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)

+{

+ return __bpf_sk_lookup(NULL, tuple, len, sock_net(ctx->sk), 0,

+ IPPROTO_TCP, netns_id, flags);

+}

+

+static const struct bpf_func_proto bpf_sock_addr_sk_lookup_tcp_proto = {

+ .func = bpf_sock_addr_sk_lookup_tcp,

+ .gpl_only = false,

+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,

+ .arg1_type = ARG_PTR_TO_CTX,

+ .arg2_type = ARG_PTR_TO_MEM,

+ .arg3_type = ARG_CONST_SIZE,

+ .arg4_type = ARG_ANYTHING,

+ .arg5_type = ARG_ANYTHING,

+};

+

+BPF_CALL_5(bpf_sock_addr_sk_lookup_udp, struct bpf_sock_addr_kern *, ctx,

+ struct bpf_sock_tuple *, tuple, u32, len, u64, netns_id, u64, flags)

+{

+ return __bpf_sk_lookup(NULL, tuple, len, sock_net(ctx->sk), 0,

+ IPPROTO_UDP, netns_id, flags);

+}

+

+static const struct bpf_func_proto bpf_sock_addr_sk_lookup_udp_proto = {

+ .func = bpf_sock_addr_sk_lookup_udp,

+ .gpl_only = false,

+ .ret_type = RET_PTR_TO_SOCKET_OR_NULL,

+ .arg1_type = ARG_PTR_TO_CTX,

+ .arg2_type = ARG_PTR_TO_MEM,

+ .arg3_type = ARG_CONST_SIZE,

+ .arg4_type = ARG_ANYTHING,

+ .arg5_type = ARG_ANYTHING,

+};

+

+ func == bpf_msg_pop_data ||

+#ifdef CONFIG_INET

+ case BPF_FUNC_sk_lookup_tcp:

+ return &bpf_sock_addr_sk_lookup_tcp_proto;

+ case BPF_FUNC_sk_lookup_udp:

+ return &bpf_sock_addr_sk_lookup_udp_proto;

+ case BPF_FUNC_sk_release:

+ return &bpf_sk_release_proto;

+#endif /* CONFIG_INET */

+#ifdef CONFIG_INET

+ case BPF_FUNC_sk_lookup_udp:

+ return &bpf_xdp_sk_lookup_udp_proto;

+ case BPF_FUNC_sk_lookup_tcp:

+ return &bpf_xdp_sk_lookup_tcp_proto;

+ case BPF_FUNC_sk_release:

+ return &bpf_sk_release_proto;

+#endif

+ case BPF_FUNC_perf_event_output:

+ return &bpf_sockopt_event_output_proto;

+ case BPF_FUNC_msg_pop_data:

+ return &bpf_msg_pop_data_proto;

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ if (size != sizeof(__u64))

+ return false;

+ break;

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ if (size != sizeof(__u64))

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ case bpf_ctx_range(struct __sk_buff, wire_len):

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ case bpf_ctx_range(struct __sk_buff, wire_len):

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ if (!capable(CAP_SYS_ADMIN))

+ return false;

+ break;

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ case bpf_ctx_range(struct __sk_buff, wire_len):

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ case bpf_ctx_range(struct __sk_buff, wire_len):

+ if (off % size != 0)

+ return false;

+

+ case bpf_ctx_range(struct sk_msg_md, family):

+ case bpf_ctx_range(struct sk_msg_md, remote_ip4):

+ case bpf_ctx_range(struct sk_msg_md, local_ip4):

+ case bpf_ctx_range_till(struct sk_msg_md, remote_ip6[0], remote_ip6[3]):

+ case bpf_ctx_range_till(struct sk_msg_md, local_ip6[0], local_ip6[3]):

+ case bpf_ctx_range(struct sk_msg_md, remote_port):

+ case bpf_ctx_range(struct sk_msg_md, local_port):

+ case bpf_ctx_range(struct sk_msg_md, size):

+ break;

+ default:

+ }

+ case bpf_ctx_range_ptr(struct __sk_buff, flow_keys):

+ case bpf_ctx_range(struct __sk_buff, tstamp):

+ case bpf_ctx_range(struct __sk_buff, wire_len):

+ *target_size = 1;

+ *insn++ = BPF_LDX_MEM(BPF_B, si->dst_reg, si->src_reg,

+ PKT_VLAN_PRESENT_OFFSET());

+ if (PKT_VLAN_PRESENT_BIT)

+ *insn++ = BPF_ALU32_IMM(BPF_RSH, si->dst_reg, PKT_VLAN_PRESENT_BIT);

+ if (PKT_VLAN_PRESENT_BIT < 7)

+ *insn++ = BPF_ALU32_IMM(BPF_AND, si->dst_reg, 1);

+ break;

+ case offsetof(struct __sk_buff, vlan_tci):

+

+ case offsetof(struct __sk_buff, tstamp):

+ BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, tstamp) != 8);

+

+ if (type == BPF_WRITE)

+ *insn++ = BPF_STX_MEM(BPF_DW,

+ si->dst_reg, si->src_reg,

+ bpf_target_off(struct sk_buff,

+ tstamp, 8,

+ target_size));

+ else

+ *insn++ = BPF_LDX_MEM(BPF_DW,

+ si->dst_reg, si->src_reg,

+ bpf_target_off(struct sk_buff,

+ tstamp, 8,

+ target_size));

+ break;

+

+ case offsetof(struct __sk_buff, wire_len):

+ BUILD_BUG_ON(FIELD_SIZEOF(struct qdisc_skb_cb, pkt_len) != 4);

+

+ off = si->off;

+ off -= offsetof(struct __sk_buff, wire_len);

+ off += offsetof(struct sk_buff, cb);

+ off += offsetof(struct qdisc_skb_cb, pkt_len);

+ *target_size = 4;

+ *insn++ = BPF_LDX_MEM(BPF_W, si->dst_reg, si->src_reg, off);

+ /* convert ctx uses the fact sg element is first in struct */

+ BUILD_BUG_ON(offsetof(struct sk_msg, sg) != 0);

+

+

+ case offsetof(struct sk_msg_md, size):

+ *insn++ = BPF_LDX_MEM(BPF_FIELD_SIZEOF(struct sk_msg_sg, size),

+ si->dst_reg, si->src_reg,

+ offsetof(struct sk_msg_sg, size));

+ break;

+ flow_keys.thoff = nhoff;

+ flow_keys.nhoff = clamp_t(u16, flow_keys.nhoff, 0, skb->len);

+ flow_keys.thoff = clamp_t(u16, flow_keys.thoff,

+ flow_keys.nhoff, skb->len);

+

+ key_vlan->vlan_priority = skb_vlan_tag_get_prio(skb);

+ if (dissector_uses_key(flow_dissector, FLOW_DISSECTOR_KEY_PORTS) &&

+ !(key_control->flags & FLOW_DIS_IS_FRAGMENT)) {

+ napi_disable(&cell->napi);

+static void neigh_mark_dead(struct neighbour *n)

+{

+ n->dead = 1;

+ if (!list_empty(&n->gc_list)) {

+ list_del_init(&n->gc_list);

+ atomic_dec(&n->tbl->gc_entries);

+ }

+}

+

+static void neigh_update_gc_list(struct neighbour *n)

+{

+ bool on_gc_list, exempt_from_gc;

+

+ write_lock_bh(&n->tbl->lock);

+ write_lock(&n->lock);

+

+ /* remove from the gc list if new state is permanent or if neighbor

+ * is externally learned; otherwise entry should be on the gc list

+ */

+ exempt_from_gc = n->nud_state & NUD_PERMANENT ||

+ n->flags & NTF_EXT_LEARNED;

+ on_gc_list = !list_empty(&n->gc_list);

+

+ if (exempt_from_gc && on_gc_list) {

+ list_del_init(&n->gc_list);

+ atomic_dec(&n->tbl->gc_entries);

+ } else if (!exempt_from_gc && !on_gc_list) {

+ /* add entries to the tail; cleaning removes from the front */

+ list_add_tail(&n->gc_list, &n->tbl->gc_list);

+ atomic_inc(&n->tbl->gc_entries);

+ }

+

+ write_unlock(&n->lock);

+ write_unlock_bh(&n->tbl->lock);

+}

+static bool neigh_update_ext_learned(struct neighbour *neigh, u32 flags,

+ int *notify)

+{

+ bool rc = false;

+ u8 ndm_flags;

+

+ if (!(flags & NEIGH_UPDATE_F_ADMIN))

+ return rc;

+

+ ndm_flags = (flags & NEIGH_UPDATE_F_EXT_LEARNED) ? NTF_EXT_LEARNED : 0;

+ if ((neigh->flags ^ ndm_flags) & NTF_EXT_LEARNED) {

+ if (ndm_flags & NTF_EXT_LEARNED)

+ neigh->flags |= NTF_EXT_LEARNED;

+ else

+ neigh->flags &= ~NTF_EXT_LEARNED;

+ rc = true;

+ *notify = 1;

+ }

+

+ return rc;

+}

+

+static bool neigh_del(struct neighbour *n, struct neighbour __rcu **np,

+ struct neigh_table *tbl)

+ if (refcount_read(&n->refcnt) == 1) {

+ neigh_mark_dead(n);

+ return neigh_del(n, np, tbl);

+ int max_clean = atomic_read(&tbl->gc_entries) - tbl->gc_thresh2;

+ unsigned long tref = jiffies - 5 * HZ;

+ struct neighbour *n, *tmp;

+ list_for_each_entry_safe(n, tmp, &tbl->gc_list, gc_list) {

+ if (refcount_read(&n->refcnt) == 1) {

+ bool remove = false;

+

+ write_lock(&n->lock);

+ if ((n->nud_state == NUD_FAILED) ||

+ time_after(tref, n->updated))

+ remove = true;

+ write_unlock(&n->lock);

+

+ if (remove && neigh_remove_one(n, tbl))

+ shrunk++;

+ if (shrunk >= max_clean)

+ break;

+ neigh_mark_dead(n);

+static struct neighbour *neigh_alloc(struct neigh_table *tbl,

+ struct net_device *dev,

+ bool exempt_from_gc)

+ if (exempt_from_gc)

+ goto do_alloc;

+

+ entries = atomic_inc_return(&tbl->gc_entries) - 1;

+do_alloc:

+ INIT_LIST_HEAD(&n->gc_list);

+

+ atomic_inc(&tbl->entries);

+ if (!exempt_from_gc)

+ atomic_dec(&tbl->gc_entries);

+static struct neighbour *___neigh_create(struct neigh_table *tbl,

+ const void *pkey,

+ struct net_device *dev,

+ bool exempt_from_gc, bool want_ref)

+ struct neighbour *n1, *rc, *n = neigh_alloc(tbl, dev, exempt_from_gc);

+ if (!exempt_from_gc)

+ list_add_tail(&n->gc_list, &n->tbl->gc_list);

+

+

+struct neighbour *__neigh_create(struct neigh_table *tbl, const void *pkey,

+ struct net_device *dev, bool want_ref)

+{

+ return ___neigh_create(tbl, pkey, dev, false, want_ref);

+}

+ n->protocol = 0;

+ neigh_mark_dead(n);

+static int __neigh_update(struct neighbour *neigh, const u8 *lladdr,

+ u8 new, u32 flags, u32 nlmsg_pid,

+ struct netlink_ext_ack *extack)

+ bool ext_learn_change = false;

+ if (neigh->dead) {

+ NL_SET_ERR_MSG(extack, "Neighbor entry is now dead");

+ }

+ ext_learn_change = neigh_update_ext_learned(neigh, flags, &notify);

+ if (!(old & NUD_VALID)) {

+ NL_SET_ERR_MSG(extack, "No link layer address given");

+ }

+ if (((new ^ old) & NUD_PERMANENT) || ext_learn_change)

+ neigh_update_gc_list(neigh);

+

+

+int neigh_update(struct neighbour *neigh, const u8 *lladdr, u8 new,

+ u32 flags, u32 nlmsg_pid)

+{

+ return __neigh_update(neigh, lladdr, new, flags, nlmsg_pid, NULL);

+}

+ INIT_LIST_HEAD(&tbl->gc_list);

+const struct nla_policy nda_policy[NDA_MAX+1] = {

+ [NDA_DST] = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },

+ [NDA_LLADDR] = { .type = NLA_BINARY, .len = MAX_ADDR_LEN },

+ [NDA_CACHEINFO] = { .len = sizeof(struct nda_cacheinfo) },

+ [NDA_PROBES] = { .type = NLA_U32 },

+ [NDA_VLAN] = { .type = NLA_U16 },

+ [NDA_PORT] = { .type = NLA_U16 },

+ [NDA_VNI] = { .type = NLA_U32 },

+ [NDA_IFINDEX] = { .type = NLA_U32 },

+ [NDA_MASTER] = { .type = NLA_U32 },

+ [NDA_PROTOCOL] = { .type = NLA_U8 },

+};

+

+ if (!dst_attr) {

+ NL_SET_ERR_MSG(extack, "Network address not specified");

+ }

+ if (nla_len(dst_attr) < (int)tbl->key_len) {

+ NL_SET_ERR_MSG(extack, "Invalid network address");

+ }

+ err = __neigh_update(neigh, NULL, NUD_FAILED,

+ NEIGH_UPDATE_F_OVERRIDE | NEIGH_UPDATE_F_ADMIN,

+ NETLINK_CB(skb).portid, extack);

+ u8 protocol = 0;

+ err = nlmsg_parse(nlh, sizeof(*ndm), tb, NDA_MAX, nda_policy, extack);

+ if (!tb[NDA_DST]) {

+ NL_SET_ERR_MSG(extack, "Network address not specified");

+ }

+ if (tb[NDA_LLADDR] && nla_len(tb[NDA_LLADDR]) < dev->addr_len) {

+ NL_SET_ERR_MSG(extack, "Invalid link address");

+ }

+ if (nla_len(tb[NDA_DST]) < (int)tbl->key_len) {

+ NL_SET_ERR_MSG(extack, "Invalid network address");

+ }

+

+ if (tb[NDA_PROTOCOL])

+ protocol = nla_get_u8(tb[NDA_PROTOCOL]);

+

+ if (protocol)

+ pn->protocol = protocol;

+ if (!dev) {

+ NL_SET_ERR_MSG(extack, "Device not specified");

+ }

+ bool exempt_from_gc;

+

+ exempt_from_gc = ndm->ndm_state & NUD_PERMANENT ||

+ ndm->ndm_flags & NTF_EXT_LEARNED;

+ neigh = ___neigh_create(tbl, dst, dev, exempt_from_gc, true);

+ err = __neigh_update(neigh, lladdr, ndm->ndm_state, flags,

+ NETLINK_CB(skb).portid, extack);

+

+ if (protocol)

+ neigh->protocol = protocol;

+

+ if (neigh->protocol && nla_put_u8(skb, NDA_PROTOCOL, neigh->protocol))

+ goto nla_put_failure;

+

+ if (pn->protocol && nla_put_u8(skb, NDA_PROTOCOL, pn->protocol))

+ goto nla_put_failure;

+

+ ndm->ndm_state || ndm->ndm_type) {

+ if (ndm->ndm_flags & ~NTF_PROXY) {

+ NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor dump request");

+ return -EINVAL;

+ }

+

+ nda_policy, extack);

+ nda_policy, extack);

+static int neigh_valid_get_req(const struct nlmsghdr *nlh,

+ struct neigh_table **tbl,

+ void **dst, int *dev_idx, u8 *ndm_flags,

+ struct netlink_ext_ack *extack)

+{

+ struct nlattr *tb[NDA_MAX + 1];

+ struct ndmsg *ndm;

+ int err, i;

+

+ if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ndm))) {

+ NL_SET_ERR_MSG(extack, "Invalid header for neighbor get request");

+ return -EINVAL;

+ }

+

+ ndm = nlmsg_data(nlh);

+ if (ndm->ndm_pad1 || ndm->ndm_pad2 || ndm->ndm_state ||

+ ndm->ndm_type) {

+ NL_SET_ERR_MSG(extack, "Invalid values in header for neighbor get request");

+ return -EINVAL;

+ }

+

+ if (ndm->ndm_flags & ~NTF_PROXY) {

+ NL_SET_ERR_MSG(extack, "Invalid flags in header for neighbor get request");

+ return -EINVAL;

+ }

+

+ err = nlmsg_parse_strict(nlh, sizeof(struct ndmsg), tb, NDA_MAX,

+ nda_policy, extack);

+ if (err < 0)

+ return err;

+

+ *ndm_flags = ndm->ndm_flags;

+ *dev_idx = ndm->ndm_ifindex;

+ *tbl = neigh_find_table(ndm->ndm_family);

+ if (*tbl == NULL) {

+ NL_SET_ERR_MSG(extack, "Unsupported family in header for neighbor get request");

+ return -EAFNOSUPPORT;

+ }

+

+ for (i = 0; i <= NDA_MAX; ++i) {

+ if (!tb[i])

+ continue;

+

+ switch (i) {

+ case NDA_DST:

+ if (nla_len(tb[i]) != (int)(*tbl)->key_len) {

+ NL_SET_ERR_MSG(extack, "Invalid network address in neighbor get request");

+ return -EINVAL;

+ }

+ *dst = nla_data(tb[i]);

+ break;

+ default:

+ NL_SET_ERR_MSG(extack, "Unsupported attribute in neighbor get request");

+ return -EINVAL;

+ }

+ }

+

+ return 0;

+}

+

+static inline size_t neigh_nlmsg_size(void)

+{

+ return NLMSG_ALIGN(sizeof(struct ndmsg))

+ + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */

+ + nla_total_size(MAX_ADDR_LEN) /* NDA_LLADDR */

+ + nla_total_size(sizeof(struct nda_cacheinfo))

+ + nla_total_size(4) /* NDA_PROBES */

+ + nla_total_size(1); /* NDA_PROTOCOL */

+}

+

+static int neigh_get_reply(struct net *net, struct neighbour *neigh,

+ u32 pid, u32 seq)

+{

+ struct sk_buff *skb;

+ int err = 0;

+

+ skb = nlmsg_new(neigh_nlmsg_size(), GFP_KERNEL);

+ if (!skb)

+ return -ENOBUFS;

+

+ err = neigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0);

+ if (err) {

+ kfree_skb(skb);

+ goto errout;

+ }

+

+ err = rtnl_unicast(skb, net, pid);

+errout:

+ return err;

+}

+

+static inline size_t pneigh_nlmsg_size(void)

+{

+ return NLMSG_ALIGN(sizeof(struct ndmsg))

+ + nla_total_size(MAX_ADDR_LEN) /* NDA_DST */

+ + nla_total_size(1); /* NDA_PROTOCOL */

+}

+

+static int pneigh_get_reply(struct net *net, struct pneigh_entry *neigh,

+ u32 pid, u32 seq, struct neigh_table *tbl)

+{

+ struct sk_buff *skb;

+ int err = 0;

+

+ skb = nlmsg_new(pneigh_nlmsg_size(), GFP_KERNEL);

+ if (!skb)

+ return -ENOBUFS;

+

+ err = pneigh_fill_info(skb, neigh, pid, seq, RTM_NEWNEIGH, 0, tbl);

+ if (err) {

+ kfree_skb(skb);

+ goto errout;

+ }

+

+ err = rtnl_unicast(skb, net, pid);

+errout:

+ return err;

+}

+

+static int neigh_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,

+ struct netlink_ext_ack *extack)

+{

+ struct net *net = sock_net(in_skb->sk);

+ struct net_device *dev = NULL;

+ struct neigh_table *tbl = NULL;

+ struct neighbour *neigh;

+ void *dst = NULL;

+ u8 ndm_flags = 0;

+ int dev_idx = 0;

+ int err;

+

+ err = neigh_valid_get_req(nlh, &tbl, &dst, &dev_idx, &ndm_flags,

+ extack);

+ if (err < 0)

+ return err;

+

+ if (dev_idx) {

+ dev = __dev_get_by_index(net, dev_idx);

+ if (!dev) {

+ NL_SET_ERR_MSG(extack, "Unknown device ifindex");

+ return -ENODEV;

+ }

+ }

+

+ if (!dst) {

+ NL_SET_ERR_MSG(extack, "Network address not specified");

+ return -EINVAL;

+ }

+

+ if (ndm_flags & NTF_PROXY) {

+ struct pneigh_entry *pn;

+

+ pn = pneigh_lookup(tbl, net, dst, dev, 0);

+ if (!pn) {

+ NL_SET_ERR_MSG(extack, "Proxy neighbour entry not found");

+ return -ENOENT;

+ }

+ return pneigh_get_reply(net, pn, NETLINK_CB(in_skb).portid,

+ nlh->nlmsg_seq, tbl);

+ }

+

+ if (!dev) {

+ NL_SET_ERR_MSG(extack, "No device specified");

+ return -EINVAL;

+ }

+

+ neigh = neigh_lookup(tbl, dst, dev);

+ if (!neigh) {

+ NL_SET_ERR_MSG(extack, "Neighbour entry not found");

+ return -ENOENT;

+ }

+

+ err = neigh_get_reply(net, neigh, NETLINK_CB(in_skb).portid,

+ nlh->nlmsg_seq);

+

+ neigh_release(neigh);

+

+ return err;

+}

+

+ neigh_mark_dead(n);

+ rtnl_register(PF_UNSPEC, RTM_GETNEIGH, neigh_get, neigh_dump_info, 0);

+ return dev_change_flags(dev, (unsigned int)new_flags, NULL);

+ [NETNSA_TARGET_NSID] = { .type = NLA_S32 },

+ + nla_total_size(sizeof(s32)) /* NETNSA_CURRENT_NSID */

+struct net_fill_args {

+ u32 portid;

+ u32 seq;

+ int flags;

+ int cmd;

+ int nsid;

+ bool add_ref;

+ int ref_nsid;

+};

+

+static int rtnl_net_fill(struct sk_buff *skb, struct net_fill_args *args)

+ nlh = nlmsg_put(skb, args->portid, args->seq, args->cmd, sizeof(*rth),

+ args->flags);

+ if (nla_put_s32(skb, NETNSA_NSID, args->nsid))

+ goto nla_put_failure;

+

+ if (args->add_ref &&

+ nla_put_s32(skb, NETNSA_CURRENT_NSID, args->ref_nsid))

+ struct net_fill_args fillargs = {

+ .portid = NETLINK_CB(skb).portid,

+ .seq = nlh->nlmsg_seq,

+ .cmd = RTM_NEWNSID,

+ };

+ struct net *peer, *target = net;

+ int err;

+ } else if (tb[NETNSA_NSID]) {

+ peer = get_net_ns_by_id(net, nla_get_u32(tb[NETNSA_NSID]));

+ if (!peer)

+ peer = ERR_PTR(-ENOENT);

+ nla = tb[NETNSA_NSID];

+ if (tb[NETNSA_TARGET_NSID]) {

+ int id = nla_get_s32(tb[NETNSA_TARGET_NSID]);

+

+ target = rtnl_get_net_ns_capable(NETLINK_CB(skb).sk, id);

+ if (IS_ERR(target)) {

+ NL_SET_BAD_ATTR(extack, tb[NETNSA_TARGET_NSID]);

+ NL_SET_ERR_MSG(extack,

+ "Target netns reference is invalid");

+ err = PTR_ERR(target);

+ goto out;

+ }

+ fillargs.add_ref = true;

+ fillargs.ref_nsid = peernet2id(net, peer);

+ }

+

+ fillargs.nsid = peernet2id(target, peer);

+ err = rtnl_net_fill(msg, &fillargs);

+ if (fillargs.add_ref)

+ put_net(target);

+ struct net *tgt_net;

+ struct net *ref_net;

+ struct net_fill_args fillargs;

+ net_cb->fillargs.nsid = id;

+ if (net_cb->fillargs.add_ref)

+ net_cb->fillargs.ref_nsid = __peernet2id(net_cb->ref_net, peer);

+ ret = rtnl_net_fill(net_cb->skb, &net_cb->fillargs);

+static int rtnl_valid_dump_net_req(const struct nlmsghdr *nlh, struct sock *sk,

+ struct rtnl_net_dump_cb *net_cb,

+ struct netlink_callback *cb)

+{

+ struct netlink_ext_ack *extack = cb->extack;

+ struct nlattr *tb[NETNSA_MAX + 1];

+ int err, i;

+

+ err = nlmsg_parse_strict(nlh, sizeof(struct rtgenmsg), tb, NETNSA_MAX,

+ rtnl_net_policy, extack);

+ if (err < 0)

+ return err;

+

+ for (i = 0; i <= NETNSA_MAX; i++) {

+ if (!tb[i])

+ continue;

+

+ if (i == NETNSA_TARGET_NSID) {

+ struct net *net;

+

+ net = rtnl_get_net_ns_capable(sk, nla_get_s32(tb[i]));

+ if (IS_ERR(net)) {

+ NL_SET_BAD_ATTR(extack, tb[i]);

+ NL_SET_ERR_MSG(extack,

+ "Invalid target network namespace id");

+ return PTR_ERR(net);

+ }

+ net_cb->fillargs.add_ref = true;

+ net_cb->ref_net = net_cb->tgt_net;

+ net_cb->tgt_net = net;

+ } else {

+ NL_SET_BAD_ATTR(extack, tb[i]);

+ NL_SET_ERR_MSG(extack,

+ "Unsupported attribute in dump request");

+ return -EINVAL;

+ }

+ }

+

+ return 0;

+}

+

+ .tgt_net = sock_net(skb->sk),

+ .fillargs = {

+ .portid = NETLINK_CB(cb->skb).portid,

+ .seq = cb->nlh->nlmsg_seq,

+ .flags = NLM_F_MULTI,

+ .cmd = RTM_NEWNSID,

+ },

+ int err = 0;

+ if (cb->strict_check) {

+ err = rtnl_valid_dump_net_req(cb->nlh, skb->sk, &net_cb, cb);

+ if (err < 0)

+ goto end;

+ spin_lock_bh(&net_cb.tgt_net->nsid_lock);

+ if (net_cb.fillargs.add_ref &&

+ !net_eq(net_cb.ref_net, net_cb.tgt_net) &&

+ !spin_trylock_bh(&net_cb.ref_net->nsid_lock)) {

+ spin_unlock_bh(&net_cb.tgt_net->nsid_lock);

+ err = -EAGAIN;

+ goto end;

+ }

+ idr_for_each(&net_cb.tgt_net->netns_ids, rtnl_net_dumpid_one, &net_cb);

+ if (net_cb.fillargs.add_ref &&

+ !net_eq(net_cb.ref_net, net_cb.tgt_net))

+ spin_unlock_bh(&net_cb.ref_net->nsid_lock);

+ spin_unlock_bh(&net_cb.tgt_net->nsid_lock);

+end:

+ if (net_cb.fillargs.add_ref)

+ put_net(net_cb.tgt_net);

+ return err < 0 ? err : skb->len;

+ struct net_fill_args fillargs = {

+ .cmd = cmd,

+ .nsid = id,

+ };

+ err = rtnl_net_fill(msg, &fillargs);

+ if (register_pernet_subsys(&net_ns_ops))

+ panic("Could not register network namespace subsystems");

+ err = dev_open(ndev, NULL);

+ if (!!(ipv6_addr_type(&ifp->addr) & IPV6_ADDR_LINKLOCAL) !=

+ !!(ipv6_addr_type(&np->remote_ip.in6) & IPV6_ADDR_LINKLOCAL))

+ call_rcu(&npinfo->rcu, rcu_cleanup_netpoll_info);

+ synchronize_rcu();

+#define RTNL_MAX_TYPE 50

+ err = dev_set_mac_address(dev, sa, extack);

+ err = dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),

+ extack);

+ err = __dev_change_flags(dev, rtnl_dev_combine_flags(dev, ifm),

+ NULL);

+struct net_device *rtnl_create_link(struct net *net, const char *ifname,

+ unsigned char name_assign_type,

+ const struct rtnl_link_ops *ops,

+ struct nlattr *tb[],

+ struct netlink_ext_ack *extack)

+ if (num_tx_queues < 1 || num_tx_queues > 4096) {

+ NL_SET_ERR_MSG(extack, "Invalid number of transmit queues");

+ }

+ if (num_rx_queues < 1 || num_rx_queues > 4096) {

+ NL_SET_ERR_MSG(extack, "Invalid number of receive queues");

+ }

+static int __rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,

+ struct nlattr **attr, struct netlink_ext_ack *extack)

+ struct nlattr *slave_attr[RTNL_SLAVE_MAX_TYPE + 1];

+ unsigned char name_assign_type = NET_NAME_USER;

+ struct nlattr *linkinfo[IFLA_INFO_MAX + 1];

+ const struct rtnl_link_ops *m_ops = NULL;

+ struct net_device *master_dev = NULL;

+ struct nlattr *tb[IFLA_MAX + 1];

+ struct net *dest_net, *link_net;

+ struct nlattr **slave_data;

+ char kind[MODULE_NAME_LEN];

+ struct nlattr **data;

+ data = NULL;

+ if (ops) {

+ if (ops->maxtype > RTNL_MAX_TYPE)

+ return -EINVAL;

+ if (ops->maxtype && linkinfo[IFLA_INFO_DATA]) {

+ err = nla_parse_nested(attr, ops->maxtype,

+ linkinfo[IFLA_INFO_DATA],

+ ops->policy, extack);

+ if (err < 0)

+ return err;

+ data = attr;

+ }

+ if (ops->validate) {

+ err = ops->validate(tb, data, extack);

+ if (err < 0)

+ return err;

+ }

+ slave_data = NULL;

+ if (m_ops) {

+ if (m_ops->slave_maxtype > RTNL_SLAVE_MAX_TYPE)

+ return -EINVAL;

+ if (m_ops->slave_maxtype &&

+ linkinfo[IFLA_INFO_SLAVE_DATA]) {

+ err = nla_parse_nested(slave_attr, m_ops->slave_maxtype,

+ linkinfo[IFLA_INFO_SLAVE_DATA],

+ m_ops->slave_policy, extack);

+ if (err < 0)

+ return err;

+ slave_data = slave_attr;

+ }

+ if (dev) {

+ int status = 0;

+ if (nlh->nlmsg_flags & NLM_F_EXCL)

+ return -EEXIST;

+ if (nlh->nlmsg_flags & NLM_F_REPLACE)

+ return -EOPNOTSUPP;

+ if (linkinfo[IFLA_INFO_DATA]) {

+ if (!ops || ops != dev->rtnl_link_ops ||

+ !ops->changelink)

+ return -EOPNOTSUPP;

+ err = ops->changelink(dev, tb, data, extack);

+ if (err < 0)

+ return err;

+ status |= DO_SETLINK_NOTIFY;

+ }

+ if (linkinfo[IFLA_INFO_SLAVE_DATA]) {

+ if (!m_ops || !m_ops->slave_changelink)

+ return -EOPNOTSUPP;

+ err = m_ops->slave_changelink(master_dev, dev, tb,

+ slave_data, extack);

+ if (err < 0)

+ return err;

+ status |= DO_SETLINK_NOTIFY;

+ return do_setlink(skb, dev, ifm, extack, tb, ifname, status);

+ }

+

+ if (!(nlh->nlmsg_flags & NLM_F_CREATE)) {

+ if (ifm->ifi_index == 0 && tb[IFLA_GROUP])

+ return rtnl_group_changelink(skb, net,

+ return -ENODEV;

+ }

+ if (tb[IFLA_MAP] || tb[IFLA_PROTINFO])

+ return -EOPNOTSUPP;

+ if (!ops) {

+ if (kind[0]) {

+ __rtnl_unlock();

+ request_module("rtnl-link-%s", kind);

+ rtnl_lock();

+ ops = rtnl_link_ops_get(kind);

+ if (ops)

+ goto replay;

+#endif

+ NL_SET_ERR_MSG(extack, "Unknown device type");

+ return -EOPNOTSUPP;

+ }

+ if (!ops->setup)

+ return -EOPNOTSUPP;

+ if (!ifname[0]) {

+ snprintf(ifname, IFNAMSIZ, "%s%%d", ops->kind);

+ name_assign_type = NET_NAME_ENUM;

+ }

+ dest_net = rtnl_link_get_net_capable(skb, net, tb, CAP_NET_ADMIN);

+ if (IS_ERR(dest_net))

+ return PTR_ERR(dest_net);

+ if (tb[IFLA_LINK_NETNSID]) {

+ int id = nla_get_s32(tb[IFLA_LINK_NETNSID]);

+ link_net = get_net_ns_by_id(dest_net, id);

+ if (!link_net) {

+ NL_SET_ERR_MSG(extack, "Unknown network namespace id");

+ err = -EINVAL;

+ err = -EPERM;

+ if (!netlink_ns_capable(skb, link_net->user_ns, CAP_NET_ADMIN))

+ goto out;

+ } else {

+ link_net = NULL;

+ }

+

+ dev = rtnl_create_link(link_net ? : dest_net, ifname,

+ name_assign_type, ops, tb, extack);

+ if (IS_ERR(dev)) {

+ err = PTR_ERR(dev);

+ goto out;

+ }

+ dev->ifindex = ifm->ifi_index;

+ if (ops->newlink) {

+ err = ops->newlink(link_net ? : net, dev, tb, data, extack);

+ /* Drivers should call free_netdev() in ->destructor

+ * and unregister it on failure after registration

+ * so that device could be finally freed in rtnl_unlock.

+ */

+ if (err < 0) {

+ /* If device is not registered at all, free it now */

+ if (dev->reg_state == NETREG_UNINITIALIZED)

+ goto out;

+ } else {

+ err = register_netdevice(dev);

+ if (err < 0) {

+ free_netdev(dev);

+ goto out;

+ }

+ }

+ err = rtnl_configure_link(dev, ifm);

+ if (err < 0)

+ goto out_unregister;

+ if (link_net) {

+ err = dev_change_net_namespace(dev, dest_net, ifname);

+ }

+ if (tb[IFLA_MASTER]) {

+ err = do_set_master(dev, nla_get_u32(tb[IFLA_MASTER]), extack);

+ if (err)

+ goto out_unregister;

+ }

+ if (link_net)

+ put_net(link_net);

+ put_net(dest_net);

+ return err;

+ if (ops->newlink) {

+ LIST_HEAD(list_kill);

+ ops->dellink(dev, &list_kill);

+ unregister_netdevice_many(&list_kill);

+ } else {

+ unregister_netdevice(dev);

+ goto out;

+}

+

+static int rtnl_newlink(struct sk_buff *skb, struct nlmsghdr *nlh,

+ struct netlink_ext_ack *extack)

+{

+ struct nlattr **attr;

+ int ret;

+

+ attr = kmalloc_array(RTNL_MAX_TYPE + 1, sizeof(*attr), GFP_KERNEL);

+ if (!attr)

+ return -ENOMEM;

+

+ ret = __rtnl_newlink(skb, nlh, attr, extack);

+ kfree(attr);

+ return ret;

+ if (ret)

+ if (dev->type != ARPHRD_ETHER)

+ return -EINVAL;

+

+static int valid_fdb_get_strict(const struct nlmsghdr *nlh,

+ struct nlattr **tb, u8 *ndm_flags,

+ int *br_idx, int *brport_idx, u8 **addr,

+ u16 *vid, struct netlink_ext_ack *extack)

+{

+ struct ndmsg *ndm;

+ int err, i;

+

+ if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*ndm))) {

+ NL_SET_ERR_MSG(extack, "Invalid header for fdb get request");

+ return -EINVAL;

+ }

+

+ ndm = nlmsg_data(nlh);

+ if (ndm->ndm_pad1 || ndm->ndm_pad2 || ndm->ndm_state ||

+ ndm->ndm_type) {

+ NL_SET_ERR_MSG(extack, "Invalid values in header for fdb get request");

+ return -EINVAL;

+ }

+

+ if (ndm->ndm_flags & ~(NTF_MASTER | NTF_SELF)) {

+ NL_SET_ERR_MSG(extack, "Invalid flags in header for fdb get request");

+ return -EINVAL;

+ }

+

+ err = nlmsg_parse_strict(nlh, sizeof(struct ndmsg), tb, NDA_MAX,

+ nda_policy, extack);

+ if (err < 0)

+ return err;

+

+ *ndm_flags = ndm->ndm_flags;

+ *brport_idx = ndm->ndm_ifindex;

+ for (i = 0; i <= NDA_MAX; ++i) {

+ if (!tb[i])

+ continue;

+

+ switch (i) {

+ case NDA_MASTER:

+ *br_idx = nla_get_u32(tb[i]);

+ break;

+ case NDA_LLADDR:

+ if (nla_len(tb[i]) != ETH_ALEN) {

+ NL_SET_ERR_MSG(extack, "Invalid address in fdb get request");

+ return -EINVAL;

+ }

+ *addr = nla_data(tb[i]);

+ break;

+ case NDA_VLAN:

+ err = fdb_vid_parse(tb[i], vid, extack);

+ if (err)

+ return err;

+ break;

+ case NDA_VNI:

+ break;

+ default:

+ NL_SET_ERR_MSG(extack, "Unsupported attribute in fdb get request");

+ return -EINVAL;

+ }

+ }

+

+ return 0;

+}

+

+static int rtnl_fdb_get(struct sk_buff *in_skb, struct nlmsghdr *nlh,

+ struct netlink_ext_ack *extack)

+{

+ struct net_device *dev = NULL, *br_dev = NULL;

+ const struct net_device_ops *ops = NULL;

+ struct net *net = sock_net(in_skb->sk);

+ struct nlattr *tb[NDA_MAX + 1];

+ struct sk_buff *skb;

+ int brport_idx = 0;

+ u8 ndm_flags = 0;

+ int br_idx = 0;

+ u8 *addr = NULL;

+ u16 vid = 0;

+ int err;

+

+ err = valid_fdb_get_strict(nlh, tb, &ndm_flags, &br_idx,

+ &brport_idx, &addr, &vid, extack);

+ if (err < 0)

+ return err;

+

+ if (!addr) {

+ NL_SET_ERR_MSG(extack, "Missing lookup address for fdb get request");

+ return -EINVAL;

+ }

+

+ if (brport_idx) {

+ dev = __dev_get_by_index(net, brport_idx);

+ if (!dev) {

+ NL_SET_ERR_MSG(extack, "Unknown device ifindex");

+ return -ENODEV;

+ }

+ }

+

+ if (br_idx) {

+ if (dev) {

+ NL_SET_ERR_MSG(extack, "Master and device are mutually exclusive");

+ return -EINVAL;

+ }

+

+ br_dev = __dev_get_by_index(net, br_idx);

+ if (!br_dev) {

+ NL_SET_ERR_MSG(extack, "Invalid master ifindex");

+ return -EINVAL;

+ }

+ ops = br_dev->netdev_ops;

+ }

+

+ if (dev) {

+ if (!ndm_flags || (ndm_flags & NTF_MASTER)) {

+ if (!(dev->priv_flags & IFF_BRIDGE_PORT)) {

+ NL_SET_ERR_MSG(extack, "Device is not a bridge port");

+ return -EINVAL;

+ }

+ br_dev = netdev_master_upper_dev_get(dev);

+ if (!br_dev) {

+ NL_SET_ERR_MSG(extack, "Master of device not found");

+ return -EINVAL;

+ }

+ ops = br_dev->netdev_ops;

+ } else {

+ if (!(ndm_flags & NTF_SELF)) {

+ NL_SET_ERR_MSG(extack, "Missing NTF_SELF");

+ return -EINVAL;

+ }

+ ops = dev->netdev_ops;

+ }

+ }

+

+ if (!br_dev && !dev) {

+ NL_SET_ERR_MSG(extack, "No device specified");

+ return -ENODEV;

+ }

+

+ if (!ops || !ops->ndo_fdb_get) {

+ NL_SET_ERR_MSG(extack, "Fdb get operation not supported by device");

+ return -EOPNOTSUPP;

+ }

+

+ skb = nlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);

+ if (!skb)

+ return -ENOBUFS;

+

+ if (br_dev)

+ dev = br_dev;

+ err = ops->ndo_fdb_get(skb, tb, dev, addr, vid,

+ NETLINK_CB(in_skb).portid,

+ nlh->nlmsg_seq, extack);

+ if (err)

+ goto out;

+

+ return rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);

+out:

+ kfree_skb(skb);

+ return err;

+}

+

+ err = br_dev->netdev_ops->ndo_bridge_setlink(dev, nlh, flags,

+ extack);

+ flags,

+ extack);

+ rtnl_register(PF_BRIDGE, RTM_GETNEIGH, rtnl_fdb_get, rtnl_fdb_dump, 0);

+#ifdef CONFIG_SKB_EXTENSIONS

+static struct kmem_cache *skbuff_ext_cache __ro_after_init;

+#endif

+ skb_ext_put(skb);

+ __skb_ext_copy(new, old);

+void sock_zerocopy_put_abort(struct ubuf_info *uarg, bool have_uref)

+ if (have_uref)

+ sock_zerocopy_put(uarg);

+int skb_zerocopy_iter_dgram(struct sk_buff *skb, struct msghdr *msg, int len)

+{

+ return __zerocopy_sg_from_iter(skb->sk, skb, &msg->msg_iter, len);

+}

+EXPORT_SYMBOL_GPL(skb_zerocopy_iter_dgram);

+

+ skb_zcopy_set(skb, uarg, NULL);

+ skb_zcopy_set(nskb, skb_uarg(orig), NULL);

+__sum16 __skb_checksum_complete_head(struct sk_buff *skb, int len)

+{

+ __sum16 sum;

+

+ sum = csum_fold(skb_checksum(skb, 0, len, skb->csum));

+ /* See comments in __skb_checksum_complete(). */

+ if (likely(!sum)) {

+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&

+ !skb->csum_complete_sw)

+ netdev_rx_csum_fault(skb->dev, skb);

+ }

+ if (!skb_shared(skb))

+ skb->csum_valid = !sum;

+ return sum;

+}

+EXPORT_SYMBOL(__skb_checksum_complete_head);

+

+/* This function assumes skb->csum already holds pseudo header's checksum,

+ * which has been changed from the hardware checksum, for example, by

+ * __skb_checksum_validate_complete(). And, the original skb->csum must

+ * have been validated unsuccessfully for CHECKSUM_COMPLETE case.

+ *

+ * It returns non-zero if the recomputed checksum is still invalid, otherwise

+ * zero. The new checksum is stored back into skb->csum unless the skb is

+ * shared.

+ */

+__sum16 __skb_checksum_complete(struct sk_buff *skb)

+{

+ __wsum csum;

+ __sum16 sum;

+

+ csum = skb_checksum(skb, 0, skb->len, 0);

+

+ sum = csum_fold(csum_add(skb->csum, csum));

+ /* This check is inverted, because we already knew the hardware

+ * checksum is invalid before calling this function. So, if the

+ * re-computed checksum is valid instead, then we have a mismatch

+ * between the original skb->csum and skb_checksum(). This means either

+ * the original hardware checksum is incorrect or we screw up skb->csum

+ * when moving skb->data around.

+ */

+ if (likely(!sum)) {

+ if (unlikely(skb->ip_summed == CHECKSUM_COMPLETE) &&

+ !skb->csum_complete_sw)

+ netdev_rx_csum_fault(skb->dev, skb);

+ }

+

+ if (!skb_shared(skb)) {

+ /* Save full packet checksum */

+ skb->csum = csum;

+ skb->ip_summed = CHECKSUM_COMPLETE;

+ skb->csum_complete_sw = 1;

+ skb->csum_valid = !sum;

+ }

+

+ return sum;

+}

+EXPORT_SYMBOL(__skb_checksum_complete);

+

+#ifdef CONFIG_SKB_EXTENSIONS

+#define SKB_EXT_ALIGN_VALUE 8

+#define SKB_EXT_CHUNKSIZEOF(x) (ALIGN((sizeof(x)), SKB_EXT_ALIGN_VALUE) / SKB_EXT_ALIGN_VALUE)

+

+static const u8 skb_ext_type_len[] = {

+#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)

+ [SKB_EXT_BRIDGE_NF] = SKB_EXT_CHUNKSIZEOF(struct nf_bridge_info),

+#endif

+#ifdef CONFIG_XFRM

+ [SKB_EXT_SEC_PATH] = SKB_EXT_CHUNKSIZEOF(struct sec_path),

+#endif

+};

+

+static __always_inline unsigned int skb_ext_total_length(void)

+{

+ return SKB_EXT_CHUNKSIZEOF(struct skb_ext) +

+#if IS_ENABLED(CONFIG_BRIDGE_NETFILTER)

+ skb_ext_type_len[SKB_EXT_BRIDGE_NF] +

+#endif

+#ifdef CONFIG_XFRM

+ skb_ext_type_len[SKB_EXT_SEC_PATH] +

+#endif

+ 0;

+}

+

+static void skb_extensions_init(void)

+{

+ BUILD_BUG_ON(SKB_EXT_NUM >= 8);

+ BUILD_BUG_ON(skb_ext_total_length() > 255);

+

+ skbuff_ext_cache = kmem_cache_create("skbuff_ext_cache",

+ SKB_EXT_ALIGN_VALUE * skb_ext_total_length(),

+ 0,

+ SLAB_HWCACHE_ALIGN|SLAB_PANIC,

+ NULL);

+}

+#else

+static void skb_extensions_init(void) {}

+#endif

+

+ skb_extensions_init();

+#ifdef CONFIG_NET_SWITCHDEV

+ skb->offload_fwd_mark = 0;

+ skb->offload_l3_fwd_mark = 0;

+#endif

+

+ * @skb: GSO skb

+ *

+ __vlan_hwaccel_clear_tag(skb);

+

+#ifdef CONFIG_SKB_EXTENSIONS

+static void *skb_ext_get_ptr(struct skb_ext *ext, enum skb_ext_id id)

+{

+ return (void *)ext + (ext->offset[id] * SKB_EXT_ALIGN_VALUE);

+}

+

+static struct skb_ext *skb_ext_alloc(void)

+{

+ struct skb_ext *new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);

+

+ if (new) {

+ memset(new->offset, 0, sizeof(new->offset));

+ refcount_set(&new->refcnt, 1);

+ }

+

+ return new;

+}

+

+static struct skb_ext *skb_ext_maybe_cow(struct skb_ext *old,

+ unsigned int old_active)

+{

+ struct skb_ext *new;

+

+ if (refcount_read(&old->refcnt) == 1)

+ return old;

+

+ new = kmem_cache_alloc(skbuff_ext_cache, GFP_ATOMIC);

+ if (!new)

+ return NULL;

+

+ memcpy(new, old, old->chunks * SKB_EXT_ALIGN_VALUE);

+ refcount_set(&new->refcnt, 1);

+

+#ifdef CONFIG_XFRM

+ if (old_active & (1 << SKB_EXT_SEC_PATH)) {

+ struct sec_path *sp = skb_ext_get_ptr(old, SKB_EXT_SEC_PATH);

+ unsigned int i;

+

+ for (i = 0; i < sp->len; i++)

+ xfrm_state_hold(sp->xvec[i]);

+ }

+#endif

+ __skb_ext_put(old);

+ return new;

+}

+

+/**

+ * skb_ext_add - allocate space for given extension, COW if needed

+ * @skb: buffer

+ * @id: extension to allocate space for

+ *

+ * Allocates enough space for the given extension.

+ * If the extension is already present, a pointer to that extension

+ * is returned.

+ *

+ * If the skb was cloned, COW applies and the returned memory can be

+ * modified without changing the extension space of clones buffers.

+ *

+ * Returns pointer to the extension or NULL on allocation failure.

+ */

+void *skb_ext_add(struct sk_buff *skb, enum skb_ext_id id)

+{

+ struct skb_ext *new, *old = NULL;

+ unsigned int newlen, newoff;

+

+ if (skb->active_extensions) {

+ old = skb->extensions;

+

+ new = skb_ext_maybe_cow(old, skb->active_extensions);

+ if (!new)

+ return NULL;

+

+ if (__skb_ext_exist(new, id))

+ goto set_active;

+

+ newoff = new->chunks;

+ } else {

+ newoff = SKB_EXT_CHUNKSIZEOF(*new);

+

+ new = skb_ext_alloc();

+ if (!new)

+ return NULL;

+ }

+

+ newlen = newoff + skb_ext_type_len[id];

+ new->chunks = newlen;

+ new->offset[id] = newoff;

+set_active:

+ skb->extensions = new;

+ skb->active_extensions |= 1 << id;

+ return skb_ext_get_ptr(new, id);

+}

+EXPORT_SYMBOL(skb_ext_add);

+

+#ifdef CONFIG_XFRM

+static void skb_ext_put_sp(struct sec_path *sp)

+{

+ unsigned int i;

+

+ for (i = 0; i < sp->len; i++)

+ xfrm_state_put(sp->xvec[i]);

+}

+#endif

+

+void __skb_ext_del(struct sk_buff *skb, enum skb_ext_id id)

+{

+ struct skb_ext *ext = skb->extensions;

+

+ skb->active_extensions &= ~(1 << id);

+ if (skb->active_extensions == 0) {

+ skb->extensions = NULL;

+ __skb_ext_put(ext);

+#ifdef CONFIG_XFRM

+ } else if (id == SKB_EXT_SEC_PATH &&

+ refcount_read(&ext->refcnt) == 1) {

+ struct sec_path *sp = skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH);

+

+ skb_ext_put_sp(sp);

+ sp->len = 0;

+#endif

+ }

+}

+EXPORT_SYMBOL(__skb_ext_del);

+

+void __skb_ext_put(struct skb_ext *ext)

+{

+ /* If this is last clone, nothing can increment

+ * it after check passes. Avoids one atomic op.

+ */

+ if (refcount_read(&ext->refcnt) == 1)

+ goto free_now;

+

+ if (!refcount_dec_and_test(&ext->refcnt))

+ return;

+free_now:

+#ifdef CONFIG_XFRM

+ if (__skb_ext_exist(ext, SKB_EXT_SEC_PATH))

+ skb_ext_put_sp(skb_ext_get_ptr(ext, SKB_EXT_SEC_PATH));

+#endif

+

+ kmem_cache_free(skbuff_ext_cache, ext);

+}

+EXPORT_SYMBOL(__skb_ext_put);

+#endif /* CONFIG_SKB_EXTENSIONS */

+ if (sk_msg_full(dst))

+ return -ENOSPC;

+

+ sk_psock_data_ready(sk, psock);

+ sk_psock_zap_ingress(psock);

+ call_rcu(&psock->rcu, sk_psock_destroy);

+ case __SK_PASS:

+ sk_other = psock->sk;

+ if (sock_flag(sk_other, SOCK_DEAD) ||

+ !sk_psock_test_state(psock, SK_PSOCK_TX_ENABLED)) {

+ goto out_free;

+ }

+ if (atomic_read(&sk_other->sk_rmem_alloc) <=

+ sk_other->sk_rcvbuf) {

+ struct tcp_skb_cb *tcp = TCP_SKB_CB(skb);

+

+ tcp->bpf.flags |= BPF_F_INGRESS;

+ skb_queue_tail(&psock->ingress_skb, skb);

+ schedule_work(&psock->work);

+ break;

+ }

+ goto out_free;

+static void sk_psock_strp_data_ready(struct sock *sk)

+ sk->sk_data_ready = sk_psock_strp_data_ready;

+ if (sk->sk_prot->rehash)

+ sk->sk_prot->rehash(sk);

+ sk_dst_reset(sk);

+ if (!ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN)) {

+ } else if (val != sk->sk_mark) {

+ sk_dst_reset(sk);

+ }

+ if (!((sk->sk_type == SOCK_STREAM &&

+ sk->sk_protocol == IPPROTO_TCP) ||

+ (sk->sk_type == SOCK_DGRAM &&

+ sk->sk_protocol == IPPROTO_UDP)))

+#if BITS_PER_LONG==32

+ seqlock_init(&sk->sk_stamp_seq);

+#endif

+ tv = ktime_to_timeval(sock_read_timestamp(sk));

+ ktime_t kt = ktime_get_real();

+ sock_write_timestamp(sk, kt);

+ tv = ktime_to_timeval(kt);

+ ts = ktime_to_timespec(sock_read_timestamp(sk));

+ ktime_t kt = ktime_get_real();

+ sock_write_timestamp(sk, kt);

+ protocol != IPPROTO_RAW &&

+EXPORT_SYMBOL(reuseport_add_sock);

+ if (__sk_stream_is_writeable(sk, 1) && sock) {

+static long long_one __maybe_unused = 1;

+static long long_max __maybe_unused = LONG_MAX;

+

+static int

+proc_dolongvec_minmax_bpf_restricted(struct ctl_table *table, int write,

+ void __user *buffer, size_t *lenp,

+ loff_t *ppos)

+{

+ if (!capable(CAP_SYS_ADMIN))

+ return -EPERM;

+

+ return proc_doulongvec_minmax(table, write, buffer, lenp, ppos);

+}

+ .maxlen = sizeof(long),

+ .proc_handler = proc_dolongvec_minmax_bpf_restricted,

+ .extra1 = &long_one,

+ .extra2 = &long_max,