From f5737cbadb7d07c4f71fc5f073ccc7d8d8985a8f Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 3 May 2019 17:01:36 +0200 Subject: net: use indirect calls helpers for ptype hook This avoids an indirect call per RX IPv6/IPv4 packet. Note that we don't want to use the indirect calls helper for taps. Signed-off-by: Paolo Abeni Signed-off-by: David S. Miller --- net/core/dev.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/net/core/dev.c b/net/core/dev.c index 22f2640f559a..108ac8137b9b 100644 --- a/net/core/dev.c +++ b/net/core/dev.c @@ -4987,7 +4987,8 @@ static int __netif_receive_skb_one_core(struct sk_buff *skb, bool pfmemalloc) ret = __netif_receive_skb_core(skb, pfmemalloc, &pt_prev); if (pt_prev) - ret = pt_prev->func(skb, skb->dev, pt_prev, orig_dev); + ret = INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, + skb->dev, pt_prev, orig_dev); return ret; } @@ -5033,7 +5034,8 @@ static inline void __netif_receive_skb_list_ptype(struct list_head *head, else list_for_each_entry_safe(skb, next, head, list) { skb_list_del_init(skb); - pt_prev->func(skb, skb->dev, pt_prev, orig_dev); + INDIRECT_CALL_INET(pt_prev->func, ipv6_rcv, ip_rcv, skb, + skb->dev, pt_prev, orig_dev); } } -- cgit v1.2.3 From 0e219ae48c3bbf382ef96adf3825457315728c03 Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 3 May 2019 17:01:37 +0200 Subject: net: use indirect calls helpers for L3 handler hooks So that we avoid another indirect call per RX packet in the common case. Signed-off-by: Paolo Abeni Signed-off-by: David S. Miller --- net/ipv4/ip_input.c | 6 +++++- net/ipv6/ip6_input.c | 7 ++++++- net/ipv6/tcp_ipv6.c | 3 ++- net/ipv6/udp.c | 3 ++- 4 files changed, 15 insertions(+), 4 deletions(-) diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c index 1132d6d1796a..8d78de4b0304 100644 --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -130,6 +130,7 @@ #include #include #include +#include #include #include @@ -188,6 +189,8 @@ bool ip_call_ra_chain(struct sk_buff *skb) return false; } +INDIRECT_CALLABLE_DECLARE(int udp_rcv(struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int tcp_v4_rcv(struct sk_buff *)); void ip_protocol_deliver_rcu(struct net *net, struct sk_buff *skb, int protocol) { const struct net_protocol *ipprot; @@ -205,7 +208,8 @@ resubmit: } nf_reset(skb); } - ret = ipprot->handler(skb); + ret = INDIRECT_CALL_2(ipprot->handler, tcp_v4_rcv, udp_rcv, + skb); if (ret < 0) { protocol = -ret; goto resubmit; diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index c7ed2b6d5a1d..adf06159837f 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -316,6 +317,9 @@ void ipv6_list_rcv(struct list_head *head, struct packet_type *pt, ip6_sublist_rcv(&sublist, curr_dev, curr_net); } +INDIRECT_CALLABLE_DECLARE(int udpv6_rcv(struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int tcp_v6_rcv(struct sk_buff *)); + /* * Deliver the packet to the host */ @@ -391,7 +395,8 @@ resubmit_final: !xfrm6_policy_check(NULL, XFRM_POLICY_IN, skb)) goto discard; - ret = ipprot->handler(skb); + ret = INDIRECT_CALL_2(ipprot->handler, tcp_v6_rcv, udpv6_rcv, + skb); if (ret > 0) { if (ipprot->flags & INET6_PROTO_FINAL) { /* Not an extension header, most likely UDP diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index 82018bdce863..d58bf84e0f9a 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -43,6 +43,7 @@ #include #include #include +#include #include #include @@ -1435,7 +1436,7 @@ static void tcp_v6_fill_cb(struct sk_buff *skb, const struct ipv6hdr *hdr, skb->tstamp || skb_hwtstamps(skb)->hwtstamp; } -static int tcp_v6_rcv(struct sk_buff *skb) +INDIRECT_CALLABLE_SCOPE int tcp_v6_rcv(struct sk_buff *skb) { struct sk_buff *skb_to_free; int sdif = inet6_sdif(skb); diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index 2464fba569b4..b3fcafaf5576 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -36,6 +36,7 @@ #include #include #include +#include #include #include @@ -1021,7 +1022,7 @@ static void udp_v6_early_demux(struct sk_buff *skb) } } -static __inline__ int udpv6_rcv(struct sk_buff *skb) +INDIRECT_CALLABLE_SCOPE int udpv6_rcv(struct sk_buff *skb) { return __udp6_lib_rcv(skb, &udp_table, IPPROTO_UDP); } -- cgit v1.2.3 From 97ff7ffb11fe7a859a490771e7ce23f1f335176b Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 3 May 2019 17:01:38 +0200 Subject: net: use indirect calls helpers at early demux stage So that we avoid another indirect call per RX packet, if early demux is enabled. Signed-off-by: Paolo Abeni Signed-off-by: David S. Miller --- net/ipv4/ip_input.c | 5 ++++- net/ipv6/ip6_input.c | 5 ++++- net/ipv6/tcp_ipv6.c | 2 +- net/ipv6/udp.c | 2 +- 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/net/ipv4/ip_input.c b/net/ipv4/ip_input.c index 8d78de4b0304..ed97724c5e33 100644 --- a/net/ipv4/ip_input.c +++ b/net/ipv4/ip_input.c @@ -309,6 +309,8 @@ drop: return true; } +INDIRECT_CALLABLE_DECLARE(int udp_v4_early_demux(struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(int tcp_v4_early_demux(struct sk_buff *)); static int ip_rcv_finish_core(struct net *net, struct sock *sk, struct sk_buff *skb, struct net_device *dev) { @@ -326,7 +328,8 @@ static int ip_rcv_finish_core(struct net *net, struct sock *sk, ipprot = rcu_dereference(inet_protos[protocol]); if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) { - err = edemux(skb); + err = INDIRECT_CALL_2(edemux, tcp_v4_early_demux, + udp_v4_early_demux, skb); if (unlikely(err)) goto drop_error; /* must reload iph, skb->head might have changed */ diff --git a/net/ipv6/ip6_input.c b/net/ipv6/ip6_input.c index adf06159837f..b50b1af1f530 100644 --- a/net/ipv6/ip6_input.c +++ b/net/ipv6/ip6_input.c @@ -48,6 +48,8 @@ #include #include +INDIRECT_CALLABLE_DECLARE(void udp_v6_early_demux(struct sk_buff *)); +INDIRECT_CALLABLE_DECLARE(void tcp_v6_early_demux(struct sk_buff *)); static void ip6_rcv_finish_core(struct net *net, struct sock *sk, struct sk_buff *skb) { @@ -58,7 +60,8 @@ static void ip6_rcv_finish_core(struct net *net, struct sock *sk, ipprot = rcu_dereference(inet6_protos[ipv6_hdr(skb)->nexthdr]); if (ipprot && (edemux = READ_ONCE(ipprot->early_demux))) - edemux(skb); + INDIRECT_CALL_2(edemux, tcp_v6_early_demux, + udp_v6_early_demux, skb); } if (!skb_valid_dst(skb)) ip6_route_input(skb); diff --git a/net/ipv6/tcp_ipv6.c b/net/ipv6/tcp_ipv6.c index d58bf84e0f9a..beaf28456301 100644 --- a/net/ipv6/tcp_ipv6.c +++ b/net/ipv6/tcp_ipv6.c @@ -1655,7 +1655,7 @@ do_time_wait: goto discard_it; } -static void tcp_v6_early_demux(struct sk_buff *skb) +INDIRECT_CALLABLE_SCOPE void tcp_v6_early_demux(struct sk_buff *skb) { const struct ipv6hdr *hdr; const struct tcphdr *th; diff --git a/net/ipv6/udp.c b/net/ipv6/udp.c index b3fcafaf5576..07fa579dfb96 100644 --- a/net/ipv6/udp.c +++ b/net/ipv6/udp.c @@ -981,7 +981,7 @@ static struct sock *__udp6_lib_demux_lookup(struct net *net, return NULL; } -static void udp_v6_early_demux(struct sk_buff *skb) +INDIRECT_CALLABLE_SCOPE void udp_v6_early_demux(struct sk_buff *skb) { struct net *net = dev_net(skb->dev); const struct udphdr *uh; -- cgit v1.2.3 From 8c3c447b3cec27cf6f77080f4d157d53b64e9555 Mon Sep 17 00:00:00 2001 From: Paolo Abeni Date: Fri, 3 May 2019 17:01:39 +0200 Subject: net: use indirect calls helpers at the socket layer This avoids an indirect call per {send,recv}msg syscall in the common (IPv6 or IPv4 socket) case. Signed-off-by: Paolo Abeni Signed-off-by: David S. Miller --- net/socket.c | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/net/socket.c b/net/socket.c index a180e1a9ff23..472fbefa5d9b 100644 --- a/net/socket.c +++ b/net/socket.c @@ -90,6 +90,7 @@ #include #include #include +#include #include #include @@ -108,6 +109,13 @@ #include #include +/* proto_ops for ipv4 and ipv6 use the same {recv,send}msg function */ +#if IS_ENABLED(CONFIG_INET) +#define INDIRECT_CALL_INET4(f, f1, ...) INDIRECT_CALL_1(f, f1, __VA_ARGS__) +#else +#define INDIRECT_CALL_INET4(f, f1, ...) f(__VA_ARGS__) +#endif + #ifdef CONFIG_NET_RX_BUSY_POLL unsigned int sysctl_net_busy_read __read_mostly; unsigned int sysctl_net_busy_poll __read_mostly; @@ -645,10 +653,12 @@ EXPORT_SYMBOL(__sock_tx_timestamp); * Sends @msg through @sock, passing through LSM. * Returns the number of bytes sent, or an error code. */ - +INDIRECT_CALLABLE_DECLARE(int inet_sendmsg(struct socket *, struct msghdr *, + size_t)); static inline int sock_sendmsg_nosec(struct socket *sock, struct msghdr *msg) { - int ret = sock->ops->sendmsg(sock, msg, msg_data_left(msg)); + int ret = INDIRECT_CALL_INET4(sock->ops->sendmsg, inet_sendmsg, sock, + msg, msg_data_left(msg)); BUG_ON(ret == -EIOCBQUEUED); return ret; } @@ -874,11 +884,13 @@ EXPORT_SYMBOL_GPL(__sock_recv_ts_and_drops); * Receives @msg from @sock, passing through LSM. Returns the total number * of bytes received, or an error. */ - +INDIRECT_CALLABLE_DECLARE(int inet_recvmsg(struct socket *, struct msghdr *, + size_t , int )); static inline int sock_recvmsg_nosec(struct socket *sock, struct msghdr *msg, int flags) { - return sock->ops->recvmsg(sock, msg, msg_data_left(msg), flags); + return INDIRECT_CALL_INET4(sock->ops->recvmsg, inet_recvmsg, sock, msg, + msg_data_left(msg), flags); } int sock_recvmsg(struct socket *sock, struct msghdr *msg, int flags) -- cgit v1.2.3