diff options
Diffstat (limited to 'include')
| -rw-r--r-- | include/linux/netdevice.h | 69 | ||||
| -rw-r--r-- | include/net/inet_sock.h | 38 |
2 files changed, 106 insertions, 1 deletions
diff --git a/include/linux/netdevice.h b/include/linux/netdevice.h index 55c2086e1f06..649a0252686e 100644 --- a/include/linux/netdevice.h +++ b/include/linux/netdevice.h @@ -530,14 +530,73 @@ struct rps_map { }; #define RPS_MAP_SIZE(_num) (sizeof(struct rps_map) + (_num * sizeof(u16))) +/* + * The rps_dev_flow structure contains the mapping of a flow to a CPU and the + * tail pointer for that CPU's input queue at the time of last enqueue. + */ +struct rps_dev_flow { + u16 cpu; + u16 fill; + unsigned int last_qtail; +}; + +/* + * The rps_dev_flow_table structure contains a table of flow mappings. + */ +struct rps_dev_flow_table { + unsigned int mask; + struct rcu_head rcu; + struct work_struct free_work; + struct rps_dev_flow flows[0]; +}; +#define RPS_DEV_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_dev_flow_table) + \ + (_num * sizeof(struct rps_dev_flow))) + +/* + * The rps_sock_flow_table contains mappings of flows to the last CPU + * on which they were processed by the application (set in recvmsg). + */ +struct rps_sock_flow_table { + unsigned int mask; + u16 ents[0]; +}; +#define RPS_SOCK_FLOW_TABLE_SIZE(_num) (sizeof(struct rps_sock_flow_table) + \ + (_num * sizeof(u16))) + +#define RPS_NO_CPU 0xffff + +static inline void rps_record_sock_flow(struct rps_sock_flow_table *table, + u32 hash) +{ + if (table && hash) { + unsigned int cpu, index = hash & table->mask; + + /* We only give a hint, preemption can change cpu under us */ + cpu = raw_smp_processor_id(); + + if (table->ents[index] != cpu) + table->ents[index] = cpu; + } +} + +static inline void rps_reset_sock_flow(struct rps_sock_flow_table *table, + u32 hash) +{ + if (table && hash) + table->ents[hash & table->mask] = RPS_NO_CPU; +} + +extern struct rps_sock_flow_table *rps_sock_flow_table; + /* This structure contains an instance of an RX queue. */ struct netdev_rx_queue { struct rps_map *rps_map; + struct rps_dev_flow_table *rps_flow_table; struct kobject kobj; struct netdev_rx_queue *first; atomic_t count; } ____cacheline_aligned_in_smp; -#endif +#endif /* CONFIG_RPS */ /* * This structure defines the management hooks for network devices. @@ -1333,11 +1392,19 @@ struct softnet_data { /* Elements below can be accessed between CPUs for RPS */ #ifdef CONFIG_RPS struct call_single_data csd ____cacheline_aligned_in_smp; + unsigned int input_queue_head; #endif struct sk_buff_head input_pkt_queue; struct napi_struct backlog; }; +static inline void incr_input_queue_head(struct softnet_data *queue) +{ +#ifdef CONFIG_RPS + queue->input_queue_head++; +#endif +} + DECLARE_PER_CPU_ALIGNED(struct softnet_data, softnet_data); #define HAVE_NETIF_QUEUE diff --git a/include/net/inet_sock.h b/include/net/inet_sock.h index 83fd34437cf1..b487bc1b99ab 100644 --- a/include/net/inet_sock.h +++ b/include/net/inet_sock.h @@ -21,6 +21,7 @@ #include <linux/string.h> #include <linux/types.h> #include <linux/jhash.h> +#include <linux/netdevice.h> #include <net/flow.h> #include <net/sock.h> @@ -101,6 +102,7 @@ struct rtable; * @uc_ttl - Unicast TTL * @inet_sport - Source port * @inet_id - ID counter for DF pkts + * @rxhash - flow hash received from netif layer * @tos - TOS * @mc_ttl - Multicasting TTL * @is_icsk - is this an inet_connection_sock? @@ -124,6 +126,9 @@ struct inet_sock { __u16 cmsg_flags; __be16 inet_sport; __u16 inet_id; +#ifdef CONFIG_RPS + __u32 rxhash; +#endif struct ip_options *opt; __u8 tos; @@ -219,4 +224,37 @@ static inline __u8 inet_sk_flowi_flags(const struct sock *sk) return inet_sk(sk)->transparent ? FLOWI_FLAG_ANYSRC : 0; } +static inline void inet_rps_record_flow(const struct sock *sk) +{ +#ifdef CONFIG_RPS + struct rps_sock_flow_table *sock_flow_table; + + rcu_read_lock(); + sock_flow_table = rcu_dereference(rps_sock_flow_table); + rps_record_sock_flow(sock_flow_table, inet_sk(sk)->rxhash); + rcu_read_unlock(); +#endif +} + +static inline void inet_rps_reset_flow(const struct sock *sk) +{ +#ifdef CONFIG_RPS + struct rps_sock_flow_table *sock_flow_table; + + rcu_read_lock(); + sock_flow_table = rcu_dereference(rps_sock_flow_table); + rps_reset_sock_flow(sock_flow_table, inet_sk(sk)->rxhash); + rcu_read_unlock(); +#endif +} + +static inline void inet_rps_save_rxhash(const struct sock *sk, u32 rxhash) +{ +#ifdef CONFIG_RPS + if (unlikely(inet_sk(sk)->rxhash != rxhash)) { + inet_rps_reset_flow(sk); + inet_sk(sk)->rxhash = rxhash; + } +#endif +} #endif /* _INET_SOCK_H */ |