struct net contains:
struct nf_hook_entries __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
which store the hook entry point locations for the various protocol
families and the hooks.
Using array results in compact c code when doing accesses, i.e.
x = rcu_dereference(net->nf.hooks[pf][hook]);
but its also wasting a lot of memory, as most families are
not used.
So split the array into those families that are used, which
are only 5 (instead of 13). In most cases, the 'pf' argument is
constant, i.e. gcc removes switch statement.
struct net before:
/* size: 5184, cachelines: 81, members: 46 */
after:
/* size: 4672, cachelines: 73, members: 46 */
Signed-off-by: Florian Westphal <fw@xxxxxxxxx>
---
include/linux/netfilter.h | 24 ++++++++++++++++++++++--
include/net/netns/netfilter.h | 6 +++++-
net/bridge/br_netfilter_hooks.c | 2 +-
net/netfilter/core.c | 39 +++++++++++++++++++++++++++++++--------
net/netfilter/nf_queue.c | 19 ++++++++++++++++++-
5 files changed, 77 insertions(+), 13 deletions(-)
diff --git a/include/linux/netfilter.h b/include/linux/netfilter.h
index b24e9b101651..80aa9a0b3d10 100644
--- a/include/linux/netfilter.h
+++ b/include/linux/netfilter.h
@@ -184,7 +184,7 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
struct net_device *indev, struct net_device *outdev,
int (*okfn)(struct net *, struct sock *, struct sk_buff *))
{
- struct nf_hook_entries *hook_head;
+ struct nf_hook_entries *hook_head = NULL;
int ret = 1;
#ifdef HAVE_JUMP_LABEL
@@ -195,7 +195,27 @@ static inline int nf_hook(u_int8_t pf, unsigned int hook, struct net *net,
#endif
rcu_read_lock();
- hook_head = rcu_dereference(net->nf.hooks[pf][hook]);
+ switch (pf) {
+ case NFPROTO_IPV4:
+ hook_head = rcu_dereference(net->nf.hooks_ipv4[hook]);
+ break;
+ case NFPROTO_IPV6:
+ hook_head = rcu_dereference(net->nf.hooks_ipv6[hook]);
+ break;
+ case NFPROTO_ARP:
+ hook_head = rcu_dereference(net->nf.hooks_arp[hook]);
+ break;
+ case NFPROTO_BRIDGE:
+ hook_head = rcu_dereference(net->nf.hooks_bridge[hook]);
+ break;
+ case NFPROTO_DECNET:
+ hook_head = rcu_dereference(net->nf.hooks_decnet[hook]);
+ break;
+ default:
+ WARN_ON_ONCE(1);
+ break;
+ }
+
if (hook_head) {
struct nf_hook_state state;
diff --git a/include/net/netns/netfilter.h b/include/net/netns/netfilter.h
index cc00af2ac2d7..b39c563c2fce 100644
--- a/include/net/netns/netfilter.h
+++ b/include/net/netns/netfilter.h
@@ -17,7 +17,11 @@ struct netns_nf {
#ifdef CONFIG_SYSCTL
struct ctl_table_header *nf_log_dir_header;
#endif
- struct nf_hook_entries __rcu *hooks[NFPROTO_NUMPROTO][NF_MAX_HOOKS];
+ struct nf_hook_entries __rcu *hooks_ipv4[NF_MAX_HOOKS];
+ struct nf_hook_entries __rcu *hooks_ipv6[NF_MAX_HOOKS];
+ struct nf_hook_entries __rcu *hooks_arp[NF_MAX_HOOKS];
+ struct nf_hook_entries __rcu *hooks_bridge[NF_MAX_HOOKS];
+ struct nf_hook_entries __rcu *hooks_decnet[NF_MAX_HOOKS];
#if IS_ENABLED(CONFIG_NF_DEFRAG_IPV4)
bool defrag_ipv4;
#endif
diff --git a/net/bridge/br_netfilter_hooks.c b/net/bridge/br_netfilter_hooks.c
index c2eea1b8737a..27f1d4f2114a 100644
--- a/net/bridge/br_netfilter_hooks.c
+++ b/net/bridge/br_netfilter_hooks.c
@@ -991,7 +991,7 @@ int br_nf_hook_thresh(unsigned int hook, struct net *net,
unsigned int i;
int ret;
- e = rcu_dereference(net->nf.hooks[NFPROTO_BRIDGE][hook]);
+ e = rcu_dereference(net->nf.hooks_bridge[hook]);
if (!e)
return okfn(net, sk, skb);
diff --git a/net/netfilter/core.c b/net/netfilter/core.c
index 52cd2901a097..fd5f550dc625 100644
--- a/net/netfilter/core.c
+++ b/net/netfilter/core.c
@@ -239,8 +239,23 @@ static void *__nf_hook_entries_try_shrink(struct nf_hook_entries __rcu **pp)
static struct nf_hook_entries __rcu **nf_hook_entry_head(struct net *net, const struct nf_hook_ops *reg)
{
- if (reg->pf != NFPROTO_NETDEV)
- return net->nf.hooks[reg->pf]+reg->hooknum;
+ switch (reg->pf) {
+ case NFPROTO_NETDEV:
+ break;
+ case NFPROTO_ARP:
+ return net->nf.hooks_arp+reg->hooknum;
+ case NFPROTO_BRIDGE:
+ return net->nf.hooks_bridge+reg->hooknum;
+ case NFPROTO_IPV4:
+ return net->nf.hooks_ipv4+reg->hooknum;
+ case NFPROTO_IPV6:
+ return net->nf.hooks_ipv6+reg->hooknum;
+ case NFPROTO_DECNET:
+ return net->nf.hooks_decnet+reg->hooknum;
+ default:
+ WARN_ON_ONCE(1);
+ return NULL;
+ }
#ifdef CONFIG_NETFILTER_INGRESS
if (reg->hooknum == NF_NETDEV_INGRESS) {
@@ -569,14 +584,22 @@ void (*nf_nat_decode_session_hook)(struct sk_buff *, struct flowi *);
EXPORT_SYMBOL(nf_nat_decode_session_hook);
#endif
-static int __net_init netfilter_net_init(struct net *net)
+
+static void __net_init __netfilter_net_init(struct nf_hook_entries *e[NF_MAX_HOOKS])
{
- int i, h;
+ int h;
- for (i = 0; i < ARRAY_SIZE(net->nf.hooks); i++) {
- for (h = 0; h < NF_MAX_HOOKS; h++)
- RCU_INIT_POINTER(net->nf.hooks[i][h], NULL);
- }
+ for (h = 0; h < NF_MAX_HOOKS; h++)
+ RCU_INIT_POINTER(e[h], NULL);
+}
+
+static int __net_init netfilter_net_init(struct net *net)
+{
+ __netfilter_net_init(net->nf.hooks_ipv4);
+ __netfilter_net_init(net->nf.hooks_ipv6);
+ __netfilter_net_init(net->nf.hooks_arp);
+ __netfilter_net_init(net->nf.hooks_bridge);
+ __netfilter_net_init(net->nf.hooks_decnet);
#ifdef CONFIG_PROC_FS
net->nf.proc_netfilter = proc_net_mkdir(net, "netfilter",
diff --git a/net/netfilter/nf_queue.c b/net/netfilter/nf_queue.c
index f7e21953b1de..4fa97febf4e1 100644
--- a/net/netfilter/nf_queue.c
+++ b/net/netfilter/nf_queue.c
@@ -204,6 +204,23 @@ static unsigned int nf_iterate(struct sk_buff *skb,
return NF_ACCEPT;
}
+static struct nf_hook_entries *nf_hook_entries_head(const struct net *net, u8 pf, u8 hooknum)
+{
+ switch (pf) {
+ case NFPROTO_BRIDGE:
+ return rcu_dereference(net->nf.hooks_bridge[hooknum]);
+ case NFPROTO_IPV4:
+ return rcu_dereference(net->nf.hooks_ipv4[hooknum]);
+ case NFPROTO_IPV6:
+ return rcu_dereference(net->nf.hooks_ipv6[hooknum]);
+ default:
+ WARN_ON_ONCE(1);
+ return NULL;
+ }
+
+ return NULL;
+}
+
/* Caller must hold rcu read-side lock */
void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
{
@@ -219,7 +236,7 @@ void nf_reinject(struct nf_queue_entry *entry, unsigned int verdict)
net = entry->state.net;
pf = entry->state.pf;
- hooks = rcu_dereference(net->nf.hooks[pf][entry->state.hook]);
+ hooks = nf_hook_entries_head(net, pf, entry->state.hook);
nf_queue_entry_release_refs(entry);
--
2.13.6
--
To unsubscribe from this list: send the line "unsubscribe netfilter-devel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
