The struct xt_counter contains two u64 values as counters for bytes and
packets. This type is exposed to userland via uapi. The kernel uses the
type as such when it communicates with userland.
However the type within an entry (such as ipt_entry) is treated
differently: Within the kernel it is a two value struct if the system
has only one CPU.
With more CPUs, the first value is per-CPU pointer which points to
per-CPU memory which holds the two u64 counter. How the struct
intepreted depends on the user.
Introduce a struct xt_counter_pad which is simply used as a place
holder, ensuring it is the same size as struct xt_counters. The kernel
function will use this type if the type might be a per-CPU pointer.
Add this padding struct to arpt_entry, ipt_entry and ip6t_entry.
Pass this type to xt_get_this_cpu_counter(), xt_percpu_counter_free()
and xt_percpu_counter_alloc(). These functions will cast it to union
xt_counter_k() and return the proper pointer to struct xt_counters_k.
This is mostly the same as previously but a bit more obvious and
introducs the struct xt_counters_k for in-kernel usage. This can be
replaces later without breaking userland.
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@xxxxxxxxxxxxx>
---
include/linux/netfilter/x_tables.h | 40 +++++++++++++------
include/uapi/linux/netfilter/x_tables.h | 4 ++
include/uapi/linux/netfilter_arp/arp_tables.h | 5 ++-
include/uapi/linux/netfilter_ipv4/ip_tables.h | 5 ++-
.../uapi/linux/netfilter_ipv6/ip6_tables.h | 5 ++-
net/ipv4/netfilter/arp_tables.c | 22 +++++-----
net/ipv4/netfilter/ip_tables.c | 22 +++++-----
net/ipv6/netfilter/ip6_tables.c | 22 +++++-----
net/netfilter/x_tables.c | 23 ++++++-----
9 files changed, 90 insertions(+), 58 deletions(-)
diff --git a/include/linux/netfilter/x_tables.h b/include/linux/netfilter/x_tables.h
index b9cd82e845d08..fc52a2ba90f6b 100644
--- a/include/linux/netfilter/x_tables.h
+++ b/include/linux/netfilter/x_tables.h
@@ -267,6 +267,17 @@ struct xt_table_info {
unsigned char entries[] __aligned(8);
};
+struct xt_counters_k {
+ /* Packet and byte counter */
+ __u64 pcnt;
+ __u64 bcnt;
+};
+
+union xt_counter_k {
+ struct xt_counters_k __percpu *pcpu;
+ struct xt_counters_k local;
+};
+
int xt_register_target(struct xt_target *target);
void xt_unregister_target(struct xt_target *target);
int xt_register_targets(struct xt_target *target, unsigned int n);
@@ -428,29 +439,32 @@ static inline unsigned long ifname_compare_aligned(const char *_a,
struct xt_percpu_counter_alloc_state {
unsigned int off;
- const char __percpu *mem;
+ void __percpu *mem;
};
bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
- struct xt_counters *counter);
-void xt_percpu_counter_free(struct xt_counters *cnt);
+ struct xt_counter_pad *xt_pad);
+void xt_percpu_counter_free(struct xt_counter_pad *xt_pad);
-static inline struct xt_counters *
-xt_get_this_cpu_counter(struct xt_counters *cnt)
+static inline struct xt_counters_k *xt_get_this_cpu_counter(struct xt_counter_pad *xt_pad)
{
- if (nr_cpu_ids > 1)
- return this_cpu_ptr((void __percpu *) (unsigned long) cnt->pcnt);
+ union xt_counter_k *xt_cnt = (union xt_counter_k *)xt_pad;
- return cnt;
+ if (nr_cpu_ids > 1)
+ return this_cpu_ptr(xt_cnt->pcpu);
+
+ return &xt_cnt->local;
}
-static inline struct xt_counters *
-xt_get_per_cpu_counter(struct xt_counters *cnt, unsigned int cpu)
+static inline struct xt_counters_k *xt_get_per_cpu_counter(struct xt_counter_pad *xt_pad,
+ unsigned int cpu)
{
- if (nr_cpu_ids > 1)
- return per_cpu_ptr((void __percpu *) (unsigned long) cnt->pcnt, cpu);
+ union xt_counter_k *xt_cnt = (union xt_counter_k *)xt_pad;
- return cnt;
+ if (nr_cpu_ids > 1)
+ return per_cpu_ptr(xt_cnt->pcpu, cpu);
+
+ return &xt_cnt->local;
}
struct nf_hook_ops *xt_hook_ops_alloc(const struct xt_table *, nf_hookfn *);
diff --git a/include/uapi/linux/netfilter/x_tables.h b/include/uapi/linux/netfilter/x_tables.h
index 796af83a963aa..70e19a140ab1e 100644
--- a/include/uapi/linux/netfilter/x_tables.h
+++ b/include/uapi/linux/netfilter/x_tables.h
@@ -111,6 +111,10 @@ struct xt_counters {
__u64 pcnt, bcnt; /* Packet and byte counters */
};
+struct xt_counter_pad {
+ __u8 pad[16];
+};
+
/* The argument to IPT_SO_ADD_COUNTERS. */
struct xt_counters_info {
/* Which table. */
diff --git a/include/uapi/linux/netfilter_arp/arp_tables.h b/include/uapi/linux/netfilter_arp/arp_tables.h
index a6ac2463f787a..4ca949a955412 100644
--- a/include/uapi/linux/netfilter_arp/arp_tables.h
+++ b/include/uapi/linux/netfilter_arp/arp_tables.h
@@ -106,7 +106,10 @@ struct arpt_entry
unsigned int comefrom;
/* Packet and byte counters. */
- struct xt_counters counters;
+ union {
+ struct xt_counters counters;
+ struct xt_counter_pad counter_pad;
+ };
/* The matches (if any), then the target. */
unsigned char elems[];
diff --git a/include/uapi/linux/netfilter_ipv4/ip_tables.h b/include/uapi/linux/netfilter_ipv4/ip_tables.h
index 1485df28b2391..a4874078ec058 100644
--- a/include/uapi/linux/netfilter_ipv4/ip_tables.h
+++ b/include/uapi/linux/netfilter_ipv4/ip_tables.h
@@ -118,7 +118,10 @@ struct ipt_entry {
unsigned int comefrom;
/* Packet and byte counters. */
- struct xt_counters counters;
+ union {
+ struct xt_counters counters;
+ struct xt_counter_pad counter_pad;
+ };
/* The matches (if any), then the target. */
unsigned char elems[];
diff --git a/include/uapi/linux/netfilter_ipv6/ip6_tables.h b/include/uapi/linux/netfilter_ipv6/ip6_tables.h
index 766e8e0bcc683..8634257e1cd59 100644
--- a/include/uapi/linux/netfilter_ipv6/ip6_tables.h
+++ b/include/uapi/linux/netfilter_ipv6/ip6_tables.h
@@ -122,7 +122,10 @@ struct ip6t_entry {
unsigned int comefrom;
/* Packet and byte counters. */
- struct xt_counters counters;
+ union {
+ struct xt_counters counters;
+ struct xt_counter_pad counter_pad;
+ };
/* The matches (if any), then the target. */
unsigned char elems[0];
diff --git a/net/ipv4/netfilter/arp_tables.c b/net/ipv4/netfilter/arp_tables.c
index 0628e68910f7f..ce3d73155ca9b 100644
--- a/net/ipv4/netfilter/arp_tables.c
+++ b/net/ipv4/netfilter/arp_tables.c
@@ -221,14 +221,14 @@ unsigned int arpt_do_table(void *priv,
arp = arp_hdr(skb);
do {
const struct xt_entry_target *t;
- struct xt_counters *counter;
+ struct xt_counters_k *counter;
if (!arp_packet_match(arp, skb->dev, indev, outdev, &e->arp)) {
e = arpt_next_entry(e);
continue;
}
- counter = xt_get_this_cpu_counter(&e->counters);
+ counter = xt_get_this_cpu_counter(&e->counter_pad);
ADD_COUNTER(*counter, arp_hdr_len(skb->dev), 1);
t = arpt_get_target_c(e);
@@ -412,7 +412,7 @@ find_check_entry(struct arpt_entry *e, struct net *net, const char *name,
struct xt_target *target;
int ret;
- if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
+ if (!xt_percpu_counter_alloc(alloc_state, &e->counter_pad))
return -ENOMEM;
t = arpt_get_target(e);
@@ -431,7 +431,7 @@ find_check_entry(struct arpt_entry *e, struct net *net, const char *name,
err:
module_put(t->u.kernel.target->me);
out:
- xt_percpu_counter_free(&e->counters);
+ xt_percpu_counter_free(&e->counter_pad);
return ret;
}
@@ -512,7 +512,7 @@ static void cleanup_entry(struct arpt_entry *e, struct net *net)
if (par.target->destroy != NULL)
par.target->destroy(&par);
module_put(par.target->me);
- xt_percpu_counter_free(&e->counters);
+ xt_percpu_counter_free(&e->counter_pad);
}
/* Checks and translates the user-supplied table segment (held in
@@ -611,11 +611,11 @@ static void get_counters(const struct xt_table_info *t,
i = 0;
xt_entry_foreach(iter, t->entries, t->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
u64 bcnt, pcnt;
unsigned int start;
- tmp = xt_get_per_cpu_counter(&iter->counters, cpu);
+ tmp = xt_get_per_cpu_counter(&iter->counter_pad, cpu);
do {
start = read_seqcount_begin(s);
bcnt = tmp->bcnt;
@@ -638,9 +638,9 @@ static void get_old_counters(const struct xt_table_info *t,
for_each_possible_cpu(cpu) {
i = 0;
xt_entry_foreach(iter, t->entries, t->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
- tmp = xt_get_per_cpu_counter(&iter->counters, cpu);
+ tmp = xt_get_per_cpu_counter(&iter->counter_pad, cpu);
ADD_COUNTER(counters[i], tmp->bcnt, tmp->pcnt);
++i;
}
@@ -1035,9 +1035,9 @@ static int do_add_counters(struct net *net, sockptr_t arg, unsigned int len)
addend = xt_write_recseq_begin();
xt_entry_foreach(iter, private->entries, private->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
- tmp = xt_get_this_cpu_counter(&iter->counters);
+ tmp = xt_get_this_cpu_counter(&iter->counter_pad);
ADD_COUNTER(*tmp, paddc[i].bcnt, paddc[i].pcnt);
++i;
}
diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c
index 20e8b46af8876..95f917f5bceef 100644
--- a/net/ipv4/netfilter/ip_tables.c
+++ b/net/ipv4/netfilter/ip_tables.c
@@ -278,7 +278,7 @@ ipt_do_table(void *priv,
do {
const struct xt_entry_target *t;
const struct xt_entry_match *ematch;
- struct xt_counters *counter;
+ struct xt_counters_k *counter;
WARN_ON(!e);
if (!ip_packet_match(ip, indev, outdev,
@@ -295,7 +295,7 @@ ipt_do_table(void *priv,
goto no_match;
}
- counter = xt_get_this_cpu_counter(&e->counters);
+ counter = xt_get_this_cpu_counter(&e->counter_pad);
ADD_COUNTER(*counter, skb->len, 1);
t = ipt_get_target_c(e);
@@ -525,7 +525,7 @@ find_check_entry(struct ipt_entry *e, struct net *net, const char *name,
struct xt_mtchk_param mtpar;
struct xt_entry_match *ematch;
- if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
+ if (!xt_percpu_counter_alloc(alloc_state, &e->counter_pad))
return -ENOMEM;
j = 0;
@@ -565,7 +565,7 @@ find_check_entry(struct ipt_entry *e, struct net *net, const char *name,
cleanup_match(ematch, net);
}
- xt_percpu_counter_free(&e->counters);
+ xt_percpu_counter_free(&e->counter_pad);
return ret;
}
@@ -653,7 +653,7 @@ cleanup_entry(struct ipt_entry *e, struct net *net)
if (par.target->destroy != NULL)
par.target->destroy(&par);
module_put(par.target->me);
- xt_percpu_counter_free(&e->counters);
+ xt_percpu_counter_free(&e->counter_pad);
}
/* Checks and translates the user-supplied table segment (held in
@@ -750,11 +750,11 @@ get_counters(const struct xt_table_info *t,
i = 0;
xt_entry_foreach(iter, t->entries, t->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
u64 bcnt, pcnt;
unsigned int start;
- tmp = xt_get_per_cpu_counter(&iter->counters, cpu);
+ tmp = xt_get_per_cpu_counter(&iter->counter_pad, cpu);
do {
start = read_seqcount_begin(s);
bcnt = tmp->bcnt;
@@ -777,9 +777,9 @@ static void get_old_counters(const struct xt_table_info *t,
for_each_possible_cpu(cpu) {
i = 0;
xt_entry_foreach(iter, t->entries, t->size) {
- const struct xt_counters *tmp;
+ const struct xt_counters_k *tmp;
- tmp = xt_get_per_cpu_counter(&iter->counters, cpu);
+ tmp = xt_get_per_cpu_counter(&iter->counter_pad, cpu);
ADD_COUNTER(counters[i], tmp->bcnt, tmp->pcnt);
++i; /* macro does multi eval of i */
}
@@ -1187,9 +1187,9 @@ do_add_counters(struct net *net, sockptr_t arg, unsigned int len)
i = 0;
addend = xt_write_recseq_begin();
xt_entry_foreach(iter, private->entries, private->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
- tmp = xt_get_this_cpu_counter(&iter->counters);
+ tmp = xt_get_this_cpu_counter(&iter->counter_pad);
ADD_COUNTER(*tmp, paddc[i].bcnt, paddc[i].pcnt);
++i;
}
diff --git a/net/ipv6/netfilter/ip6_tables.c b/net/ipv6/netfilter/ip6_tables.c
index c12d489a09840..f4877b1b2463e 100644
--- a/net/ipv6/netfilter/ip6_tables.c
+++ b/net/ipv6/netfilter/ip6_tables.c
@@ -300,7 +300,7 @@ ip6t_do_table(void *priv, struct sk_buff *skb,
do {
const struct xt_entry_target *t;
const struct xt_entry_match *ematch;
- struct xt_counters *counter;
+ struct xt_counters_k *counter;
WARN_ON(!e);
acpar.thoff = 0;
@@ -318,7 +318,7 @@ ip6t_do_table(void *priv, struct sk_buff *skb,
goto no_match;
}
- counter = xt_get_this_cpu_counter(&e->counters);
+ counter = xt_get_this_cpu_counter(&e->counter_pad);
ADD_COUNTER(*counter, skb->len, 1);
t = ip6t_get_target_c(e);
@@ -544,7 +544,7 @@ find_check_entry(struct ip6t_entry *e, struct net *net, const char *name,
struct xt_mtchk_param mtpar;
struct xt_entry_match *ematch;
- if (!xt_percpu_counter_alloc(alloc_state, &e->counters))
+ if (!xt_percpu_counter_alloc(alloc_state, &e->counter_pad))
return -ENOMEM;
j = 0;
@@ -583,7 +583,7 @@ find_check_entry(struct ip6t_entry *e, struct net *net, const char *name,
cleanup_match(ematch, net);
}
- xt_percpu_counter_free(&e->counters);
+ xt_percpu_counter_free(&e->counter_pad);
return ret;
}
@@ -670,7 +670,7 @@ static void cleanup_entry(struct ip6t_entry *e, struct net *net)
if (par.target->destroy != NULL)
par.target->destroy(&par);
module_put(par.target->me);
- xt_percpu_counter_free(&e->counters);
+ xt_percpu_counter_free(&e->counter_pad);
}
/* Checks and translates the user-supplied table segment (held in
@@ -767,11 +767,11 @@ get_counters(const struct xt_table_info *t,
i = 0;
xt_entry_foreach(iter, t->entries, t->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
u64 bcnt, pcnt;
unsigned int start;
- tmp = xt_get_per_cpu_counter(&iter->counters, cpu);
+ tmp = xt_get_per_cpu_counter(&iter->counter_pad, cpu);
do {
start = read_seqcount_begin(s);
bcnt = tmp->bcnt;
@@ -794,9 +794,9 @@ static void get_old_counters(const struct xt_table_info *t,
for_each_possible_cpu(cpu) {
i = 0;
xt_entry_foreach(iter, t->entries, t->size) {
- const struct xt_counters *tmp;
+ const struct xt_counters_k *tmp;
- tmp = xt_get_per_cpu_counter(&iter->counters, cpu);
+ tmp = xt_get_per_cpu_counter(&iter->counter_pad, cpu);
ADD_COUNTER(counters[i], tmp->bcnt, tmp->pcnt);
++i;
}
@@ -1203,9 +1203,9 @@ do_add_counters(struct net *net, sockptr_t arg, unsigned int len)
i = 0;
addend = xt_write_recseq_begin();
xt_entry_foreach(iter, private->entries, private->size) {
- struct xt_counters *tmp;
+ struct xt_counters_k *tmp;
- tmp = xt_get_this_cpu_counter(&iter->counters);
+ tmp = xt_get_this_cpu_counter(&iter->counter_pad);
ADD_COUNTER(*tmp, paddc[i].bcnt, paddc[i].pcnt);
++i;
}
diff --git a/net/netfilter/x_tables.c b/net/netfilter/x_tables.c
index ee38272cca562..5379ed82abd59 100644
--- a/net/netfilter/x_tables.c
+++ b/net/netfilter/x_tables.c
@@ -1889,7 +1889,7 @@ EXPORT_SYMBOL_GPL(xt_proto_fini);
* xt_percpu_counter_alloc - allocate x_tables rule counter
*
* @state: pointer to xt_percpu allocation state
- * @counter: pointer to counter struct inside the ip(6)/arpt_entry struct
+ * @xt_pad: pointer to the counter padding inside the ip(6)/arpt_entry struct
*
* On SMP, the packet counter [ ip(6)t_entry->counters.pcnt ] will then
* contain the address of the real (percpu) counter.
@@ -1908,9 +1908,13 @@ EXPORT_SYMBOL_GPL(xt_proto_fini);
* returns false on error.
*/
bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
- struct xt_counters *counter)
+ struct xt_counter_pad *xt_pad)
{
- BUILD_BUG_ON(XT_PCPU_BLOCK_SIZE < (sizeof(*counter) * 2));
+ union xt_counter_k *xt_cnt = (union xt_counter_k *)xt_pad;
+
+ BUILD_BUG_ON(XT_PCPU_BLOCK_SIZE < (sizeof(struct xt_counters_k) * 2));
+ BUILD_BUG_ON(sizeof(struct xt_counters_k) != sizeof(struct xt_counters));
+ BUILD_BUG_ON(sizeof(struct xt_counters_k) != sizeof(struct xt_counter_pad));
if (nr_cpu_ids <= 1)
return true;
@@ -1921,9 +1925,9 @@ bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
if (!state->mem)
return false;
}
- counter->pcnt = (__force unsigned long)(state->mem + state->off);
- state->off += sizeof(*counter);
- if (state->off > (XT_PCPU_BLOCK_SIZE - sizeof(*counter))) {
+ xt_cnt->pcpu = state->mem + state->off;
+ state->off += sizeof(struct xt_counters_k);
+ if (state->off > (XT_PCPU_BLOCK_SIZE - sizeof(struct xt_counters_k))) {
state->mem = NULL;
state->off = 0;
}
@@ -1931,12 +1935,13 @@ bool xt_percpu_counter_alloc(struct xt_percpu_counter_alloc_state *state,
}
EXPORT_SYMBOL_GPL(xt_percpu_counter_alloc);
-void xt_percpu_counter_free(struct xt_counters *counters)
+void xt_percpu_counter_free(struct xt_counter_pad *xt_pad)
{
- unsigned long pcnt = counters->pcnt;
+ union xt_counter_k *xt_cnt = (union xt_counter_k *)xt_pad;
+ unsigned long pcnt = (unsigned long)xt_cnt->pcpu;
if (nr_cpu_ids > 1 && (pcnt & (XT_PCPU_BLOCK_SIZE - 1)) == 0)
- free_percpu((void __percpu *)pcnt);
+ free_percpu(xt_cnt->pcpu);
}
EXPORT_SYMBOL_GPL(xt_percpu_counter_free);
--
2.47.2
