On Wed, Jun 11, 2003 at 10:02:14AM +1000, herbert wrote:
>
> I've got a better idea. We can make flow_cache_flush take out all
> out-of-date entries as determined by genid. I'll code it tonight.
Here is the patch with the objectless flow_cache_flush plus the
percpu stuff.
--
Debian GNU/Linux 3.0 is out! ( http://www.debian.org/ )
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Index: kernel-source-2.5/include/net/flow.h
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/include/net/flow.h,v
retrieving revision 1.2
diff -u -r1.2 flow.h
--- kernel-source-2.5/include/net/flow.h 10 Jun 2003 09:19:10 -0000 1.2
+++ kernel-source-2.5/include/net/flow.h 11 Jun 2003 08:32:39 -0000
@@ -87,7 +87,7 @@
extern void *flow_cache_lookup(struct flowi *key, u16 family, u8 dir,
flow_resolve_t resolver);
-extern void flow_cache_flush(void *object);
+extern void flow_cache_flush(void);
extern atomic_t flow_cache_genid;
#endif
Index: kernel-source-2.5/net/core/flow.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/core/flow.c,v
retrieving revision 1.4
diff -u -r1.4 flow.c
--- kernel-source-2.5/net/core/flow.c 10 Jun 2003 09:19:17 -0000 1.4
+++ kernel-source-2.5/net/core/flow.c 11 Jun 2003 08:53:58 -0000
@@ -15,6 +15,9 @@
#include <linux/interrupt.h>
#include <linux/smp.h>
#include <linux/completion.h>
+#include <linux/percpu.h>
+#include <linux/bitops.h>
+#include <linux/notifier.h>
#include <net/flow.h>
#include <asm/atomic.h>
#include <asm/semaphore.h>
@@ -54,12 +57,13 @@
#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ)
struct flow_flush_info {
- void *object;
atomic_t cpuleft;
+ unsigned long cpumap;
struct completion completion;
};
-static struct tasklet_struct flow_flush_tasklets[NR_CPUS];
-static DECLARE_MUTEX(flow_flush_sem);
+static DEFINE_PER_CPU(struct tasklet_struct, flow_flush_tasklet) = { NULL };
+
+static unsigned long flow_cache_cpu_map;
static void flow_cache_new_hashrnd(unsigned long arg)
{
@@ -182,7 +186,7 @@
}
}
- if (!fle) {
+ if (!fle && test_bit(cpu, &flow_cache_cpu_map)) {
if (flow_count(cpu) > flow_hwm)
flow_cache_shrink(cpu);
@@ -224,18 +228,20 @@
static void flow_cache_flush_tasklet(unsigned long data)
{
struct flow_flush_info *info = (void *)data;
- void *object = info->object;
int i;
int cpu;
cpu = smp_processor_id();
for (i = 0; i < flow_hash_size; i++) {
- struct flow_cache_entry *fle, **flp;
+ struct flow_cache_entry *fle;
+
+ fle = flow_table[(cpu << flow_hash_shift) + i];
+ for (; fle; fle = fle->next) {
+ unsigned genid = atomic_read(&flow_cache_genid);
- flp = &flow_table[(cpu << flow_hash_shift) + i];
- for (; (fle = *flp) != NULL; flp = &fle->next) {
- if (fle->object != object)
+ if (!fle->object || fle->genid == genid)
continue;
+
fle->object = NULL;
atomic_dec(fle->object_ref);
}
@@ -252,23 +258,27 @@
struct tasklet_struct *tasklet;
cpu = smp_processor_id();
- tasklet = &flow_flush_tasklets[cpu];
- tasklet_init(tasklet, flow_cache_flush_tasklet, (unsigned long)info);
+ if (!test_bit(cpu, &info->cpumap))
+ return;
+
+ tasklet = &per_cpu(flow_flush_tasklet, cpu);
+ tasklet->data = (unsigned long)info;
tasklet_schedule(tasklet);
}
-void flow_cache_flush(void *object)
+void flow_cache_flush(void)
{
struct flow_flush_info info;
+ static DECLARE_MUTEX(flow_flush_sem);
- info.object = object;
- atomic_set(&info.cpuleft, num_online_cpus());
+ info.cpumap = flow_cache_cpu_map;
+ atomic_set(&info.cpuleft, hweight32(info.cpumap));
init_completion(&info.completion);
down(&flow_flush_sem);
- smp_call_function(flow_cache_flush_per_cpu, &info, 1, 0);
local_bh_disable();
+ smp_call_function(flow_cache_flush_per_cpu, &info, 1, 0);
flow_cache_flush_per_cpu(&info);
local_bh_enable();
@@ -277,6 +287,32 @@
up(&flow_flush_sem);
}
+static void __devinit flow_cache_cpu_online(int cpu)
+{
+ struct tasklet_struct *tasklet;
+
+ tasklet = &per_cpu(flow_flush_tasklet, cpu);
+ tasklet_init(tasklet, flow_cache_flush_tasklet, 0);
+
+ set_bit(cpu, &flow_cache_cpu_map);
+}
+
+static int __devinit flow_cache_cpu_notify(struct notifier_block *self,
+ unsigned long action, void *hcpu)
+{
+ unsigned long cpu = (unsigned long)cpu;
+ switch (action) {
+ case CPU_UP_PREPARE:
+ flow_cache_cpu_online(cpu);
+ break;
+ }
+ return NOTIFY_OK;
+}
+
+static struct notifier_block __devinitdata flow_cache_cpu_nb = {
+ .notifier_call = flow_cache_cpu_notify,
+};
+
static int __init flow_cache_init(void)
{
unsigned long order;
@@ -315,6 +351,9 @@
panic("Failed to allocate flow cache hash table\n");
memset(flow_table, 0, PAGE_SIZE << order);
+
+ flow_cache_cpu_online(smp_processor_id());
+ register_cpu_notifier(&flow_cache_cpu_nb);
return 0;
}
Index: kernel-source-2.5/net/xfrm/xfrm_policy.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/xfrm/xfrm_policy.c,v
retrieving revision 1.6
diff -u -r1.6 xfrm_policy.c
--- kernel-source-2.5/net/xfrm/xfrm_policy.c 10 Jun 2003 09:19:24 -0000 1.6
+++ kernel-source-2.5/net/xfrm/xfrm_policy.c 11 Jun 2003 08:32:47 -0000
@@ -214,7 +214,7 @@
atomic_dec(&policy->refcnt);
if (atomic_read(&policy->refcnt) > 1)
- flow_cache_flush(policy);
+ flow_cache_flush();
xfrm_pol_put(policy);
}
