On Mon, Jun 09, 2003 at 09:18:51PM +1000, herbert wrote:
>
> OK, I was mislead by the comment in interrupt.h into thinking that
> scheduled tasklet is not guaranteed to run on the same CPU.
Can you please confirm that this is the case?
> I'll rewrite the patch.
Here is the patch with tasklets. It's much simpler.
--
Debian GNU/Linux 3.0 is out! ( http://www.debian.org/ )
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Index: kernel-source-2.5/include/net/flow.h
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/include/net/flow.h,v
retrieving revision 1.1.1.3
diff -u -r1.1.1.3 flow.h
--- kernel-source-2.5/include/net/flow.h 27 May 2003 08:38:39 -0000 1.1.1.3
+++ kernel-source-2.5/include/net/flow.h 9 Jun 2003 09:25:22 -0000
@@ -87,6 +87,7 @@
extern void *flow_cache_lookup(struct flowi *key, u16 family, u8 dir,
flow_resolve_t resolver);
+extern void flow_cache_flush(void *object);
extern atomic_t flow_cache_genid;
#endif
Index: kernel-source-2.5/include/net/xfrm.h
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/include/net/xfrm.h,v
retrieving revision 1.3
diff -u -r1.3 xfrm.h
--- kernel-source-2.5/include/net/xfrm.h 7 Jun 2003 09:36:28 -0000 1.3
+++ kernel-source-2.5/include/net/xfrm.h 9 Jun 2003 09:43:31 -0000
@@ -266,6 +266,7 @@
struct xfrm_policy
{
struct xfrm_policy *next;
+ struct list_head list;
/* This lock only affects elements except for entry. */
rwlock_t lock;
Index: kernel-source-2.5/net/core/flow.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/core/flow.c,v
retrieving revision 1.3
diff -u -r1.3 flow.c
--- kernel-source-2.5/net/core/flow.c 2 Jun 2003 10:55:50 -0000 1.3
+++ kernel-source-2.5/net/core/flow.c 9 Jun 2003 12:01:06 -0000
@@ -12,8 +12,12 @@
#include <linux/random.h>
#include <linux/init.h>
#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/smp.h>
+#include <linux/completion.h>
#include <net/flow.h>
#include <asm/atomic.h>
+#include <asm/semaphore.h>
struct flow_cache_entry {
struct flow_cache_entry *next;
@@ -49,6 +53,14 @@
#define FLOW_HASH_RND_PERIOD (10 * 60 * HZ)
+struct flow_flush_info {
+ void *object;
+ atomic_t cpuleft;
+ struct completion completion;
+};
+static struct tasklet_struct flow_flush_tasklets[NR_CPUS];
+static DECLARE_MUTEX(flow_flush_sem);
+
static void flow_cache_new_hashrnd(unsigned long arg)
{
int i;
@@ -170,6 +182,22 @@
}
}
+ if (!fle) {
+ if (flow_count(cpu) > flow_hwm)
+ flow_cache_shrink(cpu);
+
+ fle = kmem_cache_alloc(flow_cachep, SLAB_ATOMIC);
+ if (fle) {
+ fle->next = *head;
+ *head = fle;
+ fle->family = family;
+ fle->dir = dir;
+ memcpy(&fle->key, key, sizeof(*key));
+ fle->object = NULL;
+ flow_count(cpu)++;
+ }
+ }
+
{
void *obj;
atomic_t *obj_ref;
@@ -186,30 +214,67 @@
fle->object_ref = obj_ref;
if (obj)
atomic_inc(fle->object_ref);
- } else {
- if (flow_count(cpu) > flow_hwm)
- flow_cache_shrink(cpu);
-
- fle = kmem_cache_alloc(flow_cachep, SLAB_ATOMIC);
- if (fle) {
- fle->next = *head;
- *head = fle;
- fle->family = family;
- fle->dir = dir;
- memcpy(&fle->key, key, sizeof(*key));
- fle->genid = atomic_read(&flow_cache_genid);
- fle->object = obj;
- fle->object_ref = obj_ref;
- if (obj)
- atomic_inc(fle->object_ref);
-
- flow_count(cpu)++;
- }
}
local_bh_enable();
return obj;
}
+}
+
+static void flow_cache_flush_tasklet(unsigned long data)
+{
+ struct flow_flush_info *info = (void *)data;
+ void *object = info->object;
+ int i;
+ int cpu;
+
+ cpu = smp_processor_id();
+ for (i = 0; i < flow_hash_size; i++) {
+ struct flow_cache_entry *fle, **flp;
+
+ flp = &flow_table[(cpu << flow_hash_shift) + i];
+ for (; (fle = *flp) != NULL; flp = &fle->next) {
+ if (fle->object != object)
+ continue;
+ fle->object = NULL;
+ atomic_dec(fle->object_ref);
+ }
+ }
+
+ if (atomic_dec_and_test(&info->cpuleft))
+ complete(&info->completion);
+}
+
+static void flow_cache_flush_per_cpu(void *data)
+{
+ struct flow_flush_info *info = data;
+ int cpu;
+ struct tasklet_struct *tasklet;
+
+ cpu = smp_processor_id();
+ tasklet = &flow_flush_tasklets[cpu];
+ tasklet_init(tasklet, flow_cache_flush_tasklet, (unsigned long)info);
+ tasklet_schedule(tasklet);
+}
+
+void flow_cache_flush(void *object)
+{
+ struct flow_flush_info info;
+
+ info.object = object;
+ atomic_set(&info.cpuleft, num_online_cpus());
+ init_completion(&info.completion);
+
+ down(&flow_flush_sem);
+
+ smp_call_function(flow_cache_flush_per_cpu, &info, 1, 0);
+ local_bh_disable();
+ flow_cache_flush_per_cpu(&info);
+ local_bh_enable();
+
+ wait_for_completion(&info.completion);
+
+ up(&flow_flush_sem);
}
static int __init flow_cache_init(void)
Index: kernel-source-2.5/net/key/af_key.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/key/af_key.c,v
retrieving revision 1.4
diff -u -r1.4 af_key.c
--- kernel-source-2.5/net/key/af_key.c 7 Jun 2003 09:36:28 -0000 1.4
+++ kernel-source-2.5/net/key/af_key.c 9 Jun 2003 08:42:11 -0000
@@ -2016,7 +2016,6 @@
out:
if (xp) {
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
}
return err;
}
@@ -2060,7 +2059,8 @@
if (xp) {
if (hdr->sadb_msg_type == SADB_X_SPDDELETE2)
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
+ else
+ xfrm_pol_put(xp);
}
return err;
}
Index: kernel-source-2.5/net/xfrm/xfrm_policy.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/xfrm/xfrm_policy.c,v
retrieving revision 1.5
diff -u -r1.5 xfrm_policy.c
--- kernel-source-2.5/net/xfrm/xfrm_policy.c 7 Jun 2003 09:36:28 -0000 1.5
+++ kernel-source-2.5/net/xfrm/xfrm_policy.c 9 Jun 2003 09:43:49 -0000
@@ -15,6 +15,9 @@
#include <linux/config.h>
#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
#include <net/xfrm.h>
#include <net/ip.h>
@@ -29,6 +32,11 @@
kmem_cache_t *xfrm_dst_cache;
+static struct work_struct xfrm_policy_gc_work;
+static struct list_head xfrm_policy_gc_list =
+ LIST_HEAD_INIT(xfrm_policy_gc_list);
+static spinlock_t xfrm_policy_gc_lock = SPIN_LOCK_UNLOCKED;
+
int xfrm_register_type(struct xfrm_type *type, unsigned short family)
{
struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
@@ -152,7 +160,6 @@
xp = xfrm_policy_byid(0, index, 1);
if (xp) {
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
}
}
@@ -194,27 +201,56 @@
kfree(policy);
}
+static void xfrm_policy_gc_kill(struct xfrm_policy *policy)
+{
+ struct dst_entry *dst;
+
+ while ((dst = policy->bundles) != NULL) {
+ policy->bundles = dst->next;
+ dst_free(dst);
+ }
+
+ if (del_timer(&policy->timer))
+ atomic_dec(&policy->refcnt);
+
+ if (atomic_read(&policy->refcnt) > 1)
+ flow_cache_flush(policy);
+
+ xfrm_pol_put(policy);
+}
+
+static void xfrm_policy_gc_task(void *data)
+{
+ struct xfrm_policy *policy;
+ struct list_head *entry, *tmp;
+ struct list_head gc_list = LIST_HEAD_INIT(gc_list);
+
+ spin_lock_bh(&xfrm_policy_gc_lock);
+ list_splice_init(&xfrm_policy_gc_list, &gc_list);
+ spin_unlock_bh(&xfrm_policy_gc_lock);
+
+ list_for_each_safe(entry, tmp, &gc_list) {
+ policy = list_entry(entry, struct xfrm_policy, list);
+ xfrm_policy_gc_kill(policy);
+ }
+}
+
/* Rule must be locked. Release descentant resources, announce
* entry dead. The rule must be unlinked from lists to the moment.
*/
void xfrm_policy_kill(struct xfrm_policy *policy)
{
- struct dst_entry *dst;
-
write_lock_bh(&policy->lock);
if (policy->dead)
goto out;
policy->dead = 1;
- while ((dst = policy->bundles) != NULL) {
- policy->bundles = dst->next;
- dst_free(dst);
- }
-
- if (del_timer(&policy->timer))
- atomic_dec(&policy->refcnt);
+ spin_lock(&xfrm_policy_gc_lock);
+ list_add(&policy->list, &xfrm_policy_gc_list);
+ spin_unlock(&xfrm_policy_gc_lock);
+ schedule_work(&xfrm_policy_gc_work);
out:
write_unlock_bh(&policy->lock);
@@ -282,7 +318,6 @@
if (delpol) {
xfrm_policy_kill(delpol);
- xfrm_pol_put(delpol);
}
return 0;
}
@@ -344,7 +379,6 @@
write_unlock_bh(&xfrm_policy_lock);
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
write_lock_bh(&xfrm_policy_lock);
}
@@ -388,8 +422,8 @@
/* Find policy to apply to this flow. */
-void xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
- void **objp, atomic_t **obj_refp)
+static void xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
+ void **objp, atomic_t **obj_refp)
{
struct xfrm_policy *pol;
@@ -469,7 +503,6 @@
if (old_pol) {
xfrm_policy_kill(old_pol);
- xfrm_pol_put(old_pol);
}
return 0;
}
@@ -516,7 +549,6 @@
write_unlock_bh(&xfrm_policy_lock);
xfrm_policy_kill(pol);
- xfrm_pol_put(pol);
}
/* Resolve list of templates for the flow, given policy. */
@@ -1135,6 +1167,8 @@
NULL, NULL);
if (!xfrm_dst_cache)
panic("XFRM: failed to allocate xfrm_dst_cache\n");
+
+ INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task, NULL);
}
void __init xfrm_init(void)
Index: kernel-source-2.5/net/xfrm/xfrm_user.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/xfrm/xfrm_user.c,v
retrieving revision 1.3
diff -u -r1.3 xfrm_user.c
--- kernel-source-2.5/net/xfrm/xfrm_user.c 7 Jun 2003 09:36:28 -0000 1.3
+++ kernel-source-2.5/net/xfrm/xfrm_user.c 9 Jun 2003 08:42:38 -0000
@@ -784,9 +784,8 @@
NETLINK_CB(skb).pid,
MSG_DONTWAIT);
}
+ xfrm_pol_put(xp);
}
-
- xfrm_pol_put(xp);
return err;
}
