On Mon, Jun 09, 2003 at 03:07:13AM -0700, David S. Miller wrote:
>
> Please don't use 2.5.x-only features in the IPSEC code of 2.5.x,
> in this case RCU.
>
> This makes it impossible for me to backport your work to 2.4.x
Good point.
I've now replaced rcu with smp_call_function.
Cheers,
--
Debian GNU/Linux 3.0 is out! ( http://www.debian.org/ )
Email: Herbert Xu ~{PmV>HI~} <herbert@gondor.apana.org.au>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt
Index: kernel-source-2.5/include/net/flow.h
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/include/net/flow.h,v
retrieving revision 1.1.1.3
diff -u -r1.1.1.3 flow.h
--- kernel-source-2.5/include/net/flow.h 27 May 2003 08:38:39 -0000 1.1.1.3
+++ kernel-source-2.5/include/net/flow.h 9 Jun 2003 09:25:22 -0000
@@ -87,6 +87,7 @@
extern void *flow_cache_lookup(struct flowi *key, u16 family, u8 dir,
flow_resolve_t resolver);
+extern void flow_cache_flush(void *object);
extern atomic_t flow_cache_genid;
#endif
Index: kernel-source-2.5/include/net/xfrm.h
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/include/net/xfrm.h,v
retrieving revision 1.3
diff -u -r1.3 xfrm.h
--- kernel-source-2.5/include/net/xfrm.h 7 Jun 2003 09:36:28 -0000 1.3
+++ kernel-source-2.5/include/net/xfrm.h 9 Jun 2003 09:43:31 -0000
@@ -266,6 +266,7 @@
struct xfrm_policy
{
struct xfrm_policy *next;
+ struct list_head list;
/* This lock only affects elements except for entry. */
rwlock_t lock;
Index: kernel-source-2.5/net/core/flow.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/core/flow.c,v
retrieving revision 1.3
diff -u -r1.3 flow.c
--- kernel-source-2.5/net/core/flow.c 2 Jun 2003 10:55:50 -0000 1.3
+++ kernel-source-2.5/net/core/flow.c 9 Jun 2003 10:52:39 -0000
@@ -12,6 +12,8 @@
#include <linux/random.h>
#include <linux/init.h>
#include <linux/slab.h>
+#include <linux/interrupt.h>
+#include <linux/smp.h>
#include <net/flow.h>
#include <asm/atomic.h>
@@ -64,7 +66,9 @@
{
struct flow_cache_entry *fle, **flp;
int i;
+ unsigned long flags;
+ local_irq_save(flags);
for (i = 0; i < flow_hash_size; i++) {
int k = 0;
@@ -81,6 +85,7 @@
flow_count(cpu)--;
}
}
+ local_irq_restore(flags);
}
static void flow_cache_shrink(int cpu)
@@ -142,7 +147,7 @@
void *flow_cache_lookup(struct flowi *key, u16 family, u8 dir,
flow_resolve_t resolver)
{
- struct flow_cache_entry *fle, **head;
+ struct flow_cache_entry *fle, **flp, **head;
unsigned int hash;
int cpu;
@@ -153,15 +158,21 @@
hash = flow_hash_code(key, cpu);
head = &flow_table[(cpu << flow_hash_shift) + hash];
- for (fle = *head; fle; fle = fle->next) {
+ for (flp = head; (fle = *flp) != NULL; flp = &fle->next) {
if (fle->family == family &&
fle->dir == dir &&
flow_key_compare(key, &fle->key) == 0) {
if (fle->genid == atomic_read(&flow_cache_genid)) {
- void *ret = fle->object;
+ void *ret;
+ unsigned long flags;
+ local_irq_save(flags);
+
+ ret = fle->object;
if (ret)
atomic_inc(fle->object_ref);
+
+ local_irq_restore(flags);
local_bh_enable();
return ret;
@@ -170,6 +181,20 @@
}
}
+ if (!fle) {
+ if (flow_count(cpu) > flow_hwm)
+ flow_cache_shrink(cpu);
+
+ fle = kmem_cache_alloc(flow_cachep, SLAB_ATOMIC);
+ if (fle) {
+ fle->next = *head;
+ fle->family = family;
+ fle->dir = dir;
+ memcpy(&fle->key, key, sizeof(*key));
+ flow_count(cpu)++;
+ }
+ }
+
{
void *obj;
atomic_t *obj_ref;
@@ -177,39 +202,58 @@
resolver(key, family, dir, &obj, &obj_ref);
if (fle) {
+ unsigned long flags;
+
fle->genid = atomic_read(&flow_cache_genid);
- if (fle->object)
+ local_irq_save(flags);
+
+ if (fle->next == *head)
+ *head = fle;
+ else if (fle->object)
atomic_dec(fle->object_ref);
fle->object = obj;
fle->object_ref = obj_ref;
if (obj)
atomic_inc(fle->object_ref);
- } else {
- if (flow_count(cpu) > flow_hwm)
- flow_cache_shrink(cpu);
-
- fle = kmem_cache_alloc(flow_cachep, SLAB_ATOMIC);
- if (fle) {
- fle->next = *head;
- *head = fle;
- fle->family = family;
- fle->dir = dir;
- memcpy(&fle->key, key, sizeof(*key));
- fle->genid = atomic_read(&flow_cache_genid);
- fle->object = obj;
- fle->object_ref = obj_ref;
- if (obj)
- atomic_inc(fle->object_ref);
- flow_count(cpu)++;
- }
+ local_irq_restore(flags);
}
+
local_bh_enable();
return obj;
}
+}
+
+static void flow_cache_flush_per_cpu(void *object)
+{
+ int i;
+ int cpu;
+ unsigned long flags;
+
+ cpu = smp_processor_id();
+ local_irq_save(flags);
+
+ for (i = 0; i < flow_hash_size; i++) {
+ struct flow_cache_entry *fle, **flp;
+
+ flp = &flow_table[(cpu << flow_hash_shift) + i];
+ for (; (fle = *flp) != NULL; flp = &fle->next) {
+ if (fle->object != object)
+ continue;
+ fle->object = NULL;
+ atomic_dec(fle->object_ref);
+ }
+ }
+
+ local_irq_restore(flags);
+}
+
+void flow_cache_flush(void *object)
+{
+ on_each_cpu(flow_cache_flush_per_cpu, object, 1, 1);
}
static int __init flow_cache_init(void)
Index: kernel-source-2.5/net/key/af_key.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/key/af_key.c,v
retrieving revision 1.4
diff -u -r1.4 af_key.c
--- kernel-source-2.5/net/key/af_key.c 7 Jun 2003 09:36:28 -0000 1.4
+++ kernel-source-2.5/net/key/af_key.c 9 Jun 2003 08:42:11 -0000
@@ -2016,7 +2016,6 @@
out:
if (xp) {
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
}
return err;
}
@@ -2060,7 +2059,8 @@
if (xp) {
if (hdr->sadb_msg_type == SADB_X_SPDDELETE2)
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
+ else
+ xfrm_pol_put(xp);
}
return err;
}
Index: kernel-source-2.5/net/xfrm/xfrm_policy.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/xfrm/xfrm_policy.c,v
retrieving revision 1.5
diff -u -r1.5 xfrm_policy.c
--- kernel-source-2.5/net/xfrm/xfrm_policy.c 7 Jun 2003 09:36:28 -0000 1.5
+++ kernel-source-2.5/net/xfrm/xfrm_policy.c 9 Jun 2003 09:43:49 -0000
@@ -15,6 +15,9 @@
#include <linux/config.h>
#include <linux/slab.h>
+#include <linux/list.h>
+#include <linux/spinlock.h>
+#include <linux/workqueue.h>
#include <net/xfrm.h>
#include <net/ip.h>
@@ -29,6 +32,11 @@
kmem_cache_t *xfrm_dst_cache;
+static struct work_struct xfrm_policy_gc_work;
+static struct list_head xfrm_policy_gc_list =
+ LIST_HEAD_INIT(xfrm_policy_gc_list);
+static spinlock_t xfrm_policy_gc_lock = SPIN_LOCK_UNLOCKED;
+
int xfrm_register_type(struct xfrm_type *type, unsigned short family)
{
struct xfrm_policy_afinfo *afinfo = xfrm_policy_get_afinfo(family);
@@ -152,7 +160,6 @@
xp = xfrm_policy_byid(0, index, 1);
if (xp) {
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
}
}
@@ -194,27 +201,56 @@
kfree(policy);
}
+static void xfrm_policy_gc_kill(struct xfrm_policy *policy)
+{
+ struct dst_entry *dst;
+
+ while ((dst = policy->bundles) != NULL) {
+ policy->bundles = dst->next;
+ dst_free(dst);
+ }
+
+ if (del_timer(&policy->timer))
+ atomic_dec(&policy->refcnt);
+
+ if (atomic_read(&policy->refcnt) > 1)
+ flow_cache_flush(policy);
+
+ xfrm_pol_put(policy);
+}
+
+static void xfrm_policy_gc_task(void *data)
+{
+ struct xfrm_policy *policy;
+ struct list_head *entry, *tmp;
+ struct list_head gc_list = LIST_HEAD_INIT(gc_list);
+
+ spin_lock_bh(&xfrm_policy_gc_lock);
+ list_splice_init(&xfrm_policy_gc_list, &gc_list);
+ spin_unlock_bh(&xfrm_policy_gc_lock);
+
+ list_for_each_safe(entry, tmp, &gc_list) {
+ policy = list_entry(entry, struct xfrm_policy, list);
+ xfrm_policy_gc_kill(policy);
+ }
+}
+
/* Rule must be locked. Release descentant resources, announce
* entry dead. The rule must be unlinked from lists to the moment.
*/
void xfrm_policy_kill(struct xfrm_policy *policy)
{
- struct dst_entry *dst;
-
write_lock_bh(&policy->lock);
if (policy->dead)
goto out;
policy->dead = 1;
- while ((dst = policy->bundles) != NULL) {
- policy->bundles = dst->next;
- dst_free(dst);
- }
-
- if (del_timer(&policy->timer))
- atomic_dec(&policy->refcnt);
+ spin_lock(&xfrm_policy_gc_lock);
+ list_add(&policy->list, &xfrm_policy_gc_list);
+ spin_unlock(&xfrm_policy_gc_lock);
+ schedule_work(&xfrm_policy_gc_work);
out:
write_unlock_bh(&policy->lock);
@@ -282,7 +318,6 @@
if (delpol) {
xfrm_policy_kill(delpol);
- xfrm_pol_put(delpol);
}
return 0;
}
@@ -344,7 +379,6 @@
write_unlock_bh(&xfrm_policy_lock);
xfrm_policy_kill(xp);
- xfrm_pol_put(xp);
write_lock_bh(&xfrm_policy_lock);
}
@@ -388,8 +422,8 @@
/* Find policy to apply to this flow. */
-void xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
- void **objp, atomic_t **obj_refp)
+static void xfrm_policy_lookup(struct flowi *fl, u16 family, u8 dir,
+ void **objp, atomic_t **obj_refp)
{
struct xfrm_policy *pol;
@@ -469,7 +503,6 @@
if (old_pol) {
xfrm_policy_kill(old_pol);
- xfrm_pol_put(old_pol);
}
return 0;
}
@@ -516,7 +549,6 @@
write_unlock_bh(&xfrm_policy_lock);
xfrm_policy_kill(pol);
- xfrm_pol_put(pol);
}
/* Resolve list of templates for the flow, given policy. */
@@ -1135,6 +1167,8 @@
NULL, NULL);
if (!xfrm_dst_cache)
panic("XFRM: failed to allocate xfrm_dst_cache\n");
+
+ INIT_WORK(&xfrm_policy_gc_work, xfrm_policy_gc_task, NULL);
}
void __init xfrm_init(void)
Index: kernel-source-2.5/net/xfrm/xfrm_user.c
===================================================================
RCS file: /home/gondolin/herbert/src/CVS/debian/kernel-source-2.5/net/xfrm/xfrm_user.c,v
retrieving revision 1.3
diff -u -r1.3 xfrm_user.c
--- kernel-source-2.5/net/xfrm/xfrm_user.c 7 Jun 2003 09:36:28 -0000 1.3
+++ kernel-source-2.5/net/xfrm/xfrm_user.c 9 Jun 2003 08:42:38 -0000
@@ -784,9 +784,8 @@
NETLINK_CB(skb).pid,
MSG_DONTWAIT);
}
+ xfrm_pol_put(xp);
}
-
- xfrm_pol_put(xp);
return err;
}
