For the use cases where there are much more blocking readers than
writers, it will be beneficial performance-wise to use read/write lock
instead of a spinlock. However, read/write lock is non-deterministic
and can be problematic in some situations. So a complete conversion
of the underlying lock in seqlock to read/write lock will not be
appropriate.
This patch allows a seqlock user to decide to use either spinlock or
read/write lock as the underlying lock at initialization time. Once
the decision is made, it cannot be changed at a later time. To use an
underlying read/write lock, either the seqrwlock_init() function or
the DEFINE_SEQRWLOCK() macro have to be used at initialization time.
There is a slight overhead of an additional conditional branch with
that change, but it should be insignificant when compared with the
overhead of the actual locking and unlocking operations.
Signed-off-by: Waiman Long <Waiman.Long@xxxxxx>
---
include/linux/seqlock.h | 118 ++++++++++++++++++++++++++++++++++++----------
1 files changed, 92 insertions(+), 26 deletions(-)
diff --git a/include/linux/seqlock.h b/include/linux/seqlock.h
index 26be0d9..a1fd45c 100644
--- a/include/linux/seqlock.h
+++ b/include/linux/seqlock.h
@@ -20,7 +20,6 @@
* ...
* } while (read_seqretry(&foo, seq));
*
- *
* On non-SMP the spin locks disappear but the writer still needs
* to increment the sequence variables because an interrupt routine could
* change the state of the data.
@@ -176,28 +175,51 @@ static inline void write_seqcount_barrier(seqcount_t *s)
typedef struct {
struct seqcount seqcount;
- spinlock_t lock;
+ const bool use_rwlock;
+ union {
+ spinlock_t slock;
+ rwlock_t rwlock;
+ };
} seqlock_t;
/*
* These macros triggered gcc-3.x compile-time problems. We think these are
* OK now. Be cautious.
*/
-#define __SEQLOCK_UNLOCKED(lockname) \
- { \
- .seqcount = SEQCNT_ZERO, \
- .lock = __SPIN_LOCK_UNLOCKED(lockname) \
+#define __SEQLOCK_UNLOCKED(lockname) \
+ { \
+ .seqcount = SEQCNT_ZERO, \
+ .use_rwlock = false, \
+ { .slock = __SPIN_LOCK_UNLOCKED(lockname) } \
+ }
+
+#define __SEQRWLOCK_UNLOCKED(lockname) \
+ { \
+ .seqcount = SEQCNT_ZERO, \
+ .use_rwlock = true, \
+ { .rwlock = __RW_LOCK_UNLOCKED(lockname) } \
}
-#define seqlock_init(x) \
- do { \
- seqcount_init(&(x)->seqcount); \
- spin_lock_init(&(x)->lock); \
+#define seqlock_init(x) \
+ do { \
+ seqcount_init(&(x)->seqcount); \
+ spin_lock_init(&(x)->slock); \
+ *(bool *)(&(x)->use_rwlock) = false; \
+ } while (0)
+
+#define seqrwlock_init(x) \
+ do { \
+ seqcount_init(&(x)->seqcount); \
+ rwlock_init(&(x)->rwlock); \
+ *(bool *)(&(x)->use_rwlock) = true; \
} while (0)
#define DEFINE_SEQLOCK(x) \
seqlock_t x = __SEQLOCK_UNLOCKED(x)
+#define DEFINE_SEQRWLOCK(x) \
+ seqlock_t x = __SEQRWLOCK_UNLOCKED(x)
+
/*
* Read side functions for starting and finalizing a read side section.
*/
@@ -212,51 +234,86 @@ static inline unsigned read_seqretry(const seqlock_t *sl, unsigned start)
}
/*
+ * Locking and unlocking macros
+ */
+#define __SEQRLOCK(sl, suffix) \
+ do { \
+ if ((sl)->use_rwlock) \
+ read_lock ## suffix(&(sl)->rwlock); \
+ else \
+ spin_lock ## suffix(&(sl)->slock); \
+ } while (0)
+#define __SEQWLOCK(sl, suffix) \
+ do { \
+ if ((sl)->use_rwlock) \
+ write_lock ## suffix(&(sl)->rwlock); \
+ else \
+ spin_lock ## suffix(&(sl)->slock); \
+ } while (0)
+#define __SEQRUNLOCK(sl, suffix) \
+ do { \
+ if ((sl)->use_rwlock) \
+ read_unlock ## suffix(&(sl)->rwlock); \
+ else \
+ spin_unlock ## suffix(&(sl)->slock); \
+ } while (0)
+#define __SEQWUNLOCK(sl, suffix) \
+ do { \
+ if ((sl)->use_rwlock) \
+ write_unlock ## suffix(&(sl)->rwlock); \
+ else \
+ spin_unlock ## suffix(&(sl)->slock); \
+ } while (0)
+
+/*
* Lock out other writers and update the count.
* Acts like a normal spin_lock/unlock.
* Don't need preempt_disable() because that is in the spin_lock already.
*/
static inline void write_seqlock(seqlock_t *sl)
{
- spin_lock(&sl->lock);
+ __SEQWLOCK(sl, /**/);
write_seqcount_begin(&sl->seqcount);
}
static inline void write_sequnlock(seqlock_t *sl)
{
write_seqcount_end(&sl->seqcount);
- spin_unlock(&sl->lock);
+ __SEQWUNLOCK(sl, /**/);
}
static inline void write_seqlock_bh(seqlock_t *sl)
{
- spin_lock_bh(&sl->lock);
+ __SEQWLOCK(sl, _bh);
write_seqcount_begin(&sl->seqcount);
}
static inline void write_sequnlock_bh(seqlock_t *sl)
{
write_seqcount_end(&sl->seqcount);
- spin_unlock_bh(&sl->lock);
+ __SEQWUNLOCK(sl, _bh);
}
static inline void write_seqlock_irq(seqlock_t *sl)
{
- spin_lock_irq(&sl->lock);
+ __SEQWLOCK(sl, _irq);
write_seqcount_begin(&sl->seqcount);
}
static inline void write_sequnlock_irq(seqlock_t *sl)
{
write_seqcount_end(&sl->seqcount);
- spin_unlock_irq(&sl->lock);
+ __SEQWUNLOCK(sl, _irq);
}
static inline unsigned long __write_seqlock_irqsave(seqlock_t *sl)
{
unsigned long flags;
- spin_lock_irqsave(&sl->lock, flags);
+ if (sl->use_rwlock)
+ write_lock_irqsave(&sl->rwlock, flags);
+ else
+ spin_lock_irqsave(&sl->slock, flags);
write_seqcount_begin(&sl->seqcount);
return flags;
}
@@ -268,7 +325,10 @@ static inline void
write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
{
write_seqcount_end(&sl->seqcount);
- spin_unlock_irqrestore(&sl->lock, flags);
+ if (sl->use_rwlock)
+ write_unlock_irqrestore(&sl->rwlock, flags);
+ else
+ spin_unlock_irqrestore(&sl->slock, flags);
}
/*
@@ -278,39 +338,42 @@ write_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
*/
static inline void read_seqlock(seqlock_t *sl)
{
- spin_lock(&sl->lock);
+ __SEQRLOCK(sl, /**/);
}
static inline void read_sequnlock(seqlock_t *sl)
{
- spin_unlock(&sl->lock);
+ __SEQRUNLOCK(sl, /**/);
}
static inline void read_seqlock_bh(seqlock_t *sl)
{
- spin_lock_bh(&sl->lock);
+ __SEQRLOCK(sl, _bh);
}
static inline void read_sequnlock_bh(seqlock_t *sl)
{
- spin_unlock_bh(&sl->lock);
+ __SEQRUNLOCK(sl, _bh);
}
static inline void read_seqlock_irq(seqlock_t *sl)
{
- spin_lock_irq(&sl->lock);
+ __SEQRLOCK(sl, _irq);
}
static inline void read_sequnlock_irq(seqlock_t *sl)
{
- spin_unlock_irq(&sl->lock);
+ __SEQRUNLOCK(sl, _irq);
}
static inline unsigned long __read_seqlock_irqsave(seqlock_t *sl)
{
unsigned long flags;
- spin_lock_irqsave(&sl->lock, flags);
+ if (sl->use_rwlock)
+ read_lock_irqsave(&sl->rwlock, flags);
+ else
+ spin_lock_irqsave(&sl->slock, flags);
return flags;
}
@@ -320,7 +383,10 @@ static inline unsigned long __read_seqlock_irqsave(seqlock_t *sl)
static inline void
read_sequnlock_irqrestore(seqlock_t *sl, unsigned long flags)
{
- spin_unlock_irqrestore(&sl->lock, flags);
+ if (sl->use_rwlock)
+ read_unlock_irqrestore(&sl->rwlock, flags);
+ else
+ spin_unlock_irqrestore(&sl->slock, flags);
}
#endif /* __LINUX_SEQLOCK_H */
--
1.7.1
--
To unsubscribe from this list: send the line "unsubscribe linux-fsdevel" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
