Optimize the page_lock_anon_vma() fast path to be one LOCKed op,
instead of two.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@xxxxxxxxx>
---
mm/rmap.c | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++----
1 file changed, 67 insertions(+), 4 deletions(-)
Index: linux-2.6/mm/rmap.c
===================================================================
--- linux-2.6.orig/mm/rmap.c
+++ linux-2.6/mm/rmap.c
@@ -357,20 +357,75 @@ out:
return anon_vma;
}
+/*
+ * Similar to page_get_anon_vma() except it locks the anon_vma.
+ *
+ * Its a little more complex as it tries to keep the fast path to a single
+ * atomic op -- the trylock. If we fail the trylock, we fall back to getting a
+ * reference like with page_get_anon_vma() and then block on the mutex.
+ */
struct anon_vma *page_lock_anon_vma(struct page *page)
{
- struct anon_vma *anon_vma = page_get_anon_vma(page);
+ struct anon_vma *anon_vma = NULL;
+ unsigned long anon_mapping;
+
+ rcu_read_lock();
+ anon_mapping = (unsigned long) ACCESS_ONCE(page->mapping);
+ if ((anon_mapping & PAGE_MAPPING_FLAGS) != PAGE_MAPPING_ANON)
+ goto out;
+ if (!page_mapped(page))
+ goto out;
+
+ anon_vma = (struct anon_vma *) (anon_mapping - PAGE_MAPPING_ANON);
+ if (mutex_trylock(&anon_vma->root->lock)) {
+ /*
+ * If we observe a !0 refcount, then holding the lock ensures
+ * the anon_vma will not go away, see __put_anon_vma().
+ */
+ if (!atomic_read(&anon_vma->refcount)) {
+ anon_vma_unlock(anon_vma);
+ anon_vma = NULL;
+ }
+ goto out;
+ }
- if (anon_vma)
- anon_vma_lock(anon_vma);
+ /* trylock failed, we got to sleep */
+ if (!atomic_inc_not_zero(&anon_vma->refcount)) {
+ anon_vma = NULL;
+ goto out;
+ }
+
+ if (!page_mapped(page)) {
+ put_anon_vma(anon_vma);
+ anon_vma = NULL;
+ goto out;
+ }
+
+ /* we pinned the anon_vma, its safe to sleep */
+ rcu_read_unlock();
+ anon_vma_lock(anon_vma);
+
+ if (atomic_dec_and_test(&anon_vma->refcount)) {
+ /*
+ * Oops, we held the last refcount, release the lock
+ * and bail -- can't simply use put_anon_vma() because
+ * we'll deadlock on the anon_vma_lock() recursion.
+ */
+ anon_vma_unlock(anon_vma);
+ __put_anon_vma(anon_vma);
+ anon_vma = NULL;
+ }
return anon_vma;
+
+out:
+ rcu_read_unlock();
+ return anon_vma;
}
void page_unlock_anon_vma(struct anon_vma *anon_vma)
{
anon_vma_unlock(anon_vma);
- put_anon_vma(anon_vma);
}
/*
@@ -1462,6 +1517,14 @@ int try_to_munlock(struct page *page)
void __put_anon_vma(struct anon_vma *anon_vma)
{
+ /*
+ * Synchronize against page_lock_anon_vma() such that
+ * we can safely hold the lock without the anon_vma getting
+ * freed.
+ */
+ anon_vma_lock(anon_vma);
+ anon_vma_unlock(anon_vma);
+
if (anon_vma->root != anon_vma)
put_anon_vma(anon_vma->root);
anon_vma_free(anon_vma);
--
To unsubscribe, send a message with 'unsubscribe linux-mm' in
the body to majordomo@xxxxxxxxxx For more info on Linux MM,
see: http://www.linux-mm.org/ .
Fight unfair telecom policy in Canada: sign http://dissolvethecrtc.ca/
Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>