The hugetlb pmd sharing code needs additional synchronization. Ideally,
i_mmap_rwsem would be used for this. However, previous attempts at using
i_mmap_rwsem have failed. This is partly due to conflicts with the existing
uses of i_mmap_rwsem that force a locking order not compatible with it's
use for pmd sharing.
Introduce a rwsem (hinode_rwsem) that resides in the hugetlb specific inode
for the purpose of this synchronization. This patch adds the semaphore to
the inode and also provides routines for using the semaphore. The routines
only acquire the semaphore if pmd sharing is possible in an attempt to
minimize performance impacts. In addition, routines which can be used with
lockdep to help with proper locking are also added.
Use of the new semaphore and supporting routines will be provided in a
subsequent patch.
Signed-off-by: Mike Kravetz <mike.kravetz@xxxxxxxxxx>
---
fs/hugetlbfs/inode.c | 12 ++++
include/linux/hugetlb.h | 121 ++++++++++++++++++++++++++++++++++++++++
mm/hugetlb.c | 13 -----
mm/rmap.c | 1 +
4 files changed, 134 insertions(+), 13 deletions(-)
diff --git a/fs/hugetlbfs/inode.c b/fs/hugetlbfs/inode.c
index 3a1246aeedc4..0460b43240b9 100644
--- a/fs/hugetlbfs/inode.c
+++ b/fs/hugetlbfs/inode.c
@@ -85,6 +85,17 @@ static const struct fs_parameter_spec hugetlb_fs_parameters[] = {
{}
};
+#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
+static inline void init_hinode_rwsem(struct hugetlbfs_inode_info *info)
+{
+ init_rwsem(&info->hinode_rwsem);
+}
+#else
+static inline void init_hinode_rwsem(struct hugetlbfs_inode_info *info)
+{
+}
+#endif
+
#ifdef CONFIG_NUMA
static inline void hugetlb_set_vma_policy(struct vm_area_struct *vma,
struct inode *inode, pgoff_t index)
@@ -829,6 +840,7 @@ static struct inode *hugetlbfs_get_inode(struct super_block *sb,
inode->i_atime = inode->i_mtime = inode->i_ctime = current_time(inode);
inode->i_mapping->private_data = resv_map;
info->seals = F_SEAL_SEAL;
+ init_hinode_rwsem(info);
switch (mode & S_IFMT) {
default:
init_special_inode(inode, mode, dev);
diff --git a/include/linux/hugetlb.h b/include/linux/hugetlb.h
index bf79a5601091..d3939a866855 100644
--- a/include/linux/hugetlb.h
+++ b/include/linux/hugetlb.h
@@ -414,6 +414,9 @@ struct hugetlbfs_inode_info {
struct shared_policy policy;
struct inode vfs_inode;
unsigned int seals;
+#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
+ struct rw_semaphore hinode_rwsem;
+#endif
};
static inline struct hugetlbfs_inode_info *HUGETLBFS_I(struct inode *inode)
@@ -439,6 +442,101 @@ static inline struct hstate *hstate_inode(struct inode *i)
{
return HUGETLBFS_SB(i->i_sb)->hstate;
}
+
+#ifdef CONFIG_ARCH_WANT_HUGE_PMD_SHARE
+static inline bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
+{
+ unsigned long base = addr & PUD_MASK;
+ unsigned long end = base + PUD_SIZE;
+
+ /* check on proper vm_flags and page table alignment */
+ if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
+ return true;
+ return false;
+}
+
+/*
+ * hugetlb specific hinode_rwsem is used for pmd sharing synchronization.
+ * This routine will take the semaphore in read mode if necessary. If vma
+ * and addr are NULL, the routine will always acquire the semaphore. If
+ * values are supplied for vma and addr, they are used to determine if pmd
+ * sharing is actually possible, and only acquire the semaphore if possible.
+ * Returns true if lock was acquired, otherwise false.
+ */
+static inline bool hinode_lock_read(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ if (vma && !addr)
+ addr = round_up(vma->vm_start, PUD_SIZE);
+ if (vma && !vma_shareable(vma, addr))
+ return false;
+
+ down_read(&HUGETLBFS_I(inode)->hinode_rwsem);
+ return true;
+}
+
+static inline void hinode_unlock_read(struct inode *inode)
+{
+ up_read(&HUGETLBFS_I(inode)->hinode_rwsem);
+}
+
+/*
+ * Take hinode_rwsem semaphore in write mode if necessary. See,
+ * hinode_lock_read for details.
+ * Returns true is lock was acquired, otherwise false.
+ */
+static inline bool hinode_lock_write(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ if (vma && !addr)
+ addr = round_up(vma->vm_start, PUD_SIZE);
+ if (vma && !vma_shareable(vma, addr))
+ return false;
+
+ down_write(&HUGETLBFS_I(inode)->hinode_rwsem);
+ return true;
+}
+
+static inline void hinode_unlock_write(struct inode *inode)
+{
+ up_write(&HUGETLBFS_I(inode)->hinode_rwsem);
+}
+
+static inline void hinode_assert_locked(struct address_space *mapping)
+{
+ lockdep_assert_held(&HUGETLBFS_I(mapping->host)->hinode_rwsem);
+}
+
+static inline void hinode_assert_write_locked(struct address_space *mapping)
+{
+ lockdep_assert_held_write(&HUGETLBFS_I(mapping->host)->hinode_rwsem);
+}
+#else
+static inline bool hinode_lock_read(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_read(struct inode *inode)
+{
+}
+
+static inline bool hinode_lock_write(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_write(struct inode *inode)
+{
+}
+#endif
+
#else /* !CONFIG_HUGETLBFS */
#define is_file_hugepages(file) false
@@ -913,6 +1011,29 @@ static inline void set_huge_swap_pte_at(struct mm_struct *mm, unsigned long addr
pte_t *ptep, pte_t pte, unsigned long sz)
{
}
+
+static inline bool hinode_lock_read(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_read(struct inode *inode)
+{
+}
+
+static inline bool hinode_lock_write(struct inode *inode,
+ struct vm_area_struct *vma,
+ unsigned long addr)
+{
+ return false;
+}
+
+static inline void hinode_unlock_write(struct inode *inode)
+{
+}
+
#endif /* CONFIG_HUGETLB_PAGE */
static inline spinlock_t *huge_pte_lock(struct hstate *h,
diff --git a/mm/hugetlb.c b/mm/hugetlb.c
index 9a316b6d0b51..7b38c91f457b 100644
--- a/mm/hugetlb.c
+++ b/mm/hugetlb.c
@@ -5166,19 +5166,6 @@ static unsigned long page_table_shareable(struct vm_area_struct *svma,
return saddr;
}
-static bool vma_shareable(struct vm_area_struct *vma, unsigned long addr)
-{
- unsigned long base = addr & PUD_MASK;
- unsigned long end = base + PUD_SIZE;
-
- /*
- * check on proper vm_flags and page table alignment
- */
- if (vma->vm_flags & VM_MAYSHARE && range_in_vma(vma, base, end))
- return true;
- return false;
-}
-
/*
* Determine if start,end range within vma could be mapped by shared pmd.
* If yes, adjust start and end to cover range associated with possible
diff --git a/mm/rmap.c b/mm/rmap.c
index 8ef12940e357..b30d211bd999 100644
--- a/mm/rmap.c
+++ b/mm/rmap.c
@@ -22,6 +22,7 @@
*
* inode->i_mutex (while writing or truncating, not reading or faulting)
* mm->mmap_lock
+ * hugetlbfs inode->hinode_rwsem (hugetlbfs specific)
* page->flags PG_locked (lock_page)
* hugetlbfs_i_mmap_rwsem_key (in huge_pmd_share)
* mapping->i_mmap_rwsem
--
2.25.4
