The patch titled
Subject: mm: Optimize put_mems_allowed() usage
has been removed from the -mm tree. Its filename was
mm-optimize-put_mems_allowed-usage.patch
This patch was dropped because it was merged into mainline or a subsystem tree
The current -mm tree may be found at http://userweb.kernel.org/~akpm/mmotm/
------------------------------------------------------
From: Peter Zijlstra <a.p.zijlstra@xxxxxxxxx>
Subject: mm: Optimize put_mems_allowed() usage
Since put_mems_allowed() is strictly optional - it is a seqcount retry -
we don't need to evaluate the function if the allocation was in fact
successful, saving a smp_rmb() and some loads and comparisons on some
relative fast-paths.
Since the naming get/put_mems_allowed() does suggest a mandatory pairing,
rename the interface as suggested by Mel, to resemble the seqcount
interface.
This gives us: read_mems_allowed_begin() and read_mems_allowed_retry(),
where it is important to note that the return value of the latter call is
inverted from its previous incarnation.
Signed-off-by: Peter Zijlstra <a.p.zijlstra@xxxxxxxxx>
Acked-by: Mel Gorman <mgorman@xxxxxxx>
Cc: Miao Xie <miaox@xxxxxxxxxxxxxx>
Cc: David Rientjes <rientjes@xxxxxxxxxx>
Cc: Christoph Lameter <cl@xxxxxxxxx>
Cc: Tejun Heo <tj@xxxxxxxxxx>
Cc: Pekka Enberg <penberg@xxxxxxxxxxxxxx>
Signed-off-by: Andrew Morton <akpm@xxxxxxxxxxxxxxxxxxxx>
---
include/linux/cpuset.h | 27 ++++++++++++++-------------
kernel/cpuset.c | 2 +-
mm/filemap.c | 4 ++--
mm/hugetlb.c | 4 ++--
mm/mempolicy.c | 14 +++++++-------
mm/page_alloc.c | 8 ++++----
mm/slab.c | 4 ++--
mm/slub.c | 16 +++-------------
8 files changed, 35 insertions(+), 44 deletions(-)
diff -puN include/linux/cpuset.h~mm-optimize-put_mems_allowed-usage include/linux/cpuset.h
--- a/include/linux/cpuset.h~mm-optimize-put_mems_allowed-usage
+++ a/include/linux/cpuset.h
@@ -89,25 +89,26 @@ extern void rebuild_sched_domains(void);
extern void cpuset_print_task_mems_allowed(struct task_struct *p);
/*
- * get_mems_allowed is required when making decisions involving mems_allowed
- * such as during page allocation. mems_allowed can be updated in parallel
- * and depending on the new value an operation can fail potentially causing
- * process failure. A retry loop with get_mems_allowed and put_mems_allowed
- * prevents these artificial failures.
+ * read_mems_allowed_begin is required when making decisions involving
+ * mems_allowed such as during page allocation. mems_allowed can be updated in
+ * parallel and depending on the new value an operation can fail potentially
+ * causing process failure. A retry loop with read_mems_allowed_begin and
+ * read_mems_allowed_retry prevents these artificial failures.
*/
-static inline unsigned int get_mems_allowed(void)
+static inline unsigned int read_mems_allowed_begin(void)
{
return read_seqcount_begin(¤t->mems_allowed_seq);
}
/*
- * If this returns false, the operation that took place after get_mems_allowed
- * may have failed. It is up to the caller to retry the operation if
+ * If this returns true, the operation that took place after
+ * read_mems_allowed_begin may have failed artificially due to a concurrent
+ * update of mems_allowed. It is up to the caller to retry the operation if
* appropriate.
*/
-static inline bool put_mems_allowed(unsigned int seq)
+static inline bool read_mems_allowed_retry(unsigned int seq)
{
- return !read_seqcount_retry(¤t->mems_allowed_seq, seq);
+ return read_seqcount_retry(¤t->mems_allowed_seq, seq);
}
static inline void set_mems_allowed(nodemask_t nodemask)
@@ -223,14 +224,14 @@ static inline void set_mems_allowed(node
{
}
-static inline unsigned int get_mems_allowed(void)
+static inline unsigned int read_mems_allowed_begin(void)
{
return 0;
}
-static inline bool put_mems_allowed(unsigned int seq)
+static inline bool read_mems_allowed_retry(unsigned int seq)
{
- return true;
+ return false;
}
#endif /* !CONFIG_CPUSETS */
diff -puN kernel/cpuset.c~mm-optimize-put_mems_allowed-usage kernel/cpuset.c
--- a/kernel/cpuset.c~mm-optimize-put_mems_allowed-usage
+++ a/kernel/cpuset.c
@@ -976,7 +976,7 @@ static void cpuset_change_task_nodemask(
task_lock(tsk);
/*
* Determine if a loop is necessary if another thread is doing
- * get_mems_allowed(). If at least one node remains unchanged and
+ * read_mems_allowed_begin(). If at least one node remains unchanged and
* tsk does not have a mempolicy, then an empty nodemask will not be
* possible when mems_allowed is larger than a word.
*/
diff -puN mm/filemap.c~mm-optimize-put_mems_allowed-usage mm/filemap.c
--- a/mm/filemap.c~mm-optimize-put_mems_allowed-usage
+++ a/mm/filemap.c
@@ -500,10 +500,10 @@ struct page *__page_cache_alloc(gfp_t gf
if (cpuset_do_page_mem_spread()) {
unsigned int cpuset_mems_cookie;
do {
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
n = cpuset_mem_spread_node();
page = alloc_pages_exact_node(n, gfp, 0);
- } while (!put_mems_allowed(cpuset_mems_cookie) && !page);
+ } while (!page && read_mems_allowed_retry(cpuset_mems_cookie));
return page;
}
diff -puN mm/hugetlb.c~mm-optimize-put_mems_allowed-usage mm/hugetlb.c
--- a/mm/hugetlb.c~mm-optimize-put_mems_allowed-usage
+++ a/mm/hugetlb.c
@@ -544,7 +544,7 @@ static struct page *dequeue_huge_page_vm
unsigned int cpuset_mems_cookie;
retry_cpuset:
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
zonelist = huge_zonelist(vma, address,
htlb_alloc_mask, &mpol, &nodemask);
/*
@@ -573,7 +573,7 @@ retry_cpuset:
}
mpol_cond_put(mpol);
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return page;
diff -puN mm/mempolicy.c~mm-optimize-put_mems_allowed-usage mm/mempolicy.c
--- a/mm/mempolicy.c~mm-optimize-put_mems_allowed-usage
+++ a/mm/mempolicy.c
@@ -1715,7 +1715,7 @@ int node_random(const nodemask_t *maskp)
* If the effective policy is 'BIND, returns a pointer to the mempolicy's
* @nodemask for filtering the zonelist.
*
- * Must be protected by get_mems_allowed()
+ * Must be protected by read_mems_allowed_begin()
*/
struct zonelist *huge_zonelist(struct vm_area_struct *vma, unsigned long addr,
gfp_t gfp_flags, struct mempolicy **mpol,
@@ -1880,7 +1880,7 @@ alloc_pages_vma(gfp_t gfp, int order, st
retry_cpuset:
pol = get_vma_policy(current, vma, addr);
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
if (unlikely(pol->mode == MPOL_INTERLEAVE)) {
unsigned nid;
@@ -1888,7 +1888,7 @@ retry_cpuset:
nid = interleave_nid(pol, vma, addr, PAGE_SHIFT + order);
mpol_cond_put(pol);
page = alloc_page_interleave(gfp, order, nid);
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return page;
@@ -1901,7 +1901,7 @@ retry_cpuset:
struct page *page = __alloc_pages_nodemask(gfp, order,
zl, policy_nodemask(gfp, pol));
__mpol_put(pol);
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return page;
}
@@ -1910,7 +1910,7 @@ retry_cpuset:
*/
page = __alloc_pages_nodemask(gfp, order, zl,
policy_nodemask(gfp, pol));
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return page;
}
@@ -1944,7 +1944,7 @@ struct page *alloc_pages_current(gfp_t g
pol = &default_policy;
retry_cpuset:
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
/*
* No reference counting needed for current->mempolicy
@@ -1957,7 +1957,7 @@ retry_cpuset:
policy_zonelist(gfp, pol, numa_node_id()),
policy_nodemask(gfp, pol));
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return page;
diff -puN mm/page_alloc.c~mm-optimize-put_mems_allowed-usage mm/page_alloc.c
--- a/mm/page_alloc.c~mm-optimize-put_mems_allowed-usage
+++ a/mm/page_alloc.c
@@ -2497,7 +2497,7 @@ __alloc_pages_nodemask(gfp_t gfp_mask, u
return NULL;
retry_cpuset:
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
/* The preferred zone is used for statistics later */
first_zones_zonelist(zonelist, high_zoneidx,
@@ -2524,7 +2524,7 @@ out:
* the mask is being updated. If a page allocation is about to fail,
* check if the cpuset changed during allocation and if so, retry.
*/
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !page))
+ if (unlikely(!page && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return page;
@@ -2746,9 +2746,9 @@ bool skip_free_areas_node(unsigned int f
goto out;
do {
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
ret = !node_isset(nid, cpuset_current_mems_allowed);
- } while (!put_mems_allowed(cpuset_mems_cookie));
+ } while (read_mems_allowed_retry(cpuset_mems_cookie));
out:
return ret;
}
diff -puN mm/slab.c~mm-optimize-put_mems_allowed-usage mm/slab.c
--- a/mm/slab.c~mm-optimize-put_mems_allowed-usage
+++ a/mm/slab.c
@@ -3367,7 +3367,7 @@ static void *fallback_alloc(struct kmem_
local_flags = flags & (GFP_CONSTRAINT_MASK|GFP_RECLAIM_MASK);
retry_cpuset:
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
zonelist = node_zonelist(slab_node(current->mempolicy), flags);
retry:
@@ -3423,7 +3423,7 @@ retry:
}
}
- if (unlikely(!put_mems_allowed(cpuset_mems_cookie) && !obj))
+ if (unlikely(!obj && read_mems_allowed_retry(cpuset_mems_cookie)))
goto retry_cpuset;
return obj;
}
diff -puN mm/slub.c~mm-optimize-put_mems_allowed-usage mm/slub.c
--- a/mm/slub.c~mm-optimize-put_mems_allowed-usage
+++ a/mm/slub.c
@@ -1614,7 +1614,7 @@ static struct page *get_any_partial(stru
return NULL;
do {
- cpuset_mems_cookie = get_mems_allowed();
+ cpuset_mems_cookie = read_mems_allowed_begin();
zonelist = node_zonelist(slab_node(current->mempolicy), flags);
for_each_zone_zonelist(zone, z, zonelist, high_zoneidx) {
struct kmem_cache_node *n;
@@ -1624,21 +1624,11 @@ static struct page *get_any_partial(stru
if (n && cpuset_zone_allowed_hardwall(zone, flags) &&
n->nr_partial > s->min_partial) {
object = get_partial_node(s, n, c);
- if (object) {
- /*
- * Return the object even if
- * put_mems_allowed indicated that
- * the cpuset mems_allowed was
- * updated in parallel. It's a
- * harmless race between the alloc
- * and the cpuset update.
- */
- put_mems_allowed(cpuset_mems_cookie);
+ if (object)
return object;
- }
}
}
- } while (!put_mems_allowed(cpuset_mems_cookie));
+ } while (read_mems_allowed_retry(cpuset_mems_cookie));
#endif
return NULL;
}
_
Patches currently in -mm which might be from a.p.zijlstra@xxxxxxxxx are
origin.patch
linux-next.patch
fs-symlink-restrictions-on-sticky-directories.patch
fs-hardlink-creation-restrictions.patch
mm-fork-fix-overflow-in-vma-length-when-copying-mmap-on-clone.patch
nmi-watchdog-fix-for-lockup-detector-breakage-on-resume.patch
nmi-watchdog-fix-for-lockup-detector-breakage-on-resume-fix.patch
nmi-watchdog-fix-for-lockup-detector-breakage-on-resume-fix-fix-fix.patch
nmi-watchdog-fix-for-lockup-detector-breakage-on-resume-fix-fix.patch
nmi-watchdog-fix-for-lockup-detector-breakage-on-resume-v2.patch
nmi-watchdog-fix-for-lockup-detector-breakage-on-resume-v2-fix.patch
vsprintf-fix-%ps-on-non-symbols-when-using-kallsyms.patch
spinlock_debug-print-kallsyms-name-for-lock.patch
kmod-unexport-call_usermodehelper_freeinfo.patch
kmod-convert-two-call-sites-to-call_usermodehelper_fns.patch
kmod-move-call_usermodehelper_fns-to-c-file-and-unexport-all-its-helpers.patch
cpu-introduce-clear_tasks_mm_cpumask-helper.patch
kernel-cpuc-document-clear_tasks_mm_cpumask.patch
ipc-semc-alternatives-to-preempt_disable.patch
--
To unsubscribe from this list: send the line "unsubscribe mm-commits" in
the body of a message to majordomo@xxxxxxxxxxxxxxx
More majordomo info at http://vger.kernel.org/majordomo-info.html
