From: Kairui Song <kasong@xxxxxxxxxxx>
Instead of using a returning argument, we can simply store the next
cluster offset to the fixed percpu location, which reduce the stack
usage and simplify the function:
Object size:
./scripts/bloat-o-meter mm/swapfile.o mm/swapfile.o.new
add/remove: 0/0 grow/shrink: 0/2 up/down: 0/-271 (-271)
Function old new delta
get_swap_pages 2847 2733 -114
alloc_swap_scan_cluster 894 737 -157
Total: Before=30833, After=30562, chg -0.88%
Stack usage:
Before:
swapfile.c:1190:5:get_swap_pages 240 static
After:
swapfile.c:1185:5:get_swap_pages 216 static
Signed-off-by: Kairui Song <kasong@xxxxxxxxxxx>
---
include/linux/swap.h | 4 ++--
mm/swapfile.c | 57 ++++++++++++++++++++------------------------
2 files changed, 28 insertions(+), 33 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index a3b5d74b095a..0e6c6bb385f0 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -276,9 +276,9 @@ enum swap_cluster_flags {
* The first page in the swap file is the swap header, which is always marked
* bad to prevent it from being allocated as an entry. This also prevents the
* cluster to which it belongs being marked free. Therefore 0 is safe to use as
- * a sentinel to indicate next is not valid in percpu_cluster.
+ * a sentinel to indicate an entry is not valid.
*/
-#define SWAP_NEXT_INVALID 0
+#define SWAP_ENTRY_INVALID 0
#ifdef CONFIG_THP_SWAP
#define SWAP_NR_ORDERS (PMD_ORDER + 1)
diff --git a/mm/swapfile.c b/mm/swapfile.c
index a19ee8d5ffd0..f529e2ce2019 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -743,11 +743,14 @@ static void cluster_alloc_range(struct swap_info_struct *si, struct swap_cluster
ci->count += nr_pages;
}
-static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, unsigned long offset,
- unsigned int *foundp, unsigned int order,
+/* Try use a new cluster for current CPU and allocate from it. */
+static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si,
+ unsigned long offset,
+ unsigned int order,
unsigned char usage)
{
- unsigned long start = offset & ~(SWAPFILE_CLUSTER - 1);
+ unsigned int next = SWAP_ENTRY_INVALID, found = SWAP_ENTRY_INVALID;
+ unsigned long start = ALIGN_DOWN(offset, SWAPFILE_CLUSTER);
unsigned long end = min(start + SWAPFILE_CLUSTER, si->max);
unsigned int nr_pages = 1 << order;
bool need_reclaim, ret;
@@ -756,10 +759,8 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, unsigne
ci = &si->cluster_info[offset / SWAPFILE_CLUSTER];
lockdep_assert_held(&ci->lock);
- if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER) {
- offset = SWAP_NEXT_INVALID;
+ if (end < nr_pages || ci->count + nr_pages > SWAPFILE_CLUSTER)
goto out;
- }
for (end -= nr_pages; offset <= end; offset += nr_pages) {
need_reclaim = false;
@@ -773,10 +774,8 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, unsigne
* cluster has no flag set, and change of list
* won't cause fragmentation.
*/
- if (!cluster_is_usable(ci, order)) {
- offset = SWAP_NEXT_INVALID;
+ if (!cluster_is_usable(ci, order))
goto out;
- }
if (cluster_is_free(ci))
offset = start;
/* Reclaim failed but cluster is usable, try next */
@@ -784,20 +783,17 @@ static unsigned int alloc_swap_scan_cluster(struct swap_info_struct *si, unsigne
continue;
}
cluster_alloc_range(si, ci, offset, usage, order);
- *foundp = offset;
- if (ci->count == SWAPFILE_CLUSTER) {
- offset = SWAP_NEXT_INVALID;
- goto out;
- }
+ found = offset;
offset += nr_pages;
+ if (ci->count < SWAPFILE_CLUSTER && offset <= end)
+ next = offset;
break;
}
- if (offset > end)
- offset = SWAP_NEXT_INVALID;
out:
relocate_cluster(si, ci);
unlock_cluster(ci);
- return offset;
+ __this_cpu_write(si->percpu_cluster->next[order], next);
+ return found;
}
/* Return true if reclaimed a whole cluster */
@@ -866,8 +862,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int o
if (cluster_is_usable(ci, order)) {
if (cluster_is_free(ci))
offset = cluster_offset(si, ci);
- offset = alloc_swap_scan_cluster(si, offset, &found,
- order, usage);
+ found = alloc_swap_scan_cluster(si, offset,
+ order, usage);
} else {
unlock_cluster(ci);
}
@@ -878,8 +874,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int o
new_cluster:
ci = cluster_isolate_lock(si, &si->free_clusters);
if (ci) {
- offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
- &found, order, usage);
+ found = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
+ order, usage);
/*
* Allocation from free cluster must never fail and
* cluster lock must remain untouched.
@@ -896,8 +892,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int o
unsigned int frags = 0, frags_existing;
while ((ci = cluster_isolate_lock(si, &si->nonfull_clusters[order]))) {
- offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
- &found, order, usage);
+ found = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
+ order, usage);
/*
* With `fragmenting` set to true, it will surely take
* the cluster off nonfull list
@@ -917,8 +913,8 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int o
* per-CPU usage, but either way they could contain
* usable (eg. lazy-freed swap cache) slots.
*/
- offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
- &found, order, usage);
+ found = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
+ order, usage);
if (found)
goto done;
frags++;
@@ -944,21 +940,20 @@ static unsigned long cluster_alloc_swap_entry(struct swap_info_struct *si, int o
*/
while ((ci = cluster_isolate_lock(si, &si->frag_clusters[o]))) {
atomic_long_dec(&si->frag_cluster_nr[o]);
- offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
- &found, order, usage);
+ found = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
+ 0, usage);
if (found)
goto done;
}
while ((ci = cluster_isolate_lock(si, &si->nonfull_clusters[o]))) {
- offset = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
- &found, order, usage);
+ found = alloc_swap_scan_cluster(si, cluster_offset(si, ci),
+ 0, usage);
if (found)
goto done;
}
}
done:
- __this_cpu_write(si->percpu_cluster->next[order], offset);
local_unlock(&si->percpu_cluster->lock);
return found;
@@ -3150,7 +3145,7 @@ static struct swap_cluster_info *setup_clusters(struct swap_info_struct *si,
cluster = per_cpu_ptr(si->percpu_cluster, cpu);
for (i = 0; i < SWAP_NR_ORDERS; i++)
- cluster->next[i] = SWAP_NEXT_INVALID;
+ cluster->next[i] = SWAP_ENTRY_INVALID;
local_lock_init(&cluster->lock);
}
--
2.47.0
