Add CLUSTER_FLAG_SKIP_SCAN cluster flag, which is applied to a cluster
under 1 of 2 conditions. When present, the cluster will be skipped
during a scan.
- When the number of free entries is less than the number of entries
that would be required for a new allocation of the order that the
cluster serves.
- When scanning completes for the cluster, and no further scanners are
active for the cluster and no swap entries were freed for the cluster
since the last scan began. In this case, it has been proven that there
are no contiguous free entries of sufficient size to allcoate the
order that the cluster serves. In this case the cluster is made
eligible for scanning again when the next entry is freed.
The latter is implemented to permit multiple CPUs to scan the same
cluster, which in turn garrantees that if there is a free block
available in a cluster allocated for the desired order then it will be
allocated on a first come, first served basis.
As a result, the number of active scanners for a cluster must be
tracked, costing 4 bytes per cluster.
Signed-off-by: Ryan Roberts <ryan.roberts@xxxxxxx>
---
include/linux/swap.h | 3 +++
mm/swapfile.c | 36 ++++++++++++++++++++++++++++++++++--
2 files changed, 37 insertions(+), 2 deletions(-)
diff --git a/include/linux/swap.h b/include/linux/swap.h
index 34ec4668a5c9..40c308749e79 100644
--- a/include/linux/swap.h
+++ b/include/linux/swap.h
@@ -257,9 +257,12 @@ struct swap_cluster_info {
unsigned int data:24;
unsigned int flags:4;
unsigned int order:4;
+ unsigned int nr_scanners;
};
#define CLUSTER_FLAG_FREE 1 /* This cluster is free */
#define CLUSTER_FLAG_NEXT_NULL 2 /* This cluster has no next cluster */
+#define CLUSTER_FLAG_SKIP_SCAN 4 /* Skip cluster for per-order scan */
+#define CLUSTER_FLAG_DECREMENT 8 /* A swap entry was freed from cluster */
/*
* swap_info_struct::max is an unsigned int, so the maximum number of pages in
diff --git a/mm/swapfile.c b/mm/swapfile.c
index 24db03db8830..caf382b4ecd3 100644
--- a/mm/swapfile.c
+++ b/mm/swapfile.c
@@ -574,6 +574,9 @@ static void add_cluster_info_page(struct swap_info_struct *p,
VM_BUG_ON(cluster_count(&cluster_info[idx]) + count > SWAPFILE_CLUSTER);
cluster_set_count(&cluster_info[idx],
cluster_count(&cluster_info[idx]) + count);
+
+ if (SWAPFILE_CLUSTER - cluster_count(&cluster_info[idx]) < count)
+ cluster_info[idx].flags |= CLUSTER_FLAG_SKIP_SCAN;
}
/*
@@ -595,6 +598,7 @@ static void dec_cluster_info_page(struct swap_info_struct *p,
struct swap_cluster_info *cluster_info, unsigned long page_nr)
{
unsigned long idx = page_nr / SWAPFILE_CLUSTER;
+ unsigned long count = 1 << cluster_info[idx].order;
if (!cluster_info)
return;
@@ -603,6 +607,10 @@ static void dec_cluster_info_page(struct swap_info_struct *p,
cluster_set_count(&cluster_info[idx],
cluster_count(&cluster_info[idx]) - 1);
+ cluster_info[idx].flags |= CLUSTER_FLAG_DECREMENT;
+ if (SWAPFILE_CLUSTER - cluster_count(&cluster_info[idx]) >= count)
+ cluster_info[idx].flags &= ~CLUSTER_FLAG_SKIP_SCAN;
+
if (cluster_count(&cluster_info[idx]) == 0)
free_cluster(p, idx);
}
@@ -708,7 +716,8 @@ static unsigned int next_cluster_for_scan(struct swap_info_struct *si,
end = offset_to_cluster(si, *stop);
while (ci != end) {
- if ((ci->flags & CLUSTER_FLAG_FREE) == 0 && ci->order == order)
+ if ((ci->flags & (CLUSTER_FLAG_SKIP_SCAN | CLUSTER_FLAG_FREE)) == 0
+ && ci->order == order)
break;
ci = next_cluster_circular(si, ci);
}
@@ -722,6 +731,21 @@ static unsigned int next_cluster_for_scan(struct swap_info_struct *si,
return cluster_to_offset(si, ci);
}
+static inline void cluster_inc_scanners(struct swap_cluster_info *ci)
+{
+ /* Protected by si lock. */
+ ci->nr_scanners++;
+ ci->flags &= ~CLUSTER_FLAG_DECREMENT;
+}
+
+static inline void cluster_dec_scanners(struct swap_cluster_info *ci)
+{
+ /* Protected by si lock. */
+ ci->nr_scanners--;
+ if (ci->nr_scanners == 0 && (ci->flags & CLUSTER_FLAG_DECREMENT) == 0)
+ ci->flags |= CLUSTER_FLAG_SKIP_SCAN;
+}
+
/*
* Try to get swap entries with specified order from current cpu's swap entry
* pool (a cluster). This might involve allocating a new cluster for current CPU
@@ -764,6 +788,8 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
return false;
} else
return false;
+
+ cluster_inc_scanners(offset_to_cluster(si, tmp));
}
/*
@@ -780,13 +806,19 @@ static bool scan_swap_map_try_ssd_cluster(struct swap_info_struct *si,
}
unlock_cluster(ci);
if (tmp >= max) {
+ cluster_dec_scanners(ci);
cluster->next[order] = SWAP_NEXT_INVALID;
goto new_cluster;
}
*offset = tmp;
*scan_base = tmp;
tmp += nr_pages;
- cluster->next[order] = tmp < max ? tmp : SWAP_NEXT_INVALID;
+ if (tmp >= max) {
+ cluster_dec_scanners(ci);
+ cluster->next[order] = SWAP_NEXT_INVALID;
+ } else {
+ cluster->next[order] = tmp;
+ }
return true;
}
--
2.43.0
