There are several situations where copy_highpage() can end up copying
tags to a page which doesn't have its tag storage reserved.
One situation involves migration racing with mprotect(PROT_MTE): VMA is
initially untagged, migration starts and destination page is allocated
as untagged, mprotect(PROT_MTE) changes the VMA to tagged and userspace
accesses the source page, thus making it tagged. The migration code
then calls copy_highpage(), which will copy the tags from the source
page (now tagged) to the destination page (allocated as untagged).
Yes another situation can happen during THP collapse. The huge page that
will replace the HPAGE_PMD_NR contiguous mapped pages is allocated with
__GFP_TAGGED not set. copy_highpage() will copy the tags from the pages
being replaced to the huge page which doesn't have tag storage reserved.
The situation gets even more complicated when the replacement huge page
is a tag storage page. The tag storage huge page will be migrated after
a fault on access, but the tags from the original pages must be copied
over to the huge page that will be replacing the tag storage huge page.
Signed-off-by: Alexandru Elisei <alexandru.elisei@xxxxxxx>
---
arch/arm64/mm/copypage.c | 59 ++++++++++++++++++++++++++++++++++++++++
1 file changed, 59 insertions(+)
diff --git a/arch/arm64/mm/copypage.c b/arch/arm64/mm/copypage.c
index a7bb20055ce0..7899f38773b9 100644
--- a/arch/arm64/mm/copypage.c
+++ b/arch/arm64/mm/copypage.c
@@ -13,6 +13,62 @@
#include <asm/cacheflush.h>
#include <asm/cpufeature.h>
#include <asm/mte.h>
+#include <asm/mte_tag_storage.h>
+
+#ifdef CONFIG_ARM64_MTE_TAG_STORAGE
+static inline bool try_transfer_saved_tags(struct page *from, struct page *to)
+{
+ void *tags;
+ bool saved;
+
+ VM_WARN_ON_ONCE(!preemptible());
+
+ if (page_mte_tagged(from)) {
+ if (likely(page_tag_storage_reserved(to)))
+ return false;
+
+ tags = mte_allocate_tag_buf();
+ if (WARN_ON(!tags))
+ return true;
+
+ mte_copy_page_tags_to_buf(page_address(from), tags);
+ saved = mte_save_tags_for_pfn(tags, page_to_pfn(to));
+ if (!saved)
+ mte_free_tag_buf(tags);
+
+ return saved;
+ }
+
+ if (likely(!page_is_tag_storage(from)))
+ return false;
+
+ tags_by_pfn_lock();
+ tags = mte_erase_tags_for_pfn(page_to_pfn(from));
+ tags_by_pfn_unlock();
+
+ if (likely(!tags))
+ return false;
+
+ if (page_tag_storage_reserved(to)) {
+ WARN_ON_ONCE(!try_page_mte_tagging(to));
+ mte_copy_page_tags_from_buf(page_address(to), tags);
+ set_page_mte_tagged(to);
+ mte_free_tag_buf(tags);
+ return true;
+ }
+
+ saved = mte_save_tags_for_pfn(tags, page_to_pfn(to));
+ if (!saved)
+ mte_free_tag_buf(tags);
+
+ return saved;
+}
+#else
+static inline bool try_transfer_saved_tags(struct page *from, struct page *to)
+{
+ return false;
+}
+#endif
void copy_highpage(struct page *to, struct page *from)
{
@@ -24,6 +80,9 @@ void copy_highpage(struct page *to, struct page *from)
if (kasan_hw_tags_enabled())
page_kasan_tag_reset(to);
+ if (tag_storage_enabled() && try_transfer_saved_tags(from, to))
+ return;
+
if (system_supports_mte() && page_mte_tagged(from)) {
/* It's a new page, shouldn't have been tagged yet */
WARN_ON_ONCE(!try_page_mte_tagging(to));
--
2.42.1
