This handle page fault on behalf of device driver, unlike handle_mm_fault() it does not trigger migration back to system memory for device memory. Signed-off-by: Jérôme Glisse <jglisse@xxxxxxxxxx> Signed-off-by: Jatin Kumar <jakumar@xxxxxxxxxx> Signed-off-by: John Hubbard <jhubbard@xxxxxxxxxx> Signed-off-by: Mark Hairgrove <mhairgrove@xxxxxxxxxx> Signed-off-by: Sherry Cheung <SCheung@xxxxxxxxxx> Signed-off-by: Subhash Gutti <sgutti@xxxxxxxxxx> --- include/linux/hmm.h | 33 ++++++- mm/hmm.c | 262 +++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 267 insertions(+), 28 deletions(-) diff --git a/include/linux/hmm.h b/include/linux/hmm.h index 9e0f00d..c79abfc 100644 --- a/include/linux/hmm.h +++ b/include/linux/hmm.h @@ -99,6 +99,7 @@ struct hmm; * HMM_PFN_WRITE: CPU page table have the write permission set * HMM_PFN_ERROR: corresponding CPU page table entry point to poisonous memory * HMM_PFN_EMPTY: corresponding CPU page table entry is none (pte_none() true) + * HMM_PFN_FAULT: use by hmm_vma_fault() to signify which address need faulting * HMM_PFN_DEVICE: this is device memory (ie a ZONE_DEVICE page) * HMM_PFN_SPECIAL: corresponding CPU page table entry is special ie result of * vm_insert_pfn() or vm_insert_page() and thus should not be mirror by a @@ -113,10 +114,11 @@ typedef unsigned long hmm_pfn_t; #define HMM_PFN_WRITE (1 << 2) #define HMM_PFN_ERROR (1 << 3) #define HMM_PFN_EMPTY (1 << 4) -#define HMM_PFN_DEVICE (1 << 5) -#define HMM_PFN_SPECIAL (1 << 6) -#define HMM_PFN_UNADDRESSABLE (1 << 7) -#define HMM_PFN_SHIFT 8 +#define HMM_PFN_FAULT (1 << 5) +#define HMM_PFN_DEVICE (1 << 6) +#define HMM_PFN_SPECIAL (1 << 7) +#define HMM_PFN_UNADDRESSABLE (1 << 8) +#define HMM_PFN_SHIFT 9 static inline struct page *hmm_pfn_to_page(hmm_pfn_t pfn) { @@ -298,6 +300,29 @@ int hmm_vma_get_pfns(struct vm_area_struct *vma, hmm_pfn_t *pfns); +/* + * Fault memory on behalf of device driver unlike handle_mm_fault() it will not + * migrate any device memory back to system memory. The hmm_pfn_t array will be + * updated with fault result and current snapshot of the CPU page table for the + * range. Note that you must use hmm_range_monitor_start/end() to ascertain if + * you could use those. + * + * DO NOT USE hmm_vma_range_lock()/hmm_vma_range_unlock() IT WILL DEADLOCK ! + * + * The mmap_sem must be taken in read mode before entering and it might be drop + * by the function if that happen the function return false. Otherwise, if the + * mmap_sem is still held it return true. The return value does not reflect if + * the fault was successfull or not, you need to inspect the hmm_pfn_t array to + * determine fault status. + * + * See function description in mm/hmm.c for documentation. + */ +bool hmm_vma_fault(struct vm_area_struct *vma, + unsigned long start, + unsigned long end, + hmm_pfn_t *pfns); + + /* Below are for HMM internal use only ! Not to be use by device driver ! */ void hmm_mm_destroy(struct mm_struct *mm); diff --git a/mm/hmm.c b/mm/hmm.c index f2ea76b..521adfd 100644 --- a/mm/hmm.c +++ b/mm/hmm.c @@ -461,6 +461,14 @@ bool hmm_vma_range_monitor_end(struct hmm_range *range) EXPORT_SYMBOL(hmm_vma_range_monitor_end); +static void hmm_pfns_error(hmm_pfn_t *pfns, + unsigned long addr, + unsigned long end) +{ + for (; addr < end; addr += PAGE_SIZE, pfns++) + *pfns = HMM_PFN_ERROR; +} + static void hmm_pfns_empty(hmm_pfn_t *pfns, unsigned long addr, unsigned long end) @@ -477,10 +485,47 @@ static void hmm_pfns_special(hmm_pfn_t *pfns, *pfns = HMM_PFN_SPECIAL; } -static void hmm_vma_walk(struct vm_area_struct *vma, +static void hmm_pfns_clear(hmm_pfn_t *pfns, + unsigned long addr, + unsigned long end) +{ + unsigned long npfns = (end - addr) >> PAGE_SHIFT; + + memset(pfns, 0, sizeof(*pfns) * npfns); +} + +static bool hmm_pfns_fault(hmm_pfn_t *pfns, + unsigned long addr, + unsigned long end) +{ + for (; addr < end; addr += PAGE_SIZE, pfns++) + if (*pfns & HMM_PFN_FAULT) + return true; + return false; +} + +static bool hmm_vma_do_fault(struct vm_area_struct *vma, + unsigned long addr, + hmm_pfn_t *pfn) +{ + unsigned flags = FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_REMOTE; + int r; + + flags |= (*pfn & HMM_PFN_WRITE) ? FAULT_FLAG_WRITE : 0; + r = handle_mm_fault(vma, addr, flags); + if (r & VM_FAULT_RETRY) + return false; + if (r & VM_FAULT_ERROR) + *pfn = HMM_PFN_ERROR; + + return true; +} + +static bool hmm_vma_walk(struct vm_area_struct *vma, unsigned long start, unsigned long end, - hmm_pfn_t *pfns) + hmm_pfn_t *pfns, + bool fault) { unsigned long addr, next; hmm_pfn_t flag; @@ -489,6 +534,7 @@ static void hmm_vma_walk(struct vm_area_struct *vma, for (addr = start; addr < end; addr = next) { unsigned long i = (addr - start) >> PAGE_SHIFT; + bool writefault = false; pgd_t *pgdp; pud_t *pudp; pmd_t *pmdp; @@ -504,15 +550,37 @@ static void hmm_vma_walk(struct vm_area_struct *vma, next = pgd_addr_end(addr, end); pgdp = pgd_offset(vma->vm_mm, addr); if (pgd_none(*pgdp) || pgd_bad(*pgdp)) { - hmm_pfns_empty(&pfns[i], addr, next); - continue; + if (!(vma->vm_flags & VM_READ)) { + hmm_pfns_empty(&pfns[i], addr, next); + continue; + } + if (!fault || !hmm_pfns_fault(&pfns[i], addr, next)) { + hmm_pfns_empty(&pfns[i], addr, next); + continue; + } + pudp = pud_alloc(vma->vm_mm, pgdp, addr); + if (!pudp) { + hmm_pfns_error(&pfns[i], addr, next); + continue; + } } next = pud_addr_end(addr, end); pudp = pud_offset(pgdp, addr); if (pud_none(*pudp) || pud_bad(*pudp)) { - hmm_pfns_empty(&pfns[i], addr, next); - continue; + if (!(vma->vm_flags & VM_READ)) { + hmm_pfns_empty(&pfns[i], addr, next); + continue; + } + if (!fault || !hmm_pfns_fault(&pfns[i], addr, next)) { + hmm_pfns_empty(&pfns[i], addr, next); + continue; + } + pmdp = pmd_alloc(vma->vm_mm, pudp, addr); + if (!pmdp) { + hmm_pfns_error(&pfns[i], addr, next); + continue; + } } next = pmd_addr_end(addr, end); @@ -520,8 +588,23 @@ static void hmm_vma_walk(struct vm_area_struct *vma, pmd = pmd_read_atomic(pmdp); barrier(); if (pmd_none(pmd) || pmd_bad(pmd)) { - hmm_pfns_empty(&pfns[i], addr, next); - continue; + if (!(vma->vm_flags & VM_READ)) { + hmm_pfns_empty(&pfns[i], addr, next); + continue; + } + if (!fault || !hmm_pfns_fault(&pfns[i], addr, next)) { + hmm_pfns_empty(&pfns[i], addr, next); + continue; + } + /* + * Use pte_alloc() instead of pte_alloc_map, because we + * can't run pte_offset_map on the pmd, if an huge pmd + * could materialize from under us. + */ + if (unlikely(pte_alloc(vma->vm_mm, pmdp, addr))) { + hmm_pfns_error(&pfns[i], addr, next); + continue; + } } if (pmd_trans_huge(pmd) || pmd_devmap(pmd)) { unsigned long pfn = pmd_pfn(pmd) + pte_index(addr); @@ -529,12 +612,33 @@ static void hmm_vma_walk(struct vm_area_struct *vma, if (pmd_protnone(pmd)) { hmm_pfns_clear(&pfns[i], addr, next); + if (!fault || !(vma->vm_flags & VM_READ)) + continue; + if (!hmm_pfns_fault(&pfns[i], addr, next)) + continue; + + if (!hmm_vma_do_fault(vma, addr, &pfns[i])) + return false; + /* Start again for current address */ + next = addr; continue; } flags |= pmd_write(*pmdp) ? HMM_PFN_WRITE : 0; flags |= pmd_devmap(pmd) ? HMM_PFN_DEVICE : 0; - for (; addr < next; addr += PAGE_SIZE, i++, pfn++) + for (; addr < next; addr += PAGE_SIZE, i++, pfn++) { + bool fault = pfns[i] & HMM_PFN_FAULT; + bool write = pfns[i] & HMM_PFN_WRITE; + pfns[i] = hmm_pfn_from_pfn(pfn) | flags; + if (!fault || !write || flags & HMM_PFN_WRITE) + continue; + pfns[i] = HMM_PFN_FAULT | HMM_PFN_WRITE; + if (!hmm_vma_do_fault(vma, addr, &pfns[i])) + return false; + /* Start again for current address */ + next = addr; + break; + } continue; } @@ -543,41 +647,91 @@ static void hmm_vma_walk(struct vm_area_struct *vma, swp_entry_t entry; pte_t pte = *ptep; - pfns[i] = 0; - if (pte_none(pte)) { - pfns[i] = HMM_PFN_EMPTY; - continue; + if (!fault || !(pfns[i] & HMM_PFN_FAULT)) { + pfns[i] = HMM_PFN_EMPTY; + continue; + } + if (!(vma->vm_flags & VM_READ)) { + pfns[i] = HMM_PFN_EMPTY; + continue; + } + if (!hmm_vma_do_fault(vma, addr, &pfns[i])) { + hmm_pfns_clear(&pfns[i], addr, end); + pte_unmap(ptep); + return false; + } + pte = *ptep; } entry = pte_to_swp_entry(pte); if (!pte_present(pte) && !non_swap_entry(entry)) { - continue; + if (!fault || !(pfns[i] & HMM_PFN_FAULT)) { + pfns[i] = 0; + continue; + } + if (!(vma->vm_flags & VM_READ)) { + pfns[i] = 0; + continue; + } + if (!hmm_vma_do_fault(vma, addr, &pfns[i])) { + hmm_pfns_clear(&pfns[i], addr, end); + pte_unmap(ptep); + return false; + } + pte = *ptep; } + writefault = (pfns[i]&(HMM_PFN_WRITE|HMM_PFN_FAULT)) == + (HMM_PFN_WRITE|HMM_PFN_FAULT) && fault; + if (pte_present(pte)) { pfns[i] = hmm_pfn_from_pfn(pte_pfn(pte))|flag; pfns[i] |= pte_write(pte) ? HMM_PFN_WRITE : 0; - continue; - } - - /* - * This is a special swap entry, ignore migration, use - * device and report anything else as error. - */ - if (is_device_entry(entry)) { + } else if (is_device_entry(entry)) { + /* Do not fault device entry */ pfns[i] = hmm_pfn_from_pfn(swp_offset(entry)); if (is_write_device_entry(entry)) pfns[i] |= HMM_PFN_WRITE; pfns[i] |= HMM_PFN_DEVICE; pfns[i] |= HMM_PFN_UNADDRESSABLE; pfns[i] |= flag; - } else if (!is_migration_entry(entry)) { + } else if (is_migration_entry(entry) && fault) { + migration_entry_wait(vma->vm_mm, pmdp, addr); + /* Start again for current address */ + next = addr; + ptep++; + break; + } else { + /* Report error for everything else */ pfns[i] = HMM_PFN_ERROR; } + if (!(vma->vm_flags & VM_READ) || + !(vma->vm_flags & VM_WRITE)) { + writefault = false; + continue; + } + + if (writefault && !(pfns[i] & HMM_PFN_WRITE)) { + ptep++; + break; + } + writefault = false; } pte_unmap(ptep - 1); + + if (writefault && (vma->vm_flags & VM_WRITE)) { + pfns[i] = HMM_PFN_WRITE | HMM_PFN_FAULT; + if (!hmm_vma_do_fault(vma, addr, &pfns[i])) { + return false; + } + writefault = false; + /* Start again for current address */ + next = addr; + } } + + return true; } /* @@ -613,7 +767,67 @@ int hmm_vma_get_pfns(struct vm_area_struct *vma, if (end < vma->vm_start || end > vma->vm_end) return -EINVAL; - hmm_vma_walk(vma, start, end, pfns); + hmm_vma_walk(vma, start, end, pfns, false); return 0; } EXPORT_SYMBOL(hmm_vma_get_pfns); + + +/* + * hmm_vma_fault() - try to fault some address in a virtual address range + * @vma: virtual memory area containing the virtual address range + * @start: fault range virtual start address (inclusive) + * @end: fault range virtual end address (exclusive) + * @pfns: array of hmm_pfn_t, only entry with fault flag set will be faulted + * Returns: true mmap_sem is still held, false mmap_sem have been release + * + * This is similar to a regular CPU page fault except that it will not trigger + * any memory migration if the memory being faulted is not accessible by CPUs. + * + * Only pfn with fault flag set will be faulted and the hmm_pfn_t write flag + * will be use to determine if it is a write fault or not. + * + * On error, for one virtual address in the range, the function will set the + * hmm_pfn_t error flag for the corresponding pfn entry. + * + * Expected use pattern: + * retry: + * down_read(&mm->mmap_sem); + * // Find vma and address device wants to fault, initialize hmm_pfn_t + * // array accordingly + * hmm_vma_range_monitor_start(range, vma, start, end); + * if (!hmm_vma_fault(vma, start, end, pfns, allow_retry)) { + * hmm_vma_range_monitor_end(range); + * // You might want to rate limit or yield to play nicely, you may + * // also commit any valid pfn in the array assuming that you are + * // getting true from hmm_vma_range_monitor_end() + * goto retry; + * } + * // Take device driver lock that serialize device page table update + * driver_lock_device_page_table_update(); + * if (hmm_vma_range_monitor_end(range)) { + * // Commit pfns we got from hmm_vma_fault() + * } + * driver_unlock_device_page_table_update(); + * up_read(&mm->mmap_sem) + */ +bool hmm_vma_fault(struct vm_area_struct *vma, + unsigned long start, + unsigned long end, + hmm_pfn_t *pfns) +{ + /* FIXME support hugetlb fs */ + if (is_vm_hugetlb_page(vma) || (vma->vm_flags & VM_SPECIAL)) { + hmm_pfns_special(pfns, start, end); + return true; + } + + /* Sanity check, this really should not happen ! */ + if (start < vma->vm_start || start >= vma->vm_end) + return true; + if (end < vma->vm_start || end > vma->vm_end) + return true; + + return hmm_vma_walk(vma, start, end, pfns, true); +} +EXPORT_SYMBOL(hmm_vma_fault); -- 2.4.3 -- To unsubscribe, send a message with 'unsubscribe linux-mm' in the body to majordomo@xxxxxxxxx. For more info on Linux MM, see: http://www.linux-mm.org/ . Don't email: <a href=mailto:"dont@xxxxxxxxx";> email@xxxxxxxxx </a>