Re: [PATCH v2 08/15] drm/panthor: Add the MMU/VM logical block

[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index]

 



On 09/08/2023 17:53, Boris Brezillon wrote:
> MMU and VM management is related and placed in the same source file.
> 
> Page table updates are delegated to the io-pgtable-arm driver that's in
> the iommu subsystem.
> 
> The VM management logic is based on drm_gpuva_mgr, and is assuming the
> VA space is mostly managed by the usermode driver, except for a reserved
> portion of this VA-space that's used for kernel objects (like the heap
> contexts/chunks).
> 
> Both asynchronous and synchronous VM operations are supported, and
> internal helpers are exposed to allow other logical blocks to map their
> buffers in the GPU VA space.
> 
> There's one VM_BIND queue per-VM (meaning the Vulkan driver can only
> expose one sparse-binding queue), and this bind queue is managed with
> a 1:1 drm_sched_entity:drm_gpu_scheduler, such that each VM gets its own
> independent execution queue, avoiding VM operation serialization at the
> device level (things are still serialized at the VM level).
> 
> The rest is just implementation details that are hopefully well explained
> in the documentation.
> 
> v2:
> - Rename the driver (pancsf -> panthor)
> - Change the license (GPL2 -> MIT + GPL2)
> - Split the driver addition commit
> - Document the code
> - Use drm_gpuva_mgr
> - Replace VM_MAP/UNMAP by VM_BIND
> - Add support for asynchronous VM_BIND (VM_BIND queue implemented with
>   drm_sched)
> - Use drm_dev_{unplug,enter,exit}() to provide safe device removal
> - Use the panthor_irq layer to manage/process IRQs
> 
> Signed-off-by: Boris Brezillon <boris.brezillon@xxxxxxxxxxxxx>
> ---
>  drivers/gpu/drm/panthor/panthor_mmu.c | 2611 +++++++++++++++++++++++++
>  drivers/gpu/drm/panthor/panthor_mmu.h |   81 +
>  2 files changed, 2692 insertions(+)
>  create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.c
>  create mode 100644 drivers/gpu/drm/panthor/panthor_mmu.h
> 
> diff --git a/drivers/gpu/drm/panthor/panthor_mmu.c b/drivers/gpu/drm/panthor/panthor_mmu.c
> new file mode 100644
> index 000000000000..3ba784473023
> --- /dev/null
> +++ b/drivers/gpu/drm/panthor/panthor_mmu.c
> @@ -0,0 +1,2611 @@
> +// SPDX-License-Identifier: GPL-2.0 or MIT
> +/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@xxxxxxxxxx> */
> +/* Copyright 2023 Collabora ltd. */
> +
> +#include <drm/drm_debugfs.h>
> +#include <drm/drm_drv.h>
> +#include <drm/drm_exec.h>
> +#include <drm/drm_gpuva_mgr.h>
> +#include <drm/drm_managed.h>
> +#include <drm/gpu_scheduler.h>
> +#include <drm/panthor_drm.h>
> +
> +#include <linux/atomic.h>
> +#include <linux/bitfield.h>
> +#include <linux/delay.h>
> +#include <linux/dma-mapping.h>
> +#include <linux/interrupt.h>
> +#include <linux/io.h>
> +#include <linux/iopoll.h>
> +#include <linux/io-pgtable.h>
> +#include <linux/iommu.h>
> +#include <linux/kmemleak.h>
> +#include <linux/platform_device.h>
> +#include <linux/pm_runtime.h>
> +#include <linux/rwsem.h>
> +#include <linux/shmem_fs.h>
> +#include <linux/sizes.h>
> +
> +#include "panthor_device.h"
> +#include "panthor_heap.h"
> +#include "panthor_mmu.h"
> +#include "panthor_sched.h"
> +#include "panthor_gem.h"
> +#include "panthor_regs.h"
> +
> +#define MAX_AS_SLOTS			32
> +
> +struct panthor_vm;
> +
> +/**
> + * struct panthor_as_slot - Address space slot
> + */
> +struct panthor_as_slot {
> +	/** @vm: VM bound to this slot. NULL is no VM is bound. */
> +	struct panthor_vm *vm;
> +
> +	/** @lock: Lock used to serialize access to the AS registers. */
> +	spinlock_t lock;
> +};
> +
> +/**
> + * struct panthor_mmu - MMU related data
> + */
> +struct panthor_mmu {
> +	/** @irq: The MMU irq. */
> +	struct panthor_irq irq;
> +
> +	/** @as: Address space related fields.
> +	 *
> +	 * The GPU has a limited number of address spaces (AS) slots, forcing
> +	 * us to re-assign them to re-assign slots on-demand.
> +	 */
> +	struct {
> +		/** @slots_lock: Lock protecting access to all other AS fields. */
> +		struct mutex slots_lock;
> +
> +		/** @alloc_mask: Bitmask encoding the allocated slots. */
> +		unsigned long alloc_mask;
> +
> +		/** @faulty_mask: Bitmask encoding the faulty slots. */
> +		unsigned long faulty_mask;
> +
> +		/** @slots: VMs currently bound to the AS slots. */
> +		struct panthor_as_slot slots[MAX_AS_SLOTS];
> +
> +		/**
> +		 * @lru_list: List of least recently used VMs.
> +		 *
> +		 * We use this list to pick a VM to evict when all slots are
> +		 * used.
> +		 *
> +		 * There should be no more active VMs than there are AS slots,
> +		 * so this LRU is just here to keep VMs bound until there's
> +		 * a need to release a slot, thus avoid unnecessary TLB/cache
> +		 * flushes.
> +		 */
> +		struct list_head lru_list;
> +	} as;
> +
> +	/** @vm: VMs management fields */
> +	struct {
> +		/** @lock: Lock protecting access to list. */
> +		struct mutex lock;
> +
> +		/** @list: List containing all VMs. */
> +		struct list_head list;
> +
> +		/** @reset_in_progress: True if a reset is in progress. */
> +		bool reset_in_progress;
> +
> +		/** @wq: Workqueue used for the VM_BIND queues. */
> +		struct workqueue_struct *wq;
> +	} vm;
> +};
> +
> +/**
> + * struct panthor_vm_pool - VM pool object
> + */
> +struct panthor_vm_pool {
> +	/** @xa: Array used for VM handle tracking. */
> +	struct xarray xa;
> +};
> +
> +/**
> + * struct panthor_vma - GPU mapping object
> + *
> + * This is used to track GEM mappings in GPU space.
> + */
> +struct panthor_vma {
> +	/** @base: Inherits from drm_gpuva. */
> +	struct drm_gpuva base;
> +
> +	/** @node: Used to insert the mapping in the panthor_vm::shared_bos list. */
> +	struct list_head node;
> +
> +	/**
> +	 * @flags: Combination of drm_panthor_vm_bind_op_flags.
> +	 *
> +	 * Only map related flags are accepted.
> +	 */
> +	u32 flags;
> +};
> +
> +/**
> + * struct panthor_vm_op_ctx - VM operation context
> + *
> + * With VM operations potentially taking place in a dma-signaling path, we
> + * need to make sure everything that might require resource allocation is
> + * pre-allocated upfront. This is what this operation context is far.
> + *
> + * We also collect resources that have been freed, so we can release them
> + * asynchronously, and let the VM_BIND scheduler process the next VM_BIND
> + * request.
> + */
> +struct panthor_vm_op_ctx {
> +	/** @rsvd_page_tables: Pages reserved for the MMU page table update. */
> +	struct {
> +		/** @count: Number of pages reserved. */
> +		u32 count;
> +
> +		/** @ptr: Point to the first unused page in the @pages table. */
> +		u32 ptr;
> +
> +		/**
> +		 * @page: Array of pages that can be used for an MMU page table update.
> +		 *
> +		 * After an VM operation, there might be free pages left in this array.
> +		 * They should be returned to the pt_cache as part of the op_ctx cleanup.
> +		 */
> +		void **pages;
> +	} rsvd_page_tables;

Two questions:

1) Would a mempool simplify the implementation? It looks like a
reasonable match.

2) Does it really make sense to have a separate pool of memory for every
operation? Instead of having a separate pool for each operation, it
would be possible to just keep track of the total number needed for all
outstanding operations. Then a single (per device or maybe per-VM if
necessary) mempool could be resized to ensure it has the right amount of
space.

I'm also a little wary that the VM_BIND infrastructure could potentially
be abused to trigger a large amount of kernel allocation as it allocates
up-front for the worst case but those pages are not charged to the
process (AFAICT). But I haven't fully got my head round that yet.

> +
> +	/** @flags: Combination of drm_panthor_vm_bind_op_flags. */
> +	u32 flags;
> +
> +	/** @va: Virtual range targeted by the VM operation. */
> +	struct {
> +		/** @addr: Start address. */
> +		u64 addr;
> +
> +		/** @range: Range size. */
> +		u64 range;
> +	} va;
> +
> +	/**
> +	 * @returned_vmas: List of panthor_vma objects returned after a VM operation.
> +	 *
> +	 * For unmap operations, this will contain all VMAs that were covered by the
> +	 * specified VA range.
> +	 *
> +	 * For map operations, this will contain all VMAs that previously mapped to
> +	 * the specified VA range.
> +	 *
> +	 * Those VMAs, and the resources they point to will be released as part of
> +	 * the op_ctx cleanup operation.
> +	 */
> +	struct list_head returned_vmas;
> +
> +	/** @map: Fields specific to a map operation. */
> +	struct {
> +		/** @gem: GEM object information. */
> +		struct {
> +			/** @obj: GEM object to map. */
> +			struct drm_gem_object *obj;
> +
> +			/** @offset: Offset in the GEM object. */
> +			u64 offset;
> +		} gem;
> +
> +		/**
> +		 * @sgt: sg-table pointing to pages backing the GEM object.
> +		 *
> +		 * This is gathered at job creation time, such that we don't have
> +		 * to allocate in ::run_job().
> +		 */
> +		struct sg_table *sgt;
> +
> +		/**
> +		 * @prev_vma: Pre-allocated VMA object to deal with a remap situation.
> +		 *
> +		 * If the map request covers a region that's inside another VMA, the
> +		 * previous VMA will be split, requiring instantiation of a maximum of
> +		 * two new VMA objects.
> +		 */
> +		struct panthor_vma *prev_vma;
> +
> +		/**
> +		 * @new_vma: The new VMA object that will be inserted to the VA tree.
> +		 */
> +		struct panthor_vma *new_vma;
> +
> +		/**
> +		 * @next_vma: Pre-allocated VMA object to deal with a remap situation.
> +		 *
> +		 * See @prev_vma.
> +		 */
> +		struct panthor_vma *next_vma;

It's probably premature optimization, but it feels like having a cache
of these VMA structures might be an idea. I'm also struggling to
understand how both a new prev and new next VMA are needed - but I
haven't dug into the GPU VA manager.

> +	} map;
> +};
> +
> +/**
> + * struct panthor_vm - VM object
> + *
> + * A VM is an object representing a GPU (or MCU) virtual address space.
> + * It embeds the MMU page table for this address space, a tree containing
> + * all the virtual mappings of GEM objects, and other things needed to manage
> + * the VM.
> + *
> + * Except for the MCU VM, which is managed by the kernel, all other VMs are
> + * created by userspace and mostly managed by userspace, using the
> + * %DRM_IOCTL_PANTHOR_VM_BIND ioctl.
> + *
> + * A portion of the virtual address space is reserved for kernel objects,
> + * like heap chunks, and userspace gets to decide how much of the virtual
> + * address space is left to the kernel (half of the virtual address space
> + * by default).
> + */
> +struct panthor_vm {
> +	/**
> +	 * @va_mgr: GPU VA manager.
> +	 *
> +	 * We delegate all the VA management to the common drm_gpuva_mgr framework
> +	 * and only implement hooks to update the MMU page table.
> +	 */
> +	struct drm_gpuva_manager va_mgr;
> +
> +	/**
> +	 * @sched: Scheduler used for asynchronous VM_BIND request.
> +	 *
> +	 * We use a 1:1 scheduler here.
> +	 */
> +	struct drm_gpu_scheduler sched;
> +
> +	/**
> +	 * @entity: Scheduling entity representing the VM_BIND queue.
> +	 *
> +	 * There's currently one bind queue per VM. It doesn't make sense to
> +	 * allow more given the VM operations are serialized anyway.
> +	 */
> +	struct drm_sched_entity entity;
> +
> +	/** @ptdev: Device. */
> +	struct panthor_device *ptdev;
> +
> +	/** @refcount: Reference count. */
> +	struct kref refcount;
> +
> +	/** @memattr: Value to program to the AS_MEMATTR register. */
> +	u64 memattr;
> +
> +	/** @pgtbl_ops: Page table operations. */
> +	struct io_pgtable_ops *pgtbl_ops;
> +
> +	/**
> +	 * @dummy_gem: Used as a VM reservation object.
> +	 *
> +	 * We declare a drm_gem_object and no a dma_resv, so we can use drm_exec()
> +	 * for the VM reservation.
> +	 *
> +	 * All private BOs use the resv of this dummy GEM object instead of
> +	 * drm_gem_object::_resv, such that private GEM preparation is O(1)
> +	 * instead of O(N).
> +	 */
> +	struct drm_gem_object dummy_gem;
> +
> +	/**
> +	 * @op_lock: Lock used to serialize operations on a VM.
> +	 *
> +	 * The serialization of jobs queued to the VM_BIND queue is already
> +	 * taken care of by drm_sched, but we need to serialize synchronous
> +	 * and asynchronous VM_BIND request. This is what this lock is for.
> +	 */
> +	struct mutex op_lock;
> +
> +	/**
> +	 * @op_ctx: The context attached to the currently executing VM operation.
> +	 *
> +	 * NULL when no operation is in progress.
> +	 */
> +	struct panthor_vm_op_ctx *op_ctx;
> +
> +	/**
> +	 * @shared_bos: List of shared BOs.
> +	 *
> +	 * Shared BOs don't use the VM resv, and need to be prepared
> +	 * independently. This list keeps track of all VMAs that target
> +	 * non-private BOs.
> +	 *
> +	 * There might be duplicates, but drm_exec and dma_resv should
> +	 * handle that for us.
> +	 *
> +	 * TODO: This is not optimal. We should probably switch to the
> +	 * drm_gpuva_mgr solution for handling shared BOs once it's
> +	 * ready.
> +	 */
> +	struct list_head shared_bos;
> +
> +	/**
> +	 * @mm: Memory management object representing the auto-VA/kernel-VA.
> +	 *
> +	 * Used to auto-allocate VA space for kernel-managed objects (tiler
> +	 * heaps, ...).
> +	 *
> +	 * For the MCU VM, this is managing the VA range that's used to map
> +	 * all shared interfaces.
> +	 *
> +	 * For user VMs, the range is specified by userspace, and must not
> +	 * exceed half of the VA space addressable.
> +	 */
> +	struct drm_mm mm;
> +
> +	/** @mm_lock: Lock protecting the @mm field. */
> +	struct mutex mm_lock;
> +
> +	/** @as: Address space related fields. */
> +	struct {
> +		/**
> +		 * @id: ID of the address space this VM is bound to.
> +		 *
> +		 * A value of -1 means the VM is inactive/not bound.
> +		 */
> +		int id;
> +
> +		/**
> +		 * @lru_node: Used to instead the VM in the panthor_mmu::as::lru_list.
> +		 *
> +		 * Active VMs should not be inserted in the LRU list.
> +		 */
> +		struct list_head lru_node;
> +	} as;
> +
> +	/**
> +	 * @heaps: Tiler heap related fields.
> +	 */
> +	struct {
> +		/**
> +		 * @pool: The heap pool attached to this VM.
> +		 *
> +		 * Will stay NULL until someone creates a heap context on this VM.
> +		 */
> +		struct panthor_heap_pool *pool;
> +
> +		/** @lock: Lock used to protect access to @pool. */
> +		struct mutex lock;
> +	} heaps;
> +
> +	/** @node: Used to insert the VM in the panthor_mmu::vm::list. */
> +	struct list_head node;
> +
> +	/** @for_mcu: True if this is the MCU VM. */
> +	bool for_mcu;
> +
> +	/**
> +	 * @destroyed: True if the VM was destroyed.
> +	 *
> +	 * No further bind requests should be queued to a destroyed VM.
> +	 */
> +	bool destroyed;
> +
> +	/**
> +	 * @unusable: True if the VM has turned unusable because something
> +	 * bad happened during an asynchronous request.
> +	 *
> +	 * We don't try to recover from such failures, because this implies
> +	 * informing userspace about the specific operation that failed, and
> +	 * hoping the userspace driver can replay things from there. This all
> +	 * sounds very complicated for little gain.
> +	 *
> +	 * Instead, we should just flag the VM as unusable, and fail any
> +	 * further request targeting this VM.
> +	 *
> +	 * We also provide a way to query a VM state, so userspace can destroy
> +	 * it and create a new one.
> +	 *
> +	 * As an analogy, this would be mapped to a VK_ERROR_DEVICE_LOST
> +	 * situation, where the logical device needs to be re-created.
> +	 */
> +	bool unusable;
> +};
> +
> +/**
> + * struct panthor_vm_bind_job - VM bind job
> + */
> +struct panthor_vm_bind_job {
> +	/** @base: Inherit from drm_sched_job. */
> +	struct drm_sched_job base;
> +
> +	/** @refcount: Reference count. */
> +	struct kref refcount;
> +
> +	/** @cleanup_op_ctx_work: Work used to cleanup the VM operation context. */
> +	struct work_struct cleanup_op_ctx_work;
> +
> +	/** @vm: VM targeted by the VM operation. */
> +	struct panthor_vm *vm;
> +
> +	/** @ctx: Operation context. */
> +	struct panthor_vm_op_ctx ctx;
> +};
> +
> +/**
> + * @pt_cache: Cache used to allocate MMU page tables.
> + *
> + * The pre-allocation pattern forces us to over-allocate to plan for
> + * the worst case scenario, and return the pages we didn't use.
> + *
> + * Having a kmem_cache allows us to speed allocations.
> + */
> +static struct kmem_cache *pt_cache;
> +
> +/**
> + * alloc_pt() - Custom page table allocator
> + * @cookie: Cookie passed at page table allocation time.
> + * @size: Size of the page table. This size should be fixed,
> + * and determined at creation time based on the granule size.
> + * @gfp: GFP flags.
> + *
> + * We want a custom allocator so we can use a cache for page table
> + * allocations and amortize the cost of the over-reservation that's
> + * done to allow asynchronous VM operations.
> + *
> + * Return: non-NULL on success, NULL if the allocation failed for any
> + * reason.
> + */
> +static void *alloc_pt(void *cookie, size_t size, gfp_t gfp)
> +{
> +	struct panthor_vm *vm = cookie;
> +	void *page;
> +
> +	/* We're not supposed to have anything bigger than 4k here, because we picked a
> +	 * 4k granule size at init time.
> +	 */
> +	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
> +		return NULL;
> +
> +	/* Allocation of the root page table happening during init. */
> +	if (!vm->pgtbl_ops) {
> +		drm_WARN_ON(&vm->ptdev->base, vm->op_ctx);
> +		page = kmem_cache_alloc(pt_cache, gfp);
> +		goto out;
> +	}
> +
> +	/* We must have some op_ctx attached to the VM and it must have at least one
> +	 * free page.
> +	 */
> +	if (drm_WARN_ON(&vm->ptdev->base, !vm->op_ctx) ||
> +	    drm_WARN_ON(&vm->ptdev->base,
> +			vm->op_ctx->rsvd_page_tables.ptr >= vm->op_ctx->rsvd_page_tables.count))
> +		return NULL;
> +
> +	page = vm->op_ctx->rsvd_page_tables.pages[vm->op_ctx->rsvd_page_tables.ptr++];
> +	memset(page, 0, SZ_4K);
> +
> +out:
> +	/* Page table entries don't use virtual addresses, which trips out
> +	 * kmemleak. kmemleak_alloc_phys() might work, but physical addresses
> +	 * are mixed with other fields, and I fear kmemleak won't detect that
> +	 * either.
> +	 *
> +	 * Let's just ignore memory passed to the page-table driver for now.
> +	 */
> +	kmemleak_ignore(page);
> +	return page;
> +}
> +
> +/**
> + * @free_pt() - Custom page table free function
> + * @cookie: Cookie passed at page table allocation time.
> + * @data: Page table to free.
> + * @size: Size of the page table. This size should be fixed,
> + * and determined at creation time based on the granule size.
> + */
> +static void free_pt(void *cookie, void *data, size_t size)
> +{
> +	struct panthor_vm *vm = cookie;
> +
> +	if (drm_WARN_ON(&vm->ptdev->base, size != SZ_4K))
> +		return;
> +
> +	/* Return the page to the pt_cache. */
> +	kmem_cache_free(pt_cache, data);
> +}
> +
> +static int wait_ready(struct panthor_device *ptdev, u32 as_nr)
> +{
> +	int ret;
> +	u32 val;
> +
> +	/* Wait for the MMU status to indicate there is no active command, in
> +	 * case one is pending.
> +	 */
> +	ret = readl_relaxed_poll_timeout_atomic(ptdev->iomem + AS_STATUS(as_nr),
> +						val, !(val & AS_STATUS_AS_ACTIVE),
> +						10, 100000);
> +
> +	if (ret) {
> +		panthor_device_schedule_reset(ptdev);
> +		drm_err(&ptdev->base, "AS_ACTIVE bit stuck\n");
> +	}
> +
> +	return ret;
> +}
> +
> +static int write_cmd(struct panthor_device *ptdev, u32 as_nr, u32 cmd)
> +{
> +	int status;
> +
> +	/* write AS_COMMAND when MMU is ready to accept another command */
> +	status = wait_ready(ptdev, as_nr);
> +	if (!status)
> +		gpu_write(ptdev, AS_COMMAND(as_nr), cmd);
> +
> +	return status;
> +}
> +
> +static void lock_region(struct panthor_device *ptdev, u32 as_nr,
> +			u64 region_start, u64 size)
> +{
> +	u8 region_width;
> +	u64 region;
> +	u64 region_end = region_start + size;
> +
> +	if (!size)
> +		return;
> +
> +	/*
> +	 * The locked region is a naturally aligned power of 2 block encoded as
> +	 * log2 minus(1).
> +	 * Calculate the desired start/end and look for the highest bit which
> +	 * differs. The smallest naturally aligned block must include this bit
> +	 * change, the desired region starts with this bit (and subsequent bits)
> +	 * zeroed and ends with the bit (and subsequent bits) set to one.
> +	 */
> +	region_width = max(fls64(region_start ^ (region_end - 1)),
> +			   const_ilog2(AS_LOCK_REGION_MIN_SIZE)) - 1;
> +
> +	/*
> +	 * Mask off the low bits of region_start (which would be ignored by
> +	 * the hardware anyway)
> +	 */
> +	region_start &= GENMASK_ULL(63, region_width);
> +
> +	region = region_width | region_start;
> +
> +	/* Lock the region that needs to be updated */
> +	gpu_write(ptdev, AS_LOCKADDR_LO(as_nr), lower_32_bits(region));
> +	gpu_write(ptdev, AS_LOCKADDR_HI(as_nr), upper_32_bits(region));
> +	write_cmd(ptdev, as_nr, AS_COMMAND_LOCK);
> +}
> +
> +static int mmu_hw_do_operation_locked(struct panthor_device *ptdev, int as_nr,
> +				      u64 iova, u64 size, u32 op)
> +{
> +	if (as_nr < 0)
> +		return 0;
> +
> +	if (op != AS_COMMAND_UNLOCK)
> +		lock_region(ptdev, as_nr, iova, size);
> +
> +	/* Run the MMU operation */
> +	write_cmd(ptdev, as_nr, op);
> +
> +	/* Wait for the flush to complete */
> +	return wait_ready(ptdev, as_nr);
> +}
> +
> +static int mmu_hw_do_operation(struct panthor_vm *vm,
> +			       u64 iova, u64 size, u32 op)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	int ret;
> +
> +	spin_lock(&ptdev->mmu->as.slots[vm->as.id].lock);
> +	ret = mmu_hw_do_operation_locked(ptdev, vm->as.id, iova, size, op);
> +	spin_unlock(&ptdev->mmu->as.slots[vm->as.id].lock);
> +	return ret;
> +}
> +
> +static int panthor_mmu_as_enable(struct panthor_device *ptdev, u32 as_nr,
> +				 u64 transtab, u64 transcfg, u64 memattr)
> +{
> +	int ret;
> +
> +	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
> +	if (ret)
> +		return ret;
> +
> +	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), lower_32_bits(transtab));
> +	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), upper_32_bits(transtab));
> +
> +	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), lower_32_bits(memattr));
> +	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), upper_32_bits(memattr));
> +
> +	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), lower_32_bits(transcfg));
> +	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), upper_32_bits(transcfg));
> +
> +	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
> +}
> +
> +static int panthor_mmu_as_disable(struct panthor_device *ptdev, u32 as_nr)
> +{
> +	int ret;
> +
> +	ret = mmu_hw_do_operation_locked(ptdev, as_nr, 0, ~0ULL, AS_COMMAND_FLUSH_MEM);
> +	if (ret)
> +		return ret;
> +
> +	gpu_write(ptdev, AS_TRANSTAB_LO(as_nr), 0);
> +	gpu_write(ptdev, AS_TRANSTAB_HI(as_nr), 0);
> +
> +	gpu_write(ptdev, AS_MEMATTR_LO(as_nr), 0);
> +	gpu_write(ptdev, AS_MEMATTR_HI(as_nr), 0);
> +
> +	gpu_write(ptdev, AS_TRANSCFG_LO(as_nr), AS_TRANSCFG_ADRMODE_UNMAPPED);
> +	gpu_write(ptdev, AS_TRANSCFG_HI(as_nr), 0);
> +
> +	return write_cmd(ptdev, as_nr, AS_COMMAND_UPDATE);
> +}
> +
> +static u32 panthor_mmu_fault_mask(struct panthor_device *ptdev, u32 value)
> +{
> +	/* Bits 16 to 31 mean REQ_COMPLETE. */
> +	return value & GENMASK(15, 0);
> +}
> +
> +static u32 panthor_mmu_as_fault_mask(struct panthor_device *ptdev, u32 as)
> +{
> +	return BIT(as);
> +}
> +
> +/**
> + * panthor_vm_active() - Flag a VM as active
> + * @VM: VM to flag as active.
> + *
> + * Assigns an address space to a VM so it can be used by the GPU/MCU.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_active(struct panthor_vm *vm)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	struct io_pgtable_cfg *cfg = &io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg;
> +	int ret = 0, as, cookie;
> +	u64 transtab, transcfg;
> +
> +	if (!drm_dev_enter(&ptdev->base, &cookie))
> +		return -ENODEV;
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +
> +	as = vm->as.id;
> +	if (as >= 0) {
> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> +
> +		if (ptdev->mmu->as.faulty_mask & mask) {
> +			/* Unhandled pagefault on this AS, the MMU was
> +			 * disabled. We need to re-enable the MMU after
> +			 * clearing+unmasking the AS interrupts.
> +			 */
> +			gpu_write(ptdev, MMU_INT_CLEAR, mask);
> +			ptdev->mmu->as.faulty_mask &= ~mask;
> +			gpu_write(ptdev, MMU_INT_MASK, ~ptdev->mmu->as.faulty_mask);
> +			goto out_enable_as;
> +		}
> +
> +		goto out_unlock;
> +	}
> +
> +	/* Check for a free AS */
> +	if (vm->for_mcu) {
> +		drm_WARN_ON(&ptdev->base, ptdev->mmu->as.alloc_mask & BIT(0));
> +		as = 0;
> +	} else {
> +		as = ffz(ptdev->mmu->as.alloc_mask | BIT(0));
> +	}
> +
> +	if (!(BIT(as) & ptdev->gpu_info.as_present)) {
> +		struct panthor_vm *lru_vm;
> +
> +		lru_vm = list_first_entry_or_null(&ptdev->mmu->as.lru_list,
> +						  struct panthor_vm,
> +						  as.lru_node);
> +		if (drm_WARN_ON(&ptdev->base, !lru_vm)) {
> +			ret = -EBUSY;
> +			goto out_unlock;
> +		}
> +
> +		list_del_init(&lru_vm->as.lru_node);
> +		as = lru_vm->as.id;

Should this not set lru_vm->as.id = -1, so that the code knows the VM no
longer has an address space?

> +	} else {
> +		set_bit(as, &ptdev->mmu->as.alloc_mask);
> +	}
> +
> +	/* Assign the free or reclaimed AS to the FD */
> +	vm->as.id = as;
> +	ptdev->mmu->as.slots[as].vm = vm;
> +
> +out_enable_as:
> +	transtab = cfg->arm_lpae_s1_cfg.ttbr;
> +	transcfg = AS_TRANSCFG_PTW_MEMATTR_WB |
> +		   AS_TRANSCFG_PTW_RA |
> +		   AS_TRANSCFG_ADRMODE_AARCH64_4K;
> +	if (ptdev->coherent)
> +		transcfg |= AS_TRANSCFG_PTW_SH_OS;
> +
> +	ret = panthor_mmu_as_enable(vm->ptdev, vm->as.id, transtab, transcfg, vm->memattr);
> +
> +out_unlock:
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +	drm_dev_exit(cookie);
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_idle() - Flag a VM idle
> + * @VM: VM to flag as idle.
> + *
> + * When we know the GPU is done with the VM (no more jobs to process),
> + * we can relinquish the AS slot attached to this VM, if any.
> + *
> + * We don't release the slot immediately, but instead place the VM in
> + * the LRU list, so it can be evicted if another VM needs an AS slot.
> + * This way, VMs keep attached to the AS they were given until we run
> + * out of free slot, limiting the number of MMU operations (TLB flush
> + * and other AS updates).
> + */
> +void panthor_vm_idle(struct panthor_vm *vm)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	if (vm->as.id >= 0 && list_empty(&vm->as.lru_node))
> +		list_add_tail(&vm->as.lru_node, &ptdev->mmu->as.lru_list);
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +}
> +
> +static void panthor_vm_stop(struct panthor_vm *vm)
> +{
> +	drm_sched_stop(&vm->sched, NULL);
> +}
> +
> +static void panthor_vm_start(struct panthor_vm *vm)
> +{
> +	drm_sched_start(&vm->sched, true);
> +}
> +
> +/**
> + * panthor_vm_as() - Get the AS slot attached to a VM
> + * @vm: VM to get the AS slot of.
> + *
> + * Return: -1 if the VM is not assigned an AS slot yet, >= 0 otherwise.
> + */
> +int panthor_vm_as(struct panthor_vm *vm)
> +{
> +	return vm->as.id;
> +}
> +
> +static size_t get_pgsize(u64 addr, size_t size, size_t *count)
> +{
> +	/*
> +	 * io-pgtable only operates on multiple pages within a single table
> +	 * entry, so we need to split at boundaries of the table size, i.e.
> +	 * the next block size up. The distance from address A to the next
> +	 * boundary of block size B is logically B - A % B, but in unsigned
> +	 * two's complement where B is a power of two we get the equivalence
> +	 * B - A % B == (B - A) % B == (n * B - A) % B, and choose n = 0 :)
> +	 */
> +	size_t blk_offset = -addr % SZ_2M;
> +
> +	if (blk_offset || size < SZ_2M) {
> +		*count = min_not_zero(blk_offset, size) / SZ_4K;
> +		return SZ_4K;
> +	}
> +	blk_offset = -addr % SZ_1G ?: SZ_1G;
> +	*count = min(blk_offset, size) / SZ_2M;
> +	return SZ_2M;
> +}
> +
> +static int panthor_vm_flush_range(struct panthor_vm *vm, u64 iova, u64 size)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	int ret = 0, cookie;
> +
> +	if (vm->as.id < 0)
> +		return 0;
> +
> +	/* If the device is unplugged, we just silently skip the flush. */
> +	if (!drm_dev_enter(&ptdev->base, &cookie))
> +		return 0;
> +
> +	/* Flush the PTs only if we're already awake */
> +	if (pm_runtime_active(ptdev->base.dev))
> +		ret = mmu_hw_do_operation(vm, iova, size, AS_COMMAND_FLUSH_PT);
> +
> +	drm_dev_exit(cookie);
> +	return ret;
> +}
> +
> +static int panthor_vm_unmap_pages(struct panthor_vm *vm, u64 iova, size_t size)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	struct io_pgtable_ops *ops = vm->pgtbl_ops;
> +	size_t offset = 0;
> +
> +	drm_dbg(&ptdev->base, "unmap: as=%d, iova=%llx, len=%zx", vm->as.id, iova, size);
> +
> +	while (offset < size) {
> +		size_t unmapped_sz = 0, pgcount;
> +		size_t pgsize = get_pgsize(iova + offset, size - offset, &pgcount);
> +
> +		unmapped_sz = ops->unmap_pages(ops, iova + offset, pgsize, pgcount, NULL);
> +
> +		if (drm_WARN_ON(&ptdev->base, unmapped_sz != pgsize * pgcount)) {
> +			drm_err(&ptdev->base, "failed to unmap range %llx-%llx (requested range %llx-%llx)\n",
> +				iova + offset + unmapped_sz,
> +				iova + offset + pgsize * pgcount,
> +				iova, iova + size);
> +			panthor_vm_flush_range(vm, iova, offset + unmapped_sz);
> +			return  -EINVAL;
> +		}
> +		offset += unmapped_sz;
> +	}
> +
> +	return panthor_vm_flush_range(vm, iova, size);
> +}
> +
> +static int
> +panthor_vm_map_pages(struct panthor_vm *vm, u64 iova, int prot,
> +		     struct sg_table *sgt, u64 offset, ssize_t size)
> +{
> +	struct panthor_device *ptdev = vm->ptdev;
> +	unsigned int count;
> +	struct scatterlist *sgl;
> +	struct io_pgtable_ops *ops = vm->pgtbl_ops;
> +	u64 start_iova = iova;
> +	int ret;
> +
> +	if (!size)
> +		return 0;
> +
> +	for_each_sgtable_dma_sg(sgt, sgl, count) {
> +		dma_addr_t paddr = sg_dma_address(sgl);
> +		size_t len = sg_dma_len(sgl);
> +
> +		if (len <= offset) {
> +			offset -= len;
> +			continue;
> +		}
> +
> +		paddr -= offset;
> +		len -= offset;
> +
> +		if (size >= 0) {
> +			len = min_t(size_t, len, size);
> +			size -= len;
> +		}
> +
> +		drm_dbg(&ptdev->base, "map: as=%d, iova=%llx, paddr=%llx, len=%zx",
> +			vm->as.id, iova, paddr, len);
> +
> +		while (len) {
> +			size_t pgcount, mapped = 0;
> +			size_t pgsize = get_pgsize(iova | paddr, len, &pgcount);
> +
> +			ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot,
> +					     GFP_KERNEL, &mapped);
> +			iova += mapped;
> +			paddr += mapped;
> +			len -= mapped;
> +
> +			if (drm_WARN_ON(&ptdev->base, !ret && !mapped))
> +				ret = -ENOMEM;
> +
> +			if (ret) {
> +				/* If something failed, unmap what we've already mapped before
> +				 * returning. The unmap call is not supposed to fail.
> +				 */
> +				drm_WARN_ON(&ptdev->base,
> +					    panthor_vm_unmap_pages(vm, start_iova,
> +								   iova - start_iova));
> +				return ret;
> +			}
> +		}
> +
> +		if (!size)
> +			break;
> +	}
> +
> +	return panthor_vm_flush_range(vm, start_iova, iova - start_iova);
> +}
> +
> +static int flags_to_prot(u32 flags)
> +{
> +	int prot = 0;
> +
> +	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC)
> +		prot |= IOMMU_NOEXEC;
> +
> +	if (!(flags & DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED))
> +		prot |= IOMMU_CACHE;
> +
> +	if (flags & DRM_PANTHOR_VM_BIND_OP_MAP_READONLY)
> +		prot |= IOMMU_READ;
> +	else
> +		prot |= IOMMU_READ | IOMMU_WRITE;
> +
> +	return prot;
> +}
> +
> +/**
> + * panthor_vm_alloc_va() - Allocate a region in the auto-va space
> + * @VM: VM to allocate a region on.
> + * @size: Size of the region.
> + *
> + * Some GPU objects, like heap chunks, are fully managed by the kernel and
> + * need to be mapped to the userspace VM, in the region reserved for kernel
> + * objects.
> + *
> + * This function takes care of allocating a region in this reserved space.
> + *
> + * Return: A valid pointer on success, and ERR_PTR() otherwise.
> + */
> +struct drm_mm_node *
> +panthor_vm_alloc_va(struct panthor_vm *vm, size_t size)
> +{
> +	struct drm_mm_node *mm_node;
> +	int ret;
> +
> +	if (!size || (size & ~PAGE_MASK))
> +		return ERR_PTR(-EINVAL);
> +
> +	mm_node = kzalloc(sizeof(*mm_node), GFP_KERNEL);
> +	if (!mm_node)
> +		return ERR_PTR(-ENOMEM);
> +
> +	mutex_lock(&vm->mm_lock);
> +	ret = drm_mm_insert_node(&vm->mm, mm_node, size);
> +	mutex_unlock(&vm->mm_lock);
> +
> +	if (ret) {
> +		kfree(mm_node);
> +		return ERR_PTR(ret);
> +	}
> +
> +	return mm_node;
> +}
> +
> +/**
> + * panthor_vm_free_va() - Free a region allocated with panthor_vm_alloc_va()
> + * @VM: VM to free the region on.
> + * @mm_node: Memory node representing the region to free.
> + */
> +void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *mm_node)
> +{
> +	if (!mm_node)
> +		return;
> +
> +	mutex_lock(&vm->mm_lock);
> +	drm_mm_remove_node(mm_node);
> +	mutex_unlock(&vm->mm_lock);
> +
> +	kfree(mm_node);
> +}
> +
> +static void panthor_vm_cleanup_op_ctx(struct panthor_vm_op_ctx *op_ctx,
> +				      struct panthor_vm *vm)
> +{
> +	struct panthor_vma *vma, *tmp_vma;
> +
> +	u32 remaining_pt_count = op_ctx->rsvd_page_tables.count -
> +				 op_ctx->rsvd_page_tables.ptr;
> +
> +	if (remaining_pt_count) {
> +		kmem_cache_free_bulk(pt_cache, remaining_pt_count,
> +				     op_ctx->rsvd_page_tables.pages +
> +				     op_ctx->rsvd_page_tables.ptr);
> +	}
> +
> +	kfree(op_ctx->rsvd_page_tables.pages);
> +	memset(&op_ctx->rsvd_page_tables, 0, sizeof(op_ctx->rsvd_page_tables));
> +
> +	if (op_ctx->map.gem.obj) {
> +		struct panthor_gem_object *bo = to_panthor_bo(op_ctx->map.gem.obj);
> +
> +		if (!bo->base.base.import_attach)
> +			drm_gem_shmem_unpin(&bo->base);
> +
> +		drm_gem_object_put(&bo->base.base);
> +	}
> +
> +	kfree(op_ctx->map.new_vma);
> +	kfree(op_ctx->map.next_vma);
> +	kfree(op_ctx->map.prev_vma);
> +	memset(&op_ctx->map, 0, sizeof(op_ctx->map));
> +
> +	list_for_each_entry_safe(vma, tmp_vma, &op_ctx->returned_vmas, node) {
> +		struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
> +
> +		if (!bo->base.base.import_attach)
> +			drm_gem_shmem_unpin(&bo->base);
> +
> +		drm_gem_object_put(&bo->base.base);
> +		list_del(&vma->node);
> +		kfree(vma);
> +	}
> +}
> +
> +#define PANTHOR_VM_BIND_OP_MAP_FLAGS \
> +	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED | \
> +	 DRM_PANTHOR_VM_BIND_OP_TYPE_MASK)
> +
> +static int panthor_vm_prepare_map_op_ctx(struct panthor_vm_op_ctx *op_ctx,
> +					 struct panthor_vm *vm,
> +					 struct panthor_gem_object *bo,
> +					 u64 offset,
> +					 size_t size, u64 va,
> +					 u32 flags)
> +{
> +	struct sg_table *sgt = NULL;
> +	u64 pt_count;
> +	int ret;
> +
> +	if (!bo)
> +		return -EINVAL;
> +
> +	if ((flags & ~PANTHOR_VM_BIND_OP_MAP_FLAGS) ||
> +	    (flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) != DRM_PANTHOR_VM_BIND_OP_TYPE_MAP)
> +		return -EINVAL;
> +
> +	/* Make sure the VA and size are aligned and in-bounds. */
> +	if (size > bo->base.base.size || offset > bo->base.base.size - size)
> +		return -EINVAL;
> +
> +	/* If the BO has an exclusive VM attached, it can't be mapped to other VMs. */
> +	if (bo->exclusive_vm && bo->exclusive_vm != vm)
> +		return -EINVAL;
> +
> +	memset(op_ctx, 0, sizeof(*op_ctx));
> +	INIT_LIST_HEAD(&op_ctx->returned_vmas);
> +	op_ctx->flags = flags;
> +	op_ctx->va.range = size;
> +	op_ctx->va.addr = va;
> +
> +	op_ctx->map.new_vma = kzalloc(sizeof(*op_ctx->map.new_vma), GFP_KERNEL);
> +	op_ctx->map.next_vma = kzalloc(sizeof(*op_ctx->map.next_vma), GFP_KERNEL);
> +	op_ctx->map.prev_vma = kzalloc(sizeof(*op_ctx->map.prev_vma), GFP_KERNEL);
> +	if (!op_ctx->map.new_vma || !op_ctx->map.next_vma || !op_ctx->map.prev_vma) {
> +		ret = -ENOMEM;
> +		goto err_cleanup;
> +	}
> +
> +	if (!bo->base.base.import_attach) {
> +		/* Pre-reserve the BO pages, so the map operation doesn't have to
> +		 * allocate.
> +		 */
> +		ret = drm_gem_shmem_pin(&bo->base);
> +		if (ret)
> +			goto err_cleanup;
> +	}
> +
> +	sgt = drm_gem_shmem_get_pages_sgt(&bo->base);
> +	if (IS_ERR(sgt)) {
> +		if (!bo->base.base.import_attach)
> +			drm_gem_shmem_unpin(&bo->base);
> +
> +		ret = PTR_ERR(sgt);
> +		goto err_cleanup;
> +	}
> +
> +	op_ctx->map.sgt = sgt;
> +	op_ctx->map.gem.obj = &bo->base.base;
> +	op_ctx->map.gem.offset = offset;
> +	drm_gem_object_get(op_ctx->map.gem.obj);
> +
> +	/* L1, L2 and L3 page tables.
> +	 * We could optimize L3 allocation by iterating over the sgt and merging
> +	 * 2M contiguous blocks, but it's simpler to over-provision and return
> +	 * the pages if they're not used.
> +	 */
> +	pt_count = ((ALIGN(va + size, 1ull << 39) - ALIGN_DOWN(va, 1ull << 39)) >> 39) +
> +		   ((ALIGN(va + size, 1ull << 30) - ALIGN_DOWN(va, 1ull << 30)) >> 30) +
> +		   ((ALIGN(va + size, 1ull << 21) - ALIGN_DOWN(va, 1ull << 21)) >> 21);
> +
> +	op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
> +						 sizeof(*op_ctx->rsvd_page_tables.pages),
> +						 GFP_KERNEL);
> +	if (!op_ctx->rsvd_page_tables.pages)
> +		goto err_cleanup;
> +
> +	ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
> +				    op_ctx->rsvd_page_tables.pages);
> +	op_ctx->rsvd_page_tables.count = ret;
> +	if (ret != pt_count) {
> +		ret = -ENOMEM;
> +		goto err_cleanup;
> +	}
> +
> +	return 0;
> +
> +err_cleanup:
> +	panthor_vm_cleanup_op_ctx(op_ctx, vm);
> +	return ret;
> +}
> +
> +static int panthor_vm_prepare_unmap_op_ctx(struct panthor_vm_op_ctx *op_ctx,
> +					   struct panthor_vm *vm,
> +					   u64 va, size_t size)
> +{
> +	u32 pt_count = 0;
> +	int ret;
> +
> +	memset(op_ctx, 0, sizeof(*op_ctx));
> +	INIT_LIST_HEAD(&op_ctx->returned_vmas);
> +	op_ctx->va.range = size;
> +	op_ctx->va.addr = va;
> +	op_ctx->flags = DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP;
> +
> +	/* Pre-allocate L3 page tables to account for the split-2M-block
> +	 * situation on unmap.
> +	 */
> +	if (va != ALIGN(va, SZ_2M))
> +		pt_count++;
> +
> +	if (va + size != ALIGN(va + size, SZ_2M) &&
> +	    ALIGN(va + size, SZ_2M) != ALIGN(va, SZ_2M))
> +		pt_count++;
> +
> +	if (pt_count) {
> +		op_ctx->rsvd_page_tables.pages = kcalloc(pt_count,
> +							 sizeof(*op_ctx->rsvd_page_tables.pages),
> +							 GFP_KERNEL);
> +		if (!op_ctx->rsvd_page_tables.pages)
> +			goto err_cleanup;
> +
> +		ret = kmem_cache_alloc_bulk(pt_cache, GFP_KERNEL, pt_count,
> +					    op_ctx->rsvd_page_tables.pages);
> +		if (ret != pt_count) {
> +			ret = -ENOMEM;
> +			goto err_cleanup;
> +		}
> +		op_ctx->rsvd_page_tables.count = pt_count;
> +	}
> +
> +	return 0;
> +
> +err_cleanup:
> +	panthor_vm_cleanup_op_ctx(op_ctx, vm);
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_get_bo_for_va() - Get the GEM object mapped at a virtual address
> + * @vm: VM to look into.
> + * @va: Virtual address to search for.
> + * @bo_offset: Offset of the GEM object mapped at this virtual address.
> + * Only valid on success.
> + *
> + * The object returned by this function might no longer be mapped when the
> + * function returns. It's the caller responsibility to ensure there's no
> + * concurrent map/unmap operations making the returned value invalid, or
> + * make sure it doesn't matter if the object is no longer mapped.
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct panthor_gem_object *
> +panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset)
> +{
> +	struct panthor_gem_object *bo = ERR_PTR(-ENOENT);
> +	struct drm_gpuva *gpuva;
> +	struct panthor_vma *vma;
> +	int ret;
> +
> +	/* Take the VM lock to prevent concurrent map/unmap operation. */
> +	ret = dma_resv_lock(vm->dummy_gem.resv, NULL);
> +	if (drm_WARN_ON(&vm->ptdev->base, ret))
> +		return NULL;
> +
> +	gpuva = drm_gpuva_find_first(&vm->va_mgr, va, 1);
> +	vma = gpuva ? container_of(gpuva, struct panthor_vma, base) : NULL;
> +	if (vma && vma->base.gem.obj) {
> +		drm_gem_object_get(vma->base.gem.obj);
> +		bo = to_panthor_bo(vma->base.gem.obj);
> +		*bo_offset = vma->base.gem.offset;
> +	}
> +	dma_resv_unlock(vm->dummy_gem.resv);
> +
> +	return bo;
> +}
> +
> +/*
> + * Only 32 VMs per open file. If that becomes a limiting factor, we can
> + * increase this number.
> + */
> +#define PANTHOR_MAX_VMS_PER_FILE	 32
> +
> +/**
> + * panthor_vm_pool_create_vm() - Create a VM
> + * @pool: The VM to create this VM on.
> + * @kernel_va_start: Start of the region reserved for kernel objects.
> + * @kernel_va_range: Size of the region reserved for kernel objects.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_pool_create_vm(struct panthor_device *ptdev, struct panthor_vm_pool *pool,
> +			      u64 kernel_va_start, u64 kernel_va_range)
> +{
> +	struct panthor_vm *vm;
> +	int ret;
> +	u32 id;
> +
> +	vm = panthor_vm_create(ptdev, false, kernel_va_start, kernel_va_range);
> +	if (IS_ERR(vm))
> +		return PTR_ERR(vm);
> +
> +	ret = xa_alloc(&pool->xa, &id, vm,
> +		       XA_LIMIT(1, PANTHOR_MAX_VMS_PER_FILE), GFP_KERNEL);
> +
> +	if (ret) {
> +		panthor_vm_put(vm);
> +		return ret;
> +	}
> +
> +	return id;
> +}
> +
> +static void panthor_vm_destroy(struct panthor_vm *vm)
> +{
> +	if (!vm)
> +		return;
> +
> +	vm->destroyed = true;
> +
> +	mutex_lock(&vm->heaps.lock);
> +	panthor_heap_pool_destroy(vm->heaps.pool);
> +	vm->heaps.pool = NULL;
> +	mutex_unlock(&vm->heaps.lock);
> +
> +	drm_WARN_ON(&vm->ptdev->base,
> +		    panthor_vm_unmap_range(vm, vm->va_mgr.mm_start, vm->va_mgr.mm_range));
> +	panthor_vm_put(vm);
> +}
> +
> +/**
> + * panthor_vm_destroy() - Destroy a VM.
> + * @pool: VM pool.
> + * @handle: VM handle.
> + *
> + * This function doesn't free the VM object or its resources, it just kills
> + * all mappings, and makes sure nothing can be mapped after that point.
> + *
> + * If there was any active jobs at the time this function is called, these
> + * jobs should experience page faults and be killed as a result.
> + *
> + * The VM resources are freed when the last reference on the VM object is
> + * dropped.
> + */
> +int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle)
> +{
> +	struct panthor_vm *vm;
> +
> +	vm = xa_erase(&pool->xa, handle);
> +
> +	panthor_vm_destroy(vm);
> +
> +	return vm ? 0 : -EINVAL;
> +}
> +
> +/**
> + * panthor_vm_pool_get_vm() - Retrieve VM object bound to a VM handle
> + * @pool: VM pool to check.
> + * @handle: Handle of the VM to retrieve.
> + *
> + * Return: A valid pointer if the VM exists, NULL otherwise.
> + */
> +struct panthor_vm *
> +panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle)
> +{
> +	struct panthor_vm *vm;
> +
> +	vm = panthor_vm_get(xa_load(&pool->xa, handle));
> +
> +	return vm;
> +}
> +
> +/**
> + * panthor_vm_pool_destroy() - Destroy a VM pool.
> + * @pfile: File.
> + *
> + * Destroy all VMs in the pool, and release the pool resources.
> + *
> + * Note that VMs can outlive the pool they were created from if other
> + * objects hold a reference to there VMs.
> + */
> +void panthor_vm_pool_destroy(struct panthor_file *pfile)
> +{
> +	struct panthor_vm *vm;
> +	unsigned long i;
> +
> +	if (!pfile->vms)
> +		return;
> +
> +	xa_for_each(&pfile->vms->xa, i, vm)
> +		panthor_vm_destroy(vm);
> +
> +	xa_destroy(&pfile->vms->xa);
> +	kfree(pfile->vms);
> +}
> +
> +/**
> + * panthor_vm_pool_create() - Create a VM pool
> + * @pfile: File.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_pool_create(struct panthor_file *pfile)
> +{
> +	pfile->vms = kzalloc(sizeof(*pfile->vms), GFP_KERNEL);
> +	if (!pfile->vms)
> +		return -ENOMEM;
> +
> +	xa_init_flags(&pfile->vms->xa, XA_FLAGS_ALLOC1);
> +	return 0;
> +}
> +
> +/* dummy TLB ops, the real TLB flush happens in panthor_vm_flush_range() */
> +static void mmu_tlb_flush_all(void *cookie)
> +{
> +}
> +
> +static void mmu_tlb_flush_walk(unsigned long iova, size_t size, size_t granule, void *cookie)
> +{
> +}
> +
> +static const struct iommu_flush_ops mmu_tlb_ops = {
> +	.tlb_flush_all = mmu_tlb_flush_all,
> +	.tlb_flush_walk = mmu_tlb_flush_walk,
> +};
> +
> +static const char *access_type_name(struct panthor_device *ptdev,
> +				    u32 fault_status)
> +{
> +	switch (fault_status & AS_FAULTSTATUS_ACCESS_TYPE_MASK) {
> +	case AS_FAULTSTATUS_ACCESS_TYPE_ATOMIC:
> +		return "ATOMIC";
> +	case AS_FAULTSTATUS_ACCESS_TYPE_READ:
> +		return "READ";
> +	case AS_FAULTSTATUS_ACCESS_TYPE_WRITE:
> +		return "WRITE";
> +	case AS_FAULTSTATUS_ACCESS_TYPE_EX:
> +		return "EXECUTE";
> +	default:
> +		drm_WARN_ON(&ptdev->base, 1);
> +		return NULL;
> +	}
> +}
> +
> +static void panthor_mmu_irq_handler(struct panthor_device *ptdev, u32 status)
> +{
> +	status = panthor_mmu_fault_mask(ptdev, status);
> +	while (status) {
> +		u32 as = ffs(status | (status >> 16)) - 1;
> +		u32 mask = panthor_mmu_as_fault_mask(ptdev, as);
> +		u32 new_int_mask;
> +		u64 addr;
> +		u32 fault_status;
> +		u32 exception_type;
> +		u32 access_type;
> +		u32 source_id;
> +
> +		fault_status = gpu_read(ptdev, AS_FAULTSTATUS(as));
> +		addr = gpu_read(ptdev, AS_FAULTADDRESS_LO(as));
> +		addr |= (u64)gpu_read(ptdev, AS_FAULTADDRESS_HI(as)) << 32;
> +
> +		/* decode the fault status */
> +		exception_type = fault_status & 0xFF;
> +		access_type = (fault_status >> 8) & 0x3;
> +		source_id = (fault_status >> 16);
> +
> +		/* Page fault only */

This comment makes no sense - it looks like it's copied over from panfrost.

If I understand correctly we don't (currently) support growing on page
fault - and it's not really needed now the MCU can handle the tiler heaps.

> +		mutex_lock(&ptdev->mmu->as.slots_lock);
> +
> +		new_int_mask =
> +			panthor_mmu_fault_mask(ptdev, ~ptdev->mmu->as.faulty_mask);
> +
> +		/* terminal fault, print info about the fault */
> +		drm_err(&ptdev->base,
> +			"Unhandled Page fault in AS%d at VA 0x%016llX\n"
> +			"raw fault status: 0x%X\n"
> +			"decoded fault status: %s\n"
> +			"exception type 0x%X: %s\n"
> +			"access type 0x%X: %s\n"
> +			"source id 0x%X\n",
> +			as, addr,
> +			fault_status,
> +			(fault_status & (1 << 10) ? "DECODER FAULT" : "SLAVE FAULT"),
> +			exception_type, panthor_exception_name(ptdev, exception_type),
> +			access_type, access_type_name(ptdev, fault_status),
> +			source_id);
> +
> +		/* Ignore MMU interrupts on this AS until it's been
> +		 * re-enabled.
> +		 */
> +		ptdev->mmu->irq.mask = new_int_mask;
> +		gpu_write(ptdev, MMU_INT_MASK, new_int_mask);
> +
> +		/* Disable the MMU to kill jobs on this AS. */
> +		panthor_mmu_as_disable(ptdev, as);
> +		mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +		status &= ~mask;
> +	}
> +}
> +PANTHOR_IRQ_HANDLER(mmu, MMU, panthor_mmu_irq_handler);
> +
> +/**
> + * panthor_mmu_suspend() - Suspend the MMU logic
> + * @ptdev: Device.
> + *
> + * All we do here is de-assign the AS slots on all active VMs, so things
> + * get flushed to the main memory, and no further access to these VMs are
> + * possible.
> + *
> + * We also suspend the MMU IRQ.
> + */
> +void panthor_mmu_suspend(struct panthor_device *ptdev)
> +{
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
> +		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
> +
> +		if (vm) {
> +			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
> +			vm->as.id = -1;
> +			list_del_init(&vm->as.lru_node);
> +			ptdev->mmu->as.slots[i].vm = NULL;
> +		}
> +	}
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
> +}
> +
> +/**
> + * panthor_mmu_resume() - Resume the MMU logic
> + * @ptdev: Device.
> + *
> + * Resume the IRQ.
> + *
> + * We don't re-enable previously active VMs. We assume other parts of the
> + * driver will call panthor_vm_active() on the VMs they intend to use.
> + */
> +void panthor_mmu_resume(struct panthor_device *ptdev)
> +{
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	ptdev->mmu->as.alloc_mask = 0;
> +	ptdev->mmu->as.faulty_mask = 0;
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
> +}
> +
> +/**
> + * panthor_mmu_pre_reset() - Prepare for a reset
> + * @ptdev: Device.
> + *
> + * Suspend the IRQ, and make sure all VM_BIND queues are stopped, so we
> + * don't get asked to do a VM operation while the GPU is down.
> + *
> + * We don't cleanly shutdown the AS slots here, because the reset might
> + * come from an AS_ACTIVE_BIT stuck situation.
> + */
> +void panthor_mmu_pre_reset(struct panthor_device *ptdev)
> +{
> +	struct panthor_vm *vm;
> +
> +	panthor_mmu_irq_suspend(&ptdev->mmu->irq);
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	ptdev->mmu->vm.reset_in_progress = true;
> +	list_for_each_entry(vm, &ptdev->mmu->vm.list, node)
> +		panthor_vm_stop(vm);
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +}
> +
> +/**
> + * panthor_mmu_post_reset() - Restore things after a reset
> + * @ptdev: Device.
> + *
> + * Put the MMU logic back in action after a reset. That implies resuming the
> + * IRQ and re-enabling the VM_BIND queues.
> + */
> +void panthor_mmu_post_reset(struct panthor_device *ptdev)
> +{
> +	struct panthor_vm *vm;
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +
> +	/* Now that the reset is effective, we can assume that none of the
> +	 * AS slots are setup, and clear the faulty flags too.
> +	 */
> +	ptdev->mmu->as.alloc_mask = 0;
> +	ptdev->mmu->as.faulty_mask = 0;
> +
> +	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
> +		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
> +
> +		if (vm) {
> +			vm->as.id = -1;
> +			list_del_init(&vm->as.lru_node);
> +			ptdev->mmu->as.slots[i].vm = NULL;
> +		}
> +	}
> +
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	panthor_mmu_irq_resume(&ptdev->mmu->irq, panthor_mmu_fault_mask(ptdev, ~0));
> +
> +	/* Restart the VM_BIND queues. */
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
> +		panthor_vm_start(vm);
> +	}
> +	ptdev->mmu->vm.reset_in_progress = false;
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +}
> +
> +static void panthor_vm_release(struct kref *kref)
> +{
> +	struct panthor_vm *vm = container_of(kref, struct panthor_vm, refcount);
> +	struct panthor_device *ptdev = vm->ptdev;
> +
> +	mutex_lock(&vm->heaps.lock);
> +	if (drm_WARN_ON(&ptdev->base, vm->heaps.pool))
> +		panthor_heap_pool_destroy(vm->heaps.pool);
> +	mutex_unlock(&vm->heaps.lock);
> +	mutex_destroy(&vm->heaps.lock);
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_del(&vm->node);
> +	/* Restore the scheduler state so we can call drm_sched_entity_destroy()
> +	 * and drm_sched_fini(). If get there, that means we have no job left
> +	 * and no new jobs can be queued, so we can start the scheduler without
> +	 * risking interfering with the reset.
> +	 */
> +	if (ptdev->mmu->vm.reset_in_progress)
> +		panthor_vm_start(vm);
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +
> +	drm_sched_entity_destroy(&vm->entity);
> +	drm_sched_fini(&vm->sched);
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	if (vm->as.id >= 0) {
> +		int cookie;
> +
> +		if (drm_dev_enter(&ptdev->base, &cookie)) {
> +			panthor_mmu_as_disable(ptdev, vm->as.id);
> +			drm_dev_exit(cookie);
> +		}
> +
> +		ptdev->mmu->as.slots[vm->as.id].vm = NULL;
> +		clear_bit(vm->as.id, &ptdev->mmu->as.alloc_mask);
> +		list_del(&vm->as.lru_node);
> +	}
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +
> +	drm_WARN_ON(&ptdev->base,
> +		    panthor_vm_unmap_range(vm, vm->va_mgr.mm_start, vm->va_mgr.mm_range));
> +
> +	free_io_pgtable_ops(vm->pgtbl_ops);
> +
> +	drm_mm_takedown(&vm->mm);
> +	mutex_destroy(&vm->mm_lock);
> +	drm_gpuva_manager_destroy(&vm->va_mgr);
> +	drm_gem_private_object_fini(&vm->dummy_gem);
> +	mutex_destroy(&vm->op_lock);
> +	kfree(vm);
> +}
> +
> +/**
> + * panthor_vm_put() - Release a reference on a VM
> + * @vm: VM to release the reference on. Can be NULL.
> + */
> +void panthor_vm_put(struct panthor_vm *vm)
> +{
> +	if (vm)
> +		kref_put(&vm->refcount, panthor_vm_release);
> +}
> +
> +/**
> + * panthor_vm_get() - Get a VM reference
> + * @vm: VM to get the reference on. Can be NULL.
> + *
> + * Return: @vm value.
> + */
> +struct panthor_vm *panthor_vm_get(struct panthor_vm *vm)
> +{
> +	if (vm)
> +		kref_get(&vm->refcount);
> +
> +	return vm;
> +}
> +
> +/**
> + * panthor_vm_get_heap_pool() - Get the heap pool attached to a VM
> + * @vm: VM to query the heap pool on.
> + * @create: True if the heap pool should be created when it doesn't exist.
> + *
> + * Heap pools are per-VM. This function allows one to retrieve the heap pool
> + * attached to a VM.
> + *
> + * If no heap pool exists yet, and @create is true, we create one.
> + *
> + * The returned panthor_heap_pool should be released with panthor_heap_pool_put().
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct panthor_heap_pool *panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create)
> +{
> +	struct panthor_heap_pool *pool;
> +
> +	mutex_lock(&vm->heaps.lock);
> +	if (!vm->heaps.pool && create) {
> +		if (vm->destroyed)
> +			pool = ERR_PTR(-EINVAL);
> +		else
> +			pool = panthor_heap_pool_create(vm->ptdev, vm);
> +
> +		if (!IS_ERR(pool))
> +			vm->heaps.pool = panthor_heap_pool_get(pool);
> +	} else {
> +		pool = panthor_heap_pool_get(vm->heaps.pool);
> +	}
> +	mutex_unlock(&vm->heaps.lock);
> +
> +	return pool;
> +}
> +
> +static u64 mair_to_memattr(u64 mair)
> +{
> +	u64 memattr = 0;
> +	u32 i;
> +
> +	for (i = 0; i < 8; i++) {
> +		u8 in_attr = mair >> (8 * i), out_attr;
> +		u8 outer = in_attr >> 4, inner = in_attr & 0xf;
> +
> +		/* For caching to be enabled, inner and outer caching policy
> +		 * have to be both write-back, if one of them is write-through
> +		 * or non-cacheable, we just choose non-cacheable. Device
> +		 * memory is also translated to non-cacheable.
> +		 */
> +		if (!(outer & 3) || !(outer & 4) || !(inner & 4)) {
> +			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_NC |
> +				   AS_MEMATTR_AARCH64_SH_MIDGARD_INNER |
> +				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(false, false);
> +		} else {
> +			/* Use SH_CPU_INNER mode so SH_IS, which is used when
> +			 * IOMMU_CACHE is set, actually maps to the standard
> +			 * definition of inner-shareable and not Mali's
> +			 * internal-shareable mode.
> +			 */
> +			out_attr = AS_MEMATTR_AARCH64_INNER_OUTER_WB |
> +				   AS_MEMATTR_AARCH64_SH_CPU_INNER |
> +				   AS_MEMATTR_AARCH64_INNER_ALLOC_EXPL(inner & 1, inner & 2);
> +		}
> +
> +		memattr |= (u64)out_attr << (8 * i);
> +	}
> +
> +	return memattr;
> +}
> +
> +static void panthor_vma_link(struct panthor_vm *vm, struct panthor_vma *vma)
> +{
> +	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
> +
> +	mutex_lock(&bo->gpuva_list_lock);
> +	drm_gpuva_link(&vma->base);
> +	mutex_unlock(&bo->gpuva_list_lock);
> +
> +	if (!bo->exclusive_vm)
> +		list_add_tail(&vma->node, &vm->shared_bos);
> +}
> +
> +static void panthor_vma_unlink(struct panthor_vm_op_ctx *op_ctx,
> +			       struct panthor_vma *vma)
> +{
> +	struct panthor_gem_object *bo = to_panthor_bo(vma->base.gem.obj);
> +
> +	mutex_lock(&bo->gpuva_list_lock);
> +	drm_gpuva_unlink(&vma->base);
> +	mutex_unlock(&bo->gpuva_list_lock);
> +
> +	list_move_tail(&vma->node, &op_ctx->returned_vmas);
> +}
> +
> +static void panthor_vma_init(struct panthor_vma *vma,
> +			     struct drm_gem_object *obj,
> +			     u64 offset,
> +			     u64 va, u64 range, u32 flags)
> +{
> +	INIT_LIST_HEAD(&vma->node);
> +	vma->flags = flags;
> +	vma->base.gem.obj = obj;
> +	vma->base.gem.offset = offset;
> +	vma->base.va.addr = va;
> +	vma->base.va.range = range;
> +}
> +
> +#define PANTHOR_VM_MAP_FLAGS \
> +	(DRM_PANTHOR_VM_BIND_OP_MAP_READONLY | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_NOEXEC | \
> +	 DRM_PANTHOR_VM_BIND_OP_MAP_UNCACHED)
> +
> +static int panthor_gpuva_sm_step_map(struct drm_gpuva_op *op, void *priv)
> +{
> +	struct panthor_vm *vm = priv;
> +	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
> +	struct panthor_vma *vma = op_ctx->map.new_vma;
> +	int ret;
> +
> +	panthor_vma_init(vma, op->map.gem.obj, op->map.gem.offset, op->map.va.addr,
> +			 op->map.va.range, op_ctx->flags & PANTHOR_VM_MAP_FLAGS);
> +
> +	ret = panthor_vm_map_pages(vm, vma->base.va.addr, flags_to_prot(vma->flags),
> +				   op_ctx->map.sgt, vma->base.gem.offset,
> +				   vma->base.va.range);
> +	if (ret)
> +		return ret;
> +
> +	/* Ref owned by the mapping now, clear the obj field so we don't release the
> +	 * pinning/obj ref behind GPUVA's back.
> +	 */
> +	drm_gpuva_map(&vm->va_mgr, &vma->base, &op->map);
> +	panthor_vma_link(vm, op_ctx->map.new_vma);
> +	op_ctx->map.gem.obj = NULL;
> +	op_ctx->map.new_vma = NULL;
> +	return 0;
> +}
> +
> +static int panthor_gpuva_sm_step_remap(struct drm_gpuva_op *op,
> +				       void *priv)
> +{
> +	struct panthor_vma *unmap_vma = container_of(op->remap.unmap->va, struct panthor_vma, base);
> +	const u64 va_start = op->remap.prev ?
> +			     op->remap.prev->va.addr + op->remap.prev->va.range :
> +			     op->remap.unmap->va->va.addr;
> +	const u64 va_end = op->remap.next ?
> +			   op->remap.next->va.addr :
> +			   op->remap.unmap->va->va.addr + op->remap.unmap->va->va.range;
> +	struct panthor_vm *vm = priv;
> +	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
> +	struct drm_gpuva *prev_va = NULL, *next_va = NULL;
> +	int ret;
> +
> +	ret = panthor_vm_unmap_pages(vm, va_start, va_end - va_start);
> +	if (ret)
> +		return ret;
> +
> +	if (op->remap.prev) {
> +		struct panthor_gem_object *bo = to_panthor_bo(op->remap.prev->gem.obj);
> +
> +		if (!bo->base.base.import_attach) {
> +			ret = drm_gem_shmem_pin(&bo->base);
> +			if (drm_WARN_ON(&vm->ptdev->base, ret))
> +				return ret;
> +		}
> +
> +		panthor_vma_init(op_ctx->map.prev_vma,
> +				 op->remap.prev->gem.obj,
> +				 op->remap.prev->gem.offset,
> +				 op->remap.prev->va.addr,
> +				 op->remap.prev->va.range,
> +				 unmap_vma->flags);
> +		prev_va = &op_ctx->map.prev_vma->base;
> +	}
> +
> +	if (op->remap.next) {
> +		struct panthor_gem_object *bo = to_panthor_bo(op->remap.next->gem.obj);
> +
> +		if (!bo->base.base.import_attach) {
> +			ret = drm_gem_shmem_pin(&bo->base);
> +			if (drm_WARN_ON(&vm->ptdev->base, ret))
> +				return ret;
> +		}
> +
> +		panthor_vma_init(op_ctx->map.next_vma,
> +				 op->remap.next->gem.obj,
> +				 op->remap.next->gem.offset,
> +				 op->remap.next->va.addr,
> +				 op->remap.next->va.range,
> +				 unmap_vma->flags);
> +		next_va = &op_ctx->map.next_vma->base;
> +	}
> +
> +	drm_gpuva_remap(prev_va, next_va, &op->remap);
> +
> +	if (prev_va) {
> +		drm_gem_object_get(prev_va->gem.obj);
> +		panthor_vma_link(vm, op_ctx->map.prev_vma);
> +		op_ctx->map.prev_vma = NULL;
> +	}
> +
> +	if (next_va) {
> +		drm_gem_object_get(next_va->gem.obj);
> +		panthor_vma_link(vm, op_ctx->map.next_vma);
> +		op_ctx->map.next_vma = NULL;
> +	}
> +
> +	panthor_vma_unlink(op_ctx, unmap_vma);
> +	return 0;
> +}
> +
> +static int panthor_gpuva_sm_step_unmap(struct drm_gpuva_op *op,
> +				       void *priv)
> +{
> +	struct panthor_vma *unmap_vma = container_of(op->unmap.va, struct panthor_vma, base);
> +	struct panthor_vm *vm = priv;
> +	struct panthor_vm_op_ctx *op_ctx = vm->op_ctx;
> +	int ret;
> +
> +	ret = panthor_vm_unmap_pages(vm, unmap_vma->base.va.addr,
> +				     unmap_vma->base.va.range);
> +	if (drm_WARN_ON(&vm->ptdev->base, ret))
> +		return ret;
> +
> +	drm_gpuva_unmap(&op->unmap);
> +	panthor_vma_unlink(op_ctx, unmap_vma);
> +	return 0;
> +}
> +
> +static const struct drm_gpuva_fn_ops panthor_gpuva_ops = {
> +	.sm_step_map = panthor_gpuva_sm_step_map,
> +	.sm_step_remap = panthor_gpuva_sm_step_remap,
> +	.sm_step_unmap = panthor_gpuva_sm_step_unmap,
> +};
> +
> +/**
> + * panthor_vm_resv() - Get the dma_resv object attached to a VM.
> + * @vm: VM to get the dma_resv of.
> + *
> + * Return: A dma_resv object.
> + */
> +struct dma_resv *panthor_vm_resv(struct panthor_vm *vm)
> +{
> +	return vm->dummy_gem.resv;
> +}
> +
> +static int
> +panthor_vm_exec_op(struct panthor_vm *vm, struct panthor_vm_op_ctx *op,
> +		   bool flag_vm_unusable_on_failure)
> +{
> +	int ret;
> +
> +	mutex_lock(&vm->op_lock);
> +	vm->op_ctx = op;
> +	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
> +		if (vm->unusable) {
> +			ret = -EINVAL;
> +			break;
> +		}
> +
> +		ret = drm_gpuva_sm_map(&vm->va_mgr, vm, op->va.addr, op->va.range,
> +				       op->map.gem.obj, op->map.gem.offset);
> +		break;
> +
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
> +		ret = drm_gpuva_sm_unmap(&vm->va_mgr, vm, op->va.addr, op->va.range);
> +		break;
> +
> +	default:
> +		ret = -EINVAL;
> +		break;
> +	}
> +
> +	if (ret && flag_vm_unusable_on_failure)
> +		vm->unusable = true;
> +
> +	vm->op_ctx = NULL;
> +	mutex_unlock(&vm->op_lock);
> +
> +	return ret;
> +}
> +
> +static struct dma_fence *
> +panthor_vm_bind_run_job(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +	bool cookie;
> +	int ret;
> +
> +	/* Not only we report an error whose result is propagated to the
> +	 * drm_sched finished fence, but we also flag the VM as unusable, because
> +	 * a failure in the async VM_BIND results in an inconsistent state. VM needs
> +	 * to be destroyed and recreated.
> +	 */
> +	cookie = dma_fence_begin_signalling();
> +	ret = panthor_vm_exec_op(job->vm, &job->ctx, true);
> +	dma_fence_end_signalling(cookie);
> +
> +	return ret ? ERR_PTR(ret) : NULL;
> +}
> +
> +static void panthor_vm_bind_job_release(struct kref *kref)
> +{
> +	struct panthor_vm_bind_job *job = container_of(kref, struct panthor_vm_bind_job, refcount);
> +
> +	if (job->base.s_fence)
> +		drm_sched_job_cleanup(&job->base);
> +
> +	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
> +	panthor_vm_put(job->vm);
> +	kfree(job);
> +}
> +
> +/**
> + * panthor_vm_bind_job_put() - Release a VM_BIND job reference
> + * @sched_job: Job to release the reference on.
> + */
> +void panthor_vm_bind_job_put(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job =
> +		container_of(sched_job, struct panthor_vm_bind_job, base);
> +
> +	if (sched_job)
> +		kref_put(&job->refcount, panthor_vm_bind_job_release);
> +}
> +
> +static void
> +panthor_vm_bind_free_job(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job =
> +		container_of(sched_job, struct panthor_vm_bind_job, base);
> +
> +	drm_sched_job_cleanup(sched_job);
> +
> +	/* Do the heavy cleanups asynchronously, so we're out of the
> +	 * dma-signaling path and can acquire dma-resv locks safely.
> +	 */
> +	queue_work(panthor_cleanup_wq, &job->cleanup_op_ctx_work);
> +}
> +
> +static enum drm_gpu_sched_stat
> +panthor_vm_bind_timedout_job(struct drm_sched_job *sched_job)
> +{
> +	WARN(1, "VM_BIND ops are synchronous for now, there should be no timeout!");
> +	return DRM_GPU_SCHED_STAT_NOMINAL;
> +}
> +
> +static const struct drm_sched_backend_ops panthor_vm_bind_ops = {
> +	.run_job = panthor_vm_bind_run_job,
> +	.free_job = panthor_vm_bind_free_job,
> +	.timedout_job = panthor_vm_bind_timedout_job,
> +};
> +
> +/**
> + * panthor_vm_create() - Create a VM
> + * @ptdev: Device.
> + * @for_mcu: True if this is the FW MCU VM.
> + * @auto_va_start: Start of the auto-VA range.
> + * @auto_va_range: Size of the auto-VA range.
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct panthor_vm *
> +panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
> +		  u64 auto_va_start, u64 auto_va_range)
> +{
> +	u32 va_bits = GPU_MMU_FEATURES_VA_BITS(ptdev->gpu_info.mmu_features);
> +	u32 pa_bits = GPU_MMU_FEATURES_PA_BITS(ptdev->gpu_info.mmu_features);
> +	struct drm_gpu_scheduler *sched;
> +	struct io_pgtable_cfg pgtbl_cfg;
> +	u64 mair, min_va, va_range;
> +	struct panthor_vm *vm;
> +	int ret;
> +
> +	vm = kzalloc(sizeof(*vm), GFP_KERNEL);
> +	if (!vm)
> +		return ERR_PTR(-ENOMEM);
> +
> +	mutex_init(&vm->heaps.lock);
> +	kref_init(&vm->refcount);
> +	drm_gem_private_object_init(&ptdev->base, &vm->dummy_gem, 0);
> +	vm->for_mcu = for_mcu;
> +	vm->ptdev = ptdev;
> +	INIT_LIST_HEAD(&vm->shared_bos);
> +	mutex_init(&vm->op_lock);
> +
> +	if (for_mcu) {
> +		/* CSF MCU is a cortex M7, and can only address 4G */
> +		min_va = 0;
> +		va_range = SZ_4G;
> +	} else {
> +		min_va = 0;
> +		va_range = (1ull << va_bits);
> +
> +		/* If the auto_va_range is zero, we reserve half of the VA
> +		 * space for kernel stuff.
> +		 */
> +		if (!auto_va_range) {
> +			auto_va_range = va_range / 2;
> +			auto_va_start = va_range - auto_va_range;
> +		}
> +	}
> +
> +	mutex_init(&vm->mm_lock);
> +	drm_mm_init(&vm->mm, auto_va_start, auto_va_range);
> +
> +	/* We intentionally leave the reserved range to zero, because we want kernel VMAs
> +	 * to be handled the same way user VMAs are.
> +	 */
> +	drm_gpuva_manager_init(&vm->va_mgr,
> +			       for_mcu ? "panthor-MCU-VA-manager" : "panthor-GPU-VA-manager",
> +			       min_va, va_range, 0, 0,
> +			       &panthor_gpuva_ops);
> +	INIT_LIST_HEAD(&vm->node);
> +	INIT_LIST_HEAD(&vm->as.lru_node);
> +	vm->as.id = -1;
> +
> +	pgtbl_cfg = (struct io_pgtable_cfg) {
> +		.pgsize_bitmap	= SZ_4K | SZ_2M,
> +		.ias		= va_bits,
> +		.oas		= pa_bits,
> +		.coherent_walk	= ptdev->coherent,
> +		.tlb		= &mmu_tlb_ops,
> +		.iommu_dev	= ptdev->base.dev,
> +		.alloc		= alloc_pt,
> +		.free		= free_pt,
> +	};
> +
> +	vm->pgtbl_ops = alloc_io_pgtable_ops(ARM_64_LPAE_S1, &pgtbl_cfg, vm);
> +	if (!vm->pgtbl_ops) {
> +		ret = -EINVAL;
> +		goto err_gpuva_destroy;
> +	}
> +
> +	/* Bind operations are synchronous for now, no timeout needed. */
> +	ret = drm_sched_init(&vm->sched, &panthor_vm_bind_ops, ptdev->mmu->vm.wq, 1, 0,
> +			     MAX_SCHEDULE_TIMEOUT, NULL, NULL,
> +			     "panthor-vm-bind", DRM_SCHED_POLICY_SINGLE_ENTITY,
> +			     ptdev->base.dev);
> +	if (ret)
> +		goto err_free_io_pgtable;
> +
> +	sched = &vm->sched;
> +	ret = drm_sched_entity_init(&vm->entity, DRM_SCHED_PRIORITY_NORMAL,
> +				    &sched, 1, NULL);
> +	if (ret)
> +		goto err_sched_fini;
> +
> +	mair = io_pgtable_ops_to_pgtable(vm->pgtbl_ops)->cfg.arm_lpae_s1_cfg.mair;
> +	vm->memattr = mair_to_memattr(mair);
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_add_tail(&vm->node, &ptdev->mmu->vm.list);
> +
> +	/* If a reset is in progress, stop the scheduler. */
> +	if (ptdev->mmu->vm.reset_in_progress)
> +		panthor_vm_stop(vm);
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +
> +	return vm;
> +
> +err_sched_fini:
> +	drm_sched_fini(&vm->sched);
> +
> +err_free_io_pgtable:
> +	free_io_pgtable_ops(vm->pgtbl_ops);
> +
> +err_gpuva_destroy:
> +	drm_mm_takedown(&vm->mm);
> +	drm_gpuva_manager_destroy(&vm->va_mgr);
> +	drm_gem_private_object_fini(&vm->dummy_gem);
> +	kfree(vm);
> +
> +	return ERR_PTR(ret);
> +}
> +
> +static int
> +panthor_vm_bind_prepare_op_ctx(struct drm_file *file,
> +			       struct panthor_vm *vm,
> +			       const struct drm_panthor_vm_bind_op *op,
> +			       struct panthor_vm_op_ctx *op_ctx)
> +{
> +	struct drm_gem_object *gem;
> +	int ret;
> +
> +	/* Aligned on page size. */
> +	if ((op->va | op->size) & ~PAGE_MASK)
> +		return -EINVAL;
> +
> +	switch (op->flags & DRM_PANTHOR_VM_BIND_OP_TYPE_MASK) {
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_MAP:
> +		gem = drm_gem_object_lookup(file, op->bo_handle);
> +		ret = panthor_vm_prepare_map_op_ctx(op_ctx, vm,
> +						    gem ? to_panthor_bo(gem) : NULL,
> +						    op->bo_offset,
> +						    op->size,
> +						    op->va,
> +						    op->flags);
> +		drm_gem_object_put(gem);
> +		return ret;
> +
> +	case DRM_PANTHOR_VM_BIND_OP_TYPE_UNMAP:
> +		return panthor_vm_prepare_unmap_op_ctx(op_ctx, vm, op->va, op->size);
> +
> +	default:
> +		return -EINVAL;
> +	}
> +}
> +
> +static void panthor_vm_bind_job_cleanup_op_ctx_work(struct work_struct *work)
> +{
> +	struct panthor_vm_bind_job *job =
> +		container_of(work, struct panthor_vm_bind_job, cleanup_op_ctx_work);
> +
> +	panthor_vm_cleanup_op_ctx(&job->ctx, job->vm);
> +	panthor_vm_bind_job_put(&job->base);
> +}
> +
> +/**
> + * panthor_vm_bind_job_create() - Create a VM_BIND job
> + * @file: File.
> + * @vm: VM targeted by the VM_BIND job.
> + * @op: VM operation data.
> + *
> + * Return: A valid pointer on success, an ERR_PTR() otherwise.
> + */
> +struct drm_sched_job *
> +panthor_vm_bind_job_create(struct drm_file *file,
> +			   struct panthor_vm *vm,
> +			   const struct drm_panthor_vm_bind_op *op)
> +{
> +	struct panthor_vm_bind_job *job;
> +	int ret;
> +
> +	if (!vm)
> +		return ERR_PTR(-EINVAL);
> +
> +	if (vm->destroyed || vm->unusable)
> +		return ERR_PTR(-EINVAL);
> +
> +	job = kzalloc(sizeof(*job), GFP_KERNEL);
> +	if (!job)
> +		return ERR_PTR(-ENOMEM);
> +
> +	INIT_WORK(&job->cleanup_op_ctx_work, panthor_vm_bind_job_cleanup_op_ctx_work);
> +	kref_init(&job->refcount);
> +	job->vm = panthor_vm_get(vm);
> +
> +	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &job->ctx);
> +	if (ret)
> +		goto err_put_job;
> +
> +	ret = drm_sched_job_init(&job->base, &vm->entity, vm);
> +	if (ret)
> +		goto err_put_job;
> +
> +	return &job->base;
> +
> +err_put_job:
> +	panthor_vm_bind_job_put(&job->base);
> +	return ERR_PTR(ret);
> +}
> +
> +/**
> + * panthor_vm_bind_job_prepare_resvs() - Prepare VM_BIND job dma_resvs
> + * @exec: The locking/preparation context.
> + * @sched_job: The job to prepare resvs on.
> + *
> + * Locks and prepare the VM resv.
> + *
> + * If this is a map operation, locks and prepares the GEM resv.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
> +				      struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +	int ret;
> +
> +	/* Acquire the VM lock an reserve a slot for this VM bind job. */
> +	ret = drm_exec_prepare_obj(exec, &job->vm->dummy_gem, 1);
> +	if (ret)
> +		return ret;
> +
> +	if (job->ctx.map.gem.obj) {
> +		/* Lock/prepare the GEM being mapped. */
> +		ret = drm_exec_prepare_obj(exec, job->ctx.map.gem.obj, 1);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_bind_job_add_resvs_deps() - Add implicit deps to the VM_BIND job
> + * @sched_job: Job to add implicit deps on.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_bind_job_add_resvs_deps(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +	int ret;
> +
> +	/* We use explicit fencing, so no need to wait for anything else but
> +	 * DMA_RESV_USAGE_KERNEL on the BO being mapped or VM. If there are extra
> +	 * dependencies, they should be passed to the VM_BIND ioctl.
> +	 */
> +	ret = drm_sched_job_add_resv_dependencies(sched_job,
> +						  job->vm->dummy_gem.resv,
> +						  DMA_RESV_USAGE_KERNEL);
> +	if (ret)
> +		return ret;
> +
> +	if (job->ctx.map.gem.obj) {
> +		ret = drm_sched_job_add_resv_dependencies(sched_job,
> +							  job->ctx.map.gem.obj->resv,
> +							  DMA_RESV_USAGE_KERNEL);
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_bind_job_update_resvs() - Update the resv objects touched by a job
> + * @sched_job: Job to update the resvs on.
> + */
> +void panthor_vm_bind_job_update_resvs(struct drm_sched_job *sched_job)
> +{
> +	struct panthor_vm_bind_job *job = container_of(sched_job, struct panthor_vm_bind_job, base);
> +
> +	/* Explicit sync => we just register our job finished fence as bookkeep. */
> +	dma_resv_add_fence(job->vm->dummy_gem.resv,
> +			   &sched_job->s_fence->finished,
> +			   DMA_RESV_USAGE_BOOKKEEP);
> +
> +	if (job->ctx.map.gem.obj) {
> +		dma_resv_add_fence(job->ctx.map.gem.obj->resv,
> +				   &sched_job->s_fence->finished,
> +				   DMA_RESV_USAGE_BOOKKEEP);
> +	}
> +}
> +
> +/**
> + * panthor_vm_bind_exec_sync_op() - Execute a VM_BIND operation synchronously.
> + * @file: File.
> + * @vm: VM targeted by the VM operation.
> + * @op: Data describing the VM operation.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_bind_exec_sync_op(struct drm_file *file,
> +				 struct panthor_vm *vm,
> +				 struct drm_panthor_vm_bind_op *op)
> +{
> +	struct panthor_vm_op_ctx op_ctx;
> +	int ret;
> +
> +	/* No sync objects allowed on synchronous operations. */
> +	if (op->syncs.count)
> +		return -EINVAL;
> +
> +	if (!op->size)
> +		return 0;
> +
> +	ret = panthor_vm_bind_prepare_op_ctx(file, vm, op, &op_ctx);
> +	if (ret)
> +		return ret;
> +
> +	ret = panthor_vm_exec_op(vm, &op_ctx, false);
> +	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
> +
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_map_bo_range() - Map a GEM object range to a VM
> + * @vm: VM to map the GEM to.
> + * @bo: GEM object to map.
> + * @offset: Offset in the GEM object.
> + * @size: Size to map.
> + * @va: Virtual address to map the object to.
> + * @flags: Combination of drm_panthor_vm_bind_op_flags flags.
> + * Only map-related flags are valid.
> + *
> + * Internal use only. For userspace requests, use
> + * panthor_vm_bind_exec_sync_op() instead.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
> +			    u64 offset, size_t size, u64 va, u32 flags)
> +{
> +	struct panthor_vm_op_ctx op_ctx;
> +	int ret;
> +
> +	ret = panthor_vm_prepare_map_op_ctx(&op_ctx, vm, bo, offset, size, va, flags);
> +	if (ret)
> +		return ret;
> +
> +	ret = panthor_vm_exec_op(vm, &op_ctx, false);
> +	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
> +
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_unmap_range() - Unmap a portion of the VA space
> + * @vm: VM to unmap the region from.
> + * @va: Virtual address to unmap. Must be 4k aligned.
> + * @size: Size of the region to unmap. Must be 4k aligned.
> + *
> + * Internal use only. For userspace requests, use
> + * panthor_vm_bind_exec_sync_op() instead.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, size_t size)
> +{
> +	struct panthor_vm_op_ctx op_ctx;
> +	int ret;
> +
> +	ret = panthor_vm_prepare_unmap_op_ctx(&op_ctx, vm, va, size);
> +	if (ret)
> +		return ret;
> +
> +	ret = panthor_vm_exec_op(vm, &op_ctx, false);
> +	panthor_vm_cleanup_op_ctx(&op_ctx, vm);
> +
> +	return ret;
> +}
> +
> +/**
> + * panthor_vm_prepare_mapped_bos_resvs() - Prepare resvs on VM BOs.
> + * @exec: Locking/preparation context.
> + * @vm: VM targeted by the GPU job.
> + *
> + * GPU jobs assume all BOs bound to the VM at the time the job is submitted
> + * are available when the job is executed. In order to guarantee that, we
> + * need to reserve a slot on all BOs mapped to a VM and update this slot with
> + * the job fence after its submission.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec, struct panthor_vm *vm)
> +{
> +	struct panthor_vma *vma;
> +	int ret;
> +
> +	/* Acquire the VM lock an reserve a slot for this GPU job. */
> +	ret = drm_exec_prepare_obj(exec, &vm->dummy_gem, 1);
> +	if (ret)
> +		return ret;
> +
> +	list_for_each_entry(vma, &vm->shared_bos, node) {
> +		ret = drm_exec_prepare_obj(exec, vma->base.gem.obj, 1);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_add_bos_resvs_deps_to_job() - Add implicit VM deps to a GPU job
> + * @vm: VM targeted by the GPU job.
> + * @job: GPU job.
> + *
> + * We just take care of kernel access. Other accesses should be passed as
> + * explicit dependencies to the job.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
> +					 struct drm_sched_job *job)
> +{
> +	struct panthor_vma *vma;
> +	int ret;
> +
> +	/* We use explicit fencing, so no need to wait for anything else but
> +	 * DMA_RESV_USAGE_KERNEL on the BO being mapped or VM. If there are extra
> +	 * dependencies, they should be passed to the VM_BIND ioctl.
> +	 */
> +	ret = drm_sched_job_add_resv_dependencies(job,
> +						  vm->dummy_gem.resv,
> +						  DMA_RESV_USAGE_KERNEL);
> +	if (ret)
> +		return ret;
> +
> +	list_for_each_entry(vma, &vm->shared_bos, node) {
> +		ret = drm_sched_job_add_resv_dependencies(job,
> +							  vma->base.gem.obj->resv,
> +							  DMA_RESV_USAGE_KERNEL);
> +		if (ret)
> +			return ret;
> +	}
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_vm_add_job_fence_to_bos_resvs() - Add GPU job fence to GEM resvs
> + * @vm: VM targeted by the GPU job.
> + * @job: GPU job.
> + *
> + * Update the GEM resvs after a job has been submitted. All GEMs currently
> + * bound to the VMs get the job fence added to their resv as bookkeep. If
> + * another type of implicit dependency is needed, it should be updated
> + * with %DMA_BUF_IOCTL_IMPORT_SYNC_FILE after the
> + * %DRM_IOCTL_PANTHOR_GROUP_SUBMIT ioctl has returned.
> + */
> +void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
> +					   struct drm_sched_job *job)
> +{
> +	struct panthor_vma *vma;
> +
> +	/* Explicit sync => we just register our job finished fence as bookkeep. */
> +	dma_resv_add_fence(vm->dummy_gem.resv,
> +			   &job->s_fence->finished,
> +			   DMA_RESV_USAGE_BOOKKEEP);
> +
> +	list_for_each_entry(vma, &vm->shared_bos, node) {
> +		dma_resv_add_fence(vma->base.gem.obj->resv,
> +				   &job->s_fence->finished,
> +				   DMA_RESV_USAGE_BOOKKEEP);
> +	}
> +}
> +
> +/**
> + * panthor_mmu_unplug() - Unplug the MMU logic
> + * @ptdev: Device.
> + *
> + * No access to the MMU regs should be done after this function is called.
> + * We suspend the IRQ and disable all VMs to guarantee that.
> + */
> +void panthor_mmu_unplug(struct panthor_device *ptdev)
> +{
> +	if (ptdev->mmu->irq.irq > 0)

In what situation is this not true? AFAICT the driver probe will fail if
the IRQ can't be obtained.

Steve

> +		panthor_mmu_irq_suspend(&ptdev->mmu->irq);
> +
> +	mutex_lock(&ptdev->mmu->as.slots_lock);
> +	for (u32 i = 0; i < ARRAY_SIZE(ptdev->mmu->as.slots); i++) {
> +		struct panthor_vm *vm = ptdev->mmu->as.slots[i].vm;
> +
> +		if (vm) {
> +			drm_WARN_ON(&ptdev->base, panthor_mmu_as_disable(ptdev, i));
> +			vm->as.id = -1;
> +			list_del_init(&vm->as.lru_node);
> +			clear_bit(i, &ptdev->mmu->as.alloc_mask);
> +			ptdev->mmu->as.slots[i].vm = NULL;
> +		}
> +	}
> +	mutex_unlock(&ptdev->mmu->as.slots_lock);
> +}
> +
> +static void panthor_mmu_release_wq(struct drm_device *ddev, void *res)
> +{
> +	destroy_workqueue(res);
> +}
> +
> +/**
> + * panthor_mmu_init() - Initialize the MMU logic.
> + * @ptdev: Device.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_mmu_init(struct panthor_device *ptdev)
> +{
> +	struct panthor_mmu *mmu;
> +	int ret, irq;
> +
> +	mmu = drmm_kzalloc(&ptdev->base, sizeof(*mmu), GFP_KERNEL);
> +	if (!mmu)
> +		return -ENOMEM;
> +
> +	INIT_LIST_HEAD(&mmu->as.lru_list);
> +
> +	for (u32 i = 0; i < ARRAY_SIZE(mmu->as.slots); i++)
> +		spin_lock_init(&mmu->as.slots[i].lock);
> +
> +	drmm_mutex_init(&ptdev->base, &mmu->as.slots_lock);
> +	INIT_LIST_HEAD(&mmu->vm.list);
> +	drmm_mutex_init(&ptdev->base, &mmu->vm.lock);
> +
> +	ptdev->mmu = mmu;
> +
> +	irq = platform_get_irq_byname(to_platform_device(ptdev->base.dev), "mmu");
> +	if (irq <= 0)
> +		return -ENODEV;
> +
> +	ret = panthor_request_mmu_irq(ptdev, &mmu->irq, irq,
> +				      panthor_mmu_fault_mask(ptdev, ~0));
> +	if (ret)
> +		return ret;
> +
> +	mmu->vm.wq = alloc_workqueue("panthor-vm-bind", WQ_UNBOUND, 0);
> +	if (!mmu->vm.wq)
> +		return -ENOMEM;
> +
> +	return drmm_add_action_or_reset(&ptdev->base, panthor_mmu_release_wq, mmu->vm.wq);
> +}
> +
> +#ifdef CONFIG_DEBUG_FS
> +static int show_vm_gpuvas(struct panthor_vm *vm, struct seq_file *m)
> +{
> +	int ret;
> +
> +	mutex_lock(&vm->op_lock);
> +	ret = drm_debugfs_gpuva_info(m, &vm->va_mgr);
> +	mutex_unlock(&vm->op_lock);
> +
> +	return ret;
> +}
> +
> +static int show_each_vm(struct seq_file *m, void *arg)
> +{
> +	struct drm_info_node *node = (struct drm_info_node *)m->private;
> +	struct drm_device *ddev = node->minor->dev;
> +	struct panthor_device *ptdev = container_of(ddev, struct panthor_device, base);
> +	int (*show)(struct panthor_vm *, struct seq_file *) = node->info_ent->data;
> +	struct panthor_vm *vm;
> +	int ret = 0;
> +
> +	mutex_lock(&ptdev->mmu->vm.lock);
> +	list_for_each_entry(vm, &ptdev->mmu->vm.list, node) {
> +		ret = show(vm, m);
> +		if (ret < 0)
> +			break;
> +
> +		seq_puts(m, "\n");
> +	}
> +	mutex_unlock(&ptdev->mmu->vm.lock);
> +
> +	return ret;
> +}
> +
> +static struct drm_info_list panthor_mmu_debugfs_list[] = {
> +	DRM_DEBUGFS_GPUVA_INFO(show_each_vm, show_vm_gpuvas),
> +};
> +
> +/**
> + * panthor_mmu_debugfs_init() - Initialize MMU debugfs entries
> + * @minor: Minor.
> + */
> +void panthor_mmu_debugfs_init(struct drm_minor *minor)
> +{
> +	drm_debugfs_create_files(panthor_mmu_debugfs_list,
> +				 ARRAY_SIZE(panthor_mmu_debugfs_list),
> +				 minor->debugfs_root, minor);
> +}
> +#endif /* CONFIG_DEBUG_FS */
> +
> +/**
> + * panthor_mmu_pt_cache_init() - Initialize the page table cache.
> + *
> + * Return: 0 on success, a negative error code otherwise.
> + */
> +int panthor_mmu_pt_cache_init(void)
> +{
> +	pt_cache = kmem_cache_create("panthor-mmu-pt", SZ_4K, SZ_4K, 0, NULL);
> +	if (!pt_cache)
> +		return -ENOMEM;
> +
> +	return 0;
> +}
> +
> +/**
> + * panthor_mmu_pt_cache_fini() - Destroy the page table cache.
> + */
> +void panthor_mmu_pt_cache_fini(void)
> +{
> +	kmem_cache_destroy(pt_cache);
> +}
> diff --git a/drivers/gpu/drm/panthor/panthor_mmu.h b/drivers/gpu/drm/panthor/panthor_mmu.h
> new file mode 100644
> index 000000000000..d94925ccdc8c
> --- /dev/null
> +++ b/drivers/gpu/drm/panthor/panthor_mmu.h
> @@ -0,0 +1,81 @@
> +/* SPDX-License-Identifier: GPL-2.0 or MIT */
> +/* Copyright 2019 Linaro, Ltd, Rob Herring <robh@xxxxxxxxxx> */
> +/* Copyright 2023 Collabora ltd. */
> +
> +#ifndef __PANTHOR_MMU_H__
> +#define __PANTHOR_MMU_H__
> +
> +struct drm_exec;
> +struct drm_sched_job;
> +struct panthor_gem_object;
> +struct panthor_heap_pool;
> +struct panthor_vm;
> +struct panthor_vma;
> +struct panthor_mmu;
> +
> +int panthor_mmu_init(struct panthor_device *ptdev);
> +void panthor_mmu_unplug(struct panthor_device *ptdev);
> +void panthor_mmu_pre_reset(struct panthor_device *ptdev);
> +void panthor_mmu_post_reset(struct panthor_device *ptdev);
> +void panthor_mmu_suspend(struct panthor_device *ptdev);
> +void panthor_mmu_resume(struct panthor_device *ptdev);
> +
> +int panthor_vm_map_bo_range(struct panthor_vm *vm, struct panthor_gem_object *bo,
> +			    u64 offset, size_t size, u64 va, u32 flags);
> +int panthor_vm_unmap_range(struct panthor_vm *vm, u64 va, size_t size);
> +struct panthor_gem_object *
> +panthor_vm_get_bo_for_va(struct panthor_vm *vm, u64 va, u64 *bo_offset);
> +
> +int panthor_vm_active(struct panthor_vm *vm);
> +void panthor_vm_idle(struct panthor_vm *vm);
> +int panthor_vm_as(struct panthor_vm *vm);
> +
> +struct panthor_heap_pool *
> +panthor_vm_get_heap_pool(struct panthor_vm *vm, bool create);
> +
> +struct panthor_vm *panthor_vm_get(struct panthor_vm *vm);
> +void panthor_vm_put(struct panthor_vm *vm);
> +struct panthor_vm *panthor_vm_create(struct panthor_device *ptdev, bool for_mcu,
> +				     u64 auto_va_start, u64 auto_va_range);
> +
> +int panthor_vm_prepare_mapped_bos_resvs(struct drm_exec *exec,
> +					struct panthor_vm *vm);
> +int panthor_vm_add_bos_resvs_deps_to_job(struct panthor_vm *vm,
> +					 struct drm_sched_job *job);
> +void panthor_vm_add_job_fence_to_bos_resvs(struct panthor_vm *vm,
> +					   struct drm_sched_job *job);
> +
> +struct dma_resv *panthor_vm_resv(struct panthor_vm *vm);
> +
> +void panthor_vm_pool_destroy(struct panthor_file *pfile);
> +int panthor_vm_pool_create(struct panthor_file *pfile);
> +int panthor_vm_pool_create_vm(struct panthor_device *ptdev, struct panthor_vm_pool *pool,
> +			      u64 kernel_va_start, u64 kernel_va_range);
> +int panthor_vm_pool_destroy_vm(struct panthor_vm_pool *pool, u32 handle);
> +struct panthor_vm *panthor_vm_pool_get_vm(struct panthor_vm_pool *pool, u32 handle);
> +
> +struct drm_mm_node *panthor_vm_alloc_va(struct panthor_vm *vm, size_t size);
> +void panthor_vm_free_va(struct panthor_vm *vm, struct drm_mm_node *mm_node);
> +
> +int panthor_vm_bind_exec_sync_op(struct drm_file *file,
> +				 struct panthor_vm *vm,
> +				 struct drm_panthor_vm_bind_op *op);
> +
> +struct drm_sched_job *
> +panthor_vm_bind_job_create(struct drm_file *file,
> +			   struct panthor_vm *vm,
> +			   const struct drm_panthor_vm_bind_op *op);
> +void panthor_vm_bind_job_put(struct drm_sched_job *job);
> +int panthor_vm_bind_job_prepare_resvs(struct drm_exec *exec,
> +				      struct drm_sched_job *job);
> +int panthor_vm_bind_job_add_resvs_deps(struct drm_sched_job *job);
> +void panthor_vm_bind_job_update_resvs(struct drm_sched_job *job);
> +
> +int panthor_mmu_pt_cache_init(void);
> +void panthor_mmu_pt_cache_fini(void);
> +
> +#ifdef CONFIG_DEBUG_FS
> +void panthor_mmu_debugfs_init(struct drm_minor *minor);
> +#endif
> +
> +#endif




[Index of Archives]     [Linux DRI Users]     [Linux Intel Graphics]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Yosemite News]     [Linux Kernel]     [Linux SCSI]     [XFree86]     [Linux USB Devel]     [Video for Linux]     [Linux Audio Users]     [Linux Kernel]     [Linux SCSI]     [XFree86]
  Powered by Linux