Implement the job submission IOCTL with a minimum feature set.
Signed-off-by: Mikko Perttunen <mperttunen@xxxxxxxxxx>
---
v4:
* Remove all features that are not strictly necessary.
* Split into two patches.
v3:
* Remove WRITE_RELOC. Relocations are now patched implicitly
when patching is needed.
* Directly call PM runtime APIs on devices instead of using
power_on/power_off callbacks.
* Remove incorrect mutex unlock in tegra_drm_ioctl_channel_open
* Use XA_FLAGS_ALLOC1 instead of XA_FLAGS_ALLOC
* Accommodate for removal of timeout field and inlining of
syncpt_incrs array.
* Copy entire user arrays at a time instead of going through
elements one-by-one.
* Implement waiting of DMA reservations.
* Split out gather_bo implementation into a separate file.
* Fix length parameter passed to sg_init_one in gather_bo
* Cosmetic cleanup.
---
drivers/gpu/drm/tegra/Makefile | 2 +
drivers/gpu/drm/tegra/drm.c | 2 +
drivers/gpu/drm/tegra/uapi/gather_bo.c | 86 +++++
drivers/gpu/drm/tegra/uapi/gather_bo.h | 22 ++
drivers/gpu/drm/tegra/uapi/submit.c | 423 +++++++++++++++++++++++++
drivers/gpu/drm/tegra/uapi/submit.h | 17 +
6 files changed, 552 insertions(+)
create mode 100644 drivers/gpu/drm/tegra/uapi/gather_bo.c
create mode 100644 drivers/gpu/drm/tegra/uapi/gather_bo.h
create mode 100644 drivers/gpu/drm/tegra/uapi/submit.c
create mode 100644 drivers/gpu/drm/tegra/uapi/submit.h
diff --git a/drivers/gpu/drm/tegra/Makefile b/drivers/gpu/drm/tegra/Makefile
index 0abdb21b38b9..059322e88943 100644
--- a/drivers/gpu/drm/tegra/Makefile
+++ b/drivers/gpu/drm/tegra/Makefile
@@ -4,6 +4,8 @@ ccflags-$(CONFIG_DRM_TEGRA_DEBUG) += -DDEBUG
tegra-drm-y := \
drm.o \
uapi/uapi.o \
+ uapi/submit.o \
+ uapi/gather_bo.o \
gem.o \
fb.o \
dp.o \
diff --git a/drivers/gpu/drm/tegra/drm.c b/drivers/gpu/drm/tegra/drm.c
index 79416b6b6715..3077bd594560 100644
--- a/drivers/gpu/drm/tegra/drm.c
+++ b/drivers/gpu/drm/tegra/drm.c
@@ -740,6 +740,8 @@ static const struct drm_ioctl_desc tegra_drm_ioctls[] = {
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_UNMAP, tegra_drm_ioctl_channel_unmap,
DRM_RENDER_ALLOW),
+ DRM_IOCTL_DEF_DRV(TEGRA_CHANNEL_SUBMIT, tegra_drm_ioctl_channel_submit,
+ DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(TEGRA_GEM_CREATE, tegra_drm_ioctl_gem_create,
DRM_RENDER_ALLOW),
DRM_IOCTL_DEF_DRV(TEGRA_GEM_MMAP, tegra_drm_ioctl_gem_mmap,
diff --git a/drivers/gpu/drm/tegra/uapi/gather_bo.c b/drivers/gpu/drm/tegra/uapi/gather_bo.c
new file mode 100644
index 000000000000..b487a0d44648
--- /dev/null
+++ b/drivers/gpu/drm/tegra/uapi/gather_bo.c
@@ -0,0 +1,86 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2020 NVIDIA Corporation */
+
+#include <linux/scatterlist.h>
+#include <linux/slab.h>
+
+#include "gather_bo.h"
+
+static struct host1x_bo *gather_bo_get(struct host1x_bo *host_bo)
+{
+ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
+
+ kref_get(&bo->ref);
+
+ return host_bo;
+}
+
+static void gather_bo_release(struct kref *ref)
+{
+ struct gather_bo *bo = container_of(ref, struct gather_bo, ref);
+
+ kfree(bo->gather_data);
+ kfree(bo);
+}
+
+void gather_bo_put(struct host1x_bo *host_bo)
+{
+ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
+
+ kref_put(&bo->ref, gather_bo_release);
+}
+
+static struct sg_table *
+gather_bo_pin(struct device *dev, struct host1x_bo *host_bo, dma_addr_t *phys)
+{
+ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
+ struct sg_table *sgt;
+ int err;
+
+ if (phys) {
+ *phys = virt_to_phys(bo->gather_data);
+ return NULL;
+ }
+
+ sgt = kzalloc(sizeof(*sgt), GFP_KERNEL);
+ if (!sgt)
+ return ERR_PTR(-ENOMEM);
+
+ err = sg_alloc_table(sgt, 1, GFP_KERNEL);
+ if (err) {
+ kfree(sgt);
+ return ERR_PTR(err);
+ }
+
+ sg_init_one(sgt->sgl, bo->gather_data, bo->gather_data_words*4);
+
+ return sgt;
+}
+
+static void gather_bo_unpin(struct device *dev, struct sg_table *sgt)
+{
+ if (sgt) {
+ sg_free_table(sgt);
+ kfree(sgt);
+ }
+}
+
+static void *gather_bo_mmap(struct host1x_bo *host_bo)
+{
+ struct gather_bo *bo = container_of(host_bo, struct gather_bo, base);
+
+ return bo->gather_data;
+}
+
+static void gather_bo_munmap(struct host1x_bo *host_bo, void *addr)
+{
+}
+
+const struct host1x_bo_ops gather_bo_ops = {
+ .get = gather_bo_get,
+ .put = gather_bo_put,
+ .pin = gather_bo_pin,
+ .unpin = gather_bo_unpin,
+ .mmap = gather_bo_mmap,
+ .munmap = gather_bo_munmap,
+};
diff --git a/drivers/gpu/drm/tegra/uapi/gather_bo.h b/drivers/gpu/drm/tegra/uapi/gather_bo.h
new file mode 100644
index 000000000000..6b4c9d83ac91
--- /dev/null
+++ b/drivers/gpu/drm/tegra/uapi/gather_bo.h
@@ -0,0 +1,22 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2020 NVIDIA Corporation */
+
+#ifndef _TEGRA_DRM_SUBMIT_GATHER_BO_H
+#define _TEGRA_DRM_SUBMIT_GATHER_BO_H
+
+#include <linux/host1x.h>
+#include <linux/kref.h>
+
+struct gather_bo {
+ struct host1x_bo base;
+
+ struct kref ref;
+
+ u32 *gather_data;
+ size_t gather_data_words;
+};
+
+extern const struct host1x_bo_ops gather_bo_ops;
+void gather_bo_put(struct host1x_bo *host_bo);
+
+#endif
diff --git a/drivers/gpu/drm/tegra/uapi/submit.c b/drivers/gpu/drm/tegra/uapi/submit.c
new file mode 100644
index 000000000000..d625776071e8
--- /dev/null
+++ b/drivers/gpu/drm/tegra/uapi/submit.c
@@ -0,0 +1,423 @@
+// SPDX-License-Identifier: GPL-2.0-only
+/* Copyright (c) 2020 NVIDIA Corporation */
+
+#include <linux/dma-fence-array.h>
+#include <linux/file.h>
+#include <linux/host1x.h>
+#include <linux/iommu.h>
+#include <linux/kref.h>
+#include <linux/list.h>
+#include <linux/nospec.h>
+#include <linux/pm_runtime.h>
+#include <linux/sync_file.h>
+
+#include <drm/drm_drv.h>
+#include <drm/drm_file.h>
+
+#include "../uapi.h"
+#include "../drm.h"
+#include "../gem.h"
+
+#include "gather_bo.h"
+#include "submit.h"
+
+static struct tegra_drm_mapping *
+tegra_drm_mapping_get(struct tegra_drm_channel_ctx *ctx, u32 id)
+{
+ struct tegra_drm_mapping *mapping;
+
+ xa_lock(&ctx->mappings);
+ mapping = xa_load(&ctx->mappings, id);
+ if (mapping)
+ kref_get(&mapping->ref);
+ xa_unlock(&ctx->mappings);
+
+ return mapping;
+}
+
+static void *alloc_copy_user_array(void __user *from, size_t count, size_t size)
+{
+ unsigned long copy_err;
+ size_t copy_len;
+ void *data;
+
+ if (check_mul_overflow(count, size, ©_len))
+ return ERR_PTR(-EINVAL);
+
+ data = kvmalloc(copy_len, GFP_KERNEL);
+ if (!data)
+ return ERR_PTR(-ENOMEM);
+
+ copy_err = copy_from_user(data, from, copy_len);
+ if (copy_err) {
+ kvfree(data);
+ return ERR_PTR(-EFAULT);
+ }
+
+ return data;
+}
+
+static int submit_copy_gather_data(struct drm_device *drm,
+ struct gather_bo **pbo,
+ struct drm_tegra_channel_submit *args)
+{
+ unsigned long copy_err;
+ struct gather_bo *bo;
+ size_t copy_len;
+
+ if (args->gather_data_words == 0) {
+ drm_info(drm, "gather_data_words can't be 0");
+ return -EINVAL;
+ }
+
+ if (check_mul_overflow((size_t)args->gather_data_words, (size_t)4, ©_len))
+ return -EINVAL;
+
+ bo = kzalloc(sizeof(*bo), GFP_KERNEL);
+ if (!bo)
+ return -ENOMEM;
+
+ kref_init(&bo->ref);
+ host1x_bo_init(&bo->base, &gather_bo_ops);
+
+ bo->gather_data = kmalloc(copy_len, GFP_KERNEL | __GFP_NOWARN);
+ if (!bo->gather_data) {
+ kfree(bo);
+ return -ENOMEM;
+ }
+
+ copy_err = copy_from_user(bo->gather_data,
+ u64_to_user_ptr(args->gather_data_ptr),
+ copy_len);
+ if (copy_err) {
+ kfree(bo->gather_data);
+ kfree(bo);
+ return -EFAULT;
+ }
+
+ bo->gather_data_words = args->gather_data_words;
+
+ *pbo = bo;
+
+ return 0;
+}
+
+static int submit_write_reloc(struct gather_bo *bo,
+ struct drm_tegra_submit_buf *buf,
+ struct tegra_drm_mapping *mapping)
+{
+ /* TODO check that target_offset is within bounds */
+ dma_addr_t iova = mapping->iova + buf->reloc.target_offset;
+ u32 written_ptr = (u32)(iova >> buf->reloc.shift);
+
+ if (buf->flags & DRM_TEGRA_SUBMIT_BUF_RELOC_BLOCKLINEAR)
+ written_ptr |= BIT(39);
+
+ if (buf->reloc.gather_offset_words >= bo->gather_data_words)
+ return -EINVAL;
+
+ buf->reloc.gather_offset_words = array_index_nospec(
+ buf->reloc.gather_offset_words, bo->gather_data_words);
+
+ bo->gather_data[buf->reloc.gather_offset_words] = written_ptr;
+
+ return 0;
+}
+
+static int submit_process_bufs(struct drm_device *drm, struct gather_bo *bo,
+ struct tegra_drm_submit_data *job_data,
+ struct tegra_drm_channel_ctx *ctx,
+ struct drm_tegra_channel_submit *args)
+{
+ struct tegra_drm_used_mapping *mappings;
+ struct drm_tegra_submit_buf *bufs;
+ int err;
+ u32 i;
+
+ bufs = alloc_copy_user_array(u64_to_user_ptr(args->bufs_ptr),
+ args->num_bufs, sizeof(*bufs));
+ if (IS_ERR(bufs))
+ return PTR_ERR(bufs);
+
+ mappings = kcalloc(args->num_bufs, sizeof(*mappings), GFP_KERNEL);
+ if (!mappings) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ for (i = 0; i < args->num_bufs; i++) {
+ struct drm_tegra_submit_buf *buf = &bufs[i];
+ struct tegra_drm_mapping *mapping;
+
+ if (buf->flags & ~DRM_TEGRA_SUBMIT_BUF_RELOC_BLOCKLINEAR) {
+ err = -EINVAL;
+ goto drop_refs;
+ }
+
+ mapping = tegra_drm_mapping_get(ctx, buf->mapping_id);
+ if (!mapping) {
+ drm_info(drm, "invalid mapping_id for buf: %u",
+ buf->mapping_id);
+ err = -EINVAL;
+ goto drop_refs;
+ }
+
+ err = submit_write_reloc(bo, buf, mapping);
+ if (err) {
+ tegra_drm_mapping_put(mapping);
+ goto drop_refs;
+ }
+
+ mappings[i].mapping = mapping;
+ mappings[i].flags = buf->flags;
+ }
+
+ job_data->used_mappings = mappings;
+ job_data->num_used_mappings = i;
+
+ err = 0;
+
+ goto done;
+
+drop_refs:
+ for (;;) {
+ if (i-- == 0)
+ break;
+
+ tegra_drm_mapping_put(mappings[i].mapping);
+ }
+
+ kfree(mappings);
+ job_data->used_mappings = NULL;
+
+done:
+ kvfree(bufs);
+
+ return err;
+}
+
+static int submit_get_syncpt(struct drm_device *drm, struct host1x_job *job,
+ struct drm_tegra_channel_submit *args)
+{
+ struct host1x_syncpt *sp;
+
+ if (args->syncpt_incr.flags)
+ return -EINVAL;
+
+ /* Syncpt ref will be dropped on job release */
+ sp = host1x_syncpt_fd_get(args->syncpt_incr.syncpt_fd);
+ if (IS_ERR(sp))
+ return PTR_ERR(sp);
+
+ job->syncpt = sp;
+ job->syncpt_incrs = args->syncpt_incr.num_incrs;
+
+ return 0;
+}
+
+static int submit_job_add_gather(struct host1x_job *job,
+ struct tegra_drm_channel_ctx *ctx,
+ struct drm_tegra_submit_cmd_gather_uptr *cmd,
+ struct gather_bo *bo, u32 *offset,
+ struct tegra_drm_submit_data *job_data)
+{
+ u32 next_offset;
+
+ if (cmd->reserved[0] || cmd->reserved[1] || cmd->reserved[2])
+ return -EINVAL;
+
+ /* Check for maximum gather size */
+ if (cmd->words > 16383)
+ return -EINVAL;
+
+ if (check_add_overflow(*offset, cmd->words, &next_offset))
+ return -EINVAL;
+
+ if (next_offset > bo->gather_data_words)
+ return -EINVAL;
+
+ host1x_job_add_gather(job, &bo->base, cmd->words, *offset * 4);
+
+ *offset = next_offset;
+
+ return 0;
+}
+
+static int submit_create_job(struct drm_device *drm, struct host1x_job **pjob,
+ struct gather_bo *bo,
+ struct tegra_drm_channel_ctx *ctx,
+ struct drm_tegra_channel_submit *args,
+ struct tegra_drm_submit_data *job_data)
+{
+ struct drm_tegra_submit_cmd *cmds;
+ u32 i, gather_offset = 0;
+ struct host1x_job *job;
+ int err;
+
+ cmds = alloc_copy_user_array(u64_to_user_ptr(args->cmds_ptr),
+ args->num_cmds, sizeof(*cmds));
+ if (IS_ERR(cmds))
+ return PTR_ERR(cmds);
+
+ job = host1x_job_alloc(ctx->channel, args->num_cmds, 0);
+ if (!job) {
+ err = -ENOMEM;
+ goto done;
+ }
+
+ err = submit_get_syncpt(drm, job, args);
+ if (err < 0)
+ goto free_job;
+
+ job->client = &ctx->client->base;
+ job->class = ctx->client->base.class;
+ job->serialize = true;
+
+ for (i = 0; i < args->num_cmds; i++) {
+ struct drm_tegra_submit_cmd *cmd = &cmds[i];
+
+ if (cmd->type == DRM_TEGRA_SUBMIT_CMD_GATHER_UPTR) {
+ err = submit_job_add_gather(job, ctx, &cmd->gather_uptr,
+ bo, &gather_offset,
+ job_data);
+ if (err)
+ goto free_job;
+ } else if (cmd->type == DRM_TEGRA_SUBMIT_CMD_WAIT_SYNCPT) {
+ if (cmd->wait_syncpt.reserved[0] ||
+ cmd->wait_syncpt.reserved[1]) {
+ err = -EINVAL;
+ goto free_job;
+ }
+
+ host1x_job_add_wait(job, cmd->wait_syncpt.id,
+ cmd->wait_syncpt.threshold);
+ } else {
+ err = -EINVAL;
+ goto free_job;
+ }
+ }
+
+ if (gather_offset == 0) {
+ drm_info(drm, "Job must have at least one gather");
+ err = -EINVAL;
+ goto free_job;
+ }
+
+ *pjob = job;
+
+ err = 0;
+ goto done;
+
+free_job:
+ host1x_job_put(job);
+
+done:
+ kvfree(cmds);
+
+ return err;
+}
+
+static void release_job(struct host1x_job *job)
+{
+ struct tegra_drm_client *client =
+ container_of(job->client, struct tegra_drm_client, base);
+ struct tegra_drm_submit_data *job_data = job->user_data;
+ u32 i;
+
+ for (i = 0; i < job_data->num_used_mappings; i++)
+ tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
+
+ kfree(job_data->used_mappings);
+ kfree(job_data);
+
+ pm_runtime_put_autosuspend(client->base.dev);
+}
+
+int tegra_drm_ioctl_channel_submit(struct drm_device *drm, void *data,
+ struct drm_file *file)
+{
+ struct tegra_drm_file *fpriv = file->driver_priv;
+ struct drm_tegra_channel_submit *args = data;
+ struct tegra_drm_submit_data *job_data;
+ struct tegra_drm_channel_ctx *ctx;
+ struct host1x_job *job;
+ struct gather_bo *bo;
+ u32 i;
+ int err;
+
+ ctx = tegra_drm_channel_ctx_lock(fpriv, args->channel_ctx);
+ if (!ctx)
+ return -EINVAL;
+
+ /* Allocate gather BO and copy gather words in. */
+ err = submit_copy_gather_data(drm, &bo, args);
+ if (err)
+ goto unlock;
+
+ job_data = kzalloc(sizeof(*job_data), GFP_KERNEL);
+ if (!job_data) {
+ err = -ENOMEM;
+ goto put_bo;
+ }
+
+ /* Get data buffer mappings and do relocation patching. */
+ err = submit_process_bufs(drm, bo, job_data, ctx, args);
+ if (err)
+ goto free_job_data;
+
+ /* Allocate host1x_job and add gathers and waits to it. */
+ err = submit_create_job(drm, &job, bo, ctx, args,
+ job_data);
+ if (err)
+ goto free_job_data;
+
+ /* Map gather data for Host1x. */
+ err = host1x_job_pin(job, ctx->client->base.dev);
+ if (err)
+ goto put_job;
+
+ /* Boot engine. */
+ err = pm_runtime_get_sync(ctx->client->base.dev);
+ if (err < 0)
+ goto put_pm_runtime;
+
+ job->user_data = job_data;
+ job->release = release_job;
+ job->timeout = 10000;
+
+ /*
+ * job_data is now part of job reference counting, so don't release
+ * it from here.
+ */
+ job_data = NULL;
+
+ /* Submit job to hardware. */
+ err = host1x_job_submit(job);
+ if (err)
+ goto put_job;
+
+ /* Return postfences to userspace and add fences to DMA reservations. */
+ args->syncpt_incr.fence_value = job->syncpt_end;
+
+ goto put_job;
+
+put_pm_runtime:
+ if (!job->release)
+ pm_runtime_put(ctx->client->base.dev);
+ host1x_job_unpin(job);
+put_job:
+ host1x_job_put(job);
+free_job_data:
+ if (job_data && job_data->used_mappings) {
+ for (i = 0; i < job_data->num_used_mappings; i++)
+ tegra_drm_mapping_put(job_data->used_mappings[i].mapping);
+ kfree(job_data->used_mappings);
+ }
+ if (job_data)
+ kfree(job_data);
+put_bo:
+ gather_bo_put(&bo->base);
+unlock:
+ mutex_unlock(&fpriv->lock);
+ return err;
+}
diff --git a/drivers/gpu/drm/tegra/uapi/submit.h b/drivers/gpu/drm/tegra/uapi/submit.h
new file mode 100644
index 000000000000..0a165e9e4bda
--- /dev/null
+++ b/drivers/gpu/drm/tegra/uapi/submit.h
@@ -0,0 +1,17 @@
+/* SPDX-License-Identifier: GPL-2.0-only */
+/* Copyright (c) 2020 NVIDIA Corporation */
+
+#ifndef _TEGRA_DRM_UAPI_SUBMIT_H
+#define _TEGRA_DRM_UAPI_SUBMIT_H
+
+struct tegra_drm_used_mapping {
+ struct tegra_drm_mapping *mapping;
+ u32 flags;
+};
+
+struct tegra_drm_submit_data {
+ struct tegra_drm_used_mapping *used_mappings;
+ u32 num_used_mappings;
+};
+
+#endif
--
2.29.2
