From: Dave Jiang <dave.jiang@xxxxxxxxx>
Add counters per opened file for the char device in order to keep track how
many completion record faults occurred and how many of those faults failed
the writeback by the driver after attempt to fault in the page. An xarray
is added to associate the PASID with the struct idxd_user_context so the
counters can be managed.
Tested-by: Tony Zhu <tony.zhu@xxxxxxxxx>
Signed-off-by: Dave Jiang <dave.jiang@xxxxxxxxx>
Co-developed-by: Fenghua Yu <fenghua.yu@xxxxxxxxx>
Signed-off-by: Fenghua Yu <fenghua.yu@xxxxxxxxx>
---
drivers/dma/idxd/cdev.c | 50 +++++++++++++++++++++++++++++++++++++---
drivers/dma/idxd/idxd.h | 11 +++++++++
drivers/dma/idxd/init.c | 2 ++
drivers/dma/idxd/irq.c | 7 +++++-
drivers/dma/idxd/sysfs.c | 1 +
5 files changed, 67 insertions(+), 4 deletions(-)
diff --git a/drivers/dma/idxd/cdev.c b/drivers/dma/idxd/cdev.c
index eed0bc15951a..baec95911182 100644
--- a/drivers/dma/idxd/cdev.c
+++ b/drivers/dma/idxd/cdev.c
@@ -11,6 +11,7 @@
#include <linux/fs.h>
#include <linux/poll.h>
#include <linux/iommu.h>
+#include <linux/xarray.h>
#include <uapi/linux/idxd.h>
#include "registers.h"
#include "idxd.h"
@@ -36,6 +37,7 @@ struct idxd_user_context {
unsigned int pasid;
unsigned int flags;
struct iommu_sva *sva;
+ u64 counters[COUNTER_MAX];
};
static void idxd_cdev_dev_release(struct device *dev)
@@ -68,6 +70,36 @@ static inline struct idxd_wq *inode_wq(struct inode *inode)
return idxd_cdev->wq;
}
+void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index)
+{
+ struct idxd_user_context *ctx;
+
+ if (index >= COUNTER_MAX)
+ return;
+
+ mutex_lock(&wq->uc_lock);
+ ctx = xa_load(&wq->upasid_xa, pasid);
+ if (!ctx) {
+ mutex_unlock(&wq->uc_lock);
+ return;
+ }
+ ctx->counters[index]++;
+ mutex_unlock(&wq->uc_lock);
+}
+
+static void idxd_xa_pasid_remove(struct idxd_user_context *ctx)
+{
+ struct idxd_wq *wq = ctx->wq;
+ void *ptr;
+
+ mutex_lock(&wq->uc_lock);
+ ptr = xa_cmpxchg(&wq->upasid_xa, ctx->pasid, ctx, NULL, GFP_KERNEL);
+ if (ptr != (void *)ctx)
+ dev_warn(&wq->idxd->pdev->dev, "xarray cmpxchg failed for pasid %u\n",
+ ctx->pasid);
+ mutex_unlock(&wq->uc_lock);
+}
+
static int idxd_cdev_open(struct inode *inode, struct file *filp)
{
struct idxd_user_context *ctx;
@@ -108,20 +140,25 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
pasid = iommu_sva_get_pasid(sva);
if (pasid == IOMMU_PASID_INVALID) {
- iommu_sva_unbind_device(sva);
rc = -EINVAL;
- goto failed;
+ goto failed_get_pasid;
}
ctx->sva = sva;
ctx->pasid = pasid;
+ mutex_lock(&wq->uc_lock);
+ rc = xa_insert(&wq->upasid_xa, pasid, ctx, GFP_KERNEL);
+ mutex_unlock(&wq->uc_lock);
+ if (rc < 0)
+ dev_warn(dev, "PASID entry already exist in xarray.\n");
+
if (wq_dedicated(wq)) {
rc = idxd_wq_set_pasid(wq, pasid);
if (rc < 0) {
iommu_sva_unbind_device(sva);
dev_err(dev, "wq set pasid failed: %d\n", rc);
- goto failed;
+ goto failed_set_pasid;
}
}
}
@@ -130,6 +167,12 @@ static int idxd_cdev_open(struct inode *inode, struct file *filp)
mutex_unlock(&wq->wq_lock);
return 0;
+ failed_set_pasid:
+ if (device_user_pasid_enabled(idxd))
+ idxd_xa_pasid_remove(ctx);
+ failed_get_pasid:
+ if (device_user_pasid_enabled(idxd))
+ iommu_sva_unbind_device(sva);
failed:
mutex_unlock(&wq->wq_lock);
kfree(ctx);
@@ -193,6 +236,7 @@ static int idxd_cdev_release(struct inode *node, struct file *filep)
if (ctx->sva) {
idxd_cdev_evl_drain_pasid(wq, ctx->pasid);
iommu_sva_unbind_device(ctx->sva);
+ idxd_xa_pasid_remove(ctx);
}
kfree(ctx);
diff --git a/drivers/dma/idxd/idxd.h b/drivers/dma/idxd/idxd.h
index 00edf68e3528..3e0f113368fa 100644
--- a/drivers/dma/idxd/idxd.h
+++ b/drivers/dma/idxd/idxd.h
@@ -127,6 +127,12 @@ struct idxd_pmu {
#define IDXD_MAX_PRIORITY 0xf
+enum {
+ COUNTER_FAULTS = 0,
+ COUNTER_FAULT_FAILS,
+ COUNTER_MAX
+};
+
enum idxd_wq_state {
IDXD_WQ_DISABLED = 0,
IDXD_WQ_ENABLED,
@@ -215,6 +221,10 @@ struct idxd_wq {
char name[WQ_NAME_SIZE + 1];
u64 max_xfer_bytes;
u32 max_batch_size;
+
+ /* Lock to protect upasid_xa access. */
+ struct mutex uc_lock;
+ struct xarray upasid_xa;
};
struct idxd_engine {
@@ -705,6 +715,7 @@ void idxd_cdev_remove(void);
int idxd_cdev_get_major(struct idxd_device *idxd);
int idxd_wq_add_cdev(struct idxd_wq *wq);
void idxd_wq_del_cdev(struct idxd_wq *wq);
+void idxd_user_counter_increment(struct idxd_wq *wq, u32 pasid, int index);
/* perfmon */
#if IS_ENABLED(CONFIG_INTEL_IDXD_PERFMON)
diff --git a/drivers/dma/idxd/init.c b/drivers/dma/idxd/init.c
index ddb9b13f3c3c..564c025b9b7e 100644
--- a/drivers/dma/idxd/init.c
+++ b/drivers/dma/idxd/init.c
@@ -206,6 +206,8 @@ static int idxd_setup_wqs(struct idxd_device *idxd)
}
bitmap_copy(wq->opcap_bmap, idxd->opcap_bmap, IDXD_MAX_OPCAP_BITS);
}
+ mutex_init(&wq->uc_lock);
+ xa_init(&wq->upasid_xa);
idxd->wqs[i] = wq;
}
diff --git a/drivers/dma/idxd/irq.c b/drivers/dma/idxd/irq.c
index 18245f82d367..241ece8eb9fc 100644
--- a/drivers/dma/idxd/irq.c
+++ b/drivers/dma/idxd/irq.c
@@ -250,6 +250,7 @@ static void idxd_evl_fault_work(struct work_struct *work)
evl->batch_fail[entry_head->batch_id] = false;
copy_size = cr_size;
+ idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULTS);
break;
case DSA_COMP_BATCH_EVL_ERR:
bf = &evl->batch_fail[entry_head->batch_id];
@@ -261,6 +262,7 @@ static void idxd_evl_fault_work(struct work_struct *work)
*result = 1;
*bf = false;
}
+ idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULTS);
break;
case DSA_COMP_DRAIN_EVL:
copy_size = cr_size;
@@ -278,6 +280,7 @@ static void idxd_evl_fault_work(struct work_struct *work)
switch (fault->status) {
case DSA_COMP_CRA_XLAT:
if (copied != copy_size) {
+ idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULT_FAILS);
dev_err(dev, "Failed to write to completion record: (%d:%d)\n",
copy_size, copied);
if (entry_head->batch)
@@ -285,9 +288,11 @@ static void idxd_evl_fault_work(struct work_struct *work)
}
break;
case DSA_COMP_BATCH_EVL_ERR:
- if (copied != copy_size)
+ if (copied != copy_size) {
+ idxd_user_counter_increment(wq, entry_head->pasid, COUNTER_FAULT_FAILS);
dev_err(dev, "Failed to write to batch completion record: (%d:%d)\n",
copy_size, copied);
+ }
break;
case DSA_COMP_DRAIN_EVL:
if (copied != copy_size)
diff --git a/drivers/dma/idxd/sysfs.c b/drivers/dma/idxd/sysfs.c
index ea696e3c5680..137126781362 100644
--- a/drivers/dma/idxd/sysfs.c
+++ b/drivers/dma/idxd/sysfs.c
@@ -1292,6 +1292,7 @@ static void idxd_conf_wq_release(struct device *dev)
bitmap_free(wq->opcap_bmap);
kfree(wq->wqcfg);
+ xa_destroy(&wq->upasid_xa);
kfree(wq);
}
--
2.32.0
