The SMMU provides a Stall model for handling page faults in platform
devices. It is similar to PCI PRI, but doesn't require devices to have
their own translation cache. Instead, faulting transactions are parked and
the OS is given a chance to fix the page tables and retry the transaction.
Enable stall for devices that support it (opt-in by firmware). When an
event corresponds to a translation error, call the IOMMU fault handler. If
the fault is recoverable, it will call us back to terminate or continue
the stall.
Note that this patch tweaks the iommu_fault_event and page_response_msg to
extend the fault id field. Stall uses 16 bits of IDs whereas PCI PRI only
uses 9.
Signed-off-by: Jean-Philippe Brucker <jean-philippe.brucker@xxxxxxx>
---
drivers/iommu/arm-smmu-v3.c | 175 +++++++++++++++++++++++++++++++++++++++++++-
include/linux/iommu.h | 4 +-
2 files changed, 173 insertions(+), 6 deletions(-)
diff --git a/drivers/iommu/arm-smmu-v3.c b/drivers/iommu/arm-smmu-v3.c
index 2430b2140f8d..8b9f5dd06be0 100644
--- a/drivers/iommu/arm-smmu-v3.c
+++ b/drivers/iommu/arm-smmu-v3.c
@@ -338,6 +338,15 @@
#define CMDQ_PRI_1_RESP_FAIL (1UL << CMDQ_PRI_1_RESP_SHIFT)
#define CMDQ_PRI_1_RESP_SUCC (2UL << CMDQ_PRI_1_RESP_SHIFT)
+#define CMDQ_RESUME_0_SID_SHIFT 32
+#define CMDQ_RESUME_0_SID_MASK 0xffffffffUL
+#define CMDQ_RESUME_0_ACTION_SHIFT 12
+#define CMDQ_RESUME_0_ACTION_TERM (0UL << CMDQ_RESUME_0_ACTION_SHIFT)
+#define CMDQ_RESUME_0_ACTION_RETRY (1UL << CMDQ_RESUME_0_ACTION_SHIFT)
+#define CMDQ_RESUME_0_ACTION_ABORT (2UL << CMDQ_RESUME_0_ACTION_SHIFT)
+#define CMDQ_RESUME_1_STAG_SHIFT 0
+#define CMDQ_RESUME_1_STAG_MASK 0xffffUL
+
#define CMDQ_SYNC_0_CS_SHIFT 12
#define CMDQ_SYNC_0_CS_NONE (0UL << CMDQ_SYNC_0_CS_SHIFT)
#define CMDQ_SYNC_0_CS_IRQ (1UL << CMDQ_SYNC_0_CS_SHIFT)
@@ -358,6 +367,31 @@
#define EVTQ_0_ID_SHIFT 0
#define EVTQ_0_ID_MASK 0xffUL
+#define EVT_ID_TRANSLATION_FAULT 0x10
+#define EVT_ID_ADDR_SIZE_FAULT 0x11
+#define EVT_ID_ACCESS_FAULT 0x12
+#define EVT_ID_PERMISSION_FAULT 0x13
+
+#define EVTQ_0_SSV (1UL << 11)
+#define EVTQ_0_SSID_SHIFT 12
+#define EVTQ_0_SSID_MASK 0xfffffUL
+#define EVTQ_0_SID_SHIFT 32
+#define EVTQ_0_SID_MASK 0xffffffffUL
+#define EVTQ_1_STAG_SHIFT 0
+#define EVTQ_1_STAG_MASK 0xffffUL
+#define EVTQ_1_STALL (1UL << 31)
+#define EVTQ_1_PRIV (1UL << 33)
+#define EVTQ_1_EXEC (1UL << 34)
+#define EVTQ_1_READ (1UL << 35)
+#define EVTQ_1_S2 (1UL << 39)
+#define EVTQ_1_CLASS_SHIFT 40
+#define EVTQ_1_CLASS_MASK 0x3UL
+#define EVTQ_1_TT_READ (1UL << 44)
+#define EVTQ_2_ADDR_SHIFT 0
+#define EVTQ_2_ADDR_MASK 0xffffffffffffffffUL
+#define EVTQ_3_IPA_SHIFT 12
+#define EVTQ_3_IPA_MASK 0xffffffffffUL
+
/* PRI queue */
#define PRIQ_ENT_DWORDS 2
#define PRIQ_MAX_SZ_SHIFT 8
@@ -472,6 +506,13 @@ struct arm_smmu_cmdq_ent {
enum pri_resp resp;
} pri;
+ #define CMDQ_OP_RESUME 0x44
+ struct {
+ u32 sid;
+ u16 stag;
+ enum page_response_code resp;
+ } resume;
+
#define CMDQ_OP_CMD_SYNC 0x46
struct {
u32 msidata;
@@ -545,6 +586,8 @@ struct arm_smmu_strtab_ent {
bool assigned;
struct arm_smmu_s1_cfg *s1_cfg;
struct arm_smmu_s2_cfg *s2_cfg;
+
+ bool can_stall;
};
struct arm_smmu_strtab_cfg {
@@ -904,6 +947,21 @@ static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
return -EINVAL;
}
break;
+ case CMDQ_OP_RESUME:
+ cmd[0] |= (u64)ent->resume.sid << CMDQ_RESUME_0_SID_SHIFT;
+ cmd[1] |= ent->resume.stag << CMDQ_RESUME_1_STAG_SHIFT;
+ switch (ent->resume.resp) {
+ case IOMMU_PAGE_RESP_INVALID:
+ case IOMMU_PAGE_RESP_FAILURE:
+ cmd[0] |= CMDQ_RESUME_0_ACTION_ABORT;
+ break;
+ case IOMMU_PAGE_RESP_SUCCESS:
+ cmd[0] |= CMDQ_RESUME_0_ACTION_RETRY;
+ break;
+ default:
+ return -EINVAL;
+ }
+ break;
case CMDQ_OP_CMD_SYNC:
if (ent->sync.msiaddr)
cmd[0] |= CMDQ_SYNC_0_CS_IRQ;
@@ -1065,6 +1123,35 @@ static void arm_smmu_cmdq_issue_sync(struct arm_smmu_device *smmu)
dev_err_ratelimited(smmu->dev, "CMD_SYNC timeout\n");
}
+static int arm_smmu_page_response(struct iommu_domain *domain,
+ struct device *dev,
+ struct page_response_msg *resp)
+{
+ int sid = dev->iommu_fwspec->ids[0];
+ struct arm_smmu_cmdq_ent cmd = {0};
+ struct arm_smmu_master_data *master = dev->iommu_fwspec->iommu_priv;
+
+ if (master->ste.can_stall) {
+ cmd.opcode = CMDQ_OP_RESUME;
+ cmd.resume.sid = sid;
+ cmd.resume.stag = resp->page_req_group_id;
+ cmd.resume.resp = resp->resp_code;
+ } else {
+ /* TODO: put PRI response here */
+ return -EINVAL;
+ }
+
+ arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
+ /*
+ * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
+ * RESUME consumption guarantees that the stalled transaction will be
+ * terminated... at some point in the future. PRI_RESP is fire and
+ * forget.
+ */
+
+ return 0;
+}
+
/* Stream table manipulation functions */
static void
arm_smmu_write_strtab_l1_desc(__le64 *dst, struct arm_smmu_strtab_l1_desc *desc)
@@ -1182,7 +1269,8 @@ static void arm_smmu_write_strtab_ent(struct arm_smmu_device *smmu, u32 sid,
STRTAB_STE_1_STRW_SHIFT);
if (smmu->features & ARM_SMMU_FEAT_STALLS &&
- !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
+ !(smmu->features & ARM_SMMU_FEAT_STALL_FORCE) &&
+ !ste->can_stall)
dst[1] |= cpu_to_le64(STRTAB_STE_1_S1STALLD);
val |= (cfg->base & STRTAB_STE_0_S1CTXPTR_MASK
@@ -1285,10 +1373,73 @@ arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
return master;
}
+static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
+{
+ struct arm_smmu_master_data *master;
+ u8 type = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
+ u32 sid = evt[0] >> EVTQ_0_SID_SHIFT & EVTQ_0_SID_MASK;
+
+ struct iommu_fault_event fault = {
+ .page_req_group_id = evt[1] >> EVTQ_1_STAG_SHIFT & EVTQ_1_STAG_MASK,
+ .addr = evt[2] >> EVTQ_2_ADDR_SHIFT & EVTQ_2_ADDR_MASK,
+ .last_req = true,
+ };
+
+ switch (type) {
+ case EVT_ID_TRANSLATION_FAULT:
+ case EVT_ID_ADDR_SIZE_FAULT:
+ case EVT_ID_ACCESS_FAULT:
+ fault.reason = IOMMU_FAULT_REASON_PTE_FETCH;
+ break;
+ case EVT_ID_PERMISSION_FAULT:
+ fault.reason = IOMMU_FAULT_REASON_PERMISSION;
+ break;
+ default:
+ /* TODO: report other unrecoverable faults. */
+ return -EFAULT;
+ }
+
+ /* Stage-2 is always pinned at the moment */
+ if (evt[1] & EVTQ_1_S2)
+ return -EFAULT;
+
+ master = arm_smmu_find_master(smmu, sid);
+ if (!master)
+ return -EINVAL;
+
+ /*
+ * The domain is valid until the fault returns, because detach() flushes
+ * the fault queue.
+ */
+ if (evt[1] & EVTQ_1_STALL)
+ fault.type = IOMMU_FAULT_PAGE_REQ;
+ else
+ fault.type = IOMMU_FAULT_DMA_UNRECOV;
+
+ if (evt[1] & EVTQ_1_READ)
+ fault.prot |= IOMMU_FAULT_READ;
+ else
+ fault.prot |= IOMMU_FAULT_WRITE;
+
+ if (evt[1] & EVTQ_1_EXEC)
+ fault.prot |= IOMMU_FAULT_EXEC;
+
+ if (evt[1] & EVTQ_1_PRIV)
+ fault.prot |= IOMMU_FAULT_PRIV;
+
+ if (evt[0] & EVTQ_0_SSV) {
+ fault.pasid_valid = true;
+ fault.pasid = evt[0] >> EVTQ_0_SSID_SHIFT & EVTQ_0_SSID_MASK;
+ }
+
+ /* Report to device driver or populate the page tables */
+ return iommu_report_device_fault(master->dev, &fault);
+}
+
/* IRQ and event handlers */
static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
{
- int i;
+ int i, ret;
int num_handled = 0;
struct arm_smmu_device *smmu = dev;
struct arm_smmu_queue *q = &smmu->evtq.q;
@@ -1300,12 +1451,19 @@ static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
while (!queue_remove_raw(q, evt)) {
u8 id = evt[0] >> EVTQ_0_ID_SHIFT & EVTQ_0_ID_MASK;
+ spin_unlock(&q->wq.lock);
+ ret = arm_smmu_handle_evt(smmu, evt);
+ spin_lock(&q->wq.lock);
+
if (++num_handled == queue_size) {
q->batch++;
wake_up_locked(&q->wq);
num_handled = 0;
}
+ if (!ret)
+ continue;
+
dev_info(smmu->dev, "event 0x%02x received:\n", id);
for (i = 0; i < ARRAY_SIZE(evt); ++i)
dev_info(smmu->dev, "\t0x%016llx\n",
@@ -1442,7 +1600,9 @@ static int arm_smmu_flush_queues(struct notifier_block *nb,
master = dev->iommu_fwspec->iommu_priv;
if (master) {
- /* TODO: add support for PRI and Stall */
+ if (master->ste.can_stall)
+ arm_smmu_flush_queue(smmu, &smmu->evtq.q, "evtq");
+ /* TODO: add support for PRI */
return 0;
}
@@ -1756,7 +1916,8 @@ static int arm_smmu_domain_finalise_s1(struct arm_smmu_domain *smmu_domain,
.order = master->ssid_bits,
.sync = &arm_smmu_ctx_sync,
.arm_smmu = {
- .stall = !!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE),
+ .stall = !!(smmu->features & ARM_SMMU_FEAT_STALL_FORCE) ||
+ master->ste.can_stall,
.asid_bits = smmu->asid_bits,
.hw_access = !!(smmu->features & ARM_SMMU_FEAT_HA),
.hw_dirty = !!(smmu->features & ARM_SMMU_FEAT_HD),
@@ -2296,6 +2457,11 @@ static int arm_smmu_add_device(struct device *dev)
master->ssid_bits = min(smmu->ssid_bits, fwspec->num_pasid_bits);
+ if (fwspec->can_stall && smmu->features & ARM_SMMU_FEAT_STALLS) {
+ master->can_fault = true;
+ master->ste.can_stall = true;
+ }
+
group = iommu_group_get_for_dev(dev);
if (!IS_ERR(group)) {
arm_smmu_insert_master(smmu, master);
@@ -2435,6 +2601,7 @@ static struct iommu_ops arm_smmu_ops = {
.mm_attach = arm_smmu_mm_attach,
.mm_detach = arm_smmu_mm_detach,
.mm_invalidate = arm_smmu_mm_invalidate,
+ .page_response = arm_smmu_page_response,
.map = arm_smmu_map,
.unmap = arm_smmu_unmap,
.map_sg = default_iommu_map_sg,
diff --git a/include/linux/iommu.h b/include/linux/iommu.h
index 37c3b9d087ce..f5c2f4be2b42 100644
--- a/include/linux/iommu.h
+++ b/include/linux/iommu.h
@@ -227,7 +227,7 @@ struct page_response_msg {
u32 pasid;
enum page_response_code resp_code;
u32 pasid_present:1;
- u32 page_req_group_id : 9;
+ u32 page_req_group_id;
enum page_response_type type;
u32 private_data;
};
@@ -421,7 +421,7 @@ struct iommu_fault_event {
enum iommu_fault_reason reason;
u64 addr;
u32 pasid;
- u32 page_req_group_id : 9;
+ u32 page_req_group_id;
u32 last_req : 1;
u32 pasid_valid : 1;
u32 prot;
--
2.15.1
