The function pointer to the handler that processes interception of the
PQAP instruction is contained in the mdev. If the mdev is removed and
its storage de-allocated during the processing of the PQAP instruction,
the function pointer could get wiped out before the function is called
because there is currently nothing that controls access to it.
This patch introduces two new functions:
* The kvm_arch_crypto_register_hook() function registers a function pointer
for processing intercepted crypto instructions.
* The kvm_arch_crypto_register_hook() function un-registers a function
pointer that was previously registered.
Registration and unregistration of function pointers is protected by a
mutex lock. This lock is also employed when the hook is retrieved and the
hook function is called to protect against losing access to the function
while an intercepted crypto instruction is being processed.
The PQAP instruction handler function pointer is registered at the time
the vfio_ap module is loaded and unregistered when it is unloaded; so,
the lifespan of the function pointer is concurrent with the lifespan of
the vfio_ap module.
Signed-off-by: Tony Krowiak <akrowiak@xxxxxxxxxxxxx>
---
arch/s390/include/asm/kvm_host.h | 13 +++--
arch/s390/kvm/priv.c | 70 +++++++++++++++++++++++++--
drivers/s390/crypto/vfio_ap_ops.c | 37 +++++++++++---
drivers/s390/crypto/vfio_ap_private.h | 1 -
4 files changed, 105 insertions(+), 16 deletions(-)
diff --git a/arch/s390/include/asm/kvm_host.h b/arch/s390/include/asm/kvm_host.h
index 8925f3969478..d59b9309a6b8 100644
--- a/arch/s390/include/asm/kvm_host.h
+++ b/arch/s390/include/asm/kvm_host.h
@@ -803,14 +803,19 @@ struct kvm_s390_cpu_model {
unsigned short ibc;
};
-struct kvm_s390_module_hook {
- int (*hook)(struct kvm_vcpu *vcpu);
+enum kvm_s390_crypto_hook_type {
+ PQAP_HOOK
+};
+
+struct kvm_s390_crypto_hook {
+ enum kvm_s390_crypto_hook_type type;
struct module *owner;
+ int (*hook_fcn)(struct kvm_vcpu *vcpu);
+ struct list_head node;
};
struct kvm_s390_crypto {
struct kvm_s390_crypto_cb *crycb;
- struct kvm_s390_module_hook *pqap_hook;
__u32 crycbd;
__u8 aes_kw;
__u8 dea_kw;
@@ -996,6 +1001,8 @@ static inline void kvm_arch_async_page_present_queued(struct kvm_vcpu *vcpu) {}
void kvm_arch_crypto_clear_masks(struct kvm *kvm);
void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
unsigned long *aqm, unsigned long *adm);
+extern int kvm_arch_crypto_register_hook(struct kvm_s390_crypto_hook *hook);
+extern int kvm_arch_crypto_unregister_hook(struct kvm_s390_crypto_hook *hook);
extern int sie64a(struct kvm_s390_sie_block *, u64 *);
extern char sie_exit;
diff --git a/arch/s390/kvm/priv.c b/arch/s390/kvm/priv.c
index 9928f785c677..1221c04f6f6a 100644
--- a/arch/s390/kvm/priv.c
+++ b/arch/s390/kvm/priv.c
@@ -12,6 +12,7 @@
#include <linux/gfp.h>
#include <linux/errno.h>
#include <linux/compat.h>
+#include <linux/list.h>
#include <linux/mm_types.h>
#include <linux/pgtable.h>
@@ -31,6 +32,63 @@
#include "kvm-s390.h"
#include "trace.h"
+DEFINE_MUTEX(crypto_hooks_lock);
+static struct list_head crypto_hooks = { &(crypto_hooks), &(crypto_hooks) };
+
+static struct kvm_s390_crypto_hook
+*kvm_arch_crypto_find_hook(enum kvm_s390_crypto_hook_type type)
+{
+ struct kvm_s390_crypto_hook *crypto_hook;
+
+ list_for_each_entry(crypto_hook, &crypto_hooks, node) {
+ if (crypto_hook->type == type)
+ return crypto_hook;
+ }
+
+ return NULL;
+}
+
+int kvm_arch_crypto_register_hook(struct kvm_s390_crypto_hook *hook)
+{
+ struct kvm_s390_crypto_hook *crypto_hook;
+
+ mutex_lock(&crypto_hooks_lock);
+ crypto_hook = kvm_arch_crypto_find_hook(hook->type);
+ if (crypto_hook) {
+ if (crypto_hook->owner != hook->owner)
+ return -EACCES;
+ list_replace(&crypto_hook->node, &hook->node);
+ } else {
+ list_add(&hook->node, &crypto_hooks);
+ }
+ mutex_unlock(&crypto_hooks_lock);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_arch_crypto_register_hook);
+
+int kvm_arch_crypto_unregister_hook(struct kvm_s390_crypto_hook *hook)
+{
+ struct kvm_s390_crypto_hook *crypto_hook;
+
+ mutex_lock(&crypto_hooks_lock);
+ crypto_hook = kvm_arch_crypto_find_hook(hook->type);
+
+ if (crypto_hook) {
+ if (crypto_hook->owner != hook->owner)
+ return -EACCES;
+ if (crypto_hook->hook_fcn != hook->hook_fcn)
+ return -EFAULT;
+ list_del(&crypto_hook->node);
+ } else {
+ return -ENODEV;
+ }
+ mutex_unlock(&crypto_hooks_lock);
+
+ return 0;
+}
+EXPORT_SYMBOL_GPL(kvm_arch_crypto_unregister_hook);
+
static int handle_ri(struct kvm_vcpu *vcpu)
{
vcpu->stat.instruction_ri++;
@@ -610,6 +668,7 @@ static int handle_io_inst(struct kvm_vcpu *vcpu)
static int handle_pqap(struct kvm_vcpu *vcpu)
{
struct ap_queue_status status = {};
+ struct kvm_s390_crypto_hook *pqap_hook;
unsigned long reg0;
int ret;
uint8_t fc;
@@ -657,15 +716,16 @@ static int handle_pqap(struct kvm_vcpu *vcpu)
* Verify that the hook callback is registered, lock the owner
* and call the hook.
*/
- if (vcpu->kvm->arch.crypto.pqap_hook) {
- if (!try_module_get(vcpu->kvm->arch.crypto.pqap_hook->owner))
- return -EOPNOTSUPP;
- ret = vcpu->kvm->arch.crypto.pqap_hook->hook(vcpu);
- module_put(vcpu->kvm->arch.crypto.pqap_hook->owner);
+ mutex_lock(&crypto_hooks_lock);
+ pqap_hook = kvm_arch_crypto_find_hook(PQAP_HOOK);
+ if (pqap_hook) {
+ ret = pqap_hook->hook_fcn(vcpu);
if (!ret && vcpu->run->s.regs.gprs[1] & 0x00ff0000)
kvm_s390_set_psw_cc(vcpu, 3);
+ mutex_unlock(&crypto_hooks_lock);
return ret;
}
+ mutex_unlock(&crypto_hooks_lock);
/*
* A vfio_driver must register a hook.
* No hook means no driver to enable the SIE CRYCB and no queues.
diff --git a/drivers/s390/crypto/vfio_ap_ops.c b/drivers/s390/crypto/vfio_ap_ops.c
index f90c9103dac2..3466ceffc46a 100644
--- a/drivers/s390/crypto/vfio_ap_ops.c
+++ b/drivers/s390/crypto/vfio_ap_ops.c
@@ -64,6 +64,21 @@ static struct vfio_ap_queue *vfio_ap_get_queue(
return q;
}
+static struct ap_matrix_mdev *vfio_ap_find_mdev_for_apqn(int apqn)
+{
+ struct ap_matrix_mdev *matrix_mdev;
+ unsigned long apid = AP_QID_CARD(apqn);
+ unsigned long apqi = AP_QID_QUEUE(apqn);
+
+ list_for_each_entry(matrix_mdev, &matrix_dev->mdev_list, node) {
+ if (test_bit_inv(apid, matrix_mdev->matrix.apm) &&
+ test_bit_inv(apqi, matrix_mdev->matrix.aqm))
+ return matrix_mdev;
+ }
+
+ return NULL;
+}
+
/**
* vfio_ap_wait_for_irqclear
* @apqn: The AP Queue number
@@ -287,13 +302,13 @@ static int handle_pqap(struct kvm_vcpu *vcpu)
if (!(vcpu->arch.sie_block->eca & ECA_AIV))
return -EOPNOTSUPP;
+
apqn = vcpu->run->s.regs.gprs[0] & 0xffff;
mutex_lock(&matrix_dev->lock);
- if (!vcpu->kvm->arch.crypto.pqap_hook)
+ matrix_mdev = vfio_ap_find_mdev_for_apqn(apqn);
+ if (!matrix_mdev)
goto out_unlock;
- matrix_mdev = container_of(vcpu->kvm->arch.crypto.pqap_hook,
- struct ap_matrix_mdev, pqap_hook);
/*
* If the KVM pointer is in the process of being set, wait until the
@@ -353,8 +368,6 @@ static int vfio_ap_mdev_create(struct mdev_device *mdev)
vfio_ap_matrix_init(&matrix_dev->info, &matrix_mdev->matrix);
init_waitqueue_head(&matrix_mdev->wait_for_kvm);
mdev_set_drvdata(mdev, matrix_mdev);
- matrix_mdev->pqap_hook.hook = handle_pqap;
- matrix_mdev->pqap_hook.owner = THIS_MODULE;
mutex_lock(&matrix_dev->lock);
list_add(&matrix_mdev->node, &matrix_dev->mdev_list);
mutex_unlock(&matrix_dev->lock);
@@ -1123,7 +1136,6 @@ static int vfio_ap_mdev_set_kvm(struct ap_matrix_mdev *matrix_mdev,
matrix_mdev->matrix.aqm,
matrix_mdev->matrix.adm);
mutex_lock(&matrix_dev->lock);
- kvm->arch.crypto.pqap_hook = &matrix_mdev->pqap_hook;
matrix_mdev->kvm = kvm;
matrix_mdev->kvm_busy = false;
wake_up_all(&matrix_mdev->wait_for_kvm);
@@ -1193,7 +1205,6 @@ static void vfio_ap_mdev_unset_kvm(struct ap_matrix_mdev *matrix_mdev)
kvm_arch_crypto_clear_masks(matrix_mdev->kvm);
mutex_lock(&matrix_dev->lock);
vfio_ap_mdev_reset_queues(matrix_mdev->mdev);
- matrix_mdev->kvm->arch.crypto.pqap_hook = NULL;
kvm_put_kvm(matrix_mdev->kvm);
matrix_mdev->kvm = NULL;
matrix_mdev->kvm_busy = false;
@@ -1433,14 +1444,26 @@ static const struct mdev_parent_ops vfio_ap_matrix_ops = {
.ioctl = vfio_ap_mdev_ioctl,
};
+static struct kvm_s390_crypto_hook pqap_hook = {
+ .type = PQAP_HOOK,
+ .owner = THIS_MODULE,
+ .hook_fcn = handle_pqap
+};
+
int vfio_ap_mdev_register(void)
{
+ int ret;
atomic_set(&matrix_dev->available_instances, MAX_ZDEV_ENTRIES_EXT);
+ ret = kvm_arch_crypto_register_hook(&pqap_hook);
+ if (ret)
+ return ret;
+
return mdev_register_device(&matrix_dev->device, &vfio_ap_matrix_ops);
}
void vfio_ap_mdev_unregister(void)
{
+ WARN_ON(kvm_arch_crypto_unregister_hook(&pqap_hook));
mdev_unregister_device(&matrix_dev->device);
}
diff --git a/drivers/s390/crypto/vfio_ap_private.h b/drivers/s390/crypto/vfio_ap_private.h
index f82a6396acae..542259b57972 100644
--- a/drivers/s390/crypto/vfio_ap_private.h
+++ b/drivers/s390/crypto/vfio_ap_private.h
@@ -86,7 +86,6 @@ struct ap_matrix_mdev {
bool kvm_busy;
wait_queue_head_t wait_for_kvm;
struct kvm *kvm;
- struct kvm_s390_module_hook pqap_hook;
struct mdev_device *mdev;
};
--
2.30.2
