On Tue, Nov 28, 2023 at 5:01 AM Nikunj A Dadhania <nikunj@xxxxxxx> wrote: > > For enabling Secure TSC, SEV-SNP guests need to communicate with the > AMD Security Processor early during boot. Many of the required > functions are implemented in the sev-guest driver and therefore not > available at early boot. Move the required functions and provide > API to the sev guest driver for sending guest message and vmpck > routines. > > As there is no external caller for snp_issue_guest_request() anymore, > make it static and drop the prototype from sev-guest.h. > > Signed-off-by: Nikunj A Dadhania <nikunj@xxxxxxx> > --- > arch/x86/Kconfig | 1 + > arch/x86/include/asm/sev-guest.h | 91 ++++- > arch/x86/include/asm/sev.h | 10 - > arch/x86/kernel/sev.c | 451 +++++++++++++++++++++- > drivers/virt/coco/sev-guest/Kconfig | 1 - > drivers/virt/coco/sev-guest/sev-guest.c | 479 +----------------------- > 6 files changed, 550 insertions(+), 483 deletions(-) > > diff --git a/arch/x86/Kconfig b/arch/x86/Kconfig > index 3762f41bb092..b8f374ec5651 100644 > --- a/arch/x86/Kconfig > +++ b/arch/x86/Kconfig > @@ -1534,6 +1534,7 @@ config AMD_MEM_ENCRYPT > select ARCH_HAS_CC_PLATFORM > select X86_MEM_ENCRYPT > select UNACCEPTED_MEMORY > + select CRYPTO_LIB_AESGCM > help > Say yes to enable support for the encryption of system memory. > This requires an AMD processor that supports Secure Memory > diff --git a/arch/x86/include/asm/sev-guest.h b/arch/x86/include/asm/sev-guest.h > index 27cc15ad6131..16bf25c14e6f 100644 > --- a/arch/x86/include/asm/sev-guest.h > +++ b/arch/x86/include/asm/sev-guest.h > @@ -11,6 +11,11 @@ > #define __VIRT_SEVGUEST_H__ > > #include <linux/types.h> > +#include <linux/miscdevice.h> > +#include <asm/sev.h> > + > +#define SNP_REQ_MAX_RETRY_DURATION (60*HZ) > +#define SNP_REQ_RETRY_DELAY (2*HZ) > > #define MAX_AUTHTAG_LEN 32 > #define AUTHTAG_LEN 16 > @@ -58,11 +63,52 @@ struct snp_guest_msg_hdr { > u8 rsvd3[35]; > } __packed; > > +/* SNP Guest message request */ > +struct snp_req_data { > + unsigned long req_gpa; > + unsigned long resp_gpa; > +}; > + > struct snp_guest_msg { > struct snp_guest_msg_hdr hdr; > u8 payload[4000]; > } __packed; > > +struct sev_guest_platform_data { > + /* request and response are in unencrypted memory */ > + struct snp_guest_msg *request; > + struct snp_guest_msg *response; > + > + struct snp_secrets_page_layout *layout; > + struct snp_req_data input; > +}; > + > +struct snp_guest_dev { > + struct device *dev; > + struct miscdevice misc; > + > + /* Mutex to serialize the shared buffer access and command handling. */ > + struct mutex cmd_mutex; > + > + void *certs_data; > + struct aesgcm_ctx *ctx; > + > + /* > + * Avoid information leakage by double-buffering shared messages > + * in fields that are in regular encrypted memory > + */ > + struct snp_guest_msg secret_request; > + struct snp_guest_msg secret_response; > + > + struct sev_guest_platform_data *pdata; > + union { > + struct snp_report_req report; > + struct snp_derived_key_req derived_key; > + struct snp_ext_report_req ext_report; > + } req; > + unsigned int vmpck_id; > +}; > + > struct snp_guest_req { > void *req_buf; > size_t req_sz; > @@ -79,6 +125,47 @@ struct snp_guest_req { > u8 msg_type; > }; > > -int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, > - struct snp_guest_request_ioctl *rio); > +int snp_setup_psp_messaging(struct snp_guest_dev *snp_dev); > +int snp_send_guest_request(struct snp_guest_dev *dev, struct snp_guest_req *req, > + struct snp_guest_request_ioctl *rio); > +bool snp_assign_vmpck(struct snp_guest_dev *dev, unsigned int vmpck_id); > +bool snp_is_vmpck_empty(unsigned int vmpck_id); > + > +static inline void free_shared_pages(void *buf, size_t sz) > +{ > + unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; > + int ret; > + > + if (!buf) > + return; > + > + ret = set_memory_encrypted((unsigned long)buf, npages); > + if (ret) { > + WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n"); > + return; > + } > + > + __free_pages(virt_to_page(buf), get_order(sz)); > +} > + > +static inline void *alloc_shared_pages(size_t sz) > +{ > + unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; > + struct page *page; > + int ret; > + > + page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(sz)); > + if (!page) > + return NULL; > + > + ret = set_memory_decrypted((unsigned long)page_address(page), npages); > + if (ret) { > + pr_err("%s: failed to mark page shared, ret=%d\n", __func__, ret); > + __free_pages(page, get_order(sz)); > + return NULL; > + } > + > + return page_address(page); > +} > + > #endif /* __VIRT_SEVGUEST_H__ */ > diff --git a/arch/x86/include/asm/sev.h b/arch/x86/include/asm/sev.h > index 78465a8c7dc6..783150458864 100644 > --- a/arch/x86/include/asm/sev.h > +++ b/arch/x86/include/asm/sev.h > @@ -93,16 +93,6 @@ extern bool handle_vc_boot_ghcb(struct pt_regs *regs); > > #define RMPADJUST_VMSA_PAGE_BIT BIT(16) > > -/* SNP Guest message request */ > -struct snp_req_data { > - unsigned long req_gpa; > - unsigned long resp_gpa; > -}; > - > -struct sev_guest_platform_data { > - u64 secrets_gpa; > -}; > - > /* > * The secrets page contains 96-bytes of reserved field that can be used by > * the guest OS. The guest OS uses the area to save the message sequence > diff --git a/arch/x86/kernel/sev.c b/arch/x86/kernel/sev.c > index 479ea61f40f3..a413add2fd2c 100644 > --- a/arch/x86/kernel/sev.c > +++ b/arch/x86/kernel/sev.c > @@ -24,6 +24,7 @@ > #include <linux/io.h> > #include <linux/psp-sev.h> > #include <uapi/linux/sev-guest.h> > +#include <crypto/gcm.h> > > #include <asm/cpu_entry_area.h> > #include <asm/stacktrace.h> > @@ -2150,8 +2151,8 @@ static int __init init_sev_config(char *str) > } > __setup("sev=", init_sev_config); > > -int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, > - struct snp_guest_request_ioctl *rio) > +static int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *input, > + struct snp_guest_request_ioctl *rio) > { > struct ghcb_state state; > struct es_em_ctxt ctxt; > @@ -2218,7 +2219,6 @@ int snp_issue_guest_request(struct snp_guest_req *req, struct snp_req_data *inpu > > return ret; > } > -EXPORT_SYMBOL_GPL(snp_issue_guest_request); > > static struct platform_device sev_guest_device = { > .name = "sev-guest", > @@ -2227,22 +2227,451 @@ static struct platform_device sev_guest_device = { > > static int __init snp_init_platform_device(void) > { > - struct sev_guest_platform_data data; > - > if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) > return -ENODEV; > > - if (!secrets_pa) > + if (platform_device_register(&sev_guest_device)) > return -ENODEV; > > - data.secrets_gpa = secrets_pa; > - if (platform_device_add_data(&sev_guest_device, &data, sizeof(data))) > + pr_info("SNP guest platform device initialized.\n"); > + return 0; > +} > +device_initcall(snp_init_platform_device); > + > +static struct sev_guest_platform_data *platform_data; > + > +static inline u8 *snp_get_vmpck(unsigned int vmpck_id) > +{ > + if (!platform_data) > + return NULL; > + > + return platform_data->layout->vmpck0 + vmpck_id * VMPCK_KEY_LEN; > +} > + > +static inline u32 *snp_get_os_area_msg_seqno(unsigned int vmpck_id) > +{ > + if (!platform_data) > + return NULL; > + > + return &platform_data->layout->os_area.msg_seqno_0 + vmpck_id; > +} > + > +bool snp_is_vmpck_empty(unsigned int vmpck_id) > +{ > + char zero_key[VMPCK_KEY_LEN] = {0}; > + u8 *key = snp_get_vmpck(vmpck_id); > + > + if (key) > + return !memcmp(key, zero_key, VMPCK_KEY_LEN); > + > + return true; > +} > +EXPORT_SYMBOL_GPL(snp_is_vmpck_empty); > + > +/* > + * If an error is received from the host or AMD Secure Processor (ASP) there > + * are two options. Either retry the exact same encrypted request or discontinue > + * using the VMPCK. > + * > + * This is because in the current encryption scheme GHCB v2 uses AES-GCM to > + * encrypt the requests. The IV for this scheme is the sequence number. GCM > + * cannot tolerate IV reuse. > + * > + * The ASP FW v1.51 only increments the sequence numbers on a successful > + * guest<->ASP back and forth and only accepts messages at its exact sequence > + * number. > + * > + * So if the sequence number were to be reused the encryption scheme is > + * vulnerable. If the sequence number were incremented for a fresh IV the ASP > + * will reject the request. > + */ > +static void snp_disable_vmpck(struct snp_guest_dev *snp_dev) > +{ > + u8 *key = snp_get_vmpck(snp_dev->vmpck_id); > + > + pr_alert("Disabling vmpck_id %u to prevent IV reuse.\n", snp_dev->vmpck_id); > + memzero_explicit(key, VMPCK_KEY_LEN); > +} > + > +static inline u64 __snp_get_msg_seqno(struct snp_guest_dev *snp_dev) > +{ > + u32 *os_area_msg_seqno = snp_get_os_area_msg_seqno(snp_dev->vmpck_id); > + u64 count; > + > + if (!os_area_msg_seqno) { > + pr_err("SNP unable to get message sequence counter\n"); > + return 0; > + } > + > + lockdep_assert_held(&snp_dev->cmd_mutex); > + > + /* Read the current message sequence counter from secrets pages */ > + count = *os_area_msg_seqno; > + > + return count + 1; > +} > + > +/* Return a non-zero on success */ > +static u64 snp_get_msg_seqno(struct snp_guest_dev *snp_dev) > +{ > + u64 count = __snp_get_msg_seqno(snp_dev); > + > + /* > + * The message sequence counter for the SNP guest request is a 64-bit > + * value but the version 2 of GHCB specification defines a 32-bit storage > + * for it. If the counter exceeds the 32-bit value then return zero. > + * The caller should check the return value, but if the caller happens to > + * not check the value and use it, then the firmware treats zero as an > + * invalid number and will fail the message request. > + */ > + if (count >= UINT_MAX) { > + pr_err("SNP request message sequence counter overflow\n"); > + return 0; > + } > + > + return count; > +} > + > +static void snp_inc_msg_seqno(struct snp_guest_dev *snp_dev) > +{ > + u32 *os_area_msg_seqno = snp_get_os_area_msg_seqno(snp_dev->vmpck_id); > + > + if (!os_area_msg_seqno) { > + pr_err("SNP unable to get message sequence counter\n"); > + return; > + } > + > + lockdep_assert_held(&snp_dev->cmd_mutex); > + > + /* > + * The counter is also incremented by the PSP, so increment it by 2 > + * and save in secrets page. > + */ > + *os_area_msg_seqno += 2; > +} > + > +static struct aesgcm_ctx *snp_init_crypto(unsigned int vmpck_id) > +{ > + struct aesgcm_ctx *ctx; > + u8 *key; > + > + if (snp_is_vmpck_empty(vmpck_id)) { > + pr_err("VM communication key VMPCK%u is null\n", vmpck_id); > + return NULL; > + } > + > + ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); > + if (!ctx) > + return NULL; > + > + key = snp_get_vmpck(vmpck_id); > + if (aesgcm_expandkey(ctx, key, VMPCK_KEY_LEN, AUTHTAG_LEN)) { > + pr_err("Crypto context initialization failed\n"); > + kfree(ctx); > + return NULL; > + } > + > + return ctx; > +} > + > +int snp_setup_psp_messaging(struct snp_guest_dev *snp_dev) > +{ > + struct sev_guest_platform_data *pdata; > + int ret; > + > + if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) { Note that this may be going away in favor of an cpu_feature_enabled(X86_FEATURE_...) check given Kirill's "[PATCH] x86/coco, x86/sev: Use cpu_feature_enabled() to detect SEV guest flavor" > + pr_err("SNP not supported\n"); > + return 0; > + } > + > + if (platform_data) { > + pr_debug("SNP platform data already initialized.\n"); > + goto create_ctx; > + } > + > + if (!secrets_pa) { > + pr_err("SNP secrets page not found\n"); > return -ENODEV; > + } > > - if (platform_device_register(&sev_guest_device)) > + pdata = kzalloc(sizeof(struct sev_guest_platform_data), GFP_KERNEL); > + if (!pdata) { > + pr_err("Allocation of SNP guest platform data failed\n"); > + return -ENOMEM; > + } > + > + pdata->layout = (__force void *)ioremap_encrypted(secrets_pa, PAGE_SIZE); > + if (!pdata->layout) { > + pr_err("Failed to map SNP secrets page.\n"); > + goto e_free_pdata; > + } > + > + ret = -ENOMEM; > + /* Allocate the shared page used for the request and response message. */ > + pdata->request = alloc_shared_pages(sizeof(struct snp_guest_msg)); > + if (!pdata->request) > + goto e_unmap; > + > + pdata->response = alloc_shared_pages(sizeof(struct snp_guest_msg)); > + if (!pdata->response) > + goto e_free_request; > + > + /* initial the input address for guest request */ > + pdata->input.req_gpa = __pa(pdata->request); > + pdata->input.resp_gpa = __pa(pdata->response); > + platform_data = pdata; > + > +create_ctx: > + ret = -EIO; > + snp_dev->ctx = snp_init_crypto(snp_dev->vmpck_id); > + if (!snp_dev->ctx) { > + pr_err("SNP crypto context initialization failed\n"); > + platform_data = NULL; > + goto e_free_response; > + } > + > + snp_dev->pdata = platform_data; > + > + return 0; > + > +e_free_response: > + free_shared_pages(pdata->response, sizeof(struct snp_guest_msg)); > +e_free_request: > + free_shared_pages(pdata->request, sizeof(struct snp_guest_msg)); > +e_unmap: > + iounmap(pdata->layout); > +e_free_pdata: > + kfree(pdata); > + > + return ret; > +} > +EXPORT_SYMBOL_GPL(snp_setup_psp_messaging); > + > +static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, struct snp_guest_req *guest_req, > + struct sev_guest_platform_data *pdata) > +{ > + struct snp_guest_msg *resp = &snp_dev->secret_response; > + struct snp_guest_msg *req = &snp_dev->secret_request; > + struct snp_guest_msg_hdr *req_hdr = &req->hdr; > + struct snp_guest_msg_hdr *resp_hdr = &resp->hdr; > + struct aesgcm_ctx *ctx = snp_dev->ctx; > + u8 iv[GCM_AES_IV_SIZE] = {}; > + > + pr_debug("response [seqno %lld type %d version %d sz %d]\n", > + resp_hdr->msg_seqno, resp_hdr->msg_type, resp_hdr->msg_version, > + resp_hdr->msg_sz); > + > + /* Copy response from shared memory to encrypted memory. */ > + memcpy(resp, pdata->response, sizeof(*resp)); > + > + /* Verify that the sequence counter is incremented by 1 */ > + if (unlikely(resp_hdr->msg_seqno != (req_hdr->msg_seqno + 1))) > + return -EBADMSG; > + > + /* Verify response message type and version number. */ > + if (resp_hdr->msg_type != (req_hdr->msg_type + 1) || > + resp_hdr->msg_version != req_hdr->msg_version) > + return -EBADMSG; > + > + /* > + * If the message size is greater than our buffer length then return > + * an error. > + */ > + if (unlikely((resp_hdr->msg_sz + ctx->authsize) > guest_req->resp_sz)) > + return -EBADMSG; > + > + /* Decrypt the payload */ > + memcpy(iv, &resp_hdr->msg_seqno, sizeof(resp_hdr->msg_seqno)); > + if (!aesgcm_decrypt(ctx, guest_req->resp_buf, resp->payload, resp_hdr->msg_sz, > + &resp_hdr->algo, AAD_LEN, iv, resp_hdr->authtag)) > + return -EBADMSG; > + > + return 0; > +} > + > +static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, struct snp_guest_req *req) > +{ > + struct snp_guest_msg *msg = &snp_dev->secret_request; > + struct snp_guest_msg_hdr *hdr = &msg->hdr; > + struct aesgcm_ctx *ctx = snp_dev->ctx; > + u8 iv[GCM_AES_IV_SIZE] = {}; > + > + memset(msg, 0, sizeof(*msg)); > + > + hdr->algo = SNP_AEAD_AES_256_GCM; > + hdr->hdr_version = MSG_HDR_VER; > + hdr->hdr_sz = sizeof(*hdr); > + hdr->msg_type = req->msg_type; > + hdr->msg_version = req->msg_version; > + hdr->msg_seqno = seqno; > + hdr->msg_vmpck = req->vmpck_id; > + hdr->msg_sz = req->req_sz; > + > + /* Verify the sequence number is non-zero */ > + if (!hdr->msg_seqno) > + return -ENOSR; > + > + pr_debug("request [seqno %lld type %d version %d sz %d]\n", > + hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); > + > + if (WARN_ON((req->req_sz + ctx->authsize) > sizeof(msg->payload))) > + return -EBADMSG; > + > + memcpy(iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno)); > + aesgcm_encrypt(ctx, msg->payload, req->req_buf, req->req_sz, &hdr->algo, > + AAD_LEN, iv, hdr->authtag); > + > + return 0; > +} > + > +static int __handle_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, > + struct snp_guest_request_ioctl *rio, > + struct sev_guest_platform_data *pdata) > +{ > + unsigned long req_start = jiffies; > + unsigned int override_npages = 0; > + u64 override_err = 0; > + int rc; > + > +retry_request: > + /* > + * Call firmware to process the request. In this function the encrypted > + * message enters shared memory with the host. So after this call the > + * sequence number must be incremented or the VMPCK must be deleted to > + * prevent reuse of the IV. > + */ > + rc = snp_issue_guest_request(req, &pdata->input, rio); > + switch (rc) { > + case -ENOSPC: > + /* > + * If the extended guest request fails due to having too > + * small of a certificate data buffer, retry the same > + * guest request without the extended data request in > + * order to increment the sequence number and thus avoid > + * IV reuse. > + */ > + override_npages = req->data_npages; > + req->exit_code = SVM_VMGEXIT_GUEST_REQUEST; > + > + /* > + * Override the error to inform callers the given extended > + * request buffer size was too small and give the caller the > + * required buffer size. > + */ > + override_err = SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN); > + > + /* > + * If this call to the firmware succeeds, the sequence number can > + * be incremented allowing for continued use of the VMPCK. If > + * there is an error reflected in the return value, this value > + * is checked further down and the result will be the deletion > + * of the VMPCK and the error code being propagated back to the > + * user as an ioctl() return code. > + */ > + goto retry_request; > + > + /* > + * The host may return SNP_GUEST_REQ_ERR_BUSY if the request has been > + * throttled. Retry in the driver to avoid returning and reusing the > + * message sequence number on a different message. > + */ > + case -EAGAIN: > + if (jiffies - req_start > SNP_REQ_MAX_RETRY_DURATION) { > + rc = -ETIMEDOUT; > + break; > + } > + schedule_timeout_killable(SNP_REQ_RETRY_DELAY); > + goto retry_request; > + } > + > + /* > + * Increment the message sequence number. There is no harm in doing > + * this now because decryption uses the value stored in the response > + * structure and any failure will wipe the VMPCK, preventing further > + * use anyway. > + */ > + snp_inc_msg_seqno(snp_dev); > + > + if (override_err) { > + rio->exitinfo2 = override_err; > + > + /* > + * If an extended guest request was issued and the supplied certificate > + * buffer was not large enough, a standard guest request was issued to > + * prevent IV reuse. If the standard request was successful, return -EIO > + * back to the caller as would have originally been returned. > + */ > + if (!rc && override_err == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) > + rc = -EIO; > + } > + > + if (override_npages) > + req->data_npages = override_npages; > + > + return rc; > +} > + > +int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, > + struct snp_guest_request_ioctl *rio) > +{ > + struct sev_guest_platform_data *pdata; > + u64 seqno; > + int rc; > + > + if (!snp_dev || !snp_dev->pdata || !req || !rio) > return -ENODEV; > > - pr_info("SNP guest platform device initialized.\n"); > + pdata = snp_dev->pdata; > + > + /* Get message sequence and verify that its a non-zero */ > + seqno = snp_get_msg_seqno(snp_dev); > + if (!seqno) > + return -EIO; > + > + /* Clear shared memory's response for the host to populate. */ > + memset(pdata->response, 0, sizeof(struct snp_guest_msg)); > + > + /* Encrypt the userspace provided payload in pdata->secret_request. */ > + rc = enc_payload(snp_dev, seqno, req); > + if (rc) > + return rc; > + > + /* > + * Write the fully encrypted request to the shared unencrypted > + * request page. > + */ > + memcpy(pdata->request, &snp_dev->secret_request, sizeof(snp_dev->secret_request)); > + > + rc = __handle_guest_request(snp_dev, req, rio, pdata); > + if (rc) { > + if (rc == -EIO && > + rio->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) > + return rc; > + > + pr_alert("Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n", > + rc, rio->exitinfo2); > + snp_disable_vmpck(snp_dev); > + return rc; > + } > + > + rc = verify_and_dec_payload(snp_dev, req, pdata); > + if (rc) { > + pr_alert("Detected unexpected decode failure from ASP. rc: %d\n", rc); > + snp_disable_vmpck(snp_dev); > + return rc; > + } > + > return 0; > } > -device_initcall(snp_init_platform_device); > +EXPORT_SYMBOL_GPL(snp_send_guest_request); > + > +bool snp_assign_vmpck(struct snp_guest_dev *dev, unsigned int vmpck_id) > +{ > + if (WARN_ON(vmpck_id > 3)) This constant 3 should be #define'd, I believe. > + return false; > + > + dev->vmpck_id = vmpck_id; > + > + return true; > +} > +EXPORT_SYMBOL_GPL(snp_assign_vmpck); > diff --git a/drivers/virt/coco/sev-guest/Kconfig b/drivers/virt/coco/sev-guest/Kconfig > index 0b772bd921d8..a6405ab6c2c3 100644 > --- a/drivers/virt/coco/sev-guest/Kconfig > +++ b/drivers/virt/coco/sev-guest/Kconfig > @@ -2,7 +2,6 @@ config SEV_GUEST > tristate "AMD SEV Guest driver" > default m > depends on AMD_MEM_ENCRYPT > - select CRYPTO_LIB_AESGCM > select TSM_REPORTS > help > SEV-SNP firmware provides the guest a mechanism to communicate with > diff --git a/drivers/virt/coco/sev-guest/sev-guest.c b/drivers/virt/coco/sev-guest/sev-guest.c > index 0f2134deca51..1cdf7ab04d39 100644 > --- a/drivers/virt/coco/sev-guest/sev-guest.c > +++ b/drivers/virt/coco/sev-guest/sev-guest.c > @@ -31,130 +31,10 @@ > > #define DEVICE_NAME "sev-guest" > > -#define SNP_REQ_MAX_RETRY_DURATION (60*HZ) > -#define SNP_REQ_RETRY_DELAY (2*HZ) > - > -struct snp_guest_dev { > - struct device *dev; > - struct miscdevice misc; > - > - /* Mutex to serialize the shared buffer access and command handling. */ > - struct mutex cmd_mutex; > - > - void *certs_data; > - struct aesgcm_ctx *ctx; > - /* request and response are in unencrypted memory */ > - struct snp_guest_msg *request, *response; > - > - /* > - * Avoid information leakage by double-buffering shared messages > - * in fields that are in regular encrypted memory. > - */ > - struct snp_guest_msg secret_request, secret_response; > - > - struct snp_secrets_page_layout *layout; > - struct snp_req_data input; > - union { > - struct snp_report_req report; > - struct snp_derived_key_req derived_key; > - struct snp_ext_report_req ext_report; > - } req; > - unsigned int vmpck_id; > -}; > - > static u32 vmpck_id; > module_param(vmpck_id, uint, 0444); > MODULE_PARM_DESC(vmpck_id, "The VMPCK ID to use when communicating with the PSP."); > > -static inline u8 *snp_get_vmpck(struct snp_guest_dev *snp_dev) > -{ > - return snp_dev->layout->vmpck0 + snp_dev->vmpck_id * VMPCK_KEY_LEN; > -} > - > -static inline u32 *snp_get_os_area_msg_seqno(struct snp_guest_dev *snp_dev) > -{ > - return &snp_dev->layout->os_area.msg_seqno_0 + snp_dev->vmpck_id; > -} > - > -static bool snp_is_vmpck_empty(struct snp_guest_dev *snp_dev) > -{ > - char zero_key[VMPCK_KEY_LEN] = {0}; > - u8 *key = snp_get_vmpck(snp_dev); > - > - return !memcmp(key, zero_key, VMPCK_KEY_LEN); > -} > - > -/* > - * If an error is received from the host or AMD Secure Processor (ASP) there > - * are two options. Either retry the exact same encrypted request or discontinue > - * using the VMPCK. > - * > - * This is because in the current encryption scheme GHCB v2 uses AES-GCM to > - * encrypt the requests. The IV for this scheme is the sequence number. GCM > - * cannot tolerate IV reuse. > - * > - * The ASP FW v1.51 only increments the sequence numbers on a successful > - * guest<->ASP back and forth and only accepts messages at its exact sequence > - * number. > - * > - * So if the sequence number were to be reused the encryption scheme is > - * vulnerable. If the sequence number were incremented for a fresh IV the ASP > - * will reject the request. > - */ > -static void snp_disable_vmpck(struct snp_guest_dev *snp_dev) > -{ > - u8 *key = snp_get_vmpck(snp_dev); > - > - dev_alert(snp_dev->dev, "Disabling vmpck_id %u to prevent IV reuse.\n", > - snp_dev->vmpck_id); > - memzero_explicit(key, VMPCK_KEY_LEN); > -} > - > -static inline u64 __snp_get_msg_seqno(struct snp_guest_dev *snp_dev) > -{ > - u32 *os_area_msg_seqno = snp_get_os_area_msg_seqno(snp_dev); > - u64 count; > - > - lockdep_assert_held(&snp_dev->cmd_mutex); > - > - /* Read the current message sequence counter from secrets pages */ > - count = *os_area_msg_seqno; > - > - return count + 1; > -} > - > -/* Return a non-zero on success */ > -static u64 snp_get_msg_seqno(struct snp_guest_dev *snp_dev) > -{ > - u64 count = __snp_get_msg_seqno(snp_dev); > - > - /* > - * The message sequence counter for the SNP guest request is a 64-bit > - * value but the version 2 of GHCB specification defines a 32-bit storage > - * for it. If the counter exceeds the 32-bit value then return zero. > - * The caller should check the return value, but if the caller happens to > - * not check the value and use it, then the firmware treats zero as an > - * invalid number and will fail the message request. > - */ > - if (count >= UINT_MAX) { > - dev_err(snp_dev->dev, "request message sequence counter overflow\n"); > - return 0; > - } > - > - return count; > -} > - > -static void snp_inc_msg_seqno(struct snp_guest_dev *snp_dev) > -{ > - u32 *os_area_msg_seqno = snp_get_os_area_msg_seqno(snp_dev); > - > - /* > - * The counter is also incremented by the PSP, so increment it by 2 > - * and save in secrets page. > - */ > - *os_area_msg_seqno += 2; > -} > - > static inline struct snp_guest_dev *to_snp_dev(struct file *file) > { > struct miscdevice *dev = file->private_data; > @@ -162,241 +42,6 @@ static inline struct snp_guest_dev *to_snp_dev(struct file *file) > return container_of(dev, struct snp_guest_dev, misc); > } > > -static struct aesgcm_ctx *snp_init_crypto(struct snp_guest_dev *snp_dev) > -{ > - struct aesgcm_ctx *ctx; > - u8 *key; > - > - if (snp_is_vmpck_empty(snp_dev)) { > - pr_err("VM communication key VMPCK%u is null\n", vmpck_id); > - return NULL; > - } > - > - ctx = kzalloc(sizeof(*ctx), GFP_KERNEL_ACCOUNT); > - if (!ctx) > - return NULL; > - > - key = snp_get_vmpck(snp_dev); > - if (aesgcm_expandkey(ctx, key, VMPCK_KEY_LEN, AUTHTAG_LEN)) { > - pr_err("Crypto context initialization failed\n"); > - kfree(ctx); > - return NULL; > - } > - > - return ctx; > -} > - > -static int verify_and_dec_payload(struct snp_guest_dev *snp_dev, struct snp_guest_req *guest_req) > -{ > - struct snp_guest_msg *resp = &snp_dev->secret_response; > - struct snp_guest_msg *req = &snp_dev->secret_request; > - struct snp_guest_msg_hdr *req_hdr = &req->hdr; > - struct snp_guest_msg_hdr *resp_hdr = &resp->hdr; > - struct aesgcm_ctx *ctx = snp_dev->ctx; > - u8 iv[GCM_AES_IV_SIZE] = {}; > - > - pr_debug("response [seqno %lld type %d version %d sz %d]\n", > - resp_hdr->msg_seqno, resp_hdr->msg_type, resp_hdr->msg_version, > - resp_hdr->msg_sz); > - > - /* Copy response from shared memory to encrypted memory. */ > - memcpy(resp, snp_dev->response, sizeof(*resp)); > - > - /* Verify that the sequence counter is incremented by 1 */ > - if (unlikely(resp_hdr->msg_seqno != (req_hdr->msg_seqno + 1))) > - return -EBADMSG; > - > - /* Verify response message type and version number. */ > - if (resp_hdr->msg_type != (req_hdr->msg_type + 1) || > - resp_hdr->msg_version != req_hdr->msg_version) > - return -EBADMSG; > - > - /* > - * If the message size is greater than our buffer length then return > - * an error. > - */ > - if (unlikely((resp_hdr->msg_sz + ctx->authsize) > guest_req->resp_sz)) > - return -EBADMSG; > - > - /* Decrypt the payload */ > - memcpy(iv, &resp_hdr->msg_seqno, sizeof(resp_hdr->msg_seqno)); > - if (!aesgcm_decrypt(ctx, guest_req->resp_buf, resp->payload, resp_hdr->msg_sz, > - &resp_hdr->algo, AAD_LEN, iv, resp_hdr->authtag)) > - return -EBADMSG; > - > - return 0; > -} > - > -static int enc_payload(struct snp_guest_dev *snp_dev, u64 seqno, struct snp_guest_req *req) > -{ > - struct snp_guest_msg *msg = &snp_dev->secret_request; > - struct snp_guest_msg_hdr *hdr = &msg->hdr; > - struct aesgcm_ctx *ctx = snp_dev->ctx; > - u8 iv[GCM_AES_IV_SIZE] = {}; > - > - memset(msg, 0, sizeof(*msg)); > - > - hdr->algo = SNP_AEAD_AES_256_GCM; > - hdr->hdr_version = MSG_HDR_VER; > - hdr->hdr_sz = sizeof(*hdr); > - hdr->msg_type = req->msg_type; > - hdr->msg_version = req->msg_version; > - hdr->msg_seqno = seqno; > - hdr->msg_vmpck = req->vmpck_id; > - hdr->msg_sz = req->req_sz; > - > - /* Verify the sequence number is non-zero */ > - if (!hdr->msg_seqno) > - return -ENOSR; > - > - pr_debug("request [seqno %lld type %d version %d sz %d]\n", > - hdr->msg_seqno, hdr->msg_type, hdr->msg_version, hdr->msg_sz); > - > - if (WARN_ON((req->req_sz + ctx->authsize) > sizeof(msg->payload))) > - return -EBADMSG; > - > - memcpy(iv, &hdr->msg_seqno, sizeof(hdr->msg_seqno)); > - aesgcm_encrypt(ctx, msg->payload, req->req_buf, req->req_sz, &hdr->algo, > - AAD_LEN, iv, hdr->authtag); > - > - return 0; > -} > - > -static int __handle_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, > - struct snp_guest_request_ioctl *rio) > -{ > - unsigned long req_start = jiffies; > - unsigned int override_npages = 0; > - u64 override_err = 0; > - int rc; > - > -retry_request: > - /* > - * Call firmware to process the request. In this function the encrypted > - * message enters shared memory with the host. So after this call the > - * sequence number must be incremented or the VMPCK must be deleted to > - * prevent reuse of the IV. > - */ > - rc = snp_issue_guest_request(req, &snp_dev->input, rio); > - switch (rc) { > - case -ENOSPC: > - /* > - * If the extended guest request fails due to having too > - * small of a certificate data buffer, retry the same > - * guest request without the extended data request in > - * order to increment the sequence number and thus avoid > - * IV reuse. > - */ > - override_npages = req->data_npages; > - req->exit_code = SVM_VMGEXIT_GUEST_REQUEST; > - > - /* > - * Override the error to inform callers the given extended > - * request buffer size was too small and give the caller the > - * required buffer size. > - */ > - override_err = SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN); > - > - /* > - * If this call to the firmware succeeds, the sequence number can > - * be incremented allowing for continued use of the VMPCK. If > - * there is an error reflected in the return value, this value > - * is checked further down and the result will be the deletion > - * of the VMPCK and the error code being propagated back to the > - * user as an ioctl() return code. > - */ > - goto retry_request; > - > - /* > - * The host may return SNP_GUEST_VMM_ERR_BUSY if the request has been > - * throttled. Retry in the driver to avoid returning and reusing the > - * message sequence number on a different message. > - */ > - case -EAGAIN: > - if (jiffies - req_start > SNP_REQ_MAX_RETRY_DURATION) { > - rc = -ETIMEDOUT; > - break; > - } > - schedule_timeout_killable(SNP_REQ_RETRY_DELAY); > - goto retry_request; > - } > - > - /* > - * Increment the message sequence number. There is no harm in doing > - * this now because decryption uses the value stored in the response > - * structure and any failure will wipe the VMPCK, preventing further > - * use anyway. > - */ > - snp_inc_msg_seqno(snp_dev); > - > - if (override_err) { > - rio->exitinfo2 = override_err; > - > - /* > - * If an extended guest request was issued and the supplied certificate > - * buffer was not large enough, a standard guest request was issued to > - * prevent IV reuse. If the standard request was successful, return -EIO > - * back to the caller as would have originally been returned. > - */ > - if (!rc && override_err == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) > - rc = -EIO; > - } > - > - if (override_npages) > - req->data_npages = override_npages; > - > - return rc; > -} > - > -static int snp_send_guest_request(struct snp_guest_dev *snp_dev, struct snp_guest_req *req, > - struct snp_guest_request_ioctl *rio) > -{ > - u64 seqno; > - int rc; > - > - /* Get message sequence and verify that its a non-zero */ > - seqno = snp_get_msg_seqno(snp_dev); > - if (!seqno) > - return -EIO; > - > - /* Clear shared memory's response for the host to populate. */ > - memset(snp_dev->response, 0, sizeof(struct snp_guest_msg)); > - > - /* Encrypt the userspace provided payload in snp_dev->secret_request. */ > - rc = enc_payload(snp_dev, seqno, req); > - if (rc) > - return rc; > - > - /* > - * Write the fully encrypted request to the shared unencrypted > - * request page. > - */ > - memcpy(snp_dev->request, &snp_dev->secret_request, > - sizeof(snp_dev->secret_request)); > - > - rc = __handle_guest_request(snp_dev, req, rio); > - if (rc) { > - if (rc == -EIO && > - rio->exitinfo2 == SNP_GUEST_VMM_ERR(SNP_GUEST_VMM_ERR_INVALID_LEN)) > - return rc; > - > - dev_alert(snp_dev->dev, > - "Detected error from ASP request. rc: %d, exitinfo2: 0x%llx\n", > - rc, rio->exitinfo2); > - snp_disable_vmpck(snp_dev); > - return rc; > - } > - > - rc = verify_and_dec_payload(snp_dev, req); > - if (rc) { > - dev_alert(snp_dev->dev, "Detected unexpected decode failure from ASP. rc: %d\n", rc); > - snp_disable_vmpck(snp_dev); > - return rc; > - } > - > - return 0; > -} > - > struct snp_req_resp { > sockptr_t req_data; > sockptr_t resp_data; > @@ -607,7 +252,7 @@ static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long > mutex_lock(&snp_dev->cmd_mutex); > > /* Check if the VMPCK is not empty */ > - if (snp_is_vmpck_empty(snp_dev)) { > + if (snp_is_vmpck_empty(snp_dev->vmpck_id)) { > dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n"); > mutex_unlock(&snp_dev->cmd_mutex); > return -ENOTTY; > @@ -642,58 +287,11 @@ static long snp_guest_ioctl(struct file *file, unsigned int ioctl, unsigned long > return ret; > } > > -static void free_shared_pages(void *buf, size_t sz) > -{ > - unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; > - int ret; > - > - if (!buf) > - return; > - > - ret = set_memory_encrypted((unsigned long)buf, npages); > - if (ret) { > - WARN_ONCE(ret, "failed to restore encryption mask (leak it)\n"); > - return; > - } > - > - __free_pages(virt_to_page(buf), get_order(sz)); > -} > - > -static void *alloc_shared_pages(struct device *dev, size_t sz) > -{ > - unsigned int npages = PAGE_ALIGN(sz) >> PAGE_SHIFT; > - struct page *page; > - int ret; > - > - page = alloc_pages(GFP_KERNEL_ACCOUNT, get_order(sz)); > - if (!page) > - return NULL; > - > - ret = set_memory_decrypted((unsigned long)page_address(page), npages); > - if (ret) { > - dev_err(dev, "failed to mark page shared, ret=%d\n", ret); > - __free_pages(page, get_order(sz)); > - return NULL; > - } > - > - return page_address(page); > -} > - > static const struct file_operations snp_guest_fops = { > .owner = THIS_MODULE, > .unlocked_ioctl = snp_guest_ioctl, > }; > > -bool snp_assign_vmpck(struct snp_guest_dev *dev, unsigned int vmpck_id) > -{ > - if (WARN_ON(vmpck_id > 3)) > - return false; > - > - dev->vmpck_id = vmpck_id; > - > - return true; > -} > - > struct snp_msg_report_resp_hdr { > u32 status; > u32 report_size; > @@ -727,7 +325,7 @@ static int sev_report_new(struct tsm_report *report, void *data) > guard(mutex)(&snp_dev->cmd_mutex); > > /* Check if the VMPCK is not empty */ > - if (snp_is_vmpck_empty(snp_dev)) { > + if (snp_is_vmpck_empty(snp_dev->vmpck_id)) { > dev_err_ratelimited(snp_dev->dev, "VMPCK is disabled\n"); > return -ENOTTY; > } > @@ -820,76 +418,43 @@ static void unregister_sev_tsm(void *data) > > static int __init sev_guest_probe(struct platform_device *pdev) > { > - struct snp_secrets_page_layout *layout; > - struct sev_guest_platform_data *data; > struct device *dev = &pdev->dev; > struct snp_guest_dev *snp_dev; > struct miscdevice *misc; > - void __iomem *mapping; > int ret; > > if (!cc_platform_has(CC_ATTR_GUEST_SEV_SNP)) > return -ENODEV; > > - if (!dev->platform_data) > - return -ENODEV; > - > - data = (struct sev_guest_platform_data *)dev->platform_data; > - mapping = ioremap_encrypted(data->secrets_gpa, PAGE_SIZE); > - if (!mapping) > - return -ENODEV; > - > - layout = (__force void *)mapping; > - > - ret = -ENOMEM; > snp_dev = devm_kzalloc(&pdev->dev, sizeof(struct snp_guest_dev), GFP_KERNEL); > if (!snp_dev) > - goto e_unmap; > + return -ENOMEM; > > - ret = -EINVAL; > - snp_dev->layout = layout; > if (!snp_assign_vmpck(snp_dev, vmpck_id)) { > dev_err(dev, "invalid vmpck id %u\n", vmpck_id); > - goto e_unmap; > + ret = -EINVAL; > + goto e_free_snpdev; > } > > - /* Verify that VMPCK is not zero. */ > - if (snp_is_vmpck_empty(snp_dev)) { > - dev_err(dev, "vmpck id %u is null\n", vmpck_id); > - goto e_unmap; > + if (snp_setup_psp_messaging(snp_dev)) { > + dev_err(dev, "Unable to setup PSP messaging vmpck id %u\n", snp_dev->vmpck_id); > + ret = -ENODEV; > + goto e_free_snpdev; > } > > mutex_init(&snp_dev->cmd_mutex); > platform_set_drvdata(pdev, snp_dev); > snp_dev->dev = dev; > > - /* Allocate the shared page used for the request and response message. */ > - snp_dev->request = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); > - if (!snp_dev->request) > - goto e_unmap; > - > - snp_dev->response = alloc_shared_pages(dev, sizeof(struct snp_guest_msg)); > - if (!snp_dev->response) > - goto e_free_request; > - > - snp_dev->certs_data = alloc_shared_pages(dev, SEV_FW_BLOB_MAX_SIZE); > + snp_dev->certs_data = alloc_shared_pages(SEV_FW_BLOB_MAX_SIZE); > if (!snp_dev->certs_data) > - goto e_free_response; > - > - ret = -EIO; > - snp_dev->ctx = snp_init_crypto(snp_dev); > - if (!snp_dev->ctx) > - goto e_free_cert_data; > + goto e_free_ctx; > > misc = &snp_dev->misc; > misc->minor = MISC_DYNAMIC_MINOR; > misc->name = DEVICE_NAME; > misc->fops = &snp_guest_fops; > > - /* initial the input address for guest request */ > - snp_dev->input.req_gpa = __pa(snp_dev->request); > - snp_dev->input.resp_gpa = __pa(snp_dev->response); > - > ret = tsm_register(&sev_tsm_ops, snp_dev, &tsm_report_extra_type); > if (ret) > goto e_free_cert_data; > @@ -900,21 +465,18 @@ static int __init sev_guest_probe(struct platform_device *pdev) > > ret = misc_register(misc); > if (ret) > - goto e_free_ctx; > + goto e_free_cert_data; > + > + dev_info(dev, "Initialized SEV guest driver (using vmpck_id %u)\n", snp_dev->vmpck_id); > > - dev_info(dev, "Initialized SEV guest driver (using vmpck_id %u)\n", vmpck_id); > return 0; > > -e_free_ctx: > - kfree(snp_dev->ctx); > e_free_cert_data: > free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); > -e_free_response: > - free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg)); > -e_free_request: > - free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg)); > -e_unmap: > - iounmap(mapping); > +e_free_ctx: > + kfree(snp_dev->ctx); > +e_free_snpdev: > + kfree(snp_dev); > return ret; > } > > @@ -923,10 +485,9 @@ static int __exit sev_guest_remove(struct platform_device *pdev) > struct snp_guest_dev *snp_dev = platform_get_drvdata(pdev); > > free_shared_pages(snp_dev->certs_data, SEV_FW_BLOB_MAX_SIZE); > - free_shared_pages(snp_dev->response, sizeof(struct snp_guest_msg)); > - free_shared_pages(snp_dev->request, sizeof(struct snp_guest_msg)); > - kfree(snp_dev->ctx); > misc_deregister(&snp_dev->misc); > + kfree(snp_dev->ctx); > + kfree(snp_dev); > > return 0; > } > -- > 2.34.1 > -- -Dionna Glaze, PhD (she/her)