Intel Software Guard eXtensions (SGX) is a set of CPU instructions that
can be used by applications to set aside private regions of code and
data. The code outside the enclave is disallowed to access the memory
inside the enclave by the CPU access control.
SGX driver provides a ioctl API for loading and initializing enclaves.
Address range for enclaves is reserved with mmap() and they are
destroyed with munmap(). Enclave construction, measurement and
initialization is done with the provided the ioctl API.
Signed-off-by: Jarkko Sakkinen <jarkko.sakkinen@xxxxxxxxxxxxxxx>
Co-developed-by: Sean Christopherson <sean.j.christopherson@xxxxxxxxx>
Co-developed-by: Serge Ayoun <serge.ayoun@xxxxxxxxx>
Co-developed-by: Shay Katz-zamir <shay.katz-zamir@xxxxxxxxx>
Co-developed-by: Suresh Siddha <suresh.b.siddha@xxxxxxxxx>
Signed-off-by: Sean Christopherson <sean.j.christopherson@xxxxxxxxx>
Signed-off-by: Serge Ayoun <serge.ayoun@xxxxxxxxx>
Signed-off-by: Shay Katz-zamir <shay.katz-zamir@xxxxxxxxx>
Signed-off-by: Suresh Siddha <suresh.b.siddha@xxxxxxxxx>
---
arch/x86/include/uapi/asm/sgx.h | 112 ++
drivers/platform/x86/Kconfig | 2 +
drivers/platform/x86/Makefile | 1 +
drivers/platform/x86/intel_sgx/Kconfig | 22 +
drivers/platform/x86/intel_sgx/Makefile | 13 +
drivers/platform/x86/intel_sgx/sgx.h | 211 ++++
drivers/platform/x86/intel_sgx/sgx_encl.c | 1016 +++++++++++++++++
.../platform/x86/intel_sgx/sgx_encl_page.c | 288 +++++
drivers/platform/x86/intel_sgx/sgx_fault.c | 157 +++
drivers/platform/x86/intel_sgx/sgx_ioctl.c | 234 ++++
drivers/platform/x86/intel_sgx/sgx_main.c | 257 +++++
drivers/platform/x86/intel_sgx/sgx_vma.c | 58 +
12 files changed, 2371 insertions(+)
create mode 100644 arch/x86/include/uapi/asm/sgx.h
create mode 100644 drivers/platform/x86/intel_sgx/Kconfig
create mode 100644 drivers/platform/x86/intel_sgx/Makefile
create mode 100644 drivers/platform/x86/intel_sgx/sgx.h
create mode 100644 drivers/platform/x86/intel_sgx/sgx_encl.c
create mode 100644 drivers/platform/x86/intel_sgx/sgx_encl_page.c
create mode 100644 drivers/platform/x86/intel_sgx/sgx_fault.c
create mode 100644 drivers/platform/x86/intel_sgx/sgx_ioctl.c
create mode 100644 drivers/platform/x86/intel_sgx/sgx_main.c
create mode 100644 drivers/platform/x86/intel_sgx/sgx_vma.c
diff --git a/arch/x86/include/uapi/asm/sgx.h b/arch/x86/include/uapi/asm/sgx.h
new file mode 100644
index 000000000000..498daac0d756
--- /dev/null
+++ b/arch/x86/include/uapi/asm/sgx.h
@@ -0,0 +1,112 @@
+/*
+ * This file is provided under a dual BSD/GPLv2 license. When using or
+ * redistributing this file, you may do so under either license.
+ *
+ * GPL LICENSE SUMMARY
+ *
+ * Copyright(c) 2016-2017 Intel Corporation.
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of version 2 of the GNU General Public License as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but
+ * WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * General Public License for more details.
+ *
+ * Contact Information:
+ * Jarkko Sakkinen <jarkko.sakkinen@xxxxxxxxxxxxxxx>
+ * Intel Finland Oy - BIC 0357606-4 - Westendinkatu 7, 02160 Espoo
+ *
+ * BSD LICENSE
+ *
+ * Copyright(c) 2016-2017 Intel Corporation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * * Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in
+ * the documentation and/or other materials provided with the
+ * distribution.
+ * * Neither the name of Intel Corporation nor the names of its
+ * contributors may be used to endorse or promote products derived
+ * from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+ * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+ * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+ * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+ * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * Authors:
+ *
+ * Jarkko Sakkinen <jarkko.sakkinen@xxxxxxxxxxxxxxx>
+ * Suresh Siddha <suresh.b.siddha@xxxxxxxxx>
+ */
+
+#ifndef _UAPI_ASM_X86_SGX_H
+#define _UAPI_ASM_X86_SGX_H
+
+#include <linux/types.h>
+#include <linux/ioctl.h>
+
+#define SGX_MAGIC 0xA4
+
+#define SGX_IOC_ENCLAVE_CREATE \
+ _IOW(SGX_MAGIC, 0x00, struct sgx_enclave_create)
+#define SGX_IOC_ENCLAVE_ADD_PAGE \
+ _IOW(SGX_MAGIC, 0x01, struct sgx_enclave_add_page)
+#define SGX_IOC_ENCLAVE_INIT \
+ _IOW(SGX_MAGIC, 0x02, struct sgx_enclave_init)
+
+/* IOCTL return values */
+#define SGX_POWER_LOST_ENCLAVE 0x40000000
+
+/**
+ * struct sgx_enclave_create - parameter structure for the
+ * %SGX_IOC_ENCLAVE_CREATE ioctl
+ * @src: address for the SECS page data
+ */
+struct sgx_enclave_create {
+ __u64 src;
+};
+
+/**
+ * struct sgx_enclave_add_page - parameter structure for the
+ * %SGX_IOC_ENCLAVE_ADD_PAGE ioctl
+ * @addr: address within the ELRANGE
+ * @src: address for the page data
+ * @secinfo: address for the SECINFO data
+ * @mrmask: bitmask for the measured 256 byte chunks
+ */
+struct sgx_enclave_add_page {
+ __u64 addr;
+ __u64 src;
+ __u64 secinfo;
+ __u16 mrmask;
+} __attribute__((__packed__));
+
+
+/**
+ * struct sgx_enclave_init - parameter structure for the
+ * %SGX_IOC_ENCLAVE_INIT ioctl
+ * @addr: address within the ELRANGE
+ * @sigstruct: address for the SIGSTRUCT data
+ */
+struct sgx_enclave_init {
+ __u64 addr;
+ __u64 sigstruct;
+};
+
+#endif /* _UAPI_ASM_X86_SGX_H */
diff --git a/drivers/platform/x86/Kconfig b/drivers/platform/x86/Kconfig
index 107d336453b2..d5954badbc7c 100644
--- a/drivers/platform/x86/Kconfig
+++ b/drivers/platform/x86/Kconfig
@@ -1229,6 +1229,8 @@ config I2C_MULTI_INSTANTIATE
To compile this driver as a module, choose M here: the module
will be called i2c-multi-instantiate.
+source "drivers/platform/x86/intel_sgx/Kconfig"
+
endif # X86_PLATFORM_DEVICES
config PMC_ATOM
diff --git a/drivers/platform/x86/Makefile b/drivers/platform/x86/Makefile
index 50dc8f280914..f4c2c6734767 100644
--- a/drivers/platform/x86/Makefile
+++ b/drivers/platform/x86/Makefile
@@ -92,3 +92,4 @@ obj-$(CONFIG_MLX_PLATFORM) += mlx-platform.o
obj-$(CONFIG_INTEL_TURBO_MAX_3) += intel_turbo_max_3.o
obj-$(CONFIG_INTEL_CHTDC_TI_PWRBTN) += intel_chtdc_ti_pwrbtn.o
obj-$(CONFIG_I2C_MULTI_INSTANTIATE) += i2c-multi-instantiate.o
+obj-$(CONFIG_INTEL_SGX) += intel_sgx/
diff --git a/drivers/platform/x86/intel_sgx/Kconfig b/drivers/platform/x86/intel_sgx/Kconfig
new file mode 100644
index 000000000000..8e6c57c28443
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/Kconfig
@@ -0,0 +1,22 @@
+#
+# Intel SGX
+#
+
+config INTEL_SGX
+ tristate "Intel(R) SGX Driver"
+ default n
+ depends on X86_64 && CPU_SUP_INTEL
+ select INTEL_SGX_CORE
+ select MMU_NOTIFIER
+ select CRYPTO
+ select CRYPTO_SHA256
+ help
+ Intel(R) SGX is a set of CPU instructions that can be used by
+ applications to set aside private regions of code and data. The code
+ outside the enclave is disallowed to access the memory inside the
+ enclave by the CPU access control.
+
+ The firmware uses PRMRR registers to reserve an area of physical memory
+ called Enclave Page Cache (EPC). There is a hardware unit in the
+ processor called Memory Encryption Engine. The MEE encrypts and decrypts
+ the EPC pages as they enter and leave the processor package.
diff --git a/drivers/platform/x86/intel_sgx/Makefile b/drivers/platform/x86/intel_sgx/Makefile
new file mode 100644
index 000000000000..95f254e30a8b
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/Makefile
@@ -0,0 +1,13 @@
+#
+# Intel SGX
+#
+
+obj-$(CONFIG_INTEL_SGX) += intel_sgx.o
+
+intel_sgx-$(CONFIG_INTEL_SGX) += \
+ sgx_ioctl.o \
+ sgx_encl.o \
+ sgx_encl_page.o \
+ sgx_main.o \
+ sgx_fault.o \
+ sgx_vma.o \
diff --git a/drivers/platform/x86/intel_sgx/sgx.h b/drivers/platform/x86/intel_sgx/sgx.h
new file mode 100644
index 000000000000..2323469cf081
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx.h
@@ -0,0 +1,211 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#ifndef __ARCH_INTEL_SGX_H__
+#define __ARCH_INTEL_SGX_H__
+
+#include <crypto/hash.h>
+#include <linux/kref.h>
+#include <linux/mmu_notifier.h>
+#include <linux/mmu_notifier.h>
+#include <linux/radix-tree.h>
+#include <linux/radix-tree.h>
+#include <linux/rbtree.h>
+#include <linux/rwsem.h>
+#include <linux/sched.h>
+#include <linux/workqueue.h>
+#include <asm/sgx.h>
+#include <asm/sgx_pr.h>
+#include <uapi/asm/sgx.h>
+
+#define sgx_pr_ratelimited(level, encl, fmt, ...) \
+ pr_ ## level ## _ratelimited("[%d:0x%p] " fmt, \
+ pid_nr((encl)->tgid), \
+ (void *)(encl)->base, ##__VA_ARGS__)
+#define sgx_dbg(encl, fmt, ...) \
+ sgx_pr_ratelimited(debug, encl, fmt, ##__VA_ARGS__)
+#define sgx_info(encl, fmt, ...) \
+ sgx_pr_ratelimited(info, encl, fmt, ##__VA_ARGS__)
+#define sgx_warn(encl, fmt, ...) \
+ sgx_pr_ratelimited(warn, encl, fmt, ##__VA_ARGS__)
+#define sgx_err(encl, fmt, ...) \
+ sgx_pr_ratelimited(err, encl, fmt, ##__VA_ARGS__)
+#define sgx_crit(encl, fmt, ...) \
+ sgx_pr_ratelimited(crit, encl, fmt, ##__VA_ARGS__)
+
+#define SGX_EINIT_SPIN_COUNT 20
+#define SGX_EINIT_SLEEP_COUNT 50
+#define SGX_EINIT_SLEEP_TIME 20
+
+#define SGX_VA_SLOT_COUNT 512
+
+struct sgx_va_page {
+ struct sgx_epc_page *epc_page;
+ DECLARE_BITMAP(slots, SGX_VA_SLOT_COUNT);
+ struct list_head list;
+};
+
+/**
+ * enum sgx_encl_page_desc - defines bits and masks for an enclave page's desc
+ *
+ * @SGX_ENCL_PAGE_TCS:
+ * @SGX_ENCL_PAGE_LOADED:
+ * @SGX_ENCL_PAGE_RESERVED: Set when we need to temporarily prevent reclaim,
+ * e.g. the page is being directly accessed for debug
+ * purposes.
+ * @SGX_ENCL_PAGE_RECLAIMED: Set when a LOADED page is in the process of being
+ * reclaimed by the EPC manager. Once RECLAIMED is
+ * set we no longer "own" the EPC page; we're still
+ * involved in evicting the page, but we cannot free
+ * the EPC page or rely on its contents in any way.
+ *
+ * @SGX_ENCL_PAGE_VA_OFFSET_MASK: Holds the offset into the VA page that was
+ * used to evict the page.
+ * @SGX_ENCL_PAGE_ADDR_MASK: Holds the userspace virtual address of the
+ * page. Primarily used to manipulate PTEs and
+ * retrieve an enclave from a given page.
+ *
+ * enum sgx_encl_page_desc defines the layout of struct sgx_encl_page's @desc.
+ * The metadata for an enclave page is compressed into a single variable to
+ * reduce memory consumption as the size of enclaves are effectively unbounded,
+ * e.g. a userspace process can create a 512gb enclave regardless of the actual
+ * amount of EPC in the system.
+ *
+ * WARNING: Bits 11:3 are effectively a union, similar to how a union is used
+ * to store either a pointer to an EPC page or VA page depending on whether or
+ * not a struct sgx_encl_page is resident in the EPC. When the page is evicted
+ * from the EPC, bits 11:3 are used to hold the VA offset. Flags that may be
+ * set/cleared at any time must not reside in bits 11:3.
+ */
+enum sgx_encl_page_desc {
+ SGX_ENCL_PAGE_TCS = BIT(0),
+ SGX_ENCL_PAGE_LOADED = BIT(1),
+ /* Bit 2 is free, may be used at any time */
+
+ SGX_ENCL_PAGE_RESERVED = BIT(3),
+ SGX_ENCL_PAGE_RECLAIMED = BIT(4),
+ /* Bits 11:5 are free, may only be used when page is resident in EPC */
+
+ SGX_ENCL_PAGE_VA_OFFSET_MASK = GENMASK_ULL(11, 3),
+
+ SGX_ENCL_PAGE_ADDR_MASK = PAGE_MASK,
+};
+
+#define SGX_ENCL_PAGE_ADDR(encl_page) \
+ ((encl_page)->desc & SGX_ENCL_PAGE_ADDR_MASK)
+#define SGX_ENCL_PAGE_VA_OFFSET(encl_page) \
+ ((encl_page)->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK)
+#define SGX_ENCL_PAGE_BACKING_INDEX(encl_page, encl) \
+({ \
+ pgoff_t index; \
+ if (!PFN_DOWN(encl_page->desc)) \
+ index = PFN_DOWN(encl->size); \
+ else \
+ index = PFN_DOWN(encl_page->desc - encl->base); \
+ index; \
+})
+#define SGX_ENCL_PAGE_PCMD_OFFSET(encl_page, encl) \
+({ \
+ unsigned long ret; \
+ ret = SGX_ENCL_PAGE_BACKING_INDEX(encl_page, encl); \
+ ((ret & 31) * 128); \
+})
+
+struct sgx_encl_page {
+ unsigned long desc;
+ union {
+ struct sgx_epc_page *epc_page;
+ struct sgx_va_page *va_page;
+ };
+ struct sgx_encl *encl;
+ struct sgx_epc_page_impl impl;
+};
+
+enum sgx_encl_flags {
+ SGX_ENCL_INITIALIZED = BIT(0),
+ SGX_ENCL_DEBUG = BIT(1),
+ SGX_ENCL_SUSPEND = BIT(2),
+ SGX_ENCL_DEAD = BIT(3),
+};
+
+struct sgx_encl {
+ unsigned int flags;
+ uint64_t attributes;
+ uint64_t xfrm;
+ unsigned int page_cnt;
+ unsigned int secs_child_cnt;
+ struct mutex lock;
+ struct mm_struct *mm;
+ struct file *backing;
+ struct file *pcmd;
+ struct kref refcount;
+ unsigned long base;
+ unsigned long size;
+ unsigned long ssaframesize;
+ struct list_head va_pages;
+ struct radix_tree_root page_tree;
+ struct list_head add_page_reqs;
+ struct work_struct add_page_work;
+ struct sgx_encl_page secs;
+ struct pid *tgid;
+ struct mmu_notifier mmu_notifier;
+ struct notifier_block pm_notifier;
+};
+
+extern struct workqueue_struct *sgx_add_page_wq;
+extern u64 sgx_encl_size_max_32;
+extern u64 sgx_encl_size_max_64;
+extern u64 sgx_xfrm_mask;
+extern u32 sgx_misc_reserved;
+extern u32 sgx_xsave_size_tbl[64];
+
+extern const struct vm_operations_struct sgx_vm_ops;
+
+int sgx_encl_find(struct mm_struct *mm, unsigned long addr,
+ struct vm_area_struct **vma);
+void sgx_invalidate(struct sgx_encl *encl, bool flush_cpus);
+#define SGX_INVD(ret, encl, fmt, ...) \
+do { \
+ if (WARN(ret, "sgx: " fmt, ##__VA_ARGS__)) \
+ sgx_invalidate(encl, true); \
+} while (0)
+
+struct sgx_encl *sgx_encl_alloc(struct sgx_secs *secs);
+int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs);
+struct sgx_encl_page *sgx_encl_alloc_page(struct sgx_encl *encl,
+ unsigned long addr);
+void sgx_encl_free_page(struct sgx_encl_page *encl_page);
+int sgx_encl_add_page(struct sgx_encl *encl, unsigned long addr, void *data,
+ struct sgx_secinfo *secinfo, unsigned int mrmask);
+int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct,
+ struct sgx_einittoken *einittoken);
+void sgx_encl_block(struct sgx_encl_page *encl_page);
+void sgx_encl_track(struct sgx_encl *encl);
+int sgx_encl_load_page(struct sgx_encl_page *encl_page,
+ struct sgx_epc_page *epc_page);
+void sgx_encl_release(struct kref *ref);
+
+long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
+#ifdef CONFIG_COMPAT
+long sgx_compat_ioctl(struct file *filep, unsigned int cmd, unsigned long arg);
+#endif
+
+/* Utility functions */
+int sgx_test_and_clear_young(struct sgx_encl_page *page);
+void sgx_flush_cpus(struct sgx_encl *encl);
+
+struct sgx_encl_page *sgx_fault_page(struct vm_area_struct *vma,
+ unsigned long addr,
+ bool do_reserve);
+
+extern const struct sgx_epc_page_ops sgx_encl_page_ops;
+
+void sgx_set_epc_page(struct sgx_encl_page *encl_page,
+ struct sgx_epc_page *epc_page);
+void sgx_set_page_reclaimable(struct sgx_encl_page *encl_page);
+struct sgx_epc_page *sgx_alloc_va_page(unsigned int flags);
+unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page);
+void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset);
+bool sgx_va_page_full(struct sgx_va_page *va_page);
+
+#endif /* __ARCH_X86_INTEL_SGX_H__ */
diff --git a/drivers/platform/x86/intel_sgx/sgx_encl.c b/drivers/platform/x86/intel_sgx/sgx_encl.c
new file mode 100644
index 000000000000..c1a0065c4710
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx_encl.c
@@ -0,0 +1,1016 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#include <asm/mman.h>
+#include <linux/delay.h>
+#include <linux/file.h>
+#include <linux/hashtable.h>
+#include <linux/highmem.h>
+#include <linux/ratelimit.h>
+#include <linux/sched/signal.h>
+#include <linux/shmem_fs.h>
+#include <linux/slab.h>
+#include <linux/suspend.h>
+#include "sgx.h"
+
+struct sgx_add_page_req {
+ struct sgx_encl *encl;
+ struct sgx_encl_page *encl_page;
+ struct sgx_secinfo secinfo;
+ u16 mrmask;
+ struct list_head list;
+};
+
+/**
+ * sgx_encl_find - find an enclave
+ * @mm: mm struct of the current process
+ * @addr: address in the ELRANGE
+ * @vma: the resulting VMA
+ *
+ * Finds an enclave identified by the given address. Gives back the VMA, that
+ * is part of the enclave, located in that address. The VMA is given back if it
+ * is a proper enclave VMA even if an &sgx_encl instance does not exist
+ * yet (enclave creation has not been performed).
+ *
+ * Return:
+ * 0 on success,
+ * -EINVAL if an enclave was not found,
+ * -ENOENT if the enclave has not been created yet
+ */
+int sgx_encl_find(struct mm_struct *mm, unsigned long addr,
+ struct vm_area_struct **vma)
+{
+ struct vm_area_struct *result;
+ struct sgx_encl *encl;
+
+ result = find_vma(mm, addr);
+ if (!result || result->vm_ops != &sgx_vm_ops || addr < result->vm_start)
+ return -EINVAL;
+
+ encl = result->vm_private_data;
+ *vma = result;
+
+ return encl ? 0 : -ENOENT;
+}
+
+/**
+ * sgx_invalidate - kill an enclave
+ * @encl: an &sgx_encl instance
+ * @flush_cpus Set if there can be active threads inside the enclave.
+ *
+ * Mark the enclave as dead and immediately free its EPC pages (but not
+ * its resources). For active enclaves, the entry points to the enclave
+ * are destroyed first and hardware threads are kicked out so that the
+ * EPC pages can be safely manipulated.
+ */
+void sgx_invalidate(struct sgx_encl *encl, bool flush_cpus)
+{
+ struct sgx_encl_page *entry;
+ struct radix_tree_iter iter;
+ struct vm_area_struct *vma;
+ unsigned long addr;
+ void **slot;
+
+ if (encl->flags & SGX_ENCL_DEAD)
+ return;
+
+ encl->flags |= SGX_ENCL_DEAD;
+ if (flush_cpus) {
+ radix_tree_for_each_slot(slot, &encl->page_tree, &iter, 0) {
+ entry = *slot;
+ addr = SGX_ENCL_PAGE_ADDR(entry);
+ if ((entry->desc & SGX_ENCL_PAGE_LOADED) &&
+ (entry->desc & SGX_ENCL_PAGE_TCS) &&
+ !sgx_encl_find(encl->mm, addr, &vma))
+ zap_vma_ptes(vma, addr, PAGE_SIZE);
+ }
+ sgx_flush_cpus(encl);
+ }
+ radix_tree_for_each_slot(slot, &encl->page_tree, &iter, 0) {
+ entry = *slot;
+ /* If the page has RECLAIMED set, it is being reclaimed so we
+ * need to check that and let the swapper thread to free the
+ * page if this is the case.
+ */
+ if ((entry->desc & SGX_ENCL_PAGE_LOADED) &&
+ !(entry->desc & SGX_ENCL_PAGE_RECLAIMED)) {
+ if (!__sgx_free_page(entry->epc_page))
+ entry->desc &= ~SGX_ENCL_PAGE_LOADED;
+ }
+ }
+}
+
+static int sgx_measure(struct sgx_epc_page *secs_page,
+ struct sgx_epc_page *epc_page,
+ u16 mrmask)
+{
+ int ret = 0;
+ void *secs;
+ void *epc;
+ int i;
+ int j;
+
+ if (!mrmask)
+ return ret;
+
+ secs = sgx_epc_addr(secs_page);
+ epc = sgx_epc_addr(epc_page);
+
+ for (i = 0, j = 1; i < 0x1000 && !ret; i += 0x100, j <<= 1) {
+ if (!(j & mrmask))
+ continue;
+
+ ret = __eextend(secs, (void *)((unsigned long)epc + i));
+ }
+
+ return ret;
+}
+
+static int sgx_eadd(struct sgx_epc_page *secs_page,
+ struct sgx_epc_page *epc_page,
+ unsigned long linaddr,
+ struct sgx_secinfo *secinfo,
+ struct page *backing)
+{
+ struct sgx_pageinfo pginfo;
+ int ret;
+
+ pginfo.secs = (unsigned long)sgx_epc_addr(secs_page);
+ pginfo.addr = linaddr;
+ pginfo.metadata = (unsigned long)secinfo;
+
+ pginfo.contents = (unsigned long)kmap_atomic(backing);
+ ret = __eadd(&pginfo, sgx_epc_addr(epc_page));
+ kunmap_atomic((void *)(unsigned long)pginfo.contents);
+
+ return ret;
+}
+
+static bool sgx_process_add_page_req(struct sgx_add_page_req *req,
+ struct sgx_epc_page *epc_page)
+{
+ struct sgx_encl_page *encl_page = req->encl_page;
+ struct sgx_encl *encl = req->encl;
+ struct sgx_secinfo secinfo;
+ struct vm_area_struct *vma;
+ pgoff_t backing_index;
+ struct page *backing;
+ unsigned long addr;
+ int ret;
+
+ if (encl->flags & (SGX_ENCL_SUSPEND | SGX_ENCL_DEAD))
+ return false;
+
+ addr = SGX_ENCL_PAGE_ADDR(encl_page);
+ ret = sgx_encl_find(encl->mm, addr, &vma);
+ if (ret)
+ return false;
+
+ backing_index = SGX_ENCL_PAGE_BACKING_INDEX(encl_page, encl);
+ backing = sgx_get_backing(encl->backing, backing_index);
+ if (IS_ERR(backing))
+ return false;
+
+ ret = vm_insert_pfn(vma, addr, PFN_DOWN(epc_page->desc));
+ if (ret) {
+ sgx_err(encl, "%s: vm_insert_pfn() returned %d\n", __func__,
+ ret);
+ sgx_put_backing(backing, false);
+ return false;
+ }
+
+ /*
+ * The SECINFO field must be 64-byte aligned, copy it to a local
+ * variable that is guaranteed to be aligned as req->secinfo may
+ * or may not be 64-byte aligned, e.g. req may have been allocated
+ * via kzalloc which is not aware of __aligned attributes.
+ */
+ memcpy(&secinfo, &req->secinfo, sizeof(secinfo));
+
+ ret = sgx_eadd(encl->secs.epc_page, epc_page, addr, &secinfo, backing);
+
+ sgx_put_backing(backing, false);
+ if (ret) {
+ sgx_err(encl, "EADD returned %d\n", ret);
+ zap_vma_ptes(vma, addr, PAGE_SIZE);
+ return false;
+ }
+
+ ret = sgx_measure(encl->secs.epc_page, epc_page, req->mrmask);
+ if (ret) {
+ sgx_err(encl, "EEXTEND returned %d\n", ret);
+ zap_vma_ptes(vma, addr, PAGE_SIZE);
+ return false;
+ }
+
+ encl_page->encl = encl;
+ encl->secs_child_cnt++;
+ sgx_set_epc_page(encl_page, epc_page);
+ sgx_set_page_reclaimable(encl_page);
+ return true;
+}
+
+static void sgx_add_page_worker(struct work_struct *work)
+{
+ struct sgx_add_page_req *req;
+ bool skip_rest = false;
+ bool is_empty = false;
+ struct sgx_encl *encl;
+ struct sgx_epc_page *epc_page;
+
+ encl = container_of(work, struct sgx_encl, add_page_work);
+
+ do {
+ schedule();
+
+ mutex_lock(&encl->lock);
+ if (encl->flags & SGX_ENCL_DEAD)
+ skip_rest = true;
+
+ req = list_first_entry(&encl->add_page_reqs,
+ struct sgx_add_page_req, list);
+ list_del(&req->list);
+ is_empty = list_empty(&encl->add_page_reqs);
+ mutex_unlock(&encl->lock);
+
+ if (skip_rest)
+ goto next;
+
+ epc_page = sgx_alloc_page(&req->encl_page->impl, 0);
+ down_read(&encl->mm->mmap_sem);
+ mutex_lock(&encl->lock);
+
+ if (IS_ERR(epc_page)) {
+ sgx_invalidate(encl, false);
+ skip_rest = true;
+ } else if (!sgx_process_add_page_req(req, epc_page)) {
+ sgx_free_page(epc_page);
+ sgx_invalidate(encl, false);
+ skip_rest = true;
+ }
+
+ mutex_unlock(&encl->lock);
+ up_read(&encl->mm->mmap_sem);
+
+next:
+ kfree(req);
+ } while (!kref_put(&encl->refcount, sgx_encl_release) && !is_empty);
+}
+
+static u32 sgx_calc_ssaframesize(u32 miscselect, u64 xfrm)
+{
+ u32 size_max = PAGE_SIZE;
+ u32 size;
+ int i;
+
+ for (i = 2; i < 64; i++) {
+ if (!((1 << i) & xfrm))
+ continue;
+
+ size = SGX_SSA_GPRS_SIZE + sgx_xsave_size_tbl[i];
+ if (miscselect & SGX_MISC_EXINFO)
+ size += SGX_SSA_MISC_EXINFO_SIZE;
+
+ if (size > size_max)
+ size_max = size;
+ }
+
+ return (size_max + PAGE_SIZE - 1) >> PAGE_SHIFT;
+}
+
+static int sgx_validate_secs(const struct sgx_secs *secs,
+ unsigned long ssaframesize)
+{
+ int i;
+
+ if (secs->size < (2 * PAGE_SIZE) ||
+ (secs->size & (secs->size - 1)) != 0)
+ return -EINVAL;
+
+ if (secs->base & (secs->size - 1))
+ return -EINVAL;
+
+ if (secs->attributes & SGX_ATTR_RESERVED_MASK ||
+ secs->miscselect & sgx_misc_reserved)
+ return -EINVAL;
+
+ if (secs->attributes & SGX_ATTR_MODE64BIT) {
+ if (secs->size > sgx_encl_size_max_64)
+ return -EINVAL;
+ } else {
+ /* On 64-bit architecture allow 32-bit encls only in
+ * the compatibility mode.
+ */
+ if (!test_thread_flag(TIF_ADDR32))
+ return -EINVAL;
+ if (secs->size > sgx_encl_size_max_32)
+ return -EINVAL;
+ }
+
+ if ((secs->xfrm & 0x3) != 0x3 || (secs->xfrm & ~sgx_xfrm_mask))
+ return -EINVAL;
+
+ /* Check that BNDREGS and BNDCSR are equal. */
+ if (((secs->xfrm >> 3) & 1) != ((secs->xfrm >> 4) & 1))
+ return -EINVAL;
+
+ if (!secs->ssa_frame_size || ssaframesize > secs->ssa_frame_size)
+ return -EINVAL;
+
+ for (i = 0; i < SGX_SECS_RESERVED1_SIZE; i++)
+ if (secs->reserved1[i])
+ return -EINVAL;
+
+ for (i = 0; i < SGX_SECS_RESERVED2_SIZE; i++)
+ if (secs->reserved2[i])
+ return -EINVAL;
+
+ for (i = 0; i < SGX_SECS_RESERVED3_SIZE; i++)
+ if (secs->reserved3[i])
+ return -EINVAL;
+
+ for (i = 0; i < SGX_SECS_RESERVED4_SIZE; i++)
+ if (secs->reserved4[i])
+ return -EINVAL;
+
+ return 0;
+}
+
+static void sgx_mmu_notifier_release(struct mmu_notifier *mn,
+ struct mm_struct *mm)
+{
+ struct sgx_encl *encl =
+ container_of(mn, struct sgx_encl, mmu_notifier);
+
+ mutex_lock(&encl->lock);
+ encl->flags |= SGX_ENCL_DEAD;
+ mutex_unlock(&encl->lock);
+}
+
+static const struct mmu_notifier_ops sgx_mmu_notifier_ops = {
+ .release = sgx_mmu_notifier_release,
+};
+
+static int sgx_encl_grow(struct sgx_encl *encl)
+{
+ struct sgx_va_page *va_page;
+ int ret;
+
+ BUILD_BUG_ON(SGX_VA_SLOT_COUNT !=
+ (SGX_ENCL_PAGE_VA_OFFSET_MASK >> 3) + 1);
+
+ mutex_lock(&encl->lock);
+ if (!(encl->page_cnt % SGX_VA_SLOT_COUNT)) {
+ mutex_unlock(&encl->lock);
+
+ va_page = kzalloc(sizeof(*va_page), GFP_KERNEL);
+ if (!va_page)
+ return -ENOMEM;
+ va_page->epc_page = sgx_alloc_va_page(0);
+ if (IS_ERR(va_page->epc_page)) {
+ ret = PTR_ERR(va_page->epc_page);
+ kfree(va_page);
+ return ret;
+ }
+
+ mutex_lock(&encl->lock);
+ if (encl->page_cnt % SGX_VA_SLOT_COUNT) {
+ sgx_free_page(va_page->epc_page);
+ kfree(va_page);
+ } else {
+ list_add(&va_page->list, &encl->va_pages);
+ }
+ }
+ encl->page_cnt++;
+ mutex_unlock(&encl->lock);
+ return 0;
+}
+
+/**
+ * sgx_encl_alloc - allocate memory for an enclave and set attributes
+ *
+ * @secs: SECS data (must be page aligned)
+ *
+ * Allocates a new &sgx_encl instance. Validates SECS attributes, creates
+ * backing storage for the enclave and sets enclave attributes to sane initial
+ * values.
+ *
+ * Return:
+ * an &sgx_encl instance,
+ * -errno otherwise
+ */
+struct sgx_encl *sgx_encl_alloc(struct sgx_secs *secs)
+{
+ unsigned long ssaframesize;
+ struct sgx_encl *encl;
+ struct file *backing;
+ struct file *pcmd;
+
+ ssaframesize = sgx_calc_ssaframesize(secs->miscselect, secs->xfrm);
+ if (sgx_validate_secs(secs, ssaframesize))
+ return ERR_PTR(-EINVAL);
+
+ backing = shmem_file_setup("[dev/sgx]", secs->size + PAGE_SIZE,
+ VM_NORESERVE);
+ if (IS_ERR(backing))
+ return (void *)backing;
+
+ pcmd = shmem_file_setup("[dev/sgx]", (secs->size + PAGE_SIZE) >> 5,
+ VM_NORESERVE);
+ if (IS_ERR(pcmd)) {
+ fput(backing);
+ return (void *)pcmd;
+ }
+
+ encl = kzalloc(sizeof(*encl), GFP_KERNEL);
+ if (!encl) {
+ fput(backing);
+ fput(pcmd);
+ return ERR_PTR(-ENOMEM);
+ }
+
+ encl->attributes = secs->attributes;
+ encl->xfrm = secs->xfrm;
+
+ kref_init(&encl->refcount);
+ INIT_LIST_HEAD(&encl->add_page_reqs);
+ INIT_LIST_HEAD(&encl->va_pages);
+ INIT_RADIX_TREE(&encl->page_tree, GFP_KERNEL);
+ mutex_init(&encl->lock);
+ INIT_WORK(&encl->add_page_work, sgx_add_page_worker);
+
+ encl->mm = current->mm;
+ encl->base = secs->base;
+ encl->size = secs->size;
+ encl->ssaframesize = secs->ssa_frame_size;
+ encl->backing = backing;
+ encl->pcmd = pcmd;
+
+ return encl;
+}
+
+static int sgx_encl_pm_notifier(struct notifier_block *nb,
+ unsigned long action, void *data)
+{
+ struct sgx_encl *encl = container_of(nb, struct sgx_encl, pm_notifier);
+
+ if (action != PM_SUSPEND_PREPARE && action != PM_HIBERNATION_PREPARE)
+ return NOTIFY_DONE;
+
+ mutex_lock(&encl->lock);
+ sgx_invalidate(encl, false);
+ encl->flags |= SGX_ENCL_SUSPEND;
+ mutex_unlock(&encl->lock);
+ flush_work(&encl->add_page_work);
+ return NOTIFY_DONE;
+}
+
+/**
+ * sgx_encl_create - create an enclave
+ *
+ * @encl: an enclave
+ * @secs: page aligned SECS data
+ *
+ * Validates SECS attributes, allocates an EPC page for the SECS and creates
+ * the enclave by performing ECREATE.
+ *
+ * Return:
+ * 0 on success,
+ * -errno otherwise
+ */
+int sgx_encl_create(struct sgx_encl *encl, struct sgx_secs *secs)
+{
+ struct vm_area_struct *vma;
+ struct sgx_pageinfo pginfo;
+ struct sgx_secinfo secinfo;
+ struct sgx_epc_page *secs_epc;
+ long ret;
+
+ secs_epc = sgx_alloc_page(&encl->secs.impl, 0);
+ if (IS_ERR(secs_epc)) {
+ ret = PTR_ERR(secs_epc);
+ return ret;
+ }
+
+ sgx_set_epc_page(&encl->secs, secs_epc);
+ encl->secs.encl = encl;
+ encl->secs.impl.ops = &sgx_encl_page_ops;
+ encl->tgid = get_pid(task_tgid(current));
+
+ ret = sgx_encl_grow(encl);
+ if (ret)
+ return ret;
+
+ pginfo.addr = 0;
+ pginfo.contents = (unsigned long)secs;
+ pginfo.metadata = (unsigned long)&secinfo;
+ pginfo.secs = 0;
+ memset(&secinfo, 0, sizeof(secinfo));
+ ret = __ecreate((void *)&pginfo, sgx_epc_addr(secs_epc));
+
+ if (ret) {
+ sgx_dbg(encl, "ECREATE returned %ld\n", ret);
+ return ret;
+ }
+
+ if (secs->attributes & SGX_ATTR_DEBUG)
+ encl->flags |= SGX_ENCL_DEBUG;
+
+ encl->mmu_notifier.ops = &sgx_mmu_notifier_ops;
+ ret = mmu_notifier_register(&encl->mmu_notifier, encl->mm);
+ if (ret) {
+ if (ret == -EINTR)
+ ret = -ERESTARTSYS;
+ encl->mmu_notifier.ops = NULL;
+ return ret;
+ }
+
+ encl->pm_notifier.notifier_call = &sgx_encl_pm_notifier;
+ ret = register_pm_notifier(&encl->pm_notifier);
+ if (ret) {
+ encl->pm_notifier.notifier_call = NULL;
+ return ret;
+ }
+
+ down_read(¤t->mm->mmap_sem);
+ ret = sgx_encl_find(current->mm, secs->base, &vma);
+ if (ret != -ENOENT) {
+ if (!ret)
+ ret = -EINVAL;
+ up_read(¤t->mm->mmap_sem);
+ return ret;
+ }
+
+ if (vma->vm_start != secs->base ||
+ vma->vm_end != (secs->base + secs->size) ||
+ vma->vm_pgoff != 0) {
+ ret = -EINVAL;
+ up_read(¤t->mm->mmap_sem);
+ return ret;
+ }
+
+ vma->vm_private_data = encl;
+ up_read(¤t->mm->mmap_sem);
+ return 0;
+}
+
+static int sgx_validate_secinfo(struct sgx_secinfo *secinfo)
+{
+ u64 page_type = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK;
+ u64 perm = secinfo->flags & SGX_SECINFO_PERMISSION_MASK;
+ int i;
+
+ if ((secinfo->flags & SGX_SECINFO_RESERVED_MASK) ||
+ ((perm & SGX_SECINFO_W) && !(perm & SGX_SECINFO_R)) ||
+ (page_type != SGX_SECINFO_TCS &&
+ page_type != SGX_SECINFO_REG))
+ return -EINVAL;
+
+ for (i = 0; i < SGX_SECINFO_RESERVED_SIZE; i++)
+ if (secinfo->reserved[i])
+ return -EINVAL;
+
+ return 0;
+}
+
+static bool sgx_validate_offset(struct sgx_encl *encl, unsigned long offset)
+{
+ if (offset & (PAGE_SIZE - 1))
+ return false;
+
+ if (offset >= encl->size)
+ return false;
+
+ return true;
+}
+
+static int sgx_validate_tcs(struct sgx_encl *encl, struct sgx_tcs *tcs)
+{
+ int i;
+
+ if (tcs->flags & SGX_TCS_RESERVED_MASK)
+ return -EINVAL;
+
+ if (tcs->flags & SGX_TCS_DBGOPTIN)
+ return -EINVAL;
+
+ if (!sgx_validate_offset(encl, tcs->ssa_offset))
+ return -EINVAL;
+
+ if (!sgx_validate_offset(encl, tcs->fs_offset))
+ return -EINVAL;
+
+ if (!sgx_validate_offset(encl, tcs->gs_offset))
+ return -EINVAL;
+
+ if ((tcs->fs_limit & 0xFFF) != 0xFFF)
+ return -EINVAL;
+
+ if ((tcs->gs_limit & 0xFFF) != 0xFFF)
+ return -EINVAL;
+
+ for (i = 0; i < SGX_TCS_RESERVED_SIZE; i++)
+ if (tcs->reserved[i])
+ return -EINVAL;
+
+ return 0;
+}
+
+static int __sgx_encl_add_page(struct sgx_encl *encl,
+ struct sgx_encl_page *encl_page,
+ void *data,
+ struct sgx_secinfo *secinfo,
+ unsigned int mrmask)
+{
+ u64 page_type = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK;
+ struct sgx_add_page_req *req = NULL;
+ pgoff_t backing_index;
+ struct page *backing;
+ void *backing_ptr;
+ int empty;
+
+ req = kzalloc(sizeof(*req), GFP_KERNEL);
+ if (!req)
+ return -ENOMEM;
+
+ backing_index = SGX_ENCL_PAGE_BACKING_INDEX(encl_page, encl);
+ backing = sgx_get_backing(encl->backing, backing_index);
+ if (IS_ERR(backing)) {
+ kfree(req);
+ return PTR_ERR(backing);
+ }
+ backing_ptr = kmap(backing);
+ memcpy(backing_ptr, data, PAGE_SIZE);
+ kunmap(backing);
+ if (page_type == SGX_SECINFO_TCS)
+ encl_page->desc |= SGX_ENCL_PAGE_TCS;
+ memcpy(&req->secinfo, secinfo, sizeof(*secinfo));
+ req->encl = encl;
+ req->encl_page = encl_page;
+ req->mrmask = mrmask;
+ empty = list_empty(&encl->add_page_reqs);
+ kref_get(&encl->refcount);
+ list_add_tail(&req->list, &encl->add_page_reqs);
+ if (empty)
+ queue_work(sgx_add_page_wq, &encl->add_page_work);
+ sgx_put_backing(backing, true /* write */);
+ return 0;
+}
+
+/**
+ * sgx_encl_alloc_page - allocate a new enclave page
+ * @encl: an enclave
+ * @addr: page address in the ELRANGE
+ *
+ * Return:
+ * an &sgx_encl_page instance on success,
+ * -errno otherwise
+ */
+struct sgx_encl_page *sgx_encl_alloc_page(struct sgx_encl *encl,
+ unsigned long addr)
+{
+ struct sgx_encl_page *encl_page;
+ int ret;
+
+ if (radix_tree_lookup(&encl->page_tree, PFN_DOWN(addr)))
+ return ERR_PTR(-EEXIST);
+ encl_page = kzalloc(sizeof(*encl_page), GFP_KERNEL);
+ if (!encl_page)
+ return ERR_PTR(-ENOMEM);
+ encl_page->desc = addr;
+ encl_page->impl.ops = &sgx_encl_page_ops;
+ encl_page->encl = encl;
+ ret = radix_tree_insert(&encl->page_tree, PFN_DOWN(encl_page->desc),
+ encl_page);
+ if (ret) {
+ kfree(encl_page);
+ return ERR_PTR(ret);
+ }
+ return encl_page;
+}
+
+/**
+ * sgx_encl_free_page - free an enclave page
+ * @encl_page: an enclave page
+ */
+void sgx_encl_free_page(struct sgx_encl_page *encl_page)
+{
+ radix_tree_delete(&encl_page->encl->page_tree,
+ PFN_DOWN(encl_page->desc));
+ if (encl_page->desc & SGX_ENCL_PAGE_LOADED) {
+ WARN_ON(encl_page->desc & SGX_ENCL_PAGE_RECLAIMED);
+ sgx_free_page(encl_page->epc_page);
+ }
+ kfree(encl_page);
+}
+
+/**
+ * sgx_encl_add_page - add a page to the enclave
+ *
+ * @encl: an enclave
+ * @addr: page address in the ELRANGE
+ * @data: page data
+ * @secinfo: page permissions
+ * @mrmask: bitmask to select the 256 byte chunks to be measured
+ *
+ * Creates a new enclave page and enqueues an EADD operation that will be
+ * processed by a worker thread later on.
+ *
+ * Return:
+ * 0 on success,
+ * -errno otherwise
+ */
+int sgx_encl_add_page(struct sgx_encl *encl, unsigned long addr, void *data,
+ struct sgx_secinfo *secinfo, unsigned int mrmask)
+{
+ u64 page_type = secinfo->flags & SGX_SECINFO_PAGE_TYPE_MASK;
+ struct sgx_encl_page *encl_page;
+ int ret;
+
+ if (sgx_validate_secinfo(secinfo))
+ return -EINVAL;
+ if (page_type == SGX_SECINFO_TCS) {
+ ret = sgx_validate_tcs(encl, data);
+ if (ret)
+ return ret;
+ }
+ ret = sgx_encl_grow(encl);
+ if (ret)
+ return ret;
+ mutex_lock(&encl->lock);
+ if (encl->flags & (SGX_ENCL_INITIALIZED | SGX_ENCL_DEAD)) {
+ mutex_unlock(&encl->lock);
+ return -EINVAL;
+ }
+ encl_page = sgx_encl_alloc_page(encl, addr);
+ if (IS_ERR(encl_page)) {
+ mutex_unlock(&encl->lock);
+ return PTR_ERR(encl_page);
+ }
+ ret = __sgx_encl_add_page(encl, encl_page, data, secinfo, mrmask);
+ if (ret)
+ sgx_encl_free_page(encl_page);
+ mutex_unlock(&encl->lock);
+ return ret;
+}
+
+static int __sgx_get_key_hash(struct crypto_shash *tfm, const void *modulus,
+ void *hash)
+{
+ SHASH_DESC_ON_STACK(shash, tfm);
+
+ shash->tfm = tfm;
+ shash->flags = CRYPTO_TFM_REQ_MAY_SLEEP;
+
+ return crypto_shash_digest(shash, modulus, SGX_MODULUS_SIZE, hash);
+}
+
+static int sgx_get_key_hash(const void *modulus, void *hash)
+{
+ struct crypto_shash *tfm;
+ int ret;
+
+ tfm = crypto_alloc_shash("sha256", 0, CRYPTO_ALG_ASYNC);
+ if (IS_ERR(tfm))
+ return PTR_ERR(tfm);
+
+ ret = __sgx_get_key_hash(tfm, modulus, hash);
+
+ crypto_free_shash(tfm);
+ return ret;
+}
+
+/**
+ * sgx_encl_init - perform EINIT for the given enclave
+ *
+ * @encl: an enclave
+ * @sigstruct: SIGSTRUCT for the enclave
+ * @token: EINITTOKEN for the enclave
+ *
+ * Retries a few times in order to perform EINIT operation on an enclave
+ * because there could be potentially an interrupt storm.
+ *
+ * Return:
+ * 0 on success,
+ * SGX error code on EINIT failure,
+ * -errno otherwise
+ */
+int sgx_encl_init(struct sgx_encl *encl, struct sgx_sigstruct *sigstruct,
+ struct sgx_einittoken *token)
+{
+ u64 mrsigner[4];
+ int ret;
+ int i;
+ int j;
+
+ ret = sgx_get_key_hash(sigstruct->modulus, mrsigner);
+ if (ret)
+ return ret;
+
+ flush_work(&encl->add_page_work);
+
+ mutex_lock(&encl->lock);
+
+ if (encl->flags & SGX_ENCL_INITIALIZED) {
+ mutex_unlock(&encl->lock);
+ return 0;
+ }
+ if (encl->flags & SGX_ENCL_DEAD) {
+ mutex_unlock(&encl->lock);
+ return -EFAULT;
+ }
+
+ for (i = 0; i < SGX_EINIT_SLEEP_COUNT; i++) {
+ for (j = 0; j < SGX_EINIT_SPIN_COUNT; j++) {
+ ret = sgx_einit(sigstruct, token, encl->secs.epc_page,
+ mrsigner);
+ if (ret == SGX_UNMASKED_EVENT)
+ continue;
+ else
+ break;
+ }
+
+ if (ret != SGX_UNMASKED_EVENT)
+ break;
+
+ msleep_interruptible(SGX_EINIT_SLEEP_TIME);
+ if (signal_pending(current)) {
+ mutex_unlock(&encl->lock);
+ return -ERESTARTSYS;
+ }
+ }
+
+ if (ret > 0)
+ sgx_dbg(encl, "EINIT returned %d\n", ret);
+ else if (!ret)
+ encl->flags |= SGX_ENCL_INITIALIZED;
+ mutex_unlock(&encl->lock);
+
+ return ret;
+}
+
+/**
+ * sgx_encl_block - block an enclave page
+ * @encl_page: an enclave page
+ *
+ * Changes the state of the associated EPC page to blocked.
+ */
+void sgx_encl_block(struct sgx_encl_page *encl_page)
+{
+ unsigned long addr = SGX_ENCL_PAGE_ADDR(encl_page);
+ struct sgx_encl *encl = encl_page->encl;
+ struct vm_area_struct *vma;
+ int ret;
+
+ if (encl->flags & SGX_ENCL_DEAD)
+ return;
+
+ ret = sgx_encl_find(encl->mm, addr, &vma);
+ if (ret || encl != vma->vm_private_data)
+ return;
+
+ zap_vma_ptes(vma, addr, PAGE_SIZE);
+ ret = __eblock(sgx_epc_addr(encl_page->epc_page));
+ SGX_INVD(ret, encl, "EBLOCK returned %d (0x%x)", ret, ret);
+}
+
+/**
+ * sgx_encl_track - start tracking pages in the blocked state
+ * @encl: an enclave
+ *
+ * Start blocking accesses for pages in the blocked state for threads that enter
+ * inside the enclave by executing the ETRACK leaf instruction. This starts a
+ * shootdown sequence for threads that entered before ETRACK.
+ *
+ * The caller must take care (with an IPI when necessary) to make sure that the
+ * previous shootdown sequence was completed before calling this function. If
+ * this is not the case, the callee prints a critical error to the klog and
+ * kills the enclave.
+ */
+void sgx_encl_track(struct sgx_encl *encl)
+{
+ int ret = __etrack(sgx_epc_addr(encl->secs.epc_page));
+
+ SGX_INVD(ret, encl, "ETRACK returned %d\n", ret);
+}
+
+/**
+ * sgx_encl_load_page - load an enclave page
+ * @encl_page: a &sgx_encl_page
+ * @epc_page: a &sgx_epc_page
+ *
+ * Loads an enclave page from the regular memory to the EPC. The pages, which
+ * are not children of a SECS (eg SECS itself and VA pages) should set their
+ * address to zero.
+ */
+int sgx_encl_load_page(struct sgx_encl_page *encl_page,
+ struct sgx_epc_page *epc_page)
+{
+ unsigned long addr = SGX_ENCL_PAGE_ADDR(encl_page);
+ struct sgx_encl *encl = encl_page->encl;
+ struct sgx_pageinfo pginfo;
+ unsigned long pcmd_offset;
+ unsigned long va_offset;
+ pgoff_t backing_index;
+ struct page *backing;
+ struct page *pcmd;
+ void *va_ptr;
+ int ret;
+
+ backing_index = SGX_ENCL_PAGE_BACKING_INDEX(encl_page, encl);
+ pcmd_offset = SGX_ENCL_PAGE_PCMD_OFFSET(encl_page, encl);
+ va_offset = SGX_ENCL_PAGE_VA_OFFSET(encl_page);
+
+ backing = sgx_get_backing(encl->backing, backing_index);
+ if (IS_ERR(backing))
+ return PTR_ERR(backing);
+
+ pcmd = sgx_get_backing(encl->pcmd, backing_index >> 5);
+ if (IS_ERR(pcmd)) {
+ sgx_put_backing(backing, false);
+ return PTR_ERR(pcmd);
+ }
+
+
+ va_ptr = sgx_epc_addr(encl_page->va_page->epc_page) + va_offset;
+
+ pginfo.addr = addr;
+ pginfo.contents = (unsigned long)kmap_atomic(backing);
+ pginfo.metadata = (unsigned long)kmap_atomic(pcmd) + pcmd_offset;
+ pginfo.secs = addr ? (unsigned long)sgx_epc_addr(encl->secs.epc_page) :
+ 0;
+
+ ret = __eldu(&pginfo, sgx_epc_addr(epc_page), va_ptr);
+ if (ret) {
+ sgx_err(encl, "ELDU returned %d\n", ret);
+ ret = encls_to_err(ret);
+ }
+
+ kunmap_atomic((void *)(unsigned long)(pginfo.metadata - pcmd_offset));
+ kunmap_atomic((void *)(unsigned long)pginfo.contents);
+
+ sgx_put_backing(pcmd, false);
+ sgx_put_backing(backing, false);
+ return ret;
+}
+
+/**
+ * sgx_encl_release - destroy an enclave instance
+ *
+ * @kref: address of a kref inside &sgx_encl
+ *
+ * Used together with kref_put(). Frees all the resources associated with the
+ * enclave and the instance itself.
+ */
+void sgx_encl_release(struct kref *ref)
+{
+ struct sgx_encl *encl = container_of(ref, struct sgx_encl, refcount);
+ struct sgx_encl_page *entry;
+ struct radix_tree_iter iter;
+ struct sgx_va_page *va_page;
+ void **slot;
+
+ if (encl->mmu_notifier.ops) {
+ mmu_notifier_unregister_no_release(&encl->mmu_notifier,
+ encl->mm);
+ encl->mmu_notifier.ops = NULL;
+ }
+
+ if (encl->pm_notifier.notifier_call) {
+ unregister_pm_notifier(&encl->pm_notifier);
+ encl->pm_notifier.notifier_call = NULL;
+ }
+
+ radix_tree_for_each_slot(slot, &encl->page_tree, &iter, 0) {
+ entry = *slot;
+ sgx_encl_free_page(entry);
+ }
+
+ if (encl->tgid) {
+ put_pid(encl->tgid);
+ encl->tgid = NULL;
+ }
+
+ while (!list_empty(&encl->va_pages)) {
+ va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
+ list);
+ list_del(&va_page->list);
+ sgx_free_page(va_page->epc_page);
+ kfree(va_page);
+ }
+
+ if (encl->secs.desc & SGX_ENCL_PAGE_LOADED)
+ sgx_free_page(encl->secs.epc_page);
+
+ if (encl->backing) {
+ fput(encl->backing);
+ encl->backing = NULL;
+ }
+
+ if (encl->pcmd) {
+ fput(encl->pcmd);
+ encl->pcmd = NULL;
+ }
+
+ kfree(encl);
+}
diff --git a/drivers/platform/x86/intel_sgx/sgx_encl_page.c b/drivers/platform/x86/intel_sgx/sgx_encl_page.c
new file mode 100644
index 000000000000..fe52fb90d42b
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx_encl_page.c
@@ -0,0 +1,288 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#include <linux/device.h>
+#include <linux/freezer.h>
+#include <linux/highmem.h>
+#include <linux/kthread.h>
+#include <linux/ratelimit.h>
+#include <linux/sched/signal.h>
+#include <linux/slab.h>
+#include "sgx.h"
+
+static inline struct sgx_encl_page *to_encl_page(struct sgx_epc_page *epc_page)
+{
+ return container_of(epc_page->impl, struct sgx_encl_page, impl);
+}
+
+static bool sgx_encl_page_get(struct sgx_epc_page *epc_page)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ struct sgx_encl *encl = encl_page->encl;
+
+ return kref_get_unless_zero(&encl->refcount) != 0;
+}
+
+static void sgx_encl_page_put(struct sgx_epc_page *epc_page)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ struct sgx_encl *encl = encl_page->encl;
+
+ kref_put(&encl->refcount, sgx_encl_release);
+}
+
+static bool sgx_encl_page_reclaim(struct sgx_epc_page *epc_page)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ struct sgx_encl *encl = encl_page->encl;
+ bool ret;
+
+ down_read(&encl->mm->mmap_sem);
+ mutex_lock(&encl->lock);
+ /*
+ * There's a small window between the EPC manager pulling the
+ * page off the active list and calling reclaim(), during which
+ * we can free the page, e.g. via sgx_invalidate(). Check the
+ * LOADED flag to ensure the page is still resident in the EPC.
+ */
+ if (!(encl_page->desc & SGX_ENCL_PAGE_LOADED))
+ ret = false;
+ else if (encl->flags & SGX_ENCL_DEAD)
+ ret = true;
+ else if (encl_page->desc & SGX_ENCL_PAGE_RESERVED)
+ ret = false;
+ else
+ ret = !sgx_test_and_clear_young(encl_page);
+ if (ret)
+ encl_page->desc |= SGX_ENCL_PAGE_RECLAIMED;
+ mutex_unlock(&encl->lock);
+ up_read(&encl->mm->mmap_sem);
+
+ return ret;
+}
+
+static void sgx_encl_page_block(struct sgx_epc_page *epc_page)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ struct sgx_encl *encl = encl_page->encl;
+
+ down_read(&encl->mm->mmap_sem);
+ mutex_lock(&encl->lock);
+ sgx_encl_block(encl_page);
+ mutex_unlock(&encl->lock);
+ up_read(&encl->mm->mmap_sem);
+}
+
+static int sgx_ewb(struct sgx_encl *encl, struct sgx_epc_page *epc_page,
+ struct sgx_va_page *va_page, unsigned int va_offset)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ unsigned long pcmd_offset = SGX_ENCL_PAGE_PCMD_OFFSET(encl_page, encl);
+ struct sgx_pageinfo pginfo;
+ pgoff_t backing_index;
+ struct page *backing;
+ struct page *pcmd;
+ void *va;
+ int ret;
+
+ backing_index = SGX_ENCL_PAGE_BACKING_INDEX(encl_page, encl);
+
+ backing = sgx_get_backing(encl->backing, backing_index);
+ if (IS_ERR(backing)) {
+ ret = PTR_ERR(backing);
+ return ret;
+ }
+
+ pcmd = sgx_get_backing(encl->pcmd, backing_index >> 5);
+ if (IS_ERR(pcmd)) {
+ ret = PTR_ERR(pcmd);
+ sgx_put_backing(backing, true);
+ return ret;
+ }
+
+ va = sgx_epc_addr(va_page->epc_page) + va_offset;
+
+ pginfo.addr = 0;
+ pginfo.contents = (unsigned long)kmap_atomic(backing);
+ pginfo.metadata = (unsigned long)kmap_atomic(pcmd) + pcmd_offset;
+ pginfo.secs = 0;
+ ret = __ewb(&pginfo, sgx_epc_addr(epc_page), va);
+ kunmap_atomic((void *)(unsigned long)(pginfo.metadata - pcmd_offset));
+ kunmap_atomic((void *)(unsigned long)pginfo.contents);
+
+ sgx_put_backing(pcmd, true);
+ sgx_put_backing(backing, true);
+
+ return ret;
+}
+
+/**
+ * sgx_write_page - write a page to the regular memory
+ *
+ * Writes an EPC page to the shmem file associated with the enclave. Flushes
+ * CPUs and retries if there are hardware threads that can potentially have TLB
+ * entries to the page (indicated by SGX_NOT_TRACKED). Clears the reserved flag
+ * after the page is swapped.
+ *
+ * @epc_page: an EPC page
+ */
+static void sgx_write_page(struct sgx_epc_page *epc_page, bool do_free)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ struct sgx_encl *encl = encl_page->encl;
+ struct sgx_va_page *va_page;
+ unsigned int va_offset;
+ int ret;
+
+ encl_page->desc &= ~(SGX_ENCL_PAGE_LOADED | SGX_ENCL_PAGE_RECLAIMED);
+
+ if (!(encl->flags & SGX_ENCL_DEAD)) {
+ va_page = list_first_entry(&encl->va_pages, struct sgx_va_page,
+ list);
+ va_offset = sgx_alloc_va_slot(va_page);
+ if (sgx_va_page_full(va_page))
+ list_move_tail(&va_page->list, &encl->va_pages);
+
+ ret = sgx_ewb(encl, epc_page, va_page, va_offset);
+ if (ret == SGX_NOT_TRACKED) {
+ sgx_encl_track(encl);
+ ret = sgx_ewb(encl, epc_page, va_page, va_offset);
+ if (ret == SGX_NOT_TRACKED) {
+ /* slow path, IPI needed */
+ sgx_flush_cpus(encl);
+ ret = sgx_ewb(encl, epc_page, va_page,
+ va_offset);
+ }
+ }
+ SGX_INVD(ret, encl, "EWB returned %d\n", ret);
+
+ SGX_INVD(encl_page->desc & SGX_ENCL_PAGE_VA_OFFSET_MASK, encl,
+ "Flags set in VA offset area: %lx", encl_page->desc);
+ encl_page->desc |= va_offset;
+ encl_page->va_page = va_page;
+ } else if (!do_free) {
+ ret = __eremove(sgx_epc_addr(epc_page));
+ WARN(ret, "EREMOVE returned %d\n", ret);
+ }
+
+ if (do_free)
+ sgx_free_page(epc_page);
+}
+
+static void sgx_encl_page_write(struct sgx_epc_page *epc_page)
+{
+ struct sgx_encl_page *encl_page = to_encl_page(epc_page);
+ struct sgx_encl *encl = encl_page->encl;
+
+ down_read(&encl->mm->mmap_sem);
+ mutex_lock(&encl->lock);
+ sgx_write_page(epc_page, false);
+ encl->secs_child_cnt--;
+ if (!encl->secs_child_cnt && (encl->flags & SGX_ENCL_INITIALIZED))
+ sgx_write_page(encl->secs.epc_page, true);
+ mutex_unlock(&encl->lock);
+ up_read(&encl->mm->mmap_sem);
+}
+
+const struct sgx_epc_page_ops sgx_encl_page_ops = {
+ .get = sgx_encl_page_get,
+ .put = sgx_encl_page_put,
+ .reclaim = sgx_encl_page_reclaim,
+ .block = sgx_encl_page_block,
+ .write = sgx_encl_page_write,
+};
+
+/**
+ * sgx_set_epc_page - associate an EPC page with an enclave page
+ * @encl_page: an enclave page
+ * @epc_page: the EPC page to attach to @encl_page
+ */
+void sgx_set_epc_page(struct sgx_encl_page *encl_page,
+ struct sgx_epc_page *epc_page)
+{
+ encl_page->desc |= SGX_ENCL_PAGE_LOADED;
+ encl_page->epc_page = epc_page;
+}
+
+/**
+ * sgx_set_page_reclaimable - mark an EPC page reclaimable
+ * @encl_page: an enclave page with a loaded EPC page
+ */
+void sgx_set_page_reclaimable(struct sgx_encl_page *encl_page)
+{
+ sgx_test_and_clear_young(encl_page);
+
+ sgx_page_reclaimable(encl_page->epc_page);
+}
+
+/**
+ * sgx_alloc_page - allocate a VA page
+ * @flags: allocation flags
+ *
+ * Allocates an &sgx_epc_page instance and converts it to a VA page.
+ *
+ * Return:
+ * a &struct sgx_va_page instance,
+ * -errno otherwise
+ */
+struct sgx_epc_page *sgx_alloc_va_page(unsigned int flags)
+{
+ struct sgx_epc_page *epc_page;
+ int ret;
+
+ epc_page = sgx_alloc_page(NULL, flags);
+ if (IS_ERR(epc_page))
+ return (void *)epc_page;
+
+ ret = __epa(sgx_epc_addr(epc_page));
+ if (ret) {
+ pr_crit("EPA failed\n");
+ sgx_free_page(epc_page);
+ return ERR_PTR(ret);
+ }
+
+ return epc_page;
+}
+
+/**
+ * sgx_alloc_va_slot - allocate a VA slot
+ * @va_page: a &struct sgx_va_page instance
+ *
+ * Allocates a slot from a &struct sgx_va_page instance.
+ *
+ * Return: offset of the slot inside the VA page
+ */
+unsigned int sgx_alloc_va_slot(struct sgx_va_page *va_page)
+{
+ int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT);
+
+ if (slot < SGX_VA_SLOT_COUNT)
+ set_bit(slot, va_page->slots);
+
+ return slot << 3;
+}
+
+/**
+ * sgx_free_va_slot - free a VA slot
+ * @va_page: a &struct sgx_va_page instance
+ * @offset: offset of the slot inside the VA page
+ *
+ * Frees a slot from a &struct sgx_va_page instance.
+ */
+void sgx_free_va_slot(struct sgx_va_page *va_page, unsigned int offset)
+{
+ clear_bit(offset >> 3, va_page->slots);
+}
+
+/**
+ * sgx_va_page_full - is the VA page full?
+ * @va_page: a &struct sgx_va_page instance
+ *
+ * Return: true if all slots have been taken
+ */
+bool sgx_va_page_full(struct sgx_va_page *va_page)
+{
+ int slot = find_first_zero_bit(va_page->slots, SGX_VA_SLOT_COUNT);
+
+ return slot == SGX_VA_SLOT_COUNT;
+}
diff --git a/drivers/platform/x86/intel_sgx/sgx_fault.c b/drivers/platform/x86/intel_sgx/sgx_fault.c
new file mode 100644
index 000000000000..dc50b6e391f4
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx_fault.c
@@ -0,0 +1,157 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#include <linux/highmem.h>
+#include <linux/sched/mm.h>
+#include "sgx.h"
+
+static int sgx_test_and_clear_young_cb(pte_t *ptep, pgtable_t token,
+ unsigned long addr, void *data)
+{
+ pte_t pte;
+ int ret;
+
+ ret = pte_young(*ptep);
+ if (ret) {
+ pte = pte_mkold(*ptep);
+ set_pte_at((struct mm_struct *)data, addr, ptep, pte);
+ }
+
+ return ret;
+}
+
+/**
+ * sgx_test_and_clear_young() - Test and reset the accessed bit
+ * @page: enclave page to be tested for recent access
+ *
+ * Checks the Access (A) bit from the PTE corresponding to the
+ * enclave page and clears it. Returns 1 if the page has been
+ * recently accessed and 0 if not.
+ */
+int sgx_test_and_clear_young(struct sgx_encl_page *page)
+{
+ unsigned long addr = SGX_ENCL_PAGE_ADDR(page);
+ struct sgx_encl *encl = page->encl;
+ struct vm_area_struct *vma;
+ int ret;
+
+ ret = sgx_encl_find(encl->mm, addr, &vma);
+ if (ret)
+ return 0;
+
+ if (encl != vma->vm_private_data)
+ return 0;
+
+ return apply_to_page_range(vma->vm_mm, addr, PAGE_SIZE,
+ sgx_test_and_clear_young_cb, vma->vm_mm);
+}
+
+static void sgx_ipi_cb(void *info)
+{
+}
+
+void sgx_flush_cpus(struct sgx_encl *encl)
+{
+ on_each_cpu_mask(mm_cpumask(encl->mm), sgx_ipi_cb, NULL, 1);
+}
+
+static struct sgx_epc_page *__sgx_load_faulted_page(
+ struct sgx_encl_page *encl_page)
+{
+ unsigned long va_offset = SGX_ENCL_PAGE_VA_OFFSET(encl_page);
+ struct sgx_encl *encl = encl_page->encl;
+ struct sgx_epc_page *epc_page;
+ int ret;
+
+ epc_page = sgx_alloc_page(&encl_page->impl, SGX_ALLOC_ATOMIC);
+ if (IS_ERR(epc_page))
+ return epc_page;
+ ret = sgx_encl_load_page(encl_page, epc_page);
+ if (ret) {
+ sgx_free_page(epc_page);
+ return ERR_PTR(ret);
+ }
+ sgx_free_va_slot(encl_page->va_page, va_offset);
+ list_move(&encl_page->va_page->list, &encl->va_pages);
+ encl_page->desc &= ~SGX_ENCL_PAGE_VA_OFFSET_MASK;
+ sgx_set_epc_page(encl_page, epc_page);
+ return epc_page;
+}
+
+static struct sgx_encl_page *__sgx_fault_page(struct vm_area_struct *vma,
+ unsigned long addr,
+ bool do_reserve)
+{
+ struct sgx_encl *encl = vma->vm_private_data;
+ struct sgx_epc_page *epc_page;
+ struct sgx_encl_page *entry;
+ int rc = 0;
+
+ if ((encl->flags & SGX_ENCL_DEAD) ||
+ !(encl->flags & SGX_ENCL_INITIALIZED))
+ return ERR_PTR(-EFAULT);
+
+ entry = radix_tree_lookup(&encl->page_tree, addr >> PAGE_SHIFT);
+ if (!entry)
+ return ERR_PTR(-EFAULT);
+
+ /* Page is already resident in the EPC. */
+ if (entry->desc & SGX_ENCL_PAGE_LOADED) {
+ if (entry->desc & SGX_ENCL_PAGE_RESERVED) {
+ sgx_dbg(encl, "EPC page 0x%p is already reserved\n",
+ (void *)SGX_ENCL_PAGE_ADDR(entry));
+ return ERR_PTR(-EBUSY);
+ }
+ if (entry->desc & SGX_ENCL_PAGE_RECLAIMED) {
+ sgx_dbg(encl, "EPC page 0x%p is being reclaimed\n",
+ (void *)SGX_ENCL_PAGE_ADDR(entry));
+ return ERR_PTR(-EBUSY);
+ }
+ if (do_reserve)
+ entry->desc |= SGX_ENCL_PAGE_RESERVED;
+ return entry;
+ }
+
+ if (!(encl->secs.desc & SGX_ENCL_PAGE_LOADED)) {
+ epc_page = __sgx_load_faulted_page(&encl->secs);
+ if (IS_ERR(epc_page))
+ return (void *)epc_page;
+ }
+ epc_page = __sgx_load_faulted_page(entry);
+ if (IS_ERR(epc_page))
+ return (void *)epc_page;
+
+ encl->secs_child_cnt++;
+ sgx_set_page_reclaimable(entry);
+ if (do_reserve)
+ entry->desc |= SGX_ENCL_PAGE_RESERVED;
+
+ rc = vm_insert_pfn(vma, addr, PFN_DOWN(entry->epc_page->desc));
+ SGX_INVD(rc, encl, "%s: vm_insert_pfn() returned %d\n", __func__, rc);
+ if (rc)
+ return ERR_PTR(rc);
+
+ return entry;
+}
+
+struct sgx_encl_page *sgx_fault_page(struct vm_area_struct *vma,
+ unsigned long addr, bool do_reserve)
+{
+ struct sgx_encl *encl = vma->vm_private_data;
+ struct sgx_encl_page *entry;
+
+ /* If process was forked, VMA is still there but vm_private_data is set
+ * to NULL.
+ */
+ if (!encl)
+ return ERR_PTR(-EFAULT);
+ do {
+ mutex_lock(&encl->lock);
+ entry = __sgx_fault_page(vma, addr, do_reserve);
+ mutex_unlock(&encl->lock);
+ if (!do_reserve)
+ break;
+ } while (PTR_ERR(entry) == -EBUSY);
+
+ return entry;
+}
diff --git a/drivers/platform/x86/intel_sgx/sgx_ioctl.c b/drivers/platform/x86/intel_sgx/sgx_ioctl.c
new file mode 100644
index 000000000000..4edf1cc956b1
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx_ioctl.c
@@ -0,0 +1,234 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#include <asm/mman.h>
+#include <linux/delay.h>
+#include <linux/file.h>
+#include <linux/hashtable.h>
+#include <linux/highmem.h>
+#include <linux/ratelimit.h>
+#include <linux/sched/signal.h>
+#include <linux/slab.h>
+#include "sgx.h"
+
+static int sgx_encl_get(unsigned long addr, struct sgx_encl **encl)
+{
+ struct mm_struct *mm = current->mm;
+ struct vm_area_struct *vma;
+ int ret;
+
+ if (addr & (PAGE_SIZE - 1))
+ return -EINVAL;
+
+ down_read(&mm->mmap_sem);
+
+ ret = sgx_encl_find(mm, addr, &vma);
+ if (!ret) {
+ *encl = vma->vm_private_data;
+
+ if ((*encl)->flags & SGX_ENCL_SUSPEND)
+ ret = SGX_POWER_LOST_ENCLAVE;
+ else
+ kref_get(&(*encl)->refcount);
+ }
+
+ up_read(&mm->mmap_sem);
+ return ret;
+}
+
+/**
+ * sgx_ioc_enclave_create - handler for %SGX_IOC_ENCLAVE_CREATE
+ * @filep: open file to /dev/sgx
+ * @cmd: the command value
+ * @arg: pointer to an &sgx_enclave_create instance
+ *
+ * Validates SECS attributes, allocates an EPC page for the SECS and performs
+ * ECREATE.
+ *
+ * Return:
+ * 0 on success,
+ * -errno otherwise
+ */
+static long sgx_ioc_enclave_create(struct file *filep, unsigned int cmd,
+ unsigned long arg)
+{
+ struct sgx_enclave_create *createp = (struct sgx_enclave_create *)arg;
+ struct page *secs_page;
+ struct sgx_secs *secs;
+ struct sgx_encl *encl;
+ int ret;
+
+ secs_page = alloc_page(GFP_HIGHUSER);
+ if (!secs_page)
+ return -ENOMEM;
+
+ secs = kmap(secs_page);
+ ret = copy_from_user(secs, (void __user *)createp->src, sizeof(*secs));
+ if (ret)
+ goto out;
+
+ encl = sgx_encl_alloc(secs);
+ if (IS_ERR(encl)) {
+ ret = PTR_ERR(encl);
+ goto out;
+ }
+
+ ret = sgx_encl_create(encl, secs);
+ if (ret)
+ kref_put(&encl->refcount, sgx_encl_release);
+
+out:
+ kunmap(secs_page);
+ __free_page(secs_page);
+ return ret;
+}
+
+/**
+ * sgx_ioc_enclave_add_page - handler for %SGX_IOC_ENCLAVE_ADD_PAGE
+ *
+ * @filep: open file to /dev/sgx
+ * @cmd: the command value
+ * @arg: pointer to an &sgx_enclave_add_page instance
+ *
+ * Creates a new enclave page and enqueues an EADD operation that will be
+ * processed by a worker thread later on.
+ *
+ * Return:
+ * 0 on success,
+ * -errno otherwise
+ */
+static long sgx_ioc_enclave_add_page(struct file *filep, unsigned int cmd,
+ unsigned long arg)
+{
+ struct sgx_enclave_add_page *addp = (void *)arg;
+ struct sgx_secinfo secinfo;
+ struct sgx_encl *encl;
+ struct page *data_page;
+ void *data;
+ int ret;
+
+ ret = sgx_encl_get(addp->addr, &encl);
+ if (ret)
+ return ret;
+
+ if (copy_from_user(&secinfo, (void __user *)addp->secinfo,
+ sizeof(secinfo))) {
+ kref_put(&encl->refcount, sgx_encl_release);
+ return -EFAULT;
+ }
+
+ data_page = alloc_page(GFP_HIGHUSER);
+ if (!data_page) {
+ kref_put(&encl->refcount, sgx_encl_release);
+ return -ENOMEM;
+ }
+
+ data = kmap(data_page);
+
+ ret = copy_from_user((void *)data, (void __user *)addp->src, PAGE_SIZE);
+ if (ret)
+ goto out;
+
+ ret = sgx_encl_add_page(encl, addp->addr, data, &secinfo, addp->mrmask);
+ if (ret)
+ goto out;
+
+out:
+ kref_put(&encl->refcount, sgx_encl_release);
+ kunmap(data_page);
+ __free_page(data_page);
+ return ret;
+}
+
+/**
+ * sgx_ioc_enclave_init - handler for %SGX_IOC_ENCLAVE_INIT
+ *
+ * @filep: open file to /dev/sgx
+ * @cmd: the command value
+ * @arg: pointer to an &sgx_enclave_init instance
+ *
+ * Flushes the remaining enqueued EADD operations and performs EINIT. Does not
+ * allow the EINITTOKENKEY attribute for an enclave.
+ *
+ * Return:
+ * 0 on success,
+ * SGX error code on EINIT failure,
+ * -errno otherwise
+ */
+static long sgx_ioc_enclave_init(struct file *filep, unsigned int cmd,
+ unsigned long arg)
+{
+ struct sgx_enclave_init *initp = (struct sgx_enclave_init *)arg;
+ struct sgx_sigstruct *sigstruct;
+ struct sgx_einittoken *einittoken;
+ struct sgx_encl *encl;
+ struct page *initp_page;
+ int ret;
+
+ initp_page = alloc_page(GFP_HIGHUSER);
+ if (!initp_page)
+ return -ENOMEM;
+
+ sigstruct = kmap(initp_page);
+ einittoken = (struct sgx_einittoken *)
+ ((unsigned long)sigstruct + PAGE_SIZE / 2);
+ memset(einittoken, 0, sizeof(*einittoken));
+
+ ret = copy_from_user(sigstruct, (void __user *)initp->sigstruct,
+ sizeof(*sigstruct));
+ if (ret)
+ goto out;
+ if (sigstruct->attributes & SGX_ATTR_EINITTOKENKEY) {
+ ret = EINVAL;
+ goto out;
+ }
+
+ ret = sgx_encl_get(initp->addr, &encl);
+ if (ret)
+ goto out;
+
+ ret = sgx_encl_init(encl, sigstruct, einittoken);
+
+ kref_put(&encl->refcount, sgx_encl_release);
+
+out:
+ kunmap(initp_page);
+ __free_page(initp_page);
+ return ret;
+}
+
+typedef long (*sgx_ioc_t)(struct file *filep, unsigned int cmd,
+ unsigned long arg);
+
+long sgx_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
+{
+ char data[256];
+ sgx_ioc_t handler = NULL;
+ long ret;
+
+ switch (cmd) {
+ case SGX_IOC_ENCLAVE_CREATE:
+ handler = sgx_ioc_enclave_create;
+ break;
+ case SGX_IOC_ENCLAVE_ADD_PAGE:
+ handler = sgx_ioc_enclave_add_page;
+ break;
+ case SGX_IOC_ENCLAVE_INIT:
+ handler = sgx_ioc_enclave_init;
+ break;
+ default:
+ return -ENOIOCTLCMD;
+ }
+
+ if (copy_from_user(data, (void __user *)arg, _IOC_SIZE(cmd)))
+ return -EFAULT;
+
+ ret = handler(filep, cmd, (unsigned long)((void *)data));
+ if (!ret && (cmd & IOC_OUT)) {
+ if (copy_to_user((void __user *)arg, data, _IOC_SIZE(cmd)))
+ return -EFAULT;
+ }
+ if (IS_ENCLS_FAULT(ret))
+ return -EFAULT;
+ return ret;
+}
diff --git a/drivers/platform/x86/intel_sgx/sgx_main.c b/drivers/platform/x86/intel_sgx/sgx_main.c
new file mode 100644
index 000000000000..69655f82a327
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx_main.c
@@ -0,0 +1,257 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#include <linux/acpi.h>
+#include <linux/cdev.h>
+#include <linux/platform_device.h>
+#include <linux/suspend.h>
+#include "sgx.h"
+
+MODULE_DESCRIPTION("Intel SGX Driver");
+MODULE_AUTHOR("Jarkko Sakkinen <jarkko.sakkinen@xxxxxxxxxxxxxxx>");
+MODULE_LICENSE("Dual BSD/GPL");
+
+struct workqueue_struct *sgx_add_page_wq;
+u64 sgx_encl_size_max_32;
+u64 sgx_encl_size_max_64;
+u64 sgx_xfrm_mask = 0x3;
+u32 sgx_misc_reserved;
+u32 sgx_xsave_size_tbl[64];
+
+#ifdef CONFIG_COMPAT
+long sgx_compat_ioctl(struct file *filep, unsigned int cmd, unsigned long arg)
+{
+ return sgx_ioctl(filep, cmd, arg);
+}
+#endif
+
+static int sgx_mmap(struct file *file, struct vm_area_struct *vma)
+{
+ vma->vm_ops = &sgx_vm_ops;
+ vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO |
+ VM_DONTCOPY;
+
+ return 0;
+}
+
+static unsigned long sgx_get_unmapped_area(struct file *file,
+ unsigned long addr,
+ unsigned long len,
+ unsigned long pgoff,
+ unsigned long flags)
+{
+ if (len < 2 * PAGE_SIZE || (len & (len - 1)))
+ return -EINVAL;
+
+ if (len > sgx_encl_size_max_64)
+ return -EINVAL;
+
+ if (len > sgx_encl_size_max_32 && test_thread_flag(TIF_ADDR32))
+ return -EINVAL;
+
+ addr = current->mm->get_unmapped_area(file, addr, 2 * len, pgoff,
+ flags);
+ if (IS_ERR_VALUE(addr))
+ return addr;
+
+ addr = (addr + (len - 1)) & ~(len - 1);
+
+ return addr;
+}
+
+static const struct file_operations sgx_fops = {
+ .owner = THIS_MODULE,
+ .unlocked_ioctl = sgx_ioctl,
+#ifdef CONFIG_COMPAT
+ .compat_ioctl = sgx_compat_ioctl,
+#endif
+ .mmap = sgx_mmap,
+ .get_unmapped_area = sgx_get_unmapped_area,
+};
+
+static struct bus_type sgx_bus_type = {
+ .name = "sgx",
+};
+
+struct sgx_context {
+ struct device dev;
+ struct cdev cdev;
+};
+
+static dev_t sgx_devt;
+
+static void sgx_dev_release(struct device *dev)
+{
+ struct sgx_context *ctx = container_of(dev, struct sgx_context, dev);
+
+ kfree(ctx);
+}
+
+static struct sgx_context *sgx_ctx_alloc(struct device *parent)
+{
+ struct sgx_context *ctx;
+
+ ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
+ if (!ctx)
+ return ERR_PTR(-ENOMEM);
+
+ device_initialize(&ctx->dev);
+
+ ctx->dev.bus = &sgx_bus_type;
+ ctx->dev.parent = parent;
+ ctx->dev.devt = MKDEV(MAJOR(sgx_devt), 0);
+ ctx->dev.release = sgx_dev_release;
+
+ dev_set_name(&ctx->dev, "sgx");
+
+ cdev_init(&ctx->cdev, &sgx_fops);
+ ctx->cdev.owner = THIS_MODULE;
+
+ dev_set_drvdata(parent, ctx);
+
+ return ctx;
+}
+
+static struct sgx_context *sgxm_ctx_alloc(struct device *parent)
+{
+ struct sgx_context *ctx;
+ int rc;
+
+ ctx = sgx_ctx_alloc(parent);
+ if (IS_ERR(ctx))
+ return ctx;
+
+ rc = devm_add_action_or_reset(parent, (void (*)(void *))put_device,
+ &ctx->dev);
+ if (rc) {
+ kfree(ctx);
+ return ERR_PTR(rc);
+ }
+
+ return ctx;
+}
+
+static int sgx_dev_init(struct device *parent)
+{
+ struct sgx_context *sgx_dev;
+ unsigned int eax;
+ unsigned int ebx;
+ unsigned int ecx;
+ unsigned int edx;
+ int ret;
+ int i;
+
+ sgx_dev = sgxm_ctx_alloc(parent);
+
+ cpuid_count(SGX_CPUID, 0, &eax, &ebx, &ecx, &edx);
+ /* Only allow misc bits supported by the driver. */
+ sgx_misc_reserved = ~ebx | SGX_MISC_RESERVED_MASK;
+ sgx_encl_size_max_64 = 1ULL << ((edx >> 8) & 0xFF);
+ sgx_encl_size_max_32 = 1ULL << (edx & 0xFF);
+
+ if (boot_cpu_has(X86_FEATURE_OSXSAVE)) {
+ cpuid_count(SGX_CPUID, 1, &eax, &ebx, &ecx, &edx);
+ sgx_xfrm_mask = (((u64)edx) << 32) + (u64)ecx;
+
+ for (i = 2; i < 64; i++) {
+ cpuid_count(0x0D, i, &eax, &ebx, &ecx, &edx);
+ if ((1 << i) & sgx_xfrm_mask)
+ sgx_xsave_size_tbl[i] = eax + ebx;
+ }
+ }
+
+ sgx_add_page_wq = alloc_workqueue("intel_sgx-add-page-wq",
+ WQ_UNBOUND | WQ_FREEZABLE, 1);
+ if (!sgx_add_page_wq)
+ return -ENOMEM;
+
+ ret = cdev_device_add(&sgx_dev->cdev, &sgx_dev->dev);
+ if (ret)
+ goto out_workqueue;
+
+ return 0;
+out_workqueue:
+ destroy_workqueue(sgx_add_page_wq);
+ return ret;
+}
+
+static int sgx_drv_probe(struct platform_device *pdev)
+{
+ if (!sgx_enabled || !sgx_lc_enabled)
+ return -ENODEV;
+
+ return sgx_dev_init(&pdev->dev);
+}
+
+static int sgx_drv_remove(struct platform_device *pdev)
+{
+ struct sgx_context *ctx = dev_get_drvdata(&pdev->dev);
+
+ cdev_device_del(&ctx->cdev, &ctx->dev);
+ destroy_workqueue(sgx_add_page_wq);
+
+ return 0;
+}
+
+#ifdef CONFIG_ACPI
+static struct acpi_device_id sgx_device_ids[] = {
+ {"INT0E0C", 0},
+ {"", 0},
+};
+MODULE_DEVICE_TABLE(acpi, sgx_device_ids);
+#endif
+
+static struct platform_driver sgx_drv = {
+ .probe = sgx_drv_probe,
+ .remove = sgx_drv_remove,
+ .driver = {
+ .name = "intel_sgx",
+ .acpi_match_table = ACPI_PTR(sgx_device_ids),
+ },
+};
+
+static int __init sgx_drv_subsys_init(void)
+{
+ int ret;
+
+ ret = bus_register(&sgx_bus_type);
+ if (ret)
+ return ret;
+
+ ret = alloc_chrdev_region(&sgx_devt, 0, 1, "sgx");
+ if (ret < 0) {
+ bus_unregister(&sgx_bus_type);
+ return ret;
+ }
+
+ return 0;
+}
+
+static void sgx_drv_subsys_exit(void)
+{
+ bus_unregister(&sgx_bus_type);
+ unregister_chrdev_region(sgx_devt, 1);
+}
+
+static int __init sgx_drv_init(void)
+{
+ int ret;
+
+ ret = sgx_drv_subsys_init();
+ if (ret)
+ return ret;
+
+ ret = platform_driver_register(&sgx_drv);
+ if (ret)
+ sgx_drv_subsys_exit();
+
+ return ret;
+}
+module_init(sgx_drv_init);
+
+static void __exit sgx_drv_exit(void)
+{
+ platform_driver_unregister(&sgx_drv);
+ sgx_drv_subsys_exit();
+}
+module_exit(sgx_drv_exit);
diff --git a/drivers/platform/x86/intel_sgx/sgx_vma.c b/drivers/platform/x86/intel_sgx/sgx_vma.c
new file mode 100644
index 000000000000..cc0993b4fd40
--- /dev/null
+++ b/drivers/platform/x86/intel_sgx/sgx_vma.c
@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: (GPL-2.0 OR BSD-3-Clause)
+// Copyright(c) 2016-18 Intel Corporation.
+
+#include <asm/mman.h>
+#include <linux/delay.h>
+#include <linux/file.h>
+#include <linux/hashtable.h>
+#include <linux/highmem.h>
+#include <linux/mm.h>
+#include <linux/ratelimit.h>
+#include <linux/slab.h>
+#include "sgx.h"
+
+static void sgx_vma_open(struct vm_area_struct *vma)
+{
+ struct sgx_encl *encl = vma->vm_private_data;
+
+ if (!encl)
+ return;
+
+ /* kref cannot underflow because ECREATE ioctl checks that there is only
+ * one single VMA for the enclave before proceeding.
+ */
+ kref_get(&encl->refcount);
+}
+
+static void sgx_vma_close(struct vm_area_struct *vma)
+{
+ struct sgx_encl *encl = vma->vm_private_data;
+
+ if (!encl)
+ return;
+
+ mutex_lock(&encl->lock);
+ sgx_invalidate(encl, true);
+ mutex_unlock(&encl->lock);
+ kref_put(&encl->refcount, sgx_encl_release);
+}
+
+static int sgx_vma_fault(struct vm_fault *vmf)
+{
+ unsigned long addr = (unsigned long)vmf->address;
+ struct vm_area_struct *vma = vmf->vma;
+ struct sgx_encl_page *entry;
+
+ entry = sgx_fault_page(vma, addr, 0);
+
+ if (!IS_ERR(entry) || PTR_ERR(entry) == -EBUSY)
+ return VM_FAULT_NOPAGE;
+ else
+ return VM_FAULT_SIGBUS;
+}
+
+const struct vm_operations_struct sgx_vm_ops = {
+ .close = sgx_vma_close,
+ .open = sgx_vma_open,
+ .fault = sgx_vma_fault,
+};
--
2.17.1
