[PAL/Linux-SGX] Add AEX-Notify support

Documentation/manifest-syntax.rst

@@ -1346,3 +1346,20 @@ SGX EXINFO (deprecated syntax)
 This syntax specified whether a user application can retrieve faulting address
 in signal handler in case of a page fault. This syntax was renamed to
 ``sgx.use_exinfo``. The default value was ``false``.
+
+
+AEX-Notify
+^^^^^^^^^^
+
+::
+
+    sgx.experimental_enable_aex_notify = [true|false]
+    (Default: false)
+
+When enabled, this option instructs Gramine to use the AEX-Notify hardware feature,
+which is a security feature used to mitigate the `SGX-Step
+<https://lirias.kuleuven.be/retrieve/473223/>`__ type attackson SGX.
+The SGX-Step type attacks rely on frequent enclave preemptions (e.g. interrupts)
+to execute an SGX enclave in small increments and strategically extract secret data
+from the enclave through one or more side channels. The AEX-Notify feature allows
+a mitigation handler to run after each asynchronous exit (AEX).

pal/src/host/linux-sgx/generated_offsets.c

@@ -15,6 +15,7 @@ const struct generated_offset generated_offsets[] = {
     /* defines from sgx_arch.h */
     DEFINE(SGX_FLAGS_DEBUG, SGX_FLAGS_DEBUG),
     DEFINE(SGX_FLAGS_MODE64BIT, SGX_FLAGS_MODE64BIT),
+    DEFINE(SGX_FLAGS_AEXNOTIFY, SGX_FLAGS_AEXNOTIFY),
     DEFINE(SGX_XFRM_LEGACY, SGX_XFRM_LEGACY),
     DEFINE(SGX_XFRM_AVX, SGX_XFRM_AVX),
     DEFINE(SGX_XFRM_MPX, SGX_XFRM_MPX),
@@ -54,6 +55,7 @@ const struct generated_offset generated_offsets[] = {
     OFFSET_T(SGX_GPR_RFLAGS, sgx_pal_gpr_t, rflags),
     OFFSET_T(SGX_GPR_RIP, sgx_pal_gpr_t, rip),
     OFFSET_T(SGX_GPR_EXITINFO, sgx_pal_gpr_t, exitinfo),
+    OFFSET_T(SGX_GPR_AEXNOTIFY, sgx_pal_gpr_t, aexnotify),
     DEFINE(SGX_GPR_SIZE, sizeof(sgx_pal_gpr_t)),
 
     /* sgx_cpu_context_t */
@@ -103,6 +105,7 @@ const struct generated_offset generated_offsets[] = {
     OFFSET(SGX_HEAP_MIN, pal_enclave_tcb, heap_min),
     OFFSET(SGX_HEAP_MAX, pal_enclave_tcb, heap_max),
     OFFSET(SGX_CLEAR_CHILD_TID, pal_enclave_tcb, clear_child_tid),
+    OFFSET(SGX_READY_FOR_AEX_NOTIFY, pal_enclave_tcb, ready_for_aex_notify),
 
     /* struct pal_host_tcb aka PAL_HOST_TCB */
     OFFSET(PAL_HOST_TCB_TCS, pal_host_tcb, tcs),
@@ -123,6 +126,7 @@ const struct generated_offset generated_offsets[] = {
     OFFSET_T(TCS_OFS_LIMIT, sgx_arch_tcs_t, ofs_limit),
     OFFSET_T(TCS_OGS_LIMIT, sgx_arch_tcs_t, ogs_limit),
     DEFINE(TCS_SIZE, sizeof(sgx_arch_tcs_t)),
+    DEFINE(TCS_FLAGS_AEXNOTIFY, TCS_FLAGS_AEXNOTIFY),
 
     /* sgx_attributes_t */
     OFFSET_T(SGX_ATTRIBUTES_XFRM, sgx_attributes_t, xfrm),

pal/src/host/linux-sgx/host_framework.c

@@ -152,7 +152,7 @@ bool is_wrfsbase_supported(void) {
     uint32_t cpuinfo[4];
     cpuid(EXTENDED_FEATURE_FLAGS_LEAF, 0, cpuinfo);
 
-    if (!(cpuinfo[1] & 0x1)) {
+    if (!(cpuinfo[CPUID_WORD_EBX] & 0x1)) {
         log_error(
             "{RD,WR}{FS,GS}BASE instructions are not permitted on this platform. Please check the "
             "instructions under \"Building with SGX support\" from Gramine documentation.");
@@ -162,6 +162,24 @@ bool is_wrfsbase_supported(void) {
     return true;
 }
 
+bool is_aexnotify_supported(void) {
+    uint32_t cpuinfo[4];
+    cpuid(INTEL_SGX_LEAF, 1, cpuinfo);
+
+    if (!((cpuinfo[CPUID_WORD_EAX] >> 10) & 0x1)) {
+        log_debug("AEX-Notify hardware feature is not supported.");
+        return false;
+    }
+
+    cpuid(INTEL_SGX_LEAF, 0, cpuinfo);
+
+    if (!((cpuinfo[CPUID_WORD_EAX] >> 11) & 0x1)) {
+        log_debug("ENCLU[EDECCSSA] leaf instruction is not supported.");
+        return false;
+    }
+    return true;
+}
+
 int create_enclave(sgx_arch_secs_t* secs, sgx_arch_token_t* token) {
     assert(secs->size && IS_POWER_OF_2(secs->size));
     assert(IS_ALIGNED(secs->base, secs->size));

pal/src/host/linux-sgx/host_internal.h

@@ -62,6 +62,7 @@ void* realloc(void* ptr, size_t new_size);
 
 int open_sgx_driver(void);
 bool is_wrfsbase_supported(void);
+bool is_aexnotify_supported(void);
 
 int read_enclave_token(int token_file, sgx_arch_token_t* out_token);
 int create_dummy_enclave_token(sgx_sigstruct_t* sig, sgx_arch_token_t* out_token);

pal/src/host/linux-sgx/host_main.c

@@ -546,6 +546,8 @@ static int initialize_enclave(struct pal_enclave* enclave, const char* manifest_
                 gs->heap_min = (void*)enclave_heap_min;
                 gs->heap_max = (void*)pal_area->addr;
                 gs->thread = NULL;
+                /* below fields are used by AEX-Notify */
+                gs->ready_for_aex_notify = 0;
             }
         } else if (areas[i].data_src == TCS) {
             for (size_t t = 0; t < enclave->thread_num; t++) {
@@ -561,6 +563,8 @@ static int initialize_enclave(struct pal_enclave* enclave, const char* manifest_
                 tcs->ofs_limit = 0xfff;
                 tcs->ogs_limit = 0xfff;
                 tcs_addrs[t] = (void*)tcs_area->addr + g_page_size * t;
+                if (enclave_token.body.attributes.flags & SGX_FLAGS_AEXNOTIFY)
+                    tcs->flags |= TCS_FLAGS_AEXNOTIFY;
             }
         } else if (areas[i].data_src == BUF) {
             memcpy(data, areas[i].buf, areas[i].buf_size);

pal/src/host/linux-sgx/pal_exception.c

@@ -18,6 +18,29 @@
 
 #define ADDR_IN_PAL(addr) ((void*)(addr) > TEXT_START && (void*)(addr) < TEXT_END)
 
+bool g_aex_notify_enabled = false;
+
+void init_aex_notify_for_thread(void) {
+    if (!g_aex_notify_enabled)
+        return;
+
+    SET_ENCLAVE_TCB(ready_for_aex_notify, 1UL);
+    MB();
+
+    GET_ENCLAVE_TCB(gpr)->aexnotify = 1;
+}
+
+void fini_aex_notify_for_thread(void) {
+    if (!g_aex_notify_enabled)
+        return;
+
+    SET_ENCLAVE_TCB(ready_for_aex_notify, 0UL);
+    MB();
+
+    GET_ENCLAVE_TCB(gpr)->aexnotify = 0;
+}
+
+
 /* Restore an sgx_cpu_context_t as generated by .Lhandle_exception. Execution will
  * continue as specified by the rip in the context. */
 __attribute_no_sanitize_address

pal/src/host/linux-sgx/pal_linux.h

@@ -96,6 +96,10 @@ void save_xregs(PAL_XREGS_STATE* xsave_area);
 void restore_xregs(const PAL_XREGS_STATE* xsave_area);
 noreturn void _restore_sgx_context(sgx_cpu_context_t* uc, PAL_XREGS_STATE* xsave_area);
 
+extern bool g_aex_notify_enabled;
+void init_aex_notify_for_thread(void);
+void fini_aex_notify_for_thread(void);
+
 void _PalExceptionHandler(uint32_t trusted_exit_info_,
                           uint32_t untrusted_external_event, sgx_cpu_context_t* uc,
                           PAL_XREGS_STATE* xregs_state, sgx_arch_exinfo_t* exinfo);

pal/src/host/linux-sgx/pal_main.c

@@ -852,6 +852,13 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void*
         do_preheat_enclave();
     }
 
+    ret = toml_bool_in(g_pal_public_state.manifest_root, "sgx.experimental_enable_aex_notify",
+                       /*defaultval=*/false, &g_aex_notify_enabled);
+    if (ret < 0) {
+        log_error("Cannot parse 'sgx.experimental_enable_aex_notify' (the value must be `true` or `false`)");
+        ocall_exit(1, /*is_exitgroup=*/true);
+    }
+
     if ((ret = init_seal_key_material()) < 0) {
         log_error("Failed to initialize SGX sealing key material: %s", pal_strerror(ret));
         ocall_exit(1, /*is_exitgroup=*/true);
@@ -922,6 +929,7 @@ noreturn void pal_linux_main(void* uptr_libpal_uri, size_t libpal_uri_len, void*
     assert(!g_pal_linuxsgx_state.enclave_initialized);
     g_pal_linuxsgx_state.enclave_initialized = true;
 
+    init_aex_notify_for_thread();
     /* call main function */
     pal_main(instance_id, parent, first_thread, arguments, environments, post_callback);
 }

pal/src/host/linux-sgx/pal_tcb.h

@@ -47,6 +47,9 @@ struct pal_enclave_tcb {
     void*     heap_max;
     int*      clear_child_tid;
     struct untrusted_area untrusted_area_cache;
+
+    /* below fields are used by AEX-Notify */
+    uint64_t  ready_for_aex_notify; /* set when it is ready to enable AEX-Notify */
 };
 
 #ifndef DEBUG

pal/src/host/linux-sgx/pal_threading.c

@@ -160,6 +160,7 @@ void pal_start_thread(void) {
     pal_set_tcb_stack_canary(stack_protector_canary);
     PAL_TCB* pal_tcb = pal_get_tcb();
     memset(&pal_tcb->libos_tcb, 0, sizeof(pal_tcb->libos_tcb));
+    init_aex_notify_for_thread();
     callback((void*)param);
     _PalThreadExit(/*clear_child_tid=*/NULL);
     /* UNREACHABLE */
@@ -226,7 +227,7 @@ void _PalThreadYieldExecution(void) {
 /* _PalThreadExit for internal use: Thread exiting */
 noreturn void _PalThreadExit(int* clear_child_tid) {
     struct pal_handle_thread* exiting_thread = GET_ENCLAVE_TCB(thread);
-
+    fini_aex_notify_for_thread();
     /* thread is ready to exit, must inform LibOS by erasing clear_child_tid;
      * note that we don't do it now (because this thread still occupies SGX
      * TCS slot) but during handle_thread_reset in assembly code */

pal/src/host/linux-sgx/sgx_arch.h

@@ -66,6 +66,7 @@ typedef uint8_t sgx_isvfamily_id_t[SGX_ISV_FAMILY_ID_SIZE];
 #define SGX_FLAGS_MODE64BIT     0x04ULL
 #define SGX_FLAGS_PROVISION_KEY 0x10ULL
 #define SGX_FLAGS_LICENSE_KEY   0x20ULL
+#define SGX_FLAGS_AEXNOTIFY     0x400ULL
 
 /* EINIT must verify *all* SECS.ATTRIBUTES[63..0] bits (FLAGS bits) against
  * SIGSTRUCT.ATTRIBUTES[63..0].
@@ -169,6 +170,7 @@ typedef struct {
 static_assert(sizeof(sgx_arch_tcs_t) == 4096, "incorrect struct size");
 
 #define TCS_FLAGS_DBGOPTIN (01ULL)
+#define TCS_FLAGS_AEXNOTIFY (02ULL)
 
 typedef struct {
     uint64_t rax;
@@ -192,10 +194,12 @@ typedef struct {
     uint64_t ursp;
     uint64_t urbp;
     uint32_t exitinfo;
-    uint32_t reserved;
+    uint8_t  reserved[3];
+    uint8_t  aexnotify;
     uint64_t fsbase;
     uint64_t gsbase;
 } sgx_pal_gpr_t;
+static_assert(offsetof(sgx_pal_gpr_t, aexnotify) == 167, "The offset of aexnotify must be 167 in SSA.");
 
 typedef struct {
     uint64_t rax;
@@ -435,6 +439,7 @@ static inline int enclu(uint32_t eax, uint64_t rbx, uint64_t rcx, uint64_t rdx)
 #define EACCEPT     5
 #define EMODPE      6
 #define EACCEPTCOPY 7
+#define EDECCSSA    9
 
 #define SGX_LAUNCH_KEY         0
 #define SGX_PROVISION_KEY      1

python/graminelibos/manifest.py

@@ -308,6 +308,7 @@ def __init__(self, manifest_str):
         sgx.setdefault('debug', False)
         sgx.setdefault('enable_stats', False)
         sgx.setdefault('edmm_enable', False)
+        sgx.setdefault('experimental_enable_aex_notify', False)
 
         if sgx['edmm_enable']:
             sgx.setdefault('enclave_size', DEFAULT_ENCLAVE_SIZE_WITH_EDMM)
@@ -330,6 +331,8 @@ def __init__(self, manifest_str):
             sgx_cpu_features.setdefault('mpx', "disabled")
             sgx_cpu_features.setdefault('pkru', "disabled")
 
+        sgx.setdefault('enable_aex_notify', False)
+
         if not isinstance(sgx['trusted_files'], list):
             raise ValueError("Unsupported trusted files syntax, more info: " +
                   "https://gramine.readthedocs.io/en/latest/manifest-syntax.html#trusted-files")

python/graminelibos/sgx_sign.py

@@ -100,6 +100,7 @@ def collect_cpu_feature_bits(manifest_cpu_features, options_dict, val, mask, sec
 def get_enclave_attributes(manifest_sgx):
     flags_dict = {
         'debug': offs.SGX_FLAGS_DEBUG,
+        'experimental_enable_aex_notify': offs.SGX_FLAGS_AEXNOTIFY,
     }
     flags = collect_bits(manifest_sgx, flags_dict)
     if ARCHITECTURE == 'amd64':
@@ -290,6 +291,8 @@ def set_tls_field(t, offset, value):
         set_tcs_field(t, offs.TCS_OGS_BASE, '<Q', tls_area.addr - enclave_base + offs.PAGESIZE * t)
         set_tcs_field(t, offs.TCS_OFS_LIMIT, '<L', 0xfff)
         set_tcs_field(t, offs.TCS_OGS_LIMIT, '<L', 0xfff)
+        if (attr['flags'] & offs.SGX_FLAGS_AEXNOTIFY):
+            set_tcs_field(t, offs.TCS_FLAGS, '<Q', offs.TCS_FLAGS_AEXNOTIFY)
 
         set_tls_field(t, offs.SGX_COMMON_SELF, tls_area.addr + offs.PAGESIZE * t)
         set_tls_field(t, offs.SGX_COMMON_STACK_PROTECTOR_CANARY,

tools/sgx/is-sgx-available/is_sgx_available.cpp

@@ -73,6 +73,8 @@ class SgxCpuChecker {
     bool sgx_mem_concurrency_supported_ = false;
     bool cet_supported_ = false;
     bool kss_supported_ = false;
+    bool aex_notify_supported_ = false;
+    bool edeccssa_supported_ = false;
     bool miscselect_exinfo_pfgp_supported_ = false;
     bool miscselect_exinfo_cp_supported_ = false;
     uint64_t maximum_enclave_size_x86_ = 0;
@@ -125,6 +127,8 @@ class SgxCpuChecker {
         miscselect_exinfo_cp_supported_ = cpuid_12_0_ebx & (1 << 1);
         cet_supported_ = cpuid_12_1_eax & (1 << 6);
         kss_supported_ = cpuid_12_1_eax & (1 << 7);
+        aex_notify_supported_ = cpuid_12_1_eax & (1 << 10);
+        edeccssa_supported_ = cpuid_12_0_eax & (1 << 11);
         maximum_enclave_size_x86_ = saturating_exp2<uint64_t>(cpuid_12_0_edx & 0xFF);
         maximum_enclave_size_x64_ = saturating_exp2<uint64_t>((cpuid_12_0_edx >> 8) & 0xFF);
         // Check if there's any EPC region allocated by BIOS
@@ -161,6 +165,9 @@ class SgxCpuChecker {
     bool cet_supported() const { return cet_supported_; }
     // Key separation and sharing (KSS) support (CONFIGID, CONFIGSVN, ISVEXTPRODID, ISVFAMILYID report fields)
     bool kss_supported() const { return kss_supported_; }
+    // AEX-Notify support
+    bool aex_notify_supported() const { return aex_notify_supported_; }
+    bool edeccssa_supported() const { return edeccssa_supported_; }
     // Fields of MISC region of State Save Area (see Table 34-12 in the SMD).
     bool miscselect_exinfo_pfgp_supported() const { return miscselect_exinfo_pfgp_supported_; }
     bool miscselect_exinfo_cp_supported() const { return miscselect_exinfo_cp_supported_; }
@@ -210,6 +217,8 @@ void print_detailed_info(const SgxCpuChecker& cpu_checker) {
            bool2str(cpu_checker.cet_supported()));
     printf("Key separation and sharing (KSS) support (CONFIGID, CONFIGSVN, ISVEXTPRODID, "
            "ISVFAMILYID report fields): %s\n", bool2str(cpu_checker.kss_supported()));
+    printf("AEX-Notify support: %s\n", bool2str(cpu_checker.aex_notify_supported()));
+    printf("AEX-Notify (EDECCSSA) support: %s\n", bool2str(cpu_checker.edeccssa_supported()));
     printf("Max enclave size (32-bit): 0x%" PRIx64 "\n", cpu_checker.maximum_enclave_size_x86());
     printf("Max enclave size (64-bit): 0x%" PRIx64 "\n", cpu_checker.maximum_enclave_size_x64());
     printf("EPC size: 0x%" PRIx64 "\n", cpu_checker.epc_region_size());