integrating gRPC and RA-TLS cert and key issue

[lili8808]

I am going to call ra_tls_create_key_and_crt_der from my gRPC server.py for this aim I did this:

if __name__ == '__main__':
    libra_tls_attest = ctypes.CDLL(os.path.abspath("/usr/lib/x86_64-linux-gnu/gramine/runtime/glibc/libra_tls_attest.so"))
libra_tls_attest.ra_tls_create_key_and_crt_der.argtypes = (ctypes.POINTER(ctypes.c_char_p), ctypes.POINTER(ctypes.c_size_t),ctypes.POINTER(ctypes.c_char_p), ctypes.POINTER(ctypes.c_size_t))    
libra_tls_attest.ra_tls_create_key_and_crt_der.restype = ctypes.c_int
# Create the input and output variables
    der_key = ctypes.c_char_p()
    der_key_size = ctypes.c_size_t()
    der_crt = ctypes.c_char_p()
    der_crt_size = ctypes.c_size_t()
    ret = libra_tls_attest.ra_tls_create_key_and_crt_der(der_key, der_key_size, der_crt, der_crt_size)
if ret != 0:
    print("Error: ra_tls_create_key_and_crt_der returned", ret)
else:
    key_bytes =bytes(der_key.value)
    crt_bytes = bytes(der_key.value)
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    # create gRPC server credentials
    server_credentials = grpc.ssl_server_credentials(
        ((key_bytes, crt_bytes), ))
    # add the servicer created above to the server
    service_pb2_grpc.add_serviceContractServicer_to_server(serviceExecutionImp(), server)
    # listen on port 50051
    print('Starting gRPC server. Listening on port 50051.')
    server.add_secure_port('localhost:50051', server_credentials)
    server.start()
    # since server.start() will not block,
    # a sleep-loop is added to keep alive
    try:
        while True:
            time.sleep(86400)
    except KeyboardInterrupt:
        server.stop(0) 

my server.py is going to create RA-TLS certificate and key and use them to create credenitials for creating secure channel in gRPC.

added to these this is my python.manifest.template:

loader.entrypoint = "file:{{ gramine.libos }}"
libos.entrypoint = "{{ entrypoint }}"

loader.log_level = "{{ log_level }}"

loader.env.LD_LIBRARY_PATH = "{{ python.stdlib }}/lib:/lib:{{ arch_libdir }}:/usr/lib:/usr/{{ arch_libdir }}"

loader.insecure__use_cmdline_argv = true
loader.pal_internal_mem_size = "64M"
sys.enable_sigterm_injection = true

fs.mounts = [
  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
  { path = "/usr", uri = "file:/usr" },
  { path = "{{ python.stdlib }}", uri = "file:{{ python.stdlib }}" },
  { path = "{{ python.distlib }}", uri = "file:{{ python.distlib }}" },
  { path = "/etc", uri = "file:/etc" },

  { type = "tmpfs", path = "/tmp" },
]

sys.stack.size = "2M"
sys.enable_extra_runtime_domain_names_conf = true

sgx.debug = false
sgx.nonpie_binary = true
sgx.enclave_size = "512M"
sgx.thread_num = 32

sgx.remote_attestation = "{{ ra_type }}"
sgx.ra_client_spid = "{{ ra_client_spid }}"
sgx.ra_client_linkable = {{ 'true' if ra_client_linkable == '1' else 'false' }}

sgx.trusted_files = [
  "file:{{ gramine.libos }}",
  "file:{{ entrypoint }}",
  "file:{{ gramine.runtimedir() }}/",
  "file:{{ arch_libdir }}/",
  "file:/usr/{{ arch_libdir }}/",
  "file:{{ python.stdlib }}/",
  "file:{{ python.distlib }}/",
  "file:DataContractLogic.py",
  "file:/usr/local/lib/python3.8/dist-packages/grpc/",
  "file:/usr/local/lib/python3.8/dist-packages/grpc_tools/",
  "file:/usr/local/lib/python3.8/dist-packages/google/protobuf/",
  "file:/etc/mime.types",
  "file:/etc/default/apport",
  "file:server.py",
  "file:/usr/local/lib/python3.8/dist-packages/google/__init__.py/",
  "file:/usr/lib/python3/dist-packages/mysql/",
]
sgx.allowed_files = [
  "file:/etc/nsswitch.conf",
  "file:/etc/ethers",
  "file:/etc/hosts",
  "file:/etc/group",
  "file:/etc/passwd",
  "file:/etc/gai.conf"
] ```

this is the result of execution:

gramine-sgx ./python server.py
Gramine is starting. Parsing TOML manifest file, this may take some time...
Detected a huge manifest, preallocating 64MB of internal memory.
-----------------------------------------------------------------------------------------------------------------------
Gramine detected the following insecure configurations:

  - loader.insecure__use_cmdline_argv = true   (forwarding command-line args from untrusted host to the app)
  - sgx.allowed_files = [ ... ]                (some files are passed through from untrusted host without verification)

Gramine will continue application execution, but this configuration must not be used in production!
-----------------------------------------------------------------------------------------------------------------------

Detected a huge manifest, preallocating 64MB of internal memory.
E0117 12:31:22.049287000       1 ssl_transport_security.cc:846]        Invalid cert chain file.
E0117 12:31:22.061513000       1 ssl_security_connector.cc:270]        Handshaker factory creation failed with TSI_INVALID_ARGUMENT.
E0117 12:31:22.067049000       1 chttp2_server.cc:1045]                UNKNOWN:Unable to create secure server with credentials of type Ssl {created_time:"2023-01-17T12:31:22.061763+00:00"}
server.py:17: DeprecationWarning: the imp module is deprecated in favour of importlib; see the module's documentation for alternative uses
  import imp
c_char_p(130237808)
Starting gRPC server. Listening on port 50051.
Traceback (most recent call last):
  File "server.py", line 85, in <module>
    server.add_secure_port('localhost:50051', server_credentials)
  File "/usr/local/lib/python3.8/dist-packages/grpc/_server.py", line 974, in add_secure_port
    return _common.validate_port_binding_result(
  File "/usr/local/lib/python3.8/dist-packages/grpc/_common.py", line 175, in validate_port_binding_result
    raise RuntimeError(_ERROR_MESSAGE_PORT_BINDING_FAILED % address)
RuntimeError: Failed to bind to address localhost:50051; set GRPC_VERBOSITY=debug environment variable to see detailed error message.
Error in sys.excepthook:
Traceback (most recent call last):
  File "/usr/lib/python3/dist-packages/apport_python_hook.py", line 72, in apport_excepthook
    from apport.fileutils import likely_packaged, get_recent_crashes
  File "/usr/lib/python3/dist-packages/apport/__init__.py", line 5, in <module>
    from apport.report import Report
  File "/usr/lib/python3/dist-packages/apport/report.py", line 32, in <module>
    import apport.fileutils
  File "/usr/lib/python3/dist-packages/apport/fileutils.py", line 27, in <module>
    from apport.packaging_impl import impl as packaging
  File "/usr/lib/python3/dist-packages/apport/packaging_impl.py", line 23, in <module>
    import apt
  File "/usr/lib/python3/dist-packages/apt/__init__.py", line 36, in <module>
    apt_pkg.init_system()
apt_pkg.Error: E:Error reading the CPU table

Original exception was:
Traceback (most recent call last):
  File "server.py", line 85, in <module>
    server.add_secure_port('localhost:50051', server_credentials)
  File "/usr/local/lib/python3.8/dist-packages/grpc/_server.py", line 974, in add_secure_port
    return _common.validate_port_binding_result(
  File "/usr/local/lib/python3.8/dist-packages/grpc/_common.py", line 175, in validate_port_binding_result
    raise RuntimeError(_ERROR_MESSAGE_PORT_BINDING_FAILED % address)
RuntimeError: Failed to bind to address localhost:50051; set GRPC_VERBOSITY=debug environment variable to see detailed error message.

[dimakuv]

@lili8808 Could you please render your text using Markdown "code snippet" formatting? I can't read this.

[lili8808]

@dimakuv sorry for inconvinient. Done

[dimakuv]

Thanks, now it's much easier to read and follow!

[dimakuv]

@lili8808 I think you have two bugs here:

    key_bytes =bytes(der_key.value)
    crt_bytes = bytes(der_key.value)
    ...
    server_credentials = grpc.ssl_server_credentials(
        ((key_bytes, crt_bytes), ))

1. Line 2 is definitely a bug -- you're assigning der_key (the private key value) to the crt_bytes variable. It should have been der_crt.value.

2. RA-TLS produces the private key (der_key) and the corresponding certificate (der_crt) in a DER-format. However, grpc.ssl_server_credentials() requires the key and the certificate to be in PEM format, according to this page.

   • You need to re-format der_key and der_crt into pem_key and pem_crt somehow, before passing them to grpc.ssl_server_credentials().
   • You should probably use smth like this: https://cryptography.io/en/latest/hazmat/primitives/asymmetric/serialization/

[dimakuv]

@lili8808 Are you using Gramine v1.3.1? We recently fixed this SO_REUSEPORT issue, so the latest Gramine version (master branch) must work correctly. Please try master.

[lili8808]

@dimakuv Thanks , should I install it with Building the master branch? because for me building from master branch has error.

[dimakuv]

@lili8808 Yes. I don't remember if you're using MS Azure VMs (which have DCAP attestation scheme)? If yes, then a simple way to build Gramine from the master branch is something like this:

# make sure you sudo-apt installed all prerequisites

# on MS Azure Ubuntu 20.04, I had to do this
cd lib/x86_64-linux-gnu/
sudo ln -s libsgx_dcap_quoteverify.so.1 libsgx_dcap_quoteverify.so

git clone https://github.com/gramineproject/gramine.git
cd gramine
meson setup build/ --buildtype=debug -Ddirect=enabled -Dsgx=enabled -Dsgx_driver=upstream -Dtests=enabled -Ddcap=enabled
ninja -C build/
sudo ninja -C build/ install

[lili8808]

@dimakuv Thanks for you response yes I am using MS Azure VMS,
when I run make SGX=1 RA_TYPE=dcap I get this error at the end:

Measurement:
    90fb9f5181df46d7aa1c68f8956f5b85a865305e71b1ea422263c7787c5bc03a
gramine-sgx-get-token --output python.token --sig python.sig
/usr/local/bin/gramine-sgx-get-token:17: DeprecationWarning: gramine-sgx-get-token is deprecated on upstream SGX driver, and calling it will be a hard error in the future
  warnings.warn(

[dimakuv]

You can ignore this warning. It is not an error.

[lili8808]

Thanks, I have to mention that before using master branch MRENCLAVE and MRSIGNER were shown while building, now it does not show me the values of MRSIGNER, MRENCLAVE ,... when building, so my question is how can I get these values?

[lili8808]

from graminelibos import Sigstruct
import hashlib

replace file name with your application's .sig file

with open("/home/dimakuv/gramineproject/gramine/CI-Examples/helloworld/helloworld.sig", mode='rb') as file:
sig = Sigstruct.from_bytes(file.read())

mrsigner = hashlib.sha256()
mrsigner.update(sig['modulus'])
mrsigner = mrsigner.hexdigest()

print(mrsigner)
print(sig['enclave_hash'].hex())
print(sig['isv_prod_id'])
print(sig['isv_svn'])

Thanks so Much for your support.

[dimakuv]

@lili8808 Did the whole thing work?

I will be very inrerested in results of:

• your C -> Python wrapper
• your Python gRPC plugin

[lili8808]

@lili8808 Did the whole thing work?

I will be very inrerested in results of:

• your C -> Python wrapper
• your Python gRPC plugin

currently starting the gRPC server inside SGX using gramine-sgx works , The next step will be start the gRPC client and doing the verification.
this is gRPC server.py which is using RA-TLS cert and key:

if __name__ == '__main__':
    # calling ra_tls_create_key_and_crt_der from library libra_tls_attest
    libra_tls_attest = ctypes.CDLL(os.path.abspath("/usr/local/lib/x86_64-linux-gnu/gramine/runtime/glibc/libra_tls_attest.so"))
    libra_tls_attest.ra_tls_create_key_and_crt_der.argtypes = (ctypes.POINTER(ctypes.POINTER(ctypes.c_uint8)), ctypes.POINTER(ctypes.c_size_t),ctypes.POINTER(ctypes.POINTER(ctypes.c_uint8)), ctypes.POINTER(ctypes.c_size_t))
    libra_tls_attest.ra_tls_create_key_and_crt_der.restype = ctypes.c_int

# Create the input and output variables
    der_key_size = ctypes.c_size_t()
    der_crt_size = ctypes.c_size_t()
    der_key = ctypes.POINTER(ctypes.c_uint8)()
    der_crt = ctypes.POINTER(ctypes.c_uint8)()
# Call the function
    ret=libra_tls_attest.ra_tls_create_key_and_crt_der(ctypes.byref(der_key), 
        ctypes.byref(der_key_size), ctypes.byref(der_crt), ctypes.byref(der_crt_size))
# Check the return value
if ret != 0:
    print("Error: ra_tls_create_key_and_crt_der returned", ret)
else:
    # convert der_cert and der_key to bytes
    cert_bytes = bytes(ctypes.string_at(der_crt, der_crt_size.value))
    key_bytes = bytes(ctypes.string_at(der_key, der_key_size.value))
    # Use the output variables
    cert = x509.load_der_x509_certificate(cert_bytes, backend=default_backend())
    pem_cert = cert.public_bytes(encoding=serialization.Encoding.PEM)

    key = serialization.load_der_private_key(key_bytes, None, default_backend())
    pem_key = key.private_bytes(encoding=serialization.Encoding.PEM,format=serialization.PrivateFormat.PKCS8,encryption_algorithm=serialization.NoEncryption()) 
    # create a gRPC server
    server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
    # create gRPC server credentials
    server_credentials = grpc.ssl_server_credentials(
        ((pem_key, pem_cert), ))

    # add the servicer created above to the server
    service_pb2_grpc.add_ExecserviceServicer_to_server(serviceExecutionImp(), server)
    # listen on port 50051
    print('Starting gRPC server. Listening on port 50051.',flush=True)
    server.add_secure_port('localhost:50051', server_credentials)
    server.start()
    # since server.start() will not block,
    # a sleep-loop is added to keep alive
    try:
        while True:
            time.sleep(86400)
    except KeyboardInterrupt:
        server.stop(0)

which serviceExecutionImp class should be implemented for services that we have in the service.proto.

[dimakuv]

Thanks for the code snippet! I hope you'll also share your gRPC client integration with us!

[lili8808]

Thanks for the code snippet! I hope you'll also share your gRPC client integration with us!

No worries, thanks for you support, sure once I done I'll share it.