Flatten LibOS/shim/ directory to just LibOS/

Signed-off-by: Michał Kowalczyk <[email protected]>

.ci/lib/stage-clean-check.jenkinsfile

@@ -20,13 +20,13 @@ stage('clean-check') {
         # root, and keeps cache there
         rm -rf .pytest_cache
 
-        make -C LibOS/shim/test/regression clean
-        make -C LibOS/shim/test/fs clean
+        make -C LibOS/test/regression clean
+        make -C LibOS/test/fs clean
 
-        gramine-test -C LibOS/shim/test/abi/x86_64 clean
-        rm -rf LibOS/shim/test/abi/x86_64/.pytest_cache \
-               LibOS/shim/test/abi/x86_64/__pycache__ \
-               LibOS/shim/test/abi/x86_64/*.xml
+        gramine-test -C LibOS/test/abi/x86_64 clean
+        rm -rf LibOS/test/abi/x86_64/.pytest_cache \
+               LibOS/test/abi/x86_64/__pycache__ \
+               LibOS/test/abi/x86_64/*.xml
 
         make -C Pal/regression clean
 
@@ -45,10 +45,10 @@ stage('clean-check') {
         make -C CI-Examples/ra-tls-mbedtls distclean
         make -C CI-Examples/ra-tls-secret-prov distclean
 
-        make -C LibOS/shim/test/ltp clean
+        make -C LibOS/test/ltp clean
         # LTP's make clean is broken, see https://github.com/linux-test-project/ltp/issues/559
-        rm -rf /tmp/*.clean-check.clean/LibOS/shim/test/ltp/src
-        rm -rf LibOS/shim/test/ltp/src
+        rm -rf /tmp/*.clean-check.clean/LibOS/test/ltp/src
+        rm -rf LibOS/test/ltp/src
 
         # python/ subdir does not have makefiles, so no clean
         rm -rf /tmp/*.clean-check.clean/python/graminelibos/__pycache__

.ci/lib/stage-test-direct.jenkinsfile

@@ -2,20 +2,20 @@ stage('test-direct') {
     try {
         timeout(time: 20, unit: 'MINUTES') {
             sh '''
-                cd LibOS/shim/test/ltp
+                cd LibOS/test/ltp
                 make ${MAKEOPTS} all
             '''
             // Run tests in a separate block, so that Jenkins measures build time and run time
             // separately
             sh '''
-                cd LibOS/shim/test/ltp
+                cd LibOS/test/ltp
                 export GRAMINE_LTP_LIVE_OUTPUT=fcntl14,fdatasync01
                 python3 -m pytest -v -n4 --junit-xml=ltp.xml
             '''
         }
     } finally {
-        archiveArtifacts 'LibOS/shim/test/ltp/ltp.xml'
-        junit 'LibOS/shim/test/ltp/ltp.xml'
+        archiveArtifacts 'LibOS/test/ltp/ltp.xml'
+        junit 'LibOS/test/ltp/ltp.xml'
     }
     timeout(time: 5, unit: 'MINUTES') {
         sh '''

.ci/lib/stage-test.jenkinsfile

@@ -14,12 +14,12 @@ stage('test') {
     timeout(time: 15, unit: 'MINUTES') {
         try {
             sh '''
-                cd LibOS/shim/test/regression
+                cd LibOS/test/regression
                 RA_CLIENT_SPID=${ra_client_spid} gramine-test build -v
                 python3 -m pytest -v --junit-xml libos-regression.xml
             '''
         } finally {
-            junit 'LibOS/shim/test/regression/libos-regression.xml'
+            junit 'LibOS/test/regression/libos-regression.xml'
         }
     }
 
@@ -28,39 +28,39 @@ stage('test') {
             try {
                 sh '''
                     export GRAMINE_MUSL=1
-                    cd LibOS/shim/test/regression
+                    cd LibOS/test/regression
                     # For some unknown reason it fails without this clean on sgx-18.04 pipeline
                     gramine-test clean
                     RA_CLIENT_SPID=${ra_client_spid} gramine-test -n tests_musl.toml build -v
                     python3 -m pytest -v --junit-xml libos-regression-musl.xml
                 '''
             } finally {
-                junit 'LibOS/shim/test/regression/libos-regression-musl.xml'
+                junit 'LibOS/test/regression/libos-regression-musl.xml'
             }
         }
     }
 
     timeout(time: 15, unit: 'MINUTES') {
         try {
             sh '''
-                cd LibOS/shim/test/abi/x86_64
+                cd LibOS/test/abi/x86_64
                 gramine-test build -v
                 python3 -m pytest -v --junit-xml abi.xml
             '''
         } finally {
-            junit 'LibOS/shim/test/abi/x86_64/*.xml'
+            junit 'LibOS/test/abi/x86_64/*.xml'
         }
     }
 
     timeout(time: 15, unit: 'MINUTES') {
         try {
             sh '''
-                cd LibOS/shim/test/fs
+                cd LibOS/test/fs
                 gramine-test build -v
                 python3 -m pytest -v --junit-xml fs.xml
             '''
         } finally {
-            junit 'LibOS/shim/test/fs/*.xml'
+            junit 'LibOS/test/fs/*.xml'
         }
     }
 }

.ci/run-pylint

@@ -11,9 +11,9 @@ if [ -z "$PYLINT" ]; then
 fi
 
 find . -name \*.py \
-    -and -not -path ./LibOS/shim/test/ltp/src/\* \
-    -and -not -path ./LibOS/shim/test/ltp/build/\* \
-    -and -not -path ./LibOS/shim/test/ltp/install/\* \
+    -and -not -path ./LibOS/test/ltp/src/\* \
+    -and -not -path ./LibOS/test/ltp/build/\* \
+    -and -not -path ./LibOS/test/ltp/install/\* \
     -and -not -name ninja_syntax.py \
 | sed 's/./\\&/g' \
 | xargs "${PYLINT}" "$@" \

.gitmodules

@@ -1,3 +1,3 @@
-[submodule "LibOS/shim/test/ltp/src"]
-	path = LibOS/shim/test/ltp/src
+[submodule "LibOS/test/ltp/src"]
+	path = LibOS/test/ltp/src
 	url = https://github.com/linux-test-project/ltp.git

CONTRIBUTING.rst

@@ -187,7 +187,7 @@ directory), and this is the ultimate documentation for application-level
 regression tests, although most tests can be run with :command:`gramine-test`,
 or, in the worst case, should have a simple script called by Jenkins.
 
-We also have (and are actively growing) PAL and shim unit tests.
+We also have (and are actively growing) PAL and LibOS unit tests.
 
 In order to run tests, Gramine must be installed. The test binaries, which are
 also built by Meson, must be installed as well. To do that, configure your build
@@ -230,8 +230,8 @@ or build a manifest and then run the binary directly::
 For more information, run :command:`gramine-test --help` and
 :command:`gramine-test <command> --help`.
 
-The shim unit tests work similarly, and are under
-:file:`LibOS/shim/test/regression`.
+The LibOS unit tests work similarly, and are under
+:file:`LibOS/test/regression`.
 
 LTP
 ^^^
@@ -241,7 +241,7 @@ currently only supported on the Linux PAL.
 
 To run these tests::
 
-   cd LibOS/shim/test/ltp
+   cd LibOS/test/ltp
    # consider -j$(nproc) or similar to parallelize and improve the build time.
    make
    make regression
@@ -251,7 +251,7 @@ To run these tests::
    python3 -m pytest -v -k chmod01
 
 For more information on how to run the ltp tests, please refer to
-:file:`LibOS/shim/test/ltp/README.rst`.
+:file:`LibOS/test/ltp/README.rst`.
 
 
 Management Team

Documentation/Doxyfile-libos

@@ -2,5 +2,5 @@
 PROJECT_NAME = "LibOS"
 OUTPUT_DIRECTORY = _build/doxygen-libos
 INPUT = \
-    ../LibOS/shim/include \
-    ../LibOS/shim/src
+    ../LibOS/include \
+    ../LibOS/src

Documentation/attestation.rst

@@ -165,8 +165,8 @@ locking between processes when setting the user report data/target info or
 reading the report/quote.
 
 An example of this low-level interface can be found under
-``LibOS/shim/test/regression/attestation.c``. Here is a C code snippet of how
-the remote attestation flow may look like in your application::
+``LibOS/test/regression/attestation.c``. Here is a C code snippet of how the
+remote attestation flow may look like in your application::
 
     sgx_report_data_t user_report_data = {0};
     memcpy(&user_report_data, "some-dummy-data", sizeof("some-dummy-data"));

Documentation/devel/coding-style.rst

@@ -190,8 +190,8 @@ Meson
 #. No changing (overwriting) variables in different :file:`meson.build` than it
    was defined in. If you really need to do this, create a temporary variable
    in subdir and use it in the parent :file:`meson.build`. You can check
-   ``libos_sources_arch`` in :file:`LibOS/shim/src/meson.build` for example
-   usage of this pattern (appending arch-specific source files to a list).
+   ``libos_sources_arch`` in :file:`LibOS/src/meson.build` for example usage of
+   this pattern (appending arch-specific source files to a list).
 
 #. Variables named ``_prog`` refer to things obtained from ``find_program()``.
    Auxiliary commands should reside in ``Scripts/``, and the variable name is

Documentation/devel/new-syscall.rst

@@ -22,9 +22,9 @@ Now you need to add an appropriate entry in the syscalls table in
 ------------------------
 
 You can add the function body of ``shim_do_sched_setaffinity`` in a new source
-file or any existing source file in :file:`LibOS/shim/src/sys`.
+file or any existing source file in :file:`LibOS/src/sys`.
 
-For example, in :file:`LibOS/shim/src/sys/shim_sched.c`::
+For example, in :file:`LibOS/src/sys/shim_sched.c`::
 
    long shim_do_sched_setaffinity(pid_t pid, unsigned int len, unsigned long* user_mask_ptr) {
       /* code for implementing the semantics of sched_setaffinity */

Documentation/devel/onboarding.rst

@@ -351,11 +351,11 @@ fine on native Linux but fails under Gramine::
 
    - The first option is to find an already existing Pal/LibOS regression test
      that works with the buggy Gramine subsystem. For example, if you found a
-     bug in the UDP stack, look at the ``LibOS/shim/test/regression/udp.c``
-     test. Find a place in this test where you can add the code that triggers
-     the bug. Also add the corresponding check (if needed) in the Python test
-     script (``LibOS/shim/test/regression/test_libos.py`` in case of LibOS
-     regression tests).
+     bug in the UDP stack, look at the ``LibOS/test/regression/udp.c`` test.
+     Find a place in this test where you can add the code that triggers the bug.
+     Also add the corresponding check (if needed) in the Python test script
+     (``LibOS/test/regression/test_libos.py`` in case of LibOS regression
+     tests).
 
    - The second option is to add a completely new test. Sometimes the bug
      reproduction code is too big or too specific to go into one of the already
@@ -367,7 +367,7 @@ fine on native Linux but fails under Gramine::
      some LTP tests are currently disabled because they are known to trigger
      some bugs or unimplemented functionality. When you fix a bug, try to find
      LTP tests that were affected by this bug and re-enable them (see
-     ``gramine/LibOS/shim/test/ltp/ltp.cfg`` file for the list of LTP tests).
+     ``gramine/LibOS/test/ltp/ltp.cfg`` file for the list of LTP tests).
 
    - The last option is to *not* add any new tests. This option is quite rare,
      but can be used in case of hard-to-reproduce bugs. For example, some bugs
@@ -391,22 +391,22 @@ fine on native Linux but fails under Gramine::
          $ gramine-test --sgx pytest -v
 
          # build and run LibOS regression tests
-         $ cd LibOS/shim/test/regression
+         $ cd LibOS/test/regression
          $ gramine-test pytest -v
          $ gramine-test --sgx pytest -v
 
          # build and run LibOS ABI tests (currently only for x86_64)
-         $ cd LibOS/shim/test/abi/${arch}
+         $ cd LibOS/test/abi/${arch}
          $ gramine-test pytest -v
          $ gramine-test --sgx pytest -v
 
          # build and run LibOS FS tests
-         $ cd LibOS/shim/test/fs
+         $ cd LibOS/test/fs
          $ gramine-test pytest -v
          $ gramine-test --sgx pytest -v
 
          # build and run LTP tests (only in non-SGX mode)
-         $ cd LibOS/shim/test/ltp
+         $ cd LibOS/test/ltp
          $ make -j
          $ make regression
 
@@ -614,12 +614,12 @@ system call that is not implemented in Gramine (recall that ``-38`` is the
    - Implement the main emulation function ``shim_do_some_syscall()``. If the
      system call belongs to some family of already-implemented system calls, add
      this function to the already-existing Gramine C file. Otherwise, create a
-     new C file under ``LibOS/shim/src/sys/``.
+     new C file under ``LibOS/src/sys/``.
 
    - Implement the required sub-systems or components in general code of
      Gramine. For example, if you need to add new fields to the thread object,
-     modify ``LibOS/shim/include/shim_thread.h`` and
-     ``LibOS/shim/src/bookkeep/shim_thread.c``.
+     modify ``LibOS/include/shim_thread.h`` and
+     ``LibOS/src/bookkeep/shim_thread.c``.
 
    - If the system call cannot be resolved entirely inside the LibOS component
      of Gramine, and the current set of PAL API functions (``Dk..()`` functions)

Documentation/devel/performance.rst

@@ -26,7 +26,7 @@ thread/process exit. Here is an example:
 
 ::
 
-   LibOS/shim/test/regression$ perf stat gramine-sgx helloworld
+   LibOS/test/regression$ perf stat gramine-sgx helloworld
    Hello world (helloworld)!
    ----- SGX stats for thread 87219 -----
    # of EENTERs:        224

Documentation/manpages/gramine-argv-serializer.rst

@@ -21,7 +21,7 @@ option pointed to it.
 For more information on the usage, please refer to :doc:`../manifest-syntax`.
 
 For an example on how to use this utility from Python, please refer to
-:file:`LibOS/shim/test/regression/test_libos.py`.
+:file:`LibOS/test/regression/test_libos.py`.
 
 Example
 =======

LibOS/shim/include/shim_handle.h → LibOS/include/shim_handle.h

@@ -81,7 +81,7 @@ enum shim_sock_state {
  * protected by `lock`.
  * `ops`, `domain`, `type` and `protocol` are read-only and do not need any locking.
  * Access to `peek` struct is protected by `recv_lock`. This lock also ensures proper ordering of
- * stream reads (see the comment in `do_recvmsg` in "LibOS/shim/src/sys/shim_socket.c").
+ * stream reads (see the comment in `do_recvmsg` in "LibOS/src/sys/shim_socket.c").
  * `pal_handle` should be accessed using atomic operations. It can be NULL. Once it's set, it cannot
  * change anymore.
  * If you need to take both `recv_lock` and `lock`, take the former first.

LibOS/shim/include/shim_thread.h → LibOS/include/shim_thread.h

@@ -97,7 +97,7 @@ struct shim_thread {
     /* If you need both locks, take `thread->signal_dispositions->lock` before `thread->lock`. */
     struct shim_signal_dispositions* signal_dispositions;
     struct shim_signal_queue signal_queue;
-    /* For the field below, see the explanation in "LibOS/shim/src/bookkeep/shim_signal.c" near
+    /* For the field below, see the explanation in "LibOS/src/bookkeep/shim_signal.c" near
      * `g_process_pending_signals_cnt`. */
     uint64_t pending_signals;
 

LibOS/meson.build

@@ -1 +1,32 @@
-subdir('shim')
+cflags_libos = [
+    # TODO: check whether all standard functions implemented in Gramine match their APIs and remove
+    # this
+    '-fno-builtin',
+
+    '-DIN_SHIM',
+]
+
+cflags_libos += cc.get_supported_arguments(
+    # TODO: -Wnull-dereference option detection was broken in Makefile, and some problems slipped
+    # through. In meson we enabled this option globally, so we need to debug before reenabling.
+    '-Wno-null-dereference',
+
+    # Some of the code uses alignof on expressions, which is a GNU extension.
+    # Silence Clang - it complains but does support it.
+    '-Wno-gnu-alignof-expression',
+
+    # TODO: This is due to use of packed structs in IPC code, which triggers "taking address of
+    # a packed member" warning in Clang and newer GCC. That code needs to be rewritten.
+    '-Wno-address-of-packed-member',
+)
+
+if host_machine.cpu_family() == 'x86_64'
+    cflags_libos += ['-mfxsr', '-mxsave']
+endif
+
+subdir('include')
+subdir('src')
+
+if enable_tests
+    subdir('test')
+endif

LibOS/shim/src/bookkeep/shim_pid.c → LibOS/src/bookkeep/shim_pid.c

@@ -173,7 +173,7 @@ void release_id(IDTYPE id) {
             if (ret < 0) {
                 /* TODO: this is a fatal error, unfortunately it can happen if the IPC leader exits
                  * without fully waiting for this process to end. For more information check
-                 * "LibOS/shim/src/sys/shim_exit.c". Change to `log_error` + `die` after fixing. */
+                 * "LibOS/src/sys/shim_exit.c". Change to `log_error` + `die` after fixing. */
                 log_warning("IPC pid release failed");
                 DkProcessExit(1);
             }