Resolve Layer symlinks before mounting container

Author: lucashuy

samcli/lib/build/workflow_config.py

@@ -24,6 +24,30 @@
 
 LOG = logging.getLogger(__name__)
 
+LAYER_SUBFOLDERS = {
+    "python3.7": "python",
+    "python3.8": "python",
+    "python3.9": "python",
+    "python3.10": "python",
+    "python3.11": "python",
+    "nodejs4.3": "nodejs",
+    "nodejs6.10": "nodejs",
+    "nodejs8.10": "nodejs",
+    "nodejs12.x": "nodejs",
+    "nodejs14.x": "nodejs",
+    "nodejs16.x": "nodejs",
+    "nodejs18.x": "nodejs",
+    "ruby2.7": "ruby/lib",
+    "ruby3.2": "ruby/lib",
+    "java8": "java",
+    "java11": "java",
+    "java8.al2": "java",
+    "java17": "java",
+    "dotnet6": "dotnet",
+    # User is responsible for creating subfolder in these workflows
+    "makefile": "",
+}
+
 
 class UnsupportedRuntimeException(Exception):
     pass
@@ -84,34 +108,10 @@ def get_selector(
 
 
 def get_layer_subfolder(build_workflow: str) -> str:
-    subfolders_by_runtime = {
-        "python3.7": "python",
-        "python3.8": "python",
-        "python3.9": "python",
-        "python3.10": "python",
-        "python3.11": "python",
-        "nodejs4.3": "nodejs",
-        "nodejs6.10": "nodejs",
-        "nodejs8.10": "nodejs",
-        "nodejs12.x": "nodejs",
-        "nodejs14.x": "nodejs",
-        "nodejs16.x": "nodejs",
-        "nodejs18.x": "nodejs",
-        "ruby2.7": "ruby/lib",
-        "ruby3.2": "ruby/lib",
-        "java8": "java",
-        "java11": "java",
-        "java8.al2": "java",
-        "java17": "java",
-        "dotnet6": "dotnet",
-        # User is responsible for creating subfolder in these workflows
-        "makefile": "",
-    }
-
-    if build_workflow not in subfolders_by_runtime:
+    if build_workflow not in LAYER_SUBFOLDERS:
         raise UnsupportedRuntimeException("'{}' runtime is not supported for layers".format(build_workflow))
 
-    return subfolders_by_runtime[build_workflow]
+    return LAYER_SUBFOLDERS[build_workflow]
 
 
 def get_workflow_config(

samcli/local/docker/container.py

@@ -159,7 +159,7 @@ def create(self):
                     "bind": self._working_dir,
                     "mode": mount_mode,
                 },
-                **self._create_mapped_symlink_files(),
+                **Container._create_mapped_symlink_files(self._host_dir, self._working_dir),
             }
 
         kwargs = {
@@ -226,12 +226,20 @@ def create(self):
 
         return self.id
 
-    def _create_mapped_symlink_files(self) -> Dict[str, Dict[str, str]]:
+    @staticmethod
+    def _create_mapped_symlink_files(search_directory: str, bind_directory: str) -> Dict[str, Dict[str, str]]:
         """
         Resolves any top level symlinked files and folders that are found on the
         host directory and creates additional bind mounts to correctly map them
         inside of the container.
 
+        Parameters
+        ----------
+        search_directory: str
+            The folder to walk the root level of to search for symlinks
+        bind_directory: str
+            The folder inside of the container to resolve the symlink for
+
         Returns
         -------
         Dict[str, Dict[str, str]]
@@ -241,13 +249,13 @@ def _create_mapped_symlink_files(self) -> Dict[str, Dict[str, str]]:
         mount_mode = "ro,delegated"
         additional_volumes = {}
 
-        with os.scandir(self._host_dir) as directory_iterator:
+        with os.scandir(search_directory) as directory_iterator:
             for file in directory_iterator:
                 if not file.is_symlink():
                     continue
 
                 host_resolved_path = os.path.realpath(file.path)
-                container_full_path = pathlib.Path(self._working_dir, file.name).as_posix()
+                container_full_path = pathlib.Path(bind_directory, file.name).as_posix()
 
                 additional_volumes[host_resolved_path] = {
                     "bind": container_full_path,

samcli/local/docker/lambda_container.py

@@ -2,8 +2,11 @@
 Represents Lambda runtime containers.
 """
 import logging
-from typing import List
+from pathlib import Path
+from typing import Dict, List
 
+from samcli.lib.build.workflow_config import LAYER_SUBFOLDERS
+from samcli.lib.providers.provider import LayerVersion
 from samcli.lib.utils.packagetype import IMAGE
 from samcli.local.docker.lambda_debug_settings import LambdaDebugSettings
 
@@ -99,6 +102,7 @@ def __init__(
         entry, container_env_vars = LambdaContainer._get_debug_settings(runtime, debug_options)
         additional_options = LambdaContainer._get_additional_options(runtime, debug_options)
         additional_volumes = LambdaContainer._get_additional_volumes(runtime, debug_options)
+        layer_volume_mounts = LambdaContainer._get_layer_folder_mounts(layers)
 
         _work_dir = self._WORKING_DIR
         _entrypoint = None
@@ -133,11 +137,60 @@ def __init__(
             entrypoint=_entrypoint if _entrypoint else entry,
             env_vars=env_vars,
             container_opts=additional_options,
-            additional_volumes=additional_volumes,
+            additional_volumes={**additional_volumes, **layer_volume_mounts},
             container_host=container_host,
             container_host_interface=container_host_interface,
         )
 
+    @staticmethod
+    def _get_layer_folder_mounts(layers: List[LayerVersion]) -> Dict[str, Dict[str, str]]:
+        """
+        Searches the code uri of the Layer to resolve any root level symlinks before
+        the container is created
+
+        Paramters
+        ---------
+        layers: List[LayerVersion]
+            A list of layers to check for any symlinks
+
+        Returns
+        -------
+        Dict[str, Dict[str, str]]
+            A dictonary representing the resolved file or directory and the bound path
+            on the container
+        """
+        layer_mappings: Dict[str, Dict[str, str]] = {}
+
+        for layer in layers:
+            # layer.compatible_runtimes can return None
+            for runtime in layer.compatible_runtimes or []:
+                layer_folder = LAYER_SUBFOLDERS[runtime] if runtime in LAYER_SUBFOLDERS else None
+
+                # unsupported runtime for layers
+                if not layer_folder:
+                    LOG.debug("Skipping symlink check for layer %s, unsupported runtime (%s)", layer.layer_id, runtime)
+                    continue
+
+                # not locally built layer
+                if not layer.codeuri:
+                    LOG.debug(
+                        "Skipping symlink check for layer %s, layer does not have locally defined resources",
+                        layer.layer_id,
+                    )
+                    continue
+
+                # eg. `.aws-sam/build/MyLayer` + `nodejs`
+                artifact_layer_path = Path(layer.codeuri, layer_folder)
+                # eg. `/opt` + `nodejs`
+                container_bind_path = Path(LambdaImage._LAYERS_DIR, layer_folder)
+
+                mappings = LambdaContainer._create_mapped_symlink_files(
+                    str(artifact_layer_path), str(container_bind_path)
+                )
+                layer_mappings.update(mappings)
+
+        return layer_mappings
+
     @staticmethod
     def _get_exposed_ports(debug_options):
         """

tests/unit/local/docker/test_container.py

@@ -997,7 +997,7 @@ def test_no_symlinks_returns_empty(self, mock_scandir):
         mock_context.__enter__ = Mock(return_value=[self.mock_regular_file])
         mock_scandir.return_value = mock_context
 
-        volumes = self.container._create_mapped_symlink_files()
+        volumes = Container._create_mapped_symlink_files("mock_host_dir", "mock_container_dir")
 
         self.assertEqual(volumes, {})
 
@@ -1017,6 +1017,6 @@ def test_resolves_symlink(self, mock_path, mock_realpath, mock_scandir):
         mock_context.__enter__ = Mock(return_value=[self.mock_symlinked_file])
         mock_scandir.return_value = mock_context
 
-        volumes = self.container._create_mapped_symlink_files()
+        volumes = Container._create_mapped_symlink_files("mock_host_dir", "mock_container_dir")
 
         self.assertEqual(volumes, {host_path: {"bind": container_path, "mode": ANY}})

tests/unit/local/docker/test_lambda_container.py

@@ -7,6 +7,8 @@
 from parameterized import parameterized, param
 
 from samcli.commands.local.lib.debug_context import DebugContext
+from samcli.lib.build.workflow_config import LAYER_SUBFOLDERS
+from samcli.lib.providers.provider import LayerVersion
 from samcli.lib.utils.packagetype import IMAGE, ZIP
 from samcli.local.docker.lambda_container import LambdaContainer, Runtime
 from samcli.local.docker.lambda_debug_settings import DebuggingNotSupported
@@ -614,3 +616,44 @@ def test_additional_volumes_returns_volume_with_debugger_path_is_set(self, runti
         result = LambdaContainer._get_additional_volumes(runtime, debug_options)
         print(result)
         self.assertEqual(result, expected)
+
+
+class TestLambdaContainer_resolve_layers(TestCase):
+    @parameterized.expand(
+        [
+            ([],),  # no layers
+            ([LayerVersion("a:b:c", codeuri=None, compatible_runtimes=["nodejs18.x"])],),  # no codeuri
+            ([LayerVersion("a:b:c", codeuri="codeuri")],),  # no runtime
+            (
+                [LayerVersion("a:b:c", codeuri="codeuri", compatible_runtimes=["hello world"])],
+            ),  # unsupported/invalid runtime
+        ]
+    )
+    def test_returns_no_mounts_invalid_layer(self, layer):
+        result = LambdaContainer._get_layer_folder_mounts(layer)
+
+        self.assertEqual(result, {})
+
+    @patch.object(LambdaContainer, "_create_mapped_symlink_files")
+    def test_returns_no_mounts_no_links(self, create_map_mock):
+        create_map_mock.return_value = {}
+
+        layer = LayerVersion("a:b:c", codeuri="some/path", compatible_runtimes=["nodejs18.x"])
+        result = LambdaContainer._get_layer_folder_mounts([layer])
+
+        create_map_mock.assert_called_once()
+        self.assertEqual(result, {})
+
+    @patch.object(LambdaContainer, "_create_mapped_symlink_files")
+    def test_returns_mounts(self, create_map_mock):
+        code_uri = "some/path"
+        runtime = "nodejs18.x"
+        layer_folder = "nodejs"
+
+        expected_local_path = f"{code_uri}/{layer_folder}"
+        expected_container_path = f"/opt/{layer_folder}"
+
+        layer = LayerVersion("a:b:c", codeuri=code_uri, compatible_runtimes=[runtime])
+        LambdaContainer._get_layer_folder_mounts([layer])
+
+        create_map_mock.assert_called_once_with(expected_local_path, expected_container_path)