Fix fuse mount issue on ubuntu 18.04

sky/data/mounting_utils.py

@@ -185,27 +185,63 @@ def get_gcs_mount_cmd(bucket_name: str,
 def get_az_mount_install_cmd() -> str:
     """Returns a command to install AZ Container mount utility blobfuse2."""
     install_cmd = (
-        'sudo apt-get update; '
-        'sudo apt-get install -y '
-        '-o Dpkg::Options::="--force-confdef" '
-        'fuse3 libfuse3-dev || { '
-        '  echo "fuse3 not available, falling back to fuse"; '
-        '  sudo apt-get install -y '
-        '  -o Dpkg::Options::="--force-confdef" '
-        '  fuse libfuse-dev; '
-        '} && '
+        # Check architecture first - blobfuse2 only supports x86_64
         'ARCH=$(uname -m) && '
         'if [ "$ARCH" = "aarch64" ] || [ "$ARCH" = "arm64" ]; then '
         '  echo "blobfuse2 is not supported on $ARCH" && '
         f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'fi && '
+        # Try to install fuse3 from default repos
+        'sudo apt-get update && '
+        'FUSE3_INSTALLED=0 && '
+        'if sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        'fuse3 libfuse3-dev; then '
+        '  FUSE3_INSTALLED=1; '
+        '  echo "fuse3 installed from default repos"; '
         'else '
-        '  ARCH_SUFFIX="x86_64"; '
+        # If fuse3 not available, try focal for Ubuntu <= 20.04
+        '  DISTRO=$(grep "^ID=" /etc/os-release | cut -d= -f2 | '
+        'tr -d \'"\' | tr "[:upper:]" "[:lower:]") && '
+        '  VERSION=$(grep "^VERSION_ID=" /etc/os-release | cut -d= -f2 | '
+        'tr -d \'"\') && '
+        '  if [ "$DISTRO" = "ubuntu" ] && '
+        '[ "$(echo "$VERSION 20.04" | '
+        'awk \'{ print ($1 <= $2) }\')" = "1" ]; then '
+        '    echo "Trying to install fuse3 from focal for '
+        'Ubuntu $VERSION"; '
+        '    echo "deb http://archive.ubuntu.com/ubuntu '
+        'focal main universe" | '
+        'sudo tee /etc/apt/sources.list.d/focal-fuse3.list && '
+        '    sudo apt-get update && '
+        '    if sudo apt-get install -y '
+        '-o Dpkg::Options::="--force-confdef" '
+        '-o Dpkg::Options::="--force-confold" '
+        'fuse3 libfuse3-3 libfuse3-dev; then '
+        '      FUSE3_INSTALLED=1; '
+        '      echo "fuse3 installed from focal"; '
+        '      sudo rm /etc/apt/sources.list.d/focal-fuse3.list; '
+        '      sudo apt-get update; '
+        '    else '
+        '      sudo rm -f /etc/apt/sources.list.d/focal-fuse3.list; '
+        '      sudo apt-get update; '
+        '    fi; '
+        '  fi; '
         'fi && '
-        'wget -nc https://github.com/Azure/azure-storage-fuse'
-        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}'
-        f'/blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.${{ARCH_SUFFIX}}.deb '
+        # Install blobfuse2 only if fuse3 is available
+        'if [ "$FUSE3_INSTALLED" = "1" ]; then '
+        '  echo "Installing blobfuse2 with libfuse3 support"; '
+        '  wget -nc https://github.com/Azure/azure-storage-fuse'
+        f'/releases/download/blobfuse2-{BLOBFUSE2_VERSION}/'
+        f'blobfuse2-{BLOBFUSE2_VERSION}-Debian-11.0.x86_64.deb '
         '-O /tmp/blobfuse2.deb && '
-        'sudo dpkg --install /tmp/blobfuse2.deb && '
+        '  sudo dpkg --install /tmp/blobfuse2.deb; '
+        'else '
+        '  echo "Error: libfuse3 is required for Azure storage '
+        'mounting with fusermount-wrapper."; '
+        '  echo "libfuse3 could not be installed on this system."; '
+        f'  exit {exceptions.ARCH_NOT_SUPPORTED_EXIT_CODE}; '
+        'fi && '
         f'mkdir -p {_BLOBFUSE_CACHE_ROOT_DIR};')
 
     return install_cmd
@@ -277,7 +313,10 @@ def get_az_mount_cmd(container_name: str,
                f'-- {blobfuse2_cmd} -o nonempty --foreground {{}}')
     original = f'{blobfuse2_cmd} {blobfuse2_options} {mount_path}'
     # If fusermount-wrapper is available, use it to wrap the blobfuse2 command
-    # to avoid requiring root privilege.
+    # to avoid requiring privileged containers.
+    # fusermount-wrapper requires libfuse3;
+    # we install libfuse3 even on older distros like Ubuntu 18.04 by using
+    # Ubuntu 20.04 (focal) repositories.
     # TODO(aylei): feeling hacky, refactor this.
     get_mount_cmd = ('command -v fusermount-wrapper >/dev/null 2>&1 && '
                      f'echo "{wrapped}" || echo "{original}"')

tests/smoke_tests/test_region_and_zone.py

@@ -262,8 +262,6 @@ def test_docker_storage_mounts(generic_cloud: str, image_id: str):
     template_str = pathlib.Path(
         'tests/test_yamls/test_storage_mounting.yaml.j2').read_text()
     template = jinja2.Template(template_str)
-    # ubuntu 18.04 does not support fuse3, and blobfuse2 depends on fuse3.
-    azure_mount_unsupported_ubuntu_version = '18.04'
     # Commands to verify bucket upload. We need to check all three
     # storage types because the optimizer may pick any of them.
     s3_command = f'aws s3 ls {storage_name}/hello.txt'
@@ -277,37 +275,33 @@ def test_docker_storage_mounts(generic_cloud: str, image_id: str):
     # created in the centralus region when getting the storage account. We
     # should set the cluster to be launched in the same region.
     region_str = f'/centralus' if generic_cloud == 'azure' else ''
+
+    # Determine store type based on generic_cloud
+    if generic_cloud == 'aws':
+        store_type = 's3'
+    elif generic_cloud == 'gcp':
+        store_type = 'gcs'
+    else:
+        store_type = 'azure'
+
     if smoke_tests_utils.is_non_docker_remote_api_server():
         enabled_cloud_storages = smoke_tests_utils.get_enabled_cloud_storages()
         include_s3_mount = clouds.cloud_in_iterable(clouds.AWS(),
                                                     enabled_cloud_storages)
         include_gcs_mount = clouds.cloud_in_iterable(clouds.GCP(),
                                                      enabled_cloud_storages)
-        include_azure_mount = (
-            clouds.cloud_in_iterable(clouds.Azure(), enabled_cloud_storages) and
-            azure_mount_unsupported_ubuntu_version not in image_id)
+        include_azure_mount = clouds.cloud_in_iterable(clouds.Azure(),
+                                                       enabled_cloud_storages)
         content = template.render(storage_name=storage_name,
                                   include_s3_mount=include_s3_mount,
                                   include_gcs_mount=include_gcs_mount,
                                   include_azure_mount=include_azure_mount,
-                                  empty_storage_name=empty_storage_name)
-    elif azure_mount_unsupported_ubuntu_version in image_id:
-        # The store for mount_private_mount is not specified in the template.
-        # If we're running on Azure, the private mount will be created on
-        # azure blob. Also, if we're running on Kubernetes, the private mount
-        # might be created on azure blob to avoid the issue of the fuse adapter
-        # not being able to access the mount point. That will not be supported on
-        # the ubuntu 18.04 image and thus fail. For other clouds, the private mount
-        # on other storage types (GCS/S3) should succeed.
-        include_private_mount = False if (
-            generic_cloud == 'azure' or generic_cloud == 'kubernetes') else True
-        content = template.render(storage_name=storage_name,
-                                  include_azure_mount=False,
-                                  include_private_mount=include_private_mount,
-                                  empty_storage_name=empty_storage_name)
+                                  empty_storage_name=empty_storage_name,
+                                  store_type=store_type)
     else:
         content = template.render(storage_name=storage_name,
-                                  empty_storage_name=empty_storage_name)
+                                  empty_storage_name=empty_storage_name,
+                                  store_type=store_type)
     cloud_dependencies_setup_cmd = ' && '.join(
         controller_utils._get_cloud_dependencies_installation_commands(
             controller_utils.Controllers.JOBS_CONTROLLER))

tests/test_yamls/test_storage_mounting.yaml.j2

@@ -18,7 +18,7 @@ file_mounts:
   /mount_public_azure:
     source: https://azureopendatastorage.blob.core.windows.net/nyctlc
     mode: MOUNT
-  
+
   # Multiple mount points for Azure
   /mount_public_azure_2:
     source: https://azureopendatastorage.blob.core.windows.net/nyctlc
@@ -30,34 +30,37 @@ file_mounts:
     name: {{storage_name}}
     source: ~/tmp-workdir
     mode: {% if only_mount | default(false) %}MOUNT{% else %}COPY{% endif %}
+    {% if store_type is defined %}store: {{store_type}}{% endif %}
 
   # Mounting private buckets in COPY mode with a list of files as source
   /mount_private_copy_lof:
     name: {{storage_name}}
     source: ['~/tmp-workdir/tmp file', '~/tmp-workdir/tmp file2']
     mode: {% if only_mount | default(false) %}MOUNT{% else %}COPY{% endif %}
+    {% if store_type is defined %}store: {{store_type}}{% endif %}
 
-  {% if include_private_mount | default(True) %}
   # Mounting private buckets in MOUNT mode
   /mount_private_mount:
     name: {{storage_name}}
     source: ~/tmp-workdir
     mode: MOUNT
-  {% endif %}
+    {% if store_type is defined %}store: {{store_type}}{% endif %}
 
   # Mounting private buckets in MOUNT_CACHED mode
   {% if include_mount_cached | default(False) %}
   /mount_private_mount_cached:
     name: {{storage_name}}
     source: ~/tmp-workdir
     mode: MOUNT_CACHED
+    {% if store_type is defined %}store: {{store_type}}{% endif %}
   {% endif %}
 
   # Mounting empty workdir
   /mount_empty_workdir:
     name: {{empty_storage_name}}
     source: ~/empty-workdir
     mode: MOUNT
+    {% if store_type is defined %}store: {{store_type}}{% endif %}
 
 run: |
   set -ex
@@ -79,19 +82,15 @@ run: |
   ls -ltr /mount_private_copy/tmp\ file
   ls -ltr /mount_private_copy_lof/tmp\ file
   ls -ltr /mount_private_copy_lof/tmp\ file2
-  {% if include_private_mount | default(True) %}
   ls -ltr /mount_private_mount/foo
   ls -ltr /mount_private_mount/tmp\ file
-  {% endif %}
 
   # Symlinks are not copied to buckets
   ! ls /mount_private_copy/circle-link
-  {% if include_private_mount | default(True) %}
   ! ls /mount_private_mount/circle-link
 
   # Write to private bucket in MOUNT mode should pass
   echo "hello" > /mount_private_mount/hello.txt
-  {% endif %}
 
   {% if include_mount_cached | default(False) %}
   # Check private bucket contents