[DEBUG ONLY - don't merge] Test pr 674

bot/test.sh

@@ -219,6 +219,9 @@ fi
 # create tmp file for output of build step
 test_outerr=$(mktemp test.outerr.XXXX)
 
+echo "Listing known repos"
+./eessi_container.sh --list-repos
+
 echo "Executing command to test software:"
 echo "./eessi_container.sh ${COMMON_ARGS[@]} ${TEST_STEP_ARGS[@]}"
 echo "                     -- ./run_tests.sh \"${TEST_SUITE_ARGS[@]}\" \"$@\" 2>&1 | tee -a ${test_outerr}"

easystacks/software.eessi.io/2023.06/eessi-2023.06-eb-4.9.2-2022b.yml

@@ -6,4 +6,5 @@ easyconfigs:
         from-commit: d8076ebaf8cb915762adebf88d385cc672b350dc
   - gnuplot-5.4.6-GCCcore-12.2.0.eb
   - h5py-3.8.0-foss-2022b.eb
-  - MDAnalysis-2.4.2-foss-2022b.eb 
+  - MDAnalysis-2.4.2-foss-2022b.eb
+  - patchelf-0.17.2-GCCcore-12.2.0.eb

eessi_container.sh

@@ -254,7 +254,7 @@ if [[ ${LIST_REPOS} -eq 1 ]]; then
             default_label=", default"
         else
             default_label=""
-	fi
+        fi
         echo "    ${cvmfs_repo} [CVMFS config repo${default_label}]"
     done
     for cfg_repo in "${!cfg_cvmfs_repos[@]}"
@@ -323,7 +323,7 @@ do
     if [[ ! -n "${eessi_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then
         [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..."
         # cvmfs_repo_name is actually a repository ID, use that to obtain
-	#   the actual name from the EESSI_REPOS_CFG_FILE
+        #   the actual name from the EESSI_REPOS_CFG_FILE
         cfg_repo_id=${cvmfs_repo_name}
         cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name")
     fi
@@ -595,11 +595,11 @@ do
     #   that the necessary information for accessing a CVMFS repository is made
     #   available inside the container
     if [[ -n "${cfg_cvmfs_repos[${cvmfs_repo_name}]}" ]] ; then
-	cfg_repo_id=${cvmfs_repo_name}
+        cfg_repo_id=${cvmfs_repo_name}
 
-	# obtain CVMFS repository name from section for the given ID
+        # obtain CVMFS repository name from section for the given ID
         cfg_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name")
-	# derive domain part from (cfg_)repo_name (everything after first '.')
+        # derive domain part from (cfg_)repo_name (everything after first '.')
         repo_name_domain=${repo_name#*.}
 
         # cfg_cvmfs_repos is populated through reading the file pointed to by
@@ -609,15 +609,15 @@ do
         # copy repos.cfg to job directory --> makes it easier to inspect the job
         cp -a ${EESSI_REPOS_CFG_FILE} ${EESSI_TMPDIR}/repos_cfg/.
 
-	# cfg file should include sections (one per CVMFS repository to be mounted)
-	#   with each section containing the settings:
-	#   - repo_name,
-	#   - repo_version,
-	#   - config_bundle, and
-	#   - a map { filepath_in_bundle -> container_filepath }
+        # cfg file should include sections (one per CVMFS repository to be mounted)
+        #   with each section containing the settings:
+        #   - repo_name,
+        #   - repo_version,
+        #   - config_bundle, and
+        #   - a map { filepath_in_bundle -> container_filepath }
         #
-	# The config_bundle includes the files which are mapped ('->') to a target
-	# location in container:
+        # The config_bundle includes the files which are mapped ('->') to a target
+        # location in container:
         # - default.local -> /etc/cvmfs/default.local
         #   contains CVMFS settings, e.g., CVMFS_HTTP_PROXY, CVMFS_QUOTA_LIMIT, ...
         # - ${repo_name_domain}.conf -> /etc/cvmfs/domain.d/${repo_name_domain}.conf
@@ -641,7 +641,7 @@ do
         # use information to set up dir ${EESSI_TMPDIR}/repos_cfg and define
         #   BIND mounts
         # check if config_bundle exists, if so, unpack it into
-	#   ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error
+        #   ${EESSI_TMPDIR}/repos_cfg; if it doesn't, exit with an error
         # if config_bundle is relative path (no '/' at start) prepend it with
         #   EESSI_REPOS_CFG_DIR
         config_bundle_path=
@@ -726,7 +726,7 @@ do
     if [[ ${cfg_cvmfs_repos[${cvmfs_repo_name}]} ]]; then
         [[ ${VERBOSE} -eq 1 ]] && echo "repo '${cvmfs_repo_name}' is not an EESSI CVMFS repository..."
         # cvmfs_repo_name is actually a repository ID, use that to obtain
-	#   the actual name from the EESSI_REPOS_CFG_FILE
+        #   the actual name from the EESSI_REPOS_CFG_FILE
         cfg_repo_id=${cvmfs_repo_name}
         cvmfs_repo_name=$(cfg_get_value ${cfg_repo_id} "repo_name")
     fi
@@ -736,15 +736,48 @@ do
 
     # add fusemount options depending on requested access mode ('ro' - read-only; 'rw' - read & write)
     if [[ ${cvmfs_repo_access} == "ro" ]] ; then
-        export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}"
+        # need to distinguish between basic "ro" access and "ro" after a "rw" session
+        if [[ -d ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper ]]; then
+            # the overlay-upper directory is only created in a read-write-session, thus
+            # we are resuming from such a session here (otherwise there shouldn't be such
+            # directory yet as it is only created for read-write-sessions a bit further
+            # below); the overlay-upper directory can only exist because it is part of
+            # the ${RESUME} directory or tarball
+            # to be able to see the contents of the read-write session we have to mount
+            # the fuse-overlayfs (in read-only mode) on top of the CernVM-FS repository
+            echo "Overlay-upper exists, so assuming we are resuming from a previous step."
+            echo "Since repo-access mode 'ro' was requested, we are mounting it as lowerdir to make it read-only."
+
+            # make the target CernVM-FS repository available under /cvmfs_ro
+            export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}"
+
+            EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
+
+            # now, put the overlay-upper read-only on top of the repo and make it available under the usual prefix /cvmfs
+            EESSI_READONLY_OVERLAY="container:fuse-overlayfs"
+            # The contents of the previous session are available under
+            #   ${EESSI_TMPDIR} which is bind mounted to ${TMP_IN_CONTAINER}.
+            #   Hence, we have to use ${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper
+            # the left-most directory given for the lowerdir argument is put on top,
+            #   and with no upperdir=... the whole overlayfs is made available read-only
+            EESSI_READONLY_OVERLAY+=" -o lowerdir=${TMP_IN_CONTAINER}/${cvmfs_repo_name}/overlay-upper:/cvmfs_ro/${cvmfs_repo_name}"
+            EESSI_READONLY_OVERLAY+=" /cvmfs/${cvmfs_repo_name}"
+            export EESSI_READONLY_OVERLAY
+
+            EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY_OVERLAY}")
+            export EESSI_FUSE_MOUNTS
+        else
+            # basic "ro" access that doesn't require any fuseoverlay-fs
+            export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs/${cvmfs_repo_name}"
 
-        EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
-        export EESSI_FUSE_MOUNTS
+            EESSI_FUSE_MOUNTS+=("--fusemount" "${EESSI_READONLY}")
+            export EESSI_FUSE_MOUNTS
+        fi
     elif [[ ${cvmfs_repo_access} == "rw" ]] ; then
         # use repo-specific overlay directories
         mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-upper
         mkdir -p ${EESSI_TMPDIR}/${cvmfs_repo_name}/overlay-work
-	[[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})"
+        [[ ${VERBOSE} -eq 1 ]] && echo -e "TMP directory contents:\n$(ls -l ${EESSI_TMPDIR})"
 
         # set environment variables for fuse mounts in Singularity container
         export EESSI_READONLY="container:cvmfs2 ${cvmfs_repo_name} /cvmfs_ro/${cvmfs_repo_name}"
@@ -762,7 +795,7 @@ do
         export EESSI_FUSE_MOUNTS
     else
         echo -e "ERROR: access mode '${cvmfs_repo_access}' for CVMFS repository\n  '${cvmfs_repo_name}' is not known"
-	exit ${REPOSITORY_ERROR_EXITCODE}
+        exit ${REPOSITORY_ERROR_EXITCODE}
     fi
     # create repo_settings.sh file in ${EESSI_TMPDIR}/${cvmfs_repo_name} to store
     #   (intention is that the file could be just sourced to obtain the settings)

test_suite.sh

@@ -79,6 +79,22 @@ export EESSI_SOFTWARE_SUBDIR_OVERRIDE=$(python3 $TOPDIR/eessi_software_subdir.py
 
 source $TOPDIR/init/bash
 
+# DEBUG: only to test PR674, this code should never be merged
+echo "module av patchelf/0.17.2-GCCcore-12.2.0"
+module av patchelf/0.17.2-GCCcore-12.2.0
+echo "module load patchelf/0.17.2-GCCcore-12.2.0"
+module load patchelf/0.17.2-GCCcore-12.2.0
+echo "module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0"
+module --ignore_cache load patchelf/0.17.2-GCCcore-12.2.0
+echo "Command -v patchelf"
+command -v patchelf
+echo "EBROOTPATCHELF: $EBROOTPATCHELF"
+echo "patchelf --version"
+patchelf --version
+echo "ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all/patchelf/"
+ls -al /cvmfs/software.eessi.io/versions/2023.06/software/linux/${EESSI_SOFTWARE_SUBDIR_OVERRIDE}/modules/all/patchelf/
+touch /cvmfs/software.eessi.io/foo
+
 # Load the ReFrame module
 # Currently, we load the default version. Maybe we should somehow make this configurable in the future?
 module load ReFrame
@@ -194,7 +210,9 @@ fi
 
 # Run all tests
 echo "Running tests: reframe ${REFRAME_ARGS} --run"
-reframe ${REFRAME_ARGS} --run
+# reframe ${REFRAME_ARGS} --run
+# Make this run faster, we are not interestied in the test step here...
+reframe ${REFRAME_ARGS} --list
 reframe_exit_code=$?
 if [[ ${reframe_exit_code} -eq 0 ]]; then
     echo_green "ReFrame runtime ran succesfully with command: reframe ${REFRAME_ARGS} --run."