Merge pull request #87 from redhat-cop/devel

Release 0.2.0-alpha13

Author: mdujava

Diff for: .github/workflows/linter.yml

@@ -11,7 +11,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.10', '3.11', '3.12', '3.13']
 
     steps:
     - uses: actions/checkout@v2

Diff for: Makefile

@@ -14,12 +14,19 @@ update: setup_poetry
 	poetry update
 
 clean:
-	poetry env remove --all 
+	poetry env remove --all
 	rm -rf .mk_poetry* dist
 
-check: setup_poetry
+check: setup_poetry flake8 pylint
+
+flake8: setup_poetry
 	poetry run flake8 osia --max-line-length 100 --show-source --statistics
-	poetry run pylint osia
+
+pylint mypy: setup_poetry
+	poetry run $@ osia
+
+black-check: setup_poetry
+	poetry run black --check osia
 
 dist: setup_poetry
 	poetry build

Diff for: osia/cli.py

@@ -17,8 +17,11 @@
 openshift"""
 import argparse
 import logging
-from typing import List
+from typing import List, Tuple, Optional
+from subprocess import Popen
+from semantic_version import Version, SimpleSpec
 import coloredlogs
+import distro
 
 from .config.config import ARCH_AMD, ARCH_ARM, ARCH_X86_64, ARCH_AARCH64, ARCH_S390X, ARCH_PPC
 from .installer import install_cluster, delete_cluster, storage, download_installer
@@ -84,16 +87,46 @@ def _read_list(in_str: str) -> List[str]:
         a['proc'] = _identity
 
 
+def _check_fips_compatible(rhel_version: bool) -> Tuple[bool, Optional[str]]:
+    if not rhel_version:
+        return False, "FIPS installs are supported only from RHEL systems"
+    try:
+        with Popen(["fips-mode-setup", "--check"]) as proc:
+            proc.wait()
+            if proc.returncode != 0:
+                return False, "FIPS is not enabled on the system"
+    except FileNotFoundError:
+        return False, "fips-mode-setup must be installed on the system"
+
+    return True, None
+
+
 def _resolve_installer(from_args):
     if from_args.installer is None and from_args.installer_version is None:
         raise Exception('Either installer or installer-version must be passed')
     if from_args.installer:
         return from_args.installer
 
+    rhel_version = None
+    if distro.id() == "rhel":
+        rhel_version = distro.major_version()
+
+    if from_args.enable_fips:
+        supported, msg = _check_fips_compatible(rhel_version)
+        if not supported:
+            raise Exception(msg)
+
+        if Version.coerce(from_args.installer_version.split("-")[-1]) in SimpleSpec("<4.16"):
+            # ocp <4.16 does not have dedicated openshift-install-fips, it is
+            # fine to run normal installer on FIPS enabled RHEL
+            from_args.enable_fips = False
+
     return download_installer(from_args.installer_version,
                               from_args.installer_arch,
                               from_args.installers_dir,
-                              from_args.installer_source)
+                              from_args.installer_source,
+                              rhel_version=rhel_version,
+                              fips=from_args.enable_fips)
 
 
 def _merge_dictionaries(from_args):
@@ -122,6 +155,9 @@ def _exec_install_cluster(args):
 
 
 def _exec_delete_cluster(args):
+    # cleanup of fips cluster can be done from anywhere
+    args.enable_fips = None
+
     conf = _merge_dictionaries(args)
 
     if not args.skip_git:

Diff for: osia/installer/downloader/install.py

@@ -35,15 +35,16 @@
 BUILD_ROOT = "https://openshift-release-artifacts.svc.ci.openshift.org/"
 PREVIEW_ROOT = "http://mirror.openshift.com/pub/openshift-v4/{}/clients/ocp-dev-preview/"
 
-VERSION_RE = re.compile(r"^openshift-install-(?P<platform>\w+)"
-                        r"(-(?P<architecture>\w+))?-(?P<version>\d+.*)\.tar\.gz")
+VERSION_RE = re.compile(r"^openshift-install(-rhel(?P<rhel>\d+))?(-(?P<platform>(linux|mac)))?"
+                        r"(-(?P<architecture>\w+))?(-(?P<version>\d+.*))?\.tar\.gz")
 EXTRACTION_RE = re.compile(r'.*Extracting tools for .*, may take up to a minute.*')
 
 
-def _current_platform():
+def _current_platform() -> Tuple[str, str]:
     if platform.system() == "Linux" and platform.machine() == "x86_64":
         return "linux", "amd64"
-    if platform.system() == "Linux" and platform.machine() == "arm64":
+    if platform.system() == "Linux" and (
+            platform.machine() == "arm64" or platform.machine() == "aarch64"):
         return "linux", "arm64"
     if platform.system() == "Darwin" and platform.machine() == "arm64":
         return "mac", "arm64"
@@ -53,10 +54,14 @@ def _current_platform():
     raise Exception(f"Unrecognized platform {platform.system()} {platform.machine()}")
 
 
-def get_url(directory: str, arch: str) -> Tuple[Optional[str], Optional[str]]:
+def get_url(directory: str, arch: str, fips: bool = False,
+            rhel_version: str = None) -> Tuple[Optional[str], Optional[str]]:
     """Searches the http directory and returns both url to installer
     and version.
     """
+    if fips and not rhel_version:
+        raise Exception("Rhel version was not detected. Please download installer separatly.")
+
     logging.debug('Url for installers look-up %s', directory)
     lst = requests.get(directory, allow_redirects=True)
     tree = BeautifulSoup(lst.content, 'html.parser')
@@ -66,16 +71,29 @@ def get_url(directory: str, arch: str) -> Tuple[Optional[str], Optional[str]]:
     for k in links:
         logging.debug('Parsing link: %s', k.get('href'))
         match = VERSION_RE.match(k.get('href'))
-        if match and match.group('platform') == os_name:
-            if (local_arch == match.group('architecture')) \
-                    or (local_arch == arch and not match.group('architecture')):
-                installer = lst.url + k.get('href')
+
+        if match:
+            if match.group("version"):
                 version = match.group('version')
+
+            if fips and match.group("rhel") == rhel_version:
+                installer = lst.url + k.get('href')
                 break
+
+            if not fips and match.group('platform') == os_name:
+                if (local_arch == match.group('architecture')) \
+                        or (local_arch == arch and not match.group('architecture')):
+                    installer = lst.url + k.get('href')
+                    break
+    else:
+        if fips:
+            raise Exception(f"FIPS Installer not found for {rhel_version=}")
+        raise Exception("Installer not found")
     return installer, version
 
 
-def get_devel_url(version: str, arch: str) -> Tuple[Optional[str], Optional[str]]:
+def get_devel_url(version: str, arch: str, fips: bool = False,
+                  rhel_version: str = None) -> Tuple[Optional[str], Optional[str]]:
     """
     Searches developement sources and returns url to installer
     """
@@ -88,17 +106,19 @@ def get_devel_url(version: str, arch: str) -> Tuple[Optional[str], Optional[str]
         req = requests.get(BUILD_ROOT + version, allow_redirects=True)
         ast = BeautifulSoup(req.content, 'html.parser')
     logging.debug('Installer found on page, continuing')
-    return get_url(req.url, arch)
+    return get_url(req.url, arch, fips, rhel_version)
 
 
-def get_prev_url(version: str, arch: str) -> Tuple[Optional[str], Optional[str]]:
+def get_prev_url(version: str, arch: str, fips: bool = False,
+                 rhel_version: str = None) -> Tuple[Optional[str], Optional[str]]:
     """Returns installer url from dev-preview sources"""
-    return get_url(PREVIEW_ROOT.format(arch) + version + "/", arch)
+    return get_url(PREVIEW_ROOT.format(arch) + version + "/", arch, fips, rhel_version)
 
 
-def get_prod_url(version: str, arch: str) -> Tuple[Optional[str], Optional[str]]:
+def get_prod_url(version: str, arch: str, fips: bool = False,
+                 rhel_version: str = None) -> Tuple[Optional[str], Optional[str]]:
     """Returns installer url from production sources"""
-    return get_url(PROD_ROOT.format(arch) + version + "/", arch)
+    return get_url(PROD_ROOT.format(arch) + version + "/", arch, fips, rhel_version)
 
 
 def _get_storage_path(version: str, install_base: str) -> str:
@@ -114,12 +134,12 @@ def _extract_tar(buffer: NamedTemporaryFile, target: str) -> Path:
     with tarfile.open(buffer.name) as tar:
         inst_info = None
         for i in tar.getmembers():
-            if i.name == 'openshift-install':
+            if i.name in ['openshift-install', 'openshift-install-fips']:
                 inst_info = i
         if inst_info is None:
             raise Exception("error")
         stream = tar.extractfile(inst_info)
-        result = Path(target).joinpath('openshift-install')
+        result = Path(target).joinpath(inst_info.name)
         with result.open('wb') as output:
             copyfileobj(stream, output)
         result.chmod(result.stat().st_mode | stat.S_IXUSR)
@@ -131,10 +151,13 @@ def get_installer(tar_url: str, target: str):
     return get_data(tar_url, target, _extract_tar)
 
 
+# pylint: disable=too-many-arguments
 def download_installer(installer_version: str,
                        installer_arch: str,
                        dest_directory: str,
-                       source: str) -> str:
+                       source: str,
+                       fips: bool = False,
+                       rhel_version: str = None) -> str:
     """Starts search and extraction of installer"""
     logging.debug("Getting version %s of %s, storing to directory %s and devel is %r",
                   installer_version, installer_arch, dest_directory, source)
@@ -149,12 +172,17 @@ def download_installer(installer_version: str,
     else:
         raise Exception("Error for source profile " + source)
 
-    url, version = downloader(installer_version, installer_arch)
+    url, version = downloader(installer_version, installer_arch, fips, rhel_version)
     logging.debug('Installer\'s URL is  %s and full version is %s', url, version)
     root = Path(dest_directory).joinpath(version)
 
-    if root.exists() and root.joinpath('openshift-install').exists():
+    installer_exe_name = 'openshift-install'
+
+    if fips:
+        installer_exe_name = 'openshift-install-fips'
+
+    if root.exists() and root.joinpath(installer_exe_name).exists():
         logging.info('Found installer at %s', root.as_posix())
-        return root.joinpath('openshift-install').as_posix()
-    root.mkdir(parents=True)
+        return root.joinpath(installer_exe_name).as_posix()
+    root.mkdir(parents=True, exist_ok=True)
     return get_installer(url, root.as_posix())

Diff for: osia/installer/executor.py

@@ -73,7 +73,8 @@ def install_cluster(cloud_provider,
         logging.error(exception)
         if inst.check_clean():
             delete_cluster(cluster_name, installer)
-            return
+        # Do not continue in case of installer failure
+        return
 
     inst.post_installation()