diff --git a/Atomic/containers.py b/Atomic/containers.py index f2d93c72..aaa59365 100644 --- a/Atomic/containers.py +++ b/Atomic/containers.py @@ -151,7 +151,7 @@ def ps_tty(self): max_container_id = 12 if self.args.truncate else max([len(x.id) for x in container_objects]) max_image_name = 20 if self.args.truncate else max([len(x.image_name) for x in container_objects]) max_command = 20 if self.args.truncate else max([len(x.command) for x in container_objects]) - col_out = "{0:2} {1:%s} {2:%s} {3:%s} {4:16} {5:9} {6:10} {7:10}" % (max_container_id, max_image_name, max_command) + col_out = "{0:2} {1:%s} {2:%s} {3:%s} {4:16} {5:10} {6:10} {7:10}" % (max_container_id, max_image_name, max_command) if self.args.heading: util.write_out(col_out.format(" ", "CONTAINER ID", @@ -173,7 +173,7 @@ def ps_tty(self): con_obj.image_name[0:max_image_name], con_obj.command[0:max_command], con_obj.created[0:16], - con_obj.state[0:9], + con_obj.state[0:10], con_obj.backend.backend[0:10], con_obj.runtime[0:10])) diff --git a/Atomic/install.py b/Atomic/install.py index e1831fb7..855f1370 100644 --- a/Atomic/install.py +++ b/Atomic/install.py @@ -62,6 +62,8 @@ def cli(subparser): system_xor_user.add_argument("--system", dest="system", action='store_true', default=False, help=_('install a system container')) + installp.add_argument("--system-package", dest="system_package", default="auto", + help=_('control how to install the package. It accepts `auto`, `yes`, `no`, `build`')) installp.add_argument("--rootfs", dest="remote", help=_("choose an existing exploded container/image to use " "its rootfs as a remote, read-only rootfs for the " diff --git a/Atomic/syscontainers.py b/Atomic/syscontainers.py index b42245fd..480e9781 100644 --- a/Atomic/syscontainers.py +++ b/Atomic/syscontainers.py @@ -35,12 +35,16 @@ ATOMIC_LIBEXEC = os.environ.get('ATOMIC_LIBEXEC', '/usr/libexec/atomic') ATOMIC_VAR = '/var/lib/containers/atomic' +ATOMIC_USR = '/usr/lib/containers/atomic' ATOMIC_VAR_USER = "%s/.containers/atomic" % HOME OSTREE_OCIIMAGE_PREFIX = "ociimage/" SYSTEMD_UNIT_FILES_DEST = "/etc/systemd/system" SYSTEMD_UNIT_FILES_DEST_USER = "%s/.config/systemd/user" % HOME SYSTEMD_TMPFILES_DEST = "/etc/tmpfiles.d" SYSTEMD_TMPFILES_DEST_USER = "%s/.containers/tmpfiles" % HOME +SYSTEMD_UNIT_FILES_DEST_PREFIX = "%s/usr/lib/systemd/system" +SYSTEMD_TMPFILES_DEST_PREFIX = "%s/usr/lib/tmpfiles.d" +RPM_NAME_PREFIX = "atomic-container" SYSTEMD_UNIT_FILE_DEFAULT_TEMPLATE = """ [Unit] Description=$NAME @@ -55,7 +59,7 @@ WantedBy=multi-user.target """ TEMPLATE_FORCED_VARIABLES = ["DESTDIR", "NAME", "EXEC_START", "EXEC_STOP", - "HOST_UID", "HOST_GID"] + "HOST_UID", "HOST_GID", "IMAGE_ID", "IMAGE_NAME"] TEMPLATE_OVERRIDABLE_VARIABLES = ["RUN_DIRECTORY", "STATE_DIRECTORY", "UUID"] class SystemContainers(object): @@ -151,12 +155,12 @@ def _split_set_args(setvalues): def _pull_image_to_ostree(self, repo, image, upgrade): if not repo: raise ValueError("Cannot find a configured OSTree repo") - if image.startswith("ostree:"): + if image.startswith("ostree:") and image.count(':') > 1: self._check_system_ostree_image(repo, image, upgrade) - elif image.startswith("docker:"): + elif image.startswith("docker:") and image.count(':') > 1: image = self._pull_docker_image(repo, image.replace("docker:", "", 1)) - elif image.startswith("dockertar:"): - tarpath = image.replace("dockertar:", "", 1) + elif image.startswith("dockertar:/"): + tarpath = image.replace("dockertar:/", "", 1) image = self._pull_docker_tar(repo, tarpath, os.path.basename(tarpath).replace(".tar", "")) else: # Assume "oci:" self._check_system_oci_image(repo, image, upgrade) @@ -177,6 +181,34 @@ def install_user_container(self, image, name): # Same entrypoint return self.install(image, name) + def _install_rpm(self, rpm_file): + if os.path.exists("/run/ostree-booted"): + raise ValueError("This doesn't work on Atomic Host yet") + elif os.path.exists("/usr/bin/dnf"): + util.check_call(["dnf", "install", "-y", rpm_file]) + else: + util.check_call(["yum", "install", "-y", rpm_file]) + + def _uninstall_rpm(self, rpm): + if os.path.exists("/run/ostree-booted"): + raise ValueError("This doesn't work on Atomic Host yet") + elif os.path.exists("/usr/bin/dnf"): + util.check_call(["dnf", "remove", "-y", rpm]) + else: + util.check_call(["yum", "remove", "-y", rpm]) + + @staticmethod + def _find_rpm(tmp_dir): + rpm_file = None + for root, _, files in os.walk(os.path.join(tmp_dir, "build")): + if rpm_file: + break + for f in files: + if f.endswith('.rpm'): + rpm_file = os.path.join(root, f) + break + return rpm_file + def install(self, image, name): repo = self._get_ostree_repo() if not repo: @@ -185,21 +217,55 @@ def install(self, image, name): if self.args.system and self.user: raise ValueError("Only root can use --system") - image = self._pull_image_to_ostree(repo, image, False) + accepted_system_package_values = ['auto', 'build', 'no', 'yes'] + if self.args.system_package not in accepted_system_package_values: + raise ValueError("Invalid --system-package mode. Accepted values: '%s'" % "', '".join(accepted_system_package_values)) if self.get_checkout(name): util.write_out("%s already present" % (name)) return - values = {} - if self.args.setvalues is not None: - setvalues = SystemContainers._split_set_args(self.args.setvalues) - for k, v in setvalues.items(): - values[k] = v + image = self._pull_image_to_ostree(repo, image, False) + rpm_preinstalled = None + tmp_dir = None + try: + if self.args.system_package == 'auto' and self.args.system: + self.args.system_package = 'no' + + if self.args.system_package in ['build', 'yes']: + if not self.args.system: + raise ValueError("Only --system can generate rpms") + + auto = self.args.system_package == 'auto' + include_containers_file = self.args.system_package == 'build' + tmp_dir = self.generate_rpm(repo, auto, name, image, include_containers_file=include_containers_file) + if tmp_dir: + rpm_preinstalled = SystemContainers._find_rpm(tmp_dir) + # If we are only build'ing the rpm, copy it to the cwd and exit + if self.args.system_package == 'build': + destination = os.path.join(os.getcwd(), os.path.basename(rpm_preinstalled)) + shutil.move(rpm_preinstalled, destination) + util.write_out("Generated rpm %s" % destination) + + if self.args.system_package == "build": + return False - return self._checkout(repo, name, image, 0, False, values=values, remote=self.args.remote) + values = {} + if self.args.setvalues is not None: + setvalues = SystemContainers._split_set_args(self.args.setvalues) + for k, v in setvalues.items(): + values[k] = v + + self._checkout(repo, name, image, 0, False, values=values, remote=self.args.remote, rpm_preinstalled=rpm_preinstalled) + except: + if rpm_preinstalled: + self._uninstall_rpm(os.path.basename(rpm_preinstalled).replace(".rpm", "")) + raise + finally: + if tmp_dir: + shutil.rmtree(tmp_dir) - def _check_oci_configuration_file(self, conf_path, remote=None): + def _check_oci_configuration_file(self, conf_path, remote=None, include_all=False): with open(conf_path, 'r') as conf: try: configuration = json.loads(conf.read()) @@ -223,7 +289,7 @@ def _check_oci_configuration_file(self, conf_path, remote=None): continue if "source" in mount and "bind" in mount["type"]: source = mount["source"] - if not os.path.exists(source): + if include_all or not os.path.exists(source): missing_source_paths.append(source) return missing_source_paths @@ -262,13 +328,17 @@ def _generate_systemd_startstop_directives(self, name): runc_commands = ["run", "kill"] return ["%s %s '%s'" % (util.RUNC_PATH, command, name) for command in runc_commands] - def _get_systemd_destination_files(self, name): + def _get_systemd_destination_files(self, name, prefix=None): if self.user: unitfileout = os.path.join(SYSTEMD_UNIT_FILES_DEST_USER, "%s.service" % name) tmpfilesout = os.path.join(SYSTEMD_TMPFILES_DEST_USER, "%s.conf" % name) else: - unitfileout = os.path.join(SYSTEMD_UNIT_FILES_DEST, "%s.service" % name) - tmpfilesout = os.path.join(SYSTEMD_TMPFILES_DEST, "%s.conf" % name) + if prefix: + unitfileout = os.path.join(SYSTEMD_UNIT_FILES_DEST_PREFIX % prefix, "%s.service" % name) + tmpfilesout = os.path.join(SYSTEMD_TMPFILES_DEST_PREFIX % prefix, "%s.conf" % name) + else: + unitfileout = os.path.join(SYSTEMD_UNIT_FILES_DEST, "%s.service" % name) + tmpfilesout = os.path.join(SYSTEMD_TMPFILES_DEST, "%s.conf" % name) return unitfileout, tmpfilesout def _resolve_remote_path(self, remote_path): @@ -280,9 +350,10 @@ def _resolve_remote_path(self, remote_path): raise ValueError("The container's rootfs is set to remote, but the remote rootfs does not exist") return real_path - def _checkout(self, repo, name, img, deployment, upgrade, values=None, destination=None, extract_only=False, remote=None): + def _checkout(self, repo, name, img, deployment, upgrade, values=None, destination=None, extract_only=False, remote=None, prefix=None, installed_files=None, + rpm_preinstalled=None): destination = destination or "%s/%s.%d" % (self._get_system_checkout_path(), name, deployment) - unitfileout, tmpfilesout = self._get_systemd_destination_files(name) + unitfileout, tmpfilesout = self._get_systemd_destination_files(name, prefix) if not upgrade: for f in [unitfileout, tmpfilesout]: @@ -290,7 +361,8 @@ def _checkout(self, repo, name, img, deployment, upgrade, values=None, destinati raise ValueError("The file %s already exists." % f) try: - return self._do_checkout(repo, name, img, upgrade, values, destination, unitfileout, tmpfilesout, extract_only, remote) + return self._do_checkout(repo, name, img, upgrade, values, destination, unitfileout, tmpfilesout, extract_only, remote, prefix, installed_files=installed_files, + rpm_preinstalled=rpm_preinstalled) except (ValueError, OSError) as e: try: if not extract_only and not upgrade: @@ -352,8 +424,30 @@ def get_image(i): return [get_image(i) for i in matches] - def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileout, tmpfilesout, extract_only, remote): - if not values: + @staticmethod + def _write_template(inputfilename, data, values, destination): + + if destination: + try: + os.makedirs(os.path.dirname(destination)) + except OSError: + pass + + template = Template(data) + try: + result = template.substitute(values) + except KeyError as e: + raise ValueError("The template file '%s' still contains an unreplaced value for: '%s'" % \ + (inputfilename, str(e))) + + if destination is not None: + with open(destination, "w") as outfile: + outfile.write(result) + return result + + def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileout, tmpfilesout, extract_only, remote, prefix=None, installed_files=None, + rpm_preinstalled=None): + if values is None: values = {} remote_path = self._resolve_remote_path(remote) @@ -386,7 +480,7 @@ def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileou was_service_active = self._is_service_active(name) if self.display: - return + return values if self.user: rootfs = os.path.join(destination, "rootfs") @@ -437,7 +531,7 @@ def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileou os.close(rootfs_fd) if extract_only: - return + return values if self.user: values["RUN_DIRECTORY"] = os.environ.get("XDG_RUNTIME_DIR", "/run/user/%s" % (os.getuid())) @@ -456,6 +550,9 @@ def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileou # 2) What the user sets explictly as --set # 3) Values for DESTDIR and NAME manifest_file = os.path.join(exports, "manifest.json") + installed_files_template = [] + has_container_service = True + rename_files = {} if os.path.exists(manifest_file): with open(manifest_file, "r") as f: try: @@ -466,29 +563,28 @@ def _do_checkout(self, repo, name, img, upgrade, values, destination, unitfileou for key, val in manifest["defaultValues"].items(): if key not in values: values[key] = val + if rpm_preinstalled is None and "installedFilesTemplate" in manifest: + installed_files_template = manifest["installedFilesTemplate"] + if rpm_preinstalled is None and "renameFiles" in manifest: + rename_files = manifest["renameFiles"] + if "noContainerService" in manifest and manifest["noContainerService"]: + has_container_service = False + + image_manifest = self._image_manifest(repo, rev) + image_id = rev + if image_manifest: + image_manifest = json.loads(image_manifest) + image_id = SystemContainers._get_image_id_from_manifest(image_manifest) or image_id if "UUID" not in values: values["UUID"] = str(uuid.uuid4()) - values["DESTDIR"] = destination + values["DESTDIR"] = os.path.join("/", os.path.relpath(destination, prefix)) if prefix else destination values["NAME"] = name values["EXEC_START"], values["EXEC_STOP"] = self._generate_systemd_startstop_directives(name) values["HOST_UID"] = os.getuid() values["HOST_GID"] = os.getgid() - - def _write_template(inputfilename, data, values, destination): - try: - os.makedirs(os.path.dirname(destination)) - except OSError: - pass - with open(destination, "w") as outfile: - template = Template(data) - try: - result = template.substitute(values) - except KeyError as e: - os.unlink(destination) - raise ValueError("The template file '%s' still contains an unreplaced value for: '%s'" % \ - (inputfilename, str(e))) - outfile.write(result) + values["IMAGE_NAME"] = img + values["IMAGE_ID"] = image_id src = os.path.join(exports, "config.json") destination_path = os.path.join(destination, "config.json") @@ -496,7 +592,7 @@ def _write_template(inputfilename, data, values, destination): shutil.copyfile(src, destination_path) elif os.path.exists(src + ".template"): with open(src + ".template", 'r') as infile: - _write_template(src + ".template", infile.read(), values, destination_path) + SystemContainers._write_template(src + ".template", infile.read(), values, destination_path) else: self._generate_default_oci_configuration(destination) @@ -515,7 +611,7 @@ def _write_template(inputfilename, data, values, destination): # When upgrading, stop the service and remove previously installed # tmpfiles, before restarting the service. - if upgrade: + if has_container_service and upgrade: if was_service_active: self._systemctl_command("stop", name) if os.path.exists(tmpfilesout): @@ -524,22 +620,52 @@ def _write_template(inputfilename, data, values, destination): except subprocess.CalledProcessError: pass - missing_bind_paths = self._check_oci_configuration_file(destination_path, remote_path) + # rename_files may contain variables that need to be replaced. + if rename_files: + for k, v in rename_files.items(): + template = Template(v) + try: + new_v = template.substitute(values) + except KeyError as e: + raise ValueError("The template file 'manifest.json' still contains an unreplaced value for: '%s'" % \ + (str(e))) + rename_files[k] = new_v - image_manifest = self._image_manifest(repo, rev) - image_id = rev - if image_manifest: - image_manifest = json.loads(image_manifest) - image_id = SystemContainers._get_image_id_from_manifest(image_manifest) or image_id + if rpm_preinstalled: + new_installed_files = [] + else: + new_installed_files = self._rm_add_files_to_host(installed_files, exports, prefix or "/", files_template=installed_files_template, values=values, rename_files=rename_files) + + missing_bind_paths = self._check_oci_configuration_file(destination_path, remote_path, True) - with open(os.path.join(destination, "info"), 'w') as info_file: - info = {"image" : img, - "revision" : image_id, - "ostree-commit": rev, - 'created' : calendar.timegm(time.gmtime()), - "values" : values, - "remote" : remote} - info_file.write(json.dumps(info, indent=4)) + # If rpm.spec or rpm.spec.template exist, copy them to the checkout directory, processing the .template version. + if os.path.exists(os.path.join(exports, "rpm.spec.template")): + with open(os.path.join(exports, "rpm.spec.template"), "r") as f: + spec_content = f.read() + SystemContainers._write_template("rpm.spec.template", spec_content, values, os.path.join(destination, "rpm.spec")) + elif os.path.exists(os.path.join(rootfs, "rpm.spec")): + shutil.copyfile(os.path.join(rootfs, "rpm.spec"), os.path.join(destination, "rpm.spec")) + + try: + rpm_installed = os.path.basename(rpm_preinstalled) if rpm_preinstalled else None + with open(os.path.join(destination, "info"), 'w') as info_file: + info = {"image" : img, + "revision" : image_id, + "ostree-commit": rev, + 'created' : calendar.timegm(time.gmtime()), + "values" : values, + "has-container-service" : has_container_service, + "installed-files": new_installed_files, + "installed-files-template": installed_files_template, + "rename-installed-files" : rename_files, + "rpm-installed" : rpm_installed, + "remote" : remote} + info_file.write(json.dumps(info, indent=4)) + info_file.write("\n") + except (NameError, AttributeError, OSError) as e: + for i in new_installed_files: + os.remove(os.path.join(prefix or "/", os.path.relpath(i, "/"))) + raise e if os.path.exists(unitfile): with open(unitfile, 'r') as infile: @@ -551,19 +677,40 @@ def _write_template(inputfilename, data, values, destination): with open(tmpfiles, 'r') as infile: tmpfiles_template = infile.read() else: - tmpfiles_template = SystemContainers._generate_tmpfiles_data(missing_bind_paths, values["STATE_DIRECTORY"]) + tmpfiles_template = SystemContainers._generate_tmpfiles_data(missing_bind_paths) - _write_template(unitfile, systemd_template, values, unitfileout) - shutil.copyfile(unitfileout, os.path.join(destination, "%s.service" % name)) + if has_container_service: + SystemContainers._write_template(unitfile, systemd_template, values, unitfileout) + shutil.copyfile(unitfileout, os.path.join(prefix, destination, "%s.service" % name)) if (tmpfiles_template): - _write_template(unitfile, tmpfiles_template, values, tmpfilesout) - shutil.copyfile(tmpfilesout, os.path.join(destination, "tmpfiles-%s.conf" % name)) + SystemContainers._write_template(unitfile, tmpfiles_template, values, tmpfilesout) + shutil.copyfile(tmpfilesout, os.path.join(prefix, destination, "tmpfiles-%s.conf" % name)) + + if not prefix: + sym = "%s/%s" % (self._get_system_checkout_path(), name) + if os.path.exists(sym): + os.unlink(sym) + os.symlink(destination, sym) + + # if there is no container service, delete the checked out files. At this point files copied to the host + # are already handled. + if not has_container_service: + if not remote_path: + shutil.rmtree(os.path.join(destination, "rootfs")) + return values + + if prefix: + return values sym = "%s/%s" % (self._get_system_checkout_path(), name) if os.path.exists(sym): os.unlink(sym) os.symlink(destination, sym) + if rpm_preinstalled: + self._install_rpm(rpm_preinstalled) + shutil.move(rpm_preinstalled, destination) + self._systemctl_command("daemon-reload") if (tmpfiles_template): self._systemd_tmpfiles("--create", tmpfilesout) @@ -573,6 +720,11 @@ def _write_template(inputfilename, data, values, destination): elif was_service_active: self._systemctl_command("start", name) + return values + + def _get_preinstalled_containers_path(self): + return ATOMIC_USR + def _get_system_checkout_path(self): if os.environ.get("ATOMIC_OSTREE_CHECKOUT_PATH"): return os.environ.get("ATOMIC_OSTREE_CHECKOUT_PATH") @@ -615,7 +767,61 @@ def version(self, image): return [image_inspect] return None + @staticmethod + def _rm_add_files_to_host(old_installed_files, exports, prefix="/", files_template=None, values=None, rename_files=None): + # if any file was installed on the host delete it + if old_installed_files: + for i in old_installed_files: + try: + os.remove(i) + except OSError: + pass + + if not exports: + return [] + + templates_set = set(files_template or []) + + # if there is a directory hostfs/ under exports, copy these files to the host file system. + hostfs = os.path.join(exports, "hostfs") + new_installed_files = [] + if os.path.exists(hostfs): + for root, _, files in os.walk(hostfs): + rel_root_path = os.path.relpath(root, hostfs) + if not os.path.exists(os.path.join(prefix, rel_root_path)): + os.makedirs(os.path.join(prefix, rel_root_path)) + for f in files: + src_file = os.path.join(root, f) + dest_path = os.path.join(prefix, rel_root_path, f) + rel_dest_path = os.path.join("/", rel_root_path, f) + + # If rename_files is set, rename the destination file + if rename_files and rel_dest_path in rename_files: + rel_dest_path = rename_files.get(rel_dest_path) + dest_path = os.path.join(prefix or "/", os.path.relpath(rel_dest_path, "/")) + + if os.path.exists(dest_path): + for i in new_installed_files: + os.remove(new_installed_files) + raise ValueError("File %s already exists." % dest_path) + + if rel_dest_path in templates_set: + with open(src_file, 'r') as src_file_obj: + data = src_file_obj.read() + SystemContainers._write_template(src_file, data, values or {}, dest_path) + shutil.copystat(src_file, dest_path) + else: + shutil.copy2(src_file, dest_path) + + new_installed_files.append(rel_dest_path) + new_installed_files.sort() # just for an aesthetic reason in the info file output + + return new_installed_files + def update_container(self, name, setvalues=None, rebase=None): + if self._is_preinstalled_container(name): + raise ValueError("Cannot update a preinstalled container") + repo = self._get_ostree_repo() if not repo: raise ValueError("Cannot find a configured OSTree repo") @@ -637,6 +843,8 @@ def update_container(self, name, setvalues=None, rebase=None): image = rebase or info["image"] values = info["values"] revision = info["revision"] if "revision" in info else None + installed_files = info["installed-files"] if "installed-files" in info else None + rpm_installed = info["rpm-installed"] if "rpm-installed" in info else True # Check if the image id or the configuration for the container has # changed before upgrading it. @@ -662,7 +870,15 @@ def update_container(self, name, setvalues=None, rebase=None): util.write_out("Latest version already installed.") return - self._checkout(repo, name, image, next_deployment, True, values, remote=self.args.remote) + # was installed with an rpm, update in the same way. + rpm_preinstalled = None + if rpm_installed: + tmp_dir = self.generate_rpm(repo, False, name, image, include_containers_file=False) + if tmp_dir: + rpm_preinstalled = SystemContainers._find_rpm(tmp_dir) + + self._checkout(repo, name, image, next_deployment, True, values, remote=self.args.remote, installed_files=installed_files, rpm_preinstalled=rpm_preinstalled) + return def rollback(self, name): path = os.path.join(self._get_system_checkout_path(), name) @@ -670,7 +886,17 @@ def rollback(self, name): if not os.path.exists(destination): raise ValueError("Error: Cannot find a previous deployment to rollback located at %s" % destination) - was_service_active = self._is_service_active(name) + installed_files = None + rename_files = None + with open(os.path.join(self._get_system_checkout_path(), name, "info"), "r") as info_file: + info = json.loads(info_file.read()) + rpm_installed = info["rpm-installed"] if "rpm-installed" in info else True + installed_files = info["installed-files"] if "installed-files" in info and rpm_installed is None else None + installed_files_template = info["installed-files-template"] if "installed-files-template" in info and rpm_installed is None else None + has_container_service = info["has-container-service"] if "has-container-service" in info else True + rename_files = info["rename-installed-files"] if "rename-installed-files" in info else None + + was_service_active = has_container_service and self._is_service_active(name) unitfileout, tmpfilesout = self._get_systemd_destination_files(name) unitfile = os.path.join(destination, "%s.service" % name) tmpfiles = os.path.join(destination, "tmpfiles-%s.conf" % name) @@ -680,6 +906,7 @@ def rollback(self, name): "The previous checkout at %s may be corrupted." % destination) util.write_out("Rolling back container {} to the checkout at {}".format(name, destination)) + if was_service_active: self._systemctl_command("stop", name) @@ -697,9 +924,23 @@ def rollback(self, name): if (os.path.exists(tmpfiles)): shutil.copyfile(tmpfiles, tmpfilesout) + if installed_files: + self._rm_add_files_to_host(installed_files, os.path.join(destination, "rootfs/exports"), files_template=installed_files_template, rename_files=rename_files) + os.unlink(path) os.symlink(destination, path) - self._systemctl_command("daemon-reload") + + # reinstall the previous rpm if any. + rpm_installed = None + with open(os.path.join(self._get_system_checkout_path(), name, "info"), "r") as info_file: + info = json.loads(info_file.read()) + rpm_installed = info["rpm-installed"] if "rpm-installed" in info else True + + if rpm_installed: + self._install_rpm(os.path.join(self._get_system_checkout_path(), name, rpm_installed)) + + if has_container_service: + self._systemctl_command("daemon-reload") if (os.path.exists(tmpfiles)): self._systemd_tmpfiles("--create", tmpfilesout) @@ -708,6 +949,16 @@ def rollback(self, name): def get_container_runtime_info(self, container): + info_path = os.path.join(self._get_system_checkout_path(), container, "info") + if not os.path.exists(info_path): + info_path = os.path.join(self._get_preinstalled_containers_path(), container, "info") + + with open(info_path, "r") as info_file: + info = json.loads(info_file.read()) + has_container_service = info["has-container-service"] if "has-container-service" in info else True + + if not has_container_service: + return {'status' : "no service"} if self._is_service_active(container): return {'status' : "running"} elif self._is_service_failed(container): @@ -716,9 +967,8 @@ def get_container_runtime_info(self, container): # The container is newly created or stopped, and can be started with 'systemctl start' return {'status' : "inactive"} - def get_containers(self, containers=None): - checkouts = self._get_system_checkout_path() - if not os.path.exists(checkouts): + def _get_containers_at(self, checkouts, are_preinstalled, containers=None): + if not checkouts or not os.path.exists(checkouts): return [] ret = [] if containers is None: @@ -727,7 +977,9 @@ def get_containers(self, containers=None): if x[0] == ".": continue fullpath = os.path.join(checkouts, x) - if not os.path.islink(fullpath): + if not os.path.exists(fullpath): + continue + if fullpath.endswith(".0") or fullpath.endswith(".1"): continue with open(os.path.join(fullpath, "info"), "r") as info_file: @@ -742,10 +994,15 @@ def get_containers(self, containers=None): runtime = "bwrap-oci" if self.user else "runc" container = {'Image' : image, 'ImageID' : revision, 'Id' : x, 'Created' : created, 'Names' : [x], - 'Command' : command, 'Type' : 'system', 'Runtime' : runtime} + 'Command' : command, 'Type' : 'system', 'Runtime' : runtime, "Preinstalled" : are_preinstalled} ret.append(container) return ret + def get_containers(self, containers=None): + checkouts = self._get_system_checkout_path() + preinstalled = self._get_preinstalled_containers_path() + return self._get_containers_at(checkouts, False, containers) + self._get_containers_at(preinstalled, True, containers) + def get_template_variables(self, image): repo = self._get_ostree_repo() imgs = self._resolve_image(repo, image) @@ -880,7 +1137,8 @@ def get_system_images(self, get_all=False, repo=None): def _is_service_active(self, name): try: - return self._systemctl_command("is-active", name, quiet=True).replace("\n", "") == "active" + ret = self._systemctl_command("is-active", name, quiet=True) + return ret and ret.replace("\n", "") == "active" except subprocess.CalledProcessError: return False @@ -944,20 +1202,43 @@ def get_checkout(self, name): path = "%s/%s" % (self._get_system_checkout_path(), name) if os.path.exists(path): return path - else: - return None + + path = "%s/%s" % (self._get_preinstalled_containers_path(), name) + if os.path.exists(path): + return path + + return None + + def _is_preinstalled_container(self, name): + path = "%s/%s" % (self._get_system_checkout_path(), name) + if os.path.exists(path): + return False + + path = "%s/%s" % (self._get_preinstalled_containers_path(), name) + return os.path.exists(path) def uninstall(self, name): - unitfileout, tmpfilesout = self._get_systemd_destination_files(name) + if self._is_preinstalled_container(name): + self._uninstall_rpm("%s-%s" % (RPM_NAME_PREFIX, name)) - try: - self._systemctl_command("stop", name) - except subprocess.CalledProcessError: - pass - try: - self._systemctl_command("disable", name) - except subprocess.CalledProcessError: - pass + if not os.path.exists(os.path.join(self._get_system_checkout_path(), name)): + return + + with open(os.path.join(self._get_system_checkout_path(), name, "info"), "r") as info_file: + info = json.loads(info_file.read()) + has_container_service = info["has-container-service"] if "has-container-service" in info else True + rpm_installed = info["rpm-installed"] if "rpm-installed" in info else True + + unitfileout, tmpfilesout = self._get_systemd_destination_files(name) + if has_container_service: + try: + self._systemctl_command("stop", name) + except subprocess.CalledProcessError: + pass + try: + self._systemctl_command("disable", name) + except subprocess.CalledProcessError: + pass if os.path.exists(tmpfilesout): try: @@ -966,15 +1247,26 @@ def uninstall(self, name): pass os.unlink(tmpfilesout) - if os.path.lexists("%s/%s" % (self._get_system_checkout_path(), name)): - os.unlink("%s/%s" % (self._get_system_checkout_path(), name)) + checkout = self._get_system_checkout_path() + installed_files = None + with open(os.path.join(checkout, name, "info"), 'r') as info_file: + info = json.loads(info_file.read()) + installed_files = info["installed-files"] if "installed-files" in info else None + if installed_files: + self._rm_add_files_to_host(installed_files, None) + + if os.path.lexists("%s/%s" % (checkout, name)): + os.unlink("%s/%s" % (checkout, name)) for deploy in ["0", "1"]: - if os.path.exists("%s/%s.%s" % (self._get_system_checkout_path(), name, deploy)): - shutil.rmtree("%s/%s.%s" % (self._get_system_checkout_path(), name, deploy)) + if os.path.exists("%s/%s.%s" % (checkout, name, deploy)): + shutil.rmtree("%s/%s.%s" % (checkout, name, deploy)) if os.path.exists(unitfileout): os.unlink(unitfileout) + if rpm_installed: + self._uninstall_rpm(rpm_installed.replace(".rpm", "")) + def prune_ostree_images(self): repo = self._get_ostree_repo() if not repo: @@ -1248,16 +1540,12 @@ def _check_system_oci_image(self, repo, img, upgrade): return True @staticmethod - def _generate_tmpfiles_data(missing_bind_paths, state_directory): + def _generate_tmpfiles_data(missing_bind_paths): def _generate_line(x, state): return "%s %s 0700 %i %i - -\n" % (state, x, os.getuid(), os.getgid()) lines = [] for x in missing_bind_paths: - if os.path.commonprefix([x, state_directory]) == state_directory: - lines.append(_generate_line(x, "d")) - else: - lines.append(_generate_line(x, "D")) - lines.append(_generate_line(x, "R")) + lines.append(_generate_line(x, "d")) return "".join(lines) @staticmethod @@ -1272,7 +1560,7 @@ def extract(self, img, destination): repo = self._get_ostree_repo() if not repo: return False - return self._checkout(repo, img, img, 0, False, destination=destination, extract_only=True) + self._checkout(repo, img, img, 0, False, destination=destination, extract_only=True) @staticmethod def _encode_to_ostree_ref(name): @@ -1404,3 +1692,119 @@ def get_out_checksum(obj): return obj.out_checksum if hasattr(obj, 'out_checksum it.init_commit(repo, repo.load_commit(current_rev)[1], OSTree.RepoCommitTraverseFlags.REPO_COMMIT_TRAVERSE_FLAG_NONE) traverse(it) return ret + + def generate_rpm(self, repo, auto, name, image, include_containers_file=True): + temp_dir = tempfile.mkdtemp() + rpm_content = os.path.join(temp_dir, "rpmroot") + rootfs = os.path.join(rpm_content, "usr/lib/containers/atomic", name) + os.makedirs(rootfs) + success = False + try: + values = self._checkout(repo, name, image, 0, False, destination=rootfs, prefix=rpm_content) + if self.display: + return None + ret = self._generate_rpm_from_rootfs(rootfs, temp_dir, auto, name, image, values, include_containers_file) + if ret: + success = True + return ret + finally: + if not success: + shutil.rmtree(temp_dir) + + def _generate_rpm_from_rootfs(self, rootfs, temp_dir, auto, name, image, values, include_containers_file): + image_inspect = self.inspect_system_image(image) + rpm_content = os.path.join(temp_dir, "rpmroot") + spec_file = os.path.join(temp_dir, "container.spec") + + included_rpm = os.path.join(rootfs, "rootfs", "exports", "container.rpm") + if os.path.exists(included_rpm): + return included_rpm + + installed_files = None + with open(os.path.join(rootfs, "info"), "r") as info_file: + info = json.loads(info_file.read()) + installed_files = info["installed-files"] if "installed-files" in info else None + + labels = {k.lower() : v for k, v in image_inspect.get('Labels', {}).items()} + summary = labels.get('summary', name) + version = labels.get("version", "1.0") + release = labels.get("release", "1.0") + license_ = labels.get("license", "GPLv2") + url = labels.get("url") + source0 = labels.get("source0") + requires = labels.get("requires") + provides = labels.get("provides") + conflicts = labels.get("conflicts") + description = labels.get("description") + + image_id = values["IMAGE_ID"] + + if os.path.exists(os.path.join(rootfs, "rpm.spec")): + with open(os.path.join(rootfs, "rpm.spec"), "r") as f: + spec_content = f.read() + else: + # If there is no spec file and 'auto' is used, do not install an rpm + if auto: + return None + spec_content = self._generate_spec_file(rpm_content, name, summary, license_, image_id, version=version, + release=release, url=url, source0=source0, requires=requires, + provides=provides, conflicts=conflicts, description=description, + installed_files=installed_files, include_containers_file=include_containers_file) + + with open(spec_file, "w") as f: + f.write(spec_content) + + cwd = os.getcwd() + result_dir = os.path.join(temp_dir, "build") + if not os.path.exists(result_dir): + os.makedirs(result_dir) + cmd = ["rpmbuild", "--noclean", "-bb", spec_file, + "--define", "_sourcedir %s" % temp_dir, + "--define", "_specdir %s" % temp_dir, + "--define", "_builddir %s" % temp_dir, + "--define", "_srcrpmdir %s" % cwd, + "--define", "_rpmdir %s" % result_dir, + "--build-in-place", + "--buildroot=%s" % rpm_content] + util.write_out(" ".join(cmd)) + if not self.display: + util.check_call(cmd) + return temp_dir + + def _generate_spec_file(self, destdir, name, summary, license_, image_id, version="1.0", release="1", url=None, + source0=None, requires=None, conflicts=None, provides=None, description=None, + installed_files=None, include_containers_file=True): + spec = "%global __requires_exclude_from ^.*$\n" + spec = spec + "%global __provides_exclude_from ^.*$\n" + spec = spec + "%define _unpackaged_files_terminate_build 0\n" + + fields = {"Name" : "%s-%s" % (RPM_NAME_PREFIX, name), "Version" : version, "Release" : release, "Summary" : summary, + "License" : license_, "URL" : url, "Source0" : source0, "Requires" : requires, + "Provides" : provides, "Conflicts" : conflicts} + for k, v in fields.items(): + if v is not None: + spec = spec + "%s:\t%s\n" % (k, v) + + spec = spec + ("\n%%description\nImage ID: %s\n" % image_id) + if description: + spec = spec + "%s\n" % description + + spec = spec + "\n%files\n" + for root, _, files in os.walk(os.path.join(destdir, "etc")): + rel_path = os.path.relpath(root, destdir) + for f in files: + spec += "%config \"%s\"\n" % os.path.join("/", rel_path, f) + + if include_containers_file: + spec += "/usr/lib/containers/atomic/%s\n" % name + for root, _, files in os.walk(os.path.join(destdir, "usr/lib/systemd/system")): + for f in files: + spec = spec + "/usr/lib/systemd/system/%s\n" % f + for root, _, files in os.walk(os.path.join(destdir, "usr/lib/tmpfiles.d")): + for f in files: + spec = spec + "/usr/lib/tmpfiles.d/%s\n" % f + if installed_files: + for i in installed_files: + spec = spec + "%s\n" % i + + return spec diff --git a/bash/atomic b/bash/atomic index e6e7a583..ec10780a 100644 --- a/bash/atomic +++ b/bash/atomic @@ -718,6 +718,7 @@ _atomic_install() { --rootfs --storage --system + --system-package --set --user " diff --git a/docs/atomic-install.1.md b/docs/atomic-install.1.md index 15abc330..f8000b9a 100644 --- a/docs/atomic-install.1.md +++ b/docs/atomic-install.1.md @@ -12,6 +12,7 @@ atomic-install - Execute Image Install Method [**--rootfs**=*ROOTFS*] [**--set**=*NAME*=*VALUE*] [**--storage**] +[**--system-package=auto|build|yes|no**] [**--system**] IMAGE [ARG...] @@ -96,6 +97,18 @@ Note: If the image being pulled contains a label of `system.type=ostree`, atomic will automatically substitute the storage backend to be ostree. This can be overridden with the --storage option. +**--system-package=auto|build|no|yes** +Control how the container will be installed to the system. + +*auto* generates an rpm and install it to the system when the +image defines a .spec file. This is the default. + +*build* build only the software package, without installing it. + +*no* do not generate an rpm package to install the container. + +*yes* generate an rpm package and install it to the system. + **--user** If running as non-root, specify to install the image from the current OSTree repository and manage it through systemd and bubblewrap. diff --git a/tests/integration/test_system_containers.sh b/tests/integration/test_system_containers.sh index c519715e..7eac82df 100755 --- a/tests/integration/test_system_containers.sh +++ b/tests/integration/test_system_containers.sh @@ -255,7 +255,7 @@ test \! -e ${ATOMIC_OSTREE_CHECKOUT_PATH}/${NAME} test \! -e ${ATOMIC_OSTREE_CHECKOUT_PATH}/${NAME}.0 test \! -e ${ATOMIC_OSTREE_CHECKOUT_PATH}/${NAME}.1 -${ATOMIC} pull --storage ostree docker:atomic-test-secret +${ATOMIC} pull --storage ostree docker:atomic-test-secret:latest # Move directly the OSTree reference to a new one, so that we have different names and info doesn't error out mv ${ATOMIC_OSTREE_REPO}/refs/heads/ociimage/atomic-test-secret_3Alatest ${ATOMIC_OSTREE_REPO}/refs/heads/ociimage/atomic-test-secret-ostree_3Alatest ${ATOMIC} info atomic-test-secret-ostree > version.out @@ -332,7 +332,7 @@ teardown # Install from a docker local docker image export NAME="test-docker-system-container-$$" -${ATOMIC} install --name=${NAME} --set=RECEIVER=${SECRET} --system docker:atomic-test-system +${ATOMIC} install --name=${NAME} --set=RECEIVER=${SECRET} --system docker:atomic-test-system:latest test -e /etc/tmpfiles.d/${NAME}.conf test -e ${ATOMIC_OSTREE_CHECKOUT_PATH}/${NAME}.0/${NAME}.service diff --git a/tests/unit/test_pull.py b/tests/unit/test_pull.py index 541d4d07..cc5dd931 100644 --- a/tests/unit/test_pull.py +++ b/tests/unit/test_pull.py @@ -20,7 +20,7 @@ class TestAtomicPull(unittest.TestCase): class Args(): def __init__(self): - self.image = "docker:centos" + self.image = "docker:centos:latest" self.user = False def test_pull_as_privileged_user(self):