diff --git a/doc/build-helpers/images/ocitools.section.md b/doc/build-helpers/images/ocitools.section.md index 88011bda79e11..35ca069e0f13e 100644 --- a/doc/build-helpers/images/ocitools.section.md +++ b/doc/build-helpers/images/ocitools.section.md @@ -1,16 +1,125 @@ # pkgs.ociTools {#sec-pkgs-ociTools} -`pkgs.ociTools` is a set of functions for creating runtime container bundles according to the [OCI runtime specification v1.0.0](https://github.com/opencontainers/runtime-spec/blob/v1.0.0/spec.md). -It makes no assumptions about the container runner you choose to use to run the created container. +`pkgs.ociTools` is a set of functions for creating runtime container bundles according to the [OCI runtime specification v1.2](https://opencontainers.org/posts/blog/2024-02-18-oci-runtime-spec-v1-2/), +also referred to as "OCI Images". -The set of functions in `pkgs.ociTools` currently does not handle the [OCI image specification](https://github.com/opencontainers/image-spec). +For `ociTools.buildImage`, and `ociTools.buildLayeredImage`, it is a drop-in replacement for [](#sec-pkgs-dockerTools). -At a high-level an OCI implementation would download an OCI Image then unpack that image into an OCI Runtime filesystem bundle. -At this point the OCI Runtime Bundle would be run by an OCI Runtime. -`pkgs.ociTools` provides utilities to create OCI Runtime bundles. +## buildImage {#ssec-pkgs-ociTools-buildImage} + +This function can be used interchangeably with `dockerTools.buildImage` to build an OCI image. + +Additional, OCI-specific arguments can be passed via `extraOCIArgs`. See [](#ssec-pkgs-ociTools-toOCIImage) for a description of what these arguments can be. + +### Examples {#ssec-pkgs-ociTools-buildImage-examples} + +::: {.example #ex-ociTools-buildImage} +# Creating an OCI image. + +This example uses `ociTools.buildImage` to create a simple container that runs `hello`. + +```nix +{ pkgs }: +pkgs.ociTools.buildImage { + name = "hello"; + tag = "latest"; + copyToRoot = [ pkgs.hello ]; + extraOCIArgs.outputFormat = "tarball"; +} +``` +::: + +## buildLayeredImage {#ssec-pkgs-ociTools-buildLayeredImage} + +This function can be used interchangeably with `dockerTools.buildLayeredImage` to build an OCI image with the underlying layers being individual Nix store paths. + +For the reasons outlined in [](#ssec-pkgs-ociTools-toOCIImage), the store space saving will not be as impactful here as using `dockerTools.buildLayeredImage` directly. + +Additional, OCI-specific arguments can be passed via `extraOCIArgs`. See [](#ssec-pkgs-ociTools-toOCIImage) for a description of what these arguments can be. + +### Examples {#ssec-pkgs-ociTools-buildLayeredImage-examples} + +::: {.example #ex-ociTools-buildLayeredImage} +# Creating a layered OCI Image. + +This example uses `ociTools.buildImage` to create a simple container that provides multiple tools `{animal}say` tools. + +```nix +{ pkgs }: +pkgs.ociTools.buildLayeredImage { + name = "animal-sayer"; + tag = "latest"; + copyToRoot = [ + pkgs.cowsay + pkgs.ponysay + ]; +} +``` +::: + +## toOCIImage {#ssec-pkgs-ociTools-toOCIImage} + +This function is the base of the other OCI image building functions. It converts a Docker-style tarball into an OCI directory or tarball. + +Note that due to the nature of `ociTools`, which converts images built with `dockerTools` to OCI images, the image will be kept as: + +- The individual layer store paths. +- The resulting Docker-style tarball. +- The resulting OCI directory or tarball. + +This makes it less efficient on store space as using `dockerTools` directly. + +### Inputs {#ssec-pkgs-ociTools-toOCIImage-inputs} + +`toOCIImage` expects an attribute set with the following attributes as its argument: + +`docker-tarball` (Path) + +: Path of the Docker-style tarball to be converted. In most cases, this will be the + output of a `dockerTools.*` builder. + +`name` (String, _optional_) + +: Name of the resulting output path. If not specified, and if the `docker-tarball` input has a `name` + attribute, this will default to the Docker tarballs' name, with the `.tar.gz` suffix exchanged + for an `-oci` suffix. + +`outputFormat` (String, _optional_) + +: Output format of the resulting OCI image. Can either be `directory` (output is an OCI directory) or + `tarball` (output is an OCI-style tarball). + +### Examples {#ssec-pkgs-ociTools-toOCIImage-examples} + +::: {.example #ex-ociTools-toOCIImage} +# Converting arbitrary Docker tarballs into OCI images. + +This example uses `ociTools.toOCIImage` to convert a Docker-style tarball into an OCI image. + +```nix +{ ociTools, dockerTools }: +let + docker-tarball = pkgs.dockerTools.buildImage { + name = "hello"; + copyToRoot = [ pkgs.hello ]; + }; +in +ociTools.toOCIImage { + inherit docker-tarball; + name = "oci-hello"; + outputFormat = "tarball"; +} +``` +::: ## buildContainer {#ssec-pkgs-ociTools-buildContainer} +:::{.note} +The `buildContainer` interface is deprecated and will be removed in the 25.11 release. + +Please use the [](#ssec-pkgs-ociTools-buildImage) interface instead. +::: + This function creates an OCI runtime container (consisting of a `config.json` and a root filesystem directory) that runs a single command inside of it. The nix store of the container will contain all referenced dependencies of the given command. @@ -82,7 +191,9 @@ This example uses `ociTools.buildContainer` to create a simple container that ru bash, }: ociTools.buildContainer { - args = [ (lib.getExe bash) ]; + args = [ + (lib.getExe bash) + ]; readonly = false; } diff --git a/doc/redirects.json b/doc/redirects.json index f04c437298455..e917ebcb7babd 100644 --- a/doc/redirects.json +++ b/doc/redirects.json @@ -104,6 +104,18 @@ "ex-pkgs-replace-vars-with": [ "index.html#ex-pkgs-replace-vars-with" ], + "ex-ociTools-buildContainer-bash": [ + "index.html#ex-ociTools-buildContainer-bash" + ], + "ex-ociTools-buildImage": [ + "index.html#ex-ociTools-buildImage" + ], + "ex-ociTools-buildLayeredImage": [ + "index.html#ex-ociTools-buildLayeredImage" + ], + "ex-ociTools-toOCIImage": [ + "index.html#ex-ociTools-toOCIImage" + ], "ex-shfmt": [ "index.html#ex-shfmt" ], @@ -509,6 +521,9 @@ "sec-debug": [ "index.html#sec-debug" ], + "sec-pkgs-ociTools": [ + "index.html#sec-pkgs-ociTools" + ], "sec-prefer-remote-fetch": [ "index.html#sec-prefer-remote-fetch" ], @@ -617,6 +632,36 @@ "ssec-cosmic-settings-fallback": [ "index.html#ssec-cosmic-settings-fallback" ], + "ssec-pkgs-ociTools-buildContainer": [ + "index.html#ssec-pkgs-ociTools-buildContainer" + ], + "ssec-pkgs-ociTools-buildContainer-examples": [ + "index.html#ssec-pkgs-ociTools-buildContainer-examples" + ], + "ssec-pkgs-ociTools-buildContainer-inputs": [ + "index.html#ssec-pkgs-ociTools-buildContainer-inputs" + ], + "ssec-pkgs-ociTools-buildImage": [ + "index.html#ssec-pkgs-ociTools-buildImage" + ], + "ssec-pkgs-ociTools-buildImage-examples": [ + "index.html#ssec-pkgs-ociTools-buildImage-examples" + ], + "ssec-pkgs-ociTools-buildLayeredImage": [ + "index.html#ssec-pkgs-ociTools-buildLayeredImage" + ], + "ssec-pkgs-ociTools-buildLayeredImage-examples": [ + "index.html#ssec-pkgs-ociTools-buildLayeredImage-examples" + ], + "ssec-pkgs-ociTools-toOCIImage": [ + "index.html#ssec-pkgs-ociTools-toOCIImage" + ], + "ssec-pkgs-ociTools-toOCIImage-examples": [ + "index.html#ssec-pkgs-ociTools-toOCIImage-examples" + ], + "ssec-pkgs-ociTools-toOCIImage-inputs": [ + "index.html#ssec-pkgs-ociTools-toOCIImage-inputs" + ], "ssec-stdenv-dependencies": [ "index.html#ssec-stdenv-dependencies" ], @@ -2313,21 +2358,6 @@ "ex-dockerTools-streamNixShellImage-addingShellHook": [ "index.html#ex-dockerTools-streamNixShellImage-addingShellHook" ], - "sec-pkgs-ociTools": [ - "index.html#sec-pkgs-ociTools" - ], - "ssec-pkgs-ociTools-buildContainer": [ - "index.html#ssec-pkgs-ociTools-buildContainer" - ], - "ssec-pkgs-ociTools-buildContainer-inputs": [ - "index.html#ssec-pkgs-ociTools-buildContainer-inputs" - ], - "ssec-pkgs-ociTools-buildContainer-examples": [ - "index.html#ssec-pkgs-ociTools-buildContainer-examples" - ], - "ex-ociTools-buildContainer-bash": [ - "index.html#ex-ociTools-buildContainer-bash" - ], "sec-pkgs-portableService": [ "index.html#sec-pkgs-portableService" ], diff --git a/doc/release-notes/rl-2511.section.md b/doc/release-notes/rl-2511.section.md index 5192f7a7d48bb..8a8b17f190710 100644 --- a/doc/release-notes/rl-2511.section.md +++ b/doc/release-notes/rl-2511.section.md @@ -264,6 +264,10 @@ Use `python3Packages.ddgs` instead. See [release note for v9.0.0](https://github.com/deedy5/ddgs/releases/tag/v9.0.0) +- The `ociTools` builder set has been reworked to support building OCI images according to the [v1.2 runtime specification](https://opencontainers.org/posts/blog/2024-02-18-oci-runtime-spec-v1-2/). + The interface of the new `ociTools.buildImage` and `ociTools.buildLayeredImage` are drop-in-compatible with their `dockerTools` counterparts. Additionally, a generic `ociTools.toOCIImage` builder + has been added that converts Docker-style tarball images to OCI directories or tarballs. + ## Other Notable Changes {#sec-nixpkgs-release-25.11-notable-changes} @@ -394,6 +398,8 @@ - `lib.options.mkAliasOptionModuleMD` is now obsolete; use the identical [`lib.options.mkAliasOptionModule`] instead. +- `ociTools.buildContainer` has been deprecated in favor of `ociTools.buildImage`. + - `types.either` silently accepted mismatching types when used in `freeformType`. Module maintainers should fix the used type In most cases wrapping `either` with `attrsOf` should be sufficient. diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 80b652004d52d..4fab9ef4708f0 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1,11 +1,9 @@ { system, pkgs, - # Projects the test configuration into a the desired value; usually # the test runner: `config: config.test`. callTest, - }: # The return value of this function will be an attrset with arbitrary depth and # the `anything` returned by callTest at its test leaves. @@ -1108,6 +1106,7 @@ in nzbhydra2 = runTest ./nzbhydra2.nix; obs-studio = runTest ./obs-studio.nix; oci-containers = handleTestOn [ "aarch64-linux" "x86_64-linux" ] ./oci-containers.nix { }; + oci-tools = handleTest ./oci-tools.nix { }; ocis = runTest ./ocis.nix; ocsinventory-agent = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./ocsinventory-agent.nix { }; octoprint = runTest ./octoprint.nix; diff --git a/nixos/tests/oci-tools.nix b/nixos/tests/oci-tools.nix new file mode 100644 index 0000000000000..629e504f47964 --- /dev/null +++ b/nixos/tests/oci-tools.nix @@ -0,0 +1,64 @@ +# This test creates a simple image with `ociTools` and sees if it executes. + +import ./make-test-python.nix ( + { pkgs, lib, ... }: + let + basicImage = pkgs.ociTools.buildImage { + name = "bash"; + tag = "latest"; + copyToRoot = [ pkgs.bash ]; + extraOCIArgs.outputFormat = "tarball"; + }; + + layeredImage = pkgs.ociTools.buildLayeredImage rec { + name = "layered-animals"; + tag = "latest"; + contents = [ + pkgs.cowsay + pkgs.ponysay + ]; + extraOCIArgs.name = name; + }; + + convertedImage = pkgs.ociTools.toOCIImage { + # runs `hello`. + docker-tarball = pkgs.dockerTools.examples.layered-image; + }; + in + { + name = "oci-tools"; + meta.maintainers = [ pkgs.lib.maintainers.msanft ]; + + nodes.machine = + { pkgs, ... }: + { + virtualisation = { + diskSize = 3072; + docker.enable = true; + }; + environment.systemPackages = [ pkgs.skopeo ]; + }; + + testScript = '' + machine.wait_for_unit("sockets.target") + + with subtest("Image conversion works"): + with subtest("assumption"): + machine.succeed("skopeo copy oci:${convertedImage} docker-daemon:hello:latest --insecure-policy") + machine.succeed("docker run --rm hello | grep -i hello") + machine.succeed("docker image rm hello:latest") + + with subtest("Basic image building works"): + with subtest("assumption"): + machine.succeed("skopeo copy oci-archive:${basicImage} docker-daemon:bash:latest --insecure-policy") + machine.succeed("docker run --rm bash ${lib.getExe pkgs.bash} -c 'echo hello' | grep -i hello") + machine.succeed("docker image rm bash:latest") + + with subtest("Layered image building works"): + with subtest("assumption"): + machine.succeed("skopeo copy oci:${layeredImage} docker-daemon:layered-animals:latest --insecure-policy") + machine.succeed("docker run --rm layered-animals ${lib.getExe pkgs.cowsay} 'hello' | grep -i hello") + machine.succeed("docker image rm layered-animals:latest") + ''; + } +) diff --git a/pkgs/build-support/oci-tools/default.nix b/pkgs/build-support/oci-tools/default.nix deleted file mode 100644 index a9b67878d791f..0000000000000 --- a/pkgs/build-support/oci-tools/default.nix +++ /dev/null @@ -1,143 +0,0 @@ -{ - lib, - writeText, - runCommand, - writeClosure, -}: - -{ - buildContainer = - { - args, - mounts ? { }, - os ? "linux", - arch ? "x86_64", - readonly ? false, - }: - let - sysMounts = { - "/proc" = { - type = "proc"; - source = "proc"; - }; - "/dev" = { - type = "tmpfs"; - source = "tmpfs"; - options = [ - "nosuid" - "strictatime" - "mode=755" - "size=65536k" - ]; - }; - "/dev/pts" = { - type = "devpts"; - source = "devpts"; - options = [ - "nosuid" - "noexec" - "newinstance" - "ptmxmode=0666" - "mode=755" - "gid=5" - ]; - }; - "/dev/shm" = { - type = "tmpfs"; - source = "shm"; - options = [ - "nosuid" - "noexec" - "nodev" - "mode=1777" - "size=65536k" - ]; - }; - "/dev/mqueue" = { - type = "mqueue"; - source = "mqueue"; - options = [ - "nosuid" - "noexec" - "nodev" - ]; - }; - "/sys" = { - type = "sysfs"; - source = "sysfs"; - options = [ - "nosuid" - "noexec" - "nodev" - "ro" - ]; - }; - "/sys/fs/cgroup" = { - type = "cgroup"; - source = "cgroup"; - options = [ - "nosuid" - "noexec" - "nodev" - "relatime" - "ro" - ]; - }; - }; - config = writeText "config.json" ( - builtins.toJSON { - ociVersion = "1.0.0"; - platform = { - inherit os arch; - }; - - linux = { - namespaces = map (type: { inherit type; }) [ - "pid" - "network" - "mount" - "ipc" - "uts" - ]; - }; - - root = { - path = "rootfs"; - inherit readonly; - }; - - process = { - inherit args; - user = { - uid = 0; - gid = 0; - }; - cwd = "/"; - }; - - mounts = lib.mapAttrsToList ( - destination: - { - type, - source, - options ? null, - }: - { - inherit - destination - type - source - options - ; - } - ) sysMounts; - } - ); - in - runCommand "join" { } '' - set -o pipefail - mkdir -p $out/rootfs/{dev,proc,sys} - cp ${config} $out/config.json - xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/ - ''; -} diff --git a/pkgs/build-support/oci/default.nix b/pkgs/build-support/oci/default.nix new file mode 100644 index 0000000000000..a483661332cd9 --- /dev/null +++ b/pkgs/build-support/oci/default.nix @@ -0,0 +1,234 @@ +{ + lib, + stdenvNoCC, + skopeo, + dockerTools, + writeText, + writeClosure, + runCommand, + nixosTests, +}: +let + /* + toOCIImage is a generic conversion tool for converting Docker-style tarballs + into OCI images (directories or tarballs). + + # Inputs + - `docker-tarball` (drv): The Docker-style tarball to convert. In most cases, this will + be the output of a `dockerTools.buildImage` call. + - `name` (string): The name of the resulting OCI image. If not provided, the name of the + Docker-style tarball will be used (with the `.tar.gz` suffix removed). + - `outputFormat` (string): The format of the resulting OCI image. Must be one of: + - `directory`: The output will be an OCI directory. + - `tarball`: The output will be a tarball of an OCI directory. + */ + toOCIImage = lib.extendMkDerivation { + constructDrv = stdenvNoCC.mkDerivation; + excludeDrvArgNames = [ + "docker-tarball" + "name" + "outputFormat" + ]; + extendDrvArgs = + finalAttrs: + { + docker-tarball, + name ? null, + outputFormat ? "directory", + }: + let + outputName = + if name != null then + name + else if lib.hasAttr "name" docker-tarball then + "${lib.strings.removeSuffix ".tar.gz" docker-tarball.name}-oci" + else + throw "The `name` argument attribute must be provided if the `docker-tarball` input does not have a `name` attribute"; + + skopeoOutputFormats = { + directory = "oci"; + tarball = "oci-archive"; + }; + skopeoOutputFormat = + skopeoOutputFormats."${outputFormat}" + or (throw "`outputFormat` must be one of: ${lib.concatStringsSep ", " (lib.attrNames skopeoOutputFormats)}"); + in + stdenvNoCC.mkDerivation { + name = outputName; + src = docker-tarball; + dontUnpack = true; + nativeBuildInputs = [ skopeo ]; + buildPhase = '' + runHook preBuild + + skopeo copy docker-archive:$src ${skopeoOutputFormat}:$out --insecure-policy --tmpdir . + + runHook postBuild + ''; + }; + }; + + # Convenience bindings for drop-in compatibility with `dockerTools`. + mkDockerToolsDropin = + target: args: + let + dockerToolsArgs = lib.removeAttrs args [ "extraOCIArgs" ]; + in + toOCIImage ( + { + docker-tarball = (dockerTools."${target}" dockerToolsArgs); + } + // (args.extraOCIArgs or { }) + ); +in +{ + buildImage = mkDockerToolsDropin "buildImage"; + buildLayeredImage = mkDockerToolsDropin "buildLayeredImage"; + + # THE BELOW INTERFACE IS DEPRECATED AND WILL BE REMOVED WITH THE 25.11 RELEASE. + buildContainer = + lib.warn + '' + `ociTools.buildContainer` is deprecated and will be removed in the 25.11 release. + Please use `ociTools.buildImage` instead. + '' + ( + { + args, + mounts ? { }, + os ? "linux", + arch ? "x86_64", + readonly ? false, + }: + let + sysMounts = { + "/proc" = { + type = "proc"; + source = "proc"; + }; + "/dev" = { + type = "tmpfs"; + source = "tmpfs"; + options = [ + "nosuid" + "strictatime" + "mode=755" + "size=65536k" + ]; + }; + "/dev/pts" = { + type = "devpts"; + source = "devpts"; + options = [ + "nosuid" + "noexec" + "newinstance" + "ptmxmode=0666" + "mode=755" + "gid=5" + ]; + }; + "/dev/shm" = { + type = "tmpfs"; + source = "shm"; + options = [ + "nosuid" + "noexec" + "nodev" + "mode=1777" + "size=65536k" + ]; + }; + "/dev/mqueue" = { + type = "mqueue"; + source = "mqueue"; + options = [ + "nosuid" + "noexec" + "nodev" + ]; + }; + "/sys" = { + type = "sysfs"; + source = "sysfs"; + options = [ + "nosuid" + "noexec" + "nodev" + "ro" + ]; + }; + "/sys/fs/cgroup" = { + type = "cgroup"; + source = "cgroup"; + options = [ + "nosuid" + "noexec" + "nodev" + "relatime" + "ro" + ]; + }; + }; + config = writeText "config.json" ( + builtins.toJSON { + ociVersion = "1.0.0"; + platform = { + inherit os arch; + }; + + linux = { + namespaces = map (type: { inherit type; }) [ + "pid" + "network" + "mount" + "ipc" + "uts" + ]; + }; + + root = { + path = "rootfs"; + inherit readonly; + }; + + process = { + inherit args; + user = { + uid = 0; + gid = 0; + }; + cwd = "/"; + }; + + mounts = lib.mapAttrsToList ( + destination: + { + type, + source, + options ? null, + }: + { + inherit + destination + type + source + options + ; + } + ) sysMounts; + } + ); + in + runCommand "join" { } '' + set -o pipefail + mkdir -p $out/rootfs/{dev,proc,sys} + cp ${config} $out/config.json + xargs tar c < ${writeClosure args} | tar -xC $out/rootfs/ + '' + ); + + tests = { + inherit (nixosTests) oci-tools; + }; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 1b166db126cba..9c192b081e2a7 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -771,7 +771,7 @@ with pkgs; nix-gitignore = callPackage ../build-support/nix-gitignore { }; - ociTools = callPackage ../build-support/oci-tools { }; + ociTools = callPackage ../build-support/oci { }; inherit (callPackages ../build-support/setup-hooks/patch-rc-path-hooks { }) patchRcPathBash