poc: initial implementation

This is the initial implementation of the Proof of Concept for the
Omnifest Toolkit (`otk`). `otk` is implemented in Python as it's the
shared language of choice between the creators and the users of
`osbuild-mpp` which `otk` aims to improve upon.

The `otk` binary is laid out as follows:

- `otk compile` compiles an omnifest into its targets.

The `otk` package is laid out as follows:

- `otk/parse` contains parsing code for full schemas and single primitives
  such as names or variables, it also contains wrapping types for
  internal types.
- `otk/help` contains helper functions.
- `otk/transform` contains the logic for transforming trees including
  the directive implementations in the directive submodule.
- `otk/error.py` contains `otk` error types, all errors inherit from
  the `OTKError` type.
- `otk/tree.py` contains types and functions to deal with the tree
  transformations that are at the core of `otk`.
- `otk/command.py` contains the subcommands as supported by the `otk` entry
  point.

Other than the above the repository also contains:

- `test/` contains unittests for `otk`.
- `doc/` contains Sphinx documentation for `otk`.
- `example/` contains YAML example files for `otk`.

Signed-off-by: Simon de Vlieger <[email protected]>

.gitignore

@@ -0,0 +1,5 @@
+venv
+*.egg-info
+__pycache__
+.coverage
+htmlcov

Makefile

@@ -1,4 +1,15 @@
+.PHONY: lint
 lint:
 	@find . -name '*.yaml' | xargs yamllint
 
-.PHONY: lint
+.PHONY: type
+type:
+	@mypy src/ test/
+
+.PHONY: format
+format:
+	@ruff format src/ test/
+
+.PHONY: test
+test:
+	@pytest

README.md

@@ -14,7 +14,7 @@ There is documentation about the [format](./doc/format.md) and the available
 
 ## Examples
 
-Read the [examples](./example).
+Read the [examples](./example) to see what omnifests look like.
 
 ## Problem(s)
 

doc/directives.md

@@ -12,9 +12,10 @@ other parts of the omnifest.
 Variable scope is global, an `otk.define` directive anywhere in the omnifest
 tree will result in the defined names being hoisted to the global scope.
 
-Double definitions of variables are forbidden and will cause an error when
-detected. It is thus wise to 'namespace' variables by putting them inside an
-map.
+Redefinitions of variables are forbidden and will cause an error when detected.
+A redefinition of a variable is assigning a value different from the value it
+is currently holding. It is thus wise to 'namespace' variables by putting them
+inside a map.
 
 Expects a `map` for its value.
 
@@ -117,6 +118,8 @@ otk.define:
       values:
         - ${a}
         - ${b}
+        - - 5
+          - 6
 ```
 
 ### otk.op.map.join
@@ -134,7 +137,7 @@ otk.define:
   b:
     b: 2
   c:
-    otk.op.hash.merge:
+    otk.op.map.merge:
       values:
         - ${a}
         - ${b}

example/fedora/minimal-40-x86_64.yaml

@@ -0,0 +1,71 @@
+otk.version: 1
+
+otk.define:
+  version: 40
+  architecture: "aarch64"
+  packages:
+    # These packages are used in the buildroot
+    root:
+      docs: false
+      weak: false
+      packages:
+        include:
+          - "@core"
+        # TODO We can't merge nonexistent vars
+        exclude:
+          - "nonexistent"
+    # These packages are used for the operating system tree which is what ends
+    # up in the outputs.
+    tree:
+      docs: false
+      weak: false
+      packages:
+        include:
+          - "@core"
+          - "initial-setup"
+          - "libxkbcommon"
+          - "NetworkManager-wifi"
+          - "brcmfmac-firmware"
+          - "realtek-firmware"
+          - "iwlwifi-mvm-firmware"
+        # TODO We can't merge nonexistent vars
+        exclude:
+          - "nonexistent"
+    todo:
+      packages:
+        include:
+          - "@bar"
+          - "@baz"
+        # TODO We can't merge nonexistent vars
+        exclude:
+          - "nonexistent"
+    # Repositories to fetch packages from
+    repositories:
+      otk.include: repository/${version}.yaml
+    # GPG keys to verify packages with
+    keys:
+      otk.include: repository/${version}-gpg.yaml
+  filesystem:
+    root:
+      uuid: 6e4ff95f-f662-45ee-a82a-bdf44a2d0b75
+      vfs_type: ext4
+      path: /
+      options: defaults
+    boot:
+      uuid: 0194fdc2-fa2f-4cc0-81d3-ff12045b73c8
+      vfs_type: ext4
+      path: /boot
+      options: defaults
+    boot-efi:
+      uuid: 7B77-95E7
+      vfs_type: vfat
+      path: /boot/efi
+      options: defaults,uid=0,gid=0,umask=077,shortname=winnt
+      passno: 2
+
+otk.target.osbuild:
+  pipelines:
+    - otk.include: "osbuild/root.yaml"
+    - otk.include: "osbuild/pipeline/tree.yaml"
+    - otk.include: "osbuild/pipeline/raw.yaml"
+    - otk.include: "osbuild/pipeline/xz.yaml"

example/fedora/osbuild/pipeline/image.yaml

@@ -0,0 +1,2 @@
+foo:
+  bar

example/fedora/osbuild/pipeline/raw.yaml

@@ -0,0 +1,17 @@
+name: raw
+source-epoch: 1659397331
+stages:
+  #  - otk.external.osbuild.depsolve-dnf4:
+  #      architecture: ${architecture}
+  #      module_platform_id: f${version}
+  #      repositories: ${repositories}
+  #      gpgkeys: ${gpgkeys}
+  #      exclude:
+  #        docs: true
+  #      packages: ${packages.default.root}
+  - type: org.osbuild.selinux
+    options:
+      file_contexts: etc/selinux/targeted/contexts/files/file_contexts
+      labels:
+        /usr/bin/cp: system_u:object_r:install_exec_t:s0
+        /usr/bin/tar: system_u:object_r:install_exec_t:s0

example/fedora/osbuild/pipeline/tree.yaml

@@ -0,0 +1,68 @@
+name: tree
+build: name:root
+stages:
+  - otk.external.osbuild.depsolve-dnf4:
+      architecture: ${architecture}
+      module_platform_id: f${version}
+      repositories: ${packages.repositories}
+      gpgkeys: ${packages.keys}
+      docs: ${packages.tree.docs}
+      weak: ${packages.tree.weak}
+      packages: ${packages.tree.packages}
+  - type: org.osbuild.kernel-cmdline
+    options:
+      root_fs_uuid: ${filesystem.root.uuid}
+      kernel_opts: ro no_timer_check console=ttyS0,115200n8 biosdevname=0 net.ifnames=0
+  - type: org.osbuild.fix-bls
+    options:
+      prefix: ''
+  - type: org.osbuild.locale
+    options:
+      language:
+        otk.customization.locale:
+          scope: tree
+          default:
+            language: en_US
+          defined:
+            language: en_US
+  - type: org.osbuild.hostname
+    options:
+      language:
+        otk.customization.hostname:
+          scope: tree
+          default:
+            hostname: localhost.localdomain
+          defined:
+            language: localhost.localdomain
+  - type: org.osbuild.timezone
+    options:
+      language:
+        otk.customization.language:
+          scope: tree
+          default:
+            zone: UTC
+          defined:
+            zone: UTC
+  - type: org.osbuild.fstab
+    options:
+      filesystems:
+        - ${filesystem.boot}
+        - ${filesystem.boot-efi}
+        - ${filesystem.root}
+  - type: org.osbuild.grub2
+    options:
+      root_fs_uuid: ${filesystem.root.uuid}
+      boot_fs_uuid: ${filesystem.boot.uuid}
+      kernel_opts: ro no_timer_check console=ttyS0,115200n8 biosdevname=0 net.ifnames=0
+      legacy: i386-pc
+      uefi:
+        vendor: fedora
+        unified: true
+      # TODO: expose this somehow from the depsolve
+      saved_entry: "ffffffffffffffffffffffffffffffff-${version}"
+      write_cmdline: false
+      config:
+        default: saved
+  - type: org.osbuild.selinux
+    options:
+      file_contexts: etc/selinux/targeted/contexts/files/file_contexts

example/fedora/osbuild/pipeline/xz.yaml

@@ -0,0 +1,2 @@
+foo:
+  bar