initial commit of matchbox setup

.taskfiles/Talos/Taskfile.yaml

@@ -1,36 +1,211 @@
 ---
-version: '3'
+# yaml-language-server: $schema=https://taskfile.dev/schema.json
+version: "3"
 
 tasks:
-  apply-config:
-    desc: apply config files to all nodes
-    cmds:
-      - talosctl apply-config -i -n uiharu.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-uiharu.ctec.run.yaml
-      - talosctl apply-config -i -n sakura.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-sakura.ctec.run.yaml
-      - talosctl apply-config -i -n anya.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-anya.ctec.run.yaml
-      - talosctl apply-config -i -n miri.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-miri.ctec.run.yaml
-  update-config:
-    desc: apply config files to all nodes
-    cmds:
-      - talosctl apply-config -n uiharu.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-uiharu.ctec.run.yaml
-      - talosctl apply-config -n sakura.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-sakura.ctec.run.yaml
-      - talosctl apply-config -n anya.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-anya.ctec.run.yaml
-      - talosctl apply-config -n miri.ctec.run -f ./infrastructure/talos/clusterconfig/cluster-miri.ctec.run.yaml
-  reset:
-    desc: reset the talos cluster
-    cmds:
-      # - talosctl reset --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL --graceful=false --reboot -n ram.ctec.run -e ram.ctec.run
-      # - talosctl reset --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL --graceful=false --reboot -n rem.ctec.run -e rem.ctec.run
-
-      # - echo "Waiting for workers to reset... ^C to stop here"
-      # - sleep 5
-
-      # Reset the control plane nodes
-      - talosctl reset --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL --graceful=false --reboot -n anya.ctec.run -e anya.ctec.run
-      - talosctl reset --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL --graceful=false --reboot -n uiharu.ctec.run -e uiharu.ctec.run
-      - talosctl reset --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL --graceful=false --reboot -n miri.ctec.run -e miri.ctec.run
-      - talosctl reset --system-labels-to-wipe STATE --system-labels-to-wipe EPHEMERAL --graceful=false --reboot -n sakura.ctec.run -e sakura.ctec.run
   bootstrap:
-    desc: bootstrap the cluster
+    desc: Bootstrap Talos
+    summary: |
+      Args:
+        cluster: Cluster to run command against (required)
+        controller: Controller node to run command against (required)
+    prompt: Bootstrap Talos on the '{{.cluster}}' cluster... continue?
     cmds:
-      - talosctl bootstrap -n miri.ctec.run
+      - task: bootstrap-etcd
+        vars: &vars
+          cluster: "{{.cluster}}"
+          controller: "{{.controller}}"
+      - task: fetch-kubeconfig
+        vars: *vars
+      - task: bootstrap-integrations
+        vars: *vars
+    requires:
+      vars:
+        - cluster
+        - controller
+
+  bootstrap-etcd:
+    desc: Bootstrap Etcd
+    cmd: until talosctl --context {{.cluster}} --nodes {{.controller}} bootstrap; do sleep 10; done
+    requires:
+      vars:
+        - cluster
+        - controller
+
+  bootstrap-integrations:
+    desc: Bootstrap core integrations needed for Talos
+    cmds:
+      - until kubectl --context {{.cluster}} wait --for=condition=Ready=False nodes --all --timeout=600s; do sleep 10; done
+      - helmfile --kube-context {{.cluster}} --file {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/integrations/helmfile.yaml apply --skip-diff-on-install --suppress-diff
+      - until kubectl --context {{.cluster}} wait --for=condition=Ready nodes --all --timeout=600s; do sleep 10; done
+    requires:
+      vars:
+        - cluster
+    preconditions:
+      - which helmfile
+      - sh: kubectl config get-contexts {{.cluster}}
+        msg: "Kubectl context {{.cluster}} not found"
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/integrations/helmfile.yaml
+
+  fetch-kubeconfig:
+    desc: Fetch kubeconfig from Talos controllers
+    cmd: |
+      talosctl --context {{.cluster}} kubeconfig --nodes {{.controller}} \
+          --force --force-context-name {{.cluster}} {{.KUBERNETES_DIR}}/{{.cluster}}
+    requires:
+      vars:
+        - cluster
+    preconditions:
+      - talosctl config get-contexts | grep {{.cluster}}
+
+  generate-clusterconfig:
+    desc: Generate clusterconfig for Talos
+    cmds:
+      - doppler run -p talenv -c prd talhelper genconfig --
+        --secret-file {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talsecret.yaml
+        --config-file {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+        --out-dir {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig
+    requires:
+      vars:
+        - cluster
+    preconditions:
+      - which talhelper
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talsecret.yaml
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+
+  bootstrap-matchbox:
+    desc: Bootstrap required Matchbox configuration to Vyos for PXE Boot
+    vars:
+      VYOS_ADDR: gateway.ctec.run
+      VYOS_USER: vyos
+      VYOS_MATCHBOX_DIR: /config/containers/matchbox/data
+      VYOS_MATCHBOX_GROUPS_DIR: "{{.VYOS_MATCHBOX_DIR}}/groups"
+      VYOS_MATCHBOX_PROFILES_DIR: "{{.VYOS_MATCHBOX_DIR}}/profiles"
+      VYOS_MATCHBOX_ASSETS_DIR: "{{.VYOS_MATCHBOX_DIR}}/assets/talos/{{.cluster}}"
+      TALOS_VERSION:
+        sh: |
+          yq -r ".talosVersion" {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+      TALOS_BASE_URL: https://factory.talos.dev/image/737f0b7a5594b47c164b676df2a672327ba7ec0403f8cdfd284c006dab55a551/v1.6.6
+      # TALOS_BASE_URL:
+      #   sh: |
+      #     doppler run -p talenv -c prd talhelper genurl iso \
+      #     | head -n 1 \
+      #     | awk '{split($0,u," "); print u[2]}' \
+      #     | sed -E 's/\/metal-amd64.iso//'
+    cmds:
+      # Prepare folders
+      - ssh -l {{.VYOS_USER}} {{.VYOS_ADDR}} "sudo mkdir -p {{.VYOS_MATCHBOX_GROUPS_DIR}}"
+      - ssh -l {{.VYOS_USER}} {{.VYOS_ADDR}} "sudo mkdir -p {{.VYOS_MATCHBOX_PROFILES_DIR}}"
+      - ssh -l {{.VYOS_USER}} {{.VYOS_ADDR}} "sudo mkdir -p {{.VYOS_MATCHBOX_ASSETS_DIR}}"
+      - ssh -l {{.VYOS_USER}} {{.VYOS_ADDR}} "sudo chown -R {{.VYOS_USER}}:vyattacfg {{.VYOS_MATCHBOX_DIR}}"
+      # Upload Talos to Matchbox
+      # - for: ["kernel-amd64", "initramfs-amd64.xz"]
+      #   cmd: |
+      #     ssh -l {{.VYOS_USER}} {{.VYOS_ADDR}} "wget -q -O {{.VYOS_MATCHBOX_ASSETS_DIR}}/{{.ITEM}} {{.TALOS_BASE_URL}}/{{.ITEM}}"
+      # Generate new cluster config
+      - task: generate-clusterconfig
+        vars:
+          cluster: "{{.cluster}}"
+      # Copy generated clusterconfigs to Matchbox
+      - find {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig/*.yaml -type f | xargs -I{} scp -p {} vyos@{{.VYOS_ADDR}}:/{{.VYOS_MATCHBOX_ASSETS_DIR}}/
+      # Copy groups to Matchbox
+      - find {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/matchbox/groups/*.json -type f | xargs -I{} scp -p {} vyos@{{.VYOS_ADDR}}:/{{.VYOS_MATCHBOX_GROUPS_DIR}}/
+      # Copy profiles to Matchbox
+      - find {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/matchbox/profiles/*.json -type f | xargs -I{} scp -p {} vyos@{{.VYOS_ADDR}}:/{{.VYOS_MATCHBOX_PROFILES_DIR}}/
+      # Restart the Matchbox container
+      - ssh -l {{.VYOS_USER}} {{.VYOS_ADDR}} -t /opt/vyatta/bin/vyatta-op-cmd-wrapper "restart container matchbox"
+    requires:
+      vars:
+        - cluster
+
+  upgrade:
+    desc: Upgrade Talos version for a node
+    vars:
+      TALOS_VERSION:
+        sh: |
+          yq -r ".talosVersion" {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+      TALOS_IMAGE:
+        sh: |
+          doppler run -p talenv -c prd talhelper genurl installer -- \
+            --config-file {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml \
+          | grep {{.node}} \
+          | awk '{split($0,u," "); print u[2]}'
+    cmds:
+      - talosctl --context {{.cluster}} upgrade -n {{.node}} --image {{.TALOS_IMAGE }}
+    requires:
+      vars:
+        - cluster
+        - node
+    preconditions:
+      - which talhelper
+      - talosctl config get-contexts | grep {{.cluster}}
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talenv.sops.yaml
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+      - msg: "Talos image could not be determined for {{.node}}"
+        sh: 'test -n "{{.TALOS_IMAGE}}"'
+
+  upgrade-k8s:
+    desc: Upgrade Kubernetes version for a Talos cluster
+    vars:
+      KUBERNETES_VERSION:
+        sh: |
+          yq -r ".kubernetesVersion" {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+      CONTROLPLANE_NODE:
+        sh: |
+          talosctl --context main config info \
+          | grep Endpoints: \
+          | awk '{split($0,u," "); print u[2]}' \
+          | sed -E 's/,//'
+    cmds:
+      - talosctl upgrade-k8s -n {{.CONTROLPLANE_NODE}} --to {{.KUBERNETES_VERSION}}
+    requires:
+      vars:
+        - cluster
+    preconditions:
+      - which talhelper
+      - talosctl config get-contexts | grep {{.cluster}}
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talenv.sops.yaml
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/talconfig.yaml
+      - msg: "Kubernetes version could not be determined for cluster {{.cluster}}"
+        sh: 'test -n "{{.KUBERNETES_VERSION}}"'
+      - msg: "Control plane node could not be determined for cluster {{.cluster}}"
+        sh: 'test -n "{{.CONTROLPLANE_NODE}}"'
+
+  apply-clusterconfig:
+    desc: Apply clusterconfig for a Talos cluster
+    vars:
+      CLUSTERCONFIG_FILES:
+        sh: find {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig -type f -name '*.yaml' -printf '%f\n'
+    cmds:
+      - for:
+          var: CLUSTERCONFIG_FILES
+        task: _apply-machineconfig
+        vars:
+          cluster: "{{ .cluster }}"
+          filename: "{{.ITEM}}"
+          hostname: |-
+            {{ trimPrefix (printf "%s-" .cluster) .ITEM | trimSuffix ".yaml" }}
+          dry_run: "{{ .dry_run }}"
+    requires:
+      vars:
+        - cluster
+    preconditions:
+      - talosctl config get-contexts | grep {{.cluster}}
+      - test -d {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig
+
+  _apply-machineconfig:
+    internal: true
+    desc: Apply a single Talos machineConfig to a Talos node
+    cmds:
+      - talosctl --context {{.cluster}} apply-config
+        --nodes "{{.hostname}}"
+        --file "{{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig/{{.filename}}"
+        {{ if eq "true" .dry_run }}--dry-run{{ end }}
+    requires:
+      vars:
+        - cluster
+        - hostname
+        - filename
+    preconditions:
+      - talosctl config get-contexts | grep {{.cluster}}
+      - test -f {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/talos/clusterconfig/{{.filename}}

.taskfiles/bootstrap.yml

@@ -25,23 +25,23 @@ tasks:
   flux:
     desc: 3. Bootstrap flux (as per version in manifest)
     cmds:
-      - kubectl apply --context={{.cluster}} -n flux-system --server-side --kustomize {{.CLUSTER_DIR}}/{{.cluster}}/bootstrap/flux
-      - kubectl apply -f {{.CLUSTER_DIR}}/{{.cluster}}/flux/vars/cluster-settings.yaml
+      - kubectl apply --context={{.cluster}} -n flux-system --server-side --kustomize {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/flux
+      - kubectl apply -f {{.KUBERNETES_DIR}}/{{.cluster}}/flux/vars/cluster-settings.yaml
     vars:
       cluster: '{{.cluster | default "main"}}'
 
   cluster:
     desc: 8. Bootstrap Cluster
     cmds:
-      - kubectl apply --context={{.cluster}} --server-side --kustomize {{.CLUSTER_DIR}}/{{.cluster}}/flux/config/
+      - kubectl apply --context={{.cluster}} --server-side --kustomize {{.KUBERNETES_DIR}}/{{.cluster}}/flux/config/
     vars:
       cluster: '{{.cluster | default "main"}}'
 
   crds:
     summary: task {{.TASK}} [cluster=main]?
     desc: 6. Bootstrap CRDs
     cmds:
-      - kubectl apply --server-side --kustomize {{.CLUSTER_DIR}}/{{.cluster}}/bootstrap/crds/
+      - kubectl apply --server-side --kustomize {{.KUBERNETES_DIR}}/{{.cluster}}/bootstrap/crds/
     vars:
       cluster: '{{.cluster | default "main"}}'
 

Taskfile.yml

@@ -2,10 +2,8 @@
 version: "3"
 
 vars:
-  PROJECT_DIR:
-    sh: git rev-parse --show-toplevel
-  CLUSTER_DIR: "{{.PROJECT_DIR}}/kubernetes"
-  TERRAFORM_DIR: "{{.PROJECT_DIR}}/terraform"
+  KUBERNETES_DIR: "{{.ROOT_DIR}}/kubernetes"
+  TERRAFORM_DIR: "{{.ROOT_DIR}}/infrastructure/terraform"
 
 includes:
   fx:

kubernetes/main/apps/system-controllers/kubelet-csr-approver/app/values.yaml

@@ -1,5 +1,5 @@
 ---
 providerRegex: |
-  ^(uiharu|anya|rem|ram|sakura)$
+  ^(uiharu|anya|rem|ram|sakura|miri)$
 
 bypassDnsResolution: true