geniso

#!/bin/bash

set -ue

HOSTNAME=$1
FQDN=$2
POD_NETWORK_CIDR=$3
DOMAIN=$4
OUTPUT=$5

ARCH="amd64"

# Must match x.y.*
KUBE_VERSION="1.24.3"
CRIO_VERSION="1.24"

# to perform updates, also review the kubevip commands below

ISO_URL=$(curl https://builds.coreos.fedoraproject.org/streams/stable.json | jq -r .architectures.x86_64.artifacts.metal.formats.iso.disk.location)
ISO_FILENAME=$(basename $ISO_URL)

test -f $ISO_FILENAME || wget $ISO_URL

function run_container() {
    if which podman &>/dev/null ; then
	podman run --rm -v $(pwd):/work -w /work --security-opt label=disable "$@"
    else
	docker run --rm -v $(pwd):/work -w /work -u $(id -u):$(id -g) "$@"
    fi
}

function coreos-installer() {
    run_container quay.io/coreos/coreos-installer:release "$@"
}

function butane() {
    run_container quay.io/coreos/butane:release "$@"
}

BUTANE_PATH=$(mktemp -p .)

cat >$BUTANE_PATH <<EOF
variant: fcos
version: 1.4.0
passwd:
  users:
    - name: core
      ssh_authorized_keys:
        - $(cat ~/.ssh/id_rsa.pub)
storage:
  files:
    - path: /etc/hostname
      mode: 0644
      contents:
        inline: $HOSTNAME
    - path: /usr/local/bin/install_k8s
      mode: 0755
      contents:
        inline: |
          #!/bin/bash -x

          test -f /home/core/.kube/config && exit

          set -ue

          # send a static DHCP client id so DHCP lease can persist if VM is recreated
          nmcli con mod "Wired connection 1" ipv4.dhcp-client-id $HOSTNAME

          cat <<EOF > /etc/modules-load.d/crio.conf
          overlay
          br_netfilter
          EOF

          modprobe br_netfilter
          modprobe overlay

          cat <<EOF > /etc/sysctl.d/99-kubernetes-cri.conf
          net.bridge.bridge-nf-call-iptables  = 1
          net.ipv4.ip_forward                 = 1
          net.bridge.bridge-nf-call-ip6tables = 1
          EOF

          sysctl --system

          cat <<EOF > /etc/yum.repos.d/kubernetes.repo
          [kubernetes]
          name=Kubernetes
          baseurl=https://packages.cloud.google.com/yum/repos/kubernetes-el7-x86_64
          enabled=1
          gpgcheck=1
          gpgkey=https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg
          EOF

          { which conntrack && which crio ; } || {
            while pgrep ostree ; do
              echo waiting for ostree
              sleep 5
            done
            rpm-ostree install conntrack kubelet-$KUBE_VERSION kubeadm-$KUBE_VERSION kubectl-$KUBE_VERSION
            rpm-ostree ex module install cri-o:$CRIO_VERSION/default
            echo rebooting for ostree
            systemctl reboot
          }

          systemctl disable --now docker.socket
          systemctl mask docker.service
          systemctl mask containerd.service

          systemctl enable crio --now

          systemctl enable --now kubelet

          cat <<EOF >/etc/kubeadm.conf
          apiVersion: kubeadm.k8s.io/v1beta3
          kind: InitConfiguration
          nodeRegistration:
            criSocket: unix:///var/run/crio/crio.sock
            kubeletExtraArgs:
              volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
          ---
          kind: ClusterConfiguration
          apiVersion: kubeadm.k8s.io/v1beta3
          kubernetesVersion: v$KUBE_VERSION
          controlPlaneEndpoint: "$FQDN:6443"
          controllerManager:
            extraArgs:
              flex-volume-plugin-dir: "/opt/libexec/kubernetes/kubelet-plugins/volume/exec/"
          networking:
            podSubnet: "$POD_NETWORK_CIDR"
          ---
          kind: KubeletConfiguration
          apiVersion: kubelet.config.k8s.io/v1beta1
          cgroupDriver: systemd
          EOF

          kubeadm init --config /etc/kubeadm.conf
          export KUBECONFIG=/etc/kubernetes/admin.conf

          kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml

          kubectl taint nodes --all node-role.kubernetes.io/control-plane- node-role.kubernetes.io/master-

          kubectl apply -f https://kube-vip.io/manifests/rbac.yaml

          # From the output of:
          #
          # docker run --rm ghcr.io/kube-vip/kube-vip:$(curl -sL https://api.github.com/repos/kube-vip/kube-vip/releases | jq -r ".[0].name") manifest daemonset --ddns --services --arp

          kubectl apply -f - <<EOF
          apiVersion: apps/v1
          kind: DaemonSet
          metadata:
            creationTimestamp: null
            labels:
              app.kubernetes.io/name: kube-vip-ds
              app.kubernetes.io/version: v0.5.0
            name: kube-vip-ds
            namespace: kube-system
          spec:
            selector:
              matchLabels:
                app.kubernetes.io/name: kube-vip-ds
            template:
              metadata:
                creationTimestamp: null
                labels:
                  app.kubernetes.io/name: kube-vip-ds
                  app.kubernetes.io/version: v0.5.0
              spec:
                containers:
                - args:
                  - manager
                  env:
                  - name: vip_arp
                    value: "true"
                  - name: port
                    value: "6443"
                  - name: vip_cidr
                    value: "32"
                  - name: svc_enable
                    value: "true"
                  - name: vip_address
                  - name: prometheus_server
                    value: :2112
                  image: ghcr.io/kube-vip/kube-vip:v0.5.0
                  imagePullPolicy: Always
                  name: kube-vip
                  resources: {}
                  securityContext:
                    capabilities:
                      add:
                      - NET_ADMIN
                      - NET_RAW
                  volumeMounts:
                  - mountPath: /etc/kubernetes/admin.conf
                    name: kubeconfig
                hostAliases:
                - hostnames:
                  - kubernetes
                  ip: 127.0.0.1
                hostNetwork: true
                volumes:
                - hostPath:
                    path: /etc/kubernetes/admin.conf
                  name: kubeconfig
            updateStrategy: {}
          status:
            currentNumberScheduled: 0
            desiredNumberScheduled: 0
            numberMisscheduled: 0
            numberReady: 0
          EOF

          kubectl <<EOF create -f -
          apiVersion: v1
          kind: Service
          metadata:
            labels:
              k8s-app: kube-dns
              kubernetes.io/cluster-service: "true"
              kubernetes.io/name: CoreDNS
            name: kube-dns-nodeport
            namespace: kube-system
          spec:
            ports:
            - nodePort: 30053
              port: 53
              protocol: TCP
              targetPort: 53
              name: dns-tcp
            - nodePort: 30053
              port: 53
              protocol: UDP
              targetPort: 53
              name: dns-udp
            selector:
              k8s-app: kube-dns
            sessionAffinity: None
            type: NodePort
          status:
            loadBalancer: {}
          EOF

          kubectl <<EOF apply -f -
          apiVersion: v1
          data:
            Corefile: |
              .:53 {
                  errors
                  health {
                    lameduck 5s
                  }
                  ready
                  kubernetes cluster.local in-addr.arpa ip6.arpa {
                    pods insecure
                    fallthrough in-addr.arpa ip6.arpa
                    ttl 30
                  }
                  k8s_external $DOMAIN
                  prometheus :9153
                  forward . /etc/resolv.conf {
                    max_concurrent 1000
                  }
                  cache 30
                  loop
                  reload
                  loadbalance
              }
          kind: ConfigMap
          metadata:
            name: coredns
            namespace: kube-system
          EOF

          kubectl apply -f https://raw.githubusercontent.com/kube-vip/kube-vip-cloud-provider/main/manifest/kube-vip-cloud-controller.yaml

          kubectl create -f https://raw.githubusercontent.com/jcmoraisjr/haproxy-ingress/master/docs/haproxy-ingress.yaml
          # make haproxy not require a pesky class
          kubectl patch -n ingress-controller daemonset.apps haproxy-ingress --type=json -p '[{"op": "add", "path": "/spec/template/spec/containers/0/args/-", "value": "--watch-ingress-without-class"}]'
          kubectl label node --all role=ingress-controller

          mkdir -p /home/core/.kube/
          cp /etc/kubernetes/admin.conf /home/core/.kube/config
          chown -R core:core /home/core/.kube
systemd:
  units:
    - name: install_k8s.service
      enabled: true
      contents: |
        [Unit]
        Description=Installs k8s
        After=network-online.target
        Wants=network-online.target
        [Service]
        Type=oneshot
        RemainAfterExit=yes
        ExecStart=/usr/local/bin/install_k8s
        [Install]
        WantedBy=multi-user.target
EOF

IGNITION_PATH=$(mktemp -p .)

butane $BUTANE_PATH >$IGNITION_PATH

coreos-installer iso customize $ISO_FILENAME --dest-device=/dev/vda --dest-ignition=$IGNITION_PATH -o $OUTPUT

rm $IGNITION_PATH $BUTANE_PATH