dfdaemon-daemonset.yaml

{{- if .Values.dfdaemon.enable }}
apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: {{ template "dragonfly.dfdaemon.fullname" . }}
  labels:
    app: {{ template "dragonfly.fullname" . }}
    chart: {{ .Chart.Name }}-{{ .Chart.Version }}
    release: {{ .Release.Name }}
    heritage: {{ .Release.Service }}
    component: {{ .Values.dfdaemon.name }}
  annotations:
    checksum/config: {{ include (print $.Template.BasePath "/dfdaemon/dfdaemon-configmap.yaml") . | sha256sum }}
  {{- if .Values.dfdaemon.daemonsetAnnotations }}
{{ toYaml .Values.dfdaemon.daemonsetAnnotations | indent 4 }}
  {{- end }}
spec:
  selector:
    matchLabels:
      app: {{ template "dragonfly.fullname" . }}
      component: "{{ .Values.dfdaemon.name }}"
      release: {{ .Release.Name }}
  template:
    metadata:
      labels:
        app: {{ template "dragonfly.fullname" . }}
        component: "{{ .Values.dfdaemon.name }}"
        release: {{ .Release.Name }}
        {{- if .Values.dfdaemon.podLabels }}
{{ toYaml .Values.dfdaemon.podLabels | indent 8 }}
        {{- end }}
      {{- if .Values.dfdaemon.podAnnotations }}
      annotations:
{{ toYaml .Values.dfdaemon.podAnnotations | indent 8 }}
      {{- end }}
    spec:
      {{- if .Values.dfdaemon.nodeSelector }}
      nodeSelector:
{{ toYaml .Values.dfdaemon.nodeSelector | indent 8 }}
      {{- end }}
      hostNetwork: {{ .Values.dfdaemon.hostNetwork }}
      {{- if .Values.dfdaemon.hostNetwork }}
      dnsPolicy: "ClusterFirstWithHostNet"
      {{- end }}
      hostPID: {{ or (and .Values.containerRuntime.docker.enable .Values.containerRuntime.docker.restart) .Values.containerRuntime.containerd.enable .Values.containerRuntime.crio.enable }}
      {{- if .Values.dfdaemon.tolerations }}
      tolerations:
{{ toYaml .Values.dfdaemon.tolerations | indent 8 }}
      {{- end }}
      {{- if .Values.dfdaemon.affinity }}
      affinity:
{{ toYaml .Values.dfdaemon.affinity | indent 8 }}
      {{- end }}
      {{- if quote .Values.dfdaemon.terminationGracePeriodSeconds }}
      terminationGracePeriodSeconds: {{ .Values.dfdaemon.terminationGracePeriodSeconds }}
      {{- end }}
      {{- if and (.Capabilities.APIVersions.Has "scheduling.k8s.io/v1beta1") (.Values.dfdaemon.priorityClassName) }}
      priorityClassName: {{ .Values.dfdaemon.priorityClassName }}
      {{- end }}
      {{- if .Values.dfdaemon.hostAliases }}
      hostAliases:
{{ toYaml .Values.dfdaemon.hostAliases | indent 8 }}
      {{- end }}
      containers:
      - name: dfdaemon
        image: "{{ .Values.dfdaemon.image }}:{{ .Values.dfdaemon.tag }}"
        imagePullPolicy: {{ .Values.dfdaemon.pullPolicy | quote }}
        resources:
{{ toYaml .Values.dfdaemon.resources | indent 12 }}
        ports:
        - containerPort: {{ .Values.dfdaemon.containerPort }}
          {{- if and (not .Values.dfdaemon.hostNetwork) (empty .Values.dfdaemon.config.proxy.tcpListen.namespace) }}
          hostPort: {{ .Values.dfdaemon.hostPort }}
          hostIP: 127.0.0.1
          {{- end }}
          protocol: TCP
        readinessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr={{ .Values.dfdaemon.config.download.peerGRPC.tcpListen.listen }}:{{ .Values.dfdaemon.config.download.peerGRPC.tcpListen.port }}"]
          initialDelaySeconds: 5
          periodSeconds: 10
        livenessProbe:
          exec:
            command: ["/bin/grpc_health_probe", "-addr={{ .Values.dfdaemon.config.download.peerGRPC.tcpListen.listen }}:{{ .Values.dfdaemon.config.download.peerGRPC.tcpListen.port }}"]
          initialDelaySeconds: 15
          periodSeconds: 10
        {{- if and .Values.containerRuntime.docker.enable (not .Values.containerRuntime.docker.restart) }}
        {{- if .Values.containerRuntime.docker.injectHosts }}
        lifecycle:
          postStart:
            exec:
              command:
              - "/bin/sh"
              - "-c"
              - |
                # inject hosts after dfdaemon started
                domains="{{- join " " .Values.containerRuntime.docker.registryDomains }}"
                # remove static dns in pod /etc/hosts, which injected by host network
                sed -i '/# Dragonfly SNI Host/d' /etc/hosts

                if [[ -n "$domains" ]]; then
                  for domain in $domains; do
                    # inject static dns into /host/etc/hosts
                    if grep "127.0.0.1 $domain" /host/etc/hosts; then
                      echo "Dragonfly SNI Host $domain Found in /host/etc/hosts"
                      continue
                    else
                      echo "Try to add dragonfly SNI host $domain"
                      echo "127.0.0.1 $domain # Dragonfly SNI Host $domain" >> /host/etc/hosts
                      echo "Dragonfly SNI host $domain added"
                    fi
                  done
                fi
          preStop:
            exec:
              command:
              - "/bin/sh"
              - "-c"
              - |
                # when stop dfdaemon, clean up injected hosts info in /etc/hosts for current node
                echo "$(sed '/# Dragonfly SNI Host/d' /host/etc/hosts)" > /host/etc/hosts
        {{- end }}
        {{- end }}
        volumeMounts:
        - name: config
          mountPath: "/etc/dragonfly"
        {{- if and .Values.containerRuntime.docker.enable (not .Values.containerRuntime.docker.restart) }}
        {{- if and .Values.containerRuntime.docker.injectHosts }}
        - name: etc
          mountPath: /host/etc
        {{- end }}
        {{- end }}
        {{- if .Values.containerRuntime.docker.enable }}
        - name: d7y-ca
          mountPath: /etc/dragonfly-ca
        {{- end }}
        {{- if .Values.dfdaemon.extraVolumeMounts }}
        {{- toYaml .Values.dfdaemon.extraVolumeMounts | nindent 8 }}
        {{- end }}
        {{- if and (not .Values.dfdaemon.hostNetwork) .Values.dfdaemon.config.proxy.tcpListen.namespace }}
        - name: run
          mountPath: /run/dragonfly
        - name: data
          mountPath: {{ .Values.dfdaemon.config.dataDir }}
        securityContext:
          capabilities:
            add:
            - SYS_ADMIN
        {{- end }}
      {{- if or (and (not .Values.dfdaemon.hostNetwork) .Values.dfdaemon.config.proxy.tcpListen.namespace) .Values.containerRuntime.containerd.enable .Values.containerRuntime.docker.enable }}
      initContainers:
      {{- if and (not .Values.dfdaemon.hostNetwork) .Values.dfdaemon.config.proxy.tcpListen.namespace }}
      - name: mount-netns
        image: "{{ .Values.dfdaemon.image }}:{{ .Values.dfdaemon.tag }}"
        imagePullPolicy: {{ .Values.dfdaemon.pullPolicy | quote }}
        resources:
{{ toYaml .Values.dfdaemon.resources | indent 12 }}
        # FIXME dfget daemon only need /proc/1/ns/net and CAP_SYS_ADMIN
        # but containerd resolves the symbolic of /proc/1/ns/net from v1.5.0
        # due to /proc/1/ns/net is not a regular symbolic link, it always failed.
        # for keeping only CAP_SYS_ADMIN capability, use init container to bind mount only netns to /run/dragonfly/net
        # https://github.com/containerd/containerd/blob/v1.5.0/pkg/cri/opts/spec_linux.go#L171
        command:
        - /bin/sh
        - -cx
        - |-
          if [ ! -e "/run/dragonfly/net" ]; then
            touch /run/dragonfly/net
          fi
          i1=$(stat -L -c %i /host/ns/net)
          i2=$(stat -L -c %i /run/dragonfly/net)
          if [ "$i1" != "$i2" ]; then
            /bin/mount -o bind /host/ns/net /run/dragonfly/net
          fi
        volumeMounts:
        - name: hostns
          mountPath: /host/ns
        - name: run
          mountPath: /run/dragonfly
          # bind mount need Bidirectional to propagate into host
          mountPropagation: Bidirectional
        securityContext:
          # open /proc/1/ns need privilege permission
          privileged: true
      {{- end }}
      {{- if .Values.containerRuntime.extraInitContainers }}
{{ toYaml .Values.containerRuntime.extraInitContainers | indent 6 }}
      {{- else if .Values.containerRuntime.docker.enable }}
      - name: update-docker-config
        image: "{{ .Values.containerRuntime.initContainerImage }}"
        imagePullPolicy: {{ .Values.dfdaemon.pullPolicy | quote }}
        resources:
{{ toYaml .Values.dfdaemon.resources | indent 12 }}
        command:
        - /bin/sh
        - -cx
        - |-
          mkdir -p /tmp/dragonfly-ca
          cd /tmp/dragonfly-ca

          openssl genrsa -out cakey.pem 2048

          cat << EOF > root.conf
          [ req ]
          default_bits        = 2048
          default_keyfile     = key.pem
          default_md          = sha256
          distinguished_name  = req_distinguished_name
          req_extensions      = req_ext
          string_mask         = nombstr
          x509_extensions     = x509_ext
          [ req_distinguished_name ]
          countryName                 = Country Name (2 letter code)
          countryName_default         = {{.Values.containerRuntime.docker.caCert.countryName}}
          stateOrProvinceName         = State or Province Name (full name)
          stateOrProvinceName_default = {{.Values.containerRuntime.docker.caCert.stateOrProvinceName}}
          localityName                = Locality Name (eg, city)
          localityName_default        = {{.Values.containerRuntime.docker.caCert.localityName}}
          organizationName            = Organization Name (eg, company)
          organizationName_default    = {{.Values.containerRuntime.docker.caCert.organizationName}}
          commonName                  = Common Name (e.g. server FQDN or YOUR name)
          commonName_max              = 64
          commonName_default          = {{.Values.containerRuntime.docker.caCert.commonName}}
          [ x509_ext ]
          authorityKeyIdentifier = keyid,issuer
          basicConstraints       = CA:TRUE
          keyUsage               = digitalSignature, keyEncipherment, keyCertSign, cRLSign
          subjectKeyIdentifier   = hash
          [ req_ext ]
          basicConstraints     = CA:TRUE
          keyUsage             = digitalSignature, keyEncipherment, keyCertSign, cRLSign
          subjectKeyIdentifier = hash
          EOF

          openssl req -batch -new -x509 -key ./cakey.pem -out ./cacert.pem -days 65536 -config ./root.conf
          openssl x509 -inform PEM -in ./cacert.pem -outform DER -out ./CA.cer

          openssl x509 -in ./cacert.pem -noout -text
          # update ca for golang program(docker in host), refer: https://github.com/golang/go/blob/go1.17/src/crypto/x509/root_linux.go#L8
          ca_list="/etc/ssl/certs/ca-certificates.crt /etc/pki/tls/certs/ca-bundle.crt /etc/ssl/ca-bundle.pem /etc/pki/tls/cacert.pem /etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem /etc/ssl/cert.pem"
          for ca in $ca_list; do
            ca="/host$ca"
            if [[ -e "$ca" ]]; then
              echo "CA $ca" found
              if grep "Dragonfly Authority CA" "$ca"; then
                echo "Dragonfly Authority ca found"
                if [[ -e /host/etc/dragonfly-ca/cakey.pem && -e /host/etc/dragonfly-ca/cacert.pem ]]; then
                  echo "CA cert and key ready"
                  break
                else
                  echo "Warning: CA cert and key not ready"
                fi
              fi
              echo "Try to add Dragonfly CA"
              echo "# Dragonfly Authority CA" > cacert.toadd.pem
              cat cacert.pem >> cacert.toadd.pem
              cat cacert.toadd.pem >> "$ca"
              echo "Dragonfly CA added"
              cp -f ./cakey.pem ./cacert.pem /host/etc/dragonfly-ca/
              break
            fi
          done

          {{- if not .Values.containerRuntime.docker.restart }}
          domains="{{- join " " .Values.containerRuntime.docker.registryDomains }}"
          if [[ -n "$domains" ]]; then
            for domain in $domains; do
              # inject docker cert by registry domain
              dir=/host/etc/docker/certs.d/$domain
              mkdir -p "$dir"
              echo copy CA cert to $dir
              cp -f /host/etc/dragonfly-ca/cacert.pem "$dir/ca.crt"
            done
          fi
          {{- end }}
          {{- if .Values.containerRuntime.docker.restart }}
          # inject docker proxy setting and restart docker
          # currently, without host pid in container, we can not nsenter with pid and can not invoke systemctl correctly.
          status=$(nsenter -t 1 -m systemctl -- status docker --no-pager | grep http-proxy.conf)
          if [[ -n "$status" ]]; then
            echo Docker proxy already enabled, skip
          else
            echo Try to inject proxy and restart docker
            path=$(nsenter -t 1 -m systemctl -- show -p FragmentPath docker.service | grep -o "/.*")
            if [[ -z "$path" ]]; then
                echo docker.service not found
                exit 1
            fi
            nsenter -t 1 -m mkdir -- -p "$path".d
            nsenter -t 1 -m sh -- -c "echo '[Service]' > $path.d/http-proxy.conf"
            nsenter -t 1 -m sh -- -c "echo 'Environment=\"HTTP_PROXY=http://127.0.0.1:{{ .Values.dfdaemon.hostPort}}\"' >> $path.d/http-proxy.conf"
            nsenter -t 1 -m sh -- -c "echo 'Environment=\"HTTPS_PROXY=http://127.0.0.1:{{ .Values.dfdaemon.hostPort}}\"' >> $path.d/http-proxy.conf"
            nsenter -t 1 -m sh -- -c "echo 'Environment=\"NO_PROXY={{ join "," .Values.containerRuntime.docker.skipHosts }}\"' >> $path.d/http-proxy.conf"
            nsenter -t 1 -m systemctl -- daemon-reload
            nsenter -t 1 -m systemctl -- restart docker.service
          fi
          {{- end }}
        volumeMounts:
        - name: etc
          mountPath: /host/etc
        {{- if .Values.containerRuntime.docker.restart }}
        securityContext:
          # nsenter need privilege permission
          privileged: true
        {{- end }}
      {{- else if .Values.containerRuntime.containerd.enable }}
      - name: update-containerd
        image: "{{ .Values.containerRuntime.initContainerImage }}"
        imagePullPolicy: {{ .Values.dfdaemon.pullPolicy | quote }}
        resources:
{{ toYaml .Values.dfdaemon.resources | indent 12 }}
        command:
        - /bin/sh
        - -cx
        - |-
          etcContainerd=/host{{ .Values.containerRuntime.containerd.configPathDir }}
          if [[ -e $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }} ]]; then
            echo containerd config found
          else
            echo $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }} not found
            exit 1
          fi
          cat $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}

          registries="{{- join " " .Values.containerRuntime.containerd.registries }}"
          if [[ -n "$domains" ]]; then
            echo empty registry domains
            exit 1
          fi
          # detect containerd config version
          need_restart=0
          if grep "version[^=]*=[^2]*2" $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}; then
            # inject v2 mirror setting

            # get config_path if set
            {{- if .Values.containerRuntime.containerd.injectConfigPath }}
            config_path=$etcContainerd/certs.d
            {{- else }}
            config_path=$(cat $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }} | tr '"' ' ' | grep config_path | awk '{print $3}')
            {{- end }}
            if [[ -z "$config_path" ]]; then
              echo config_path is not enabled, just add one mirror in {{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}
              # parse registry domain
              registry={{ .Values.dfdaemon.config.proxy.registryMirror.url}}
              domain=$(echo $registry | sed -e "s,https://,," | sed "s,:.*,,")
              # inject registry
              if grep "registry.mirrors.\"$domain\"" $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}; then
                # TODO merge mirrors
                echo "registry $registry found in {{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}, skip"
              else
                cat << EOF >> $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$domain"]
            endpoint = ["http://127.0.0.1:{{ .Values.dfdaemon.hostPort}}","$registry"]
          EOF
                echo "Registry $domain added"
                need_restart=1
              fi
            else
              echo config_path is enabled, add mirror in $config_path
              # TODO check whether config_path is enabled, if not, add it
              tmp=$(cat $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }} | tr '"' ' ' | grep config_path | awk '{print $3}')
              if [[ -z "$tmp" ]]; then
                echo inject config_path into $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}
                cat << EOF >> $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}
          [plugins."io.containerd.grpc.v1.cri".registry]
            config_path = "{{ .Values.containerRuntime.containerd.configPathDir }}/certs.d"
          EOF
              fi
              mkdir -p $etcContainerd/certs.d
              for registry in $registries; do
                # parse registry domain
                domain=$(echo $registry | sed -e "s,http.://,," | sed "s,:.*,,")
                # inject registry
                mkdir -p $etcContainerd/certs.d/$domain
                if [[ -e "$etcContainerd/certs.d/$domain/hosts.toml" ]]; then
                  echo "registry $registry found in {{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}, skip"
                  continue
                else
                  cat << EOF >> $etcContainerd/certs.d/$domain/hosts.toml
          server = "$registry"
          [host."http://127.0.0.1:{{ .Values.dfdaemon.hostPort}}"]
            capabilities = ["pull", "resolve"]
            [host."http://127.0.0.1:{{ .Values.dfdaemon.hostPort}}".header]
            X-Dragonfly-Registry = ["$registry"]
          EOF
                  echo "Registry $domain added"
                  {{- if not .Values.containerRuntime.containerd.injectConfigPath }}
                  need_restart=1
                  {{- end }}
                fi
              done
            fi
          else
            # inject legacy v1 mirror setting
            echo containerd config is version 1, just only support one mirror in {{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}
            # parse registry domain
            registry={{ .Values.dfdaemon.config.proxy.registryMirror.url}}
            domain=$(echo {{ .Values.dfdaemon.config.proxy.registryMirror.url}} | sed -e "s,http.://,," | sed "s,:.*,,")
            # inject registry
            if grep "registry.mirrors.\"$domain\"" $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}; then
              # TODO merge mirrors
              echo "registry $registry found in {{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}, skip"
            else
              cat << EOF >> $etcContainerd/{{ default "config.toml" .Values.containerRuntime.containerd.configFileName }}
          [plugins.cri.registry.mirrors."$domain"]
            endpoint = ["http://127.0.0.1:{{ .Values.dfdaemon.hostPort}}","$registry"]
          EOF
              echo "Registry $domain added"
              need_restart=1
            fi
          fi

          # restart containerd
          # currently, without host pid in container, we can not nsenter with pid and can not invoke systemctl correctly.
          if [[ "$need_restart" -gt 0 ]]; then
            nsenter -t 1 -m systemctl -- restart containerd.service
          fi
        volumeMounts:
        - name: containerd-conf
          mountPath: /host{{ .Values.containerRuntime.containerd.configPathDir }}
        securityContext:
          # nsenter need privilege permission
          privileged: true
      {{- else if .Values.containerRuntime.crio.enable }}
      - name: update-crio
        image: "{{ .Values.containerRuntime.initContainerImage }}"
        imagePullPolicy: {{ .Values.dfdaemon.pullPolicy | quote }}
        resources:
{{ toYaml .Values.dfdaemon.resources | indent 12 }}
        command:
        - /bin/sh
        - -cx
        - |-
          registries="{{- join " " .Values.containerRuntime.crio.registries }}"
          if [[ -n "$domains" ]]; then
            echo Empty registry domains
            exit 1
          fi

          confd="/host/etc/containers/registries.conf.d"
          if [[ ! -e "$confd" ]]; then
            mkdir -p "$confd"
          fi

          for registry in $registries; do
            # parse registry domain
            domain=$(echo $registry | sed "s,http.://,," | sed "s,:.*,,")
            schema=$(echo $registry | sed "s,://.*,,")
            port=$(echo $registry | sed "s,http.://,," | sed "s,[^:]*,," | sed "s,:,,")
            insecure=false
            if [[ "$schema" == "http" ]]; then
              insecure=true
            fi
            if [[ -z "$port" ]]; then
              if [[ "$schema" == "https" ]]; then
                port=443
              elif [[ "$schema" == "http" ]]; then
                port=80
              fi
            fi
            echo schema: $schema, domain: $domain, port: $port
            # inject registry
            if [[ -e "$confd/$domain.conf" ]]; then
              echo "registry $registry found in $confd, skip"
              continue
            else
              cat << EOF > "$confd/$domain.conf"
          [[registry]]
          prefix = "$domain"
          location = "$domain:$port"
          insecure = $insecure
          [[registry.mirror]]
          location = "127.0.0.1:{{ .Values.dfdaemon.hostPort}}"
          insecure = true
          EOF
              echo "Registry $domain added"
            fi
          done
          nsenter -t 1 -m systemctl -- reload crio.service
        volumeMounts:
        - name: etc
          mountPath: /host/etc
        securityContext:
          # nsenter need privilege permission
          privileged: true
      {{- end }}
      {{- end }}
      volumes:
      - name: config
        configMap:
          name: {{ template "dragonfly.dfdaemon.fullname" . }}
      {{- if and (not .Values.dfdaemon.hostNetwork) .Values.dfdaemon.config.proxy.tcpListen.namespace }}
      - name: hostns
        hostPath:
          path: /proc/1/ns
      - name: run
        hostPath:
          path: /run/dragonfly
          type: DirectoryOrCreate
      {{- end }}
      {{- if .Values.containerRuntime.docker.enable }}
      - name: etc
        hostPath:
          path: /etc
      - name: d7y-ca
        hostPath:
          path: /etc/dragonfly-ca
          type: DirectoryOrCreate
      {{- else if .Values.containerRuntime.containerd.enable }}
      - name: containerd-conf
        hostPath:
          path: {{ .Values.containerRuntime.containerd.configPathDir }}
      {{- else if .Values.containerRuntime.crio.enable }}
      - name: etc
        hostPath:
          path: /etc
      {{- end }}
      - name: data
        {{- if .Values.dfdaemon.mountDataDirAsHostPath }}
        hostPath:
          path: {{ .Values.dfdaemon.config.dataDir }}
          type: DirectoryOrCreate
        {{- else }}
        emptyDir: {}
        {{- end }}
      {{- if .Values.dfdaemon.extraVolumes }}
      {{- toYaml .Values.dfdaemon.extraVolumes | nindent 6 }}
      {{- end }}
{{- end }}