diff --git a/.github/workflows/terraform.yaml b/.github/workflows/terraform.yaml index 331c930..bb862e5 100644 --- a/.github/workflows/terraform.yaml +++ b/.github/workflows/terraform.yaml @@ -41,6 +41,7 @@ jobs: TF_VAR_spaces_access_id: ${{ secrets.SPACES_ACCESS_ID }} TF_VAR_spaces_secret_key: ${{ secrets.SPACES_SECRET_KEY }} TF_VAR_sendgrid_api_key: ${{ secrets.SENDGRID_API_KEY }} + TF_VAR_git_repo: ${{ github.server_url }}/${{ github.repository }} if: github.ref == 'refs/heads/main' needs: validate steps: diff --git a/flux/bootstrap/cluster-kustomization.yaml b/flux/bootstrap/cluster-kustomization.yaml new file mode 100644 index 0000000..60ae60e --- /dev/null +++ b/flux/bootstrap/cluster-kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.toolkit.fluxcd.io/v1 +kind: Kustomization +metadata: + name: cluster + namespace: flux-system +spec: + interval: 10m + prune: true + sourceRef: + kind: GitRepository + name: flux-system + path: ./flux + postBuild: + substituteFrom: + - kind: ConfigMap + name: shared-values + - kind: ConfigMap + name: runtime-values diff --git a/flux/bootstrap/shared-values.yaml b/flux/bootstrap/shared-values.yaml new file mode 100644 index 0000000..8c62af8 --- /dev/null +++ b/flux/bootstrap/shared-values.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: shared-values + namespace: flux-system +data: + key: value diff --git a/flux/kustomization.yaml b/flux/kustomization.yaml new file mode 100644 index 0000000..18cc241 --- /dev/null +++ b/flux/kustomization.yaml @@ -0,0 +1,18 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - bootstrap/cluster-kustomization.yaml + - bootstrap/shared-values.yaml + - repositories/emberstack.yaml + - repositories/envoy-proxy.yaml + - repositories/external-dns.yaml + - repositories/gitlab.yaml + - repositories/jetstack.yaml + - releases/cert-manager.yaml + - releases/cluster-issuer.yaml + - releases/dns01-certificate.yaml + - releases/envoy-gateway.yaml + - releases/external-dns.yaml + - releases/gateway-config.yaml + - releases/gitlab.yaml + - releases/reflector.yaml \ No newline at end of file diff --git a/flux/releases/cert-manager.yaml b/flux/releases/cert-manager.yaml new file mode 100644 index 0000000..82e13bc --- /dev/null +++ b/flux/releases/cert-manager.yaml @@ -0,0 +1,26 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: cert-manager + namespace: flux-system +spec: + interval: 1m + chartRef: + kind: OCIRepository + name: cert-manager + namespace: flux-system + targetNamespace: cert-manager + install: + createNamespace: true + values: + crds: + enabled: true + keep: true + + replicaCount: 1 + + prometheus: + enabled: true + + webhook: + timeoutSeconds: 25 diff --git a/flux/releases/cluster-issuer.yaml b/flux/releases/cluster-issuer.yaml new file mode 100644 index 0000000..9af598a --- /dev/null +++ b/flux/releases/cluster-issuer.yaml @@ -0,0 +1,21 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: cluster-issuer + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: cert-manager + chart: + spec: + chart: ./helm/cluster-issuer + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + targetNamespace: cert-manager + install: + createNamespace: true + values: + email: ${email} diff --git a/flux/releases/dns01-certificate.yaml b/flux/releases/dns01-certificate.yaml new file mode 100644 index 0000000..42a709a --- /dev/null +++ b/flux/releases/dns01-certificate.yaml @@ -0,0 +1,24 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: dns01-certificate + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: cert-manager + - name: cluster-issuer + chart: + spec: + chart: ./helm/dns01-certificate + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + targetNamespace: cert-manager + install: + createNamespace: true + values: + name: wildcard-certificate + secretName: wildcard-certificate + issuer: letsencrypt diff --git a/flux/releases/envoy-gateway.yaml b/flux/releases/envoy-gateway.yaml new file mode 100644 index 0000000..05b920a --- /dev/null +++ b/flux/releases/envoy-gateway.yaml @@ -0,0 +1,17 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: envoy-gateway + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: cert-manager + - name: dns01-certificate + chartRef: + kind: OCIRepository + name: envoy-proxy + namespace: flux-system + targetNamespace: envoy-gateway-system + install: + createNamespace: true diff --git a/flux/releases/external-dns.yaml b/flux/releases/external-dns.yaml new file mode 100644 index 0000000..a135400 --- /dev/null +++ b/flux/releases/external-dns.yaml @@ -0,0 +1,36 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: external-dns + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: reflector + chart: + spec: + chart: external-dns + version: 1.20.0 + sourceRef: + kind: HelmRepository + name: external-dns + namespace: flux-system + targetNamespace: external-dns + install: + createNamespace: true + values: + provider: + name: digitalocean + env: + - name: DO_TOKEN + valueFrom: + secretKeyRef: + name: do-dns-secret + key: password + domainFilters: + - ${domain} + txtOwnerId: ${cluster_name} + sources: + - service + - gateway-httproute + policy: sync diff --git a/flux/releases/gateway-config.yaml b/flux/releases/gateway-config.yaml new file mode 100644 index 0000000..120d4e8 --- /dev/null +++ b/flux/releases/gateway-config.yaml @@ -0,0 +1,22 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: gateway-config + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: envoy-gateway + chart: + spec: + chart: ./helm/gateway-config + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + targetNamespace: envoy-gateway-system + install: + createNamespace: true + values: + name: gateway + certificateName: wildcard-certificate diff --git a/flux/releases/gitlab.yaml b/flux/releases/gitlab.yaml new file mode 100644 index 0000000..f280c74 --- /dev/null +++ b/flux/releases/gitlab.yaml @@ -0,0 +1,175 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: gitlab + namespace: flux-system +spec: + interval: 1m + dependsOn: + - name: cert-manager + - name: dns01-certificate + - name: reflector + - name: cluster-issuer + - name: envoy-gateway + - name: gateway-config + chart: + spec: + chart: gitlab + version: 9.10.3 + sourceRef: + kind: HelmRepository + name: gitlab + namespace: flux-system + targetNamespace: gitlab + install: + createNamespace: true + values: + global: + edition: ce + + initialRootPassword: + secret: gitlab-initial-root-password + key: password + + hosts: + domain: ${domain} + https: true + gitlab: + name: ${gitlab_host}.${domain} + registry: + name: ${registry_host}.${domain} + pages: + name: ${pages_host}.${domain} + + gatewayApi: + enabled: true + installEnvoy: false + configureCertmanager: false + gatewayRef: + name: gateway + namespace: envoy-gateway-system + + ingress: + enabled: false + configureCertmanager: false + + psql: + host: ${postgres_host} + port: ${postgres_port} + database: ${postgres_database} + username: ${postgres_username} + password: + secret: gitlab-postgres-secret + key: password + + redis: + scheme: rediss + host: ${redis_host} + port: ${redis_port} + auth: + enabled: true + secret: gitlab-redis-secret + key: password + + pages: + enabled: true + + appConfig: + object_store: + enabled: true + proxy_download: true + connection: + secret: gitlab-s3-main-secret + key: connection + + artifacts: + bucket: ${buckets_artifacts} + connection: {} + uploads: + bucket: ${buckets_uploads} + connection: {} + packages: + bucket: ${buckets_packages} + connection: {} + lfs: + bucket: ${buckets_lfs} + connection: {} + registry: + bucket: ${buckets_registry} + connection: {} + pages: + bucket: ${buckets_pages} + connection: {} + + minio: + enabled: false + + email: + display_name: GitLab + from: gitlab@${domain} + reply_to: noreply@${domain} + + smtp: + enabled: true + domain: smtp.sendgrid.net + address: smtp.sendgrid.net + port: 2525 + user_name: apikey + password: + secret: gitlab-sendgrid-secret + key: password + tls: false + starttls_auto: true + openssl_verify_mode: peer + + time_zone: UTC + + extraEnv: + PGSSLMODE: require + + + postgresql: + install: false + redis: + install: false + gitlab: + webservice: + gatewayRoute: + sectionName: https + metrics: + enabled: true + serviceMonitor: + enabled: true + workhorse: + metrics: + enabled: true + serviceMonitor: + enabled: true + sidekiq: + metrics: + enabled: true + podMonitor: + enabled: true + gitlab-exporter: + metrics: + enabled: true + serviceMonitor: + enabled: true + gitlab-shell: + sshDaemon: gitlab-sshd + metrics: + enabled: true + serviceMonitor: + enabled: true + gitlab-runner: + install: false + registry: + metrics: + enabled: true + serviceMonitor: + enabled: true + prometheus: + install: false + nginx-ingress: + enabled: false + installCertmanager: false \ No newline at end of file diff --git a/flux/releases/reflector.yaml b/flux/releases/reflector.yaml new file mode 100644 index 0000000..d1a97e1 --- /dev/null +++ b/flux/releases/reflector.yaml @@ -0,0 +1,18 @@ +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: reflector + namespace: flux-system +spec: + interval: 1m + chart: + spec: + chart: reflector + version: 10.0.35 + sourceRef: + kind: HelmRepository + name: emberstack + namespace: flux-system + targetNamespace: kube-system + install: + createNamespace: true diff --git a/flux/repositories/cert-manager.yaml b/flux/repositories/cert-manager.yaml new file mode 100644 index 0000000..97df86d --- /dev/null +++ b/flux/repositories/cert-manager.yaml @@ -0,0 +1,9 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: cert-manager + namespace: flux-system +spec: + url: oci://quay.io/jetstack/charts/cert-manager + ref: + tag: 1.20.2 \ No newline at end of file diff --git a/flux/repositories/emberstack.yaml b/flux/repositories/emberstack.yaml new file mode 100644 index 0000000..9ebffb6 --- /dev/null +++ b/flux/repositories/emberstack.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: emberstack + namespace: flux-system +spec: + url: https://emberstack.github.io/helm-charts \ No newline at end of file diff --git a/flux/repositories/envoy-proxy.yaml b/flux/repositories/envoy-proxy.yaml new file mode 100644 index 0000000..59aea4b --- /dev/null +++ b/flux/repositories/envoy-proxy.yaml @@ -0,0 +1,9 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: OCIRepository +metadata: + name: envoy-proxy + namespace: flux-system +spec: + url: oci://docker.io/envoyproxy/gateway-helm + ref: + tag: 1.7.2 \ No newline at end of file diff --git a/flux/repositories/external-dns.yaml b/flux/repositories/external-dns.yaml new file mode 100644 index 0000000..ed3c2a5 --- /dev/null +++ b/flux/repositories/external-dns.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: external-dns + namespace: flux-system +spec: + url: https://kubernetes-sigs.github.io/external-dns \ No newline at end of file diff --git a/flux/repositories/gitlab.yaml b/flux/repositories/gitlab.yaml new file mode 100644 index 0000000..ec31d4b --- /dev/null +++ b/flux/repositories/gitlab.yaml @@ -0,0 +1,7 @@ +apiVersion: source.toolkit.fluxcd.io/v1 +kind: HelmRepository +metadata: + name: gitlab + namespace: flux-system +spec: + url: https://charts.gitlab.io \ No newline at end of file diff --git a/terraform/dns.tf b/terraform/dns.tf index 446aa10..97e0ed5 100644 --- a/terraform/dns.tf +++ b/terraform/dns.tf @@ -1,22 +1,3 @@ -locals { - lb_ip = try(data.kubernetes_service_v1.envoy_gateway.status[0].load_balancer[0].ingress[0].ip, null) - records = toset([ - var.gitlab_host, var.registry_host, "*.${var.pages_host}", var.grafana_host - ]) -} - resource "digitalocean_domain" "main" { name = var.domain_name } - -# resource "digitalocean_record" "main" { -# for_each = local.records - -# domain = digitalocean_domain.main.name -# type = "A" -# name = each.key -# value = local.lb_ip -# ttl = 300 - -# depends_on = [ time_sleep.wait_for_gateway_debug ] -# } \ No newline at end of file diff --git a/terraform/fluxcd.tf b/terraform/fluxcd.tf new file mode 100644 index 0000000..5e5a9ba --- /dev/null +++ b/terraform/fluxcd.tf @@ -0,0 +1,34 @@ +resource "flux_bootstrap_git" "flux_cd" { + path = "flux" +} + +resource "kubernetes_config_map_v1" "runtime_values" { + metadata { + name = "runtime-values" + namespace = "flux-system" + } + + data = { + domain = var.domain_name + gitlab_host = var.gitlab_host + registry_host = var.registry_host + pages_host = var.pages_host + email = var.email + cluster_name = var.cluster_name + + postgres_host = digitalocean_database_connection_pool.main.private_host + postgres_port = digitalocean_database_connection_pool.main.port + postgres_database = digitalocean_database_db.gitlab.name + postgres_username = digitalocean_database_cluster.postgres.user + redis_host = digitalocean_database_cluster.valkey.private_host + redis_port = digitalocean_database_cluster.valkey.port + buckets_artifacts = cloudflare_r2_bucket.gitlab["artifacts"].name + buckets_uploads = cloudflare_r2_bucket.gitlab["uploads"].name + buckets_packages = cloudflare_r2_bucket.gitlab["packages"].name + buckets_lfs = cloudflare_r2_bucket.gitlab["lfs"].name + buckets_registry = cloudflare_r2_bucket.gitlab["registry"].name + buckets_pages = cloudflare_r2_bucket.gitlab["pages"].name + } + + depends_on = [ flux_bootstrap_git.flux_cd ] +} \ No newline at end of file diff --git a/terraform/helm.tf b/terraform/helm.tf deleted file mode 100644 index 778416a..0000000 --- a/terraform/helm.tf +++ /dev/null @@ -1,163 +0,0 @@ -resource "helm_release" "reflector" { - name = "reflector" - namespace = "kube-system" - repository = "https://emberstack.github.io/helm-charts" - chart = "reflector" - version = "10.0.35" -} - -resource "helm_release" "external_dns" { - name = "external-dns" - namespace = kubernetes_namespace_v1.external_dns.metadata[0].name - repository = "https://kubernetes-sigs.github.io/external-dns" - chart = "external-dns" - version = "1.20.0" - - values = [ - templatefile("${path.module}/../helm/external-dns/values.yaml", - { - domain = var.domain_name, - cluster_name = var.cluster_name - }) - ] - - depends_on = [ - kubernetes_namespace_v1.external_dns, - kubernetes_secret_v1.do_dns_secret, - helm_release.reflector - ] -} - -resource "helm_release" "cert_manager" { - name = "cert-manager" - namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name - repository = "oci://quay.io/jetstack/charts" - chart = "cert-manager" - version = "1.20.2" - - values = [ - file("${path.module}/../helm/cert-manager/values.yaml") - ] -} - -resource "helm_release" "cluster_issuer" { - name = "cluster-issuer" - namespace = kubernetes_namespace_v1.cert_manager.metadata[0].name - chart = "${path.module}/../helm/cluster-issuer/chart" - - values = [ - templatefile("${path.module}/../helm/cluster-issuer/values.yaml", - { - email = var.email - }) - ] - - depends_on = [helm_release.cert_manager] -} - -resource "helm_release" "dns01_certificate" { - name = "dns01-certificate" - namespace = kubernetes_namespace_v1.envoy_gateway_system.metadata[0].name - chart = "${path.module}/../helm/dns01-certificate/chart" - - values = [ - yamlencode({ - domain = var.domain_name - dnsNameTemplates = [ - "{{ .Values.domain }}", - "*.{{ .Values.domain }}", - "*.pages.{{ .Values.domain }}" - ] - reflectNamespaces = "${kubernetes_namespace_v1.gitlab.metadata[0].name},${kubernetes_namespace_v1.monitoring.metadata[0].name}" - }) - ] - - depends_on = [helm_release.cert_manager, helm_release.cluster_issuer, helm_release.reflector] -} - -resource "helm_release" "envoy_gateway" { - name = "envoy-gateway" - namespace = kubernetes_namespace_v1.envoy_gateway_system.metadata[0].name - repository = "oci://docker.io/envoyproxy" - chart = "gateway-helm" - version = "1.7.2" - - wait = true - - depends_on = [ helm_release.cert_manager, helm_release.dns01_certificate ] - -} - -resource "helm_release" "gateway_config" { - name = "gateway-config" - namespace = kubernetes_namespace_v1.envoy_gateway_system.metadata[0].name - chart = "${path.module}/../helm/gateway-config/chart" - - depends_on = [ helm_release.envoy_gateway ] -} - -resource "helm_release" "gitlab" { - name = "gitlab" - namespace = kubernetes_namespace_v1.gitlab.metadata[0].name - repository = "https://charts.gitlab.io/" - chart = "gitlab" - version = "9.10.3" - - timeout = 1800 - wait = true - wait_for_jobs = true - - values = [ - templatefile("${path.module}/../helm/gitlab/values.yaml", { - domain = var.domain_name - gitlab_host = var.gitlab_host - registry_host = var.registry_host - pages_host = var.pages_host - postgres_host = digitalocean_database_connection_pool.main.private_host - postgres_port = digitalocean_database_connection_pool.main.port - postgres_database = digitalocean_database_db.gitlab.name - postgres_username = digitalocean_database_cluster.postgres.user - redis_host = digitalocean_database_cluster.valkey.private_host - redis_port = digitalocean_database_cluster.valkey.port - buckets = {for key, bucket in cloudflare_r2_bucket.gitlab : key => bucket.name} - }) - ] - - depends_on = [ - digitalocean_database_cluster.postgres, - digitalocean_database_postgresql_config.postgres, - digitalocean_database_db.gitlab, - digitalocean_database_user.gitlab, - digitalocean_database_connection_pool.main, - digitalocean_database_cluster.valkey, - helm_release.cert_manager, - helm_release.dns01_certificate, - helm_release.reflector, - helm_release.cluster_issuer, - helm_release.envoy_gateway, - helm_release.gateway_config, - kubernetes_secret_v1.gitlab_initial_root_password, - kubernetes_secret_v1.gitlab_postgres, - kubernetes_secret_v1.gitlab_redis, - kubernetes_secret_v1.gitlab_s3_main, - kubernetes_secret_v1.gitlab_sendgrid_secret - ] -} - -resource "helm_release" "kube_prometheus_stack" { - name = "kube-prometheus-stack" - namespace = kubernetes_namespace_v1.monitoring.metadata[0].name - repository = "https://prometheus-community.github.io/helm-charts" - chart = "kube-prometheus-stack" - version = "83.6.0" - - values = [ - templatefile("${path.module}/../helm/kube-prometheus-stack/values.yaml", - { - grafana_host = var.grafana_host - domain = var.domain_name - }) - ] - - depends_on = [ helm_release.envoy_gateway ] -} \ No newline at end of file diff --git a/terraform/kubernetes.tf b/terraform/kubernetes.tf index 0c3b5a5..4dc2bd8 100644 --- a/terraform/kubernetes.tf +++ b/terraform/kubernetes.tf @@ -178,17 +178,3 @@ resource "kubernetes_secret_v1" "gitlab_sendgrid_secret" { type = "Opaque" } - -resource "time_sleep" "wait_for_gateway_debug" { - depends_on = [ helm_release.envoy_gateway, helm_release.gateway_config ] - create_duration = "120s" -} - -data "kubernetes_service_v1" "envoy_gateway" { - metadata { - name = "gateway" - namespace = kubernetes_namespace_v1.envoy_gateway_system.metadata[0].name - } - - depends_on = [ time_sleep.wait_for_gateway_debug ] -} diff --git a/terraform/providers.tf b/terraform/providers.tf index e0a26ea..08d35b7 100644 --- a/terraform/providers.tf +++ b/terraform/providers.tf @@ -16,7 +16,7 @@ provider "kubernetes" { ) } -provider "helm" { +provider "flux" { kubernetes = { host = digitalocean_kubernetes_cluster.main.endpoint token = digitalocean_kubernetes_cluster.main.kube_config[0].token @@ -24,4 +24,7 @@ provider "helm" { digitalocean_kubernetes_cluster.main.kube_config[0].cluster_ca_certificate ) } + git = { + url = var.git_repo + } } \ No newline at end of file diff --git a/terraform/variables.tf b/terraform/variables.tf index cbb364d..988eae2 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -49,6 +49,11 @@ variable "sendgrid_api_key" { } +variable "git_repo" { + type = string +} + + variable "region" { type = string default = "ams3" diff --git a/terraform/versions.tf b/terraform/versions.tf index 581f474..e48d918 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -14,17 +14,13 @@ terraform { source = "hashicorp/kubernetes" version = "~> 3.0.1" } - helm = { - source = "hashicorp/helm" - version = "~> 3.1.1" + flux = { + source = "fluxcd/flux" + version = "~> 1.8.6" } random = { source = "hashicorp/random" version = "~> 3.8.1" } - time = { - source = "hashicorp/time" - version = "~> 0.13.1" - } } } \ No newline at end of file